LLM client module for PocketFlow
Provides unified interface for multiple LLM providers with streaming, caching, token tracking, and error handling.
Types
LlmClient = ref object provider*: LlmProvider baseUrl*: string apiKey*: string model*: string costTracker*: CostTracker cache*: Cache
- Client for interacting with LLM providers
LlmOptions = object temperature*: float maxTokens*: int topP*: float stream*: bool streamCallback*: StreamCallback useCache*: bool timeout*: int
- Configuration options for LLM requests
LlmProvider = enum OpenAI, Ollama, Anthropic, Google, Custom
- Supported LLM providers
StreamCallback = proc (chunk: string): Future[void] {....gcsafe.}
- Callback for streaming responses
Procs
proc chat(client: LlmClient; messages: JsonNode; temperature: float = 0.7): Future[ string] {....gcsafe, stackTrace: false, raises: [Exception, ValueError], tags: [RootEffect], forbids: [].}
-
Sends a chat completion request (non-streaming)
Args: messages: JArray of message objects with "role" and "content" temperature: Sampling temperature (0.0-2.0)
Returns: The assistant's response text
proc chatWithOptions(client: LlmClient; messages: JsonNode; options: LlmOptions): Future[ string] {....gcsafe, stackTrace: false, raises: [Exception, ValueError, KeyError], tags: [RootEffect, TimeEffect], forbids: [].}
proc embeddings(client: LlmClient; texts: seq[string]; model: string = ""): Future[ seq[seq[float]]] {....gcsafe, stackTrace: false, raises: [Exception, ValueError, LLMError, KeyError, HttpRequestError, LibraryError, SslError, OSError, IOError, ProtocolError, JsonParsingError], tags: [RootEffect, TimeEffect, ReadIOEffect, WriteIOEffect], forbids: [].}
-
Generates embeddings for text inputs
Args: texts: Sequence of texts to embed model: Embedding model (uses defaults if empty)
Returns: Sequence of embedding vectors
proc generate(client: LlmClient; prompt: string; temperature: float = 0.7): Future[ string] {....gcsafe, stackTrace: false, raises: [Exception, ValueError], tags: [RootEffect], forbids: [].}
-
Helper for simple single-turn generation
Args: prompt: The user prompt temperature: Sampling temperature
Returns: The generated response
proc newLlmClient(provider: LlmProvider = OpenAI; baseUrl: string = ""; apiKey: string = ""; model: string = ""; costTracker: CostTracker = nil; cache: Cache = nil): LlmClient {. ...raises: [], tags: [], forbids: [].}
-
Creates a new LLM client
Args: provider: The LLM provider to use baseUrl: Custom base URL (uses defaults if empty) apiKey: API key for authentication model: Model name (uses defaults if empty) costTracker: Optional custom cost tracker cache: Optional custom cache