From 9356e1f7d018231aefebcfe10bb54ce4d22a55d6 Mon Sep 17 00:00:00 2001 From: Aditya Gupta Date: Thu, 16 Jul 2026 13:10:59 +0530 Subject: [PATCH] feat(llm): Added ollama provider --- apps/gui/src/app/actions.ts | 4 + .../config/ProviderInstancesConfig.tsx | 69 +++++--- .../src/lib/simulation/provider-resolver.ts | 11 ++ package.json | 1 + packages/llm/src/index.ts | 1 + packages/llm/src/llm.ts | 9 + packages/llm/src/provider-manager.ts | 30 +++- packages/llm/src/providers/ollama.ts | 162 ++++++++++++++++++ packages/memory/src/handoff.ts | 6 +- pnpm-lock.yaml | 55 +++++- 10 files changed, 311 insertions(+), 37 deletions(-) create mode 100644 packages/llm/src/providers/ollama.ts diff --git a/apps/gui/src/app/actions.ts b/apps/gui/src/app/actions.ts index eefc5c1..23d8152 100644 --- a/apps/gui/src/app/actions.ts +++ b/apps/gui/src/app/actions.ts @@ -254,6 +254,7 @@ export async function createProviderInstance( modelName?: string, type: "generative" | "embedding" = "generative", maxContext?: number, + endpointUrl?: string, ): Promise { return ProviderManager.create( name, @@ -262,6 +263,7 @@ export async function createProviderInstance( modelName, type, maxContext, + endpointUrl, ); } @@ -281,6 +283,7 @@ export async function updateProviderInstance( modelName?: string, type: "generative" | "embedding" = "generative", maxContext?: number, + endpointUrl?: string, ): Promise { ProviderManager.update( id, @@ -290,6 +293,7 @@ export async function updateProviderInstance( modelName, type, maxContext, + endpointUrl, ); } diff --git a/apps/gui/src/components/config/ProviderInstancesConfig.tsx b/apps/gui/src/components/config/ProviderInstancesConfig.tsx index 4b9da0e..192016b 100644 --- a/apps/gui/src/components/config/ProviderInstancesConfig.tsx +++ b/apps/gui/src/components/config/ProviderInstancesConfig.tsx @@ -64,6 +64,7 @@ export function ProviderInstancesConfig({ "generative", ); const [editMaxContext, setEditMaxContext] = useState(32768); + const [editEndpointUrl, setEditEndpointUrl] = useState(""); const [loading, setLoading] = useState(false); const [error, setError] = useState(""); @@ -76,6 +77,7 @@ export function ProviderInstancesConfig({ setEditIsActive(false); setEditType("generative"); setEditMaxContext(32768); + setEditEndpointUrl(""); } else if (selectedInstanceId === "new") { setEditName(""); const defaultProvider = "google-genai"; @@ -86,6 +88,7 @@ export function ProviderInstancesConfig({ setEditModel(pMeta?.defaultModel || "gemini-2.5-flash"); setEditIsActive(false); setEditMaxContext(32768); + setEditEndpointUrl(""); } else { const inst = instances.find((i) => i.id === selectedInstanceId); if (inst) { @@ -109,6 +112,7 @@ export function ProviderInstancesConfig({ ? inst.maxContext : 32768, ); + setEditEndpointUrl(inst.endpointUrl || ""); } } }, [selectedInstanceId, instances, availableProviders]); @@ -149,7 +153,7 @@ export function ProviderInstancesConfig({ let targetInstanceId = selectedInstanceId; if (selectedInstanceId === "new") { - if (!editKey.trim()) { + if (editProvider !== "ollama" && !editKey.trim()) { setError("API Key is required for new instances."); setLoading(false); return; @@ -157,10 +161,13 @@ export function ProviderInstancesConfig({ const created = await createProviderInstance( editName, editProvider, - editKey, + editProvider === "ollama" ? "none" : editKey, editModel || undefined, editType, editType === "generative" ? editMaxContext : 0, + editProvider === "ollama" + ? editEndpointUrl || "http://localhost:11434" + : undefined, ); if (editIsActive) { await setActiveProviderInstance(created.id); @@ -194,10 +201,13 @@ export function ProviderInstancesConfig({ selectedInstanceId, editName, editProvider, - editKey || undefined, + editProvider === "ollama" ? "none" : editKey || undefined, editModel || undefined, editType, editType === "generative" ? editMaxContext : 0, + editProvider === "ollama" + ? editEndpointUrl || "http://localhost:11434" + : undefined, ); if (editIsActive) { await setActiveProviderInstance(selectedInstanceId); @@ -336,11 +346,11 @@ export function ProviderInstancesConfig({ } items={[ { - label: "Generative (Text Completion)", + label: "Generative (Text Generation)", value: "generative", }, { - label: "Embedding (Vector generation)", + label: "Embedding (Vector Embeddings)", value: "embedding", }, ]} @@ -351,10 +361,10 @@ export function ProviderInstancesConfig({ - Generative (Chat / Text Completion) + Generative (Text Generation) - Embedding (Vector generation) + Embedding (Vector Embeddings) @@ -398,21 +408,36 @@ export function ProviderInstancesConfig({ )} -
- - setEditKey(e.target.value)} - placeholder={ - selectedInstanceId === "new" - ? "AIzaSy..." - : "•••••••• (unchanged)" - } - required={selectedInstanceId === "new"} - /> -
+ {editProvider !== "ollama" && ( +
+ + setEditKey(e.target.value)} + placeholder={ + selectedInstanceId === "new" + ? "AIzaSy..." + : "•••••••• (unchanged)" + } + required={selectedInstanceId === "new"} + /> +
+ )} + + {editProvider === "ollama" && ( +
+ + setEditEndpointUrl(e.target.value)} + placeholder="e.g. http://localhost:11434" + required + /> +
+ )}
diff --git a/apps/gui/src/lib/simulation/provider-resolver.ts b/apps/gui/src/lib/simulation/provider-resolver.ts index 325d33a..b7fcc57 100644 --- a/apps/gui/src/lib/simulation/provider-resolver.ts +++ b/apps/gui/src/lib/simulation/provider-resolver.ts @@ -1,6 +1,8 @@ import { GeminiProvider, MockLLMProvider, + OllamaProvider, + OllamaEmbeddingProvider, ProviderManager, OpenRouterProvider, GeminiEmbeddingProvider, @@ -58,6 +60,13 @@ function buildLLMProvider(inst: ModelProviderInstance): ILLMProvider { inst.name, inst.maxContext, ); + } else if (inst.providerName === "ollama") { + return new OllamaProvider( + inst.endpointUrl, + inst.modelName, + inst.name, + inst.maxContext, + ); } return new MockLLMProvider([]); } @@ -67,6 +76,8 @@ function buildEmbeddingProvider( ): IEmbeddingProvider { if (inst.providerName === "google-genai") { return new GeminiEmbeddingProvider(inst.apiKey, inst.modelName); + } else if (inst.providerName === "ollama") { + return new OllamaEmbeddingProvider(inst.endpointUrl, inst.modelName); } return new MockEmbeddingProvider(inst.modelName); } diff --git a/package.json b/package.json index 9d8ab7a..7d83620 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ }, "dependencies": { "@langchain/google-genai": "^2.2.0", + "@langchain/ollama": "^0.2.3", "@langchain/openrouter": "^0.4.3", "@types/node": "^20.19.43", "dotenv": "^17.4.2" diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index 6817141..a7c747b 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -2,5 +2,6 @@ export * from "./llm.js"; export * from "./config.js"; export * from "./providers/google-genai.js"; export * from "./providers/mock.js"; +export * from "./providers/ollama.js"; export * from "./providers/openrouter.js"; export * from "./provider-manager.js"; diff --git a/packages/llm/src/llm.ts b/packages/llm/src/llm.ts index b8635d2..ad8a7aa 100644 --- a/packages/llm/src/llm.ts +++ b/packages/llm/src/llm.ts @@ -57,6 +57,7 @@ export interface ModelProviderInstance { modelName?: string; type: "generative" | "embedding"; maxContext?: number; + endpointUrl?: string; } export interface ModelProviderMeta { @@ -83,6 +84,14 @@ export const AVAILABLE_PROVIDERS: ModelProviderMeta[] = [ defaultModel: "google/gemini-2.5-flash", defaultEmbeddingModel: "openai/text-embedding-3-small", }, + { + id: "ollama", + displayName: "Ollama", + description: + "Local model runner — no API key required, uses the Ollama server base URL instead", + defaultModel: "llama3.1", + defaultEmbeddingModel: "nomic-embed-text", + }, { id: "mock", displayName: "Mock LLM Provider", diff --git a/packages/llm/src/provider-manager.ts b/packages/llm/src/provider-manager.ts index 522c5bf..4401990 100644 --- a/packages/llm/src/provider-manager.ts +++ b/packages/llm/src/provider-manager.ts @@ -82,6 +82,14 @@ function getSettingsDb() { // ignore } + try { + db.prepare( + `ALTER TABLE provider_instances ADD COLUMN endpointUrl TEXT`, + ).run(); + } catch { + // ignore + } + // Auto-bootstrap environment variables if DB contains 0 instances try { if (!hasBootstrapped) { @@ -172,6 +180,7 @@ export class ProviderManager { modelName?: string; type: string; maxContext?: number; + endpointUrl?: string; }[]; return rows.map((r) => ({ id: r.id, @@ -187,6 +196,7 @@ export class ProviderManager { : r.type === "embedding" ? 0 : 32768, + endpointUrl: r.endpointUrl || undefined, })); } finally { db.close(); @@ -200,6 +210,7 @@ export class ProviderManager { modelName?: string, type: "generative" | "embedding" = "generative", maxContext?: number, + endpointUrl?: string, ): ModelProviderInstance { const db = getSettingsDb(); try { @@ -220,8 +231,8 @@ export class ProviderManager { db.prepare( ` - INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext, endpointUrl) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `, ).run( id, @@ -232,6 +243,7 @@ export class ProviderManager { modelName || null, type, actualMaxContext, + endpointUrl || null, ); return { @@ -243,6 +255,7 @@ export class ProviderManager { modelName, type, maxContext: actualMaxContext, + endpointUrl, }; } finally { db.close(); @@ -299,6 +312,7 @@ export class ProviderManager { modelName?: string, type: "generative" | "embedding" = "generative", maxContext?: number, + endpointUrl?: string, ): void { const db = getSettingsDb(); try { @@ -312,7 +326,7 @@ export class ProviderManager { db.prepare( ` UPDATE provider_instances - SET name = ?, providerName = ?, apiKey = ?, modelName = ?, type = ?, maxContext = ? + SET name = ?, providerName = ?, apiKey = ?, modelName = ?, type = ?, maxContext = ?, endpointUrl = ? WHERE id = ? `, ).run( @@ -322,13 +336,14 @@ export class ProviderManager { modelName || null, type, actualMaxContext, + endpointUrl || null, id, ); } else { db.prepare( ` UPDATE provider_instances - SET name = ?, providerName = ?, modelName = ?, type = ?, maxContext = ? + SET name = ?, providerName = ?, modelName = ?, type = ?, maxContext = ?, endpointUrl = ? WHERE id = ? `, ).run( @@ -337,6 +352,7 @@ export class ProviderManager { modelName || null, type, actualMaxContext, + endpointUrl || null, id, ); } @@ -364,6 +380,7 @@ export class ProviderManager { modelName?: string; type: string; maxContext?: number; + endpointUrl?: string; } | undefined; @@ -447,6 +464,7 @@ export class ProviderManager { modelName?: string; type: string; maxContext?: number; + endpointUrl?: string; } | undefined; @@ -466,6 +484,7 @@ export class ProviderManager { : retryRow.type === "embedding" ? 0 : 32768, + endpointUrl: retryRow.endpointUrl || undefined, }; } } @@ -483,6 +502,7 @@ export class ProviderManager { modelName?: string; type: string; maxContext?: number; + endpointUrl?: string; } | undefined; if (firstRow) { @@ -503,6 +523,7 @@ export class ProviderManager { : firstRow.type === "embedding" ? 0 : 32768, + endpointUrl: firstRow.endpointUrl || undefined, }; } return null; @@ -522,6 +543,7 @@ export class ProviderManager { : row.type === "embedding" ? 0 : 32768, + endpointUrl: row.endpointUrl || undefined, }; } catch { const googleKey = process.env.GOOGLE_API_KEY; diff --git a/packages/llm/src/providers/ollama.ts b/packages/llm/src/providers/ollama.ts new file mode 100644 index 0000000..9db58e8 --- /dev/null +++ b/packages/llm/src/providers/ollama.ts @@ -0,0 +1,162 @@ +import { z } from "zod"; +import { ChatOllama, OllamaEmbeddings } from "@langchain/ollama"; +import { + ILLMProvider, + LLMRequest, + LLMResponse, + LLMCallRecord, + IEmbeddingProvider, +} from "../llm.js"; +import { ProviderManager } from "../provider-manager.js"; + +export class OllamaProvider implements ILLMProvider { + static readonly providerId = "ollama"; + static readonly displayName = "Ollama"; + static readonly description = + "Local model runner supporting open-source LLMs via the Ollama server"; + static readonly defaultModel = "llama3.1"; + + providerName = "Ollama"; + private model: ChatOllama; + private modelNameUsed: string; + private providerInstanceName?: string; + private maxContextUsed?: number; + lastCalls: LLMCallRecord[] = []; + + /** + * Creates an OllamaProvider. + * + * Resolution order for configuration: + * 1. Explicit constructor arguments + * 2. Active "generative" instance in ProviderManager whose providerName === "ollama" + * 3. Defaults (baseUrl: http://localhost:11434, model: llama3.1) + * + * No API key is required for Ollama. The `endpointUrl` in + * ModelProviderInstance stores the Ollama server base URL + * (e.g. "http://localhost:11434"). + */ + constructor( + baseUrl?: string, + modelName?: string, + providerInstanceName?: string, + maxContext?: number, + ) { + let url = baseUrl; + let model = modelName; + this.providerInstanceName = providerInstanceName; + this.maxContextUsed = maxContext; + + if (!url || !model) { + const active = ProviderManager.getActive("generative"); + if (active && active.providerName === OllamaProvider.providerId) { + if (!url) { + url = active.endpointUrl; + } + if (!model) { + model = active.modelName; + } + if (!this.providerInstanceName) { + this.providerInstanceName = active.name; + } + if (this.maxContextUsed === undefined) { + this.maxContextUsed = active.maxContext; + } + } + } + + this.modelNameUsed = model || OllamaProvider.defaultModel; + this.model = new ChatOllama({ + baseUrl: url || "http://localhost:11434", + model: this.modelNameUsed, + }); + } + + async generateStructuredResponse( + request: LLMRequest, + ): Promise>> { + const structuredModel = this.model.withStructuredOutput(request.schema, { + includeRaw: true, + }); + const result = (await structuredModel.invoke([ + { role: "system", content: request.systemPrompt }, + { role: "user", content: request.userContext }, + ])) as unknown as { + parsed?: z.infer; + raw?: { + usage_metadata?: { + input_tokens?: number; + output_tokens?: number; + total_tokens?: number; + }; + }; + }; + + const parsed = result?.parsed; + const raw = result?.raw; + + const usage = { + inputTokens: raw?.usage_metadata?.input_tokens || 0, + outputTokens: raw?.usage_metadata?.output_tokens || 0, + totalTokens: raw?.usage_metadata?.total_tokens || 0, + modelName: this.modelNameUsed, + providerInstanceName: this.providerInstanceName || "Default", + maxContext: + this.maxContextUsed !== undefined ? this.maxContextUsed : 32768, + }; + + this.lastCalls.push({ + systemPrompt: request.systemPrompt, + userContext: request.userContext, + usage, + }); + + return { success: true, data: parsed, usage }; + } +} + +export class OllamaEmbeddingProvider implements IEmbeddingProvider { + static readonly providerId = "ollama"; + static readonly displayName = "Ollama Embeddings"; + + providerName = "Ollama"; + private model: OllamaEmbeddings; + + /** + * Creates an OllamaEmbeddingProvider. + * + * Resolution order: + * 1. Explicit constructor arguments + * 2. Active "embedding" instance in ProviderManager + * 3. Defaults (baseUrl: http://localhost:11434, model: nomic-embed-text) + * + * The `endpointUrl` field in ModelProviderInstance stores the base URL. + */ + constructor(baseUrl?: string, modelName?: string) { + let url = baseUrl; + let model = modelName; + + if (!url || !model) { + const active = ProviderManager.getActive("embedding"); + if ( + active && + active.providerName === OllamaEmbeddingProvider.providerId + ) { + if (!url) { + url = active.endpointUrl; + } + if (!model) { + model = active.modelName; + } + } + } + + this.model = new OllamaEmbeddings({ + baseUrl: url || "http://localhost:11434", + model: model || "nomic-embed-text", + }); + } + + async embed(text: string): Promise { + return this.model.embedQuery(text); + } +} diff --git a/packages/memory/src/handoff.ts b/packages/memory/src/handoff.ts index 733ff26..4e61028 100644 --- a/packages/memory/src/handoff.ts +++ b/packages/memory/src/handoff.ts @@ -259,11 +259,7 @@ ${candidatesList} } const result = response.data; - const db = ( - this.bufferRepo as unknown as { - db: { transaction: (fn: () => void) => void }; - } - ).db; + const db = (this.bufferRepo as any).db; const ledgerEntries: LedgerEntry[] = []; for (const chunk of result.chunks) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 058feeb..4d9b0b3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -263,9 +263,12 @@ importers: "@langchain/google-genai": specifier: ^2.2.0 version: 2.2.0(@langchain/core@1.2.1(openai@6.45.0(ws@8.21.0)(zod@4.4.3))(ws@8.21.0)) + "@langchain/ollama": + specifier: ^0.2.3 + version: 0.2.4 "@langchain/openrouter": specifier: ^0.4.3 - version: 0.4.3(@langchain/core@1.2.1(openai@6.45.0(ws@8.21.0)(zod@4.4.3))(ws@8.21.0))(ws@8.21.0)(zod@4.4.3) + version: 0.4.3(ws@8.21.0)(zod@4.4.3) "@types/node": specifier: ^20.19.43 version: 20.19.43 @@ -2065,6 +2068,15 @@ packages: peerDependencies: "@langchain/core": ^1.2.0 + "@langchain/ollama@0.2.4": + resolution: + { + integrity: sha512-XThDrZurNPcUO6sasN13rkes1aGgu5gWAtDkkyIGT3ZeMOvrYgPKGft+bbhvsigTIH9C01TfPzrSp8LAmvHIjA==, + } + engines: { node: ">=18" } + peerDependencies: + "@langchain/core": ">=0.3.58 <0.4.0" + "@langchain/openai@1.5.3": resolution: { @@ -7468,6 +7480,12 @@ packages: integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==, } + ollama@0.5.18: + resolution: + { + integrity: sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg==, + } + on-finished@2.4.1: resolution: { @@ -9136,6 +9154,14 @@ packages: integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==, } + uuid@10.0.0: + resolution: + { + integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==, + } + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). + hasBin: true + uuid@14.0.1: resolution: { @@ -9282,6 +9308,12 @@ packages: integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==, } + whatwg-fetch@3.6.20: + resolution: + { + integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==, + } + which@2.0.2: resolution: { @@ -10356,9 +10388,13 @@ snapshots: "@google/generative-ai": 0.24.1 "@langchain/core": 1.2.1(openai@6.45.0(ws@8.21.0)(zod@4.4.3))(ws@8.21.0) - "@langchain/openai@1.5.3(@langchain/core@1.2.1(openai@6.45.0(ws@8.21.0)(zod@4.4.3))(ws@8.21.0))(ws@8.21.0)": + "@langchain/ollama@0.2.4": + dependencies: + ollama: 0.5.18 + uuid: 10.0.0 + + "@langchain/openai@1.5.3(ws@8.21.0)": dependencies: - "@langchain/core": 1.2.1(openai@6.45.0(ws@8.21.0)(zod@4.4.3))(ws@8.21.0) js-tiktoken: 1.0.21 openai: 6.45.0(ws@8.21.0)(zod@4.4.3) zod: 4.4.3 @@ -10368,10 +10404,9 @@ snapshots: - "@smithy/signature-v4" - ws - "@langchain/openrouter@0.4.3(@langchain/core@1.2.1(openai@6.45.0(ws@8.21.0)(zod@4.4.3))(ws@8.21.0))(ws@8.21.0)(zod@4.4.3)": + "@langchain/openrouter@0.4.3(ws@8.21.0)(zod@4.4.3)": dependencies: - "@langchain/core": 1.2.1(openai@6.45.0(ws@8.21.0)(zod@4.4.3))(ws@8.21.0) - "@langchain/openai": 1.5.3(@langchain/core@1.2.1(openai@6.45.0(ws@8.21.0)(zod@4.4.3))(ws@8.21.0))(ws@8.21.0) + "@langchain/openai": 1.5.3(ws@8.21.0) eventsource-parser: 3.1.0 openai: 6.45.0(ws@8.21.0)(zod@4.4.3) transitivePeerDependencies: @@ -14096,6 +14131,10 @@ snapshots: ohash@2.0.11: {} + ollama@0.5.18: + dependencies: + whatwg-fetch: 3.6.20 + on-finished@2.4.1: dependencies: ee-first: 1.1.1 @@ -15272,6 +15311,8 @@ snapshots: util-deprecate@1.0.2: {} + uuid@10.0.0: {} + uuid@14.0.1: {} validate-npm-package-name@7.0.2: {} @@ -15352,6 +15393,8 @@ snapshots: web-namespaces@2.0.1: {} + whatwg-fetch@3.6.20: {} + which@2.0.2: dependencies: isexe: 2.0.0