sortedcord/omnia-consolidated
1
0
Fork 0
mirror of https://github.com/sortedcord/omnia.git synced 2026-07-22 12:02:49 +05:30
Code Issues Packages Projects Releases Wiki Activity

feat(llm): Added ollama provider

Browse Source
This commit is contained in:
Aditya Gupta
2026-07-16 13:10:59 +05:30
parent ccafe56dfe
commit 9356e1f7d0
10 changed files with 311 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/llm/src/index.ts
View File

@@ -2,5 +2,6 @@ export * from "./llm.js";
export * from "./config.js";
export * from "./providers/google-genai.js";
export * from "./providers/mock.js";
export * from "./providers/ollama.js";
export * from "./providers/openrouter.js";
export * from "./provider-manager.js";

9
packages/llm/src/llm.ts
View File

@@ -57,6 +57,7 @@ export interface ModelProviderInstance {
modelName?: string;
type: "generative" | "embedding";
maxContext?: number;
endpointUrl?: string;
}
export interface ModelProviderMeta {
@@ -83,6 +84,14 @@ export const AVAILABLE_PROVIDERS: ModelProviderMeta[] = [
defaultModel: "google/gemini-2.5-flash",
defaultEmbeddingModel: "openai/text-embedding-3-small",
},
{
id: "ollama",
displayName: "Ollama",
description:
"Local model runner — no API key required, uses the Ollama server base URL instead",
defaultModel: "llama3.1",
defaultEmbeddingModel: "nomic-embed-text",
},
{
id: "mock",
displayName: "Mock LLM Provider",

30
packages/llm/src/provider-manager.ts
View File

@@ -82,6 +82,14 @@ function getSettingsDb() {
// ignore
}
try {
db.prepare(
`ALTER TABLE provider_instances ADD COLUMN endpointUrl TEXT`,
).run();
} catch {
// ignore
}
// Auto-bootstrap environment variables if DB contains 0 instances
try {
if (!hasBootstrapped) {
@@ -172,6 +180,7 @@ export class ProviderManager {
modelName?: string;
type: string;
maxContext?: number;
endpointUrl?: string;
}[];
return rows.map((r) => ({
id: r.id,
@@ -187,6 +196,7 @@ export class ProviderManager {
: r.type === "embedding"
? 0
: 32768,
endpointUrl: r.endpointUrl || undefined,
}));
} finally {
db.close();
@@ -200,6 +210,7 @@ export class ProviderManager {
modelName?: string,
type: "generative" | "embedding" = "generative",
maxContext?: number,
endpointUrl?: string,
): ModelProviderInstance {
const db = getSettingsDb();
try {
@@ -220,8 +231,8 @@ export class ProviderManager {
db.prepare(
`
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext, endpointUrl)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(
id,
@@ -232,6 +243,7 @@ export class ProviderManager {
modelName || null,
type,
actualMaxContext,
endpointUrl || null,
);
return {
@@ -243,6 +255,7 @@ export class ProviderManager {
modelName,
type,
maxContext: actualMaxContext,
endpointUrl,
};
} finally {
db.close();
@@ -299,6 +312,7 @@ export class ProviderManager {
modelName?: string,
type: "generative" | "embedding" = "generative",
maxContext?: number,
endpointUrl?: string,
): void {
const db = getSettingsDb();
try {
@@ -312,7 +326,7 @@ export class ProviderManager {
db.prepare(
`
UPDATE provider_instances
SET name = ?, providerName = ?, apiKey = ?, modelName = ?, type = ?, maxContext = ?
SET name = ?, providerName = ?, apiKey = ?, modelName = ?, type = ?, maxContext = ?, endpointUrl = ?
WHERE id = ?
`,
).run(
@@ -322,13 +336,14 @@ export class ProviderManager {
modelName || null,
type,
actualMaxContext,
endpointUrl || null,
id,
);
} else {
db.prepare(
`
UPDATE provider_instances
SET name = ?, providerName = ?, modelName = ?, type = ?, maxContext = ?
SET name = ?, providerName = ?, modelName = ?, type = ?, maxContext = ?, endpointUrl = ?
WHERE id = ?
`,
).run(
@@ -337,6 +352,7 @@ export class ProviderManager {
modelName || null,
type,
actualMaxContext,
endpointUrl || null,
id,
);
}
@@ -364,6 +380,7 @@ export class ProviderManager {
modelName?: string;
type: string;
maxContext?: number;
endpointUrl?: string;
}
| undefined;
@@ -447,6 +464,7 @@ export class ProviderManager {
modelName?: string;
type: string;
maxContext?: number;
endpointUrl?: string;
}
| undefined;
@@ -466,6 +484,7 @@ export class ProviderManager {
: retryRow.type === "embedding"
? 0
: 32768,
endpointUrl: retryRow.endpointUrl || undefined,
};
}
}
@@ -483,6 +502,7 @@ export class ProviderManager {
modelName?: string;
type: string;
maxContext?: number;
endpointUrl?: string;
}
| undefined;
if (firstRow) {
@@ -503,6 +523,7 @@ export class ProviderManager {
: firstRow.type === "embedding"
? 0
: 32768,
endpointUrl: firstRow.endpointUrl || undefined,
};
}
return null;
@@ -522,6 +543,7 @@ export class ProviderManager {
: row.type === "embedding"
? 0
: 32768,
endpointUrl: row.endpointUrl || undefined,
};
} catch {
const googleKey = process.env.GOOGLE_API_KEY;

162
packages/llm/src/providers/ollama.ts Normal file
View File

@@ -0,0 +1,162 @@
import { z } from "zod";
import { ChatOllama, OllamaEmbeddings } from "@langchain/ollama";
import {
ILLMProvider,
LLMRequest,
LLMResponse,
LLMCallRecord,
IEmbeddingProvider,
} from "../llm.js";
import { ProviderManager } from "../provider-manager.js";
export class OllamaProvider implements ILLMProvider {
static readonly providerId = "ollama";
static readonly displayName = "Ollama";
static readonly description =
"Local model runner supporting open-source LLMs via the Ollama server";
static readonly defaultModel = "llama3.1";
providerName = "Ollama";
private model: ChatOllama;
private modelNameUsed: string;
private providerInstanceName?: string;
private maxContextUsed?: number;
lastCalls: LLMCallRecord[] = [];
/**
* Creates an OllamaProvider.
*
* Resolution order for configuration:
* 1. Explicit constructor arguments
* 2. Active "generative" instance in ProviderManager whose providerName === "ollama"
* 3. Defaults (baseUrl: http://localhost:11434, model: llama3.1)
*
* No API key is required for Ollama. The `endpointUrl` in
* ModelProviderInstance stores the Ollama server base URL
* (e.g. "http://localhost:11434").
*/
constructor(
baseUrl?: string,
modelName?: string,
providerInstanceName?: string,
maxContext?: number,
) {
let url = baseUrl;
let model = modelName;
this.providerInstanceName = providerInstanceName;
this.maxContextUsed = maxContext;
if (!url || !model) {
const active = ProviderManager.getActive("generative");
if (active && active.providerName === OllamaProvider.providerId) {
if (!url) {
url = active.endpointUrl;
}
if (!model) {
model = active.modelName;
}
if (!this.providerInstanceName) {
this.providerInstanceName = active.name;
}
if (this.maxContextUsed === undefined) {
this.maxContextUsed = active.maxContext;
}
}
}
this.modelNameUsed = model || OllamaProvider.defaultModel;
this.model = new ChatOllama({
baseUrl: url || "http://localhost:11434",
model: this.modelNameUsed,
});
}
async generateStructuredResponse<T extends z.ZodTypeAny>(
request: LLMRequest<T>,
): Promise<LLMResponse<z.infer<T>>> {
const structuredModel = this.model.withStructuredOutput(request.schema, {
includeRaw: true,
});
const result = (await structuredModel.invoke([
{ role: "system", content: request.systemPrompt },
{ role: "user", content: request.userContext },
])) as unknown as {
parsed?: z.infer<T>;
raw?: {
usage_metadata?: {
input_tokens?: number;
output_tokens?: number;
total_tokens?: number;
};
};
};
const parsed = result?.parsed;
const raw = result?.raw;
const usage = {
inputTokens: raw?.usage_metadata?.input_tokens || 0,
outputTokens: raw?.usage_metadata?.output_tokens || 0,
totalTokens: raw?.usage_metadata?.total_tokens || 0,
modelName: this.modelNameUsed,
providerInstanceName: this.providerInstanceName || "Default",
maxContext:
this.maxContextUsed !== undefined ? this.maxContextUsed : 32768,
};
this.lastCalls.push({
systemPrompt: request.systemPrompt,
userContext: request.userContext,
usage,
});
return { success: true, data: parsed, usage };
}
}
export class OllamaEmbeddingProvider implements IEmbeddingProvider {
static readonly providerId = "ollama";
static readonly displayName = "Ollama Embeddings";
providerName = "Ollama";
private model: OllamaEmbeddings;
/**
* Creates an OllamaEmbeddingProvider.
*
* Resolution order:
* 1. Explicit constructor arguments
* 2. Active "embedding" instance in ProviderManager
* 3. Defaults (baseUrl: http://localhost:11434, model: nomic-embed-text)
*
* The `endpointUrl` field in ModelProviderInstance stores the base URL.
*/
constructor(baseUrl?: string, modelName?: string) {
let url = baseUrl;
let model = modelName;
if (!url || !model) {
const active = ProviderManager.getActive("embedding");
if (
active &&
active.providerName === OllamaEmbeddingProvider.providerId
) {
if (!url) {
url = active.endpointUrl;
}
if (!model) {
model = active.modelName;
}
}
}
this.model = new OllamaEmbeddings({
baseUrl: url || "http://localhost:11434",
model: model || "nomic-embed-text",
});
}
async embed(text: string): Promise<number[]> {
return this.model.embedQuery(text);
}
}