mirror of
https://github.com/sortedcord/omnia.git
synced 2026-07-23 04:22:49 +05:30
refactor!(llm): implement new model registration system
This commit is contained in:
112
packages/llm/src/base-provider.ts
Normal file
112
packages/llm/src/base-provider.ts
Normal file
@@ -0,0 +1,112 @@
|
||||
import { z } from "zod";
|
||||
import type {
|
||||
ILLMProvider,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
LLMCallRecord,
|
||||
} from "./llm.js";
|
||||
import { ProviderManager } from "./provider-manager.js";
|
||||
import { getLlmConfig } from "./config.js";
|
||||
|
||||
export interface ResolvedCredentials {
|
||||
key: string | undefined;
|
||||
model: string | undefined;
|
||||
providerInstanceName: string | undefined;
|
||||
maxContext: number | undefined;
|
||||
}
|
||||
|
||||
export function resolveCredentials(opts: {
|
||||
explicitKey?: string;
|
||||
explicitModel?: string;
|
||||
explicitProviderInstanceName?: string;
|
||||
explicitMaxContext?: number;
|
||||
providerId: string;
|
||||
envVarName: string;
|
||||
type: "generative" | "embedding";
|
||||
}): ResolvedCredentials {
|
||||
let key = opts.explicitKey;
|
||||
let model = opts.explicitModel;
|
||||
let providerInstanceName = opts.explicitProviderInstanceName;
|
||||
let maxContext = opts.explicitMaxContext;
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive(opts.type);
|
||||
if (active && active.providerName === opts.providerId) {
|
||||
key = active.apiKey;
|
||||
if (!model) model = active.modelName;
|
||||
if (!providerInstanceName) providerInstanceName = active.name;
|
||||
if (maxContext === undefined) maxContext = active.maxContext;
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
const cfg = getLlmConfig();
|
||||
key = cfg[opts.envVarName];
|
||||
if (!providerInstanceName && key) {
|
||||
providerInstanceName = "Environment Variable";
|
||||
}
|
||||
}
|
||||
|
||||
return { key, model, providerInstanceName, maxContext };
|
||||
}
|
||||
|
||||
export abstract class BaseLLMProvider implements ILLMProvider {
|
||||
abstract providerName: string;
|
||||
protected abstract readonly model: unknown;
|
||||
protected abstract modelNameUsed: string;
|
||||
protected abstract providerInstanceName?: string;
|
||||
protected abstract maxContextUsed?: number;
|
||||
protected abstract defaultMaxContext: number;
|
||||
lastCalls: LLMCallRecord[] = [];
|
||||
|
||||
async generateStructuredResponse<T extends z.ZodTypeAny>(
|
||||
request: LLMRequest<T>,
|
||||
): Promise<LLMResponse<z.infer<T>>> {
|
||||
const structuredModel = (
|
||||
this.model as {
|
||||
withStructuredOutput(
|
||||
s: z.ZodTypeAny,
|
||||
o: { includeRaw: true },
|
||||
): {
|
||||
invoke(m: unknown): Promise<unknown>;
|
||||
};
|
||||
}
|
||||
).withStructuredOutput(request.schema, { includeRaw: true });
|
||||
const result = (await structuredModel.invoke([
|
||||
{ role: "system", content: request.systemPrompt },
|
||||
{ role: "user", content: request.userContext },
|
||||
])) as unknown as {
|
||||
parsed?: z.infer<T>;
|
||||
raw?: {
|
||||
usage_metadata?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const parsed = result?.parsed;
|
||||
const raw = result?.raw;
|
||||
|
||||
const usage = {
|
||||
inputTokens: raw?.usage_metadata?.input_tokens || 0,
|
||||
outputTokens: raw?.usage_metadata?.output_tokens || 0,
|
||||
totalTokens: raw?.usage_metadata?.total_tokens || 0,
|
||||
modelName: this.modelNameUsed,
|
||||
providerInstanceName: this.providerInstanceName || "Default",
|
||||
maxContext:
|
||||
this.maxContextUsed !== undefined
|
||||
? this.maxContextUsed
|
||||
: this.defaultMaxContext,
|
||||
};
|
||||
|
||||
this.lastCalls.push({
|
||||
systemPrompt: request.systemPrompt,
|
||||
userContext: request.userContext,
|
||||
usage,
|
||||
});
|
||||
|
||||
return { success: true, data: parsed, usage };
|
||||
}
|
||||
}
|
||||
98
packages/llm/src/bootstrap.ts
Normal file
98
packages/llm/src/bootstrap.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import type BetterSqlite3 from "better-sqlite3";
|
||||
import { ProviderRegistry } from "./registry.js";
|
||||
|
||||
export function seedFromEnvVars(db: BetterSqlite3.Database): boolean {
|
||||
const totalCount = db
|
||||
.prepare("SELECT COUNT(*) as count FROM provider_instances")
|
||||
.get() as { count: number };
|
||||
if (totalCount.count > 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let hasActiveGenerative = false;
|
||||
let hasActiveEmbedding = false;
|
||||
|
||||
const insertStmt = db.prepare(
|
||||
`INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
|
||||
const insertMany = db.transaction(
|
||||
(
|
||||
entries: {
|
||||
id: string;
|
||||
name: string;
|
||||
providerId: string;
|
||||
key: string;
|
||||
isActive: number;
|
||||
modelName: string;
|
||||
type: "generative" | "embedding";
|
||||
maxContext: number;
|
||||
}[],
|
||||
) => {
|
||||
for (const e of entries) {
|
||||
insertStmt.run(
|
||||
e.id,
|
||||
e.name,
|
||||
e.providerId,
|
||||
e.key,
|
||||
e.isActive,
|
||||
e.modelName,
|
||||
e.type,
|
||||
e.maxContext,
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
const entries: {
|
||||
id: string;
|
||||
name: string;
|
||||
providerId: string;
|
||||
key: string;
|
||||
isActive: number;
|
||||
modelName: string;
|
||||
type: "generative" | "embedding";
|
||||
maxContext: number;
|
||||
}[] = [];
|
||||
|
||||
for (const def of ProviderRegistry.all()) {
|
||||
if (!def.envVar) continue;
|
||||
const key = process.env[def.envVar]?.trim();
|
||||
if (!key) continue;
|
||||
|
||||
if (def.capabilities.generative) {
|
||||
entries.push({
|
||||
id: `provider-default-${def.id}`,
|
||||
name: `${def.displayName} (Env)`,
|
||||
providerId: def.id,
|
||||
key,
|
||||
isActive: hasActiveGenerative ? 0 : 1,
|
||||
modelName: def.defaultModel,
|
||||
type: "generative",
|
||||
maxContext: def.defaultMaxContext,
|
||||
});
|
||||
if (!hasActiveGenerative) hasActiveGenerative = true;
|
||||
}
|
||||
|
||||
if (def.capabilities.embedding) {
|
||||
const embedModel = def.defaultEmbeddingModel || "";
|
||||
entries.push({
|
||||
id: `provider-default-${def.id}-embed`,
|
||||
name: `${def.displayName} Embed (Env)`,
|
||||
providerId: def.id,
|
||||
key,
|
||||
isActive: hasActiveEmbedding ? 0 : 1,
|
||||
modelName: embedModel,
|
||||
type: "embedding",
|
||||
maxContext: 0,
|
||||
});
|
||||
if (!hasActiveEmbedding) hasActiveEmbedding = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (entries.length > 0) {
|
||||
insertMany(entries);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -1,12 +1,25 @@
|
||||
import { z } from "zod";
|
||||
import { ProviderRegistry } from "./registry.js";
|
||||
|
||||
const LLMConfigSchema = z.object({
|
||||
GOOGLE_API_KEY: z.string().optional(),
|
||||
OPENROUTER_API_KEY: z.string().optional(),
|
||||
ANTHROPIC_API_KEY: z.string().optional(),
|
||||
OPENAI_API_KEY: z.string().optional(),
|
||||
GROQ_API_KEY: z.string().optional(),
|
||||
DEEPSEEK_API_KEY: z.string().optional(),
|
||||
});
|
||||
let _config: Record<string, string | undefined> | null = null;
|
||||
|
||||
export const llmConfig = LLMConfigSchema.parse(process.env);
|
||||
export function getLlmConfig(): Record<string, string | undefined> {
|
||||
if (!_config) {
|
||||
const envVars: string[] = [];
|
||||
for (const def of ProviderRegistry.all()) {
|
||||
if (def.envVar && !envVars.includes(def.envVar)) {
|
||||
envVars.push(def.envVar);
|
||||
}
|
||||
}
|
||||
const shape: Record<string, z.ZodOptional<z.ZodString>> = {};
|
||||
for (const key of envVars) {
|
||||
shape[key] = z.string().optional();
|
||||
}
|
||||
_config = z.object(shape).parse(process.env);
|
||||
}
|
||||
return _config;
|
||||
}
|
||||
|
||||
export function resetLlmConfig(): void {
|
||||
_config = null;
|
||||
}
|
||||
|
||||
75
packages/llm/src/db.ts
Normal file
75
packages/llm/src/db.ts
Normal file
@@ -0,0 +1,75 @@
|
||||
import Database from "better-sqlite3";
|
||||
import type BetterSqlite3 from "better-sqlite3";
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
|
||||
let _db: BetterSqlite3.Database | null = null;
|
||||
let _dbPathOverride: string | null = null;
|
||||
|
||||
export function setDbPath(p: string | null) {
|
||||
if (_dbPathOverride !== p) {
|
||||
_db?.close();
|
||||
_db = null;
|
||||
_dbPathOverride = p;
|
||||
}
|
||||
}
|
||||
|
||||
function findDbPath(): string {
|
||||
let current = process.cwd();
|
||||
while (current !== "/" && current !== path.parse(current).root) {
|
||||
if (fs.existsSync(path.join(current, "pnpm-workspace.yaml"))) {
|
||||
const dbDir = path.resolve(current, "data");
|
||||
if (!fs.existsSync(dbDir)) {
|
||||
fs.mkdirSync(dbDir, { recursive: true });
|
||||
}
|
||||
return path.join(dbDir, "settings.db");
|
||||
}
|
||||
current = path.dirname(current);
|
||||
}
|
||||
const dbDir = path.resolve(process.cwd(), "data");
|
||||
if (!fs.existsSync(dbDir)) {
|
||||
fs.mkdirSync(dbDir, { recursive: true });
|
||||
}
|
||||
return path.join(dbDir, "settings.db");
|
||||
}
|
||||
|
||||
function runMigrations(db: BetterSqlite3.Database): void {
|
||||
const version = db.pragma("user_version", { simple: true }) as number;
|
||||
|
||||
if (version < 1) {
|
||||
db.prepare(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS provider_instances (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
providerName TEXT NOT NULL,
|
||||
apiKey TEXT NOT NULL,
|
||||
isActive INTEGER NOT NULL DEFAULT 0,
|
||||
modelName TEXT,
|
||||
type TEXT NOT NULL DEFAULT 'generative',
|
||||
maxContext INTEGER,
|
||||
endpointUrl TEXT
|
||||
)
|
||||
`,
|
||||
).run();
|
||||
db.pragma("user_version = 1");
|
||||
}
|
||||
|
||||
db.prepare(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS provider_mappings (
|
||||
task TEXT PRIMARY KEY,
|
||||
providerInstanceId TEXT NOT NULL
|
||||
)
|
||||
`,
|
||||
).run();
|
||||
}
|
||||
|
||||
export function getDb(): BetterSqlite3.Database {
|
||||
if (!_db) {
|
||||
const dbPath = _dbPathOverride ?? findDbPath();
|
||||
_db = new Database(dbPath);
|
||||
runMigrations(_db);
|
||||
}
|
||||
return _db;
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
export * from "./llm.js";
|
||||
export * from "./config.js";
|
||||
export * from "./registry.js";
|
||||
export * from "./provider-factory.js";
|
||||
export * from "./model-lister.js";
|
||||
export * from "./provider-manager.js";
|
||||
export * from "./providers/google-genai.js";
|
||||
export * from "./providers/mock.js";
|
||||
export * from "./providers/ollama.js";
|
||||
@@ -9,4 +12,3 @@ export * from "./providers/anthropic.js";
|
||||
export * from "./providers/openai.js";
|
||||
export * from "./providers/groq.js";
|
||||
export * from "./providers/deepseek.js";
|
||||
export * from "./provider-manager.js";
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { z } from "zod";
|
||||
import { ProviderRegistry } from "./registry.js";
|
||||
|
||||
export interface LLMRequest<T extends z.ZodTypeAny> {
|
||||
systemPrompt: string;
|
||||
@@ -68,63 +69,21 @@ export interface ModelProviderMeta {
|
||||
defaultEmbeddingModel: string;
|
||||
}
|
||||
|
||||
export const AVAILABLE_PROVIDERS: ModelProviderMeta[] = [
|
||||
{
|
||||
id: "google-genai",
|
||||
displayName: "Google Gemini",
|
||||
description: "Official Gemini integration using Google Gen AI SDK",
|
||||
defaultModel: "gemini-2.5-flash",
|
||||
defaultEmbeddingModel: "gemini-embedding-001",
|
||||
export function getAvailableProviders(): ModelProviderMeta[] {
|
||||
return ProviderRegistry.all().map((def) => ({
|
||||
id: def.id,
|
||||
displayName: def.displayName,
|
||||
description: def.description,
|
||||
defaultModel: def.defaultModel,
|
||||
defaultEmbeddingModel: def.defaultEmbeddingModel || "",
|
||||
}));
|
||||
}
|
||||
|
||||
export const AVAILABLE_PROVIDERS = {
|
||||
get count(): number {
|
||||
return getAvailableProviders().length;
|
||||
},
|
||||
{
|
||||
id: "openai",
|
||||
displayName: "OpenAI",
|
||||
description: "Official OpenAI integration using @langchain/openai SDK",
|
||||
defaultModel: "gpt-4o-mini",
|
||||
defaultEmbeddingModel: "text-embedding-3-small",
|
||||
toArray(): ModelProviderMeta[] {
|
||||
return getAvailableProviders();
|
||||
},
|
||||
{
|
||||
id: "anthropic",
|
||||
displayName: "Anthropic Claude",
|
||||
description: "Official Claude integration using @langchain/anthropic SDK",
|
||||
defaultModel: "claude-3-5-sonnet-latest",
|
||||
defaultEmbeddingModel: "",
|
||||
},
|
||||
{
|
||||
id: "groq",
|
||||
displayName: "Groq",
|
||||
description: "Official Groq integration using @langchain/groq SDK",
|
||||
defaultModel: "llama-3.3-70b-versatile",
|
||||
defaultEmbeddingModel: "",
|
||||
},
|
||||
{
|
||||
id: "deepseek",
|
||||
displayName: "DeepSeek",
|
||||
description: "Official DeepSeek integration using @langchain/deepseek SDK",
|
||||
defaultModel: "deepseek-chat",
|
||||
defaultEmbeddingModel: "",
|
||||
},
|
||||
{
|
||||
id: "openrouter",
|
||||
displayName: "OpenRouter",
|
||||
description:
|
||||
"Multi-model router supporting Anthropic, OpenAI, DeepSeek, and local models",
|
||||
defaultModel: "google/gemini-2.5-flash",
|
||||
defaultEmbeddingModel: "openai/text-embedding-3-small",
|
||||
},
|
||||
{
|
||||
id: "ollama",
|
||||
displayName: "Ollama",
|
||||
description:
|
||||
"Local model runner — no API key required, uses the Ollama server base URL instead",
|
||||
defaultModel: "llama3.1",
|
||||
defaultEmbeddingModel: "nomic-embed-text",
|
||||
},
|
||||
{
|
||||
id: "mock",
|
||||
displayName: "Mock LLM Provider",
|
||||
description: "Stateless mock provider for testing and offline development",
|
||||
defaultModel: "mock",
|
||||
defaultEmbeddingModel: "mock-embeddings",
|
||||
},
|
||||
];
|
||||
};
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
* Results are cached in-memory with a 5-minute TTL to avoid repeated calls.
|
||||
*/
|
||||
|
||||
import { ProviderRegistry } from "./registry.js";
|
||||
|
||||
export interface ModelInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
@@ -14,10 +16,9 @@ interface CacheEntry {
|
||||
fetchedAt: number;
|
||||
}
|
||||
|
||||
const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const FETCH_TIMEOUT_MS = 10_000; // 10 seconds
|
||||
const CACHE_TTL_MS = 5 * 60 * 1000;
|
||||
const FETCH_TIMEOUT_MS = 10_000;
|
||||
|
||||
// Cache key is "providerName:apiKey-or-endpoint" (we don't hash since it's in-process)
|
||||
const modelCache = new Map<string, CacheEntry>();
|
||||
|
||||
function cacheKey(
|
||||
@@ -28,7 +29,7 @@ function cacheKey(
|
||||
return `${providerName}:${endpointUrl || apiKey}`;
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(
|
||||
export async function fetchWithTimeout(
|
||||
url: string,
|
||||
init?: RequestInit,
|
||||
): Promise<Response> {
|
||||
@@ -41,41 +42,7 @@ async function fetchWithTimeout(
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchGeminiModels(apiKey: string): Promise<ModelInfo[]> {
|
||||
const models: ModelInfo[] = [];
|
||||
let pageToken: string | undefined;
|
||||
|
||||
do {
|
||||
const url = new URL(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models",
|
||||
);
|
||||
url.searchParams.set("key", apiKey);
|
||||
url.searchParams.set("pageSize", "100");
|
||||
if (pageToken) {
|
||||
url.searchParams.set("pageToken", pageToken);
|
||||
}
|
||||
|
||||
const res = await fetchWithTimeout(url.toString());
|
||||
if (!res.ok) return models;
|
||||
|
||||
const json = (await res.json()) as {
|
||||
models?: { name: string; displayName?: string }[];
|
||||
nextPageToken?: string;
|
||||
};
|
||||
|
||||
for (const m of json.models ?? []) {
|
||||
// m.name is like "models/gemini-2.5-flash"; strip "models/" prefix
|
||||
const id = m.name.replace(/^models\//, "");
|
||||
models.push({ id, name: m.displayName || id });
|
||||
}
|
||||
|
||||
pageToken = json.nextPageToken;
|
||||
} while (pageToken);
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
async function fetchOpenAICompatibleModels(
|
||||
export async function fetchOpenAICompatibleModels(
|
||||
baseUrl: string,
|
||||
apiKey: string,
|
||||
): Promise<ModelInfo[]> {
|
||||
@@ -98,146 +65,25 @@ async function fetchOpenAICompatibleModels(
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchAnthropicModels(apiKey: string): Promise<ModelInfo[]> {
|
||||
const models: ModelInfo[] = [];
|
||||
let afterId: string | undefined;
|
||||
|
||||
do {
|
||||
const url = new URL("https://api.anthropic.com/v1/models");
|
||||
url.searchParams.set("limit", "1000");
|
||||
if (afterId) {
|
||||
url.searchParams.set("after_id", afterId);
|
||||
}
|
||||
|
||||
const res = await fetchWithTimeout(url.toString(), {
|
||||
headers: {
|
||||
"x-api-key": apiKey,
|
||||
"anthropic-version": "2023-06-01",
|
||||
Accept: "application/json",
|
||||
},
|
||||
});
|
||||
if (!res.ok) return models;
|
||||
|
||||
const json = (await res.json()) as {
|
||||
data?: { id: string; display_name?: string }[];
|
||||
has_more?: boolean;
|
||||
last_id?: string;
|
||||
};
|
||||
|
||||
for (const m of json.data ?? []) {
|
||||
models.push({ id: m.id, name: m.display_name || m.id });
|
||||
}
|
||||
|
||||
afterId = json.has_more ? json.last_id : undefined;
|
||||
} while (afterId);
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
async function fetchOllamaModels(endpointUrl: string): Promise<ModelInfo[]> {
|
||||
const base = endpointUrl.replace(/\/$/, "");
|
||||
const res = await fetchWithTimeout(`${base}/api/tags`);
|
||||
if (!res.ok) return [];
|
||||
|
||||
const json = (await res.json()) as {
|
||||
models?: { name: string; model?: string }[];
|
||||
};
|
||||
|
||||
return (json.models ?? []).map((m) => ({
|
||||
id: m.name,
|
||||
name: m.name,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchOpenRouterModels(apiKey: string): Promise<ModelInfo[]> {
|
||||
const res = await fetchWithTimeout(
|
||||
"https://openrouter.ai/api/v1/models",
|
||||
apiKey
|
||||
? {
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
Accept: "application/json",
|
||||
},
|
||||
}
|
||||
: { headers: { Accept: "application/json" } },
|
||||
);
|
||||
if (!res.ok) return [];
|
||||
|
||||
const json = (await res.json()) as {
|
||||
data?: { id: string; name?: string; owned_by?: string }[];
|
||||
};
|
||||
|
||||
return (json.data ?? []).map((m) => ({
|
||||
id: m.id,
|
||||
name: m.name || m.id,
|
||||
ownedBy: m.owned_by,
|
||||
}));
|
||||
}
|
||||
|
||||
export class ModelLister {
|
||||
/**
|
||||
* List available models for a given provider. Results are cached for 5 minutes.
|
||||
*
|
||||
* @param providerName The provider ID (e.g. "openai", "google-genai")
|
||||
* @param apiKey The API key for the provider (or "none" for Ollama)
|
||||
* @param endpointUrl The endpoint URL (required for Ollama, ignored otherwise)
|
||||
* @returns Array of ModelInfo objects, or [] on any error
|
||||
*/
|
||||
static async listModels(
|
||||
providerName: string,
|
||||
apiKey: string,
|
||||
endpointUrl?: string,
|
||||
): Promise<ModelInfo[]> {
|
||||
if (providerName === "mock") {
|
||||
return [{ id: "mock", name: "Mock Model" }];
|
||||
}
|
||||
|
||||
const key = cacheKey(providerName, apiKey, endpointUrl);
|
||||
const cached = modelCache.get(key);
|
||||
if (cached && Date.now() - cached.fetchedAt < CACHE_TTL_MS) {
|
||||
return cached.models;
|
||||
}
|
||||
|
||||
const def = ProviderRegistry.get(providerName);
|
||||
let models: ModelInfo[] = [];
|
||||
try {
|
||||
switch (providerName) {
|
||||
case "google-genai":
|
||||
models = await fetchGeminiModels(apiKey);
|
||||
break;
|
||||
case "openai":
|
||||
models = await fetchOpenAICompatibleModels(
|
||||
"https://api.openai.com/v1",
|
||||
apiKey,
|
||||
);
|
||||
break;
|
||||
case "anthropic":
|
||||
models = await fetchAnthropicModels(apiKey);
|
||||
break;
|
||||
case "groq":
|
||||
models = await fetchOpenAICompatibleModels(
|
||||
"https://api.groq.com/openai/v1",
|
||||
apiKey,
|
||||
);
|
||||
break;
|
||||
case "deepseek":
|
||||
models = await fetchOpenAICompatibleModels(
|
||||
"https://api.deepseek.com",
|
||||
apiKey,
|
||||
);
|
||||
break;
|
||||
case "ollama":
|
||||
models = await fetchOllamaModels(
|
||||
endpointUrl || "http://localhost:11434",
|
||||
);
|
||||
break;
|
||||
case "openrouter":
|
||||
models = await fetchOpenRouterModels(apiKey);
|
||||
break;
|
||||
default:
|
||||
models = [];
|
||||
if (def?.listModels) {
|
||||
models = await def.listModels(apiKey, endpointUrl);
|
||||
}
|
||||
} catch {
|
||||
// Network error, invalid key, timeout — return empty array for graceful degradation
|
||||
models = [];
|
||||
}
|
||||
|
||||
@@ -245,7 +91,6 @@ export class ModelLister {
|
||||
return models;
|
||||
}
|
||||
|
||||
/** Invalidate the cache entry for a specific provider+key combination. */
|
||||
static invalidateCache(
|
||||
providerName: string,
|
||||
apiKey: string,
|
||||
@@ -254,7 +99,6 @@ export class ModelLister {
|
||||
modelCache.delete(cacheKey(providerName, apiKey, endpointUrl));
|
||||
}
|
||||
|
||||
/** Clear the entire model cache. */
|
||||
static clearCache(): void {
|
||||
modelCache.clear();
|
||||
}
|
||||
|
||||
21
packages/llm/src/provider-factory.ts
Normal file
21
packages/llm/src/provider-factory.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import type {
|
||||
ILLMProvider,
|
||||
IEmbeddingProvider,
|
||||
ModelProviderInstance,
|
||||
} from "./llm.js";
|
||||
import { MockLLMProvider, MockEmbeddingProvider } from "./providers/mock.js";
|
||||
import { ProviderRegistry } from "./registry.js";
|
||||
|
||||
export function buildLLMProvider(inst: ModelProviderInstance): ILLMProvider {
|
||||
const def = ProviderRegistry.get(inst.providerName);
|
||||
return def?.generativeCreate?.(inst) ?? new MockLLMProvider([]);
|
||||
}
|
||||
|
||||
export function buildEmbeddingProvider(
|
||||
inst: ModelProviderInstance,
|
||||
): IEmbeddingProvider {
|
||||
const def = ProviderRegistry.get(inst.providerName);
|
||||
return (
|
||||
def?.embeddingCreate?.(inst) ?? new MockEmbeddingProvider(inst.modelName)
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,27 +1,86 @@
|
||||
import { z } from "zod";
|
||||
import { ChatAnthropic } from "@langchain/anthropic";
|
||||
import {
|
||||
ILLMProvider,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
LLMCallRecord,
|
||||
} from "../llm.js";
|
||||
import { llmConfig } from "../config.js";
|
||||
import { ProviderManager } from "../provider-manager.js";
|
||||
import { ILLMProvider } from "../llm.js";
|
||||
import type { ModelProviderInstance } from "../llm.js";
|
||||
import { BaseLLMProvider, resolveCredentials } from "../base-provider.js";
|
||||
import { registerProvider, registerGenerative } from "../registry.js";
|
||||
import { fetchWithTimeout, type ModelInfo } from "../model-lister.js";
|
||||
|
||||
export class AnthropicProvider implements ILLMProvider {
|
||||
static readonly providerId = "anthropic";
|
||||
static readonly displayName = "Anthropic Claude";
|
||||
static readonly description =
|
||||
"Official Claude integration using @langchain/anthropic SDK";
|
||||
static readonly defaultModel = "claude-3-5-sonnet-latest";
|
||||
async function fetchAnthropicModels(apiKey: string): Promise<ModelInfo[]> {
|
||||
const models: ModelInfo[] = [];
|
||||
let afterId: string | undefined;
|
||||
|
||||
do {
|
||||
const url = new URL("https://api.anthropic.com/v1/models");
|
||||
url.searchParams.set("limit", "1000");
|
||||
if (afterId) {
|
||||
url.searchParams.set("after_id", afterId);
|
||||
}
|
||||
|
||||
const res = await fetchWithTimeout(url.toString(), {
|
||||
headers: {
|
||||
"x-api-key": apiKey,
|
||||
"anthropic-version": "2023-06-01",
|
||||
Accept: "application/json",
|
||||
},
|
||||
});
|
||||
if (!res.ok) return models;
|
||||
|
||||
const json = (await res.json()) as {
|
||||
data?: { id: string; display_name?: string }[];
|
||||
has_more?: boolean;
|
||||
last_id?: string;
|
||||
};
|
||||
|
||||
for (const m of json.data ?? []) {
|
||||
models.push({ id: m.id, name: m.display_name || m.id });
|
||||
}
|
||||
|
||||
afterId = json.has_more ? json.last_id : undefined;
|
||||
} while (afterId);
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
export class AnthropicProvider extends BaseLLMProvider {
|
||||
static {
|
||||
registerProvider({
|
||||
id: "anthropic",
|
||||
displayName: "Anthropic Claude",
|
||||
description: "Official Claude integration using @langchain/anthropic SDK",
|
||||
envVar: "ANTHROPIC_API_KEY",
|
||||
capabilities: { generative: true, embedding: false },
|
||||
defaultModel: "claude-3-5-sonnet-latest",
|
||||
defaultMaxContext: 200000,
|
||||
fallbackPriority: 2,
|
||||
listModels: fetchAnthropicModels,
|
||||
});
|
||||
registerGenerative(
|
||||
"anthropic",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new AnthropicProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): ILLMProvider {
|
||||
return new AnthropicProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
);
|
||||
}
|
||||
|
||||
providerName = "Anthropic";
|
||||
private model: ChatAnthropic;
|
||||
private modelNameUsed: string;
|
||||
private providerInstanceName?: string;
|
||||
private maxContextUsed?: number;
|
||||
lastCalls: LLMCallRecord[] = [];
|
||||
protected readonly model: ChatAnthropic;
|
||||
protected modelNameUsed: string;
|
||||
protected providerInstanceName?: string;
|
||||
protected maxContextUsed?: number;
|
||||
protected defaultMaxContext = 200000;
|
||||
|
||||
constructor(
|
||||
apiKey?: string,
|
||||
@@ -29,86 +88,29 @@ export class AnthropicProvider implements ILLMProvider {
|
||||
providerInstanceName?: string,
|
||||
maxContext?: number,
|
||||
) {
|
||||
let key = apiKey;
|
||||
let model = modelName;
|
||||
this.providerInstanceName = providerInstanceName;
|
||||
this.maxContextUsed = maxContext;
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive("generative");
|
||||
if (active && active.providerName === AnthropicProvider.providerId) {
|
||||
key = active.apiKey;
|
||||
if (!model) {
|
||||
model = active.modelName;
|
||||
}
|
||||
if (!this.providerInstanceName) {
|
||||
this.providerInstanceName = active.name;
|
||||
}
|
||||
if (this.maxContextUsed === undefined) {
|
||||
this.maxContextUsed = active.maxContext;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
key = llmConfig.ANTHROPIC_API_KEY;
|
||||
if (!this.providerInstanceName && key) {
|
||||
this.providerInstanceName = "Environment Variable";
|
||||
}
|
||||
}
|
||||
|
||||
super();
|
||||
const {
|
||||
key,
|
||||
model,
|
||||
providerInstanceName: resolvedName,
|
||||
maxContext: resolvedMax,
|
||||
} = resolveCredentials({
|
||||
explicitKey: apiKey,
|
||||
explicitModel: modelName,
|
||||
explicitProviderInstanceName: providerInstanceName,
|
||||
explicitMaxContext: maxContext,
|
||||
providerId: "anthropic",
|
||||
envVarName: "ANTHROPIC_API_KEY",
|
||||
type: "generative",
|
||||
});
|
||||
if (!key) {
|
||||
throw new Error(
|
||||
"ANTHROPIC_API_KEY is required to initialize AnthropicProvider",
|
||||
);
|
||||
}
|
||||
|
||||
this.modelNameUsed = model || AnthropicProvider.defaultModel;
|
||||
this.model = new ChatAnthropic({
|
||||
apiKey: key,
|
||||
model: this.modelNameUsed,
|
||||
});
|
||||
}
|
||||
|
||||
async generateStructuredResponse<T extends z.ZodTypeAny>(
|
||||
request: LLMRequest<T>,
|
||||
): Promise<LLMResponse<z.infer<T>>> {
|
||||
const structuredModel = this.model.withStructuredOutput(request.schema, {
|
||||
includeRaw: true,
|
||||
});
|
||||
const result = (await structuredModel.invoke([
|
||||
{ role: "system", content: request.systemPrompt },
|
||||
{ role: "user", content: request.userContext },
|
||||
])) as unknown as {
|
||||
parsed?: z.infer<T>;
|
||||
raw?: {
|
||||
usage_metadata?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const parsed = result?.parsed;
|
||||
const raw = result?.raw;
|
||||
|
||||
const usage = {
|
||||
inputTokens: raw?.usage_metadata?.input_tokens || 0,
|
||||
outputTokens: raw?.usage_metadata?.output_tokens || 0,
|
||||
totalTokens: raw?.usage_metadata?.total_tokens || 0,
|
||||
modelName: this.modelNameUsed,
|
||||
providerInstanceName: this.providerInstanceName || "Default",
|
||||
maxContext:
|
||||
this.maxContextUsed !== undefined ? this.maxContextUsed : 200000,
|
||||
};
|
||||
|
||||
this.lastCalls.push({
|
||||
systemPrompt: request.systemPrompt,
|
||||
userContext: request.userContext,
|
||||
usage,
|
||||
});
|
||||
|
||||
return { success: true, data: parsed, usage };
|
||||
this.providerInstanceName = resolvedName;
|
||||
this.maxContextUsed = resolvedMax;
|
||||
this.modelNameUsed = model || "claude-3-5-sonnet-latest";
|
||||
this.model = new ChatAnthropic({ apiKey: key, model: this.modelNameUsed });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +1,52 @@
|
||||
import { z } from "zod";
|
||||
import { ChatDeepSeek } from "@langchain/deepseek";
|
||||
import {
|
||||
ILLMProvider,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
LLMCallRecord,
|
||||
} from "../llm.js";
|
||||
import { llmConfig } from "../config.js";
|
||||
import { ProviderManager } from "../provider-manager.js";
|
||||
import { ILLMProvider } from "../llm.js";
|
||||
import type { ModelProviderInstance } from "../llm.js";
|
||||
import { BaseLLMProvider, resolveCredentials } from "../base-provider.js";
|
||||
import { registerProvider, registerGenerative } from "../registry.js";
|
||||
import { fetchOpenAICompatibleModels } from "../model-lister.js";
|
||||
|
||||
export class DeepSeekProvider implements ILLMProvider {
|
||||
static readonly providerId = "deepseek";
|
||||
static readonly displayName = "DeepSeek";
|
||||
static readonly description =
|
||||
"Official DeepSeek integration using @langchain/deepseek SDK";
|
||||
static readonly defaultModel = "deepseek-chat";
|
||||
export class DeepSeekProvider extends BaseLLMProvider {
|
||||
static {
|
||||
registerProvider({
|
||||
id: "deepseek",
|
||||
displayName: "DeepSeek",
|
||||
description:
|
||||
"Official DeepSeek integration using @langchain/deepseek SDK",
|
||||
envVar: "DEEPSEEK_API_KEY",
|
||||
capabilities: { generative: true, embedding: false },
|
||||
defaultModel: "deepseek-chat",
|
||||
defaultMaxContext: 64000,
|
||||
fallbackPriority: 4,
|
||||
listModels: (apiKey) =>
|
||||
fetchOpenAICompatibleModels("https://api.deepseek.com", apiKey),
|
||||
});
|
||||
registerGenerative(
|
||||
"deepseek",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new DeepSeekProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): ILLMProvider {
|
||||
return new DeepSeekProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
);
|
||||
}
|
||||
|
||||
providerName = "DeepSeek";
|
||||
private model: ChatDeepSeek;
|
||||
private modelNameUsed: string;
|
||||
private providerInstanceName?: string;
|
||||
private maxContextUsed?: number;
|
||||
lastCalls: LLMCallRecord[] = [];
|
||||
protected readonly model: ChatDeepSeek;
|
||||
protected modelNameUsed: string;
|
||||
protected providerInstanceName?: string;
|
||||
protected maxContextUsed?: number;
|
||||
protected defaultMaxContext = 64000;
|
||||
|
||||
constructor(
|
||||
apiKey?: string,
|
||||
@@ -29,86 +54,29 @@ export class DeepSeekProvider implements ILLMProvider {
|
||||
providerInstanceName?: string,
|
||||
maxContext?: number,
|
||||
) {
|
||||
let key = apiKey;
|
||||
let model = modelName;
|
||||
this.providerInstanceName = providerInstanceName;
|
||||
this.maxContextUsed = maxContext;
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive("generative");
|
||||
if (active && active.providerName === DeepSeekProvider.providerId) {
|
||||
key = active.apiKey;
|
||||
if (!model) {
|
||||
model = active.modelName;
|
||||
}
|
||||
if (!this.providerInstanceName) {
|
||||
this.providerInstanceName = active.name;
|
||||
}
|
||||
if (this.maxContextUsed === undefined) {
|
||||
this.maxContextUsed = active.maxContext;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
key = llmConfig.DEEPSEEK_API_KEY;
|
||||
if (!this.providerInstanceName && key) {
|
||||
this.providerInstanceName = "Environment Variable";
|
||||
}
|
||||
}
|
||||
|
||||
super();
|
||||
const {
|
||||
key,
|
||||
model,
|
||||
providerInstanceName: resolvedName,
|
||||
maxContext: resolvedMax,
|
||||
} = resolveCredentials({
|
||||
explicitKey: apiKey,
|
||||
explicitModel: modelName,
|
||||
explicitProviderInstanceName: providerInstanceName,
|
||||
explicitMaxContext: maxContext,
|
||||
providerId: "deepseek",
|
||||
envVarName: "DEEPSEEK_API_KEY",
|
||||
type: "generative",
|
||||
});
|
||||
if (!key) {
|
||||
throw new Error(
|
||||
"DEEPSEEK_API_KEY is required to initialize DeepSeekProvider",
|
||||
);
|
||||
}
|
||||
|
||||
this.modelNameUsed = model || DeepSeekProvider.defaultModel;
|
||||
this.model = new ChatDeepSeek({
|
||||
apiKey: key,
|
||||
model: this.modelNameUsed,
|
||||
});
|
||||
}
|
||||
|
||||
async generateStructuredResponse<T extends z.ZodTypeAny>(
|
||||
request: LLMRequest<T>,
|
||||
): Promise<LLMResponse<z.infer<T>>> {
|
||||
const structuredModel = this.model.withStructuredOutput(request.schema, {
|
||||
includeRaw: true,
|
||||
});
|
||||
const result = (await structuredModel.invoke([
|
||||
{ role: "system", content: request.systemPrompt },
|
||||
{ role: "user", content: request.userContext },
|
||||
])) as unknown as {
|
||||
parsed?: z.infer<T>;
|
||||
raw?: {
|
||||
usage_metadata?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const parsed = result?.parsed;
|
||||
const raw = result?.raw;
|
||||
|
||||
const usage = {
|
||||
inputTokens: raw?.usage_metadata?.input_tokens || 0,
|
||||
outputTokens: raw?.usage_metadata?.output_tokens || 0,
|
||||
totalTokens: raw?.usage_metadata?.total_tokens || 0,
|
||||
modelName: this.modelNameUsed,
|
||||
providerInstanceName: this.providerInstanceName || "Default",
|
||||
maxContext:
|
||||
this.maxContextUsed !== undefined ? this.maxContextUsed : 64000,
|
||||
};
|
||||
|
||||
this.lastCalls.push({
|
||||
systemPrompt: request.systemPrompt,
|
||||
userContext: request.userContext,
|
||||
usage,
|
||||
});
|
||||
|
||||
return { success: true, data: parsed, usage };
|
||||
this.providerInstanceName = resolvedName;
|
||||
this.maxContextUsed = resolvedMax;
|
||||
this.modelNameUsed = model || "deepseek-chat";
|
||||
this.model = new ChatDeepSeek({ apiKey: key, model: this.modelNameUsed });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +1,96 @@
|
||||
import { z } from "zod";
|
||||
import {
|
||||
ChatGoogleGenerativeAI,
|
||||
GoogleGenerativeAIEmbeddings,
|
||||
} from "@langchain/google-genai";
|
||||
import {
|
||||
import type {
|
||||
ILLMProvider,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
LLMCallRecord,
|
||||
IEmbeddingProvider,
|
||||
ModelProviderInstance,
|
||||
} from "../llm.js";
|
||||
import { llmConfig } from "../config.js";
|
||||
import {
|
||||
registerProvider,
|
||||
registerGenerative,
|
||||
registerEmbedding,
|
||||
} from "../registry.js";
|
||||
import { fetchWithTimeout, type ModelInfo } from "../model-lister.js";
|
||||
import { BaseLLMProvider, resolveCredentials } from "../base-provider.js";
|
||||
import { getLlmConfig } from "../config.js";
|
||||
import { ProviderManager } from "../provider-manager.js";
|
||||
|
||||
export class GeminiProvider implements ILLMProvider {
|
||||
static readonly providerId = "google-genai";
|
||||
static readonly displayName = "Google Gemini";
|
||||
static readonly description =
|
||||
"Official Gemini integration using Google Gen AI SDK";
|
||||
static readonly defaultModel = "gemini-2.5-flash";
|
||||
async function fetchGeminiModels(apiKey: string): Promise<ModelInfo[]> {
|
||||
const models: ModelInfo[] = [];
|
||||
let pageToken: string | undefined;
|
||||
|
||||
do {
|
||||
const url = new URL(
|
||||
"https://generativelanguage.googleapis.com/v1beta/models",
|
||||
);
|
||||
url.searchParams.set("key", apiKey);
|
||||
url.searchParams.set("pageSize", "100");
|
||||
if (pageToken) {
|
||||
url.searchParams.set("pageToken", pageToken);
|
||||
}
|
||||
|
||||
const res = await fetchWithTimeout(url.toString());
|
||||
if (!res.ok) return models;
|
||||
|
||||
const json = (await res.json()) as {
|
||||
models?: { name: string; displayName?: string }[];
|
||||
nextPageToken?: string;
|
||||
};
|
||||
|
||||
for (const m of json.models ?? []) {
|
||||
const id = m.name.replace(/^models\//, "");
|
||||
models.push({ id, name: m.displayName || id });
|
||||
}
|
||||
|
||||
pageToken = json.nextPageToken;
|
||||
} while (pageToken);
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
export class GeminiProvider extends BaseLLMProvider {
|
||||
static {
|
||||
registerProvider({
|
||||
id: "google-genai",
|
||||
displayName: "Google Gemini",
|
||||
description: "Official Gemini integration using Google Gen AI SDK",
|
||||
envVar: "GOOGLE_API_KEY",
|
||||
capabilities: { generative: true, embedding: true },
|
||||
defaultModel: "gemini-2.5-flash",
|
||||
defaultEmbeddingModel: "gemini-embedding-001",
|
||||
defaultMaxContext: 32768,
|
||||
fallbackPriority: 0,
|
||||
listModels: fetchGeminiModels,
|
||||
});
|
||||
registerGenerative(
|
||||
"google-genai",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new GeminiProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
providerName = "Gemini";
|
||||
private model: ChatGoogleGenerativeAI;
|
||||
private modelNameUsed: string;
|
||||
private providerInstanceName?: string;
|
||||
private maxContextUsed?: number;
|
||||
lastCalls: LLMCallRecord[] = [];
|
||||
protected readonly model: ChatGoogleGenerativeAI;
|
||||
protected modelNameUsed: string;
|
||||
protected providerInstanceName?: string;
|
||||
protected maxContextUsed?: number;
|
||||
protected readonly defaultMaxContext = 32768;
|
||||
|
||||
static create(inst: ModelProviderInstance): ILLMProvider {
|
||||
return new GeminiProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
);
|
||||
}
|
||||
|
||||
constructor(
|
||||
apiKey?: string,
|
||||
@@ -33,113 +98,66 @@ export class GeminiProvider implements ILLMProvider {
|
||||
providerInstanceName?: string,
|
||||
maxContext?: number,
|
||||
) {
|
||||
let key = apiKey;
|
||||
let model = modelName;
|
||||
this.providerInstanceName = providerInstanceName;
|
||||
this.maxContextUsed = maxContext;
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive("generative");
|
||||
if (active && active.providerName === GeminiProvider.providerId) {
|
||||
key = active.apiKey;
|
||||
if (!model) {
|
||||
model = active.modelName;
|
||||
}
|
||||
if (!this.providerInstanceName) {
|
||||
this.providerInstanceName = active.name;
|
||||
}
|
||||
if (this.maxContextUsed === undefined) {
|
||||
this.maxContextUsed = active.maxContext;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
key = llmConfig.GOOGLE_API_KEY;
|
||||
if (!this.providerInstanceName && key) {
|
||||
this.providerInstanceName = "Environment Variable";
|
||||
}
|
||||
}
|
||||
|
||||
super();
|
||||
const {
|
||||
key,
|
||||
model,
|
||||
providerInstanceName: pn,
|
||||
maxContext: mc,
|
||||
} = resolveCredentials({
|
||||
explicitKey: apiKey,
|
||||
explicitModel: modelName,
|
||||
explicitProviderInstanceName: providerInstanceName,
|
||||
explicitMaxContext: maxContext,
|
||||
providerId: "google-genai",
|
||||
envVarName: "GOOGLE_API_KEY",
|
||||
type: "generative",
|
||||
});
|
||||
if (!key) {
|
||||
throw new Error(
|
||||
"GOOGLE_API_KEY is required to initialize GeminiProvider",
|
||||
);
|
||||
}
|
||||
|
||||
this.providerInstanceName = pn;
|
||||
this.maxContextUsed = mc;
|
||||
this.modelNameUsed = model || "gemini-2.5-flash";
|
||||
this.model = new ChatGoogleGenerativeAI({
|
||||
apiKey: key,
|
||||
model: this.modelNameUsed,
|
||||
});
|
||||
}
|
||||
|
||||
async generateStructuredResponse<T extends z.ZodTypeAny>(
|
||||
request: LLMRequest<T>,
|
||||
): Promise<LLMResponse<z.infer<T>>> {
|
||||
const structuredModel = this.model.withStructuredOutput(request.schema, {
|
||||
includeRaw: true,
|
||||
});
|
||||
const result = (await structuredModel.invoke([
|
||||
{ role: "system", content: request.systemPrompt },
|
||||
{ role: "user", content: request.userContext },
|
||||
])) as unknown as {
|
||||
parsed?: z.infer<T>;
|
||||
raw?: {
|
||||
usage_metadata?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const parsed = result?.parsed;
|
||||
const raw = result?.raw;
|
||||
|
||||
const usage = {
|
||||
inputTokens: raw?.usage_metadata?.input_tokens || 0,
|
||||
outputTokens: raw?.usage_metadata?.output_tokens || 0,
|
||||
totalTokens: raw?.usage_metadata?.total_tokens || 0,
|
||||
modelName: this.modelNameUsed,
|
||||
providerInstanceName: this.providerInstanceName || "Default",
|
||||
maxContext:
|
||||
this.maxContextUsed !== undefined ? this.maxContextUsed : 32768,
|
||||
};
|
||||
|
||||
this.lastCalls.push({
|
||||
systemPrompt: request.systemPrompt,
|
||||
userContext: request.userContext,
|
||||
usage,
|
||||
});
|
||||
|
||||
return { success: true, data: parsed, usage };
|
||||
}
|
||||
}
|
||||
|
||||
export class GeminiEmbeddingProvider implements IEmbeddingProvider {
|
||||
static readonly providerId = "google-genai";
|
||||
static readonly displayName = "Google Gemini Embeddings";
|
||||
static {
|
||||
registerEmbedding(
|
||||
"google-genai",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new GeminiEmbeddingProvider(inst.apiKey, inst.modelName),
|
||||
);
|
||||
}
|
||||
|
||||
providerName = "Gemini";
|
||||
private model: GoogleGenerativeAIEmbeddings;
|
||||
|
||||
static create(inst: ModelProviderInstance): IEmbeddingProvider {
|
||||
return new GeminiEmbeddingProvider(inst.apiKey, inst.modelName);
|
||||
}
|
||||
|
||||
constructor(apiKey?: string, modelName?: string) {
|
||||
let key = apiKey;
|
||||
let model = modelName;
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive("embedding");
|
||||
if (active) {
|
||||
if (active && active.providerName === "google-genai") {
|
||||
key = active.apiKey;
|
||||
if (!model) {
|
||||
model = active.modelName;
|
||||
}
|
||||
if (!model) model = active.modelName;
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
key = llmConfig.GOOGLE_API_KEY;
|
||||
key = getLlmConfig().GOOGLE_API_KEY;
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
|
||||
@@ -1,27 +1,51 @@
|
||||
import { z } from "zod";
|
||||
import { ChatGroq } from "@langchain/groq";
|
||||
import {
|
||||
ILLMProvider,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
LLMCallRecord,
|
||||
} from "../llm.js";
|
||||
import { llmConfig } from "../config.js";
|
||||
import { ProviderManager } from "../provider-manager.js";
|
||||
import { ILLMProvider } from "../llm.js";
|
||||
import type { ModelProviderInstance } from "../llm.js";
|
||||
import { BaseLLMProvider, resolveCredentials } from "../base-provider.js";
|
||||
import { registerProvider, registerGenerative } from "../registry.js";
|
||||
import { fetchOpenAICompatibleModels } from "../model-lister.js";
|
||||
|
||||
export class GroqProvider implements ILLMProvider {
|
||||
static readonly providerId = "groq";
|
||||
static readonly displayName = "Groq";
|
||||
static readonly description =
|
||||
"Official Groq integration using @langchain/groq SDK";
|
||||
static readonly defaultModel = "llama-3.3-70b-versatile";
|
||||
export class GroqProvider extends BaseLLMProvider {
|
||||
static {
|
||||
registerProvider({
|
||||
id: "groq",
|
||||
displayName: "Groq",
|
||||
description: "Official Groq integration using @langchain/groq SDK",
|
||||
envVar: "GROQ_API_KEY",
|
||||
capabilities: { generative: true, embedding: false },
|
||||
defaultModel: "llama-3.3-70b-versatile",
|
||||
defaultMaxContext: 8192,
|
||||
fallbackPriority: 3,
|
||||
listModels: (apiKey) =>
|
||||
fetchOpenAICompatibleModels("https://api.groq.com/openai/v1", apiKey),
|
||||
});
|
||||
registerGenerative(
|
||||
"groq",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new GroqProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): ILLMProvider {
|
||||
return new GroqProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
);
|
||||
}
|
||||
|
||||
providerName = "Groq";
|
||||
private model: ChatGroq;
|
||||
private modelNameUsed: string;
|
||||
private providerInstanceName?: string;
|
||||
private maxContextUsed?: number;
|
||||
lastCalls: LLMCallRecord[] = [];
|
||||
protected readonly model: ChatGroq;
|
||||
protected modelNameUsed: string;
|
||||
protected providerInstanceName?: string;
|
||||
protected maxContextUsed?: number;
|
||||
protected defaultMaxContext = 8192;
|
||||
|
||||
constructor(
|
||||
apiKey?: string,
|
||||
@@ -29,84 +53,27 @@ export class GroqProvider implements ILLMProvider {
|
||||
providerInstanceName?: string,
|
||||
maxContext?: number,
|
||||
) {
|
||||
let key = apiKey;
|
||||
let model = modelName;
|
||||
this.providerInstanceName = providerInstanceName;
|
||||
this.maxContextUsed = maxContext;
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive("generative");
|
||||
if (active && active.providerName === GroqProvider.providerId) {
|
||||
key = active.apiKey;
|
||||
if (!model) {
|
||||
model = active.modelName;
|
||||
}
|
||||
if (!this.providerInstanceName) {
|
||||
this.providerInstanceName = active.name;
|
||||
}
|
||||
if (this.maxContextUsed === undefined) {
|
||||
this.maxContextUsed = active.maxContext;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
key = llmConfig.GROQ_API_KEY;
|
||||
if (!this.providerInstanceName && key) {
|
||||
this.providerInstanceName = "Environment Variable";
|
||||
}
|
||||
}
|
||||
|
||||
super();
|
||||
const {
|
||||
key,
|
||||
model,
|
||||
providerInstanceName: resolvedName,
|
||||
maxContext: resolvedMax,
|
||||
} = resolveCredentials({
|
||||
explicitKey: apiKey,
|
||||
explicitModel: modelName,
|
||||
explicitProviderInstanceName: providerInstanceName,
|
||||
explicitMaxContext: maxContext,
|
||||
providerId: "groq",
|
||||
envVarName: "GROQ_API_KEY",
|
||||
type: "generative",
|
||||
});
|
||||
if (!key) {
|
||||
throw new Error("GROQ_API_KEY is required to initialize GroqProvider");
|
||||
}
|
||||
|
||||
this.modelNameUsed = model || GroqProvider.defaultModel;
|
||||
this.model = new ChatGroq({
|
||||
apiKey: key,
|
||||
model: this.modelNameUsed,
|
||||
});
|
||||
}
|
||||
|
||||
async generateStructuredResponse<T extends z.ZodTypeAny>(
|
||||
request: LLMRequest<T>,
|
||||
): Promise<LLMResponse<z.infer<T>>> {
|
||||
const structuredModel = this.model.withStructuredOutput(request.schema, {
|
||||
includeRaw: true,
|
||||
});
|
||||
const result = (await structuredModel.invoke([
|
||||
{ role: "system", content: request.systemPrompt },
|
||||
{ role: "user", content: request.userContext },
|
||||
])) as unknown as {
|
||||
parsed?: z.infer<T>;
|
||||
raw?: {
|
||||
usage_metadata?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const parsed = result?.parsed;
|
||||
const raw = result?.raw;
|
||||
|
||||
const usage = {
|
||||
inputTokens: raw?.usage_metadata?.input_tokens || 0,
|
||||
outputTokens: raw?.usage_metadata?.output_tokens || 0,
|
||||
totalTokens: raw?.usage_metadata?.total_tokens || 0,
|
||||
modelName: this.modelNameUsed,
|
||||
providerInstanceName: this.providerInstanceName || "Default",
|
||||
maxContext:
|
||||
this.maxContextUsed !== undefined ? this.maxContextUsed : 8192,
|
||||
};
|
||||
|
||||
this.lastCalls.push({
|
||||
systemPrompt: request.systemPrompt,
|
||||
userContext: request.userContext,
|
||||
usage,
|
||||
});
|
||||
|
||||
return { success: true, data: parsed, usage };
|
||||
this.providerInstanceName = resolvedName;
|
||||
this.maxContextUsed = resolvedMax;
|
||||
this.modelNameUsed = model || "llama-3.3-70b-versatile";
|
||||
this.model = new ChatGroq({ apiKey: key, model: this.modelNameUsed });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,13 +6,33 @@ import {
|
||||
LLMCallRecord,
|
||||
IEmbeddingProvider,
|
||||
} from "../llm.js";
|
||||
import type { ModelProviderInstance } from "../llm.js";
|
||||
import {
|
||||
registerProvider,
|
||||
registerGenerative,
|
||||
registerEmbedding,
|
||||
} from "../registry.js";
|
||||
|
||||
export class MockLLMProvider implements ILLMProvider {
|
||||
static readonly providerId = "mock";
|
||||
static readonly displayName = "Mock LLM Provider";
|
||||
static readonly description =
|
||||
"Stateless mock provider for testing and offline development";
|
||||
static readonly defaultModel = "mock";
|
||||
static {
|
||||
registerProvider({
|
||||
id: "mock",
|
||||
displayName: "Mock LLM Provider",
|
||||
description:
|
||||
"Stateless mock provider for testing and offline development",
|
||||
capabilities: { generative: true, embedding: true },
|
||||
defaultModel: "mock",
|
||||
defaultEmbeddingModel: "mock-embeddings",
|
||||
defaultMaxContext: 0,
|
||||
fallbackPriority: 1000,
|
||||
listModels: () => Promise.resolve([{ id: "mock", name: "Mock Model" }]),
|
||||
});
|
||||
registerGenerative("mock", () => new MockLLMProvider([]));
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): ILLMProvider {
|
||||
return new MockLLMProvider([]);
|
||||
}
|
||||
|
||||
providerName = "mock";
|
||||
private callCount = 0;
|
||||
@@ -46,7 +66,17 @@ export class MockLLMProvider implements ILLMProvider {
|
||||
}
|
||||
|
||||
export class MockEmbeddingProvider implements IEmbeddingProvider {
|
||||
static readonly providerId = "mock";
|
||||
static {
|
||||
registerEmbedding(
|
||||
"mock",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new MockEmbeddingProvider(inst.modelName),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): IEmbeddingProvider {
|
||||
return new MockEmbeddingProvider(inst.modelName);
|
||||
}
|
||||
|
||||
providerName = "mock";
|
||||
|
||||
|
||||
@@ -1,27 +1,72 @@
|
||||
import { z } from "zod";
|
||||
import { ChatOllama, OllamaEmbeddings } from "@langchain/ollama";
|
||||
import {
|
||||
ILLMProvider,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
LLMCallRecord,
|
||||
IEmbeddingProvider,
|
||||
} from "../llm.js";
|
||||
import { ILLMProvider, IEmbeddingProvider } from "../llm.js";
|
||||
import type { ModelProviderInstance } from "../llm.js";
|
||||
import { ProviderManager } from "../provider-manager.js";
|
||||
import { BaseLLMProvider } from "../base-provider.js";
|
||||
import {
|
||||
registerProvider,
|
||||
registerGenerative,
|
||||
registerEmbedding,
|
||||
} from "../registry.js";
|
||||
import { fetchWithTimeout, type ModelInfo } from "../model-lister.js";
|
||||
|
||||
export class OllamaProvider implements ILLMProvider {
|
||||
static readonly providerId = "ollama";
|
||||
static readonly displayName = "Ollama";
|
||||
static readonly description =
|
||||
"Local model runner supporting open-source LLMs via the Ollama server";
|
||||
static readonly defaultModel = "llama3.1";
|
||||
async function fetchOllamaModels(endpointUrl: string): Promise<ModelInfo[]> {
|
||||
const base = endpointUrl.replace(/\/$/, "");
|
||||
const res = await fetchWithTimeout(`${base}/api/tags`);
|
||||
if (!res.ok) return [];
|
||||
|
||||
const json = (await res.json()) as {
|
||||
models?: { name: string; model?: string }[];
|
||||
};
|
||||
|
||||
return (json.models ?? []).map((m) => ({
|
||||
id: m.name,
|
||||
name: m.name,
|
||||
}));
|
||||
}
|
||||
|
||||
export class OllamaProvider extends BaseLLMProvider {
|
||||
static {
|
||||
registerProvider({
|
||||
id: "ollama",
|
||||
displayName: "Ollama",
|
||||
description:
|
||||
"Local model runner supporting open-source LLMs via the Ollama server",
|
||||
capabilities: { generative: true, embedding: true },
|
||||
defaultModel: "llama3.1",
|
||||
defaultEmbeddingModel: "nomic-embed-text",
|
||||
defaultMaxContext: 32768,
|
||||
fallbackPriority: 100,
|
||||
listModels: (_apiKey, endpointUrl) =>
|
||||
fetchOllamaModels(endpointUrl || "http://localhost:11434"),
|
||||
});
|
||||
registerGenerative(
|
||||
"ollama",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new OllamaProvider(
|
||||
inst.endpointUrl,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): ILLMProvider {
|
||||
return new OllamaProvider(
|
||||
inst.endpointUrl,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
);
|
||||
}
|
||||
|
||||
providerName = "Ollama";
|
||||
private model: ChatOllama;
|
||||
private modelNameUsed: string;
|
||||
private providerInstanceName?: string;
|
||||
private maxContextUsed?: number;
|
||||
lastCalls: LLMCallRecord[] = [];
|
||||
protected readonly model: ChatOllama;
|
||||
protected modelNameUsed: string;
|
||||
protected providerInstanceName?: string;
|
||||
protected maxContextUsed?: number;
|
||||
protected defaultMaxContext = 32768;
|
||||
|
||||
/**
|
||||
* Creates an OllamaProvider.
|
||||
@@ -41,6 +86,7 @@ export class OllamaProvider implements ILLMProvider {
|
||||
providerInstanceName?: string,
|
||||
maxContext?: number,
|
||||
) {
|
||||
super();
|
||||
let url = baseUrl;
|
||||
let model = modelName;
|
||||
this.providerInstanceName = providerInstanceName;
|
||||
@@ -48,7 +94,7 @@ export class OllamaProvider implements ILLMProvider {
|
||||
|
||||
if (!url || !model) {
|
||||
const active = ProviderManager.getActive("generative");
|
||||
if (active && active.providerName === OllamaProvider.providerId) {
|
||||
if (active && active.providerName === "ollama") {
|
||||
if (!url) {
|
||||
url = active.endpointUrl;
|
||||
}
|
||||
@@ -64,59 +110,26 @@ export class OllamaProvider implements ILLMProvider {
|
||||
}
|
||||
}
|
||||
|
||||
this.modelNameUsed = model || OllamaProvider.defaultModel;
|
||||
this.modelNameUsed = model || "llama3.1";
|
||||
this.model = new ChatOllama({
|
||||
baseUrl: url || "http://localhost:11434",
|
||||
model: this.modelNameUsed,
|
||||
});
|
||||
}
|
||||
|
||||
async generateStructuredResponse<T extends z.ZodTypeAny>(
|
||||
request: LLMRequest<T>,
|
||||
): Promise<LLMResponse<z.infer<T>>> {
|
||||
const structuredModel = this.model.withStructuredOutput(request.schema, {
|
||||
includeRaw: true,
|
||||
});
|
||||
const result = (await structuredModel.invoke([
|
||||
{ role: "system", content: request.systemPrompt },
|
||||
{ role: "user", content: request.userContext },
|
||||
])) as unknown as {
|
||||
parsed?: z.infer<T>;
|
||||
raw?: {
|
||||
usage_metadata?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const parsed = result?.parsed;
|
||||
const raw = result?.raw;
|
||||
|
||||
const usage = {
|
||||
inputTokens: raw?.usage_metadata?.input_tokens || 0,
|
||||
outputTokens: raw?.usage_metadata?.output_tokens || 0,
|
||||
totalTokens: raw?.usage_metadata?.total_tokens || 0,
|
||||
modelName: this.modelNameUsed,
|
||||
providerInstanceName: this.providerInstanceName || "Default",
|
||||
maxContext:
|
||||
this.maxContextUsed !== undefined ? this.maxContextUsed : 32768,
|
||||
};
|
||||
|
||||
this.lastCalls.push({
|
||||
systemPrompt: request.systemPrompt,
|
||||
userContext: request.userContext,
|
||||
usage,
|
||||
});
|
||||
|
||||
return { success: true, data: parsed, usage };
|
||||
}
|
||||
}
|
||||
|
||||
export class OllamaEmbeddingProvider implements IEmbeddingProvider {
|
||||
static readonly providerId = "ollama";
|
||||
static readonly displayName = "Ollama Embeddings";
|
||||
static {
|
||||
registerEmbedding(
|
||||
"ollama",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new OllamaEmbeddingProvider(inst.endpointUrl, inst.modelName),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): IEmbeddingProvider {
|
||||
return new OllamaEmbeddingProvider(inst.endpointUrl, inst.modelName);
|
||||
}
|
||||
|
||||
providerName = "Ollama";
|
||||
private model: OllamaEmbeddings;
|
||||
@@ -137,10 +150,7 @@ export class OllamaEmbeddingProvider implements IEmbeddingProvider {
|
||||
|
||||
if (!url || !model) {
|
||||
const active = ProviderManager.getActive("embedding");
|
||||
if (
|
||||
active &&
|
||||
active.providerName === OllamaEmbeddingProvider.providerId
|
||||
) {
|
||||
if (active && active.providerName === "ollama") {
|
||||
if (!url) {
|
||||
url = active.endpointUrl;
|
||||
}
|
||||
|
||||
@@ -1,28 +1,58 @@
|
||||
import { z } from "zod";
|
||||
import { ChatOpenAI, OpenAIEmbeddings } from "@langchain/openai";
|
||||
import {
|
||||
ILLMProvider,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
LLMCallRecord,
|
||||
IEmbeddingProvider,
|
||||
} from "../llm.js";
|
||||
import { llmConfig } from "../config.js";
|
||||
import { ILLMProvider, IEmbeddingProvider } from "../llm.js";
|
||||
import type { ModelProviderInstance } from "../llm.js";
|
||||
import { getLlmConfig } from "../config.js";
|
||||
import { ProviderManager } from "../provider-manager.js";
|
||||
import { BaseLLMProvider, resolveCredentials } from "../base-provider.js";
|
||||
import {
|
||||
registerProvider,
|
||||
registerGenerative,
|
||||
registerEmbedding,
|
||||
} from "../registry.js";
|
||||
import { fetchOpenAICompatibleModels } from "../model-lister.js";
|
||||
|
||||
export class OpenAIProvider implements ILLMProvider {
|
||||
static readonly providerId = "openai";
|
||||
static readonly displayName = "OpenAI";
|
||||
static readonly description =
|
||||
"Official OpenAI integration using @langchain/openai SDK";
|
||||
static readonly defaultModel = "gpt-4o-mini";
|
||||
export class OpenAIProvider extends BaseLLMProvider {
|
||||
static {
|
||||
registerProvider({
|
||||
id: "openai",
|
||||
displayName: "OpenAI",
|
||||
description: "Official OpenAI integration using @langchain/openai SDK",
|
||||
envVar: "OPENAI_API_KEY",
|
||||
capabilities: { generative: true, embedding: true },
|
||||
defaultModel: "gpt-4o-mini",
|
||||
defaultEmbeddingModel: "text-embedding-3-small",
|
||||
defaultMaxContext: 128000,
|
||||
fallbackPriority: 1,
|
||||
listModels: (apiKey) =>
|
||||
fetchOpenAICompatibleModels("https://api.openai.com/v1", apiKey),
|
||||
});
|
||||
registerGenerative(
|
||||
"openai",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new OpenAIProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): ILLMProvider {
|
||||
return new OpenAIProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
);
|
||||
}
|
||||
|
||||
providerName = "OpenAI";
|
||||
private model: ChatOpenAI;
|
||||
private modelNameUsed: string;
|
||||
private providerInstanceName?: string;
|
||||
private maxContextUsed?: number;
|
||||
lastCalls: LLMCallRecord[] = [];
|
||||
protected readonly model: ChatOpenAI;
|
||||
protected modelNameUsed: string;
|
||||
protected providerInstanceName?: string;
|
||||
protected maxContextUsed?: number;
|
||||
protected defaultMaxContext = 128000;
|
||||
|
||||
constructor(
|
||||
apiKey?: string,
|
||||
@@ -30,93 +60,45 @@ export class OpenAIProvider implements ILLMProvider {
|
||||
providerInstanceName?: string,
|
||||
maxContext?: number,
|
||||
) {
|
||||
let key = apiKey;
|
||||
let model = modelName;
|
||||
this.providerInstanceName = providerInstanceName;
|
||||
this.maxContextUsed = maxContext;
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive("generative");
|
||||
if (active && active.providerName === OpenAIProvider.providerId) {
|
||||
key = active.apiKey;
|
||||
if (!model) {
|
||||
model = active.modelName;
|
||||
}
|
||||
if (!this.providerInstanceName) {
|
||||
this.providerInstanceName = active.name;
|
||||
}
|
||||
if (this.maxContextUsed === undefined) {
|
||||
this.maxContextUsed = active.maxContext;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
key = llmConfig.OPENAI_API_KEY;
|
||||
if (!this.providerInstanceName && key) {
|
||||
this.providerInstanceName = "Environment Variable";
|
||||
}
|
||||
}
|
||||
|
||||
super();
|
||||
const {
|
||||
key,
|
||||
model,
|
||||
providerInstanceName: resolvedName,
|
||||
maxContext: resolvedMax,
|
||||
} = resolveCredentials({
|
||||
explicitKey: apiKey,
|
||||
explicitModel: modelName,
|
||||
explicitProviderInstanceName: providerInstanceName,
|
||||
explicitMaxContext: maxContext,
|
||||
providerId: "openai",
|
||||
envVarName: "OPENAI_API_KEY",
|
||||
type: "generative",
|
||||
});
|
||||
if (!key) {
|
||||
throw new Error(
|
||||
"OPENAI_API_KEY is required to initialize OpenAIProvider",
|
||||
);
|
||||
}
|
||||
|
||||
this.modelNameUsed = model || OpenAIProvider.defaultModel;
|
||||
this.model = new ChatOpenAI({
|
||||
apiKey: key,
|
||||
model: this.modelNameUsed,
|
||||
});
|
||||
}
|
||||
|
||||
async generateStructuredResponse<T extends z.ZodTypeAny>(
|
||||
request: LLMRequest<T>,
|
||||
): Promise<LLMResponse<z.infer<T>>> {
|
||||
const structuredModel = this.model.withStructuredOutput(request.schema, {
|
||||
includeRaw: true,
|
||||
});
|
||||
const result = (await structuredModel.invoke([
|
||||
{ role: "system", content: request.systemPrompt },
|
||||
{ role: "user", content: request.userContext },
|
||||
])) as unknown as {
|
||||
parsed?: z.infer<T>;
|
||||
raw?: {
|
||||
usage_metadata?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const parsed = result?.parsed;
|
||||
const raw = result?.raw;
|
||||
|
||||
const usage = {
|
||||
inputTokens: raw?.usage_metadata?.input_tokens || 0,
|
||||
outputTokens: raw?.usage_metadata?.output_tokens || 0,
|
||||
totalTokens: raw?.usage_metadata?.total_tokens || 0,
|
||||
modelName: this.modelNameUsed,
|
||||
providerInstanceName: this.providerInstanceName || "Default",
|
||||
maxContext:
|
||||
this.maxContextUsed !== undefined ? this.maxContextUsed : 128000,
|
||||
};
|
||||
|
||||
this.lastCalls.push({
|
||||
systemPrompt: request.systemPrompt,
|
||||
userContext: request.userContext,
|
||||
usage,
|
||||
});
|
||||
|
||||
return { success: true, data: parsed, usage };
|
||||
this.providerInstanceName = resolvedName;
|
||||
this.maxContextUsed = resolvedMax;
|
||||
this.modelNameUsed = model || "gpt-4o-mini";
|
||||
this.model = new ChatOpenAI({ apiKey: key, model: this.modelNameUsed });
|
||||
}
|
||||
}
|
||||
|
||||
export class OpenAIEmbeddingProvider implements IEmbeddingProvider {
|
||||
static readonly providerId = "openai";
|
||||
static readonly displayName = "OpenAI Embeddings";
|
||||
static {
|
||||
registerEmbedding(
|
||||
"openai",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new OpenAIEmbeddingProvider(inst.apiKey, inst.modelName),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): IEmbeddingProvider {
|
||||
return new OpenAIEmbeddingProvider(inst.apiKey, inst.modelName);
|
||||
}
|
||||
|
||||
providerName = "OpenAI";
|
||||
private model: OpenAIEmbeddings;
|
||||
@@ -127,10 +109,7 @@ export class OpenAIEmbeddingProvider implements IEmbeddingProvider {
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive("embedding");
|
||||
if (
|
||||
active &&
|
||||
active.providerName === OpenAIEmbeddingProvider.providerId
|
||||
) {
|
||||
if (active && active.providerName === "openai") {
|
||||
key = active.apiKey;
|
||||
if (!model) {
|
||||
model = active.modelName;
|
||||
@@ -139,7 +118,7 @@ export class OpenAIEmbeddingProvider implements IEmbeddingProvider {
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
key = llmConfig.OPENAI_API_KEY;
|
||||
key = getLlmConfig().OPENAI_API_KEY;
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
|
||||
@@ -1,27 +1,77 @@
|
||||
import { z } from "zod";
|
||||
import { ChatOpenRouter } from "@langchain/openrouter";
|
||||
import {
|
||||
ILLMProvider,
|
||||
LLMRequest,
|
||||
LLMResponse,
|
||||
LLMCallRecord,
|
||||
} from "../llm.js";
|
||||
import { llmConfig } from "../config.js";
|
||||
import { ProviderManager } from "../provider-manager.js";
|
||||
import { ILLMProvider } from "../llm.js";
|
||||
import type { ModelProviderInstance } from "../llm.js";
|
||||
import { BaseLLMProvider, resolveCredentials } from "../base-provider.js";
|
||||
import { registerProvider, registerGenerative } from "../registry.js";
|
||||
import { fetchWithTimeout, type ModelInfo } from "../model-lister.js";
|
||||
|
||||
export class OpenRouterProvider implements ILLMProvider {
|
||||
static readonly providerId = "openrouter";
|
||||
static readonly displayName = "OpenRouter";
|
||||
static readonly description =
|
||||
"Multi-model router supporting Anthropic, OpenAI, DeepSeek, and local models";
|
||||
static readonly defaultModel = "google/gemini-2.5-flash";
|
||||
async function fetchOpenRouterModels(apiKey: string): Promise<ModelInfo[]> {
|
||||
const res = await fetchWithTimeout(
|
||||
"https://openrouter.ai/api/v1/models",
|
||||
apiKey
|
||||
? {
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
Accept: "application/json",
|
||||
},
|
||||
}
|
||||
: { headers: { Accept: "application/json" } },
|
||||
);
|
||||
if (!res.ok) return [];
|
||||
|
||||
const json = (await res.json()) as {
|
||||
data?: { id: string; name?: string; owned_by?: string }[];
|
||||
};
|
||||
|
||||
return (json.data ?? []).map((m) => ({
|
||||
id: m.id,
|
||||
name: m.name || m.id,
|
||||
ownedBy: m.owned_by,
|
||||
}));
|
||||
}
|
||||
|
||||
export class OpenRouterProvider extends BaseLLMProvider {
|
||||
static {
|
||||
registerProvider({
|
||||
id: "openrouter",
|
||||
displayName: "OpenRouter",
|
||||
description:
|
||||
"Multi-model router supporting Anthropic, OpenAI, DeepSeek, and local models",
|
||||
envVar: "OPENROUTER_API_KEY",
|
||||
capabilities: { generative: true, embedding: false },
|
||||
defaultModel: "google/gemini-2.5-flash",
|
||||
defaultEmbeddingModel: "openai/text-embedding-3-small",
|
||||
defaultMaxContext: 32768,
|
||||
fallbackPriority: 5,
|
||||
listModels: fetchOpenRouterModels,
|
||||
});
|
||||
registerGenerative(
|
||||
"openrouter",
|
||||
(inst: ModelProviderInstance) =>
|
||||
new OpenRouterProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
static create(inst: ModelProviderInstance): ILLMProvider {
|
||||
return new OpenRouterProvider(
|
||||
inst.apiKey,
|
||||
inst.modelName,
|
||||
inst.name,
|
||||
inst.maxContext,
|
||||
);
|
||||
}
|
||||
|
||||
providerName = "OpenRouter";
|
||||
private model: ChatOpenRouter;
|
||||
private modelNameUsed: string;
|
||||
private providerInstanceName?: string;
|
||||
private maxContextUsed?: number;
|
||||
lastCalls: LLMCallRecord[] = [];
|
||||
protected readonly model: ChatOpenRouter;
|
||||
protected modelNameUsed: string;
|
||||
protected providerInstanceName?: string;
|
||||
protected maxContextUsed?: number;
|
||||
protected defaultMaxContext = 32768;
|
||||
|
||||
constructor(
|
||||
apiKey?: string,
|
||||
@@ -29,86 +79,29 @@ export class OpenRouterProvider implements ILLMProvider {
|
||||
providerInstanceName?: string,
|
||||
maxContext?: number,
|
||||
) {
|
||||
let key = apiKey;
|
||||
let model = modelName;
|
||||
this.providerInstanceName = providerInstanceName;
|
||||
this.maxContextUsed = maxContext;
|
||||
|
||||
if (!key) {
|
||||
const active = ProviderManager.getActive("generative");
|
||||
if (active && active.providerName === OpenRouterProvider.providerId) {
|
||||
key = active.apiKey;
|
||||
if (!model) {
|
||||
model = active.modelName;
|
||||
}
|
||||
if (!this.providerInstanceName) {
|
||||
this.providerInstanceName = active.name;
|
||||
}
|
||||
if (this.maxContextUsed === undefined) {
|
||||
this.maxContextUsed = active.maxContext;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!key) {
|
||||
key = llmConfig.OPENROUTER_API_KEY;
|
||||
if (!this.providerInstanceName && key) {
|
||||
this.providerInstanceName = "Environment Variable";
|
||||
}
|
||||
}
|
||||
|
||||
super();
|
||||
const {
|
||||
key,
|
||||
model,
|
||||
providerInstanceName: resolvedName,
|
||||
maxContext: resolvedMax,
|
||||
} = resolveCredentials({
|
||||
explicitKey: apiKey,
|
||||
explicitModel: modelName,
|
||||
explicitProviderInstanceName: providerInstanceName,
|
||||
explicitMaxContext: maxContext,
|
||||
providerId: "openrouter",
|
||||
envVarName: "OPENROUTER_API_KEY",
|
||||
type: "generative",
|
||||
});
|
||||
if (!key) {
|
||||
throw new Error(
|
||||
"OPENROUTER_API_KEY is required to initialize OpenRouterProvider",
|
||||
);
|
||||
}
|
||||
|
||||
this.providerInstanceName = resolvedName;
|
||||
this.maxContextUsed = resolvedMax;
|
||||
this.modelNameUsed = model || "google/gemini-2.5-flash";
|
||||
this.model = new ChatOpenRouter({
|
||||
apiKey: key,
|
||||
model: this.modelNameUsed,
|
||||
});
|
||||
}
|
||||
|
||||
async generateStructuredResponse<T extends z.ZodTypeAny>(
|
||||
request: LLMRequest<T>,
|
||||
): Promise<LLMResponse<z.infer<T>>> {
|
||||
const structuredModel = this.model.withStructuredOutput(request.schema, {
|
||||
includeRaw: true,
|
||||
});
|
||||
const result = (await structuredModel.invoke([
|
||||
{ role: "system", content: request.systemPrompt },
|
||||
{ role: "user", content: request.userContext },
|
||||
])) as unknown as {
|
||||
parsed?: z.infer<T>;
|
||||
raw?: {
|
||||
usage_metadata?: {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const parsed = result?.parsed;
|
||||
const raw = result?.raw;
|
||||
|
||||
const usage = {
|
||||
inputTokens: raw?.usage_metadata?.input_tokens || 0,
|
||||
outputTokens: raw?.usage_metadata?.output_tokens || 0,
|
||||
totalTokens: raw?.usage_metadata?.total_tokens || 0,
|
||||
modelName: this.modelNameUsed,
|
||||
providerInstanceName: this.providerInstanceName || "Default",
|
||||
maxContext:
|
||||
this.maxContextUsed !== undefined ? this.maxContextUsed : 32768,
|
||||
};
|
||||
|
||||
this.lastCalls.push({
|
||||
systemPrompt: request.systemPrompt,
|
||||
userContext: request.userContext,
|
||||
usage,
|
||||
});
|
||||
|
||||
return { success: true, data: parsed, usage };
|
||||
this.model = new ChatOpenRouter({ apiKey: key, model: this.modelNameUsed });
|
||||
}
|
||||
}
|
||||
|
||||
79
packages/llm/src/registry.ts
Normal file
79
packages/llm/src/registry.ts
Normal file
@@ -0,0 +1,79 @@
|
||||
import type {
|
||||
ILLMProvider,
|
||||
IEmbeddingProvider,
|
||||
ModelProviderInstance,
|
||||
ModelProviderMeta,
|
||||
} from "./llm.js";
|
||||
import type { ModelInfo } from "./model-lister.js";
|
||||
|
||||
export interface ProviderDefinition {
|
||||
id: string;
|
||||
displayName: string;
|
||||
description: string;
|
||||
envVar?: string;
|
||||
capabilities: { generative: boolean; embedding: boolean };
|
||||
defaultModel: string;
|
||||
defaultEmbeddingModel?: string;
|
||||
defaultMaxContext: number;
|
||||
fallbackPriority: number;
|
||||
listModels?: (apiKey: string, endpointUrl?: string) => Promise<ModelInfo[]>;
|
||||
generativeCreate?: (inst: ModelProviderInstance) => ILLMProvider;
|
||||
embeddingCreate?: (inst: ModelProviderInstance) => IEmbeddingProvider;
|
||||
}
|
||||
|
||||
const _entries = new Map<string, ProviderDefinition>();
|
||||
|
||||
type ProviderMeta = Omit<
|
||||
ProviderDefinition,
|
||||
"generativeCreate" | "embeddingCreate"
|
||||
>;
|
||||
|
||||
export function registerProvider(meta: ProviderMeta) {
|
||||
const existing = _entries.get(meta.id);
|
||||
_entries.set(meta.id, {
|
||||
...existing,
|
||||
...meta,
|
||||
generativeCreate: existing?.generativeCreate,
|
||||
embeddingCreate: existing?.embeddingCreate,
|
||||
});
|
||||
}
|
||||
|
||||
export function registerGenerative(
|
||||
id: string,
|
||||
createFn: (inst: ModelProviderInstance) => ILLMProvider,
|
||||
) {
|
||||
const existing = _entries.get(id);
|
||||
if (existing) {
|
||||
existing.generativeCreate = createFn;
|
||||
} else {
|
||||
_entries.set(id, { id, generativeCreate: createFn } as ProviderDefinition);
|
||||
}
|
||||
}
|
||||
|
||||
export function registerEmbedding(
|
||||
id: string,
|
||||
createFn: (inst: ModelProviderInstance) => IEmbeddingProvider,
|
||||
) {
|
||||
const existing = _entries.get(id);
|
||||
if (existing) {
|
||||
existing.embeddingCreate = createFn;
|
||||
} else {
|
||||
_entries.set(id, { id, embeddingCreate: createFn } as ProviderDefinition);
|
||||
}
|
||||
}
|
||||
|
||||
export const ProviderRegistry = {
|
||||
all: (): ProviderDefinition[] => [..._entries.values()],
|
||||
get: (id: string): ProviderDefinition | undefined => _entries.get(id),
|
||||
has: (id: string): boolean => _entries.has(id),
|
||||
} as const;
|
||||
|
||||
export function toProviderMeta(def: ProviderDefinition): ModelProviderMeta {
|
||||
return {
|
||||
id: def.id,
|
||||
displayName: def.displayName,
|
||||
description: def.description,
|
||||
defaultModel: def.defaultModel,
|
||||
defaultEmbeddingModel: def.defaultEmbeddingModel || "",
|
||||
};
|
||||
}
|
||||
56
packages/llm/src/row-mapper.ts
Normal file
56
packages/llm/src/row-mapper.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import type { ModelProviderInstance } from "./llm.js";
|
||||
import type { ProviderDefinition } from "./registry.js";
|
||||
|
||||
export type DbRow = {
|
||||
id: string;
|
||||
name: string;
|
||||
providerName: string;
|
||||
apiKey: string;
|
||||
isActive: number;
|
||||
modelName?: string;
|
||||
type: string;
|
||||
maxContext?: number;
|
||||
endpointUrl?: string;
|
||||
};
|
||||
|
||||
export function mapRow(r: DbRow): ModelProviderInstance {
|
||||
return {
|
||||
id: r.id,
|
||||
name: r.name,
|
||||
providerName: r.providerName,
|
||||
apiKey: r.apiKey,
|
||||
isActive: r.isActive === 1,
|
||||
modelName: r.modelName || undefined,
|
||||
type: (r.type as "generative" | "embedding") || "generative",
|
||||
maxContext:
|
||||
r.maxContext !== undefined && r.maxContext !== null
|
||||
? r.maxContext
|
||||
: r.type === "embedding"
|
||||
? 0
|
||||
: 32768,
|
||||
endpointUrl: r.endpointUrl || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function synthInstance(
|
||||
def: ProviderDefinition,
|
||||
type: "generative" | "embedding",
|
||||
apiKey: string,
|
||||
): ModelProviderInstance {
|
||||
const modelName =
|
||||
type === "embedding" ? def.defaultEmbeddingModel : def.defaultModel;
|
||||
return {
|
||||
id: `provider-default-env-fallback-${def.id}-${type}`,
|
||||
name:
|
||||
type === "embedding"
|
||||
? `${def.displayName} Embed (Env Fallback)`
|
||||
: `${def.displayName} (Env Fallback)`,
|
||||
providerName: def.id,
|
||||
apiKey,
|
||||
isActive: true,
|
||||
modelName,
|
||||
type,
|
||||
maxContext: type === "embedding" ? 0 : def.defaultMaxContext,
|
||||
endpointUrl: undefined,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user