sortedcord/omnia-consolidated
1
0
Fork 0
mirror of https://github.com/sortedcord/omnia.git synced 2026-07-22 03:52:48 +05:30
Code Issues Packages Projects Releases Wiki Activity

feat(llm): Added Groq and Deepseek providers

Browse Source
This commit is contained in:
Aditya Gupta
2026-07-16 18:25:07 +05:30
parent 13155cba23
commit 2b56c01e4c
13 changed files with 829 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
packages/llm/README.md
View File

@@ -279,6 +279,48 @@ Also exports `GeminiEmbeddingProvider` (implements `IEmbeddingProvider`) using t
4. None found → throw Error
```
### Groq — `GroqProvider`
| Property | Value |
| --------------------------- | -------------------------------------------- |
| **File** | [`providers/groq.ts`](src/providers/groq.ts) |
| **Provider ID** | `groq` |
| **SDK** | `@langchain/groq` (`ChatGroq`) |
| **Default Model** | `llama-3.3-70b-versatile` |
| **Default Embedding Model** | _(none)_ |
| **Default Max Context** | `8192` |
| **Type** | Generative only (no embedding provider) |
**Key resolution** in the constructor follows this cascade:
```
1. Explicit apiKey argument → use it
2. ProviderManager.getActive() → if providerName matches "groq"
3. GROQ_API_KEY env var → final fallback
4. None found → throw Error
```
### DeepSeek — `DeepSeekProvider`
| Property | Value |
| --------------------------- | ---------------------------------------------------- |
| **File** | [`providers/deepseek.ts`](src/providers/deepseek.ts) |
| **Provider ID** | `deepseek` |
| **SDK** | `@langchain/deepseek` (`ChatDeepSeek`) |
| **Default Model** | `deepseek-chat` |
| **Default Embedding Model** | _(none)_ |
| **Default Max Context** | `64000` |
| **Type** | Generative only (no embedding provider) |
**Key resolution** in the constructor follows this cascade:
```
1. Explicit apiKey argument → use it
2. ProviderManager.getActive() → if providerName matches "deepseek"
3. DEEPSEEK_API_KEY env var → final fallback
4. None found → throw Error
```
### OpenAI — `OpenAIProvider`
| Property | Value |
@@ -404,6 +446,8 @@ The `buildLLMProvider()` and `buildEmbeddingProvider()` functions perform the fi
| `"openrouter"` | `OpenRouterProvider` | _(falls through to mock)_ |
| `"ollama"` | `OllamaProvider` | `OllamaEmbeddingProvider` |
| `"anthropic"` | `AnthropicProvider` | _(falls through to mock)_ |
| `"groq"` | `GroqProvider` | _(falls through to mock)_ |
| `"deepseek"` | `DeepSeekProvider` | _(falls through to mock)_ |
| _(anything else)_ | `MockLLMProvider` | `MockEmbeddingProvider` |
## Structured Output
@@ -432,6 +476,8 @@ This sends the Zod schema to the model as a structured output constraint. The re
| `OPENAI_API_KEY` | No | OpenAI API key |
| `OPENROUTER_API_KEY` | No | OpenRouter API key |
| `ANTHROPIC_API_KEY` | No | Anthropic Claude API key |
| `GROQ_API_KEY` | No | Groq API key |
| `DEEPSEEK_API_KEY` | No | DeepSeek API key |
Both are optional because providers can also be configured through the database via the GUI settings page.
@@ -450,6 +496,8 @@ packages/llm/
│ ├── openrouter.ts # OpenRouterProvider
│ ├── anthropic.ts # AnthropicProvider
│ ├── openai.ts # OpenAIProvider + OpenAIEmbeddingProvider
│ ├── groq.ts # GroqProvider
│ ├── deepseek.ts # DeepSeekProvider
│ └── mock.ts # MockLLMProvider + MockEmbeddingProvider
├── tests/
│ ├── mock.test.ts

2
packages/llm/src/config.ts
View File

@@ -5,6 +5,8 @@ const LLMConfigSchema = z.object({
OPENROUTER_API_KEY: z.string().optional(),
ANTHROPIC_API_KEY: z.string().optional(),
OPENAI_API_KEY: z.string().optional(),
GROQ_API_KEY: z.string().optional(),
DEEPSEEK_API_KEY: z.string().optional(),
});
export const llmConfig = LLMConfigSchema.parse(process.env);

2
packages/llm/src/index.ts
View File

@@ -6,4 +6,6 @@ export * from "./providers/ollama.js";
export * from "./providers/openrouter.js";
export * from "./providers/anthropic.js";
export * from "./providers/openai.js";
export * from "./providers/groq.js";
export * from "./providers/deepseek.js";
export * from "./provider-manager.js";

14
packages/llm/src/llm.ts
View File

@@ -90,6 +90,20 @@ export const AVAILABLE_PROVIDERS: ModelProviderMeta[] = [
defaultModel: "claude-3-5-sonnet-latest",
defaultEmbeddingModel: "",
},
{
id: "groq",
displayName: "Groq",
description: "Official Groq integration using @langchain/groq SDK",
defaultModel: "llama-3.3-70b-versatile",
defaultEmbeddingModel: "",
},
{
id: "deepseek",
displayName: "DeepSeek",
description: "Official DeepSeek integration using @langchain/deepseek SDK",
defaultModel: "deepseek-chat",
defaultEmbeddingModel: "",
},
{
id: "openrouter",
displayName: "OpenRouter",

122
packages/llm/src/provider-manager.ts
View File

@@ -101,6 +101,8 @@ function getSettingsDb() {
const openRouterKey = process.env.OPENROUTER_API_KEY;
const anthropicKey = process.env.ANTHROPIC_API_KEY;
const openaiKey = process.env.OPENAI_API_KEY;
const groqKey = process.env.GROQ_API_KEY;
const deepseekKey = process.env.DEEPSEEK_API_KEY;
let hasInsertedGenerative = false;
let hasInsertedEmbedding = false;
@@ -209,6 +211,52 @@ function getSettingsDb() {
}
}
if (groqKey && groqKey.trim()) {
const id = "provider-default-groq";
const isActive = hasInsertedGenerative ? 0 : 1;
db.prepare(
`
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(
id,
"Groq (Env)",
"groq",
groqKey.trim(),
isActive,
"llama-3.3-70b-versatile",
"generative",
8192,
);
if (isActive === 1) {
hasInsertedGenerative = true;
}
}
if (deepseekKey && deepseekKey.trim()) {
const id = "provider-default-deepseek";
const isActive = hasInsertedGenerative ? 0 : 1;
db.prepare(
`
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(
id,
"DeepSeek (Env)",
"deepseek",
deepseekKey.trim(),
isActive,
"deepseek-chat",
"generative",
64000,
);
if (isActive === 1) {
hasInsertedGenerative = true;
}
}
if (openRouterKey && openRouterKey.trim()) {
const id = "provider-default-openrouter";
const isActive = hasInsertedGenerative ? 0 : 1;
@@ -464,6 +512,8 @@ export class ProviderManager {
const openRouterKey = process.env.OPENROUTER_API_KEY;
const anthropicKey = process.env.ANTHROPIC_API_KEY;
const openaiKey = process.env.OPENAI_API_KEY;
const groqKey = process.env.GROQ_API_KEY;
const deepseekKey = process.env.DEEPSEEK_API_KEY;
let hasInsertedGenerative = false;
let hasInsertedEmbedding = false;
@@ -572,6 +622,52 @@ export class ProviderManager {
}
}
if (groqKey && groqKey.trim()) {
const id = "provider-default-groq";
const isActive = hasInsertedGenerative ? 0 : 1;
db.prepare(
`
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(
id,
"Groq (Env)",
"groq",
groqKey.trim(),
isActive,
"llama-3.3-70b-versatile",
"generative",
8192,
);
if (isActive === 1) {
hasInsertedGenerative = true;
}
}
if (deepseekKey && deepseekKey.trim()) {
const id = "provider-default-deepseek";
const isActive = hasInsertedGenerative ? 0 : 1;
db.prepare(
`
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName, type, maxContext)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
).run(
id,
"DeepSeek (Env)",
"deepseek",
deepseekKey.trim(),
isActive,
"deepseek-chat",
"generative",
64000,
);
if (isActive === 1) {
hasInsertedGenerative = true;
}
}
if (openRouterKey && openRouterKey.trim()) {
const id = "provider-default-openrouter";
const isActive = hasInsertedGenerative ? 0 : 1;
@@ -757,6 +853,32 @@ export class ProviderManager {
maxContext: 200000,
};
}
const groqKey = process.env.GROQ_API_KEY;
if (groqKey && groqKey.trim()) {
return {
id: "provider-default-env-fallback",
name: "Groq (Env Fallback)",
providerName: "groq",
apiKey: groqKey.trim(),
isActive: true,
modelName: "llama-3.3-70b-versatile",
type: "generative",
maxContext: 8192,
};
}
const deepseekKey = process.env.DEEPSEEK_API_KEY;
if (deepseekKey && deepseekKey.trim()) {
return {
id: "provider-default-env-fallback",
name: "DeepSeek (Env Fallback)",
providerName: "deepseek",
apiKey: deepseekKey.trim(),
isActive: true,
modelName: "deepseek-chat",
type: "generative",
maxContext: 64000,
};
}
const openRouterKey = process.env.OPENROUTER_API_KEY;
if (openRouterKey && openRouterKey.trim()) {
return {

114
packages/llm/src/providers/deepseek.ts Normal file
View File

@@ -0,0 +1,114 @@
import { z } from "zod";
import { ChatDeepSeek } from "@langchain/deepseek";
import {
ILLMProvider,
LLMRequest,
LLMResponse,
LLMCallRecord,
} from "../llm.js";
import { llmConfig } from "../config.js";
import { ProviderManager } from "../provider-manager.js";
export class DeepSeekProvider implements ILLMProvider {
static readonly providerId = "deepseek";
static readonly displayName = "DeepSeek";
static readonly description =
"Official DeepSeek integration using @langchain/deepseek SDK";
static readonly defaultModel = "deepseek-chat";
providerName = "DeepSeek";
private model: ChatDeepSeek;
private modelNameUsed: string;
private providerInstanceName?: string;
private maxContextUsed?: number;
lastCalls: LLMCallRecord[] = [];
constructor(
apiKey?: string,
modelName?: string,
providerInstanceName?: string,
maxContext?: number,
) {
let key = apiKey;
let model = modelName;
this.providerInstanceName = providerInstanceName;
this.maxContextUsed = maxContext;
if (!key) {
const active = ProviderManager.getActive("generative");
if (active && active.providerName === DeepSeekProvider.providerId) {
key = active.apiKey;
if (!model) {
model = active.modelName;
}
if (!this.providerInstanceName) {
this.providerInstanceName = active.name;
}
if (this.maxContextUsed === undefined) {
this.maxContextUsed = active.maxContext;
}
}
}
if (!key) {
key = llmConfig.DEEPSEEK_API_KEY;
if (!this.providerInstanceName && key) {
this.providerInstanceName = "Environment Variable";
}
}
if (!key) {
throw new Error(
"DEEPSEEK_API_KEY is required to initialize DeepSeekProvider",
);
}
this.modelNameUsed = model || DeepSeekProvider.defaultModel;
this.model = new ChatDeepSeek({
apiKey: key,
model: this.modelNameUsed,
});
}
async generateStructuredResponse<T extends z.ZodTypeAny>(
request: LLMRequest<T>,
): Promise<LLMResponse<z.infer<T>>> {
const structuredModel = this.model.withStructuredOutput(request.schema, {
includeRaw: true,
});
const result = (await structuredModel.invoke([
{ role: "system", content: request.systemPrompt },
{ role: "user", content: request.userContext },
])) as unknown as {
parsed?: z.infer<T>;
raw?: {
usage_metadata?: {
input_tokens?: number;
output_tokens?: number;
total_tokens?: number;
};
};
};
const parsed = result?.parsed;
const raw = result?.raw;
const usage = {
inputTokens: raw?.usage_metadata?.input_tokens || 0,
outputTokens: raw?.usage_metadata?.output_tokens || 0,
totalTokens: raw?.usage_metadata?.total_tokens || 0,
modelName: this.modelNameUsed,
providerInstanceName: this.providerInstanceName || "Default",
maxContext:
this.maxContextUsed !== undefined ? this.maxContextUsed : 64000,
};
this.lastCalls.push({
systemPrompt: request.systemPrompt,
userContext: request.userContext,
usage,
});
return { success: true, data: parsed, usage };
}
}

112
packages/llm/src/providers/groq.ts Normal file
View File

@@ -0,0 +1,112 @@
import { z } from "zod";
import { ChatGroq } from "@langchain/groq";
import {
ILLMProvider,
LLMRequest,
LLMResponse,
LLMCallRecord,
} from "../llm.js";
import { llmConfig } from "../config.js";
import { ProviderManager } from "../provider-manager.js";
export class GroqProvider implements ILLMProvider {
static readonly providerId = "groq";
static readonly displayName = "Groq";
static readonly description =
"Official Groq integration using @langchain/groq SDK";
static readonly defaultModel = "llama-3.3-70b-versatile";
providerName = "Groq";
private model: ChatGroq;
private modelNameUsed: string;
private providerInstanceName?: string;
private maxContextUsed?: number;
lastCalls: LLMCallRecord[] = [];
constructor(
apiKey?: string,
modelName?: string,
providerInstanceName?: string,
maxContext?: number,
) {
let key = apiKey;
let model = modelName;
this.providerInstanceName = providerInstanceName;
this.maxContextUsed = maxContext;
if (!key) {
const active = ProviderManager.getActive("generative");
if (active && active.providerName === GroqProvider.providerId) {
key = active.apiKey;
if (!model) {
model = active.modelName;
}
if (!this.providerInstanceName) {
this.providerInstanceName = active.name;
}
if (this.maxContextUsed === undefined) {
this.maxContextUsed = active.maxContext;
}
}
}
if (!key) {
key = llmConfig.GROQ_API_KEY;
if (!this.providerInstanceName && key) {
this.providerInstanceName = "Environment Variable";
}
}
if (!key) {
throw new Error("GROQ_API_KEY is required to initialize GroqProvider");
}
this.modelNameUsed = model || GroqProvider.defaultModel;
this.model = new ChatGroq({
apiKey: key,
model: this.modelNameUsed,
});
}
async generateStructuredResponse<T extends z.ZodTypeAny>(
request: LLMRequest<T>,
): Promise<LLMResponse<z.infer<T>>> {
const structuredModel = this.model.withStructuredOutput(request.schema, {
includeRaw: true,
});
const result = (await structuredModel.invoke([
{ role: "system", content: request.systemPrompt },
{ role: "user", content: request.userContext },
])) as unknown as {
parsed?: z.infer<T>;
raw?: {
usage_metadata?: {
input_tokens?: number;
output_tokens?: number;
total_tokens?: number;
};
};
};
const parsed = result?.parsed;
const raw = result?.raw;
const usage = {
inputTokens: raw?.usage_metadata?.input_tokens || 0,
outputTokens: raw?.usage_metadata?.output_tokens || 0,
totalTokens: raw?.usage_metadata?.total_tokens || 0,
modelName: this.modelNameUsed,
providerInstanceName: this.providerInstanceName || "Default",
maxContext:
this.maxContextUsed !== undefined ? this.maxContextUsed : 8192,
};
this.lastCalls.push({
systemPrompt: request.systemPrompt,
userContext: request.userContext,
usage,
});
return { success: true, data: parsed, usage };
}
}

98
packages/llm/tests/deepseek.test.ts Normal file
View File

@@ -0,0 +1,98 @@
import { describe, test, expect, vi } from "vitest";
import { z } from "zod";
import { DeepSeekProvider } from "../src/providers/deepseek.js";
import { llmConfig } from "../src/config.js";
// Mock the ChatDeepSeek class
vi.mock("@langchain/deepseek", () => {
return {
ChatDeepSeek: class {
config: unknown;
constructor(config: unknown) {
this.config = config;
}
withStructuredOutput = vi.fn().mockImplementation(() => {
return {
invoke: vi.fn().mockImplementation(async () => {
return {
parsed: {
name: "mocked response",
success: true,
},
raw: {
usage_metadata: {
input_tokens: 10,
output_tokens: 5,
total_tokens: 15,
},
},
};
}),
};
});
},
};
});
describe("DeepSeekProvider Unit Tests (Tier 1)", () => {
test("initializes successfully with a provided apiKey", () => {
const provider = new DeepSeekProvider("dummy-key");
expect(provider.providerName).toBe("DeepSeek");
});
test("initializes successfully with apiKey from config", () => {
const originalKey = llmConfig.DEEPSEEK_API_KEY;
llmConfig.DEEPSEEK_API_KEY = "env-dummy-key";
try {
const provider = new DeepSeekProvider();
expect(provider.providerName).toBe("DeepSeek");
} finally {
llmConfig.DEEPSEEK_API_KEY = originalKey;
}
});
test("throws error if no API key is provided or in config", () => {
const originalKey = llmConfig.DEEPSEEK_API_KEY;
llmConfig.DEEPSEEK_API_KEY = undefined;
try {
expect(() => new DeepSeekProvider()).toThrow(
"DEEPSEEK_API_KEY is required to initialize DeepSeekProvider",
);
} finally {
llmConfig.DEEPSEEK_API_KEY = originalKey;
}
});
test("generateStructuredResponse invokes the model with structured output, records usage and updates lastCalls", async () => {
const provider = new DeepSeekProvider("dummy-key");
const TestSchema = z.object({
name: z.string(),
success: z.boolean(),
});
const response = await provider.generateStructuredResponse({
systemPrompt: "system prompt",
userContext: "user context",
schema: TestSchema,
});
expect(response.success).toBe(true);
expect(response.data).toEqual({
name: "mocked response",
success: true,
});
expect(response.usage).toEqual({
inputTokens: 10,
outputTokens: 5,
totalTokens: 15,
modelName: "deepseek-chat",
providerInstanceName: "Default",
maxContext: 64000,
});
expect(provider.lastCalls.length).toBe(1);
});
});

98
packages/llm/tests/groq.test.ts Normal file
View File

@@ -0,0 +1,98 @@
import { describe, test, expect, vi } from "vitest";
import { z } from "zod";
import { GroqProvider } from "../src/providers/groq.js";
import { llmConfig } from "../src/config.js";
// Mock the ChatGroq class
vi.mock("@langchain/groq", () => {
return {
ChatGroq: class {
config: unknown;
constructor(config: unknown) {
this.config = config;
}
withStructuredOutput = vi.fn().mockImplementation(() => {
return {
invoke: vi.fn().mockImplementation(async () => {
return {
parsed: {
name: "mocked response",
success: true,
},
raw: {
usage_metadata: {
input_tokens: 10,
output_tokens: 5,
total_tokens: 15,
},
},
};
}),
};
});
},
};
});
describe("GroqProvider Unit Tests (Tier 1)", () => {
test("initializes successfully with a provided apiKey", () => {
const provider = new GroqProvider("dummy-key");
expect(provider.providerName).toBe("Groq");
});
test("initializes successfully with apiKey from config", () => {
const originalKey = llmConfig.GROQ_API_KEY;
llmConfig.GROQ_API_KEY = "env-dummy-key";
try {
const provider = new GroqProvider();
expect(provider.providerName).toBe("Groq");
} finally {
llmConfig.GROQ_API_KEY = originalKey;
}
});
test("throws error if no API key is provided or in config", () => {
const originalKey = llmConfig.GROQ_API_KEY;
llmConfig.GROQ_API_KEY = undefined;
try {
expect(() => new GroqProvider()).toThrow(
"GROQ_API_KEY is required to initialize GroqProvider",
);
} finally {
llmConfig.GROQ_API_KEY = originalKey;
}
});
test("generateStructuredResponse invokes the model with structured output, records usage and updates lastCalls", async () => {
const provider = new GroqProvider("dummy-key");
const TestSchema = z.object({
name: z.string(),
success: z.boolean(),
});
const response = await provider.generateStructuredResponse({
systemPrompt: "system prompt",
userContext: "user context",
schema: TestSchema,
});
expect(response.success).toBe(true);
expect(response.data).toEqual({
name: "mocked response",
success: true,
});
expect(response.usage).toEqual({
inputTokens: 10,
outputTokens: 5,
totalTokens: 15,
modelName: "llama-3.3-70b-versatile",
providerInstanceName: "Default",
maxContext: 8192,
});
expect(provider.lastCalls.length).toBe(1);
});
});

135
packages/llm/tests/openai.test.ts Normal file
View File

@@ -0,0 +1,135 @@
import { describe, test, expect, vi } from "vitest";
import { z } from "zod";
import {
OpenAIProvider,
OpenAIEmbeddingProvider,
} from "../src/providers/openai.js";
import { llmConfig } from "../src/config.js";
// Mock the ChatOpenAI and OpenAIEmbeddings classes
vi.mock("@langchain/openai", () => {
return {
ChatOpenAI: class {
config: unknown;
constructor(config: unknown) {
this.config = config;
}
withStructuredOutput = vi.fn().mockImplementation(() => {
return {
invoke: vi.fn().mockImplementation(async () => {
return {
parsed: {
name: "mocked response",
success: true,
},
raw: {
usage_metadata: {
input_tokens: 10,
output_tokens: 5,
total_tokens: 15,
},
},
};
}),
};
});
},
OpenAIEmbeddings: class {
config: unknown;
constructor(config: unknown) {
this.config = config;
}
embedQuery = vi.fn().mockImplementation(async (text: string) => {
return [0.1, 0.2, 0.3];
});
},
};
});
describe("OpenAIProvider Unit Tests (Tier 1)", () => {
test("initializes successfully with a provided apiKey", () => {
const provider = new OpenAIProvider("dummy-key");
expect(provider.providerName).toBe("OpenAI");
});
test("initializes successfully with apiKey from config", () => {
const originalKey = llmConfig.OPENAI_API_KEY;
llmConfig.OPENAI_API_KEY = "env-dummy-key";
try {
const provider = new OpenAIProvider();
expect(provider.providerName).toBe("OpenAI");
} finally {
llmConfig.OPENAI_API_KEY = originalKey;
}
});
test("throws error if no API key is provided or in config", () => {
const originalKey = llmConfig.OPENAI_API_KEY;
llmConfig.OPENAI_API_KEY = undefined;
try {
expect(() => new OpenAIProvider()).toThrow(
"OPENAI_API_KEY is required to initialize OpenAIProvider",
);
} finally {
llmConfig.OPENAI_API_KEY = originalKey;
}
});
test("generateStructuredResponse invokes the model with structured output, records usage and updates lastCalls", async () => {
const provider = new OpenAIProvider("dummy-key");
const TestSchema = z.object({
name: z.string(),
success: z.boolean(),
});
const response = await provider.generateStructuredResponse({
systemPrompt: "system prompt",
userContext: "user context",
schema: TestSchema,
});
expect(response.success).toBe(true);
expect(response.data).toEqual({
name: "mocked response",
success: true,
});
expect(response.usage).toEqual({
inputTokens: 10,
outputTokens: 5,
totalTokens: 15,
modelName: "gpt-4o-mini",
providerInstanceName: "Default",
maxContext: 128000,
});
expect(provider.lastCalls.length).toBe(1);
});
});
describe("OpenAIEmbeddingProvider Unit Tests (Tier 1)", () => {
test("initializes successfully with a provided apiKey", () => {
const provider = new OpenAIEmbeddingProvider("dummy-key");
expect(provider.providerName).toBe("OpenAI");
});
test("initializes successfully with apiKey from config", () => {
const originalKey = llmConfig.OPENAI_API_KEY;
llmConfig.OPENAI_API_KEY = "env-dummy-key";
try {
const provider = new OpenAIEmbeddingProvider();
expect(provider.providerName).toBe("OpenAI");
} finally {
llmConfig.OPENAI_API_KEY = originalKey;
}
});
test("embed returns dummy array successfully", async () => {
const provider = new OpenAIEmbeddingProvider("dummy-key");
const result = await provider.embed("hello");
expect(result).toEqual([0.1, 0.2, 0.3]);
});
});