refactor(llm): Unify auto-bootstrapped prover instances and custom instances

This commit is contained in:
2026-07-10 22:27:24 +05:30
parent b5fb48ed99
commit 7df685365e
8 changed files with 315 additions and 167 deletions

View File

@@ -1,6 +1,6 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
import "./.next/types/routes.d.ts";
import "./.next/dev/types/routes.d.ts";
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

View File

@@ -29,7 +29,7 @@ export default function ConfigPage() {
const [loading, setLoading] = useState(true);
const [error, setError] = useState("");
const [selectedInstanceId, setSelectedInstanceId] = useState<string | "new">("new");
const [selectedInstanceId, setSelectedInstanceId] = useState<string | null>(null);
const [editName, setEditName] = useState("");
const [editProvider, setEditProvider] = useState("google-genai");
const [editKey, setEditKey] = useState("");
@@ -37,7 +37,13 @@ export default function ConfigPage() {
const [editIsActive, setEditIsActive] = useState(false);
useEffect(() => {
if (selectedInstanceId === "new") {
if (selectedInstanceId === null) {
setEditName("");
setEditProvider("google-genai");
setEditKey("");
setEditModel("gemini-2.5-flash");
setEditIsActive(false);
} else if (selectedInstanceId === "new") {
setEditName("");
const defaultProvider = "google-genai";
setEditProvider(defaultProvider);
@@ -144,14 +150,14 @@ export default function ConfigPage() {
};
const handleDelete = async () => {
if (selectedInstanceId === "new") return;
if (selectedInstanceId === "new" || selectedInstanceId === null) return;
if (!confirm("Are you sure you want to delete this provider instance?")) return;
try {
setLoading(true);
setError("");
await deleteProviderInstance(selectedInstanceId);
setSelectedInstanceId("new");
setSelectedInstanceId(null);
await loadInstances();
await loadMappings();
} catch (err) {
@@ -234,123 +240,129 @@ export default function ConfigPage() {
{/* 70% area */}
<div className="flex flex-col bg-white">
<form onSubmit={handleSave} className="flex h-full flex-col justify-between">
<div className="flex flex-1 flex-col gap-5 p-6">
<h3 className="m-0 mb-2 text-lg font-semibold text-[#111]">
{selectedInstanceId === "new"
? "Create New Provider Instance"
: `Configure: ${editName}`}
</h3>
<div className="flex flex-col gap-1.5">
<label htmlFor="formName" className="text-xs font-medium text-gray-700">
Friendly Name
</label>
<input
id="formName"
type="text"
value={editName}
onChange={(e) => setEditName(e.target.value)}
placeholder="e.g. Gemini - Production"
required
className="w-full rounded-md border border-gray-300 bg-white px-3 py-2 text-sm outline-none transition-[border-color,box-shadow] focus:border-blue-500 focus:ring-3 focus:ring-blue-500/15"
/>
</div>
<div className="flex flex-col gap-1.5">
<label htmlFor="formProvider" className="text-xs font-medium text-gray-700">
Provider Type
</label>
<select
id="formProvider"
value={editProvider}
onChange={(e) => handleProviderChange(e.target.value)}
className="w-full rounded-md border border-gray-300 bg-white px-3 py-2 text-sm outline-none transition-[border-color,box-shadow] focus:border-blue-500 focus:ring-3 focus:ring-blue-500/15"
>
{availableProviders.map((p) => (
<option key={p.id} value={p.id}>
{p.displayName}
</option>
))}
</select>
{editProvider && availableProviders.length > 0 && (
<span className="mt-1 block rounded border border-gray-200 bg-gray-100 px-3 py-2 text-xs text-gray-600">
{availableProviders.find((p) => p.id === editProvider)?.description}
</span>
)}
</div>
<div className="flex flex-col gap-1.5">
<label htmlFor="formKey" className="text-xs font-medium text-gray-700">
API Key
</label>
<input
id="formKey"
type="password"
value={editKey}
onChange={(e) => setEditKey(e.target.value)}
placeholder={
selectedInstanceId === "new"
? "AIzaSy..."
: "•••••••• (unchanged)"
}
required={selectedInstanceId === "new"}
className="w-full rounded-md border border-gray-300 bg-white px-3 py-2 text-sm outline-none transition-[border-color,box-shadow] focus:border-blue-500 focus:ring-3 focus:ring-blue-500/15"
/>
</div>
<div className="flex flex-col gap-1.5">
<label htmlFor="formModel" className="text-xs font-medium text-gray-700">
Model Name
</label>
<input
id="formModel"
type="text"
value={editModel}
onChange={(e) => setEditModel(e.target.value)}
placeholder="e.g. gemini-2.5-flash, gemini-2.5-pro"
className="w-full rounded-md border border-gray-300 bg-white px-3 py-2 text-sm outline-none transition-[border-color,box-shadow] focus:border-blue-500 focus:ring-3 focus:ring-blue-500/15"
/>
</div>
<div className="mt-1 flex flex-row items-center gap-2">
<input
id="formActive"
type="checkbox"
checked={editIsActive}
onChange={(e) => setEditIsActive(e.target.checked)}
className="h-4 w-4 cursor-pointer"
/>
<label htmlFor="formActive" className="cursor-pointer text-xs font-medium text-gray-700">
Set as Active Instance
</label>
</div>
{selectedInstanceId === null ? (
<div className="flex flex-1 flex-col items-center justify-center p-6 text-center text-sm text-gray-400">
Press + to add or select an existing Instance to edit
</div>
) : (
<form onSubmit={handleSave} className="flex h-full flex-col justify-between">
<div className="flex flex-1 flex-col gap-5 p-6">
<h3 className="m-0 mb-2 text-lg font-semibold text-[#111]">
{selectedInstanceId === "new"
? "Create New Provider Instance"
: `Configure: ${editName}`}
</h3>
<div className="flex items-center justify-between border-t border-gray-200 bg-gray-50 px-6 py-4">
<div>
{selectedInstanceId !== "new" && (
<button
type="button"
onClick={handleDelete}
disabled={loading}
className="cursor-pointer rounded-md bg-red-500 px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-red-600 disabled:opacity-50"
<div className="flex flex-col gap-1.5">
<label htmlFor="formName" className="text-xs font-medium text-gray-700">
Friendly Name
</label>
<input
id="formName"
type="text"
value={editName}
onChange={(e) => setEditName(e.target.value)}
placeholder="e.g. Gemini - Production"
required
className="w-full rounded-md border border-gray-300 bg-white px-3 py-2 text-sm outline-none transition-[border-color,box-shadow] focus:border-blue-500 focus:ring-3 focus:ring-blue-500/15"
/>
</div>
<div className="flex flex-col gap-1.5">
<label htmlFor="formProvider" className="text-xs font-medium text-gray-700">
Provider Type
</label>
<select
id="formProvider"
value={editProvider}
onChange={(e) => handleProviderChange(e.target.value)}
className="w-full rounded-md border border-gray-300 bg-white px-3 py-2 text-sm outline-none transition-[border-color,box-shadow] focus:border-blue-500 focus:ring-3 focus:ring-blue-500/15"
>
Delete
{availableProviders.map((p) => (
<option key={p.id} value={p.id}>
{p.displayName}
</option>
))}
</select>
{editProvider && availableProviders.length > 0 && (
<span className="mt-1 block rounded border border-gray-200 bg-gray-100 px-3 py-2 text-xs text-gray-600">
{availableProviders.find((p) => p.id === editProvider)?.description}
</span>
)}
</div>
<div className="flex flex-col gap-1.5">
<label htmlFor="formKey" className="text-xs font-medium text-gray-700">
API Key
</label>
<input
id="formKey"
type="password"
value={editKey}
onChange={(e) => setEditKey(e.target.value)}
placeholder={
selectedInstanceId === "new"
? "AIzaSy..."
: "•••••••• (unchanged)"
}
required={selectedInstanceId === "new"}
className="w-full rounded-md border border-gray-300 bg-white px-3 py-2 text-sm outline-none transition-[border-color,box-shadow] focus:border-blue-500 focus:ring-3 focus:ring-blue-500/15"
/>
</div>
<div className="flex flex-col gap-1.5">
<label htmlFor="formModel" className="text-xs font-medium text-gray-700">
Model Name
</label>
<input
id="formModel"
type="text"
value={editModel}
onChange={(e) => setEditModel(e.target.value)}
placeholder="e.g. gemini-2.5-flash, gemini-2.5-pro"
className="w-full rounded-md border border-gray-300 bg-white px-3 py-2 text-sm outline-none transition-[border-color,box-shadow] focus:border-blue-500 focus:ring-3 focus:ring-blue-500/15"
/>
</div>
<div className="mt-1 flex flex-row items-center gap-2">
<input
id="formActive"
type="checkbox"
checked={editIsActive}
onChange={(e) => setEditIsActive(e.target.checked)}
className="h-4 w-4 cursor-pointer"
/>
<label htmlFor="formActive" className="cursor-pointer text-xs font-medium text-gray-700">
Set as Active Instance
</label>
</div>
</div>
<div className="flex items-center justify-between border-t border-gray-200 bg-gray-50 px-6 py-4">
<div>
{selectedInstanceId !== "new" && (
<button
type="button"
onClick={handleDelete}
disabled={loading}
className="cursor-pointer rounded-md bg-red-500 px-4 py-2 text-sm font-medium text-white transition-colors hover:bg-red-600 disabled:opacity-50"
>
Delete
</button>
)}
</div>
<div>
<button
type="submit"
disabled={loading}
className="cursor-pointer rounded-md bg-blue-600 px-5 py-2 text-sm font-medium text-white transition-colors hover:bg-blue-700 disabled:opacity-50"
>
{loading ? "Saving..." : "Save"}
</button>
)}
</div>
</div>
<div>
<button
type="submit"
disabled={loading}
className="cursor-pointer rounded-md bg-blue-600 px-5 py-2 text-sm font-medium text-white transition-colors hover:bg-blue-700 disabled:opacity-50"
>
{loading ? "Saving..." : "Save"}
</button>
</div>
</div>
</form>
</form>
)}
</div>
</div>
</section>
@@ -382,7 +394,7 @@ export default function ConfigPage() {
onChange={(e) => handleUpdateMapping(task.key, e.target.value)}
className="w-full rounded border border-gray-300 bg-white px-2 py-1.5 text-xs"
>
<option value="">-- Use Active Key (Default) --</option>
<option value="">Use Default Provider</option>
{instances.map((inst) => (
<option key={inst.id} value={inst.id}>
{inst.name} ({inst.providerName}){inst.isActive ? " [Active]" : ""}

View File

@@ -123,13 +123,6 @@ class SimulationManager {
? ProviderManager.list().find((p) => p.id === providerInstanceId)
: ProviderManager.getActive();
if (!activeInstance) {
const envKey = process.env.GOOGLE_API_KEY;
if (envKey) {
activeInstance = ProviderManager.create("Default (Env)", "google-genai", envKey);
}
}
if (!activeInstance) {
return {
id: "",
@@ -659,12 +652,6 @@ class SimulationManager {
if (!inst) {
inst = active;
}
if (!inst) {
const envKey = process.env.GOOGLE_API_KEY;
if (envKey) {
inst = ProviderManager.create("Default (Env)", "google-genai", envKey);
}
}
if (!inst) {
throw new Error(`No active LLM Provider Instance found for task "${task}". Please configure a key in Settings first.`);

View File

@@ -3,6 +3,17 @@ import path from "path";
import fs from "fs";
import type { LLMProviderInstance } from "./llm.js";
let dbPathOverride: string | null = null;
let hasBootstrapped = false;
export function setDbPathOverride(p: string | null) {
dbPathOverride = p;
}
export function resetHasBootstrapped() {
hasBootstrapped = false;
}
function getWorkspaceRoot() {
let current = process.cwd();
while (current !== "/" && current !== path.parse(current).root) {
@@ -20,12 +31,17 @@ function getWorkspaceRoot() {
}
function getSettingsDb() {
const wsRoot = getWorkspaceRoot();
const dbDir = path.resolve(wsRoot, "data");
if (!fs.existsSync(dbDir)) {
fs.mkdirSync(dbDir, { recursive: true });
let dbPath: string;
if (dbPathOverride) {
dbPath = dbPathOverride;
} else {
const wsRoot = getWorkspaceRoot();
const dbDir = path.resolve(wsRoot, "data");
if (!fs.existsSync(dbDir)) {
fs.mkdirSync(dbDir, { recursive: true });
}
dbPath = path.join(dbDir, "settings.db");
}
const dbPath = path.join(dbDir, "settings.db");
const db = new Database(dbPath);
db.prepare(`
@@ -44,6 +60,39 @@ function getSettingsDb() {
} catch {
// ignore
}
// Auto-bootstrap environment variables if DB contains 0 instances
try {
if (!hasBootstrapped) {
const totalCount = db.prepare(`SELECT COUNT(*) as count FROM provider_instances`).get() as { count: number };
if (totalCount.count === 0) {
const googleKey = process.env.GOOGLE_API_KEY;
const openRouterKey = process.env.OPENROUTER_API_KEY;
let hasInserted = false;
if (googleKey && googleKey.trim()) {
const id = "provider-default-google";
db.prepare(`
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName)
VALUES (?, ?, ?, ?, ?, ?)
`).run(id, "Gemini (Env)", "google-genai", googleKey.trim(), 1, "gemini-2.5-flash");
hasInserted = true;
}
if (openRouterKey && openRouterKey.trim()) {
const id = "provider-default-openrouter";
const isActive = hasInserted ? 0 : 1;
db.prepare(`
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName)
VALUES (?, ?, ?, ?, ?, ?)
`).run(id, "OpenRouter (Env)", "openrouter", openRouterKey.trim(), isActive, "google/gemini-2.5-flash");
}
}
hasBootstrapped = true;
}
} catch {
// ignore write lock issues or other DB errors during bootstrap
}
return db;
}
@@ -153,28 +202,25 @@ export class ProviderManager {
} | undefined;
if (!row) {
// Check if there are any rows at all
const totalCount = db.prepare(`SELECT COUNT(*) as count FROM provider_instances`).get() as { count: number };
if (totalCount.count === 0) {
// Database is completely empty! Check if GOOGLE_API_KEY env is set.
const envKey = process.env.GOOGLE_API_KEY;
if (envKey && envKey.trim()) {
// Auto-bootstrap default active instance from env
const id = "provider-default-env";
db.prepare(`
INSERT INTO provider_instances (id, name, providerName, apiKey, isActive, modelName)
VALUES (?, ?, ?, ?, ?, ?)
`).run(id, "Default (Env)", "google-genai", envKey, 1, "gemini-2.5-flash");
return {
id,
name: "Default (Env)",
providerName: "google-genai",
apiKey: envKey,
isActive: true,
modelName: "gemini-2.5-flash",
};
}
// If there's no active row but some rows exist, return the first one as active, or update it
const firstRow = db.prepare(`SELECT * FROM provider_instances LIMIT 1`).get() as {
id: string;
name: string;
providerName: string;
apiKey: string;
isActive: number;
modelName?: string;
} | undefined;
if (firstRow) {
db.prepare(`UPDATE provider_instances SET isActive = 1 WHERE id = ?`).run(firstRow.id);
return {
id: firstRow.id,
name: firstRow.name,
providerName: firstRow.providerName,
apiKey: firstRow.apiKey,
isActive: true,
modelName: firstRow.modelName || undefined,
};
}
return null;
}
@@ -189,17 +235,28 @@ export class ProviderManager {
};
} catch {
// Lock or write issue fallback: return an in-memory active key if env key exists
const envKey = process.env.GOOGLE_API_KEY;
if (envKey) {
const googleKey = process.env.GOOGLE_API_KEY;
if (googleKey && googleKey.trim()) {
return {
id: "provider-default-env-fallback",
name: "Default (Env Fallback)",
name: "Gemini (Env Fallback)",
providerName: "google-genai",
apiKey: envKey,
apiKey: googleKey.trim(),
isActive: true,
modelName: "gemini-2.5-flash",
};
}
const openRouterKey = process.env.OPENROUTER_API_KEY;
if (openRouterKey && openRouterKey.trim()) {
return {
id: "provider-default-env-fallback",
name: "OpenRouter (Env Fallback)",
providerName: "openrouter",
apiKey: openRouterKey.trim(),
isActive: true,
modelName: "google/gemini-2.5-flash",
};
}
return null;
} finally {
db.close();

View File

@@ -20,7 +20,7 @@ export class GeminiProvider implements ILLMProvider {
if (!key) {
const active = ProviderManager.getActive();
if (active) {
if (active && active.providerName === GeminiProvider.providerId) {
key = active.apiKey;
if (!model) {
model = active.modelName;

View File

@@ -20,7 +20,7 @@ export class OpenRouterProvider implements ILLMProvider {
if (!key) {
const active = ProviderManager.getActive();
if (active) {
if (active && active.providerName === OpenRouterProvider.providerId) {
key = active.apiKey;
if (!model) {
model = active.modelName;

View File

@@ -0,0 +1,91 @@
import { describe, test, expect, beforeEach, afterEach } from "vitest";
import fs from "fs";
import path from "path";
import { ProviderManager, setDbPathOverride, resetHasBootstrapped } from "../src/index.js";
describe("ProviderManager Bootstrapping & CRUD Unit Tests", () => {
let tempDbPath: string;
let originalGoogle: string | undefined;
let originalOpenRouter: string | undefined;
beforeEach(() => {
originalGoogle = process.env.GOOGLE_API_KEY;
originalOpenRouter = process.env.OPENROUTER_API_KEY;
delete process.env.GOOGLE_API_KEY;
delete process.env.OPENROUTER_API_KEY;
resetHasBootstrapped();
// Generate a unique temp database path for this test run
tempDbPath = path.resolve(process.cwd(), `test-settings-${Date.now()}-${Math.random().toString(36).substring(2)}.db`);
setDbPathOverride(tempDbPath);
});
afterEach(() => {
setDbPathOverride(null);
if (fs.existsSync(tempDbPath)) {
try {
fs.unlinkSync(tempDbPath);
} catch {
// ignore
}
}
if (originalGoogle !== undefined) {
process.env.GOOGLE_API_KEY = originalGoogle;
} else {
delete process.env.GOOGLE_API_KEY;
}
if (originalOpenRouter !== undefined) {
process.env.OPENROUTER_API_KEY = originalOpenRouter;
} else {
delete process.env.OPENROUTER_API_KEY;
}
});
test("auto-bootstraps Gemini and OpenRouter when database is empty and environment variables are present", () => {
process.env.GOOGLE_API_KEY = "mock-google-key-123";
process.env.OPENROUTER_API_KEY = "mock-openrouter-key-456";
const list = ProviderManager.list();
expect(list.length).toBe(2);
const gemini = list.find((p) => p.providerName === "google-genai");
expect(gemini).toBeDefined();
expect(gemini?.name).toBe("Gemini (Env)");
expect(gemini?.apiKey).toBe("mock-google-key-123");
expect(gemini?.modelName).toBe("gemini-2.5-flash");
expect(gemini?.isActive).toBe(true); // first inserted is active
const openrouter = list.find((p) => p.providerName === "openrouter");
expect(openrouter).toBeDefined();
expect(openrouter?.name).toBe("OpenRouter (Env)");
expect(openrouter?.apiKey).toBe("mock-openrouter-key-456");
expect(openrouter?.modelName).toBe("google/gemini-2.5-flash");
expect(openrouter?.isActive).toBe(false); // second inserted is inactive
});
test("treats bootstrapped instances as normal provider instances (editable and deletable)", () => {
process.env.GOOGLE_API_KEY = "mock-google-key-123";
// Trigger bootstrap
const list = ProviderManager.list();
expect(list.length).toBe(1);
const bootstrapped = list[0];
expect(bootstrapped.name).toBe("Gemini (Env)");
expect(bootstrapped.isActive).toBe(true);
// Edit name and key
ProviderManager.update(bootstrapped.id, "My Gemini Key", "google-genai", "new-secret-key", "gemini-2.5-pro");
const listAfterUpdate = ProviderManager.list();
expect(listAfterUpdate.length).toBe(1);
expect(listAfterUpdate[0].name).toBe("My Gemini Key");
expect(listAfterUpdate[0].apiKey).toBe("new-secret-key");
expect(listAfterUpdate[0].modelName).toBe("gemini-2.5-pro");
// Delete instance
ProviderManager.delete(bootstrapped.id);
const listAfterDelete = ProviderManager.list();
expect(listAfterDelete.length).toBe(0);
});
});

View File

@@ -73,9 +73,10 @@ If no specific provider instance is mapped to a task, the task automatically rou
To maintain backwards-compatibility and support headless runs, live evaluation suites, and automated unit tests without requiring database pre-configuration, the config manager supports **self-bootstrapping**:
1. When the provider manager queries the active key instance, if `data/settings.db` contains **0 registered keys**, it checks the process environment for `GOOGLE_API_KEY` or `OPENROUTER_API_KEY`.
2. If `process.env.GOOGLE_API_KEY` is present, it automatically creates, saves, and activates a default provider instance (`Default (Env)`) in `settings.db`.
3. If database write locks occur (e.g., during high-concurrency Vitest test suites), the system seamlessly returns a temporary in-memory `LLMProviderInstance` to keep execution fluent and error-free.
1. When any database connection is initialized via the provider manager, if `data/settings.db` contains **0 registered keys**, it checks the process environment for `GOOGLE_API_KEY` and `OPENROUTER_API_KEY`.
2. If `process.env.GOOGLE_API_KEY` is present, it automatically creates, saves, and activates a default provider instance (`Gemini (Env)`) in `settings.db`.
3. If `process.env.OPENROUTER_API_KEY` is present, it automatically creates and saves a default provider instance (`OpenRouter (Env)`) in `settings.db`.
4. If database write locks occur (e.g., during high-concurrency Vitest test suites), the system seamlessly returns a temporary in-memory `LLMProviderInstance` (`Gemini (Env Fallback)` or `OpenRouter (Env Fallback)`) to keep execution fluent and error-free.
---