feat(llm): Dynamically fetch available models from providers

This commit is contained in:
2026-07-16 19:16:55 +05:30
parent 2b56c01e4c
commit 8ad94d3fc2
8 changed files with 1083 additions and 8 deletions

View File

@@ -9,6 +9,8 @@ import {
ModelProviderInstance,
AVAILABLE_PROVIDERS,
ModelProviderMeta,
ModelLister,
ModelInfo,
} from "@omnia/llm";
function resolveScenarioPath(relative: string): string {
@@ -317,3 +319,32 @@ export async function regenerateEmbeddings(
): Promise<void> {
await simulationManager.regenerateAllEmbeddings(newProviderInstanceId);
}
/**
* Fetch available models for a provider given its credentials.
* Used when creating a new instance (before it's saved to the DB).
*/
export async function fetchAvailableModels(
providerName: string,
apiKey: string,
endpointUrl?: string,
): Promise<ModelInfo[]> {
return ModelLister.listModels(providerName, apiKey, endpointUrl);
}
/**
* Fetch available models for an existing saved provider instance.
* The API key is retrieved from the DB server-side — never sent to the client.
*/
export async function fetchAvailableModelsForInstance(
instanceId: string,
): Promise<ModelInfo[]> {
const instances = ProviderManager.list();
const inst = instances.find((i) => i.id === instanceId);
if (!inst) return [];
return ModelLister.listModels(
inst.providerName,
inst.apiKey,
inst.endpointUrl,
);
}

View File

@@ -1,14 +1,20 @@
"use client";
import { useEffect, useState } from "react";
import { useEffect, useRef, useState } from "react";
import {
createProviderInstance,
updateProviderInstance,
setActiveProviderInstance,
regenerateEmbeddings,
deleteProviderInstance,
fetchAvailableModels,
fetchAvailableModelsForInstance,
} from "@/app/actions";
import type { ModelProviderInstance, ModelProviderMeta } from "@omnia/llm";
import type {
ModelInfo,
ModelProviderInstance,
ModelProviderMeta,
} from "@omnia/llm";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
@@ -22,6 +28,14 @@ import {
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import {
Combobox,
ComboboxContent,
ComboboxEmpty,
ComboboxInput,
ComboboxItem,
ComboboxList,
} from "@/components/ui/combobox";
import {
Card,
CardHeader,
@@ -38,6 +52,7 @@ import {
} from "@/components/ui/item";
import { Empty, EmptyTitle, EmptyDescription } from "@/components/ui/empty";
import { cn } from "@/lib/utils";
import { RefreshCwIcon } from "lucide-react";
interface ProviderInstancesConfigProps {
instances: ModelProviderInstance[];
@@ -68,6 +83,61 @@ export function ProviderInstancesConfig({
const [loading, setLoading] = useState(false);
const [error, setError] = useState("");
// Model listing state
const [availableModels, setAvailableModels] = useState<ModelInfo[]>([]);
const [modelsLoading, setModelsLoading] = useState(false);
const debounceTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
// Fetch models for an existing saved instance
const fetchModelsForExistingInstance = async (instanceId: string) => {
setModelsLoading(true);
setAvailableModels([]);
try {
const models = await fetchAvailableModelsForInstance(instanceId);
setAvailableModels(models);
} catch {
setAvailableModels([]);
} finally {
setModelsLoading(false);
}
};
// Fetch models for a new instance (using live key/endpoint from form)
const fetchModelsForNewInstance = (
provider: string,
apiKey: string,
endpointUrl: string,
) => {
if (debounceTimer.current) clearTimeout(debounceTimer.current);
const isOllama = provider === "ollama";
const hasCredentials = isOllama
? endpointUrl.trim().length > 0
: apiKey.trim().length > 8; // don't spam API with partial keys
if (!hasCredentials) {
setAvailableModels([]);
return;
}
debounceTimer.current = setTimeout(async () => {
setModelsLoading(true);
setAvailableModels([]);
try {
const models = await fetchAvailableModels(
provider,
isOllama ? "none" : apiKey,
isOllama ? endpointUrl : undefined,
);
setAvailableModels(models);
} catch {
setAvailableModels([]);
} finally {
setModelsLoading(false);
}
}, 600);
};
useEffect(() => {
if (selectedInstanceId === null) {
setEditName("");
@@ -78,6 +148,7 @@ export function ProviderInstancesConfig({
setEditType("generative");
setEditMaxContext(32768);
setEditEndpointUrl("");
setAvailableModels([]);
} else if (selectedInstanceId === "new") {
setEditName("");
const defaultProvider = "google-genai";
@@ -89,6 +160,7 @@ export function ProviderInstancesConfig({
setEditIsActive(false);
setEditMaxContext(32768);
setEditEndpointUrl("");
setAvailableModels([]);
} else {
const inst = instances.find((i) => i.id === selectedInstanceId);
if (inst) {
@@ -113,10 +185,21 @@ export function ProviderInstancesConfig({
: 32768,
);
setEditEndpointUrl(inst.endpointUrl || "");
setAvailableModels([]);
// Auto-fetch models for existing instances
fetchModelsForExistingInstance(selectedInstanceId);
}
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedInstanceId, instances, availableProviders]);
// Re-fetch models when provider/key/endpoint changes on new instance form
useEffect(() => {
if (selectedInstanceId !== "new") return;
fetchModelsForNewInstance(editProvider, editKey, editEndpointUrl);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [editProvider, editKey, editEndpointUrl, selectedInstanceId]);
const handleProviderChange = (providerId: string | null) => {
if (!providerId) return;
setEditProvider(providerId);
@@ -126,6 +209,7 @@ export function ProviderInstancesConfig({
? pMeta?.defaultEmbeddingModel || ""
: pMeta?.defaultModel || "",
);
setAvailableModels([]);
};
const handleTypeChange = (type: "generative" | "embedding") => {
@@ -138,6 +222,14 @@ export function ProviderInstancesConfig({
);
};
const handleRefreshModels = () => {
if (selectedInstanceId && selectedInstanceId !== "new") {
fetchModelsForExistingInstance(selectedInstanceId);
} else {
fetchModelsForNewInstance(editProvider, editKey, editEndpointUrl);
}
};
const handleSave = async (e: React.FormEvent) => {
e.preventDefault();
if (!editName.trim()) {
@@ -440,13 +532,77 @@ export function ProviderInstancesConfig({
)}
<div className="flex flex-col gap-1.5">
<Label htmlFor="formModel">Model Name</Label>
<Input
id="formModel"
<div className="flex items-center justify-between">
<Label htmlFor="formModel">Model</Label>
<button
type="button"
onClick={handleRefreshModels}
disabled={modelsLoading}
className="flex items-center gap-1 rounded px-1.5 py-0.5 text-xs text-muted-foreground transition-colors hover:bg-muted hover:text-foreground disabled:opacity-40"
title="Refresh model list"
>
<RefreshCwIcon
className={cn(
"size-3",
modelsLoading && "animate-spin",
)}
/>
{modelsLoading
? "Fetching…"
: availableModels.length > 0
? `${availableModels.length} models`
: "Fetch models"}
</button>
</div>
<Combobox
value={editModel}
onChange={(e) => setEditModel(e.target.value)}
placeholder="e.g. gemini-2.5-flash, gemini-2.5-pro"
/>
onValueChange={(v) => {
if (v) setEditModel(v);
}}
items={availableModels.map((m) => m.id)}
>
<ComboboxInput
id="formModel"
placeholder={
modelsLoading
? "Fetching models…"
: availableModels.length > 0
? "Search or select a model…"
: "e.g. gemini-2.5-flash"
}
disabled={modelsLoading}
showClear={false}
value={editModel}
onChange={(e) =>
setEditModel((e.target as HTMLInputElement).value)
}
className="w-full"
/>
<ComboboxContent>
<ComboboxEmpty>
{modelsLoading
? "Fetching models…"
: "No models found. Type a custom model name above."}
</ComboboxEmpty>
<ComboboxList>
{(modelId: string) => {
const model = availableModels.find(
(m) => m.id === modelId,
);
return (
<ComboboxItem key={modelId} value={modelId}>
<span className="flex-1 truncate">{modelId}</span>
{model?.ownedBy && (
<span className="ml-2 shrink-0 text-xs text-muted-foreground">
{model.ownedBy}
</span>
)}
</ComboboxItem>
);
}}
</ComboboxList>
</ComboboxContent>
</Combobox>
</div>
{editType === "generative" && (

View File

@@ -0,0 +1,300 @@
"use client";
import * as React from "react";
import { Combobox as ComboboxPrimitive } from "@base-ui/react";
import { CheckIcon, ChevronDownIcon, XIcon } from "lucide-react";
import { cn } from "@/lib/utils";
import { Button } from "@/components/ui/button";
import {
InputGroup,
InputGroupAddon,
InputGroupButton,
InputGroupInput,
} from "@/components/ui/input-group";
const Combobox = ComboboxPrimitive.Root;
function ComboboxValue({ ...props }: ComboboxPrimitive.Value.Props) {
return <ComboboxPrimitive.Value data-slot="combobox-value" {...props} />;
}
function ComboboxTrigger({
className,
children,
...props
}: ComboboxPrimitive.Trigger.Props) {
return (
<ComboboxPrimitive.Trigger
data-slot="combobox-trigger"
className={cn("[&_svg:not([class*='size-'])]:size-4", className)}
{...props}
>
{children}
<ChevronDownIcon className="pointer-events-none size-4 text-muted-foreground" />
</ComboboxPrimitive.Trigger>
);
}
function ComboboxClear({ className, ...props }: ComboboxPrimitive.Clear.Props) {
return (
<ComboboxPrimitive.Clear
data-slot="combobox-clear"
render={<InputGroupButton variant="ghost" size="icon-xs" />}
className={cn(className)}
{...props}
>
<XIcon className="pointer-events-none" />
</ComboboxPrimitive.Clear>
);
}
function ComboboxInput({
className,
children,
disabled = false,
showTrigger = true,
showClear = false,
...props
}: ComboboxPrimitive.Input.Props & {
showTrigger?: boolean;
showClear?: boolean;
}) {
return (
<InputGroup className={cn("w-auto", className)}>
<ComboboxPrimitive.Input
render={<InputGroupInput disabled={disabled} />}
{...props}
/>
<InputGroupAddon align="inline-end">
{showTrigger && (
<InputGroupButton
size="icon-xs"
variant="ghost"
render={<ComboboxTrigger />}
data-slot="input-group-button"
className="group-has-data-[slot=combobox-clear]/input-group:hidden data-pressed:bg-transparent"
disabled={disabled}
/>
)}
{showClear && <ComboboxClear disabled={disabled} />}
</InputGroupAddon>
{children}
</InputGroup>
);
}
function ComboboxContent({
className,
side = "bottom",
sideOffset = 6,
align = "start",
alignOffset = 0,
anchor,
...props
}: ComboboxPrimitive.Popup.Props &
Pick<
ComboboxPrimitive.Positioner.Props,
"side" | "align" | "sideOffset" | "alignOffset" | "anchor"
>) {
return (
<ComboboxPrimitive.Portal>
<ComboboxPrimitive.Positioner
side={side}
sideOffset={sideOffset}
align={align}
alignOffset={alignOffset}
anchor={anchor}
className="isolate z-50"
>
<ComboboxPrimitive.Popup
data-slot="combobox-content"
data-chips={!!anchor}
className={cn(
"cn-menu-target cn-menu-translucent group/combobox-content relative max-h-(--available-height) w-(--anchor-width) max-w-(--available-width) min-w-[calc(var(--anchor-width)+var(--spacing-7,1.75rem))] origin-(--transform-origin) overflow-hidden rounded border-2 bg-popover text-popover-foreground shadow-md duration-100 data-[chips=true]:min-w-(--anchor-width) data-[side=bottom]:slide-in-from-top-2 data-[side=inline-end]:slide-in-from-left-2 data-[side=inline-start]:slide-in-from-right-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 *:data-[slot=input-group]:m-1 *:data-[slot=input-group]:mb-0 *:data-[slot=input-group]:h-8 *:data-[slot=input-group]:border-input/30 *:data-[slot=input-group]:bg-input/30 *:data-[slot=input-group]:shadow-none data-open:animate-in data-open:fade-in-0 data-open:zoom-in-95 data-closed:animate-out data-closed:fade-out-0 data-closed:zoom-out-95",
className,
)}
{...props}
/>
</ComboboxPrimitive.Positioner>
</ComboboxPrimitive.Portal>
);
}
function ComboboxList({ className, ...props }: ComboboxPrimitive.List.Props) {
return (
<ComboboxPrimitive.List
data-slot="combobox-list"
className={cn(
"no-scrollbar max-h-72 scroll-py-1 overflow-y-auto overscroll-contain p-1 data-empty:p-0",
className,
)}
{...props}
/>
);
}
function ComboboxItem({
className,
children,
...props
}: ComboboxPrimitive.Item.Props) {
return (
<ComboboxPrimitive.Item
data-slot="combobox-item"
className={cn(
"relative flex w-full cursor-default items-center gap-2 rounded-sm py-1 pr-8 pl-1.5 text-sm outline-hidden select-none data-highlighted:bg-accent data-highlighted:text-accent-foreground not-data-[variant=destructive]:data-highlighted:**:text-accent-foreground data-disabled:pointer-events-none data-disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
className,
)}
{...props}
>
{children}
<ComboboxPrimitive.ItemIndicator
render={
<span className="pointer-events-none absolute right-2 flex size-4 items-center justify-center" />
}
>
<CheckIcon className="pointer-events-none" />
</ComboboxPrimitive.ItemIndicator>
</ComboboxPrimitive.Item>
);
}
function ComboboxGroup({ className, ...props }: ComboboxPrimitive.Group.Props) {
return (
<ComboboxPrimitive.Group
data-slot="combobox-group"
className={cn(className)}
{...props}
/>
);
}
function ComboboxLabel({
className,
...props
}: ComboboxPrimitive.GroupLabel.Props) {
return (
<ComboboxPrimitive.GroupLabel
data-slot="combobox-label"
className={cn("px-2 py-1.5 text-xs text-muted-foreground", className)}
{...props}
/>
);
}
function ComboboxCollection({ ...props }: ComboboxPrimitive.Collection.Props) {
return (
<ComboboxPrimitive.Collection data-slot="combobox-collection" {...props} />
);
}
function ComboboxEmpty({ className, ...props }: ComboboxPrimitive.Empty.Props) {
return (
<ComboboxPrimitive.Empty
data-slot="combobox-empty"
className={cn(
"hidden w-full justify-center py-2 text-center text-sm text-muted-foreground group-data-empty/combobox-content:flex",
className,
)}
{...props}
/>
);
}
function ComboboxSeparator({
className,
...props
}: ComboboxPrimitive.Separator.Props) {
return (
<ComboboxPrimitive.Separator
data-slot="combobox-separator"
className={cn("-mx-1 my-1 h-px bg-border", className)}
{...props}
/>
);
}
function ComboboxChips({
className,
...props
}: React.ComponentPropsWithRef<typeof ComboboxPrimitive.Chips> &
ComboboxPrimitive.Chips.Props) {
return (
<ComboboxPrimitive.Chips
data-slot="combobox-chips"
className={cn(
"flex min-h-8 flex-wrap items-center gap-1 rounded border-2 bg-input bg-clip-padding px-2.5 py-1 text-sm shadow-sm transition-colors focus-within:outline-2 focus-within:outline-offset-2 focus-within:outline-primary has-aria-invalid:border-destructive has-data-[slot=combobox-chip]:px-1",
className,
)}
{...props}
/>
);
}
function ComboboxChip({
className,
children,
showRemove = true,
...props
}: ComboboxPrimitive.Chip.Props & {
showRemove?: boolean;
}) {
return (
<ComboboxPrimitive.Chip
data-slot="combobox-chip"
className={cn(
"flex h-[calc(var(--spacing,0.25rem)*5.25)] w-fit items-center justify-center gap-1 rounded-sm border-2 bg-muted px-1.5 text-xs font-medium whitespace-nowrap text-foreground has-disabled:pointer-events-none has-disabled:cursor-not-allowed has-disabled:opacity-50 has-data-[slot=combobox-chip-remove]:pr-0",
className,
)}
{...props}
>
{children}
{showRemove && (
<ComboboxPrimitive.ChipRemove
render={<InputGroupButton variant="ghost" size="icon-xs" />}
className="-ml-1 opacity-50 hover:opacity-100"
data-slot="combobox-chip-remove"
>
<XIcon className="pointer-events-none" />
</ComboboxPrimitive.ChipRemove>
)}
</ComboboxPrimitive.Chip>
);
}
function ComboboxChipsInput({
className,
...props
}: ComboboxPrimitive.Input.Props) {
return (
<ComboboxPrimitive.Input
data-slot="combobox-chip-input"
className={cn("min-w-16 flex-1 outline-none", className)}
{...props}
/>
);
}
function useComboboxAnchor() {
return React.useRef<HTMLDivElement | null>(null);
}
export {
Combobox,
ComboboxInput,
ComboboxContent,
ComboboxList,
ComboboxItem,
ComboboxGroup,
ComboboxLabel,
ComboboxCollection,
ComboboxEmpty,
ComboboxSeparator,
ComboboxChips,
ComboboxChip,
ComboboxChipsInput,
ComboboxTrigger,
ComboboxValue,
useComboboxAnchor,
};

View File

@@ -0,0 +1,172 @@
"use client";
import * as React from "react";
import { cva, type VariantProps } from "class-variance-authority";
import { cn } from "@/lib/utils";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
function InputGroup({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="input-group"
role="group"
className={cn(
"group/input-group relative flex h-8 w-full min-w-0 items-center rounded border-2 bg-input shadow-sm transition-colors outline-none in-data-[slot=combobox-content]:focus-within:border-inherit in-data-[slot=combobox-content]:focus-within:ring-0 has-disabled:bg-input/50 has-disabled:opacity-50 has-[[data-slot=input-group-control]:focus-visible]:outline-2 has-[[data-slot=input-group-control]:focus-visible]:outline-offset-2 has-[[data-slot=input-group-control]:focus-visible]:outline-primary has-[[data-slot][aria-invalid=true]]:border-destructive has-[>[data-align=block-end]]:h-auto has-[>[data-align=block-end]]:flex-col has-[>[data-align=block-start]]:h-auto has-[>[data-align=block-start]]:flex-col has-[>textarea]:h-auto has-[>[data-align=block-end]]:[&>input]:pt-3 has-[>[data-align=block-start]]:[&>input]:pb-3 has-[>[data-align=inline-end]]:[&>input]:pr-1.5 has-[>[data-align=inline-start]]:[&>input]:pl-1.5",
className,
)}
{...props}
/>
);
}
const inputGroupAddonVariants = cva(
"flex h-auto cursor-text items-center justify-center gap-2 py-1.5 text-sm font-medium text-muted-foreground select-none group-data-[disabled=true]/input-group:opacity-50 [&>kbd]:rounded-[calc(var(--radius)-5px)] [&>svg:not([class*='size-'])]:size-4",
{
variants: {
align: {
"inline-start":
"order-first pl-2 has-[>button]:ml-[-0.3rem] has-[>kbd]:ml-[-0.15rem]",
"inline-end":
"order-last pr-2 has-[>button]:mr-[-0.3rem] has-[>kbd]:mr-[-0.15rem]",
"block-start":
"order-first w-full justify-start px-2.5 pt-2 group-has-[>input]/input-group:pt-2 [.border-b]:pb-2",
"block-end":
"order-last w-full justify-start px-2.5 pb-2 group-has-[>input]/input-group:pb-2 [.border-t]:pt-2",
},
},
defaultVariants: {
align: "inline-start",
},
},
);
function InputGroupAddon({
className,
align = "inline-start",
...props
}: React.ComponentProps<"div"> & VariantProps<typeof inputGroupAddonVariants>) {
return (
<div
role="group"
data-slot="input-group-addon"
data-align={align}
className={cn(inputGroupAddonVariants({ align }), className)}
onClick={(e) => {
if ((e.target as HTMLElement).closest("button")) {
return;
}
e.currentTarget.parentElement?.querySelector("input")?.focus();
}}
{...props}
/>
);
}
const inputGroupButtonVariants = cva(
"flex items-center gap-2 text-sm shadow-none",
{
variants: {
size: {
xs: "h-6 gap-1 rounded-[calc(var(--radius)-3px)] px-1.5 [&>svg:not([class*='size-'])]:size-3.5",
sm: "",
"icon-xs":
"size-6 rounded-[calc(var(--radius)-3px)] p-0 has-[>svg]:p-0",
"icon-sm": "size-8 p-0 has-[>svg]:p-0",
},
},
defaultVariants: {
size: "xs",
},
},
);
function InputGroupButton({
className,
type = "button",
variant = "ghost",
size = "xs",
render,
...props
}: Omit<React.ComponentProps<typeof Button>, "size" | "type"> &
VariantProps<typeof inputGroupButtonVariants> & {
type?: "button" | "submit" | "reset";
render?: React.ReactElement;
}) {
if (render) {
return React.cloneElement(render, {
className: cn(
inputGroupButtonVariants({ size }),
(render.props as any)?.className,
className,
),
type,
...props,
} as any);
}
return (
<Button
type={type}
data-size={size}
variant={variant}
className={cn(inputGroupButtonVariants({ size }), className)}
{...props}
/>
);
}
function InputGroupText({ className, ...props }: React.ComponentProps<"span">) {
return (
<span
className={cn(
"flex items-center gap-2 text-sm text-muted-foreground [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4",
className,
)}
{...props}
/>
);
}
function InputGroupInput({
className,
...props
}: React.ComponentProps<"input">) {
return (
<Input
data-slot="input-group-control"
className={cn(
"flex-1 rounded-none border-0 bg-transparent shadow-none outline-none focus-visible:outline-none disabled:bg-transparent aria-invalid:outline-none dark:bg-transparent dark:disabled:bg-transparent",
className,
)}
{...props}
/>
);
}
function InputGroupTextarea({
className,
...props
}: React.ComponentProps<"textarea">) {
return (
<Textarea
data-slot="input-group-control"
className={cn(
"flex-1 resize-none rounded-none border-0 bg-transparent py-2 shadow-none outline-none focus-visible:outline-none disabled:bg-transparent aria-invalid:outline-none dark:bg-transparent dark:disabled:bg-transparent",
className,
)}
{...props}
/>
);
}
export {
InputGroup,
InputGroupAddon,
InputGroupButton,
InputGroupText,
InputGroupInput,
InputGroupTextarea,
};

View File

@@ -450,6 +450,38 @@ The `buildLLMProvider()` and `buildEmbeddingProvider()` functions perform the fi
| `"deepseek"` | `DeepSeekProvider` | _(falls through to mock)_ |
| _(anything else)_ | `MockLLMProvider` | `MockEmbeddingProvider` |
## Model Listing and Discovery
The `ModelLister` class provides a unified interface to dynamically query available models from the remote provider APIs.
### Caching and TTL
All list requests are cached in-memory with a **5-minute TTL** (`300,000ms`) to prevent rapid, repetitive remote API requests and avoid rate limit exhaustion.
- **Cache Key**: Generated using `providerName` combined with either the `apiKey` or `endpointUrl`.
- **Invalidation**: Call `ModelLister.invalidateCache(providerName, apiKey, endpointUrl)` to clear cache for specific instances, or `ModelLister.clearCache()` to wipe all lists.
### Provider Integration Details
| Provider | Endpoint | Auth Header | Pagination |
| ----------------- | ---------------------------- | ----------------------- | --------------------------- |
| **Google Gemini** | `GET /v1beta/models?key=KEY` | Query Param | ✅ Loop via `nextPageToken` |
| **OpenAI** | `GET /v1/models` | `Authorization: Bearer` | ❌ |
| **Anthropic** | `GET /v1/models` | `x-api-key` | ✅ Loop via `after_id` |
| **Groq** | `GET /openai/v1/models` | `Authorization: Bearer` | ❌ |
| **DeepSeek** | `GET /models` | `Authorization: Bearer` | ❌ |
| **Ollama** | `GET /api/tags` | None (Local) | ❌ |
| **OpenRouter** | `GET /api/v1/models` | Optional Bearer | ❌ |
| **Mock** | Instant return (no fetch) | — | — |
### Methods
- `listModels(providerName: string, apiKey: string, endpointUrl?: string): Promise<ModelInfo[]>`
- `invalidateCache(providerName: string, apiKey: string, endpointUrl?: string): void`
- `clearCache(): void`
---
## Structured Output
All real providers use LangChain's `.withStructuredOutput(schema, { includeRaw: true })` pattern:
@@ -489,6 +521,7 @@ packages/llm/
│ ├── index.ts # Re-exports everything
│ ├── llm.ts # Interfaces, types, AVAILABLE_PROVIDERS
│ ├── config.ts # Env var parsing (Zod)
│ ├── model-lister.ts # ModelLister with cache and API fetching
│ ├── provider-manager.ts # ProviderManager (SQLite CRUD)
│ └── providers/
│ ├── google-genai.ts # GeminiProvider + GeminiEmbeddingProvider
@@ -502,6 +535,7 @@ packages/llm/
├── tests/
│ ├── mock.test.ts
│ ├── openrouter.test.ts
│ ├── model-lister.test.ts # ModelLister cache and fetch logic unit tests
│ └── provider-manager.test.ts
└── package.json
```

View File

@@ -1,5 +1,6 @@
export * from "./llm.js";
export * from "./config.js";
export * from "./model-lister.js";
export * from "./providers/google-genai.js";
export * from "./providers/mock.js";
export * from "./providers/ollama.js";

View File

@@ -0,0 +1,261 @@
/**
* ModelLister — fetches available models from each provider's REST API.
* Results are cached in-memory with a 5-minute TTL to avoid repeated calls.
*/
export interface ModelInfo {
id: string;
name: string;
ownedBy?: string;
}
interface CacheEntry {
models: ModelInfo[];
fetchedAt: number;
}
const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
const FETCH_TIMEOUT_MS = 10_000; // 10 seconds
// Cache key is "providerName:apiKey-or-endpoint" (we don't hash since it's in-process)
const modelCache = new Map<string, CacheEntry>();
function cacheKey(
providerName: string,
apiKey: string,
endpointUrl?: string,
): string {
return `${providerName}:${endpointUrl || apiKey}`;
}
async function fetchWithTimeout(
url: string,
init?: RequestInit,
): Promise<Response> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
try {
return await fetch(url, { ...init, signal: controller.signal });
} finally {
clearTimeout(timer);
}
}
async function fetchGeminiModels(apiKey: string): Promise<ModelInfo[]> {
const models: ModelInfo[] = [];
let pageToken: string | undefined;
do {
const url = new URL(
"https://generativelanguage.googleapis.com/v1beta/models",
);
url.searchParams.set("key", apiKey);
url.searchParams.set("pageSize", "100");
if (pageToken) {
url.searchParams.set("pageToken", pageToken);
}
const res = await fetchWithTimeout(url.toString());
if (!res.ok) return models;
const json = (await res.json()) as {
models?: { name: string; displayName?: string }[];
nextPageToken?: string;
};
for (const m of json.models ?? []) {
// m.name is like "models/gemini-2.5-flash"; strip "models/" prefix
const id = m.name.replace(/^models\//, "");
models.push({ id, name: m.displayName || id });
}
pageToken = json.nextPageToken;
} while (pageToken);
return models;
}
async function fetchOpenAICompatibleModels(
baseUrl: string,
apiKey: string,
): Promise<ModelInfo[]> {
const res = await fetchWithTimeout(`${baseUrl}/models`, {
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json",
},
});
if (!res.ok) return [];
const json = (await res.json()) as {
data?: { id: string; owned_by?: string; name?: string }[];
};
return (json.data ?? []).map((m) => ({
id: m.id,
name: m.name || m.id,
ownedBy: m.owned_by,
}));
}
async function fetchAnthropicModels(apiKey: string): Promise<ModelInfo[]> {
const models: ModelInfo[] = [];
let afterId: string | undefined;
do {
const url = new URL("https://api.anthropic.com/v1/models");
url.searchParams.set("limit", "1000");
if (afterId) {
url.searchParams.set("after_id", afterId);
}
const res = await fetchWithTimeout(url.toString(), {
headers: {
"x-api-key": apiKey,
"anthropic-version": "2023-06-01",
Accept: "application/json",
},
});
if (!res.ok) return models;
const json = (await res.json()) as {
data?: { id: string; display_name?: string }[];
has_more?: boolean;
last_id?: string;
};
for (const m of json.data ?? []) {
models.push({ id: m.id, name: m.display_name || m.id });
}
afterId = json.has_more ? json.last_id : undefined;
} while (afterId);
return models;
}
async function fetchOllamaModels(endpointUrl: string): Promise<ModelInfo[]> {
const base = endpointUrl.replace(/\/$/, "");
const res = await fetchWithTimeout(`${base}/api/tags`);
if (!res.ok) return [];
const json = (await res.json()) as {
models?: { name: string; model?: string }[];
};
return (json.models ?? []).map((m) => ({
id: m.name,
name: m.name,
}));
}
async function fetchOpenRouterModels(apiKey: string): Promise<ModelInfo[]> {
const res = await fetchWithTimeout(
"https://openrouter.ai/api/v1/models",
apiKey
? {
headers: {
Authorization: `Bearer ${apiKey}`,
Accept: "application/json",
},
}
: { headers: { Accept: "application/json" } },
);
if (!res.ok) return [];
const json = (await res.json()) as {
data?: { id: string; name?: string; owned_by?: string }[];
};
return (json.data ?? []).map((m) => ({
id: m.id,
name: m.name || m.id,
ownedBy: m.owned_by,
}));
}
export class ModelLister {
/**
* List available models for a given provider. Results are cached for 5 minutes.
*
* @param providerName The provider ID (e.g. "openai", "google-genai")
* @param apiKey The API key for the provider (or "none" for Ollama)
* @param endpointUrl The endpoint URL (required for Ollama, ignored otherwise)
* @returns Array of ModelInfo objects, or [] on any error
*/
static async listModels(
providerName: string,
apiKey: string,
endpointUrl?: string,
): Promise<ModelInfo[]> {
if (providerName === "mock") {
return [{ id: "mock", name: "Mock Model" }];
}
const key = cacheKey(providerName, apiKey, endpointUrl);
const cached = modelCache.get(key);
if (cached && Date.now() - cached.fetchedAt < CACHE_TTL_MS) {
return cached.models;
}
let models: ModelInfo[] = [];
try {
switch (providerName) {
case "google-genai":
models = await fetchGeminiModels(apiKey);
break;
case "openai":
models = await fetchOpenAICompatibleModels(
"https://api.openai.com/v1",
apiKey,
);
break;
case "anthropic":
models = await fetchAnthropicModels(apiKey);
break;
case "groq":
models = await fetchOpenAICompatibleModels(
"https://api.groq.com/openai/v1",
apiKey,
);
break;
case "deepseek":
models = await fetchOpenAICompatibleModels(
"https://api.deepseek.com",
apiKey,
);
break;
case "ollama":
models = await fetchOllamaModels(
endpointUrl || "http://localhost:11434",
);
break;
case "openrouter":
models = await fetchOpenRouterModels(apiKey);
break;
default:
models = [];
}
} catch {
// Network error, invalid key, timeout — return empty array for graceful degradation
models = [];
}
modelCache.set(key, { models, fetchedAt: Date.now() });
return models;
}
/** Invalidate the cache entry for a specific provider+key combination. */
static invalidateCache(
providerName: string,
apiKey: string,
endpointUrl?: string,
): void {
modelCache.delete(cacheKey(providerName, apiKey, endpointUrl));
}
/** Clear the entire model cache. */
static clearCache(): void {
modelCache.clear();
}
}

View File

@@ -0,0 +1,120 @@
import { describe, test, expect, vi, beforeEach, afterEach } from "vitest";
import { ModelLister } from "@omnia/llm";
describe("ModelLister Unit Tests (Tier 1)", () => {
beforeEach(() => {
vi.stubGlobal("fetch", vi.fn());
ModelLister.clearCache();
});
afterEach(() => {
vi.restoreAllMocks();
});
test("returns mock provider list instantly without fetch", async () => {
const models = await ModelLister.listModels("mock", "none");
expect(models).toEqual([{ id: "mock", name: "Mock Model" }]);
expect(fetch).not.toHaveBeenCalled();
});
test("fetches and caches OpenAI-compatible models", async () => {
const mockResponse = {
data: [
{ id: "gpt-4o", owned_by: "openai" },
{ id: "gpt-4o-mini", owned_by: "openai" },
],
};
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => mockResponse,
});
vi.stubGlobal("fetch", mockFetch);
// First call: Should fetch
const models = await ModelLister.listModels("openai", "test-key");
expect(models).toEqual([
{ id: "gpt-4o", name: "gpt-4o", ownedBy: "openai" },
{ id: "gpt-4o-mini", name: "gpt-4o-mini", ownedBy: "openai" },
]);
expect(mockFetch).toHaveBeenCalledTimes(1);
expect(mockFetch).toHaveBeenCalledWith(
"https://api.openai.com/v1/models",
expect.objectContaining({
headers: {
Authorization: "Bearer test-key",
Accept: "application/json",
},
}),
);
// Second call: Should read from cache
const cachedModels = await ModelLister.listModels("openai", "test-key");
expect(cachedModels).toEqual(models);
expect(mockFetch).toHaveBeenCalledTimes(1);
});
test("respects cache invalidation", async () => {
const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ data: [{ id: "model-1" }] }),
});
vi.stubGlobal("fetch", mockFetch);
await ModelLister.listModels("openai", "test-key");
expect(mockFetch).toHaveBeenCalledTimes(1);
// Invalidate
ModelLister.invalidateCache("openai", "test-key");
// Second call: Should fetch again
await ModelLister.listModels("openai", "test-key");
expect(mockFetch).toHaveBeenCalledTimes(2);
});
test("gracefully returns empty array on fetch failure", async () => {
const mockFetch = vi.fn().mockResolvedValue({
ok: false,
status: 500,
});
vi.stubGlobal("fetch", mockFetch);
const models = await ModelLister.listModels("openai", "bad-key");
expect(models).toEqual([]);
expect(mockFetch).toHaveBeenCalledTimes(1);
});
test("handles Gemini pagination correctly", async () => {
const mockFetch = vi
.fn()
.mockResolvedValueOnce({
ok: true,
json: async () => ({
models: [
{
name: "models/gemini-2.5-flash",
displayName: "Gemini 2.5 Flash",
},
],
nextPageToken: "token-1",
}),
})
.mockResolvedValueOnce({
ok: true,
json: async () => ({
models: [
{ name: "models/gemini-2.5-pro", displayName: "Gemini 2.5 Pro" },
],
}),
});
vi.stubGlobal("fetch", mockFetch);
const models = await ModelLister.listModels("google-genai", "gemini-key");
expect(models).toEqual([
{ id: "gemini-2.5-flash", name: "Gemini 2.5 Flash" },
{ id: "gemini-2.5-pro", name: "Gemini 2.5 Pro" },
]);
expect(mockFetch).toHaveBeenCalledTimes(2);
});
});