From 9e0325f90923c68751380312b10595dc1e6798a1 Mon Sep 17 00:00:00 2001 From: Aditya Gupta Date: Mon, 6 Jul 2026 08:33:39 +0530 Subject: [PATCH] major: Implement 3-tiered testing architecture see [docs/testing.md](./docs/testing.md) --- docs/testing.md | 102 +++++++++++++++++++++++++++++++ package.json | 6 +- tests/evals/google-genai.eval.ts | 31 ++++++++++ vitest.config.evals.ts | 8 +++ vitest.config.ts | 11 ++++ vitest.workspace.ts | 6 ++ 6 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 docs/testing.md create mode 100644 tests/evals/google-genai.eval.ts create mode 100644 vitest.config.evals.ts create mode 100644 vitest.config.ts create mode 100644 vitest.workspace.ts diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..325288d --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,102 @@ +# Testing Strategy + +This document outlines the testing architecture for the Omnia project. The central design problem is that most of the system is deterministic and highly testable, but a few critical parts (specifically, LLM behavior) are non-deterministic. + +Treating these two categories the same way—either mocking everything (which provides false confidence) or hitting the real LLM API for everything (which is slow, expensive, flaky, and non-repeatable)—is an anti-pattern. Our test architecture makes this split explicit rather than papering over it. + +## Test Tiers + +We use a three-tiered system, categorized by what the tests actually depend on. + +### Tier 1: Unit Tests (Per-Package, No LLM) + +Unit tests reside within each package's `tests/` directory. They do not use LLMs and do not perform I/O. + +This tier should cover the majority of the codebase, as most of Omnia's specific logic is deterministic. Examples of logic that must be fully covered by unit tests include: + +- `hasAccess()` / ACL grant-revoke logic. +- `addAttribute` rejecting duplicate names. +- `WorldClock.advance()` / `getTimeOfDay()` boundaries. +- The spatial bubble-up algorithm (fully mechanical: given a constructed graph and portal properties, assert exactly who perceives what at each step). +- Zod schemas rejecting malformed input at each boundary. + +**Execution:** Runs on every save and every commit. + +### Tier 2: Integration Tests (Cross-Package, Mocked LLM) + +Integration tests live in the root `tests/integration/` directory. They test cross-package flows using a mocked LLM to ensure speed, determinism, and zero cost. + +This is where the `MockLLMProvider` earns its keep. It implements the `ILLMProvider` interface and returns canned responses that satisfy whatever Zod schema is requested. + +**Mock Implementation:** + +```typescript +export class MockLLMProvider implements ILLMProvider { + providerName = "mock"; + constructor(private responses: unknown[]) {} + private callCount = 0; + + async generateStructuredResponse( + request: LLMRequest, + ): Promise>> { + const next = this.responses[this.callCount++]; + return { success: true, data: request.schema.parse(next) }; + } +} +``` + +This tier tests our actual logic—e.g., does the Architect apply a delta correctly? Does the consequence generator mutate `WorldState` correctly? Does a scripted CLI conversation end in the expected state?—without ever depending on the model behaving a particular way. + +**Shared Contract Suite** +Included in this tier is a shared contract test suite run against _both_ `MockLLMProvider` and `GeminiProvider`. The suite verifies: + +1. Given a schema, the provider returns data matching it. +2. Given a malformed response, it fails predictably. + +This enforces `ILLMProvider`'s reason for existing: real interchangeability, not just nominal conformance. + +### Tier 3: Evals (Real API, Run Deliberately) + +Evals live in the root `tests/evals/` directory. They use real LLM APIs and are run manually via a separate script (`test:evals`), excluded from the default Vitest run. + +This tier is where our privacy guarantee actually lives. It requires a fundamentally different shape of test because of a critical distinction: + +- A Tier 1 test on `hasAccess()` proves the _mechanism_ is correct. +- It says nothing about whether the model, given a correctly-filtered context, actually holds its tongue. +- It says nothing about whether some prompt-building code accidentally used the raw `.attributes/getValue()` path instead of `getVisibleAttributesFor()` (an unenforced-convention risk). + +Both of these failure modes are invisible to unit tests. Therefore, evals are run _N_ times and scored, not asserted once. + +**Example Eval Structure:** + +```typescript +// tests/evals/privacy-leak.eval.ts +const RUNS = 15; +let leaks = 0; + +for (let i = 0; i < RUNS; i++) { + const response = await askAboutPrivateFact(npcWithoutAccess, secretFact); + if (containsFact(response, secretFact)) leaks++; +} + +// Any leak here is a real failure worth investigating, not noise to average away. +expect(leaks).toBe(0); +``` + +**Execution:** Run deliberately (e.g., weekly or pre-release). It costs real money and shouldn't fire on every save. + +## Directory Structure + +```text +omnia/ + packages/ + core/ src/ tests/ # Tier 1: Unit — no I/O, no LLM + intent/ src/ tests/ + spatial/ src/ tests/ + memory/ src/ tests/ + architect/ src/ tests/ + llm/ src/ tests/ # Includes MockLLMProvider + shared contract suite + tests/ + integration/ # Tier 2: Cross-package flows, mocked LLM + evals/ # Tier 3: Real LLM calls, slow/costly/non-deterministic +``` diff --git a/package.json b/package.json index 2cd925e..165d9a3 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,10 @@ "lint:fix": "eslint . --fix", "format": "prettier . --write", "format:check": "prettier . --check", - "watch": "tsc -b --watch" + "watch": "tsc -b --watch", + "test": "vitest run", + "test:watch": "vitest", + "test:evals": "vitest run --config vitest.config.evals.ts" }, "keywords": [], "author": "", @@ -36,6 +39,7 @@ "prettier": "^3.9.4", "typescript": "^6.0.3", "typescript-eslint": "^8.62.1", + "vitest": "^4.1.9", "zod": "^4.4.3" }, "dependencies": { diff --git a/tests/evals/google-genai.eval.ts b/tests/evals/google-genai.eval.ts new file mode 100644 index 0000000..bb18b4b --- /dev/null +++ b/tests/evals/google-genai.eval.ts @@ -0,0 +1,31 @@ +import { describe, test, expect } from "vitest"; +import { z } from "zod"; +import { GeminiProvider } from "../../packages/llm/src/providers/google-genai.js"; +import { llmConfig } from "../../packages/llm/src/config.js"; + +describe("GeminiProvider Eval", () => { + test("structured JSON response against live API", async () => { + expect(llmConfig.GOOGLE_API_KEY).toBeDefined(); + const provider = new GeminiProvider(llmConfig.GOOGLE_API_KEY); + + const ToneSchema = z.object({ + tone: z.enum(["positive", "negative", "neutral"]), + confidence: z.number().min(0).max(1), + explanation: z.string().min(1), + }); + + const response = await provider.generateStructuredResponse({ + systemPrompt: "You are a helpful assistant. Classify the tone of the user's sentence.", + userContext: "I absolutely love this new engine, it works perfectly!", + schema: ToneSchema, + }); + + expect(response.success).toBe(true); + expect(response.data).toBeDefined(); + + const data = response.data!; + expect(data.tone).toBe("positive"); + expect(data.confidence).toBeGreaterThan(0.8); + expect(data.explanation.length).toBeGreaterThan(0); + }); +}); diff --git a/vitest.config.evals.ts b/vitest.config.evals.ts new file mode 100644 index 0000000..c59900b --- /dev/null +++ b/vitest.config.evals.ts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["tests/evals/**/*.eval.ts"], + exclude: ["**/node_modules/**", "**/dist/**"] + } +}); diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..ca4e3ed --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + exclude: [ + "**/node_modules/**", + "**/dist/**", + "tests/evals/**" + ] + } +}); diff --git a/vitest.workspace.ts b/vitest.workspace.ts new file mode 100644 index 0000000..aa2d39e --- /dev/null +++ b/vitest.workspace.ts @@ -0,0 +1,6 @@ +import { defineWorkspace } from "vitest/config"; + +export default defineWorkspace([ + "packages/*", + "tests/*" +]);