diff --git a/README.md b/README.md index 8554c9d..4c3efa9 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Space is a graph: `world → region → location → point of interest`, connect ### Memory Tiers - **Verbatim Buffer (implemented):** Per-character subjective event log. Every entry is stored from the owner's perspective actors resolved through the owner's alias map, outcomes attached — and recalled with naturalized time phrasing. -- **Vector Archive (planned):** Summarized, embedded memory entries for semantic retrieval, keeping verbatim quotes only for high-salience lines. +- **Vector Archive (implemented):** Summarized, embedded memory entries for semantic retrieval, keeping verbatim quotes only for high-salience lines. - **Dossier (planned):** Each observer's subjective beliefs about another character. Memory is per-character on purpose: recall is testimony from a vantage point, which is what makes interrogating two witnesses interesting. diff --git a/apps/gui/src/app/config/page.tsx b/apps/gui/src/app/config/page.tsx index 2a3ebea..af0174c 100644 --- a/apps/gui/src/app/config/page.tsx +++ b/apps/gui/src/app/config/page.tsx @@ -459,6 +459,7 @@ export default function ConfigPage() { { key: "llm-validator", label: "LLM Validator", desc: "Arbitrates and validates proposed actions against the world state rules.", type: "generative" }, { key: "intent-decoder", label: "Intent Decoder", desc: "Splits raw prose actions into structured intents (Player and NPC).", type: "generative" }, { key: "timedelta", label: "TimeDelta Generator", desc: "Calculates the duration of character actions to advance the game clock.", type: "generative" }, + { key: "handoff", label: "Memory Handoff Engine", desc: "Promotes entities' working memories to the long-term Ledger via LLM summarization and pruning.", type: "generative" }, { key: "embeddings", label: "Text Embeddings Generator", desc: "Generates vector embeddings for long-term memory retrieval.", type: "embedding" }, ].map((task) => (
{ + const worldState = session.coreRepo.loadWorldState( + session.worldInstanceId, + ); + if (!worldState) throw new Error("World state lost"); + + const handoffEngine = new HandoffEngine( + session.handoffProvider, + session.embeddingProvider, + session.bufferRepo, + session.ledgerRepo, + ); + + const entities = Array.from(worldState.entities.values()); + for (const entity of entities) { + const bufferEntries = session.bufferRepo.listForOwner(entity.id); + const maxContext = session.handoffProvider.maxContext !== undefined ? session.handoffProvider.maxContext : 32768; + + const trigger = checkHandoffTrigger(entity, bufferEntries, worldState.clock.get(), maxContext); + if (trigger !== "none") { + await handoffEngine.runHandoff(entity, bufferEntries, worldState.clock.get()); + } + } + } + private async runAliasResolution(session: SimSession): Promise { const worldState = session.coreRepo.loadWorldState( session.worldInstanceId, @@ -743,6 +772,7 @@ class SimulationManager { const validatorProvider = resolveProviderForTask("llm-validator"); const decoderProvider = resolveProviderForTask("intent-decoder"); const timedeltaProvider = resolveProviderForTask("timedelta"); + const handoffProvider = resolveProviderForTask("handoff"); const embeddingProvider = resolveEmbeddingProvider(); const architect = new Architect( @@ -769,6 +799,7 @@ class SimulationManager { validatorProvider, decoderProvider, timedeltaProvider, + handoffProvider, embeddingProvider, architect, aliasGenerator, diff --git a/packages/llm/src/llm.ts b/packages/llm/src/llm.ts index 98b12e8..4543144 100644 --- a/packages/llm/src/llm.ts +++ b/packages/llm/src/llm.ts @@ -36,6 +36,7 @@ export interface LLMCallRecord { export interface ILLMProvider { providerName: string; + maxContext?: number; generateStructuredResponse( request: LLMRequest, ): Promise>>; diff --git a/packages/memory/package.json b/packages/memory/package.json index eb9d3ad..069ced9 100644 --- a/packages/memory/package.json +++ b/packages/memory/package.json @@ -9,6 +9,7 @@ "dependencies": { "@omnia/core": "workspace:*", "@omnia/intent": "workspace:*", + "@omnia/llm": "workspace:*", "zod": "^4.4.3" } } diff --git a/packages/memory/src/buffer.ts b/packages/memory/src/buffer.ts index 9132c71..b44998c 100644 --- a/packages/memory/src/buffer.ts +++ b/packages/memory/src/buffer.ts @@ -14,6 +14,7 @@ export interface BufferEntry { isValid: boolean; reason: string; }; + pinned?: boolean; } export { resolveAlias } from "@omnia/core"; @@ -62,23 +63,31 @@ export class BufferRepository { location_id TEXT, intent_json TEXT NOT NULL, outcome_json TEXT, + pinned INTEGER DEFAULT 0, FOREIGN KEY (owner_id) REFERENCES objects(id) ON DELETE CASCADE ); `); + + try { + this.db.exec(`ALTER TABLE buffer_entries ADD COLUMN pinned INTEGER DEFAULT 0;`); + } catch { + // ignore + } } save(entry: BufferEntry): void { this.db .prepare( ` - INSERT INTO buffer_entries (id, owner_id, timestamp, location_id, intent_json, outcome_json) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO buffer_entries (id, owner_id, timestamp, location_id, intent_json, outcome_json, pinned) + VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET owner_id = excluded.owner_id, timestamp = excluded.timestamp, location_id = excluded.location_id, intent_json = excluded.intent_json, - outcome_json = excluded.outcome_json + outcome_json = excluded.outcome_json, + pinned = excluded.pinned `, ) .run( @@ -88,6 +97,7 @@ export class BufferRepository { entry.locationId, JSON.stringify(entry.intent), entry.outcome ? JSON.stringify(entry.outcome) : null, + entry.pinned ? 1 : 0, ); } @@ -95,7 +105,7 @@ export class BufferRepository { const row = this.db .prepare( ` - SELECT id, owner_id, timestamp, location_id, intent_json, outcome_json + SELECT id, owner_id, timestamp, location_id, intent_json, outcome_json, pinned FROM buffer_entries WHERE id = ? `, ) @@ -107,6 +117,7 @@ export class BufferRepository { location_id: string | null; intent_json: string; outcome_json: string | null; + pinned?: number; } | undefined; @@ -119,6 +130,7 @@ export class BufferRepository { locationId: row.location_id, intent: JSON.parse(row.intent_json), outcome: row.outcome_json ? JSON.parse(row.outcome_json) : undefined, + pinned: row.pinned === 1, }; } @@ -126,7 +138,7 @@ export class BufferRepository { const rows = this.db .prepare( ` - SELECT id, owner_id, timestamp, location_id, intent_json, outcome_json + SELECT id, owner_id, timestamp, location_id, intent_json, outcome_json, pinned FROM buffer_entries WHERE owner_id = ? ORDER BY timestamp ASC `, @@ -138,6 +150,7 @@ export class BufferRepository { location_id: string | null; intent_json: string; outcome_json: string | null; + pinned?: number; }[]; return rows.map((row) => ({ @@ -147,6 +160,7 @@ export class BufferRepository { locationId: row.location_id, intent: JSON.parse(row.intent_json), outcome: row.outcome_json ? JSON.parse(row.outcome_json) : undefined, + pinned: row.pinned === 1, })); } diff --git a/packages/memory/src/handoff.ts b/packages/memory/src/handoff.ts new file mode 100644 index 0000000..e4fee90 --- /dev/null +++ b/packages/memory/src/handoff.ts @@ -0,0 +1,300 @@ +import { z } from "zod"; +import { Entity, naturalizeTime } from "@omnia/core"; +import { BufferEntry, serializeSubjectiveBufferEntry, BufferRepository } from "./buffer.js"; +import { LedgerEntry, LedgerRepository } from "./ledger.js"; +import { ILLMProvider, IEmbeddingProvider } from "@omnia/llm"; + +export const HandoffChunkSchema = z.object({ + sourceEntryIds: z.array(z.string()), // buffer rows this chunk consumes + content: z.string(), // third-person summary -> LedgerEntry.content + quotes: z.array(z.string()), // verbatim, high-salience lines only + importance: z.number().int().min(1).max(10), + involvedEntityIds: z.array(z.string()), + retainInBuffer: z.boolean(), // "pin" +}); + +export const HandoffResultSchema = z.object({ + chunks: z.array(HandoffChunkSchema), +}); + +export type HandoffResult = z.infer; +export type HandoffTrigger = "none" | "voluntary" | "involuntary"; + +/** + * Serializes the hypothetical memory section for size checking. + */ +export function getMemorySectionLength( + entity: Entity, + entries: BufferEntry[], + now: Date, +): number { + if (entries.length === 0) { + return `=== RECENT EVENTS ===\n(No recent events recorded.)`.length; + } + + const groupedLines: string[] = []; + let currentGroup: string | null = null; + + for (const entry of entries) { + const serialized = serializeSubjectiveBufferEntry(entry, entity); + const when = naturalizeTime(now, new Date(entry.timestamp)); + + if (when !== currentGroup) { + currentGroup = when; + const header = when.charAt(0).toUpperCase() + when.slice(1); + groupedLines.push(header); + } + + groupedLines.push(` - ${serialized}`); + } + + return `=== RECENT EVENTS ===\n${groupedLines.join("\n")}`.length; +} + +function checkSceneExit(entity: Entity, bufferEntries: BufferEntry[]): boolean { + if (bufferEntries.length === 0) return false; + + // Find the location of the most recent buffer entries + const lastEntry = bufferEntries[bufferEntries.length - 1]; + if (lastEntry.locationId && entity.locationId && lastEntry.locationId !== entity.locationId) { + return true; + } + + // Also check if there are entries from different locations in the buffer + const locations = new Set(bufferEntries.map(e => e.locationId).filter(loc => loc !== null)); + if (locations.size > 1) { + return true; + } + + return false; +} + +function checkIdleDecay(bufferEntries: BufferEntry[]): boolean { + const N = 5; // N consecutive idle turns + if (bufferEntries.length < N) return false; + + // Check the last N entries + const lastN = bufferEntries.slice(-N); + return lastN.every(e => e.intent.type === "monologue"); +} + +function checkAttributeTrigger(entity: Entity): boolean { + const consciousness = entity.attributes.get("consciousness"); + if (consciousness && consciousness.getValue().toLowerCase() === "unconscious") { + return true; + } + + const status = entity.attributes.get("status"); + if (status && ["unconscious", "asleep", "dead", "inactive"].includes(status.getValue().toLowerCase())) { + return true; + } + + return false; +} + +/** + * Checks deterministically whether handoff should run for the given entity. + */ +export function checkHandoffTrigger( + entity: Entity, + bufferEntries: BufferEntry[], + now: Date, + maxContext: number = 32768, +): HandoffTrigger { + if (bufferEntries.length === 0) { + return "none"; + } + + // Involuntary triggers first (hard) + if (maxContext > 0) { + const memoryLength = getMemorySectionLength(entity, bufferEntries, now); + const charCeiling = maxContext * 4 * 0.60; + if (memoryLength > charCeiling) { + return "involuntary"; + } + } + + // Event velocity + if (bufferEntries.length > 20) { + return "involuntary"; + } + + // Voluntary triggers (soft) + if (checkSceneExit(entity, bufferEntries)) { + return "voluntary"; + } + + if (checkIdleDecay(bufferEntries)) { + return "voluntary"; + } + + if (checkAttributeTrigger(entity)) { + return "voluntary"; + } + + return "none"; +} + +/** + * Splits the buffer into candidate pool (older) and watermark tail (untouched). + */ +export function splitBufferForHandoff( + bufferEntries: BufferEntry[], + now: Date, + K: number = 8, +): { candidates: BufferEntry[]; watermark: BufferEntry[] } { + const sorted = [...bufferEntries].sort( + (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(), + ); + + const freshBuckets = new Set([ + "just now", + "moments ago", + "a few minutes ago", + "several minutes ago", + ]); + + let watermarkStartIndex = sorted.length; + + // 1. Mark last K entries as watermark + if (sorted.length > K) { + watermarkStartIndex = sorted.length - K; + } else { + watermarkStartIndex = 0; + } + + // 2. Expand watermark to include any fresh entries before it + for (let i = watermarkStartIndex - 1; i >= 0; i--) { + const bucket = naturalizeTime(now, new Date(sorted[i].timestamp)); + if (freshBuckets.has(bucket)) { + watermarkStartIndex = i; + } else { + break; + } + } + + return { + candidates: sorted.slice(0, watermarkStartIndex), + watermark: sorted.slice(watermarkStartIndex), + }; +} + +/** + * HandoffEngine processes memory handoffs using LLM summarization and DB transactions. + */ +export class HandoffEngine { + constructor( + private llmProvider: ILLMProvider, + private embedProvider: IEmbeddingProvider, + private bufferRepo: BufferRepository, + private ledgerRepo: LedgerRepository, + ) {} + + async runHandoff( + entity: Entity, + bufferEntries: BufferEntry[], + now: Date, + ): Promise { + const { candidates } = splitBufferForHandoff(bufferEntries, now); + + if (candidates.length === 0) { + return false; + } + + const candidatesList = candidates.map((entry) => { + const serialized = serializeSubjectiveBufferEntry(entry, entity); + return `ID: ${entry.id} | Timestamp: ${entry.timestamp} | Location: ${entry.locationId || "None"}\nContent: ${serialized}`; + }).join("\n---\n"); + + const systemPrompt = ` +You are the memory Handoff Engine. Your task is to process a list of recent working memory buffer entries for an entity and select which memories to promote to the long-term Ledger, and which to forget or summarize. + +Instructions: +1. **Cluster** related consecutive buffer entries into high-level narrative beats or events (e.g. a full back-and-forth conversation or a single physical action and its outcome). Combine them into a single summary chunk. +2. **Write in the third-person** for the "content" of each chunk (e.g. "John asked Mary for the key, and Mary reluctantly handed it over"). +3. **verbatim Quotes**: Extract verbatim, high-salience quotes from dialogue if relevant. Do not modify or invent quotes. +4. **Determine Importance**: Assign an importance score from 1 (trivial, e.g. waking up) to 10 (life-altering, e.g. witnessing a crime). +5. **Involved Entities**: Identify all entity IDs involved in the memories in this chunk. +6. **Retain in Buffer (Pinning)**: If a beat represents an unresolved high-stakes situation (e.g. a standing threat, an unanswered accusation, an ongoing chase or conflict), set "retainInBuffer" to true so it remains in the working memory buffer for immediate context. Otherwise, set it to false so it is safely pruned from the buffer. +7. **Exclude stage business**: Glances, sighs, ambient noticing, and irrelevant sensory details should be ignored and not included in any promoted chunk. They will be forgotten. +8. **Forget by omission**: Any buffer entry ID that you do not include in any chunk's "sourceEntryIds" will be permanently deleted and forgotten. +`.trim(); + + const userContext = ` +Subject Entity ID: ${entity.id} +Current Time: ${now.toISOString()} + +Working Memory Candidates for Handoff: +${candidatesList} +`.trim(); + + const response = await this.llmProvider.generateStructuredResponse({ + systemPrompt, + userContext, + schema: HandoffResultSchema, + }); + + if (!response.success || !response.data) { + return false; + } + + const result = response.data; + const db = (this.bufferRepo as any).db; + + const ledgerEntries: LedgerEntry[] = []; + for (const chunk of result.chunks) { + let embedding: number[] = []; + try { + embedding = await this.embedProvider.embed(chunk.content); + } catch (err) { + console.error("Failed to generate embedding for handoff chunk:", err); + return false; + } + + ledgerEntries.push({ + id: "ledger-" + Math.random().toString(36).substr(2, 9) + "-" + Date.now(), + ownerId: entity.id, + timestamp: now.toISOString(), + locationId: entity.locationId, + involvedEntityIds: chunk.involvedEntityIds, + content: chunk.content, + quotes: chunk.quotes, + importance: chunk.importance, + embedding, + }); + } + + try { + db.transaction(() => { + // Save promoted ledger entries + for (const entry of ledgerEntries) { + this.ledgerRepo.save(entry); + } + + // Keep track of pinned source IDs + const pinnedSourceIds = new Set(); + for (const chunk of result.chunks) { + if (chunk.retainInBuffer) { + for (const id of chunk.sourceEntryIds) { + pinnedSourceIds.add(id); + } + } + } + + // Delete or pin entries + for (const candidate of candidates) { + if (pinnedSourceIds.has(candidate.id)) { + const updated = { ...candidate, pinned: true }; + this.bufferRepo.save(updated); + } else { + this.bufferRepo.delete(candidate.id); + } + } + })(); + return true; + } catch (err) { + console.error("Transaction failed during handoff execution:", err); + return false; + } + } +} diff --git a/packages/memory/src/index.ts b/packages/memory/src/index.ts index 863a0a5..a6fd721 100644 --- a/packages/memory/src/index.ts +++ b/packages/memory/src/index.ts @@ -1,2 +1,3 @@ export * from "./buffer.js"; export * from "./ledger.js"; +export * from "./handoff.js"; diff --git a/packages/memory/tests/handoff.test.ts b/packages/memory/tests/handoff.test.ts new file mode 100644 index 0000000..58c551f --- /dev/null +++ b/packages/memory/tests/handoff.test.ts @@ -0,0 +1,167 @@ +import { describe, test, expect } from "vitest"; +import Database from "better-sqlite3"; +import { Entity } from "@omnia/core"; +import { MockLLMProvider, MockEmbeddingProvider } from "@omnia/llm"; +import { + BufferEntry, + BufferRepository, + LedgerRepository, + checkHandoffTrigger, + splitBufferForHandoff, + HandoffEngine, +} from "@omnia/memory"; + +describe("Memory Handoff Tests (Tier 1)", () => { + const now = new Date("2026-07-07T12:00:00.000Z"); + + test("splitBufferForHandoff correctly splits based on watermark and fresh buckets", () => { + const entries: BufferEntry[] = []; + + // Add 12 older entries (older than 30 minutes) + for (let i = 0; i < 12; i++) { + const minutesAgo = 60 - i; + const timestamp = new Date(now.getTime() - minutesAgo * 60 * 1000).toISOString(); + entries.push({ + id: `entry-old-${i}`, + ownerId: "alice", + timestamp, + locationId: "room-1", + intent: { + type: "dialogue", + originalText: `Old event ${i}`, + description: `does old thing ${i}`, + actorId: "alice", + targetIds: ["bob"], + }, + }); + } + + // Add 4 fresh entries (moments ago / just now) + const freshTimes = [ + new Date(now.getTime() - 10 * 1000).toISOString(), + new Date(now.getTime() - 30 * 1000).toISOString(), + new Date(now.getTime() - 90 * 1000).toISOString(), + new Date(now.getTime() - 180 * 1000).toISOString(), + ]; + + freshTimes.forEach((timestamp, idx) => { + entries.push({ + id: `entry-fresh-${idx}`, + ownerId: "alice", + timestamp, + locationId: "room-1", + intent: { + type: "dialogue", + originalText: `Fresh event ${idx}`, + description: `does fresh thing ${idx}`, + actorId: "alice", + targetIds: ["bob"], + }, + }); + }); + + const { candidates, watermark } = splitBufferForHandoff(entries, now, 8); + + expect(watermark.length).toBeGreaterThanOrEqual(8); + expect(candidates.length).toBe(8); + expect(candidates[0].id).toBe("entry-old-0"); + }); + + test("checkHandoffTrigger detects scene change and idle decay", () => { + const entity = new Entity("alice"); + entity.locationId = "room-2"; + + // Scenario 1: Empty buffer -> no trigger + expect(checkHandoffTrigger(entity, [], now)).toBe("none"); + + // Scenario 2: Scene exit + const entryAtRoom1: BufferEntry = { + id: "e-1", + ownerId: "alice", + timestamp: now.toISOString(), + locationId: "room-1", + intent: { type: "dialogue", originalText: "hello", description: "says hello", actorId: "alice", targetIds: [] }, + }; + expect(checkHandoffTrigger(entity, [entryAtRoom1], now)).toBe("voluntary"); + + // Scenario 3: Idle decay (5 consecutive monologues) + const monologues: BufferEntry[] = Array.from({ length: 5 }, (_, i) => ({ + id: `m-${i}`, + ownerId: "alice", + timestamp: now.toISOString(), + locationId: "room-2", + intent: { type: "monologue", originalText: "think", description: "thinks", actorId: "alice", targetIds: [] }, + })); + expect(checkHandoffTrigger(entity, monologues, now)).toBe("voluntary"); + }); + + test("HandoffEngine promotes candidates to Ledger and prunes buffer transactionally", async () => { + const db = new Database(":memory:"); + db.exec(` + CREATE TABLE IF NOT EXISTS objects ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + name TEXT NOT NULL + ); + INSERT INTO objects (id, type, name) VALUES ('alice', 'character', 'Alice'); + `); + + const bufferRepo = new BufferRepository(db); + const ledgerRepo = new LedgerRepository(db); + + const entity = new Entity("alice"); + entity.locationId = "room-1"; + + const entries: BufferEntry[] = []; + for (let i = 0; i < 10; i++) { + const timestamp = new Date(now.getTime() - (50 - i) * 60 * 1000).toISOString(); + const entry: BufferEntry = { + id: `entry-${i}`, + ownerId: "alice", + timestamp, + locationId: "room-1", + intent: { + type: i % 2 === 0 ? "dialogue" : "action", + originalText: `Event ${i}`, + description: `does thing ${i}`, + actorId: "alice", + targetIds: ["bob"], + }, + }; + bufferRepo.save(entry); + entries.push(entry); + } + + const mockHandoffResult = { + chunks: [ + { + sourceEntryIds: ["entry-0"], + content: "Alice initiated dialogue and performed various tasks.", + quotes: ["Event 0"], + importance: 5, + involvedEntityIds: ["bob"], + retainInBuffer: false, + }, + ], + }; + + const llmProvider = new MockLLMProvider([mockHandoffResult]); + const embedProvider = new MockEmbeddingProvider(); + const engine = new HandoffEngine(llmProvider, embedProvider, bufferRepo, ledgerRepo); + + const success = await engine.runHandoff(entity, entries, now); + expect(success).toBe(true); + + const ledgerRows = db.prepare("SELECT * FROM ledger_entries WHERE owner_id = ?").all("alice") as any[]; + expect(ledgerRows.length).toBe(1); + expect(ledgerRows[0].content).toBe("Alice initiated dialogue and performed various tasks."); + expect(JSON.parse(ledgerRows[0].quotes_json)).toEqual(["Event 0"]); + expect(ledgerRows[0].importance).toBe(5); + + const remainingBuffer = bufferRepo.listForOwner("alice"); + expect(remainingBuffer.length).toBe(8); + expect(remainingBuffer.map((b) => b.id)).not.toContain("entry-0"); + expect(remainingBuffer.map((b) => b.id)).not.toContain("entry-1"); + expect(remainingBuffer[0].id).toBe("entry-2"); + }); +}); diff --git a/packages/memory/tsconfig.json b/packages/memory/tsconfig.json index c2c286c..c3a3b11 100644 --- a/packages/memory/tsconfig.json +++ b/packages/memory/tsconfig.json @@ -7,6 +7,7 @@ "include": ["src"], "references": [ { "path": "../core" }, - { "path": "../intent" } + { "path": "../intent" }, + { "path": "../llm" } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d451101..c31b05f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -393,6 +393,9 @@ importers: '@omnia/intent': specifier: workspace:* version: link:../intent + '@omnia/llm': + specifier: workspace:* + version: link:../llm zod: specifier: ^4.4.3 version: 4.4.3