feat(memory): Implemented tier two memory retrieval using cognition model

This commit is contained in:
2026-07-11 13:57:27 +05:30
parent 1509b69ca7
commit a5fa43e2e6
7 changed files with 659 additions and 62 deletions

View File

@@ -9,6 +9,8 @@ import {
BufferEntry,
BufferRepository,
serializeSubjectiveBufferEntry,
LedgerEntry,
LedgerRepository,
} from "@omnia/memory";
/**
@@ -37,12 +39,17 @@ export class ActorPromptBuilder {
/**
* @param bufferRepo Used to fetch the actor's recent memory. Optional —
* if absent, the memory section is omitted.
* @param ledgerRepo Used to fetch long-term memories. Optional.
* @param memoryLimit Maximum number of recent buffer entries to inject.
* Defaults to 20.
* @param ledgerLimit Maximum number of long-term memories to retrieve.
* Defaults to 5.
*/
constructor(
private bufferRepo?: BufferRepository,
private ledgerRepo?: LedgerRepository,
private memoryLimit = 20,
private ledgerLimit = 5,
) {}
/**
@@ -74,15 +81,14 @@ Guidelines:
- Keep your prose vivid but concise. A single response may contain more than one intent (e.g., you may think, then speak, then act) — write them in natural narrative order.
- Not every response requires an outward action. It is perfectly valid to only think (a monologue) and do nothing perceivable.
- Never speak or act on another entity's behalf — you only control your own character.
".
`.trim();
}
private buildUserContext(worldState: WorldState, entity: Entity): string {
const sections: string[] = [];
const now = worldState.clock.get();
// --- Subjective present time ---
const now = worldState.clock.get();
sections.push(
`=== CURRENT MOMENT ===\nIt is ${now.toISOString()} right now.`,
);
@@ -92,30 +98,43 @@ Guidelines:
`=== THE WORLD AS YOU PERCEIVE IT ===\n${serializeSubjectiveWorldState(worldState, entity.id)}`,
);
// Fetch recent buffer entries once
let recentEntries: BufferEntry[] = [];
if (this.bufferRepo) {
try {
recentEntries = this.bufferRepo.listForOwner(entity.id);
} catch {}
}
// --- Recent memory ---
const memorySection = this.buildMemorySection(
entity,
worldState.clock.get(),
);
const memorySection = this.buildMemorySection(entity, recentEntries, now);
if (memorySection) {
sections.push(memorySection);
}
// --- Recalled Long-Term memory ---
const ledgerSection = this.buildLedgerSection(
worldState,
entity,
recentEntries,
now,
);
if (ledgerSection) {
sections.push(ledgerSection);
}
return sections.join("\n\n");
}
private buildMemorySection(entity: Entity, now: Date): string | null {
private buildMemorySection(
entity: Entity,
entries: BufferEntry[],
now: Date,
): string | null {
if (!this.bufferRepo) return null;
let entries: BufferEntry[];
try {
entries = this.bufferRepo.listForOwner(entity.id);
} catch {
return null;
}
if (entries.length === 0) {
return `=== YOUR RECENT MEMORY ===\n(You have no memories yet.)`;
return `=== RECENT EVENTS ===\n(No recent events recorded.)`;
}
const recent = entries.slice(-this.memoryLimit);
@@ -135,6 +154,110 @@ Guidelines:
groupedLines.push(` - ${serialized}`);
}
return `=== YOUR RECENT MEMORY ===\n${groupedLines.join("\n")}`;
return `=== RECENT EVENTS ===\n${groupedLines.join("\n")}`;
}
private buildLedgerSection(
worldState: WorldState,
entity: Entity,
recentBuffer: BufferEntry[],
now: Date,
): string | null {
if (!this.ledgerRepo) return null;
// 1. Get co-located entities (in the same location as entity)
const coLocatedEntityIds: string[] = [];
if (entity.locationId) {
for (const e of worldState.entities.values()) {
if (e.id !== entity.id && e.locationId === entity.locationId) {
coLocatedEntityIds.push(e.id);
}
}
}
// 2. Compute Active Focus entities based on recent interactions (last 10 entries)
const activeFocus = new Set<string>();
const maxFocus = 3;
// We scan the recent buffer entries to see who we recently talked to or who talked to us
for (let i = recentBuffer.length - 1; i >= 0; i--) {
const entry = recentBuffer[i];
const intent = entry.intent;
if (
intent.actorId !== entity.id &&
coLocatedEntityIds.includes(intent.actorId)
) {
activeFocus.add(intent.actorId);
}
for (const targetId of intent.targetIds) {
if (targetId !== entity.id && coLocatedEntityIds.includes(targetId)) {
activeFocus.add(targetId);
}
}
if (activeFocus.size >= maxFocus) break;
}
// If co-located entities is small, auto-focus all of them
if (activeFocus.size < maxFocus && coLocatedEntityIds.length <= maxFocus) {
for (const id of coLocatedEntityIds) {
if (id !== entity.id) {
activeFocus.add(id);
}
}
}
const activeFocusIds = Array.from(activeFocus);
// 3. Retrieve memories using Active Focus
let recalled: LedgerEntry[];
try {
recalled = this.ledgerRepo.retrieve(
entity.id,
entity.locationId,
activeFocusIds,
undefined, // no query embedding for now (Recency + Importance ranking)
now,
this.ledgerLimit,
{ includeAssociativeNeighbors: true },
);
} catch {
return null;
}
if (recalled.length === 0) return null;
// 4. Format them identical to the recent memory format
const groupedLines: string[] = [];
let currentGroup: string | null = null;
for (const entry of recalled) {
const when = naturalizeTime(now, new Date(entry.timestamp));
let content = entry.content;
// Resolve system IDs to subjective aliases in the content
for (const targetId of entry.involvedEntityIds) {
const alias = entity.aliases.get(targetId) ?? targetId;
content = content.replace(new RegExp(targetId, "g"), alias);
}
if (entry.locationId) {
content += ` (at ${entry.locationId})`;
}
if (when !== currentGroup) {
currentGroup = when;
const header = when.charAt(0).toUpperCase() + when.slice(1);
groupedLines.push(header);
}
groupedLines.push(` - ${content}`);
if (entry.quotes && entry.quotes.length > 0) {
for (const quote of entry.quotes) {
groupedLines.push(` Quote: "${quote}"`);
}
}
}
return `=== YOUR MEMORIES ===\n${groupedLines.join("\n")}`;
}
}

View File

@@ -3,6 +3,7 @@ import { ILLMProvider } from "@omnia/llm";
import {
BufferEntry,
BufferRepository,
LedgerRepository,
} from "@omnia/memory";
import {
Intent,
@@ -73,6 +74,7 @@ export class ActorAgent {
constructor(
llmProvider: ILLMProvider | { actor: ILLMProvider; decoder: ILLMProvider },
bufferRepo?: BufferRepository,
ledgerRepo?: LedgerRepository,
memoryLimit?: number,
generator?: IActorProseGenerator,
) {
@@ -87,7 +89,7 @@ export class ActorAgent {
decoderProv = llmProvider;
}
this.promptBuilder = new ActorPromptBuilder(bufferRepo, memoryLimit);
this.promptBuilder = new ActorPromptBuilder(bufferRepo, ledgerRepo, memoryLimit);
this.decoder = new IntentDecoder(decoderProv);
this.generator = generator ?? new LLMActorProseGenerator(actorProv);
this.llmProvider = actorProv;

View File

@@ -0,0 +1,100 @@
import { describe, it, expect, beforeEach, afterEach } from "vitest";
import Database from "better-sqlite3";
import { WorldState, Entity, AttributeVisibility } from "@omnia/core";
import { BufferRepository, LedgerRepository } from "@omnia/memory";
import { ActorPromptBuilder } from "../src/actor-prompt-builder";
describe("ActorPromptBuilder with Long-Term Memory Integration", () => {
let db: Database.Database;
let bufferRepo: BufferRepository;
let ledgerRepo: LedgerRepository;
beforeEach(() => {
db = new Database(":memory:");
// Core database schemas for testing
db.exec(`
CREATE TABLE objects (
id TEXT PRIMARY KEY
);
`);
db.exec(`
INSERT INTO objects (id) VALUES ('alice'), ('bob'), ('charlie');
`);
bufferRepo = new BufferRepository(db);
ledgerRepo = new LedgerRepository(db);
});
afterEach(() => {
db.close();
});
it("should inject both recent memory and recalled long-term memory with subjective aliases resolved", () => {
const world = new WorldState("world-123", new Date("2024-01-10T12:00:00.000Z"));
const alice = new Entity("alice", "tavern");
// Add subjective alias for bob
alice.aliases.set("bob", "Strider");
world.addEntity(alice);
const bob = new Entity("bob", "tavern");
world.addEntity(bob);
// 1. Populate recent buffer memory
bufferRepo.save({
id: "buf1",
ownerId: "alice",
timestamp: "2024-01-10T11:58:00.000Z", // 2 mins ago
locationId: "tavern",
intent: {
type: "dialogue",
actorId: "alice",
targetIds: ["bob"],
originalText: "Hello there",
description: "Alice greets Bob",
},
});
// 2. Populate ledger repository (long-term memory)
ledgerRepo.save({
id: "ledger1",
ownerId: "alice",
timestamp: "2024-01-08T12:00:00.000Z", // 2 days ago
locationId: "tavern",
involvedEntityIds: ["bob"],
content: "alice met bob at the tavern.",
quotes: ["I am a ranger."],
importance: 9,
embedding: [],
});
const builder = new ActorPromptBuilder(bufferRepo, ledgerRepo, 20, 5);
const { userContext } = builder.build(world, alice);
// Check recent memory exists
expect(userContext).toContain("=== RECENT EVENTS ===");
expect(userContext).toContain("Alice greets Bob");
// Bob should be resolved to Strider
expect(userContext).toContain("spoke to Strider");
// Check long-term memory exists
expect(userContext).toContain("=== YOUR MEMORIES ===");
// Bob should be resolved to Strider in the ledger content
expect(userContext).toContain("alice met Strider at the tavern.");
expect(userContext).toContain('Quote: "I am a ranger."');
});
it("should not explode if ledger contains no memories or is empty", () => {
const world = new WorldState("world-123", new Date("2024-01-10T12:00:00.000Z"));
const alice = new Entity("alice", "tavern");
world.addEntity(alice);
const builder = new ActorPromptBuilder(bufferRepo, ledgerRepo, 20, 5);
const { userContext } = builder.build(world, alice);
expect(userContext).toContain("=== RECENT EVENTS ===");
expect(userContext).not.toContain("=== YOUR MEMORIES ===");
});
});

View File

@@ -92,6 +92,31 @@ export class LedgerRepository {
})();
}
private mapRowToEntry(row: any, involvedEntityIds: string[]): LedgerEntry {
let embedding: number[] = [];
if (row.embedding) {
const buffer = row.embedding as Buffer;
const floatArray = new Float32Array(
buffer.buffer,
buffer.byteOffset,
buffer.byteLength / Float32Array.BYTES_PER_ELEMENT
);
embedding = Array.from(floatArray);
}
return {
id: row.id,
ownerId: row.owner_id,
timestamp: row.timestamp,
locationId: row.location_id,
involvedEntityIds,
content: row.content,
quotes: JSON.parse(row.quotes_json || "[]"),
importance: row.importance,
embedding: embedding,
};
}
load(id: string): LedgerEntry | null {
const row = this.db
.prepare(
@@ -113,28 +138,7 @@ export class LedgerRepository {
)
.all(id) as { entity_id: string }[];
let embedding: number[] = [];
if (row.embedding) {
const buffer = row.embedding as Buffer;
const floatArray = new Float32Array(
buffer.buffer,
buffer.byteOffset,
buffer.byteLength / Float32Array.BYTES_PER_ELEMENT
);
embedding = Array.from(floatArray);
}
return {
id: row.id,
ownerId: row.owner_id,
timestamp: row.timestamp,
locationId: row.location_id,
involvedEntityIds: entitiesRows.map((er) => er.entity_id),
content: row.content,
quotes: JSON.parse(row.quotes_json),
importance: row.importance,
embedding: embedding,
};
return this.mapRowToEntry(row, entitiesRows.map((er) => er.entity_id));
}
/**
@@ -202,32 +206,174 @@ export class LedgerRepository {
entitiesMap.get(er.entry_id)!.push(er.entity_id);
}
return rows.map((row) => {
let embedding: number[] = [];
if (row.embedding) {
const buffer = row.embedding as Buffer;
const floatArray = new Float32Array(
buffer.buffer,
buffer.byteOffset,
buffer.byteLength / Float32Array.BYTES_PER_ELEMENT
);
embedding = Array.from(floatArray);
return rows.map((row) => this.mapRowToEntry(row, entitiesMap.get(row.id) || []));
}
private fetchRawNeighbors(ownerId: string, timestamp: string): LedgerEntry[] {
const neighbors: LedgerEntry[] = [];
// Preceding entry
const preceding = this.db
.prepare(
`
SELECT id, owner_id, timestamp, location_id, content, quotes_json, importance, embedding
FROM ledger_entries
WHERE owner_id = ? AND timestamp < ?
ORDER BY timestamp DESC
LIMIT 1
`
)
.get(ownerId, timestamp) as any;
if (preceding) {
neighbors.push(this.mapRowToEntry(preceding, []));
}
// Succeeding entry
const succeeding = this.db
.prepare(
`
SELECT id, owner_id, timestamp, location_id, content, quotes_json, importance, embedding
FROM ledger_entries
WHERE owner_id = ? AND timestamp > ?
ORDER BY timestamp ASC
LIMIT 1
`
)
.get(ownerId, timestamp) as any;
if (succeeding) {
neighbors.push(this.mapRowToEntry(succeeding, []));
}
return neighbors;
}
/**
* Phase 1 + Phase 2 Retrieval Pipeline
* 1. Fetches candidates via Phase 1 heuristic filtering.
* 2. Ranks them using: Score = Recency + Importance + Semantic Match.
* 3. Selects the top `limit` memories.
* 4. Optionally pulls in the immediate chronological neighbors (associative chain).
* 5. Returns all gathered entries sorted chronologically (timestamp ASC).
*/
retrieve(
ownerId: string,
currentLocationId: string | null,
currentInvolvedEntityIds: string[],
queryEmbedding?: number[],
now: Date = new Date(),
limit: number = 5,
options?: {
includeAssociativeNeighbors?: boolean;
recencyWeight?: number;
importanceWeight?: number;
relevanceWeight?: number;
decayRate?: number;
}
): LedgerEntry[] {
const includeAssociativeNeighbors = options?.includeAssociativeNeighbors ?? false;
const recencyWeight = options?.recencyWeight ?? 1.0;
const importanceWeight = options?.importanceWeight ?? 1.0;
const relevanceWeight = options?.relevanceWeight ?? 1.0;
const decayRate = options?.decayRate ?? 0.99;
// Fetch candidate pool (limit 100 to provide enough options for Phase 2 ranking)
const candidates = this.getRelevant(ownerId, currentLocationId, currentInvolvedEntityIds, 100);
if (candidates.length === 0) return [];
// Score candidates
const scored = candidates.map((entry) => {
// Recency calculation with exponential decay
const deltaMs = now.getTime() - new Date(entry.timestamp).getTime();
const hoursElapsed = Math.max(0, deltaMs / (3600 * 1000));
const recency = Math.pow(decayRate, hoursElapsed);
// Importance score normalized (0.0 to 1.0)
const importanceNorm = entry.importance / 10.0;
// Semantic relevance
let relevance = 0;
if (queryEmbedding && entry.embedding && entry.embedding.length > 0) {
relevance = cosineSimilarity(queryEmbedding, entry.embedding);
}
return {
id: row.id,
ownerId: row.owner_id,
timestamp: row.timestamp,
locationId: row.location_id,
involvedEntityIds: entitiesMap.get(row.id) || [],
content: row.content,
quotes: JSON.parse(row.quotes_json),
importance: row.importance,
embedding: embedding,
};
const score =
recencyWeight * recency +
importanceWeight * importanceNorm +
relevanceWeight * relevance;
return { entry, score };
});
// Rank and take top memories
scored.sort((a, b) => b.score - a.score);
const selected = scored.slice(0, limit).map((s) => s.entry);
let finalEntries = [...selected];
// Optionally retrieve associative neighbors
if (includeAssociativeNeighbors && selected.length > 0) {
const neighborMap = new Map<string, LedgerEntry>();
for (const entry of selected) {
const rawNeighbors = this.fetchRawNeighbors(ownerId, entry.timestamp);
for (const rn of rawNeighbors) {
if (!finalEntries.some((fe) => fe.id === rn.id) && !neighborMap.has(rn.id)) {
neighborMap.set(rn.id, rn);
}
}
}
const neighborsToPopulate = Array.from(neighborMap.values());
if (neighborsToPopulate.length > 0) {
const neighborIds = neighborsToPopulate.map((n) => n.id);
const placeholders = neighborIds.map(() => "?").join(",");
const entitiesRows = this.db
.prepare(
`
SELECT entry_id, entity_id FROM ledger_involved_entities
WHERE entry_id IN (${placeholders})
`
)
.all(...neighborIds) as { entry_id: string; entity_id: string }[];
const entitiesMap = new Map<string, string[]>();
for (const er of entitiesRows) {
if (!entitiesMap.has(er.entry_id)) {
entitiesMap.set(er.entry_id, []);
}
entitiesMap.get(er.entry_id)!.push(er.entity_id);
}
for (const n of neighborsToPopulate) {
n.involvedEntityIds = entitiesMap.get(n.id) || [];
finalEntries.push(n);
}
}
}
// Sort chronologically ASC for the final prompt output
finalEntries.sort((a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime());
return finalEntries;
}
delete(id: string): void {
this.db.prepare(`DELETE FROM ledger_entries WHERE id = ?`).run(id);
}
}
function cosineSimilarity(a: number[], b: number[]): number {
if (a.length !== b.length || a.length === 0) return 0;
let dot = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dot += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
if (normA === 0 || normB === 0) return 0;
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
}

View File

@@ -121,4 +121,108 @@ describe("LedgerRepository", () => {
expect(ids).toContain("mem_social"); // due to involvedEntityIds
expect(ids).not.toContain("mem_irrelevant");
});
it("should retrieve ranked memories with recency, importance, and semantic match", () => {
const now = new Date("2024-01-10T12:00:00.000Z");
repo.save({
id: "mem1",
ownerId: "alice",
timestamp: "2024-01-01T12:00:00.000Z",
locationId: "loc1",
involvedEntityIds: [],
content: "Alice fought a dragon.",
quotes: [],
importance: 10,
embedding: [0, 1, 0],
});
repo.save({
id: "mem2",
ownerId: "alice",
timestamp: "2024-01-10T11:00:00.000Z",
locationId: "loc1",
involvedEntityIds: [],
content: "Alice ate a sandwich.",
quotes: [],
importance: 2,
embedding: [1, 0, 0],
});
repo.save({
id: "mem3",
ownerId: "alice",
timestamp: "2024-01-10T11:50:00.000Z",
locationId: "loc1",
involvedEntityIds: [],
content: "Alice read a book.",
quotes: [],
importance: 5,
embedding: [0.707, 0.707, 0],
});
// Query: [1, 0, 0]
// mem3 score: recency (~0.998) + importance (0.5) + relevance (0.707) = ~2.205
// mem2 score: recency (~0.99) + importance (0.2) + relevance (1.0) = ~2.19
// mem1 score: recency (~0.114) + importance (1.0) + relevance (0.0) = ~1.114
// If limit = 2, should return mem2 and mem3, sorted chronologically (mem2 first, then mem3)
const results = repo.retrieve("alice", "loc1", [], [1, 0, 0], now, 2);
expect(results).toHaveLength(2);
expect(results[0].id).toBe("mem2");
expect(results[1].id).toBe("mem3");
});
it("should pull in associative neighbors when specified", () => {
repo.save({
id: "mem_preceding",
ownerId: "alice",
timestamp: "2024-01-10T10:00:00.000Z",
locationId: "loc_other",
involvedEntityIds: [],
content: "Alice woke up.",
quotes: [],
importance: 2,
embedding: [],
});
repo.save({
id: "mem_target",
ownerId: "alice",
timestamp: "2024-01-10T11:00:00.000Z",
locationId: "loc1",
involvedEntityIds: [],
content: "Alice arrived at tavern.",
quotes: [],
importance: 2,
embedding: [],
});
repo.save({
id: "mem_succeeding",
ownerId: "alice",
timestamp: "2024-01-10T12:00:00.000Z",
locationId: "loc_other",
involvedEntityIds: [],
content: "Alice ordered ale.",
quotes: [],
importance: 2,
embedding: [],
});
// Without neighbors: only returns mem_target
const withoutNeighbors = repo.retrieve("alice", "loc1", [], undefined, new Date("2024-01-10T14:00:00.000Z"), 1, {
includeAssociativeNeighbors: false,
});
expect(withoutNeighbors).toHaveLength(1);
expect(withoutNeighbors[0].id).toBe("mem_target");
// With neighbors: returns preceding, target, and succeeding sorted chronologically
const withNeighbors = repo.retrieve("alice", "loc1", [], undefined, new Date("2024-01-10T14:00:00.000Z"), 1, {
includeAssociativeNeighbors: true,
});
expect(withNeighbors).toHaveLength(3);
expect(withNeighbors[0].id).toBe("mem_preceding");
expect(withNeighbors[1].id).toBe("mem_target");
expect(withNeighbors[2].id).toBe("mem_succeeding");
});
});