mirror of
https://github.com/sortedcord/omnia.git
synced 2026-07-22 03:52:48 +05:30
Compare commits
2 Commits
b4bf70dbae
...
0512be647c
| Author | SHA1 | Date | |
|---|---|---|---|
| 0512be647c | |||
| 1ed1edf4cf |
@@ -183,7 +183,7 @@ omnia/
|
||||
intent/ intent types (dialogue/action/monologue) and the prose decoder
|
||||
architect/ World Architect: LLM validation plus time-delta generation
|
||||
actor/ actor agent: epistemically-bounded prompts, pluggable prose generators
|
||||
memory/ verbatim buffer; later the vector archive, dossier, and affect vectors
|
||||
memory/ Cognitive Buffer; Memory Ledger (vector archive), dossier, and affect vectors
|
||||
spatial/ location and POI graph, portal-based perception
|
||||
llm/ ILLMProvider interface plus Gemini and deterministic mock implementations
|
||||
scenario/ scenario JSON schema and loader (JSON → SQLite)
|
||||
|
||||
@@ -23,6 +23,7 @@ function IntentTag({
|
||||
}) {
|
||||
const labels: Record<string, string> = {
|
||||
monologue: "thought",
|
||||
thought: "thought",
|
||||
dialogue: "dialogue",
|
||||
action: "action",
|
||||
};
|
||||
|
||||
@@ -61,7 +61,7 @@ export function PromptModal({ entry, onClose }: PromptModalProps) {
|
||||
{
|
||||
label: "Memory Ledger",
|
||||
type: "memories",
|
||||
content: ledgerStr || "(No memory buffer entries.)",
|
||||
content: ledgerStr || "(No memory ledger entries.)",
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
@@ -72,8 +72,8 @@
|
||||
],
|
||||
"initialMemories": [
|
||||
{
|
||||
"id": "alpha-wake",
|
||||
"timestamp": "2026-07-09T07:58:00.000Z",
|
||||
"id": "ab3f29d2-cf11-4111-9a99-b13c126d123e",
|
||||
"timestamp": "2026-07-01T07:58:00.000Z",
|
||||
"locationId": "white-room",
|
||||
"intent": {
|
||||
"type": "monologue",
|
||||
@@ -82,6 +82,18 @@
|
||||
"actorId": "7c9b83b3-8cfb-4e89-8d77-626a5757d591",
|
||||
"targetIds": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "zz3f29d2-as11-9811-9a99-b13c126d123e",
|
||||
"timestamp": "2026-07-01T09:58:00.000Z",
|
||||
"locationId": "white-room",
|
||||
"intent": {
|
||||
"type": "action",
|
||||
"originalText": "he wakes up from his sleep.",
|
||||
"description": "",
|
||||
"actorId": "bf3f29d2-cf11-4b11-9a99-b13c126d400e",
|
||||
"targetIds": []
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -119,12 +131,12 @@
|
||||
],
|
||||
"initialMemories": [
|
||||
{
|
||||
"id": "beta-wake",
|
||||
"timestamp": "2026-07-09T07:58:30.000Z",
|
||||
"id": "zx1f29d2-cf11-4111-9a99-b13c126d123e",
|
||||
"timestamp": "2026-07-09T07:58:00.000Z",
|
||||
"locationId": "white-room",
|
||||
"intent": {
|
||||
"type": "action",
|
||||
"originalText": "Why can't I remember anything before the research agreement. It's like my memory was erased.",
|
||||
"originalText": "I wake up in an unfamiliar place.",
|
||||
"description": "",
|
||||
"actorId": "bf3f29d2-cf11-4b11-9a99-b13c126d400e",
|
||||
"targetIds": []
|
||||
|
||||
@@ -112,7 +112,11 @@ Guidelines:
|
||||
}
|
||||
|
||||
// --- Cognitive Buffer ---
|
||||
const memorySection = this.buildMemorySection(entity, recentEntries, now);
|
||||
const memorySection = this.buildCognitiveBufferSection(
|
||||
entity,
|
||||
recentEntries,
|
||||
now,
|
||||
);
|
||||
if (memorySection) {
|
||||
sections.push(memorySection);
|
||||
}
|
||||
@@ -131,7 +135,7 @@ Guidelines:
|
||||
return sections.join("\n\n");
|
||||
}
|
||||
|
||||
private buildMemorySection(
|
||||
private buildCognitiveBufferSection(
|
||||
entity: Entity,
|
||||
entries: BufferEntry[],
|
||||
now: Date,
|
||||
@@ -139,7 +143,7 @@ Guidelines:
|
||||
if (!this.bufferRepo) return null;
|
||||
|
||||
if (entries.length === 0) {
|
||||
return `=== COGNITIVE BUFFER ===\n(No recent events recorded.)`;
|
||||
return `=== COGNITIVE BUFFER ===\n(No entries recorded.)`;
|
||||
}
|
||||
|
||||
const recent = entries.slice(-this.memoryLimit);
|
||||
@@ -147,9 +151,16 @@ Guidelines:
|
||||
let currentGroup: string | null = null;
|
||||
|
||||
for (const entry of recent) {
|
||||
const serialized = serializeSubjectiveBufferEntry(entry, entity);
|
||||
let serialized = serializeSubjectiveBufferEntry(entry, entity);
|
||||
const when = naturalizeTime(now, new Date(entry.timestamp));
|
||||
|
||||
if (
|
||||
entry.intent.actorId === entity.id &&
|
||||
entry.intent.type === "dialogue"
|
||||
) {
|
||||
serialized = `You said: ${serialized}`;
|
||||
}
|
||||
|
||||
if (when !== currentGroup) {
|
||||
currentGroup = when;
|
||||
const header = when.charAt(0).toUpperCase() + when.slice(1);
|
||||
|
||||
@@ -78,7 +78,7 @@ describe("ActorPromptBuilder with Memory Ledger Integration", () => {
|
||||
|
||||
// Check Cognitive Buffer exists
|
||||
expect(userContext).toContain("=== COGNITIVE BUFFER ===");
|
||||
expect(userContext).toContain("Alice greets Bob");
|
||||
expect(userContext).toContain("You said: Alice greets Bob");
|
||||
|
||||
// Check Memory Ledger exists
|
||||
expect(userContext).toContain("=== MEMORY LEDGER ===");
|
||||
|
||||
@@ -47,22 +47,22 @@ export class Architect {
|
||||
* Processes, validates, generates deltas, applies them to the world state,
|
||||
* and persists the changes to the database.
|
||||
*
|
||||
* "monologue" intents are internal thoughts — they bypass validation and
|
||||
* "monologue" and "thought" intents are internal thoughts — they bypass validation and
|
||||
* time-delta generation entirely: the clock does not advance, the world
|
||||
* state is not mutated or persisted. The caller is responsible for writing
|
||||
* the monologue to the actor's memory buffer.
|
||||
* the monologue/thought to the actor's Cognitive Buffer.
|
||||
*/
|
||||
async processIntent(
|
||||
worldState: WorldState,
|
||||
intent: Intent,
|
||||
): Promise<ProcessResult> {
|
||||
// 0. Monologue intents are purely internal — short-circuit before any
|
||||
// 0. Monologue/thought intents are purely internal — short-circuit before any
|
||||
// validation or world mutation.
|
||||
if (intent.type === "monologue") {
|
||||
if (intent.type === "monologue" || intent.type === "thought") {
|
||||
return {
|
||||
isValid: true,
|
||||
reason:
|
||||
"Monologue intent bypasses validation (internal thought, not perceivable).",
|
||||
"Monologue/thought intent bypasses validation (internal thought, not perceivable).",
|
||||
timeDelta: {
|
||||
minutesToAdvance: 0,
|
||||
explanation: "Internal thought — no time elapsed.",
|
||||
|
||||
@@ -26,10 +26,11 @@ export class TimeDeltaGenerator implements IDeltaGenerator<TimeDelta> {
|
||||
explanation: "Dialogue action; 1 minute granted for quick exchange.",
|
||||
};
|
||||
}
|
||||
if (intent.type === "monologue") {
|
||||
if (intent.type === "monologue" || intent.type === "thought") {
|
||||
return {
|
||||
minutesToAdvance: 0,
|
||||
explanation: "Monologue action; no time advanced for internal thought.",
|
||||
explanation:
|
||||
"Monologue/thought action; no time advanced for internal thought.",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ export class LLMValidator {
|
||||
/**
|
||||
* Validates an action intent against the objective world state.
|
||||
*
|
||||
* "monologue" intents must never reach this validator — they are internal
|
||||
* "monologue" and "thought" intents must never reach this validator — they are internal
|
||||
* thoughts that bypass validation entirely (see Architect.processIntent).
|
||||
* This guard exists as a defensive safeguard.
|
||||
*/
|
||||
@@ -24,12 +24,12 @@ export class LLMValidator {
|
||||
worldState: WorldState,
|
||||
intent: Intent,
|
||||
): Promise<ValidationResult> {
|
||||
// Defensive guard: monologue intents bypass validation.
|
||||
if (intent.type === "monologue") {
|
||||
// Defensive guard: monologue and thought intents bypass validation.
|
||||
if (intent.type === "monologue" || intent.type === "thought") {
|
||||
return {
|
||||
isValid: true,
|
||||
reason:
|
||||
"Monologue intents are internal thoughts and bypass validation.",
|
||||
"Monologue/thought intents are internal thoughts and bypass validation.",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ For each intent you must:
|
||||
1. Classify its type:
|
||||
- "dialogue": if actor speaking, talking, whispering, murmuring, etc
|
||||
- "action": Any physical or logical action performed in the world (e.g., moving, opening, looking).
|
||||
- "monologue": An inner thought, reflection, or internal monologue/self narration.
|
||||
- "monologue" (or "thought"): An inner thought, reflection, or internal monologue/self narration.
|
||||
2. Extract the original text fragment from the prose that corresponds to this intent.
|
||||
3. Populate "description" and "selfDescription":
|
||||
- "description": No subject or name — a bare third-person verb phrase only (e.g. "clears their throat", "shakes their head slowly")
|
||||
|
||||
@@ -6,9 +6,15 @@ import { z } from "zod";
|
||||
* - "action": A physical or logical action performed in the world.
|
||||
* - "monologue": An inner thought or internal monologue. Not perceivable by
|
||||
* any other entity. Bypasses the Architect/validators entirely and is
|
||||
* written directly to the actor's memory buffer with no outcome.
|
||||
* written directly to the actor's Cognitive Buffer with no outcome.
|
||||
* - "thought": Equivalent/alias to "monologue".
|
||||
*/
|
||||
export const IntentTypeSchema = z.enum(["dialogue", "action", "monologue"]);
|
||||
export const IntentTypeSchema = z.enum([
|
||||
"dialogue",
|
||||
"action",
|
||||
"monologue",
|
||||
"thought",
|
||||
]);
|
||||
export type IntentType = z.infer<typeof IntentTypeSchema>;
|
||||
|
||||
/**
|
||||
@@ -30,7 +36,7 @@ export const LLMIntentSchema = z.object({
|
||||
/**
|
||||
* Entity IDs of the receiving parties (e.g., who is being spoken to,
|
||||
* what object is being interacted with). Always an empty array for
|
||||
* "monologue" intents, since they are not perceivable by anyone.
|
||||
* "monologue" and "thought" intents, since they are not perceivable by anyone.
|
||||
*/
|
||||
targetIds: z.array(z.string()),
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { Intent } from "@omnia/intent";
|
||||
|
||||
export interface BufferEntry {
|
||||
id: string;
|
||||
ownerId: string; // Whose subjective memory buffer this lives in
|
||||
ownerId: string; // Whose Cognitive Buffer this entry lives in
|
||||
timestamp: string; // WorldClock.get().toISOString() at write time
|
||||
locationId: string | null; // Actor's location when this happened
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ export function getMemorySectionLength(
|
||||
now: Date,
|
||||
): number {
|
||||
if (entries.length === 0) {
|
||||
return `=== COGNITIVE BUFFER ===\n(No recent events recorded.)`.length;
|
||||
return `=== COGNITIVE BUFFER ===\n(No entries recorded.)`.length;
|
||||
}
|
||||
|
||||
const groupedLines: string[] = [];
|
||||
@@ -85,7 +85,9 @@ function checkIdleDecay(bufferEntries: BufferEntry[]): boolean {
|
||||
|
||||
// Check the last N entries
|
||||
const lastN = bufferEntries.slice(-N);
|
||||
return lastN.every((e) => e.intent.type === "monologue");
|
||||
return lastN.every(
|
||||
(e) => e.intent.type === "monologue" || e.intent.type === "thought",
|
||||
);
|
||||
}
|
||||
|
||||
function checkAttributeTrigger(entity: Entity): boolean {
|
||||
@@ -246,7 +248,7 @@ Instructions:
|
||||
Subject Entity ID: ${entity.id}
|
||||
Current Time: ${now.toISOString()}
|
||||
|
||||
Working Memory Candidates for Handoff:
|
||||
Cognitive Buffer Candidates for Handoff:
|
||||
${candidatesList}
|
||||
`.trim();
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ export const ScenarioMemoryEntrySchema = z.object({
|
||||
timestamp: z.string(), // ISO string
|
||||
locationId: z.string().nullable(),
|
||||
intent: z.object({
|
||||
type: z.enum(["dialogue", "action", "monologue"]),
|
||||
type: z.enum(["dialogue", "action", "monologue", "thought"]),
|
||||
originalText: z.string(),
|
||||
description: z.string(),
|
||||
selfDescription: z.string().optional(),
|
||||
|
||||
@@ -128,17 +128,22 @@ describe("Talking Room Demo Scenario Test (Tier 1)", () => {
|
||||
|
||||
// 7. Assert initial pre-seeded memories
|
||||
const alphaMemories = bufferRepo.listForOwner(alphaId);
|
||||
expect(alphaMemories).toHaveLength(1);
|
||||
expect(alphaMemories[0].id).toBe("alpha-wake");
|
||||
expect(alphaMemories).toHaveLength(2);
|
||||
expect(alphaMemories[0].id).toBe("ab3f29d2-cf11-4111-9a99-b13c126d123e");
|
||||
expect(alphaMemories[0].intent.type).toBe("monologue");
|
||||
expect(alphaMemories[0].intent.originalText).toContain("jail");
|
||||
expect(alphaMemories[0].intent.description).toBe("");
|
||||
|
||||
expect(alphaMemories[1].id).toBe("zz3f29d2-as11-9811-9a99-b13c126d123e");
|
||||
expect(alphaMemories[1].intent.type).toBe("action");
|
||||
expect(alphaMemories[1].intent.originalText).toContain("sleep");
|
||||
expect(alphaMemories[1].intent.description).toBe("");
|
||||
|
||||
const betaMemories = bufferRepo.listForOwner(betaId);
|
||||
expect(betaMemories).toHaveLength(1);
|
||||
expect(betaMemories[0].id).toBe("beta-wake");
|
||||
expect(betaMemories[0].id).toBe("zx1f29d2-cf11-4111-9a99-b13c126d123e");
|
||||
expect(betaMemories[0].intent.type).toBe("action");
|
||||
expect(betaMemories[0].intent.originalText).toContain("agreement");
|
||||
expect(betaMemories[0].intent.originalText).toContain("unfamiliar");
|
||||
expect(betaMemories[0].intent.description).toBe("");
|
||||
|
||||
db.close();
|
||||
|
||||
@@ -166,7 +166,7 @@ describe("Actor Agent + Monologue Intent Integration (Tier 2)", () => {
|
||||
const expectedTime = new Date(startTime.getTime() + 3 * 60_000);
|
||||
expect(world.clock.get().toISOString()).toBe(expectedTime.toISOString());
|
||||
|
||||
// 7. All three intents persisted to Alice's memory buffer.
|
||||
// 7. All three intents persisted to Alice's Cognitive Buffer.
|
||||
const aliceMemory = bufferRepo.listForOwner("alice");
|
||||
expect(aliceMemory).toHaveLength(3);
|
||||
expect(aliceMemory[0].intent.type).toBe("monologue");
|
||||
|
||||
BIN
web/docs/src/assets/img/mem1.png
Normal file
BIN
web/docs/src/assets/img/mem1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 36 KiB |
BIN
web/docs/src/assets/img/mem2.png
Normal file
BIN
web/docs/src/assets/img/mem2.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 85 KiB |
@@ -36,14 +36,14 @@ Entity aliases are persisted in the `objects` table via the `aliases_json TEXT`
|
||||
|
||||
## Subjective Buffer Entry
|
||||
|
||||
A subjective `BufferEntry` records a discrete event from the perspective of an entity (the `owner`). It wraps a structured `Intent` and appends execution metadata.
|
||||
A subjective `BufferEntry` records a discrete event from the perspective of an entity (the `owner`). It wraps a structured `Intent` and appends execution metadata. These entries make up the entity's **Cognitive Buffer**.
|
||||
|
||||
### The Shape of a Buffer Entry
|
||||
|
||||
```typescript
|
||||
interface BufferEntry {
|
||||
id: string;
|
||||
ownerId: string; // Whose subjective memory buffer this lives in
|
||||
ownerId: string; // Whose Cognitive Buffer this entry lives in
|
||||
timestamp: string; // WorldClock.get().toISOString() at write time
|
||||
locationId: string | null; // Actor's location when this happened
|
||||
|
||||
@@ -61,7 +61,7 @@ interface BufferEntry {
|
||||
|
||||
## Buffer Serialization (Epistemic Substitute)
|
||||
|
||||
To prevent leaking system IDs, buffer memories are serialized using `serializeSubjectiveBufferEntry` with the `resolveAlias` helper:
|
||||
To prevent leaking system IDs, Cognitive Buffer entries are serialized using `serializeSubjectiveBufferEntry` with the `resolveAlias` helper:
|
||||
|
||||
```typescript
|
||||
export function resolveAlias(viewer: Entity, targetId: string): string {
|
||||
@@ -89,7 +89,7 @@ CREATE TABLE IF NOT EXISTS buffer_entries (
|
||||
```
|
||||
|
||||
- **JSON Storage**: `intent` and `outcome` are serialized/deserialized as raw JSON, validated by Zod at creation time.
|
||||
- **Cascade Deletes**: Deleting an entity removes all associated subjective memory entries.
|
||||
- **Cascade Deletes**: Deleting an entity removes all associated Cognitive Buffer entries.
|
||||
|
||||
## Time Naturalization
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ omnia/
|
||||
intent/ intent types (dialogue/action/monologue) and the prose decoder
|
||||
architect/ World Architect: LLM validation plus time-delta generation
|
||||
actor/ actor agent: epistemically-bounded prompts, pluggable prose generators
|
||||
memory/ verbatim buffer; later the vector archive, dossier, and affect vectors
|
||||
memory/ Cognitive Buffer; Memory Ledger (vector archive), dossier, and affect vectors
|
||||
spatial/ location and POI graph, portal-based perception
|
||||
llm/ ILLMProvider interface plus Gemini and deterministic mock implementations
|
||||
scenario/ scenario JSON schema and loader (JSON → SQLite)
|
||||
|
||||
42
web/docs/src/content/docs/diary/2_Finalizing_memory.md
Normal file
42
web/docs/src/content/docs/diary/2_Finalizing_memory.md
Normal file
@@ -0,0 +1,42 @@
|
||||
---
|
||||
title: I'm done with confusing memory jargon
|
||||
author: sortedcord
|
||||
time: 2026-07-18T16:12:00+05:30
|
||||
---
|
||||
|
||||
Memory has always been a core part of Omnia's architecture. I've always leaned towards a three tier system and it feels good to finally have a system frozen that's not subject to any more dramatic shifts.
|
||||
|
||||
Up until now, I had been juggling terms like "short-term", "long-term", "buffer", "ledger" and memory quite randomly and interchagebly. Not just that, I think even [NLAVS](https://github.com/sortedcord/NLAVS) is also guilty of this.
|
||||
|
||||
## How it was
|
||||
|
||||

|
||||
|
||||
Notice how the tier 2 bar is the longest here. This made sense to me. You'd have a short term memory structure, ideally for events that are happening in real time and this would be small. At any given time you can't realistically carry a lot of information. And then at the bottom we have the long term tier. And just to make the graph complete, there's this intermediate layer which would contain most of the memories of an agent.
|
||||
|
||||
Sounded right on markdown and looked cool in excali. But once you start talking about how they actually work, how does a unit of memory actually go between these layers is when all hell breaks loose. This wasn't something I paid much attention towards.
|
||||
|
||||
## A colorful diagram that's a wanna be complex flowchart
|
||||
|
||||
While developing [Handoff](../architecture/handoff.md); which is the mechanism for promoting "short term" memories to "long term" memories I realized that blindly modelling this after how humans are supposed to remember isn't a good approach to this, or rather, I'd say is a naiive solution to a data engineering problem.
|
||||
|
||||

|
||||
|
||||
Don't let this fool you. I didn't just move boxes around, add new boxes and renamed stuff and called it a day (even though that's what i did ;). But the main realization I had and the main framework that Omnia will follow from now on out is that **memories for an agent shouldn't be discrete rather they should be episodic.**
|
||||
|
||||
When one remembers something, they don't remember it in isolated bits and pieces. You remember them in sequences that are chained together. Remembering something often leads to remembering events that either caused it or were caused by it. Like a chain. This chain is the episode this is what makes Omnia's memory system different: **Memory isn't pure hierarchy.**
|
||||
|
||||
Each tier of memory serves its own purpose and has its own benefits and trade-offs but integrating them well together allows them to complement each other quite well.
|
||||
|
||||
Putting this diagram into words:
|
||||
|
||||
1. [**Tier 1: Cognitive Buffer**](../architecture/memory.md) is basically how the agent perceives its surroundings as-is. It is analgoues to a vision to text thing where whatever the agent "sees" is written down here, in the sense that this is high resolution and very verbose. So it contains actions, dialogues of others and itself but also its own thoughts, feelings, monologues.
|
||||
2. [**Handoff**](../architecture/handoff.md) is what determines how important each event in the cognitive buffer is to be promoted to Memories. It's also responsible for compressing multiple actions into a single Memory Ledger entry.
|
||||
3. [**Tier 2: Memory Ledger**](../architecture/memory-tier2.md) contains the bulk of an agent's memory. All memories are stored by when it happened, where it happened and who/what were involved in it. Given the size it will grow to, there was a need to implement a retrieval system (more than 1 of them).
|
||||
4. [**Tier 3: Dossiers**](../architecture/) is the component of Omnia that allows for a per-actor subjective reality mapping. Every fact an actor knows whether it's about themselves or something else is stored as a dossier.
|
||||
|
||||
Dossiers also dictate what series of memories from the memory ledger populate the context for an actor allowing for episodic retreival of memories alongside the general ledger retriever that works on semantic similarity.
|
||||
|
||||
The main difference between the cognitive buffer+memory ledger and dossiers is that the former are immutable. Once something happens, it stays in the memory. It can become less relevant or harder to remember overtime (through time decay) but it stays in the memory as an immutable form.
|
||||
|
||||
Dossiers however, are mutable. They are an ever-changing object in the actor's memory that define what they think about something at that intance of time. This 'thought' or 'impression' can be about anything - themselves, another actor, location, abstract idea, etc.
|
||||
Reference in New Issue
Block a user