sortedcord/omnia-consolidated
1
0
Fork 0
mirror of https://github.com/sortedcord/omnia.git synced 2026-07-22 03:52:48 +05:30
Code Issues Packages Projects Releases Wiki Activity

refactor(llm, memory): Use generic types prompt builder and prompt component for actor,intent and handoff

Browse Source
This commit is contained in:
Aditya Gupta
2026-07-19 18:20:28 +05:30
parent 1e34becec7
commit ee25bf4a4c
11 changed files with 267 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
apps/gui/src/lib/simulation/alias-handoff.ts
View File

@@ -39,44 +39,13 @@ export async function runHandoffResolution(session: SimSession): Promise<void> {
worldState.clock.get(),
);
if (ran) {
const lastResult = (handoffEngine as any).lastResult;
const lastCall =
session.handoffProvider.lastCalls?.[
(session.handoffProvider.lastCalls?.length || 0) - 1
];
const info = session.entities.find((e) => e.id === entity.id);
const entityName = info?.name || entity.id;
let handoffPrompt = undefined;
if (lastCall) {
const header = "Cognitive Buffer Candidates for Handoff:";
const userContext = lastCall.userContext;
const idx = userContext.indexOf(header);
let contextStr = userContext;
let candidatesStr = "";
if (idx !== -1) {
contextStr = userContext.substring(0, idx).trim();
candidatesStr = userContext.substring(idx).trim();
}
handoffPrompt = {
systemPrompt: lastCall.systemPrompt,
userContext: lastCall.userContext,
components: [
{
label: "System Prompt",
type: "system",
content: lastCall.systemPrompt,
},
{ label: "Entity Context", type: "world", content: contextStr },
{
label: "Cognitive Candidates",
type: "input",
content: candidatesStr,
},
],
};
}
session.log.push({
turn: session.turn,
@@ -86,9 +55,15 @@ export async function runHandoffResolution(session: SimSession): Promise<void> {
intents: [],
timestamp: worldState.clock.get().toISOString(),
isHandoff: true,
rawPrompt: handoffPrompt,
rawPrompt: lastResult
? {
systemPrompt: lastResult.systemPrompt || "",
userContext: lastResult.userContext || "",
components: lastResult.promptComponents,
}
: undefined,
usage: lastCall?.usage,
handoffResult: lastCall?.response,
handoffResult: lastResult?.response || lastCall?.response,
});
}
}

21
apps/gui/src/lib/simulation/turn-executor.ts
View File

@@ -168,7 +168,7 @@ export async function processNpcTurn(
rawPrompt: {
systemPrompt: result.systemPrompt || "",
userContext: result.userContext || "",
sections: result.promptComponents,
components: result.promptComponents,
},
};
@@ -267,7 +267,7 @@ export async function executePlayerAction(
rawPrompt: {
systemPrompt: result.systemPrompt || ctx.systemPrompt,
userContext: result.userContext || ctx.userContext,
sections: result.promptComponents,
components: result.promptComponents,
},
};
@@ -279,9 +279,26 @@ export async function executePlayerAction(
session.decoderProvider.lastCalls[
session.decoderProvider.lastCalls.length - 1
];
const proseHeader = "=== NARRATIVE PROSE ===";
const userContext = call.userContext;
const idx = userContext.indexOf(proseHeader);
let contextStr = userContext;
let proseStr = "";
if (idx !== -1) {
contextStr = userContext.substring(0, idx).trim();
proseStr = userContext.substring(idx).trim();
}
entry.decoderPrompt = {
systemPrompt: call.systemPrompt,
userContext: call.userContext,
components: [
{ label: "System Prompt", type: "system", content: call.systemPrompt },
{ label: "Decoder Context", type: "world", content: contextStr },
{ label: "Narrative Prose", type: "input", content: proseStr },
],
};
entry.decoderUsage = call.usage;
}

3
packages/actor/src/actor-prompt-builder.ts
View File

@@ -13,7 +13,8 @@ import {
LedgerEntry,
LedgerRepository,
} from "@omnia/memory";
import { hydrate, PromptComponent } from "@omnia/voice";
import { hydrate } from "@omnia/voice";
import { PromptComponent } from "@omnia/llm";
/**
* Zod schema for the structured response expected from the actor LLM.

3
packages/actor/src/actor.ts
View File

@@ -1,8 +1,7 @@
import { Entity, WorldState } from "@omnia/core";
import { ILLMProvider } from "@omnia/llm";
import { ILLMProvider, PromptComponent } from "@omnia/llm";
import { BufferEntry, BufferRepository, LedgerRepository } from "@omnia/memory";
import { Intent, IntentDecoder, IntentSequence } from "@omnia/intent";
import { PromptComponent } from "@omnia/voice";
import {
ActorPromptBuilder,
ActorResponseSchema,

90
packages/intent/src/intent-decoder.ts
View File

@@ -1,19 +1,18 @@
import { WorldState, resolveAlias } from "@omnia/core";
import { WorldState } from "@omnia/core";
import { ILLMProvider } from "@omnia/llm";
import { dehydrate, expandContractions } from "@omnia/voice";
import { Intent, IntentSequence, LLMIntentSequenceSchema } from "./intent.js";
import { IntentDecoderPromptBuilder } from "./intent-prompt-builder.js";
export class IntentDecoder {
constructor(private llmProvider: ILLMProvider) {}
private promptBuilder: IntentDecoderPromptBuilder;
constructor(private llmProvider: ILLMProvider) {
this.promptBuilder = new IntentDecoderPromptBuilder();
}
/**
* Decodes narrative prose into an ordered sequence of structured intents.
*
* Responsibilities (from docs/intents.md):
* - Split prose into multiple intents when applicable.
* - Classify each intent as "dialogue", "action", or "monologue".
* - Parse narrative text into structured JSON with minimal information loss.
* - Contextually resolve receiving parties (targets).
*/
async decode(
worldState: WorldState,
@@ -24,72 +23,12 @@ export class IntentDecoder {
const processedProse = expandContractions(narrativeProse);
const actor = worldState.getEntity(actorId);
// 1. Get other entities co-located in the same context
const otherEntitiesLines: string[] = [];
for (const otherEntity of worldState.entities.values()) {
if (
otherEntity.id !== actorId &&
otherEntity.locationId === actor?.locationId
) {
const alias = actor
? resolveAlias(actor, otherEntity.id)
: otherEntity.id;
otherEntitiesLines.push(` - Alias="${alias}" ID=${otherEntity.id}`);
}
}
const otherEntitiesContext =
otherEntitiesLines.length > 0
? otherEntitiesLines.join("\n")
: " (No other entities in context)";
// 2. Format historical context (2-3 recent intents received by the actor)
const historicalLines: string[] = [];
for (const prior of recentIntents) {
const targetIds =
prior.actorId !== actorId ? [prior.actorId] : prior.targetIds;
const targetsStr = targetIds
.map((tid) => {
const alias = actor ? resolveAlias(actor, tid) : tid;
return `(Alias="${alias}", ID="${tid}")`;
})
.join(", ");
// In the historical context, prior.content is already dehydrated, so we hydrate it for the actor's view
// Wait, we can keep it simple or just use the content. We'll show the content.
historicalLines.push(
` - Content: "${prior.content}", Type: ${prior.type}, Target Entities: ${targetsStr || "None"}`,
);
}
const historicalContext =
historicalLines.length > 0
? historicalLines.join("\n")
: " (No prior intents in context)";
const systemPrompt = `
You are the Intent Decoder for a narrative simulation engine.
Your job is to take a block of narrative prose written by an actor agent and decompose it into an ordered sequence of discrete intents.
For each intent you must:
1. Classify its type:
- "dialogue": if actor speaking, talking, whispering, murmuring, etc
- "action": Any physical action performed in the world (e.g., moving, opening, looking). DO NOT CLASSIFY SPEAKING MODIFIERS AS ACTIONS
- "monologue" (or "thought"): An inner thought, reflection, or monologue/self narration.
2. Extract the original narrative text fragment from the prose that corresponds to this intent and populate it as "content". Do not paraphrase, do not convert to third person, and do not convert to second person. Keep the original text fragment exactly as written in the prose (first-person voice).
3. Identify targetIds — the entity IDs of the receiving parties. Use the "Other entities in context" list to resolve any subjective names, aliases, or descriptions used in the prose to their correct entity IDs. If no specific target, use an empty array.
4. Identify modifiers — a list of strings representing additional qualities or modifiers extracted from the narrative prose. This includes emotions, tone of voice, speed, manner of action, or statement type (e.g., "question", "anxious", "whispering", "slowly", "quietly", "forcefully"). If no modifiers are present, use an empty array.
5. For dialogue intents always use the following format for content field:
I say "<dialogue>" (optionally: to him/her/alias).
`.trim();
const userContext = `
Intent Source: ${actorId}
Other entities in context:
${otherEntitiesContext}
Historical Context:
${historicalContext}
=== NARRATIVE PROSE ===
${processedProse}
`.trim();
const { systemPrompt, userContext, components } = this.promptBuilder.build(
worldState,
actorId,
processedProse,
recentIntents,
);
const response = await this.llmProvider.generateStructuredResponse({
systemPrompt,
@@ -126,6 +65,9 @@ ${processedProse}
return {
intents: fullIntents,
systemPrompt,
userContext,
promptComponents: components,
};
}
}

102
packages/intent/src/intent-prompt-builder.ts Normal file
View File

@@ -0,0 +1,102 @@
import { WorldState, resolveAlias } from "@omnia/core";
import { PromptBreakdown, PromptComponent } from "@omnia/llm";
import { Intent } from "./intent.js";
// TODO: Builder a generic interface for prompt builders in @omnia/llm: IPromptBuilder or something
/**
* Prompt builder for the Intent Decoder.
* Separates prompt generation, structure, and component breakdowns.
*/
export class IntentDecoderPromptBuilder {
build(
worldState: WorldState,
actorId: string,
processedProse: string,
recentIntents: Intent[],
): PromptBreakdown {
const actor = worldState.getEntity(actorId);
// 1. Get other entities co-located in the same context
const otherEntitiesLines: string[] = [];
for (const otherEntity of worldState.entities.values()) {
if (
otherEntity.id !== actorId &&
otherEntity.locationId === actor?.locationId
) {
const alias = actor
? resolveAlias(actor, otherEntity.id)
: otherEntity.id;
otherEntitiesLines.push(` - Alias="${alias}" ID=${otherEntity.id}`);
}
}
const otherEntitiesContext =
otherEntitiesLines.length > 0
? otherEntitiesLines.join("\n")
: " (No other entities in context)";
// 2. Format historical context (2-3 recent intents received by the actor)
const historicalLines: string[] = [];
for (const prior of recentIntents) {
const targetIds =
prior.actorId !== actorId ? [prior.actorId] : prior.targetIds;
const targetsStr = targetIds
.map((tid) => {
const alias = actor ? resolveAlias(actor, tid) : tid;
return `(Alias="${alias}", ID="${tid}")`;
})
.join(", ");
historicalLines.push(
` - Content: "${prior.content}", Type: ${prior.type}, Target Entities: ${targetsStr || "None"}`,
);
}
const historicalContext =
historicalLines.length > 0
? historicalLines.join("\n")
: " (No prior intents in context)";
const systemPrompt = `
You are the Intent Decoder for a narrative simulation engine.
Your job is to take a block of narrative prose written by an actor agent and decompose it into an ordered sequence of discrete intents.
For each intent you must:
1. Classify its type:
- "dialogue": if actor speaking, talking, whispering, murmuring, etc
- "action": Any physical action performed in the world (e.g., moving, opening, looking). DO NOT CLASSIFY SPEAKING MODIFIERS AS ACTIONS
- "monologue" (or "thought"): An inner thought, reflection, or monologue/self narration.
2. Extract the original narrative text fragment from the prose that corresponds to this intent and populate it as "content". Do not paraphrase, do not convert to third person, and do not convert to second person. Keep the original text fragment exactly as written in the prose (first-person voice).
3. Identify targetIds — the entity IDs of the receiving parties. Use the "Other entities in context" list to resolve any subjective names, aliases, or descriptions used in the prose to their correct entity IDs. If no specific target, use an empty array.
4. Identify modifiers — a list of strings representing additional qualities or modifiers extracted from the narrative prose. This includes emotions, tone of voice, speed, manner of action, or statement type (e.g., "question", "anxious", "whispering", "slowly", "quietly", "forcefully"). If no modifiers are present, use an empty array.
5. For dialogue intents always use the following format for content field:
I say "<dialogue>" (optionally: to him/her/alias).
`.trim();
const decoderContext = `
Intent Source: ${actorId}
Other entities in context:
${otherEntitiesContext}
Historical Context:
${historicalContext}
`.trim();
const narrativeProseSection = `=== NARRATIVE PROSE ===\n${processedProse}`;
const userContext = `${decoderContext}\n\n${narrativeProseSection}`;
const components: PromptComponent[] = [
{ label: "System Prompt", type: "system", content: systemPrompt },
{ label: "Decoder Context", type: "world", content: decoderContext },
{
label: "Narrative Prose",
type: "input",
content: narrativeProseSection,
},
];
return {
systemPrompt,
userContext,
components,
};
}
}

8
packages/intent/src/intent.ts
View File

@@ -56,8 +56,14 @@ export const LLMIntentSequenceSchema = z.object({
* The full output of the Intent Decoder: an ordered sequence of intents
* extracted from a single narrative prose block.
*/
import { PromptComponent } from "@omnia/llm";
export const IntentSequenceSchema = z.object({
intents: z.array(IntentSchema),
});
export type IntentSequence = z.infer<typeof IntentSequenceSchema>;
export type IntentSequence = z.infer<typeof IntentSequenceSchema> & {
systemPrompt?: string;
userContext?: string;
promptComponents?: PromptComponent[];
};

16
packages/llm/src/llm.ts
View File

@@ -1,6 +1,22 @@
import { z } from "zod";
import { ProviderRegistry } from "./registry.js";
export interface PromptComponent {
label: string;
type: "system" | "world" | "events" | "memories" | "input" | "other";
content: string;
}
export interface PromptBreakdown {
systemPrompt: string;
userContext: string;
components?: PromptComponent[];
}
export interface IPromptBuilder<TArgs extends any[]> {
build(...args: TArgs): PromptBreakdown;
}
export interface LLMRequest<T extends z.ZodTypeAny> {
systemPrompt: string;
userContext: string;

59
packages/memory/src/handoff-prompt-builder.ts Normal file
View File

@@ -0,0 +1,59 @@
import { Entity } from "@omnia/core";
import { PromptBreakdown, PromptComponent } from "@omnia/llm";
import { BufferEntry, serializeSubjectiveBufferEntry } from "./buffer.js";
/**
* Prompt builder for the Handoff Engine.
* Separates prompt generation, structure, and component breakdowns.
*/
export class HandoffPromptBuilder {
build(entity: Entity, candidates: BufferEntry[], now: Date): PromptBreakdown {
const candidatesList = candidates
.map((entry) => {
const serialized = serializeSubjectiveBufferEntry(entry, entity);
return `ID: ${entry.id} | Timestamp: ${entry.timestamp} | Location: ${entry.locationId || "None"}\nContent: ${serialized}`;
})
.join("\n---\n");
const systemPrompt = `
You are the memory Handoff Engine. Your task is to process a list of Cognitive Buffer entries for an entity and select which memories to promote to the Memory Ledger, and which to forget or summarize.
Instructions:
1. **Cluster** related consecutive buffer entries into high-level narrative beats or events (e.g. physical action and its outcome or trivial actions). Combine them into a single chunk.
2. **Write in the third-person** for the events of other entities. (eg. Alan did that. Sarah did this, etc)
2. **Write in first-person for the events that you yourself did. (eg. I did this, I did that.)
3. **verbatim Quotes**: Extract verbatim, high-salience quotes from dialogue if relevant. Do not modify or invent quotes.
4. **Determine Importance**: Assign an importance score from 1 (trivial, e.g. waking up) to 10 (life-altering, e.g. witnessing a crime).
4. Discard small body movements like looking around, sighing, etc that do not contextually hold any meaning after it is done.
5. **Involved Entities**: Identify all entity IDs involved in the memories in this chunk.
6. **Retain in Cognitive Buffer (Pinning)**: If a beat represents an unresolved high-stakes situation (e.g. a standing threat, an unanswered accusation, an ongoing chase or conflict), set "retainInBuffer" to true so it remains in the Cognitive Buffer for immediate context. Otherwise, set it to false so it is safely pruned from the Cognitive Buffer.
7. **Exclude stage business**: Glances, sighs, ambient noticing, and irrelevant sensory details should be ignored and not included in any promoted chunk. They will be forgotten.
8. **Forget by omission**: Any buffer entry ID that you do not include in any chunk's "sourceEntryIds" will be permanently deleted and forgotten.
`.trim();
const entityContext = `
Subject Entity ID: ${entity.id}
Current Time: ${now.toISOString()}
`.trim();
const candidatesSection = `Cognitive Buffer Candidates for Handoff:\n${candidatesList}`;
const userContext = `${entityContext}\n\n${candidatesSection}`;
const components: PromptComponent[] = [
{ label: "System Prompt", type: "system", content: systemPrompt },
{ label: "Entity Context", type: "world", content: entityContext },
{
label: "Cognitive Candidates",
type: "input",
content: candidatesSection,
},
];
return {
systemPrompt,
userContext,
components,
};
}
}

68
packages/memory/src/handoff.ts
View File

@@ -6,7 +6,8 @@ import {
BufferRepository,
} from "./buffer.js";
import { LedgerEntry, LedgerRepository } from "./ledger.js";
import { ILLMProvider, IEmbeddingProvider } from "@omnia/llm";
import { ILLMProvider, IEmbeddingProvider, PromptComponent } from "@omnia/llm";
import { HandoffPromptBuilder } from "./handoff-prompt-builder.js";
export const HandoffChunkSchema = z.object({
sourceEntryIds: z.array(z.string()), // buffer rows this chunk consumes
@@ -202,55 +203,44 @@ export function splitBufferForHandoff(
/**
* HandoffEngine processes memory handoffs using LLM summarization and DB transactions.
*/
export interface HandoffRunResult {
success: boolean;
systemPrompt?: string;
userContext?: string;
promptComponents?: PromptComponent[];
response?: any;
}
export class HandoffEngine {
public lastResult: HandoffRunResult | null = null;
private promptBuilder: HandoffPromptBuilder;
constructor(
private llmProvider: ILLMProvider,
private embedProvider: IEmbeddingProvider,
private bufferRepo: BufferRepository,
private ledgerRepo: LedgerRepository,
) {}
) {
this.promptBuilder = new HandoffPromptBuilder();
}
async runHandoff(
entity: Entity,
bufferEntries: BufferEntry[],
now: Date,
): Promise<boolean> {
this.lastResult = null;
const { candidates } = splitBufferForHandoff(bufferEntries, now);
if (candidates.length === 0) {
return false;
}
const candidatesList = candidates
.map((entry) => {
const serialized = serializeSubjectiveBufferEntry(entry, entity);
return `ID: ${entry.id} | Timestamp: ${entry.timestamp} | Location: ${entry.locationId || "None"}\nContent: ${serialized}`;
})
.join("\n---\n");
const systemPrompt = `
You are the memory Handoff Engine. Your task is to process a list of Cognitive Buffer entries for an entity and select which memories to promote to the Memory Ledger, and which to forget or summarize.
Instructions:
1. **Cluster** related consecutive buffer entries into high-level narrative beats or events (e.g. physical action and its outcome or trivial actions). Combine them into a single chunk.
2. **Write in the third-person** for the events of other entities. (eg. Alan did that. Sarah did this, etc)
2. **Write in first-person for the events that you yourself did. (eg. I did this, I did that.)
3. **verbatim Quotes**: Extract verbatim, high-salience quotes from dialogue if relevant. Do not modify or invent quotes.
4. **Determine Importance**: Assign an importance score from 1 (trivial, e.g. waking up) to 10 (life-altering, e.g. witnessing a crime).
4. Discard small body movements like looking around, sighing, etc that do not contextually hold any meaning after it is done.
5. **Involved Entities**: Identify all entity IDs involved in the memories in this chunk.
6. **Retain in Cognitive Buffer (Pinning)**: If a beat represents an unresolved high-stakes situation (e.g. a standing threat, an unanswered accusation, an ongoing chase or conflict), set "retainInBuffer" to true so it remains in the Cognitive Buffer for immediate context. Otherwise, set it to false so it is safely pruned from the Cognitive Buffer.
7. **Exclude stage business**: Glances, sighs, ambient noticing, and irrelevant sensory details should be ignored and not included in any promoted chunk. They will be forgotten.
8. **Forget by omission**: Any buffer entry ID that you do not include in any chunk's "sourceEntryIds" will be permanently deleted and forgotten.
`.trim();
const userContext = `
Subject Entity ID: ${entity.id}
Current Time: ${now.toISOString()}
Cognitive Buffer Candidates for Handoff:
${candidatesList}
`.trim();
const { systemPrompt, userContext, components } = this.promptBuilder.build(
entity,
candidates,
now,
);
const response = await this.llmProvider.generateStructuredResponse({
systemPrompt,
@@ -259,9 +249,23 @@ ${candidatesList}
});
if (!response.success || !response.data) {
this.lastResult = {
success: false,
systemPrompt,
userContext,
promptComponents: components,
};
return false;
}
this.lastResult = {
success: true,
systemPrompt,
userContext,
promptComponents: components,
response: response.data,
};
const result = response.data;
const db = (this.bufferRepo as any).db;

12
packages/voice/src/index.ts
View File

@@ -1,15 +1,3 @@
export * from "./dehydration.js";
export * from "./hydration.js";
export * from "./contractions.js";
export interface PromptComponent {
label: string;
type: "system" | "world" | "events" | "memories" | "input" | "other";
content: string;
}
export interface PromptBreakdown {
systemPrompt: string;
userContext: string;
components?: PromptComponent[];
}