mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-10 19:43:19 +08:00
重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
12
openclaw/extensions/memory-core/api.ts
Normal file
12
openclaw/extensions/memory-core/api.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
export type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core";
|
||||
export type {
|
||||
MemoryEmbeddingProbeResult,
|
||||
MemoryProviderStatus,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
export {
|
||||
dedupeDreamDiaryEntries,
|
||||
removeBackfillDiaryEntries,
|
||||
writeBackfillDiaryEntries,
|
||||
} from "./src/dreaming-narrative.js";
|
||||
export { previewGroundedRemMarkdown } from "./src/rem-evidence.js";
|
||||
24
openclaw/extensions/memory-core/cli-metadata.ts
Normal file
24
openclaw/extensions/memory-core/cli-metadata.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { definePluginEntry } from "openclaw/plugin-sdk/core";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "memory-core",
|
||||
name: "Memory (Core)",
|
||||
description: "File-backed memory search tools and CLI",
|
||||
register(api) {
|
||||
api.registerCli(
|
||||
async ({ program }) => {
|
||||
const { registerMemoryCli } = await import("./src/cli.js");
|
||||
registerMemoryCli(program);
|
||||
},
|
||||
{
|
||||
descriptors: [
|
||||
{
|
||||
name: "memory",
|
||||
description: "Search, inspect, and reindex memory files",
|
||||
hasSubcommands: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
182
openclaw/extensions/memory-core/index.test.ts
Normal file
182
openclaw/extensions/memory-core/index.test.ts
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildMemoryFlushPlan,
|
||||
buildPromptSection,
|
||||
DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES,
|
||||
DEFAULT_MEMORY_FLUSH_PROMPT,
|
||||
DEFAULT_MEMORY_FLUSH_SOFT_TOKENS,
|
||||
} from "./index.js";
|
||||
|
||||
describe("buildPromptSection", () => {
|
||||
it("returns empty when no memory tools are available", () => {
|
||||
expect(buildPromptSection({ availableTools: new Set() })).toEqual([]);
|
||||
});
|
||||
|
||||
it("describes the two-step flow when both memory tools are available", () => {
|
||||
const result = buildPromptSection({
|
||||
availableTools: new Set(["memory_search", "memory_get"]),
|
||||
});
|
||||
expect(result[0]).toBe("## Memory Recall");
|
||||
expect(result[1]).toContain("run memory_search");
|
||||
expect(result[1]).toContain("then use memory_get");
|
||||
expect(result[1]).toContain("indexed session transcripts");
|
||||
expect(result).toContain(
|
||||
"Citations: include Source: <path#line> when it helps the user verify memory snippets.",
|
||||
);
|
||||
expect(result.at(-1)).toBe("");
|
||||
});
|
||||
|
||||
it("limits the guidance to memory_search when only search is available", () => {
|
||||
const result = buildPromptSection({ availableTools: new Set(["memory_search"]) });
|
||||
expect(result[0]).toBe("## Memory Recall");
|
||||
expect(result[1]).toContain("run memory_search");
|
||||
expect(result[1]).toContain("indexed session transcripts");
|
||||
expect(result[1]).not.toContain("then use memory_get");
|
||||
});
|
||||
|
||||
it("limits the guidance to memory_get when only get is available", () => {
|
||||
const result = buildPromptSection({ availableTools: new Set(["memory_get"]) });
|
||||
expect(result[0]).toBe("## Memory Recall");
|
||||
expect(result[1]).toContain("run memory_get");
|
||||
expect(result[1]).not.toContain("run memory_search");
|
||||
});
|
||||
|
||||
it("includes citations-off instruction when citationsMode is off", () => {
|
||||
const result = buildPromptSection({
|
||||
availableTools: new Set(["memory_search"]),
|
||||
citationsMode: "off",
|
||||
});
|
||||
expect(result).toContain(
|
||||
"Citations are disabled: do not mention file paths or line numbers in replies unless the user explicitly asks.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildMemoryFlushPlan", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: {
|
||||
userTimezone: "America/New_York",
|
||||
timeFormat: "12",
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
it("replaces YYYY-MM-DD using user timezone and appends current time", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
...cfg,
|
||||
agents: {
|
||||
...cfg.agents,
|
||||
defaults: {
|
||||
...cfg.agents?.defaults,
|
||||
compaction: {
|
||||
memoryFlush: {
|
||||
prompt: "Store durable notes in memory/YYYY-MM-DD.md",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
nowMs: Date.UTC(2026, 1, 16, 15, 0, 0),
|
||||
});
|
||||
|
||||
expect(plan?.prompt).toContain("memory/2026-02-16.md");
|
||||
expect(plan?.prompt).toContain(
|
||||
"Current time: Monday, February 16th, 2026 - 10:00 AM (America/New_York) / 2026-02-16 15:00 UTC",
|
||||
);
|
||||
expect(plan?.relativePath).toBe("memory/2026-02-16.md");
|
||||
});
|
||||
|
||||
it("does not append a duplicate current time line", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
...cfg,
|
||||
agents: {
|
||||
...cfg.agents,
|
||||
defaults: {
|
||||
...cfg.agents?.defaults,
|
||||
compaction: {
|
||||
memoryFlush: {
|
||||
prompt: "Store notes.\nCurrent time: already present",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
nowMs: Date.UTC(2026, 1, 16, 15, 0, 0),
|
||||
});
|
||||
|
||||
expect(plan?.prompt).toContain("Current time: already present");
|
||||
expect((plan?.prompt.match(/Current time:/g) ?? []).length).toBe(1);
|
||||
});
|
||||
|
||||
it("defaults to safe prompts and gating values", () => {
|
||||
const plan = buildMemoryFlushPlan();
|
||||
expect(plan).not.toBeNull();
|
||||
expect(plan?.softThresholdTokens).toBe(DEFAULT_MEMORY_FLUSH_SOFT_TOKENS);
|
||||
expect(plan?.forceFlushTranscriptBytes).toBe(DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES);
|
||||
expect(plan?.prompt).toContain("memory/");
|
||||
expect(plan?.prompt).toContain("MEMORY.md");
|
||||
expect(plan?.systemPrompt).toContain("MEMORY.md");
|
||||
});
|
||||
|
||||
it("respects disable flag", () => {
|
||||
expect(
|
||||
buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: { compaction: { memoryFlush: { enabled: false } } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("falls back to defaults when numeric values are invalid", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: {
|
||||
compaction: {
|
||||
reserveTokensFloor: Number.NaN,
|
||||
memoryFlush: {
|
||||
softThresholdTokens: -100,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(plan?.softThresholdTokens).toBe(DEFAULT_MEMORY_FLUSH_SOFT_TOKENS);
|
||||
expect(plan?.forceFlushTranscriptBytes).toBe(DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES);
|
||||
expect(plan?.reserveTokensFloor).toBe(20_000);
|
||||
});
|
||||
|
||||
it("parses forceFlushTranscriptBytes from byte-size strings", () => {
|
||||
const plan = buildMemoryFlushPlan({
|
||||
cfg: {
|
||||
agents: {
|
||||
defaults: {
|
||||
compaction: {
|
||||
memoryFlush: {
|
||||
forceFlushTranscriptBytes: "3mb",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(plan?.forceFlushTranscriptBytes).toBe(3 * 1024 * 1024);
|
||||
});
|
||||
|
||||
it("keeps overwrite guards in the default prompt", () => {
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toMatch(/APPEND/i);
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("do not overwrite");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("timestamped variant");
|
||||
expect(DEFAULT_MEMORY_FLUSH_PROMPT).toContain("YYYY-MM-DD.md");
|
||||
});
|
||||
});
|
||||
75
openclaw/extensions/memory-core/index.ts
Normal file
75
openclaw/extensions/memory-core/index.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { registerMemoryCli } from "./src/cli.js";
|
||||
import { registerDreamingCommand } from "./src/dreaming-command.js";
|
||||
import { registerShortTermPromotionDreaming } from "./src/dreaming.js";
|
||||
import {
|
||||
buildMemoryFlushPlan,
|
||||
DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES,
|
||||
DEFAULT_MEMORY_FLUSH_PROMPT,
|
||||
DEFAULT_MEMORY_FLUSH_SOFT_TOKENS,
|
||||
} from "./src/flush-plan.js";
|
||||
import { registerBuiltInMemoryEmbeddingProviders } from "./src/memory/provider-adapters.js";
|
||||
import { buildPromptSection } from "./src/prompt-section.js";
|
||||
import { listMemoryCorePublicArtifacts } from "./src/public-artifacts.js";
|
||||
import { memoryRuntime } from "./src/runtime-provider.js";
|
||||
import { createMemoryGetTool, createMemorySearchTool } from "./src/tools.js";
|
||||
export {
|
||||
buildMemoryFlushPlan,
|
||||
DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES,
|
||||
DEFAULT_MEMORY_FLUSH_PROMPT,
|
||||
DEFAULT_MEMORY_FLUSH_SOFT_TOKENS,
|
||||
} from "./src/flush-plan.js";
|
||||
export { buildPromptSection } from "./src/prompt-section.js";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: "memory-core",
|
||||
name: "Memory (Core)",
|
||||
description: "File-backed memory search tools and CLI",
|
||||
kind: "memory",
|
||||
register(api) {
|
||||
registerBuiltInMemoryEmbeddingProviders(api);
|
||||
registerShortTermPromotionDreaming(api);
|
||||
registerDreamingCommand(api);
|
||||
api.registerMemoryCapability({
|
||||
promptBuilder: buildPromptSection,
|
||||
flushPlanResolver: buildMemoryFlushPlan,
|
||||
runtime: memoryRuntime,
|
||||
publicArtifacts: {
|
||||
listArtifacts: listMemoryCorePublicArtifacts,
|
||||
},
|
||||
});
|
||||
|
||||
api.registerTool(
|
||||
(ctx) =>
|
||||
createMemorySearchTool({
|
||||
config: ctx.config,
|
||||
agentSessionKey: ctx.sessionKey,
|
||||
}),
|
||||
{ names: ["memory_search"] },
|
||||
);
|
||||
|
||||
api.registerTool(
|
||||
(ctx) =>
|
||||
createMemoryGetTool({
|
||||
config: ctx.config,
|
||||
agentSessionKey: ctx.sessionKey,
|
||||
}),
|
||||
{ names: ["memory_get"] },
|
||||
);
|
||||
|
||||
api.registerCli(
|
||||
({ program }) => {
|
||||
registerMemoryCli(program);
|
||||
},
|
||||
{
|
||||
descriptors: [
|
||||
{
|
||||
name: "memory",
|
||||
description: "Search, inspect, and reindex memory files",
|
||||
hasSubcommands: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
1
openclaw/extensions/memory-core/manager-runtime.ts
Normal file
1
openclaw/extensions/memory-core/manager-runtime.ts
Normal file
|
|
@ -0,0 +1 @@
|
|||
export { closeAllMemoryIndexManagers, MemoryIndexManager } from "./src/memory/manager-runtime.js";
|
||||
139
openclaw/extensions/memory-core/openclaw.plugin.json
Normal file
139
openclaw/extensions/memory-core/openclaw.plugin.json
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
{
|
||||
"id": "memory-core",
|
||||
"kind": "memory",
|
||||
"commandAliases": [
|
||||
{
|
||||
"name": "dreaming",
|
||||
"kind": "runtime-slash",
|
||||
"cliCommand": "memory"
|
||||
}
|
||||
],
|
||||
"uiHints": {
|
||||
"dreaming.frequency": {
|
||||
"label": "Dreaming Frequency",
|
||||
"placeholder": "0 3 * * *",
|
||||
"help": "Optional cron cadence for the full dreaming sweep (light, REM, then deep)."
|
||||
}
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"dreaming": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"frequency": {
|
||||
"type": "string"
|
||||
},
|
||||
"timezone": {
|
||||
"type": "string"
|
||||
},
|
||||
"verboseLogging": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"storage": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": ["inline", "separate", "both"]
|
||||
},
|
||||
"separateReports": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"phases": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"light": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"lookbackDays": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"dedupeSimilarity": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"deep": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"minScore": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"minRecallCount": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"minUniqueQueries": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"recencyHalfLifeDays": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"maxAgeDays": {
|
||||
"type": "integer",
|
||||
"minimum": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"rem": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"lookbackDays": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"minPatternStrength": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
24
openclaw/extensions/memory-core/package.json
Normal file
24
openclaw/extensions/memory-core/package.json
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
{
|
||||
"name": "@openclaw/memory-core",
|
||||
"version": "2026.4.20",
|
||||
"private": true,
|
||||
"description": "OpenClaw core memory search plugin",
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*",
|
||||
"openclaw": "workspace:*"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"openclaw": ">=2026.4.20"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"openclaw": {
|
||||
"optional": true
|
||||
}
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
32
openclaw/extensions/memory-core/runtime-api.ts
Normal file
32
openclaw/extensions/memory-core/runtime-api.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
export { getMemorySearchManager, MemoryIndexManager } from "./src/memory/index.js";
|
||||
export { memoryRuntime } from "./src/runtime-provider.js";
|
||||
export {
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
getBuiltinMemoryEmbeddingProviderDoctorMetadata,
|
||||
listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata,
|
||||
registerBuiltInMemoryEmbeddingProviders,
|
||||
} from "./src/memory/provider-adapters.js";
|
||||
export { createEmbeddingProvider } from "./src/memory/embeddings.js";
|
||||
export {
|
||||
resolveMemoryCacheSummary,
|
||||
resolveMemoryFtsState,
|
||||
resolveMemoryVectorState,
|
||||
type Tone,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
export { checkQmdBinaryAvailability } from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
|
||||
export { hasConfiguredMemorySecretInput } from "openclaw/plugin-sdk/memory-core-host-secret";
|
||||
export { auditDreamingArtifacts, repairDreamingArtifacts } from "./src/dreaming-repair.js";
|
||||
export {
|
||||
auditShortTermPromotionArtifacts,
|
||||
removeGroundedShortTermCandidates,
|
||||
repairShortTermPromotionArtifacts,
|
||||
} from "./src/short-term-promotion.js";
|
||||
export type { BuiltinMemoryEmbeddingProviderDoctorMetadata } from "./src/memory/provider-adapters.js";
|
||||
export type {
|
||||
DreamingArtifactsAuditSummary,
|
||||
RepairDreamingArtifactsResult,
|
||||
} from "./src/dreaming-repair.js";
|
||||
export type {
|
||||
RepairShortTermPromotionArtifactsResult,
|
||||
ShortTermAuditSummary,
|
||||
} from "./src/short-term-promotion.js";
|
||||
26
openclaw/extensions/memory-core/src/cli.host.runtime.ts
Normal file
26
openclaw/extensions/memory-core/src/cli.host.runtime.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
export {
|
||||
colorize,
|
||||
defaultRuntime,
|
||||
formatErrorMessage,
|
||||
isRich,
|
||||
resolveCommandSecretRefsViaGateway,
|
||||
setVerbose,
|
||||
shortenHomeInString,
|
||||
shortenHomePath,
|
||||
theme,
|
||||
withManager,
|
||||
withProgress,
|
||||
withProgressTotals,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-cli";
|
||||
export {
|
||||
loadConfig,
|
||||
resolveDefaultAgentId,
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
resolveStateDir,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
export {
|
||||
listMemoryFiles,
|
||||
normalizeExtraMemoryPaths,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-files";
|
||||
export { getMemorySearchManager } from "./memory/index.js";
|
||||
1935
openclaw/extensions/memory-core/src/cli.runtime.ts
Normal file
1935
openclaw/extensions/memory-core/src/cli.runtime.ts
Normal file
File diff suppressed because it is too large
Load diff
1624
openclaw/extensions/memory-core/src/cli.test.ts
Normal file
1624
openclaw/extensions/memory-core/src/cli.test.ts
Normal file
File diff suppressed because it is too large
Load diff
222
openclaw/extensions/memory-core/src/cli.ts
Normal file
222
openclaw/extensions/memory-core/src/cli.ts
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
import type { Command } from "commander";
|
||||
import {
|
||||
formatDocsLink,
|
||||
formatHelpExamples,
|
||||
theme,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-cli";
|
||||
import type {
|
||||
MemoryCommandOptions,
|
||||
MemoryPromoteCommandOptions,
|
||||
MemoryPromoteExplainOptions,
|
||||
MemoryRemBackfillOptions,
|
||||
MemoryRemHarnessOptions,
|
||||
MemorySearchCommandOptions,
|
||||
} from "./cli.types.js";
|
||||
import {
|
||||
DEFAULT_PROMOTION_MIN_RECALL_COUNT,
|
||||
DEFAULT_PROMOTION_MIN_SCORE,
|
||||
DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES,
|
||||
} from "./short-term-promotion.js";
|
||||
|
||||
type MemoryCliRuntime = typeof import("./cli.runtime.js");
|
||||
|
||||
let memoryCliRuntimePromise: Promise<MemoryCliRuntime> | null = null;
|
||||
|
||||
async function loadMemoryCliRuntime(): Promise<MemoryCliRuntime> {
|
||||
memoryCliRuntimePromise ??= import("./cli.runtime.js");
|
||||
return await memoryCliRuntimePromise;
|
||||
}
|
||||
|
||||
export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryStatus(opts);
|
||||
}
|
||||
|
||||
async function runMemoryIndex(opts: MemoryCommandOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryIndex(opts);
|
||||
}
|
||||
|
||||
async function runMemorySearch(queryArg: string | undefined, opts: MemorySearchCommandOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemorySearch(queryArg, opts);
|
||||
}
|
||||
|
||||
async function runMemoryPromote(opts: MemoryPromoteCommandOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryPromote(opts);
|
||||
}
|
||||
|
||||
async function runMemoryPromoteExplain(
|
||||
selectorArg: string | undefined,
|
||||
opts: MemoryPromoteExplainOptions,
|
||||
) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryPromoteExplain(selectorArg, opts);
|
||||
}
|
||||
|
||||
async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryRemHarness(opts);
|
||||
}
|
||||
|
||||
async function runMemoryRemBackfill(opts: MemoryRemBackfillOptions) {
|
||||
const runtime = await loadMemoryCliRuntime();
|
||||
await runtime.runMemoryRemBackfill(opts);
|
||||
}
|
||||
|
||||
export function registerMemoryCli(program: Command) {
|
||||
const memory = program
|
||||
.command("memory")
|
||||
.description("Search, inspect, and reindex memory files")
|
||||
.addHelpText(
|
||||
"after",
|
||||
() =>
|
||||
`\n${theme.heading("Examples:")}\n${formatHelpExamples([
|
||||
["openclaw memory status", "Show index and provider status."],
|
||||
[
|
||||
"openclaw memory status --fix",
|
||||
"Repair stale recall locks and normalize promotion metadata.",
|
||||
],
|
||||
["openclaw memory status --deep", "Probe embedding provider readiness."],
|
||||
["openclaw memory index --force", "Force a full reindex."],
|
||||
['openclaw memory search "meeting notes"', "Quick search using positional query."],
|
||||
[
|
||||
'openclaw memory search --query "deployment" --max-results 20',
|
||||
"Limit results for focused troubleshooting.",
|
||||
],
|
||||
[
|
||||
`openclaw memory promote --limit 10 --min-score ${DEFAULT_PROMOTION_MIN_SCORE}`,
|
||||
"Review weighted short-term candidates for long-term memory.",
|
||||
],
|
||||
[
|
||||
"openclaw memory promote --apply",
|
||||
"Append top-ranked short-term candidates into MEMORY.md.",
|
||||
],
|
||||
[
|
||||
'openclaw memory promote-explain "router vlan"',
|
||||
"Explain why a specific candidate would or would not promote.",
|
||||
],
|
||||
[
|
||||
"openclaw memory rem-harness --json",
|
||||
"Preview REM reflections, candidate truths, and deep promotion output.",
|
||||
],
|
||||
[
|
||||
"openclaw memory rem-backfill --path ./memory",
|
||||
"Write grounded historical REM entries into DREAMS.md for UI review.",
|
||||
],
|
||||
[
|
||||
"openclaw memory rem-backfill --path ./memory --stage-short-term",
|
||||
"Also seed durable grounded candidates into the live short-term promotion store.",
|
||||
],
|
||||
["openclaw memory status --json", "Output machine-readable JSON (good for scripts)."],
|
||||
])}\n\n${theme.muted("Docs:")} ${formatDocsLink("/cli/memory", "docs.openclaw.ai/cli/memory")}\n`,
|
||||
);
|
||||
|
||||
memory
|
||||
.command("status")
|
||||
.description("Show memory search index status")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--json", "Print JSON")
|
||||
.option("--deep", "Probe embedding provider availability")
|
||||
.option("--index", "Reindex if dirty (implies --deep)")
|
||||
.option("--fix", "Repair stale recall locks and normalize promotion metadata")
|
||||
.option("--verbose", "Verbose logging", false)
|
||||
.action(async (opts: MemoryCommandOptions & { force?: boolean }) => {
|
||||
await runMemoryStatus(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("index")
|
||||
.description("Reindex memory files")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--force", "Force full reindex", false)
|
||||
.option("--verbose", "Verbose logging", false)
|
||||
.action(async (opts: MemoryCommandOptions) => {
|
||||
await runMemoryIndex(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("search")
|
||||
.description("Search memory files")
|
||||
.argument("[query]", "Search query")
|
||||
.option("--query <text>", "Search query (alternative to positional argument)")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--max-results <n>", "Max results", (value: string) => Number(value))
|
||||
.option("--min-score <n>", "Minimum score", (value: string) => Number(value))
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (queryArg: string | undefined, opts: MemorySearchCommandOptions) => {
|
||||
await runMemorySearch(queryArg, opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("promote")
|
||||
.description("Rank short-term recalls and optionally append top entries to MEMORY.md")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--limit <n>", "Max candidates", (value: string) => Number(value))
|
||||
.option(
|
||||
"--min-score <n>",
|
||||
`Minimum weighted score (default: ${DEFAULT_PROMOTION_MIN_SCORE})`,
|
||||
(value: string) => Number(value),
|
||||
)
|
||||
.option(
|
||||
"--min-recall-count <n>",
|
||||
`Minimum recall count (default: ${DEFAULT_PROMOTION_MIN_RECALL_COUNT})`,
|
||||
(value: string) => Number(value),
|
||||
)
|
||||
.option(
|
||||
"--min-unique-queries <n>",
|
||||
`Minimum distinct query count (default: ${DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES})`,
|
||||
(value: string) => Number(value),
|
||||
)
|
||||
.option("--apply", "Append selected candidates to MEMORY.md", false)
|
||||
.option("--include-promoted", "Include already promoted candidates", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: MemoryPromoteCommandOptions) => {
|
||||
await runMemoryPromote(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("promote-explain")
|
||||
.description("Explain a specific promotion candidate and its score breakdown")
|
||||
.argument("<selector>", "Candidate key, path fragment, or snippet fragment")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--include-promoted", "Include already promoted candidates", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (selectorArg: string | undefined, opts: MemoryPromoteExplainOptions) => {
|
||||
await runMemoryPromoteExplain(selectorArg, opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("rem-harness")
|
||||
.description("Preview REM reflections, candidate truths, and deep promotions without writing")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--path <file-or-dir>", "Seed the harness from historical daily memory file(s)")
|
||||
.option("--grounded", "Also render a grounded day-level REM preview")
|
||||
.option("--include-promoted", "Include already promoted deep candidates", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: MemoryRemHarnessOptions) => {
|
||||
await runMemoryRemHarness(opts);
|
||||
});
|
||||
|
||||
memory
|
||||
.command("rem-backfill")
|
||||
.description("Write grounded historical REM summaries into DREAMS.md for UI review")
|
||||
.option("--agent <id>", "Agent id (default: default agent)")
|
||||
.option("--path <file-or-dir>", "Historical daily memory file(s) or directory")
|
||||
.option("--rollback", "Remove previously written grounded REM backfill entries", false)
|
||||
.option(
|
||||
"--stage-short-term",
|
||||
"Also seed grounded durable candidates into the short-term promotion store",
|
||||
false,
|
||||
)
|
||||
.option(
|
||||
"--rollback-short-term",
|
||||
"Remove previously seeded grounded short-term candidates",
|
||||
false,
|
||||
)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: MemoryRemBackfillOptions) => {
|
||||
await runMemoryRemBackfill(opts);
|
||||
});
|
||||
}
|
||||
41
openclaw/extensions/memory-core/src/cli.types.ts
Normal file
41
openclaw/extensions/memory-core/src/cli.types.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
export type MemoryCommandOptions = {
|
||||
agent?: string;
|
||||
json?: boolean;
|
||||
deep?: boolean;
|
||||
index?: boolean;
|
||||
force?: boolean;
|
||||
fix?: boolean;
|
||||
verbose?: boolean;
|
||||
};
|
||||
|
||||
export type MemorySearchCommandOptions = MemoryCommandOptions & {
|
||||
query?: string;
|
||||
maxResults?: number;
|
||||
minScore?: number;
|
||||
};
|
||||
|
||||
export type MemoryPromoteCommandOptions = MemoryCommandOptions & {
|
||||
limit?: number;
|
||||
minScore?: number;
|
||||
minRecallCount?: number;
|
||||
minUniqueQueries?: number;
|
||||
apply?: boolean;
|
||||
includePromoted?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryPromoteExplainOptions = MemoryCommandOptions & {
|
||||
includePromoted?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryRemHarnessOptions = MemoryCommandOptions & {
|
||||
includePromoted?: boolean;
|
||||
path?: string;
|
||||
grounded?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryRemBackfillOptions = MemoryCommandOptions & {
|
||||
path?: string;
|
||||
rollback?: boolean;
|
||||
stageShortTerm?: boolean;
|
||||
rollbackShortTerm?: boolean;
|
||||
};
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
classifyConceptTagScript,
|
||||
deriveConceptTags,
|
||||
summarizeConceptTagScriptCoverage,
|
||||
} from "./concept-vocabulary.js";
|
||||
|
||||
describe("concept vocabulary", () => {
|
||||
it("extracts Unicode-aware concept tags for common European languages", () => {
|
||||
const tags = deriveConceptTags({
|
||||
path: "memory/2026-04-04.md",
|
||||
snippet:
|
||||
"Configuración de gateway, configuration du routeur, Sicherung und Überwachung Glacier.",
|
||||
});
|
||||
|
||||
expect(tags).toEqual(
|
||||
expect.arrayContaining([
|
||||
"gateway",
|
||||
"configuración",
|
||||
"configuration",
|
||||
"routeur",
|
||||
"sicherung",
|
||||
"überwachung",
|
||||
"glacier",
|
||||
]),
|
||||
);
|
||||
expect(tags).not.toContain("de");
|
||||
expect(tags).not.toContain("du");
|
||||
expect(tags).not.toContain("und");
|
||||
expect(tags).not.toContain("2026-04-04.md");
|
||||
});
|
||||
|
||||
it("extracts protected and segmented CJK concept tags", () => {
|
||||
const tags = deriveConceptTags({
|
||||
path: "memory/2026-04-04.md",
|
||||
snippet:
|
||||
"障害対応ルーター設定とバックアップ確認。路由器备份与网关同步。라우터 백업 페일오버 점검.",
|
||||
});
|
||||
|
||||
expect(tags).toEqual(
|
||||
expect.arrayContaining([
|
||||
"障害対応",
|
||||
"ルーター",
|
||||
"バックアップ",
|
||||
"路由器",
|
||||
"备份",
|
||||
"网关",
|
||||
"라우터",
|
||||
"백업",
|
||||
]),
|
||||
);
|
||||
expect(tags).not.toContain("ルー");
|
||||
expect(tags).not.toContain("ター");
|
||||
});
|
||||
|
||||
it("classifies concept tags by script family", () => {
|
||||
expect(classifyConceptTagScript("routeur")).toBe("latin");
|
||||
expect(classifyConceptTagScript("路由器")).toBe("cjk");
|
||||
expect(classifyConceptTagScript("qmd路由器")).toBe("mixed");
|
||||
});
|
||||
|
||||
it("summarizes entry coverage across latin, cjk, and mixed tags", () => {
|
||||
expect(
|
||||
summarizeConceptTagScriptCoverage([
|
||||
["routeur", "sauvegarde"],
|
||||
["路由器", "备份"],
|
||||
["qmd", "路由器"],
|
||||
["сервер"],
|
||||
]),
|
||||
).toEqual({
|
||||
latinEntryCount: 1,
|
||||
cjkEntryCount: 1,
|
||||
mixedEntryCount: 1,
|
||||
otherEntryCount: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
473
openclaw/extensions/memory-core/src/concept-vocabulary.ts
Normal file
473
openclaw/extensions/memory-core/src/concept-vocabulary.ts
Normal file
|
|
@ -0,0 +1,473 @@
|
|||
import path from "node:path";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
export const MAX_CONCEPT_TAGS = 8;
|
||||
|
||||
export type ConceptTagScriptFamily = "latin" | "cjk" | "mixed" | "other";
|
||||
|
||||
export type ConceptTagScriptCoverage = {
|
||||
latinEntryCount: number;
|
||||
cjkEntryCount: number;
|
||||
mixedEntryCount: number;
|
||||
otherEntryCount: number;
|
||||
};
|
||||
|
||||
const LANGUAGE_STOP_WORDS = {
|
||||
shared: [
|
||||
"about",
|
||||
"after",
|
||||
"agent",
|
||||
"again",
|
||||
"also",
|
||||
"because",
|
||||
"before",
|
||||
"being",
|
||||
"between",
|
||||
"build",
|
||||
"called",
|
||||
"could",
|
||||
"daily",
|
||||
"default",
|
||||
"deploy",
|
||||
"during",
|
||||
"every",
|
||||
"file",
|
||||
"files",
|
||||
"from",
|
||||
"have",
|
||||
"into",
|
||||
"just",
|
||||
"line",
|
||||
"lines",
|
||||
"long",
|
||||
"main",
|
||||
"make",
|
||||
"memory",
|
||||
"month",
|
||||
"more",
|
||||
"most",
|
||||
"move",
|
||||
"much",
|
||||
"next",
|
||||
"note",
|
||||
"notes",
|
||||
"over",
|
||||
"part",
|
||||
"past",
|
||||
"port",
|
||||
"same",
|
||||
"score",
|
||||
"search",
|
||||
"session",
|
||||
"sessions",
|
||||
"short",
|
||||
"should",
|
||||
"since",
|
||||
"some",
|
||||
"than",
|
||||
"that",
|
||||
"their",
|
||||
"there",
|
||||
"these",
|
||||
"they",
|
||||
"this",
|
||||
"through",
|
||||
"today",
|
||||
"using",
|
||||
"with",
|
||||
"work",
|
||||
"workspace",
|
||||
"year",
|
||||
],
|
||||
english: ["and", "are", "for", "into", "its", "our", "then", "were"],
|
||||
spanish: [
|
||||
"al",
|
||||
"con",
|
||||
"como",
|
||||
"de",
|
||||
"del",
|
||||
"el",
|
||||
"en",
|
||||
"es",
|
||||
"la",
|
||||
"las",
|
||||
"los",
|
||||
"para",
|
||||
"por",
|
||||
"que",
|
||||
"se",
|
||||
"sin",
|
||||
"su",
|
||||
"sus",
|
||||
"una",
|
||||
"uno",
|
||||
"unos",
|
||||
"unas",
|
||||
"y",
|
||||
],
|
||||
french: [
|
||||
"au",
|
||||
"aux",
|
||||
"avec",
|
||||
"dans",
|
||||
"de",
|
||||
"des",
|
||||
"du",
|
||||
"en",
|
||||
"est",
|
||||
"et",
|
||||
"la",
|
||||
"le",
|
||||
"les",
|
||||
"ou",
|
||||
"pour",
|
||||
"que",
|
||||
"qui",
|
||||
"sans",
|
||||
"ses",
|
||||
"son",
|
||||
"sur",
|
||||
"une",
|
||||
"un",
|
||||
],
|
||||
german: [
|
||||
"auf",
|
||||
"aus",
|
||||
"bei",
|
||||
"das",
|
||||
"dem",
|
||||
"den",
|
||||
"der",
|
||||
"des",
|
||||
"die",
|
||||
"ein",
|
||||
"eine",
|
||||
"einem",
|
||||
"einen",
|
||||
"einer",
|
||||
"für",
|
||||
"im",
|
||||
"in",
|
||||
"mit",
|
||||
"nach",
|
||||
"oder",
|
||||
"ohne",
|
||||
"über",
|
||||
"und",
|
||||
"von",
|
||||
"zu",
|
||||
"zum",
|
||||
"zur",
|
||||
],
|
||||
cjk: [
|
||||
"が",
|
||||
"から",
|
||||
"する",
|
||||
"して",
|
||||
"した",
|
||||
"で",
|
||||
"と",
|
||||
"に",
|
||||
"の",
|
||||
"は",
|
||||
"へ",
|
||||
"まで",
|
||||
"も",
|
||||
"や",
|
||||
"を",
|
||||
"与",
|
||||
"为",
|
||||
"了",
|
||||
"及",
|
||||
"和",
|
||||
"在",
|
||||
"将",
|
||||
"或",
|
||||
"把",
|
||||
"是",
|
||||
"用",
|
||||
"的",
|
||||
"과",
|
||||
"는",
|
||||
"도",
|
||||
"로",
|
||||
"를",
|
||||
"에",
|
||||
"에서",
|
||||
"와",
|
||||
"은",
|
||||
"으로",
|
||||
"을",
|
||||
"이",
|
||||
"하다",
|
||||
"한",
|
||||
"할",
|
||||
"해",
|
||||
"했다",
|
||||
"했다",
|
||||
],
|
||||
pathNoise: [
|
||||
"cjs",
|
||||
"cpp",
|
||||
"cts",
|
||||
"jsx",
|
||||
"json",
|
||||
"md",
|
||||
"mjs",
|
||||
"mts",
|
||||
"text",
|
||||
"toml",
|
||||
"ts",
|
||||
"tsx",
|
||||
"txt",
|
||||
"yaml",
|
||||
"yml",
|
||||
],
|
||||
} as const;
|
||||
|
||||
const CONCEPT_STOP_WORDS = new Set(
|
||||
Object.values(LANGUAGE_STOP_WORDS)
|
||||
.flatMap((words) => words)
|
||||
.map((word) => normalizeLowercaseStringOrEmpty(word)),
|
||||
);
|
||||
|
||||
const PROTECTED_GLOSSARY = [
|
||||
"backup",
|
||||
"backups",
|
||||
"embedding",
|
||||
"embeddings",
|
||||
"failover",
|
||||
"gateway",
|
||||
"glacier",
|
||||
"gpt",
|
||||
"kv",
|
||||
"network",
|
||||
"openai",
|
||||
"qmd",
|
||||
"router",
|
||||
"s3",
|
||||
"vlan",
|
||||
"sauvegarde",
|
||||
"routeur",
|
||||
"passerelle",
|
||||
"konfiguration",
|
||||
"sicherung",
|
||||
"überwachung",
|
||||
"configuración",
|
||||
"respaldo",
|
||||
"enrutador",
|
||||
"puerta-de-enlace",
|
||||
"バックアップ",
|
||||
"フェイルオーバー",
|
||||
"ルーター",
|
||||
"ネットワーク",
|
||||
"ゲートウェイ",
|
||||
"障害対応",
|
||||
"路由器",
|
||||
"备份",
|
||||
"故障转移",
|
||||
"网络",
|
||||
"网关",
|
||||
"라우터",
|
||||
"백업",
|
||||
"페일오버",
|
||||
"네트워크",
|
||||
"게이트웨이",
|
||||
"장애대응",
|
||||
].map((word) => normalizeLowercaseStringOrEmpty(word.normalize("NFKC")));
|
||||
|
||||
const COMPOUND_TOKEN_RE = /[\p{L}\p{N}]+(?:[._/-][\p{L}\p{N}]+)+/gu;
|
||||
const LETTER_OR_NUMBER_RE = /[\p{L}\p{N}]/u;
|
||||
const LATIN_RE = /\p{Script=Latin}/u;
|
||||
const HAN_RE = /\p{Script=Han}/u;
|
||||
const HIRAGANA_RE = /\p{Script=Hiragana}/u;
|
||||
const KATAKANA_RE = /\p{Script=Katakana}/u;
|
||||
const HANGUL_RE = /\p{Script=Hangul}/u;
|
||||
|
||||
const DEFAULT_WORD_SEGMENTER =
|
||||
typeof Intl.Segmenter === "function" ? new Intl.Segmenter("und", { granularity: "word" }) : null;
|
||||
|
||||
function containsLetterOrNumber(value: string): boolean {
|
||||
return LETTER_OR_NUMBER_RE.test(value);
|
||||
}
|
||||
|
||||
export function classifyConceptTagScript(tag: string): ConceptTagScriptFamily {
|
||||
const normalized = tag.normalize("NFKC");
|
||||
const hasLatin = LATIN_RE.test(normalized);
|
||||
const hasCjk =
|
||||
HAN_RE.test(normalized) ||
|
||||
HIRAGANA_RE.test(normalized) ||
|
||||
KATAKANA_RE.test(normalized) ||
|
||||
HANGUL_RE.test(normalized);
|
||||
if (hasLatin && hasCjk) {
|
||||
return "mixed";
|
||||
}
|
||||
if (hasCjk) {
|
||||
return "cjk";
|
||||
}
|
||||
if (hasLatin) {
|
||||
return "latin";
|
||||
}
|
||||
return "other";
|
||||
}
|
||||
|
||||
function minimumTokenLengthForScript(script: ConceptTagScriptFamily): number {
|
||||
if (script === "cjk") {
|
||||
return 2;
|
||||
}
|
||||
return 3;
|
||||
}
|
||||
|
||||
function isKanaOnlyToken(value: string): boolean {
|
||||
return (
|
||||
!HAN_RE.test(value) &&
|
||||
!HANGUL_RE.test(value) &&
|
||||
(HIRAGANA_RE.test(value) || KATAKANA_RE.test(value))
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeConceptToken(rawToken: string): string | null {
|
||||
const normalized = normalizeLowercaseStringOrEmpty(
|
||||
rawToken
|
||||
.normalize("NFKC")
|
||||
.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")
|
||||
.replaceAll("_", "-"),
|
||||
);
|
||||
if (!normalized || !containsLetterOrNumber(normalized) || normalized.length > 32) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
/^\d+$/.test(normalized) ||
|
||||
/^\d{4}-\d{2}-\d{2}$/u.test(normalized) ||
|
||||
/^\d{4}-\d{2}-\d{2}\.[\p{L}\p{N}]+$/u.test(normalized)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const script = classifyConceptTagScript(normalized);
|
||||
if (normalized.length < minimumTokenLengthForScript(script)) {
|
||||
return null;
|
||||
}
|
||||
if (isKanaOnlyToken(normalized) && normalized.length < 3) {
|
||||
return null;
|
||||
}
|
||||
if (CONCEPT_STOP_WORDS.has(normalized)) {
|
||||
return null;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function collectGlossaryMatches(source: string): string[] {
|
||||
const normalizedSource = normalizeLowercaseStringOrEmpty(source.normalize("NFKC"));
|
||||
const matches: string[] = [];
|
||||
for (const entry of PROTECTED_GLOSSARY) {
|
||||
if (!normalizedSource.includes(entry)) {
|
||||
continue;
|
||||
}
|
||||
matches.push(entry);
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
|
||||
function collectCompoundTokens(source: string): string[] {
|
||||
return source.match(COMPOUND_TOKEN_RE) ?? [];
|
||||
}
|
||||
|
||||
function collectSegmentTokens(source: string): string[] {
|
||||
if (DEFAULT_WORD_SEGMENTER) {
|
||||
return Array.from(DEFAULT_WORD_SEGMENTER.segment(source), (part) =>
|
||||
part.isWordLike ? part.segment : "",
|
||||
).filter(Boolean);
|
||||
}
|
||||
return source.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
|
||||
}
|
||||
|
||||
function pushNormalizedTag(tags: string[], rawToken: string, limit: number): void {
|
||||
const normalized = normalizeConceptToken(rawToken);
|
||||
if (!normalized || tags.includes(normalized)) {
|
||||
return;
|
||||
}
|
||||
tags.push(normalized);
|
||||
if (tags.length > limit) {
|
||||
tags.splice(limit);
|
||||
}
|
||||
}
|
||||
|
||||
export function deriveConceptTags(params: {
|
||||
path: string;
|
||||
snippet: string;
|
||||
limit?: number;
|
||||
}): string[] {
|
||||
const source = `${path.basename(params.path)} ${params.snippet}`;
|
||||
const limit = Number.isFinite(params.limit)
|
||||
? Math.max(0, Math.floor(params.limit as number))
|
||||
: MAX_CONCEPT_TAGS;
|
||||
if (limit === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const tags: string[] = [];
|
||||
for (const rawToken of [
|
||||
...collectGlossaryMatches(source),
|
||||
...collectCompoundTokens(source),
|
||||
...collectSegmentTokens(source),
|
||||
]) {
|
||||
pushNormalizedTag(tags, rawToken, limit);
|
||||
if (tags.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
|
||||
export function summarizeConceptTagScriptCoverage(
|
||||
conceptTagsByEntry: string[][],
|
||||
): ConceptTagScriptCoverage {
|
||||
const coverage: ConceptTagScriptCoverage = {
|
||||
latinEntryCount: 0,
|
||||
cjkEntryCount: 0,
|
||||
mixedEntryCount: 0,
|
||||
otherEntryCount: 0,
|
||||
};
|
||||
|
||||
for (const conceptTags of conceptTagsByEntry) {
|
||||
let hasLatin = false;
|
||||
let hasCjk = false;
|
||||
let hasOther = false;
|
||||
for (const tag of conceptTags) {
|
||||
const family = classifyConceptTagScript(tag);
|
||||
if (family === "mixed") {
|
||||
hasLatin = true;
|
||||
hasCjk = true;
|
||||
continue;
|
||||
}
|
||||
if (family === "latin") {
|
||||
hasLatin = true;
|
||||
continue;
|
||||
}
|
||||
if (family === "cjk") {
|
||||
hasCjk = true;
|
||||
continue;
|
||||
}
|
||||
hasOther = true;
|
||||
}
|
||||
|
||||
if (hasLatin && hasCjk) {
|
||||
coverage.mixedEntryCount += 1;
|
||||
} else if (hasCjk) {
|
||||
coverage.cjkEntryCount += 1;
|
||||
} else if (hasLatin) {
|
||||
coverage.latinEntryCount += 1;
|
||||
} else if (hasOther) {
|
||||
coverage.otherEntryCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return coverage;
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
normalizeConceptToken,
|
||||
collectGlossaryMatches,
|
||||
collectCompoundTokens,
|
||||
collectSegmentTokens,
|
||||
};
|
||||
52
openclaw/extensions/memory-core/src/config.test.ts
Normal file
52
openclaw/extensions/memory-core/src/config.test.ts
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import fs from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { validateJsonSchemaValue } from "../../../src/plugins/schema-validator.js";
|
||||
|
||||
const manifest = JSON.parse(
|
||||
fs.readFileSync(new URL("../openclaw.plugin.json", import.meta.url), "utf-8"),
|
||||
) as { configSchema: Record<string, unknown> };
|
||||
|
||||
describe("memory-core manifest config schema", () => {
|
||||
it("accepts dreaming phase thresholds used by QA and runtime", () => {
|
||||
const result = validateJsonSchemaValue({
|
||||
schema: manifest.configSchema,
|
||||
cacheKey: "memory-core.manifest.dreaming-phase-thresholds",
|
||||
value: {
|
||||
dreaming: {
|
||||
enabled: true,
|
||||
timezone: "Europe/London",
|
||||
verboseLogging: true,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
phases: {
|
||||
light: {
|
||||
enabled: true,
|
||||
lookbackDays: 2,
|
||||
limit: 20,
|
||||
dedupeSimilarity: 0.9,
|
||||
},
|
||||
deep: {
|
||||
enabled: true,
|
||||
limit: 10,
|
||||
minScore: 0,
|
||||
minRecallCount: 3,
|
||||
minUniqueQueries: 3,
|
||||
recencyHalfLifeDays: 14,
|
||||
maxAgeDays: 30,
|
||||
},
|
||||
rem: {
|
||||
enabled: true,
|
||||
lookbackDays: 7,
|
||||
limit: 10,
|
||||
minPatternStrength: 0.75,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
});
|
||||
});
|
||||
200
openclaw/extensions/memory-core/src/dreaming-command.test.ts
Normal file
200
openclaw/extensions/memory-core/src/dreaming-command.test.ts
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
import type {
|
||||
OpenClawPluginCommandDefinition,
|
||||
PluginCommandContext,
|
||||
} from "openclaw/plugin-sdk/core";
|
||||
import type { OpenClawConfig, OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { registerDreamingCommand } from "./dreaming-command.js";
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function resolveStoredDreaming(config: OpenClawConfig): Record<string, unknown> {
|
||||
const entry = asRecord(config.plugins?.entries?.["memory-core"]);
|
||||
const pluginConfig = asRecord(entry?.config);
|
||||
return asRecord(pluginConfig?.dreaming) ?? {};
|
||||
}
|
||||
|
||||
function createHarness(initialConfig: OpenClawConfig = {}) {
|
||||
const registered: { command?: OpenClawPluginCommandDefinition } = {};
|
||||
let runtimeConfig: OpenClawConfig = initialConfig;
|
||||
|
||||
const runtime = {
|
||||
config: {
|
||||
loadConfig: vi.fn(() => runtimeConfig),
|
||||
writeConfigFile: vi.fn(async (nextConfig: OpenClawConfig) => {
|
||||
runtimeConfig = nextConfig;
|
||||
}),
|
||||
},
|
||||
} as unknown as OpenClawPluginApi["runtime"];
|
||||
|
||||
const api = {
|
||||
runtime,
|
||||
registerCommand: vi.fn((definition: OpenClawPluginCommandDefinition) => {
|
||||
registered.command = definition;
|
||||
}),
|
||||
} as unknown as OpenClawPluginApi;
|
||||
|
||||
registerDreamingCommand(api);
|
||||
|
||||
if (!registered.command) {
|
||||
throw new Error("memory-core did not register /dreaming");
|
||||
}
|
||||
|
||||
return {
|
||||
command: registered.command,
|
||||
runtime,
|
||||
getRuntimeConfig: () => runtimeConfig,
|
||||
};
|
||||
}
|
||||
|
||||
function createCommandContext(
|
||||
args?: string,
|
||||
overrides?: Partial<Pick<PluginCommandContext, "gatewayClientScopes">>,
|
||||
): PluginCommandContext {
|
||||
return {
|
||||
channel: "webchat",
|
||||
isAuthorizedSender: true,
|
||||
commandBody: args ? `/dreaming ${args}` : "/dreaming",
|
||||
args,
|
||||
config: {},
|
||||
gatewayClientScopes: overrides?.gatewayClientScopes,
|
||||
requestConversationBinding: async () => ({ status: "error", message: "unsupported" }),
|
||||
detachConversationBinding: async () => ({ removed: false }),
|
||||
getCurrentConversationBinding: async () => null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("memory-core /dreaming command", () => {
|
||||
it("registers with an enable/disable description", () => {
|
||||
const { command } = createHarness();
|
||||
expect(command.name).toBe("dreaming");
|
||||
expect(command.acceptsArgs).toBe(true);
|
||||
expect(command.description).toContain("Enable or disable");
|
||||
});
|
||||
|
||||
it("shows phase explanations when invoked without args", async () => {
|
||||
const { command } = createHarness();
|
||||
const result = await command.handler(createCommandContext());
|
||||
|
||||
expect(result.text).toContain("Usage: /dreaming status");
|
||||
expect(result.text).toContain("Dreaming status:");
|
||||
expect(result.text).toContain("- implementation detail: each sweep runs light -> REM -> deep.");
|
||||
expect(result.text).toContain(
|
||||
"- deep is the only stage that writes durable entries to MEMORY.md.",
|
||||
);
|
||||
});
|
||||
|
||||
it("persists global enablement under plugins.entries.memory-core.config.dreaming.enabled", async () => {
|
||||
const { command, runtime, getRuntimeConfig } = createHarness({
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-core": {
|
||||
config: {
|
||||
dreaming: {
|
||||
phases: {
|
||||
deep: {
|
||||
minScore: 0.9,
|
||||
},
|
||||
},
|
||||
frequency: "0 */6 * * *",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await command.handler(createCommandContext("off"));
|
||||
|
||||
expect(runtime.config.writeConfigFile).toHaveBeenCalledTimes(1);
|
||||
expect(resolveStoredDreaming(getRuntimeConfig())).toMatchObject({
|
||||
enabled: false,
|
||||
frequency: "0 */6 * * *",
|
||||
});
|
||||
expect(result.text).toContain("Dreaming disabled.");
|
||||
});
|
||||
|
||||
it("blocks unscoped gateway callers from persisting dreaming config", async () => {
|
||||
const { command, runtime } = createHarness();
|
||||
|
||||
const result = await command.handler(
|
||||
createCommandContext("off", {
|
||||
gatewayClientScopes: [],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.text).toContain("requires operator.admin");
|
||||
expect(runtime.config.writeConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("blocks write-scoped gateway callers from persisting dreaming config", async () => {
|
||||
const { command, runtime } = createHarness();
|
||||
|
||||
const result = await command.handler(
|
||||
createCommandContext("off", {
|
||||
gatewayClientScopes: ["operator.write"],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(result.text).toContain("requires operator.admin");
|
||||
expect(runtime.config.writeConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows admin-scoped gateway callers to persist dreaming config", async () => {
|
||||
const { command, runtime, getRuntimeConfig } = createHarness();
|
||||
|
||||
const result = await command.handler(
|
||||
createCommandContext("on", {
|
||||
gatewayClientScopes: ["operator.admin"],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(runtime.config.writeConfigFile).toHaveBeenCalledTimes(1);
|
||||
expect(resolveStoredDreaming(getRuntimeConfig())).toMatchObject({
|
||||
enabled: true,
|
||||
});
|
||||
expect(result.text).toContain("Dreaming enabled.");
|
||||
});
|
||||
|
||||
it("returns status without mutating config", async () => {
|
||||
const { command, runtime } = createHarness({
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-core": {
|
||||
config: {
|
||||
dreaming: {
|
||||
frequency: "15 */8 * * *",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
userTimezone: "America/Los_Angeles",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await command.handler(createCommandContext("status"));
|
||||
|
||||
expect(result.text).toContain("Dreaming status:");
|
||||
expect(result.text).toContain("- enabled: off (America/Los_Angeles)");
|
||||
expect(result.text).toContain("- sweep cadence: 15 */8 * * *");
|
||||
expect(result.text).toContain("- promotion policy: score>=0.8, recalls>=3, uniqueQueries>=3");
|
||||
expect(runtime.config.writeConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shows usage for invalid args and does not mutate config", async () => {
|
||||
const { command, runtime } = createHarness();
|
||||
const result = await command.handler(createCommandContext("unknown-mode"));
|
||||
|
||||
expect(result.text).toContain("Usage: /dreaming status");
|
||||
expect(runtime.config.writeConfigFile).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
127
openclaw/extensions/memory-core/src/dreaming-command.ts
Normal file
127
openclaw/extensions/memory-core/src/dreaming-command.ts
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
import type { OpenClawConfig, OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core";
|
||||
import { resolveMemoryDreamingConfig } from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { asRecord } from "./dreaming-shared.js";
|
||||
import { resolveShortTermPromotionDreamingConfig } from "./dreaming.js";
|
||||
|
||||
function resolveMemoryCorePluginConfig(cfg: OpenClawConfig): Record<string, unknown> {
|
||||
const entry = asRecord(cfg.plugins?.entries?.["memory-core"]);
|
||||
return asRecord(entry?.config) ?? {};
|
||||
}
|
||||
|
||||
function updateDreamingEnabledInConfig(cfg: OpenClawConfig, enabled: boolean): OpenClawConfig {
|
||||
const entries = { ...cfg.plugins?.entries };
|
||||
const existingEntry = asRecord(entries["memory-core"]) ?? {};
|
||||
const existingConfig = asRecord(existingEntry.config) ?? {};
|
||||
const existingSleep = asRecord(existingConfig.dreaming) ?? {};
|
||||
entries["memory-core"] = {
|
||||
...existingEntry,
|
||||
config: {
|
||||
...existingConfig,
|
||||
dreaming: {
|
||||
...existingSleep,
|
||||
enabled,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...cfg,
|
||||
plugins: {
|
||||
...cfg.plugins,
|
||||
entries,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatEnabled(value: boolean): string {
|
||||
return value ? "on" : "off";
|
||||
}
|
||||
|
||||
function formatPhaseGuide(): string {
|
||||
return [
|
||||
"- implementation detail: each sweep runs light -> REM -> deep.",
|
||||
"- deep is the only stage that writes durable entries to MEMORY.md.",
|
||||
"- DREAMS.md is for human-readable dreaming summaries and diary entries.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function formatStatus(cfg: OpenClawConfig): string {
|
||||
const pluginConfig = resolveMemoryCorePluginConfig(cfg);
|
||||
const dreaming = resolveMemoryDreamingConfig({
|
||||
pluginConfig,
|
||||
cfg,
|
||||
});
|
||||
const deep = resolveShortTermPromotionDreamingConfig({ pluginConfig, cfg });
|
||||
const timezone = dreaming.timezone ? ` (${dreaming.timezone})` : "";
|
||||
|
||||
return [
|
||||
"Dreaming status:",
|
||||
`- enabled: ${formatEnabled(dreaming.enabled)}${timezone}`,
|
||||
`- sweep cadence: ${dreaming.frequency}`,
|
||||
`- promotion policy: score>=${deep.minScore}, recalls>=${deep.minRecallCount}, uniqueQueries>=${deep.minUniqueQueries}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function formatUsage(includeStatus: string): string {
|
||||
return [
|
||||
"Usage: /dreaming status",
|
||||
"Usage: /dreaming on|off",
|
||||
"",
|
||||
includeStatus,
|
||||
"",
|
||||
"Phases:",
|
||||
formatPhaseGuide(),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function requiresAdminToMutateDreaming(gatewayClientScopes?: readonly string[]): boolean {
|
||||
return Array.isArray(gatewayClientScopes) && !gatewayClientScopes.includes("operator.admin");
|
||||
}
|
||||
|
||||
export function registerDreamingCommand(api: OpenClawPluginApi): void {
|
||||
api.registerCommand({
|
||||
name: "dreaming",
|
||||
description: "Enable or disable memory dreaming.",
|
||||
acceptsArgs: true,
|
||||
handler: async (ctx) => {
|
||||
const args = ctx.args?.trim() ?? "";
|
||||
const [firstToken = ""] = args
|
||||
.split(/\s+/)
|
||||
.filter(Boolean)
|
||||
.map((token) => normalizeLowercaseStringOrEmpty(token));
|
||||
const currentConfig = api.runtime.config.loadConfig();
|
||||
|
||||
if (
|
||||
!firstToken ||
|
||||
firstToken === "help" ||
|
||||
firstToken === "options" ||
|
||||
firstToken === "phases"
|
||||
) {
|
||||
return { text: formatUsage(formatStatus(currentConfig)) };
|
||||
}
|
||||
|
||||
if (firstToken === "status") {
|
||||
return { text: formatStatus(currentConfig) };
|
||||
}
|
||||
|
||||
if (firstToken === "on" || firstToken === "off") {
|
||||
if (requiresAdminToMutateDreaming(ctx.gatewayClientScopes)) {
|
||||
return { text: "⚠️ /dreaming on|off requires operator.admin for gateway clients." };
|
||||
}
|
||||
const enabled = firstToken === "on";
|
||||
const nextConfig = updateDreamingEnabledInConfig(currentConfig, enabled);
|
||||
await api.runtime.config.writeConfigFile(nextConfig);
|
||||
return {
|
||||
text: [
|
||||
`Dreaming ${enabled ? "enabled" : "disabled"}.`,
|
||||
"",
|
||||
formatStatus(nextConfig),
|
||||
].join("\n"),
|
||||
};
|
||||
}
|
||||
|
||||
return { text: formatUsage(formatStatus(currentConfig)) };
|
||||
},
|
||||
});
|
||||
}
|
||||
113
openclaw/extensions/memory-core/src/dreaming-markdown.test.ts
Normal file
113
openclaw/extensions/memory-core/src/dreaming-markdown.test.ts
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { writeDailyDreamingPhaseBlock, writeDeepDreamingReport } from "./dreaming-markdown.js";
|
||||
import { createMemoryCoreTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempWorkspace } = createMemoryCoreTestHarness();
|
||||
|
||||
describe("dreaming markdown storage", () => {
|
||||
const nowMs = Date.parse("2026-04-05T10:00:00Z");
|
||||
const timezone = "UTC";
|
||||
|
||||
it("writes inline light dreaming output into the daily memory file", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
|
||||
const result = await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: remember the API key is fake"],
|
||||
nowMs,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.inlinePath).toBe(path.join(workspaceDir, "memory", "2026-04-05.md"));
|
||||
const content = await fs.readFile(result.inlinePath!, "utf-8");
|
||||
expect(content).toContain("## Light Sleep");
|
||||
expect(content).toContain("- Candidate: remember the API key is fake");
|
||||
});
|
||||
|
||||
it("keeps multiple inline phases in the shared daily memory file", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- Candidate: first block"],
|
||||
nowMs,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
bodyLines: ["- Theme: `focus` kept surfacing."],
|
||||
nowMs,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
const dreamsPath = path.join(workspaceDir, "memory", "2026-04-05.md");
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content).toContain("## Light Sleep");
|
||||
expect(content).toContain("## REM Sleep");
|
||||
expect(content).toContain("- Candidate: first block");
|
||||
expect(content).toContain("- Theme: `focus` kept surfacing.");
|
||||
});
|
||||
|
||||
it("keeps daily phase output separate from lowercase dreams.md diaries", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
const lowercasePath = path.join(workspaceDir, "dreams.md");
|
||||
await fs.writeFile(lowercasePath, "# Scratch\n\n", "utf-8");
|
||||
|
||||
const result = await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "rem",
|
||||
bodyLines: ["- Theme: `glacier` kept surfacing."],
|
||||
nowMs,
|
||||
timezone,
|
||||
storage: {
|
||||
mode: "inline",
|
||||
separateReports: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.inlinePath).toBe(path.join(workspaceDir, "memory", "2026-04-05.md"));
|
||||
const content = await fs.readFile(result.inlinePath!, "utf-8");
|
||||
expect(content).toContain("## REM Sleep");
|
||||
expect(content).toContain("- Theme: `glacier` kept surfacing.");
|
||||
await expect(fs.readFile(lowercasePath, "utf-8")).resolves.toBe("# Scratch\n\n");
|
||||
});
|
||||
|
||||
it("still writes deep reports to the per-phase report directory", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-markdown-");
|
||||
|
||||
const reportPath = await writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: ["- Promoted: durable preference"],
|
||||
storage: {
|
||||
mode: "separate",
|
||||
separateReports: false,
|
||||
},
|
||||
nowMs: Date.parse("2026-04-05T10:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
|
||||
expect(reportPath).toBe(path.join(workspaceDir, "memory", "dreaming", "deep", "2026-04-05.md"));
|
||||
const content = await fs.readFile(reportPath!, "utf-8");
|
||||
expect(content).toContain("# Deep Sleep");
|
||||
expect(content).toContain("- Promoted: durable preference");
|
||||
|
||||
await expect(fs.access(path.join(workspaceDir, "DREAMS.md"))).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
148
openclaw/extensions/memory-core/src/dreaming-markdown.ts
Normal file
148
openclaw/extensions/memory-core/src/dreaming-markdown.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
formatMemoryDreamingDay,
|
||||
type MemoryDreamingPhaseName,
|
||||
type MemoryDreamingStorageConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events";
|
||||
import {
|
||||
replaceManagedMarkdownBlock,
|
||||
withTrailingNewline,
|
||||
} from "openclaw/plugin-sdk/memory-host-markdown";
|
||||
|
||||
const DAILY_PHASE_HEADINGS: Record<Exclude<MemoryDreamingPhaseName, "deep">, string> = {
|
||||
light: "## Light Sleep",
|
||||
rem: "## REM Sleep",
|
||||
};
|
||||
|
||||
const DAILY_PHASE_LABELS: Record<Exclude<MemoryDreamingPhaseName, "deep">, string> = {
|
||||
light: "light",
|
||||
rem: "rem",
|
||||
};
|
||||
|
||||
function resolvePhaseMarkers(phase: Exclude<MemoryDreamingPhaseName, "deep">): {
|
||||
start: string;
|
||||
end: string;
|
||||
} {
|
||||
const label = DAILY_PHASE_LABELS[phase];
|
||||
return {
|
||||
start: `<!-- openclaw:dreaming:${label}:start -->`,
|
||||
end: `<!-- openclaw:dreaming:${label}:end -->`,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDailyMemoryPath(workspaceDir: string, epochMs: number, timezone?: string): string {
|
||||
const isoDay = formatMemoryDreamingDay(epochMs, timezone);
|
||||
return path.join(workspaceDir, "memory", `${isoDay}.md`);
|
||||
}
|
||||
|
||||
function resolveSeparateReportPath(
|
||||
workspaceDir: string,
|
||||
phase: MemoryDreamingPhaseName,
|
||||
epochMs: number,
|
||||
timezone?: string,
|
||||
): string {
|
||||
const isoDay = formatMemoryDreamingDay(epochMs, timezone);
|
||||
return path.join(workspaceDir, "memory", "dreaming", phase, `${isoDay}.md`);
|
||||
}
|
||||
|
||||
function shouldWriteInline(storage: MemoryDreamingStorageConfig): boolean {
|
||||
return storage.mode === "inline" || storage.mode === "both";
|
||||
}
|
||||
|
||||
function shouldWriteSeparate(storage: MemoryDreamingStorageConfig): boolean {
|
||||
return storage.mode === "separate" || storage.mode === "both" || storage.separateReports;
|
||||
}
|
||||
|
||||
export async function writeDailyDreamingPhaseBlock(params: {
|
||||
workspaceDir: string;
|
||||
phase: Exclude<MemoryDreamingPhaseName, "deep">;
|
||||
bodyLines: string[];
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
storage: MemoryDreamingStorageConfig;
|
||||
}): Promise<{ inlinePath?: string; reportPath?: string }> {
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const body = params.bodyLines.length > 0 ? params.bodyLines.join("\n") : "- No notable updates.";
|
||||
let inlinePath: string | undefined;
|
||||
let reportPath: string | undefined;
|
||||
|
||||
if (shouldWriteInline(params.storage)) {
|
||||
inlinePath = resolveDailyMemoryPath(params.workspaceDir, nowMs, params.timezone);
|
||||
await fs.mkdir(path.dirname(inlinePath), { recursive: true });
|
||||
const original = await fs.readFile(inlinePath, "utf-8").catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
const markers = resolvePhaseMarkers(params.phase);
|
||||
const updated = replaceManagedMarkdownBlock({
|
||||
original,
|
||||
heading: DAILY_PHASE_HEADINGS[params.phase],
|
||||
startMarker: markers.start,
|
||||
endMarker: markers.end,
|
||||
body,
|
||||
});
|
||||
await fs.writeFile(inlinePath, withTrailingNewline(updated), "utf-8");
|
||||
}
|
||||
|
||||
if (shouldWriteSeparate(params.storage)) {
|
||||
reportPath = resolveSeparateReportPath(
|
||||
params.workspaceDir,
|
||||
params.phase,
|
||||
nowMs,
|
||||
params.timezone,
|
||||
);
|
||||
await fs.mkdir(path.dirname(reportPath), { recursive: true });
|
||||
const report = [
|
||||
`# ${params.phase === "light" ? "Light Sleep" : "REM Sleep"}`,
|
||||
"",
|
||||
body,
|
||||
"",
|
||||
].join("\n");
|
||||
await fs.writeFile(reportPath, report, "utf-8");
|
||||
}
|
||||
|
||||
await appendMemoryHostEvent(params.workspaceDir, {
|
||||
type: "memory.dream.completed",
|
||||
timestamp: new Date(nowMs).toISOString(),
|
||||
phase: params.phase,
|
||||
...(inlinePath ? { inlinePath } : {}),
|
||||
...(reportPath ? { reportPath } : {}),
|
||||
lineCount: params.bodyLines.length,
|
||||
storageMode: params.storage.mode,
|
||||
});
|
||||
|
||||
return {
|
||||
...(inlinePath ? { inlinePath } : {}),
|
||||
...(reportPath ? { reportPath } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeDeepDreamingReport(params: {
|
||||
workspaceDir: string;
|
||||
bodyLines: string[];
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
storage: MemoryDreamingStorageConfig;
|
||||
}): Promise<string | undefined> {
|
||||
if (!shouldWriteSeparate(params.storage)) {
|
||||
return undefined;
|
||||
}
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
const reportPath = resolveSeparateReportPath(params.workspaceDir, "deep", nowMs, params.timezone);
|
||||
await fs.mkdir(path.dirname(reportPath), { recursive: true });
|
||||
const body = params.bodyLines.length > 0 ? params.bodyLines.join("\n") : "- No durable changes.";
|
||||
await fs.writeFile(reportPath, `# Deep Sleep\n\n${body}\n`, "utf-8");
|
||||
await appendMemoryHostEvent(params.workspaceDir, {
|
||||
type: "memory.dream.completed",
|
||||
timestamp: new Date(nowMs).toISOString(),
|
||||
phase: "deep",
|
||||
reportPath,
|
||||
lineCount: params.bodyLines.length,
|
||||
storageMode: params.storage.mode,
|
||||
});
|
||||
return reportPath;
|
||||
}
|
||||
886
openclaw/extensions/memory-core/src/dreaming-narrative.test.ts
Normal file
886
openclaw/extensions/memory-core/src/dreaming-narrative.test.ts
Normal file
|
|
@ -0,0 +1,886 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import * as configRuntimeModule from "openclaw/plugin-sdk/config-runtime";
|
||||
import {
|
||||
RequestScopedSubagentRuntimeError,
|
||||
SUBAGENT_RUNTIME_REQUEST_SCOPE_ERROR_CODE,
|
||||
} from "openclaw/plugin-sdk/error-runtime";
|
||||
import * as memoryCoreHostRuntimeCoreModule from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveGlobalMap } from "../../../src/shared/global-singleton.js";
|
||||
import {
|
||||
appendNarrativeEntry,
|
||||
buildBackfillDiaryEntry,
|
||||
buildDiaryEntry,
|
||||
buildNarrativePrompt,
|
||||
dedupeDreamDiaryEntries,
|
||||
extractNarrativeText,
|
||||
formatNarrativeDate,
|
||||
formatBackfillDiaryDate,
|
||||
generateAndAppendDreamNarrative,
|
||||
removeBackfillDiaryEntries,
|
||||
type NarrativePhaseData,
|
||||
writeBackfillDiaryEntries,
|
||||
} from "./dreaming-narrative.js";
|
||||
import { createMemoryCoreTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempWorkspace } = createMemoryCoreTestHarness();
|
||||
const DREAMS_FILE_LOCKS_KEY = Symbol.for("openclaw.memoryCore.dreamingNarrative.fileLocks");
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
resolveGlobalMap<string, unknown>(DREAMS_FILE_LOCKS_KEY).clear();
|
||||
});
|
||||
|
||||
describe("buildNarrativePrompt", () => {
|
||||
it("builds a prompt from snippets only", () => {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "light",
|
||||
snippets: ["user prefers dark mode", "API key rotation scheduled"],
|
||||
};
|
||||
const prompt = buildNarrativePrompt(data);
|
||||
expect(prompt).toContain("user prefers dark mode");
|
||||
expect(prompt).toContain("API key rotation scheduled");
|
||||
expect(prompt).not.toContain("Recurring themes");
|
||||
});
|
||||
|
||||
it("includes themes when provided", () => {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "rem",
|
||||
snippets: ["config migration path"],
|
||||
themes: ["infrastructure", "deployment"],
|
||||
};
|
||||
const prompt = buildNarrativePrompt(data);
|
||||
expect(prompt).toContain("Recurring themes");
|
||||
expect(prompt).toContain("infrastructure");
|
||||
expect(prompt).toContain("deployment");
|
||||
});
|
||||
|
||||
it("includes promotions for deep phase", () => {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "deep",
|
||||
snippets: ["trading bot uses bracket orders"],
|
||||
promotions: ["always use stop-loss on options trades"],
|
||||
};
|
||||
const prompt = buildNarrativePrompt(data);
|
||||
expect(prompt).toContain("crystallized");
|
||||
expect(prompt).toContain("always use stop-loss on options trades");
|
||||
});
|
||||
|
||||
it("caps snippets at 12", () => {
|
||||
const snippets = Array.from({ length: 20 }, (_, i) => `snippet-${i}`);
|
||||
const prompt = buildNarrativePrompt({ phase: "light", snippets });
|
||||
expect(prompt).toContain("snippet-11");
|
||||
expect(prompt).not.toContain("snippet-12");
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractNarrativeText", () => {
|
||||
it("extracts string content from assistant message", () => {
|
||||
const messages = [
|
||||
{ role: "user", content: "hello" },
|
||||
{ role: "assistant", content: "The workspace hummed quietly." },
|
||||
];
|
||||
expect(extractNarrativeText(messages)).toBe("The workspace hummed quietly.");
|
||||
});
|
||||
|
||||
it("extracts from content array with text blocks", () => {
|
||||
const messages = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: "First paragraph." },
|
||||
{ type: "text", text: "Second paragraph." },
|
||||
],
|
||||
},
|
||||
];
|
||||
expect(extractNarrativeText(messages)).toBe("First paragraph.\nSecond paragraph.");
|
||||
});
|
||||
|
||||
it("returns null when no assistant message exists", () => {
|
||||
const messages = [{ role: "user", content: "hello" }];
|
||||
expect(extractNarrativeText(messages)).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for empty assistant content", () => {
|
||||
const messages = [{ role: "assistant", content: " " }];
|
||||
expect(extractNarrativeText(messages)).toBeNull();
|
||||
});
|
||||
|
||||
it("picks the last assistant message", () => {
|
||||
const messages = [
|
||||
{ role: "assistant", content: "First response." },
|
||||
{ role: "user", content: "more" },
|
||||
{ role: "assistant", content: "Final response." },
|
||||
];
|
||||
expect(extractNarrativeText(messages)).toBe("Final response.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatNarrativeDate", () => {
|
||||
it("formats a UTC date", () => {
|
||||
const date = formatNarrativeDate(Date.parse("2026-04-05T03:00:00Z"), "UTC");
|
||||
expect(date).toContain("April");
|
||||
expect(date).toContain("2026");
|
||||
expect(date).toContain("3:00");
|
||||
expect(date).toContain("UTC");
|
||||
});
|
||||
|
||||
it("applies an explicit timezone", () => {
|
||||
// 2026-04-11T21:46:55Z in America/Los_Angeles (PDT, UTC-7) → 2:46 PM
|
||||
const date = formatNarrativeDate(Date.parse("2026-04-11T21:46:55Z"), "America/Los_Angeles");
|
||||
expect(date).toContain("2:46");
|
||||
expect(date).toContain("PM");
|
||||
expect(date).toContain("PDT");
|
||||
});
|
||||
|
||||
it("uses host local timezone when timezone is undefined (#65027)", () => {
|
||||
// Force a non-UTC host timezone so this test is meaningful on UTC CI
|
||||
// runners where the old `?? "UTC"` fallback would silently pass.
|
||||
const originalTZ = process.env.TZ;
|
||||
try {
|
||||
process.env.TZ = "America/Los_Angeles"; // PDT = UTC-7
|
||||
const epochMs = Date.parse("2026-04-11T21:46:55Z");
|
||||
const result = formatNarrativeDate(epochMs);
|
||||
// 21:46 UTC → 14:46 PDT → "2:46 PM"
|
||||
expect(result).toContain("2:46");
|
||||
expect(result).toContain("PM");
|
||||
expect(result).toContain("PDT");
|
||||
} finally {
|
||||
if (originalTZ === undefined) {
|
||||
delete process.env.TZ;
|
||||
} else {
|
||||
process.env.TZ = originalTZ;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildDiaryEntry", () => {
|
||||
it("formats narrative with date and separators", () => {
|
||||
const entry = buildDiaryEntry("The code drifted gently.", "April 5, 2026, 3:00 AM");
|
||||
expect(entry).toContain("---");
|
||||
expect(entry).toContain("*April 5, 2026, 3:00 AM*");
|
||||
expect(entry).toContain("The code drifted gently.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("backfill diary entries", () => {
|
||||
it("formats a backfill date without time", () => {
|
||||
expect(formatBackfillDiaryDate("2026-01-01", "UTC")).toBe("January 1, 2026");
|
||||
});
|
||||
|
||||
it("preserves the iso day label in high-positive-offset timezones", () => {
|
||||
expect(formatBackfillDiaryDate("2026-01-01", "Pacific/Kiritimati")).toBe("January 1, 2026");
|
||||
});
|
||||
|
||||
it("builds a marked backfill diary entry", () => {
|
||||
const entry = buildBackfillDiaryEntry({
|
||||
isoDay: "2026-01-01",
|
||||
sourcePath: "memory/2026-01-01.md",
|
||||
bodyLines: ["What Happened", "1. A durable preference appeared."],
|
||||
timezone: "UTC",
|
||||
});
|
||||
expect(entry).toContain("*January 1, 2026*");
|
||||
expect(entry).toContain("openclaw:dreaming:backfill-entry");
|
||||
expect(entry).toContain("What Happened");
|
||||
});
|
||||
|
||||
it("writes and replaces backfill diary entries", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-backfill-");
|
||||
const first = await writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
timezone: "UTC",
|
||||
entries: [
|
||||
{
|
||||
isoDay: "2026-01-01",
|
||||
sourcePath: "memory/2026-01-01.md",
|
||||
bodyLines: ["What Happened", "1. First pass."],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(first.written).toBe(1);
|
||||
expect(first.replaced).toBe(0);
|
||||
|
||||
const second = await writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
timezone: "UTC",
|
||||
entries: [
|
||||
{
|
||||
isoDay: "2026-01-02",
|
||||
sourcePath: "memory/2026-01-02.md",
|
||||
bodyLines: ["Reflections", "1. Second pass."],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(second.written).toBe(1);
|
||||
expect(second.replaced).toBe(1);
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).not.toContain("First pass.");
|
||||
expect(content).toContain("Second pass.");
|
||||
expect(content.match(/openclaw:dreaming:backfill-entry/g)?.length).toBe(1);
|
||||
});
|
||||
|
||||
it("removes only backfill diary entries", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-backfill-");
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Keep this real dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
await writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
timezone: "UTC",
|
||||
entries: [
|
||||
{
|
||||
isoDay: "2026-01-01",
|
||||
sourcePath: "memory/2026-01-01.md",
|
||||
bodyLines: ["What Happened", "1. Remove this backfill."],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const removed = await removeBackfillDiaryEntries({ workspaceDir });
|
||||
expect(removed.removed).toBe(1);
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("Keep this real dream.");
|
||||
expect(content).not.toContain("Remove this backfill.");
|
||||
});
|
||||
|
||||
it("refuses to overwrite a symlinked DREAMS.md during backfill writes", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-backfill-");
|
||||
const targetPath = path.join(workspaceDir, "outside.txt");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(targetPath, "outside\n", "utf-8");
|
||||
await fs.symlink(targetPath, dreamsPath);
|
||||
|
||||
await expect(
|
||||
writeBackfillDiaryEntries({
|
||||
workspaceDir,
|
||||
timezone: "UTC",
|
||||
entries: [
|
||||
{
|
||||
isoDay: "2026-01-01",
|
||||
sourcePath: "memory/2026-01-01.md",
|
||||
bodyLines: ["What Happened", "1. First pass."],
|
||||
},
|
||||
],
|
||||
}),
|
||||
).rejects.toThrow("Refusing to write symlinked DREAMS.md");
|
||||
await expect(fs.readFile(targetPath, "utf-8")).resolves.toBe("outside\n");
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendNarrativeEntry", () => {
|
||||
it("creates DREAMS.md with diary header on fresh workspace", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const dreamsPath = await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Fragments of authentication logic kept surfacing.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
expect(dreamsPath).toBe(path.join(workspaceDir, "DREAMS.md"));
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content).toContain("# Dream Diary");
|
||||
expect(content).toContain("Fragments of authentication logic kept surfacing.");
|
||||
expect(content).toContain("<!-- openclaw:dreaming:diary:start -->");
|
||||
expect(content).toContain("<!-- openclaw:dreaming:diary:end -->");
|
||||
});
|
||||
|
||||
it("appends a second entry within the diary markers", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "First dream.",
|
||||
nowMs: Date.parse("2026-04-04T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Second dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("First dream.");
|
||||
expect(content).toContain("Second dream.");
|
||||
// Both entries should be between start and end markers.
|
||||
const start = content.indexOf("<!-- openclaw:dreaming:diary:start -->");
|
||||
const end = content.indexOf("<!-- openclaw:dreaming:diary:end -->");
|
||||
const firstIdx = content.indexOf("First dream.");
|
||||
const secondIdx = content.indexOf("Second dream.");
|
||||
expect(firstIdx).toBeGreaterThan(start);
|
||||
expect(secondIdx).toBeGreaterThan(firstIdx);
|
||||
expect(secondIdx).toBeLessThan(end);
|
||||
});
|
||||
|
||||
it("prepends diary before existing managed blocks", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(
|
||||
dreamsPath,
|
||||
"## Light Sleep\n<!-- openclaw:dreaming:light:start -->\n- Candidate: test\n<!-- openclaw:dreaming:light:end -->\n",
|
||||
"utf-8",
|
||||
);
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "The workspace was quiet tonight.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
const diaryIdx = content.indexOf("# Dream Diary");
|
||||
const lightIdx = content.indexOf("## Light Sleep");
|
||||
// Diary should come before the managed block.
|
||||
expect(diaryIdx).toBeLessThan(lightIdx);
|
||||
expect(content).toContain("The workspace was quiet tonight.");
|
||||
});
|
||||
|
||||
it("reuses existing dreams file when present", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(dreamsPath, "# Existing\n", "utf-8");
|
||||
const result = await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Appended dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
expect(result).toBe(dreamsPath);
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content).toContain("Appended dream.");
|
||||
// Original content should still be there, after the diary.
|
||||
expect(content).toContain("# Existing");
|
||||
});
|
||||
|
||||
it("keeps existing diary content intact when the atomic replace fails", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(dreamsPath, "# Existing\n", "utf-8");
|
||||
const renameError = Object.assign(new Error("replace failed"), { code: "ENOSPC" });
|
||||
const renameSpy = vi.spyOn(fs, "rename").mockRejectedValueOnce(renameError);
|
||||
|
||||
await expect(
|
||||
appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Appended dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
}),
|
||||
).rejects.toThrow("replace failed");
|
||||
|
||||
expect(renameSpy).toHaveBeenCalledOnce();
|
||||
await expect(fs.readFile(dreamsPath, "utf-8")).resolves.toBe("# Existing\n");
|
||||
});
|
||||
|
||||
it("preserves restrictive dreams file permissions across atomic replace", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(dreamsPath, "# Existing\n", { encoding: "utf-8", mode: 0o600 });
|
||||
await fs.chmod(dreamsPath, 0o600);
|
||||
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Appended dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
|
||||
const stat = await fs.stat(dreamsPath);
|
||||
expect(stat.mode & 0o777).toBe(0o600);
|
||||
});
|
||||
|
||||
it("dedupes only exact diary duplicates while keeping distinct timestamps", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-dedupe-");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(
|
||||
dreamsPath,
|
||||
[
|
||||
"# Dream Diary",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:diary:start -->",
|
||||
"---",
|
||||
"",
|
||||
"*April 11, 2026, 8:00 AM*",
|
||||
"",
|
||||
"The server room smelled like rain.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"*April 11, 2026, 8:00 AM*",
|
||||
"",
|
||||
"<!-- transient comment -->",
|
||||
"",
|
||||
"The server room smelled like rain.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"*April 11, 2026, 8:30 AM*",
|
||||
"",
|
||||
"The server room smelled like rain.",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:diary:end -->",
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const result = await dedupeDreamDiaryEntries({ workspaceDir });
|
||||
|
||||
expect(result.removed).toBe(1);
|
||||
expect(result.kept).toBe(2);
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content.match(/The server room smelled like rain\./g)?.length).toBe(2);
|
||||
expect(content).toContain("*April 11, 2026, 8:00 AM*");
|
||||
expect(content).toContain("*April 11, 2026, 8:30 AM*");
|
||||
});
|
||||
|
||||
it("serializes append and dedupe so concurrent rewrites keep the new entry", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-dedupe-");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(
|
||||
dreamsPath,
|
||||
[
|
||||
"# Dream Diary",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:diary:start -->",
|
||||
"---",
|
||||
"",
|
||||
"*April 11, 2026, 8:00 AM*",
|
||||
"",
|
||||
"The server room smelled like rain.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"*April 11, 2026, 8:00 AM*",
|
||||
"",
|
||||
"The server room smelled like rain.",
|
||||
"",
|
||||
"<!-- openclaw:dreaming:diary:end -->",
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
await Promise.all([
|
||||
dedupeDreamDiaryEntries({ workspaceDir }),
|
||||
appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "A fresh signal arrived after the cleanup started.",
|
||||
nowMs: Date.parse("2026-04-11T14:30:00Z"),
|
||||
timezone: "UTC",
|
||||
}),
|
||||
]);
|
||||
|
||||
const content = await fs.readFile(dreamsPath, "utf-8");
|
||||
expect(content.match(/The server room smelled like rain\./g)?.length).toBe(1);
|
||||
expect(content).toContain("A fresh signal arrived after the cleanup started.");
|
||||
});
|
||||
|
||||
it("keeps dedupe a no-op when no exact duplicates exist", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-dedupe-");
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Only one entry exists.",
|
||||
nowMs: Date.parse("2026-04-11T14:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
|
||||
const result = await dedupeDreamDiaryEntries({ workspaceDir });
|
||||
|
||||
expect(result.removed).toBe(0);
|
||||
expect(result.kept).toBe(1);
|
||||
await expect(fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8")).resolves.toContain(
|
||||
"Only one entry exists.",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not rewrite the diary file when dedupe finds nothing to remove", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-dedupe-");
|
||||
const dreamsPath = await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Only one entry exists.",
|
||||
nowMs: Date.parse("2026-04-11T14:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
const before = await fs.stat(dreamsPath);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
const result = await dedupeDreamDiaryEntries({ workspaceDir });
|
||||
const after = await fs.stat(dreamsPath);
|
||||
|
||||
expect(result.removed).toBe(0);
|
||||
expect(after.mtimeMs).toBe(before.mtimeMs);
|
||||
});
|
||||
|
||||
it("cleans up the per-file lock entry after diary updates finish", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-dedupe-");
|
||||
const dreamsLocks = resolveGlobalMap<string, unknown>(DREAMS_FILE_LOCKS_KEY);
|
||||
|
||||
expect(dreamsLocks.size).toBe(0);
|
||||
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Only one entry exists.",
|
||||
nowMs: Date.parse("2026-04-11T14:00:00Z"),
|
||||
timezone: "UTC",
|
||||
});
|
||||
|
||||
expect(dreamsLocks.size).toBe(0);
|
||||
});
|
||||
|
||||
it("surfaces temp cleanup failure after atomic replace error", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(dreamsPath, "# Existing\n", "utf-8");
|
||||
vi.spyOn(fs, "rename").mockRejectedValueOnce(
|
||||
Object.assign(new Error("replace failed"), { code: "ENOSPC" }),
|
||||
);
|
||||
vi.spyOn(fs, "rm").mockRejectedValueOnce(
|
||||
Object.assign(new Error("cleanup failed"), { code: "EACCES" }),
|
||||
);
|
||||
|
||||
await expect(
|
||||
appendNarrativeEntry({
|
||||
workspaceDir,
|
||||
narrative: "Appended dream.",
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
}),
|
||||
).rejects.toThrow("cleanup also failed");
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateAndAppendDreamNarrative", () => {
|
||||
function createMockSubagent(responseText: string) {
|
||||
return {
|
||||
run: vi.fn().mockResolvedValue({ runId: "run-123" }),
|
||||
waitForRun: vi.fn().mockResolvedValue({ status: "ok" }),
|
||||
getSessionMessages: vi.fn().mockResolvedValue({
|
||||
messages: [
|
||||
{ role: "user", content: "prompt" },
|
||||
{ role: "assistant", content: responseText },
|
||||
],
|
||||
}),
|
||||
deleteSession: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
function createMockLogger() {
|
||||
return {
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
it("generates narrative and writes diary entry", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("The repository whispered of forgotten endpoints.");
|
||||
const logger = createMockLogger();
|
||||
const nowMs = Date.parse("2026-04-05T03:00:00Z");
|
||||
const workspaceHash = createHash("sha1").update(workspaceDir).digest("hex").slice(0, 12);
|
||||
const expectedSessionKey = `dreaming-narrative-light-${workspaceHash}-${nowMs}`;
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: {
|
||||
phase: "light",
|
||||
snippets: ["API endpoints need authentication"],
|
||||
},
|
||||
nowMs,
|
||||
timezone: "UTC",
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.run).toHaveBeenCalledOnce();
|
||||
expect(subagent.run.mock.calls[0][0]).toMatchObject({
|
||||
idempotencyKey: expectedSessionKey,
|
||||
sessionKey: expectedSessionKey,
|
||||
deliver: false,
|
||||
});
|
||||
expect(subagent.waitForRun).toHaveBeenCalledOnce();
|
||||
expect(subagent.deleteSession).toHaveBeenCalledOnce();
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("The repository whispered of forgotten endpoints.");
|
||||
expect(logger.info).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips narrative when no snippets are available", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("Should not appear.");
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "light", snippets: [] },
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.run).not.toHaveBeenCalled();
|
||||
const exists = await fs
|
||||
.access(path.join(workspaceDir, "DREAMS.md"))
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
expect(exists).toBe(false);
|
||||
});
|
||||
|
||||
it("handles subagent timeout gracefully", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.waitForRun.mockResolvedValue({ status: "timeout" });
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "deep", snippets: ["some memory"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
// Should not throw, should warn.
|
||||
expect(logger.warn).toHaveBeenCalled();
|
||||
const exists = await fs
|
||||
.access(path.join(workspaceDir, "DREAMS.md"))
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
expect(exists).toBe(false);
|
||||
});
|
||||
|
||||
it("waits once more before cleanup after timeout and logs cleanup failures", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.waitForRun
|
||||
.mockResolvedValueOnce({ status: "timeout" })
|
||||
.mockResolvedValueOnce({ status: "ok" });
|
||||
subagent.deleteSession.mockRejectedValue(new Error("still active"));
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "rem", snippets: ["some memory"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.waitForRun).toHaveBeenCalledTimes(2);
|
||||
expect(subagent.waitForRun.mock.calls[1][0]).toMatchObject({ timeoutMs: 120_000 });
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining("narrative session cleanup failed for rem phase"),
|
||||
);
|
||||
});
|
||||
|
||||
it("handles subagent error gracefully", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.run.mockRejectedValue(
|
||||
new Error("connection failed", {
|
||||
cause: new RequestScopedSubagentRuntimeError(),
|
||||
}),
|
||||
);
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "rem", snippets: ["pattern surfaced"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
// Should not throw.
|
||||
expect(logger.warn).toHaveBeenCalled();
|
||||
await expect(fs.access(path.join(workspaceDir, "DREAMS.md"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to a local narrative when subagent runtime is request-scoped", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.run.mockRejectedValue(new RequestScopedSubagentRuntimeError());
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "light", snippets: ["API endpoints need authentication"] },
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
logger,
|
||||
});
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("API endpoints need authentication");
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("request-scoped"));
|
||||
expect(logger.warn).not.toHaveBeenCalledWith(expect.stringContaining(workspaceDir));
|
||||
expect(subagent.deleteSession).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("falls back when the request-scoped runtime error is detected by stable code", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
const crossBoundaryError = new Error("different wrapper text");
|
||||
crossBoundaryError.name = "RequestScopedSubagentRuntimeError";
|
||||
Object.assign(crossBoundaryError, {
|
||||
code: SUBAGENT_RUNTIME_REQUEST_SCOPE_ERROR_CODE,
|
||||
});
|
||||
subagent.run.mockRejectedValue(crossBoundaryError);
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "deep", snippets: [], promotions: ["A durable candidate surfaced."] },
|
||||
nowMs: Date.parse("2026-04-05T03:00:00Z"),
|
||||
timezone: "UTC",
|
||||
logger,
|
||||
});
|
||||
|
||||
const content = await fs.readFile(path.join(workspaceDir, "DREAMS.md"), "utf-8");
|
||||
expect(content).toContain("A durable candidate surfaced.");
|
||||
});
|
||||
|
||||
it("does not fall back for non-Error objects that only spoof the stable code", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.run.mockRejectedValue({
|
||||
code: SUBAGENT_RUNTIME_REQUEST_SCOPE_ERROR_CODE,
|
||||
name: "RequestScopedSubagentRuntimeError",
|
||||
message: "spoofed",
|
||||
});
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "deep", snippets: ["should not persist"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
await expect(fs.access(path.join(workspaceDir, "DREAMS.md"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
expect(logger.warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining("narrative generation failed"),
|
||||
);
|
||||
});
|
||||
|
||||
it("cleans up session even on failure", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("");
|
||||
subagent.getSessionMessages.mockRejectedValue(new Error("fetch failed"));
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "light", snippets: ["memory fragment"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
expect(subagent.deleteSession).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("scrubs stale dreaming entries and orphan transcripts after cleanup", async () => {
|
||||
const workspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const stateDir = await createTempWorkspace("openclaw-dreaming-state-");
|
||||
const sessionsDir = path.join(stateDir, "agents", "main", "sessions");
|
||||
await fs.mkdir(sessionsDir, { recursive: true });
|
||||
const storePath = path.join(sessionsDir, "sessions.json");
|
||||
const orphanPath = path.join(sessionsDir, "orphan.jsonl");
|
||||
const livePath = path.join(sessionsDir, "still-live.jsonl");
|
||||
await fs.writeFile(
|
||||
storePath,
|
||||
`${JSON.stringify({
|
||||
"agent:main:dreaming-narrative-light-1": {
|
||||
sessionId: "missing",
|
||||
},
|
||||
"agent:main:kept-session": {
|
||||
sessionId: "still-live",
|
||||
},
|
||||
"agent:main:telegram:group:dreaming-narrative-room": {
|
||||
sessionId: "still-missing-non-dreaming",
|
||||
},
|
||||
})}\n`,
|
||||
"utf-8",
|
||||
);
|
||||
await fs.writeFile(orphanPath, '{"runId":"dreaming-narrative-light-123"}\n', "utf-8");
|
||||
await fs.writeFile(livePath, '{"runId":"dreaming-narrative-light-keep"}\n', "utf-8");
|
||||
const oldDate = new Date(Date.now() - 600_000);
|
||||
await fs.utimes(orphanPath, oldDate, oldDate);
|
||||
await fs.utimes(livePath, oldDate, oldDate);
|
||||
|
||||
vi.spyOn(configRuntimeModule, "loadConfig").mockReturnValue({ session: {} } as never);
|
||||
vi.spyOn(configRuntimeModule, "resolveStorePath").mockImplementation(((
|
||||
_store: string | undefined,
|
||||
{ agentId }: { agentId: string },
|
||||
) => {
|
||||
expect(agentId).toBe("main");
|
||||
return storePath;
|
||||
}) as typeof configRuntimeModule.resolveStorePath);
|
||||
vi.spyOn(memoryCoreHostRuntimeCoreModule, "resolveStateDir").mockReturnValue(stateDir);
|
||||
|
||||
const subagent = createMockSubagent("The repository whispered of forgotten endpoints.");
|
||||
const logger = createMockLogger();
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir,
|
||||
data: { phase: "light", snippets: ["memory fragment"] },
|
||||
logger,
|
||||
});
|
||||
|
||||
const updatedStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
expect(updatedStore).not.toHaveProperty("agent:main:dreaming-narrative-light-1");
|
||||
expect(updatedStore).toHaveProperty("agent:main:kept-session");
|
||||
expect(updatedStore).toHaveProperty("agent:main:telegram:group:dreaming-narrative-room");
|
||||
const sessionFiles = await fs.readdir(sessionsDir);
|
||||
expect(sessionFiles.some((name) => name.startsWith("orphan.jsonl.deleted."))).toBe(true);
|
||||
expect(sessionFiles).toContain("still-live.jsonl");
|
||||
expect(logger.info).toHaveBeenCalledWith(expect.stringContaining("dreaming cleanup scrubbed"));
|
||||
});
|
||||
|
||||
it("isolates narrative sessions across workspaces even at the same timestamp", async () => {
|
||||
const firstWorkspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const secondWorkspaceDir = await createTempWorkspace("openclaw-dreaming-narrative-");
|
||||
const subagent = createMockSubagent("A quiet memory took shape.");
|
||||
const logger = createMockLogger();
|
||||
const nowMs = Date.parse("2026-04-05T03:00:00Z");
|
||||
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir: firstWorkspaceDir,
|
||||
data: { phase: "light", snippets: ["first workspace fragment"] },
|
||||
nowMs,
|
||||
logger,
|
||||
});
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent,
|
||||
workspaceDir: secondWorkspaceDir,
|
||||
data: { phase: "light", snippets: ["second workspace fragment"] },
|
||||
nowMs,
|
||||
logger,
|
||||
});
|
||||
|
||||
const firstSessionKey = subagent.run.mock.calls[0]?.[0]?.sessionKey;
|
||||
const secondSessionKey = subagent.run.mock.calls[1]?.[0]?.sessionKey;
|
||||
expect(firstSessionKey).toBeTypeOf("string");
|
||||
expect(secondSessionKey).toBeTypeOf("string");
|
||||
expect(firstSessionKey).not.toBe(secondSessionKey);
|
||||
expect(firstSessionKey).toContain("dreaming-narrative-light-");
|
||||
expect(secondSessionKey).toContain("dreaming-narrative-light-");
|
||||
expect(subagent.deleteSession.mock.calls[0]?.[0]?.sessionKey).toBe(firstSessionKey);
|
||||
expect(subagent.deleteSession.mock.calls[1]?.[0]?.sessionKey).toBe(secondSessionKey);
|
||||
});
|
||||
});
|
||||
946
openclaw/extensions/memory-core/src/dreaming-narrative.ts
Normal file
946
openclaw/extensions/memory-core/src/dreaming-narrative.ts
Normal file
|
|
@ -0,0 +1,946 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import type { Dirent } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
loadConfig,
|
||||
loadSessionStore,
|
||||
resolveStorePath,
|
||||
updateSessionStore,
|
||||
} from "openclaw/plugin-sdk/config-runtime";
|
||||
import {
|
||||
extractErrorCode,
|
||||
formatErrorMessage,
|
||||
RequestScopedSubagentRuntimeError,
|
||||
readErrorName,
|
||||
SUBAGENT_RUNTIME_REQUEST_SCOPE_ERROR_CODE,
|
||||
} from "openclaw/plugin-sdk/error-runtime";
|
||||
import { resolveGlobalMap } from "openclaw/plugin-sdk/global-singleton";
|
||||
import { createAsyncLock } from "openclaw/plugin-sdk/infra-runtime";
|
||||
import { resolveStateDir } from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────
|
||||
|
||||
type SubagentSurface = {
|
||||
run: (params: {
|
||||
idempotencyKey: string;
|
||||
sessionKey: string;
|
||||
message: string;
|
||||
extraSystemPrompt?: string;
|
||||
deliver?: boolean;
|
||||
}) => Promise<{ runId: string }>;
|
||||
waitForRun: (params: {
|
||||
runId: string;
|
||||
timeoutMs?: number;
|
||||
}) => Promise<{ status: string; error?: string }>;
|
||||
getSessionMessages: (params: {
|
||||
sessionKey: string;
|
||||
limit?: number;
|
||||
}) => Promise<{ messages: unknown[] }>;
|
||||
deleteSession: (params: { sessionKey: string }) => Promise<void>;
|
||||
};
|
||||
|
||||
export type NarrativePhaseData = {
|
||||
phase: "light" | "deep" | "rem";
|
||||
/** Short memory snippets the phase processed. */
|
||||
snippets: string[];
|
||||
/** Concept tags / themes that surfaced (REM and light). */
|
||||
themes?: string[];
|
||||
/** Snippets that were promoted to durable memory (deep). */
|
||||
promotions?: string[];
|
||||
};
|
||||
|
||||
type Logger = {
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
};
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────
|
||||
|
||||
const NARRATIVE_SYSTEM_PROMPT = [
|
||||
"You are keeping a dream diary. Write a single entry in first person.",
|
||||
"",
|
||||
"Voice & tone:",
|
||||
"- You are a curious, gentle, slightly whimsical mind reflecting on the day.",
|
||||
"- Write like a poet who happens to be a programmer — sensory, warm, occasionally funny.",
|
||||
"- Mix the technical and the tender: code and constellations, APIs and afternoon light.",
|
||||
"- Let the fragments surprise you into unexpected connections and small epiphanies.",
|
||||
"",
|
||||
"What you might include (vary each entry, never all at once):",
|
||||
"- A tiny poem or haiku woven naturally into the prose",
|
||||
"- A small sketch described in words — a doodle in the margin of the diary",
|
||||
"- A quiet rumination or philosophical aside",
|
||||
"- Sensory details: the hum of a server, the color of a sunset in hex, rain on a window",
|
||||
"- Gentle humor or playful wordplay",
|
||||
"- An observation that connects two distant memories in an unexpected way",
|
||||
"",
|
||||
"Rules:",
|
||||
"- Draw from the memory fragments provided — weave them into the entry.",
|
||||
'- Never say "I\'m dreaming", "in my dream", "as I dream", or any meta-commentary about dreaming.',
|
||||
'- Never mention "AI", "agent", "LLM", "model", "language model", or any technical self-reference.',
|
||||
"- Do NOT use markdown headers, bullet points, or any formatting — just flowing prose.",
|
||||
"- Keep it between 80-180 words. Quality over quantity.",
|
||||
"- Output ONLY the diary entry. No preamble, no sign-off, no commentary.",
|
||||
].join("\n");
|
||||
|
||||
const NARRATIVE_TIMEOUT_MS = 60_000;
|
||||
const NARRATIVE_DELETE_SETTLE_TIMEOUT_MS = 120_000;
|
||||
const DREAMING_SESSION_KEY_PREFIX = "dreaming-narrative-";
|
||||
const DREAMING_TRANSCRIPT_RUN_MARKER = '"runId":"dreaming-narrative-';
|
||||
const DREAMING_ORPHAN_MIN_AGE_MS = 300_000;
|
||||
const SAFE_SESSION_ID_RE = /^[a-z0-9][a-z0-9._-]{0,127}$/i;
|
||||
const DREAMS_FILENAMES = ["DREAMS.md", "dreams.md"] as const;
|
||||
const DIARY_START_MARKER = "<!-- openclaw:dreaming:diary:start -->";
|
||||
const DIARY_END_MARKER = "<!-- openclaw:dreaming:diary:end -->";
|
||||
const BACKFILL_ENTRY_MARKER = "openclaw:dreaming:backfill-entry";
|
||||
const DREAMS_FILE_LOCKS_KEY = Symbol.for("openclaw.memoryCore.dreamingNarrative.fileLocks");
|
||||
|
||||
type DreamsFileLockEntry = {
|
||||
withLock: ReturnType<typeof createAsyncLock>;
|
||||
refs: number;
|
||||
};
|
||||
|
||||
const dreamsFileLocks = resolveGlobalMap<string, DreamsFileLockEntry>(DREAMS_FILE_LOCKS_KEY);
|
||||
|
||||
function isRequestScopedSubagentRuntimeError(err: unknown): boolean {
|
||||
return (
|
||||
err instanceof RequestScopedSubagentRuntimeError ||
|
||||
(err instanceof Error &&
|
||||
err.name === "RequestScopedSubagentRuntimeError" &&
|
||||
extractErrorCode(err) === SUBAGENT_RUNTIME_REQUEST_SCOPE_ERROR_CODE)
|
||||
);
|
||||
}
|
||||
|
||||
function formatFallbackWriteFailure(err: unknown): string {
|
||||
const code = extractErrorCode(err);
|
||||
const name = readErrorName(err);
|
||||
if (code && name) {
|
||||
return `code=${code} name=${name}`;
|
||||
}
|
||||
if (code) {
|
||||
return `code=${code}`;
|
||||
}
|
||||
if (name) {
|
||||
return `name=${name}`;
|
||||
}
|
||||
return "unknown error";
|
||||
}
|
||||
|
||||
function buildRequestScopedFallbackNarrative(data: NarrativePhaseData): string {
|
||||
return (
|
||||
data.snippets.map((value) => value.trim()).find((value) => value.length > 0) ??
|
||||
(data.promotions ?? []).map((value) => value.trim()).find((value) => value.length > 0) ??
|
||||
"A memory trace surfaced, but details were unavailable in this run."
|
||||
);
|
||||
}
|
||||
|
||||
async function startNarrativeRunOrFallback(params: {
|
||||
subagent: SubagentSurface;
|
||||
sessionKey: string;
|
||||
message: string;
|
||||
data: NarrativePhaseData;
|
||||
workspaceDir: string;
|
||||
nowMs: number;
|
||||
timezone?: string;
|
||||
logger: Logger;
|
||||
}): Promise<string | null> {
|
||||
try {
|
||||
const run = await params.subagent.run({
|
||||
idempotencyKey: params.sessionKey,
|
||||
sessionKey: params.sessionKey,
|
||||
message: params.message,
|
||||
extraSystemPrompt: NARRATIVE_SYSTEM_PROMPT,
|
||||
deliver: false,
|
||||
});
|
||||
return run.runId;
|
||||
} catch (runErr) {
|
||||
if (!isRequestScopedSubagentRuntimeError(runErr)) {
|
||||
throw runErr;
|
||||
}
|
||||
try {
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir: params.workspaceDir,
|
||||
narrative: buildRequestScopedFallbackNarrative(params.data),
|
||||
nowMs: params.nowMs,
|
||||
timezone: params.timezone,
|
||||
});
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation used fallback for ${params.data.phase} phase because subagent runtime is request-scoped.`,
|
||||
);
|
||||
} catch (fallbackErr) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative fallback failed for ${params.data.phase} phase (${formatFallbackWriteFailure(fallbackErr)})`,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function buildNarrativeSessionKey(params: {
|
||||
workspaceDir: string;
|
||||
phase: NarrativePhaseData["phase"];
|
||||
nowMs: number;
|
||||
}): string {
|
||||
const workspaceHash = createHash("sha1").update(params.workspaceDir).digest("hex").slice(0, 12);
|
||||
return `dreaming-narrative-${params.phase}-${workspaceHash}-${params.nowMs}`;
|
||||
}
|
||||
|
||||
// ── Prompt building ────────────────────────────────────────────────────
|
||||
|
||||
export function buildNarrativePrompt(data: NarrativePhaseData): string {
|
||||
const lines: string[] = [];
|
||||
lines.push("Write a dream diary entry from these memory fragments:\n");
|
||||
|
||||
for (const snippet of data.snippets.slice(0, 12)) {
|
||||
lines.push(`- ${snippet}`);
|
||||
}
|
||||
|
||||
if (data.themes?.length) {
|
||||
lines.push("\nRecurring themes:");
|
||||
for (const theme of data.themes.slice(0, 6)) {
|
||||
lines.push(`- ${theme}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (data.promotions?.length) {
|
||||
lines.push("\nMemories that crystallized into something lasting:");
|
||||
for (const promo of data.promotions.slice(0, 5)) {
|
||||
lines.push(`- ${promo}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
// ── Message extraction ─────────────────────────────────────────────────
|
||||
|
||||
export function extractNarrativeText(messages: unknown[]): string | null {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const msg = messages[i];
|
||||
if (!msg || typeof msg !== "object" || Array.isArray(msg)) {
|
||||
continue;
|
||||
}
|
||||
const record = msg as Record<string, unknown>;
|
||||
if (record.role !== "assistant") {
|
||||
continue;
|
||||
}
|
||||
const content = record.content;
|
||||
if (typeof content === "string" && content.trim().length > 0) {
|
||||
return content.trim();
|
||||
}
|
||||
if (Array.isArray(content)) {
|
||||
const text = content
|
||||
.filter(
|
||||
(part: unknown) =>
|
||||
part &&
|
||||
typeof part === "object" &&
|
||||
!Array.isArray(part) &&
|
||||
(part as Record<string, unknown>).type === "text" &&
|
||||
typeof (part as Record<string, unknown>).text === "string",
|
||||
)
|
||||
.map((part) => (part as { text: string }).text)
|
||||
.join("\n")
|
||||
.trim();
|
||||
if (text.length > 0) {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Date formatting ────────────────────────────────────────────────────
|
||||
|
||||
export function formatNarrativeDate(epochMs: number, timezone?: string): string {
|
||||
const opts: Intl.DateTimeFormatOptions = {
|
||||
timeZone: timezone,
|
||||
year: "numeric",
|
||||
month: "long",
|
||||
day: "numeric",
|
||||
hour: "numeric",
|
||||
minute: "2-digit",
|
||||
hour12: true,
|
||||
// Always include the timezone abbreviation so the reader knows which
|
||||
// timezone the timestamp refers to. Without this, users who haven't
|
||||
// configured a timezone see bare times that look local but are actually
|
||||
// UTC, causing confusion (see #65027).
|
||||
timeZoneName: "short",
|
||||
};
|
||||
return new Intl.DateTimeFormat("en-US", opts).format(new Date(epochMs));
|
||||
}
|
||||
|
||||
// ── DREAMS.md file I/O ─────────────────────────────────────────────────
|
||||
|
||||
async function resolveDreamsPath(workspaceDir: string): Promise<string> {
|
||||
for (const name of DREAMS_FILENAMES) {
|
||||
const target = path.join(workspaceDir, name);
|
||||
try {
|
||||
await fs.access(target);
|
||||
return target;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException)?.code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
return path.join(workspaceDir, DREAMS_FILENAMES[0]);
|
||||
}
|
||||
|
||||
async function readDreamsFile(dreamsPath: string): Promise<string> {
|
||||
try {
|
||||
return await fs.readFile(dreamsPath, "utf-8");
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function ensureDiarySection(existing: string): string {
|
||||
if (existing.includes(DIARY_START_MARKER) && existing.includes(DIARY_END_MARKER)) {
|
||||
return existing;
|
||||
}
|
||||
const diarySection = `# Dream Diary\n\n${DIARY_START_MARKER}\n${DIARY_END_MARKER}\n`;
|
||||
if (existing.trim().length === 0) {
|
||||
return diarySection;
|
||||
}
|
||||
return diarySection + "\n" + existing;
|
||||
}
|
||||
|
||||
function replaceDiaryContent(existing: string, diaryContent: string): string {
|
||||
const ensured = ensureDiarySection(existing);
|
||||
const startIdx = ensured.indexOf(DIARY_START_MARKER);
|
||||
const endIdx = ensured.indexOf(DIARY_END_MARKER);
|
||||
if (startIdx < 0 || endIdx < 0 || endIdx < startIdx) {
|
||||
return ensured;
|
||||
}
|
||||
const before = ensured.slice(0, startIdx + DIARY_START_MARKER.length);
|
||||
const after = ensured.slice(endIdx);
|
||||
const normalized = diaryContent.trim().length > 0 ? `\n${diaryContent.trim()}\n` : "\n";
|
||||
return before + normalized + after;
|
||||
}
|
||||
|
||||
function splitDiaryBlocks(diaryContent: string): string[] {
|
||||
return diaryContent
|
||||
.split(/\n---\n/)
|
||||
.map((block) => block.trim())
|
||||
.filter((block) => block.length > 0);
|
||||
}
|
||||
|
||||
function normalizeDiaryBlockFingerprint(block: string): string {
|
||||
const lines = block
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0);
|
||||
let dateLine = "";
|
||||
const bodyLines: string[] = [];
|
||||
for (const line of lines) {
|
||||
if (!dateLine && line.startsWith("*") && line.endsWith("*") && line.length > 2) {
|
||||
dateLine = line.slice(1, -1).trim();
|
||||
continue;
|
||||
}
|
||||
if (line.startsWith("<!--") || line.startsWith("#")) {
|
||||
continue;
|
||||
}
|
||||
bodyLines.push(line);
|
||||
}
|
||||
const normalizedDate = dateLine.replace(/\s+/g, " ").trim();
|
||||
const normalizedBody = bodyLines
|
||||
.join("\n")
|
||||
.replace(/[ \t]+\n/g, "\n")
|
||||
.trim();
|
||||
return `${normalizedDate}\n${normalizedBody}`;
|
||||
}
|
||||
|
||||
function joinDiaryBlocks(blocks: string[]): string {
|
||||
if (blocks.length === 0) {
|
||||
return "";
|
||||
}
|
||||
return blocks.map((block) => `---\n\n${block.trim()}\n`).join("\n");
|
||||
}
|
||||
|
||||
function stripBackfillDiaryBlocks(existing: string): { updated: string; removed: number } {
|
||||
const ensured = ensureDiarySection(existing);
|
||||
const startIdx = ensured.indexOf(DIARY_START_MARKER);
|
||||
const endIdx = ensured.indexOf(DIARY_END_MARKER);
|
||||
if (startIdx < 0 || endIdx < 0 || endIdx < startIdx) {
|
||||
return { updated: ensured, removed: 0 };
|
||||
}
|
||||
const inner = ensured.slice(startIdx + DIARY_START_MARKER.length, endIdx);
|
||||
const kept: string[] = [];
|
||||
let removed = 0;
|
||||
for (const block of splitDiaryBlocks(inner)) {
|
||||
if (block.includes(BACKFILL_ENTRY_MARKER)) {
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
kept.push(block);
|
||||
}
|
||||
return {
|
||||
updated: replaceDiaryContent(ensured, joinDiaryBlocks(kept)),
|
||||
removed,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatBackfillDiaryDate(isoDay: string, _timezone?: string): string {
|
||||
const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(isoDay);
|
||||
if (!match) {
|
||||
return isoDay;
|
||||
}
|
||||
const [, year, month, day] = match;
|
||||
const opts: Intl.DateTimeFormatOptions = {
|
||||
// Preserve the source iso day exactly; backfill labels should not drift by timezone.
|
||||
timeZone: "UTC",
|
||||
year: "numeric",
|
||||
month: "long",
|
||||
day: "numeric",
|
||||
};
|
||||
const epochMs = Date.UTC(Number(year), Number(month) - 1, Number(day), 12);
|
||||
return new Intl.DateTimeFormat("en-US", opts).format(new Date(epochMs));
|
||||
}
|
||||
|
||||
async function assertSafeDreamsPath(dreamsPath: string): Promise<void> {
|
||||
const stat = await fs.lstat(dreamsPath).catch((err: NodeJS.ErrnoException) => {
|
||||
if (err.code === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (!stat) {
|
||||
return;
|
||||
}
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw new Error("Refusing to write symlinked DREAMS.md");
|
||||
}
|
||||
if (!stat.isFile()) {
|
||||
throw new Error("Refusing to write non-file DREAMS.md");
|
||||
}
|
||||
}
|
||||
|
||||
async function writeDreamsFileAtomic(dreamsPath: string, content: string): Promise<void> {
|
||||
await assertSafeDreamsPath(dreamsPath);
|
||||
const existing = await fs.stat(dreamsPath).catch((err: NodeJS.ErrnoException) => {
|
||||
if (err.code === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
const mode = existing?.mode ?? 0o600;
|
||||
const tempPath = `${dreamsPath}.${process.pid}.${Date.now()}.tmp`;
|
||||
await fs.writeFile(tempPath, content, { encoding: "utf-8", flag: "wx", mode });
|
||||
await fs.chmod(tempPath, mode).catch(() => undefined);
|
||||
try {
|
||||
await fs.rename(tempPath, dreamsPath);
|
||||
await fs.chmod(dreamsPath, mode).catch(() => undefined);
|
||||
} catch (err) {
|
||||
const cleanupError = await fs.rm(tempPath, { force: true }).catch((rmErr) => rmErr);
|
||||
if (cleanupError) {
|
||||
throw new Error(
|
||||
`Atomic DREAMS.md write failed (${formatErrorMessage(err)}); cleanup also failed (${formatErrorMessage(cleanupError)})`,
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function updateDreamsFile<T>(params: {
|
||||
workspaceDir: string;
|
||||
updater: (
|
||||
existing: string,
|
||||
dreamsPath: string,
|
||||
) =>
|
||||
| Promise<{ content: string; result: T; shouldWrite?: boolean }>
|
||||
| {
|
||||
content: string;
|
||||
result: T;
|
||||
shouldWrite?: boolean;
|
||||
};
|
||||
}): Promise<T> {
|
||||
const dreamsPath = await resolveDreamsPath(params.workspaceDir);
|
||||
await fs.mkdir(path.dirname(dreamsPath), { recursive: true });
|
||||
let lockEntry = dreamsFileLocks.get(dreamsPath);
|
||||
if (!lockEntry) {
|
||||
lockEntry = { withLock: createAsyncLock(), refs: 0 };
|
||||
dreamsFileLocks.set(dreamsPath, lockEntry);
|
||||
}
|
||||
lockEntry.refs += 1;
|
||||
try {
|
||||
return await lockEntry.withLock(async () => {
|
||||
const existing = await readDreamsFile(dreamsPath);
|
||||
const { content, result, shouldWrite = true } = await params.updater(existing, dreamsPath);
|
||||
if (shouldWrite) {
|
||||
await writeDreamsFileAtomic(dreamsPath, content.endsWith("\n") ? content : `${content}\n`);
|
||||
}
|
||||
return result;
|
||||
});
|
||||
} finally {
|
||||
lockEntry.refs -= 1;
|
||||
if (lockEntry.refs <= 0 && dreamsFileLocks.get(dreamsPath) === lockEntry) {
|
||||
dreamsFileLocks.delete(dreamsPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildBackfillDiaryEntry(params: {
|
||||
isoDay: string;
|
||||
bodyLines: string[];
|
||||
sourcePath?: string;
|
||||
timezone?: string;
|
||||
}): string {
|
||||
const dateStr = formatBackfillDiaryDate(params.isoDay, params.timezone);
|
||||
const marker = `<!-- ${BACKFILL_ENTRY_MARKER} day=${params.isoDay}${params.sourcePath ? ` source=${params.sourcePath}` : ""} -->`;
|
||||
const body = params.bodyLines
|
||||
.map((line) => line.trimEnd())
|
||||
.join("\n")
|
||||
.trim();
|
||||
return [`*${dateStr}*`, marker, body].filter((part) => part.length > 0).join("\n\n");
|
||||
}
|
||||
|
||||
export async function writeBackfillDiaryEntries(params: {
|
||||
workspaceDir: string;
|
||||
entries: Array<{
|
||||
isoDay: string;
|
||||
bodyLines: string[];
|
||||
sourcePath?: string;
|
||||
}>;
|
||||
timezone?: string;
|
||||
}): Promise<{ dreamsPath: string; written: number; replaced: number }> {
|
||||
return await updateDreamsFile({
|
||||
workspaceDir: params.workspaceDir,
|
||||
updater: (existing, dreamsPath) => {
|
||||
const stripped = stripBackfillDiaryBlocks(existing);
|
||||
const startIdx = stripped.updated.indexOf(DIARY_START_MARKER);
|
||||
const endIdx = stripped.updated.indexOf(DIARY_END_MARKER);
|
||||
const inner =
|
||||
startIdx >= 0 && endIdx > startIdx
|
||||
? stripped.updated.slice(startIdx + DIARY_START_MARKER.length, endIdx)
|
||||
: "";
|
||||
const preservedBlocks = splitDiaryBlocks(inner);
|
||||
const nextBlocks = [
|
||||
...preservedBlocks,
|
||||
...params.entries.map((entry) =>
|
||||
buildBackfillDiaryEntry({
|
||||
isoDay: entry.isoDay,
|
||||
bodyLines: entry.bodyLines,
|
||||
sourcePath: entry.sourcePath,
|
||||
timezone: params.timezone,
|
||||
}),
|
||||
),
|
||||
];
|
||||
return {
|
||||
content: replaceDiaryContent(stripped.updated, joinDiaryBlocks(nextBlocks)),
|
||||
result: {
|
||||
dreamsPath,
|
||||
written: params.entries.length,
|
||||
replaced: stripped.removed,
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function removeBackfillDiaryEntries(params: {
|
||||
workspaceDir: string;
|
||||
}): Promise<{ dreamsPath: string; removed: number }> {
|
||||
return await updateDreamsFile({
|
||||
workspaceDir: params.workspaceDir,
|
||||
updater: (existing, dreamsPath) => {
|
||||
const stripped = stripBackfillDiaryBlocks(existing);
|
||||
return {
|
||||
content: stripped.updated,
|
||||
result: {
|
||||
dreamsPath,
|
||||
removed: stripped.removed,
|
||||
},
|
||||
shouldWrite: stripped.removed > 0 || existing.length > 0,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function dedupeDreamDiaryEntries(params: {
|
||||
workspaceDir: string;
|
||||
}): Promise<{ dreamsPath: string; removed: number; kept: number }> {
|
||||
return await updateDreamsFile({
|
||||
workspaceDir: params.workspaceDir,
|
||||
updater: (existing, dreamsPath) => {
|
||||
const ensured = ensureDiarySection(existing);
|
||||
const startIdx = ensured.indexOf(DIARY_START_MARKER);
|
||||
const endIdx = ensured.indexOf(DIARY_END_MARKER);
|
||||
if (startIdx < 0 || endIdx < 0 || endIdx < startIdx) {
|
||||
return {
|
||||
content: ensured,
|
||||
result: { dreamsPath, removed: 0, kept: 0 },
|
||||
shouldWrite: false,
|
||||
};
|
||||
}
|
||||
const inner = ensured.slice(startIdx + DIARY_START_MARKER.length, endIdx);
|
||||
const blocks = splitDiaryBlocks(inner);
|
||||
const seen = new Set<string>();
|
||||
const keptBlocks: string[] = [];
|
||||
let removed = 0;
|
||||
for (const block of blocks) {
|
||||
const fingerprint = normalizeDiaryBlockFingerprint(block);
|
||||
if (seen.has(fingerprint)) {
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
seen.add(fingerprint);
|
||||
keptBlocks.push(block);
|
||||
}
|
||||
return {
|
||||
content: replaceDiaryContent(ensured, joinDiaryBlocks(keptBlocks)),
|
||||
result: {
|
||||
dreamsPath,
|
||||
removed,
|
||||
kept: keptBlocks.length,
|
||||
},
|
||||
shouldWrite: removed > 0,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function buildDiaryEntry(narrative: string, dateStr: string): string {
|
||||
return `\n---\n\n*${dateStr}*\n\n${narrative}\n`;
|
||||
}
|
||||
|
||||
export async function appendNarrativeEntry(params: {
|
||||
workspaceDir: string;
|
||||
narrative: string;
|
||||
nowMs: number;
|
||||
timezone?: string;
|
||||
}): Promise<string> {
|
||||
const dateStr = formatNarrativeDate(params.nowMs, params.timezone);
|
||||
const entry = buildDiaryEntry(params.narrative, dateStr);
|
||||
return await updateDreamsFile({
|
||||
workspaceDir: params.workspaceDir,
|
||||
updater: (existing, dreamsPath) => {
|
||||
let updated: string;
|
||||
if (existing.includes(DIARY_START_MARKER) && existing.includes(DIARY_END_MARKER)) {
|
||||
const endIdx = existing.lastIndexOf(DIARY_END_MARKER);
|
||||
updated = existing.slice(0, endIdx) + entry + "\n" + existing.slice(endIdx);
|
||||
} else if (existing.includes(DIARY_START_MARKER)) {
|
||||
const startIdx = existing.indexOf(DIARY_START_MARKER) + DIARY_START_MARKER.length;
|
||||
updated =
|
||||
existing.slice(0, startIdx) +
|
||||
entry +
|
||||
"\n" +
|
||||
DIARY_END_MARKER +
|
||||
"\n" +
|
||||
existing.slice(startIdx);
|
||||
} else {
|
||||
const diarySection = `# Dream Diary\n\n${DIARY_START_MARKER}${entry}\n${DIARY_END_MARKER}\n`;
|
||||
updated = existing.trim().length === 0 ? diarySection : `${diarySection}\n${existing}`;
|
||||
}
|
||||
return { content: updated, result: dreamsPath };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ── Orchestrator ───────────────────────────────────────────────────────
|
||||
|
||||
async function safePathExists(pathname: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.stat(pathname);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeComparablePath(pathname: string): string {
|
||||
return process.platform === "win32" ? pathname.toLowerCase() : pathname;
|
||||
}
|
||||
|
||||
async function normalizeSessionFileForComparison(params: {
|
||||
sessionsDir: string;
|
||||
sessionFile: string;
|
||||
}): Promise<string | null> {
|
||||
const trimmed = params.sessionFile.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
const resolved = path.isAbsolute(trimmed) ? trimmed : path.resolve(params.sessionsDir, trimmed);
|
||||
try {
|
||||
return normalizeComparablePath(await fs.realpath(resolved));
|
||||
} catch {
|
||||
return normalizeComparablePath(path.resolve(resolved));
|
||||
}
|
||||
}
|
||||
|
||||
function isDreamingSessionStoreKey(sessionKey: string): boolean {
|
||||
const firstSeparator = sessionKey.indexOf(":");
|
||||
if (firstSeparator < 0) {
|
||||
return sessionKey.startsWith(DREAMING_SESSION_KEY_PREFIX);
|
||||
}
|
||||
const secondSeparator = sessionKey.indexOf(":", firstSeparator + 1);
|
||||
const sessionSegment = secondSeparator < 0 ? sessionKey : sessionKey.slice(secondSeparator + 1);
|
||||
return sessionSegment.startsWith(DREAMING_SESSION_KEY_PREFIX);
|
||||
}
|
||||
|
||||
async function normalizeSessionEntryPathForComparison(params: {
|
||||
sessionsDir: string;
|
||||
entry: { sessionFile?: string; sessionId?: string } | undefined;
|
||||
}): Promise<string | null> {
|
||||
const sessionFile = typeof params.entry?.sessionFile === "string" ? params.entry.sessionFile : "";
|
||||
if (sessionFile) {
|
||||
return normalizeSessionFileForComparison({
|
||||
sessionsDir: params.sessionsDir,
|
||||
sessionFile,
|
||||
});
|
||||
}
|
||||
const sessionId =
|
||||
typeof params.entry?.sessionId === "string" ? params.entry.sessionId.trim() : "";
|
||||
if (!SAFE_SESSION_ID_RE.test(sessionId)) {
|
||||
return null;
|
||||
}
|
||||
return normalizeSessionFileForComparison({
|
||||
sessionsDir: params.sessionsDir,
|
||||
sessionFile: `${sessionId}.jsonl`,
|
||||
});
|
||||
}
|
||||
|
||||
async function scrubDreamingNarrativeArtifacts(logger: Logger): Promise<void> {
|
||||
const cfg = loadConfig();
|
||||
const agentsDir = path.join(resolveStateDir(), "agents");
|
||||
let agentEntries: Dirent[] = [];
|
||||
try {
|
||||
agentEntries = await fs.readdir(agentsDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
let prunedEntries = 0;
|
||||
let archivedOrphans = 0;
|
||||
|
||||
for (const agentEntry of agentEntries) {
|
||||
if (!agentEntry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const storePath = resolveStorePath(cfg.session?.store, { agentId: agentEntry.name });
|
||||
const sessionsDir = path.dirname(storePath);
|
||||
let store: Record<string, { sessionFile?: string; sessionId?: string } | undefined>;
|
||||
try {
|
||||
store = loadSessionStore(storePath) as Record<
|
||||
string,
|
||||
{ sessionFile?: string; sessionId?: string } | undefined
|
||||
>;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const referencedSessionFiles = new Set<string>();
|
||||
let needsStoreUpdate = false;
|
||||
for (const [key, entry] of Object.entries(store)) {
|
||||
const normalizedSessionFile = await normalizeSessionEntryPathForComparison({
|
||||
sessionsDir,
|
||||
entry,
|
||||
});
|
||||
if (normalizedSessionFile) {
|
||||
referencedSessionFiles.add(normalizedSessionFile);
|
||||
}
|
||||
if (!isDreamingSessionStoreKey(key)) {
|
||||
continue;
|
||||
}
|
||||
if (!normalizedSessionFile || !(await safePathExists(normalizedSessionFile))) {
|
||||
needsStoreUpdate = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (needsStoreUpdate) {
|
||||
referencedSessionFiles.clear();
|
||||
prunedEntries += await updateSessionStore(storePath, async (lockedStore) => {
|
||||
let prunedForAgent = 0;
|
||||
for (const [key, entry] of Object.entries(lockedStore)) {
|
||||
const normalizedSessionFile = await normalizeSessionEntryPathForComparison({
|
||||
sessionsDir,
|
||||
entry,
|
||||
});
|
||||
if (normalizedSessionFile) {
|
||||
referencedSessionFiles.add(normalizedSessionFile);
|
||||
}
|
||||
if (!isDreamingSessionStoreKey(key)) {
|
||||
continue;
|
||||
}
|
||||
if (!normalizedSessionFile || !(await safePathExists(normalizedSessionFile))) {
|
||||
delete lockedStore[key];
|
||||
prunedForAgent += 1;
|
||||
}
|
||||
}
|
||||
return prunedForAgent;
|
||||
});
|
||||
}
|
||||
|
||||
let sessionFiles: Dirent[] = [];
|
||||
try {
|
||||
sessionFiles = await fs.readdir(sessionsDir, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const fileEntry of sessionFiles) {
|
||||
if (!fileEntry.isFile() || !fileEntry.name.endsWith(".jsonl")) {
|
||||
continue;
|
||||
}
|
||||
const transcriptPath = path.join(sessionsDir, fileEntry.name);
|
||||
const normalizedTranscriptPath =
|
||||
(await normalizeSessionFileForComparison({
|
||||
sessionsDir,
|
||||
sessionFile: fileEntry.name,
|
||||
})) ?? normalizeComparablePath(transcriptPath);
|
||||
if (referencedSessionFiles.has(normalizedTranscriptPath)) {
|
||||
continue;
|
||||
}
|
||||
let stat;
|
||||
try {
|
||||
stat = await fs.stat(transcriptPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (Date.now() - stat.mtimeMs < DREAMING_ORPHAN_MIN_AGE_MS) {
|
||||
continue;
|
||||
}
|
||||
let content = "";
|
||||
try {
|
||||
content = await fs.readFile(transcriptPath, "utf-8");
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!content.includes(DREAMING_TRANSCRIPT_RUN_MARKER)) {
|
||||
continue;
|
||||
}
|
||||
const archivedPath = `${transcriptPath}.deleted.${Date.now()}`;
|
||||
try {
|
||||
await fs.rename(transcriptPath, archivedPath);
|
||||
archivedOrphans += 1;
|
||||
} catch {
|
||||
// best-effort scrubber
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (prunedEntries > 0 || archivedOrphans > 0) {
|
||||
logger.info(
|
||||
`memory-core: dreaming cleanup scrubbed ${prunedEntries} stale session entr${prunedEntries === 1 ? "y" : "ies"} and archived ${archivedOrphans} orphan transcript${archivedOrphans === 1 ? "" : "s"}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateAndAppendDreamNarrative(params: {
|
||||
subagent: SubagentSurface;
|
||||
workspaceDir: string;
|
||||
data: NarrativePhaseData;
|
||||
nowMs?: number;
|
||||
timezone?: string;
|
||||
logger: Logger;
|
||||
}): Promise<void> {
|
||||
const nowMs = Number.isFinite(params.nowMs) ? (params.nowMs as number) : Date.now();
|
||||
|
||||
if (params.data.snippets.length === 0 && !params.data.promotions?.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionKey = buildNarrativeSessionKey({
|
||||
workspaceDir: params.workspaceDir,
|
||||
phase: params.data.phase,
|
||||
nowMs,
|
||||
});
|
||||
const message = buildNarrativePrompt(params.data);
|
||||
let runId: string | null = null;
|
||||
let waitStatus: string | null = null;
|
||||
|
||||
try {
|
||||
runId = await startNarrativeRunOrFallback({
|
||||
subagent: params.subagent,
|
||||
sessionKey,
|
||||
message,
|
||||
data: params.data,
|
||||
workspaceDir: params.workspaceDir,
|
||||
nowMs,
|
||||
timezone: params.timezone,
|
||||
logger: params.logger,
|
||||
});
|
||||
if (!runId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await params.subagent.waitForRun({
|
||||
runId,
|
||||
timeoutMs: NARRATIVE_TIMEOUT_MS,
|
||||
});
|
||||
waitStatus = result.status;
|
||||
|
||||
if (result.status !== "ok") {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation ended with status=${result.status} for ${params.data.phase} phase.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const { messages } = await params.subagent.getSessionMessages({
|
||||
sessionKey,
|
||||
limit: 5,
|
||||
});
|
||||
|
||||
const narrative = extractNarrativeText(messages);
|
||||
if (!narrative) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation produced no text for ${params.data.phase} phase.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await appendNarrativeEntry({
|
||||
workspaceDir: params.workspaceDir,
|
||||
narrative,
|
||||
nowMs,
|
||||
timezone: params.timezone,
|
||||
});
|
||||
|
||||
params.logger.info(
|
||||
`memory-core: dream diary entry written for ${params.data.phase} phase [workspace=${params.workspaceDir}].`,
|
||||
);
|
||||
} catch (err) {
|
||||
// Narrative generation is best-effort — never fail the parent phase.
|
||||
params.logger.warn(
|
||||
`memory-core: narrative generation failed for ${params.data.phase} phase: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
} finally {
|
||||
if (runId && waitStatus === "timeout") {
|
||||
try {
|
||||
const settle = await params.subagent.waitForRun({
|
||||
runId,
|
||||
timeoutMs: NARRATIVE_DELETE_SETTLE_TIMEOUT_MS,
|
||||
});
|
||||
if (settle.status !== "ok" && settle.status !== "error") {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative cleanup wait ended with status=${settle.status} for ${params.data.phase} phase.`,
|
||||
);
|
||||
}
|
||||
} catch (cleanupWaitErr) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative cleanup wait failed for ${params.data.phase} phase: ${formatErrorMessage(cleanupWaitErr)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await params.subagent.deleteSession({ sessionKey });
|
||||
} catch (cleanupErr) {
|
||||
params.logger.warn(
|
||||
`memory-core: narrative session cleanup failed for ${params.data.phase} phase: ${formatErrorMessage(cleanupErr)}`,
|
||||
);
|
||||
}
|
||||
|
||||
await scrubDreamingNarrativeArtifacts(params.logger).catch((scrubErr: unknown) => {
|
||||
params.logger.warn(
|
||||
`memory-core: dreaming cleanup scrub failed for ${params.data.phase} phase: ${formatErrorMessage(scrubErr)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
1863
openclaw/extensions/memory-core/src/dreaming-phases.test.ts
Normal file
1863
openclaw/extensions/memory-core/src/dreaming-phases.test.ts
Normal file
File diff suppressed because it is too large
Load diff
1741
openclaw/extensions/memory-core/src/dreaming-phases.ts
Normal file
1741
openclaw/extensions/memory-core/src/dreaming-phases.ts
Normal file
File diff suppressed because it is too large
Load diff
128
openclaw/extensions/memory-core/src/dreaming-repair.test.ts
Normal file
128
openclaw/extensions/memory-core/src/dreaming-repair.test.ts
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { auditDreamingArtifacts, repairDreamingArtifacts } from "./dreaming-repair.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
async function createWorkspace(): Promise<string> {
|
||||
const workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "dreaming-repair-test-"));
|
||||
tempDirs.push(workspaceDir);
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", ".dreams"), { recursive: true });
|
||||
return workspaceDir;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
while (tempDirs.length > 0) {
|
||||
const dir = tempDirs.pop();
|
||||
if (dir) {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("dreaming artifact repair", () => {
|
||||
it("detects self-ingested dreaming corpus lines", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
await fs
|
||||
.writeFile(
|
||||
path.join(workspaceDir, "memory", ".dreams", "session-corpus", "2026-04-11.txt"),
|
||||
[
|
||||
"[main/dreaming-main.jsonl#L4] regular session text",
|
||||
"[main/dreaming-narrative-light.jsonl#L1] Write a dream diary entry from these memory fragments:",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
)
|
||||
.catch(async () => {
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", ".dreams", "session-corpus"), {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", ".dreams", "session-corpus", "2026-04-11.txt"),
|
||||
[
|
||||
"[main/dreaming-main.jsonl#L4] regular session text",
|
||||
"[main/dreaming-narrative-light.jsonl#L1] Write a dream diary entry from these memory fragments:",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
});
|
||||
|
||||
const audit = await auditDreamingArtifacts({ workspaceDir });
|
||||
|
||||
expect(audit.sessionCorpusFileCount).toBe(1);
|
||||
expect(audit.suspiciousSessionCorpusFileCount).toBe(1);
|
||||
expect(audit.suspiciousSessionCorpusLineCount).toBe(1);
|
||||
expect(audit.issues).toEqual([
|
||||
expect.objectContaining({
|
||||
code: "dreaming-session-corpus-self-ingested",
|
||||
fixable: true,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not flag ordinary transcript text that merely mentions dreaming-narrative", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", ".dreams", "session-corpus"), {
|
||||
recursive: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", ".dreams", "session-corpus", "2026-04-11.txt"),
|
||||
[
|
||||
"[main/chat.jsonl#L4] regular session text",
|
||||
"[main/chat.jsonl#L5] We should inspect the dreaming-narrative session behavior tomorrow.",
|
||||
].join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const audit = await auditDreamingArtifacts({ workspaceDir });
|
||||
|
||||
expect(audit.suspiciousSessionCorpusFileCount).toBe(0);
|
||||
expect(audit.suspiciousSessionCorpusLineCount).toBe(0);
|
||||
expect(audit.issues).toEqual([]);
|
||||
});
|
||||
|
||||
it("rejects relative workspace paths during audit and repair", async () => {
|
||||
await expect(auditDreamingArtifacts({ workspaceDir: "relative/workspace" })).rejects.toThrow(
|
||||
"workspaceDir must be an absolute path",
|
||||
);
|
||||
await expect(repairDreamingArtifacts({ workspaceDir: "relative/workspace" })).rejects.toThrow(
|
||||
"workspaceDir must be an absolute path",
|
||||
);
|
||||
});
|
||||
|
||||
it("archives derived dreaming artifacts without touching the diary by default", async () => {
|
||||
const workspaceDir = await createWorkspace();
|
||||
const sessionCorpusDir = path.join(workspaceDir, "memory", ".dreams", "session-corpus");
|
||||
await fs.mkdir(sessionCorpusDir, { recursive: true });
|
||||
await fs.writeFile(path.join(sessionCorpusDir, "2026-04-11.txt"), "corpus\n", "utf-8");
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", ".dreams", "session-ingestion.json"),
|
||||
JSON.stringify({ version: 3, files: {}, seenMessages: {} }, null, 2),
|
||||
"utf-8",
|
||||
);
|
||||
const dreamsPath = path.join(workspaceDir, "DREAMS.md");
|
||||
await fs.writeFile(dreamsPath, "# Dream Diary\n", "utf-8");
|
||||
|
||||
const repair = await repairDreamingArtifacts({
|
||||
workspaceDir,
|
||||
now: new Date("2026-04-11T21:30:00.000Z"),
|
||||
});
|
||||
|
||||
expect(repair.changed).toBe(true);
|
||||
expect(repair.archivedSessionCorpus).toBe(true);
|
||||
expect(repair.archivedSessionIngestion).toBe(true);
|
||||
expect(repair.archivedDreamsDiary).toBe(false);
|
||||
expect(repair.archiveDir).toBe(
|
||||
path.join(workspaceDir, ".openclaw-repair", "dreaming", "2026-04-11T21-30-00-000Z"),
|
||||
);
|
||||
await expect(fs.access(sessionCorpusDir)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(
|
||||
fs.access(path.join(workspaceDir, "memory", ".dreams", "session-ingestion.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(fs.readFile(dreamsPath, "utf-8")).resolves.toContain("# Dream Diary");
|
||||
const archivedEntries = await fs.readdir(repair.archiveDir!);
|
||||
expect(archivedEntries.some((entry) => entry.startsWith("session-corpus."))).toBe(true);
|
||||
expect(archivedEntries.some((entry) => entry.startsWith("session-ingestion.json."))).toBe(true);
|
||||
});
|
||||
});
|
||||
280
openclaw/extensions/memory-core/src/dreaming-repair.ts
Normal file
280
openclaw/extensions/memory-core/src/dreaming-repair.ts
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export type DreamingArtifactsAuditIssue = {
|
||||
severity: "warn" | "error";
|
||||
code:
|
||||
| "dreaming-session-corpus-unreadable"
|
||||
| "dreaming-session-corpus-self-ingested"
|
||||
| "dreaming-session-ingestion-unreadable"
|
||||
| "dreaming-diary-unreadable";
|
||||
message: string;
|
||||
fixable: boolean;
|
||||
};
|
||||
|
||||
export type DreamingArtifactsAuditSummary = {
|
||||
dreamsPath?: string;
|
||||
sessionCorpusDir: string;
|
||||
sessionCorpusFileCount: number;
|
||||
suspiciousSessionCorpusFileCount: number;
|
||||
suspiciousSessionCorpusLineCount: number;
|
||||
sessionIngestionPath: string;
|
||||
sessionIngestionExists: boolean;
|
||||
issues: DreamingArtifactsAuditIssue[];
|
||||
};
|
||||
|
||||
export type RepairDreamingArtifactsResult = {
|
||||
changed: boolean;
|
||||
archiveDir?: string;
|
||||
archivedDreamsDiary: boolean;
|
||||
archivedSessionCorpus: boolean;
|
||||
archivedSessionIngestion: boolean;
|
||||
archivedPaths: string[];
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
const DREAMS_FILENAMES = ["DREAMS.md", "dreams.md"] as const;
|
||||
const SESSION_CORPUS_RELATIVE_DIR = path.join("memory", ".dreams", "session-corpus");
|
||||
const SESSION_INGESTION_RELATIVE_PATH = path.join("memory", ".dreams", "session-ingestion.json");
|
||||
const REPAIR_ARCHIVE_RELATIVE_DIR = path.join(".openclaw-repair", "dreaming");
|
||||
const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-";
|
||||
const DREAMING_NARRATIVE_PROMPT_PREFIX = "Write a dream diary entry from these memory fragments";
|
||||
|
||||
function requireAbsoluteWorkspaceDir(rawWorkspaceDir: string): string {
|
||||
const trimmed = rawWorkspaceDir.trim();
|
||||
if (!trimmed) {
|
||||
throw new Error("workspaceDir is required");
|
||||
}
|
||||
if (!path.isAbsolute(trimmed)) {
|
||||
throw new Error("workspaceDir must be an absolute path");
|
||||
}
|
||||
return path.resolve(trimmed);
|
||||
}
|
||||
|
||||
async function resolveExistingDreamsPath(workspaceDir: string): Promise<string | undefined> {
|
||||
for (const fileName of DREAMS_FILENAMES) {
|
||||
const candidate = path.join(workspaceDir, fileName);
|
||||
try {
|
||||
await fs.access(candidate);
|
||||
return candidate;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function listSessionCorpusFiles(sessionCorpusDir: string): Promise<string[]> {
|
||||
const entries = await fs.readdir(sessionCorpusDir, { withFileTypes: true });
|
||||
return entries
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".txt"))
|
||||
.map((entry) => path.join(sessionCorpusDir, entry.name))
|
||||
.toSorted();
|
||||
}
|
||||
|
||||
function isSuspiciousSessionCorpusLine(line: string): boolean {
|
||||
return (
|
||||
line.includes(DREAMING_NARRATIVE_PROMPT_PREFIX) &&
|
||||
(line.includes(DREAMING_NARRATIVE_RUN_PREFIX) || line.includes("dreaming-narrative-"))
|
||||
);
|
||||
}
|
||||
|
||||
function buildArchiveTimestamp(now: Date): string {
|
||||
return now.toISOString().replace(/[:.]/g, "-");
|
||||
}
|
||||
|
||||
async function ensureArchivablePath(targetPath: string): Promise<"file" | "dir" | null> {
|
||||
const stat = await fs.lstat(targetPath).catch((err: NodeJS.ErrnoException) => {
|
||||
if (err.code === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (!stat) {
|
||||
return null;
|
||||
}
|
||||
if (stat.isSymbolicLink()) {
|
||||
throw new Error(`Refusing to archive symlinked path: ${targetPath}`);
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
return "dir";
|
||||
}
|
||||
if (stat.isFile()) {
|
||||
return "file";
|
||||
}
|
||||
throw new Error(`Refusing to archive non-file artifact: ${targetPath}`);
|
||||
}
|
||||
|
||||
async function moveToArchive(params: {
|
||||
targetPath: string;
|
||||
archiveDir: string;
|
||||
}): Promise<string | null> {
|
||||
const kind = await ensureArchivablePath(params.targetPath);
|
||||
if (!kind) {
|
||||
return null;
|
||||
}
|
||||
await fs.mkdir(params.archiveDir, { recursive: true });
|
||||
const baseName = path.basename(params.targetPath);
|
||||
const destination = path.join(params.archiveDir, `${baseName}.${randomUUID()}`);
|
||||
await fs.rename(params.targetPath, destination);
|
||||
return destination;
|
||||
}
|
||||
|
||||
export async function auditDreamingArtifacts(params: {
|
||||
workspaceDir: string;
|
||||
}): Promise<DreamingArtifactsAuditSummary> {
|
||||
const workspaceDir = requireAbsoluteWorkspaceDir(params.workspaceDir);
|
||||
const dreamsPath = await resolveExistingDreamsPath(workspaceDir);
|
||||
const sessionCorpusDir = path.join(workspaceDir, SESSION_CORPUS_RELATIVE_DIR);
|
||||
const sessionIngestionPath = path.join(workspaceDir, SESSION_INGESTION_RELATIVE_PATH);
|
||||
const issues: DreamingArtifactsAuditIssue[] = [];
|
||||
let sessionCorpusFileCount = 0;
|
||||
let suspiciousSessionCorpusFileCount = 0;
|
||||
let suspiciousSessionCorpusLineCount = 0;
|
||||
let sessionIngestionExists = false;
|
||||
|
||||
if (dreamsPath) {
|
||||
try {
|
||||
await fs.access(dreamsPath);
|
||||
} catch (err) {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
code: "dreaming-diary-unreadable",
|
||||
message: `Dream diary could not be inspected: ${(err as NodeJS.ErrnoException).code ?? "error"}.`,
|
||||
fixable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const corpusFiles = await listSessionCorpusFiles(sessionCorpusDir);
|
||||
sessionCorpusFileCount = corpusFiles.length;
|
||||
for (const corpusFile of corpusFiles) {
|
||||
const content = await fs.readFile(corpusFile, "utf-8");
|
||||
const suspiciousLines = content
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0 && isSuspiciousSessionCorpusLine(line));
|
||||
if (suspiciousLines.length > 0) {
|
||||
suspiciousSessionCorpusFileCount += 1;
|
||||
suspiciousSessionCorpusLineCount += suspiciousLines.length;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
code: "dreaming-session-corpus-unreadable",
|
||||
message: `Dreaming session corpus could not be inspected: ${(err as NodeJS.ErrnoException).code ?? "error"}.`,
|
||||
fixable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.access(sessionIngestionPath);
|
||||
sessionIngestionExists = true;
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
code: "dreaming-session-ingestion-unreadable",
|
||||
message: `Dreaming session-ingestion state could not be inspected: ${(err as NodeJS.ErrnoException).code ?? "error"}.`,
|
||||
fixable: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (suspiciousSessionCorpusLineCount > 0) {
|
||||
issues.push({
|
||||
severity: "warn",
|
||||
code: "dreaming-session-corpus-self-ingested",
|
||||
message: `Dreaming session corpus appears to contain self-ingested narrative content (${suspiciousSessionCorpusLineCount} suspicious line${suspiciousSessionCorpusLineCount === 1 ? "" : "s"}).`,
|
||||
fixable: true,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...(dreamsPath ? { dreamsPath } : {}),
|
||||
sessionCorpusDir,
|
||||
sessionCorpusFileCount,
|
||||
suspiciousSessionCorpusFileCount,
|
||||
suspiciousSessionCorpusLineCount,
|
||||
sessionIngestionPath,
|
||||
sessionIngestionExists,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
export async function repairDreamingArtifacts(params: {
|
||||
workspaceDir: string;
|
||||
archiveDiary?: boolean;
|
||||
now?: Date;
|
||||
}): Promise<RepairDreamingArtifactsResult> {
|
||||
const workspaceDir = requireAbsoluteWorkspaceDir(params.workspaceDir);
|
||||
const warnings: string[] = [];
|
||||
const archivedPaths: string[] = [];
|
||||
let archiveDir: string | undefined;
|
||||
let archivedDreamsDiary = false;
|
||||
let archivedSessionCorpus = false;
|
||||
let archivedSessionIngestion = false;
|
||||
|
||||
const ensureArchiveDir = () => {
|
||||
archiveDir ??= path.join(
|
||||
workspaceDir,
|
||||
REPAIR_ARCHIVE_RELATIVE_DIR,
|
||||
buildArchiveTimestamp(params.now ?? new Date()),
|
||||
);
|
||||
return archiveDir;
|
||||
};
|
||||
|
||||
const archivePathIfPresent = async (targetPath: string): Promise<string | null> => {
|
||||
try {
|
||||
return await moveToArchive({ targetPath, archiveDir: ensureArchiveDir() });
|
||||
} catch (err) {
|
||||
warnings.push(err instanceof Error ? err.message : String(err));
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const sessionCorpusDestination = await archivePathIfPresent(
|
||||
path.join(workspaceDir, SESSION_CORPUS_RELATIVE_DIR),
|
||||
);
|
||||
if (sessionCorpusDestination) {
|
||||
archivedSessionCorpus = true;
|
||||
archivedPaths.push(sessionCorpusDestination);
|
||||
}
|
||||
|
||||
const sessionIngestionDestination = await archivePathIfPresent(
|
||||
path.join(workspaceDir, SESSION_INGESTION_RELATIVE_PATH),
|
||||
);
|
||||
if (sessionIngestionDestination) {
|
||||
archivedSessionIngestion = true;
|
||||
archivedPaths.push(sessionIngestionDestination);
|
||||
}
|
||||
|
||||
if (params.archiveDiary) {
|
||||
const dreamsPath = await resolveExistingDreamsPath(workspaceDir);
|
||||
if (dreamsPath) {
|
||||
const dreamsDestination = await archivePathIfPresent(dreamsPath);
|
||||
if (dreamsDestination) {
|
||||
archivedDreamsDiary = true;
|
||||
archivedPaths.push(dreamsDestination);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const changed = archivedDreamsDiary || archivedSessionCorpus || archivedSessionIngestion;
|
||||
return {
|
||||
changed,
|
||||
...(archiveDir ? { archiveDir } : {}),
|
||||
archivedDreamsDiary,
|
||||
archivedSessionCorpus,
|
||||
archivedSessionIngestion,
|
||||
archivedPaths,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
22
openclaw/extensions/memory-core/src/dreaming-shared.ts
Normal file
22
openclaw/extensions/memory-core/src/dreaming-shared.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
export { asNullableRecord as asRecord } from "openclaw/plugin-sdk/text-runtime";
|
||||
export { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
||||
export function normalizeTrimmedString(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
export function includesSystemEventToken(cleanedBody: string, eventText: string): boolean {
|
||||
const normalizedBody = normalizeTrimmedString(cleanedBody);
|
||||
const normalizedEventText = normalizeTrimmedString(eventText);
|
||||
if (!normalizedBody || !normalizedEventText) {
|
||||
return false;
|
||||
}
|
||||
if (normalizedBody === normalizedEventText) {
|
||||
return true;
|
||||
}
|
||||
return normalizedBody.split(/\r?\n/).some((line) => line.trim() === normalizedEventText);
|
||||
}
|
||||
1659
openclaw/extensions/memory-core/src/dreaming.test.ts
Normal file
1659
openclaw/extensions/memory-core/src/dreaming.test.ts
Normal file
File diff suppressed because it is too large
Load diff
788
openclaw/extensions/memory-core/src/dreaming.ts
Normal file
788
openclaw/extensions/memory-core/src/dreaming.ts
Normal file
|
|
@ -0,0 +1,788 @@
|
|||
import { peekSystemEventEntries } from "openclaw/plugin-sdk/infra-runtime";
|
||||
import type { OpenClawConfig, OpenClawPluginApi } from "openclaw/plugin-sdk/memory-core";
|
||||
import {
|
||||
DEFAULT_MEMORY_DREAMING_FREQUENCY as DEFAULT_MEMORY_DREAMING_CRON_EXPR,
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_LIMIT as DEFAULT_MEMORY_DREAMING_LIMIT,
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_MIN_RECALL_COUNT as DEFAULT_MEMORY_DREAMING_MIN_RECALL_COUNT,
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_MIN_SCORE as DEFAULT_MEMORY_DREAMING_MIN_SCORE,
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_MIN_UNIQUE_QUERIES as DEFAULT_MEMORY_DREAMING_MIN_UNIQUE_QUERIES,
|
||||
DEFAULT_MEMORY_DEEP_DREAMING_RECENCY_HALF_LIFE_DAYS as DEFAULT_MEMORY_DREAMING_RECENCY_HALF_LIFE_DAYS,
|
||||
resolveMemoryCorePluginConfig,
|
||||
resolveMemoryDeepDreamingConfig,
|
||||
resolveMemoryDreamingWorkspaces,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-status";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { writeDeepDreamingReport } from "./dreaming-markdown.js";
|
||||
import { generateAndAppendDreamNarrative, type NarrativePhaseData } from "./dreaming-narrative.js";
|
||||
import { runDreamingSweepPhases } from "./dreaming-phases.js";
|
||||
import {
|
||||
asRecord,
|
||||
formatErrorMessage,
|
||||
includesSystemEventToken,
|
||||
normalizeTrimmedString,
|
||||
} from "./dreaming-shared.js";
|
||||
import {
|
||||
applyShortTermPromotions,
|
||||
repairShortTermPromotionArtifacts,
|
||||
rankShortTermPromotionCandidates,
|
||||
} from "./short-term-promotion.js";
|
||||
|
||||
const MANAGED_DREAMING_CRON_NAME = "Memory Dreaming Promotion";
|
||||
const MANAGED_DREAMING_CRON_TAG = "[managed-by=memory-core.short-term-promotion]";
|
||||
const DREAMING_SYSTEM_EVENT_TEXT = "__openclaw_memory_core_short_term_promotion_dream__";
|
||||
const LEGACY_LIGHT_SLEEP_CRON_NAME = "Memory Light Dreaming";
|
||||
const LEGACY_LIGHT_SLEEP_CRON_TAG = "[managed-by=memory-core.dreaming.light]";
|
||||
const LEGACY_LIGHT_SLEEP_EVENT_TEXT = "__openclaw_memory_core_light_sleep__";
|
||||
const LEGACY_REM_SLEEP_CRON_NAME = "Memory REM Dreaming";
|
||||
const LEGACY_REM_SLEEP_CRON_TAG = "[managed-by=memory-core.dreaming.rem]";
|
||||
const LEGACY_REM_SLEEP_EVENT_TEXT = "__openclaw_memory_core_rem_sleep__";
|
||||
const RUNTIME_CRON_RECONCILE_INTERVAL_MS = 60_000;
|
||||
const HEARTBEAT_ISOLATED_SESSION_SUFFIX = ":heartbeat";
|
||||
|
||||
type Logger = Pick<OpenClawPluginApi["logger"], "info" | "warn" | "error">;
|
||||
|
||||
type CronSchedule = { kind: "cron"; expr: string; tz?: string };
|
||||
type CronPayload = { kind: "systemEvent"; text: string };
|
||||
type ManagedCronJobCreate = {
|
||||
name: string;
|
||||
description: string;
|
||||
enabled: boolean;
|
||||
schedule: CronSchedule;
|
||||
sessionTarget: "main";
|
||||
wakeMode: "now";
|
||||
payload: CronPayload;
|
||||
};
|
||||
|
||||
type ManagedCronJobPatch = {
|
||||
name?: string;
|
||||
description?: string;
|
||||
enabled?: boolean;
|
||||
schedule?: CronSchedule;
|
||||
sessionTarget?: "main";
|
||||
wakeMode?: "now";
|
||||
payload?: CronPayload;
|
||||
};
|
||||
|
||||
type ManagedCronJobLike = {
|
||||
id: string;
|
||||
name?: string;
|
||||
description?: string;
|
||||
enabled?: boolean;
|
||||
schedule?: {
|
||||
kind?: string;
|
||||
expr?: string;
|
||||
tz?: string;
|
||||
};
|
||||
sessionTarget?: string;
|
||||
wakeMode?: string;
|
||||
payload?: {
|
||||
kind?: string;
|
||||
text?: string;
|
||||
};
|
||||
createdAtMs?: number;
|
||||
};
|
||||
|
||||
type CronServiceLike = {
|
||||
list: (opts?: { includeDisabled?: boolean }) => Promise<ManagedCronJobLike[]>;
|
||||
add: (input: ManagedCronJobCreate) => Promise<unknown>;
|
||||
update: (id: string, patch: ManagedCronJobPatch) => Promise<unknown>;
|
||||
remove: (id: string) => Promise<{ removed?: boolean }>;
|
||||
};
|
||||
|
||||
type StartupCronSourceRefs = {
|
||||
context: Record<string, unknown>;
|
||||
deps: Record<string, unknown> | null;
|
||||
};
|
||||
|
||||
export type ShortTermPromotionDreamingConfig = {
|
||||
enabled: boolean;
|
||||
cron: string;
|
||||
timezone?: string;
|
||||
limit: number;
|
||||
minScore: number;
|
||||
minRecallCount: number;
|
||||
minUniqueQueries: number;
|
||||
recencyHalfLifeDays?: number;
|
||||
maxAgeDays?: number;
|
||||
verboseLogging: boolean;
|
||||
storage?: {
|
||||
mode: "inline" | "separate" | "both";
|
||||
separateReports: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
type ReconcileResult =
|
||||
| { status: "unavailable"; removed: number }
|
||||
| { status: "disabled"; removed: number }
|
||||
| { status: "added"; removed: number }
|
||||
| { status: "updated"; removed: number }
|
||||
| { status: "noop"; removed: number };
|
||||
|
||||
type LegacyPhaseMigrationMode = "enabled" | "disabled";
|
||||
|
||||
function formatRepairSummary(repair: {
|
||||
rewroteStore: boolean;
|
||||
removedInvalidEntries: number;
|
||||
removedStaleLock: boolean;
|
||||
}): string {
|
||||
const actions: string[] = [];
|
||||
if (repair.rewroteStore) {
|
||||
actions.push(
|
||||
`rewrote recall store${repair.removedInvalidEntries > 0 ? ` (-${repair.removedInvalidEntries} invalid)` : ""}`,
|
||||
);
|
||||
}
|
||||
if (repair.removedStaleLock) {
|
||||
actions.push("removed stale promotion lock");
|
||||
}
|
||||
return actions.join(", ");
|
||||
}
|
||||
|
||||
function resolveManagedCronDescription(config: ShortTermPromotionDreamingConfig): string {
|
||||
const recencyHalfLifeDays =
|
||||
config.recencyHalfLifeDays ?? DEFAULT_MEMORY_DREAMING_RECENCY_HALF_LIFE_DAYS;
|
||||
return `${MANAGED_DREAMING_CRON_TAG} Promote weighted short-term recalls into MEMORY.md (limit=${config.limit}, minScore=${config.minScore.toFixed(3)}, minRecallCount=${config.minRecallCount}, minUniqueQueries=${config.minUniqueQueries}, recencyHalfLifeDays=${recencyHalfLifeDays}, maxAgeDays=${config.maxAgeDays ?? "none"}).`;
|
||||
}
|
||||
|
||||
function buildManagedDreamingCronJob(
|
||||
config: ShortTermPromotionDreamingConfig,
|
||||
): ManagedCronJobCreate {
|
||||
return {
|
||||
name: MANAGED_DREAMING_CRON_NAME,
|
||||
description: resolveManagedCronDescription(config),
|
||||
enabled: true,
|
||||
schedule: {
|
||||
kind: "cron",
|
||||
expr: config.cron,
|
||||
...(config.timezone ? { tz: config.timezone } : {}),
|
||||
},
|
||||
sessionTarget: "main",
|
||||
wakeMode: "now",
|
||||
payload: {
|
||||
kind: "systemEvent",
|
||||
text: DREAMING_SYSTEM_EVENT_TEXT,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function isManagedDreamingJob(job: ManagedCronJobLike): boolean {
|
||||
const description = normalizeTrimmedString(job.description);
|
||||
if (description?.includes(MANAGED_DREAMING_CRON_TAG)) {
|
||||
return true;
|
||||
}
|
||||
const name = normalizeTrimmedString(job.name);
|
||||
const payloadText = normalizeTrimmedString(job.payload?.text);
|
||||
return name === MANAGED_DREAMING_CRON_NAME && payloadText === DREAMING_SYSTEM_EVENT_TEXT;
|
||||
}
|
||||
|
||||
function isLegacyPhaseDreamingJob(job: ManagedCronJobLike): boolean {
|
||||
const description = normalizeTrimmedString(job.description);
|
||||
if (
|
||||
description?.includes(LEGACY_LIGHT_SLEEP_CRON_TAG) ||
|
||||
description?.includes(LEGACY_REM_SLEEP_CRON_TAG)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const name = normalizeTrimmedString(job.name);
|
||||
const payloadText = normalizeTrimmedString(job.payload?.text);
|
||||
if (name === LEGACY_LIGHT_SLEEP_CRON_NAME && payloadText === LEGACY_LIGHT_SLEEP_EVENT_TEXT) {
|
||||
return true;
|
||||
}
|
||||
return name === LEGACY_REM_SLEEP_CRON_NAME && payloadText === LEGACY_REM_SLEEP_EVENT_TEXT;
|
||||
}
|
||||
|
||||
function compareOptionalStrings(a: string | undefined, b: string | undefined): boolean {
|
||||
return a === b;
|
||||
}
|
||||
|
||||
async function migrateLegacyPhaseDreamingCronJobs(params: {
|
||||
cron: CronServiceLike;
|
||||
legacyJobs: ManagedCronJobLike[];
|
||||
logger: Logger;
|
||||
mode: LegacyPhaseMigrationMode;
|
||||
}): Promise<number> {
|
||||
let migrated = 0;
|
||||
for (const job of params.legacyJobs) {
|
||||
try {
|
||||
const result = await params.cron.remove(job.id);
|
||||
if (result.removed === true) {
|
||||
migrated += 1;
|
||||
}
|
||||
} catch (err) {
|
||||
params.logger.warn(
|
||||
`memory-core: failed to migrate legacy phase dreaming cron job ${job.id}: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (migrated > 0) {
|
||||
if (params.mode === "enabled") {
|
||||
params.logger.info(
|
||||
`memory-core: migrated ${migrated} legacy phase dreaming cron job(s) to the unified dreaming controller.`,
|
||||
);
|
||||
} else {
|
||||
params.logger.info(
|
||||
`memory-core: completed legacy phase dreaming cron migration while unified dreaming is disabled (${migrated} job(s) removed).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return migrated;
|
||||
}
|
||||
|
||||
function buildManagedDreamingPatch(
|
||||
job: ManagedCronJobLike,
|
||||
desired: ManagedCronJobCreate,
|
||||
): ManagedCronJobPatch | null {
|
||||
const patch: ManagedCronJobPatch = {};
|
||||
|
||||
if (!compareOptionalStrings(normalizeTrimmedString(job.name), desired.name)) {
|
||||
patch.name = desired.name;
|
||||
}
|
||||
if (!compareOptionalStrings(normalizeTrimmedString(job.description), desired.description)) {
|
||||
patch.description = desired.description;
|
||||
}
|
||||
if (job.enabled !== true) {
|
||||
patch.enabled = true;
|
||||
}
|
||||
|
||||
const scheduleKind = normalizeLowercaseStringOrEmpty(normalizeTrimmedString(job.schedule?.kind));
|
||||
const scheduleExpr = normalizeTrimmedString(job.schedule?.expr);
|
||||
const scheduleTz = normalizeTrimmedString(job.schedule?.tz);
|
||||
if (
|
||||
scheduleKind !== "cron" ||
|
||||
!compareOptionalStrings(scheduleExpr, desired.schedule.expr) ||
|
||||
!compareOptionalStrings(scheduleTz, desired.schedule.tz)
|
||||
) {
|
||||
patch.schedule = desired.schedule;
|
||||
}
|
||||
|
||||
const sessionTarget = normalizeLowercaseStringOrEmpty(normalizeTrimmedString(job.sessionTarget));
|
||||
if (sessionTarget !== "main") {
|
||||
patch.sessionTarget = "main";
|
||||
}
|
||||
const wakeMode = normalizeLowercaseStringOrEmpty(normalizeTrimmedString(job.wakeMode));
|
||||
if (wakeMode !== "now") {
|
||||
patch.wakeMode = "now";
|
||||
}
|
||||
|
||||
const payloadKind = normalizeLowercaseStringOrEmpty(normalizeTrimmedString(job.payload?.kind));
|
||||
const payloadText = normalizeTrimmedString(job.payload?.text);
|
||||
if (payloadKind !== "systemevent" || !compareOptionalStrings(payloadText, desired.payload.text)) {
|
||||
patch.payload = desired.payload;
|
||||
}
|
||||
|
||||
return Object.keys(patch).length > 0 ? patch : null;
|
||||
}
|
||||
|
||||
function sortManagedJobs(managed: ManagedCronJobLike[]): ManagedCronJobLike[] {
|
||||
return managed.toSorted((a, b) => {
|
||||
const aCreated =
|
||||
typeof a.createdAtMs === "number" && Number.isFinite(a.createdAtMs)
|
||||
? a.createdAtMs
|
||||
: Number.MAX_SAFE_INTEGER;
|
||||
const bCreated =
|
||||
typeof b.createdAtMs === "number" && Number.isFinite(b.createdAtMs)
|
||||
? b.createdAtMs
|
||||
: Number.MAX_SAFE_INTEGER;
|
||||
if (aCreated !== bCreated) {
|
||||
return aCreated - bCreated;
|
||||
}
|
||||
return a.id.localeCompare(b.id);
|
||||
});
|
||||
}
|
||||
|
||||
function resolveCronServiceFromCandidate(candidate: unknown): CronServiceLike | null {
|
||||
if (!candidate || typeof candidate !== "object") {
|
||||
return null;
|
||||
}
|
||||
const cron = candidate as Partial<CronServiceLike>;
|
||||
if (
|
||||
typeof cron.list !== "function" ||
|
||||
typeof cron.add !== "function" ||
|
||||
typeof cron.update !== "function" ||
|
||||
typeof cron.remove !== "function"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return cron as CronServiceLike;
|
||||
}
|
||||
|
||||
function resolveStartupCronSourceFromEvent(event: unknown): StartupCronSourceRefs | null {
|
||||
const payload = asRecord(event);
|
||||
if (!payload) {
|
||||
return null;
|
||||
}
|
||||
if (payload.type !== "gateway" || payload.action !== "startup") {
|
||||
return null;
|
||||
}
|
||||
const context = asRecord(payload.context);
|
||||
if (!context) {
|
||||
return null;
|
||||
}
|
||||
return { context, deps: asRecord(context.deps) };
|
||||
}
|
||||
|
||||
function resolveCronServiceFromStartupSource(
|
||||
source: StartupCronSourceRefs | null,
|
||||
): CronServiceLike | null {
|
||||
if (!source) {
|
||||
return null;
|
||||
}
|
||||
return (
|
||||
resolveCronServiceFromCandidate(source.context.cron) ??
|
||||
resolveCronServiceFromCandidate(source.deps?.cron)
|
||||
);
|
||||
}
|
||||
|
||||
function resolveCronServiceFromStartupEvent(event: unknown): CronServiceLike | null {
|
||||
return resolveCronServiceFromStartupSource(resolveStartupCronSourceFromEvent(event));
|
||||
}
|
||||
|
||||
function resolveStartupConfigFromEvent(event: unknown, fallback: OpenClawConfig): OpenClawConfig {
|
||||
const startupEvent = asRecord(event);
|
||||
const startupContext = asRecord(startupEvent?.context);
|
||||
const startupCfg = asRecord(startupContext?.cfg);
|
||||
if (!startupCfg) {
|
||||
return fallback;
|
||||
}
|
||||
return startupCfg as OpenClawConfig;
|
||||
}
|
||||
|
||||
function resolveDreamingTriggerSessionKeys(sessionKey?: string): string[] {
|
||||
const normalized = normalizeTrimmedString(sessionKey);
|
||||
if (!normalized) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const keys = [normalized];
|
||||
// Isolated heartbeat runs execute in a sibling `:heartbeat` session while cron
|
||||
// system events stay queued on the base main session.
|
||||
if (normalized.endsWith(HEARTBEAT_ISOLATED_SESSION_SUFFIX)) {
|
||||
const baseSessionKey = normalized.slice(0, -HEARTBEAT_ISOLATED_SESSION_SUFFIX.length).trim();
|
||||
if (baseSessionKey) {
|
||||
keys.push(baseSessionKey);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(new Set(keys));
|
||||
}
|
||||
|
||||
function hasPendingManagedDreamingCronEvent(sessionKey?: string): boolean {
|
||||
return resolveDreamingTriggerSessionKeys(sessionKey).some((candidateSessionKey) =>
|
||||
peekSystemEventEntries(candidateSessionKey).some(
|
||||
(event) =>
|
||||
event.contextKey?.startsWith("cron:") === true &&
|
||||
normalizeTrimmedString(event.text) === DREAMING_SYSTEM_EVENT_TEXT,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveShortTermPromotionDreamingConfig(params: {
|
||||
pluginConfig?: Record<string, unknown>;
|
||||
cfg?: OpenClawConfig;
|
||||
}): ShortTermPromotionDreamingConfig {
|
||||
const resolved = resolveMemoryDeepDreamingConfig(params);
|
||||
return {
|
||||
enabled: resolved.enabled,
|
||||
cron: resolved.cron,
|
||||
...(resolved.timezone ? { timezone: resolved.timezone } : {}),
|
||||
limit: resolved.limit,
|
||||
minScore: resolved.minScore,
|
||||
minRecallCount: resolved.minRecallCount,
|
||||
minUniqueQueries: resolved.minUniqueQueries,
|
||||
recencyHalfLifeDays: resolved.recencyHalfLifeDays,
|
||||
...(typeof resolved.maxAgeDays === "number" ? { maxAgeDays: resolved.maxAgeDays } : {}),
|
||||
verboseLogging: resolved.verboseLogging,
|
||||
storage: resolved.storage,
|
||||
};
|
||||
}
|
||||
|
||||
export async function reconcileShortTermDreamingCronJob(params: {
|
||||
cron: CronServiceLike | null;
|
||||
config: ShortTermPromotionDreamingConfig;
|
||||
logger: Logger;
|
||||
}): Promise<ReconcileResult> {
|
||||
const cron = params.cron;
|
||||
if (!cron) {
|
||||
return { status: "unavailable", removed: 0 };
|
||||
}
|
||||
|
||||
const allJobs = await cron.list({ includeDisabled: true });
|
||||
const managed = allJobs.filter(isManagedDreamingJob);
|
||||
const legacyPhaseJobs = allJobs.filter(isLegacyPhaseDreamingJob);
|
||||
|
||||
if (!params.config.enabled) {
|
||||
let removed = await migrateLegacyPhaseDreamingCronJobs({
|
||||
cron,
|
||||
legacyJobs: legacyPhaseJobs,
|
||||
logger: params.logger,
|
||||
mode: "disabled",
|
||||
});
|
||||
for (const job of managed) {
|
||||
try {
|
||||
const result = await cron.remove(job.id);
|
||||
if (result.removed === true) {
|
||||
removed += 1;
|
||||
}
|
||||
} catch (err) {
|
||||
params.logger.warn(
|
||||
`memory-core: failed to remove managed dreaming cron job ${job.id}: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (removed > 0) {
|
||||
params.logger.info(`memory-core: removed ${removed} managed dreaming cron job(s).`);
|
||||
}
|
||||
return { status: "disabled", removed };
|
||||
}
|
||||
|
||||
const desired = buildManagedDreamingCronJob(params.config);
|
||||
if (managed.length === 0) {
|
||||
await cron.add(desired);
|
||||
const migratedLegacy = await migrateLegacyPhaseDreamingCronJobs({
|
||||
cron,
|
||||
legacyJobs: legacyPhaseJobs,
|
||||
logger: params.logger,
|
||||
mode: "enabled",
|
||||
});
|
||||
params.logger.info("memory-core: created managed dreaming cron job.");
|
||||
return { status: "added", removed: migratedLegacy };
|
||||
}
|
||||
|
||||
const [primary, ...duplicates] = sortManagedJobs(managed);
|
||||
let removed = await migrateLegacyPhaseDreamingCronJobs({
|
||||
cron,
|
||||
legacyJobs: legacyPhaseJobs,
|
||||
logger: params.logger,
|
||||
mode: "enabled",
|
||||
});
|
||||
for (const duplicate of duplicates) {
|
||||
try {
|
||||
const result = await cron.remove(duplicate.id);
|
||||
if (result.removed === true) {
|
||||
removed += 1;
|
||||
}
|
||||
} catch (err) {
|
||||
params.logger.warn(
|
||||
`memory-core: failed to prune duplicate managed dreaming cron job ${duplicate.id}: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const patch = buildManagedDreamingPatch(primary, desired);
|
||||
if (!patch) {
|
||||
if (removed > 0) {
|
||||
params.logger.info("memory-core: pruned duplicate managed dreaming cron jobs.");
|
||||
}
|
||||
return { status: "noop", removed };
|
||||
}
|
||||
|
||||
await cron.update(primary.id, patch);
|
||||
params.logger.info("memory-core: updated managed dreaming cron job.");
|
||||
return { status: "updated", removed };
|
||||
}
|
||||
|
||||
export async function runShortTermDreamingPromotionIfTriggered(params: {
|
||||
cleanedBody: string;
|
||||
trigger?: string;
|
||||
workspaceDir?: string;
|
||||
cfg?: OpenClawConfig;
|
||||
config: ShortTermPromotionDreamingConfig;
|
||||
logger: Logger;
|
||||
subagent?: Parameters<typeof generateAndAppendDreamNarrative>[0]["subagent"];
|
||||
}): Promise<{ handled: true; reason: string } | undefined> {
|
||||
if (params.trigger !== "heartbeat") {
|
||||
return undefined;
|
||||
}
|
||||
if (!includesSystemEventToken(params.cleanedBody, DREAMING_SYSTEM_EVENT_TEXT)) {
|
||||
return undefined;
|
||||
}
|
||||
if (!params.config.enabled) {
|
||||
return { handled: true, reason: "memory-core: short-term dreaming disabled" };
|
||||
}
|
||||
|
||||
const recencyHalfLifeDays =
|
||||
params.config.recencyHalfLifeDays ?? DEFAULT_MEMORY_DREAMING_RECENCY_HALF_LIFE_DAYS;
|
||||
const workspaceCandidates = params.cfg
|
||||
? resolveMemoryDreamingWorkspaces(params.cfg).map((entry) => entry.workspaceDir)
|
||||
: [];
|
||||
const seenWorkspaces = new Set<string>();
|
||||
const workspaces = workspaceCandidates.filter((workspaceDir) => {
|
||||
if (seenWorkspaces.has(workspaceDir)) {
|
||||
return false;
|
||||
}
|
||||
seenWorkspaces.add(workspaceDir);
|
||||
return true;
|
||||
});
|
||||
const fallbackWorkspaceDir = normalizeTrimmedString(params.workspaceDir);
|
||||
if (workspaces.length === 0 && fallbackWorkspaceDir) {
|
||||
workspaces.push(fallbackWorkspaceDir);
|
||||
}
|
||||
if (workspaces.length === 0) {
|
||||
params.logger.warn(
|
||||
"memory-core: dreaming promotion skipped because no memory workspace is available.",
|
||||
);
|
||||
return { handled: true, reason: "memory-core: short-term dreaming missing workspace" };
|
||||
}
|
||||
if (params.config.limit === 0) {
|
||||
params.logger.info("memory-core: dreaming promotion skipped because limit=0.");
|
||||
return { handled: true, reason: "memory-core: short-term dreaming disabled by limit" };
|
||||
}
|
||||
|
||||
if (params.config.verboseLogging) {
|
||||
params.logger.info(
|
||||
`memory-core: dreaming verbose enabled (cron=${params.config.cron}, limit=${params.config.limit}, minScore=${params.config.minScore.toFixed(3)}, minRecallCount=${params.config.minRecallCount}, minUniqueQueries=${params.config.minUniqueQueries}, recencyHalfLifeDays=${recencyHalfLifeDays}, maxAgeDays=${params.config.maxAgeDays ?? "none"}, workspaces=${workspaces.length}).`,
|
||||
);
|
||||
}
|
||||
|
||||
let totalCandidates = 0;
|
||||
let totalApplied = 0;
|
||||
let failedWorkspaces = 0;
|
||||
const pluginConfig = params.cfg ? resolveMemoryCorePluginConfig(params.cfg) : undefined;
|
||||
for (const workspaceDir of workspaces) {
|
||||
try {
|
||||
const sweepNowMs = Date.now();
|
||||
await runDreamingSweepPhases({
|
||||
workspaceDir,
|
||||
pluginConfig,
|
||||
cfg: params.cfg,
|
||||
logger: params.logger,
|
||||
subagent: params.subagent,
|
||||
nowMs: sweepNowMs,
|
||||
});
|
||||
|
||||
const reportLines: string[] = [];
|
||||
const repair = await repairShortTermPromotionArtifacts({ workspaceDir });
|
||||
if (repair.changed) {
|
||||
params.logger.info(
|
||||
`memory-core: normalized recall artifacts before dreaming (${formatRepairSummary(repair)}) [workspace=${workspaceDir}].`,
|
||||
);
|
||||
reportLines.push(`- Repaired recall artifacts: ${formatRepairSummary(repair)}.`);
|
||||
}
|
||||
const candidates = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
limit: params.config.limit,
|
||||
minScore: params.config.minScore,
|
||||
minRecallCount: params.config.minRecallCount,
|
||||
minUniqueQueries: params.config.minUniqueQueries,
|
||||
recencyHalfLifeDays,
|
||||
maxAgeDays: params.config.maxAgeDays,
|
||||
nowMs: sweepNowMs,
|
||||
});
|
||||
totalCandidates += candidates.length;
|
||||
reportLines.push(`- Ranked ${candidates.length} candidate(s) for durable promotion.`);
|
||||
if (params.config.verboseLogging) {
|
||||
const candidateSummary =
|
||||
candidates.length > 0
|
||||
? candidates
|
||||
.map(
|
||||
(candidate) =>
|
||||
`${candidate.path}:${candidate.startLine}-${candidate.endLine} score=${candidate.score.toFixed(3)} recalls=${candidate.recallCount} queries=${candidate.uniqueQueries} components={freq=${candidate.components.frequency.toFixed(3)},rel=${candidate.components.relevance.toFixed(3)},div=${candidate.components.diversity.toFixed(3)},rec=${candidate.components.recency.toFixed(3)},cons=${candidate.components.consolidation.toFixed(3)},concept=${candidate.components.conceptual.toFixed(3)}}`,
|
||||
)
|
||||
.join(" | ")
|
||||
: "none";
|
||||
params.logger.info(
|
||||
`memory-core: dreaming candidate details [workspace=${workspaceDir}] ${candidateSummary}`,
|
||||
);
|
||||
}
|
||||
const applied = await applyShortTermPromotions({
|
||||
workspaceDir,
|
||||
candidates,
|
||||
limit: params.config.limit,
|
||||
minScore: params.config.minScore,
|
||||
minRecallCount: params.config.minRecallCount,
|
||||
minUniqueQueries: params.config.minUniqueQueries,
|
||||
maxAgeDays: params.config.maxAgeDays,
|
||||
timezone: params.config.timezone,
|
||||
nowMs: sweepNowMs,
|
||||
});
|
||||
totalApplied += applied.applied;
|
||||
reportLines.push(`- Promoted ${applied.applied} candidate(s) into MEMORY.md.`);
|
||||
if (params.config.verboseLogging) {
|
||||
const appliedSummary =
|
||||
applied.appliedCandidates.length > 0
|
||||
? applied.appliedCandidates
|
||||
.map(
|
||||
(candidate) =>
|
||||
`${candidate.path}:${candidate.startLine}-${candidate.endLine} score=${candidate.score.toFixed(3)} recalls=${candidate.recallCount}`,
|
||||
)
|
||||
.join(" | ")
|
||||
: "none";
|
||||
params.logger.info(
|
||||
`memory-core: dreaming applied details [workspace=${workspaceDir}] ${appliedSummary}`,
|
||||
);
|
||||
}
|
||||
await writeDeepDreamingReport({
|
||||
workspaceDir,
|
||||
bodyLines: reportLines,
|
||||
nowMs: sweepNowMs,
|
||||
timezone: params.config.timezone,
|
||||
storage: params.config.storage ?? { mode: "separate", separateReports: false },
|
||||
});
|
||||
// Generate dream diary narrative from promoted memories.
|
||||
if (params.subagent && (candidates.length > 0 || applied.applied > 0)) {
|
||||
const data: NarrativePhaseData = {
|
||||
phase: "deep",
|
||||
snippets: candidates.map((c) => c.snippet).filter(Boolean),
|
||||
promotions: applied.appliedCandidates.map((c) => c.snippet).filter(Boolean),
|
||||
};
|
||||
await generateAndAppendDreamNarrative({
|
||||
subagent: params.subagent,
|
||||
workspaceDir,
|
||||
data,
|
||||
nowMs: sweepNowMs,
|
||||
timezone: params.config.timezone,
|
||||
logger: params.logger,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
failedWorkspaces += 1;
|
||||
params.logger.error(
|
||||
`memory-core: dreaming promotion failed for workspace ${workspaceDir}: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
params.logger.info(
|
||||
`memory-core: dreaming promotion complete (workspaces=${workspaces.length}, candidates=${totalCandidates}, applied=${totalApplied}, failed=${failedWorkspaces}).`,
|
||||
);
|
||||
|
||||
return { handled: true, reason: "memory-core: short-term dreaming processed" };
|
||||
}
|
||||
|
||||
export function registerShortTermPromotionDreaming(api: OpenClawPluginApi): void {
|
||||
let startupCronSource: StartupCronSourceRefs | null = null;
|
||||
let unavailableCronWarningEmitted = false;
|
||||
let lastRuntimeReconcileAtMs = 0;
|
||||
let lastRuntimeConfigKey: string | null = null;
|
||||
let lastRuntimeCronRef: CronServiceLike | null = null;
|
||||
|
||||
const runtimeConfigKey = (config: ShortTermPromotionDreamingConfig): string =>
|
||||
[
|
||||
config.enabled ? "enabled" : "disabled",
|
||||
config.cron,
|
||||
config.timezone ?? "",
|
||||
String(config.limit),
|
||||
String(config.minScore),
|
||||
String(config.minRecallCount),
|
||||
String(config.minUniqueQueries),
|
||||
String(config.recencyHalfLifeDays ?? ""),
|
||||
String(config.maxAgeDays ?? ""),
|
||||
config.verboseLogging ? "verbose" : "quiet",
|
||||
config.storage?.mode ?? "",
|
||||
config.storage?.separateReports ? "separate" : "inline",
|
||||
].join("|");
|
||||
|
||||
const reconcileManagedDreamingCron = async (params: {
|
||||
reason: "startup" | "runtime";
|
||||
startupEvent?: unknown;
|
||||
}): Promise<ShortTermPromotionDreamingConfig> => {
|
||||
const startupCfg =
|
||||
params.reason === "startup" && params.startupEvent !== undefined
|
||||
? resolveStartupConfigFromEvent(params.startupEvent, api.config)
|
||||
: api.config;
|
||||
const config = resolveShortTermPromotionDreamingConfig({
|
||||
pluginConfig:
|
||||
resolveMemoryCorePluginConfig(startupCfg) ??
|
||||
resolveMemoryCorePluginConfig(api.config) ??
|
||||
api.pluginConfig,
|
||||
cfg: startupCfg,
|
||||
});
|
||||
if (params.reason === "startup" && params.startupEvent !== undefined) {
|
||||
startupCronSource = resolveStartupCronSourceFromEvent(params.startupEvent);
|
||||
}
|
||||
const cron = resolveCronServiceFromStartupSource(startupCronSource);
|
||||
const configKey = runtimeConfigKey(config);
|
||||
if (!cron && config.enabled && !unavailableCronWarningEmitted) {
|
||||
api.logger.warn(
|
||||
"memory-core: managed dreaming cron could not be reconciled (cron service unavailable).",
|
||||
);
|
||||
unavailableCronWarningEmitted = true;
|
||||
}
|
||||
if (cron) {
|
||||
unavailableCronWarningEmitted = false;
|
||||
}
|
||||
if (params.reason === "runtime") {
|
||||
const now = Date.now();
|
||||
const withinThrottleWindow =
|
||||
now - lastRuntimeReconcileAtMs < RUNTIME_CRON_RECONCILE_INTERVAL_MS;
|
||||
if (
|
||||
withinThrottleWindow &&
|
||||
lastRuntimeConfigKey === configKey &&
|
||||
lastRuntimeCronRef === cron
|
||||
) {
|
||||
return config;
|
||||
}
|
||||
lastRuntimeReconcileAtMs = now;
|
||||
lastRuntimeConfigKey = configKey;
|
||||
lastRuntimeCronRef = cron;
|
||||
}
|
||||
await reconcileShortTermDreamingCronJob({
|
||||
cron,
|
||||
config,
|
||||
logger: api.logger,
|
||||
});
|
||||
return config;
|
||||
};
|
||||
|
||||
api.registerHook(
|
||||
"gateway:startup",
|
||||
async (event: unknown) => {
|
||||
try {
|
||||
await reconcileManagedDreamingCron({
|
||||
reason: "startup",
|
||||
startupEvent: event,
|
||||
});
|
||||
} catch (err) {
|
||||
api.logger.error(
|
||||
`memory-core: dreaming startup reconciliation failed: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
{ name: "memory-core-short-term-dreaming-cron" },
|
||||
);
|
||||
|
||||
api.on("before_agent_reply", async (event, ctx) => {
|
||||
try {
|
||||
if (ctx.trigger !== "heartbeat") {
|
||||
return undefined;
|
||||
}
|
||||
const config = await reconcileManagedDreamingCron({
|
||||
reason: "runtime",
|
||||
});
|
||||
if (
|
||||
!hasPendingManagedDreamingCronEvent(ctx.sessionKey) ||
|
||||
!includesSystemEventToken(event.cleanedBody, DREAMING_SYSTEM_EVENT_TEXT)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return await runShortTermDreamingPromotionIfTriggered({
|
||||
cleanedBody: event.cleanedBody,
|
||||
trigger: ctx.trigger,
|
||||
workspaceDir: ctx.workspaceDir,
|
||||
cfg: api.config,
|
||||
config,
|
||||
logger: api.logger,
|
||||
subagent: config.enabled ? api.runtime?.subagent : undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
api.logger.error(`memory-core: dreaming trigger failed: ${formatErrorMessage(err)}`);
|
||||
return undefined;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
buildManagedDreamingCronJob,
|
||||
buildManagedDreamingPatch,
|
||||
isManagedDreamingJob,
|
||||
resolveCronServiceFromStartupEvent,
|
||||
constants: {
|
||||
MANAGED_DREAMING_CRON_NAME,
|
||||
MANAGED_DREAMING_CRON_TAG,
|
||||
DREAMING_SYSTEM_EVENT_TEXT,
|
||||
DEFAULT_DREAMING_CRON_EXPR: DEFAULT_MEMORY_DREAMING_CRON_EXPR,
|
||||
DEFAULT_DREAMING_LIMIT: DEFAULT_MEMORY_DREAMING_LIMIT,
|
||||
DEFAULT_DREAMING_MIN_SCORE: DEFAULT_MEMORY_DREAMING_MIN_SCORE,
|
||||
DEFAULT_DREAMING_MIN_RECALL_COUNT: DEFAULT_MEMORY_DREAMING_MIN_RECALL_COUNT,
|
||||
DEFAULT_DREAMING_MIN_UNIQUE_QUERIES: DEFAULT_MEMORY_DREAMING_MIN_UNIQUE_QUERIES,
|
||||
DEFAULT_DREAMING_RECENCY_HALF_LIFE_DAYS: DEFAULT_MEMORY_DREAMING_RECENCY_HALF_LIFE_DAYS,
|
||||
},
|
||||
};
|
||||
139
openclaw/extensions/memory-core/src/flush-plan.ts
Normal file
139
openclaw/extensions/memory-core/src/flush-plan.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import {
|
||||
DEFAULT_PI_COMPACTION_RESERVE_TOKENS_FLOOR,
|
||||
parseNonNegativeByteSize,
|
||||
resolveCronStyleNow,
|
||||
SILENT_REPLY_TOKEN,
|
||||
type MemoryFlushPlan,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-runtime-core";
|
||||
|
||||
export const DEFAULT_MEMORY_FLUSH_SOFT_TOKENS = 4000;
|
||||
export const DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES = 2 * 1024 * 1024;
|
||||
|
||||
const MEMORY_FLUSH_TARGET_HINT =
|
||||
"Store durable memories only in memory/YYYY-MM-DD.md (create memory/ if needed).";
|
||||
const MEMORY_FLUSH_APPEND_ONLY_HINT =
|
||||
"If memory/YYYY-MM-DD.md already exists, APPEND new content only and do not overwrite existing entries.";
|
||||
const MEMORY_FLUSH_READ_ONLY_HINT =
|
||||
"Treat workspace bootstrap/reference files such as MEMORY.md, DREAMS.md, SOUL.md, TOOLS.md, and AGENTS.md as read-only during this flush; never overwrite, replace, or edit them.";
|
||||
const MEMORY_FLUSH_REQUIRED_HINTS = [
|
||||
MEMORY_FLUSH_TARGET_HINT,
|
||||
MEMORY_FLUSH_APPEND_ONLY_HINT,
|
||||
MEMORY_FLUSH_READ_ONLY_HINT,
|
||||
];
|
||||
|
||||
export const DEFAULT_MEMORY_FLUSH_PROMPT = [
|
||||
"Pre-compaction memory flush.",
|
||||
MEMORY_FLUSH_TARGET_HINT,
|
||||
MEMORY_FLUSH_READ_ONLY_HINT,
|
||||
MEMORY_FLUSH_APPEND_ONLY_HINT,
|
||||
"Do NOT create timestamped variant files (e.g., YYYY-MM-DD-HHMM.md); always use the canonical YYYY-MM-DD.md filename.",
|
||||
`If nothing to store, reply with ${SILENT_REPLY_TOKEN}.`,
|
||||
].join(" ");
|
||||
|
||||
export const DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT = [
|
||||
"Pre-compaction memory flush turn.",
|
||||
"The session is near auto-compaction; capture durable memories to disk.",
|
||||
MEMORY_FLUSH_TARGET_HINT,
|
||||
MEMORY_FLUSH_READ_ONLY_HINT,
|
||||
MEMORY_FLUSH_APPEND_ONLY_HINT,
|
||||
`You may reply, but usually ${SILENT_REPLY_TOKEN} is correct.`,
|
||||
].join(" ");
|
||||
|
||||
function formatDateStampInTimezone(nowMs: number, timezone: string): string {
|
||||
const parts = new Intl.DateTimeFormat("en-US", {
|
||||
timeZone: timezone,
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
}).formatToParts(new Date(nowMs));
|
||||
const year = parts.find((part) => part.type === "year")?.value;
|
||||
const month = parts.find((part) => part.type === "month")?.value;
|
||||
const day = parts.find((part) => part.type === "day")?.value;
|
||||
if (year && month && day) {
|
||||
return `${year}-${month}-${day}`;
|
||||
}
|
||||
return new Date(nowMs).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function normalizeNonNegativeInt(value: unknown): number | null {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
return null;
|
||||
}
|
||||
const int = Math.floor(value);
|
||||
return int >= 0 ? int : null;
|
||||
}
|
||||
|
||||
function ensureNoReplyHint(text: string): string {
|
||||
if (text.includes(SILENT_REPLY_TOKEN)) {
|
||||
return text;
|
||||
}
|
||||
return `${text}\n\nIf no user-visible reply is needed, start with ${SILENT_REPLY_TOKEN}.`;
|
||||
}
|
||||
|
||||
function ensureMemoryFlushSafetyHints(text: string): string {
|
||||
let next = text.trim();
|
||||
for (const hint of MEMORY_FLUSH_REQUIRED_HINTS) {
|
||||
if (!next.includes(hint)) {
|
||||
next = next ? `${next}\n\n${hint}` : hint;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function appendCurrentTimeLine(text: string, timeLine: string): string {
|
||||
const trimmed = text.trimEnd();
|
||||
if (!trimmed) {
|
||||
return timeLine;
|
||||
}
|
||||
if (trimmed.includes("Current time:")) {
|
||||
return trimmed;
|
||||
}
|
||||
return `${trimmed}\n${timeLine}`;
|
||||
}
|
||||
|
||||
export function buildMemoryFlushPlan(
|
||||
params: {
|
||||
cfg?: OpenClawConfig;
|
||||
nowMs?: number;
|
||||
} = {},
|
||||
): MemoryFlushPlan | null {
|
||||
const resolved = params;
|
||||
const nowMs = Number.isFinite(resolved.nowMs) ? (resolved.nowMs as number) : Date.now();
|
||||
const cfg = resolved.cfg;
|
||||
const defaults = cfg?.agents?.defaults?.compaction?.memoryFlush;
|
||||
if (defaults?.enabled === false) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const softThresholdTokens =
|
||||
normalizeNonNegativeInt(defaults?.softThresholdTokens) ?? DEFAULT_MEMORY_FLUSH_SOFT_TOKENS;
|
||||
const forceFlushTranscriptBytes =
|
||||
parseNonNegativeByteSize(defaults?.forceFlushTranscriptBytes) ??
|
||||
DEFAULT_MEMORY_FLUSH_FORCE_TRANSCRIPT_BYTES;
|
||||
const reserveTokensFloor =
|
||||
normalizeNonNegativeInt(cfg?.agents?.defaults?.compaction?.reserveTokensFloor) ??
|
||||
DEFAULT_PI_COMPACTION_RESERVE_TOKENS_FLOOR;
|
||||
|
||||
const { timeLine, userTimezone } = resolveCronStyleNow(cfg ?? {}, nowMs);
|
||||
const dateStamp = formatDateStampInTimezone(nowMs, userTimezone);
|
||||
const relativePath = `memory/${dateStamp}.md`;
|
||||
|
||||
const promptBase = ensureNoReplyHint(
|
||||
ensureMemoryFlushSafetyHints(defaults?.prompt?.trim() || DEFAULT_MEMORY_FLUSH_PROMPT),
|
||||
);
|
||||
const systemPrompt = ensureNoReplyHint(
|
||||
ensureMemoryFlushSafetyHints(
|
||||
defaults?.systemPrompt?.trim() || DEFAULT_MEMORY_FLUSH_SYSTEM_PROMPT,
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
softThresholdTokens,
|
||||
forceFlushTranscriptBytes,
|
||||
reserveTokensFloor,
|
||||
prompt: appendCurrentTimeLine(promptBase.replaceAll("YYYY-MM-DD", dateStamp), timeLine),
|
||||
systemPrompt: systemPrompt.replaceAll("YYYY-MM-DD", dateStamp),
|
||||
relativePath,
|
||||
};
|
||||
}
|
||||
99
openclaw/extensions/memory-core/src/memory-events.test.ts
Normal file
99
openclaw/extensions/memory-core/src/memory-events.test.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { readMemoryHostEvents } from "openclaw/plugin-sdk/memory-host-events";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js";
|
||||
import {
|
||||
applyShortTermPromotions,
|
||||
rankShortTermPromotionCandidates,
|
||||
recordShortTermRecalls,
|
||||
} from "./short-term-promotion.js";
|
||||
import { createMemoryCoreTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempWorkspace } = createMemoryCoreTestHarness();
|
||||
|
||||
describe("memory host event journal integration", () => {
|
||||
it("records recall and promotion events from short-term promotion flows", async () => {
|
||||
const workspaceDir = await createTempWorkspace("memory-core-events-");
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", "2026-04-05.md"),
|
||||
"# Daily\n\nalpha\nbeta\ngamma\n",
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await recordShortTermRecalls({
|
||||
workspaceDir,
|
||||
query: "alpha memory",
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-05.md",
|
||||
startLine: 3,
|
||||
endLine: 4,
|
||||
score: 0.92,
|
||||
snippet: "alpha beta",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
nowMs: Date.UTC(2026, 3, 5, 12, 0, 0),
|
||||
});
|
||||
|
||||
const candidates = await rankShortTermPromotionCandidates({
|
||||
workspaceDir,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.UTC(2026, 3, 5, 12, 5, 0),
|
||||
});
|
||||
const applied = await applyShortTermPromotions({
|
||||
workspaceDir,
|
||||
candidates,
|
||||
minScore: 0,
|
||||
minRecallCount: 0,
|
||||
minUniqueQueries: 0,
|
||||
nowMs: Date.UTC(2026, 3, 5, 12, 10, 0),
|
||||
});
|
||||
|
||||
expect(applied.applied).toBe(1);
|
||||
|
||||
const events = await readMemoryHostEvents({ workspaceDir });
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual([
|
||||
"memory.recall.recorded",
|
||||
"memory.promotion.applied",
|
||||
]);
|
||||
expect(events[0]).toMatchObject({
|
||||
type: "memory.recall.recorded",
|
||||
resultCount: 1,
|
||||
query: "alpha memory",
|
||||
});
|
||||
expect(events[1]).toMatchObject({
|
||||
type: "memory.promotion.applied",
|
||||
applied: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("records dreaming completion events when phase artifacts are written", async () => {
|
||||
const workspaceDir = await createTempWorkspace("memory-core-dream-events-");
|
||||
|
||||
const written = await writeDailyDreamingPhaseBlock({
|
||||
workspaceDir,
|
||||
phase: "light",
|
||||
bodyLines: ["- staged note", "- second note"],
|
||||
nowMs: Date.UTC(2026, 3, 5, 13, 0, 0),
|
||||
storage: { mode: "both", separateReports: true },
|
||||
});
|
||||
|
||||
const events = await readMemoryHostEvents({ workspaceDir });
|
||||
|
||||
expect(written.inlinePath).toBeTruthy();
|
||||
expect(written.reportPath).toBeTruthy();
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]).toMatchObject({
|
||||
type: "memory.dream.completed",
|
||||
phase: "light",
|
||||
lineCount: 2,
|
||||
storageMode: "both",
|
||||
});
|
||||
});
|
||||
});
|
||||
121
openclaw/extensions/memory-core/src/memory-tool-manager-mock.ts
Normal file
121
openclaw/extensions/memory-core/src/memory-tool-manager-mock.ts
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
import type { MemorySearchRuntimeDebug } from "openclaw/plugin-sdk/memory-core-host-runtime-files";
|
||||
import { vi } from "vitest";
|
||||
|
||||
export type SearchImpl = (opts?: {
|
||||
maxResults?: number;
|
||||
minScore?: number;
|
||||
sessionKey?: string;
|
||||
qmdSearchModeOverride?: "query" | "search" | "vsearch";
|
||||
onDebug?: (debug: MemorySearchRuntimeDebug) => void;
|
||||
}) => Promise<unknown[]>;
|
||||
export type MemoryReadParams = { relPath: string; from?: number; lines?: number };
|
||||
export type MemoryReadResult = {
|
||||
text: string;
|
||||
path: string;
|
||||
truncated?: boolean;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
nextFrom?: number;
|
||||
};
|
||||
type MemoryBackend = "builtin" | "qmd";
|
||||
|
||||
let backend: MemoryBackend = "builtin";
|
||||
let workspaceDir = "/workspace";
|
||||
let customStatus: Record<string, unknown> | undefined;
|
||||
let searchImpl: SearchImpl = async () => [];
|
||||
let readFileImpl: (params: MemoryReadParams) => Promise<MemoryReadResult> = async (params) => ({
|
||||
text: "",
|
||||
path: params.relPath,
|
||||
from: params.from ?? 1,
|
||||
lines: params.lines ?? 120,
|
||||
});
|
||||
|
||||
const stubManager = {
|
||||
search: vi.fn(async (_query: string, opts?: Parameters<SearchImpl>[0]) => await searchImpl(opts)),
|
||||
readFile: vi.fn(async (params: MemoryReadParams) => await readFileImpl(params)),
|
||||
status: () => ({
|
||||
backend,
|
||||
files: 1,
|
||||
chunks: 1,
|
||||
dirty: false,
|
||||
workspaceDir,
|
||||
dbPath: "/workspace/.memory/index.sqlite",
|
||||
provider: "builtin",
|
||||
model: "builtin",
|
||||
requestedProvider: "builtin",
|
||||
sources: ["memory" as const],
|
||||
sourceCounts: [{ source: "memory" as const, files: 1, chunks: 1 }],
|
||||
custom: customStatus,
|
||||
}),
|
||||
sync: vi.fn(),
|
||||
probeVectorAvailability: vi.fn(async () => true),
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
const getMemorySearchManagerMock = vi.fn(async () => ({ manager: stubManager }));
|
||||
const readAgentMemoryFileMock = vi.fn(
|
||||
async (params: MemoryReadParams) => await readFileImpl(params),
|
||||
);
|
||||
|
||||
vi.mock("./tools.runtime.js", () => ({
|
||||
resolveMemoryBackendConfig: ({
|
||||
cfg,
|
||||
}: {
|
||||
cfg?: { memory?: { backend?: string; qmd?: unknown } };
|
||||
}) => ({
|
||||
backend,
|
||||
qmd: cfg?.memory?.qmd,
|
||||
}),
|
||||
getMemorySearchManager: getMemorySearchManagerMock,
|
||||
readAgentMemoryFile: readAgentMemoryFileMock,
|
||||
}));
|
||||
|
||||
export function setMemoryBackend(next: MemoryBackend): void {
|
||||
backend = next;
|
||||
}
|
||||
|
||||
export function setMemoryWorkspaceDir(next: string): void {
|
||||
workspaceDir = next;
|
||||
}
|
||||
|
||||
export function setMemoryStatusCustom(next: Record<string, unknown> | undefined): void {
|
||||
customStatus = next;
|
||||
}
|
||||
|
||||
export function setMemorySearchImpl(next: SearchImpl): void {
|
||||
searchImpl = next;
|
||||
}
|
||||
|
||||
export function setMemoryReadFileImpl(
|
||||
next: (params: MemoryReadParams) => Promise<MemoryReadResult>,
|
||||
): void {
|
||||
readFileImpl = next;
|
||||
}
|
||||
|
||||
export function resetMemoryToolMockState(overrides?: {
|
||||
backend?: MemoryBackend;
|
||||
searchImpl?: SearchImpl;
|
||||
readFileImpl?: (params: MemoryReadParams) => Promise<MemoryReadResult>;
|
||||
}): void {
|
||||
backend = overrides?.backend ?? "builtin";
|
||||
workspaceDir = "/workspace";
|
||||
customStatus = undefined;
|
||||
searchImpl = overrides?.searchImpl ?? (async () => []);
|
||||
readFileImpl =
|
||||
overrides?.readFileImpl ??
|
||||
(async (params: MemoryReadParams) => ({
|
||||
text: "",
|
||||
path: params.relPath,
|
||||
from: params.from ?? 1,
|
||||
lines: params.lines ?? 120,
|
||||
}));
|
||||
vi.clearAllMocks();
|
||||
}
|
||||
|
||||
export function getMemorySearchManagerMockCalls(): number {
|
||||
return getMemorySearchManagerMock.mock.calls.length;
|
||||
}
|
||||
|
||||
export function getReadAgentMemoryFileMockCalls(): number {
|
||||
return readAgentMemoryFileMock.mock.calls.length;
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
import { vi, type Mock } from "vitest";
|
||||
import "./test-runtime-mocks.js";
|
||||
|
||||
// Avoid exporting vitest mock types (TS2742 under pnpm + d.ts emit).
|
||||
type EmbedBatchMock = Mock<(texts: string[]) => Promise<number[][]>>;
|
||||
type EmbedQueryMock = Mock<() => Promise<number[]>>;
|
||||
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
embedBatch: vi.fn(async (texts: string[]) => texts.map(() => [0, 1, 0])),
|
||||
embedQuery: vi.fn(async () => [0, 1, 0]),
|
||||
}));
|
||||
|
||||
export function getEmbedBatchMock(): EmbedBatchMock {
|
||||
return hoisted.embedBatch;
|
||||
}
|
||||
|
||||
export function getEmbedQueryMock(): EmbedQueryMock {
|
||||
return hoisted.embedQuery;
|
||||
}
|
||||
|
||||
export function resetEmbeddingMocks(): void {
|
||||
hoisted.embedBatch.mockReset();
|
||||
hoisted.embedQuery.mockReset();
|
||||
hoisted.embedBatch.mockImplementation(async (texts: string[]) => texts.map(() => [0, 1, 0]));
|
||||
hoisted.embedQuery.mockImplementation(async () => [0, 1, 0]);
|
||||
}
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
createEmbeddingProvider: async () => ({
|
||||
requestedProvider: "openai",
|
||||
provider: {
|
||||
id: "mock",
|
||||
model: "mock-embed",
|
||||
maxInputTokens: 8192,
|
||||
embedQuery: hoisted.embedQuery,
|
||||
embedBatch: hoisted.embedBatch,
|
||||
},
|
||||
}),
|
||||
}));
|
||||
172
openclaw/extensions/memory-core/src/memory/embeddings.ts
Normal file
172
openclaw/extensions/memory-core/src/memory/embeddings.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
import {
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
getMemoryEmbeddingProvider,
|
||||
listMemoryEmbeddingProviders,
|
||||
type MemoryEmbeddingProvider,
|
||||
type MemoryEmbeddingProviderAdapter,
|
||||
type MemoryEmbeddingProviderCreateOptions,
|
||||
type MemoryEmbeddingProviderRuntime,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { formatErrorMessage } from "../dreaming-shared.js";
|
||||
import { canAutoSelectLocal } from "./provider-adapters.js";
|
||||
|
||||
export { DEFAULT_LOCAL_MODEL } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
|
||||
export type EmbeddingProvider = MemoryEmbeddingProvider;
|
||||
export type EmbeddingProviderId = string;
|
||||
export type EmbeddingProviderRequest = string;
|
||||
export type EmbeddingProviderFallback = string;
|
||||
export type EmbeddingProviderRuntime = MemoryEmbeddingProviderRuntime;
|
||||
|
||||
export type EmbeddingProviderResult = {
|
||||
provider: EmbeddingProvider | null;
|
||||
requestedProvider: EmbeddingProviderRequest;
|
||||
fallbackFrom?: string;
|
||||
fallbackReason?: string;
|
||||
providerUnavailableReason?: string;
|
||||
runtime?: EmbeddingProviderRuntime;
|
||||
};
|
||||
|
||||
type CreateEmbeddingProviderOptions = MemoryEmbeddingProviderCreateOptions & {
|
||||
provider: EmbeddingProviderRequest;
|
||||
fallback: EmbeddingProviderFallback;
|
||||
};
|
||||
|
||||
function formatProviderError(adapter: MemoryEmbeddingProviderAdapter, err: unknown): string {
|
||||
return adapter.formatSetupError?.(err) ?? formatErrorMessage(err);
|
||||
}
|
||||
|
||||
function shouldContinueAutoSelection(
|
||||
adapter: MemoryEmbeddingProviderAdapter,
|
||||
err: unknown,
|
||||
): boolean {
|
||||
return adapter.shouldContinueAutoSelection?.(err) ?? false;
|
||||
}
|
||||
|
||||
function getAdapter(
|
||||
id: string,
|
||||
config?: MemoryEmbeddingProviderCreateOptions["config"],
|
||||
): MemoryEmbeddingProviderAdapter {
|
||||
const adapter = getMemoryEmbeddingProvider(id, config);
|
||||
if (!adapter) {
|
||||
throw new Error(`Unknown memory embedding provider: ${id}`);
|
||||
}
|
||||
return adapter;
|
||||
}
|
||||
|
||||
function listAutoSelectAdapters(
|
||||
options: CreateEmbeddingProviderOptions,
|
||||
): MemoryEmbeddingProviderAdapter[] {
|
||||
return listMemoryEmbeddingProviders(options.config)
|
||||
.filter((adapter) => typeof adapter.autoSelectPriority === "number")
|
||||
.filter((adapter) =>
|
||||
adapter.id === "local" ? canAutoSelectLocal(options.local?.modelPath) : true,
|
||||
)
|
||||
.toSorted(
|
||||
(a, b) =>
|
||||
(a.autoSelectPriority ?? Number.MAX_SAFE_INTEGER) -
|
||||
(b.autoSelectPriority ?? Number.MAX_SAFE_INTEGER),
|
||||
);
|
||||
}
|
||||
|
||||
function resolveProviderModel(
|
||||
adapter: MemoryEmbeddingProviderAdapter,
|
||||
requestedModel: string,
|
||||
): string {
|
||||
const trimmed = requestedModel.trim();
|
||||
if (trimmed) {
|
||||
return trimmed;
|
||||
}
|
||||
return adapter.defaultModel ?? "";
|
||||
}
|
||||
|
||||
export function resolveEmbeddingProviderFallbackModel(
|
||||
providerId: string,
|
||||
fallbackSourceModel: string,
|
||||
config?: MemoryEmbeddingProviderCreateOptions["config"],
|
||||
): string {
|
||||
const adapter = getMemoryEmbeddingProvider(providerId, config);
|
||||
return adapter?.defaultModel ?? fallbackSourceModel;
|
||||
}
|
||||
|
||||
async function createWithAdapter(
|
||||
adapter: MemoryEmbeddingProviderAdapter,
|
||||
options: CreateEmbeddingProviderOptions,
|
||||
): Promise<EmbeddingProviderResult> {
|
||||
const result = await adapter.create({
|
||||
...options,
|
||||
model: resolveProviderModel(adapter, options.model),
|
||||
});
|
||||
return {
|
||||
provider: result.provider,
|
||||
requestedProvider: options.provider,
|
||||
runtime: result.runtime,
|
||||
};
|
||||
}
|
||||
|
||||
export async function createEmbeddingProvider(
|
||||
options: CreateEmbeddingProviderOptions,
|
||||
): Promise<EmbeddingProviderResult> {
|
||||
if (options.provider === "auto") {
|
||||
const reasons: string[] = [];
|
||||
for (const adapter of listAutoSelectAdapters(options)) {
|
||||
try {
|
||||
const result = await createWithAdapter(adapter, {
|
||||
...options,
|
||||
provider: adapter.id,
|
||||
});
|
||||
return {
|
||||
...result,
|
||||
requestedProvider: "auto",
|
||||
};
|
||||
} catch (err) {
|
||||
const message = formatProviderError(adapter, err);
|
||||
if (shouldContinueAutoSelection(adapter, err)) {
|
||||
reasons.push(message);
|
||||
continue;
|
||||
}
|
||||
const wrapped = new Error(message) as Error & { cause?: unknown };
|
||||
wrapped.cause = err;
|
||||
throw wrapped;
|
||||
}
|
||||
}
|
||||
return {
|
||||
provider: null,
|
||||
requestedProvider: "auto",
|
||||
providerUnavailableReason:
|
||||
reasons.length > 0 ? reasons.join("\n\n") : "No embeddings provider available.",
|
||||
};
|
||||
}
|
||||
|
||||
const primaryAdapter = getAdapter(options.provider, options.config);
|
||||
try {
|
||||
return await createWithAdapter(primaryAdapter, options);
|
||||
} catch (primaryErr) {
|
||||
const reason = formatProviderError(primaryAdapter, primaryErr);
|
||||
if (options.fallback && options.fallback !== "none" && options.fallback !== options.provider) {
|
||||
const fallbackAdapter = getAdapter(options.fallback, options.config);
|
||||
try {
|
||||
const fallbackResult = await createWithAdapter(fallbackAdapter, {
|
||||
...options,
|
||||
provider: options.fallback,
|
||||
});
|
||||
return {
|
||||
...fallbackResult,
|
||||
requestedProvider: options.provider,
|
||||
fallbackFrom: options.provider,
|
||||
fallbackReason: reason,
|
||||
};
|
||||
} catch (fallbackErr) {
|
||||
const fallbackReason = formatProviderError(fallbackAdapter, fallbackErr);
|
||||
const wrapped = new Error(
|
||||
`${reason}\n\nFallback to ${options.fallback} failed: ${fallbackReason}`,
|
||||
) as Error & { cause?: unknown };
|
||||
wrapped.cause = primaryErr;
|
||||
throw wrapped;
|
||||
}
|
||||
}
|
||||
const wrapped = new Error(reason) as Error & { cause?: unknown };
|
||||
wrapped.cause = primaryErr;
|
||||
throw wrapped;
|
||||
}
|
||||
}
|
||||
98
openclaw/extensions/memory-core/src/memory/hybrid.test.ts
Normal file
98
openclaw/extensions/memory-core/src/memory/hybrid.test.ts
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
|
||||
|
||||
describe("memory hybrid helpers", () => {
|
||||
it("buildFtsQuery tokenizes and AND-joins", () => {
|
||||
expect(buildFtsQuery("hello world")).toBe('"hello" AND "world"');
|
||||
expect(buildFtsQuery("FOO_bar baz-1")).toBe('"FOO_bar" AND "baz" AND "1"');
|
||||
expect(buildFtsQuery("金银价格")).toBe('"金银价格"');
|
||||
expect(buildFtsQuery("価格 2026年")).toBe('"価格" AND "2026年"');
|
||||
expect(buildFtsQuery(" ")).toBeNull();
|
||||
});
|
||||
|
||||
it("bm25RankToScore is monotonic and clamped", () => {
|
||||
expect(bm25RankToScore(0)).toBeCloseTo(1);
|
||||
expect(bm25RankToScore(1)).toBeCloseTo(0.5);
|
||||
expect(bm25RankToScore(10)).toBeLessThan(bm25RankToScore(1));
|
||||
expect(bm25RankToScore(-100)).toBeCloseTo(1, 1);
|
||||
});
|
||||
|
||||
it("bm25RankToScore preserves FTS5 BM25 relevance ordering", () => {
|
||||
const strongest = bm25RankToScore(-4.2);
|
||||
const middle = bm25RankToScore(-2.1);
|
||||
const weakest = bm25RankToScore(-0.5);
|
||||
|
||||
expect(strongest).toBeGreaterThan(middle);
|
||||
expect(middle).toBeGreaterThan(weakest);
|
||||
expect(strongest).not.toBe(middle);
|
||||
expect(middle).not.toBe(weakest);
|
||||
});
|
||||
|
||||
it("mergeHybridResults unions by id and combines weighted scores", async () => {
|
||||
const merged = await mergeHybridResults({
|
||||
vectorWeight: 0.7,
|
||||
textWeight: 0.3,
|
||||
vector: [
|
||||
{
|
||||
id: "a",
|
||||
path: "memory/a.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
source: "memory",
|
||||
snippet: "vec-a",
|
||||
vectorScore: 0.9,
|
||||
},
|
||||
],
|
||||
keyword: [
|
||||
{
|
||||
id: "b",
|
||||
path: "memory/b.md",
|
||||
startLine: 3,
|
||||
endLine: 4,
|
||||
source: "memory",
|
||||
snippet: "kw-b",
|
||||
textScore: 1.0,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(merged).toHaveLength(2);
|
||||
const a = merged.find((r) => r.path === "memory/a.md");
|
||||
const b = merged.find((r) => r.path === "memory/b.md");
|
||||
expect(a?.score).toBeCloseTo(0.7 * 0.9);
|
||||
expect(b?.score).toBeCloseTo(0.3 * 1.0);
|
||||
});
|
||||
|
||||
it("mergeHybridResults prefers keyword snippet when ids overlap", async () => {
|
||||
const merged = await mergeHybridResults({
|
||||
vectorWeight: 0.5,
|
||||
textWeight: 0.5,
|
||||
vector: [
|
||||
{
|
||||
id: "a",
|
||||
path: "memory/a.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
source: "memory",
|
||||
snippet: "vec-a",
|
||||
vectorScore: 0.2,
|
||||
},
|
||||
],
|
||||
keyword: [
|
||||
{
|
||||
id: "a",
|
||||
path: "memory/a.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
source: "memory",
|
||||
snippet: "kw-a",
|
||||
textScore: 1.0,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(merged).toHaveLength(1);
|
||||
expect(merged[0]?.snippet).toBe("kw-a");
|
||||
expect(merged[0]?.score).toBeCloseTo(0.5 * 0.2 + 0.5 * 1.0);
|
||||
});
|
||||
});
|
||||
155
openclaw/extensions/memory-core/src/memory/hybrid.ts
Normal file
155
openclaw/extensions/memory-core/src/memory/hybrid.ts
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
import { applyMMRToHybridResults, type MMRConfig, DEFAULT_MMR_CONFIG } from "./mmr.js";
|
||||
import {
|
||||
applyTemporalDecayToHybridResults,
|
||||
type TemporalDecayConfig,
|
||||
DEFAULT_TEMPORAL_DECAY_CONFIG,
|
||||
} from "./temporal-decay.js";
|
||||
|
||||
export type HybridSource = string;
|
||||
|
||||
export { type MMRConfig, DEFAULT_MMR_CONFIG };
|
||||
export { type TemporalDecayConfig, DEFAULT_TEMPORAL_DECAY_CONFIG };
|
||||
|
||||
export type HybridVectorResult = {
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
source: HybridSource;
|
||||
snippet: string;
|
||||
vectorScore: number;
|
||||
};
|
||||
|
||||
export type HybridKeywordResult = {
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
source: HybridSource;
|
||||
snippet: string;
|
||||
textScore: number;
|
||||
};
|
||||
|
||||
export function buildFtsQuery(raw: string): string | null {
|
||||
const tokens =
|
||||
raw
|
||||
.match(/[\p{L}\p{N}_]+/gu)
|
||||
?.map((t) => t.trim())
|
||||
.filter(Boolean) ?? [];
|
||||
if (tokens.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
|
||||
return quoted.join(" AND ");
|
||||
}
|
||||
|
||||
export function bm25RankToScore(rank: number): number {
|
||||
if (!Number.isFinite(rank)) {
|
||||
return 1 / (1 + 999);
|
||||
}
|
||||
if (rank < 0) {
|
||||
const relevance = -rank;
|
||||
return relevance / (1 + relevance);
|
||||
}
|
||||
return 1 / (1 + rank);
|
||||
}
|
||||
|
||||
export async function mergeHybridResults(params: {
|
||||
vector: HybridVectorResult[];
|
||||
keyword: HybridKeywordResult[];
|
||||
vectorWeight: number;
|
||||
textWeight: number;
|
||||
workspaceDir?: string;
|
||||
/** MMR configuration for diversity-aware re-ranking */
|
||||
mmr?: Partial<MMRConfig>;
|
||||
/** Temporal decay configuration for recency-aware scoring */
|
||||
temporalDecay?: Partial<TemporalDecayConfig>;
|
||||
/** Test hook for deterministic time-dependent behavior */
|
||||
nowMs?: number;
|
||||
}): Promise<
|
||||
Array<{
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
snippet: string;
|
||||
source: HybridSource;
|
||||
}>
|
||||
> {
|
||||
const byId = new Map<
|
||||
string,
|
||||
{
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
source: HybridSource;
|
||||
snippet: string;
|
||||
vectorScore: number;
|
||||
textScore: number;
|
||||
}
|
||||
>();
|
||||
|
||||
for (const r of params.vector) {
|
||||
byId.set(r.id, {
|
||||
id: r.id,
|
||||
path: r.path,
|
||||
startLine: r.startLine,
|
||||
endLine: r.endLine,
|
||||
source: r.source,
|
||||
snippet: r.snippet,
|
||||
vectorScore: r.vectorScore,
|
||||
textScore: 0,
|
||||
});
|
||||
}
|
||||
|
||||
for (const r of params.keyword) {
|
||||
const existing = byId.get(r.id);
|
||||
if (existing) {
|
||||
existing.textScore = r.textScore;
|
||||
if (r.snippet && r.snippet.length > 0) {
|
||||
existing.snippet = r.snippet;
|
||||
}
|
||||
} else {
|
||||
byId.set(r.id, {
|
||||
id: r.id,
|
||||
path: r.path,
|
||||
startLine: r.startLine,
|
||||
endLine: r.endLine,
|
||||
source: r.source,
|
||||
snippet: r.snippet,
|
||||
vectorScore: 0,
|
||||
textScore: r.textScore,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const merged = Array.from(byId.values()).map((entry) => {
|
||||
const score = params.vectorWeight * entry.vectorScore + params.textWeight * entry.textScore;
|
||||
return {
|
||||
path: entry.path,
|
||||
startLine: entry.startLine,
|
||||
endLine: entry.endLine,
|
||||
score,
|
||||
snippet: entry.snippet,
|
||||
source: entry.source,
|
||||
};
|
||||
});
|
||||
|
||||
const temporalDecayConfig = { ...DEFAULT_TEMPORAL_DECAY_CONFIG, ...params.temporalDecay };
|
||||
const decayed = await applyTemporalDecayToHybridResults({
|
||||
results: merged,
|
||||
temporalDecay: temporalDecayConfig,
|
||||
workspaceDir: params.workspaceDir,
|
||||
nowMs: params.nowMs,
|
||||
});
|
||||
const sorted = decayed.toSorted((a, b) => b.score - a.score);
|
||||
|
||||
// Apply MMR re-ranking if enabled
|
||||
const mmrConfig = { ...DEFAULT_MMR_CONFIG, ...params.mmr };
|
||||
if (mmrConfig.enabled) {
|
||||
return applyMMRToHybridResults(sorted, mmrConfig);
|
||||
}
|
||||
|
||||
return sorted;
|
||||
}
|
||||
525
openclaw/extensions/memory-core/src/memory/index.test.ts
Normal file
525
openclaw/extensions/memory-core/src/memory/index.test.ts
Normal file
|
|
@ -0,0 +1,525 @@
|
|||
import { mkdirSync, rmSync } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-core";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
clearMemoryEmbeddingProviders as clearRegistry,
|
||||
listMemoryEmbeddingProviders as listRegisteredAdapters,
|
||||
registerMemoryEmbeddingProvider as registerAdapter,
|
||||
} from "../../../../src/plugins/memory-embedding-providers.js";
|
||||
import "./test-runtime-mocks.js";
|
||||
import type { MemoryIndexManager } from "./index.js";
|
||||
import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js";
|
||||
import {
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
registerBuiltInMemoryEmbeddingProviders,
|
||||
} from "./provider-adapters.js";
|
||||
|
||||
let embedBatchCalls = 0;
|
||||
let embedBatchInputCalls = 0;
|
||||
let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = [];
|
||||
let forceNoProvider = false;
|
||||
|
||||
vi.mock("./embeddings.js", () => {
|
||||
const embedText = (text: string) => {
|
||||
const lower = text.toLowerCase();
|
||||
const alpha = lower.split("alpha").length - 1;
|
||||
const beta = lower.split("beta").length - 1;
|
||||
const image = lower.split("image").length - 1;
|
||||
const audio = lower.split("audio").length - 1;
|
||||
return [alpha, beta, image, audio];
|
||||
};
|
||||
return {
|
||||
createEmbeddingProvider: async (options: {
|
||||
provider?: string;
|
||||
model?: string;
|
||||
outputDimensionality?: number;
|
||||
}) => {
|
||||
providerCalls.push({
|
||||
provider: options.provider,
|
||||
model: options.model,
|
||||
outputDimensionality: options.outputDimensionality,
|
||||
});
|
||||
if (forceNoProvider) {
|
||||
return {
|
||||
provider: null,
|
||||
requestedProvider: options.provider ?? "auto",
|
||||
providerUnavailableReason: "No API key found for provider",
|
||||
};
|
||||
}
|
||||
const providerId = options.provider === "gemini" ? "gemini" : "mock";
|
||||
const model = options.model ?? "mock-embed";
|
||||
return {
|
||||
requestedProvider: options.provider ?? "openai",
|
||||
provider: {
|
||||
id: providerId,
|
||||
model,
|
||||
embedQuery: async (text: string) => embedText(text),
|
||||
embedBatch: async (texts: string[]) => {
|
||||
embedBatchCalls += 1;
|
||||
return texts.map(embedText);
|
||||
},
|
||||
...(providerId === "gemini"
|
||||
? {
|
||||
embedBatchInputs: async (
|
||||
inputs: Array<{
|
||||
text: string;
|
||||
parts?: Array<
|
||||
| { type: "text"; text: string }
|
||||
| { type: "inline-data"; mimeType: string; data: string }
|
||||
>;
|
||||
}>,
|
||||
) => {
|
||||
embedBatchInputCalls += 1;
|
||||
return inputs.map((input) => {
|
||||
const inlineData = input.parts?.find((part) => part.type === "inline-data");
|
||||
if (inlineData?.type === "inline-data" && inlineData.data.length > 9000) {
|
||||
throw new Error("payload too large");
|
||||
}
|
||||
const mimeType =
|
||||
inlineData?.type === "inline-data" ? inlineData.mimeType : undefined;
|
||||
if (mimeType?.startsWith("image/")) {
|
||||
return [0, 0, 1, 0];
|
||||
}
|
||||
if (mimeType?.startsWith("audio/")) {
|
||||
return [0, 0, 0, 1];
|
||||
}
|
||||
return embedText(input.text);
|
||||
});
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
...(providerId === "gemini"
|
||||
? {
|
||||
runtime: {
|
||||
id: "gemini",
|
||||
cacheKeyData: {
|
||||
provider: "gemini",
|
||||
baseUrl: "https://generativelanguage.googleapis.com/v1beta",
|
||||
model,
|
||||
outputDimensionality: options.outputDimensionality,
|
||||
headers: [],
|
||||
},
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
describe("memory index", () => {
|
||||
it("registers the builtin local embedding provider", () => {
|
||||
const adapter = listRegisteredAdapters().find((entry) => entry.id === "local");
|
||||
|
||||
expect(adapter).toBeDefined();
|
||||
expect(adapter).toEqual(
|
||||
expect.objectContaining({
|
||||
id: "local",
|
||||
defaultModel: DEFAULT_LOCAL_MODEL,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
let fixtureRoot = "";
|
||||
let workspaceDir = "";
|
||||
let memoryDir = "";
|
||||
let indexVectorPath = "";
|
||||
let indexMainPath = "";
|
||||
let indexMultimodalPath = "";
|
||||
|
||||
const managersForCleanup = new Set<MemoryIndexManager>();
|
||||
|
||||
beforeAll(async () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-"));
|
||||
workspaceDir = path.join(fixtureRoot, "workspace");
|
||||
memoryDir = path.join(workspaceDir, "memory");
|
||||
indexMainPath = path.join(workspaceDir, "index-main.sqlite");
|
||||
indexVectorPath = path.join(workspaceDir, "index-vector.sqlite");
|
||||
indexMultimodalPath = path.join(workspaceDir, "index-multimodal.sqlite");
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close()));
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await closeAllMemorySearchManagers();
|
||||
clearRegistry();
|
||||
managersForCleanup.clear();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Perf: most suites don't need atomic swap behavior for full reindexes.
|
||||
// Keep atomic reindex tests on the safe path.
|
||||
vi.stubEnv("OPENCLAW_TEST_MEMORY_UNSAFE_REINDEX", "1");
|
||||
clearRegistry();
|
||||
registerBuiltInMemoryEmbeddingProviders({ registerMemoryEmbeddingProvider: registerAdapter });
|
||||
embedBatchCalls = 0;
|
||||
embedBatchInputCalls = 0;
|
||||
providerCalls = [];
|
||||
forceNoProvider = false;
|
||||
|
||||
rmSync(workspaceDir, { recursive: true, force: true });
|
||||
mkdirSync(memoryDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(memoryDir, "2026-01-12.md"),
|
||||
"# Log\nAlpha memory line.\nZebra memory line.",
|
||||
);
|
||||
});
|
||||
|
||||
function resetManagerForTest(manager: MemoryIndexManager) {
|
||||
// These tests reuse managers for performance. Clear the index + embedding
|
||||
// cache to keep each test fully isolated.
|
||||
const db = (
|
||||
manager as unknown as {
|
||||
db: {
|
||||
exec: (sql: string) => void;
|
||||
prepare: (sql: string) => { get: (name: string) => { name?: string } | undefined };
|
||||
};
|
||||
}
|
||||
).db;
|
||||
(manager as unknown as { resetIndex: () => void }).resetIndex();
|
||||
const embeddingCacheTable = db
|
||||
.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?")
|
||||
.get("embedding_cache");
|
||||
if (embeddingCacheTable?.name === "embedding_cache") {
|
||||
db.exec("DELETE FROM embedding_cache");
|
||||
}
|
||||
(manager as unknown as { dirty: boolean }).dirty = true;
|
||||
(manager as unknown as { sessionsDirty: boolean }).sessionsDirty = false;
|
||||
}
|
||||
|
||||
type TestCfg = Parameters<typeof getMemorySearchManager>[0]["cfg"];
|
||||
|
||||
function createCfg(params: {
|
||||
storePath: string;
|
||||
extraPaths?: string[];
|
||||
sources?: Array<"memory" | "sessions">;
|
||||
sessionMemory?: boolean;
|
||||
provider?: "openai" | "gemini";
|
||||
model?: string;
|
||||
outputDimensionality?: number;
|
||||
multimodal?: {
|
||||
enabled?: boolean;
|
||||
modalities?: Array<"image" | "audio" | "all">;
|
||||
maxFileBytes?: number;
|
||||
};
|
||||
vectorEnabled?: boolean;
|
||||
cacheEnabled?: boolean;
|
||||
minScore?: number;
|
||||
onSearch?: boolean;
|
||||
hybrid?: { enabled: boolean; vectorWeight?: number; textWeight?: number };
|
||||
}): TestCfg {
|
||||
return {
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: workspaceDir,
|
||||
memorySearch: {
|
||||
provider: params.provider ?? "openai",
|
||||
model: params.model ?? "mock-embed",
|
||||
outputDimensionality: params.outputDimensionality,
|
||||
store: { path: params.storePath, vector: { enabled: params.vectorEnabled ?? false } },
|
||||
// Perf: keep test indexes to a single chunk to reduce sqlite work.
|
||||
chunking: { tokens: 4000, overlap: 0 },
|
||||
sync: { watch: false, onSessionStart: false, onSearch: params.onSearch ?? true },
|
||||
query: {
|
||||
minScore: params.minScore ?? 0,
|
||||
hybrid: params.hybrid ?? { enabled: false },
|
||||
},
|
||||
cache: params.cacheEnabled ? { enabled: true } : undefined,
|
||||
extraPaths: params.extraPaths,
|
||||
multimodal: params.multimodal,
|
||||
sources: params.sources,
|
||||
experimental: { sessionMemory: params.sessionMemory ?? false },
|
||||
},
|
||||
},
|
||||
list: [{ id: "main", default: true }],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function requireManager(
|
||||
result: Awaited<ReturnType<typeof getMemorySearchManager>>,
|
||||
missingMessage = "manager missing",
|
||||
): MemoryIndexManager {
|
||||
expect(result.manager).not.toBeNull();
|
||||
if (!result.manager) {
|
||||
throw new Error(missingMessage);
|
||||
}
|
||||
return result.manager as MemoryIndexManager;
|
||||
}
|
||||
|
||||
async function getPersistentManager(cfg: TestCfg): Promise<MemoryIndexManager> {
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
const manager = requireManager(result);
|
||||
managersForCleanup.add(manager);
|
||||
resetManagerForTest(manager);
|
||||
return manager;
|
||||
}
|
||||
|
||||
async function getFreshManager(cfg: TestCfg): Promise<MemoryIndexManager> {
|
||||
const { getRequiredMemoryIndexManager } = await import("./test-manager-helpers.js");
|
||||
return await getRequiredMemoryIndexManager({ cfg, agentId: "main" });
|
||||
}
|
||||
|
||||
async function expectHybridKeywordSearchFindsMemory(cfg: TestCfg) {
|
||||
const manager = await getFreshManager(cfg);
|
||||
try {
|
||||
const status = manager.status();
|
||||
if (!status.fts?.available) {
|
||||
return;
|
||||
}
|
||||
|
||||
await manager.sync({ reason: "test" });
|
||||
const results = await manager.search("zebra");
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results[0]?.path).toContain("memory/2026-01-12.md");
|
||||
} finally {
|
||||
await manager.close?.();
|
||||
}
|
||||
}
|
||||
|
||||
it.skip("indexes memory files and searches", async () => {
|
||||
const cfg = createCfg({
|
||||
storePath: indexMainPath,
|
||||
hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 },
|
||||
});
|
||||
const manager = await getFreshManager(cfg);
|
||||
try {
|
||||
await manager.sync({ reason: "test" });
|
||||
const results = await manager.search("alpha");
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results[0]?.path).toContain("memory/2026-01-12.md");
|
||||
const status = manager.status();
|
||||
expect(status.sourceCounts).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
source: "memory",
|
||||
files: status.files,
|
||||
chunks: status.chunks,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
} finally {
|
||||
await manager.close?.();
|
||||
}
|
||||
});
|
||||
|
||||
it("indexes multimodal image and audio files from extra paths with Gemini structured inputs", async () => {
|
||||
const mediaDir = path.join(workspaceDir, "media-memory");
|
||||
await fs.mkdir(mediaDir, { recursive: true });
|
||||
await fs.writeFile(path.join(mediaDir, "diagram.png"), Buffer.from("png"));
|
||||
await fs.writeFile(path.join(mediaDir, "meeting.wav"), Buffer.from("wav"));
|
||||
|
||||
const cfg = createCfg({
|
||||
storePath: indexMultimodalPath,
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
extraPaths: [mediaDir],
|
||||
multimodal: { enabled: true, modalities: ["image", "audio"] },
|
||||
});
|
||||
const manager = await getPersistentManager(cfg);
|
||||
await manager.sync({ reason: "test" });
|
||||
|
||||
expect(embedBatchInputCalls).toBeGreaterThan(0);
|
||||
|
||||
const imageResults = await manager.search("image");
|
||||
expect(imageResults.some((result) => result.path.endsWith("diagram.png"))).toBe(true);
|
||||
|
||||
const audioResults = await manager.search("audio");
|
||||
expect(audioResults.some((result) => result.path.endsWith("meeting.wav"))).toBe(true);
|
||||
});
|
||||
|
||||
it.skip("finds keyword matches via hybrid search when query embedding is zero", async () => {
|
||||
await expectHybridKeywordSearchFindsMemory(
|
||||
createCfg({
|
||||
storePath: indexMainPath,
|
||||
hybrid: { enabled: true, vectorWeight: 0, textWeight: 1 },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.skip("preserves keyword-only hybrid hits when minScore exceeds text weight", async () => {
|
||||
await expectHybridKeywordSearchFindsMemory(
|
||||
createCfg({
|
||||
storePath: indexMainPath,
|
||||
minScore: 0.35,
|
||||
hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("reports vector availability after probe", async () => {
|
||||
const cfg = createCfg({ storePath: indexVectorPath, vectorEnabled: true });
|
||||
const manager = await getPersistentManager(cfg);
|
||||
const available = await manager.probeVectorAvailability();
|
||||
const status = manager.status();
|
||||
expect(status.vector?.enabled).toBe(true);
|
||||
expect(typeof status.vector?.available).toBe("boolean");
|
||||
expect(status.vector?.available).toBe(available);
|
||||
});
|
||||
|
||||
it("builds FTS index and returns search results when no embedding provider is available", async () => {
|
||||
forceNoProvider = true;
|
||||
|
||||
const cfg = createCfg({
|
||||
storePath: path.join(workspaceDir, "index-fts-only.sqlite"),
|
||||
minScore: 0.35,
|
||||
hybrid: { enabled: true },
|
||||
});
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
const manager = requireManager(result);
|
||||
managersForCleanup.add(manager);
|
||||
resetManagerForTest(manager);
|
||||
if (!manager.status().fts?.available) {
|
||||
return;
|
||||
}
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(memoryDir, "2026-01-12.md"),
|
||||
"# Log\nAlpha memory line.\nZebra memory line.",
|
||||
);
|
||||
await manager.sync({ reason: "test" });
|
||||
|
||||
const status = manager.status();
|
||||
expect(status.chunks).toBeGreaterThan(0);
|
||||
expect(embedBatchCalls).toBe(0);
|
||||
|
||||
const results = await manager.search("Alpha");
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
expect(results[0]?.snippet).toMatch(/Alpha/i);
|
||||
|
||||
const noResults = await manager.search("nonexistent_xyz_keyword");
|
||||
expect(noResults.length).toBe(0);
|
||||
});
|
||||
|
||||
it("prefers exact session transcript hits in FTS-only mode", async () => {
|
||||
forceNoProvider = true;
|
||||
const stateDir = path.join(workspaceDir, ".state-session-ranking");
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
try {
|
||||
const cfg = createCfg({
|
||||
storePath: path.join(workspaceDir, "index-fts-session-ranking.sqlite"),
|
||||
sources: ["memory", "sessions"],
|
||||
sessionMemory: true,
|
||||
minScore: 0,
|
||||
hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 },
|
||||
});
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
const manager = requireManager(result);
|
||||
managersForCleanup.add(manager);
|
||||
resetManagerForTest(manager);
|
||||
if (!manager.status().fts?.available) {
|
||||
return;
|
||||
}
|
||||
|
||||
const memoryPath = path.join(workspaceDir, "MEMORY.md");
|
||||
await fs.writeFile(memoryPath, "Project Nebula stale codename: ORBIT-9.\n", "utf8");
|
||||
const staleAt = new Date("2020-01-01T00:00:00.000Z");
|
||||
await fs.utimes(memoryPath, staleAt, staleAt);
|
||||
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent("main");
|
||||
await fs.mkdir(sessionsDir, { recursive: true });
|
||||
const transcriptPath = path.join(sessionsDir, "session-ranking.jsonl");
|
||||
const now = Date.parse("2026-04-07T15:25:04.113Z");
|
||||
await fs.writeFile(
|
||||
transcriptPath,
|
||||
[
|
||||
JSON.stringify({
|
||||
type: "session",
|
||||
id: "session-ranking",
|
||||
timestamp: new Date(now - 60_000).toISOString(),
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "user",
|
||||
timestamp: new Date(now - 30_000).toISOString(),
|
||||
content: [{ type: "text", text: "What is the current Project Nebula codename?" }],
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
timestamp: new Date(now).toISOString(),
|
||||
content: [{ type: "text", text: "The current Project Nebula codename is ORBIT-10." }],
|
||||
},
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await manager.sync({ reason: "test", force: true });
|
||||
const results = await manager.search("current Project Nebula codename ORBIT-10", {
|
||||
minScore: 0,
|
||||
maxResults: 3,
|
||||
});
|
||||
|
||||
expect(results[0]?.source).toBe("sessions");
|
||||
expect(results[0]?.snippet).toContain("ORBIT-10");
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
|
||||
it("bootstraps an empty index on first search so session transcript hits are available", async () => {
|
||||
forceNoProvider = true;
|
||||
const stateDir = path.join(workspaceDir, ".state-session-bootstrap");
|
||||
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
|
||||
try {
|
||||
const cfg = createCfg({
|
||||
storePath: path.join(workspaceDir, "index-fts-session-bootstrap.sqlite"),
|
||||
sources: ["memory", "sessions"],
|
||||
sessionMemory: true,
|
||||
minScore: 0,
|
||||
hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 },
|
||||
});
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
const manager = requireManager(result);
|
||||
managersForCleanup.add(manager);
|
||||
resetManagerForTest(manager);
|
||||
if (!manager.status().fts?.available) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent("main");
|
||||
await fs.mkdir(sessionsDir, { recursive: true });
|
||||
const transcriptPath = path.join(sessionsDir, "session-bootstrap.jsonl");
|
||||
await fs.writeFile(
|
||||
transcriptPath,
|
||||
[
|
||||
JSON.stringify({
|
||||
type: "session",
|
||||
id: "session-bootstrap",
|
||||
timestamp: "2026-04-07T15:24:04.113Z",
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: "message",
|
||||
message: {
|
||||
role: "assistant",
|
||||
timestamp: "2026-04-07T15:25:04.113Z",
|
||||
content: [{ type: "text", text: "The current Project Nebula codename is ORBIT-10." }],
|
||||
},
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const results = await manager.search("current Project Nebula codename ORBIT-10", {
|
||||
minScore: 0,
|
||||
maxResults: 3,
|
||||
});
|
||||
|
||||
expect(results[0]?.source).toBe("sessions");
|
||||
expect(results[0]?.snippet).toContain("ORBIT-10");
|
||||
} finally {
|
||||
vi.unstubAllEnvs();
|
||||
}
|
||||
});
|
||||
});
|
||||
11
openclaw/extensions/memory-core/src/memory/index.ts
Normal file
11
openclaw/extensions/memory-core/src/memory/index.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
export { MemoryIndexManager } from "./manager.js";
|
||||
export type {
|
||||
MemoryEmbeddingProbeResult,
|
||||
MemorySearchManager,
|
||||
MemorySearchResult,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
export {
|
||||
closeAllMemorySearchManagers,
|
||||
getMemorySearchManager,
|
||||
type MemorySearchManagerResult,
|
||||
} from "./search-manager.js";
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
export function startAsyncSearchSync(params: {
|
||||
enabled: boolean;
|
||||
dirty: boolean;
|
||||
sessionsDirty: boolean;
|
||||
sync: (params: { reason: string }) => Promise<void>;
|
||||
onError: (err: unknown) => void;
|
||||
}): void {
|
||||
if (!params.enabled || (!params.dirty && !params.sessionsDirty)) {
|
||||
return;
|
||||
}
|
||||
void params.sync({ reason: "search" }).catch((err) => {
|
||||
params.onError(err);
|
||||
});
|
||||
}
|
||||
|
||||
export async function awaitPendingManagerWork(params: {
|
||||
pendingSync?: Promise<void> | null;
|
||||
pendingProviderInit?: Promise<void> | null;
|
||||
}): Promise<void> {
|
||||
if (params.pendingSync) {
|
||||
try {
|
||||
await params.pendingSync;
|
||||
} catch {}
|
||||
}
|
||||
if (params.pendingProviderInit) {
|
||||
try {
|
||||
await params.pendingProviderInit;
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
export async function moveMemoryIndexFiles(sourceBase: string, targetBase: string): Promise<void> {
|
||||
const suffixes = ["", "-wal", "-shm"];
|
||||
for (const suffix of suffixes) {
|
||||
const source = `${sourceBase}${suffix}`;
|
||||
const target = `${targetBase}${suffix}`;
|
||||
try {
|
||||
await fs.rename(source, target);
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function removeMemoryIndexFiles(basePath: string): Promise<void> {
|
||||
const suffixes = ["", "-wal", "-shm"];
|
||||
await Promise.all(suffixes.map((suffix) => fs.rm(`${basePath}${suffix}`, { force: true })));
|
||||
}
|
||||
|
||||
export async function swapMemoryIndexFiles(targetPath: string, tempPath: string): Promise<void> {
|
||||
const backupPath = `${targetPath}.backup-${randomUUID()}`;
|
||||
await moveMemoryIndexFiles(targetPath, backupPath);
|
||||
try {
|
||||
await moveMemoryIndexFiles(tempPath, targetPath);
|
||||
} catch (err) {
|
||||
await moveMemoryIndexFiles(backupPath, targetPath);
|
||||
throw err;
|
||||
}
|
||||
await removeMemoryIndexFiles(backupPath);
|
||||
}
|
||||
|
||||
export async function runMemoryAtomicReindex<T>(params: {
|
||||
targetPath: string;
|
||||
tempPath: string;
|
||||
build: () => Promise<T>;
|
||||
}): Promise<T> {
|
||||
try {
|
||||
const result = await params.build();
|
||||
await swapMemoryIndexFiles(params.targetPath, params.tempPath);
|
||||
return result;
|
||||
} catch (err) {
|
||||
await removeMemoryIndexFiles(params.tempPath);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
MEMORY_BATCH_FAILURE_LIMIT,
|
||||
recordMemoryBatchFailure,
|
||||
resetMemoryBatchFailureState,
|
||||
} from "./manager-batch-state.js";
|
||||
|
||||
describe("memory batch state", () => {
|
||||
it("resets failures after recovery", () => {
|
||||
expect(
|
||||
resetMemoryBatchFailureState({
|
||||
enabled: true,
|
||||
count: 1,
|
||||
lastError: "batch failed",
|
||||
lastProvider: "openai",
|
||||
}),
|
||||
).toEqual({
|
||||
enabled: true,
|
||||
count: 0,
|
||||
lastError: undefined,
|
||||
lastProvider: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("disables batching after repeated failures", () => {
|
||||
const once = recordMemoryBatchFailure(
|
||||
{ enabled: true, count: 0 },
|
||||
{ provider: "openai", message: "batch failed", attempts: 1 },
|
||||
);
|
||||
expect(once).toEqual({
|
||||
enabled: true,
|
||||
count: 1,
|
||||
lastError: "batch failed",
|
||||
lastProvider: "openai",
|
||||
});
|
||||
|
||||
const twice = recordMemoryBatchFailure(once, {
|
||||
provider: "openai",
|
||||
message: "batch failed again",
|
||||
attempts: 1,
|
||||
});
|
||||
expect(twice).toEqual({
|
||||
enabled: false,
|
||||
count: MEMORY_BATCH_FAILURE_LIMIT,
|
||||
lastError: "batch failed again",
|
||||
lastProvider: "openai",
|
||||
});
|
||||
});
|
||||
|
||||
it("force-disables batching immediately", () => {
|
||||
expect(
|
||||
recordMemoryBatchFailure(
|
||||
{ enabled: true, count: 0 },
|
||||
{ provider: "gemini", message: "not available", forceDisable: true },
|
||||
),
|
||||
).toEqual({
|
||||
enabled: false,
|
||||
count: MEMORY_BATCH_FAILURE_LIMIT,
|
||||
lastError: "not available",
|
||||
lastProvider: "gemini",
|
||||
});
|
||||
});
|
||||
|
||||
it("leaves disabled state unchanged", () => {
|
||||
expect(
|
||||
recordMemoryBatchFailure(
|
||||
{ enabled: false, count: MEMORY_BATCH_FAILURE_LIMIT, lastError: "x", lastProvider: "y" },
|
||||
{ provider: "openai", message: "ignored" },
|
||||
),
|
||||
).toEqual({
|
||||
enabled: false,
|
||||
count: MEMORY_BATCH_FAILURE_LIMIT,
|
||||
lastError: "x",
|
||||
lastProvider: "y",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
export const MEMORY_BATCH_FAILURE_LIMIT = 2;
|
||||
|
||||
export type MemoryBatchFailureState = {
|
||||
enabled: boolean;
|
||||
count: number;
|
||||
lastError?: string;
|
||||
lastProvider?: string;
|
||||
};
|
||||
|
||||
export function resetMemoryBatchFailureState(
|
||||
state: MemoryBatchFailureState,
|
||||
): MemoryBatchFailureState {
|
||||
return {
|
||||
...state,
|
||||
count: 0,
|
||||
lastError: undefined,
|
||||
lastProvider: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function recordMemoryBatchFailure(
|
||||
state: MemoryBatchFailureState,
|
||||
params: {
|
||||
provider: string;
|
||||
message: string;
|
||||
attempts?: number;
|
||||
forceDisable?: boolean;
|
||||
},
|
||||
): MemoryBatchFailureState {
|
||||
if (!state.enabled) {
|
||||
return state;
|
||||
}
|
||||
const increment = params.forceDisable
|
||||
? MEMORY_BATCH_FAILURE_LIMIT
|
||||
: Math.max(1, params.attempts ?? 1);
|
||||
const count = state.count + increment;
|
||||
const enabled = !(params.forceDisable || count >= MEMORY_BATCH_FAILURE_LIMIT);
|
||||
return {
|
||||
enabled,
|
||||
count,
|
||||
lastError: params.message,
|
||||
lastProvider: params.provider,
|
||||
};
|
||||
}
|
||||
145
openclaw/extensions/memory-core/src/memory/manager-cache.test.ts
Normal file
145
openclaw/extensions/memory-core/src/memory/manager-cache.test.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
closeManagedCacheEntries,
|
||||
getOrCreateManagedCacheEntry,
|
||||
resolveSingletonManagedCache,
|
||||
type ManagedCache,
|
||||
} from "./manager-cache.js";
|
||||
|
||||
type TestEntry = {
|
||||
id: string;
|
||||
close: () => Promise<void>;
|
||||
};
|
||||
|
||||
function createTestCache(): ManagedCache<TestEntry> {
|
||||
return resolveSingletonManagedCache<TestEntry>(Symbol("openclaw.manager-cache.test"));
|
||||
}
|
||||
|
||||
function createEntry(id: string): TestEntry {
|
||||
return {
|
||||
id,
|
||||
close: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
function createDeferred<T>() {
|
||||
let resolve!: (value: T | PromiseLike<T>) => void;
|
||||
let reject!: (reason?: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
describe("manager cache", () => {
|
||||
const cachesForCleanup: ManagedCache<TestEntry>[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
cachesForCleanup.splice(0).map((cache) =>
|
||||
closeManagedCacheEntries({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
}),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it("deduplicates concurrent creation for the same cache key", async () => {
|
||||
const cache = createTestCache();
|
||||
cachesForCleanup.push(cache);
|
||||
let createCalls = 0;
|
||||
|
||||
const results = await Promise.all(
|
||||
Array.from(
|
||||
{ length: 12 },
|
||||
async () =>
|
||||
await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
create: async () => {
|
||||
createCalls += 1;
|
||||
await Promise.resolve();
|
||||
return createEntry("shared");
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
expect(results).toHaveLength(12);
|
||||
expect(new Set(results).size).toBe(1);
|
||||
expect(createCalls).toBe(1);
|
||||
});
|
||||
|
||||
it("waits for pending creation before global teardown closes cached entries", async () => {
|
||||
const cache = createTestCache();
|
||||
const first = createEntry("first");
|
||||
const second = createEntry("second");
|
||||
cachesForCleanup.push(cache);
|
||||
const gate = createDeferred<void>();
|
||||
|
||||
const pendingFirst = getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
create: async () => {
|
||||
await gate.promise;
|
||||
return first;
|
||||
},
|
||||
});
|
||||
|
||||
const teardown = closeManagedCacheEntries({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
});
|
||||
gate.resolve();
|
||||
|
||||
await teardown;
|
||||
expect(first.close).toHaveBeenCalledTimes(1);
|
||||
|
||||
const resolvedFirst = await pendingFirst;
|
||||
const resolvedSecond = await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
create: async () => second,
|
||||
});
|
||||
|
||||
expect(resolvedFirst).toBe(first);
|
||||
expect(resolvedSecond).toBe(second);
|
||||
expect(resolvedSecond).not.toBe(resolvedFirst);
|
||||
});
|
||||
|
||||
it("bypasses identity caching for status-only callers", async () => {
|
||||
const cache = createTestCache();
|
||||
cachesForCleanup.push(cache);
|
||||
let createCalls = 0;
|
||||
|
||||
const first = await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
bypassCache: true,
|
||||
create: async () => {
|
||||
createCalls += 1;
|
||||
return createEntry(`status-${createCalls}`);
|
||||
},
|
||||
});
|
||||
const second = await getOrCreateManagedCacheEntry({
|
||||
cache: cache.cache,
|
||||
pending: cache.pending,
|
||||
key: "same",
|
||||
bypassCache: true,
|
||||
create: async () => {
|
||||
createCalls += 1;
|
||||
return createEntry(`status-${createCalls}`);
|
||||
},
|
||||
});
|
||||
|
||||
expect(first).not.toBe(second);
|
||||
expect(createCalls).toBe(2);
|
||||
expect(cache.cache.size).toBe(0);
|
||||
});
|
||||
});
|
||||
77
openclaw/extensions/memory-core/src/memory/manager-cache.ts
Normal file
77
openclaw/extensions/memory-core/src/memory/manager-cache.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import { resolveGlobalSingleton } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
|
||||
type Closable = {
|
||||
close?: () => Promise<void> | void;
|
||||
};
|
||||
|
||||
export type ManagedCache<T> = {
|
||||
cache: Map<string, T>;
|
||||
pending: Map<string, Promise<T>>;
|
||||
};
|
||||
|
||||
export function resolveSingletonManagedCache<T>(cacheKey: symbol): ManagedCache<T> {
|
||||
return resolveGlobalSingleton<ManagedCache<T>>(cacheKey, () => ({
|
||||
cache: new Map<string, T>(),
|
||||
pending: new Map<string, Promise<T>>(),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function getOrCreateManagedCacheEntry<T>(params: {
|
||||
cache: Map<string, T>;
|
||||
pending: Map<string, Promise<T>>;
|
||||
key: string;
|
||||
bypassCache?: boolean;
|
||||
create: () => Promise<T> | T;
|
||||
}): Promise<T> {
|
||||
if (params.bypassCache) {
|
||||
return await params.create();
|
||||
}
|
||||
const existing = params.cache.get(params.key);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const pending = params.pending.get(params.key);
|
||||
if (pending) {
|
||||
return pending;
|
||||
}
|
||||
const createPromise = (async () => {
|
||||
const refreshed = params.cache.get(params.key);
|
||||
if (refreshed) {
|
||||
return refreshed;
|
||||
}
|
||||
const entry = await params.create();
|
||||
params.cache.set(params.key, entry);
|
||||
return entry;
|
||||
})();
|
||||
params.pending.set(params.key, createPromise);
|
||||
try {
|
||||
return await createPromise;
|
||||
} finally {
|
||||
if (params.pending.get(params.key) === createPromise) {
|
||||
params.pending.delete(params.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function closeManagedCacheEntries<T extends Closable>(params: {
|
||||
cache: Map<string, T>;
|
||||
pending: Map<string, Promise<T>>;
|
||||
onCloseError?: (err: unknown) => void;
|
||||
}): Promise<void> {
|
||||
const pending = Array.from(params.pending.values());
|
||||
if (pending.length > 0) {
|
||||
await Promise.allSettled(pending);
|
||||
}
|
||||
const entries = Array.from(params.cache.values());
|
||||
params.cache.clear();
|
||||
for (const entry of entries) {
|
||||
if (typeof entry.close !== "function") {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await entry.close();
|
||||
} catch (err) {
|
||||
params.onCloseError?.(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
15
openclaw/extensions/memory-core/src/memory/manager-db.ts
Normal file
15
openclaw/extensions/memory-core/src/memory/manager-db.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { ensureDir, requireNodeSqlite } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export function openMemoryDatabaseAtPath(dbPath: string, allowExtension: boolean): DatabaseSync {
|
||||
const dir = path.dirname(dbPath);
|
||||
ensureDir(dir);
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const db = new DatabaseSync(dbPath, { allowExtension });
|
||||
// busy_timeout is per-connection and resets to 0 on restart.
|
||||
// Set it on every open so concurrent processes retry instead of
|
||||
// failing immediately with SQLITE_BUSY.
|
||||
db.exec("PRAGMA busy_timeout = 5000");
|
||||
return db;
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
import {
|
||||
ensureMemoryIndexSchema,
|
||||
requireNodeSqlite,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
collectMemoryCachedEmbeddings,
|
||||
loadMemoryEmbeddingCache,
|
||||
upsertMemoryEmbeddingCache,
|
||||
} from "./manager-embedding-cache.js";
|
||||
|
||||
describe("memory embedding cache", () => {
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
|
||||
function createDb() {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
ensureMemoryIndexSchema({
|
||||
db,
|
||||
embeddingCacheTable: "embedding_cache",
|
||||
cacheEnabled: true,
|
||||
ftsTable: "chunks_fts",
|
||||
ftsEnabled: false,
|
||||
ftsTokenizer: "unicode61",
|
||||
});
|
||||
return db;
|
||||
}
|
||||
|
||||
it("loads cached embeddings for the active provider key", () => {
|
||||
const db = createDb();
|
||||
try {
|
||||
upsertMemoryEmbeddingCache({
|
||||
db,
|
||||
enabled: true,
|
||||
provider: { id: "openai", model: "text-embedding-3-small" },
|
||||
providerKey: "provider-key",
|
||||
entries: [
|
||||
{ hash: "a", embedding: [0.1, 0.2] },
|
||||
{ hash: "b", embedding: [0.3, 0.4] },
|
||||
],
|
||||
now: 123,
|
||||
});
|
||||
|
||||
const cached = loadMemoryEmbeddingCache({
|
||||
db,
|
||||
enabled: true,
|
||||
provider: { id: "openai", model: "text-embedding-3-small" },
|
||||
providerKey: "provider-key",
|
||||
hashes: ["a", "b", "a"],
|
||||
});
|
||||
|
||||
expect(cached).toEqual(
|
||||
new Map([
|
||||
["a", [0.1, 0.2]],
|
||||
["b", [0.3, 0.4]],
|
||||
]),
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("reuses cached embeddings on forced reindex instead of scheduling new embeds", () => {
|
||||
const cached = new Map<string, number[]>([
|
||||
["alpha", [0.1, 0.2]],
|
||||
["beta", [0.3, 0.4]],
|
||||
]);
|
||||
const embedMissing = vi.fn();
|
||||
|
||||
const plan = collectMemoryCachedEmbeddings({
|
||||
chunks: [{ hash: "alpha" }, { hash: "beta" }],
|
||||
cached,
|
||||
});
|
||||
|
||||
if (plan.missing.length > 0) {
|
||||
embedMissing(plan.missing);
|
||||
}
|
||||
|
||||
expect(plan.embeddings).toEqual([
|
||||
[0.1, 0.2],
|
||||
[0.3, 0.4],
|
||||
]);
|
||||
expect(plan.missing).toHaveLength(0);
|
||||
expect(embedMissing).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
import type { DatabaseSync, SQLInputValue } from "node:sqlite";
|
||||
import {
|
||||
parseEmbedding,
|
||||
type MemoryChunk,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
type EmbeddingCacheDb = Pick<DatabaseSync, "prepare">;
|
||||
|
||||
type EmbeddingProviderRef = {
|
||||
id: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
export function loadMemoryEmbeddingCache(params: {
|
||||
db: EmbeddingCacheDb;
|
||||
enabled: boolean;
|
||||
provider: EmbeddingProviderRef | null;
|
||||
providerKey: string | null;
|
||||
hashes: string[];
|
||||
tableName?: string;
|
||||
}): Map<string, number[]> {
|
||||
const provider = params.provider;
|
||||
if (!params.enabled || !provider || !params.providerKey || params.hashes.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
const unique: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const hash of params.hashes) {
|
||||
if (!hash || seen.has(hash)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(hash);
|
||||
unique.push(hash);
|
||||
}
|
||||
if (unique.length === 0) {
|
||||
return new Map();
|
||||
}
|
||||
|
||||
const tableName = params.tableName ?? "embedding_cache";
|
||||
const out = new Map<string, number[]>();
|
||||
const baseParams: SQLInputValue[] = [provider.id, provider.model, params.providerKey];
|
||||
const batchSize = 400;
|
||||
for (let start = 0; start < unique.length; start += batchSize) {
|
||||
const batch = unique.slice(start, start + batchSize);
|
||||
const placeholders = batch.map(() => "?").join(", ");
|
||||
const rows = params.db
|
||||
.prepare(
|
||||
`SELECT hash, embedding FROM ${tableName}\n` +
|
||||
` WHERE provider = ? AND model = ? AND provider_key = ? AND hash IN (${placeholders})`,
|
||||
)
|
||||
.all(...baseParams, ...batch) as Array<{ hash: string; embedding: string }>;
|
||||
for (const row of rows) {
|
||||
out.set(row.hash, parseEmbedding(row.embedding));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function upsertMemoryEmbeddingCache(params: {
|
||||
db: EmbeddingCacheDb;
|
||||
enabled: boolean;
|
||||
provider: EmbeddingProviderRef | null;
|
||||
providerKey: string | null;
|
||||
entries: Array<{ hash: string; embedding: number[] }>;
|
||||
now?: number;
|
||||
tableName?: string;
|
||||
}): void {
|
||||
const provider = params.provider;
|
||||
if (!params.enabled || !provider || !params.providerKey || params.entries.length === 0) {
|
||||
return;
|
||||
}
|
||||
const tableName = params.tableName ?? "embedding_cache";
|
||||
const now = params.now ?? Date.now();
|
||||
const stmt = params.db.prepare(
|
||||
`INSERT INTO ${tableName} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` +
|
||||
` VALUES (?, ?, ?, ?, ?, ?, ?)\n` +
|
||||
` ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET\n` +
|
||||
` embedding=excluded.embedding,\n` +
|
||||
` dims=excluded.dims,\n` +
|
||||
` updated_at=excluded.updated_at`,
|
||||
);
|
||||
for (const entry of params.entries) {
|
||||
const embedding = entry.embedding ?? [];
|
||||
stmt.run(
|
||||
provider.id,
|
||||
provider.model,
|
||||
params.providerKey,
|
||||
entry.hash,
|
||||
JSON.stringify(embedding),
|
||||
embedding.length,
|
||||
now,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function collectMemoryCachedEmbeddings<T extends Pick<MemoryChunk, "hash">>(params: {
|
||||
chunks: T[];
|
||||
cached: Map<string, number[]>;
|
||||
}): {
|
||||
embeddings: number[][];
|
||||
missing: Array<{ index: number; chunk: T }>;
|
||||
} {
|
||||
const embeddings: number[][] = Array.from({ length: params.chunks.length }, () => []);
|
||||
const missing: Array<{ index: number; chunk: T }> = [];
|
||||
|
||||
for (let index = 0; index < params.chunks.length; index += 1) {
|
||||
const chunk = params.chunks[index];
|
||||
const hit = chunk?.hash ? params.cached.get(chunk.hash) : undefined;
|
||||
if (hit && hit.length > 0) {
|
||||
embeddings[index] = hit;
|
||||
} else if (chunk) {
|
||||
missing.push({ index, chunk });
|
||||
}
|
||||
}
|
||||
|
||||
return { embeddings, missing };
|
||||
}
|
||||
|
|
@ -0,0 +1,668 @@
|
|||
import fs from "node:fs/promises";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
enforceEmbeddingMaxInputTokens,
|
||||
hasNonTextEmbeddingParts,
|
||||
type EmbeddingInput,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { createSubsystemLogger } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { type SessionFileEntry } from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
|
||||
import {
|
||||
buildMultimodalChunkForIndexing,
|
||||
chunkMarkdown,
|
||||
hashText,
|
||||
remapChunkLines,
|
||||
type MemoryChunk,
|
||||
type MemoryFileEntry,
|
||||
type MemorySource,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import {
|
||||
MEMORY_BATCH_FAILURE_LIMIT,
|
||||
recordMemoryBatchFailure,
|
||||
resetMemoryBatchFailureState,
|
||||
} from "./manager-batch-state.js";
|
||||
import {
|
||||
collectMemoryCachedEmbeddings,
|
||||
loadMemoryEmbeddingCache,
|
||||
upsertMemoryEmbeddingCache,
|
||||
} from "./manager-embedding-cache.js";
|
||||
import {
|
||||
buildMemoryEmbeddingBatches,
|
||||
buildTextEmbeddingInputs,
|
||||
filterNonEmptyMemoryChunks,
|
||||
isRetryableMemoryEmbeddingError,
|
||||
resolveMemoryEmbeddingRetryDelay,
|
||||
runMemoryEmbeddingRetryLoop,
|
||||
} from "./manager-embedding-policy.js";
|
||||
import { deleteMemoryFtsRows } from "./manager-fts-state.js";
|
||||
import { MemoryManagerSyncOps } from "./manager-sync-ops.js";
|
||||
import { logMemoryVectorDegradedWrite } from "./manager-vector-warning.js";
|
||||
import { replaceMemoryVectorRow } from "./manager-vector-write.js";
|
||||
|
||||
const VECTOR_TABLE = "chunks_vec";
|
||||
const FTS_TABLE = "chunks_fts";
|
||||
const EMBEDDING_CACHE_TABLE = "embedding_cache";
|
||||
const EMBEDDING_BATCH_MAX_TOKENS = 8000;
|
||||
const EMBEDDING_INDEX_CONCURRENCY = 4;
|
||||
const EMBEDDING_RETRY_MAX_ATTEMPTS = 3;
|
||||
const EMBEDDING_RETRY_BASE_DELAY_MS = 500;
|
||||
const EMBEDDING_RETRY_MAX_DELAY_MS = 8000;
|
||||
const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 60_000;
|
||||
const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 60_000;
|
||||
const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 60_000;
|
||||
const EMBEDDING_BATCH_TIMEOUT_LOCAL_MS = 10 * 60_000;
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
|
||||
export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
|
||||
protected abstract batchFailureCount: number;
|
||||
protected abstract batchFailureLastError?: string;
|
||||
protected abstract batchFailureLastProvider?: string;
|
||||
protected abstract batchFailureLock: Promise<void>;
|
||||
|
||||
protected pruneEmbeddingCacheIfNeeded(): void {
|
||||
if (!this.cache.enabled) {
|
||||
return;
|
||||
}
|
||||
const max = this.cache.maxEntries;
|
||||
if (!max || max <= 0) {
|
||||
return;
|
||||
}
|
||||
const row = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get() as
|
||||
| { c: number }
|
||||
| undefined;
|
||||
const count = row?.c ?? 0;
|
||||
if (count <= max) {
|
||||
return;
|
||||
}
|
||||
const excess = count - max;
|
||||
this.db
|
||||
.prepare(
|
||||
`DELETE FROM ${EMBEDDING_CACHE_TABLE}\n` +
|
||||
` WHERE rowid IN (\n` +
|
||||
` SELECT rowid FROM ${EMBEDDING_CACHE_TABLE}\n` +
|
||||
` ORDER BY updated_at ASC\n` +
|
||||
` LIMIT ?\n` +
|
||||
` )`,
|
||||
)
|
||||
.run(excess);
|
||||
}
|
||||
|
||||
private async embedChunksInBatches(chunks: MemoryChunk[]): Promise<number[][]> {
|
||||
if (chunks.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const { embeddings, missing } = this.collectCachedEmbeddings(chunks);
|
||||
|
||||
if (missing.length === 0) {
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
const missingChunks = missing.map((m) => m.chunk);
|
||||
const batches = buildMemoryEmbeddingBatches(missingChunks, EMBEDDING_BATCH_MAX_TOKENS);
|
||||
const toCache: Array<{ hash: string; embedding: number[] }> = [];
|
||||
const provider = this.provider;
|
||||
if (!provider) {
|
||||
throw new Error("Cannot embed batch in FTS-only mode (no embedding provider)");
|
||||
}
|
||||
let cursor = 0;
|
||||
for (const batch of batches) {
|
||||
const inputs = buildTextEmbeddingInputs(batch);
|
||||
const hasStructuredInputs = inputs.some((input) => hasNonTextEmbeddingParts(input));
|
||||
if (hasStructuredInputs && !provider.embedBatchInputs) {
|
||||
throw new Error(
|
||||
`Embedding provider "${provider.id}" does not support multimodal memory inputs.`,
|
||||
);
|
||||
}
|
||||
const batchEmbeddings = hasStructuredInputs
|
||||
? await this.embedBatchInputsWithRetry(inputs)
|
||||
: await this.embedBatchWithRetry(batch.map((chunk) => chunk.text));
|
||||
for (let i = 0; i < batch.length; i += 1) {
|
||||
const item = missing[cursor + i];
|
||||
const embedding = batchEmbeddings[i] ?? [];
|
||||
if (item) {
|
||||
embeddings[item.index] = embedding;
|
||||
toCache.push({ hash: item.chunk.hash, embedding });
|
||||
}
|
||||
}
|
||||
cursor += batch.length;
|
||||
}
|
||||
upsertMemoryEmbeddingCache({
|
||||
db: this.db,
|
||||
enabled: this.cache.enabled,
|
||||
provider: this.provider,
|
||||
providerKey: this.providerKey,
|
||||
entries: toCache,
|
||||
tableName: EMBEDDING_CACHE_TABLE,
|
||||
});
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
protected computeProviderKey(): string {
|
||||
// FTS-only mode: no provider, use a constant key
|
||||
if (!this.provider) {
|
||||
return hashText(JSON.stringify({ provider: "none", model: "fts-only" }));
|
||||
}
|
||||
if (this.providerRuntime?.cacheKeyData) {
|
||||
return hashText(JSON.stringify(this.providerRuntime.cacheKeyData));
|
||||
}
|
||||
return hashText(JSON.stringify({ provider: this.provider.id, model: this.provider.model }));
|
||||
}
|
||||
|
||||
private buildBatchDebug(source: MemorySource, chunks: MemoryChunk[]) {
|
||||
return (message: string, data?: Record<string, unknown>) =>
|
||||
log.debug(
|
||||
message,
|
||||
data ? { ...data, source, chunks: chunks.length } : { source, chunks: chunks.length },
|
||||
);
|
||||
}
|
||||
|
||||
private async embedChunksWithBatch(
|
||||
chunks: MemoryChunk[],
|
||||
_entry: MemoryFileEntry | SessionFileEntry,
|
||||
source: MemorySource,
|
||||
): Promise<number[][]> {
|
||||
const provider = this.provider;
|
||||
const batchEmbed = this.providerRuntime?.batchEmbed;
|
||||
if (!provider || !batchEmbed) {
|
||||
return this.embedChunksInBatches(chunks);
|
||||
}
|
||||
if (chunks.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const { embeddings, missing } = this.collectCachedEmbeddings(chunks);
|
||||
if (missing.length === 0) {
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
const missingChunks = missing.map((item) => item.chunk);
|
||||
const batchResult = await this.runBatchWithFallback({
|
||||
provider: provider.id,
|
||||
run: async () =>
|
||||
await batchEmbed({
|
||||
agentId: this.agentId,
|
||||
chunks: missingChunks,
|
||||
wait: this.batch.wait,
|
||||
concurrency: this.batch.concurrency,
|
||||
pollIntervalMs: this.batch.pollIntervalMs,
|
||||
timeoutMs: this.batch.timeoutMs,
|
||||
debug: this.buildBatchDebug(source, chunks),
|
||||
}),
|
||||
fallback: async () => await this.embedChunksInBatches(chunks),
|
||||
});
|
||||
if (!batchResult) {
|
||||
return this.embedChunksInBatches(chunks);
|
||||
}
|
||||
const toCache: Array<{ hash: string; embedding: number[] }> = [];
|
||||
for (let index = 0; index < missing.length; index += 1) {
|
||||
const item = missing[index];
|
||||
const embedding = batchResult[index] ?? [];
|
||||
if (!item) {
|
||||
continue;
|
||||
}
|
||||
embeddings[item.index] = embedding;
|
||||
toCache.push({ hash: item.chunk.hash, embedding });
|
||||
}
|
||||
upsertMemoryEmbeddingCache({
|
||||
db: this.db,
|
||||
enabled: this.cache.enabled,
|
||||
provider,
|
||||
providerKey: this.providerKey,
|
||||
entries: toCache,
|
||||
tableName: EMBEDDING_CACHE_TABLE,
|
||||
});
|
||||
return embeddings;
|
||||
}
|
||||
|
||||
private collectCachedEmbeddings(chunks: MemoryChunk[]): {
|
||||
embeddings: number[][];
|
||||
missing: Array<{ index: number; chunk: MemoryChunk }>;
|
||||
} {
|
||||
return collectMemoryCachedEmbeddings({
|
||||
chunks,
|
||||
cached: loadMemoryEmbeddingCache({
|
||||
db: this.db,
|
||||
enabled: this.cache.enabled,
|
||||
provider: this.provider,
|
||||
providerKey: this.providerKey,
|
||||
hashes: chunks.map((chunk) => chunk.hash),
|
||||
tableName: EMBEDDING_CACHE_TABLE,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
protected async embedBatchWithRetry(texts: string[]): Promise<number[][]> {
|
||||
if (texts.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const provider = this.provider;
|
||||
if (!provider) {
|
||||
throw new Error("Cannot embed batch in FTS-only mode (no embedding provider)");
|
||||
}
|
||||
return await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => {
|
||||
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
||||
log.debug("memory embeddings: batch start", {
|
||||
provider: provider.id,
|
||||
items: texts.length,
|
||||
timeoutMs,
|
||||
});
|
||||
return await this.withTimeout(
|
||||
provider.embedBatch(texts),
|
||||
timeoutMs,
|
||||
`memory embeddings batch timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
);
|
||||
},
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
await this.waitForEmbeddingRetry(delayMs, "retrying");
|
||||
},
|
||||
maxAttempts: EMBEDDING_RETRY_MAX_ATTEMPTS,
|
||||
baseDelayMs: EMBEDDING_RETRY_BASE_DELAY_MS,
|
||||
});
|
||||
}
|
||||
|
||||
protected async embedBatchInputsWithRetry(inputs: EmbeddingInput[]): Promise<number[][]> {
|
||||
if (inputs.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const provider = this.provider;
|
||||
const embedBatchInputs = provider?.embedBatchInputs;
|
||||
if (!embedBatchInputs) {
|
||||
return await this.embedBatchWithRetry(inputs.map((input) => input.text));
|
||||
}
|
||||
return await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => {
|
||||
const timeoutMs = this.resolveEmbeddingTimeout("batch");
|
||||
log.debug("memory embeddings: structured batch start", {
|
||||
provider: provider.id,
|
||||
items: inputs.length,
|
||||
timeoutMs,
|
||||
});
|
||||
return await this.withTimeout(
|
||||
embedBatchInputs(inputs),
|
||||
timeoutMs,
|
||||
`memory embeddings batch timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
);
|
||||
},
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
await this.waitForEmbeddingRetry(delayMs, "retrying structured batch");
|
||||
},
|
||||
maxAttempts: EMBEDDING_RETRY_MAX_ATTEMPTS,
|
||||
baseDelayMs: EMBEDDING_RETRY_BASE_DELAY_MS,
|
||||
});
|
||||
}
|
||||
|
||||
private async waitForEmbeddingRetry(delayMs: number, action: string): Promise<void> {
|
||||
const waitMs = resolveMemoryEmbeddingRetryDelay(
|
||||
delayMs,
|
||||
Math.random(),
|
||||
EMBEDDING_RETRY_MAX_DELAY_MS,
|
||||
);
|
||||
log.warn(`memory embeddings rate limited; ${action} in ${waitMs}ms`);
|
||||
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
||||
}
|
||||
|
||||
private resolveEmbeddingTimeout(kind: "query" | "batch"): number {
|
||||
const isLocal = this.provider?.id === "local";
|
||||
if (kind === "query") {
|
||||
return isLocal ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS;
|
||||
}
|
||||
return isLocal ? EMBEDDING_BATCH_TIMEOUT_LOCAL_MS : EMBEDDING_BATCH_TIMEOUT_REMOTE_MS;
|
||||
}
|
||||
|
||||
protected async embedQueryWithTimeout(text: string): Promise<number[]> {
|
||||
if (!this.provider) {
|
||||
throw new Error("Cannot embed query in FTS-only mode (no embedding provider)");
|
||||
}
|
||||
const timeoutMs = this.resolveEmbeddingTimeout("query");
|
||||
log.debug("memory embeddings: query start", { provider: this.provider.id, timeoutMs });
|
||||
return await this.withTimeout(
|
||||
this.provider.embedQuery(text),
|
||||
timeoutMs,
|
||||
`memory embeddings query timed out after ${Math.round(timeoutMs / 1000)}s`,
|
||||
);
|
||||
}
|
||||
|
||||
protected async withTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
message: string,
|
||||
): Promise<T> {
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
||||
return await promise;
|
||||
}
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
||||
});
|
||||
try {
|
||||
return (await Promise.race([promise, timeoutPromise])) as T;
|
||||
} finally {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async withBatchFailureLock<T>(fn: () => Promise<T>): Promise<T> {
|
||||
let release: () => void;
|
||||
const wait = this.batchFailureLock;
|
||||
this.batchFailureLock = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
await wait;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release!();
|
||||
}
|
||||
}
|
||||
|
||||
private async resetBatchFailureCount(): Promise<void> {
|
||||
await this.withBatchFailureLock(async () => {
|
||||
if (this.batchFailureCount > 0) {
|
||||
log.debug("memory embeddings: batch recovered; resetting failure count");
|
||||
}
|
||||
const nextState = resetMemoryBatchFailureState({
|
||||
enabled: this.batch.enabled,
|
||||
count: this.batchFailureCount,
|
||||
lastError: this.batchFailureLastError,
|
||||
lastProvider: this.batchFailureLastProvider,
|
||||
});
|
||||
this.batch.enabled = nextState.enabled;
|
||||
this.batchFailureCount = nextState.count;
|
||||
this.batchFailureLastError = nextState.lastError;
|
||||
this.batchFailureLastProvider = nextState.lastProvider;
|
||||
});
|
||||
}
|
||||
|
||||
private async recordBatchFailure(params: {
|
||||
provider: string;
|
||||
message: string;
|
||||
attempts?: number;
|
||||
forceDisable?: boolean;
|
||||
}): Promise<{ disabled: boolean; count: number }> {
|
||||
return await this.withBatchFailureLock(async () => {
|
||||
if (!this.batch.enabled) {
|
||||
return { disabled: true, count: this.batchFailureCount };
|
||||
}
|
||||
const nextState = recordMemoryBatchFailure(
|
||||
{
|
||||
enabled: this.batch.enabled,
|
||||
count: this.batchFailureCount,
|
||||
lastError: this.batchFailureLastError,
|
||||
lastProvider: this.batchFailureLastProvider,
|
||||
},
|
||||
params,
|
||||
);
|
||||
this.batch.enabled = nextState.enabled;
|
||||
this.batchFailureCount = nextState.count;
|
||||
this.batchFailureLastError = nextState.lastError;
|
||||
this.batchFailureLastProvider = nextState.lastProvider;
|
||||
return { disabled: !nextState.enabled, count: nextState.count };
|
||||
});
|
||||
}
|
||||
|
||||
private isBatchTimeoutError(message: string): boolean {
|
||||
return /timed out|timeout/i.test(message);
|
||||
}
|
||||
|
||||
private async runBatchWithTimeoutRetry<T>(params: {
|
||||
provider: string;
|
||||
run: () => Promise<T>;
|
||||
}): Promise<T> {
|
||||
try {
|
||||
return await params.run();
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
if (this.isBatchTimeoutError(message)) {
|
||||
log.warn(`memory embeddings: ${params.provider} batch timed out; retrying once`);
|
||||
try {
|
||||
return await params.run();
|
||||
} catch (retryErr) {
|
||||
(retryErr as { batchAttempts?: number }).batchAttempts = 2;
|
||||
throw retryErr;
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private async runBatchWithFallback<T>(params: {
|
||||
provider: string;
|
||||
run: () => Promise<T>;
|
||||
fallback: () => Promise<number[][]>;
|
||||
}): Promise<T | number[][]> {
|
||||
if (!this.batch.enabled) {
|
||||
return await params.fallback();
|
||||
}
|
||||
try {
|
||||
const result = await this.runBatchWithTimeoutRetry({
|
||||
provider: params.provider,
|
||||
run: params.run,
|
||||
});
|
||||
await this.resetBatchFailureCount();
|
||||
return result;
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
const attempts = (err as { batchAttempts?: number }).batchAttempts ?? 1;
|
||||
const forceDisable = /asyncBatchEmbedContent not available/i.test(message);
|
||||
const failure = await this.recordBatchFailure({
|
||||
provider: params.provider,
|
||||
message,
|
||||
attempts,
|
||||
forceDisable,
|
||||
});
|
||||
const suffix = failure.disabled ? "disabling batch" : "keeping batch enabled";
|
||||
log.warn(
|
||||
`memory embeddings: ${params.provider} batch failed (${failure.count}/${MEMORY_BATCH_FAILURE_LIMIT}); ${suffix}; falling back to non-batch embeddings: ${message}`,
|
||||
);
|
||||
return await params.fallback();
|
||||
}
|
||||
}
|
||||
|
||||
protected getIndexConcurrency(): number {
|
||||
return this.batch.enabled ? this.batch.concurrency : EMBEDDING_INDEX_CONCURRENCY;
|
||||
}
|
||||
|
||||
private clearIndexedFileData(pathname: string, source: MemorySource): void {
|
||||
if (this.vector.enabled) {
|
||||
try {
|
||||
this.db
|
||||
.prepare(
|
||||
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`,
|
||||
)
|
||||
.run(pathname, source);
|
||||
} catch {}
|
||||
}
|
||||
if (this.fts.enabled && this.fts.available) {
|
||||
try {
|
||||
deleteMemoryFtsRows({
|
||||
db: this.db,
|
||||
tableName: FTS_TABLE,
|
||||
path: pathname,
|
||||
source,
|
||||
currentModel: this.provider?.model,
|
||||
});
|
||||
} catch {}
|
||||
}
|
||||
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(pathname, source);
|
||||
}
|
||||
|
||||
private upsertFileRecord(entry: MemoryFileEntry | SessionFileEntry, source: MemorySource): void {
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
source=excluded.source,
|
||||
hash=excluded.hash,
|
||||
mtime=excluded.mtime,
|
||||
size=excluded.size`,
|
||||
)
|
||||
.run(entry.path, source, entry.hash, entry.mtimeMs, entry.size);
|
||||
}
|
||||
|
||||
private deleteFileRecord(pathname: string, source: MemorySource): void {
|
||||
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(pathname, source);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write chunks (and optional embeddings) for a file into the index.
|
||||
* Handles both the chunks table, the vector table, and the FTS table.
|
||||
* Pass an empty embeddings array to skip vector writes (FTS-only mode).
|
||||
*/
|
||||
private writeChunks(
|
||||
entry: MemoryFileEntry | SessionFileEntry,
|
||||
source: MemorySource,
|
||||
model: string,
|
||||
chunks: MemoryChunk[],
|
||||
embeddings: number[][],
|
||||
vectorReady: boolean,
|
||||
): void {
|
||||
const now = Date.now();
|
||||
this.clearIndexedFileData(entry.path, source);
|
||||
for (let i = 0; i < chunks.length; i++) {
|
||||
const chunk = chunks[i];
|
||||
const embedding = embeddings[i] ?? [];
|
||||
const id = hashText(
|
||||
`${source}:${entry.path}:${chunk.startLine}:${chunk.endLine}:${chunk.hash}:${model}`,
|
||||
);
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
hash=excluded.hash,
|
||||
model=excluded.model,
|
||||
text=excluded.text,
|
||||
embedding=excluded.embedding,
|
||||
updated_at=excluded.updated_at`,
|
||||
)
|
||||
.run(
|
||||
id,
|
||||
entry.path,
|
||||
source,
|
||||
chunk.startLine,
|
||||
chunk.endLine,
|
||||
chunk.hash,
|
||||
model,
|
||||
chunk.text,
|
||||
JSON.stringify(embedding),
|
||||
now,
|
||||
);
|
||||
if (vectorReady && embedding.length > 0) {
|
||||
replaceMemoryVectorRow({
|
||||
db: this.db,
|
||||
tableName: VECTOR_TABLE,
|
||||
id,
|
||||
embedding,
|
||||
});
|
||||
}
|
||||
if (this.fts.enabled && this.fts.available) {
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO ${FTS_TABLE} (text, id, path, source, model, start_line, end_line)\n` +
|
||||
` VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||
)
|
||||
.run(chunk.text, id, entry.path, source, model, chunk.startLine, chunk.endLine);
|
||||
}
|
||||
}
|
||||
this.vectorDegradedWriteWarningShown = logMemoryVectorDegradedWrite({
|
||||
vectorEnabled: this.vector.enabled,
|
||||
vectorReady,
|
||||
chunkCount: chunks.length,
|
||||
warningShown: this.vectorDegradedWriteWarningShown,
|
||||
loadError: this.vector.loadError,
|
||||
warn: (message) => log.warn(message),
|
||||
});
|
||||
this.upsertFileRecord(entry, source);
|
||||
}
|
||||
|
||||
protected async indexFile(
|
||||
entry: MemoryFileEntry | SessionFileEntry,
|
||||
options: { source: MemorySource; content?: string },
|
||||
) {
|
||||
// FTS-only mode: no embedding provider, but we can still build a FTS index
|
||||
if (!this.provider) {
|
||||
// Multimodal files require an embedding provider; skip in FTS-only mode.
|
||||
if ("kind" in entry && entry.kind === "multimodal") {
|
||||
return;
|
||||
}
|
||||
const content = options.content ?? (await fs.readFile(entry.absPath, "utf-8"));
|
||||
const chunks = filterNonEmptyMemoryChunks(chunkMarkdown(content, this.settings.chunking));
|
||||
if (options.source === "sessions" && "lineMap" in entry) {
|
||||
remapChunkLines(chunks, entry.lineMap);
|
||||
}
|
||||
this.writeChunks(entry, options.source, "fts-only", chunks, [], false);
|
||||
return;
|
||||
}
|
||||
|
||||
let chunks: MemoryChunk[];
|
||||
let structuredInputBytes: number | undefined;
|
||||
if ("kind" in entry && entry.kind === "multimodal") {
|
||||
if (!this.provider) {
|
||||
log.debug("Skipping multimodal indexing in FTS-only mode", {
|
||||
path: entry.path,
|
||||
source: options.source,
|
||||
});
|
||||
this.clearIndexedFileData(entry.path, options.source);
|
||||
this.upsertFileRecord(entry, options.source);
|
||||
return;
|
||||
}
|
||||
const multimodalChunk = await buildMultimodalChunkForIndexing(entry);
|
||||
if (!multimodalChunk) {
|
||||
this.clearIndexedFileData(entry.path, options.source);
|
||||
this.deleteFileRecord(entry.path, options.source);
|
||||
return;
|
||||
}
|
||||
structuredInputBytes = multimodalChunk.structuredInputBytes;
|
||||
chunks = [multimodalChunk.chunk];
|
||||
} else {
|
||||
const content = options.content ?? (await fs.readFile(entry.absPath, "utf-8"));
|
||||
const baseChunks = filterNonEmptyMemoryChunks(chunkMarkdown(content, this.settings.chunking));
|
||||
chunks = this.provider
|
||||
? enforceEmbeddingMaxInputTokens(this.provider, baseChunks, EMBEDDING_BATCH_MAX_TOKENS)
|
||||
: baseChunks;
|
||||
if (options.source === "sessions" && "lineMap" in entry) {
|
||||
remapChunkLines(chunks, entry.lineMap);
|
||||
}
|
||||
}
|
||||
if (!this.provider) {
|
||||
this.writeChunks(entry, options.source, "fts-only", chunks, [], false);
|
||||
return;
|
||||
}
|
||||
|
||||
let embeddings: number[][];
|
||||
try {
|
||||
embeddings = this.batch.enabled
|
||||
? await this.embedChunksWithBatch(chunks, entry, options.source)
|
||||
: await this.embedChunksInBatches(chunks);
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
if (
|
||||
"kind" in entry &&
|
||||
entry.kind === "multimodal" &&
|
||||
/(413|payload too large|request too large|input too large|too many tokens|input limit|request size)/i.test(
|
||||
message,
|
||||
)
|
||||
) {
|
||||
log.warn("memory embeddings: skipping multimodal file rejected as too large", {
|
||||
path: entry.path,
|
||||
bytes: structuredInputBytes,
|
||||
provider: this.provider.id,
|
||||
model: this.provider.model,
|
||||
error: message,
|
||||
});
|
||||
this.clearIndexedFileData(entry.path, options.source);
|
||||
this.upsertFileRecord(entry, options.source);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const sample = embeddings.find((embedding) => embedding.length > 0);
|
||||
const vectorReady = sample ? await this.ensureVectorReady(sample.length) : false;
|
||||
this.writeChunks(entry, options.source, this.provider.model, chunks, embeddings, vectorReady);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
buildMemoryEmbeddingBatches,
|
||||
filterNonEmptyMemoryChunks,
|
||||
isRetryableMemoryEmbeddingError,
|
||||
isStructuredInputTooLargeMemoryEmbeddingError,
|
||||
resolveMemoryEmbeddingRetryDelay,
|
||||
runMemoryEmbeddingRetryLoop,
|
||||
} from "./manager-embedding-policy.js";
|
||||
|
||||
function chunk(text: string) {
|
||||
return {
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
text,
|
||||
hash: text,
|
||||
};
|
||||
}
|
||||
|
||||
describe("memory embedding policy", () => {
|
||||
it("splits large files across multiple embedding batches", () => {
|
||||
const line = "a".repeat(4200);
|
||||
const batches = buildMemoryEmbeddingBatches([chunk(line), chunk(line)], 8000);
|
||||
|
||||
expect(batches).toHaveLength(2);
|
||||
expect(batches.every((batch) => batch.length === 1)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps small files in a single embedding batch", () => {
|
||||
const line = "b".repeat(120);
|
||||
const batches = buildMemoryEmbeddingBatches(
|
||||
[chunk(line), chunk(line), chunk(line), chunk(line)],
|
||||
8000,
|
||||
);
|
||||
|
||||
expect(batches).toHaveLength(1);
|
||||
expect(batches[0]).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("filters empty chunks before embedding", () => {
|
||||
const chunks = filterNonEmptyMemoryChunks([chunk("\n\n"), chunk("hello"), chunk(" ")]);
|
||||
|
||||
expect(chunks.map((entry) => entry.text)).toEqual(["hello"]);
|
||||
});
|
||||
|
||||
it("retries transient rate limit and 5xx errors", async () => {
|
||||
const run = vi.fn(async () => {
|
||||
const call = run.mock.calls.length;
|
||||
if (call === 1) {
|
||||
throw new Error("openai embeddings failed: 429 rate limit");
|
||||
}
|
||||
if (call === 2) {
|
||||
throw new Error("openai embeddings failed: 502 Bad Gateway (cloudflare)");
|
||||
}
|
||||
return "ok";
|
||||
});
|
||||
const waits: number[] = [];
|
||||
|
||||
const result = await runMemoryEmbeddingRetryLoop({
|
||||
run,
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
waits.push(delayMs);
|
||||
},
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
});
|
||||
|
||||
expect(result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(3);
|
||||
expect(waits).toEqual([500, 1000]);
|
||||
});
|
||||
|
||||
it("retries too-many-tokens-per-day errors", async () => {
|
||||
let calls = 0;
|
||||
const waits: number[] = [];
|
||||
|
||||
const result = await runMemoryEmbeddingRetryLoop({
|
||||
run: async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
throw new Error("AWS Bedrock embeddings failed: Too many tokens per day");
|
||||
}
|
||||
return "ok";
|
||||
},
|
||||
isRetryable: isRetryableMemoryEmbeddingError,
|
||||
waitForRetry: async (delayMs) => {
|
||||
waits.push(delayMs);
|
||||
},
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
});
|
||||
|
||||
expect(result).toBe("ok");
|
||||
expect(calls).toBe(2);
|
||||
expect(waits).toEqual([500]);
|
||||
});
|
||||
|
||||
it("classifies oversized structured-input errors", () => {
|
||||
expect(isStructuredInputTooLargeMemoryEmbeddingError("payload too large")).toBe(true);
|
||||
expect(
|
||||
isStructuredInputTooLargeMemoryEmbeddingError(
|
||||
"gemini embeddings failed: request size exceeded input limit",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(isStructuredInputTooLargeMemoryEmbeddingError("connection reset by peer")).toBe(false);
|
||||
});
|
||||
|
||||
it("caps retry jittered delays", () => {
|
||||
expect(resolveMemoryEmbeddingRetryDelay(500, 0, 8000)).toBe(500);
|
||||
expect(resolveMemoryEmbeddingRetryDelay(500, 1, 8000)).toBe(600);
|
||||
expect(resolveMemoryEmbeddingRetryDelay(10_000, 1, 8000)).toBe(8000);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
|
||||
type MemoryEmbeddingTextPart = {
|
||||
type: "text";
|
||||
text: string;
|
||||
};
|
||||
|
||||
type MemoryEmbeddingInlineDataPart = {
|
||||
type: "inline-data";
|
||||
mimeType: string;
|
||||
data: string;
|
||||
};
|
||||
|
||||
type MemoryEmbeddingInput = {
|
||||
text: string;
|
||||
parts?: Array<MemoryEmbeddingTextPart | MemoryEmbeddingInlineDataPart>;
|
||||
};
|
||||
|
||||
type MemoryEmbeddingChunk = {
|
||||
text: string;
|
||||
embeddingInput?: MemoryEmbeddingInput;
|
||||
};
|
||||
|
||||
function estimateUtf8Bytes(text: string): number {
|
||||
if (!text) {
|
||||
return 0;
|
||||
}
|
||||
return Buffer.byteLength(text, "utf8");
|
||||
}
|
||||
|
||||
function estimateStructuredEmbeddingInputBytes(input: MemoryEmbeddingInput): number {
|
||||
if (!input.parts?.length) {
|
||||
return estimateUtf8Bytes(input.text);
|
||||
}
|
||||
let total = 0;
|
||||
for (const part of input.parts) {
|
||||
if (part.type === "text") {
|
||||
total += estimateUtf8Bytes(part.text);
|
||||
} else {
|
||||
total += estimateUtf8Bytes(part.mimeType);
|
||||
total += estimateUtf8Bytes(part.data);
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
export function filterNonEmptyMemoryChunks<T extends MemoryEmbeddingChunk>(chunks: T[]): T[] {
|
||||
return chunks.filter((chunk) => chunk.text.trim().length > 0);
|
||||
}
|
||||
|
||||
export function buildMemoryEmbeddingBatches<T extends MemoryEmbeddingChunk>(
|
||||
chunks: T[],
|
||||
maxTokens: number,
|
||||
): T[][] {
|
||||
const batches: T[][] = [];
|
||||
let current: T[] = [];
|
||||
let currentTokens = 0;
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const estimate = chunk.embeddingInput
|
||||
? estimateStructuredEmbeddingInputBytes(chunk.embeddingInput)
|
||||
: estimateUtf8Bytes(chunk.text);
|
||||
const wouldExceed = current.length > 0 && currentTokens + estimate > maxTokens;
|
||||
if (wouldExceed) {
|
||||
batches.push(current);
|
||||
current = [];
|
||||
currentTokens = 0;
|
||||
}
|
||||
if (current.length === 0 && estimate > maxTokens) {
|
||||
batches.push([chunk]);
|
||||
continue;
|
||||
}
|
||||
current.push(chunk);
|
||||
currentTokens += estimate;
|
||||
}
|
||||
|
||||
if (current.length > 0) {
|
||||
batches.push(current);
|
||||
}
|
||||
return batches;
|
||||
}
|
||||
|
||||
export function isRetryableMemoryEmbeddingError(message: string): boolean {
|
||||
return /(rate[_ ]limit|too many requests|429|resource has been exhausted|5\d\d|cloudflare|tokens per day)/i.test(
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
export function isStructuredInputTooLargeMemoryEmbeddingError(message: string): boolean {
|
||||
return /(413|payload too large|request too large|input too large|too many tokens|input limit|request size)/i.test(
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveMemoryEmbeddingRetryDelay(
|
||||
delayMs: number,
|
||||
randomValue: number,
|
||||
maxDelayMs: number,
|
||||
): number {
|
||||
return Math.min(maxDelayMs, Math.round(delayMs * (1 + randomValue * 0.2)));
|
||||
}
|
||||
|
||||
export async function runMemoryEmbeddingRetryLoop<T>(params: {
|
||||
run: () => Promise<T>;
|
||||
isRetryable: (message: string) => boolean;
|
||||
waitForRetry: (delayMs: number) => Promise<void>;
|
||||
maxAttempts: number;
|
||||
baseDelayMs: number;
|
||||
}): Promise<T> {
|
||||
let attempt = 0;
|
||||
let delayMs = params.baseDelayMs;
|
||||
while (true) {
|
||||
try {
|
||||
return await params.run();
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
if (!params.isRetryable(message) || attempt >= params.maxAttempts) {
|
||||
throw err;
|
||||
}
|
||||
await params.waitForRetry(delayMs);
|
||||
delayMs *= 2;
|
||||
attempt += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildTextEmbeddingInputs(chunks: MemoryEmbeddingChunk[]): MemoryEmbeddingInput[] {
|
||||
return chunks.map((chunk) => chunk.embeddingInput ?? { text: chunk.text });
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
import { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { deleteMemoryFtsRows } from "./manager-fts-state.js";
|
||||
|
||||
describe("memory FTS state", () => {
|
||||
let db: DatabaseSync | null = null;
|
||||
|
||||
afterEach(() => {
|
||||
db?.close();
|
||||
db = null;
|
||||
});
|
||||
|
||||
it("only removes rows for the active model when a provider is active", () => {
|
||||
db = new DatabaseSync(":memory:");
|
||||
db.exec("CREATE TABLE chunks_fts (path TEXT, source TEXT, model TEXT)");
|
||||
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"memory",
|
||||
"mock-embed",
|
||||
);
|
||||
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"memory",
|
||||
"other-model",
|
||||
);
|
||||
|
||||
deleteMemoryFtsRows({
|
||||
db,
|
||||
path: "memory/2026-01-12.md",
|
||||
source: "memory",
|
||||
currentModel: "mock-embed",
|
||||
});
|
||||
|
||||
const rows = db.prepare("SELECT model FROM chunks_fts ORDER BY model").all() as Array<{
|
||||
model: string;
|
||||
}>;
|
||||
expect(rows).toEqual([{ model: "other-model" }]);
|
||||
});
|
||||
|
||||
it("removes all rows for the path in FTS-only mode", () => {
|
||||
db = new DatabaseSync(":memory:");
|
||||
db.exec("CREATE TABLE chunks_fts (path TEXT, source TEXT, model TEXT)");
|
||||
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"memory",
|
||||
"mock-embed",
|
||||
);
|
||||
db.prepare("INSERT INTO chunks_fts (path, source, model) VALUES (?, ?, ?)").run(
|
||||
"memory/2026-01-12.md",
|
||||
"memory",
|
||||
"fts-only",
|
||||
);
|
||||
|
||||
deleteMemoryFtsRows({
|
||||
db,
|
||||
path: "memory/2026-01-12.md",
|
||||
source: "memory",
|
||||
});
|
||||
|
||||
const count = db.prepare("SELECT COUNT(*) as c FROM chunks_fts").get() as { c: number };
|
||||
expect(count.c).toBe(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export function deleteMemoryFtsRows(params: {
|
||||
db: DatabaseSync;
|
||||
tableName?: string;
|
||||
path: string;
|
||||
source: MemorySource;
|
||||
currentModel?: string;
|
||||
}): void {
|
||||
const tableName = params.tableName ?? "chunks_fts";
|
||||
if (params.currentModel) {
|
||||
params.db
|
||||
.prepare(`DELETE FROM ${tableName} WHERE path = ? AND source = ? AND model = ?`)
|
||||
.run(params.path, params.source, params.currentModel);
|
||||
return;
|
||||
}
|
||||
params.db
|
||||
.prepare(`DELETE FROM ${tableName} WHERE path = ? AND source = ?`)
|
||||
.run(params.path, params.source);
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
import type {
|
||||
OpenClawConfig,
|
||||
ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import {
|
||||
resolveEmbeddingProviderFallbackModel,
|
||||
type EmbeddingProvider,
|
||||
type EmbeddingProviderResult,
|
||||
type EmbeddingProviderRuntime,
|
||||
} from "./embeddings.js";
|
||||
|
||||
export type MemoryResolvedProviderState = {
|
||||
provider: EmbeddingProvider | null;
|
||||
fallbackFrom?: string;
|
||||
fallbackReason?: string;
|
||||
providerUnavailableReason?: string;
|
||||
providerRuntime?: EmbeddingProviderRuntime;
|
||||
};
|
||||
|
||||
export function resolveMemoryPrimaryProviderRequest(params: {
|
||||
settings: ResolvedMemorySearchConfig;
|
||||
}): {
|
||||
provider: string;
|
||||
model: string;
|
||||
remote: ResolvedMemorySearchConfig["remote"];
|
||||
outputDimensionality: ResolvedMemorySearchConfig["outputDimensionality"];
|
||||
fallback: ResolvedMemorySearchConfig["fallback"];
|
||||
local: ResolvedMemorySearchConfig["local"];
|
||||
} {
|
||||
return {
|
||||
provider: params.settings.provider,
|
||||
model: params.settings.model,
|
||||
remote: params.settings.remote,
|
||||
outputDimensionality: params.settings.outputDimensionality,
|
||||
fallback: params.settings.fallback,
|
||||
local: params.settings.local,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveMemoryProviderState(
|
||||
result: Pick<
|
||||
EmbeddingProviderResult,
|
||||
"provider" | "fallbackFrom" | "fallbackReason" | "providerUnavailableReason" | "runtime"
|
||||
>,
|
||||
): MemoryResolvedProviderState {
|
||||
return {
|
||||
provider: result.provider,
|
||||
fallbackFrom: result.fallbackFrom,
|
||||
fallbackReason: result.fallbackReason,
|
||||
providerUnavailableReason: result.providerUnavailableReason,
|
||||
providerRuntime: result.runtime,
|
||||
};
|
||||
}
|
||||
|
||||
export function applyMemoryFallbackProviderState(params: {
|
||||
current: MemoryResolvedProviderState;
|
||||
fallbackFrom: string;
|
||||
reason: string;
|
||||
result: Pick<EmbeddingProviderResult, "provider" | "runtime">;
|
||||
}): MemoryResolvedProviderState {
|
||||
return {
|
||||
...params.current,
|
||||
fallbackFrom: params.fallbackFrom,
|
||||
fallbackReason: params.reason,
|
||||
provider: params.result.provider,
|
||||
providerRuntime: params.result.runtime,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveMemoryFallbackProviderRequest(params: {
|
||||
cfg: OpenClawConfig;
|
||||
settings: ResolvedMemorySearchConfig;
|
||||
currentProviderId: string | null;
|
||||
}): {
|
||||
provider: string;
|
||||
model: string;
|
||||
remote: ResolvedMemorySearchConfig["remote"];
|
||||
outputDimensionality: ResolvedMemorySearchConfig["outputDimensionality"];
|
||||
fallback: "none";
|
||||
local: ResolvedMemorySearchConfig["local"];
|
||||
} | null {
|
||||
const fallback = params.settings.fallback;
|
||||
if (
|
||||
!fallback ||
|
||||
fallback === "none" ||
|
||||
!params.currentProviderId ||
|
||||
fallback === params.currentProviderId
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
provider: fallback,
|
||||
model: resolveEmbeddingProviderFallbackModel(fallback, params.settings.model, params.cfg),
|
||||
remote: params.settings.remote,
|
||||
outputDimensionality: params.settings.outputDimensionality,
|
||||
fallback: "none",
|
||||
local: params.settings.local,
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
resolveConfiguredScopeHash,
|
||||
resolveConfiguredSourcesForMeta,
|
||||
shouldRunFullMemoryReindex,
|
||||
type MemoryIndexMeta,
|
||||
} from "./manager-reindex-state.js";
|
||||
|
||||
function createMeta(overrides: Partial<MemoryIndexMeta> = {}): MemoryIndexMeta {
|
||||
return {
|
||||
model: "mock-embed-v1",
|
||||
provider: "openai",
|
||||
providerKey: "provider-key-v1",
|
||||
sources: ["memory"],
|
||||
scopeHash: "scope-v1",
|
||||
chunkTokens: 4000,
|
||||
chunkOverlap: 0,
|
||||
ftsTokenizer: "unicode61",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createFullReindexParams(
|
||||
overrides: {
|
||||
meta?: MemoryIndexMeta | null;
|
||||
provider?: { id: string; model: string } | null;
|
||||
providerKey?: string;
|
||||
configuredSources?: MemorySource[];
|
||||
configuredScopeHash?: string;
|
||||
chunkTokens?: number;
|
||||
chunkOverlap?: number;
|
||||
vectorReady?: boolean;
|
||||
ftsTokenizer?: string;
|
||||
} = {},
|
||||
) {
|
||||
return {
|
||||
meta: createMeta(),
|
||||
provider: { id: "openai", model: "mock-embed-v1" },
|
||||
providerKey: "provider-key-v1",
|
||||
configuredSources: ["memory"] as MemorySource[],
|
||||
configuredScopeHash: "scope-v1",
|
||||
chunkTokens: 4000,
|
||||
chunkOverlap: 0,
|
||||
vectorReady: false,
|
||||
ftsTokenizer: "unicode61",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("memory reindex state", () => {
|
||||
it("requires a full reindex when the embedding model changes", () => {
|
||||
expect(
|
||||
shouldRunFullMemoryReindex(
|
||||
createFullReindexParams({
|
||||
provider: { id: "openai", model: "mock-embed-v2" },
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("requires a full reindex when the provider cache key changes", () => {
|
||||
expect(
|
||||
shouldRunFullMemoryReindex(
|
||||
createFullReindexParams({
|
||||
provider: { id: "gemini", model: "gemini-embedding-2-preview" },
|
||||
providerKey: "provider-key-dims-768",
|
||||
meta: createMeta({
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
providerKey: "provider-key-dims-3072",
|
||||
}),
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("requires a full reindex when extraPaths change", () => {
|
||||
const workspaceDir = "/tmp/workspace";
|
||||
const firstScopeHash = resolveConfiguredScopeHash({
|
||||
workspaceDir,
|
||||
extraPaths: ["/tmp/workspace/a"],
|
||||
multimodal: {
|
||||
enabled: false,
|
||||
modalities: [],
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
},
|
||||
});
|
||||
const secondScopeHash = resolveConfiguredScopeHash({
|
||||
workspaceDir,
|
||||
extraPaths: ["/tmp/workspace/b"],
|
||||
multimodal: {
|
||||
enabled: false,
|
||||
modalities: [],
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
shouldRunFullMemoryReindex(
|
||||
createFullReindexParams({
|
||||
meta: createMeta({ scopeHash: firstScopeHash }),
|
||||
configuredScopeHash: secondScopeHash,
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("requires a full reindex when configured sources add sessions", () => {
|
||||
expect(
|
||||
shouldRunFullMemoryReindex(
|
||||
createFullReindexParams({
|
||||
configuredSources: ["memory", "sessions"],
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("requires a full reindex when multimodal settings change", () => {
|
||||
const workspaceDir = "/tmp/workspace";
|
||||
const firstScopeHash = resolveConfiguredScopeHash({
|
||||
workspaceDir,
|
||||
extraPaths: ["/tmp/workspace/media"],
|
||||
multimodal: {
|
||||
enabled: false,
|
||||
modalities: [],
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
},
|
||||
});
|
||||
const secondScopeHash = resolveConfiguredScopeHash({
|
||||
workspaceDir,
|
||||
extraPaths: ["/tmp/workspace/media"],
|
||||
multimodal: {
|
||||
enabled: true,
|
||||
modalities: ["image"],
|
||||
maxFileBytes: 20 * 1024 * 1024,
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
shouldRunFullMemoryReindex(
|
||||
createFullReindexParams({
|
||||
meta: createMeta({ scopeHash: firstScopeHash }),
|
||||
configuredScopeHash: secondScopeHash,
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps older indexes with missing sources compatible with memory-only config", () => {
|
||||
expect(
|
||||
shouldRunFullMemoryReindex(
|
||||
createFullReindexParams({
|
||||
meta: createMeta({ sources: undefined }),
|
||||
configuredSources: resolveConfiguredSourcesForMeta(new Set(["memory"])),
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
import {
|
||||
hashText,
|
||||
normalizeExtraMemoryPaths,
|
||||
type MemorySource,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export type MemoryIndexMeta = {
|
||||
model: string;
|
||||
provider: string;
|
||||
providerKey?: string;
|
||||
sources?: MemorySource[];
|
||||
scopeHash?: string;
|
||||
chunkTokens: number;
|
||||
chunkOverlap: number;
|
||||
vectorDims?: number;
|
||||
ftsTokenizer?: string;
|
||||
};
|
||||
|
||||
export function resolveConfiguredSourcesForMeta(sources: Iterable<MemorySource>): MemorySource[] {
|
||||
const normalized = Array.from(sources)
|
||||
.filter((source): source is MemorySource => source === "memory" || source === "sessions")
|
||||
.toSorted();
|
||||
return normalized.length > 0 ? normalized : ["memory"];
|
||||
}
|
||||
|
||||
export function normalizeMetaSources(meta: MemoryIndexMeta): MemorySource[] {
|
||||
if (!Array.isArray(meta.sources)) {
|
||||
// Backward compatibility for older indexes that did not persist sources.
|
||||
return ["memory"];
|
||||
}
|
||||
const normalized = Array.from(
|
||||
new Set(
|
||||
meta.sources.filter(
|
||||
(source): source is MemorySource => source === "memory" || source === "sessions",
|
||||
),
|
||||
),
|
||||
).toSorted();
|
||||
return normalized.length > 0 ? normalized : ["memory"];
|
||||
}
|
||||
|
||||
export function configuredMetaSourcesDiffer(params: {
|
||||
meta: MemoryIndexMeta;
|
||||
configuredSources: MemorySource[];
|
||||
}): boolean {
|
||||
const metaSources = normalizeMetaSources(params.meta);
|
||||
if (metaSources.length !== params.configuredSources.length) {
|
||||
return true;
|
||||
}
|
||||
return metaSources.some((source, index) => source !== params.configuredSources[index]);
|
||||
}
|
||||
|
||||
export function resolveConfiguredScopeHash(params: {
|
||||
workspaceDir: string;
|
||||
extraPaths?: string[];
|
||||
multimodal: {
|
||||
enabled: boolean;
|
||||
modalities: string[];
|
||||
maxFileBytes: number;
|
||||
};
|
||||
}): string {
|
||||
const extraPaths = normalizeExtraMemoryPaths(params.workspaceDir, params.extraPaths)
|
||||
.map((value) => value.replace(/\\/g, "/"))
|
||||
.toSorted();
|
||||
return hashText(
|
||||
JSON.stringify({
|
||||
extraPaths,
|
||||
multimodal: {
|
||||
enabled: params.multimodal.enabled,
|
||||
modalities: [...params.multimodal.modalities].toSorted(),
|
||||
maxFileBytes: params.multimodal.maxFileBytes,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function shouldRunFullMemoryReindex(params: {
|
||||
meta: MemoryIndexMeta | null;
|
||||
provider: { id: string; model: string } | null;
|
||||
providerKey?: string;
|
||||
configuredSources: MemorySource[];
|
||||
configuredScopeHash: string;
|
||||
chunkTokens: number;
|
||||
chunkOverlap: number;
|
||||
vectorReady: boolean;
|
||||
ftsTokenizer: string;
|
||||
}): boolean {
|
||||
const { meta } = params;
|
||||
return (
|
||||
!meta ||
|
||||
(params.provider ? meta.model !== params.provider.model : meta.model !== "fts-only") ||
|
||||
(params.provider ? meta.provider !== params.provider.id : meta.provider !== "none") ||
|
||||
meta.providerKey !== params.providerKey ||
|
||||
configuredMetaSourcesDiffer({
|
||||
meta,
|
||||
configuredSources: params.configuredSources,
|
||||
}) ||
|
||||
meta.scopeHash !== params.configuredScopeHash ||
|
||||
meta.chunkTokens !== params.chunkTokens ||
|
||||
meta.chunkOverlap !== params.chunkOverlap ||
|
||||
(params.vectorReady && !meta.vectorDims) ||
|
||||
(meta.ftsTokenizer ?? "unicode61") !== params.ftsTokenizer
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
export { closeAllMemoryIndexManagers, MemoryIndexManager } from "./manager.js";
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { resolveMemorySearchPreflight } from "./manager-search-preflight.js";
|
||||
|
||||
describe("memory manager search preflight", () => {
|
||||
it("skips search and provider init for blank queries", () => {
|
||||
expect(
|
||||
resolveMemorySearchPreflight({
|
||||
query: " ",
|
||||
hasIndexedContent: true,
|
||||
}),
|
||||
).toEqual({
|
||||
normalizedQuery: "",
|
||||
shouldInitializeProvider: false,
|
||||
shouldSearch: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("skips provider init when the index is empty", () => {
|
||||
expect(
|
||||
resolveMemorySearchPreflight({
|
||||
query: "hello",
|
||||
hasIndexedContent: false,
|
||||
}),
|
||||
).toEqual({
|
||||
normalizedQuery: "hello",
|
||||
shouldInitializeProvider: false,
|
||||
shouldSearch: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("allows provider init when query and indexed content are present", () => {
|
||||
expect(
|
||||
resolveMemorySearchPreflight({
|
||||
query: " hello ",
|
||||
hasIndexedContent: true,
|
||||
}),
|
||||
).toEqual({
|
||||
normalizedQuery: "hello",
|
||||
shouldInitializeProvider: true,
|
||||
shouldSearch: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
export function resolveMemorySearchPreflight(params: {
|
||||
query: string;
|
||||
hasIndexedContent: boolean;
|
||||
}):
|
||||
| {
|
||||
normalizedQuery: string;
|
||||
shouldInitializeProvider: boolean;
|
||||
shouldSearch: true;
|
||||
}
|
||||
| {
|
||||
normalizedQuery: string;
|
||||
shouldInitializeProvider: false;
|
||||
shouldSearch: false;
|
||||
} {
|
||||
const normalizedQuery = params.query.trim();
|
||||
if (!normalizedQuery) {
|
||||
return {
|
||||
normalizedQuery,
|
||||
shouldInitializeProvider: false,
|
||||
shouldSearch: false,
|
||||
};
|
||||
}
|
||||
if (!params.hasIndexedContent) {
|
||||
return {
|
||||
normalizedQuery,
|
||||
shouldInitializeProvider: false,
|
||||
shouldSearch: false,
|
||||
};
|
||||
}
|
||||
return {
|
||||
normalizedQuery,
|
||||
shouldInitializeProvider: true,
|
||||
shouldSearch: true,
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
import {
|
||||
ensureMemoryIndexSchema,
|
||||
requireNodeSqlite,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { bm25RankToScore, buildFtsQuery } from "./hybrid.js";
|
||||
import { searchKeyword } from "./manager-search.js";
|
||||
|
||||
describe("searchKeyword trigram fallback", () => {
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
|
||||
function supportsTrigramFts(): boolean {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
try {
|
||||
const result = ensureMemoryIndexSchema({
|
||||
db,
|
||||
embeddingCacheTable: "embedding_cache",
|
||||
cacheEnabled: false,
|
||||
ftsTable: "chunks_fts",
|
||||
ftsEnabled: true,
|
||||
ftsTokenizer: "trigram",
|
||||
});
|
||||
return result.ftsAvailable;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function createTrigramDb() {
|
||||
const db = new DatabaseSync(":memory:");
|
||||
const result = ensureMemoryIndexSchema({
|
||||
db,
|
||||
embeddingCacheTable: "embedding_cache",
|
||||
cacheEnabled: false,
|
||||
ftsTable: "chunks_fts",
|
||||
ftsEnabled: true,
|
||||
ftsTokenizer: "trigram",
|
||||
});
|
||||
if (!result.ftsAvailable) {
|
||||
db.close();
|
||||
throw new Error(`FTS5 trigram unavailable: ${result.ftsError ?? "unknown error"}`);
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
async function runSearch(params: {
|
||||
rows: Array<{ id: string; path: string; text: string }>;
|
||||
query: string;
|
||||
boostFallbackRanking?: boolean;
|
||||
}) {
|
||||
const db = createTrigramDb();
|
||||
try {
|
||||
const insert = db.prepare(
|
||||
"INSERT INTO chunks_fts (text, id, path, source, model, start_line, end_line) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
);
|
||||
for (const row of params.rows) {
|
||||
insert.run(row.text, row.id, row.path, "memory", "mock-embed", 1, 1);
|
||||
}
|
||||
return await searchKeyword({
|
||||
db,
|
||||
ftsTable: "chunks_fts",
|
||||
providerModel: "mock-embed",
|
||||
query: params.query,
|
||||
ftsTokenizer: "trigram",
|
||||
limit: 10,
|
||||
snippetMaxChars: 200,
|
||||
sourceFilter: { sql: "", params: [] },
|
||||
buildFtsQuery,
|
||||
bm25RankToScore,
|
||||
boostFallbackRanking: params.boostFallbackRanking,
|
||||
});
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
const itWithTrigramFts = supportsTrigramFts() ? it : it.skip;
|
||||
|
||||
itWithTrigramFts("finds short Chinese queries with substring fallback", async () => {
|
||||
const results = await runSearch({
|
||||
rows: [{ id: "1", path: "memory/zh.md", text: "今天玩成语接龙游戏" }],
|
||||
query: "成语",
|
||||
});
|
||||
expect(results.map((row) => row.id)).toContain("1");
|
||||
expect(results[0]?.textScore).toBe(1);
|
||||
});
|
||||
|
||||
itWithTrigramFts("finds short Japanese and Korean queries with substring fallback", async () => {
|
||||
const japaneseResults = await runSearch({
|
||||
rows: [{ id: "jp", path: "memory/jp.md", text: "今日はしりとり大会" }],
|
||||
query: "しり とり",
|
||||
});
|
||||
expect(japaneseResults.map((row) => row.id)).toEqual(["jp"]);
|
||||
|
||||
const koreanResults = await runSearch({
|
||||
rows: [{ id: "ko", path: "memory/ko.md", text: "오늘 끝말잇기 게임을 했다" }],
|
||||
query: "끝말",
|
||||
});
|
||||
expect(koreanResults.map((row) => row.id)).toEqual(["ko"]);
|
||||
});
|
||||
|
||||
itWithTrigramFts(
|
||||
"keeps MATCH semantics for long trigram terms while requiring short CJK substrings",
|
||||
async () => {
|
||||
const results = await runSearch({
|
||||
rows: [
|
||||
{ id: "match", path: "memory/good.md", text: "今天玩成语接龙游戏" },
|
||||
{ id: "partial", path: "memory/partial.md", text: "今天玩成语接龙" },
|
||||
],
|
||||
query: "成语接龙 游戏",
|
||||
});
|
||||
expect(results.map((row) => row.id)).toEqual(["match"]);
|
||||
expect(results[0]?.textScore).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
|
||||
itWithTrigramFts("applies fallback lexical boosts without exceeding bounded scores", async () => {
|
||||
const results = await runSearch({
|
||||
rows: [
|
||||
{
|
||||
id: "strong",
|
||||
path: "memory/project-memory-notes.md",
|
||||
text: "Project memory notes covering workspace context and retrieval behavior.",
|
||||
},
|
||||
{
|
||||
id: "weak",
|
||||
path: "memory/notes.md",
|
||||
text: "Project memory context.",
|
||||
},
|
||||
],
|
||||
query: "project memory context",
|
||||
boostFallbackRanking: true,
|
||||
});
|
||||
expect(results.map((row) => row.id)).toEqual(["weak", "strong"]);
|
||||
const rawResults = await runSearch({
|
||||
rows: [
|
||||
{
|
||||
id: "strong",
|
||||
path: "memory/project-memory-notes.md",
|
||||
text: "Project memory notes covering workspace context and retrieval behavior.",
|
||||
},
|
||||
{
|
||||
id: "weak",
|
||||
path: "memory/notes.md",
|
||||
text: "Project memory context.",
|
||||
},
|
||||
],
|
||||
query: "project memory context",
|
||||
boostFallbackRanking: false,
|
||||
});
|
||||
|
||||
const boostedById = new Map(results.map((row) => [row.id, row]));
|
||||
const rawById = new Map(rawResults.map((row) => [row.id, row]));
|
||||
expect(rawById.get("strong")?.textScore).toBeLessThan(rawById.get("weak")?.textScore ?? 0);
|
||||
expect(boostedById.get("strong")?.score).toBeGreaterThan(boostedById.get("weak")?.score ?? 0);
|
||||
expect(boostedById.get("strong")?.textScore).toBe(rawById.get("strong")?.textScore);
|
||||
expect(boostedById.get("weak")?.textScore).toBe(rawById.get("weak")?.textScore);
|
||||
expect(boostedById.get("strong")?.score).toBeLessThanOrEqual(1);
|
||||
expect(boostedById.get("weak")?.score).toBeLessThanOrEqual(1);
|
||||
});
|
||||
|
||||
itWithTrigramFts("does not overweight repeated query tokens in fallback scoring", async () => {
|
||||
const unique = await runSearch({
|
||||
rows: [{ id: "1", path: "memory/project.md", text: "Project memory context." }],
|
||||
query: "project memory context",
|
||||
boostFallbackRanking: true,
|
||||
});
|
||||
const repeated = await runSearch({
|
||||
rows: [{ id: "1", path: "memory/project.md", text: "Project memory context." }],
|
||||
query: "project project project memory context",
|
||||
boostFallbackRanking: true,
|
||||
});
|
||||
|
||||
expect(repeated[0]?.score).toBe(unique[0]?.score);
|
||||
});
|
||||
});
|
||||
308
openclaw/extensions/memory-core/src/memory/manager-search.ts
Normal file
308
openclaw/extensions/memory-core/src/memory/manager-search.ts
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { truncateUtf16Safe } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import {
|
||||
cosineSimilarity,
|
||||
parseEmbedding,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
const vectorToBlob = (embedding: number[]): Buffer =>
|
||||
Buffer.from(new Float32Array(embedding).buffer);
|
||||
const FTS_QUERY_TOKEN_RE = /[\p{L}\p{N}_]+/gu;
|
||||
const SHORT_CJK_TRIGRAM_RE = /[\u3040-\u30ff\u3400-\u9fff\uac00-\ud7af\u3131-\u3163]/u;
|
||||
|
||||
export type SearchSource = string;
|
||||
|
||||
export type SearchRowResult = {
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
snippet: string;
|
||||
source: SearchSource;
|
||||
};
|
||||
|
||||
function normalizeSearchTokens(raw: string): string[] {
|
||||
return (
|
||||
raw
|
||||
.match(FTS_QUERY_TOKEN_RE)
|
||||
?.map((token) => token.trim().toLowerCase())
|
||||
.filter(Boolean) ?? []
|
||||
);
|
||||
}
|
||||
|
||||
function scoreFallbackKeywordResult(params: {
|
||||
query: string;
|
||||
path: string;
|
||||
text: string;
|
||||
ftsScore: number;
|
||||
}): number {
|
||||
const queryTokens = [...new Set(normalizeSearchTokens(params.query))];
|
||||
if (queryTokens.length === 0) {
|
||||
return params.ftsScore;
|
||||
}
|
||||
|
||||
const textTokens = normalizeSearchTokens(params.text);
|
||||
const textTokenSet = new Set(textTokens);
|
||||
const pathLower = params.path.toLowerCase();
|
||||
const overlap = queryTokens.filter((token) => textTokenSet.has(token)).length;
|
||||
const uniqueQueryOverlap = overlap / Math.max(new Set(queryTokens).size, 1);
|
||||
const density = overlap / Math.max(textTokenSet.size, 1);
|
||||
const pathBoost = queryTokens.reduce(
|
||||
(score, token) => score + (pathLower.includes(token) ? 0.18 : 0),
|
||||
0,
|
||||
);
|
||||
const textLengthBoost = Math.min(params.text.length / 160, 0.18);
|
||||
|
||||
const lexicalBoost = uniqueQueryOverlap * 0.45 + density * 0.2 + pathBoost + textLengthBoost;
|
||||
return Math.min(1, params.ftsScore + lexicalBoost);
|
||||
}
|
||||
|
||||
function escapeLikePattern(term: string): string {
|
||||
return term.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
|
||||
}
|
||||
|
||||
function buildMatchQueryFromTerms(terms: string[]): string | null {
|
||||
if (terms.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const quoted = terms.map((term) => `"${term.replaceAll('"', "")}"`);
|
||||
return quoted.join(" AND ");
|
||||
}
|
||||
|
||||
function planKeywordSearch(params: {
|
||||
query: string;
|
||||
ftsTokenizer?: "unicode61" | "trigram";
|
||||
buildFtsQuery: (raw: string) => string | null;
|
||||
}): { matchQuery: string | null; substringTerms: string[] } {
|
||||
if (params.ftsTokenizer !== "trigram") {
|
||||
return {
|
||||
matchQuery: params.buildFtsQuery(params.query),
|
||||
substringTerms: [],
|
||||
};
|
||||
}
|
||||
|
||||
const tokens =
|
||||
params.query
|
||||
.match(FTS_QUERY_TOKEN_RE)
|
||||
?.map((token) => token.trim())
|
||||
.filter(Boolean) ?? [];
|
||||
if (tokens.length === 0) {
|
||||
return { matchQuery: null, substringTerms: [] };
|
||||
}
|
||||
|
||||
const matchTerms: string[] = [];
|
||||
const substringTerms: string[] = [];
|
||||
for (const token of tokens) {
|
||||
if (SHORT_CJK_TRIGRAM_RE.test(token) && Array.from(token).length < 3) {
|
||||
substringTerms.push(token);
|
||||
continue;
|
||||
}
|
||||
matchTerms.push(token);
|
||||
}
|
||||
|
||||
return {
|
||||
matchQuery: buildMatchQueryFromTerms(matchTerms),
|
||||
substringTerms,
|
||||
};
|
||||
}
|
||||
|
||||
export async function searchVector(params: {
|
||||
db: DatabaseSync;
|
||||
vectorTable: string;
|
||||
providerModel: string;
|
||||
queryVec: number[];
|
||||
limit: number;
|
||||
snippetMaxChars: number;
|
||||
ensureVectorReady: (dimensions: number) => Promise<boolean>;
|
||||
sourceFilterVec: { sql: string; params: SearchSource[] };
|
||||
sourceFilterChunks: { sql: string; params: SearchSource[] };
|
||||
}): Promise<SearchRowResult[]> {
|
||||
if (params.queryVec.length === 0 || params.limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
if (await params.ensureVectorReady(params.queryVec.length)) {
|
||||
const rows = params.db
|
||||
.prepare(
|
||||
`SELECT c.id, c.path, c.start_line, c.end_line, c.text,\n` +
|
||||
` c.source,\n` +
|
||||
` vec_distance_cosine(v.embedding, ?) AS dist\n` +
|
||||
` FROM ${params.vectorTable} v\n` +
|
||||
` JOIN chunks c ON c.id = v.id\n` +
|
||||
` WHERE c.model = ?${params.sourceFilterVec.sql}\n` +
|
||||
` ORDER BY dist ASC\n` +
|
||||
` LIMIT ?`,
|
||||
)
|
||||
.all(
|
||||
vectorToBlob(params.queryVec),
|
||||
params.providerModel,
|
||||
...params.sourceFilterVec.params,
|
||||
params.limit,
|
||||
) as Array<{
|
||||
id: string;
|
||||
path: string;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
text: string;
|
||||
source: SearchSource;
|
||||
dist: number;
|
||||
}>;
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
path: row.path,
|
||||
startLine: row.start_line,
|
||||
endLine: row.end_line,
|
||||
score: 1 - row.dist,
|
||||
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
||||
source: row.source,
|
||||
}));
|
||||
}
|
||||
|
||||
const candidates = listChunks({
|
||||
db: params.db,
|
||||
providerModel: params.providerModel,
|
||||
sourceFilter: params.sourceFilterChunks,
|
||||
});
|
||||
const scored = candidates
|
||||
.map((chunk) => ({
|
||||
chunk,
|
||||
score: cosineSimilarity(params.queryVec, chunk.embedding),
|
||||
}))
|
||||
.filter((entry) => Number.isFinite(entry.score));
|
||||
return scored
|
||||
.toSorted((a, b) => b.score - a.score)
|
||||
.slice(0, params.limit)
|
||||
.map((entry) => ({
|
||||
id: entry.chunk.id,
|
||||
path: entry.chunk.path,
|
||||
startLine: entry.chunk.startLine,
|
||||
endLine: entry.chunk.endLine,
|
||||
score: entry.score,
|
||||
snippet: truncateUtf16Safe(entry.chunk.text, params.snippetMaxChars),
|
||||
source: entry.chunk.source,
|
||||
}));
|
||||
}
|
||||
|
||||
export function listChunks(params: {
|
||||
db: DatabaseSync;
|
||||
providerModel: string;
|
||||
sourceFilter: { sql: string; params: SearchSource[] };
|
||||
}): Array<{
|
||||
id: string;
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
text: string;
|
||||
embedding: number[];
|
||||
source: SearchSource;
|
||||
}> {
|
||||
const rows = params.db
|
||||
.prepare(
|
||||
`SELECT id, path, start_line, end_line, text, embedding, source\n` +
|
||||
` FROM chunks\n` +
|
||||
` WHERE model = ?${params.sourceFilter.sql}`,
|
||||
)
|
||||
.all(params.providerModel, ...params.sourceFilter.params) as Array<{
|
||||
id: string;
|
||||
path: string;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
text: string;
|
||||
embedding: string;
|
||||
source: SearchSource;
|
||||
}>;
|
||||
|
||||
return rows.map((row) => ({
|
||||
id: row.id,
|
||||
path: row.path,
|
||||
startLine: row.start_line,
|
||||
endLine: row.end_line,
|
||||
text: row.text,
|
||||
embedding: parseEmbedding(row.embedding),
|
||||
source: row.source,
|
||||
}));
|
||||
}
|
||||
|
||||
export async function searchKeyword(params: {
|
||||
db: DatabaseSync;
|
||||
ftsTable: string;
|
||||
providerModel: string | undefined;
|
||||
query: string;
|
||||
ftsTokenizer?: "unicode61" | "trigram";
|
||||
limit: number;
|
||||
snippetMaxChars: number;
|
||||
sourceFilter: { sql: string; params: SearchSource[] };
|
||||
buildFtsQuery: (raw: string) => string | null;
|
||||
bm25RankToScore: (rank: number) => number;
|
||||
boostFallbackRanking?: boolean;
|
||||
}): Promise<Array<SearchRowResult & { textScore: number }>> {
|
||||
if (params.limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
const plan = planKeywordSearch({
|
||||
query: params.query,
|
||||
ftsTokenizer: params.ftsTokenizer,
|
||||
buildFtsQuery: params.buildFtsQuery,
|
||||
});
|
||||
if (!plan.matchQuery && plan.substringTerms.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// When providerModel is undefined (FTS-only mode), search all models
|
||||
const modelClause = params.providerModel ? " AND model = ?" : "";
|
||||
const modelParams = params.providerModel ? [params.providerModel] : [];
|
||||
const substringClause = plan.substringTerms.map(() => " AND text LIKE ? ESCAPE '\\'").join("");
|
||||
const substringParams = plan.substringTerms.map((term) => `%${escapeLikePattern(term)}%`);
|
||||
const whereClause = plan.matchQuery
|
||||
? `${params.ftsTable} MATCH ?${substringClause}${modelClause}${params.sourceFilter.sql}`
|
||||
: `1=1${substringClause}${modelClause}${params.sourceFilter.sql}`;
|
||||
const queryParams = [
|
||||
...(plan.matchQuery ? [plan.matchQuery] : []),
|
||||
...substringParams,
|
||||
...modelParams,
|
||||
...params.sourceFilter.params,
|
||||
params.limit,
|
||||
];
|
||||
const rankExpression = plan.matchQuery ? `bm25(${params.ftsTable})` : "0";
|
||||
|
||||
const rows = params.db
|
||||
.prepare(
|
||||
`SELECT id, path, source, start_line, end_line, text,\n` +
|
||||
` ${rankExpression} AS rank\n` +
|
||||
` FROM ${params.ftsTable}\n` +
|
||||
` WHERE ${whereClause}\n` +
|
||||
` ORDER BY rank ASC\n` +
|
||||
` LIMIT ?`,
|
||||
)
|
||||
.all(...queryParams) as Array<{
|
||||
id: string;
|
||||
path: string;
|
||||
source: SearchSource;
|
||||
start_line: number;
|
||||
end_line: number;
|
||||
text: string;
|
||||
rank: number;
|
||||
}>;
|
||||
|
||||
return rows.map((row) => {
|
||||
const textScore = plan.matchQuery ? params.bm25RankToScore(row.rank) : 1;
|
||||
const score = params.boostFallbackRanking
|
||||
? scoreFallbackKeywordResult({
|
||||
query: params.query,
|
||||
path: row.path,
|
||||
text: row.text,
|
||||
ftsScore: textScore,
|
||||
})
|
||||
: textScore;
|
||||
return {
|
||||
id: row.id,
|
||||
path: row.path,
|
||||
startLine: row.start_line,
|
||||
endLine: row.end_line,
|
||||
score,
|
||||
textScore,
|
||||
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
||||
source: row.source,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
export function shouldSyncSessionsForReindex(params: {
|
||||
hasSessionSource: boolean;
|
||||
sessionsDirty: boolean;
|
||||
dirtySessionFileCount: number;
|
||||
sync?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessionFiles?: string[];
|
||||
};
|
||||
needsFullReindex?: boolean;
|
||||
}): boolean {
|
||||
if (!params.hasSessionSource) {
|
||||
return false;
|
||||
}
|
||||
if (params.sync?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) {
|
||||
return true;
|
||||
}
|
||||
if (params.sync?.force) {
|
||||
return true;
|
||||
}
|
||||
if (params.needsFullReindex) {
|
||||
return true;
|
||||
}
|
||||
const reason = params.sync?.reason;
|
||||
if (reason === "session-start" || reason === "watch") {
|
||||
return false;
|
||||
}
|
||||
return params.sessionsDirty && params.dirtySessionFileCount > 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { resolveMemorySessionSyncPlan } from "./manager-session-sync-state.js";
|
||||
|
||||
describe("memory session sync state", () => {
|
||||
it("tracks active paths and bulk hashes for full scans", () => {
|
||||
const plan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: false,
|
||||
files: ["/tmp/a.jsonl", "/tmp/b.jsonl"],
|
||||
targetSessionFiles: null,
|
||||
sessionsDirtyFiles: new Set(),
|
||||
existingRows: [
|
||||
{ path: "sessions/a.jsonl", hash: "hash-a" },
|
||||
{ path: "sessions/b.jsonl", hash: "hash-b" },
|
||||
],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(true);
|
||||
expect(plan.activePaths).toEqual(new Set(["sessions/a.jsonl", "sessions/b.jsonl"]));
|
||||
expect(plan.existingRows).toEqual([
|
||||
{ path: "sessions/a.jsonl", hash: "hash-a" },
|
||||
{ path: "sessions/b.jsonl", hash: "hash-b" },
|
||||
]);
|
||||
expect(plan.existingHashes).toEqual(
|
||||
new Map([
|
||||
["sessions/a.jsonl", "hash-a"],
|
||||
["sessions/b.jsonl", "hash-b"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("treats targeted session syncs as refresh-only and skips unrelated pruning", () => {
|
||||
const plan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: false,
|
||||
files: ["/tmp/targeted-first.jsonl"],
|
||||
targetSessionFiles: new Set(["/tmp/targeted-first.jsonl"]),
|
||||
sessionsDirtyFiles: new Set(["/tmp/targeted-first.jsonl"]),
|
||||
existingRows: [
|
||||
{ path: "sessions/targeted-first.jsonl", hash: "hash-first" },
|
||||
{ path: "sessions/targeted-second.jsonl", hash: "hash-second" },
|
||||
],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(true);
|
||||
expect(plan.activePaths).toBeNull();
|
||||
expect(plan.existingRows).toBeNull();
|
||||
expect(plan.existingHashes).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps dirty-only incremental mode when no targeted sync is requested", () => {
|
||||
const plan = resolveMemorySessionSyncPlan({
|
||||
needsFullReindex: false,
|
||||
files: ["/tmp/incremental.jsonl"],
|
||||
targetSessionFiles: null,
|
||||
sessionsDirtyFiles: new Set(["/tmp/incremental.jsonl"]),
|
||||
existingRows: [],
|
||||
sessionPathForFile: (file) => `sessions/${file.split("/").at(-1)}`,
|
||||
});
|
||||
|
||||
expect(plan.indexAll).toBe(false);
|
||||
expect(plan.activePaths).toEqual(new Set(["sessions/incremental.jsonl"]));
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
import { type MemorySourceFileStateRow } from "./manager-source-state.js";
|
||||
|
||||
export function resolveMemorySessionSyncPlan(params: {
|
||||
needsFullReindex: boolean;
|
||||
files: string[];
|
||||
targetSessionFiles: Set<string> | null;
|
||||
sessionsDirtyFiles: Set<string>;
|
||||
existingRows?: MemorySourceFileStateRow[] | null;
|
||||
sessionPathForFile: (file: string) => string;
|
||||
}): {
|
||||
activePaths: Set<string> | null;
|
||||
existingRows: MemorySourceFileStateRow[] | null;
|
||||
existingHashes: Map<string, string> | null;
|
||||
indexAll: boolean;
|
||||
} {
|
||||
const activePaths = params.targetSessionFiles
|
||||
? null
|
||||
: new Set(params.files.map((file) => params.sessionPathForFile(file)));
|
||||
const existingRows = activePaths === null ? null : (params.existingRows ?? []);
|
||||
return {
|
||||
activePaths,
|
||||
existingRows,
|
||||
existingHashes: existingRows ? new Map(existingRows.map((row) => [row.path, row.hash])) : null,
|
||||
indexAll:
|
||||
params.needsFullReindex ||
|
||||
Boolean(params.targetSessionFiles) ||
|
||||
params.sessionsDirtyFiles.size === 0,
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
loadMemorySourceFileState,
|
||||
MEMORY_SOURCE_FILE_HASH_SQL,
|
||||
MEMORY_SOURCE_FILE_STATE_SQL,
|
||||
resolveMemorySourceExistingHash,
|
||||
} from "./manager-source-state.js";
|
||||
|
||||
describe("memory source state", () => {
|
||||
it("loads source hashes with one bulk query", () => {
|
||||
const calls: Array<{ sql: string; args: unknown[] }> = [];
|
||||
const state = loadMemorySourceFileState({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: (...args) => {
|
||||
calls.push({ sql, args });
|
||||
return [
|
||||
{ path: "memory/one.md", hash: "hash-1" },
|
||||
{ path: "memory/two.md", hash: "hash-2" },
|
||||
];
|
||||
},
|
||||
get: () => undefined,
|
||||
}),
|
||||
},
|
||||
source: "memory",
|
||||
});
|
||||
|
||||
expect(calls).toEqual([{ sql: MEMORY_SOURCE_FILE_STATE_SQL, args: ["memory"] }]);
|
||||
expect(state.rows).toEqual([
|
||||
{ path: "memory/one.md", hash: "hash-1" },
|
||||
{ path: "memory/two.md", hash: "hash-2" },
|
||||
]);
|
||||
expect(state.hashes).toEqual(
|
||||
new Map([
|
||||
["memory/one.md", "hash-1"],
|
||||
["memory/two.md", "hash-2"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("uses bulk snapshot hashes when present", () => {
|
||||
const calls: Array<{ sql: string; args: unknown[] }> = [];
|
||||
const hash = resolveMemorySourceExistingHash({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: () => [],
|
||||
get: (...args) => {
|
||||
calls.push({ sql, args });
|
||||
return { hash: "unexpected" };
|
||||
},
|
||||
}),
|
||||
},
|
||||
source: "sessions",
|
||||
path: "sessions/thread.jsonl",
|
||||
existingHashes: new Map([["sessions/thread.jsonl", "hash-from-snapshot"]]),
|
||||
});
|
||||
|
||||
expect(hash).toBe("hash-from-snapshot");
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("falls back to per-file lookups without a bulk snapshot", () => {
|
||||
const calls: Array<{ sql: string; args: unknown[] }> = [];
|
||||
const hash = resolveMemorySourceExistingHash({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: () => [],
|
||||
get: (...args) => {
|
||||
calls.push({ sql, args });
|
||||
return { hash: "hash-from-row" };
|
||||
},
|
||||
}),
|
||||
},
|
||||
source: "sessions",
|
||||
path: "sessions/thread.jsonl",
|
||||
existingHashes: null,
|
||||
});
|
||||
|
||||
expect(hash).toBe("hash-from-row");
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
sql: MEMORY_SOURCE_FILE_HASH_SQL,
|
||||
args: ["sessions/thread.jsonl", "sessions"],
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
import type { SQLInputValue } from "node:sqlite";
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
export type MemorySourceFileStateRow = {
|
||||
path: string;
|
||||
hash: string;
|
||||
};
|
||||
|
||||
type MemorySourceStateDb = {
|
||||
prepare: (sql: string) => {
|
||||
all: (...args: SQLInputValue[]) => unknown;
|
||||
get: (...args: SQLInputValue[]) => unknown;
|
||||
};
|
||||
};
|
||||
|
||||
export const MEMORY_SOURCE_FILE_STATE_SQL = `SELECT path, hash FROM files WHERE source = ?`;
|
||||
export const MEMORY_SOURCE_FILE_HASH_SQL = `SELECT hash FROM files WHERE path = ? AND source = ?`;
|
||||
|
||||
export function loadMemorySourceFileState(params: {
|
||||
db: MemorySourceStateDb;
|
||||
source: MemorySource;
|
||||
}): {
|
||||
rows: MemorySourceFileStateRow[];
|
||||
hashes: Map<string, string>;
|
||||
} {
|
||||
const rows = params.db.prepare(MEMORY_SOURCE_FILE_STATE_SQL).all(params.source) as
|
||||
| MemorySourceFileStateRow[]
|
||||
| undefined;
|
||||
const normalizedRows = rows ?? [];
|
||||
return {
|
||||
rows: normalizedRows,
|
||||
hashes: new Map(normalizedRows.map((row) => [row.path, row.hash])),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveMemorySourceExistingHash(params: {
|
||||
db: MemorySourceStateDb;
|
||||
source: MemorySource;
|
||||
path: string;
|
||||
existingHashes?: Map<string, string> | null;
|
||||
}): string | undefined {
|
||||
if (params.existingHashes) {
|
||||
return params.existingHashes.get(params.path);
|
||||
}
|
||||
return (
|
||||
params.db.prepare(MEMORY_SOURCE_FILE_HASH_SQL).get(params.path, params.source) as
|
||||
| { hash: string }
|
||||
| undefined
|
||||
)?.hash;
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
import type { SQLInputValue } from "node:sqlite";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
collectMemoryStatusAggregate,
|
||||
MEMORY_STATUS_AGGREGATE_SQL,
|
||||
resolveInitialMemoryDirty,
|
||||
resolveStatusProviderInfo,
|
||||
} from "./manager-status-state.js";
|
||||
|
||||
describe("memory manager status state", () => {
|
||||
it("keeps memory clean for status-only managers after prior indexing", () => {
|
||||
expect(
|
||||
resolveInitialMemoryDirty({
|
||||
hasMemorySource: true,
|
||||
statusOnly: true,
|
||||
hasIndexedMeta: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("marks status-only managers dirty when no prior index metadata exists", () => {
|
||||
expect(
|
||||
resolveInitialMemoryDirty({
|
||||
hasMemorySource: true,
|
||||
statusOnly: true,
|
||||
hasIndexedMeta: false,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("reports the requested provider before provider initialization", () => {
|
||||
expect(
|
||||
resolveStatusProviderInfo({
|
||||
provider: null,
|
||||
providerInitialized: false,
|
||||
requestedProvider: "openai",
|
||||
configuredModel: "mock-embed",
|
||||
}),
|
||||
).toEqual({
|
||||
provider: "openai",
|
||||
model: "mock-embed",
|
||||
searchMode: "hybrid",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports fts-only mode when initialization finished without a provider", () => {
|
||||
expect(
|
||||
resolveStatusProviderInfo({
|
||||
provider: null,
|
||||
providerInitialized: true,
|
||||
requestedProvider: "openai",
|
||||
configuredModel: "mock-embed",
|
||||
}),
|
||||
).toEqual({
|
||||
provider: "none",
|
||||
model: undefined,
|
||||
searchMode: "fts-only",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses one aggregation query for status counts and source breakdowns", () => {
|
||||
const calls: Array<{ sql: string; params: SQLInputValue[] }> = [];
|
||||
const aggregate = collectMemoryStatusAggregate({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: (...params) => {
|
||||
calls.push({ sql, params });
|
||||
return [
|
||||
{ kind: "files" as const, source: "memory" as const, c: 2 },
|
||||
{ kind: "chunks" as const, source: "memory" as const, c: 5 },
|
||||
{ kind: "files" as const, source: "sessions" as const, c: 1 },
|
||||
{ kind: "chunks" as const, source: "sessions" as const, c: 3 },
|
||||
];
|
||||
},
|
||||
}),
|
||||
},
|
||||
sources: ["memory", "sessions"],
|
||||
sourceFilterSql: " AND source IN (?, ?)",
|
||||
sourceFilterParams: ["memory", "sessions"],
|
||||
});
|
||||
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
sql: MEMORY_STATUS_AGGREGATE_SQL.replaceAll("__FILTER__", " AND source IN (?, ?)"),
|
||||
params: ["memory", "sessions", "memory", "sessions"],
|
||||
},
|
||||
]);
|
||||
expect(aggregate).toEqual({
|
||||
files: 3,
|
||||
chunks: 8,
|
||||
sourceCounts: [
|
||||
{ source: "memory", files: 2, chunks: 5 },
|
||||
{ source: "sessions", files: 1, chunks: 3 },
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
import type { SQLInputValue } from "node:sqlite";
|
||||
import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
type StatusProvider = {
|
||||
id: string;
|
||||
model: string;
|
||||
};
|
||||
|
||||
type StatusAggregateRow = {
|
||||
kind: "files" | "chunks";
|
||||
source: MemorySource;
|
||||
c: number;
|
||||
};
|
||||
|
||||
type StatusAggregateDb = {
|
||||
prepare: (sql: string) => {
|
||||
all: (...args: SQLInputValue[]) => StatusAggregateRow[];
|
||||
};
|
||||
};
|
||||
|
||||
export const MEMORY_STATUS_AGGREGATE_SQL =
|
||||
`SELECT 'files' AS kind, source, COUNT(*) as c FROM files WHERE 1=1__FILTER__ GROUP BY source\n` +
|
||||
`UNION ALL\n` +
|
||||
`SELECT 'chunks' AS kind, source, COUNT(*) as c FROM chunks WHERE 1=1__FILTER__ GROUP BY source`;
|
||||
|
||||
export function resolveInitialMemoryDirty(params: {
|
||||
hasMemorySource: boolean;
|
||||
statusOnly: boolean;
|
||||
hasIndexedMeta: boolean;
|
||||
}): boolean {
|
||||
return params.hasMemorySource && (params.statusOnly ? !params.hasIndexedMeta : true);
|
||||
}
|
||||
|
||||
export function resolveStatusProviderInfo(params: {
|
||||
provider: StatusProvider | null;
|
||||
providerInitialized: boolean;
|
||||
requestedProvider: string;
|
||||
configuredModel?: string;
|
||||
}): {
|
||||
provider: string;
|
||||
model?: string;
|
||||
searchMode: "hybrid" | "fts-only";
|
||||
} {
|
||||
if (params.provider) {
|
||||
return {
|
||||
provider: params.provider.id,
|
||||
model: params.provider.model,
|
||||
searchMode: "hybrid",
|
||||
};
|
||||
}
|
||||
if (params.providerInitialized) {
|
||||
return {
|
||||
provider: "none",
|
||||
model: undefined,
|
||||
searchMode: "fts-only",
|
||||
};
|
||||
}
|
||||
return {
|
||||
provider: params.requestedProvider,
|
||||
model: params.configuredModel || undefined,
|
||||
searchMode: "hybrid",
|
||||
};
|
||||
}
|
||||
|
||||
export function collectMemoryStatusAggregate(params: {
|
||||
db: StatusAggregateDb;
|
||||
sources: Iterable<MemorySource>;
|
||||
sourceFilterSql?: string;
|
||||
sourceFilterParams?: MemorySource[];
|
||||
}): {
|
||||
files: number;
|
||||
chunks: number;
|
||||
sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>;
|
||||
} {
|
||||
const sources = Array.from(params.sources);
|
||||
const bySource = new Map<MemorySource, { files: number; chunks: number }>();
|
||||
for (const source of sources) {
|
||||
bySource.set(source, { files: 0, chunks: 0 });
|
||||
}
|
||||
const sourceFilterSql = params.sourceFilterSql ?? "";
|
||||
const sourceFilterParams = params.sourceFilterParams ?? [];
|
||||
const aggregateRows = params.db
|
||||
.prepare(MEMORY_STATUS_AGGREGATE_SQL.replaceAll("__FILTER__", sourceFilterSql))
|
||||
.all(...sourceFilterParams, ...sourceFilterParams);
|
||||
let files = 0;
|
||||
let chunks = 0;
|
||||
for (const row of aggregateRows) {
|
||||
const count = row.c ?? 0;
|
||||
const entry = bySource.get(row.source) ?? { files: 0, chunks: 0 };
|
||||
if (row.kind === "files") {
|
||||
entry.files = count;
|
||||
files += count;
|
||||
} else {
|
||||
entry.chunks = count;
|
||||
chunks += count;
|
||||
}
|
||||
bySource.set(row.source, entry);
|
||||
}
|
||||
return {
|
||||
files,
|
||||
chunks,
|
||||
sourceCounts: sources.map((source) => Object.assign({ source }, bySource.get(source)!)),
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,189 @@
|
|||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
createSubsystemLogger,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import type { MemorySyncProgressUpdate } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
|
||||
export type MemoryReadonlyRecoveryState = {
|
||||
closed: boolean;
|
||||
db: DatabaseSync;
|
||||
vector: {
|
||||
dims?: number;
|
||||
};
|
||||
readonlyRecoveryAttempts: number;
|
||||
readonlyRecoverySuccesses: number;
|
||||
readonlyRecoveryFailures: number;
|
||||
readonlyRecoveryLastError?: string;
|
||||
runSync: (params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}) => Promise<void>;
|
||||
openDatabase: () => DatabaseSync;
|
||||
resetVectorState: () => void;
|
||||
ensureSchema: () => void;
|
||||
readMeta: () => { vectorDims?: number } | undefined;
|
||||
};
|
||||
|
||||
export function isMemoryReadonlyDbError(err: unknown): boolean {
|
||||
const readonlyPattern =
|
||||
/attempt to write a readonly database|database is read-only|SQLITE_READONLY/i;
|
||||
const messages = new Set<string>();
|
||||
|
||||
const pushValue = (value: unknown): void => {
|
||||
if (typeof value !== "string") {
|
||||
return;
|
||||
}
|
||||
const normalized = value.trim();
|
||||
if (!normalized) {
|
||||
return;
|
||||
}
|
||||
messages.add(normalized);
|
||||
};
|
||||
|
||||
pushValue(formatErrorMessage(err));
|
||||
if (err && typeof err === "object") {
|
||||
const record = err as Record<string, unknown>;
|
||||
pushValue(record.message);
|
||||
pushValue(record.code);
|
||||
pushValue(record.name);
|
||||
if (record.cause && typeof record.cause === "object") {
|
||||
const cause = record.cause as Record<string, unknown>;
|
||||
pushValue(cause.message);
|
||||
pushValue(cause.code);
|
||||
pushValue(cause.name);
|
||||
}
|
||||
}
|
||||
|
||||
return [...messages].some((value) => readonlyPattern.test(value));
|
||||
}
|
||||
|
||||
export function extractMemoryErrorReason(err: unknown): string {
|
||||
if (err instanceof Error && err.message.trim()) {
|
||||
return err.message;
|
||||
}
|
||||
if (err && typeof err === "object") {
|
||||
const record = err as Record<string, unknown>;
|
||||
if (typeof record.message === "string" && record.message.trim()) {
|
||||
return record.message;
|
||||
}
|
||||
if (typeof record.code === "string" && record.code.trim()) {
|
||||
return record.code;
|
||||
}
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
|
||||
export async function runMemorySyncWithReadonlyRecovery(
|
||||
state: MemoryReadonlyRecoveryState,
|
||||
params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
},
|
||||
): Promise<void> {
|
||||
try {
|
||||
await state.runSync(params);
|
||||
return;
|
||||
} catch (err) {
|
||||
if (!isMemoryReadonlyDbError(err) || state.closed) {
|
||||
throw err;
|
||||
}
|
||||
const reason = extractMemoryErrorReason(err);
|
||||
state.readonlyRecoveryAttempts += 1;
|
||||
state.readonlyRecoveryLastError = reason;
|
||||
log.warn(`memory sync readonly handle detected; reopening sqlite connection`, { reason });
|
||||
try {
|
||||
state.db.close();
|
||||
} catch {}
|
||||
const previousVectorDims = state.vector.dims;
|
||||
state.db = state.openDatabase();
|
||||
state.resetVectorState();
|
||||
state.ensureSchema();
|
||||
const meta = state.readMeta();
|
||||
state.vector.dims = meta?.vectorDims ?? previousVectorDims;
|
||||
try {
|
||||
await state.runSync(params);
|
||||
state.readonlyRecoverySuccesses += 1;
|
||||
} catch (retryErr) {
|
||||
state.readonlyRecoveryFailures += 1;
|
||||
throw retryErr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function enqueueMemoryTargetedSessionSync(
|
||||
state: {
|
||||
isClosed: () => boolean;
|
||||
getSyncing: () => Promise<void> | null;
|
||||
getQueuedSessionFiles: () => Set<string>;
|
||||
getQueuedSessionSync: () => Promise<void> | null;
|
||||
setQueuedSessionSync: (value: Promise<void> | null) => void;
|
||||
sync: (params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}) => Promise<void>;
|
||||
},
|
||||
sessionFiles?: string[],
|
||||
): Promise<void> {
|
||||
const queuedSessionFiles = state.getQueuedSessionFiles();
|
||||
for (const sessionFile of sessionFiles ?? []) {
|
||||
const trimmed = sessionFile.trim();
|
||||
if (trimmed) {
|
||||
queuedSessionFiles.add(trimmed);
|
||||
}
|
||||
}
|
||||
if (queuedSessionFiles.size === 0) {
|
||||
return state.getSyncing() ?? Promise.resolve();
|
||||
}
|
||||
if (!state.getQueuedSessionSync()) {
|
||||
state.setQueuedSessionSync(
|
||||
(async () => {
|
||||
try {
|
||||
await state.getSyncing()?.catch(() => undefined);
|
||||
while (!state.isClosed() && state.getQueuedSessionFiles().size > 0) {
|
||||
const pendingSessionFiles = Array.from(state.getQueuedSessionFiles());
|
||||
state.getQueuedSessionFiles().clear();
|
||||
await state.sync({
|
||||
reason: "queued-session-files",
|
||||
sessionFiles: pendingSessionFiles,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
state.setQueuedSessionSync(null);
|
||||
}
|
||||
})(),
|
||||
);
|
||||
}
|
||||
return state.getQueuedSessionSync() ?? Promise.resolve();
|
||||
}
|
||||
|
||||
export function _createMemorySyncControlConfigForTests(
|
||||
workspaceDir: string,
|
||||
indexPath: string,
|
||||
): OpenClawConfig {
|
||||
return {
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: workspaceDir,
|
||||
memorySearch: {
|
||||
provider: "openai",
|
||||
model: "mock-embed",
|
||||
store: { path: indexPath, vector: { enabled: false } },
|
||||
cache: { enabled: false },
|
||||
query: { minScore: 0, hybrid: { enabled: false } },
|
||||
sync: { watch: false, onSessionStart: false, onSearch: false },
|
||||
},
|
||||
},
|
||||
list: [{ id: "main", default: true }],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
}
|
||||
1334
openclaw/extensions/memory-core/src/memory/manager-sync-ops.ts
Normal file
1334
openclaw/extensions/memory-core/src/memory/manager-sync-ops.ts
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,78 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
clearMemorySyncedSessionFiles,
|
||||
runMemoryTargetedSessionSync,
|
||||
} from "./manager-targeted-sync.js";
|
||||
|
||||
describe("memory targeted session sync", () => {
|
||||
it("preserves unrelated dirty sessions after targeted cleanup", () => {
|
||||
const secondSessionPath = "/tmp/targeted-dirty-second.jsonl";
|
||||
const sessionsDirtyFiles = new Set(["/tmp/targeted-dirty-first.jsonl", secondSessionPath]);
|
||||
|
||||
const sessionsDirty = clearMemorySyncedSessionFiles({
|
||||
sessionsDirtyFiles,
|
||||
targetSessionFiles: ["/tmp/targeted-dirty-first.jsonl"],
|
||||
});
|
||||
|
||||
expect(sessionsDirtyFiles.has(secondSessionPath)).toBe(true);
|
||||
expect(sessionsDirty).toBe(true);
|
||||
});
|
||||
|
||||
it("runs a full reindex after fallback activates during targeted sync", async () => {
|
||||
const activateFallbackProvider = vi.fn(async () => true);
|
||||
const runSafeReindex = vi.fn(async () => {});
|
||||
const runUnsafeReindex = vi.fn(async () => {});
|
||||
|
||||
await runMemoryTargetedSessionSync({
|
||||
hasSessionSource: true,
|
||||
targetSessionFiles: new Set(["/tmp/targeted-fallback.jsonl"]),
|
||||
reason: "post-compaction",
|
||||
progress: undefined,
|
||||
useUnsafeReindex: false,
|
||||
sessionsDirtyFiles: new Set(),
|
||||
syncSessionFiles: async () => {
|
||||
throw new Error("embedding backend failed");
|
||||
},
|
||||
shouldFallbackOnError: () => true,
|
||||
activateFallbackProvider,
|
||||
runSafeReindex,
|
||||
runUnsafeReindex,
|
||||
});
|
||||
|
||||
expect(activateFallbackProvider).toHaveBeenCalledWith("embedding backend failed");
|
||||
expect(runSafeReindex).toHaveBeenCalledWith({
|
||||
reason: "post-compaction",
|
||||
force: true,
|
||||
progress: undefined,
|
||||
});
|
||||
expect(runUnsafeReindex).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the unsafe reindex path when enabled", async () => {
|
||||
const runSafeReindex = vi.fn(async () => {});
|
||||
const runUnsafeReindex = vi.fn(async () => {});
|
||||
|
||||
await runMemoryTargetedSessionSync({
|
||||
hasSessionSource: true,
|
||||
targetSessionFiles: new Set(["/tmp/targeted-fallback.jsonl"]),
|
||||
reason: "post-compaction",
|
||||
progress: undefined,
|
||||
useUnsafeReindex: true,
|
||||
sessionsDirtyFiles: new Set(),
|
||||
syncSessionFiles: async () => {
|
||||
throw new Error("embedding backend failed");
|
||||
},
|
||||
shouldFallbackOnError: () => true,
|
||||
activateFallbackProvider: async () => true,
|
||||
runSafeReindex,
|
||||
runUnsafeReindex,
|
||||
});
|
||||
|
||||
expect(runUnsafeReindex).toHaveBeenCalledWith({
|
||||
reason: "post-compaction",
|
||||
force: true,
|
||||
progress: undefined,
|
||||
});
|
||||
expect(runSafeReindex).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import type { MemorySyncProgressUpdate } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
type TargetedSyncProgress = {
|
||||
completed: number;
|
||||
total: number;
|
||||
label?: string;
|
||||
report: (update: MemorySyncProgressUpdate) => void;
|
||||
};
|
||||
|
||||
export function clearMemorySyncedSessionFiles(params: {
|
||||
sessionsDirtyFiles: Set<string>;
|
||||
targetSessionFiles?: Iterable<string> | null;
|
||||
}): boolean {
|
||||
if (!params.targetSessionFiles) {
|
||||
params.sessionsDirtyFiles.clear();
|
||||
} else {
|
||||
for (const targetSessionFile of params.targetSessionFiles) {
|
||||
params.sessionsDirtyFiles.delete(targetSessionFile);
|
||||
}
|
||||
}
|
||||
return params.sessionsDirtyFiles.size > 0;
|
||||
}
|
||||
|
||||
export async function runMemoryTargetedSessionSync(params: {
|
||||
hasSessionSource: boolean;
|
||||
targetSessionFiles: Set<string> | null;
|
||||
reason?: string;
|
||||
progress?: TargetedSyncProgress;
|
||||
useUnsafeReindex: boolean;
|
||||
sessionsDirtyFiles: Set<string>;
|
||||
syncSessionFiles: (params: {
|
||||
needsFullReindex: boolean;
|
||||
targetSessionFiles?: string[];
|
||||
progress?: TargetedSyncProgress;
|
||||
}) => Promise<void>;
|
||||
shouldFallbackOnError: (message: string) => boolean;
|
||||
activateFallbackProvider: (reason: string) => Promise<boolean>;
|
||||
runSafeReindex: (params: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
progress?: TargetedSyncProgress;
|
||||
}) => Promise<void>;
|
||||
runUnsafeReindex: (params: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
progress?: TargetedSyncProgress;
|
||||
}) => Promise<void>;
|
||||
}): Promise<{ handled: boolean; sessionsDirty: boolean }> {
|
||||
if (!params.hasSessionSource || !params.targetSessionFiles) {
|
||||
return {
|
||||
handled: false,
|
||||
sessionsDirty: params.sessionsDirtyFiles.size > 0,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await params.syncSessionFiles({
|
||||
needsFullReindex: false,
|
||||
targetSessionFiles: Array.from(params.targetSessionFiles),
|
||||
progress: params.progress,
|
||||
});
|
||||
return {
|
||||
handled: true,
|
||||
sessionsDirty: clearMemorySyncedSessionFiles({
|
||||
sessionsDirtyFiles: params.sessionsDirtyFiles,
|
||||
targetSessionFiles: params.targetSessionFiles,
|
||||
}),
|
||||
};
|
||||
} catch (err) {
|
||||
const reason = formatErrorMessage(err);
|
||||
const activated =
|
||||
params.shouldFallbackOnError(reason) && (await params.activateFallbackProvider(reason));
|
||||
if (!activated) {
|
||||
throw err;
|
||||
}
|
||||
const reindexParams = {
|
||||
reason: params.reason,
|
||||
force: true,
|
||||
progress: params.progress,
|
||||
};
|
||||
if (params.useUnsafeReindex) {
|
||||
await params.runUnsafeReindex(reindexParams);
|
||||
} else {
|
||||
await params.runSafeReindex(reindexParams);
|
||||
}
|
||||
return {
|
||||
handled: true,
|
||||
sessionsDirty: params.sessionsDirtyFiles.size > 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { logMemoryVectorDegradedWrite } from "./manager-vector-warning.js";
|
||||
|
||||
describe("memory vector degradation warnings", () => {
|
||||
it("emits the degraded warning only once for a manager", () => {
|
||||
const warn = vi.fn();
|
||||
|
||||
const first = logMemoryVectorDegradedWrite({
|
||||
vectorEnabled: true,
|
||||
vectorReady: false,
|
||||
chunkCount: 3,
|
||||
warningShown: false,
|
||||
loadError: "load failed",
|
||||
warn,
|
||||
});
|
||||
const second = logMemoryVectorDegradedWrite({
|
||||
vectorEnabled: true,
|
||||
vectorReady: false,
|
||||
chunkCount: 2,
|
||||
warningShown: first,
|
||||
loadError: "load failed",
|
||||
warn,
|
||||
});
|
||||
|
||||
expect(first).toBe(true);
|
||||
expect(second).toBe(true);
|
||||
expect(warn).toHaveBeenCalledTimes(1);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
"chunks_vec not updated — sqlite-vec unavailable: load failed. Vector recall degraded. Further duplicate warnings suppressed.",
|
||||
);
|
||||
});
|
||||
|
||||
it("skips the warning when vector writes are available", () => {
|
||||
const warn = vi.fn();
|
||||
|
||||
const shown = logMemoryVectorDegradedWrite({
|
||||
vectorEnabled: true,
|
||||
vectorReady: true,
|
||||
chunkCount: 1,
|
||||
warningShown: false,
|
||||
warn,
|
||||
});
|
||||
|
||||
expect(shown).toBe(false);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
export function logMemoryVectorDegradedWrite(params: {
|
||||
vectorEnabled: boolean;
|
||||
vectorReady: boolean;
|
||||
chunkCount: number;
|
||||
warningShown: boolean;
|
||||
loadError?: string;
|
||||
warn: (message: string) => void;
|
||||
}): boolean {
|
||||
if (
|
||||
!params.vectorEnabled ||
|
||||
params.vectorReady ||
|
||||
params.chunkCount <= 0 ||
|
||||
params.warningShown
|
||||
) {
|
||||
return params.warningShown;
|
||||
}
|
||||
const errDetail = params.loadError ? `: ${params.loadError}` : "";
|
||||
params.warn(
|
||||
`chunks_vec not updated — sqlite-vec unavailable${errDetail}. Vector recall degraded. Further duplicate warnings suppressed.`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import type { SQLInputValue } from "node:sqlite";
|
||||
|
||||
type VectorWriteDb = {
|
||||
prepare: (sql: string) => {
|
||||
run: (...params: SQLInputValue[]) => unknown;
|
||||
};
|
||||
};
|
||||
|
||||
const vectorToBlob = (embedding: number[]): Buffer =>
|
||||
Buffer.from(new Float32Array(embedding).buffer);
|
||||
|
||||
export function replaceMemoryVectorRow(params: {
|
||||
db: VectorWriteDb;
|
||||
id: string;
|
||||
embedding: number[];
|
||||
tableName?: string;
|
||||
}): void {
|
||||
const tableName = params.tableName ?? "chunks_vec";
|
||||
try {
|
||||
params.db.prepare(`DELETE FROM ${tableName} WHERE id = ?`).run(params.id);
|
||||
} catch {}
|
||||
params.db
|
||||
.prepare(`INSERT INTO ${tableName} (id, embedding) VALUES (?, ?)`)
|
||||
.run(params.id, vectorToBlob(params.embedding));
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { awaitPendingManagerWork, startAsyncSearchSync } from "./manager-async-state.js";
|
||||
|
||||
describe("memory search async sync", () => {
|
||||
it("does not await sync when searching", async () => {
|
||||
let releaseSync = () => {};
|
||||
const pending = new Promise<void>((resolve) => {
|
||||
releaseSync = () => resolve();
|
||||
});
|
||||
const syncMock = vi.fn(async () => {
|
||||
return pending;
|
||||
});
|
||||
const onError = vi.fn();
|
||||
|
||||
startAsyncSearchSync({
|
||||
enabled: true,
|
||||
dirty: true,
|
||||
sessionsDirty: false,
|
||||
sync: syncMock,
|
||||
onError,
|
||||
});
|
||||
|
||||
expect(syncMock).toHaveBeenCalledTimes(1);
|
||||
releaseSync();
|
||||
await pending;
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("waits for in-flight search sync during close", async () => {
|
||||
let releaseSync = () => {};
|
||||
const pendingSync = new Promise<void>((resolve) => {
|
||||
releaseSync = () => resolve();
|
||||
});
|
||||
|
||||
let closed = false;
|
||||
const closePromise = awaitPendingManagerWork({ pendingSync }).then(() => {
|
||||
closed = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(closed).toBe(false);
|
||||
|
||||
releaseSync();
|
||||
await closePromise;
|
||||
});
|
||||
|
||||
it("skips background search sync when search-triggered sync is disabled", () => {
|
||||
const syncMock = vi.fn(async () => {});
|
||||
startAsyncSearchSync({
|
||||
enabled: false,
|
||||
dirty: true,
|
||||
sessionsDirty: false,
|
||||
sync: syncMock,
|
||||
onError: vi.fn(),
|
||||
});
|
||||
expect(syncMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { runMemoryAtomicReindex } from "./manager-atomic-reindex.js";
|
||||
|
||||
describe("memory manager atomic reindex", () => {
|
||||
let fixtureRoot = "";
|
||||
let caseId = 0;
|
||||
let indexPath: string;
|
||||
let tempIndexPath: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-atomic-"));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
const workspaceDir = path.join(fixtureRoot, `case-${caseId++}`);
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
indexPath = path.join(workspaceDir, "index.sqlite");
|
||||
tempIndexPath = `${indexPath}.tmp`;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("keeps the prior index when a full reindex fails", async () => {
|
||||
writeChunkMarker(indexPath, "before");
|
||||
writeChunkMarker(tempIndexPath, "after");
|
||||
|
||||
await expect(
|
||||
runMemoryAtomicReindex({
|
||||
targetPath: indexPath,
|
||||
tempPath: tempIndexPath,
|
||||
build: async () => {
|
||||
throw new Error("embedding failure");
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("embedding failure");
|
||||
|
||||
expect(readChunkMarker(indexPath)).toBe("before");
|
||||
await expect(fs.access(tempIndexPath)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("replaces the old index after a successful temp reindex", async () => {
|
||||
writeChunkMarker(indexPath, "before");
|
||||
writeChunkMarker(tempIndexPath, "after");
|
||||
|
||||
await runMemoryAtomicReindex({
|
||||
targetPath: indexPath,
|
||||
tempPath: tempIndexPath,
|
||||
build: async () => undefined,
|
||||
});
|
||||
|
||||
expect(readChunkMarker(indexPath)).toBe("after");
|
||||
await expect(fs.access(tempIndexPath)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
function writeChunkMarker(dbPath: string, marker: string): void {
|
||||
const db = new DatabaseSync(dbPath);
|
||||
try {
|
||||
db.exec("CREATE TABLE chunks (id TEXT PRIMARY KEY, text TEXT NOT NULL)");
|
||||
db.prepare("INSERT INTO chunks (id, text) VALUES (?, ?)").run("chunk-1", marker);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function readChunkMarker(dbPath: string): string | undefined {
|
||||
const db = new DatabaseSync(dbPath);
|
||||
try {
|
||||
return (
|
||||
db.prepare("SELECT text FROM chunks WHERE id = ?").get("chunk-1") as
|
||||
| { text: string }
|
||||
| undefined
|
||||
)?.text;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { MemoryIndexManager } from "./manager.js";
|
||||
import "./test-runtime-mocks.js";
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
createEmbeddingProvider: async () => ({
|
||||
requestedProvider: "auto",
|
||||
provider: null,
|
||||
providerUnavailableReason: "No embeddings provider available.",
|
||||
}),
|
||||
resolveEmbeddingProviderFallbackModel: () => "fts-only",
|
||||
}));
|
||||
|
||||
type MemoryIndexModule = typeof import("./index.js");
|
||||
|
||||
describe("memory manager FTS-only reindex", () => {
|
||||
let fixtureRoot = "";
|
||||
let caseId = 0;
|
||||
let workspaceDir = "";
|
||||
let indexPath = "";
|
||||
let manager: MemoryIndexManager | null = null;
|
||||
let getMemorySearchManager: MemoryIndexModule["getMemorySearchManager"];
|
||||
let closeAllMemorySearchManagers: MemoryIndexModule["closeAllMemorySearchManagers"];
|
||||
|
||||
beforeAll(async () => {
|
||||
vi.resetModules();
|
||||
({ getMemorySearchManager, closeAllMemorySearchManagers } = await import("./index.js"));
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fts-only-"));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
workspaceDir = path.join(fixtureRoot, `case-${caseId++}`);
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "Alpha topic\n\nKeep this note.");
|
||||
indexPath = path.join(workspaceDir, "index.sqlite");
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (manager) {
|
||||
await manager.close();
|
||||
manager = null;
|
||||
}
|
||||
await closeAllMemorySearchManagers();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!fixtureRoot) {
|
||||
vi.resetModules();
|
||||
return;
|
||||
}
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
async function createManager(): Promise<MemoryIndexManager> {
|
||||
const cfg = {
|
||||
memory: {
|
||||
backend: "builtin",
|
||||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: workspaceDir,
|
||||
memorySearch: {
|
||||
provider: "auto",
|
||||
model: "",
|
||||
store: { path: indexPath },
|
||||
cache: { enabled: false },
|
||||
sync: { watch: false, onSessionStart: false, onSearch: false },
|
||||
},
|
||||
},
|
||||
list: [{ id: "main", default: true }],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
if (!result.manager) {
|
||||
throw new Error(result.error ?? "manager missing");
|
||||
}
|
||||
manager = result.manager as unknown as MemoryIndexManager;
|
||||
return manager;
|
||||
}
|
||||
|
||||
function countChunksContaining(term: string): number {
|
||||
const db = new DatabaseSync(indexPath);
|
||||
try {
|
||||
const row = db
|
||||
.prepare(`SELECT COUNT(*) as c FROM chunks WHERE text LIKE ?`)
|
||||
.get(`%${term}%`) as { c: number } | undefined;
|
||||
return row?.c ?? 0;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
it("preserves indexed chunks across forced reindex in FTS-only mode", async () => {
|
||||
const memoryManager = await createManager();
|
||||
|
||||
await memoryManager.sync({ force: true });
|
||||
const firstStatus = memoryManager.status();
|
||||
expect(firstStatus.chunks).toBeGreaterThan(0);
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
|
||||
await memoryManager.sync({ force: true });
|
||||
const secondStatus = memoryManager.status();
|
||||
expect(secondStatus.chunks).toBeGreaterThan(0);
|
||||
expect(countChunksContaining("Alpha topic")).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("refreshes FTS-only indexed content after memory file updates", async () => {
|
||||
const memoryManager = await createManager();
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "MEMORY.md"),
|
||||
"Beta refresh marker\n\nUpdated memory content.",
|
||||
);
|
||||
await memoryManager.sync({ force: true });
|
||||
|
||||
expect(countChunksContaining("refresh marker")).toBeGreaterThan(0);
|
||||
expect(countChunksContaining("Alpha topic")).toBe(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
import type {
|
||||
OpenClawConfig,
|
||||
ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
applyMemoryFallbackProviderState,
|
||||
resolveMemoryFallbackProviderRequest,
|
||||
resolveMemoryPrimaryProviderRequest,
|
||||
resolveMemoryProviderState,
|
||||
} from "./manager-provider-state.js";
|
||||
|
||||
const DEFAULT_OLLAMA_EMBEDDING_MODEL = "nomic-embed-text";
|
||||
const DEFAULT_LMSTUDIO_EMBEDDING_MODEL = "text-embedding-nomic-embed-text-v1.5";
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) =>
|
||||
providerId === "ollama"
|
||||
? DEFAULT_OLLAMA_EMBEDDING_MODEL
|
||||
: providerId === "lmstudio"
|
||||
? DEFAULT_LMSTUDIO_EMBEDDING_MODEL
|
||||
: fallbackSourceModel,
|
||||
}));
|
||||
|
||||
type EmbeddingProvider = {
|
||||
id: string;
|
||||
model: string;
|
||||
embedQuery: (text: string) => Promise<number[]>;
|
||||
embedBatch: (texts: string[]) => Promise<number[][]>;
|
||||
};
|
||||
|
||||
type EmbeddingProviderRuntime = {
|
||||
id: string;
|
||||
cacheKeyData: { provider: string; model: string };
|
||||
};
|
||||
|
||||
function createProvider(id: string): EmbeddingProvider {
|
||||
return {
|
||||
id,
|
||||
model: `${id}-model`,
|
||||
embedQuery: async () => [0.1, 0.2, 0.3],
|
||||
embedBatch: async (texts: string[]) => texts.map(() => [0.1, 0.2, 0.3]),
|
||||
};
|
||||
}
|
||||
|
||||
function createSettings(params: {
|
||||
provider: "openai" | "mistral";
|
||||
fallback?: "none" | "mistral" | "ollama" | "lmstudio";
|
||||
}): ResolvedMemorySearchConfig {
|
||||
return {
|
||||
provider: params.provider,
|
||||
model: params.provider === "mistral" ? "mistral/mistral-embed" : "text-embedding-3-small",
|
||||
fallback: params.fallback ?? "none",
|
||||
remote: undefined,
|
||||
outputDimensionality: undefined,
|
||||
local: undefined,
|
||||
} as unknown as ResolvedMemorySearchConfig;
|
||||
}
|
||||
|
||||
describe("memory manager mistral provider wiring", () => {
|
||||
it("stores mistral client when mistral provider is selected", () => {
|
||||
const mistralRuntime: EmbeddingProviderRuntime = {
|
||||
id: "mistral",
|
||||
cacheKeyData: { provider: "mistral", model: "mistral-embed" },
|
||||
};
|
||||
|
||||
const state = resolveMemoryProviderState({
|
||||
provider: createProvider("mistral"),
|
||||
runtime: mistralRuntime,
|
||||
fallbackFrom: undefined,
|
||||
fallbackReason: undefined,
|
||||
providerUnavailableReason: undefined,
|
||||
});
|
||||
|
||||
expect(state.provider?.id).toBe("mistral");
|
||||
expect(state.providerRuntime).toBe(mistralRuntime);
|
||||
});
|
||||
|
||||
it("stores mistral client after fallback activation", () => {
|
||||
const openAiRuntime: EmbeddingProviderRuntime = {
|
||||
id: "openai",
|
||||
cacheKeyData: { provider: "openai", model: "text-embedding-3-small" },
|
||||
};
|
||||
const mistralRuntime: EmbeddingProviderRuntime = {
|
||||
id: "mistral",
|
||||
cacheKeyData: { provider: "mistral", model: "mistral-embed" },
|
||||
};
|
||||
const current = resolveMemoryProviderState({
|
||||
provider: createProvider("openai"),
|
||||
runtime: openAiRuntime,
|
||||
fallbackFrom: undefined,
|
||||
fallbackReason: undefined,
|
||||
providerUnavailableReason: undefined,
|
||||
});
|
||||
|
||||
const fallbackState = applyMemoryFallbackProviderState({
|
||||
current,
|
||||
fallbackFrom: "openai",
|
||||
reason: "forced test",
|
||||
result: {
|
||||
provider: createProvider("mistral"),
|
||||
runtime: mistralRuntime,
|
||||
},
|
||||
});
|
||||
|
||||
expect(fallbackState.fallbackFrom).toBe("openai");
|
||||
expect(fallbackState.fallbackReason).toBe("forced test");
|
||||
expect(fallbackState.provider?.id).toBe("mistral");
|
||||
expect(fallbackState.providerRuntime).toBe(mistralRuntime);
|
||||
});
|
||||
|
||||
it("uses default ollama model when activating ollama fallback", () => {
|
||||
const request = resolveMemoryFallbackProviderRequest({
|
||||
cfg: {} as OpenClawConfig,
|
||||
settings: createSettings({ provider: "openai", fallback: "ollama" }),
|
||||
currentProviderId: "openai",
|
||||
});
|
||||
|
||||
expect(request?.provider).toBe("ollama");
|
||||
expect(request?.model).toBe(DEFAULT_OLLAMA_EMBEDDING_MODEL);
|
||||
expect(request?.fallback).toBe("none");
|
||||
});
|
||||
|
||||
it("includes outputDimensionality in the primary provider request", () => {
|
||||
const request = resolveMemoryPrimaryProviderRequest({
|
||||
settings: {
|
||||
...createSettings({ provider: "mistral" }),
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
outputDimensionality: 1536,
|
||||
} as ResolvedMemorySearchConfig,
|
||||
});
|
||||
|
||||
expect(request.provider).toBe("gemini");
|
||||
expect(request.model).toBe("gemini-embedding-2-preview");
|
||||
expect(request.outputDimensionality).toBe(1536);
|
||||
});
|
||||
|
||||
it("uses default lmstudio model when activating lmstudio fallback", async () => {
|
||||
const request = resolveMemoryFallbackProviderRequest({
|
||||
cfg: {} as OpenClawConfig,
|
||||
settings: createSettings({ provider: "openai", fallback: "lmstudio" }),
|
||||
currentProviderId: "openai",
|
||||
});
|
||||
|
||||
expect(request?.provider).toBe("lmstudio");
|
||||
expect(request?.model).toBe(DEFAULT_LMSTUDIO_EMBEDDING_MODEL);
|
||||
expect(request?.fallback).toBe("none");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,334 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { readMemoryFile } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
describe("MemoryIndexManager.readFile", () => {
|
||||
let workspaceDir: string;
|
||||
let memoryDir: string;
|
||||
let extraDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-read-"));
|
||||
memoryDir = path.join(workspaceDir, "memory");
|
||||
extraDir = path.join(workspaceDir, "extra");
|
||||
await fs.mkdir(memoryDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
[memoryDir, extraDir].map(async (root) => {
|
||||
const entries = await fs.readdir(root).catch(() => []);
|
||||
await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
await fs.rm(path.join(root, entry), { recursive: true, force: true });
|
||||
}),
|
||||
);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("returns empty text when the requested file does not exist", async () => {
|
||||
const relPath = "memory/2099-01-01.md";
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
});
|
||||
expect(result).toEqual({ text: "", path: relPath });
|
||||
});
|
||||
|
||||
it("returns content slices when the file exists", async () => {
|
||||
const relPath = "memory/2026-02-20.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, ["line 1", "line 2", "line 3"].join("\n"), "utf-8");
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
from: 2,
|
||||
lines: 1,
|
||||
});
|
||||
expect(result).toEqual({
|
||||
text: "line 2\n\n[More content available. Use from=3 to continue.]",
|
||||
path: relPath,
|
||||
from: 2,
|
||||
lines: 1,
|
||||
truncated: true,
|
||||
nextFrom: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a default-sized excerpt when no line range is provided", async () => {
|
||||
const relPath = "memory/default-window.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
absPath,
|
||||
Array.from({ length: 150 }, (_, index) => `line ${index + 1}`).join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
});
|
||||
|
||||
expect(result.path).toBe(relPath);
|
||||
expect(result.from).toBe(1);
|
||||
expect(result.lines).toBe(120);
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.nextFrom).toBe(121);
|
||||
expect(result.text).toContain("line 1");
|
||||
expect(result.text).toContain("line 120");
|
||||
expect(result.text).not.toContain("line 121");
|
||||
expect(result.text).toContain("Use from=121 to continue.");
|
||||
});
|
||||
|
||||
it("returns a bounded window when from is provided without lines", async () => {
|
||||
const relPath = "memory/from-only.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
absPath,
|
||||
Array.from({ length: 160 }, (_, index) => `line ${index + 1}`).join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
from: 21,
|
||||
});
|
||||
|
||||
expect(result.from).toBe(21);
|
||||
expect(result.lines).toBe(120);
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.nextFrom).toBe(141);
|
||||
expect(result.text).toContain("line 21");
|
||||
expect(result.text).toContain("line 140");
|
||||
expect(result.text).not.toContain("line 141");
|
||||
});
|
||||
|
||||
it("honors injected defaultLines and maxChars overrides", async () => {
|
||||
const relPath = "memory/agent-limits.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
absPath,
|
||||
Array.from({ length: 40 }, (_, index) => `line ${index + 1}: ${"x".repeat(40)}`).join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
defaultLines: 5,
|
||||
maxChars: 220,
|
||||
});
|
||||
|
||||
expect(result.from).toBe(1);
|
||||
expect(result.lines).toBeLessThanOrEqual(5);
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.nextFrom).toBeGreaterThan(1);
|
||||
expect(result.text).toContain("Use from=");
|
||||
});
|
||||
|
||||
it("returns empty text when the requested slice is past EOF", async () => {
|
||||
const relPath = "memory/window.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, ["alpha", "beta"].join("\n"), "utf-8");
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
from: 10,
|
||||
lines: 5,
|
||||
});
|
||||
expect(result).toEqual({ text: "", path: relPath, from: 10, lines: 0 });
|
||||
});
|
||||
|
||||
it("caps returned text to the default max chars and exposes continuation metadata", async () => {
|
||||
const relPath = "memory/char-cap.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
absPath,
|
||||
Array.from({ length: 200 }, (_, index) => `${index + 1}: ${"x".repeat(200)}`).join("\n"),
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.nextFrom).toBeGreaterThan(1);
|
||||
expect(result.lines).toBeLessThan(120);
|
||||
expect(result.text.length).toBeLessThanOrEqual(12_000 + 64);
|
||||
expect(result.text).toContain("Use from=");
|
||||
});
|
||||
|
||||
it("suggests read fallback for pathological single-line truncation in workspace memory files", async () => {
|
||||
const relPath = "memory/oversized-line.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, `1: ${"x".repeat(20_000)}`, "utf-8");
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.lines).toBe(1);
|
||||
expect(result.nextFrom).toBeUndefined();
|
||||
expect(result.text).toContain("use read on the source file");
|
||||
expect(result.text).not.toContain("Use from=");
|
||||
});
|
||||
|
||||
it("does not advertise line continuation when a single oversized line is cut mid-line", async () => {
|
||||
const relPath = "memory/oversized-line-with-tail.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, [`1: ${"x".repeat(20_000)}`, "line 2"].join("\n"), "utf-8");
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.lines).toBe(1);
|
||||
expect(result.nextFrom).toBeUndefined();
|
||||
expect(result.text).not.toContain("Use from=");
|
||||
});
|
||||
|
||||
it("omits truncation metadata when the full excerpt fits and no more lines remain", async () => {
|
||||
const relPath = "memory/complete.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, ["alpha", "beta", "gamma"].join("\n"), "utf-8");
|
||||
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
text: "alpha\nbeta\ngamma",
|
||||
path: relPath,
|
||||
from: 1,
|
||||
lines: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns empty text when the file disappears after stat", async () => {
|
||||
const relPath = "memory/transient.md";
|
||||
const absPath = path.join(workspaceDir, relPath);
|
||||
await fs.mkdir(path.dirname(absPath), { recursive: true });
|
||||
await fs.writeFile(absPath, "first\nsecond", "utf-8");
|
||||
|
||||
const realReadFile = fs.readFile;
|
||||
let injected = false;
|
||||
const readSpy = vi
|
||||
.spyOn(fs, "readFile")
|
||||
.mockImplementation(async (...args: Parameters<typeof realReadFile>) => {
|
||||
const [target, options] = args;
|
||||
if (!injected && typeof target === "string" && path.resolve(target) === absPath) {
|
||||
injected = true;
|
||||
const err = new Error("missing") as NodeJS.ErrnoException;
|
||||
err.code = "ENOENT";
|
||||
throw err;
|
||||
}
|
||||
return realReadFile(target, options);
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath,
|
||||
});
|
||||
expect(result).toEqual({ text: "", path: relPath });
|
||||
} finally {
|
||||
readSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects non-memory paths", async () => {
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [],
|
||||
relPath: "NOTES.md",
|
||||
}),
|
||||
).rejects.toThrow("path required");
|
||||
});
|
||||
|
||||
it("allows additional memory paths and blocks symlinks", async () => {
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "extra.md"), "Extra content.");
|
||||
await fs.writeFile(path.join(extraDir, "oversized.md"), `1: ${"y".repeat(20_000)}`);
|
||||
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: "extra/extra.md",
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
path: "extra/extra.md",
|
||||
text: "Extra content.",
|
||||
from: 1,
|
||||
lines: 1,
|
||||
});
|
||||
|
||||
const oversized = await readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: "extra/oversized.md",
|
||||
});
|
||||
expect(oversized.truncated).toBe(true);
|
||||
expect(oversized.text).not.toContain("use read on the source file");
|
||||
|
||||
const linkPath = path.join(extraDir, "linked.md");
|
||||
let symlinkOk = true;
|
||||
try {
|
||||
await fs.symlink(path.join(extraDir, "extra.md"), linkPath, "file");
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "EPERM" || code === "EACCES") {
|
||||
symlinkOk = false;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (symlinkOk) {
|
||||
await expect(
|
||||
readMemoryFile({
|
||||
workspaceDir,
|
||||
extraPaths: [extraDir],
|
||||
relPath: "extra/linked.md",
|
||||
}),
|
||||
).rejects.toThrow("path required");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,290 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { openMemoryDatabaseAtPath } from "./manager-db.js";
|
||||
import {
|
||||
_createMemorySyncControlConfigForTests,
|
||||
enqueueMemoryTargetedSessionSync,
|
||||
runMemorySyncWithReadonlyRecovery,
|
||||
type MemoryReadonlyRecoveryState,
|
||||
} from "./manager-sync-control.js";
|
||||
|
||||
type ReadonlyRecoveryHarness = MemoryReadonlyRecoveryState & {
|
||||
syncing: Promise<void> | null;
|
||||
queuedSessionFiles: Set<string>;
|
||||
queuedSessionSync: Promise<void> | null;
|
||||
vectorDegradedWriteWarningShown: boolean;
|
||||
ensureProviderInitialized: ReturnType<typeof vi.fn>;
|
||||
enqueueTargetedSessionSync: ReturnType<typeof vi.fn>;
|
||||
runSync: ReturnType<typeof vi.fn>;
|
||||
openDatabase: ReturnType<typeof vi.fn>;
|
||||
resetVectorState: ReturnType<typeof vi.fn>;
|
||||
ensureSchema: ReturnType<typeof vi.fn>;
|
||||
readMeta: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
describe("memory manager readonly recovery", () => {
|
||||
let workspaceDir = "";
|
||||
let indexPath = "";
|
||||
|
||||
function createQueuedSyncHarness(syncing: Promise<void>) {
|
||||
const queuedSessionFiles = new Set<string>();
|
||||
let queuedSessionSync: Promise<void> | null = null;
|
||||
const sync = vi.fn(async () => {});
|
||||
return {
|
||||
queuedSessionFiles,
|
||||
get queuedSessionSync() {
|
||||
return queuedSessionSync;
|
||||
},
|
||||
sync,
|
||||
state: {
|
||||
isClosed: () => false,
|
||||
getSyncing: () => syncing,
|
||||
getQueuedSessionFiles: () => queuedSessionFiles,
|
||||
getQueuedSessionSync: () => queuedSessionSync,
|
||||
setQueuedSessionSync: (value: Promise<void> | null) => {
|
||||
queuedSessionSync = value;
|
||||
},
|
||||
sync,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function _createMemoryConfig(): OpenClawConfig {
|
||||
return _createMemorySyncControlConfigForTests(workspaceDir, indexPath);
|
||||
}
|
||||
|
||||
function createReadonlyRecoveryHarness() {
|
||||
const reopenedClose = vi.fn();
|
||||
const initialClose = vi.fn();
|
||||
const reopenedDb = { close: reopenedClose } as unknown as DatabaseSync;
|
||||
const initialDb = { close: initialClose } as unknown as DatabaseSync;
|
||||
const harness: ReadonlyRecoveryHarness = {
|
||||
closed: false,
|
||||
syncing: null,
|
||||
queuedSessionFiles: new Set<string>(),
|
||||
queuedSessionSync: null,
|
||||
db: initialDb,
|
||||
vector: {
|
||||
dims: 123,
|
||||
},
|
||||
vectorDegradedWriteWarningShown: true,
|
||||
readonlyRecoveryAttempts: 0,
|
||||
readonlyRecoverySuccesses: 0,
|
||||
readonlyRecoveryFailures: 0,
|
||||
readonlyRecoveryLastError: undefined,
|
||||
ensureProviderInitialized: vi.fn(async () => {}),
|
||||
enqueueTargetedSessionSync: vi.fn(async () => {}),
|
||||
runSync: vi.fn(async (_params) => undefined) as ReadonlyRecoveryHarness["runSync"],
|
||||
openDatabase: vi.fn(() => reopenedDb),
|
||||
resetVectorState: vi.fn(function (this: ReadonlyRecoveryHarness) {
|
||||
this.vector.dims = undefined;
|
||||
this.vectorDegradedWriteWarningShown = false;
|
||||
}) as ReadonlyRecoveryHarness["resetVectorState"],
|
||||
ensureSchema: vi.fn(() => undefined) as ReadonlyRecoveryHarness["ensureSchema"],
|
||||
readMeta: vi.fn(() => undefined),
|
||||
};
|
||||
return {
|
||||
harness,
|
||||
initialDb,
|
||||
initialClose,
|
||||
reopenedDb,
|
||||
reopenedClose,
|
||||
};
|
||||
}
|
||||
|
||||
async function runSyncWithReadonlyRecovery(
|
||||
harness: ReadonlyRecoveryHarness,
|
||||
params?: { reason?: string; force?: boolean; sessionFiles?: string[] },
|
||||
) {
|
||||
return await runMemorySyncWithReadonlyRecovery(harness, params);
|
||||
}
|
||||
|
||||
function expectReadonlyRecoveryStatus(
|
||||
instance: {
|
||||
readonlyRecoveryAttempts: number;
|
||||
readonlyRecoverySuccesses: number;
|
||||
readonlyRecoveryFailures: number;
|
||||
readonlyRecoveryLastError?: string;
|
||||
},
|
||||
lastError: string,
|
||||
) {
|
||||
expect({
|
||||
attempts: instance.readonlyRecoveryAttempts,
|
||||
successes: instance.readonlyRecoverySuccesses,
|
||||
failures: instance.readonlyRecoveryFailures,
|
||||
lastError: instance.readonlyRecoveryLastError,
|
||||
}).toEqual({
|
||||
attempts: 1,
|
||||
successes: 1,
|
||||
failures: 0,
|
||||
lastError,
|
||||
});
|
||||
}
|
||||
|
||||
async function expectReadonlyRetry(params: { firstError: unknown; expectedLastError: string }) {
|
||||
const { harness, initialClose } = createReadonlyRecoveryHarness();
|
||||
harness.runSync.mockRejectedValueOnce(params.firstError).mockResolvedValueOnce(undefined);
|
||||
|
||||
await runSyncWithReadonlyRecovery(harness, {
|
||||
reason: "test",
|
||||
});
|
||||
|
||||
expect(harness.runSync).toHaveBeenCalledTimes(2);
|
||||
expect(harness.openDatabase).toHaveBeenCalledTimes(1);
|
||||
expect(harness.resetVectorState).toHaveBeenCalledTimes(1);
|
||||
expect(harness.vector.dims).toBe(123);
|
||||
expect(initialClose).toHaveBeenCalledTimes(1);
|
||||
expectReadonlyRecoveryStatus(harness, params.expectedLastError);
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-readonly-"));
|
||||
indexPath = path.join(workspaceDir, "index.sqlite");
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "Hello memory.");
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("reopens sqlite and retries once when sync hits SQLITE_READONLY", async () => {
|
||||
await expectReadonlyRetry({
|
||||
firstError: new Error("attempt to write a readonly database"),
|
||||
expectedLastError: "attempt to write a readonly database",
|
||||
});
|
||||
});
|
||||
|
||||
it("reopens sqlite and retries when readonly appears in error code", async () => {
|
||||
await expectReadonlyRetry({
|
||||
firstError: { message: "write failed", code: "SQLITE_READONLY" },
|
||||
expectedLastError: "write failed",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not retry non-readonly sync errors", async () => {
|
||||
const { harness, initialClose } = createReadonlyRecoveryHarness();
|
||||
harness.runSync.mockRejectedValueOnce(new Error("embedding timeout"));
|
||||
|
||||
await expect(
|
||||
runSyncWithReadonlyRecovery(harness, {
|
||||
reason: "test",
|
||||
}),
|
||||
).rejects.toThrow("embedding timeout");
|
||||
expect(harness.runSync).toHaveBeenCalledTimes(1);
|
||||
expect(harness.openDatabase).not.toHaveBeenCalled();
|
||||
expect(harness.resetVectorState).not.toHaveBeenCalled();
|
||||
expect(initialClose).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("clears the degraded warning latch before retrying", async () => {
|
||||
const { harness } = createReadonlyRecoveryHarness();
|
||||
harness.runSync.mockRejectedValueOnce(new Error("attempt to write a readonly database"));
|
||||
|
||||
await expect(
|
||||
runSyncWithReadonlyRecovery(harness, {
|
||||
reason: "test",
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(harness.vectorDegradedWriteWarningShown).toBe(false);
|
||||
});
|
||||
|
||||
it("prefers reopened vector dims when metadata is available", async () => {
|
||||
const { harness } = createReadonlyRecoveryHarness();
|
||||
harness.readMeta.mockReturnValueOnce({ vectorDims: 768 });
|
||||
harness.runSync.mockRejectedValueOnce(new Error("attempt to write a readonly database"));
|
||||
|
||||
await expect(
|
||||
runSyncWithReadonlyRecovery(harness, {
|
||||
reason: "test",
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(harness.vector.dims).toBe(768);
|
||||
});
|
||||
|
||||
it("sets busy_timeout on memory sqlite connections", async () => {
|
||||
const db = openMemoryDatabaseAtPath(indexPath, false);
|
||||
const row = db.prepare("PRAGMA busy_timeout").get() as
|
||||
| { busy_timeout?: number; timeout?: number }
|
||||
| undefined;
|
||||
const busyTimeout = row?.busy_timeout ?? row?.timeout;
|
||||
expect(busyTimeout).toBe(5000);
|
||||
db.close();
|
||||
});
|
||||
|
||||
it("queues targeted session files behind an in-flight sync", async () => {
|
||||
let releaseSync = () => {};
|
||||
const pendingSync = new Promise<void>((resolve) => {
|
||||
releaseSync = () => resolve();
|
||||
});
|
||||
const harness = createQueuedSyncHarness(pendingSync);
|
||||
|
||||
const queued = enqueueMemoryTargetedSessionSync(harness.state, [
|
||||
" /tmp/first.jsonl ",
|
||||
"",
|
||||
"/tmp/second.jsonl",
|
||||
]);
|
||||
|
||||
expect(harness.sync).not.toHaveBeenCalled();
|
||||
|
||||
releaseSync();
|
||||
await queued;
|
||||
|
||||
expect(harness.sync).toHaveBeenCalledTimes(1);
|
||||
expect(harness.sync).toHaveBeenCalledWith({
|
||||
reason: "queued-session-files",
|
||||
sessionFiles: ["/tmp/first.jsonl", "/tmp/second.jsonl"],
|
||||
});
|
||||
expect(harness.queuedSessionSync).toBeNull();
|
||||
});
|
||||
|
||||
it("merges repeated queued requests while the active sync is still running", async () => {
|
||||
let releaseSync = () => {};
|
||||
const pendingSync = new Promise<void>((resolve) => {
|
||||
releaseSync = () => resolve();
|
||||
});
|
||||
const harness = createQueuedSyncHarness(pendingSync);
|
||||
|
||||
const first = enqueueMemoryTargetedSessionSync(harness.state, [
|
||||
"/tmp/first.jsonl",
|
||||
"/tmp/second.jsonl",
|
||||
]);
|
||||
const second = enqueueMemoryTargetedSessionSync(harness.state, [
|
||||
"/tmp/second.jsonl",
|
||||
"/tmp/third.jsonl",
|
||||
]);
|
||||
|
||||
expect(first).toBe(second);
|
||||
|
||||
releaseSync();
|
||||
await second;
|
||||
|
||||
expect(harness.sync).toHaveBeenCalledTimes(1);
|
||||
expect(harness.sync).toHaveBeenCalledWith({
|
||||
reason: "queued-session-files",
|
||||
sessionFiles: ["/tmp/first.jsonl", "/tmp/second.jsonl", "/tmp/third.jsonl"],
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to the active sync when no usable session files were queued", async () => {
|
||||
let releaseSync = () => {};
|
||||
const pendingSync = new Promise<void>((resolve) => {
|
||||
releaseSync = () => resolve();
|
||||
});
|
||||
const harness = createQueuedSyncHarness(pendingSync);
|
||||
|
||||
const queued = enqueueMemoryTargetedSessionSync(harness.state, ["", " "]);
|
||||
|
||||
expect(queued).toBe(pendingSync);
|
||||
releaseSync();
|
||||
await queued;
|
||||
expect(harness.sync).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { shouldSyncSessionsForReindex } from "./manager-session-reindex.js";
|
||||
|
||||
describe("memory manager session reindex gating", () => {
|
||||
it("keeps session syncing enabled for full reindexes triggered from session-start/watch", () => {
|
||||
expect(
|
||||
shouldSyncSessionsForReindex({
|
||||
hasSessionSource: true,
|
||||
sessionsDirty: false,
|
||||
dirtySessionFileCount: 0,
|
||||
sync: { reason: "session-start" },
|
||||
needsFullReindex: true,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldSyncSessionsForReindex({
|
||||
hasSessionSource: true,
|
||||
sessionsDirty: false,
|
||||
dirtySessionFileCount: 0,
|
||||
sync: { reason: "watch" },
|
||||
needsFullReindex: true,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldSyncSessionsForReindex({
|
||||
hasSessionSource: true,
|
||||
sessionsDirty: false,
|
||||
dirtySessionFileCount: 0,
|
||||
sync: { reason: "session-start" },
|
||||
needsFullReindex: false,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldSyncSessionsForReindex({
|
||||
hasSessionSource: true,
|
||||
sessionsDirty: false,
|
||||
dirtySessionFileCount: 0,
|
||||
sync: { reason: "watch" },
|
||||
needsFullReindex: false,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { runDetachedMemorySync } from "./manager-sync-ops.js";
|
||||
|
||||
describe("memory manager sync failures", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("does not raise unhandledRejection when watch-triggered sync fails", async () => {
|
||||
const unhandled: unknown[] = [];
|
||||
const handler = (reason: unknown) => {
|
||||
unhandled.push(reason);
|
||||
};
|
||||
process.on("unhandledRejection", handler);
|
||||
const syncSpy = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("openai embeddings failed: 400 bad request"));
|
||||
setTimeout(() => {
|
||||
runDetachedMemorySync(syncSpy, "watch");
|
||||
}, 1);
|
||||
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
vi.useRealTimers();
|
||||
await syncSpy.mock.results[0]?.value?.catch(() => undefined);
|
||||
|
||||
process.off("unhandledRejection", handler);
|
||||
expect(unhandled).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
842
openclaw/extensions/memory-core/src/memory/manager.ts
Normal file
842
openclaw/extensions/memory-core/src/memory/manager.ts
Normal file
|
|
@ -0,0 +1,842 @@
|
|||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { type FSWatcher } from "chokidar";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
createSubsystemLogger,
|
||||
resolveAgentDir,
|
||||
resolveAgentWorkspaceDir,
|
||||
resolveMemorySearchConfig,
|
||||
type OpenClawConfig,
|
||||
type ResolvedMemorySearchConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { extractKeywords } from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
|
||||
import {
|
||||
readMemoryFile,
|
||||
type MemoryEmbeddingProbeResult,
|
||||
type MemoryProviderStatus,
|
||||
type MemorySearchManager,
|
||||
type MemorySearchRuntimeDebug,
|
||||
type MemorySearchResult,
|
||||
type MemorySource,
|
||||
type MemorySyncProgressUpdate,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import {
|
||||
createEmbeddingProvider,
|
||||
type EmbeddingProvider,
|
||||
type EmbeddingProviderId,
|
||||
type EmbeddingProviderRequest,
|
||||
type EmbeddingProviderResult,
|
||||
type EmbeddingProviderRuntime,
|
||||
} from "./embeddings.js";
|
||||
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
|
||||
import { awaitPendingManagerWork, startAsyncSearchSync } from "./manager-async-state.js";
|
||||
import { MEMORY_BATCH_FAILURE_LIMIT } from "./manager-batch-state.js";
|
||||
import {
|
||||
closeManagedCacheEntries,
|
||||
getOrCreateManagedCacheEntry,
|
||||
resolveSingletonManagedCache,
|
||||
} from "./manager-cache.js";
|
||||
import { MemoryManagerEmbeddingOps } from "./manager-embedding-ops.js";
|
||||
import {
|
||||
resolveMemoryPrimaryProviderRequest,
|
||||
resolveMemoryProviderState,
|
||||
} from "./manager-provider-state.js";
|
||||
import { resolveMemorySearchPreflight } from "./manager-search-preflight.js";
|
||||
import { searchKeyword, searchVector } from "./manager-search.js";
|
||||
import {
|
||||
collectMemoryStatusAggregate,
|
||||
resolveInitialMemoryDirty,
|
||||
resolveStatusProviderInfo,
|
||||
} from "./manager-status-state.js";
|
||||
import {
|
||||
enqueueMemoryTargetedSessionSync,
|
||||
extractMemoryErrorReason,
|
||||
isMemoryReadonlyDbError,
|
||||
runMemorySyncWithReadonlyRecovery,
|
||||
type MemoryReadonlyRecoveryState,
|
||||
} from "./manager-sync-control.js";
|
||||
import { applyTemporalDecayToHybridResults } from "./temporal-decay.js";
|
||||
const SNIPPET_MAX_CHARS = 700;
|
||||
const VECTOR_TABLE = "chunks_vec";
|
||||
const FTS_TABLE = "chunks_fts";
|
||||
const EMBEDDING_CACHE_TABLE = "embedding_cache";
|
||||
const MEMORY_INDEX_MANAGER_CACHE_KEY = Symbol.for("openclaw.memoryIndexManagerCache");
|
||||
const log = createSubsystemLogger("memory");
|
||||
|
||||
const { cache: INDEX_CACHE, pending: INDEX_CACHE_PENDING } =
|
||||
resolveSingletonManagedCache<MemoryIndexManager>(MEMORY_INDEX_MANAGER_CACHE_KEY);
|
||||
export async function closeAllMemoryIndexManagers(): Promise<void> {
|
||||
await closeManagedCacheEntries({
|
||||
cache: INDEX_CACHE,
|
||||
pending: INDEX_CACHE_PENDING,
|
||||
onCloseError: (err) => {
|
||||
log.warn(`failed to close memory index manager: ${String(err)}`);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements MemorySearchManager {
|
||||
private readonly cacheKey: string;
|
||||
protected readonly cfg: OpenClawConfig;
|
||||
protected readonly agentId: string;
|
||||
protected readonly workspaceDir: string;
|
||||
protected readonly settings: ResolvedMemorySearchConfig;
|
||||
protected provider: EmbeddingProvider | null;
|
||||
private readonly requestedProvider: EmbeddingProviderRequest;
|
||||
private providerInitPromise: Promise<void> | null = null;
|
||||
private providerInitialized = false;
|
||||
protected fallbackFrom?: EmbeddingProviderId;
|
||||
protected fallbackReason?: string;
|
||||
private providerUnavailableReason?: string;
|
||||
protected providerRuntime?: EmbeddingProviderRuntime;
|
||||
protected batch: {
|
||||
enabled: boolean;
|
||||
wait: boolean;
|
||||
concurrency: number;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
};
|
||||
protected batchFailureCount = 0;
|
||||
protected batchFailureLastError?: string;
|
||||
protected batchFailureLastProvider?: string;
|
||||
protected batchFailureLock: Promise<void> = Promise.resolve();
|
||||
protected db: DatabaseSync;
|
||||
protected readonly sources: Set<MemorySource>;
|
||||
protected providerKey: string;
|
||||
protected readonly cache: { enabled: boolean; maxEntries?: number };
|
||||
protected readonly vector: {
|
||||
enabled: boolean;
|
||||
available: boolean | null;
|
||||
extensionPath?: string;
|
||||
loadError?: string;
|
||||
dims?: number;
|
||||
};
|
||||
protected readonly fts: {
|
||||
enabled: boolean;
|
||||
available: boolean;
|
||||
loadError?: string;
|
||||
};
|
||||
protected vectorReady: Promise<boolean> | null = null;
|
||||
protected watcher: FSWatcher | null = null;
|
||||
protected watchTimer: NodeJS.Timeout | null = null;
|
||||
protected sessionWatchTimer: NodeJS.Timeout | null = null;
|
||||
protected sessionUnsubscribe: (() => void) | null = null;
|
||||
protected intervalTimer: NodeJS.Timeout | null = null;
|
||||
protected closed = false;
|
||||
protected dirty = false;
|
||||
protected sessionsDirty = false;
|
||||
protected sessionsDirtyFiles = new Set<string>();
|
||||
protected sessionPendingFiles = new Set<string>();
|
||||
protected sessionDeltas = new Map<
|
||||
string,
|
||||
{ lastSize: number; pendingBytes: number; pendingMessages: number }
|
||||
>();
|
||||
private sessionWarm = new Set<string>();
|
||||
private syncing: Promise<void> | null = null;
|
||||
private queuedSessionFiles = new Set<string>();
|
||||
private queuedSessionSync: Promise<void> | null = null;
|
||||
private readonlyRecoveryAttempts = 0;
|
||||
private readonlyRecoverySuccesses = 0;
|
||||
private readonlyRecoveryFailures = 0;
|
||||
private readonlyRecoveryLastError?: string;
|
||||
|
||||
private static async loadProviderResult(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
settings: ResolvedMemorySearchConfig;
|
||||
}): Promise<EmbeddingProviderResult> {
|
||||
return await createEmbeddingProvider({
|
||||
config: params.cfg,
|
||||
agentDir: resolveAgentDir(params.cfg, params.agentId),
|
||||
...resolveMemoryPrimaryProviderRequest({ settings: params.settings }),
|
||||
});
|
||||
}
|
||||
|
||||
static async get(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
purpose?: "default" | "status";
|
||||
}): Promise<MemoryIndexManager | null> {
|
||||
const { cfg, agentId } = params;
|
||||
const settings = resolveMemorySearchConfig(cfg, agentId);
|
||||
if (!settings) {
|
||||
return null;
|
||||
}
|
||||
const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
|
||||
const purpose = params.purpose === "status" ? "status" : "default";
|
||||
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}:${purpose}`;
|
||||
const statusOnly = params.purpose === "status";
|
||||
return await getOrCreateManagedCacheEntry({
|
||||
cache: INDEX_CACHE,
|
||||
pending: INDEX_CACHE_PENDING,
|
||||
key,
|
||||
bypassCache: statusOnly,
|
||||
create: async () =>
|
||||
new MemoryIndexManager({
|
||||
cacheKey: key,
|
||||
cfg,
|
||||
agentId,
|
||||
workspaceDir,
|
||||
settings,
|
||||
purpose: params.purpose,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
private constructor(params: {
|
||||
cacheKey: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
workspaceDir: string;
|
||||
settings: ResolvedMemorySearchConfig;
|
||||
providerResult?: EmbeddingProviderResult;
|
||||
purpose?: "default" | "status";
|
||||
}) {
|
||||
super();
|
||||
this.cacheKey = params.cacheKey;
|
||||
this.cfg = params.cfg;
|
||||
this.agentId = params.agentId;
|
||||
this.workspaceDir = params.workspaceDir;
|
||||
this.settings = params.settings;
|
||||
this.provider = null;
|
||||
this.requestedProvider = params.settings.provider;
|
||||
if (params.providerResult) {
|
||||
this.applyProviderResult(params.providerResult);
|
||||
}
|
||||
this.sources = new Set(params.settings.sources);
|
||||
this.db = this.openDatabase();
|
||||
this.providerKey = this.computeProviderKey();
|
||||
this.cache = {
|
||||
enabled: params.settings.cache.enabled,
|
||||
maxEntries: params.settings.cache.maxEntries,
|
||||
};
|
||||
this.fts = { enabled: params.settings.query.hybrid.enabled, available: false };
|
||||
this.ensureSchema();
|
||||
this.vector = {
|
||||
enabled: params.settings.store.vector.enabled,
|
||||
available: null,
|
||||
extensionPath: params.settings.store.vector.extensionPath,
|
||||
};
|
||||
const meta = this.readMeta();
|
||||
if (meta?.vectorDims) {
|
||||
this.vector.dims = meta.vectorDims;
|
||||
}
|
||||
const statusOnly = params.purpose === "status";
|
||||
if (!statusOnly) {
|
||||
this.ensureWatcher();
|
||||
this.ensureSessionListener();
|
||||
this.ensureIntervalSync();
|
||||
}
|
||||
this.dirty = resolveInitialMemoryDirty({
|
||||
hasMemorySource: this.sources.has("memory"),
|
||||
statusOnly,
|
||||
hasIndexedMeta: Boolean(meta),
|
||||
});
|
||||
this.batch = this.resolveBatchConfig();
|
||||
}
|
||||
|
||||
private applyProviderResult(providerResult: EmbeddingProviderResult): void {
|
||||
const providerState = resolveMemoryProviderState(providerResult);
|
||||
this.provider = providerState.provider;
|
||||
this.fallbackFrom = providerState.fallbackFrom;
|
||||
this.fallbackReason = providerState.fallbackReason;
|
||||
this.providerUnavailableReason = providerState.providerUnavailableReason;
|
||||
this.providerRuntime = providerState.providerRuntime;
|
||||
this.providerInitialized = true;
|
||||
}
|
||||
|
||||
private async ensureProviderInitialized(): Promise<void> {
|
||||
if (this.providerInitialized) {
|
||||
return;
|
||||
}
|
||||
if (!this.providerInitPromise) {
|
||||
this.providerInitPromise = (async () => {
|
||||
const providerResult = await MemoryIndexManager.loadProviderResult({
|
||||
cfg: this.cfg,
|
||||
agentId: this.agentId,
|
||||
settings: this.settings,
|
||||
});
|
||||
this.applyProviderResult(providerResult);
|
||||
this.providerKey = this.computeProviderKey();
|
||||
this.batch = this.resolveBatchConfig();
|
||||
})();
|
||||
}
|
||||
try {
|
||||
await this.providerInitPromise;
|
||||
} finally {
|
||||
if (this.providerInitialized) {
|
||||
this.providerInitPromise = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async warmSession(sessionKey?: string): Promise<void> {
|
||||
if (!this.settings.sync.onSessionStart) {
|
||||
return;
|
||||
}
|
||||
const key = sessionKey?.trim() || "";
|
||||
if (key && this.sessionWarm.has(key)) {
|
||||
return;
|
||||
}
|
||||
void this.sync({ reason: "session-start" }).catch((err) => {
|
||||
log.warn(`memory sync failed (session-start): ${String(err)}`);
|
||||
});
|
||||
if (key) {
|
||||
this.sessionWarm.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
async search(
|
||||
query: string,
|
||||
opts?: {
|
||||
maxResults?: number;
|
||||
minScore?: number;
|
||||
sessionKey?: string;
|
||||
qmdSearchModeOverride?: "query" | "search" | "vsearch";
|
||||
onDebug?: (debug: MemorySearchRuntimeDebug) => void;
|
||||
},
|
||||
): Promise<MemorySearchResult[]> {
|
||||
opts?.onDebug?.({ backend: "builtin" });
|
||||
let hasIndexedContent = this.hasIndexedContent();
|
||||
if (!hasIndexedContent) {
|
||||
try {
|
||||
// A fresh process can receive its first search before background watch/session
|
||||
// syncs have built the index. Force one synchronous bootstrap so the first
|
||||
// lookup after restart does not fail closed with empty results.
|
||||
await this.sync({ reason: "search", force: true });
|
||||
} catch (err) {
|
||||
log.warn(`memory sync failed (search-bootstrap): ${String(err)}`);
|
||||
}
|
||||
hasIndexedContent = this.hasIndexedContent();
|
||||
}
|
||||
const preflight = resolveMemorySearchPreflight({
|
||||
query,
|
||||
hasIndexedContent,
|
||||
});
|
||||
if (!preflight.shouldSearch) {
|
||||
return [];
|
||||
}
|
||||
const cleaned = preflight.normalizedQuery;
|
||||
void this.warmSession(opts?.sessionKey);
|
||||
startAsyncSearchSync({
|
||||
enabled: this.settings.sync.onSearch,
|
||||
dirty: this.dirty,
|
||||
sessionsDirty: this.sessionsDirty,
|
||||
sync: async (params) => await this.sync(params),
|
||||
onError: (err) => {
|
||||
log.warn(`memory sync failed (search): ${String(err)}`);
|
||||
},
|
||||
});
|
||||
if (preflight.shouldInitializeProvider) {
|
||||
await this.ensureProviderInitialized();
|
||||
}
|
||||
const minScore = opts?.minScore ?? this.settings.query.minScore;
|
||||
const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
|
||||
const hybrid = this.settings.query.hybrid;
|
||||
const candidates = Math.min(
|
||||
200,
|
||||
Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)),
|
||||
);
|
||||
|
||||
// FTS-only mode: no embedding provider available
|
||||
if (!this.provider) {
|
||||
if (!this.fts.enabled || !this.fts.available) {
|
||||
log.warn("memory search: no provider and FTS unavailable");
|
||||
return [];
|
||||
}
|
||||
|
||||
const fullQueryResults = await this.searchKeyword(cleaned, candidates, {
|
||||
boostFallbackRanking: true,
|
||||
}).catch(() => []);
|
||||
const resultSets =
|
||||
fullQueryResults.length > 0
|
||||
? [fullQueryResults]
|
||||
: await Promise.all(
|
||||
// Fallback: broaden recall for conversational queries when the
|
||||
// exact AND query is too strict to return any results.
|
||||
(() => {
|
||||
const keywords = extractKeywords(cleaned, {
|
||||
ftsTokenizer: this.settings.store.fts.tokenizer,
|
||||
});
|
||||
const searchTerms = keywords.length > 0 ? keywords : [cleaned];
|
||||
return searchTerms.map((term) =>
|
||||
this.searchKeyword(term, candidates, { boostFallbackRanking: true }).catch(
|
||||
() => [],
|
||||
),
|
||||
);
|
||||
})(),
|
||||
);
|
||||
|
||||
// Merge and deduplicate results, keeping highest score for each chunk
|
||||
const seenIds = new Map<string, (typeof resultSets)[0][0]>();
|
||||
for (const results of resultSets) {
|
||||
for (const result of results) {
|
||||
const existing = seenIds.get(result.id);
|
||||
if (!existing || result.score > existing.score) {
|
||||
seenIds.set(result.id, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const merged = [...seenIds.values()];
|
||||
const decayed = await applyTemporalDecayToHybridResults({
|
||||
results: merged,
|
||||
temporalDecay: hybrid.temporalDecay,
|
||||
workspaceDir: this.workspaceDir,
|
||||
});
|
||||
const sorted = decayed.toSorted((a, b) => b.score - a.score);
|
||||
return this.selectScoredResults(sorted, maxResults, minScore, 0);
|
||||
}
|
||||
|
||||
// If FTS isn't available, hybrid mode cannot use keyword search; degrade to vector-only.
|
||||
const keywordResults =
|
||||
hybrid.enabled && this.fts.enabled && this.fts.available
|
||||
? await this.searchKeyword(cleaned, candidates).catch(() => [])
|
||||
: [];
|
||||
|
||||
const queryVec = await this.embedQueryWithTimeout(cleaned);
|
||||
const hasVector = queryVec.some((v) => v !== 0);
|
||||
const vectorResults = hasVector
|
||||
? await this.searchVector(queryVec, candidates).catch(() => [])
|
||||
: [];
|
||||
|
||||
if (!hybrid.enabled || !this.fts.enabled || !this.fts.available) {
|
||||
return vectorResults.filter((entry) => entry.score >= minScore).slice(0, maxResults);
|
||||
}
|
||||
|
||||
const merged = await this.mergeHybridResults({
|
||||
vector: vectorResults,
|
||||
keyword: keywordResults,
|
||||
vectorWeight: hybrid.vectorWeight,
|
||||
textWeight: hybrid.textWeight,
|
||||
mmr: hybrid.mmr,
|
||||
temporalDecay: hybrid.temporalDecay,
|
||||
});
|
||||
const strict = merged.filter((entry) => entry.score >= minScore);
|
||||
if (strict.length > 0 || keywordResults.length === 0) {
|
||||
return strict.slice(0, maxResults);
|
||||
}
|
||||
|
||||
// Hybrid defaults can produce keyword-only matches with max score equal to
|
||||
// textWeight (for example 0.3). If minScore is higher (for example 0.35),
|
||||
// these exact lexical hits get filtered out even when they are the only
|
||||
// relevant results.
|
||||
const relaxedMinScore = Math.min(minScore, hybrid.textWeight);
|
||||
const keywordKeys = new Set(
|
||||
keywordResults.map(
|
||||
(entry) => `${entry.source}:${entry.path}:${entry.startLine}:${entry.endLine}`,
|
||||
),
|
||||
);
|
||||
return this.selectScoredResults(
|
||||
merged.filter((entry) =>
|
||||
keywordKeys.has(`${entry.source}:${entry.path}:${entry.startLine}:${entry.endLine}`),
|
||||
),
|
||||
maxResults,
|
||||
minScore,
|
||||
relaxedMinScore,
|
||||
);
|
||||
}
|
||||
|
||||
private selectScoredResults<T extends MemorySearchResult & { score: number }>(
|
||||
results: T[],
|
||||
maxResults: number,
|
||||
minScore: number,
|
||||
relaxedMinScore = minScore,
|
||||
): T[] {
|
||||
const strict = results.filter((entry) => entry.score >= minScore);
|
||||
if (strict.length > 0) {
|
||||
return strict.slice(0, maxResults);
|
||||
}
|
||||
return results.filter((entry) => entry.score >= relaxedMinScore).slice(0, maxResults);
|
||||
}
|
||||
|
||||
private hasIndexedContent(): boolean {
|
||||
const chunkRow = this.db.prepare(`SELECT 1 as found FROM chunks LIMIT 1`).get() as
|
||||
| {
|
||||
found?: number;
|
||||
}
|
||||
| undefined;
|
||||
if (chunkRow?.found === 1) {
|
||||
return true;
|
||||
}
|
||||
if (!this.fts.enabled || !this.fts.available) {
|
||||
return false;
|
||||
}
|
||||
const ftsRow = this.db.prepare(`SELECT 1 as found FROM ${FTS_TABLE} LIMIT 1`).get() as
|
||||
| {
|
||||
found?: number;
|
||||
}
|
||||
| undefined;
|
||||
return ftsRow?.found === 1;
|
||||
}
|
||||
|
||||
private async searchVector(
|
||||
queryVec: number[],
|
||||
limit: number,
|
||||
): Promise<Array<MemorySearchResult & { id: string }>> {
|
||||
// This method should never be called without a provider
|
||||
if (!this.provider) {
|
||||
return [];
|
||||
}
|
||||
const results = await searchVector({
|
||||
db: this.db,
|
||||
vectorTable: VECTOR_TABLE,
|
||||
providerModel: this.provider.model,
|
||||
queryVec,
|
||||
limit,
|
||||
snippetMaxChars: SNIPPET_MAX_CHARS,
|
||||
ensureVectorReady: async (dimensions) => await this.ensureVectorReady(dimensions),
|
||||
sourceFilterVec: this.buildSourceFilter("c"),
|
||||
sourceFilterChunks: this.buildSourceFilter(),
|
||||
});
|
||||
return results.map((entry) => entry as MemorySearchResult & { id: string });
|
||||
}
|
||||
|
||||
private buildFtsQuery(raw: string): string | null {
|
||||
return buildFtsQuery(raw);
|
||||
}
|
||||
|
||||
private async searchKeyword(
|
||||
query: string,
|
||||
limit: number,
|
||||
options?: { boostFallbackRanking?: boolean },
|
||||
): Promise<Array<MemorySearchResult & { id: string; textScore: number }>> {
|
||||
if (!this.fts.enabled || !this.fts.available) {
|
||||
return [];
|
||||
}
|
||||
const sourceFilter = this.buildSourceFilter();
|
||||
// In FTS-only mode (no provider), search all models; otherwise filter by current provider's model
|
||||
const providerModel = this.provider?.model;
|
||||
const results = await searchKeyword({
|
||||
db: this.db,
|
||||
ftsTable: FTS_TABLE,
|
||||
providerModel,
|
||||
query,
|
||||
ftsTokenizer: this.settings.store.fts.tokenizer,
|
||||
limit,
|
||||
snippetMaxChars: SNIPPET_MAX_CHARS,
|
||||
sourceFilter,
|
||||
buildFtsQuery: (raw) => this.buildFtsQuery(raw),
|
||||
bm25RankToScore,
|
||||
boostFallbackRanking: options?.boostFallbackRanking,
|
||||
});
|
||||
return results.map((entry) => entry as MemorySearchResult & { id: string; textScore: number });
|
||||
}
|
||||
|
||||
private mergeHybridResults(params: {
|
||||
vector: Array<MemorySearchResult & { id: string }>;
|
||||
keyword: Array<MemorySearchResult & { id: string; textScore: number }>;
|
||||
vectorWeight: number;
|
||||
textWeight: number;
|
||||
mmr?: { enabled: boolean; lambda: number };
|
||||
temporalDecay?: { enabled: boolean; halfLifeDays: number };
|
||||
}): Promise<MemorySearchResult[]> {
|
||||
return mergeHybridResults({
|
||||
vector: params.vector.map((r) => ({
|
||||
id: r.id,
|
||||
path: r.path,
|
||||
startLine: r.startLine,
|
||||
endLine: r.endLine,
|
||||
source: r.source,
|
||||
snippet: r.snippet,
|
||||
vectorScore: r.score,
|
||||
})),
|
||||
keyword: params.keyword.map((r) => ({
|
||||
id: r.id,
|
||||
path: r.path,
|
||||
startLine: r.startLine,
|
||||
endLine: r.endLine,
|
||||
source: r.source,
|
||||
snippet: r.snippet,
|
||||
textScore: r.textScore,
|
||||
})),
|
||||
vectorWeight: params.vectorWeight,
|
||||
textWeight: params.textWeight,
|
||||
mmr: params.mmr,
|
||||
temporalDecay: params.temporalDecay,
|
||||
workspaceDir: this.workspaceDir,
|
||||
}).then((entries) => entries.map((entry) => entry as MemorySearchResult));
|
||||
}
|
||||
|
||||
async sync(params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}): Promise<void> {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
await this.ensureProviderInitialized();
|
||||
if (this.syncing) {
|
||||
if (params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) {
|
||||
return this.enqueueTargetedSessionSync(params.sessionFiles);
|
||||
}
|
||||
return this.syncing;
|
||||
}
|
||||
this.syncing = this.runSyncWithReadonlyRecovery(params).finally(() => {
|
||||
this.syncing = null;
|
||||
});
|
||||
return this.syncing ?? Promise.resolve();
|
||||
}
|
||||
|
||||
private enqueueTargetedSessionSync(sessionFiles?: string[]): Promise<void> {
|
||||
return enqueueMemoryTargetedSessionSync(
|
||||
{
|
||||
isClosed: () => this.closed,
|
||||
getSyncing: () => this.syncing,
|
||||
getQueuedSessionFiles: () => this.queuedSessionFiles,
|
||||
getQueuedSessionSync: () => this.queuedSessionSync,
|
||||
setQueuedSessionSync: (value) => {
|
||||
this.queuedSessionSync = value;
|
||||
},
|
||||
sync: async (params) => await this.sync(params),
|
||||
},
|
||||
sessionFiles,
|
||||
);
|
||||
}
|
||||
|
||||
private isReadonlyDbError(err: unknown): boolean {
|
||||
return isMemoryReadonlyDbError(err);
|
||||
}
|
||||
|
||||
private extractErrorReason(err: unknown): string {
|
||||
return extractMemoryErrorReason(err);
|
||||
}
|
||||
|
||||
private async runSyncWithReadonlyRecovery(params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}): Promise<void> {
|
||||
const getClosed = () => this.closed;
|
||||
const getDb = () => this.db;
|
||||
const setDb = (value: DatabaseSync) => {
|
||||
this.db = value;
|
||||
};
|
||||
const getReadonlyRecoveryAttempts = () => this.readonlyRecoveryAttempts;
|
||||
const setReadonlyRecoveryAttempts = (value: number) => {
|
||||
this.readonlyRecoveryAttempts = value;
|
||||
};
|
||||
const getReadonlyRecoverySuccesses = () => this.readonlyRecoverySuccesses;
|
||||
const setReadonlyRecoverySuccesses = (value: number) => {
|
||||
this.readonlyRecoverySuccesses = value;
|
||||
};
|
||||
const getReadonlyRecoveryFailures = () => this.readonlyRecoveryFailures;
|
||||
const setReadonlyRecoveryFailures = (value: number) => {
|
||||
this.readonlyRecoveryFailures = value;
|
||||
};
|
||||
const getReadonlyRecoveryLastError = () => this.readonlyRecoveryLastError;
|
||||
const setReadonlyRecoveryLastError = (value: string | undefined) => {
|
||||
this.readonlyRecoveryLastError = value;
|
||||
};
|
||||
const state: MemoryReadonlyRecoveryState = {
|
||||
get closed() {
|
||||
return getClosed();
|
||||
},
|
||||
get db() {
|
||||
return getDb();
|
||||
},
|
||||
set db(value) {
|
||||
setDb(value);
|
||||
},
|
||||
vector: this.vector,
|
||||
get readonlyRecoveryAttempts() {
|
||||
return getReadonlyRecoveryAttempts();
|
||||
},
|
||||
set readonlyRecoveryAttempts(value) {
|
||||
setReadonlyRecoveryAttempts(value);
|
||||
},
|
||||
get readonlyRecoverySuccesses() {
|
||||
return getReadonlyRecoverySuccesses();
|
||||
},
|
||||
set readonlyRecoverySuccesses(value) {
|
||||
setReadonlyRecoverySuccesses(value);
|
||||
},
|
||||
get readonlyRecoveryFailures() {
|
||||
return getReadonlyRecoveryFailures();
|
||||
},
|
||||
set readonlyRecoveryFailures(value) {
|
||||
setReadonlyRecoveryFailures(value);
|
||||
},
|
||||
get readonlyRecoveryLastError() {
|
||||
return getReadonlyRecoveryLastError();
|
||||
},
|
||||
set readonlyRecoveryLastError(value) {
|
||||
setReadonlyRecoveryLastError(value);
|
||||
},
|
||||
runSync: (nextParams) => this.runSync(nextParams),
|
||||
openDatabase: () => this.openDatabase(),
|
||||
resetVectorState: () => this.resetVectorState(),
|
||||
ensureSchema: () => this.ensureSchema(),
|
||||
readMeta: () => this.readMeta() ?? undefined,
|
||||
};
|
||||
await runMemorySyncWithReadonlyRecovery(state, params);
|
||||
}
|
||||
|
||||
async readFile(params: {
|
||||
relPath: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<{ text: string; path: string }> {
|
||||
return await readMemoryFile({
|
||||
workspaceDir: this.workspaceDir,
|
||||
extraPaths: this.settings.extraPaths,
|
||||
relPath: params.relPath,
|
||||
from: params.from,
|
||||
lines: params.lines,
|
||||
});
|
||||
}
|
||||
|
||||
status(): MemoryProviderStatus {
|
||||
const sourceFilter = this.buildSourceFilter();
|
||||
const aggregateState = collectMemoryStatusAggregate({
|
||||
db: {
|
||||
prepare: (sql) => ({
|
||||
all: (...args) =>
|
||||
this.db.prepare(sql).all(...args) as Array<{
|
||||
kind: "files" | "chunks";
|
||||
source: MemorySource;
|
||||
c: number;
|
||||
}>,
|
||||
}),
|
||||
},
|
||||
sources: this.sources,
|
||||
sourceFilterSql: sourceFilter.sql,
|
||||
sourceFilterParams: sourceFilter.params,
|
||||
});
|
||||
|
||||
const providerInfo = resolveStatusProviderInfo({
|
||||
provider: this.provider,
|
||||
providerInitialized: this.providerInitialized,
|
||||
requestedProvider: this.requestedProvider,
|
||||
configuredModel: this.settings.model || undefined,
|
||||
});
|
||||
|
||||
return {
|
||||
backend: "builtin",
|
||||
files: aggregateState.files,
|
||||
chunks: aggregateState.chunks,
|
||||
dirty: this.dirty || this.sessionsDirty,
|
||||
workspaceDir: this.workspaceDir,
|
||||
dbPath: this.settings.store.path,
|
||||
provider: providerInfo.provider,
|
||||
model: providerInfo.model,
|
||||
requestedProvider: this.requestedProvider,
|
||||
sources: Array.from(this.sources),
|
||||
extraPaths: this.settings.extraPaths,
|
||||
sourceCounts: aggregateState.sourceCounts,
|
||||
cache: this.cache.enabled
|
||||
? {
|
||||
enabled: true,
|
||||
entries:
|
||||
(
|
||||
this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get() as
|
||||
| { c: number }
|
||||
| undefined
|
||||
)?.c ?? 0,
|
||||
maxEntries: this.cache.maxEntries,
|
||||
}
|
||||
: { enabled: false, maxEntries: this.cache.maxEntries },
|
||||
fts: {
|
||||
enabled: this.fts.enabled,
|
||||
available: this.fts.available,
|
||||
error: this.fts.loadError,
|
||||
},
|
||||
fallback: this.fallbackReason
|
||||
? { from: this.fallbackFrom ?? "local", reason: this.fallbackReason }
|
||||
: undefined,
|
||||
vector: {
|
||||
enabled: this.vector.enabled,
|
||||
available: this.vector.available ?? undefined,
|
||||
extensionPath: this.vector.extensionPath,
|
||||
loadError: this.vector.loadError,
|
||||
dims: this.vector.dims,
|
||||
},
|
||||
batch: {
|
||||
enabled: this.batch.enabled,
|
||||
failures: this.batchFailureCount,
|
||||
limit: MEMORY_BATCH_FAILURE_LIMIT,
|
||||
wait: this.batch.wait,
|
||||
concurrency: this.batch.concurrency,
|
||||
pollIntervalMs: this.batch.pollIntervalMs,
|
||||
timeoutMs: this.batch.timeoutMs,
|
||||
lastError: this.batchFailureLastError,
|
||||
lastProvider: this.batchFailureLastProvider,
|
||||
},
|
||||
custom: {
|
||||
searchMode: providerInfo.searchMode,
|
||||
providerUnavailableReason: this.providerUnavailableReason,
|
||||
readonlyRecovery: {
|
||||
attempts: this.readonlyRecoveryAttempts,
|
||||
successes: this.readonlyRecoverySuccesses,
|
||||
failures: this.readonlyRecoveryFailures,
|
||||
lastError: this.readonlyRecoveryLastError,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async probeVectorAvailability(): Promise<boolean> {
|
||||
if (!this.vector.enabled) {
|
||||
return false;
|
||||
}
|
||||
await this.ensureProviderInitialized();
|
||||
// FTS-only mode: vector search not available
|
||||
if (!this.provider) {
|
||||
return false;
|
||||
}
|
||||
return this.ensureVectorReady();
|
||||
}
|
||||
|
||||
async probeEmbeddingAvailability(): Promise<MemoryEmbeddingProbeResult> {
|
||||
await this.ensureProviderInitialized();
|
||||
// FTS-only mode: embeddings not available but search still works
|
||||
if (!this.provider) {
|
||||
return {
|
||||
ok: false,
|
||||
error: this.providerUnavailableReason ?? "No embedding provider available (FTS-only mode)",
|
||||
};
|
||||
}
|
||||
try {
|
||||
await this.embedBatchWithRetry(["ping"]);
|
||||
return { ok: true };
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
this.closed = true;
|
||||
const pendingSync = this.syncing;
|
||||
const pendingProviderInit = this.providerInitPromise;
|
||||
if (this.watchTimer) {
|
||||
clearTimeout(this.watchTimer);
|
||||
this.watchTimer = null;
|
||||
}
|
||||
if (this.sessionWatchTimer) {
|
||||
clearTimeout(this.sessionWatchTimer);
|
||||
this.sessionWatchTimer = null;
|
||||
}
|
||||
if (this.intervalTimer) {
|
||||
clearInterval(this.intervalTimer);
|
||||
this.intervalTimer = null;
|
||||
}
|
||||
if (this.watcher) {
|
||||
await this.watcher.close();
|
||||
this.watcher = null;
|
||||
}
|
||||
if (this.sessionUnsubscribe) {
|
||||
this.sessionUnsubscribe();
|
||||
this.sessionUnsubscribe = null;
|
||||
}
|
||||
await awaitPendingManagerWork({ pendingSync, pendingProviderInit });
|
||||
this.db.close();
|
||||
INDEX_CACHE.delete(this.cacheKey);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
import { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { replaceMemoryVectorRow } from "./manager-vector-write.js";
|
||||
|
||||
describe("memory vector dedupe", () => {
|
||||
let db: DatabaseSync | null = null;
|
||||
|
||||
afterEach(() => {
|
||||
db?.close();
|
||||
db = null;
|
||||
});
|
||||
|
||||
it("deletes existing vector rows before inserting replacements", () => {
|
||||
db = new DatabaseSync(":memory:");
|
||||
db.exec("CREATE TABLE chunks_vec (id TEXT PRIMARY KEY, embedding BLOB)");
|
||||
|
||||
replaceMemoryVectorRow({
|
||||
db,
|
||||
id: "chunk-1",
|
||||
embedding: [1, 0, 0],
|
||||
});
|
||||
|
||||
db.exec(`
|
||||
CREATE TRIGGER fail_if_vector_row_not_deleted
|
||||
BEFORE INSERT ON chunks_vec
|
||||
WHEN EXISTS (SELECT 1 FROM chunks_vec WHERE id = NEW.id)
|
||||
BEGIN
|
||||
SELECT RAISE(FAIL, 'vector row not deleted before insert');
|
||||
END;
|
||||
`);
|
||||
|
||||
expect(() =>
|
||||
replaceMemoryVectorRow({
|
||||
db: db!,
|
||||
id: "chunk-1",
|
||||
embedding: [2, 0, 0],
|
||||
}),
|
||||
).not.toThrow();
|
||||
|
||||
const row = db
|
||||
.prepare("SELECT COUNT(*) as c, length(embedding) as bytes FROM chunks_vec WHERE id = ?")
|
||||
.get("chunk-1") as { c: number; bytes: number } | undefined;
|
||||
expect(row?.c).toBe(1);
|
||||
expect(row?.bytes).toBe(12);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type {
|
||||
MemorySearchConfig,
|
||||
OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { MemoryIndexManager } from "./index.js";
|
||||
import { registerBuiltInMemoryEmbeddingProviders } from "./provider-adapters.js";
|
||||
|
||||
type WatchIgnoredFn = (watchPath: string, stats?: { isDirectory?: () => boolean }) => boolean;
|
||||
|
||||
const { watchMock } = vi.hoisted(() => ({
|
||||
watchMock: vi.fn(() => ({
|
||||
on: vi.fn(),
|
||||
close: vi.fn(async () => undefined),
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("chokidar", () => ({
|
||||
default: { watch: watchMock },
|
||||
watch: watchMock,
|
||||
}));
|
||||
|
||||
vi.mock("./sqlite-vec.js", () => ({
|
||||
loadSqliteVecExtension: async () => ({ ok: false, error: "sqlite-vec disabled in tests" }),
|
||||
}));
|
||||
|
||||
vi.mock("./embeddings.js", () => ({
|
||||
createEmbeddingProvider: async () => ({
|
||||
requestedProvider: "openai",
|
||||
provider: {
|
||||
id: "mock",
|
||||
model: "mock-embed",
|
||||
embedQuery: async () => [1, 0],
|
||||
embedBatch: async (texts: string[]) => texts.map(() => [1, 0]),
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
type MemoryIndexModule = typeof import("./index.js");
|
||||
type MemoryEmbeddingProvidersModule =
|
||||
typeof import("../../../../src/plugins/memory-embedding-providers.js");
|
||||
|
||||
let getMemorySearchManager: MemoryIndexModule["getMemorySearchManager"];
|
||||
let closeAllMemorySearchManagers: MemoryIndexModule["closeAllMemorySearchManagers"];
|
||||
let clearRegistry: MemoryEmbeddingProvidersModule["clearMemoryEmbeddingProviders"];
|
||||
let registerAdapter: MemoryEmbeddingProvidersModule["registerMemoryEmbeddingProvider"];
|
||||
|
||||
describe("memory watcher config", () => {
|
||||
let manager: MemoryIndexManager | null = null;
|
||||
let workspaceDir = "";
|
||||
let extraDir = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
vi.resetModules();
|
||||
({ getMemorySearchManager, closeAllMemorySearchManagers } = await import("./index.js"));
|
||||
({
|
||||
clearMemoryEmbeddingProviders: clearRegistry,
|
||||
registerMemoryEmbeddingProvider: registerAdapter,
|
||||
} = await import("../../../../src/plugins/memory-embedding-providers.js"));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
clearRegistry();
|
||||
registerBuiltInMemoryEmbeddingProviders({ registerMemoryEmbeddingProvider: registerAdapter });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
watchMock.mockClear();
|
||||
if (manager) {
|
||||
await manager.close();
|
||||
manager = null;
|
||||
}
|
||||
await closeAllMemorySearchManagers();
|
||||
clearRegistry();
|
||||
if (workspaceDir) {
|
||||
await fs.rm(workspaceDir, { recursive: true, force: true });
|
||||
workspaceDir = "";
|
||||
extraDir = "";
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
async function setupWatcherWorkspace(seedFile: { name: string; contents: string }) {
|
||||
workspaceDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-memory-watch-"));
|
||||
extraDir = path.join(workspaceDir, "extra");
|
||||
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, seedFile.name), seedFile.contents);
|
||||
}
|
||||
|
||||
function createWatcherConfig(overrides?: Partial<MemorySearchConfig>): OpenClawConfig {
|
||||
const defaults: NonNullable<NonNullable<OpenClawConfig["agents"]>["defaults"]> = {
|
||||
workspace: workspaceDir,
|
||||
memorySearch: {
|
||||
provider: "openai",
|
||||
model: "mock-embed",
|
||||
store: { path: path.join(workspaceDir, "index.sqlite"), vector: { enabled: false } },
|
||||
sync: { watch: true, watchDebounceMs: 25, onSessionStart: false, onSearch: false },
|
||||
query: { minScore: 0, hybrid: { enabled: false } },
|
||||
extraPaths: [extraDir],
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
return {
|
||||
agents: {
|
||||
defaults,
|
||||
list: [{ id: "main", default: true }],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
}
|
||||
|
||||
async function expectWatcherManager(cfg: OpenClawConfig) {
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
expect(result.manager).not.toBeNull();
|
||||
if (!result.manager) {
|
||||
throw new Error("manager missing");
|
||||
}
|
||||
manager = result.manager as unknown as MemoryIndexManager;
|
||||
}
|
||||
|
||||
it("watches the memory directory and ignores non-markdown churn", async () => {
|
||||
await setupWatcherWorkspace({ name: "notes.md", contents: "hello" });
|
||||
const cfg = createWatcherConfig();
|
||||
|
||||
await expectWatcherManager(cfg);
|
||||
|
||||
expect(watchMock).toHaveBeenCalledTimes(1);
|
||||
const [watchedPaths, options] = watchMock.mock.calls[0] as unknown as [
|
||||
string[],
|
||||
Record<string, unknown>,
|
||||
];
|
||||
expect(watchedPaths).toEqual(
|
||||
expect.arrayContaining([
|
||||
path.join(workspaceDir, "MEMORY.md"),
|
||||
path.join(workspaceDir, "memory.md"),
|
||||
path.join(workspaceDir, "memory"),
|
||||
path.join(extraDir, "**", "*.md"),
|
||||
]),
|
||||
);
|
||||
expect(options.ignoreInitial).toBe(true);
|
||||
expect(options.awaitWriteFinish).toEqual({ stabilityThreshold: 25, pollInterval: 100 });
|
||||
|
||||
const ignored = options.ignored as WatchIgnoredFn | undefined;
|
||||
expect(ignored).toBeTypeOf("function");
|
||||
expect(ignored?.(path.join(workspaceDir, "memory", "node_modules", "pkg", "index.md"))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(ignored?.(path.join(workspaceDir, "memory", ".venv", "lib", "python.md"))).toBe(true);
|
||||
expect(ignored?.(path.join(workspaceDir, "memory", "project", "notes.tmp"))).toBe(true);
|
||||
expect(ignored?.(path.join(workspaceDir, "memory", "project", "notes.json"))).toBe(true);
|
||||
expect(ignored?.(path.join(workspaceDir, "memory", "project", "notes.md"))).toBe(false);
|
||||
expect(
|
||||
ignored?.(path.join(workspaceDir, "memory", "project"), { isDirectory: () => true }),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("watches multimodal extensions with case-insensitive globs", async () => {
|
||||
await setupWatcherWorkspace({ name: "PHOTO.PNG", contents: "png" });
|
||||
const cfg = createWatcherConfig({
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
fallback: "none",
|
||||
multimodal: { enabled: true, modalities: ["image", "audio"] },
|
||||
});
|
||||
|
||||
await expectWatcherManager(cfg);
|
||||
|
||||
expect(watchMock).toHaveBeenCalledTimes(1);
|
||||
const [watchedPaths, options] = watchMock.mock.calls[0] as unknown as [
|
||||
string[],
|
||||
Record<string, unknown>,
|
||||
];
|
||||
expect(watchedPaths).toEqual(
|
||||
expect.arrayContaining([
|
||||
path.join(extraDir, "**", "*.[pP][nN][gG]"),
|
||||
path.join(extraDir, "**", "*.[wW][aA][vV]"),
|
||||
]),
|
||||
);
|
||||
|
||||
const ignored = options.ignored as WatchIgnoredFn | undefined;
|
||||
expect(ignored).toBeTypeOf("function");
|
||||
expect(ignored?.(path.join(extraDir, "nested", "PHOTO.PNG"))).toBe(false);
|
||||
expect(ignored?.(path.join(extraDir, "nested", "voice.WAV"))).toBe(false);
|
||||
expect(ignored?.(path.join(extraDir, "nested", "metadata.json"))).toBe(true);
|
||||
});
|
||||
});
|
||||
444
openclaw/extensions/memory-core/src/memory/mmr.test.ts
Normal file
444
openclaw/extensions/memory-core/src/memory/mmr.test.ts
Normal file
|
|
@ -0,0 +1,444 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
tokenize,
|
||||
jaccardSimilarity,
|
||||
textSimilarity,
|
||||
computeMMRScore,
|
||||
mmrRerank,
|
||||
applyMMRToHybridResults,
|
||||
DEFAULT_MMR_CONFIG,
|
||||
type MMRItem,
|
||||
} from "./mmr.js";
|
||||
|
||||
describe("tokenize", () => {
|
||||
it("normalizes, filters, and deduplicates token sets", () => {
|
||||
const cases = [
|
||||
{
|
||||
name: "alphanumeric lowercase",
|
||||
input: "Hello World 123",
|
||||
expected: ["hello", "world", "123"],
|
||||
},
|
||||
{ name: "empty string", input: "", expected: [] },
|
||||
{ name: "special chars only", input: "!@#$%^&*()", expected: [] },
|
||||
{
|
||||
name: "underscores",
|
||||
input: "hello_world test_case",
|
||||
expected: ["hello_world", "test_case"],
|
||||
},
|
||||
{
|
||||
name: "dedupe repeated tokens",
|
||||
input: "hello hello world world",
|
||||
expected: ["hello", "world"],
|
||||
},
|
||||
{
|
||||
name: "CJK characters produce unigrams and bigrams",
|
||||
input: "今天讨论",
|
||||
expected: ["今", "天", "讨", "论", "今天", "天讨", "讨论"],
|
||||
},
|
||||
{
|
||||
name: "mixed ASCII and CJK",
|
||||
input: "hello 你好世界 test",
|
||||
expected: ["hello", "test", "你", "好", "世", "界", "你好", "好世", "世界"],
|
||||
},
|
||||
{
|
||||
name: "single CJK character (no bigrams)",
|
||||
input: "龙",
|
||||
expected: ["龙"],
|
||||
},
|
||||
{
|
||||
name: "non-adjacent CJK chars do not form bigrams",
|
||||
input: "我a好",
|
||||
expected: ["a", "我", "好"],
|
||||
// No "我好" bigram — they are separated by "a"
|
||||
},
|
||||
{
|
||||
name: "Japanese hiragana",
|
||||
input: "こんにちは",
|
||||
expected: ["こ", "ん", "に", "ち", "は", "こん", "んに", "にち", "ちは"],
|
||||
},
|
||||
{
|
||||
name: "Korean hangul",
|
||||
input: "안녕하세요",
|
||||
expected: ["안", "녕", "하", "세", "요", "안녕", "녕하", "하세", "세요"],
|
||||
},
|
||||
] as const;
|
||||
|
||||
for (const testCase of cases) {
|
||||
expect(tokenize(testCase.input), testCase.name).toEqual(new Set(testCase.expected));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("jaccardSimilarity", () => {
|
||||
it("computes expected scores for overlap edge cases", () => {
|
||||
const cases = [
|
||||
{
|
||||
name: "identical sets",
|
||||
left: new Set(["a", "b", "c"]),
|
||||
right: new Set(["a", "b", "c"]),
|
||||
expected: 1,
|
||||
},
|
||||
{ name: "disjoint sets", left: new Set(["a", "b"]), right: new Set(["c", "d"]), expected: 0 },
|
||||
{ name: "two empty sets", left: new Set<string>(), right: new Set<string>(), expected: 1 },
|
||||
{
|
||||
name: "left non-empty right empty",
|
||||
left: new Set(["a"]),
|
||||
right: new Set<string>(),
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "left empty right non-empty",
|
||||
left: new Set<string>(),
|
||||
right: new Set(["a"]),
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "partial overlap",
|
||||
left: new Set(["a", "b", "c"]),
|
||||
right: new Set(["b", "c", "d"]),
|
||||
expected: 0.5,
|
||||
},
|
||||
] as const;
|
||||
|
||||
for (const testCase of cases) {
|
||||
expect(jaccardSimilarity(testCase.left, testCase.right), testCase.name).toBe(
|
||||
testCase.expected,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("is symmetric", () => {
|
||||
const setA = new Set(["a", "b"]);
|
||||
const setB = new Set(["b", "c"]);
|
||||
expect(jaccardSimilarity(setA, setB)).toBe(jaccardSimilarity(setB, setA));
|
||||
});
|
||||
});
|
||||
|
||||
describe("textSimilarity", () => {
|
||||
it("computes expected text-level similarity cases", () => {
|
||||
const cases = [
|
||||
{ name: "identical", left: "hello world", right: "hello world", expected: 1 },
|
||||
{ name: "same words reordered", left: "hello world", right: "world hello", expected: 1 },
|
||||
{ name: "different text", left: "hello world", right: "foo bar", expected: 0 },
|
||||
{ name: "case insensitive", left: "Hello World", right: "hello world", expected: 1 },
|
||||
{
|
||||
name: "CJK similar texts share tokens",
|
||||
left: "今天我们讨论了项目进展",
|
||||
right: "今天我们讨论了会议安排",
|
||||
// Shared unigrams: 今,天,我,们,讨,论,了 (7) + shared bigrams: 今天,天我,我们,们讨,讨论,论了 (6) = 13 shared
|
||||
// Total unique tokens > 13, so similarity > 0 and < 1
|
||||
expected: -1, // placeholder — just check > 0
|
||||
},
|
||||
{
|
||||
name: "CJK completely different texts",
|
||||
left: "苹果香蕉",
|
||||
right: "钢铁煤炭",
|
||||
expected: 0,
|
||||
},
|
||||
] as const;
|
||||
|
||||
for (const testCase of cases) {
|
||||
if (testCase.expected === -1) {
|
||||
// Placeholder: just assert positive similarity
|
||||
const sim = textSimilarity(testCase.left, testCase.right);
|
||||
expect(sim, testCase.name).toBeGreaterThan(0);
|
||||
expect(sim, testCase.name).toBeLessThan(1);
|
||||
} else {
|
||||
expect(textSimilarity(testCase.left, testCase.right), testCase.name).toBe(
|
||||
testCase.expected,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeMMRScore", () => {
|
||||
it("balances relevance and diversity across lambda settings", () => {
|
||||
const cases = [
|
||||
{
|
||||
name: "lambda=1 relevance only",
|
||||
relevance: 0.8,
|
||||
similarity: 0.5,
|
||||
lambda: 1,
|
||||
expected: 0.8,
|
||||
},
|
||||
{
|
||||
name: "lambda=0 diversity only",
|
||||
relevance: 0.8,
|
||||
similarity: 0.5,
|
||||
lambda: 0,
|
||||
expected: -0.5,
|
||||
},
|
||||
{ name: "lambda=0.5 mixed", relevance: 0.8, similarity: 0.6, lambda: 0.5, expected: 0.1 },
|
||||
{ name: "default lambda math", relevance: 1.0, similarity: 0.5, lambda: 0.7, expected: 0.55 },
|
||||
] as const;
|
||||
|
||||
for (const testCase of cases) {
|
||||
expect(
|
||||
computeMMRScore(testCase.relevance, testCase.similarity, testCase.lambda),
|
||||
testCase.name,
|
||||
).toBeCloseTo(testCase.expected);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("empty input behavior", () => {
|
||||
it("returns empty array for empty input", () => {
|
||||
expect(mmrRerank([])).toEqual([]);
|
||||
expect(applyMMRToHybridResults([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("mmrRerank", () => {
|
||||
describe("edge cases", () => {
|
||||
it("returns single item unchanged", () => {
|
||||
const items: MMRItem[] = [{ id: "1", score: 0.9, content: "hello" }];
|
||||
expect(mmrRerank(items)).toEqual(items);
|
||||
});
|
||||
|
||||
it("returns copy, not original array", () => {
|
||||
const items: MMRItem[] = [{ id: "1", score: 0.9, content: "hello" }];
|
||||
const result = mmrRerank(items);
|
||||
expect(result).not.toBe(items);
|
||||
});
|
||||
|
||||
it("returns items unchanged when disabled", () => {
|
||||
const items: MMRItem[] = [
|
||||
{ id: "1", score: 0.9, content: "hello" },
|
||||
{ id: "2", score: 0.8, content: "hello" },
|
||||
];
|
||||
const result = mmrRerank(items, { enabled: false });
|
||||
expect(result).toEqual(items);
|
||||
});
|
||||
});
|
||||
|
||||
describe("lambda edge cases", () => {
|
||||
const diverseItems: MMRItem[] = [
|
||||
{ id: "1", score: 1.0, content: "apple banana cherry" },
|
||||
{ id: "2", score: 0.9, content: "apple banana date" },
|
||||
{ id: "3", score: 0.8, content: "elderberry fig grape" },
|
||||
];
|
||||
|
||||
it("lambda=1 returns pure relevance order", () => {
|
||||
const result = mmrRerank(diverseItems, { lambda: 1 });
|
||||
expect(result.map((i) => i.id)).toEqual(["1", "2", "3"]);
|
||||
});
|
||||
|
||||
it("lambda=0 maximizes diversity", () => {
|
||||
const result = mmrRerank(diverseItems, { enabled: true, lambda: 0 });
|
||||
// First item is still highest score (no penalty yet)
|
||||
expect(result[0].id).toBe("1");
|
||||
// Second should be most different from first
|
||||
expect(result[1].id).toBe("3"); // elderberry... is most different
|
||||
});
|
||||
|
||||
it("clamps lambda > 1 to 1", () => {
|
||||
const result = mmrRerank(diverseItems, { lambda: 1.5 });
|
||||
expect(result.map((i) => i.id)).toEqual(["1", "2", "3"]);
|
||||
});
|
||||
|
||||
it("clamps lambda < 0 to 0", () => {
|
||||
const result = mmrRerank(diverseItems, { enabled: true, lambda: -0.5 });
|
||||
expect(result[0].id).toBe("1");
|
||||
expect(result[1].id).toBe("3");
|
||||
});
|
||||
});
|
||||
|
||||
describe("diversity behavior", () => {
|
||||
it("promotes diverse results over similar high-scoring ones", () => {
|
||||
const items: MMRItem[] = [
|
||||
{ id: "1", score: 1.0, content: "machine learning neural networks" },
|
||||
{ id: "2", score: 0.95, content: "machine learning deep learning" },
|
||||
{ id: "3", score: 0.9, content: "database systems sql queries" },
|
||||
{ id: "4", score: 0.85, content: "machine learning algorithms" },
|
||||
];
|
||||
|
||||
const result = mmrRerank(items, { enabled: true, lambda: 0.5 });
|
||||
|
||||
// First is always highest score
|
||||
expect(result[0].id).toBe("1");
|
||||
// Second should be the diverse database item, not another ML item
|
||||
expect(result[1].id).toBe("3");
|
||||
});
|
||||
|
||||
it("handles items with identical content", () => {
|
||||
const items: MMRItem[] = [
|
||||
{ id: "1", score: 1.0, content: "identical content" },
|
||||
{ id: "2", score: 0.9, content: "identical content" },
|
||||
{ id: "3", score: 0.8, content: "different stuff" },
|
||||
];
|
||||
|
||||
const result = mmrRerank(items, { enabled: true, lambda: 0.5 });
|
||||
expect(result[0].id).toBe("1");
|
||||
// Second should be different, not identical duplicate
|
||||
expect(result[1].id).toBe("3");
|
||||
});
|
||||
|
||||
it("handles all identical content gracefully", () => {
|
||||
const items: MMRItem[] = [
|
||||
{ id: "1", score: 1.0, content: "same" },
|
||||
{ id: "2", score: 0.9, content: "same" },
|
||||
{ id: "3", score: 0.8, content: "same" },
|
||||
];
|
||||
|
||||
const result = mmrRerank(items, { lambda: 0.7 });
|
||||
// Should still complete without error, order by score as tiebreaker
|
||||
expect(result).toHaveLength(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("tie-breaking", () => {
|
||||
it("uses original score as tiebreaker", () => {
|
||||
const items: MMRItem[] = [
|
||||
{ id: "1", score: 1.0, content: "unique content one" },
|
||||
{ id: "2", score: 0.9, content: "unique content two" },
|
||||
{ id: "3", score: 0.8, content: "unique content three" },
|
||||
];
|
||||
|
||||
// With very different content and lambda=1, should be pure score order
|
||||
const result = mmrRerank(items, { lambda: 1 });
|
||||
expect(result.map((i) => i.id)).toEqual(["1", "2", "3"]);
|
||||
});
|
||||
|
||||
it("preserves all items even with same MMR scores", () => {
|
||||
const items: MMRItem[] = [
|
||||
{ id: "1", score: 0.5, content: "a" },
|
||||
{ id: "2", score: 0.5, content: "b" },
|
||||
{ id: "3", score: 0.5, content: "c" },
|
||||
];
|
||||
|
||||
const result = mmrRerank(items, { lambda: 0.7 });
|
||||
expect(result).toHaveLength(3);
|
||||
expect(new Set(result.map((i) => i.id))).toEqual(new Set(["1", "2", "3"]));
|
||||
});
|
||||
});
|
||||
|
||||
describe("score normalization", () => {
|
||||
it("handles items with same scores", () => {
|
||||
const items: MMRItem[] = [
|
||||
{ id: "1", score: 0.5, content: "hello world" },
|
||||
{ id: "2", score: 0.5, content: "foo bar" },
|
||||
];
|
||||
|
||||
const result = mmrRerank(items, { lambda: 0.7 });
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("handles negative scores", () => {
|
||||
const items: MMRItem[] = [
|
||||
{ id: "1", score: -0.5, content: "hello world" },
|
||||
{ id: "2", score: -1.0, content: "foo bar" },
|
||||
];
|
||||
|
||||
const result = mmrRerank(items, { lambda: 0.7 });
|
||||
expect(result).toHaveLength(2);
|
||||
// Higher score (less negative) should come first
|
||||
expect(result[0].id).toBe("1");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyMMRToHybridResults", () => {
|
||||
type HybridResult = {
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
snippet: string;
|
||||
source: string;
|
||||
};
|
||||
|
||||
it("preserves all original fields", () => {
|
||||
const results: HybridResult[] = [
|
||||
{
|
||||
path: "/test/file.ts",
|
||||
startLine: 1,
|
||||
endLine: 10,
|
||||
score: 0.9,
|
||||
snippet: "hello world",
|
||||
source: "memory",
|
||||
},
|
||||
];
|
||||
|
||||
const reranked = applyMMRToHybridResults(results);
|
||||
expect(reranked[0]).toEqual(results[0]);
|
||||
});
|
||||
|
||||
it("creates unique IDs from path and startLine", () => {
|
||||
const results: HybridResult[] = [
|
||||
{
|
||||
path: "/test/a.ts",
|
||||
startLine: 1,
|
||||
endLine: 10,
|
||||
score: 0.9,
|
||||
snippet: "same content here",
|
||||
source: "memory",
|
||||
},
|
||||
{
|
||||
path: "/test/a.ts",
|
||||
startLine: 20,
|
||||
endLine: 30,
|
||||
score: 0.8,
|
||||
snippet: "same content here",
|
||||
source: "memory",
|
||||
},
|
||||
];
|
||||
|
||||
// Should work without ID collision
|
||||
const reranked = applyMMRToHybridResults(results);
|
||||
expect(reranked).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("re-ranks results for diversity", () => {
|
||||
const results: HybridResult[] = [
|
||||
{
|
||||
path: "/a.ts",
|
||||
startLine: 1,
|
||||
endLine: 10,
|
||||
score: 1.0,
|
||||
snippet: "function add numbers together",
|
||||
source: "memory",
|
||||
},
|
||||
{
|
||||
path: "/b.ts",
|
||||
startLine: 1,
|
||||
endLine: 10,
|
||||
score: 0.95,
|
||||
snippet: "function add values together",
|
||||
source: "memory",
|
||||
},
|
||||
{
|
||||
path: "/c.ts",
|
||||
startLine: 1,
|
||||
endLine: 10,
|
||||
score: 0.9,
|
||||
snippet: "database connection pool",
|
||||
source: "memory",
|
||||
},
|
||||
];
|
||||
|
||||
const reranked = applyMMRToHybridResults(results, { enabled: true, lambda: 0.5 });
|
||||
|
||||
// First stays the same (highest score)
|
||||
expect(reranked[0].path).toBe("/a.ts");
|
||||
// Second should be the diverse one
|
||||
expect(reranked[1].path).toBe("/c.ts");
|
||||
});
|
||||
|
||||
it("respects disabled config", () => {
|
||||
const results: HybridResult[] = [
|
||||
{ path: "/a.ts", startLine: 1, endLine: 10, score: 0.9, snippet: "test", source: "memory" },
|
||||
{ path: "/b.ts", startLine: 1, endLine: 10, score: 0.8, snippet: "test", source: "memory" },
|
||||
];
|
||||
|
||||
const reranked = applyMMRToHybridResults(results, { enabled: false });
|
||||
expect(reranked).toEqual(results);
|
||||
});
|
||||
});
|
||||
|
||||
describe("DEFAULT_MMR_CONFIG", () => {
|
||||
it("has expected default values", () => {
|
||||
expect(DEFAULT_MMR_CONFIG.enabled).toBe(false);
|
||||
expect(DEFAULT_MMR_CONFIG.lambda).toBe(0.7);
|
||||
});
|
||||
});
|
||||
250
openclaw/extensions/memory-core/src/memory/mmr.ts
Normal file
250
openclaw/extensions/memory-core/src/memory/mmr.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
/**
|
||||
* Maximal Marginal Relevance (MMR) re-ranking algorithm.
|
||||
*
|
||||
* MMR balances relevance with diversity by iteratively selecting results
|
||||
* that maximize: λ * relevance - (1-λ) * max_similarity_to_selected
|
||||
*
|
||||
* @see Carbonell & Goldstein, "The Use of MMR, Diversity-Based Reranking" (1998)
|
||||
*/
|
||||
|
||||
export type MMRItem = {
|
||||
id: string;
|
||||
score: number;
|
||||
content: string;
|
||||
};
|
||||
|
||||
export type MMRConfig = {
|
||||
/** Enable/disable MMR re-ranking. Default: false (opt-in) */
|
||||
enabled: boolean;
|
||||
/** Lambda parameter: 0 = max diversity, 1 = max relevance. Default: 0.7 */
|
||||
lambda: number;
|
||||
};
|
||||
|
||||
export const DEFAULT_MMR_CONFIG: MMRConfig = {
|
||||
enabled: false,
|
||||
lambda: 0.7,
|
||||
};
|
||||
|
||||
/**
|
||||
* Regex matching CJK-family characters that lack whitespace word boundaries:
|
||||
* - CJK Unified Ideographs (Chinese hanzi, Japanese kanji, Korean hanja)
|
||||
* - CJK Extension A
|
||||
* - Hiragana & Katakana (Japanese)
|
||||
* - Hangul Syllables & Jamo (Korean)
|
||||
*/
|
||||
const CJK_RE = /[\u3040-\u309f\u30a0-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\u1100-\u11ff]/;
|
||||
|
||||
/**
|
||||
* Tokenize text for Jaccard similarity computation.
|
||||
* Extracts alphanumeric tokens, CJK-family characters (unigrams),
|
||||
* and consecutive CJK character pairs (bigrams).
|
||||
*
|
||||
* Bigrams are only created from characters that are adjacent in the
|
||||
* original text, so mixed content like "我喜欢hello你好" will NOT
|
||||
* produce the spurious bigram "欢你".
|
||||
*/
|
||||
export function tokenize(text: string): Set<string> {
|
||||
const lower = normalizeLowercaseStringOrEmpty(text);
|
||||
const ascii = lower.match(/[a-z0-9_]+/g) ?? [];
|
||||
|
||||
// Track CJK characters with their original positions
|
||||
const chars = Array.from(lower);
|
||||
const cjkData: { char: string; index: number }[] = [];
|
||||
for (let i = 0; i < chars.length; i++) {
|
||||
if (CJK_RE.test(chars[i])) {
|
||||
cjkData.push({ char: chars[i], index: i });
|
||||
}
|
||||
}
|
||||
|
||||
// Build bigrams only from originally adjacent CJK characters
|
||||
const bigrams: string[] = [];
|
||||
for (let i = 0; i < cjkData.length - 1; i++) {
|
||||
if (cjkData[i + 1].index === cjkData[i].index + 1) {
|
||||
bigrams.push(cjkData[i].char + cjkData[i + 1].char);
|
||||
}
|
||||
}
|
||||
|
||||
const unigrams = cjkData.map((d) => d.char);
|
||||
return new Set([...ascii, ...bigrams, ...unigrams]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute Jaccard similarity between two token sets.
|
||||
* Returns a value in [0, 1] where 1 means identical sets.
|
||||
*/
|
||||
export function jaccardSimilarity(setA: Set<string>, setB: Set<string>): number {
|
||||
if (setA.size === 0 && setB.size === 0) {
|
||||
return 1;
|
||||
}
|
||||
if (setA.size === 0 || setB.size === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let intersectionSize = 0;
|
||||
const smaller = setA.size <= setB.size ? setA : setB;
|
||||
const larger = setA.size <= setB.size ? setB : setA;
|
||||
|
||||
for (const token of smaller) {
|
||||
if (larger.has(token)) {
|
||||
intersectionSize++;
|
||||
}
|
||||
}
|
||||
|
||||
const unionSize = setA.size + setB.size - intersectionSize;
|
||||
return unionSize === 0 ? 0 : intersectionSize / unionSize;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute text similarity between two content strings using Jaccard on tokens.
|
||||
*/
|
||||
export function textSimilarity(contentA: string, contentB: string): number {
|
||||
return jaccardSimilarity(tokenize(contentA), tokenize(contentB));
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the maximum similarity between an item and all selected items.
|
||||
*/
|
||||
function maxSimilarityToSelected(
|
||||
item: MMRItem,
|
||||
selectedItems: MMRItem[],
|
||||
tokenCache: Map<string, Set<string>>,
|
||||
): number {
|
||||
if (selectedItems.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let maxSim = 0;
|
||||
const itemTokens = tokenCache.get(item.id) ?? tokenize(item.content);
|
||||
|
||||
for (const selected of selectedItems) {
|
||||
const selectedTokens = tokenCache.get(selected.id) ?? tokenize(selected.content);
|
||||
const sim = jaccardSimilarity(itemTokens, selectedTokens);
|
||||
if (sim > maxSim) {
|
||||
maxSim = sim;
|
||||
}
|
||||
}
|
||||
|
||||
return maxSim;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute MMR score for a candidate item.
|
||||
* MMR = λ * relevance - (1-λ) * max_similarity_to_selected
|
||||
*/
|
||||
export function computeMMRScore(relevance: number, maxSimilarity: number, lambda: number): number {
|
||||
return lambda * relevance - (1 - lambda) * maxSimilarity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-rank items using Maximal Marginal Relevance (MMR).
|
||||
*
|
||||
* The algorithm iteratively selects items that balance relevance with diversity:
|
||||
* 1. Start with the highest-scoring item
|
||||
* 2. For each remaining slot, select the item that maximizes the MMR score
|
||||
* 3. MMR score = λ * relevance - (1-λ) * max_similarity_to_already_selected
|
||||
*
|
||||
* @param items - Items to re-rank, must have score and content
|
||||
* @param config - MMR configuration (lambda, enabled)
|
||||
* @returns Re-ranked items in MMR order
|
||||
*/
|
||||
export function mmrRerank<T extends MMRItem>(items: T[], config: Partial<MMRConfig> = {}): T[] {
|
||||
const { enabled = DEFAULT_MMR_CONFIG.enabled, lambda = DEFAULT_MMR_CONFIG.lambda } = config;
|
||||
|
||||
// Early exits
|
||||
if (!enabled || items.length <= 1) {
|
||||
return [...items];
|
||||
}
|
||||
|
||||
// Clamp lambda to valid range
|
||||
const clampedLambda = Math.max(0, Math.min(1, lambda));
|
||||
|
||||
// If lambda is 1, just return sorted by relevance (no diversity penalty)
|
||||
if (clampedLambda === 1) {
|
||||
return [...items].toSorted((a, b) => b.score - a.score);
|
||||
}
|
||||
|
||||
// Pre-tokenize all items for efficiency
|
||||
const tokenCache = new Map<string, Set<string>>();
|
||||
for (const item of items) {
|
||||
tokenCache.set(item.id, tokenize(item.content));
|
||||
}
|
||||
|
||||
// Normalize scores to [0, 1] for fair comparison with similarity
|
||||
const maxScore = Math.max(...items.map((i) => i.score));
|
||||
const minScore = Math.min(...items.map((i) => i.score));
|
||||
const scoreRange = maxScore - minScore;
|
||||
|
||||
const normalizeScore = (score: number): number => {
|
||||
if (scoreRange === 0) {
|
||||
return 1; // All scores equal
|
||||
}
|
||||
return (score - minScore) / scoreRange;
|
||||
};
|
||||
|
||||
const selected: T[] = [];
|
||||
const remaining = new Set(items);
|
||||
|
||||
// Select items iteratively
|
||||
while (remaining.size > 0) {
|
||||
let bestItem: T | null = null;
|
||||
let bestMMRScore = -Infinity;
|
||||
|
||||
for (const candidate of remaining) {
|
||||
const normalizedRelevance = normalizeScore(candidate.score);
|
||||
const maxSim = maxSimilarityToSelected(candidate, selected, tokenCache);
|
||||
const mmrScore = computeMMRScore(normalizedRelevance, maxSim, clampedLambda);
|
||||
|
||||
// Use original score as tiebreaker (higher is better)
|
||||
if (
|
||||
mmrScore > bestMMRScore ||
|
||||
(mmrScore === bestMMRScore && candidate.score > (bestItem?.score ?? -Infinity))
|
||||
) {
|
||||
bestMMRScore = mmrScore;
|
||||
bestItem = candidate;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestItem) {
|
||||
selected.push(bestItem);
|
||||
remaining.delete(bestItem);
|
||||
} else {
|
||||
// Should never happen, but safety exit
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return selected;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply MMR re-ranking to hybrid search results.
|
||||
* Adapts the generic MMR function to work with the hybrid search result format.
|
||||
*/
|
||||
export function applyMMRToHybridResults<
|
||||
T extends { score: number; snippet: string; path: string; startLine: number },
|
||||
>(results: T[], config: Partial<MMRConfig> = {}): T[] {
|
||||
if (results.length === 0) {
|
||||
return results;
|
||||
}
|
||||
|
||||
// Create a map from ID to original item for type-safe retrieval
|
||||
const itemById = new Map<string, T>();
|
||||
|
||||
// Create MMR items with unique IDs
|
||||
const mmrItems: MMRItem[] = results.map((r, index) => {
|
||||
const id = `${r.path}:${r.startLine}:${index}`;
|
||||
itemById.set(id, r);
|
||||
return {
|
||||
id,
|
||||
score: r.score,
|
||||
content: r.snippet,
|
||||
};
|
||||
});
|
||||
|
||||
const reranked = mmrRerank(mmrItems, config);
|
||||
|
||||
// Map back to original items using the ID
|
||||
return reranked.map((item) => itemById.get(item.id)!);
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { filterUnregisteredMemoryEmbeddingProviderAdapters } from "./provider-adapter-registration.js";
|
||||
|
||||
describe("filterUnregisteredMemoryEmbeddingProviderAdapters", () => {
|
||||
it("keeps builtin adapters that are not already registered", () => {
|
||||
const adapters = filterUnregisteredMemoryEmbeddingProviderAdapters({
|
||||
builtinAdapters: [
|
||||
{ id: "local" },
|
||||
{ id: "openai" },
|
||||
{ id: "gemini" },
|
||||
{ id: "voyage" },
|
||||
{ id: "mistral" },
|
||||
],
|
||||
registeredAdapters: [],
|
||||
});
|
||||
|
||||
expect(adapters.map((adapter) => adapter.id)).toEqual([
|
||||
"local",
|
||||
"openai",
|
||||
"gemini",
|
||||
"voyage",
|
||||
"mistral",
|
||||
]);
|
||||
});
|
||||
|
||||
it("skips builtin adapters that are already registered", () => {
|
||||
const adapters = filterUnregisteredMemoryEmbeddingProviderAdapters({
|
||||
builtinAdapters: [
|
||||
{ id: "local" },
|
||||
{ id: "openai" },
|
||||
{ id: "gemini" },
|
||||
{ id: "voyage" },
|
||||
{ id: "mistral" },
|
||||
],
|
||||
registeredAdapters: [{ id: "local" }, { id: "gemini" }],
|
||||
});
|
||||
|
||||
expect(adapters.map((adapter) => adapter.id)).toEqual(["openai", "voyage", "mistral"]);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
type AdapterLike = {
|
||||
id: string;
|
||||
};
|
||||
|
||||
export function filterUnregisteredMemoryEmbeddingProviderAdapters<T extends AdapterLike>(params: {
|
||||
builtinAdapters: readonly T[];
|
||||
registeredAdapters: readonly AdapterLike[];
|
||||
}): T[] {
|
||||
const existingIds = new Set(params.registeredAdapters.map((adapter) => adapter.id));
|
||||
return params.builtinAdapters.filter((adapter) => !existingIds.has(adapter.id));
|
||||
}
|
||||
166
openclaw/extensions/memory-core/src/memory/provider-adapters.ts
Normal file
166
openclaw/extensions/memory-core/src/memory/provider-adapters.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
import fsSync from "node:fs";
|
||||
import {
|
||||
createLocalEmbeddingProvider,
|
||||
DEFAULT_LOCAL_MODEL,
|
||||
listMemoryEmbeddingProviders,
|
||||
listRegisteredMemoryEmbeddingProviderAdapters,
|
||||
type MemoryEmbeddingProviderAdapter,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-embeddings";
|
||||
import { resolveUserPath } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { getProviderEnvVars } from "openclaw/plugin-sdk/provider-env-vars";
|
||||
import { formatErrorMessage } from "../dreaming-shared.js";
|
||||
import { filterUnregisteredMemoryEmbeddingProviderAdapters } from "./provider-adapter-registration.js";
|
||||
|
||||
export type BuiltinMemoryEmbeddingProviderDoctorMetadata = {
|
||||
providerId: string;
|
||||
authProviderId: string;
|
||||
envVars: string[];
|
||||
transport: "local" | "remote";
|
||||
autoSelectPriority?: number;
|
||||
};
|
||||
|
||||
function isNodeLlamaCppMissing(err: unknown): boolean {
|
||||
if (!(err instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
const code = (err as Error & { code?: unknown }).code;
|
||||
return code === "ERR_MODULE_NOT_FOUND" && err.message.includes("node-llama-cpp");
|
||||
}
|
||||
|
||||
function listRemoteEmbeddingSetupHints(): string[] {
|
||||
try {
|
||||
return listMemoryEmbeddingProviders()
|
||||
.filter(
|
||||
(adapter) =>
|
||||
adapter.transport === "remote" && typeof adapter.autoSelectPriority === "number",
|
||||
)
|
||||
.toSorted((a, b) => (a.autoSelectPriority ?? 0) - (b.autoSelectPriority ?? 0))
|
||||
.map((adapter) => `Or set agents.defaults.memorySearch.provider = "${adapter.id}" (remote).`);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function formatLocalSetupError(err: unknown): string {
|
||||
const detail = formatErrorMessage(err);
|
||||
const missing = isNodeLlamaCppMissing(err);
|
||||
return [
|
||||
"Local embeddings unavailable.",
|
||||
missing
|
||||
? "Reason: optional dependency node-llama-cpp is missing (or failed to install)."
|
||||
: detail
|
||||
? `Reason: ${detail}`
|
||||
: undefined,
|
||||
missing && detail ? `Detail: ${detail}` : null,
|
||||
"To enable local embeddings:",
|
||||
"1) Use Node 24 (recommended for installs/updates; Node 22 LTS, currently 22.14+, remains supported)",
|
||||
missing
|
||||
? "2) Reinstall OpenClaw (this should install node-llama-cpp): npm i -g openclaw@latest"
|
||||
: null,
|
||||
"3) If you use pnpm: pnpm approve-builds (select node-llama-cpp), then pnpm rebuild node-llama-cpp",
|
||||
...listRemoteEmbeddingSetupHints(),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function canAutoSelectLocal(modelPath?: string): boolean {
|
||||
const trimmed = modelPath?.trim();
|
||||
if (!trimmed) {
|
||||
return false;
|
||||
}
|
||||
if (/^(hf:|https?:)/i.test(trimmed)) {
|
||||
return false;
|
||||
}
|
||||
const resolved = resolveUserPath(trimmed);
|
||||
try {
|
||||
return fsSync.statSync(resolved).isFile();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const localAdapter: MemoryEmbeddingProviderAdapter = {
|
||||
id: "local",
|
||||
defaultModel: DEFAULT_LOCAL_MODEL,
|
||||
transport: "local",
|
||||
autoSelectPriority: 10,
|
||||
formatSetupError: formatLocalSetupError,
|
||||
shouldContinueAutoSelection: () => true,
|
||||
create: async (options) => {
|
||||
const provider = await createLocalEmbeddingProvider({
|
||||
...options,
|
||||
provider: "local",
|
||||
fallback: "none",
|
||||
});
|
||||
return {
|
||||
provider,
|
||||
runtime: {
|
||||
id: "local",
|
||||
cacheKeyData: {
|
||||
provider: "local",
|
||||
model: provider.model,
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export const builtinMemoryEmbeddingProviderAdapters = [localAdapter] as const;
|
||||
|
||||
export { DEFAULT_LOCAL_MODEL };
|
||||
|
||||
export function getBuiltinMemoryEmbeddingProviderAdapter(
|
||||
id: string,
|
||||
): MemoryEmbeddingProviderAdapter | undefined {
|
||||
return listMemoryEmbeddingProviders().find((adapter) => adapter.id === id);
|
||||
}
|
||||
|
||||
export function registerBuiltInMemoryEmbeddingProviders(register: {
|
||||
registerMemoryEmbeddingProvider: (adapter: MemoryEmbeddingProviderAdapter) => void;
|
||||
}): void {
|
||||
// Only inspect providers already registered in the current load. Falling back
|
||||
// to capability discovery here can recursively trigger plugin loading while
|
||||
// memory-core itself is still registering.
|
||||
for (const adapter of filterUnregisteredMemoryEmbeddingProviderAdapters({
|
||||
builtinAdapters: builtinMemoryEmbeddingProviderAdapters,
|
||||
registeredAdapters: listRegisteredMemoryEmbeddingProviderAdapters(),
|
||||
})) {
|
||||
register.registerMemoryEmbeddingProvider(adapter);
|
||||
}
|
||||
}
|
||||
|
||||
export function getBuiltinMemoryEmbeddingProviderDoctorMetadata(
|
||||
providerId: string,
|
||||
): BuiltinMemoryEmbeddingProviderDoctorMetadata | null {
|
||||
const adapter = getBuiltinMemoryEmbeddingProviderAdapter(providerId);
|
||||
if (!adapter) {
|
||||
return null;
|
||||
}
|
||||
const authProviderId = adapter.authProviderId ?? adapter.id;
|
||||
return {
|
||||
providerId: adapter.id,
|
||||
authProviderId,
|
||||
envVars: getProviderEnvVars(authProviderId),
|
||||
transport: adapter.transport === "local" ? "local" : "remote",
|
||||
autoSelectPriority: adapter.autoSelectPriority,
|
||||
};
|
||||
}
|
||||
|
||||
export function listBuiltinAutoSelectMemoryEmbeddingProviderDoctorMetadata(): Array<BuiltinMemoryEmbeddingProviderDoctorMetadata> {
|
||||
return listMemoryEmbeddingProviders()
|
||||
.filter((adapter) => typeof adapter.autoSelectPriority === "number")
|
||||
.toSorted((a, b) => (a.autoSelectPriority ?? 0) - (b.autoSelectPriority ?? 0))
|
||||
.map((adapter) => {
|
||||
const authProviderId = adapter.authProviderId ?? adapter.id;
|
||||
return {
|
||||
providerId: adapter.id,
|
||||
authProviderId,
|
||||
envVars: getProviderEnvVars(authProviderId),
|
||||
transport: adapter.transport === "local" ? "local" : "remote",
|
||||
autoSelectPriority: adapter.autoSelectPriority,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export { canAutoSelectLocal, formatLocalSetupError };
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { resolveQmdCollectionPatternFlags } from "./qmd-compat.js";
|
||||
|
||||
describe("resolveQmdCollectionPatternFlags", () => {
|
||||
it("prefers modern --glob by default and falls back to legacy --mask", () => {
|
||||
expect(resolveQmdCollectionPatternFlags(null)).toEqual(["--glob", "--mask"]);
|
||||
expect(resolveQmdCollectionPatternFlags("--glob")).toEqual(["--glob", "--mask"]);
|
||||
});
|
||||
|
||||
it("keeps preferring legacy --mask after a legacy-only qmd succeeds", () => {
|
||||
expect(resolveQmdCollectionPatternFlags("--mask")).toEqual(["--mask", "--glob"]);
|
||||
});
|
||||
});
|
||||
7
openclaw/extensions/memory-core/src/memory/qmd-compat.ts
Normal file
7
openclaw/extensions/memory-core/src/memory/qmd-compat.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
export type QmdCollectionPatternFlag = "--glob" | "--mask";
|
||||
|
||||
export function resolveQmdCollectionPatternFlags(
|
||||
preferredFlag: QmdCollectionPatternFlag | null,
|
||||
): QmdCollectionPatternFlag[] {
|
||||
return preferredFlag === "--mask" ? ["--mask", "--glob"] : ["--glob", "--mask"];
|
||||
}
|
||||
|
|
@ -0,0 +1,392 @@
|
|||
import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { Mock } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { logWarnMock, logDebugMock, logInfoMock } = vi.hoisted(() => ({
|
||||
logWarnMock: vi.fn(),
|
||||
logDebugMock: vi.fn(),
|
||||
logInfoMock: vi.fn(),
|
||||
}));
|
||||
|
||||
interface MockChild extends EventEmitter {
|
||||
stdout: EventEmitter;
|
||||
stderr: EventEmitter;
|
||||
kill: (signal?: NodeJS.Signals) => void;
|
||||
closeWith: (code?: number | null) => void;
|
||||
}
|
||||
|
||||
function createMockChild(params?: { autoClose?: boolean }): MockChild {
|
||||
const stdout = new EventEmitter();
|
||||
const stderr = new EventEmitter();
|
||||
const child = new EventEmitter() as unknown as MockChild;
|
||||
child.stdout = stdout;
|
||||
child.stderr = stderr;
|
||||
child.closeWith = (code = 0) => {
|
||||
child.emit("close", code);
|
||||
};
|
||||
child.kill = () => {};
|
||||
if (params?.autoClose !== false) {
|
||||
queueMicrotask(() => {
|
||||
child.emit("close", 0);
|
||||
});
|
||||
}
|
||||
return child;
|
||||
}
|
||||
|
||||
function emitAndClose(
|
||||
child: MockChild,
|
||||
stream: "stdout" | "stderr",
|
||||
data: string,
|
||||
code: number = 0,
|
||||
) {
|
||||
queueMicrotask(() => {
|
||||
child[stream].emit("data", data);
|
||||
child.closeWith(code);
|
||||
});
|
||||
}
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/memory-core-host-engine-foundation", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("openclaw/plugin-sdk/memory-core-host-engine-foundation")
|
||||
>("openclaw/plugin-sdk/memory-core-host-engine-foundation");
|
||||
return {
|
||||
...actual,
|
||||
createSubsystemLogger: () => {
|
||||
const logger = {
|
||||
warn: logWarnMock,
|
||||
debug: logDebugMock,
|
||||
info: logInfoMock,
|
||||
child: () => logger,
|
||||
};
|
||||
return logger;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("node:child_process", async () => {
|
||||
const actual = await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
return {
|
||||
...actual,
|
||||
spawn: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
import { spawn as mockedSpawn } from "node:child_process";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { resolveMemoryBackendConfig } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
import { QmdMemoryManager } from "./qmd-manager.js";
|
||||
|
||||
const spawnMock = mockedSpawn as unknown as Mock;
|
||||
|
||||
describe("QmdMemoryManager slugified path resolution", () => {
|
||||
let tmpRoot: string;
|
||||
let workspaceDir: string;
|
||||
let stateDir: string;
|
||||
let cfg: OpenClawConfig;
|
||||
const agentId = "main";
|
||||
const openManagers = new Set<QmdMemoryManager>();
|
||||
|
||||
function trackManager<T extends QmdMemoryManager | null>(manager: T): T {
|
||||
if (manager) {
|
||||
openManagers.add(manager);
|
||||
}
|
||||
return manager;
|
||||
}
|
||||
|
||||
async function createManager(params?: { cfg?: OpenClawConfig }) {
|
||||
const cfgToUse = params?.cfg ?? cfg;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg: cfgToUse, agentId });
|
||||
const manager = trackManager(
|
||||
await QmdMemoryManager.create({
|
||||
cfg: cfgToUse,
|
||||
agentId,
|
||||
resolved,
|
||||
mode: "status",
|
||||
}),
|
||||
);
|
||||
if (!manager) {
|
||||
throw new Error("manager missing");
|
||||
}
|
||||
return { manager, resolved };
|
||||
}
|
||||
|
||||
function installIndexedPathStub(params: {
|
||||
manager: QmdMemoryManager;
|
||||
collection: string;
|
||||
normalizedPath: string;
|
||||
actualPath?: string;
|
||||
exactPaths?: string[];
|
||||
allPaths?: string[];
|
||||
}) {
|
||||
const inner = params.manager as unknown as {
|
||||
db: {
|
||||
prepare: (query: string) => {
|
||||
get: (...args: unknown[]) => unknown;
|
||||
all: (...args: unknown[]) => unknown;
|
||||
};
|
||||
close: () => void;
|
||||
};
|
||||
};
|
||||
inner.db = {
|
||||
prepare: (query: string) => ({
|
||||
get: (...args: unknown[]) => {
|
||||
if (query.includes("collection = ? AND active = 1 AND path = ?")) {
|
||||
expect(args[0]).toBe(params.collection);
|
||||
const requestedPath = args[1];
|
||||
expect(typeof requestedPath).toBe("string");
|
||||
const exactCandidates = new Set([
|
||||
...(params.exactPaths ?? []),
|
||||
...(params.actualPath ? [params.actualPath] : []),
|
||||
]);
|
||||
return typeof requestedPath === "string" && exactCandidates.has(requestedPath)
|
||||
? { path: requestedPath }
|
||||
: undefined;
|
||||
}
|
||||
throw new Error(`unexpected sqlite query: ${query}`);
|
||||
},
|
||||
all: (...args: unknown[]) => {
|
||||
if (query.includes("collection = ? AND path = ? AND active = 1")) {
|
||||
expect(args).toEqual([params.collection, params.normalizedPath]);
|
||||
return (params.exactPaths ?? []).map((pathValue) => ({ path: pathValue }));
|
||||
}
|
||||
if (query.includes("collection = ? AND active = 1")) {
|
||||
expect(args).toEqual([params.collection]);
|
||||
return (params.allPaths ?? [params.actualPath]).map((pathValue) => ({
|
||||
path: pathValue,
|
||||
}));
|
||||
}
|
||||
throw new Error(`unexpected sqlite query: ${query}`);
|
||||
},
|
||||
}),
|
||||
close: () => {},
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
spawnMock.mockReset();
|
||||
spawnMock.mockImplementation(() => createMockChild());
|
||||
logWarnMock.mockClear();
|
||||
logDebugMock.mockClear();
|
||||
logInfoMock.mockClear();
|
||||
|
||||
tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qmd-slugified-"));
|
||||
workspaceDir = path.join(tmpRoot, "workspace");
|
||||
stateDir = path.join(tmpRoot, "state");
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||
|
||||
cfg = {
|
||||
agents: {
|
||||
list: [{ id: agentId, default: true, workspace: workspaceDir }],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
Array.from(openManagers, async (manager) => {
|
||||
await manager.close();
|
||||
}),
|
||||
);
|
||||
openManagers.clear();
|
||||
await fs.rm(tmpRoot, { recursive: true, force: true });
|
||||
delete process.env.OPENCLAW_STATE_DIR;
|
||||
});
|
||||
|
||||
it("maps slugified workspace qmd URIs back to the indexed filesystem path", async () => {
|
||||
const actualRelative = "extra-docs/Category/Sub Category/Topic Name/Topic Name.md";
|
||||
const actualFile = path.join(workspaceDir, actualRelative);
|
||||
await fs.mkdir(path.dirname(actualFile), { recursive: true });
|
||||
await fs.writeFile(actualFile, "line-1\nline-2\nline-3", "utf-8");
|
||||
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "search") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(
|
||||
child,
|
||||
"stdout",
|
||||
JSON.stringify([
|
||||
{
|
||||
file: "qmd://workspace-main/extra-docs/category/sub-category/topic-name/topic-name.md",
|
||||
score: 0.73,
|
||||
snippet: "@@ -2,1\nline-2",
|
||||
},
|
||||
]),
|
||||
);
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
|
||||
const { manager } = await createManager();
|
||||
installIndexedPathStub({
|
||||
manager,
|
||||
collection: "workspace-main",
|
||||
normalizedPath: "extra-docs/category/sub-category/topic-name/topic-name.md",
|
||||
actualPath: actualRelative,
|
||||
});
|
||||
|
||||
const results = await manager.search("line-2", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
});
|
||||
expect(results).toEqual([
|
||||
{
|
||||
path: actualRelative,
|
||||
startLine: 2,
|
||||
endLine: 2,
|
||||
score: 0.73,
|
||||
snippet: "@@ -2,1\nline-2",
|
||||
source: "memory",
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(manager.readFile({ relPath: results[0].path })).resolves.toEqual({
|
||||
path: actualRelative,
|
||||
text: "line-1\nline-2\nline-3",
|
||||
from: 1,
|
||||
lines: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it("maps slugified extra collection qmd URIs back to qmd/<collection>/ paths", async () => {
|
||||
const extraRoot = path.join(tmpRoot, "vault");
|
||||
await fs.mkdir(extraRoot, { recursive: true });
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
paths: [{ path: extraRoot, pattern: "**/*.md", name: "vault" }],
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const actualRelative = "Topics/Sub Category/Topic Name.md";
|
||||
const actualFile = path.join(extraRoot, actualRelative);
|
||||
await fs.mkdir(path.dirname(actualFile), { recursive: true });
|
||||
await fs.writeFile(actualFile, "vault memory", "utf-8");
|
||||
|
||||
const { manager, resolved } = await createManager({ cfg });
|
||||
const collectionName =
|
||||
resolved.qmd?.collections.find((collection) => collection.path === extraRoot)?.name ??
|
||||
"vault";
|
||||
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "search") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(
|
||||
child,
|
||||
"stdout",
|
||||
JSON.stringify([
|
||||
{
|
||||
file: `qmd://${collectionName}/topics/sub-category/topic-name.md`,
|
||||
score: 0.81,
|
||||
snippet: "@@ -1,1\nvault memory",
|
||||
},
|
||||
]),
|
||||
);
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
installIndexedPathStub({
|
||||
manager,
|
||||
collection: collectionName,
|
||||
normalizedPath: "topics/sub-category/topic-name.md",
|
||||
actualPath: actualRelative,
|
||||
});
|
||||
|
||||
const results = await manager.search("vault memory", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
});
|
||||
expect(results).toEqual([
|
||||
{
|
||||
path: `qmd/${collectionName}/${actualRelative}`,
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.81,
|
||||
snippet: "@@ -1,1\nvault memory",
|
||||
source: "memory",
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(manager.readFile({ relPath: results[0].path })).resolves.toEqual({
|
||||
path: `qmd/${collectionName}/${actualRelative}`,
|
||||
text: "vault memory",
|
||||
from: 1,
|
||||
lines: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers an exact indexed path over normalized slug recovery", async () => {
|
||||
const exactRelative = "notes/topic-name.md";
|
||||
const slugCollisionRelative = "notes/Topic Name.md";
|
||||
const exactFile = path.join(workspaceDir, exactRelative);
|
||||
const collisionFile = path.join(workspaceDir, slugCollisionRelative);
|
||||
await fs.mkdir(path.dirname(exactFile), { recursive: true });
|
||||
await fs.writeFile(exactFile, "exact slugified path", "utf-8");
|
||||
await fs.writeFile(collisionFile, "mixed case path", "utf-8");
|
||||
|
||||
spawnMock.mockImplementation((_cmd: string, args: string[]) => {
|
||||
if (args[0] === "search") {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
emitAndClose(
|
||||
child,
|
||||
"stdout",
|
||||
JSON.stringify([
|
||||
{
|
||||
file: "qmd://workspace-main/notes/topic-name.md",
|
||||
score: 0.79,
|
||||
snippet: "@@ -1,1\nexact slugified path",
|
||||
},
|
||||
]),
|
||||
);
|
||||
return child;
|
||||
}
|
||||
return createMockChild();
|
||||
});
|
||||
|
||||
const { manager } = await createManager();
|
||||
installIndexedPathStub({
|
||||
manager,
|
||||
collection: "workspace-main",
|
||||
normalizedPath: exactRelative,
|
||||
exactPaths: [exactRelative],
|
||||
allPaths: [exactRelative, slugCollisionRelative],
|
||||
});
|
||||
|
||||
const results = await manager.search("exact slugified path", {
|
||||
sessionKey: "agent:main:slack:dm:u123",
|
||||
});
|
||||
expect(results).toEqual([
|
||||
{
|
||||
path: exactRelative,
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 0.79,
|
||||
snippet: "@@ -1,1\nexact slugified path",
|
||||
source: "memory",
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(manager.readFile({ relPath: results[0].path })).resolves.toEqual({
|
||||
path: exactRelative,
|
||||
text: "exact slugified path",
|
||||
from: 1,
|
||||
lines: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
4389
openclaw/extensions/memory-core/src/memory/qmd-manager.test.ts
Normal file
4389
openclaw/extensions/memory-core/src/memory/qmd-manager.test.ts
Normal file
File diff suppressed because it is too large
Load diff
2898
openclaw/extensions/memory-core/src/memory/qmd-manager.ts
Normal file
2898
openclaw/extensions/memory-core/src/memory/qmd-manager.ts
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,493 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import type { checkQmdBinaryAvailability as checkQmdBinaryAvailabilityFn } from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
type CheckQmdBinaryAvailability = typeof checkQmdBinaryAvailabilityFn;
|
||||
|
||||
function createManagerStatus(params: {
|
||||
backend: "qmd" | "builtin";
|
||||
provider: string;
|
||||
model: string;
|
||||
requestedProvider: string;
|
||||
withMemorySourceCounts?: boolean;
|
||||
}) {
|
||||
const base = {
|
||||
backend: params.backend,
|
||||
provider: params.provider,
|
||||
model: params.model,
|
||||
requestedProvider: params.requestedProvider,
|
||||
files: 0,
|
||||
chunks: 0,
|
||||
dirty: false,
|
||||
workspaceDir: "/tmp",
|
||||
dbPath: "/tmp/index.sqlite",
|
||||
};
|
||||
if (!params.withMemorySourceCounts) {
|
||||
return base;
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
sources: ["memory" as const],
|
||||
sourceCounts: [{ source: "memory" as const, files: 0, chunks: 0 }],
|
||||
};
|
||||
}
|
||||
|
||||
function createManagerMock(params: {
|
||||
backend: "qmd" | "builtin";
|
||||
provider: string;
|
||||
model: string;
|
||||
requestedProvider: string;
|
||||
searchResults?: Array<{
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
snippet: string;
|
||||
source: "memory";
|
||||
}>;
|
||||
withMemorySourceCounts?: boolean;
|
||||
}) {
|
||||
return {
|
||||
search: vi.fn(async () => params.searchResults ?? []),
|
||||
readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })),
|
||||
status: vi.fn(() =>
|
||||
createManagerStatus({
|
||||
backend: params.backend,
|
||||
provider: params.provider,
|
||||
model: params.model,
|
||||
requestedProvider: params.requestedProvider,
|
||||
withMemorySourceCounts: params.withMemorySourceCounts,
|
||||
}),
|
||||
),
|
||||
sync: vi.fn(async () => {}),
|
||||
probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })),
|
||||
probeVectorAvailability: vi.fn(async () => true),
|
||||
close: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
const mockPrimary = vi.hoisted(() => ({
|
||||
...createManagerMock({
|
||||
backend: "qmd",
|
||||
provider: "qmd",
|
||||
model: "qmd",
|
||||
requestedProvider: "qmd",
|
||||
withMemorySourceCounts: true,
|
||||
}),
|
||||
}));
|
||||
|
||||
const fallbackManager = vi.hoisted(() => ({
|
||||
...createManagerMock({
|
||||
backend: "builtin",
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
requestedProvider: "openai",
|
||||
searchResults: [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
score: 1,
|
||||
snippet: "fallback",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
}),
|
||||
}));
|
||||
|
||||
const fallbackSearch = fallbackManager.search;
|
||||
const mockMemoryIndexGet = vi.hoisted(() => vi.fn(async () => fallbackManager));
|
||||
const mockCloseAllMemoryIndexManagers = vi.hoisted(() => vi.fn(async () => {}));
|
||||
const checkQmdBinaryAvailability = vi.hoisted(() =>
|
||||
vi.fn<CheckQmdBinaryAvailability>(async () => ({ available: true })),
|
||||
);
|
||||
|
||||
vi.mock("./qmd-manager.js", () => ({
|
||||
QmdMemoryManager: {
|
||||
create: vi.fn(async () => mockPrimary),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/memory-core-host-engine-qmd", () => ({
|
||||
checkQmdBinaryAvailability,
|
||||
}));
|
||||
|
||||
vi.mock("../../manager-runtime.js", () => ({
|
||||
MemoryIndexManager: {
|
||||
get: mockMemoryIndexGet,
|
||||
},
|
||||
closeAllMemoryIndexManagers: mockCloseAllMemoryIndexManagers,
|
||||
}));
|
||||
|
||||
import { QmdMemoryManager } from "./qmd-manager.js";
|
||||
import { closeAllMemorySearchManagers, getMemorySearchManager } from "./search-manager.js";
|
||||
const createQmdManagerMock = vi.mocked(QmdMemoryManager.create);
|
||||
|
||||
type SearchManagerResult = Awaited<ReturnType<typeof getMemorySearchManager>>;
|
||||
type SearchManager = NonNullable<SearchManagerResult["manager"]>;
|
||||
|
||||
function createQmdCfg(agentId: string): OpenClawConfig {
|
||||
return {
|
||||
memory: { backend: "qmd", qmd: {} },
|
||||
agents: { list: [{ id: agentId, default: true, workspace: "/tmp/workspace" }] },
|
||||
};
|
||||
}
|
||||
|
||||
function createBuiltinCfg(agentId: string): OpenClawConfig {
|
||||
return {
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: "/tmp/workspace",
|
||||
memorySearch: {
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
store: {
|
||||
path: "/tmp/index.sqlite",
|
||||
vector: { enabled: false },
|
||||
},
|
||||
sync: { watch: false, onSessionStart: false, onSearch: false },
|
||||
query: { minScore: 0, hybrid: { enabled: false } },
|
||||
sources: ["memory"],
|
||||
experimental: { sessionMemory: false },
|
||||
},
|
||||
},
|
||||
list: [{ id: agentId, default: true, workspace: "/tmp/workspace" }],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
}
|
||||
|
||||
function requireManager(result: SearchManagerResult): SearchManager {
|
||||
expect(result.manager).toBeTruthy();
|
||||
if (!result.manager) {
|
||||
throw new Error("manager missing");
|
||||
}
|
||||
return result.manager;
|
||||
}
|
||||
|
||||
async function createFailedQmdSearchHarness(params: { agentId: string; errorMessage: string }) {
|
||||
const cfg = createQmdCfg(params.agentId);
|
||||
mockPrimary.search.mockRejectedValueOnce(new Error(params.errorMessage));
|
||||
const first = await getMemorySearchManager({ cfg, agentId: params.agentId });
|
||||
return { cfg, manager: requireManager(first), firstResult: first };
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await closeAllMemorySearchManagers();
|
||||
mockPrimary.search.mockClear();
|
||||
mockPrimary.readFile.mockClear();
|
||||
mockPrimary.status.mockClear();
|
||||
mockPrimary.sync.mockClear();
|
||||
mockPrimary.probeEmbeddingAvailability.mockClear();
|
||||
mockPrimary.probeVectorAvailability.mockClear();
|
||||
mockPrimary.close.mockClear();
|
||||
fallbackSearch.mockClear();
|
||||
fallbackManager.readFile.mockClear();
|
||||
fallbackManager.status.mockClear();
|
||||
fallbackManager.sync.mockClear();
|
||||
fallbackManager.probeEmbeddingAvailability.mockClear();
|
||||
fallbackManager.probeVectorAvailability.mockClear();
|
||||
fallbackManager.close.mockClear();
|
||||
mockCloseAllMemoryIndexManagers.mockClear();
|
||||
mockMemoryIndexGet.mockClear();
|
||||
mockMemoryIndexGet.mockResolvedValue(fallbackManager);
|
||||
checkQmdBinaryAvailability.mockClear();
|
||||
checkQmdBinaryAvailability.mockResolvedValue({ available: true });
|
||||
createQmdManagerMock.mockClear();
|
||||
});
|
||||
|
||||
describe("getMemorySearchManager caching", () => {
|
||||
it("reuses the same QMD manager instance for repeated calls", async () => {
|
||||
const cfg = createQmdCfg("main");
|
||||
|
||||
const first = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
const second = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
|
||||
expect(first.manager).toBe(second.manager);
|
||||
expect(createQmdManagerMock.mock.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("evicts failed qmd wrapper so next call retries qmd", async () => {
|
||||
const retryAgentId = "retry-agent";
|
||||
const {
|
||||
cfg,
|
||||
manager: firstManager,
|
||||
firstResult: first,
|
||||
} = await createFailedQmdSearchHarness({
|
||||
agentId: retryAgentId,
|
||||
errorMessage: "qmd query failed",
|
||||
});
|
||||
|
||||
const fallbackResults = await firstManager.search("hello");
|
||||
expect(fallbackResults).toHaveLength(1);
|
||||
expect(fallbackResults[0]?.path).toBe("MEMORY.md");
|
||||
|
||||
const second = await getMemorySearchManager({ cfg, agentId: retryAgentId });
|
||||
requireManager(second);
|
||||
expect(second.manager).not.toBe(first.manager);
|
||||
expect(createQmdManagerMock.mock.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("falls back immediately when the qmd binary is unavailable", async () => {
|
||||
const cfg = createQmdCfg("missing-qmd");
|
||||
checkQmdBinaryAvailability.mockResolvedValueOnce({
|
||||
available: false,
|
||||
error: "spawn qmd ENOENT",
|
||||
});
|
||||
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "missing-qmd" });
|
||||
const manager = requireManager(result);
|
||||
const searchResults = await manager.search("hello");
|
||||
|
||||
expect(createQmdManagerMock).not.toHaveBeenCalled();
|
||||
expect(mockMemoryIndexGet).toHaveBeenCalled();
|
||||
expect(searchResults).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("probes qmd availability from the agent workspace", async () => {
|
||||
const agentId = "workspace-probe";
|
||||
const cfg = createQmdCfg(agentId);
|
||||
|
||||
await getMemorySearchManager({ cfg, agentId });
|
||||
|
||||
expect(checkQmdBinaryAvailability).toHaveBeenCalledWith({
|
||||
command: "qmd",
|
||||
env: process.env,
|
||||
cwd: "/tmp/workspace",
|
||||
});
|
||||
});
|
||||
|
||||
it("creates a missing agent workspace before probing qmd availability", async () => {
|
||||
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qmd-workspace-"));
|
||||
const workspace = path.join(tempRoot, "missing", "workspace");
|
||||
const agentId = "missing-workspace";
|
||||
const cfg = {
|
||||
memory: { backend: "qmd", qmd: {} },
|
||||
agents: { list: [{ id: agentId, default: true, workspace }] },
|
||||
} as OpenClawConfig;
|
||||
|
||||
try {
|
||||
await getMemorySearchManager({ cfg, agentId });
|
||||
|
||||
const stat = await fs.stat(workspace);
|
||||
expect(stat.isDirectory()).toBe(true);
|
||||
expect(checkQmdBinaryAvailability).toHaveBeenCalledWith({
|
||||
command: "qmd",
|
||||
env: process.env,
|
||||
cwd: workspace,
|
||||
});
|
||||
} finally {
|
||||
await fs.rm(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("returns a cached qmd manager without probing the binary again", async () => {
|
||||
const agentId = "cached-qmd";
|
||||
const cfg = createQmdCfg(agentId);
|
||||
|
||||
const first = await getMemorySearchManager({ cfg, agentId });
|
||||
const second = await getMemorySearchManager({ cfg, agentId });
|
||||
|
||||
requireManager(first);
|
||||
requireManager(second);
|
||||
expect(first.manager).toBe(second.manager);
|
||||
expect(checkQmdBinaryAvailability).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not cache qmd managers for status-only requests", async () => {
|
||||
const agentId = "status-agent";
|
||||
const cfg = createQmdCfg(agentId);
|
||||
|
||||
const first = await getMemorySearchManager({ cfg, agentId, purpose: "status" });
|
||||
const second = await getMemorySearchManager({ cfg, agentId, purpose: "status" });
|
||||
|
||||
requireManager(first);
|
||||
requireManager(second);
|
||||
expect(first.manager?.status()).toMatchObject({
|
||||
backend: "qmd",
|
||||
provider: "qmd",
|
||||
model: "qmd",
|
||||
requestedProvider: "qmd",
|
||||
});
|
||||
expect(createQmdManagerMock.mock.calls).toHaveLength(2);
|
||||
expect(mockMemoryIndexGet).not.toHaveBeenCalled();
|
||||
|
||||
await first.manager?.close?.();
|
||||
await second.manager?.close?.();
|
||||
expect(mockPrimary.close).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not cache builtin managers for status-only requests", async () => {
|
||||
const agentId = "builtin-status-agent";
|
||||
const cfg = createBuiltinCfg(agentId);
|
||||
const firstBuiltinManager = createManagerMock({
|
||||
backend: "builtin",
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
requestedProvider: "openai",
|
||||
});
|
||||
const secondBuiltinManager = createManagerMock({
|
||||
backend: "builtin",
|
||||
provider: "openai",
|
||||
model: "text-embedding-3-small",
|
||||
requestedProvider: "openai",
|
||||
});
|
||||
mockMemoryIndexGet
|
||||
.mockResolvedValueOnce(firstBuiltinManager)
|
||||
.mockResolvedValueOnce(secondBuiltinManager);
|
||||
|
||||
const first = await getMemorySearchManager({ cfg, agentId, purpose: "status" });
|
||||
const second = await getMemorySearchManager({ cfg, agentId, purpose: "status" });
|
||||
|
||||
expect(first.manager).toBe(firstBuiltinManager);
|
||||
expect(second.manager).toBe(secondBuiltinManager);
|
||||
expect(second.manager).not.toBe(first.manager);
|
||||
expect(mockMemoryIndexGet).toHaveBeenCalledTimes(2);
|
||||
|
||||
await first.manager?.close?.();
|
||||
await second.manager?.close?.();
|
||||
expect(firstBuiltinManager.close).toHaveBeenCalledTimes(1);
|
||||
expect(secondBuiltinManager.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reports real qmd index counts for status-only requests", async () => {
|
||||
const agentId = "status-counts-agent";
|
||||
const cfg = createQmdCfg(agentId);
|
||||
mockPrimary.status.mockReturnValueOnce({
|
||||
...createManagerStatus({
|
||||
backend: "qmd",
|
||||
provider: "qmd",
|
||||
model: "qmd",
|
||||
requestedProvider: "qmd",
|
||||
withMemorySourceCounts: true,
|
||||
}),
|
||||
files: 10,
|
||||
chunks: 42,
|
||||
sourceCounts: [{ source: "memory" as const, files: 10, chunks: 42 }],
|
||||
});
|
||||
|
||||
const result = await getMemorySearchManager({ cfg, agentId, purpose: "status" });
|
||||
const manager = requireManager(result);
|
||||
|
||||
expect(manager.status()).toMatchObject({
|
||||
backend: "qmd",
|
||||
files: 10,
|
||||
chunks: 42,
|
||||
sourceCounts: [{ source: "memory", files: 10, chunks: 42 }],
|
||||
});
|
||||
expect(createQmdManagerMock.mock.calls[0]?.[0]).toEqual(
|
||||
expect.objectContaining({ agentId, mode: "status" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("reuses cached full qmd manager for status-only requests", async () => {
|
||||
const agentId = "status-reuses-full-agent";
|
||||
const cfg = createQmdCfg(agentId);
|
||||
|
||||
const full = await getMemorySearchManager({ cfg, agentId });
|
||||
const status = await getMemorySearchManager({ cfg, agentId, purpose: "status" });
|
||||
|
||||
requireManager(full);
|
||||
requireManager(status);
|
||||
expect(status.manager).not.toBe(full.manager);
|
||||
expect(createQmdManagerMock.mock.calls).toHaveLength(1);
|
||||
await status.manager?.close?.();
|
||||
expect(mockPrimary.close).not.toHaveBeenCalled();
|
||||
|
||||
const fullAgain = await getMemorySearchManager({ cfg, agentId });
|
||||
expect(fullAgain.manager).toBe(full.manager);
|
||||
});
|
||||
|
||||
it("gets a fresh qmd manager for later status requests after close", async () => {
|
||||
const agentId = "status-eviction-agent";
|
||||
const cfg = createQmdCfg(agentId);
|
||||
|
||||
const first = await getMemorySearchManager({ cfg, agentId, purpose: "status" });
|
||||
const firstManager = requireManager(first);
|
||||
await firstManager.close?.();
|
||||
|
||||
const second = await getMemorySearchManager({ cfg, agentId, purpose: "status" });
|
||||
requireManager(second);
|
||||
|
||||
expect(createQmdManagerMock.mock.calls).toHaveLength(2);
|
||||
expect(mockPrimary.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not evict a newer cached wrapper when closing an older failed wrapper", async () => {
|
||||
const retryAgentId = "retry-agent-close";
|
||||
const {
|
||||
cfg,
|
||||
manager: firstManager,
|
||||
firstResult: first,
|
||||
} = await createFailedQmdSearchHarness({
|
||||
agentId: retryAgentId,
|
||||
errorMessage: "qmd query failed",
|
||||
});
|
||||
await firstManager.search("hello");
|
||||
|
||||
const second = await getMemorySearchManager({ cfg, agentId: retryAgentId });
|
||||
const secondManager = requireManager(second);
|
||||
expect(second.manager).not.toBe(first.manager);
|
||||
|
||||
await firstManager.close?.();
|
||||
|
||||
const third = await getMemorySearchManager({ cfg, agentId: retryAgentId });
|
||||
expect(third.manager).toBe(secondManager);
|
||||
expect(createQmdManagerMock.mock.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("falls back to builtin search when qmd fails with sqlite busy", async () => {
|
||||
const retryAgentId = "retry-agent-busy";
|
||||
const { manager: firstManager } = await createFailedQmdSearchHarness({
|
||||
agentId: retryAgentId,
|
||||
errorMessage: "qmd index busy while reading results: SQLITE_BUSY: database is locked",
|
||||
});
|
||||
|
||||
const results = await firstManager.search("hello");
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.path).toBe("MEMORY.md");
|
||||
expect(fallbackSearch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps original qmd error when fallback manager initialization fails", async () => {
|
||||
const retryAgentId = "retry-agent-no-fallback-auth";
|
||||
const { manager: firstManager } = await createFailedQmdSearchHarness({
|
||||
agentId: retryAgentId,
|
||||
errorMessage: "qmd query failed",
|
||||
});
|
||||
mockMemoryIndexGet.mockRejectedValueOnce(new Error("No API key found for provider openai"));
|
||||
|
||||
await expect(firstManager.search("hello")).rejects.toThrow("qmd query failed");
|
||||
});
|
||||
|
||||
it("closes cached managers on global teardown", async () => {
|
||||
const cfg = createQmdCfg("teardown-agent");
|
||||
const first = await getMemorySearchManager({ cfg, agentId: "teardown-agent" });
|
||||
const firstManager = requireManager(first);
|
||||
|
||||
await closeAllMemorySearchManagers();
|
||||
|
||||
expect(mockPrimary.close).toHaveBeenCalledTimes(1);
|
||||
expect(mockCloseAllMemoryIndexManagers).toHaveBeenCalledTimes(1);
|
||||
|
||||
const second = await getMemorySearchManager({ cfg, agentId: "teardown-agent" });
|
||||
expect(second.manager).toBeTruthy();
|
||||
expect(second.manager).not.toBe(firstManager);
|
||||
expect(createQmdManagerMock.mock.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("closes builtin index managers on teardown after runtime is loaded", async () => {
|
||||
const retryAgentId = "teardown-with-fallback";
|
||||
const { manager } = await createFailedQmdSearchHarness({
|
||||
agentId: retryAgentId,
|
||||
errorMessage: "qmd query failed",
|
||||
});
|
||||
await manager.search("hello");
|
||||
|
||||
await closeAllMemorySearchManagers();
|
||||
|
||||
expect(mockCloseAllMemoryIndexManagers).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
365
openclaw/extensions/memory-core/src/memory/search-manager.ts
Normal file
365
openclaw/extensions/memory-core/src/memory/search-manager.ts
Normal file
|
|
@ -0,0 +1,365 @@
|
|||
import fs from "node:fs/promises";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
createSubsystemLogger,
|
||||
resolveAgentWorkspaceDir,
|
||||
resolveGlobalSingleton,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
|
||||
import { checkQmdBinaryAvailability } from "openclaw/plugin-sdk/memory-core-host-engine-qmd";
|
||||
import {
|
||||
resolveMemoryBackendConfig,
|
||||
type MemoryEmbeddingProbeResult,
|
||||
type MemorySearchManager,
|
||||
type MemorySearchRuntimeDebug,
|
||||
type MemorySyncProgressUpdate,
|
||||
type ResolvedQmdConfig,
|
||||
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
|
||||
|
||||
const MEMORY_SEARCH_MANAGER_CACHE_KEY = Symbol.for("openclaw.memorySearchManagerCache");
|
||||
type MemorySearchManagerCacheStore = {
|
||||
qmdManagerCache: Map<string, MemorySearchManager>;
|
||||
};
|
||||
|
||||
function getMemorySearchManagerCacheStore(): MemorySearchManagerCacheStore {
|
||||
// Keep caches reachable across `vi.resetModules()` so later cleanup can close older instances.
|
||||
return resolveGlobalSingleton<MemorySearchManagerCacheStore>(
|
||||
MEMORY_SEARCH_MANAGER_CACHE_KEY,
|
||||
() => ({
|
||||
qmdManagerCache: new Map<string, MemorySearchManager>(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
const { qmdManagerCache: QMD_MANAGER_CACHE } = getMemorySearchManagerCacheStore();
|
||||
let managerRuntimePromise: Promise<typeof import("../../manager-runtime.js")> | null = null;
|
||||
let qmdManagerModulePromise: Promise<typeof import("./qmd-manager.js")> | null = null;
|
||||
|
||||
function loadManagerRuntime() {
|
||||
managerRuntimePromise ??= import("../../manager-runtime.js");
|
||||
return managerRuntimePromise;
|
||||
}
|
||||
|
||||
function loadQmdManagerModule() {
|
||||
qmdManagerModulePromise ??= import("./qmd-manager.js");
|
||||
return qmdManagerModulePromise;
|
||||
}
|
||||
|
||||
export type MemorySearchManagerResult = {
|
||||
manager: MemorySearchManager | null;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export async function getMemorySearchManager(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
purpose?: "default" | "status";
|
||||
}): Promise<MemorySearchManagerResult> {
|
||||
const resolved = resolveMemoryBackendConfig(params);
|
||||
if (resolved.backend === "qmd" && resolved.qmd) {
|
||||
const statusOnly = params.purpose === "status";
|
||||
const baseCacheKey = buildQmdCacheKey(params.agentId, resolved.qmd);
|
||||
const cacheKey = `${baseCacheKey}:${statusOnly ? "status" : "full"}`;
|
||||
const cached = QMD_MANAGER_CACHE.get(cacheKey);
|
||||
if (cached) {
|
||||
return { manager: cached };
|
||||
}
|
||||
if (statusOnly) {
|
||||
const fullCached = QMD_MANAGER_CACHE.get(`${baseCacheKey}:full`);
|
||||
if (fullCached) {
|
||||
// Status callers often close the manager they receive. Wrap the live
|
||||
// full manager with a no-op close so health/status probes do not tear
|
||||
// down the active QMD manager for the process.
|
||||
return { manager: new BorrowedMemoryManager(fullCached) };
|
||||
}
|
||||
}
|
||||
|
||||
const workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId);
|
||||
try {
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
} catch (err) {
|
||||
log.warn(
|
||||
`qmd workspace unavailable (${workspaceDir}); falling back to builtin: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
return await getBuiltinMemorySearchManager(params);
|
||||
}
|
||||
|
||||
const qmdBinary = await checkQmdBinaryAvailability({
|
||||
command: resolved.qmd.command,
|
||||
env: process.env,
|
||||
cwd: workspaceDir,
|
||||
});
|
||||
if (!qmdBinary.available) {
|
||||
log.warn(
|
||||
`qmd binary unavailable (${resolved.qmd.command}); falling back to builtin: ${qmdBinary.error ?? "unknown error"}`,
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
const { QmdMemoryManager } = await loadQmdManagerModule();
|
||||
const primary = await QmdMemoryManager.create({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
resolved,
|
||||
mode: statusOnly ? "status" : "full",
|
||||
});
|
||||
if (primary) {
|
||||
if (statusOnly) {
|
||||
return { manager: primary };
|
||||
}
|
||||
const wrapper = new FallbackMemoryManager(
|
||||
{
|
||||
primary,
|
||||
fallbackFactory: async () => {
|
||||
const { MemoryIndexManager } = await loadManagerRuntime();
|
||||
return await MemoryIndexManager.get(params);
|
||||
},
|
||||
},
|
||||
() => {
|
||||
QMD_MANAGER_CACHE.delete(cacheKey);
|
||||
},
|
||||
);
|
||||
QMD_MANAGER_CACHE.set(cacheKey, wrapper);
|
||||
return { manager: wrapper };
|
||||
}
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
log.warn(`qmd memory unavailable; falling back to builtin: ${message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return await getBuiltinMemorySearchManager(params);
|
||||
}
|
||||
|
||||
async function getBuiltinMemorySearchManager(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
purpose?: "default" | "status";
|
||||
}): Promise<MemorySearchManagerResult> {
|
||||
try {
|
||||
const { MemoryIndexManager } = await loadManagerRuntime();
|
||||
const manager = await MemoryIndexManager.get(params);
|
||||
return { manager };
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
return { manager: null, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
class BorrowedMemoryManager implements MemorySearchManager {
|
||||
constructor(private readonly inner: MemorySearchManager) {}
|
||||
|
||||
async search(
|
||||
query: string,
|
||||
opts?: {
|
||||
maxResults?: number;
|
||||
minScore?: number;
|
||||
sessionKey?: string;
|
||||
qmdSearchModeOverride?: "query" | "search" | "vsearch";
|
||||
onDebug?: (debug: MemorySearchRuntimeDebug) => void;
|
||||
},
|
||||
) {
|
||||
return await this.inner.search(query, opts);
|
||||
}
|
||||
|
||||
async readFile(params: { relPath: string; from?: number; lines?: number }) {
|
||||
return await this.inner.readFile(params);
|
||||
}
|
||||
|
||||
status() {
|
||||
return this.inner.status();
|
||||
}
|
||||
|
||||
async sync(params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}) {
|
||||
await this.inner.sync?.(params);
|
||||
}
|
||||
|
||||
async probeEmbeddingAvailability(): Promise<MemoryEmbeddingProbeResult> {
|
||||
return await this.inner.probeEmbeddingAvailability();
|
||||
}
|
||||
|
||||
async probeVectorAvailability() {
|
||||
return await this.inner.probeVectorAvailability();
|
||||
}
|
||||
|
||||
async close() {}
|
||||
}
|
||||
|
||||
export async function closeAllMemorySearchManagers(): Promise<void> {
|
||||
const managers = Array.from(QMD_MANAGER_CACHE.values());
|
||||
QMD_MANAGER_CACHE.clear();
|
||||
for (const manager of managers) {
|
||||
try {
|
||||
await manager.close?.();
|
||||
} catch (err) {
|
||||
log.warn(`failed to close qmd memory manager: ${String(err)}`);
|
||||
}
|
||||
}
|
||||
if (managerRuntimePromise !== null) {
|
||||
const { closeAllMemoryIndexManagers } = await loadManagerRuntime();
|
||||
await closeAllMemoryIndexManagers();
|
||||
}
|
||||
}
|
||||
|
||||
class FallbackMemoryManager implements MemorySearchManager {
|
||||
private fallback: MemorySearchManager | null = null;
|
||||
private primaryFailed = false;
|
||||
private lastError?: string;
|
||||
private cacheEvicted = false;
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
primary: MemorySearchManager;
|
||||
fallbackFactory: () => Promise<MemorySearchManager | null>;
|
||||
},
|
||||
private readonly onClose?: () => void,
|
||||
) {}
|
||||
|
||||
async search(
|
||||
query: string,
|
||||
opts?: {
|
||||
maxResults?: number;
|
||||
minScore?: number;
|
||||
sessionKey?: string;
|
||||
qmdSearchModeOverride?: "query" | "search" | "vsearch";
|
||||
onDebug?: (debug: MemorySearchRuntimeDebug) => void;
|
||||
},
|
||||
) {
|
||||
if (!this.primaryFailed) {
|
||||
try {
|
||||
return await this.deps.primary.search(query, opts);
|
||||
} catch (err) {
|
||||
this.primaryFailed = true;
|
||||
this.lastError = formatErrorMessage(err);
|
||||
log.warn(`qmd memory failed; switching to builtin index: ${this.lastError}`);
|
||||
await this.deps.primary.close?.().catch(() => {});
|
||||
// Evict the failed wrapper so the next request can retry QMD with a fresh manager.
|
||||
this.evictCacheEntry();
|
||||
}
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
if (fallback) {
|
||||
return await fallback.search(query, opts);
|
||||
}
|
||||
throw new Error(this.lastError ?? "memory search unavailable");
|
||||
}
|
||||
|
||||
async readFile(params: { relPath: string; from?: number; lines?: number }) {
|
||||
if (!this.primaryFailed) {
|
||||
return await this.deps.primary.readFile(params);
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
if (fallback) {
|
||||
return await fallback.readFile(params);
|
||||
}
|
||||
throw new Error(this.lastError ?? "memory read unavailable");
|
||||
}
|
||||
|
||||
status() {
|
||||
if (!this.primaryFailed) {
|
||||
return this.deps.primary.status();
|
||||
}
|
||||
const fallbackStatus = this.fallback?.status();
|
||||
const fallbackInfo = { from: "qmd", reason: this.lastError ?? "unknown" };
|
||||
if (fallbackStatus) {
|
||||
const custom = fallbackStatus.custom ?? {};
|
||||
return {
|
||||
...fallbackStatus,
|
||||
fallback: fallbackInfo,
|
||||
custom: {
|
||||
...custom,
|
||||
fallback: { disabled: true, reason: this.lastError ?? "unknown" },
|
||||
},
|
||||
};
|
||||
}
|
||||
const primaryStatus = this.deps.primary.status();
|
||||
const custom = primaryStatus.custom ?? {};
|
||||
return {
|
||||
...primaryStatus,
|
||||
fallback: fallbackInfo,
|
||||
custom: {
|
||||
...custom,
|
||||
fallback: { disabled: true, reason: this.lastError ?? "unknown" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async sync(params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
sessionFiles?: string[];
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}) {
|
||||
if (!this.primaryFailed) {
|
||||
await this.deps.primary.sync?.(params);
|
||||
return;
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
await fallback?.sync?.(params);
|
||||
}
|
||||
|
||||
async probeEmbeddingAvailability(): Promise<MemoryEmbeddingProbeResult> {
|
||||
if (!this.primaryFailed) {
|
||||
return await this.deps.primary.probeEmbeddingAvailability();
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
if (fallback) {
|
||||
return await fallback.probeEmbeddingAvailability();
|
||||
}
|
||||
return { ok: false, error: this.lastError ?? "memory embeddings unavailable" };
|
||||
}
|
||||
|
||||
async probeVectorAvailability() {
|
||||
if (!this.primaryFailed) {
|
||||
return await this.deps.primary.probeVectorAvailability();
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
return (await fallback?.probeVectorAvailability()) ?? false;
|
||||
}
|
||||
|
||||
async close() {
|
||||
await this.deps.primary.close?.();
|
||||
await this.fallback?.close?.();
|
||||
this.evictCacheEntry();
|
||||
}
|
||||
|
||||
private async ensureFallback(): Promise<MemorySearchManager | null> {
|
||||
if (this.fallback) {
|
||||
return this.fallback;
|
||||
}
|
||||
let fallback: MemorySearchManager | null;
|
||||
try {
|
||||
fallback = await this.deps.fallbackFactory();
|
||||
if (!fallback) {
|
||||
log.warn("memory fallback requested but builtin index is unavailable");
|
||||
return null;
|
||||
}
|
||||
} catch (err) {
|
||||
const message = formatErrorMessage(err);
|
||||
log.warn(`memory fallback unavailable: ${message}`);
|
||||
return null;
|
||||
}
|
||||
this.fallback = fallback;
|
||||
return this.fallback;
|
||||
}
|
||||
|
||||
private evictCacheEntry(): void {
|
||||
if (this.cacheEvicted) {
|
||||
return;
|
||||
}
|
||||
this.cacheEvicted = true;
|
||||
this.onClose?.();
|
||||
}
|
||||
}
|
||||
|
||||
function buildQmdCacheKey(agentId: string, config: ResolvedQmdConfig): string {
|
||||
// ResolvedQmdConfig is assembled in a stable field order in resolveMemoryBackendConfig.
|
||||
// Fast stringify avoids deep key-sorting overhead on this hot path.
|
||||
return `${agentId}:${JSON.stringify(config)}`;
|
||||
}
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createMemoryCoreTestHarness } from "../test-helpers.js";
|
||||
import { mergeHybridResults } from "./hybrid.js";
|
||||
import {
|
||||
applyTemporalDecayToHybridResults,
|
||||
applyTemporalDecayToScore,
|
||||
calculateTemporalDecayMultiplier,
|
||||
} from "./temporal-decay.js";
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const NOW_MS = Date.UTC(2026, 1, 10, 0, 0, 0);
|
||||
const { createTempWorkspace } = createMemoryCoreTestHarness();
|
||||
|
||||
function createVectorMemoryEntry(params: {
|
||||
id: string;
|
||||
path: string;
|
||||
snippet: string;
|
||||
vectorScore: number;
|
||||
}) {
|
||||
return {
|
||||
id: params.id,
|
||||
path: params.path,
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
source: "memory" as const,
|
||||
snippet: params.snippet,
|
||||
vectorScore: params.vectorScore,
|
||||
};
|
||||
}
|
||||
|
||||
async function mergeVectorResultsWithTemporalDecay(
|
||||
vector: Parameters<typeof mergeHybridResults>[0]["vector"],
|
||||
) {
|
||||
return mergeHybridResults({
|
||||
vectorWeight: 1,
|
||||
textWeight: 0,
|
||||
temporalDecay: { enabled: true, halfLifeDays: 30 },
|
||||
mmr: { enabled: false },
|
||||
nowMs: NOW_MS,
|
||||
vector,
|
||||
keyword: [],
|
||||
});
|
||||
}
|
||||
|
||||
describe("temporal decay", () => {
|
||||
it("matches exponential decay formula", () => {
|
||||
const halfLifeDays = 30;
|
||||
const ageInDays = 10;
|
||||
const lambda = Math.LN2 / halfLifeDays;
|
||||
const expectedMultiplier = Math.exp(-lambda * ageInDays);
|
||||
|
||||
expect(calculateTemporalDecayMultiplier({ ageInDays, halfLifeDays })).toBeCloseTo(
|
||||
expectedMultiplier,
|
||||
);
|
||||
expect(applyTemporalDecayToScore({ score: 0.8, ageInDays, halfLifeDays })).toBeCloseTo(
|
||||
0.8 * expectedMultiplier,
|
||||
);
|
||||
});
|
||||
|
||||
it("is 0.5 exactly at half-life", () => {
|
||||
expect(calculateTemporalDecayMultiplier({ ageInDays: 30, halfLifeDays: 30 })).toBeCloseTo(0.5);
|
||||
});
|
||||
|
||||
it("does not decay evergreen memory files", async () => {
|
||||
const dir = await createTempWorkspace("openclaw-temporal-decay-");
|
||||
|
||||
const rootMemoryPath = path.join(dir, "MEMORY.md");
|
||||
const topicPath = path.join(dir, "memory", "projects.md");
|
||||
await fs.mkdir(path.dirname(topicPath), { recursive: true });
|
||||
await fs.writeFile(rootMemoryPath, "evergreen");
|
||||
await fs.writeFile(topicPath, "topic evergreen");
|
||||
|
||||
const veryOld = new Date(Date.UTC(2010, 0, 1));
|
||||
await fs.utimes(rootMemoryPath, veryOld, veryOld);
|
||||
await fs.utimes(topicPath, veryOld, veryOld);
|
||||
|
||||
const decayed = await applyTemporalDecayToHybridResults({
|
||||
results: [
|
||||
{ path: "MEMORY.md", score: 1, source: "memory" },
|
||||
{ path: "memory/projects.md", score: 0.75, source: "memory" },
|
||||
],
|
||||
workspaceDir: dir,
|
||||
temporalDecay: { enabled: true, halfLifeDays: 30 },
|
||||
nowMs: NOW_MS,
|
||||
});
|
||||
|
||||
expect(decayed[0]?.score).toBeCloseTo(1);
|
||||
expect(decayed[1]?.score).toBeCloseTo(0.75);
|
||||
});
|
||||
|
||||
it("applies decay in hybrid merging before ranking", async () => {
|
||||
const merged = await mergeVectorResultsWithTemporalDecay([
|
||||
createVectorMemoryEntry({
|
||||
id: "old",
|
||||
path: "memory/2025-01-01.md",
|
||||
snippet: "old but high",
|
||||
vectorScore: 0.95,
|
||||
}),
|
||||
createVectorMemoryEntry({
|
||||
id: "new",
|
||||
path: "memory/2026-02-10.md",
|
||||
snippet: "new and relevant",
|
||||
vectorScore: 0.8,
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(merged[0]?.path).toBe("memory/2026-02-10.md");
|
||||
expect(merged[0]?.score ?? 0).toBeGreaterThan(merged[1]?.score ?? 0);
|
||||
});
|
||||
|
||||
it("handles future dates, zero age, and very old memories", async () => {
|
||||
const merged = await mergeVectorResultsWithTemporalDecay([
|
||||
createVectorMemoryEntry({
|
||||
id: "future",
|
||||
path: "memory/2099-01-01.md",
|
||||
snippet: "future",
|
||||
vectorScore: 0.9,
|
||||
}),
|
||||
createVectorMemoryEntry({
|
||||
id: "today",
|
||||
path: "memory/2026-02-10.md",
|
||||
snippet: "today",
|
||||
vectorScore: 0.8,
|
||||
}),
|
||||
createVectorMemoryEntry({
|
||||
id: "very-old",
|
||||
path: "memory/2000-01-01.md",
|
||||
snippet: "ancient",
|
||||
vectorScore: 1,
|
||||
}),
|
||||
]);
|
||||
|
||||
const byPath = new Map(merged.map((entry) => [entry.path, entry]));
|
||||
expect(byPath.get("memory/2099-01-01.md")?.score).toBeCloseTo(0.9);
|
||||
expect(byPath.get("memory/2026-02-10.md")?.score).toBeCloseTo(0.8);
|
||||
expect(byPath.get("memory/2000-01-01.md")?.score ?? 1).toBeLessThan(0.001);
|
||||
});
|
||||
|
||||
it("uses file mtime fallback for non-memory sources", async () => {
|
||||
const dir = await createTempWorkspace("openclaw-temporal-decay-");
|
||||
const sessionPath = path.join(dir, "sessions", "thread.jsonl");
|
||||
await fs.mkdir(path.dirname(sessionPath), { recursive: true });
|
||||
await fs.writeFile(sessionPath, "{}\n");
|
||||
const oldMtime = new Date(NOW_MS - 30 * DAY_MS);
|
||||
await fs.utimes(sessionPath, oldMtime, oldMtime);
|
||||
|
||||
const decayed = await applyTemporalDecayToHybridResults({
|
||||
results: [{ path: "sessions/thread.jsonl", score: 1, source: "sessions" }],
|
||||
workspaceDir: dir,
|
||||
temporalDecay: { enabled: true, halfLifeDays: 30 },
|
||||
nowMs: NOW_MS,
|
||||
});
|
||||
|
||||
expect(decayed[0]?.score).toBeCloseTo(0.5, 2);
|
||||
});
|
||||
});
|
||||
167
openclaw/extensions/memory-core/src/memory/temporal-decay.ts
Normal file
167
openclaw/extensions/memory-core/src/memory/temporal-decay.ts
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export type TemporalDecayConfig = {
|
||||
enabled: boolean;
|
||||
halfLifeDays: number;
|
||||
};
|
||||
|
||||
export const DEFAULT_TEMPORAL_DECAY_CONFIG: TemporalDecayConfig = {
|
||||
enabled: false,
|
||||
halfLifeDays: 30,
|
||||
};
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const DATED_MEMORY_PATH_RE = /(?:^|\/)memory\/(\d{4})-(\d{2})-(\d{2})\.md$/;
|
||||
|
||||
export function toDecayLambda(halfLifeDays: number): number {
|
||||
if (!Number.isFinite(halfLifeDays) || halfLifeDays <= 0) {
|
||||
return 0;
|
||||
}
|
||||
return Math.LN2 / halfLifeDays;
|
||||
}
|
||||
|
||||
export function calculateTemporalDecayMultiplier(params: {
|
||||
ageInDays: number;
|
||||
halfLifeDays: number;
|
||||
}): number {
|
||||
const lambda = toDecayLambda(params.halfLifeDays);
|
||||
const clampedAge = Math.max(0, params.ageInDays);
|
||||
if (lambda <= 0 || !Number.isFinite(clampedAge)) {
|
||||
return 1;
|
||||
}
|
||||
return Math.exp(-lambda * clampedAge);
|
||||
}
|
||||
|
||||
export function applyTemporalDecayToScore(params: {
|
||||
score: number;
|
||||
ageInDays: number;
|
||||
halfLifeDays: number;
|
||||
}): number {
|
||||
return params.score * calculateTemporalDecayMultiplier(params);
|
||||
}
|
||||
|
||||
function parseMemoryDateFromPath(filePath: string): Date | null {
|
||||
const normalized = filePath.replaceAll("\\", "/").replace(/^\.\//, "");
|
||||
const match = DATED_MEMORY_PATH_RE.exec(normalized);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const year = Number(match[1]);
|
||||
const month = Number(match[2]);
|
||||
const day = Number(match[3]);
|
||||
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const timestamp = Date.UTC(year, month - 1, day);
|
||||
const parsed = new Date(timestamp);
|
||||
if (
|
||||
parsed.getUTCFullYear() !== year ||
|
||||
parsed.getUTCMonth() !== month - 1 ||
|
||||
parsed.getUTCDate() !== day
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function isEvergreenMemoryPath(filePath: string): boolean {
|
||||
const normalized = filePath.replaceAll("\\", "/").replace(/^\.\//, "");
|
||||
if (normalized === "MEMORY.md" || normalized === "memory.md") {
|
||||
return true;
|
||||
}
|
||||
if (!normalized.startsWith("memory/")) {
|
||||
return false;
|
||||
}
|
||||
return !DATED_MEMORY_PATH_RE.test(normalized);
|
||||
}
|
||||
|
||||
async function extractTimestamp(params: {
|
||||
filePath: string;
|
||||
source?: string;
|
||||
workspaceDir?: string;
|
||||
}): Promise<Date | null> {
|
||||
const fromPath = parseMemoryDateFromPath(params.filePath);
|
||||
if (fromPath) {
|
||||
return fromPath;
|
||||
}
|
||||
|
||||
// Memory root/topic files are evergreen knowledge and should not decay.
|
||||
if (params.source === "memory" && isEvergreenMemoryPath(params.filePath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!params.workspaceDir) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const absolutePath = path.isAbsolute(params.filePath)
|
||||
? params.filePath
|
||||
: path.resolve(params.workspaceDir, params.filePath);
|
||||
|
||||
try {
|
||||
const stat = await fs.stat(absolutePath);
|
||||
if (!Number.isFinite(stat.mtimeMs)) {
|
||||
return null;
|
||||
}
|
||||
return new Date(stat.mtimeMs);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function ageInDaysFromTimestamp(timestamp: Date, nowMs: number): number {
|
||||
const ageMs = Math.max(0, nowMs - timestamp.getTime());
|
||||
return ageMs / DAY_MS;
|
||||
}
|
||||
|
||||
export async function applyTemporalDecayToHybridResults<
|
||||
T extends { path: string; score: number; source: string },
|
||||
>(params: {
|
||||
results: T[];
|
||||
temporalDecay?: Partial<TemporalDecayConfig>;
|
||||
workspaceDir?: string;
|
||||
nowMs?: number;
|
||||
}): Promise<T[]> {
|
||||
const config = { ...DEFAULT_TEMPORAL_DECAY_CONFIG, ...params.temporalDecay };
|
||||
if (!config.enabled) {
|
||||
return [...params.results];
|
||||
}
|
||||
|
||||
const nowMs = params.nowMs ?? Date.now();
|
||||
const timestampPromiseCache = new Map<string, Promise<Date | null>>();
|
||||
|
||||
return Promise.all(
|
||||
params.results.map(async (entry) => {
|
||||
const cacheKey = `${entry.source}:${entry.path}`;
|
||||
let timestampPromise = timestampPromiseCache.get(cacheKey);
|
||||
if (!timestampPromise) {
|
||||
timestampPromise = extractTimestamp({
|
||||
filePath: entry.path,
|
||||
source: entry.source,
|
||||
workspaceDir: params.workspaceDir,
|
||||
});
|
||||
timestampPromiseCache.set(cacheKey, timestampPromise);
|
||||
}
|
||||
|
||||
const timestamp = await timestampPromise;
|
||||
if (!timestamp) {
|
||||
return entry;
|
||||
}
|
||||
|
||||
const decayedScore = applyTemporalDecayToScore({
|
||||
score: entry.score,
|
||||
ageInDays: ageInDaysFromTimestamp(timestamp, nowMs),
|
||||
halfLifeDays: config.halfLifeDays,
|
||||
});
|
||||
|
||||
return {
|
||||
...entry,
|
||||
score: decayedScore,
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
import * as ssrf from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { vi } from "vitest";
|
||||
|
||||
export function mockPublicPinnedHostname() {
|
||||
return vi.spyOn(ssrf, "resolvePinnedHostnameWithPolicy").mockImplementation(async (hostname) => {
|
||||
const normalized = normalizeLowercaseStringOrEmpty(hostname).replace(/\.$/, "");
|
||||
const addresses = ["93.184.216.34"];
|
||||
const lookup = ((host: string, options?: unknown, callback?: unknown) => {
|
||||
const cb =
|
||||
typeof options === "function"
|
||||
? (options as (err: NodeJS.ErrnoException | null, address: unknown) => void)
|
||||
: (callback as (err: NodeJS.ErrnoException | null, address: unknown) => void);
|
||||
if (!cb) {
|
||||
return;
|
||||
}
|
||||
if (normalizeLowercaseStringOrEmpty(host).replace(/\.$/, "") !== normalized) {
|
||||
cb(null, []);
|
||||
return;
|
||||
}
|
||||
cb(
|
||||
null,
|
||||
addresses.map((address) => ({
|
||||
address,
|
||||
family: address.includes(":") ? 6 : 4,
|
||||
})),
|
||||
);
|
||||
}) as never;
|
||||
return {
|
||||
hostname: normalized,
|
||||
addresses,
|
||||
lookup,
|
||||
};
|
||||
});
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue