mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-10 09:10:46 +08:00
重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
165
openclaw/extensions/memory-wiki/src/apply.test.ts
Normal file
165
openclaw/extensions/memory-wiki/src/apply.test.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyMemoryWikiMutation } from "./apply.js";
|
||||
import { parseWikiMarkdown, renderWikiMarkdown } from "./markdown.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("applyMemoryWikiMutation", () => {
|
||||
it("creates synthesis pages with managed summary blocks and refreshed indexes", async () => {
|
||||
const { rootDir, config } = await createVault({ prefix: "memory-wiki-apply-" });
|
||||
|
||||
const result = await applyMemoryWikiMutation({
|
||||
config,
|
||||
mutation: {
|
||||
op: "create_synthesis",
|
||||
title: "Alpha Synthesis",
|
||||
body: "Alpha summary body.",
|
||||
sourceIds: ["source.alpha", "source.beta"],
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.postgres",
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
status: "supported",
|
||||
confidence: 0.86,
|
||||
evidence: [
|
||||
{
|
||||
sourceId: "source.alpha",
|
||||
lines: "12-18",
|
||||
weight: 0.9,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
contradictions: ["Needs a better primary source"],
|
||||
questions: ["What changed after launch?"],
|
||||
confidence: 0.7,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.changed).toBe(true);
|
||||
expect(result.pagePath).toBe("syntheses/alpha-synthesis.md");
|
||||
expect(result.pageId).toBe("synthesis.alpha-synthesis");
|
||||
expect(result.compile.pageCounts.synthesis).toBe(1);
|
||||
|
||||
const page = await fs.readFile(path.join(rootDir, result.pagePath), "utf8");
|
||||
const parsed = parseWikiMarkdown(page);
|
||||
|
||||
expect(parsed.frontmatter).toMatchObject({
|
||||
pageType: "synthesis",
|
||||
id: "synthesis.alpha-synthesis",
|
||||
title: "Alpha Synthesis",
|
||||
sourceIds: ["source.alpha", "source.beta"],
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.postgres",
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
status: "supported",
|
||||
confidence: 0.86,
|
||||
evidence: [
|
||||
{
|
||||
sourceId: "source.alpha",
|
||||
lines: "12-18",
|
||||
weight: 0.9,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
contradictions: ["Needs a better primary source"],
|
||||
questions: ["What changed after launch?"],
|
||||
confidence: 0.7,
|
||||
status: "active",
|
||||
});
|
||||
expect(parsed.body).toContain("## Summary");
|
||||
expect(parsed.body).toContain("<!-- openclaw:wiki:generated:start -->");
|
||||
expect(parsed.body).toContain("Alpha summary body.");
|
||||
expect(parsed.body).toContain("## Notes");
|
||||
expect(parsed.body).toContain("<!-- openclaw:human:start -->");
|
||||
await expect(fs.readFile(path.join(rootDir, "index.md"), "utf8")).resolves.toContain(
|
||||
"[Alpha Synthesis](syntheses/alpha-synthesis.md)",
|
||||
);
|
||||
});
|
||||
|
||||
it("updates page metadata without overwriting existing human notes", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-apply-",
|
||||
});
|
||||
|
||||
const targetPath = path.join(rootDir, "entities", "alpha.md");
|
||||
await fs.mkdir(path.dirname(targetPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
targetPath,
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha",
|
||||
title: "Alpha",
|
||||
sourceIds: ["source.old"],
|
||||
confidence: 0.3,
|
||||
},
|
||||
body: `# Alpha
|
||||
|
||||
## Notes
|
||||
<!-- openclaw:human:start -->
|
||||
keep this note
|
||||
<!-- openclaw:human:end -->
|
||||
`,
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await applyMemoryWikiMutation({
|
||||
config,
|
||||
mutation: {
|
||||
op: "update_metadata",
|
||||
lookup: "entity.alpha",
|
||||
sourceIds: ["source.new"],
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.status",
|
||||
text: "Alpha is still active for existing tenants.",
|
||||
status: "contested",
|
||||
evidence: [{ sourceId: "source.new", lines: "4-9" }],
|
||||
},
|
||||
],
|
||||
contradictions: ["Conflicts with source.beta"],
|
||||
questions: ["Is Alpha still active?"],
|
||||
confidence: null,
|
||||
status: "review",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.changed).toBe(true);
|
||||
expect(result.pagePath).toBe("entities/alpha.md");
|
||||
expect(result.compile.pageCounts.entity).toBe(1);
|
||||
|
||||
const updated = await fs.readFile(targetPath, "utf8");
|
||||
const parsed = parseWikiMarkdown(updated);
|
||||
|
||||
expect(parsed.frontmatter).toMatchObject({
|
||||
pageType: "entity",
|
||||
id: "entity.alpha",
|
||||
title: "Alpha",
|
||||
sourceIds: ["source.new"],
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.status",
|
||||
text: "Alpha is still active for existing tenants.",
|
||||
status: "contested",
|
||||
evidence: [{ sourceId: "source.new", lines: "4-9" }],
|
||||
},
|
||||
],
|
||||
contradictions: ["Conflicts with source.beta"],
|
||||
questions: ["Is Alpha still active?"],
|
||||
status: "review",
|
||||
});
|
||||
expect(parsed.frontmatter).not.toHaveProperty("confidence");
|
||||
expect(parsed.body).toContain("keep this note");
|
||||
expect(parsed.body).toContain("<!-- openclaw:human:start -->");
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "entities", "index.md"), "utf8"),
|
||||
).resolves.toContain("[Alpha](entities/alpha.md)");
|
||||
});
|
||||
});
|
||||
318
openclaw/extensions/memory-wiki/src/apply.ts
Normal file
318
openclaw/extensions/memory-wiki/src/apply.ts
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
replaceManagedMarkdownBlock,
|
||||
withTrailingNewline,
|
||||
} from "openclaw/plugin-sdk/memory-host-markdown";
|
||||
import { compileMemoryWikiVault, type CompileMemoryWikiResult } from "./compile.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import {
|
||||
parseWikiMarkdown,
|
||||
renderWikiMarkdown,
|
||||
slugifyWikiSegment,
|
||||
normalizeSourceIds,
|
||||
normalizeWikiClaims,
|
||||
type WikiClaim,
|
||||
} from "./markdown.js";
|
||||
import {
|
||||
readQueryableWikiPages,
|
||||
resolveQueryableWikiPageByLookup,
|
||||
type QueryableWikiPage,
|
||||
} from "./query.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
const GENERATED_START = "<!-- openclaw:wiki:generated:start -->";
|
||||
const GENERATED_END = "<!-- openclaw:wiki:generated:end -->";
|
||||
const HUMAN_START = "<!-- openclaw:human:start -->";
|
||||
const HUMAN_END = "<!-- openclaw:human:end -->";
|
||||
|
||||
export type CreateSynthesisMemoryWikiMutation = {
|
||||
op: "create_synthesis";
|
||||
title: string;
|
||||
body: string;
|
||||
sourceIds: string[];
|
||||
claims?: WikiClaim[];
|
||||
contradictions?: string[];
|
||||
questions?: string[];
|
||||
confidence?: number;
|
||||
status?: string;
|
||||
};
|
||||
|
||||
export type UpdateMetadataMemoryWikiMutation = {
|
||||
op: "update_metadata";
|
||||
lookup: string;
|
||||
sourceIds?: string[];
|
||||
claims?: WikiClaim[];
|
||||
contradictions?: string[];
|
||||
questions?: string[];
|
||||
confidence?: number | null;
|
||||
status?: string;
|
||||
};
|
||||
|
||||
export type ApplyMemoryWikiMutation =
|
||||
| CreateSynthesisMemoryWikiMutation
|
||||
| UpdateMetadataMemoryWikiMutation;
|
||||
|
||||
export type ApplyMemoryWikiMutationResult = {
|
||||
changed: boolean;
|
||||
operation: ApplyMemoryWikiMutation["op"];
|
||||
pagePath: string;
|
||||
pageId?: string;
|
||||
compile: CompileMemoryWikiResult;
|
||||
};
|
||||
|
||||
export function normalizeMemoryWikiMutationInput(rawParams: unknown): ApplyMemoryWikiMutation {
|
||||
const params = rawParams as {
|
||||
op: ApplyMemoryWikiMutation["op"];
|
||||
title?: string;
|
||||
body?: string;
|
||||
lookup?: string;
|
||||
sourceIds?: string[];
|
||||
claims?: WikiClaim[];
|
||||
contradictions?: string[];
|
||||
questions?: string[];
|
||||
confidence?: number | null;
|
||||
status?: string;
|
||||
};
|
||||
if (params.op === "create_synthesis") {
|
||||
if (!params.title?.trim()) {
|
||||
throw new Error("wiki mutation requires title for create_synthesis.");
|
||||
}
|
||||
if (!params.body?.trim()) {
|
||||
throw new Error("wiki mutation requires body for create_synthesis.");
|
||||
}
|
||||
if (!params.sourceIds || params.sourceIds.length === 0) {
|
||||
throw new Error("wiki mutation requires at least one sourceId for create_synthesis.");
|
||||
}
|
||||
return {
|
||||
op: "create_synthesis",
|
||||
title: params.title,
|
||||
body: params.body,
|
||||
sourceIds: params.sourceIds,
|
||||
...(Array.isArray(params.claims) ? { claims: normalizeWikiClaims(params.claims) } : {}),
|
||||
...(params.contradictions ? { contradictions: params.contradictions } : {}),
|
||||
...(params.questions ? { questions: params.questions } : {}),
|
||||
...(typeof params.confidence === "number" ? { confidence: params.confidence } : {}),
|
||||
...(params.status ? { status: params.status } : {}),
|
||||
};
|
||||
}
|
||||
if (!params.lookup?.trim()) {
|
||||
throw new Error("wiki mutation requires lookup for update_metadata.");
|
||||
}
|
||||
return {
|
||||
op: "update_metadata",
|
||||
lookup: params.lookup,
|
||||
...(params.sourceIds ? { sourceIds: params.sourceIds } : {}),
|
||||
...(Array.isArray(params.claims) ? { claims: normalizeWikiClaims(params.claims) } : {}),
|
||||
...(params.contradictions ? { contradictions: params.contradictions } : {}),
|
||||
...(params.questions ? { questions: params.questions } : {}),
|
||||
...(params.confidence !== undefined ? { confidence: params.confidence } : {}),
|
||||
...(params.status ? { status: params.status } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeUniqueStrings(values: string[] | undefined): string[] | undefined {
|
||||
if (!values) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = values
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
.filter((value, index, all) => all.indexOf(value) === index);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function ensureHumanNotesBlock(body: string): string {
|
||||
if (body.includes(HUMAN_START) && body.includes(HUMAN_END)) {
|
||||
return body;
|
||||
}
|
||||
const trimmed = body.trimEnd();
|
||||
const prefix = trimmed.length > 0 ? `${trimmed}\n\n` : "";
|
||||
return `${prefix}## Notes\n${HUMAN_START}\n${HUMAN_END}\n`;
|
||||
}
|
||||
|
||||
function buildSynthesisBody(params: {
|
||||
title: string;
|
||||
originalBody?: string;
|
||||
generatedBody: string;
|
||||
}): string {
|
||||
const base = params.originalBody?.trim().length
|
||||
? params.originalBody
|
||||
: `# ${params.title}\n\n## Notes\n${HUMAN_START}\n${HUMAN_END}\n`;
|
||||
const withGenerated = replaceManagedMarkdownBlock({
|
||||
original: base,
|
||||
heading: "## Summary",
|
||||
startMarker: GENERATED_START,
|
||||
endMarker: GENERATED_END,
|
||||
body: params.generatedBody,
|
||||
});
|
||||
return ensureHumanNotesBlock(withGenerated);
|
||||
}
|
||||
|
||||
async function writeWikiPage(params: {
|
||||
absolutePath: string;
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
}): Promise<boolean> {
|
||||
const rendered = withTrailingNewline(
|
||||
renderWikiMarkdown({
|
||||
frontmatter: params.frontmatter,
|
||||
body: params.body,
|
||||
}),
|
||||
);
|
||||
const existing = await fs.readFile(params.absolutePath, "utf8").catch(() => "");
|
||||
if (existing === rendered) {
|
||||
return false;
|
||||
}
|
||||
await fs.mkdir(path.dirname(params.absolutePath), { recursive: true });
|
||||
await fs.writeFile(params.absolutePath, rendered, "utf8");
|
||||
return true;
|
||||
}
|
||||
|
||||
async function resolveWritablePage(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
lookup: string;
|
||||
}): Promise<QueryableWikiPage | null> {
|
||||
const pages = await readQueryableWikiPages(params.config.vault.path);
|
||||
return resolveQueryableWikiPageByLookup(pages, params.lookup);
|
||||
}
|
||||
|
||||
async function applyCreateSynthesisMutation(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
mutation: CreateSynthesisMemoryWikiMutation;
|
||||
}): Promise<{ changed: boolean; pagePath: string; pageId: string }> {
|
||||
const slug = slugifyWikiSegment(params.mutation.title);
|
||||
const pagePath = path.join("syntheses", `${slug}.md`).replace(/\\/g, "/");
|
||||
const absolutePath = path.join(params.config.vault.path, pagePath);
|
||||
const existing = await fs.readFile(absolutePath, "utf8").catch(() => "");
|
||||
const parsed = parseWikiMarkdown(existing);
|
||||
const pageId =
|
||||
(typeof parsed.frontmatter.id === "string" && parsed.frontmatter.id.trim()) ||
|
||||
`synthesis.${slug}`;
|
||||
const changed = await writeWikiPage({
|
||||
absolutePath,
|
||||
frontmatter: {
|
||||
...parsed.frontmatter,
|
||||
pageType: "synthesis",
|
||||
id: pageId,
|
||||
title: params.mutation.title,
|
||||
sourceIds: normalizeSourceIds(params.mutation.sourceIds),
|
||||
...(params.mutation.claims ? { claims: normalizeWikiClaims(params.mutation.claims) } : {}),
|
||||
...(normalizeUniqueStrings(params.mutation.contradictions)
|
||||
? { contradictions: normalizeUniqueStrings(params.mutation.contradictions) }
|
||||
: {}),
|
||||
...(normalizeUniqueStrings(params.mutation.questions)
|
||||
? { questions: normalizeUniqueStrings(params.mutation.questions) }
|
||||
: {}),
|
||||
...(typeof params.mutation.confidence === "number"
|
||||
? { confidence: params.mutation.confidence }
|
||||
: {}),
|
||||
status: params.mutation.status?.trim() || "active",
|
||||
updatedAt: new Date().toISOString(),
|
||||
},
|
||||
body: buildSynthesisBody({
|
||||
title: params.mutation.title,
|
||||
originalBody: parsed.body,
|
||||
generatedBody: params.mutation.body.trim(),
|
||||
}),
|
||||
});
|
||||
return { changed, pagePath, pageId };
|
||||
}
|
||||
|
||||
function buildUpdatedFrontmatter(params: {
|
||||
original: Record<string, unknown>;
|
||||
mutation: UpdateMetadataMemoryWikiMutation;
|
||||
}): Record<string, unknown> {
|
||||
const frontmatter: Record<string, unknown> = {
|
||||
...params.original,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
if (params.mutation.sourceIds) {
|
||||
frontmatter.sourceIds = normalizeSourceIds(params.mutation.sourceIds);
|
||||
}
|
||||
if (params.mutation.claims) {
|
||||
const claims = normalizeWikiClaims(params.mutation.claims);
|
||||
if (claims.length > 0) {
|
||||
frontmatter.claims = claims;
|
||||
} else {
|
||||
delete frontmatter.claims;
|
||||
}
|
||||
}
|
||||
if (params.mutation.contradictions) {
|
||||
const contradictions = normalizeUniqueStrings(params.mutation.contradictions) ?? [];
|
||||
if (contradictions.length > 0) {
|
||||
frontmatter.contradictions = contradictions;
|
||||
} else {
|
||||
delete frontmatter.contradictions;
|
||||
}
|
||||
}
|
||||
if (params.mutation.questions) {
|
||||
const questions = normalizeUniqueStrings(params.mutation.questions) ?? [];
|
||||
if (questions.length > 0) {
|
||||
frontmatter.questions = questions;
|
||||
} else {
|
||||
delete frontmatter.questions;
|
||||
}
|
||||
}
|
||||
if (params.mutation.confidence === null) {
|
||||
delete frontmatter.confidence;
|
||||
} else if (typeof params.mutation.confidence === "number") {
|
||||
frontmatter.confidence = params.mutation.confidence;
|
||||
}
|
||||
if (params.mutation.status?.trim()) {
|
||||
frontmatter.status = params.mutation.status.trim();
|
||||
}
|
||||
return frontmatter;
|
||||
}
|
||||
|
||||
async function applyUpdateMetadataMutation(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
mutation: UpdateMetadataMemoryWikiMutation;
|
||||
}): Promise<{ changed: boolean; pagePath: string; pageId?: string }> {
|
||||
const page = await resolveWritablePage({
|
||||
config: params.config,
|
||||
lookup: params.mutation.lookup,
|
||||
});
|
||||
if (!page) {
|
||||
throw new Error(`Wiki page not found: ${params.mutation.lookup}`);
|
||||
}
|
||||
const parsed = parseWikiMarkdown(page.raw);
|
||||
const changed = await writeWikiPage({
|
||||
absolutePath: page.absolutePath,
|
||||
frontmatter: buildUpdatedFrontmatter({
|
||||
original: parsed.frontmatter,
|
||||
mutation: params.mutation,
|
||||
}),
|
||||
body: parsed.body,
|
||||
});
|
||||
return {
|
||||
changed,
|
||||
pagePath: page.relativePath,
|
||||
...(page.id ? { pageId: page.id } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function applyMemoryWikiMutation(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
mutation: ApplyMemoryWikiMutation;
|
||||
}): Promise<ApplyMemoryWikiMutationResult> {
|
||||
await initializeMemoryWikiVault(params.config);
|
||||
const result =
|
||||
params.mutation.op === "create_synthesis"
|
||||
? await applyCreateSynthesisMutation({
|
||||
config: params.config,
|
||||
mutation: params.mutation,
|
||||
})
|
||||
: await applyUpdateMetadataMutation({
|
||||
config: params.config,
|
||||
mutation: params.mutation,
|
||||
});
|
||||
const compile = await compileMemoryWikiVault(params.config);
|
||||
return {
|
||||
changed: result.changed,
|
||||
operation: params.mutation.op,
|
||||
pagePath: result.pagePath,
|
||||
...(result.pageId ? { pageId: result.pageId } : {}),
|
||||
compile,
|
||||
};
|
||||
}
|
||||
345
openclaw/extensions/memory-wiki/src/bridge.test.ts
Normal file
345
openclaw/extensions/memory-wiki/src/bridge.test.ts
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { MemoryPluginPublicArtifact } from "openclaw/plugin-sdk/memory-host-core";
|
||||
import {
|
||||
appendMemoryHostEvent,
|
||||
resolveMemoryHostEventLogPath,
|
||||
} from "openclaw/plugin-sdk/memory-host-events";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
clearMemoryPluginState,
|
||||
registerMemoryCapability,
|
||||
} from "../../../src/plugins/memory-state.js";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
import { syncMemoryWikiBridgeSources } from "./bridge.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("syncMemoryWikiBridgeSources", () => {
|
||||
let fixtureRoot = "";
|
||||
let caseId = 0;
|
||||
|
||||
beforeAll(async () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-bridge-suite-"));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!fixtureRoot) {
|
||||
return;
|
||||
}
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clearMemoryPluginState();
|
||||
});
|
||||
|
||||
function nextCaseRoot(name: string): string {
|
||||
return path.join(fixtureRoot, `case-${caseId++}-${name}`);
|
||||
}
|
||||
|
||||
async function createBridgeWorkspace(name: string): Promise<string> {
|
||||
const workspaceDir = nextCaseRoot(name);
|
||||
await fs.mkdir(workspaceDir, { recursive: true });
|
||||
return workspaceDir;
|
||||
}
|
||||
|
||||
function registerBridgeArtifacts(artifacts: MemoryPluginPublicArtifact[]) {
|
||||
registerMemoryCapability("memory-core", {
|
||||
publicArtifacts: {
|
||||
async listArtifacts() {
|
||||
return artifacts;
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
it("imports public memory artifacts and stays idempotent across reruns", async () => {
|
||||
const workspaceDir = await createBridgeWorkspace("workspace");
|
||||
const { rootDir: vaultDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot("vault"),
|
||||
config: {
|
||||
vaultMode: "bridge",
|
||||
bridge: {
|
||||
enabled: true,
|
||||
readMemoryArtifacts: true,
|
||||
indexMemoryRoot: true,
|
||||
indexDailyNotes: true,
|
||||
indexDreamReports: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await fs.mkdir(path.join(workspaceDir, "memory", "dreaming"), { recursive: true });
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", "2026-04-05.md"),
|
||||
"# Daily Note\n",
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(workspaceDir, "memory", "dreaming", "2026-04-05.md"),
|
||||
"# Dream Report\n",
|
||||
"utf8",
|
||||
);
|
||||
registerBridgeArtifacts([
|
||||
{
|
||||
kind: "memory-root",
|
||||
workspaceDir,
|
||||
relativePath: "MEMORY.md",
|
||||
absolutePath: path.join(workspaceDir, "MEMORY.md"),
|
||||
agentIds: ["main"],
|
||||
contentType: "markdown",
|
||||
},
|
||||
{
|
||||
kind: "daily-note",
|
||||
workspaceDir,
|
||||
relativePath: "memory/2026-04-05.md",
|
||||
absolutePath: path.join(workspaceDir, "memory", "2026-04-05.md"),
|
||||
agentIds: ["main"],
|
||||
contentType: "markdown",
|
||||
},
|
||||
{
|
||||
kind: "dream-report",
|
||||
workspaceDir,
|
||||
relativePath: "memory/dreaming/2026-04-05.md",
|
||||
absolutePath: path.join(workspaceDir, "memory", "dreaming", "2026-04-05.md"),
|
||||
agentIds: ["main"],
|
||||
contentType: "markdown",
|
||||
},
|
||||
]);
|
||||
|
||||
const appConfig: OpenClawConfig = {
|
||||
agents: {
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir }],
|
||||
},
|
||||
};
|
||||
|
||||
const first = await syncMemoryWikiBridgeSources({ config, appConfig });
|
||||
|
||||
expect(first.workspaces).toBe(1);
|
||||
expect(first.artifactCount).toBe(3);
|
||||
expect(first.importedCount).toBe(3);
|
||||
expect(first.updatedCount).toBe(0);
|
||||
expect(first.skippedCount).toBe(0);
|
||||
expect(first.removedCount).toBe(0);
|
||||
expect(first.pagePaths).toHaveLength(3);
|
||||
|
||||
const sourcePages = await fs.readdir(path.join(vaultDir, "sources"));
|
||||
expect(sourcePages.filter((name) => name.startsWith("bridge-"))).toHaveLength(3);
|
||||
|
||||
const memoryPage = await fs.readFile(path.join(vaultDir, first.pagePaths[0] ?? ""), "utf8");
|
||||
expect(memoryPage).toContain("sourceType: memory-bridge");
|
||||
expect(memoryPage).toContain("## Bridge Source");
|
||||
|
||||
const second = await syncMemoryWikiBridgeSources({ config, appConfig });
|
||||
|
||||
expect(second.importedCount).toBe(0);
|
||||
expect(second.updatedCount).toBe(0);
|
||||
expect(second.skippedCount).toBe(3);
|
||||
expect(second.removedCount).toBe(0);
|
||||
|
||||
const logLines = (await fs.readFile(path.join(vaultDir, ".openclaw-wiki", "log.jsonl"), "utf8"))
|
||||
.trim()
|
||||
.split("\n");
|
||||
expect(logLines).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("returns a no-op result outside bridge mode", async () => {
|
||||
const { config } = await createVault({ rootDir: nextCaseRoot("isolated") });
|
||||
|
||||
const result = await syncMemoryWikiBridgeSources({ config });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
importedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
removedCount: 0,
|
||||
artifactCount: 0,
|
||||
workspaces: 0,
|
||||
pagePaths: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a no-op result when bridge mode is enabled without exported memory artifacts", async () => {
|
||||
const workspaceDir = await createBridgeWorkspace("no-memory-core");
|
||||
const { config } = await createVault({
|
||||
rootDir: nextCaseRoot("no-memory-core-vault"),
|
||||
config: {
|
||||
vaultMode: "bridge",
|
||||
bridge: {
|
||||
enabled: true,
|
||||
readMemoryArtifacts: true,
|
||||
indexMemoryRoot: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8");
|
||||
|
||||
const appConfig: OpenClawConfig = {
|
||||
agents: {
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir }],
|
||||
},
|
||||
};
|
||||
|
||||
const result = await syncMemoryWikiBridgeSources({ config, appConfig });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
importedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
removedCount: 0,
|
||||
artifactCount: 0,
|
||||
workspaces: 0,
|
||||
pagePaths: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("imports the public memory event journal when followMemoryEvents is enabled", async () => {
|
||||
const workspaceDir = await createBridgeWorkspace("events-workspace");
|
||||
const { rootDir: vaultDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot("events-vault"),
|
||||
config: {
|
||||
vaultMode: "bridge",
|
||||
bridge: {
|
||||
enabled: true,
|
||||
followMemoryEvents: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await appendMemoryHostEvent(workspaceDir, {
|
||||
type: "memory.recall.recorded",
|
||||
timestamp: "2026-04-05T12:00:00.000Z",
|
||||
query: "bridge events",
|
||||
resultCount: 1,
|
||||
results: [
|
||||
{
|
||||
path: "memory/2026-04-05.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
score: 0.8,
|
||||
},
|
||||
],
|
||||
});
|
||||
registerBridgeArtifacts([
|
||||
{
|
||||
kind: "event-log",
|
||||
workspaceDir,
|
||||
relativePath: "memory/.dreams/events.jsonl",
|
||||
absolutePath: resolveMemoryHostEventLogPath(workspaceDir),
|
||||
agentIds: ["main"],
|
||||
contentType: "json",
|
||||
},
|
||||
]);
|
||||
|
||||
const appConfig: OpenClawConfig = {
|
||||
agents: {
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir }],
|
||||
},
|
||||
};
|
||||
|
||||
const result = await syncMemoryWikiBridgeSources({ config, appConfig });
|
||||
|
||||
expect(result.artifactCount).toBe(1);
|
||||
expect(result.importedCount).toBe(1);
|
||||
expect(result.removedCount).toBe(0);
|
||||
const page = await fs.readFile(path.join(vaultDir, result.pagePaths[0] ?? ""), "utf8");
|
||||
expect(page).toContain("sourceType: memory-bridge-events");
|
||||
expect(page).toContain('"type":"memory.recall.recorded"');
|
||||
});
|
||||
|
||||
it("prunes stale bridge pages when the source artifact disappears", async () => {
|
||||
const workspaceDir = await createBridgeWorkspace("prune-workspace");
|
||||
const { rootDir: vaultDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot("prune-vault"),
|
||||
config: {
|
||||
vaultMode: "bridge",
|
||||
bridge: {
|
||||
enabled: true,
|
||||
indexMemoryRoot: true,
|
||||
indexDailyNotes: false,
|
||||
indexDreamReports: false,
|
||||
followMemoryEvents: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8");
|
||||
registerBridgeArtifacts([
|
||||
{
|
||||
kind: "memory-root",
|
||||
workspaceDir,
|
||||
relativePath: "MEMORY.md",
|
||||
absolutePath: path.join(workspaceDir, "MEMORY.md"),
|
||||
agentIds: ["main"],
|
||||
contentType: "markdown",
|
||||
},
|
||||
]);
|
||||
const appConfig: OpenClawConfig = {
|
||||
agents: {
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir }],
|
||||
},
|
||||
};
|
||||
|
||||
const first = await syncMemoryWikiBridgeSources({ config, appConfig });
|
||||
const firstPagePath = first.pagePaths[0] ?? "";
|
||||
await expect(fs.stat(path.join(vaultDir, firstPagePath))).resolves.toBeTruthy();
|
||||
|
||||
await fs.rm(path.join(workspaceDir, "MEMORY.md"));
|
||||
registerBridgeArtifacts([]);
|
||||
const second = await syncMemoryWikiBridgeSources({ config, appConfig });
|
||||
|
||||
expect(second.artifactCount).toBe(0);
|
||||
expect(second.removedCount).toBe(1);
|
||||
await expect(fs.stat(path.join(vaultDir, firstPagePath))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
});
|
||||
|
||||
it("caps composed bridge source filenames to the filesystem component limit", async () => {
|
||||
const workspaceDir = await createBridgeWorkspace(`${"漢".repeat(50)}-workspace`);
|
||||
const { rootDir: vaultDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot("long-bridge-vault"),
|
||||
config: {
|
||||
vaultMode: "bridge",
|
||||
bridge: {
|
||||
enabled: true,
|
||||
readMemoryArtifacts: true,
|
||||
indexDailyNotes: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const relativePath = `${"語".repeat(50)}/${"録".repeat(50)}.md`;
|
||||
const absolutePath = path.join(workspaceDir, relativePath);
|
||||
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
|
||||
await fs.writeFile(absolutePath, "# Deep Unicode Note\n", "utf8");
|
||||
registerBridgeArtifacts([
|
||||
{
|
||||
kind: "daily-note",
|
||||
workspaceDir,
|
||||
relativePath,
|
||||
absolutePath,
|
||||
agentIds: ["main"],
|
||||
contentType: "markdown",
|
||||
},
|
||||
]);
|
||||
|
||||
const appConfig: OpenClawConfig = {
|
||||
agents: {
|
||||
list: [{ id: "main", default: true, workspace: workspaceDir }],
|
||||
},
|
||||
};
|
||||
|
||||
const result = await syncMemoryWikiBridgeSources({ config, appConfig });
|
||||
const pagePath = result.pagePaths[0] ?? "";
|
||||
|
||||
expect(result.importedCount).toBe(1);
|
||||
expect(Buffer.byteLength(path.basename(pagePath))).toBeLessThanOrEqual(255);
|
||||
await expect(fs.stat(path.join(vaultDir, pagePath))).resolves.toBeTruthy();
|
||||
});
|
||||
});
|
||||
290
openclaw/extensions/memory-wiki/src/bridge.ts
Normal file
290
openclaw/extensions/memory-wiki/src/bridge.ts
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
listActiveMemoryPublicArtifacts,
|
||||
type MemoryPluginPublicArtifact,
|
||||
} from "openclaw/plugin-sdk/memory-host-core";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
import {
|
||||
createWikiPageFilename,
|
||||
renderMarkdownFence,
|
||||
renderWikiMarkdown,
|
||||
slugifyWikiSegment,
|
||||
} from "./markdown.js";
|
||||
import { writeImportedSourcePage } from "./source-page-shared.js";
|
||||
import { resolveArtifactKey } from "./source-path-shared.js";
|
||||
import {
|
||||
pruneImportedSourceEntries,
|
||||
readMemoryWikiSourceSyncState,
|
||||
writeMemoryWikiSourceSyncState,
|
||||
} from "./source-sync-state.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
type BridgeArtifact = {
|
||||
syncKey: string;
|
||||
artifactType: "markdown" | "memory-events";
|
||||
workspaceDir: string;
|
||||
relativePath: string;
|
||||
absolutePath: string;
|
||||
};
|
||||
|
||||
export type BridgeMemoryWikiResult = {
|
||||
importedCount: number;
|
||||
updatedCount: number;
|
||||
skippedCount: number;
|
||||
removedCount: number;
|
||||
artifactCount: number;
|
||||
workspaces: number;
|
||||
pagePaths: string[];
|
||||
};
|
||||
|
||||
function shouldImportArtifact(
|
||||
artifact: MemoryPluginPublicArtifact,
|
||||
bridgeConfig: ResolvedMemoryWikiConfig["bridge"],
|
||||
): boolean {
|
||||
switch (artifact.kind) {
|
||||
case "memory-root":
|
||||
return bridgeConfig.indexMemoryRoot;
|
||||
case "daily-note":
|
||||
return bridgeConfig.indexDailyNotes;
|
||||
case "dream-report":
|
||||
return bridgeConfig.indexDreamReports;
|
||||
case "event-log":
|
||||
return bridgeConfig.followMemoryEvents;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function collectBridgeArtifacts(
|
||||
bridgeConfig: ResolvedMemoryWikiConfig["bridge"],
|
||||
artifacts: MemoryPluginPublicArtifact[],
|
||||
): Promise<BridgeArtifact[]> {
|
||||
const collected: BridgeArtifact[] = [];
|
||||
for (const artifact of artifacts) {
|
||||
if (!shouldImportArtifact(artifact, bridgeConfig)) {
|
||||
continue;
|
||||
}
|
||||
const syncKey = await resolveArtifactKey(artifact.absolutePath);
|
||||
collected.push({
|
||||
syncKey,
|
||||
artifactType: artifact.kind === "event-log" ? "memory-events" : "markdown",
|
||||
workspaceDir: artifact.workspaceDir,
|
||||
relativePath: artifact.relativePath,
|
||||
absolutePath: artifact.absolutePath,
|
||||
});
|
||||
}
|
||||
const deduped = new Map<string, BridgeArtifact>();
|
||||
for (const artifact of collected) {
|
||||
deduped.set(artifact.syncKey, artifact);
|
||||
}
|
||||
return [...deduped.values()];
|
||||
}
|
||||
|
||||
function resolveBridgeTitle(artifact: BridgeArtifact, agentIds: string[]): string {
|
||||
if (artifact.artifactType === "memory-events") {
|
||||
if (agentIds.length === 0) {
|
||||
return "Memory Bridge: event journal";
|
||||
}
|
||||
return `Memory Bridge (${agentIds.join(", ")}): event journal`;
|
||||
}
|
||||
const base = artifact.relativePath
|
||||
.replace(/\.md$/i, "")
|
||||
.replace(/^memory\//, "")
|
||||
.replace(/\//g, " / ");
|
||||
if (agentIds.length === 0) {
|
||||
return `Memory Bridge: ${base}`;
|
||||
}
|
||||
return `Memory Bridge (${agentIds.join(", ")}): ${base}`;
|
||||
}
|
||||
|
||||
function resolveBridgePagePath(params: { workspaceDir: string; relativePath: string }): {
|
||||
pageId: string;
|
||||
pagePath: string;
|
||||
workspaceSlug: string;
|
||||
artifactSlug: string;
|
||||
} {
|
||||
const workspaceBaseSlug = slugifyWikiSegment(path.basename(params.workspaceDir));
|
||||
const workspaceHash = createHash("sha1").update(path.resolve(params.workspaceDir)).digest("hex");
|
||||
const artifactBaseSlug = slugifyWikiSegment(
|
||||
params.relativePath.replace(/\.md$/i, "").replace(/\//g, "-"),
|
||||
);
|
||||
const artifactHash = createHash("sha1").update(params.relativePath).digest("hex");
|
||||
const workspaceSlug = `${workspaceBaseSlug}-${workspaceHash.slice(0, 8)}`;
|
||||
const artifactSlug = `${artifactBaseSlug}-${artifactHash.slice(0, 8)}`;
|
||||
const fileName = createWikiPageFilename(`bridge-${workspaceSlug}-${artifactSlug}`);
|
||||
return {
|
||||
pageId: `source.bridge.${workspaceSlug}.${artifactSlug}`,
|
||||
pagePath: path.join("sources", fileName).replace(/\\/g, "/"),
|
||||
workspaceSlug,
|
||||
artifactSlug,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeBridgeSourcePage(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
artifact: BridgeArtifact;
|
||||
agentIds: string[];
|
||||
sourceUpdatedAtMs: number;
|
||||
sourceSize: number;
|
||||
state: Awaited<ReturnType<typeof readMemoryWikiSourceSyncState>>;
|
||||
}): Promise<{ pagePath: string; changed: boolean; created: boolean }> {
|
||||
const { pageId, pagePath } = resolveBridgePagePath({
|
||||
workspaceDir: params.artifact.workspaceDir,
|
||||
relativePath: params.artifact.relativePath,
|
||||
});
|
||||
const title = resolveBridgeTitle(params.artifact, params.agentIds);
|
||||
const renderFingerprint = createHash("sha1")
|
||||
.update(
|
||||
JSON.stringify({
|
||||
artifactType: params.artifact.artifactType,
|
||||
workspaceDir: params.artifact.workspaceDir,
|
||||
relativePath: params.artifact.relativePath,
|
||||
agentIds: params.agentIds,
|
||||
}),
|
||||
)
|
||||
.digest("hex");
|
||||
return writeImportedSourcePage({
|
||||
vaultRoot: params.config.vault.path,
|
||||
syncKey: params.artifact.syncKey,
|
||||
sourcePath: params.artifact.absolutePath,
|
||||
sourceUpdatedAtMs: params.sourceUpdatedAtMs,
|
||||
sourceSize: params.sourceSize,
|
||||
renderFingerprint,
|
||||
pagePath,
|
||||
group: "bridge",
|
||||
state: params.state,
|
||||
buildRendered: (raw, updatedAt) => {
|
||||
const contentLanguage =
|
||||
params.artifact.artifactType === "memory-events" ? "json" : "markdown";
|
||||
return renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: pageId,
|
||||
title,
|
||||
sourceType:
|
||||
params.artifact.artifactType === "memory-events"
|
||||
? "memory-bridge-events"
|
||||
: "memory-bridge",
|
||||
sourcePath: params.artifact.absolutePath,
|
||||
bridgeRelativePath: params.artifact.relativePath,
|
||||
bridgeWorkspaceDir: params.artifact.workspaceDir,
|
||||
bridgeAgentIds: params.agentIds,
|
||||
status: "active",
|
||||
updatedAt,
|
||||
},
|
||||
body: [
|
||||
`# ${title}`,
|
||||
"",
|
||||
"## Bridge Source",
|
||||
`- Workspace: \`${params.artifact.workspaceDir}\``,
|
||||
`- Relative path: \`${params.artifact.relativePath}\``,
|
||||
`- Kind: \`${params.artifact.artifactType}\``,
|
||||
`- Agents: ${params.agentIds.length > 0 ? params.agentIds.join(", ") : "unknown"}`,
|
||||
`- Updated: ${updatedAt}`,
|
||||
"",
|
||||
"## Content",
|
||||
renderMarkdownFence(raw, contentLanguage),
|
||||
"",
|
||||
"## Notes",
|
||||
"<!-- openclaw:human:start -->",
|
||||
"<!-- openclaw:human:end -->",
|
||||
"",
|
||||
].join("\n"),
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function syncMemoryWikiBridgeSources(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
}): Promise<BridgeMemoryWikiResult> {
|
||||
await initializeMemoryWikiVault(params.config);
|
||||
if (
|
||||
params.config.vaultMode !== "bridge" ||
|
||||
!params.config.bridge.enabled ||
|
||||
!params.config.bridge.readMemoryArtifacts ||
|
||||
!params.appConfig
|
||||
) {
|
||||
return {
|
||||
importedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
removedCount: 0,
|
||||
artifactCount: 0,
|
||||
workspaces: 0,
|
||||
pagePaths: [],
|
||||
};
|
||||
}
|
||||
|
||||
const publicArtifacts = await listActiveMemoryPublicArtifacts({ cfg: params.appConfig });
|
||||
const state = await readMemoryWikiSourceSyncState(params.config.vault.path);
|
||||
const results: Array<{ pagePath: string; changed: boolean; created: boolean }> = [];
|
||||
let artifactCount = 0;
|
||||
const activeKeys = new Set<string>();
|
||||
const artifacts = await collectBridgeArtifacts(params.config.bridge, publicArtifacts);
|
||||
const agentIdsByWorkspace = new Map<string, string[]>();
|
||||
for (const artifact of publicArtifacts) {
|
||||
agentIdsByWorkspace.set(artifact.workspaceDir, artifact.agentIds);
|
||||
}
|
||||
artifactCount = artifacts.length;
|
||||
for (const artifact of artifacts) {
|
||||
const stats = await fs.stat(artifact.absolutePath);
|
||||
activeKeys.add(artifact.syncKey);
|
||||
results.push(
|
||||
await writeBridgeSourcePage({
|
||||
config: params.config,
|
||||
artifact,
|
||||
agentIds: agentIdsByWorkspace.get(artifact.workspaceDir) ?? [],
|
||||
sourceUpdatedAtMs: stats.mtimeMs,
|
||||
sourceSize: stats.size,
|
||||
state,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const workspaceCount = new Set(publicArtifacts.map((artifact) => artifact.workspaceDir)).size;
|
||||
|
||||
const removedCount = await pruneImportedSourceEntries({
|
||||
vaultRoot: params.config.vault.path,
|
||||
group: "bridge",
|
||||
activeKeys,
|
||||
state,
|
||||
});
|
||||
await writeMemoryWikiSourceSyncState(params.config.vault.path, state);
|
||||
const importedCount = results.filter((result) => result.changed && result.created).length;
|
||||
const updatedCount = results.filter((result) => result.changed && !result.created).length;
|
||||
const skippedCount = results.filter((result) => !result.changed).length;
|
||||
const pagePaths = results
|
||||
.map((result) => result.pagePath)
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
|
||||
if (importedCount > 0 || updatedCount > 0 || removedCount > 0) {
|
||||
await appendMemoryWikiLog(params.config.vault.path, {
|
||||
type: "ingest",
|
||||
timestamp: new Date().toISOString(),
|
||||
details: {
|
||||
sourceType: "memory-bridge",
|
||||
workspaces: workspaceCount,
|
||||
artifactCount,
|
||||
importedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
removedCount,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
importedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
removedCount,
|
||||
artifactCount,
|
||||
workspaces: workspaceCount,
|
||||
pagePaths,
|
||||
};
|
||||
}
|
||||
903
openclaw/extensions/memory-wiki/src/chatgpt-import.ts
Normal file
903
openclaw/extensions/memory-wiki/src/chatgpt-import.ts
Normal file
|
|
@ -0,0 +1,903 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
replaceManagedMarkdownBlock,
|
||||
withTrailingNewline,
|
||||
} from "openclaw/plugin-sdk/memory-host-markdown";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
import {
|
||||
parseWikiMarkdown,
|
||||
renderWikiMarkdown,
|
||||
WIKI_RELATED_END_MARKER,
|
||||
WIKI_RELATED_START_MARKER,
|
||||
} from "./markdown.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
const CHATGPT_PREFERENCE_SIGNAL_RE =
|
||||
/\b(prefer|prefers|preference|want|wants|need|needs|avoid|avoids|hate|hates|love|loves|default to|should default to|always use|don't want|does not want|likes|dislikes)\b/i;
|
||||
const HUMAN_START_MARKER = "<!-- openclaw:human:start -->";
|
||||
const HUMAN_END_MARKER = "<!-- openclaw:human:end -->";
|
||||
|
||||
const CHATGPT_RISK_RULES: Array<{ label: string; pattern: RegExp }> = [
|
||||
{
|
||||
label: "relationships",
|
||||
pattern:
|
||||
/\b(relationship|dating|breakup|jealous|sex|intimacy|partner|apology|trust|boyfriend|girlfriend|husband|wife)\b/i,
|
||||
},
|
||||
{
|
||||
label: "health",
|
||||
pattern:
|
||||
/\b(supplement|medication|diagnosis|symptom|therapy|depression|anxiety|mri|migraine|injury|pain|cortisol|sleep)\b/i,
|
||||
},
|
||||
{
|
||||
label: "legal_tax",
|
||||
pattern:
|
||||
/\b(contract|tax|legal|law|lawsuit|visa|immigration|license|insurance|claim|non-residence|residency)\b/i,
|
||||
},
|
||||
{
|
||||
label: "finance",
|
||||
pattern:
|
||||
/\b(investment|invest|portfolio|dividend|yield|coupon|valuation|mortgage|loan|crypto|covered call|call option|put option)\b/i,
|
||||
},
|
||||
{
|
||||
label: "drugs",
|
||||
pattern: /\b(vape|weed|cannabis|nicotine|opioid|ketamine)\b/i,
|
||||
},
|
||||
];
|
||||
|
||||
type ChatGptMessage = {
|
||||
role: string;
|
||||
text: string;
|
||||
};
|
||||
|
||||
type ChatGptRiskAssessment = {
|
||||
level: "low" | "medium" | "high";
|
||||
reasons: string[];
|
||||
};
|
||||
|
||||
type ChatGptConversationRecord = {
|
||||
conversationId: string;
|
||||
title: string;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
sourcePath: string;
|
||||
pageId: string;
|
||||
pagePath: string;
|
||||
labels: string[];
|
||||
risk: ChatGptRiskAssessment;
|
||||
userMessageCount: number;
|
||||
assistantMessageCount: number;
|
||||
preferenceSignals: string[];
|
||||
firstUserLine?: string;
|
||||
lastUserLine?: string;
|
||||
transcript: ChatGptMessage[];
|
||||
};
|
||||
|
||||
type ChatGptImportOperation = "create" | "update" | "skip";
|
||||
|
||||
export type ChatGptImportAction = {
|
||||
conversationId: string;
|
||||
title: string;
|
||||
pagePath: string;
|
||||
operation: ChatGptImportOperation;
|
||||
riskLevel: ChatGptRiskAssessment["level"];
|
||||
labels: string[];
|
||||
userMessageCount: number;
|
||||
assistantMessageCount: number;
|
||||
preferenceSignals: string[];
|
||||
};
|
||||
|
||||
type ChatGptImportRunEntry = {
|
||||
path: string;
|
||||
snapshotPath?: string;
|
||||
};
|
||||
|
||||
type ChatGptImportRunRecord = {
|
||||
version: 1;
|
||||
runId: string;
|
||||
importType: "chatgpt";
|
||||
exportPath: string;
|
||||
sourcePath: string;
|
||||
appliedAt: string;
|
||||
conversationCount: number;
|
||||
createdCount: number;
|
||||
updatedCount: number;
|
||||
skippedCount: number;
|
||||
createdPaths: string[];
|
||||
updatedPaths: ChatGptImportRunEntry[];
|
||||
rolledBackAt?: string;
|
||||
};
|
||||
|
||||
export type ChatGptImportResult = {
|
||||
dryRun: boolean;
|
||||
exportPath: string;
|
||||
sourcePath: string;
|
||||
conversationCount: number;
|
||||
createdCount: number;
|
||||
updatedCount: number;
|
||||
skippedCount: number;
|
||||
actions: ChatGptImportAction[];
|
||||
pagePaths: string[];
|
||||
runId?: string;
|
||||
indexUpdatedFiles: string[];
|
||||
};
|
||||
|
||||
export type ChatGptRollbackResult = {
|
||||
runId: string;
|
||||
removedCount: number;
|
||||
restoredCount: number;
|
||||
pagePaths: string[];
|
||||
indexUpdatedFiles: string[];
|
||||
alreadyRolledBack: boolean;
|
||||
};
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function normalizeWhitespace(value: string): string {
|
||||
return value.trim().replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
function resolveConversationSourcePath(exportInputPath: string): {
|
||||
exportPath: string;
|
||||
conversationsPath: string;
|
||||
} {
|
||||
const resolved = path.resolve(exportInputPath);
|
||||
const conversationsPath = resolved.endsWith(".json")
|
||||
? resolved
|
||||
: path.join(resolved, "conversations.json");
|
||||
return {
|
||||
exportPath: resolved,
|
||||
conversationsPath,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadConversations(exportInputPath: string): Promise<{
|
||||
exportPath: string;
|
||||
conversationsPath: string;
|
||||
conversations: Record<string, unknown>[];
|
||||
}> {
|
||||
const { exportPath, conversationsPath } = resolveConversationSourcePath(exportInputPath);
|
||||
const raw = await fs.readFile(conversationsPath, "utf8");
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (Array.isArray(parsed)) {
|
||||
return {
|
||||
exportPath,
|
||||
conversationsPath,
|
||||
conversations: parsed.filter(
|
||||
(entry): entry is Record<string, unknown> => asRecord(entry) !== null,
|
||||
),
|
||||
};
|
||||
}
|
||||
const record = asRecord(parsed);
|
||||
if (record) {
|
||||
for (const value of Object.values(record)) {
|
||||
if (Array.isArray(value)) {
|
||||
return {
|
||||
exportPath,
|
||||
conversationsPath,
|
||||
conversations: value.filter(
|
||||
(entry): entry is Record<string, unknown> => asRecord(entry) !== null,
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(`Unrecognized ChatGPT conversations export format: ${conversationsPath}`);
|
||||
}
|
||||
|
||||
function isoFromUnix(raw: unknown): string | undefined {
|
||||
if (typeof raw !== "number" && typeof raw !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const numeric = Number(raw);
|
||||
if (!Number.isFinite(numeric)) {
|
||||
return undefined;
|
||||
}
|
||||
return new Date(numeric * 1000).toISOString();
|
||||
}
|
||||
|
||||
function cleanMessageText(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
return "";
|
||||
}
|
||||
if (
|
||||
(trimmed.includes("asset_pointer") ||
|
||||
trimmed.includes("image_asset_pointer") ||
|
||||
trimmed.includes("dalle") ||
|
||||
trimmed.includes("file_service")) &&
|
||||
trimmed.length > 40
|
||||
) {
|
||||
return "";
|
||||
}
|
||||
if (
|
||||
trimmed.startsWith("{") &&
|
||||
trimmed.length > 80 &&
|
||||
(trimmed.includes(":") || trimmed.includes("content_type"))
|
||||
) {
|
||||
const textMatch = trimmed.match(/["']text["']\s*:\s*(["'])(.+?)\1/s);
|
||||
return textMatch?.[2] ? normalizeWhitespace(textMatch[2]) : "";
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function extractMessageText(message: Record<string, unknown>): string {
|
||||
const content = asRecord(message.content);
|
||||
if (content) {
|
||||
const parts = content.parts;
|
||||
if (Array.isArray(parts)) {
|
||||
const collected: string[] = [];
|
||||
for (const part of parts) {
|
||||
if (typeof part === "string") {
|
||||
const cleaned = cleanMessageText(part);
|
||||
if (cleaned) {
|
||||
collected.push(cleaned);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const partRecord = asRecord(part);
|
||||
if (partRecord && typeof partRecord.text === "string" && partRecord.text.trim()) {
|
||||
collected.push(partRecord.text.trim());
|
||||
}
|
||||
}
|
||||
return collected.join("\n").trim();
|
||||
}
|
||||
if (typeof content.text === "string") {
|
||||
return cleanMessageText(content.text);
|
||||
}
|
||||
}
|
||||
return typeof message.text === "string" ? cleanMessageText(message.text) : "";
|
||||
}
|
||||
|
||||
function activeBranchMessages(conversation: Record<string, unknown>): ChatGptMessage[] {
|
||||
const mapping = asRecord(conversation.mapping);
|
||||
if (!mapping) {
|
||||
return [];
|
||||
}
|
||||
let currentNode =
|
||||
typeof conversation.current_node === "string" ? conversation.current_node : undefined;
|
||||
const seen = new Set<string>();
|
||||
const chain: ChatGptMessage[] = [];
|
||||
while (currentNode && !seen.has(currentNode)) {
|
||||
seen.add(currentNode);
|
||||
const node = asRecord(mapping[currentNode]);
|
||||
if (!node) {
|
||||
break;
|
||||
}
|
||||
const message = asRecord(node.message);
|
||||
if (message) {
|
||||
const author = asRecord(message.author);
|
||||
const role = typeof author?.role === "string" ? author.role : "unknown";
|
||||
const text = extractMessageText(message);
|
||||
if (text) {
|
||||
chain.push({ role, text });
|
||||
}
|
||||
}
|
||||
currentNode = typeof node.parent === "string" ? node.parent : undefined;
|
||||
}
|
||||
return chain.toReversed();
|
||||
}
|
||||
|
||||
function inferRisk(title: string, sampleText: string): ChatGptRiskAssessment {
|
||||
const blob = `${title}\n${sampleText}`.toLowerCase();
|
||||
const reasons = CHATGPT_RISK_RULES.filter((rule) => rule.pattern.test(blob)).map(
|
||||
(rule) => rule.label,
|
||||
);
|
||||
if (reasons.length > 0) {
|
||||
return { level: "high", reasons: [...new Set(reasons)] };
|
||||
}
|
||||
if (/\b(career|job|salary|interview|offer|resume|cover letter)\b/i.test(blob)) {
|
||||
return { level: "medium", reasons: ["work_career"] };
|
||||
}
|
||||
return { level: "low", reasons: [] };
|
||||
}
|
||||
|
||||
function inferLabels(title: string, sampleText: string): string[] {
|
||||
const blob = `${title}\n${sampleText}`.toLowerCase();
|
||||
const labels = new Set<string>(["domain/personal"]);
|
||||
const addAreaTopic = (area: string, topics: string[]) => {
|
||||
labels.add(area);
|
||||
for (const topic of topics) {
|
||||
labels.add(topic);
|
||||
}
|
||||
};
|
||||
const hasTranslation =
|
||||
/\b(translate|translation|traduc\w*|traducc\w*|traduç\w*|traducci[oó]n|traduccio|traducció|traduzione)\b/i.test(
|
||||
blob,
|
||||
);
|
||||
const hasLearning =
|
||||
/\b(anki|flashcards?|grammar|conjugat\w*|declension|pronunciation|vocab(?:ular(?:y|io))?|lesson|tutor|teacher|jlpt|kanji|hiragana|katakana|study|learn|practice)\b/i.test(
|
||||
blob,
|
||||
);
|
||||
const hasLanguageName =
|
||||
/\b(japanese|portuguese|catalan|castellano|espa[nñ]ol|franc[eé]s|french|italian|german|spanish)\b/i.test(
|
||||
blob,
|
||||
);
|
||||
if (hasTranslation) {
|
||||
labels.add("topic/translation");
|
||||
}
|
||||
if (
|
||||
hasLearning ||
|
||||
(hasLanguageName && /\b(learn|study|practice|lesson|tutor|grammar)\b/i.test(blob))
|
||||
) {
|
||||
addAreaTopic("area/language-learning", ["topic/language-learning"]);
|
||||
}
|
||||
if (
|
||||
/\b(hike|trail|hotel|flight|trip|travel|airport|itinerary|booking|airbnb|train|stay)\b/i.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
labels.add("area/travel");
|
||||
labels.add("topic/travel");
|
||||
}
|
||||
if (
|
||||
/\b(recipe|cook|cooking|bread|sourdough|pizza|espresso|coffee|mousse|cast iron|meatballs?)\b/i.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
addAreaTopic("area/cooking", ["topic/cooking"]);
|
||||
}
|
||||
if (
|
||||
/\b(garden|orchard|plant|soil|compost|agroforestry|permaculture|mulch|beds?|irrigation|seeds?)\b/i.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
addAreaTopic("area/gardening", ["topic/gardening"]);
|
||||
}
|
||||
if (/\b(dating|relationship|partner|jealous|breakup|trust)\b/i.test(blob)) {
|
||||
addAreaTopic("area/relationships", ["topic/relationships"]);
|
||||
}
|
||||
if (
|
||||
/\b(investment|invest|portfolio|dividend|yield|coupon|valuation|return|mortgage|loan|kraken|crypto|covered call|call option|put option|option chain|bond|stocks?)\b/i.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
addAreaTopic("area/finance", ["topic/finance"]);
|
||||
}
|
||||
if (
|
||||
/\b(contract|mou|tax|impuesto|legal|law|lawsuit|visa|immigration|license|licencia|dispute|claim|insurance|non-residence|residency)\b/i.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
addAreaTopic("area/legal-tax", ["topic/legal-tax"]);
|
||||
}
|
||||
if (
|
||||
/\b(supplement|medication|diagnos(?:is|e)|symptom|therapy|depress(?:ion|ed)|anxiet(?:y|ies)|mri|migraine|injur(?:y|ies)|pain|cortisol|sleep|dentist|dermatolog(?:ist|y))\b/i.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
addAreaTopic("area/health", ["topic/health"]);
|
||||
}
|
||||
if (
|
||||
/\b(book (an )?appointment|rebook|open (a )?new account|driving test|exam|gestor(?:a)?|itv)\b/i.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
addAreaTopic("area/life-admin", ["topic/life-admin"]);
|
||||
}
|
||||
if (/\b(frc|robot|robotics|wpilib|limelight|chiefdelphi)\b/i.test(blob)) {
|
||||
addAreaTopic("area/work", ["topic/robotics"]);
|
||||
} else if (
|
||||
/\b(docker|git|python|node|npm|pip|sql|postgres|api|bug|stack trace|permission denied)\b/i.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
addAreaTopic("area/work", ["topic/software"]);
|
||||
} else if (/\b(job|interview|cover letter|resume|cv)\b/i.test(blob)) {
|
||||
addAreaTopic("area/work", ["topic/career"]);
|
||||
}
|
||||
if (/\b(wifi|wi-fi|starlink|router|mesh|network|orbi|milesight|coverage)\b/i.test(blob)) {
|
||||
addAreaTopic("area/home", ["topic/home-infrastructure"]);
|
||||
}
|
||||
if (
|
||||
/\b(p38|range rover|porsche|bmw|bobcat|excavator|auger|trailer|chainsaw|stihl)\b/i.test(blob)
|
||||
) {
|
||||
addAreaTopic("area/vehicles", ["topic/vehicles"]);
|
||||
}
|
||||
if (![...labels].some((label) => label.startsWith("area/"))) {
|
||||
labels.add("area/other");
|
||||
}
|
||||
return [...labels];
|
||||
}
|
||||
|
||||
function collectPreferenceSignals(userTexts: string[]): string[] {
|
||||
const signals: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const text of userTexts.slice(0, 25)) {
|
||||
for (const rawLine of text.split(/\r?\n/)) {
|
||||
const line = normalizeWhitespace(rawLine);
|
||||
if (!line || !CHATGPT_PREFERENCE_SIGNAL_RE.test(line)) {
|
||||
continue;
|
||||
}
|
||||
const key = line.toLowerCase();
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
signals.push(line);
|
||||
if (signals.length >= 10) {
|
||||
return signals;
|
||||
}
|
||||
}
|
||||
}
|
||||
return signals;
|
||||
}
|
||||
|
||||
function buildTranscript(messages: ChatGptMessage[]): string {
|
||||
if (messages.length === 0) {
|
||||
return "_No active-branch transcript could be reconstructed._";
|
||||
}
|
||||
return messages
|
||||
.flatMap((message) => [
|
||||
`### ${message.role[0]?.toUpperCase() ?? "U"}${message.role.slice(1)}`,
|
||||
"",
|
||||
message.text,
|
||||
"",
|
||||
])
|
||||
.join("\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function resolveConversationPagePath(record: { conversationId: string; createdAt?: string }): {
|
||||
pageId: string;
|
||||
pagePath: string;
|
||||
} {
|
||||
const conversationSlug = record.conversationId.replace(/[^a-zA-Z0-9]/g, "").toLowerCase();
|
||||
const pageId = `source.chatgpt.${conversationSlug || createHash("sha1").update(record.conversationId).digest("hex").slice(0, 12)}`;
|
||||
const datePrefix = record.createdAt?.slice(0, 10) ?? "undated";
|
||||
const shortId = conversationSlug.slice(0, 8) || "export";
|
||||
return {
|
||||
pageId,
|
||||
pagePath: path
|
||||
.join("sources", `chatgpt-${datePrefix}-${conversationSlug || shortId}.md`)
|
||||
.replace(/\\/g, "/"),
|
||||
};
|
||||
}
|
||||
|
||||
function toConversationRecord(
|
||||
conversation: Record<string, unknown>,
|
||||
sourcePath: string,
|
||||
): ChatGptConversationRecord | null {
|
||||
const conversationId =
|
||||
typeof conversation.conversation_id === "string" ? conversation.conversation_id.trim() : "";
|
||||
if (!conversationId) {
|
||||
return null;
|
||||
}
|
||||
const title =
|
||||
typeof conversation.title === "string" && conversation.title.trim()
|
||||
? conversation.title.trim()
|
||||
: "Untitled conversation";
|
||||
const transcript = activeBranchMessages(conversation);
|
||||
const userTexts = transcript.filter((entry) => entry.role === "user").map((entry) => entry.text);
|
||||
const assistantTexts = transcript.filter((entry) => entry.role === "assistant");
|
||||
const sampleText = userTexts.slice(0, 6).join("\n");
|
||||
const risk = inferRisk(title, sampleText);
|
||||
const labels = inferLabels(title, sampleText);
|
||||
const { pageId, pagePath } = resolveConversationPagePath({
|
||||
conversationId,
|
||||
createdAt: isoFromUnix(conversation.create_time),
|
||||
});
|
||||
return {
|
||||
conversationId,
|
||||
title,
|
||||
createdAt: isoFromUnix(conversation.create_time),
|
||||
updatedAt: isoFromUnix(conversation.update_time) ?? isoFromUnix(conversation.create_time),
|
||||
sourcePath,
|
||||
pageId,
|
||||
pagePath,
|
||||
labels,
|
||||
risk,
|
||||
userMessageCount: userTexts.length,
|
||||
assistantMessageCount: assistantTexts.length,
|
||||
preferenceSignals: risk.level === "low" ? collectPreferenceSignals(userTexts) : [],
|
||||
firstUserLine: userTexts[0]?.split(/\r?\n/)[0]?.trim(),
|
||||
lastUserLine: userTexts.at(-1)?.split(/\r?\n/)[0]?.trim(),
|
||||
transcript,
|
||||
};
|
||||
}
|
||||
|
||||
function renderConversationPage(record: ChatGptConversationRecord): string {
|
||||
const autoDigestLines =
|
||||
record.risk.level === "low"
|
||||
? [
|
||||
`- User messages: ${record.userMessageCount}`,
|
||||
`- Assistant messages: ${record.assistantMessageCount}`,
|
||||
...(record.firstUserLine ? [`- First user line: ${record.firstUserLine}`] : []),
|
||||
...(record.lastUserLine ? [`- Last user line: ${record.lastUserLine}`] : []),
|
||||
...(record.preferenceSignals.length > 0
|
||||
? ["- Preference signals:", ...record.preferenceSignals.map((line) => ` - ${line}`)]
|
||||
: ["- Preference signals: none detected"]),
|
||||
]
|
||||
: [
|
||||
"- Auto digest withheld from durable-candidate generation until reviewed.",
|
||||
`- Risk reasons: ${record.risk.reasons.length > 0 ? record.risk.reasons.join(", ") : "none recorded"}`,
|
||||
];
|
||||
return renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: record.pageId,
|
||||
title: `ChatGPT Export: ${record.title}`,
|
||||
sourceType: "chatgpt-export",
|
||||
sourceSystem: "chatgpt",
|
||||
sourcePath: record.sourcePath,
|
||||
conversationId: record.conversationId,
|
||||
riskLevel: record.risk.level,
|
||||
riskReasons: record.risk.reasons,
|
||||
labels: record.labels,
|
||||
status: "draft",
|
||||
...(record.createdAt ? { createdAt: record.createdAt } : {}),
|
||||
...(record.updatedAt ? { updatedAt: record.updatedAt } : {}),
|
||||
},
|
||||
body: [
|
||||
`# ChatGPT Export: ${record.title}`,
|
||||
"",
|
||||
"## Source",
|
||||
`- Conversation id: \`${record.conversationId}\``,
|
||||
`- Export file: \`${record.sourcePath}\``,
|
||||
...(record.createdAt ? [`- Created: ${record.createdAt}`] : []),
|
||||
...(record.updatedAt ? [`- Updated: ${record.updatedAt}`] : []),
|
||||
"",
|
||||
"## Auto Triage",
|
||||
`- Risk level: \`${record.risk.level}\``,
|
||||
`- Labels: ${record.labels.join(", ")}`,
|
||||
`- Active-branch messages: ${record.transcript.length}`,
|
||||
"",
|
||||
"## Auto Digest",
|
||||
...autoDigestLines,
|
||||
"",
|
||||
"## Active Branch Transcript",
|
||||
buildTranscript(record.transcript),
|
||||
"",
|
||||
"## Notes",
|
||||
HUMAN_START_MARKER,
|
||||
HUMAN_END_MARKER,
|
||||
"",
|
||||
].join("\n"),
|
||||
});
|
||||
}
|
||||
|
||||
function replaceSimpleManagedBlock(params: {
|
||||
original: string;
|
||||
startMarker: string;
|
||||
endMarker: string;
|
||||
replacement: string;
|
||||
}): string {
|
||||
const escapedStart = params.startMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const escapedEnd = params.endMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const blockPattern = new RegExp(`${escapedStart}[\\s\\S]*?${escapedEnd}`);
|
||||
return params.original.replace(blockPattern, params.replacement);
|
||||
}
|
||||
|
||||
function extractSimpleManagedBlock(params: {
|
||||
body: string;
|
||||
startMarker: string;
|
||||
endMarker: string;
|
||||
}): string | null {
|
||||
const escapedStart = params.startMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const escapedEnd = params.endMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const blockPattern = new RegExp(`${escapedStart}[\\s\\S]*?${escapedEnd}`);
|
||||
return params.body.match(blockPattern)?.[0] ?? null;
|
||||
}
|
||||
|
||||
function extractManagedBlockBody(params: {
|
||||
body: string;
|
||||
startMarker: string;
|
||||
endMarker: string;
|
||||
}): string | null {
|
||||
const escapedStart = params.startMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const escapedEnd = params.endMarker.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const blockPattern = new RegExp(`${escapedStart}\\n?([\\s\\S]*?)\\n?${escapedEnd}`);
|
||||
const captured = params.body.match(blockPattern)?.[1];
|
||||
return typeof captured === "string" ? captured.trim() : null;
|
||||
}
|
||||
|
||||
function preserveExistingPageBlocks(rendered: string, existing: string): string {
|
||||
if (!existing.trim()) {
|
||||
return withTrailingNewline(rendered);
|
||||
}
|
||||
const parsedExisting = parseWikiMarkdown(existing);
|
||||
const parsedRendered = parseWikiMarkdown(rendered);
|
||||
let nextBody = parsedRendered.body;
|
||||
|
||||
const humanBlock = extractSimpleManagedBlock({
|
||||
body: parsedExisting.body,
|
||||
startMarker: HUMAN_START_MARKER,
|
||||
endMarker: HUMAN_END_MARKER,
|
||||
});
|
||||
if (humanBlock) {
|
||||
nextBody = replaceSimpleManagedBlock({
|
||||
original: nextBody,
|
||||
startMarker: HUMAN_START_MARKER,
|
||||
endMarker: HUMAN_END_MARKER,
|
||||
replacement: humanBlock,
|
||||
});
|
||||
}
|
||||
|
||||
const relatedBody = extractManagedBlockBody({
|
||||
body: parsedExisting.body,
|
||||
startMarker: WIKI_RELATED_START_MARKER,
|
||||
endMarker: WIKI_RELATED_END_MARKER,
|
||||
});
|
||||
if (relatedBody) {
|
||||
nextBody = replaceManagedMarkdownBlock({
|
||||
original: nextBody,
|
||||
heading: "## Related",
|
||||
startMarker: WIKI_RELATED_START_MARKER,
|
||||
endMarker: WIKI_RELATED_END_MARKER,
|
||||
body: relatedBody,
|
||||
});
|
||||
}
|
||||
|
||||
return withTrailingNewline(
|
||||
renderWikiMarkdown({
|
||||
frontmatter: parsedRendered.frontmatter,
|
||||
body: nextBody,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function buildRunId(exportPath: string, nowIso: string): string {
|
||||
const seed = `${exportPath}:${nowIso}:${Math.random()}`;
|
||||
return `chatgpt-${createHash("sha1").update(seed).digest("hex").slice(0, 12)}`;
|
||||
}
|
||||
|
||||
function resolveImportRunsDir(vaultRoot: string): string {
|
||||
return path.join(vaultRoot, ".openclaw-wiki", "import-runs");
|
||||
}
|
||||
|
||||
function resolveImportRunPath(vaultRoot: string, runId: string): string {
|
||||
return path.join(resolveImportRunsDir(vaultRoot), `${runId}.json`);
|
||||
}
|
||||
|
||||
function normalizeConversationActions(
|
||||
records: ChatGptConversationRecord[],
|
||||
operations: Map<string, ChatGptImportOperation>,
|
||||
): ChatGptImportAction[] {
|
||||
return records.map((record) => ({
|
||||
conversationId: record.conversationId,
|
||||
title: record.title,
|
||||
pagePath: record.pagePath,
|
||||
operation: operations.get(record.pagePath) ?? "skip",
|
||||
riskLevel: record.risk.level,
|
||||
labels: record.labels,
|
||||
userMessageCount: record.userMessageCount,
|
||||
assistantMessageCount: record.assistantMessageCount,
|
||||
preferenceSignals: record.preferenceSignals,
|
||||
}));
|
||||
}
|
||||
|
||||
async function writeImportRunRecord(
|
||||
vaultRoot: string,
|
||||
record: ChatGptImportRunRecord,
|
||||
): Promise<void> {
|
||||
const recordPath = resolveImportRunPath(vaultRoot, record.runId);
|
||||
await fs.mkdir(path.dirname(recordPath), { recursive: true });
|
||||
await fs.writeFile(recordPath, `${JSON.stringify(record, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
async function readImportRunRecord(
|
||||
vaultRoot: string,
|
||||
runId: string,
|
||||
): Promise<ChatGptImportRunRecord> {
|
||||
const recordPath = resolveImportRunPath(vaultRoot, runId);
|
||||
const raw = await fs.readFile(recordPath, "utf8");
|
||||
return JSON.parse(raw) as ChatGptImportRunRecord;
|
||||
}
|
||||
|
||||
async function writeTrackedImportPage(params: {
|
||||
vaultRoot: string;
|
||||
runDir: string;
|
||||
relativePath: string;
|
||||
content: string;
|
||||
record: ChatGptImportRunRecord;
|
||||
}): Promise<ChatGptImportOperation> {
|
||||
const absolutePath = path.join(params.vaultRoot, params.relativePath);
|
||||
const existing = await fs.readFile(absolutePath, "utf8").catch(() => "");
|
||||
const rendered = preserveExistingPageBlocks(params.content, existing);
|
||||
if (existing === rendered) {
|
||||
return "skip";
|
||||
}
|
||||
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
|
||||
if (!existing) {
|
||||
await fs.writeFile(absolutePath, rendered, "utf8");
|
||||
params.record.createdPaths.push(params.relativePath);
|
||||
return "create";
|
||||
}
|
||||
const snapshotHash = createHash("sha1").update(params.relativePath).digest("hex").slice(0, 12);
|
||||
const snapshotRelativePath = path.join("snapshots", `${snapshotHash}.md`).replace(/\\/g, "/");
|
||||
const snapshotAbsolutePath = path.join(params.runDir, snapshotRelativePath);
|
||||
await fs.mkdir(path.dirname(snapshotAbsolutePath), { recursive: true });
|
||||
await fs.writeFile(snapshotAbsolutePath, existing, "utf8");
|
||||
await fs.writeFile(absolutePath, rendered, "utf8");
|
||||
params.record.updatedPaths.push({
|
||||
path: params.relativePath,
|
||||
snapshotPath: snapshotRelativePath,
|
||||
});
|
||||
return "update";
|
||||
}
|
||||
|
||||
export async function importChatGptConversations(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
exportPath: string;
|
||||
dryRun?: boolean;
|
||||
nowMs?: number;
|
||||
}): Promise<ChatGptImportResult> {
|
||||
await initializeMemoryWikiVault(params.config, { nowMs: params.nowMs });
|
||||
const { exportPath, conversationsPath, conversations } = await loadConversations(
|
||||
params.exportPath,
|
||||
);
|
||||
const records = conversations
|
||||
.map((conversation) => toConversationRecord(conversation, conversationsPath))
|
||||
.filter((entry): entry is ChatGptConversationRecord => entry !== null)
|
||||
.toSorted((left, right) => left.pagePath.localeCompare(right.pagePath));
|
||||
|
||||
const operations = new Map<string, ChatGptImportOperation>();
|
||||
let createdCount = 0;
|
||||
let updatedCount = 0;
|
||||
let skippedCount = 0;
|
||||
let runId: string | undefined;
|
||||
const nowIso = new Date(params.nowMs ?? Date.now()).toISOString();
|
||||
|
||||
let importRunRecord: ChatGptImportRunRecord | undefined;
|
||||
let importRunDir = "";
|
||||
|
||||
if (!params.dryRun) {
|
||||
runId = buildRunId(exportPath, nowIso);
|
||||
importRunDir = path.join(resolveImportRunsDir(params.config.vault.path), runId);
|
||||
importRunRecord = {
|
||||
version: 1,
|
||||
runId,
|
||||
importType: "chatgpt",
|
||||
exportPath,
|
||||
sourcePath: conversationsPath,
|
||||
appliedAt: nowIso,
|
||||
conversationCount: records.length,
|
||||
createdCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
createdPaths: [],
|
||||
updatedPaths: [],
|
||||
};
|
||||
}
|
||||
|
||||
for (const record of records) {
|
||||
const rendered = renderConversationPage(record);
|
||||
const absolutePath = path.join(params.config.vault.path, record.pagePath);
|
||||
const existing = await fs.readFile(absolutePath, "utf8").catch(() => "");
|
||||
const stabilized = preserveExistingPageBlocks(rendered, existing);
|
||||
const operation: ChatGptImportOperation =
|
||||
existing === stabilized ? "skip" : existing ? "update" : "create";
|
||||
operations.set(record.pagePath, operation);
|
||||
if (operation === "create") {
|
||||
createdCount += 1;
|
||||
} else if (operation === "update") {
|
||||
updatedCount += 1;
|
||||
} else {
|
||||
skippedCount += 1;
|
||||
}
|
||||
if (!params.dryRun && importRunRecord) {
|
||||
await writeTrackedImportPage({
|
||||
vaultRoot: params.config.vault.path,
|
||||
runDir: importRunDir,
|
||||
relativePath: record.pagePath,
|
||||
content: rendered,
|
||||
record: importRunRecord,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let indexUpdatedFiles: string[] = [];
|
||||
if (!params.dryRun && importRunRecord) {
|
||||
importRunRecord.createdCount = createdCount;
|
||||
importRunRecord.updatedCount = updatedCount;
|
||||
importRunRecord.skippedCount = skippedCount;
|
||||
if (importRunRecord.createdPaths.length > 0 || importRunRecord.updatedPaths.length > 0) {
|
||||
const compile = await compileMemoryWikiVault(params.config);
|
||||
indexUpdatedFiles = compile.updatedFiles;
|
||||
await writeImportRunRecord(params.config.vault.path, importRunRecord);
|
||||
await appendMemoryWikiLog(params.config.vault.path, {
|
||||
type: "ingest",
|
||||
timestamp: nowIso,
|
||||
details: {
|
||||
sourceType: "chatgpt-export",
|
||||
runId: importRunRecord.runId,
|
||||
exportPath,
|
||||
sourcePath: conversationsPath,
|
||||
conversationCount: records.length,
|
||||
createdCount: importRunRecord.createdPaths.length,
|
||||
updatedCount: importRunRecord.updatedPaths.length,
|
||||
skippedCount,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
runId = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dryRun: Boolean(params.dryRun),
|
||||
exportPath,
|
||||
sourcePath: conversationsPath,
|
||||
conversationCount: records.length,
|
||||
createdCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
actions: normalizeConversationActions(records, operations),
|
||||
pagePaths: records.map((record) => record.pagePath),
|
||||
...(runId ? { runId } : {}),
|
||||
indexUpdatedFiles,
|
||||
};
|
||||
}
|
||||
|
||||
export async function rollbackChatGptImportRun(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
runId: string;
|
||||
}): Promise<ChatGptRollbackResult> {
|
||||
await initializeMemoryWikiVault(params.config);
|
||||
const record = await readImportRunRecord(params.config.vault.path, params.runId);
|
||||
if (record.rolledBackAt) {
|
||||
return {
|
||||
runId: record.runId,
|
||||
removedCount: 0,
|
||||
restoredCount: 0,
|
||||
pagePaths: [
|
||||
...record.createdPaths,
|
||||
...record.updatedPaths.map((entry) => entry.path),
|
||||
].toSorted((left, right) => left.localeCompare(right)),
|
||||
indexUpdatedFiles: [],
|
||||
alreadyRolledBack: true,
|
||||
};
|
||||
}
|
||||
let removedCount = 0;
|
||||
for (const relativePath of record.createdPaths) {
|
||||
await fs
|
||||
.rm(path.join(params.config.vault.path, relativePath), { force: true })
|
||||
.catch(() => undefined);
|
||||
removedCount += 1;
|
||||
}
|
||||
let restoredCount = 0;
|
||||
const runDir = path.join(resolveImportRunsDir(params.config.vault.path), record.runId);
|
||||
for (const entry of record.updatedPaths) {
|
||||
if (!entry.snapshotPath) {
|
||||
continue;
|
||||
}
|
||||
const snapshotPath = path.join(runDir, entry.snapshotPath);
|
||||
const snapshot = await fs.readFile(snapshotPath, "utf8");
|
||||
const targetPath = path.join(params.config.vault.path, entry.path);
|
||||
await fs.mkdir(path.dirname(targetPath), { recursive: true });
|
||||
await fs.writeFile(targetPath, snapshot, "utf8");
|
||||
restoredCount += 1;
|
||||
}
|
||||
const compile = await compileMemoryWikiVault(params.config);
|
||||
record.rolledBackAt = new Date().toISOString();
|
||||
await writeImportRunRecord(params.config.vault.path, record);
|
||||
await appendMemoryWikiLog(params.config.vault.path, {
|
||||
type: "ingest",
|
||||
timestamp: record.rolledBackAt,
|
||||
details: {
|
||||
sourceType: "chatgpt-export",
|
||||
runId: record.runId,
|
||||
rollback: true,
|
||||
removedCount,
|
||||
restoredCount,
|
||||
},
|
||||
});
|
||||
return {
|
||||
runId: record.runId,
|
||||
removedCount,
|
||||
restoredCount,
|
||||
pagePaths: [...record.createdPaths, ...record.updatedPaths.map((entry) => entry.path)].toSorted(
|
||||
(left, right) => left.localeCompare(right),
|
||||
),
|
||||
indexUpdatedFiles: compile.updatedFiles,
|
||||
alreadyRolledBack: false,
|
||||
};
|
||||
}
|
||||
60
openclaw/extensions/memory-wiki/src/claim-health.test.ts
Normal file
60
openclaw/extensions/memory-wiki/src/claim-health.test.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { buildPageContradictionClusters } from "./claim-health.js";
|
||||
import type { WikiPageSummary } from "./markdown.js";
|
||||
|
||||
function createPage(params: {
|
||||
relativePath: string;
|
||||
title: string;
|
||||
contradictions: string[];
|
||||
}): WikiPageSummary {
|
||||
return {
|
||||
absolutePath: `/tmp/${params.relativePath}`,
|
||||
relativePath: params.relativePath,
|
||||
kind: "entity",
|
||||
title: params.title,
|
||||
sourceIds: [],
|
||||
linkTargets: [],
|
||||
claims: [],
|
||||
contradictions: params.contradictions,
|
||||
questions: [],
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildPageContradictionClusters", () => {
|
||||
it("clusters Unicode contradiction notes that differ only by punctuation", () => {
|
||||
const clusters = buildPageContradictionClusters([
|
||||
createPage({
|
||||
relativePath: "entities/alpha.md",
|
||||
title: "Alpha",
|
||||
contradictions: ["模型冲突:版本 A"],
|
||||
}),
|
||||
createPage({
|
||||
relativePath: "entities/beta.md",
|
||||
title: "Beta",
|
||||
contradictions: ["模型冲突 版本 A"],
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(clusters).toHaveLength(1);
|
||||
expect(clusters[0]?.entries).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("keeps combining-mark contradiction notes in separate clusters", () => {
|
||||
const clusters = buildPageContradictionClusters([
|
||||
createPage({
|
||||
relativePath: "entities/alpha.md",
|
||||
title: "Alpha",
|
||||
contradictions: ["किताब"],
|
||||
}),
|
||||
createPage({
|
||||
relativePath: "entities/beta.md",
|
||||
title: "Beta",
|
||||
contradictions: ["कीताब"],
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(clusters).toHaveLength(2);
|
||||
expect(clusters.map((cluster) => cluster.key).toSorted()).toEqual(["किताब", "कीताब"]);
|
||||
expect(clusters.every((cluster) => cluster.entries)).toBe(true);
|
||||
});
|
||||
});
|
||||
239
openclaw/extensions/memory-wiki/src/claim-health.ts
Normal file
239
openclaw/extensions/memory-wiki/src/claim-health.ts
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import type { WikiClaim, WikiPageSummary } from "./markdown.js";
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
export const WIKI_AGING_DAYS = 30;
|
||||
export const WIKI_STALE_DAYS = 90;
|
||||
|
||||
const CONTESTED_CLAIM_STATUSES = new Set(["contested", "contradicted", "refuted", "superseded"]);
|
||||
|
||||
export type WikiFreshnessLevel = "fresh" | "aging" | "stale" | "unknown";
|
||||
|
||||
export type WikiFreshness = {
|
||||
level: WikiFreshnessLevel;
|
||||
reason: string;
|
||||
daysSinceTouch?: number;
|
||||
lastTouchedAt?: string;
|
||||
};
|
||||
|
||||
export type WikiClaimHealth = {
|
||||
key: string;
|
||||
pagePath: string;
|
||||
pageTitle: string;
|
||||
pageId?: string;
|
||||
claimId?: string;
|
||||
text: string;
|
||||
status: string;
|
||||
confidence?: number;
|
||||
evidenceCount: number;
|
||||
missingEvidence: boolean;
|
||||
freshness: WikiFreshness;
|
||||
};
|
||||
|
||||
export type WikiClaimContradictionCluster = {
|
||||
key: string;
|
||||
label: string;
|
||||
entries: WikiClaimHealth[];
|
||||
};
|
||||
|
||||
export type WikiPageContradictionCluster = {
|
||||
key: string;
|
||||
label: string;
|
||||
entries: Array<{
|
||||
pagePath: string;
|
||||
pageTitle: string;
|
||||
pageId?: string;
|
||||
note: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
function parseTimestamp(value?: string): number | null {
|
||||
if (!value?.trim()) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function clampDaysSinceTouch(daysSinceTouch: number): number {
|
||||
return Math.max(0, daysSinceTouch);
|
||||
}
|
||||
|
||||
function normalizeClaimTextKey(text: string): string {
|
||||
return normalizeLowercaseStringOrEmpty(text.replace(/\s+/g, " "));
|
||||
}
|
||||
|
||||
function normalizeTextKey(text: string): string {
|
||||
return normalizeLowercaseStringOrEmpty(text)
|
||||
.replace(/[^\p{L}\p{N}\p{M}]+/gu, " ")
|
||||
.replace(/\s+/g, " ");
|
||||
}
|
||||
|
||||
function buildFreshnessFromTimestamp(params: { timestamp?: string; now?: Date }): WikiFreshness {
|
||||
const now = params.now ?? new Date();
|
||||
const timestampMs = parseTimestamp(params.timestamp);
|
||||
if (timestampMs === null || !params.timestamp) {
|
||||
return {
|
||||
level: "unknown",
|
||||
reason: "missing updatedAt",
|
||||
};
|
||||
}
|
||||
const daysSinceTouch = clampDaysSinceTouch(Math.floor((now.getTime() - timestampMs) / DAY_MS));
|
||||
if (daysSinceTouch >= WIKI_STALE_DAYS) {
|
||||
return {
|
||||
level: "stale",
|
||||
reason: `last touched ${params.timestamp}`,
|
||||
daysSinceTouch,
|
||||
lastTouchedAt: params.timestamp,
|
||||
};
|
||||
}
|
||||
if (daysSinceTouch >= WIKI_AGING_DAYS) {
|
||||
return {
|
||||
level: "aging",
|
||||
reason: `last touched ${params.timestamp}`,
|
||||
daysSinceTouch,
|
||||
lastTouchedAt: params.timestamp,
|
||||
};
|
||||
}
|
||||
return {
|
||||
level: "fresh",
|
||||
reason: `last touched ${params.timestamp}`,
|
||||
daysSinceTouch,
|
||||
lastTouchedAt: params.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveLatestTimestamp(candidates: Array<string | undefined>): string | undefined {
|
||||
let bestValue: string | undefined;
|
||||
let bestMs = -1;
|
||||
for (const candidate of candidates) {
|
||||
const parsed = parseTimestamp(candidate);
|
||||
if (parsed === null || !candidate || parsed <= bestMs) {
|
||||
continue;
|
||||
}
|
||||
bestMs = parsed;
|
||||
bestValue = candidate;
|
||||
}
|
||||
return bestValue;
|
||||
}
|
||||
|
||||
export function normalizeClaimStatus(status?: string): string {
|
||||
return normalizeLowercaseStringOrEmpty(status) || "supported";
|
||||
}
|
||||
|
||||
export function isClaimContestedStatus(status?: string): boolean {
|
||||
return CONTESTED_CLAIM_STATUSES.has(normalizeClaimStatus(status));
|
||||
}
|
||||
|
||||
export function assessPageFreshness(page: WikiPageSummary, now?: Date): WikiFreshness {
|
||||
return buildFreshnessFromTimestamp({ timestamp: page.updatedAt, now });
|
||||
}
|
||||
|
||||
export function assessClaimFreshness(params: {
|
||||
page: WikiPageSummary;
|
||||
claim: WikiClaim;
|
||||
now?: Date;
|
||||
}): WikiFreshness {
|
||||
const latestTimestamp = resolveLatestTimestamp([
|
||||
params.claim.updatedAt,
|
||||
params.page.updatedAt,
|
||||
...params.claim.evidence.map((evidence) => evidence.updatedAt),
|
||||
]);
|
||||
return buildFreshnessFromTimestamp({ timestamp: latestTimestamp, now: params.now });
|
||||
}
|
||||
|
||||
export function buildWikiClaimHealth(params: {
|
||||
page: WikiPageSummary;
|
||||
claim: WikiClaim;
|
||||
index: number;
|
||||
now?: Date;
|
||||
}): WikiClaimHealth {
|
||||
const claimId = params.claim.id?.trim();
|
||||
return {
|
||||
key: `${params.page.relativePath}#${claimId ?? `claim-${params.index + 1}`}`,
|
||||
pagePath: params.page.relativePath,
|
||||
pageTitle: params.page.title,
|
||||
...(params.page.id ? { pageId: params.page.id } : {}),
|
||||
...(claimId ? { claimId } : {}),
|
||||
text: params.claim.text,
|
||||
status: normalizeClaimStatus(params.claim.status),
|
||||
...(typeof params.claim.confidence === "number" ? { confidence: params.claim.confidence } : {}),
|
||||
evidenceCount: params.claim.evidence.length,
|
||||
missingEvidence: params.claim.evidence.length === 0,
|
||||
freshness: assessClaimFreshness({ page: params.page, claim: params.claim, now: params.now }),
|
||||
};
|
||||
}
|
||||
|
||||
export function collectWikiClaimHealth(pages: WikiPageSummary[], now?: Date): WikiClaimHealth[] {
|
||||
return pages.flatMap((page) =>
|
||||
page.claims.map((claim, index) => buildWikiClaimHealth({ page, claim, index, now })),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildClaimContradictionClusters(params: {
|
||||
pages: WikiPageSummary[];
|
||||
now?: Date;
|
||||
}): WikiClaimContradictionCluster[] {
|
||||
const claimHealth = collectWikiClaimHealth(params.pages, params.now);
|
||||
const byId = new Map<string, WikiClaimHealth[]>();
|
||||
for (const claim of claimHealth) {
|
||||
if (!claim.claimId) {
|
||||
continue;
|
||||
}
|
||||
const current = byId.get(claim.claimId) ?? [];
|
||||
current.push(claim);
|
||||
byId.set(claim.claimId, current);
|
||||
}
|
||||
|
||||
return [...byId.entries()]
|
||||
.flatMap(([claimId, entries]) => {
|
||||
if (entries.length < 2) {
|
||||
return [];
|
||||
}
|
||||
const distinctTexts = new Set(entries.map((entry) => normalizeClaimTextKey(entry.text)));
|
||||
const distinctStatuses = new Set(entries.map((entry) => entry.status));
|
||||
if (distinctTexts.size < 2 && distinctStatuses.size < 2) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
key: claimId,
|
||||
label: claimId,
|
||||
entries: [...entries].toSorted((left, right) =>
|
||||
left.pagePath.localeCompare(right.pagePath),
|
||||
),
|
||||
},
|
||||
];
|
||||
})
|
||||
.toSorted((left, right) => left.label.localeCompare(right.label));
|
||||
}
|
||||
|
||||
export function buildPageContradictionClusters(
|
||||
pages: WikiPageSummary[],
|
||||
): WikiPageContradictionCluster[] {
|
||||
const byNote = new Map<string, WikiPageContradictionCluster["entries"]>();
|
||||
for (const page of pages) {
|
||||
for (const note of page.contradictions) {
|
||||
const key = normalizeTextKey(note);
|
||||
if (!key) {
|
||||
continue;
|
||||
}
|
||||
const current = byNote.get(key) ?? [];
|
||||
current.push({
|
||||
pagePath: page.relativePath,
|
||||
pageTitle: page.title,
|
||||
...(page.id ? { pageId: page.id } : {}),
|
||||
note,
|
||||
});
|
||||
byNote.set(key, current);
|
||||
}
|
||||
}
|
||||
return [...byNote.entries()]
|
||||
.map(([key, entries]) => ({
|
||||
key,
|
||||
label: entries[0]?.note ?? key,
|
||||
entries: [...entries].toSorted((left, right) => left.pagePath.localeCompare(right.pagePath)),
|
||||
}))
|
||||
.toSorted((left, right) => left.label.localeCompare(right.label));
|
||||
}
|
||||
250
openclaw/extensions/memory-wiki/src/cli.test.ts
Normal file
250
openclaw/extensions/memory-wiki/src/cli.test.ts
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { Command } from "commander";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { registerWikiCli, runWikiChatGptImport, runWikiChatGptRollback } from "./cli.js";
|
||||
import type { MemoryWikiPluginConfig } from "./config.js";
|
||||
import { parseWikiMarkdown, renderWikiMarkdown } from "./markdown.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
let suiteRoot = "";
|
||||
let caseIndex = 0;
|
||||
|
||||
describe("memory-wiki cli", () => {
|
||||
beforeAll(async () => {
|
||||
suiteRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-cli-suite-"));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (suiteRoot) {
|
||||
await fs.rm(suiteRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
vi.spyOn(process.stdout, "write").mockImplementation(
|
||||
(() => true) as typeof process.stdout.write,
|
||||
);
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
process.exitCode = undefined;
|
||||
});
|
||||
|
||||
async function createCliVault(options?: {
|
||||
config?: MemoryWikiPluginConfig;
|
||||
initialize?: boolean;
|
||||
}) {
|
||||
return createVault({
|
||||
prefix: "memory-wiki-cli-",
|
||||
rootDir: path.join(suiteRoot, `case-${caseIndex++}`),
|
||||
initialize: options?.initialize,
|
||||
config: options?.config,
|
||||
});
|
||||
}
|
||||
|
||||
async function createChatGptExport(rootDir: string) {
|
||||
const exportDir = path.join(rootDir, "chatgpt-export");
|
||||
await fs.mkdir(exportDir, { recursive: true });
|
||||
const conversations = [
|
||||
{
|
||||
conversation_id: "12345678-1234-1234-1234-1234567890ab",
|
||||
title: "Travel preference check",
|
||||
create_time: 1_712_363_200,
|
||||
update_time: 1_712_366_800,
|
||||
current_node: "assistant-1",
|
||||
mapping: {
|
||||
root: {},
|
||||
"user-1": {
|
||||
parent: "root",
|
||||
message: {
|
||||
author: { role: "user" },
|
||||
content: {
|
||||
parts: ["I prefer aisle seats and I don't want a hotel far from the airport."],
|
||||
},
|
||||
},
|
||||
},
|
||||
"assistant-1": {
|
||||
parent: "user-1",
|
||||
message: {
|
||||
author: { role: "assistant" },
|
||||
content: {
|
||||
parts: ["Noted. I will keep travel options close to the airport."],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
await fs.writeFile(
|
||||
path.join(exportDir, "conversations.json"),
|
||||
`${JSON.stringify(conversations, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
return exportDir;
|
||||
}
|
||||
|
||||
it("registers apply synthesis and writes a synthesis page", async () => {
|
||||
const { rootDir, config } = await createCliVault();
|
||||
const program = new Command();
|
||||
program.name("test");
|
||||
registerWikiCli(program, config);
|
||||
|
||||
await program.parseAsync(
|
||||
[
|
||||
"wiki",
|
||||
"apply",
|
||||
"synthesis",
|
||||
"CLI Alpha",
|
||||
"--body",
|
||||
"Alpha from CLI.",
|
||||
"--source-id",
|
||||
"source.alpha",
|
||||
"--source-id",
|
||||
"source.beta",
|
||||
],
|
||||
{ from: "user" },
|
||||
);
|
||||
|
||||
const page = await fs.readFile(path.join(rootDir, "syntheses", "cli-alpha.md"), "utf8");
|
||||
expect(page).toContain("Alpha from CLI.");
|
||||
expect(page).toContain("source.alpha");
|
||||
await expect(fs.readFile(path.join(rootDir, "index.md"), "utf8")).resolves.toContain(
|
||||
"[CLI Alpha](syntheses/cli-alpha.md)",
|
||||
);
|
||||
});
|
||||
|
||||
it("registers apply metadata and preserves the page body", async () => {
|
||||
const { rootDir, config } = await createCliVault();
|
||||
const targetPath = path.join(rootDir, "entities", "alpha.md");
|
||||
await fs.mkdir(path.dirname(targetPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
targetPath,
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha",
|
||||
title: "Alpha",
|
||||
sourceIds: ["source.old"],
|
||||
confidence: 0.2,
|
||||
},
|
||||
body: `# Alpha
|
||||
|
||||
## Notes
|
||||
<!-- openclaw:human:start -->
|
||||
cli note
|
||||
<!-- openclaw:human:end -->
|
||||
`,
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const program = new Command();
|
||||
program.name("test");
|
||||
registerWikiCli(program, config);
|
||||
|
||||
await program.parseAsync(
|
||||
[
|
||||
"wiki",
|
||||
"apply",
|
||||
"metadata",
|
||||
"entity.alpha",
|
||||
"--source-id",
|
||||
"source.new",
|
||||
"--contradiction",
|
||||
"Conflicts with source.beta",
|
||||
"--question",
|
||||
"Still active?",
|
||||
"--status",
|
||||
"review",
|
||||
"--clear-confidence",
|
||||
],
|
||||
{ from: "user" },
|
||||
);
|
||||
|
||||
const page = await fs.readFile(path.join(rootDir, "entities", "alpha.md"), "utf8");
|
||||
const parsed = parseWikiMarkdown(page);
|
||||
expect(parsed.frontmatter).toMatchObject({
|
||||
sourceIds: ["source.new"],
|
||||
contradictions: ["Conflicts with source.beta"],
|
||||
questions: ["Still active?"],
|
||||
status: "review",
|
||||
});
|
||||
expect(parsed.frontmatter).not.toHaveProperty("confidence");
|
||||
expect(parsed.body).toContain("cli note");
|
||||
});
|
||||
|
||||
it("runs wiki doctor and sets a non-zero exit code when warnings exist", async () => {
|
||||
const { rootDir, config } = await createCliVault({
|
||||
config: {
|
||||
vaultMode: "bridge",
|
||||
bridge: { enabled: false },
|
||||
},
|
||||
});
|
||||
const program = new Command();
|
||||
program.name("test");
|
||||
registerWikiCli(program, config);
|
||||
await fs.rm(rootDir, { recursive: true, force: true });
|
||||
|
||||
await program.parseAsync(["wiki", "doctor", "--json"], { from: "user" });
|
||||
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it("imports ChatGPT exports with dry-run, apply, and rollback", async () => {
|
||||
const { rootDir, config } = await createCliVault({ initialize: true });
|
||||
const exportDir = await createChatGptExport(rootDir);
|
||||
|
||||
const dryRun = await runWikiChatGptImport({
|
||||
config,
|
||||
exportPath: exportDir,
|
||||
dryRun: true,
|
||||
json: true,
|
||||
});
|
||||
expect(dryRun.dryRun).toBe(true);
|
||||
expect(dryRun.createdCount).toBe(1);
|
||||
await expect(fs.readdir(path.join(rootDir, "sources"))).resolves.toEqual([]);
|
||||
|
||||
const applied = await runWikiChatGptImport({
|
||||
config,
|
||||
exportPath: exportDir,
|
||||
json: true,
|
||||
});
|
||||
expect(applied.runId).toBeTruthy();
|
||||
expect(applied.createdCount).toBe(1);
|
||||
const sourceFiles = (await fs.readdir(path.join(rootDir, "sources"))).filter(
|
||||
(entry) => entry !== "index.md",
|
||||
);
|
||||
expect(sourceFiles).toHaveLength(1);
|
||||
const pageContent = await fs.readFile(path.join(rootDir, "sources", sourceFiles[0]), "utf8");
|
||||
expect(pageContent).toContain("ChatGPT Export: Travel preference check");
|
||||
expect(pageContent).toContain("I prefer aisle seats");
|
||||
expect(pageContent).toContain("Preference signals:");
|
||||
|
||||
const secondDryRun = await runWikiChatGptImport({
|
||||
config,
|
||||
exportPath: exportDir,
|
||||
dryRun: true,
|
||||
json: true,
|
||||
});
|
||||
expect(secondDryRun.createdCount).toBe(0);
|
||||
expect(secondDryRun.updatedCount).toBe(0);
|
||||
expect(secondDryRun.skippedCount).toBe(1);
|
||||
|
||||
const rollback = await runWikiChatGptRollback({
|
||||
config,
|
||||
runId: applied.runId!,
|
||||
json: true,
|
||||
});
|
||||
expect(rollback.alreadyRolledBack).toBe(false);
|
||||
await expect(
|
||||
fs
|
||||
.readdir(path.join(rootDir, "sources"))
|
||||
.then((entries) => entries.filter((entry) => entry !== "index.md")),
|
||||
).resolves.toEqual([]);
|
||||
});
|
||||
});
|
||||
905
openclaw/extensions/memory-wiki/src/cli.ts
Normal file
905
openclaw/extensions/memory-wiki/src/cli.ts
Normal file
|
|
@ -0,0 +1,905 @@
|
|||
import fs from "node:fs/promises";
|
||||
import type { Command } from "commander";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
import { applyMemoryWikiMutation } from "./apply.js";
|
||||
import {
|
||||
importChatGptConversations,
|
||||
rollbackChatGptImportRun,
|
||||
type ChatGptImportResult,
|
||||
type ChatGptRollbackResult,
|
||||
} from "./chatgpt-import.js";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import {
|
||||
resolveMemoryWikiConfig,
|
||||
WIKI_SEARCH_BACKENDS,
|
||||
WIKI_SEARCH_CORPORA,
|
||||
type MemoryWikiPluginConfig,
|
||||
type ResolvedMemoryWikiConfig,
|
||||
} from "./config.js";
|
||||
import { ingestMemoryWikiSource } from "./ingest.js";
|
||||
import { lintMemoryWikiVault } from "./lint.js";
|
||||
import {
|
||||
probeObsidianCli,
|
||||
runObsidianCommand,
|
||||
runObsidianDaily,
|
||||
runObsidianOpen,
|
||||
runObsidianSearch,
|
||||
} from "./obsidian.js";
|
||||
import { getMemoryWikiPage, searchMemoryWiki } from "./query.js";
|
||||
import { syncMemoryWikiImportedSources } from "./source-sync.js";
|
||||
import {
|
||||
buildMemoryWikiDoctorReport,
|
||||
renderMemoryWikiDoctor,
|
||||
renderMemoryWikiStatus,
|
||||
resolveMemoryWikiStatus,
|
||||
} from "./status.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
type WikiStatusCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiDoctorCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiInitCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiCompileCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiLintCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiIngestCommandOptions = {
|
||||
json?: boolean;
|
||||
title?: string;
|
||||
};
|
||||
|
||||
type WikiSearchCommandOptions = {
|
||||
json?: boolean;
|
||||
maxResults?: number;
|
||||
backend?: ResolvedMemoryWikiConfig["search"]["backend"];
|
||||
corpus?: ResolvedMemoryWikiConfig["search"]["corpus"];
|
||||
};
|
||||
|
||||
type WikiGetCommandOptions = {
|
||||
json?: boolean;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
backend?: ResolvedMemoryWikiConfig["search"]["backend"];
|
||||
corpus?: ResolvedMemoryWikiConfig["search"]["corpus"];
|
||||
};
|
||||
|
||||
type WikiApplySynthesisCommandOptions = {
|
||||
json?: boolean;
|
||||
body?: string;
|
||||
bodyFile?: string;
|
||||
sourceId?: string[];
|
||||
contradiction?: string[];
|
||||
question?: string[];
|
||||
confidence?: number;
|
||||
status?: string;
|
||||
};
|
||||
|
||||
type WikiApplyMetadataCommandOptions = {
|
||||
json?: boolean;
|
||||
sourceId?: string[];
|
||||
contradiction?: string[];
|
||||
question?: string[];
|
||||
confidence?: number;
|
||||
clearConfidence?: boolean;
|
||||
status?: string;
|
||||
};
|
||||
|
||||
type WikiBridgeImportCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiUnsafeLocalImportCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiChatGptImportCommandOptions = {
|
||||
json?: boolean;
|
||||
dryRun?: boolean;
|
||||
export?: string;
|
||||
};
|
||||
|
||||
type WikiChatGptRollbackCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiObsidianSearchCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiObsidianOpenCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiObsidianCommandCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
type WikiObsidianDailyCommandOptions = {
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
function isResolvedMemoryWikiConfig(
|
||||
config: MemoryWikiPluginConfig | ResolvedMemoryWikiConfig | undefined,
|
||||
): config is ResolvedMemoryWikiConfig {
|
||||
return Boolean(
|
||||
config &&
|
||||
"vaultMode" in config &&
|
||||
"vault" in config &&
|
||||
"bridge" in config &&
|
||||
"obsidian" in config &&
|
||||
"unsafeLocal" in config,
|
||||
);
|
||||
}
|
||||
|
||||
function writeOutput(output: string, writer: Pick<NodeJS.WriteStream, "write"> = process.stdout) {
|
||||
writer.write(output.endsWith("\n") ? output : `${output}\n`);
|
||||
}
|
||||
|
||||
function normalizeCliStringList(values?: string[]): string[] | undefined {
|
||||
if (!values) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = values
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
.filter((value, index, all) => all.indexOf(value) === index);
|
||||
return normalized.length > 0 ? normalized : undefined;
|
||||
}
|
||||
|
||||
function collectCliValues(value: string, acc: string[] = []) {
|
||||
acc.push(value);
|
||||
return acc;
|
||||
}
|
||||
|
||||
function parseWikiSearchEnumOption<T extends string>(
|
||||
value: string,
|
||||
allowed: readonly T[],
|
||||
label: string,
|
||||
): T {
|
||||
if ((allowed as readonly string[]).includes(value)) {
|
||||
return value as T;
|
||||
}
|
||||
throw new Error(`Invalid ${label}: ${value}. Expected one of: ${allowed.join(", ")}`);
|
||||
}
|
||||
|
||||
async function resolveWikiApplyBody(params: { body?: string; bodyFile?: string }): Promise<string> {
|
||||
if (params.body?.trim()) {
|
||||
return params.body;
|
||||
}
|
||||
if (params.bodyFile?.trim()) {
|
||||
return await fs.readFile(params.bodyFile, "utf8");
|
||||
}
|
||||
throw new Error("wiki apply synthesis requires --body or --body-file.");
|
||||
}
|
||||
|
||||
type MemoryWikiMutationResult = Awaited<ReturnType<typeof applyMemoryWikiMutation>>;
|
||||
|
||||
function formatMemoryWikiMutationSummary(result: MemoryWikiMutationResult, json?: boolean): string {
|
||||
if (json) {
|
||||
return JSON.stringify(result, null, 2);
|
||||
}
|
||||
return `${result.changed ? "Updated" : "No changes for"} ${result.pagePath} via ${result.operation}. ${result.compile.updatedFiles.length > 0 ? `Refreshed ${result.compile.updatedFiles.length} index file${result.compile.updatedFiles.length === 1 ? "" : "s"}.` : "Indexes unchanged."}`;
|
||||
}
|
||||
|
||||
function formatJsonOrText<T>(
|
||||
result: T,
|
||||
json: boolean | undefined,
|
||||
render: (result: T) => string,
|
||||
): string {
|
||||
return json ? JSON.stringify(result, null, 2) : render(result);
|
||||
}
|
||||
|
||||
async function runWikiCommandWithSummary<T>(params: {
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
run: () => Promise<T>;
|
||||
render: (result: T) => string;
|
||||
}): Promise<T> {
|
||||
const result = await params.run();
|
||||
writeOutput(formatJsonOrText(result, params.json, params.render), params.stdout);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function runSyncedWikiCommandWithSummary<T>(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
run: () => Promise<T>;
|
||||
render: (result: T) => string;
|
||||
}): Promise<T> {
|
||||
await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
|
||||
return runWikiCommandWithSummary(params);
|
||||
}
|
||||
|
||||
function addWikiSearchConfigOptions<T extends Command>(command: T): T {
|
||||
return command
|
||||
.option(
|
||||
"--backend <backend>",
|
||||
`Search backend (${WIKI_SEARCH_BACKENDS.join(", ")})`,
|
||||
(value: string) => parseWikiSearchEnumOption(value, WIKI_SEARCH_BACKENDS, "backend"),
|
||||
)
|
||||
.option(
|
||||
"--corpus <corpus>",
|
||||
`Search corpus (${WIKI_SEARCH_CORPORA.join(", ")})`,
|
||||
(value: string) => parseWikiSearchEnumOption(value, WIKI_SEARCH_CORPORA, "corpus"),
|
||||
);
|
||||
}
|
||||
|
||||
function addWikiApplyMutationOptions<T extends Command>(command: T): T {
|
||||
return command
|
||||
.option("--source-id <id>", "Source id", collectCliValues)
|
||||
.option("--contradiction <text>", "Contradiction note", collectCliValues)
|
||||
.option("--question <text>", "Open question", collectCliValues)
|
||||
.option("--confidence <n>", "Confidence score between 0 and 1", (value: string) =>
|
||||
Number(value),
|
||||
)
|
||||
.option("--status <status>", "Page status");
|
||||
}
|
||||
|
||||
export async function runWikiStatus(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
|
||||
const status = await resolveMemoryWikiStatus(params.config, {
|
||||
appConfig: params.appConfig,
|
||||
});
|
||||
writeOutput(
|
||||
params.json ? JSON.stringify(status, null, 2) : renderMemoryWikiStatus(status),
|
||||
params.stdout,
|
||||
);
|
||||
return status;
|
||||
}
|
||||
|
||||
export async function runWikiDoctor(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
|
||||
const report = buildMemoryWikiDoctorReport(
|
||||
await resolveMemoryWikiStatus(params.config, {
|
||||
appConfig: params.appConfig,
|
||||
}),
|
||||
);
|
||||
if (!report.healthy) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
writeOutput(
|
||||
params.json ? JSON.stringify(report, null, 2) : renderMemoryWikiDoctor(report),
|
||||
params.stdout,
|
||||
);
|
||||
return report;
|
||||
}
|
||||
|
||||
export async function runWikiInit(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () => initializeMemoryWikiVault(params.config),
|
||||
render: (value) =>
|
||||
`Initialized wiki vault at ${value.rootDir} (${value.createdDirectories.length} dirs, ${value.createdFiles.length} files).`,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiCompile(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runSyncedWikiCommandWithSummary({
|
||||
config: params.config,
|
||||
appConfig: params.appConfig,
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () => compileMemoryWikiVault(params.config),
|
||||
render: (value) =>
|
||||
`Compiled wiki vault at ${value.vaultRoot} (${value.pages.length} pages, ${value.updatedFiles.length} indexes updated).`,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiLint(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runSyncedWikiCommandWithSummary({
|
||||
config: params.config,
|
||||
appConfig: params.appConfig,
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () => lintMemoryWikiVault(params.config),
|
||||
render: (value) =>
|
||||
`Linted wiki vault at ${value.vaultRoot} (${value.issueCount} issues, report: ${value.reportPath}).`,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiIngest(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
inputPath: string;
|
||||
title?: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () =>
|
||||
ingestMemoryWikiSource({
|
||||
config: params.config,
|
||||
inputPath: params.inputPath,
|
||||
title: params.title,
|
||||
}),
|
||||
render: (value) =>
|
||||
`Ingested ${value.sourcePath} into ${value.pagePath}. Refreshed ${value.indexUpdatedFiles.length} index file${value.indexUpdatedFiles.length === 1 ? "" : "s"}.`,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiSearch(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
query: string;
|
||||
maxResults?: number;
|
||||
searchBackend?: ResolvedMemoryWikiConfig["search"]["backend"];
|
||||
searchCorpus?: ResolvedMemoryWikiConfig["search"]["corpus"];
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
|
||||
const results = await searchMemoryWiki({
|
||||
config: params.config,
|
||||
appConfig: params.appConfig,
|
||||
query: params.query,
|
||||
maxResults: params.maxResults,
|
||||
searchBackend: params.searchBackend,
|
||||
searchCorpus: params.searchCorpus,
|
||||
});
|
||||
const summary = params.json
|
||||
? JSON.stringify(results, null, 2)
|
||||
: results.length === 0
|
||||
? "No wiki or memory results."
|
||||
: results
|
||||
.map(
|
||||
(result, index) =>
|
||||
`${index + 1}. ${result.title} (${result.corpus}/${result.kind})\nPath: ${result.path}${typeof result.startLine === "number" && typeof result.endLine === "number" ? `\nLines: ${result.startLine}-${result.endLine}` : ""}${result.provenanceLabel ? `\nProvenance: ${result.provenanceLabel}` : ""}\nSnippet: ${result.snippet}`,
|
||||
)
|
||||
.join("\n\n");
|
||||
writeOutput(summary, params.stdout);
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function runWikiGet(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
lookup: string;
|
||||
fromLine?: number;
|
||||
lineCount?: number;
|
||||
searchBackend?: ResolvedMemoryWikiConfig["search"]["backend"];
|
||||
searchCorpus?: ResolvedMemoryWikiConfig["search"]["corpus"];
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
|
||||
const result = await getMemoryWikiPage({
|
||||
config: params.config,
|
||||
appConfig: params.appConfig,
|
||||
lookup: params.lookup,
|
||||
fromLine: params.fromLine,
|
||||
lineCount: params.lineCount,
|
||||
searchBackend: params.searchBackend,
|
||||
searchCorpus: params.searchCorpus,
|
||||
});
|
||||
const summary = params.json
|
||||
? JSON.stringify(result, null, 2)
|
||||
: (result?.content ?? `Wiki page not found: ${params.lookup}`);
|
||||
writeOutput(summary, params.stdout);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function runWikiApplySynthesis(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
title: string;
|
||||
body?: string;
|
||||
bodyFile?: string;
|
||||
sourceIds?: string[];
|
||||
contradictions?: string[];
|
||||
questions?: string[];
|
||||
confidence?: number;
|
||||
status?: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
const sourceIds = normalizeCliStringList(params.sourceIds);
|
||||
if (!sourceIds) {
|
||||
throw new Error("wiki apply synthesis requires at least one --source-id.");
|
||||
}
|
||||
const body = await resolveWikiApplyBody({ body: params.body, bodyFile: params.bodyFile });
|
||||
await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
|
||||
const result = await applyMemoryWikiMutation({
|
||||
config: params.config,
|
||||
mutation: {
|
||||
op: "create_synthesis",
|
||||
title: params.title,
|
||||
body,
|
||||
sourceIds,
|
||||
...(normalizeCliStringList(params.contradictions)
|
||||
? { contradictions: normalizeCliStringList(params.contradictions) }
|
||||
: {}),
|
||||
...(normalizeCliStringList(params.questions)
|
||||
? { questions: normalizeCliStringList(params.questions) }
|
||||
: {}),
|
||||
...(typeof params.confidence === "number" ? { confidence: params.confidence } : {}),
|
||||
...(params.status?.trim() ? { status: params.status.trim() } : {}),
|
||||
},
|
||||
});
|
||||
writeOutput(formatMemoryWikiMutationSummary(result, params.json), params.stdout);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function runWikiApplyMetadata(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
lookup: string;
|
||||
sourceIds?: string[];
|
||||
contradictions?: string[];
|
||||
questions?: string[];
|
||||
confidence?: number;
|
||||
clearConfidence?: boolean;
|
||||
status?: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
await syncMemoryWikiImportedSources({ config: params.config, appConfig: params.appConfig });
|
||||
const result = await applyMemoryWikiMutation({
|
||||
config: params.config,
|
||||
mutation: {
|
||||
op: "update_metadata",
|
||||
lookup: params.lookup,
|
||||
...(normalizeCliStringList(params.sourceIds)
|
||||
? { sourceIds: normalizeCliStringList(params.sourceIds) }
|
||||
: {}),
|
||||
...(normalizeCliStringList(params.contradictions)
|
||||
? { contradictions: normalizeCliStringList(params.contradictions) }
|
||||
: {}),
|
||||
...(normalizeCliStringList(params.questions)
|
||||
? { questions: normalizeCliStringList(params.questions) }
|
||||
: {}),
|
||||
...(params.clearConfidence
|
||||
? { confidence: null }
|
||||
: typeof params.confidence === "number"
|
||||
? { confidence: params.confidence }
|
||||
: {}),
|
||||
...(params.status?.trim() ? { status: params.status.trim() } : {}),
|
||||
},
|
||||
});
|
||||
writeOutput(formatMemoryWikiMutationSummary(result, params.json), params.stdout);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function runWikiBridgeImport(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () =>
|
||||
syncMemoryWikiImportedSources({
|
||||
config: params.config,
|
||||
appConfig: params.appConfig,
|
||||
}),
|
||||
render: (value) =>
|
||||
`Bridge import synced ${value.artifactCount} artifacts across ${value.workspaces} workspaces (${value.importedCount} new, ${value.updatedCount} updated, ${value.skippedCount} unchanged, ${value.removedCount} removed). Indexes ${value.indexesRefreshed ? `refreshed (${value.indexUpdatedFiles.length} files)` : `not refreshed (${value.indexRefreshReason})`}.`,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiUnsafeLocalImport(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () =>
|
||||
syncMemoryWikiImportedSources({
|
||||
config: params.config,
|
||||
appConfig: params.appConfig,
|
||||
}),
|
||||
render: (value) =>
|
||||
`Unsafe-local import synced ${value.artifactCount} artifacts (${value.importedCount} new, ${value.updatedCount} updated, ${value.skippedCount} unchanged, ${value.removedCount} removed). Indexes ${value.indexesRefreshed ? `refreshed (${value.indexUpdatedFiles.length} files)` : `not refreshed (${value.indexRefreshReason})`}.`,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiObsidianStatus(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () => probeObsidianCli(),
|
||||
render: (value) =>
|
||||
value.available
|
||||
? `Obsidian CLI available at ${value.command}`
|
||||
: "Obsidian CLI is not available on PATH.",
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiObsidianSearch(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
query: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () => runObsidianSearch({ config: params.config, query: params.query }),
|
||||
render: (value) => value.stdout.trim(),
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiObsidianOpenCli(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
vaultPath: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () => runObsidianOpen({ config: params.config, vaultPath: params.vaultPath }),
|
||||
render: (value) => value.stdout.trim() || "Opened in Obsidian.",
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiObsidianCommandCli(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
id: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () => runObsidianCommand({ config: params.config, id: params.id }),
|
||||
render: (value) => value.stdout.trim() || "Command sent to Obsidian.",
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiObsidianDailyCli(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () => runObsidianDaily({ config: params.config }),
|
||||
render: (value) => value.stdout.trim() || "Opened today's daily note.",
|
||||
});
|
||||
}
|
||||
|
||||
function formatChatGptImportSummary(result: ChatGptImportResult): string {
|
||||
if (result.dryRun) {
|
||||
return `ChatGPT import dry run scanned ${result.conversationCount} conversations (${result.createdCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged).`;
|
||||
}
|
||||
const runSuffix = result.runId ? ` Run id: ${result.runId}.` : "";
|
||||
return `ChatGPT import applied ${result.conversationCount} conversations (${result.createdCount} new, ${result.updatedCount} updated, ${result.skippedCount} unchanged). Refreshed ${result.indexUpdatedFiles.length} index file${result.indexUpdatedFiles.length === 1 ? "" : "s"}.${runSuffix}`;
|
||||
}
|
||||
|
||||
function formatChatGptRollbackSummary(result: ChatGptRollbackResult): string {
|
||||
if (result.alreadyRolledBack) {
|
||||
return `ChatGPT import run ${result.runId} was already rolled back.`;
|
||||
}
|
||||
return `Rolled back ChatGPT import run ${result.runId} (${result.removedCount} removed, ${result.restoredCount} restored). Refreshed ${result.indexUpdatedFiles.length} index file${result.indexUpdatedFiles.length === 1 ? "" : "s"}.`;
|
||||
}
|
||||
|
||||
export async function runWikiChatGptImport(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
exportPath: string;
|
||||
dryRun?: boolean;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () =>
|
||||
importChatGptConversations({
|
||||
config: params.config,
|
||||
exportPath: params.exportPath,
|
||||
dryRun: params.dryRun,
|
||||
}),
|
||||
render: formatChatGptImportSummary,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runWikiChatGptRollback(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
runId: string;
|
||||
json?: boolean;
|
||||
stdout?: Pick<NodeJS.WriteStream, "write">;
|
||||
}) {
|
||||
return runWikiCommandWithSummary({
|
||||
json: params.json,
|
||||
stdout: params.stdout,
|
||||
run: () =>
|
||||
rollbackChatGptImportRun({
|
||||
config: params.config,
|
||||
runId: params.runId,
|
||||
}),
|
||||
render: formatChatGptRollbackSummary,
|
||||
});
|
||||
}
|
||||
|
||||
export function registerWikiCli(
|
||||
program: Command,
|
||||
pluginConfig?: MemoryWikiPluginConfig | ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
) {
|
||||
const config = isResolvedMemoryWikiConfig(pluginConfig)
|
||||
? pluginConfig
|
||||
: resolveMemoryWikiConfig(pluginConfig);
|
||||
const wiki = program.command("wiki").description("Inspect and initialize the memory wiki vault");
|
||||
|
||||
wiki
|
||||
.command("status")
|
||||
.description("Show wiki vault status")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiStatusCommandOptions) => {
|
||||
await runWikiStatus({ config, appConfig, json: opts.json });
|
||||
});
|
||||
|
||||
wiki
|
||||
.command("doctor")
|
||||
.description("Audit wiki vault setup and report actionable fixes")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiDoctorCommandOptions) => {
|
||||
await runWikiDoctor({ config, appConfig, json: opts.json });
|
||||
});
|
||||
|
||||
wiki
|
||||
.command("init")
|
||||
.description("Initialize the wiki vault layout")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiInitCommandOptions) => {
|
||||
await runWikiInit({ config, json: opts.json });
|
||||
});
|
||||
|
||||
wiki
|
||||
.command("compile")
|
||||
.description("Refresh generated wiki indexes")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiCompileCommandOptions) => {
|
||||
await runWikiCompile({ config, appConfig, json: opts.json });
|
||||
});
|
||||
|
||||
wiki
|
||||
.command("lint")
|
||||
.description("Lint the wiki vault and write a report")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiLintCommandOptions) => {
|
||||
await runWikiLint({ config, appConfig, json: opts.json });
|
||||
});
|
||||
|
||||
wiki
|
||||
.command("ingest")
|
||||
.description("Ingest a local file into the wiki sources folder")
|
||||
.argument("<path>", "Local file path to ingest")
|
||||
.option("--title <title>", "Override the source title")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (inputPath: string, opts: WikiIngestCommandOptions) => {
|
||||
await runWikiIngest({ config, inputPath, title: opts.title, json: opts.json });
|
||||
});
|
||||
|
||||
addWikiSearchConfigOptions(
|
||||
wiki
|
||||
.command("search")
|
||||
.description("Search wiki pages and, when configured, the active memory corpus")
|
||||
.argument("<query>", "Search query")
|
||||
.option("--max-results <n>", "Maximum results", (value: string) => Number(value)),
|
||||
)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (query: string, opts: WikiSearchCommandOptions) => {
|
||||
await runWikiSearch({
|
||||
config,
|
||||
appConfig,
|
||||
query,
|
||||
maxResults: opts.maxResults,
|
||||
searchBackend: opts.backend,
|
||||
searchCorpus: opts.corpus,
|
||||
json: opts.json,
|
||||
});
|
||||
});
|
||||
|
||||
addWikiSearchConfigOptions(
|
||||
wiki
|
||||
.command("get")
|
||||
.description("Read a wiki page by id or relative path, with optional active-memory fallback")
|
||||
.argument("<lookup>", "Relative path or page id")
|
||||
.option("--from <n>", "Start line", (value: string) => Number(value))
|
||||
.option("--lines <n>", "Number of lines", (value: string) => Number(value)),
|
||||
)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (lookup: string, opts: WikiGetCommandOptions) => {
|
||||
await runWikiGet({
|
||||
config,
|
||||
appConfig,
|
||||
lookup,
|
||||
fromLine: opts.from,
|
||||
lineCount: opts.lines,
|
||||
searchBackend: opts.backend,
|
||||
searchCorpus: opts.corpus,
|
||||
json: opts.json,
|
||||
});
|
||||
});
|
||||
|
||||
const apply = wiki.command("apply").description("Apply narrow wiki mutations");
|
||||
addWikiApplyMutationOptions(
|
||||
apply
|
||||
.command("synthesis")
|
||||
.description("Create or refresh a synthesis page with managed summary content")
|
||||
.argument("<title>", "Synthesis title")
|
||||
.option("--body <text>", "Summary body text")
|
||||
.option("--body-file <path>", "Read summary body text from a file"),
|
||||
)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (title: string, opts: WikiApplySynthesisCommandOptions) => {
|
||||
await runWikiApplySynthesis({
|
||||
config,
|
||||
appConfig,
|
||||
title,
|
||||
body: opts.body,
|
||||
bodyFile: opts.bodyFile,
|
||||
sourceIds: opts.sourceId,
|
||||
contradictions: opts.contradiction,
|
||||
questions: opts.question,
|
||||
confidence: opts.confidence,
|
||||
status: opts.status,
|
||||
json: opts.json,
|
||||
});
|
||||
});
|
||||
addWikiApplyMutationOptions(
|
||||
apply
|
||||
.command("metadata")
|
||||
.description("Update metadata on an existing page")
|
||||
.argument("<lookup>", "Relative path or page id"),
|
||||
)
|
||||
.option("--clear-confidence", "Remove any stored confidence value")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (lookup: string, opts: WikiApplyMetadataCommandOptions) => {
|
||||
await runWikiApplyMetadata({
|
||||
config,
|
||||
appConfig,
|
||||
lookup,
|
||||
sourceIds: opts.sourceId,
|
||||
contradictions: opts.contradiction,
|
||||
questions: opts.question,
|
||||
confidence: opts.confidence,
|
||||
clearConfidence: opts.clearConfidence,
|
||||
status: opts.status,
|
||||
json: opts.json,
|
||||
});
|
||||
});
|
||||
|
||||
const bridge = wiki
|
||||
.command("bridge")
|
||||
.description("Import public memory artifacts into the wiki vault");
|
||||
bridge
|
||||
.command("import")
|
||||
.description("Sync bridge-backed memory artifacts into wiki source pages")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiBridgeImportCommandOptions) => {
|
||||
await runWikiBridgeImport({ config, appConfig, json: opts.json });
|
||||
});
|
||||
|
||||
const unsafeLocal = wiki
|
||||
.command("unsafe-local")
|
||||
.description("Import explicitly configured private local paths into wiki source pages");
|
||||
unsafeLocal
|
||||
.command("import")
|
||||
.description("Sync unsafe-local configured paths into wiki source pages")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiUnsafeLocalImportCommandOptions) => {
|
||||
await runWikiUnsafeLocalImport({ config, appConfig, json: opts.json });
|
||||
});
|
||||
|
||||
const chatgpt = wiki
|
||||
.command("chatgpt")
|
||||
.description("Import ChatGPT export history into wiki source pages");
|
||||
chatgpt
|
||||
.command("import")
|
||||
.description("Import a ChatGPT export into draft wiki source pages")
|
||||
.requiredOption("--export <path>", "ChatGPT export directory or conversations.json path")
|
||||
.option("--dry-run", "Preview changes without writing", false)
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiChatGptImportCommandOptions) => {
|
||||
await runWikiChatGptImport({
|
||||
config,
|
||||
exportPath: opts.export!,
|
||||
dryRun: opts.dryRun,
|
||||
json: opts.json,
|
||||
});
|
||||
});
|
||||
chatgpt
|
||||
.command("rollback")
|
||||
.description("Roll back a previously applied ChatGPT import run")
|
||||
.argument("<run-id>", "Import run id")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (runId: string, opts: WikiChatGptRollbackCommandOptions) => {
|
||||
await runWikiChatGptRollback({
|
||||
config,
|
||||
runId,
|
||||
json: opts.json,
|
||||
});
|
||||
});
|
||||
|
||||
const obsidian = wiki.command("obsidian").description("Run official Obsidian CLI helpers");
|
||||
obsidian
|
||||
.command("status")
|
||||
.description("Probe the Obsidian CLI")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiStatusCommandOptions) => {
|
||||
await runWikiObsidianStatus({ config, json: opts.json });
|
||||
});
|
||||
obsidian
|
||||
.command("search")
|
||||
.description("Search the current Obsidian vault")
|
||||
.argument("<query>", "Search query")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (query: string, opts: WikiObsidianSearchCommandOptions) => {
|
||||
await runWikiObsidianSearch({ config, query, json: opts.json });
|
||||
});
|
||||
obsidian
|
||||
.command("open")
|
||||
.description("Open a file in Obsidian by vault-relative path")
|
||||
.argument("<path>", "Vault-relative path")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (vaultPath: string, opts: WikiObsidianOpenCommandOptions) => {
|
||||
await runWikiObsidianOpenCli({ config, vaultPath, json: opts.json });
|
||||
});
|
||||
obsidian
|
||||
.command("command")
|
||||
.description("Execute an Obsidian command palette command by id")
|
||||
.argument("<id>", "Obsidian command id")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (id: string, opts: WikiObsidianCommandCommandOptions) => {
|
||||
await runWikiObsidianCommandCli({ config, id, json: opts.json });
|
||||
});
|
||||
obsidian
|
||||
.command("daily")
|
||||
.description("Open today's daily note in Obsidian")
|
||||
.option("--json", "Print JSON")
|
||||
.action(async (opts: WikiObsidianDailyCommandOptions) => {
|
||||
await runWikiObsidianDailyCli({ config, json: opts.json });
|
||||
});
|
||||
}
|
||||
350
openclaw/extensions/memory-wiki/src/compile.test.ts
Normal file
350
openclaw/extensions/memory-wiki/src/compile.test.ts
Normal file
|
|
@ -0,0 +1,350 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import { renderWikiMarkdown } from "./markdown.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("compileMemoryWikiVault", () => {
|
||||
let suiteRoot = "";
|
||||
let caseId = 0;
|
||||
|
||||
beforeAll(async () => {
|
||||
suiteRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-compile-suite-"));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (suiteRoot) {
|
||||
await fs.rm(suiteRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function nextCaseRoot() {
|
||||
return path.join(suiteRoot, `case-${caseId++}`);
|
||||
}
|
||||
|
||||
it("writes root and directory indexes for native markdown", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot(),
|
||||
initialize: true,
|
||||
});
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.alpha",
|
||||
title: "Alpha",
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.doc",
|
||||
text: "Alpha is the canonical source page.",
|
||||
status: "supported",
|
||||
evidence: [{ sourceId: "source.alpha", lines: "1-3" }],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await compileMemoryWikiVault(config);
|
||||
|
||||
expect(result.pageCounts.source).toBe(1);
|
||||
expect(result.claimCount).toBe(1);
|
||||
await expect(fs.readFile(path.join(rootDir, "index.md"), "utf8")).resolves.toContain(
|
||||
"[Alpha](sources/alpha.md)",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "index.md"), "utf8")).resolves.toContain(
|
||||
"- Claims: 1",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "sources", "index.md"), "utf8")).resolves.toContain(
|
||||
"[Alpha](sources/alpha.md)",
|
||||
);
|
||||
const agentDigest = JSON.parse(
|
||||
await fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"), "utf8"),
|
||||
) as {
|
||||
claimCount: number;
|
||||
pages: Array<{ path: string; claimCount: number; topClaims: Array<{ text: string }> }>;
|
||||
};
|
||||
expect(agentDigest.claimCount).toBe(1);
|
||||
expect(agentDigest.pages).toContainEqual(
|
||||
expect.objectContaining({
|
||||
path: "sources/alpha.md",
|
||||
claimCount: 1,
|
||||
topClaims: [expect.objectContaining({ text: "Alpha is the canonical source page." })],
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "claims.jsonl"), "utf8"),
|
||||
).resolves.toContain('"text":"Alpha is the canonical source page."');
|
||||
});
|
||||
|
||||
it("renders obsidian-friendly links when configured", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot(),
|
||||
initialize: true,
|
||||
config: {
|
||||
vault: { renderMode: "obsidian" },
|
||||
},
|
||||
});
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha" },
|
||||
body: "# Alpha\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await compileMemoryWikiVault(config);
|
||||
|
||||
await expect(fs.readFile(path.join(rootDir, "index.md"), "utf8")).resolves.toContain(
|
||||
"[[sources/alpha|Alpha]]",
|
||||
);
|
||||
});
|
||||
|
||||
it("writes related blocks from source ids and shared sources", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot(),
|
||||
initialize: true,
|
||||
});
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha" },
|
||||
body: "# Alpha\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "beta.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.beta",
|
||||
title: "Beta",
|
||||
sourceIds: ["source.alpha"],
|
||||
},
|
||||
body: "# Beta\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "concepts", "gamma.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "concept",
|
||||
id: "concept.gamma",
|
||||
title: "Gamma",
|
||||
sourceIds: ["source.alpha"],
|
||||
},
|
||||
body: "# Gamma\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await compileMemoryWikiVault(config);
|
||||
|
||||
await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain(
|
||||
"## Related",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain(
|
||||
"[Alpha](sources/alpha.md)",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain(
|
||||
"[Gamma](concepts/gamma.md)",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "sources", "alpha.md"), "utf8")).resolves.toContain(
|
||||
"[Beta](entities/beta.md)",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "sources", "alpha.md"), "utf8")).resolves.toContain(
|
||||
"[Gamma](concepts/gamma.md)",
|
||||
);
|
||||
});
|
||||
|
||||
it("writes dashboard report pages when createDashboards is enabled", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot(),
|
||||
initialize: true,
|
||||
});
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha",
|
||||
title: "Alpha",
|
||||
sourceIds: ["source.alpha"],
|
||||
questions: ["What changed after launch?"],
|
||||
contradictions: ["Conflicts with source.beta"],
|
||||
confidence: 0.3,
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.db",
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
status: "supported",
|
||||
confidence: 0.4,
|
||||
evidence: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "concepts", "alpha-db.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "concept",
|
||||
id: "concept.alpha.db",
|
||||
title: "Alpha DB",
|
||||
sourceIds: ["source.alpha"],
|
||||
updatedAt: "2025-10-01T00:00:00.000Z",
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.db",
|
||||
text: "Alpha uses MySQL for production writes.",
|
||||
status: "contested",
|
||||
confidence: 0.62,
|
||||
evidence: [
|
||||
{
|
||||
sourceId: "source.alpha",
|
||||
lines: "9-11",
|
||||
updatedAt: "2025-10-01T00:00:00.000Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha DB\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.alpha",
|
||||
title: "Alpha Source",
|
||||
updatedAt: "2026-01-01T00:00:00.000Z",
|
||||
},
|
||||
body: "# Alpha Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await compileMemoryWikiVault(config);
|
||||
|
||||
expect(result.pageCounts.report).toBeGreaterThanOrEqual(5);
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "reports", "open-questions.md"), "utf8"),
|
||||
).resolves.toContain("[Alpha](entities/alpha.md): What changed after launch?");
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "reports", "contradictions.md"), "utf8"),
|
||||
).resolves.toContain("Conflicts with source.beta: [Alpha](entities/alpha.md)");
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "reports", "contradictions.md"), "utf8"),
|
||||
).resolves.toContain("`claim.alpha.db`");
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "reports", "low-confidence.md"), "utf8"),
|
||||
).resolves.toContain("[Alpha](entities/alpha.md): confidence 0.30");
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "reports", "low-confidence.md"), "utf8"),
|
||||
).resolves.toContain("Alpha uses PostgreSQL for production writes.");
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "reports", "claim-health.md"), "utf8"),
|
||||
).resolves.toContain("Missing Evidence");
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "reports", "claim-health.md"), "utf8"),
|
||||
).resolves.toContain("Alpha uses PostgreSQL for production writes.");
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "reports", "stale-pages.md"), "utf8"),
|
||||
).resolves.toContain("[Alpha](entities/alpha.md): missing updatedAt");
|
||||
const agentDigest = JSON.parse(
|
||||
await fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"), "utf8"),
|
||||
) as {
|
||||
claimHealth: { missingEvidence: number; freshness: { unknown: number } };
|
||||
contradictionClusters: Array<{ key: string }>;
|
||||
};
|
||||
expect(agentDigest.claimHealth.missingEvidence).toBeGreaterThanOrEqual(1);
|
||||
expect(agentDigest.claimHealth.freshness.unknown).toBeGreaterThanOrEqual(1);
|
||||
expect(agentDigest.contradictionClusters).toContainEqual(
|
||||
expect.objectContaining({ key: "claim.alpha.db" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("skips dashboard report pages when createDashboards is disabled", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot(),
|
||||
initialize: true,
|
||||
config: {
|
||||
render: { createDashboards: false },
|
||||
},
|
||||
});
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha",
|
||||
title: "Alpha",
|
||||
sourceIds: ["source.alpha"],
|
||||
questions: ["What changed after launch?"],
|
||||
},
|
||||
body: "# Alpha\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await compileMemoryWikiVault(config);
|
||||
|
||||
await expect(fs.access(path.join(rootDir, "reports", "open-questions.md"))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("ignores generated related links when computing backlinks on repeated compile", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot(),
|
||||
initialize: true,
|
||||
});
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "beta.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "entity", id: "entity.beta", title: "Beta" },
|
||||
body: "# Beta\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "concepts", "gamma.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "concept", id: "concept.gamma", title: "Gamma" },
|
||||
body: "# Gamma\n\nSee [Beta](entities/beta.md).\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await compileMemoryWikiVault(config);
|
||||
const second = await compileMemoryWikiVault(config);
|
||||
|
||||
expect(second.updatedFiles).toEqual([]);
|
||||
await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain(
|
||||
"[Gamma](concepts/gamma.md)",
|
||||
);
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "concepts", "gamma.md"), "utf8"),
|
||||
).resolves.not.toContain("### Referenced By");
|
||||
});
|
||||
});
|
||||
1068
openclaw/extensions/memory-wiki/src/compile.ts
Normal file
1068
openclaw/extensions/memory-wiki/src/compile.ts
Normal file
File diff suppressed because it is too large
Load diff
82
openclaw/extensions/memory-wiki/src/config-compat.test.ts
Normal file
82
openclaw/extensions/memory-wiki/src/config-compat.test.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
import {
|
||||
legacyConfigRules,
|
||||
migrateMemoryWikiLegacyConfig,
|
||||
normalizeCompatibilityConfig,
|
||||
} from "./config-compat.js";
|
||||
|
||||
describe("memory-wiki config compatibility", () => {
|
||||
it("detects the legacy bridge artifact toggle", () => {
|
||||
expect(
|
||||
legacyConfigRules[0]?.match({
|
||||
readMemoryCore: true,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("migrates readMemoryCore to readMemoryArtifacts", () => {
|
||||
const config = {
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-wiki": {
|
||||
config: {
|
||||
bridge: {
|
||||
enabled: true,
|
||||
readMemoryCore: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const migration = migrateMemoryWikiLegacyConfig(config);
|
||||
|
||||
expect(migration?.changes).toEqual([
|
||||
"Moved plugins.entries.memory-wiki.config.bridge.readMemoryCore → plugins.entries.memory-wiki.config.bridge.readMemoryArtifacts.",
|
||||
]);
|
||||
expect(
|
||||
(
|
||||
migration?.config.plugins?.entries?.["memory-wiki"] as {
|
||||
config?: { bridge?: Record<string, unknown> };
|
||||
}
|
||||
).config?.bridge,
|
||||
).toEqual({
|
||||
enabled: true,
|
||||
readMemoryArtifacts: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the canonical bridge toggle when both keys are present", () => {
|
||||
const config = {
|
||||
plugins: {
|
||||
entries: {
|
||||
"memory-wiki": {
|
||||
config: {
|
||||
bridge: {
|
||||
readMemoryCore: false,
|
||||
readMemoryArtifacts: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
const migration = normalizeCompatibilityConfig({ cfg: config });
|
||||
|
||||
expect(migration.changes).toEqual([
|
||||
"Removed legacy plugins.entries.memory-wiki.config.bridge.readMemoryCore; kept explicit plugins.entries.memory-wiki.config.bridge.readMemoryArtifacts.",
|
||||
]);
|
||||
expect(
|
||||
(
|
||||
migration.config.plugins?.entries?.["memory-wiki"] as {
|
||||
config?: { bridge?: Record<string, unknown> };
|
||||
}
|
||||
).config?.bridge,
|
||||
).toEqual({
|
||||
readMemoryArtifacts: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
75
openclaw/extensions/memory-wiki/src/config-compat.ts
Normal file
75
openclaw/extensions/memory-wiki/src/config-compat.ts
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import type { OpenClawConfig } from "../api.js";
|
||||
|
||||
type LegacyConfigRule = {
|
||||
path: Array<string | number>;
|
||||
message: string;
|
||||
match: (value: unknown) => boolean;
|
||||
};
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null;
|
||||
}
|
||||
|
||||
function hasLegacyBridgeArtifactToggle(value: unknown): boolean {
|
||||
return Object.prototype.hasOwnProperty.call(asRecord(value) ?? {}, "readMemoryCore");
|
||||
}
|
||||
|
||||
export const legacyConfigRules: LegacyConfigRule[] = [
|
||||
{
|
||||
path: ["plugins", "entries", "memory-wiki", "config", "bridge"],
|
||||
message:
|
||||
'plugins.entries.memory-wiki.config.bridge.readMemoryCore is legacy; use plugins.entries.memory-wiki.config.bridge.readMemoryArtifacts. Run "openclaw doctor --fix".',
|
||||
match: hasLegacyBridgeArtifactToggle,
|
||||
},
|
||||
];
|
||||
|
||||
export function migrateMemoryWikiLegacyConfig(config: OpenClawConfig): {
|
||||
config: OpenClawConfig;
|
||||
changes: string[];
|
||||
} | null {
|
||||
const rawEntry = asRecord(config.plugins?.entries?.["memory-wiki"]);
|
||||
const rawPluginConfig = asRecord(rawEntry?.config);
|
||||
const rawBridge = asRecord(rawPluginConfig?.bridge);
|
||||
if (!rawBridge || !hasLegacyBridgeArtifactToggle(rawBridge)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const nextConfig = structuredClone(config);
|
||||
const nextPlugins = asRecord(nextConfig.plugins) ?? {};
|
||||
nextConfig.plugins = nextPlugins;
|
||||
const nextEntries = asRecord(nextPlugins.entries) ?? {};
|
||||
nextPlugins.entries = nextEntries;
|
||||
const nextEntry = asRecord(nextEntries["memory-wiki"]) ?? {};
|
||||
nextEntries["memory-wiki"] = nextEntry;
|
||||
const nextPluginConfig = asRecord(nextEntry.config) ?? {};
|
||||
nextEntry.config = nextPluginConfig;
|
||||
const nextBridge = asRecord(nextPluginConfig.bridge) ?? {};
|
||||
nextPluginConfig.bridge = nextBridge;
|
||||
|
||||
const legacyValue = nextBridge.readMemoryCore;
|
||||
const hasCanonical = Object.prototype.hasOwnProperty.call(nextBridge, "readMemoryArtifacts");
|
||||
if (!hasCanonical) {
|
||||
nextBridge.readMemoryArtifacts = legacyValue;
|
||||
}
|
||||
delete nextBridge.readMemoryCore;
|
||||
|
||||
return {
|
||||
config: nextConfig,
|
||||
changes: hasCanonical
|
||||
? [
|
||||
"Removed legacy plugins.entries.memory-wiki.config.bridge.readMemoryCore; kept explicit plugins.entries.memory-wiki.config.bridge.readMemoryArtifacts.",
|
||||
]
|
||||
: [
|
||||
"Moved plugins.entries.memory-wiki.config.bridge.readMemoryCore → plugins.entries.memory-wiki.config.bridge.readMemoryArtifacts.",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeCompatibilityConfig({ cfg }: { cfg: OpenClawConfig }): {
|
||||
config: OpenClawConfig;
|
||||
changes: string[];
|
||||
} {
|
||||
return migrateMemoryWikiLegacyConfig(cfg) ?? { config: cfg, changes: [] };
|
||||
}
|
||||
95
openclaw/extensions/memory-wiki/src/config.test.ts
Normal file
95
openclaw/extensions/memory-wiki/src/config.test.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import fs from "node:fs";
|
||||
import AjvPkg from "ajv";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
DEFAULT_WIKI_RENDER_MODE,
|
||||
DEFAULT_WIKI_SEARCH_BACKEND,
|
||||
DEFAULT_WIKI_SEARCH_CORPUS,
|
||||
DEFAULT_WIKI_VAULT_MODE,
|
||||
resolveDefaultMemoryWikiVaultPath,
|
||||
resolveMemoryWikiConfig,
|
||||
} from "./config.js";
|
||||
|
||||
function compileManifestConfigSchema() {
|
||||
const manifest = JSON.parse(
|
||||
fs.readFileSync(new URL("../openclaw.plugin.json", import.meta.url), "utf8"),
|
||||
) as { configSchema: Record<string, unknown> };
|
||||
const Ajv = AjvPkg as unknown as new (opts?: object) => import("ajv").default;
|
||||
const ajv = new Ajv({ allErrors: true, strict: false, useDefaults: true });
|
||||
return ajv.compile(manifest.configSchema);
|
||||
}
|
||||
|
||||
describe("resolveMemoryWikiConfig", () => {
|
||||
it("returns isolated defaults", () => {
|
||||
const config = resolveMemoryWikiConfig(undefined, { homedir: "/Users/tester" });
|
||||
|
||||
expect(config.vaultMode).toBe(DEFAULT_WIKI_VAULT_MODE);
|
||||
expect(config.vault.renderMode).toBe(DEFAULT_WIKI_RENDER_MODE);
|
||||
expect(config.vault.path).toBe(resolveDefaultMemoryWikiVaultPath("/Users/tester"));
|
||||
expect(config.search.backend).toBe(DEFAULT_WIKI_SEARCH_BACKEND);
|
||||
expect(config.search.corpus).toBe(DEFAULT_WIKI_SEARCH_CORPUS);
|
||||
expect(config.context.includeCompiledDigestPrompt).toBe(false);
|
||||
});
|
||||
|
||||
it("expands ~/ paths and preserves explicit modes", () => {
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{
|
||||
vaultMode: "bridge",
|
||||
vault: {
|
||||
path: "~/vaults/wiki",
|
||||
renderMode: "obsidian",
|
||||
},
|
||||
},
|
||||
{ homedir: "/Users/tester" },
|
||||
);
|
||||
|
||||
expect(config.vaultMode).toBe("bridge");
|
||||
expect(config.vault.path).toBe("/Users/tester/vaults/wiki");
|
||||
expect(config.vault.renderMode).toBe("obsidian");
|
||||
});
|
||||
|
||||
it("normalizes the bridge artifact toggle", () => {
|
||||
const canonical = resolveMemoryWikiConfig({
|
||||
bridge: {
|
||||
readMemoryArtifacts: false,
|
||||
},
|
||||
});
|
||||
|
||||
expect(canonical.bridge.readMemoryArtifacts).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("memory-wiki manifest config schema", () => {
|
||||
it("accepts the documented config shape", () => {
|
||||
const validate = compileManifestConfigSchema();
|
||||
const config = {
|
||||
vaultMode: "unsafe-local",
|
||||
vault: {
|
||||
path: "~/wiki",
|
||||
renderMode: "obsidian",
|
||||
},
|
||||
obsidian: {
|
||||
enabled: true,
|
||||
useOfficialCli: true,
|
||||
},
|
||||
bridge: {
|
||||
enabled: true,
|
||||
readMemoryArtifacts: true,
|
||||
followMemoryEvents: true,
|
||||
},
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: true,
|
||||
paths: ["extensions/memory-core/src"],
|
||||
},
|
||||
search: {
|
||||
backend: "shared",
|
||||
corpus: "all",
|
||||
},
|
||||
context: {
|
||||
includeCompiledDigestPrompt: true,
|
||||
},
|
||||
};
|
||||
|
||||
expect(validate(config)).toBe(true);
|
||||
});
|
||||
});
|
||||
258
openclaw/extensions/memory-wiki/src/config.ts
Normal file
258
openclaw/extensions/memory-wiki/src/config.ts
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { buildPluginConfigSchema, z, type OpenClawPluginConfigSchema } from "../api.js";
|
||||
|
||||
export const WIKI_VAULT_MODES = ["isolated", "bridge", "unsafe-local"] as const;
|
||||
export const WIKI_RENDER_MODES = ["native", "obsidian"] as const;
|
||||
export const WIKI_SEARCH_BACKENDS = ["shared", "local"] as const;
|
||||
export const WIKI_SEARCH_CORPORA = ["wiki", "memory", "all"] as const;
|
||||
|
||||
export type WikiVaultMode = (typeof WIKI_VAULT_MODES)[number];
|
||||
export type WikiRenderMode = (typeof WIKI_RENDER_MODES)[number];
|
||||
export type WikiSearchBackend = (typeof WIKI_SEARCH_BACKENDS)[number];
|
||||
export type WikiSearchCorpus = (typeof WIKI_SEARCH_CORPORA)[number];
|
||||
|
||||
export type MemoryWikiPluginConfig = {
|
||||
vaultMode?: WikiVaultMode;
|
||||
vault?: {
|
||||
path?: string;
|
||||
renderMode?: WikiRenderMode;
|
||||
};
|
||||
obsidian?: {
|
||||
enabled?: boolean;
|
||||
useOfficialCli?: boolean;
|
||||
vaultName?: string;
|
||||
openAfterWrites?: boolean;
|
||||
};
|
||||
bridge?: {
|
||||
enabled?: boolean;
|
||||
readMemoryArtifacts?: boolean;
|
||||
indexDreamReports?: boolean;
|
||||
indexDailyNotes?: boolean;
|
||||
indexMemoryRoot?: boolean;
|
||||
followMemoryEvents?: boolean;
|
||||
};
|
||||
unsafeLocal?: {
|
||||
allowPrivateMemoryCoreAccess?: boolean;
|
||||
paths?: string[];
|
||||
};
|
||||
ingest?: {
|
||||
autoCompile?: boolean;
|
||||
maxConcurrentJobs?: number;
|
||||
allowUrlIngest?: boolean;
|
||||
};
|
||||
search?: {
|
||||
backend?: WikiSearchBackend;
|
||||
corpus?: WikiSearchCorpus;
|
||||
};
|
||||
context?: {
|
||||
includeCompiledDigestPrompt?: boolean;
|
||||
};
|
||||
render?: {
|
||||
preserveHumanBlocks?: boolean;
|
||||
createBacklinks?: boolean;
|
||||
createDashboards?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
export type ResolvedMemoryWikiConfig = {
|
||||
vaultMode: WikiVaultMode;
|
||||
vault: {
|
||||
path: string;
|
||||
renderMode: WikiRenderMode;
|
||||
};
|
||||
obsidian: {
|
||||
enabled: boolean;
|
||||
useOfficialCli: boolean;
|
||||
vaultName?: string;
|
||||
openAfterWrites: boolean;
|
||||
};
|
||||
bridge: {
|
||||
enabled: boolean;
|
||||
readMemoryArtifacts: boolean;
|
||||
indexDreamReports: boolean;
|
||||
indexDailyNotes: boolean;
|
||||
indexMemoryRoot: boolean;
|
||||
followMemoryEvents: boolean;
|
||||
};
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: boolean;
|
||||
paths: string[];
|
||||
};
|
||||
ingest: {
|
||||
autoCompile: boolean;
|
||||
maxConcurrentJobs: number;
|
||||
allowUrlIngest: boolean;
|
||||
};
|
||||
search: {
|
||||
backend: WikiSearchBackend;
|
||||
corpus: WikiSearchCorpus;
|
||||
};
|
||||
context: {
|
||||
includeCompiledDigestPrompt: boolean;
|
||||
};
|
||||
render: {
|
||||
preserveHumanBlocks: boolean;
|
||||
createBacklinks: boolean;
|
||||
createDashboards: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
export const DEFAULT_WIKI_VAULT_MODE: WikiVaultMode = "isolated";
|
||||
export const DEFAULT_WIKI_RENDER_MODE: WikiRenderMode = "native";
|
||||
export const DEFAULT_WIKI_SEARCH_BACKEND: WikiSearchBackend = "shared";
|
||||
export const DEFAULT_WIKI_SEARCH_CORPUS: WikiSearchCorpus = "wiki";
|
||||
|
||||
const MemoryWikiConfigSource = z.strictObject({
|
||||
vaultMode: z.enum(WIKI_VAULT_MODES).optional(),
|
||||
vault: z
|
||||
.strictObject({
|
||||
path: z.string().optional(),
|
||||
renderMode: z.enum(WIKI_RENDER_MODES).optional(),
|
||||
})
|
||||
.optional(),
|
||||
obsidian: z
|
||||
.strictObject({
|
||||
enabled: z.boolean().optional(),
|
||||
useOfficialCli: z.boolean().optional(),
|
||||
vaultName: z.string().optional(),
|
||||
openAfterWrites: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
bridge: z
|
||||
.strictObject({
|
||||
enabled: z.boolean().optional(),
|
||||
readMemoryArtifacts: z.boolean().optional(),
|
||||
indexDreamReports: z.boolean().optional(),
|
||||
indexDailyNotes: z.boolean().optional(),
|
||||
indexMemoryRoot: z.boolean().optional(),
|
||||
followMemoryEvents: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
unsafeLocal: z
|
||||
.strictObject({
|
||||
allowPrivateMemoryCoreAccess: z.boolean().optional(),
|
||||
paths: z.array(z.string()).optional(),
|
||||
})
|
||||
.optional(),
|
||||
ingest: z
|
||||
.strictObject({
|
||||
autoCompile: z.boolean().optional(),
|
||||
maxConcurrentJobs: z.number().int().min(1).optional(),
|
||||
allowUrlIngest: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
search: z
|
||||
.strictObject({
|
||||
backend: z.enum(WIKI_SEARCH_BACKENDS).optional(),
|
||||
corpus: z.enum(WIKI_SEARCH_CORPORA).optional(),
|
||||
})
|
||||
.optional(),
|
||||
context: z
|
||||
.strictObject({
|
||||
includeCompiledDigestPrompt: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
render: z
|
||||
.strictObject({
|
||||
preserveHumanBlocks: z.boolean().optional(),
|
||||
createBacklinks: z.boolean().optional(),
|
||||
createDashboards: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
});
|
||||
|
||||
const memoryWikiConfigSchemaBase = buildPluginConfigSchema(MemoryWikiConfigSource, {
|
||||
safeParse(value: unknown) {
|
||||
if (value === undefined) {
|
||||
return { success: true, data: resolveMemoryWikiConfig(undefined) };
|
||||
}
|
||||
const result = MemoryWikiConfigSource.safeParse(value);
|
||||
if (result.success) {
|
||||
return { success: true, data: resolveMemoryWikiConfig(result.data) };
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
error: {
|
||||
issues: result.error.issues.map((issue) => ({
|
||||
path: issue.path.filter((segment): segment is string | number => {
|
||||
const kind = typeof segment;
|
||||
return kind === "string" || kind === "number";
|
||||
}),
|
||||
message: issue.message,
|
||||
})),
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
export const memoryWikiConfigSchema: OpenClawPluginConfigSchema = memoryWikiConfigSchemaBase;
|
||||
|
||||
function expandHomePath(inputPath: string, homedir: string): string {
|
||||
if (inputPath === "~") {
|
||||
return homedir;
|
||||
}
|
||||
if (inputPath.startsWith("~/")) {
|
||||
return path.join(homedir, inputPath.slice(2));
|
||||
}
|
||||
return inputPath;
|
||||
}
|
||||
|
||||
export function resolveDefaultMemoryWikiVaultPath(homedir = os.homedir()): string {
|
||||
return path.join(homedir, ".openclaw", "wiki", "main");
|
||||
}
|
||||
|
||||
export function resolveMemoryWikiConfig(
|
||||
config: MemoryWikiPluginConfig | undefined,
|
||||
options?: { homedir?: string },
|
||||
): ResolvedMemoryWikiConfig {
|
||||
const homedir = options?.homedir ?? os.homedir();
|
||||
const parsed = config ? MemoryWikiConfigSource.safeParse(config) : null;
|
||||
const safeConfig = parsed?.success ? parsed.data : (config ?? {});
|
||||
|
||||
return {
|
||||
vaultMode: safeConfig.vaultMode ?? DEFAULT_WIKI_VAULT_MODE,
|
||||
vault: {
|
||||
path: expandHomePath(
|
||||
safeConfig.vault?.path ?? resolveDefaultMemoryWikiVaultPath(homedir),
|
||||
homedir,
|
||||
),
|
||||
renderMode: safeConfig.vault?.renderMode ?? DEFAULT_WIKI_RENDER_MODE,
|
||||
},
|
||||
obsidian: {
|
||||
enabled: safeConfig.obsidian?.enabled ?? false,
|
||||
useOfficialCli: safeConfig.obsidian?.useOfficialCli ?? false,
|
||||
...(safeConfig.obsidian?.vaultName ? { vaultName: safeConfig.obsidian.vaultName } : {}),
|
||||
openAfterWrites: safeConfig.obsidian?.openAfterWrites ?? false,
|
||||
},
|
||||
bridge: {
|
||||
enabled: safeConfig.bridge?.enabled ?? false,
|
||||
readMemoryArtifacts: safeConfig.bridge?.readMemoryArtifacts ?? true,
|
||||
indexDreamReports: safeConfig.bridge?.indexDreamReports ?? true,
|
||||
indexDailyNotes: safeConfig.bridge?.indexDailyNotes ?? true,
|
||||
indexMemoryRoot: safeConfig.bridge?.indexMemoryRoot ?? true,
|
||||
followMemoryEvents: safeConfig.bridge?.followMemoryEvents ?? true,
|
||||
},
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: safeConfig.unsafeLocal?.allowPrivateMemoryCoreAccess ?? false,
|
||||
paths: safeConfig.unsafeLocal?.paths ?? [],
|
||||
},
|
||||
ingest: {
|
||||
autoCompile: safeConfig.ingest?.autoCompile ?? true,
|
||||
maxConcurrentJobs: safeConfig.ingest?.maxConcurrentJobs ?? 1,
|
||||
allowUrlIngest: safeConfig.ingest?.allowUrlIngest ?? true,
|
||||
},
|
||||
search: {
|
||||
backend: safeConfig.search?.backend ?? DEFAULT_WIKI_SEARCH_BACKEND,
|
||||
corpus: safeConfig.search?.corpus ?? DEFAULT_WIKI_SEARCH_CORPUS,
|
||||
},
|
||||
context: {
|
||||
includeCompiledDigestPrompt: safeConfig.context?.includeCompiledDigestPrompt ?? false,
|
||||
},
|
||||
render: {
|
||||
preserveHumanBlocks: safeConfig.render?.preserveHumanBlocks ?? true,
|
||||
createBacklinks: safeConfig.render?.createBacklinks ?? true,
|
||||
createDashboards: safeConfig.render?.createDashboards ?? true,
|
||||
},
|
||||
};
|
||||
}
|
||||
37
openclaw/extensions/memory-wiki/src/corpus-supplement.ts
Normal file
37
openclaw/extensions/memory-wiki/src/corpus-supplement.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
import type { OpenClawConfig } from "../api.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { getMemoryWikiPage, searchMemoryWiki } from "./query.js";
|
||||
|
||||
export function createWikiCorpusSupplement(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
}) {
|
||||
return {
|
||||
search: async (input: { query: string; maxResults?: number; agentSessionKey?: string }) =>
|
||||
await searchMemoryWiki({
|
||||
config: params.config,
|
||||
appConfig: params.appConfig,
|
||||
agentSessionKey: input.agentSessionKey,
|
||||
query: input.query,
|
||||
maxResults: input.maxResults,
|
||||
searchBackend: "local",
|
||||
searchCorpus: "wiki",
|
||||
}),
|
||||
get: async (input: {
|
||||
lookup: string;
|
||||
fromLine?: number;
|
||||
lineCount?: number;
|
||||
agentSessionKey?: string;
|
||||
}) =>
|
||||
await getMemoryWikiPage({
|
||||
config: params.config,
|
||||
appConfig: params.appConfig,
|
||||
agentSessionKey: input.agentSessionKey,
|
||||
lookup: input.lookup,
|
||||
fromLine: input.fromLine,
|
||||
lineCount: input.lineCount,
|
||||
searchBackend: "local",
|
||||
searchCorpus: "wiki",
|
||||
}),
|
||||
};
|
||||
}
|
||||
429
openclaw/extensions/memory-wiki/src/gateway.test.ts
Normal file
429
openclaw/extensions/memory-wiki/src/gateway.test.ts
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
applyMemoryWikiMutation,
|
||||
normalizeMemoryWikiMutationInput,
|
||||
type ApplyMemoryWikiMutation,
|
||||
} from "./apply.js";
|
||||
import { registerMemoryWikiGatewayMethods } from "./gateway.js";
|
||||
import { listMemoryWikiImportInsights } from "./import-insights.js";
|
||||
import { listMemoryWikiImportRuns } from "./import-runs.js";
|
||||
import { ingestMemoryWikiSource } from "./ingest.js";
|
||||
import { listMemoryWikiPalace } from "./memory-palace.js";
|
||||
import { searchMemoryWiki } from "./query.js";
|
||||
import { syncMemoryWikiImportedSources } from "./source-sync.js";
|
||||
import { resolveMemoryWikiStatus } from "./status.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
vi.mock("./apply.js", () => ({
|
||||
applyMemoryWikiMutation: vi.fn(),
|
||||
normalizeMemoryWikiMutationInput: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./compile.js", () => ({
|
||||
compileMemoryWikiVault: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./ingest.js", () => ({
|
||||
ingestMemoryWikiSource: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./import-insights.js", () => ({
|
||||
listMemoryWikiImportInsights: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./import-runs.js", () => ({
|
||||
listMemoryWikiImportRuns: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./lint.js", () => ({
|
||||
lintMemoryWikiVault: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./memory-palace.js", () => ({
|
||||
listMemoryWikiPalace: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./obsidian.js", () => ({
|
||||
probeObsidianCli: vi.fn(),
|
||||
runObsidianCommand: vi.fn(),
|
||||
runObsidianDaily: vi.fn(),
|
||||
runObsidianOpen: vi.fn(),
|
||||
runObsidianSearch: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./query.js", () => ({
|
||||
getMemoryWikiPage: vi.fn(),
|
||||
searchMemoryWiki: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./source-sync.js", () => ({
|
||||
syncMemoryWikiImportedSources: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./status.js", () => ({
|
||||
buildMemoryWikiDoctorReport: vi.fn(),
|
||||
resolveMemoryWikiStatus: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./vault.js", () => ({
|
||||
initializeMemoryWikiVault: vi.fn(),
|
||||
}));
|
||||
|
||||
const { createPluginApi, createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
function findGatewayHandler(
|
||||
registerGatewayMethod: ReturnType<typeof vi.fn>,
|
||||
method: string,
|
||||
):
|
||||
| ((ctx: {
|
||||
params: Record<string, unknown>;
|
||||
respond: (ok: boolean, payload?: unknown, error?: unknown) => void;
|
||||
}) => Promise<void>)
|
||||
| undefined {
|
||||
return registerGatewayMethod.mock.calls.find((call) => call[0] === method)?.[1];
|
||||
}
|
||||
|
||||
describe("memory-wiki gateway methods", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(syncMemoryWikiImportedSources).mockResolvedValue({
|
||||
importedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
removedCount: 0,
|
||||
artifactCount: 0,
|
||||
workspaces: 0,
|
||||
pagePaths: [],
|
||||
indexesRefreshed: false,
|
||||
indexUpdatedFiles: [],
|
||||
indexRefreshReason: "no-import-changes",
|
||||
});
|
||||
vi.mocked(resolveMemoryWikiStatus).mockResolvedValue({
|
||||
vaultMode: "isolated",
|
||||
vaultExists: true,
|
||||
} as never);
|
||||
vi.mocked(ingestMemoryWikiSource).mockResolvedValue({
|
||||
pagePath: "sources/alpha-notes.md",
|
||||
} as never);
|
||||
vi.mocked(listMemoryWikiImportRuns).mockResolvedValue({
|
||||
runs: [],
|
||||
totalRuns: 0,
|
||||
activeRuns: 0,
|
||||
rolledBackRuns: 0,
|
||||
} as never);
|
||||
vi.mocked(listMemoryWikiImportInsights).mockResolvedValue({
|
||||
sourceType: "chatgpt",
|
||||
totalItems: 0,
|
||||
totalClusters: 0,
|
||||
clusters: [],
|
||||
} as never);
|
||||
vi.mocked(listMemoryWikiPalace).mockResolvedValue({
|
||||
totalItems: 0,
|
||||
totalClaims: 0,
|
||||
totalQuestions: 0,
|
||||
totalContradictions: 0,
|
||||
clusters: [],
|
||||
} as never);
|
||||
vi.mocked(normalizeMemoryWikiMutationInput).mockReturnValue({
|
||||
op: "create_synthesis",
|
||||
title: "Gateway Alpha",
|
||||
body: "Gateway summary.",
|
||||
sourceIds: ["source.alpha"],
|
||||
} satisfies ApplyMemoryWikiMutation);
|
||||
vi.mocked(applyMemoryWikiMutation).mockResolvedValue({
|
||||
operation: "create_synthesis",
|
||||
pagePath: "syntheses/gateway-alpha.md",
|
||||
} as never);
|
||||
vi.mocked(searchMemoryWiki).mockResolvedValue({
|
||||
items: [],
|
||||
total: 0,
|
||||
} as never);
|
||||
});
|
||||
|
||||
it("returns wiki status over the gateway", async () => {
|
||||
const { config } = await createVault({ prefix: "memory-wiki-gateway-" });
|
||||
const { api, registerGatewayMethod } = createPluginApi();
|
||||
|
||||
registerMemoryWikiGatewayMethods({ api, config });
|
||||
const handler = findGatewayHandler(registerGatewayMethod, "wiki.status");
|
||||
if (!handler) {
|
||||
throw new Error("wiki.status handler missing");
|
||||
}
|
||||
const respond = vi.fn();
|
||||
|
||||
await handler({
|
||||
params: {},
|
||||
respond,
|
||||
});
|
||||
|
||||
expect(syncMemoryWikiImportedSources).toHaveBeenCalledWith({ config, appConfig: undefined });
|
||||
expect(resolveMemoryWikiStatus).toHaveBeenCalledWith(config, {
|
||||
appConfig: undefined,
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
vaultMode: "isolated",
|
||||
vaultExists: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns recent import runs over the gateway", async () => {
|
||||
const { config } = await createVault({ prefix: "memory-wiki-gateway-" });
|
||||
const { api, registerGatewayMethod } = createPluginApi();
|
||||
vi.mocked(listMemoryWikiImportRuns).mockResolvedValue({
|
||||
runs: [
|
||||
{
|
||||
runId: "chatgpt-abc123",
|
||||
importType: "chatgpt",
|
||||
appliedAt: "2026-04-10T10:00:00.000Z",
|
||||
exportPath: "/tmp/chatgpt",
|
||||
sourcePath: "/tmp/chatgpt/conversations.json",
|
||||
conversationCount: 12,
|
||||
createdCount: 4,
|
||||
updatedCount: 2,
|
||||
skippedCount: 6,
|
||||
status: "applied",
|
||||
pagePaths: ["sources/chatgpt-2026-04-10-alpha.md"],
|
||||
samplePaths: ["sources/chatgpt-2026-04-10-alpha.md"],
|
||||
},
|
||||
],
|
||||
totalRuns: 1,
|
||||
activeRuns: 1,
|
||||
rolledBackRuns: 0,
|
||||
} as never);
|
||||
|
||||
registerMemoryWikiGatewayMethods({ api, config });
|
||||
const handler = findGatewayHandler(registerGatewayMethod, "wiki.importRuns");
|
||||
if (!handler) {
|
||||
throw new Error("wiki.importRuns handler missing");
|
||||
}
|
||||
const respond = vi.fn();
|
||||
|
||||
await handler({
|
||||
params: {
|
||||
limit: 5,
|
||||
},
|
||||
respond,
|
||||
});
|
||||
|
||||
expect(listMemoryWikiImportRuns).toHaveBeenCalledWith(config, { limit: 5 });
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
totalRuns: 1,
|
||||
activeRuns: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns import insights over the gateway", async () => {
|
||||
const { config } = await createVault({ prefix: "memory-wiki-gateway-" });
|
||||
const { api, registerGatewayMethod } = createPluginApi();
|
||||
vi.mocked(listMemoryWikiImportInsights).mockResolvedValue({
|
||||
sourceType: "chatgpt",
|
||||
totalItems: 2,
|
||||
totalClusters: 1,
|
||||
clusters: [
|
||||
{
|
||||
key: "topic/travel",
|
||||
label: "Travel",
|
||||
itemCount: 2,
|
||||
highRiskCount: 1,
|
||||
withheldCount: 1,
|
||||
preferenceSignalCount: 0,
|
||||
updatedAt: "2026-04-10T10:00:00.000Z",
|
||||
items: [
|
||||
{
|
||||
pagePath: "sources/chatgpt-2026-04-10-alpha.md",
|
||||
title: "BA flight receipts process",
|
||||
riskLevel: "low",
|
||||
labels: ["domain/personal", "area/travel", "topic/travel"],
|
||||
topicKey: "topic/travel",
|
||||
topicLabel: "Travel",
|
||||
digestStatus: "available",
|
||||
firstUserLine: "how do i get receipts?",
|
||||
lastUserLine: "that option does not exist",
|
||||
preferenceSignals: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
} as never);
|
||||
|
||||
registerMemoryWikiGatewayMethods({ api, config });
|
||||
const handler = findGatewayHandler(registerGatewayMethod, "wiki.importInsights");
|
||||
if (!handler) {
|
||||
throw new Error("wiki.importInsights handler missing");
|
||||
}
|
||||
const respond = vi.fn();
|
||||
|
||||
await handler({
|
||||
params: {},
|
||||
respond,
|
||||
});
|
||||
|
||||
expect(syncMemoryWikiImportedSources).toHaveBeenCalledWith({ config, appConfig: undefined });
|
||||
expect(listMemoryWikiImportInsights).toHaveBeenCalledWith(config);
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
sourceType: "chatgpt",
|
||||
totalItems: 2,
|
||||
totalClusters: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns memory palace overview over the gateway", async () => {
|
||||
const { config } = await createVault({ prefix: "memory-wiki-gateway-" });
|
||||
const { api, registerGatewayMethod } = createPluginApi();
|
||||
vi.mocked(listMemoryWikiPalace).mockResolvedValue({
|
||||
totalItems: 3,
|
||||
totalClaims: 4,
|
||||
totalQuestions: 1,
|
||||
totalContradictions: 1,
|
||||
clusters: [
|
||||
{
|
||||
key: "synthesis",
|
||||
label: "Syntheses",
|
||||
itemCount: 1,
|
||||
claimCount: 2,
|
||||
questionCount: 1,
|
||||
contradictionCount: 0,
|
||||
items: [
|
||||
{
|
||||
pagePath: "syntheses/travel-system.md",
|
||||
title: "Travel system",
|
||||
kind: "synthesis",
|
||||
claimCount: 2,
|
||||
questionCount: 1,
|
||||
contradictionCount: 0,
|
||||
claims: ["prefers direct receipts"],
|
||||
questions: ["should this become a playbook?"],
|
||||
contradictions: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
} as never);
|
||||
|
||||
registerMemoryWikiGatewayMethods({ api, config });
|
||||
const handler = findGatewayHandler(registerGatewayMethod, "wiki.palace");
|
||||
if (!handler) {
|
||||
throw new Error("wiki.palace handler missing");
|
||||
}
|
||||
const respond = vi.fn();
|
||||
|
||||
await handler({
|
||||
params: {},
|
||||
respond,
|
||||
});
|
||||
|
||||
expect(syncMemoryWikiImportedSources).toHaveBeenCalledWith({ config, appConfig: undefined });
|
||||
expect(listMemoryWikiPalace).toHaveBeenCalledWith(config);
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
totalItems: 3,
|
||||
totalClaims: 4,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("validates required query params for wiki.search", async () => {
|
||||
const { config } = await createVault({ prefix: "memory-wiki-gateway-" });
|
||||
const { api, registerGatewayMethod } = createPluginApi();
|
||||
|
||||
registerMemoryWikiGatewayMethods({ api, config });
|
||||
const handler = findGatewayHandler(registerGatewayMethod, "wiki.search");
|
||||
if (!handler) {
|
||||
throw new Error("wiki.search handler missing");
|
||||
}
|
||||
const respond = vi.fn();
|
||||
|
||||
await handler({
|
||||
params: {},
|
||||
respond,
|
||||
});
|
||||
|
||||
expect(searchMemoryWiki).not.toHaveBeenCalled();
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({ message: "query is required." }),
|
||||
);
|
||||
});
|
||||
|
||||
it("forwards ingest requests over the gateway", async () => {
|
||||
const { config } = await createVault({ prefix: "memory-wiki-gateway-" });
|
||||
const { api, registerGatewayMethod } = createPluginApi();
|
||||
|
||||
registerMemoryWikiGatewayMethods({ api, config });
|
||||
const handler = findGatewayHandler(registerGatewayMethod, "wiki.ingest");
|
||||
if (!handler) {
|
||||
throw new Error("wiki.ingest handler missing");
|
||||
}
|
||||
const respond = vi.fn();
|
||||
|
||||
await handler({
|
||||
params: {
|
||||
inputPath: "/tmp/alpha-notes.txt",
|
||||
title: "Alpha",
|
||||
},
|
||||
respond,
|
||||
});
|
||||
|
||||
expect(ingestMemoryWikiSource).toHaveBeenCalledWith({
|
||||
config,
|
||||
inputPath: "/tmp/alpha-notes.txt",
|
||||
title: "Alpha",
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
pagePath: "sources/alpha-notes.md",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("applies wiki mutations over the gateway", async () => {
|
||||
const { config } = await createVault({ prefix: "memory-wiki-gateway-" });
|
||||
const { api, registerGatewayMethod } = createPluginApi();
|
||||
|
||||
registerMemoryWikiGatewayMethods({ api, config });
|
||||
const handler = findGatewayHandler(registerGatewayMethod, "wiki.apply");
|
||||
if (!handler) {
|
||||
throw new Error("wiki.apply handler missing");
|
||||
}
|
||||
const respond = vi.fn();
|
||||
const params = {
|
||||
op: "create_synthesis",
|
||||
title: "Gateway Alpha",
|
||||
body: "Gateway summary.",
|
||||
sourceIds: ["source.alpha"],
|
||||
};
|
||||
|
||||
await handler({
|
||||
params,
|
||||
respond,
|
||||
});
|
||||
|
||||
expect(normalizeMemoryWikiMutationInput).toHaveBeenCalledWith(params);
|
||||
expect(applyMemoryWikiMutation).toHaveBeenCalledWith({
|
||||
config,
|
||||
mutation: expect.objectContaining({
|
||||
op: "create_synthesis",
|
||||
title: "Gateway Alpha",
|
||||
}),
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
operation: "create_synthesis",
|
||||
pagePath: "syntheses/gateway-alpha.md",
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
408
openclaw/extensions/memory-wiki/src/gateway.ts
Normal file
408
openclaw/extensions/memory-wiki/src/gateway.ts
Normal file
|
|
@ -0,0 +1,408 @@
|
|||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import type { OpenClawConfig, OpenClawPluginApi } from "../api.js";
|
||||
import { applyMemoryWikiMutation, normalizeMemoryWikiMutationInput } from "./apply.js";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import {
|
||||
WIKI_SEARCH_BACKENDS,
|
||||
WIKI_SEARCH_CORPORA,
|
||||
type ResolvedMemoryWikiConfig,
|
||||
} from "./config.js";
|
||||
import { listMemoryWikiImportInsights } from "./import-insights.js";
|
||||
import { listMemoryWikiImportRuns } from "./import-runs.js";
|
||||
import { ingestMemoryWikiSource } from "./ingest.js";
|
||||
import { lintMemoryWikiVault } from "./lint.js";
|
||||
import { listMemoryWikiPalace } from "./memory-palace.js";
|
||||
import {
|
||||
probeObsidianCli,
|
||||
runObsidianCommand,
|
||||
runObsidianDaily,
|
||||
runObsidianOpen,
|
||||
runObsidianSearch,
|
||||
} from "./obsidian.js";
|
||||
import { getMemoryWikiPage, searchMemoryWiki } from "./query.js";
|
||||
import { syncMemoryWikiImportedSources } from "./source-sync.js";
|
||||
import { buildMemoryWikiDoctorReport, resolveMemoryWikiStatus } from "./status.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
const READ_SCOPE = "operator.read" as const;
|
||||
const WRITE_SCOPE = "operator.write" as const;
|
||||
type GatewayMethodContext = Parameters<
|
||||
Parameters<OpenClawPluginApi["registerGatewayMethod"]>[1]
|
||||
>[0];
|
||||
type GatewayRespond = GatewayMethodContext["respond"];
|
||||
|
||||
function readStringParam(params: Record<string, unknown>, key: string): string | undefined;
|
||||
function readStringParam(
|
||||
params: Record<string, unknown>,
|
||||
key: string,
|
||||
options: { required: true },
|
||||
): string;
|
||||
function readStringParam(
|
||||
params: Record<string, unknown>,
|
||||
key: string,
|
||||
options?: { required?: boolean },
|
||||
): string | undefined {
|
||||
const value = params[key];
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
return value.trim();
|
||||
}
|
||||
if (options?.required) {
|
||||
throw new Error(`${key} is required.`);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function readNumberParam(params: Record<string, unknown>, key: string): number | undefined {
|
||||
const value = params[key];
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed)) {
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function readEnumParam<T extends string>(
|
||||
params: Record<string, unknown>,
|
||||
key: string,
|
||||
allowed: readonly T[],
|
||||
): T | undefined {
|
||||
const value = readStringParam(params, key);
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
if ((allowed as readonly string[]).includes(value)) {
|
||||
return value as T;
|
||||
}
|
||||
throw new Error(`${key} must be one of: ${allowed.join(", ")}.`);
|
||||
}
|
||||
|
||||
function respondError(respond: GatewayRespond, error: unknown) {
|
||||
const message = formatErrorMessage(error);
|
||||
respond(false, undefined, { code: "internal_error", message });
|
||||
}
|
||||
|
||||
async function syncImportedSourcesIfNeeded(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
) {
|
||||
await syncMemoryWikiImportedSources({ config, appConfig });
|
||||
}
|
||||
|
||||
export function registerMemoryWikiGatewayMethods(params: {
|
||||
api: OpenClawPluginApi;
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
}) {
|
||||
const { api, config, appConfig } = params;
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.status",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
respond(
|
||||
true,
|
||||
await resolveMemoryWikiStatus(config, {
|
||||
appConfig,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.importRuns",
|
||||
async ({ params: requestParams, respond }) => {
|
||||
try {
|
||||
const limit = readNumberParam(requestParams, "limit");
|
||||
respond(true, await listMemoryWikiImportRuns(config, limit !== undefined ? { limit } : {}));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.importInsights",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
respond(true, await listMemoryWikiImportInsights(config));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.palace",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
respond(true, await listMemoryWikiPalace(config));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.init",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
respond(true, await initializeMemoryWikiVault(config));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.doctor",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const status = await resolveMemoryWikiStatus(config, {
|
||||
appConfig,
|
||||
});
|
||||
respond(true, buildMemoryWikiDoctorReport(status));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.compile",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
respond(true, await compileMemoryWikiVault(config));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.ingest",
|
||||
async ({ params: requestParams, respond }) => {
|
||||
try {
|
||||
const inputPath = readStringParam(requestParams, "inputPath", { required: true });
|
||||
const title = readStringParam(requestParams, "title");
|
||||
respond(
|
||||
true,
|
||||
await ingestMemoryWikiSource({
|
||||
config,
|
||||
inputPath,
|
||||
...(title ? { title } : {}),
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.lint",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
respond(true, await lintMemoryWikiVault(config));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.bridge.import",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
respond(
|
||||
true,
|
||||
await syncMemoryWikiImportedSources({
|
||||
config: { ...config, vaultMode: "bridge" },
|
||||
appConfig,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.unsafeLocal.import",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
respond(
|
||||
true,
|
||||
await syncMemoryWikiImportedSources({
|
||||
config: { ...config, vaultMode: "unsafe-local" },
|
||||
appConfig,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.search",
|
||||
async ({ params: requestParams, respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const query = readStringParam(requestParams, "query", { required: true });
|
||||
const maxResults = readNumberParam(requestParams, "maxResults");
|
||||
const searchBackend = readEnumParam(requestParams, "backend", WIKI_SEARCH_BACKENDS);
|
||||
const searchCorpus = readEnumParam(requestParams, "corpus", WIKI_SEARCH_CORPORA);
|
||||
respond(
|
||||
true,
|
||||
await searchMemoryWiki({
|
||||
config,
|
||||
appConfig,
|
||||
query,
|
||||
maxResults,
|
||||
searchBackend,
|
||||
searchCorpus,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.apply",
|
||||
async ({ params: requestParams, respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
respond(
|
||||
true,
|
||||
await applyMemoryWikiMutation({
|
||||
config,
|
||||
mutation: normalizeMemoryWikiMutationInput(requestParams),
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.get",
|
||||
async ({ params: requestParams, respond }) => {
|
||||
try {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const lookup = readStringParam(requestParams, "lookup", { required: true });
|
||||
const fromLine = readNumberParam(requestParams, "fromLine");
|
||||
const lineCount = readNumberParam(requestParams, "lineCount");
|
||||
const searchBackend = readEnumParam(requestParams, "backend", WIKI_SEARCH_BACKENDS);
|
||||
const searchCorpus = readEnumParam(requestParams, "corpus", WIKI_SEARCH_CORPORA);
|
||||
respond(
|
||||
true,
|
||||
await getMemoryWikiPage({
|
||||
config,
|
||||
appConfig,
|
||||
lookup,
|
||||
fromLine,
|
||||
lineCount,
|
||||
searchBackend,
|
||||
searchCorpus,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.obsidian.status",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
respond(true, await probeObsidianCli());
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.obsidian.search",
|
||||
async ({ params: requestParams, respond }) => {
|
||||
try {
|
||||
const query = readStringParam(requestParams, "query", { required: true });
|
||||
respond(true, await runObsidianSearch({ config, query }));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: READ_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.obsidian.open",
|
||||
async ({ params: requestParams, respond }) => {
|
||||
try {
|
||||
const vaultPath = readStringParam(requestParams, "path", { required: true });
|
||||
respond(true, await runObsidianOpen({ config, vaultPath }));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.obsidian.command",
|
||||
async ({ params: requestParams, respond }) => {
|
||||
try {
|
||||
const id = readStringParam(requestParams, "id", { required: true });
|
||||
respond(true, await runObsidianCommand({ config, id }));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
|
||||
api.registerGatewayMethod(
|
||||
"wiki.obsidian.daily",
|
||||
async ({ respond }) => {
|
||||
try {
|
||||
respond(true, await runObsidianDaily({ config }));
|
||||
} catch (error) {
|
||||
respondError(respond, error);
|
||||
}
|
||||
},
|
||||
{ scope: WRITE_SCOPE },
|
||||
);
|
||||
}
|
||||
142
openclaw/extensions/memory-wiki/src/import-insights.test.ts
Normal file
142
openclaw/extensions/memory-wiki/src/import-insights.test.ts
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { listMemoryWikiImportInsights } from "./import-insights.js";
|
||||
import { renderWikiMarkdown } from "./markdown.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("listMemoryWikiImportInsights", () => {
|
||||
it("clusters ChatGPT import pages by topic and extracts digest fields", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-import-insights-",
|
||||
initialize: true,
|
||||
});
|
||||
await fs.mkdir(path.join(rootDir, "sources"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "chatgpt-travel.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.chatgpt.travel",
|
||||
title: "ChatGPT Export: BA flight receipts process",
|
||||
sourceType: "chatgpt-export",
|
||||
riskLevel: "low",
|
||||
riskReasons: [],
|
||||
labels: ["domain/personal", "area/travel", "topic/travel"],
|
||||
createdAt: "2026-01-11T14:07:58.552Z",
|
||||
updatedAt: "2026-01-11T14:08:45.377Z",
|
||||
},
|
||||
body: [
|
||||
"# ChatGPT Export: BA flight receipts process",
|
||||
"",
|
||||
"## Auto Digest",
|
||||
"- User messages: 2",
|
||||
"- Assistant messages: 2",
|
||||
"- First user line: how do i get receipts?",
|
||||
"- Last user line: that option does not exist",
|
||||
"- Preference signals:",
|
||||
" - prefers direct airline receipts",
|
||||
"",
|
||||
"## Active Branch Transcript",
|
||||
"### User",
|
||||
"",
|
||||
"how do i get receipts?",
|
||||
"",
|
||||
"### Assistant",
|
||||
"",
|
||||
"Try the BA receipt request flow first.",
|
||||
"",
|
||||
].join("\n"),
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "chatgpt-health.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.chatgpt.health",
|
||||
title: "ChatGPT Export: Migraine Medication Advice",
|
||||
sourceType: "chatgpt-export",
|
||||
riskLevel: "high",
|
||||
riskReasons: ["health"],
|
||||
labels: ["domain/personal", "area/health", "topic/health"],
|
||||
updatedAt: "2026-01-31T20:18:00.000Z",
|
||||
},
|
||||
body: [
|
||||
"# ChatGPT Export: Migraine Medication Advice",
|
||||
"",
|
||||
"## Auto Digest",
|
||||
"- Auto digest withheld from durable-candidate generation until reviewed.",
|
||||
"- Risk reasons: health content",
|
||||
"- First user line: i have a migraine, pink or yellow?",
|
||||
"- Last user line: should i take this now?",
|
||||
"- Preference signals:",
|
||||
" - prefers color-coded medication guidance",
|
||||
"",
|
||||
"## Active Branch Transcript",
|
||||
"### User",
|
||||
"",
|
||||
"i have a migraine, pink or yellow?",
|
||||
"",
|
||||
"### Assistant",
|
||||
"",
|
||||
"You're right, let's reset and stick to safe dosing guidance.",
|
||||
"",
|
||||
].join("\n"),
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await listMemoryWikiImportInsights(config);
|
||||
|
||||
expect(result.sourceType).toBe("chatgpt");
|
||||
expect(result.totalItems).toBe(2);
|
||||
expect(result.totalClusters).toBe(2);
|
||||
expect(result.clusters[0]).toMatchObject({
|
||||
key: "topic/health",
|
||||
label: "Health",
|
||||
itemCount: 1,
|
||||
highRiskCount: 1,
|
||||
withheldCount: 1,
|
||||
});
|
||||
expect(result.clusters[1]).toMatchObject({
|
||||
key: "topic/travel",
|
||||
label: "Travel",
|
||||
itemCount: 1,
|
||||
preferenceSignalCount: 1,
|
||||
});
|
||||
expect(result.clusters[1]?.items[0]).toMatchObject({
|
||||
title: "BA flight receipts process",
|
||||
riskReasons: [],
|
||||
activeBranchMessages: 0,
|
||||
userMessageCount: 2,
|
||||
assistantMessageCount: 2,
|
||||
firstUserLine: "how do i get receipts?",
|
||||
lastUserLine: "that option does not exist",
|
||||
assistantOpener: "Try the BA receipt request flow first.",
|
||||
summary: "Try the BA receipt request flow first.",
|
||||
candidateSignals: ["prefers direct airline receipts"],
|
||||
correctionSignals: [],
|
||||
preferenceSignals: ["prefers direct airline receipts"],
|
||||
digestStatus: "available",
|
||||
});
|
||||
const healthItem = result.clusters
|
||||
.flatMap((cluster) => cluster.items)
|
||||
.find((item) => item.title === "Migraine Medication Advice");
|
||||
expect(healthItem).toMatchObject({
|
||||
summary:
|
||||
"Sensitive health chat withheld from durable-memory extraction because it touches health.",
|
||||
candidateSignals: [],
|
||||
correctionSignals: [],
|
||||
preferenceSignals: [],
|
||||
userMessageCount: 1,
|
||||
assistantMessageCount: 1,
|
||||
});
|
||||
expect(healthItem?.firstUserLine).toBeUndefined();
|
||||
expect(healthItem?.lastUserLine).toBeUndefined();
|
||||
expect(healthItem?.assistantOpener).toBeUndefined();
|
||||
});
|
||||
});
|
||||
440
openclaw/extensions/memory-wiki/src/import-insights.ts
Normal file
440
openclaw/extensions/memory-wiki/src/import-insights.ts
Normal file
|
|
@ -0,0 +1,440 @@
|
|||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { parseWikiMarkdown } from "./markdown.js";
|
||||
import { readQueryableWikiPages } from "./query.js";
|
||||
|
||||
export type MemoryWikiImportInsightItem = {
|
||||
pagePath: string;
|
||||
title: string;
|
||||
riskLevel: "low" | "medium" | "high" | "unknown";
|
||||
riskReasons: string[];
|
||||
labels: string[];
|
||||
topicKey: string;
|
||||
topicLabel: string;
|
||||
digestStatus: "available" | "withheld";
|
||||
activeBranchMessages: number;
|
||||
userMessageCount: number;
|
||||
assistantMessageCount: number;
|
||||
firstUserLine?: string;
|
||||
lastUserLine?: string;
|
||||
assistantOpener?: string;
|
||||
summary: string;
|
||||
candidateSignals: string[];
|
||||
correctionSignals: string[];
|
||||
preferenceSignals: string[];
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
export type MemoryWikiImportInsightCluster = {
|
||||
key: string;
|
||||
label: string;
|
||||
itemCount: number;
|
||||
highRiskCount: number;
|
||||
withheldCount: number;
|
||||
preferenceSignalCount: number;
|
||||
updatedAt?: string;
|
||||
items: MemoryWikiImportInsightItem[];
|
||||
};
|
||||
|
||||
export type MemoryWikiImportInsightsStatus = {
|
||||
sourceType: "chatgpt";
|
||||
totalItems: number;
|
||||
totalClusters: number;
|
||||
clusters: MemoryWikiImportInsightCluster[];
|
||||
};
|
||||
|
||||
function normalizeStringArray(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.filter(
|
||||
(entry): entry is string => typeof entry === "string" && entry.trim().length > 0,
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeFiniteInt(value: unknown): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
return 0;
|
||||
}
|
||||
return Math.max(0, Math.floor(value));
|
||||
}
|
||||
|
||||
function normalizeTimestamp(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function humanizeLabelSuffix(label: string): string {
|
||||
const suffix = label.includes("/") ? label.split("/").slice(1).join("/") : label;
|
||||
return suffix
|
||||
.split(/[/-]/g)
|
||||
.filter((part) => part.length > 0)
|
||||
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
function resolveTopic(labels: string[]): { key: string; label: string } {
|
||||
const preferred =
|
||||
labels.find((label) => label.startsWith("topic/")) ??
|
||||
labels.find((label) => label.startsWith("area/")) ??
|
||||
labels.find((label) => label.startsWith("domain/")) ??
|
||||
"topic/other";
|
||||
return {
|
||||
key: preferred,
|
||||
label: humanizeLabelSuffix(preferred),
|
||||
};
|
||||
}
|
||||
|
||||
function extractHeadingSection(body: string, heading: string): string[] {
|
||||
const lines = body.split(/\r?\n/);
|
||||
const headingLine = `## ${heading}`;
|
||||
const startIndex = lines.findIndex((line) => line.trim() === headingLine);
|
||||
if (startIndex < 0) {
|
||||
return [];
|
||||
}
|
||||
const section: string[] = [];
|
||||
for (const line of lines.slice(startIndex + 1)) {
|
||||
if (line.startsWith("## ")) {
|
||||
break;
|
||||
}
|
||||
if (line.trim().length > 0) {
|
||||
section.push(line.trimEnd());
|
||||
}
|
||||
}
|
||||
return section;
|
||||
}
|
||||
|
||||
function extractDigestField(lines: string[], prefix: string): string | undefined {
|
||||
const needle = `- ${prefix}:`;
|
||||
const line = lines.find((entry) => entry.startsWith(needle));
|
||||
if (!line) {
|
||||
return undefined;
|
||||
}
|
||||
const value = line.slice(needle.length).trim();
|
||||
return value.length > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
function extractIntegerField(lines: string[], prefix: string): number {
|
||||
const raw = extractDigestField(lines, prefix);
|
||||
if (!raw) {
|
||||
return 0;
|
||||
}
|
||||
const match = raw.match(/\d+/);
|
||||
return match ? normalizeFiniteInt(Number(match[0])) : 0;
|
||||
}
|
||||
|
||||
function extractPreferenceSignals(lines: string[]): string[] {
|
||||
const startIndex = lines.findIndex((line) => line.startsWith("- Preference signals:"));
|
||||
if (startIndex < 0) {
|
||||
return [];
|
||||
}
|
||||
if (lines[startIndex]?.includes("none detected")) {
|
||||
return [];
|
||||
}
|
||||
const signals: string[] = [];
|
||||
for (const line of lines.slice(startIndex + 1)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed.startsWith("- ")) {
|
||||
break;
|
||||
}
|
||||
const signal = trimmed.slice(2).trim();
|
||||
if (signal.length > 0) {
|
||||
signals.push(signal);
|
||||
}
|
||||
}
|
||||
return signals;
|
||||
}
|
||||
|
||||
type TranscriptTurn = {
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
};
|
||||
|
||||
function parseTranscriptTurns(body: string): TranscriptTurn[] {
|
||||
const transcriptLines = extractHeadingSection(body, "Active Branch Transcript");
|
||||
if (transcriptLines.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const turns: TranscriptTurn[] = [];
|
||||
let currentRole: TranscriptTurn["role"] | null = null;
|
||||
let currentLines: string[] = [];
|
||||
|
||||
const flush = () => {
|
||||
if (!currentRole) {
|
||||
currentLines = [];
|
||||
return;
|
||||
}
|
||||
const text = currentLines.join("\n").trim();
|
||||
if (text) {
|
||||
turns.push({ role: currentRole, text });
|
||||
}
|
||||
currentLines = [];
|
||||
};
|
||||
|
||||
for (const rawLine of transcriptLines) {
|
||||
const line = rawLine.trimEnd();
|
||||
if (line.trim() === "### User") {
|
||||
flush();
|
||||
currentRole = "user";
|
||||
continue;
|
||||
}
|
||||
if (line.trim() === "### Assistant") {
|
||||
flush();
|
||||
currentRole = "assistant";
|
||||
continue;
|
||||
}
|
||||
if (currentRole) {
|
||||
currentLines.push(line);
|
||||
}
|
||||
}
|
||||
flush();
|
||||
return turns;
|
||||
}
|
||||
|
||||
function firstParagraph(text: string): string | undefined {
|
||||
const candidate = text
|
||||
.split(/\n\s*\n/)
|
||||
.map((entry) => entry.trim())
|
||||
.find((entry) => entry.length > 0);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function shortenSentence(value: string, maxLength = 180): string {
|
||||
const compact = value.replace(/\s+/g, " ").trim();
|
||||
if (compact.length <= maxLength) {
|
||||
return compact;
|
||||
}
|
||||
return `${compact.slice(0, maxLength - 1).trimEnd()}…`;
|
||||
}
|
||||
|
||||
function extractCorrectionSignals(turns: TranscriptTurn[]): string[] {
|
||||
const correctionPatterns = [
|
||||
"you're right",
|
||||
"you’re right",
|
||||
"bad assumption",
|
||||
"let's reset",
|
||||
"let’s reset",
|
||||
"does not exist anymore",
|
||||
"that was a bad assumption",
|
||||
"what actually works today",
|
||||
];
|
||||
return turns
|
||||
.filter((turn) => turn.role === "assistant")
|
||||
.flatMap((turn) => {
|
||||
const first = firstParagraph(turn.text);
|
||||
if (!first) {
|
||||
return [];
|
||||
}
|
||||
const normalized = first.toLowerCase();
|
||||
return correctionPatterns.some((pattern) => normalized.includes(pattern))
|
||||
? [shortenSentence(first, 160)]
|
||||
: [];
|
||||
})
|
||||
.slice(0, 2);
|
||||
}
|
||||
|
||||
function deriveCandidateSignals(params: {
|
||||
preferenceSignals: string[];
|
||||
correctionSignals: string[];
|
||||
}): string[] {
|
||||
const output: string[] = [];
|
||||
for (const signal of params.preferenceSignals) {
|
||||
if (!output.includes(signal)) {
|
||||
output.push(signal);
|
||||
}
|
||||
}
|
||||
for (const correction of params.correctionSignals) {
|
||||
const summary = `Correction detected: ${correction}`;
|
||||
if (!output.includes(summary)) {
|
||||
output.push(summary);
|
||||
}
|
||||
}
|
||||
return output.slice(0, 4);
|
||||
}
|
||||
|
||||
function deriveSummary(params: {
|
||||
title: string;
|
||||
digestStatus: "available" | "withheld";
|
||||
assistantOpener?: string;
|
||||
firstUserLine?: string;
|
||||
riskReasons: string[];
|
||||
topicLabel: string;
|
||||
}): string {
|
||||
if (params.digestStatus === "withheld") {
|
||||
if (params.riskReasons.length > 0) {
|
||||
return `Sensitive ${params.topicLabel.toLowerCase()} chat withheld from durable-memory extraction because it touches ${params.riskReasons.join(", ")}.`;
|
||||
}
|
||||
return `Sensitive ${params.topicLabel.toLowerCase()} chat withheld from durable-memory extraction pending review.`;
|
||||
}
|
||||
if (params.assistantOpener) {
|
||||
return shortenSentence(params.assistantOpener, 180);
|
||||
}
|
||||
if (params.firstUserLine) {
|
||||
return shortenSentence(params.firstUserLine, 180);
|
||||
}
|
||||
return params.title;
|
||||
}
|
||||
|
||||
function shouldExposeImportContent(digestStatus: "available" | "withheld"): boolean {
|
||||
return digestStatus === "available";
|
||||
}
|
||||
|
||||
function normalizeRiskLevel(value: unknown): MemoryWikiImportInsightItem["riskLevel"] {
|
||||
if (value === "low" || value === "medium" || value === "high") {
|
||||
return value;
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function compareItemsByUpdated(
|
||||
left: MemoryWikiImportInsightItem,
|
||||
right: MemoryWikiImportInsightItem,
|
||||
): number {
|
||||
const leftKey = left.updatedAt ?? left.createdAt ?? "";
|
||||
const rightKey = right.updatedAt ?? right.createdAt ?? "";
|
||||
if (rightKey !== leftKey) {
|
||||
return rightKey.localeCompare(leftKey);
|
||||
}
|
||||
return left.title.localeCompare(right.title);
|
||||
}
|
||||
|
||||
export async function listMemoryWikiImportInsights(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
): Promise<MemoryWikiImportInsightsStatus> {
|
||||
const pages = await readQueryableWikiPages(config.vault.path);
|
||||
const items = pages
|
||||
.flatMap((page) => {
|
||||
if (page.pageType !== "source") {
|
||||
return [];
|
||||
}
|
||||
const parsed = parseWikiMarkdown(page.raw);
|
||||
if (parsed.frontmatter.sourceType !== "chatgpt-export") {
|
||||
return [];
|
||||
}
|
||||
const labels = normalizeStringArray(parsed.frontmatter.labels);
|
||||
const topic = resolveTopic(labels);
|
||||
const triageLines = extractHeadingSection(parsed.body, "Auto Triage");
|
||||
const digestLines = extractHeadingSection(parsed.body, "Auto Digest");
|
||||
const transcriptTurns = parseTranscriptTurns(parsed.body);
|
||||
const digestStatus = digestLines.some((line) =>
|
||||
line.toLowerCase().includes("withheld from durable-candidate generation"),
|
||||
)
|
||||
? "withheld"
|
||||
: "available";
|
||||
const exposeImportContent = shouldExposeImportContent(digestStatus);
|
||||
const userTurns = transcriptTurns.filter((turn) => turn.role === "user");
|
||||
const assistantTurns = transcriptTurns.filter((turn) => turn.role === "assistant");
|
||||
const assistantOpener = exposeImportContent
|
||||
? firstParagraph(assistantTurns[0]?.text ?? "")
|
||||
: undefined;
|
||||
const correctionSignals = exposeImportContent
|
||||
? extractCorrectionSignals(transcriptTurns)
|
||||
: [];
|
||||
const preferenceSignals = exposeImportContent ? extractPreferenceSignals(digestLines) : [];
|
||||
const candidateSignals = exposeImportContent
|
||||
? deriveCandidateSignals({
|
||||
preferenceSignals,
|
||||
correctionSignals,
|
||||
})
|
||||
: [];
|
||||
const firstUserLine = exposeImportContent
|
||||
? extractDigestField(digestLines, "First user line")
|
||||
: undefined;
|
||||
const lastUserLine = exposeImportContent
|
||||
? extractDigestField(digestLines, "Last user line")
|
||||
: undefined;
|
||||
return [
|
||||
{
|
||||
pagePath: page.relativePath,
|
||||
title: page.title.replace(/^ChatGPT Export:\s*/i, ""),
|
||||
riskLevel: normalizeRiskLevel(parsed.frontmatter.riskLevel),
|
||||
riskReasons: normalizeStringArray(parsed.frontmatter.riskReasons),
|
||||
labels,
|
||||
topicKey: topic.key,
|
||||
topicLabel: topic.label,
|
||||
digestStatus,
|
||||
activeBranchMessages: extractIntegerField(triageLines, "Active-branch messages"),
|
||||
userMessageCount: Math.max(
|
||||
extractIntegerField(digestLines, "User messages"),
|
||||
userTurns.length,
|
||||
),
|
||||
assistantMessageCount: Math.max(
|
||||
extractIntegerField(digestLines, "Assistant messages"),
|
||||
assistantTurns.length,
|
||||
),
|
||||
...(firstUserLine ? { firstUserLine } : {}),
|
||||
...(lastUserLine ? { lastUserLine } : {}),
|
||||
...(assistantOpener ? { assistantOpener } : {}),
|
||||
summary: deriveSummary({
|
||||
title: page.title.replace(/^ChatGPT Export:\s*/i, ""),
|
||||
digestStatus,
|
||||
...(assistantOpener ? { assistantOpener } : {}),
|
||||
...(firstUserLine ? { firstUserLine } : {}),
|
||||
riskReasons: normalizeStringArray(parsed.frontmatter.riskReasons),
|
||||
topicLabel: topic.label,
|
||||
}),
|
||||
candidateSignals,
|
||||
correctionSignals,
|
||||
preferenceSignals,
|
||||
...(normalizeTimestamp(parsed.frontmatter.createdAt)
|
||||
? { createdAt: normalizeTimestamp(parsed.frontmatter.createdAt) }
|
||||
: {}),
|
||||
...(normalizeTimestamp(parsed.frontmatter.updatedAt)
|
||||
? { updatedAt: normalizeTimestamp(parsed.frontmatter.updatedAt) }
|
||||
: {}),
|
||||
} satisfies MemoryWikiImportInsightItem,
|
||||
];
|
||||
})
|
||||
.toSorted(compareItemsByUpdated);
|
||||
|
||||
const clustersByKey = new Map<string, MemoryWikiImportInsightItem[]>();
|
||||
for (const item of items) {
|
||||
const list = clustersByKey.get(item.topicKey) ?? [];
|
||||
list.push(item);
|
||||
clustersByKey.set(item.topicKey, list);
|
||||
}
|
||||
|
||||
const clusters = [...clustersByKey.entries()]
|
||||
.map(([key, clusterItems]) => {
|
||||
const sortedItems = [...clusterItems].toSorted(compareItemsByUpdated);
|
||||
const updatedAt = sortedItems
|
||||
.map((item) => item.updatedAt ?? item.createdAt)
|
||||
.find((value): value is string => typeof value === "string" && value.length > 0);
|
||||
return Object.assign(
|
||||
{
|
||||
key,
|
||||
label: sortedItems[0]?.topicLabel ?? humanizeLabelSuffix(key),
|
||||
itemCount: sortedItems.length,
|
||||
highRiskCount: sortedItems.filter((item) => item.riskLevel === `high`).length,
|
||||
withheldCount: sortedItems.filter((item) => item.digestStatus === `withheld`).length,
|
||||
preferenceSignalCount: sortedItems.reduce(
|
||||
(sum, item) => sum + item.preferenceSignals.length,
|
||||
0,
|
||||
),
|
||||
},
|
||||
updatedAt ? { updatedAt } : {},
|
||||
{ items: sortedItems },
|
||||
) satisfies MemoryWikiImportInsightCluster;
|
||||
})
|
||||
.toSorted((left, right) => {
|
||||
const leftKey = left.updatedAt ?? "";
|
||||
const rightKey = right.updatedAt ?? "";
|
||||
if (rightKey !== leftKey) {
|
||||
return rightKey.localeCompare(leftKey);
|
||||
}
|
||||
if (right.itemCount !== left.itemCount) {
|
||||
return right.itemCount - left.itemCount;
|
||||
}
|
||||
return left.label.localeCompare(right.label);
|
||||
});
|
||||
|
||||
return {
|
||||
sourceType: "chatgpt",
|
||||
totalItems: items.length,
|
||||
totalClusters: clusters.length,
|
||||
clusters,
|
||||
};
|
||||
}
|
||||
141
openclaw/extensions/memory-wiki/src/import-runs.ts
Normal file
141
openclaw/extensions/memory-wiki/src/import-runs.ts
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
|
||||
export type MemoryWikiImportRunSummary = {
|
||||
runId: string;
|
||||
importType: string;
|
||||
appliedAt: string;
|
||||
exportPath: string;
|
||||
sourcePath: string;
|
||||
conversationCount: number;
|
||||
createdCount: number;
|
||||
updatedCount: number;
|
||||
skippedCount: number;
|
||||
status: "applied" | "rolled_back";
|
||||
rolledBackAt?: string;
|
||||
pagePaths: string[];
|
||||
samplePaths: string[];
|
||||
};
|
||||
|
||||
export type MemoryWikiImportRunsStatus = {
|
||||
runs: MemoryWikiImportRunSummary[];
|
||||
totalRuns: number;
|
||||
activeRuns: number;
|
||||
rolledBackRuns: number;
|
||||
};
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function asStringArray(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.filter(
|
||||
(entry): entry is string => typeof entry === "string" && entry.trim().length > 0,
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeImportRunSummary(raw: unknown): MemoryWikiImportRunSummary | null {
|
||||
const record = asRecord(raw);
|
||||
if (!record) {
|
||||
return null;
|
||||
}
|
||||
const runId = typeof record?.runId === "string" ? record.runId.trim() : "";
|
||||
const importType = typeof record?.importType === "string" ? record.importType.trim() : "";
|
||||
const appliedAt = typeof record?.appliedAt === "string" ? record.appliedAt.trim() : "";
|
||||
const exportPath = typeof record?.exportPath === "string" ? record.exportPath.trim() : "";
|
||||
const sourcePath = typeof record?.sourcePath === "string" ? record.sourcePath.trim() : "";
|
||||
if (!runId || !importType || !appliedAt || !exportPath || !sourcePath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const createdPaths = asStringArray(record.createdPaths);
|
||||
const updatedPaths = Array.isArray(record.updatedPaths)
|
||||
? record.updatedPaths
|
||||
.map((entry) => asRecord(entry))
|
||||
.map((entry) => (typeof entry?.path === "string" ? entry.path.trim() : ""))
|
||||
.filter((entry): entry is string => entry.length > 0)
|
||||
: [];
|
||||
const pagePaths = [...new Set([...createdPaths, ...updatedPaths])];
|
||||
const conversationCount =
|
||||
typeof record.conversationCount === "number" && Number.isFinite(record.conversationCount)
|
||||
? Math.max(0, Math.floor(record.conversationCount))
|
||||
: createdPaths.length + updatedPaths.length;
|
||||
const createdCount =
|
||||
typeof record.createdCount === "number" && Number.isFinite(record.createdCount)
|
||||
? Math.max(0, Math.floor(record.createdCount))
|
||||
: createdPaths.length;
|
||||
const updatedCount =
|
||||
typeof record.updatedCount === "number" && Number.isFinite(record.updatedCount)
|
||||
? Math.max(0, Math.floor(record.updatedCount))
|
||||
: updatedPaths.length;
|
||||
const skippedCount =
|
||||
typeof record.skippedCount === "number" && Number.isFinite(record.skippedCount)
|
||||
? Math.max(0, Math.floor(record.skippedCount))
|
||||
: Math.max(0, conversationCount - createdCount - updatedCount);
|
||||
const rolledBackAt =
|
||||
typeof record.rolledBackAt === "string" && record.rolledBackAt.trim().length > 0
|
||||
? record.rolledBackAt.trim()
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
runId,
|
||||
importType,
|
||||
appliedAt,
|
||||
exportPath,
|
||||
sourcePath,
|
||||
conversationCount,
|
||||
createdCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
status: rolledBackAt ? "rolled_back" : "applied",
|
||||
...(rolledBackAt ? { rolledBackAt } : {}),
|
||||
pagePaths,
|
||||
samplePaths: pagePaths.slice(0, 5),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveImportRunsDir(vaultRoot: string): string {
|
||||
return path.join(vaultRoot, ".openclaw-wiki", "import-runs");
|
||||
}
|
||||
|
||||
export async function listMemoryWikiImportRuns(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
options?: { limit?: number },
|
||||
): Promise<MemoryWikiImportRunsStatus> {
|
||||
const limit = Math.max(1, Math.floor(options?.limit ?? 10));
|
||||
const importRunsDir = resolveImportRunsDir(config.vault.path);
|
||||
const entries = await fs
|
||||
.readdir(importRunsDir, { withFileTypes: true })
|
||||
.catch((error: NodeJS.ErrnoException) => {
|
||||
if (error?.code === "ENOENT") {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
const runs = (
|
||||
await Promise.all(
|
||||
entries
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
|
||||
.map(async (entry) => {
|
||||
const raw = await fs.readFile(path.join(importRunsDir, entry.name), "utf8");
|
||||
return normalizeImportRunSummary(JSON.parse(raw) as unknown);
|
||||
}),
|
||||
)
|
||||
)
|
||||
.filter((entry): entry is MemoryWikiImportRunSummary => entry !== null)
|
||||
.toSorted((left, right) => right.appliedAt.localeCompare(left.appliedAt));
|
||||
|
||||
return {
|
||||
runs: runs.slice(0, limit),
|
||||
totalRuns: runs.length,
|
||||
activeRuns: runs.filter((entry) => entry.status === "applied").length,
|
||||
rolledBackRuns: runs.filter((entry) => entry.status === "rolled_back").length,
|
||||
};
|
||||
}
|
||||
34
openclaw/extensions/memory-wiki/src/ingest.test.ts
Normal file
34
openclaw/extensions/memory-wiki/src/ingest.test.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ingestMemoryWikiSource } from "./ingest.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createTempDir, createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("ingestMemoryWikiSource", () => {
|
||||
it("copies a local text file into sources markdown", async () => {
|
||||
const rootDir = await createTempDir("memory-wiki-ingest-");
|
||||
const inputPath = path.join(rootDir, "meeting-notes.txt");
|
||||
await fs.writeFile(inputPath, "hello from source\n", "utf8");
|
||||
const { config } = await createVault({
|
||||
rootDir: path.join(rootDir, "vault"),
|
||||
});
|
||||
|
||||
const result = await ingestMemoryWikiSource({
|
||||
config,
|
||||
inputPath,
|
||||
nowMs: Date.UTC(2026, 3, 5, 12, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.pageId).toBe("source.meeting-notes");
|
||||
expect(result.pagePath).toBe("sources/meeting-notes.md");
|
||||
expect(result.indexUpdatedFiles.length).toBeGreaterThan(0);
|
||||
await expect(
|
||||
fs.readFile(path.join(config.vault.path, "sources", "meeting-notes.md"), "utf8"),
|
||||
).resolves.toContain("hello from source");
|
||||
await expect(fs.readFile(path.join(config.vault.path, "index.md"), "utf8")).resolves.toContain(
|
||||
"[meeting notes](sources/meeting-notes.md)",
|
||||
);
|
||||
});
|
||||
});
|
||||
112
openclaw/extensions/memory-wiki/src/ingest.ts
Normal file
112
openclaw/extensions/memory-wiki/src/ingest.ts
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
import { renderMarkdownFence, renderWikiMarkdown, slugifyWikiSegment } from "./markdown.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
export type IngestMemoryWikiSourceResult = {
|
||||
sourcePath: string;
|
||||
pageId: string;
|
||||
pagePath: string;
|
||||
title: string;
|
||||
bytes: number;
|
||||
created: boolean;
|
||||
indexUpdatedFiles: string[];
|
||||
};
|
||||
|
||||
function pathExists(filePath: string): Promise<boolean> {
|
||||
return fs
|
||||
.access(filePath)
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
}
|
||||
|
||||
function resolveSourceTitle(sourcePath: string, explicitTitle?: string): string {
|
||||
if (explicitTitle?.trim()) {
|
||||
return explicitTitle.trim();
|
||||
}
|
||||
return path.basename(sourcePath, path.extname(sourcePath)).replace(/[-_]+/g, " ").trim();
|
||||
}
|
||||
|
||||
function assertUtf8Text(buffer: Buffer, sourcePath: string): string {
|
||||
const preview = buffer.subarray(0, Math.min(buffer.length, 4096));
|
||||
if (preview.includes(0)) {
|
||||
throw new Error(`Cannot ingest binary file as markdown source: ${sourcePath}`);
|
||||
}
|
||||
return buffer.toString("utf8");
|
||||
}
|
||||
|
||||
export async function ingestMemoryWikiSource(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
inputPath: string;
|
||||
title?: string;
|
||||
nowMs?: number;
|
||||
}): Promise<IngestMemoryWikiSourceResult> {
|
||||
await initializeMemoryWikiVault(params.config, { nowMs: params.nowMs });
|
||||
const sourcePath = path.resolve(params.inputPath);
|
||||
const buffer = await fs.readFile(sourcePath);
|
||||
const content = assertUtf8Text(buffer, sourcePath);
|
||||
const title = resolveSourceTitle(sourcePath, params.title);
|
||||
const slug = slugifyWikiSegment(title);
|
||||
const pageId = `source.${slug}`;
|
||||
const pageRelativePath = path.join("sources", `${slug}.md`);
|
||||
const pagePath = path.join(params.config.vault.path, pageRelativePath);
|
||||
const created = !(await pathExists(pagePath));
|
||||
const timestamp = new Date(params.nowMs ?? Date.now()).toISOString();
|
||||
|
||||
const markdown = renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: pageId,
|
||||
title,
|
||||
sourceType: "local-file",
|
||||
sourcePath,
|
||||
ingestedAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
status: "active",
|
||||
},
|
||||
body: [
|
||||
`# ${title}`,
|
||||
"",
|
||||
"## Source",
|
||||
`- Type: \`local-file\``,
|
||||
`- Path: \`${sourcePath}\``,
|
||||
`- Bytes: ${buffer.byteLength}`,
|
||||
`- Updated: ${timestamp}`,
|
||||
"",
|
||||
"## Content",
|
||||
renderMarkdownFence(content, "text"),
|
||||
"",
|
||||
"## Notes",
|
||||
"<!-- openclaw:human:start -->",
|
||||
"<!-- openclaw:human:end -->",
|
||||
"",
|
||||
].join("\n"),
|
||||
});
|
||||
|
||||
await fs.writeFile(pagePath, markdown, "utf8");
|
||||
await appendMemoryWikiLog(params.config.vault.path, {
|
||||
type: "ingest",
|
||||
timestamp,
|
||||
details: {
|
||||
inputPath: sourcePath,
|
||||
pageId,
|
||||
pagePath: pageRelativePath.split(path.sep).join("/"),
|
||||
bytes: buffer.byteLength,
|
||||
created,
|
||||
},
|
||||
});
|
||||
const compile = await compileMemoryWikiVault(params.config);
|
||||
|
||||
return {
|
||||
sourcePath,
|
||||
pageId,
|
||||
pagePath: pageRelativePath.split(path.sep).join("/"),
|
||||
title,
|
||||
bytes: buffer.byteLength,
|
||||
created,
|
||||
indexUpdatedFiles: compile.updatedFiles,
|
||||
};
|
||||
}
|
||||
119
openclaw/extensions/memory-wiki/src/lint.test.ts
Normal file
119
openclaw/extensions/memory-wiki/src/lint.test.ts
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { lintMemoryWikiVault } from "./lint.js";
|
||||
import { renderWikiMarkdown } from "./markdown.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("lintMemoryWikiVault", () => {
|
||||
it("detects duplicate ids, provenance gaps, contradictions, and open questions", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-lint-",
|
||||
config: {
|
||||
vault: { renderMode: "obsidian" },
|
||||
},
|
||||
});
|
||||
await Promise.all(
|
||||
["entities", "concepts", "sources", "syntheses"].map((dir) =>
|
||||
fs.mkdir(path.join(rootDir, dir), { recursive: true }),
|
||||
),
|
||||
);
|
||||
|
||||
const duplicate = renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha",
|
||||
title: "Alpha",
|
||||
contradictions: ["Conflicts with source.beta"],
|
||||
questions: ["Is Alpha still active?"],
|
||||
confidence: 0.2,
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.db",
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
confidence: 0.2,
|
||||
evidence: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha\n\n[[missing-page]]\n",
|
||||
});
|
||||
await fs.writeFile(path.join(rootDir, "entities", "alpha.md"), duplicate, "utf8");
|
||||
await fs.writeFile(path.join(rootDir, "concepts", "alpha.md"), duplicate, "utf8");
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "bridge-alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.bridge.alpha",
|
||||
title: "Bridge Alpha",
|
||||
sourceType: "memory-bridge",
|
||||
},
|
||||
body: "# Bridge Alpha\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "syntheses", "alpha-db.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "synthesis",
|
||||
id: "synthesis.alpha.db",
|
||||
title: "Alpha Database",
|
||||
sourceIds: ["source.bridge.alpha"],
|
||||
updatedAt: "2025-10-01T00:00:00.000Z",
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.db",
|
||||
text: "Alpha uses MySQL for production writes.",
|
||||
status: "contested",
|
||||
confidence: 0.7,
|
||||
evidence: [
|
||||
{
|
||||
sourceId: "source.bridge.alpha",
|
||||
lines: "1-3",
|
||||
updatedAt: "2025-10-01T00:00:00.000Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha Database\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await lintMemoryWikiVault(config);
|
||||
|
||||
expect(result.issueCount).toBeGreaterThan(0);
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("duplicate-id");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("missing-source-ids");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("missing-import-provenance");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("broken-wikilink");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("contradiction-present");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("claim-conflict");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("open-question");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("low-confidence");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("claim-missing-evidence");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("claim-low-confidence");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("stale-page");
|
||||
expect(result.issues.map((issue) => issue.code)).toContain("stale-claim");
|
||||
expect(
|
||||
result.issuesByCategory.contradictions.some((issue) => issue.code === "claim-conflict"),
|
||||
).toBe(true);
|
||||
expect(result.issuesByCategory["open-questions"].length).toBeGreaterThanOrEqual(2);
|
||||
expect(
|
||||
result.issuesByCategory.provenance.some(
|
||||
(issue) => issue.code === "missing-import-provenance",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
result.issuesByCategory.provenance.some((issue) => issue.code === "claim-missing-evidence"),
|
||||
).toBe(true);
|
||||
await expect(fs.readFile(result.reportPath, "utf8")).resolves.toContain("### Errors");
|
||||
await expect(fs.readFile(result.reportPath, "utf8")).resolves.toContain("### Contradictions");
|
||||
await expect(fs.readFile(result.reportPath, "utf8")).resolves.toContain("### Open Questions");
|
||||
});
|
||||
});
|
||||
375
openclaw/extensions/memory-wiki/src/lint.ts
Normal file
375
openclaw/extensions/memory-wiki/src/lint.ts
Normal file
|
|
@ -0,0 +1,375 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
replaceManagedMarkdownBlock,
|
||||
withTrailingNewline,
|
||||
} from "openclaw/plugin-sdk/memory-host-markdown";
|
||||
import {
|
||||
assessPageFreshness,
|
||||
buildClaimContradictionClusters,
|
||||
collectWikiClaimHealth,
|
||||
} from "./claim-health.js";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
import { renderWikiMarkdown, type WikiPageSummary } from "./markdown.js";
|
||||
|
||||
export type MemoryWikiLintIssue = {
|
||||
severity: "error" | "warning";
|
||||
category: "structure" | "provenance" | "links" | "contradictions" | "open-questions" | "quality";
|
||||
code:
|
||||
| "missing-id"
|
||||
| "duplicate-id"
|
||||
| "missing-page-type"
|
||||
| "page-type-mismatch"
|
||||
| "missing-title"
|
||||
| "missing-source-ids"
|
||||
| "missing-import-provenance"
|
||||
| "broken-wikilink"
|
||||
| "contradiction-present"
|
||||
| "claim-conflict"
|
||||
| "open-question"
|
||||
| "low-confidence"
|
||||
| "claim-low-confidence"
|
||||
| "claim-missing-evidence"
|
||||
| "stale-page"
|
||||
| "stale-claim";
|
||||
path: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type LintMemoryWikiResult = {
|
||||
vaultRoot: string;
|
||||
issueCount: number;
|
||||
issues: MemoryWikiLintIssue[];
|
||||
issuesByCategory: Record<MemoryWikiLintIssue["category"], MemoryWikiLintIssue[]>;
|
||||
reportPath: string;
|
||||
};
|
||||
|
||||
function toExpectedPageType(page: WikiPageSummary): string {
|
||||
return page.kind;
|
||||
}
|
||||
|
||||
function collectBrokenLinkIssues(pages: WikiPageSummary[]): MemoryWikiLintIssue[] {
|
||||
const validTargets = new Set<string>();
|
||||
for (const page of pages) {
|
||||
const withoutExtension = page.relativePath.replace(/\.md$/i, "");
|
||||
validTargets.add(withoutExtension);
|
||||
validTargets.add(path.basename(withoutExtension));
|
||||
}
|
||||
|
||||
const issues: MemoryWikiLintIssue[] = [];
|
||||
for (const page of pages) {
|
||||
for (const linkTarget of page.linkTargets) {
|
||||
if (!validTargets.has(linkTarget)) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "links",
|
||||
code: "broken-wikilink",
|
||||
path: page.relativePath,
|
||||
message: `Broken wikilink target \`${linkTarget}\`.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return issues;
|
||||
}
|
||||
|
||||
function collectPageIssues(pages: WikiPageSummary[]): MemoryWikiLintIssue[] {
|
||||
const issues: MemoryWikiLintIssue[] = [];
|
||||
const pagesById = new Map<string, WikiPageSummary[]>();
|
||||
const claimHealth = collectWikiClaimHealth(pages);
|
||||
|
||||
for (const page of pages) {
|
||||
if (!page.id) {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
category: "structure",
|
||||
code: "missing-id",
|
||||
path: page.relativePath,
|
||||
message: "Missing `id` frontmatter.",
|
||||
});
|
||||
} else {
|
||||
const current = pagesById.get(page.id) ?? [];
|
||||
current.push(page);
|
||||
pagesById.set(page.id, current);
|
||||
}
|
||||
|
||||
if (!page.pageType) {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
category: "structure",
|
||||
code: "missing-page-type",
|
||||
path: page.relativePath,
|
||||
message: "Missing `pageType` frontmatter.",
|
||||
});
|
||||
} else if (page.pageType !== toExpectedPageType(page)) {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
category: "structure",
|
||||
code: "page-type-mismatch",
|
||||
path: page.relativePath,
|
||||
message: `Expected pageType \`${toExpectedPageType(page)}\`, found \`${page.pageType}\`.`,
|
||||
});
|
||||
}
|
||||
|
||||
if (!page.title.trim()) {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
category: "structure",
|
||||
code: "missing-title",
|
||||
path: page.relativePath,
|
||||
message: "Missing page title.",
|
||||
});
|
||||
}
|
||||
|
||||
if (page.kind !== "source" && page.kind !== "report" && page.sourceIds.length === 0) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "provenance",
|
||||
code: "missing-source-ids",
|
||||
path: page.relativePath,
|
||||
message: "Non-source page is missing `sourceIds` provenance.",
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
(page.sourceType === "memory-bridge" || page.sourceType === "memory-bridge-events") &&
|
||||
(!page.sourcePath || !page.bridgeRelativePath || !page.bridgeWorkspaceDir)
|
||||
) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "provenance",
|
||||
code: "missing-import-provenance",
|
||||
path: page.relativePath,
|
||||
message:
|
||||
"Bridge-imported source page is missing `sourcePath`, `bridgeRelativePath`, or `bridgeWorkspaceDir` provenance.",
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
(page.provenanceMode === "unsafe-local" || page.sourceType === "memory-unsafe-local") &&
|
||||
(!page.sourcePath || !page.unsafeLocalConfiguredPath || !page.unsafeLocalRelativePath)
|
||||
) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "provenance",
|
||||
code: "missing-import-provenance",
|
||||
path: page.relativePath,
|
||||
message:
|
||||
"Unsafe-local source page is missing `sourcePath`, `unsafeLocalConfiguredPath`, or `unsafeLocalRelativePath` provenance.",
|
||||
});
|
||||
}
|
||||
|
||||
if (page.contradictions.length > 0) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "contradictions",
|
||||
code: "contradiction-present",
|
||||
path: page.relativePath,
|
||||
message: `Page lists ${page.contradictions.length} contradiction${page.contradictions.length === 1 ? "" : "s"} to resolve.`,
|
||||
});
|
||||
}
|
||||
|
||||
if (page.questions.length > 0) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "open-questions",
|
||||
code: "open-question",
|
||||
path: page.relativePath,
|
||||
message: `Page lists ${page.questions.length} open question${page.questions.length === 1 ? "" : "s"}.`,
|
||||
});
|
||||
}
|
||||
|
||||
if (typeof page.confidence === "number" && page.confidence < 0.5) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "quality",
|
||||
code: "low-confidence",
|
||||
path: page.relativePath,
|
||||
message: `Page confidence is low (${page.confidence.toFixed(2)}).`,
|
||||
});
|
||||
}
|
||||
|
||||
const freshness = assessPageFreshness(page);
|
||||
if (page.kind !== "report" && (freshness.level === "stale" || freshness.level === "unknown")) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "quality",
|
||||
code: "stale-page",
|
||||
path: page.relativePath,
|
||||
message: `Page freshness needs review (${freshness.reason}).`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const claim of claimHealth) {
|
||||
if (claim.missingEvidence) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "provenance",
|
||||
code: "claim-missing-evidence",
|
||||
path: claim.pagePath,
|
||||
message: `Claim ${claim.claimId ? `\`${claim.claimId}\`` : `\`${claim.text}\``} is missing structured evidence.`,
|
||||
});
|
||||
}
|
||||
if (typeof claim.confidence === "number" && claim.confidence < 0.5) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "quality",
|
||||
code: "claim-low-confidence",
|
||||
path: claim.pagePath,
|
||||
message: `Claim ${claim.claimId ? `\`${claim.claimId}\`` : `\`${claim.text}\``} has low confidence (${claim.confidence.toFixed(2)}).`,
|
||||
});
|
||||
}
|
||||
if (claim.freshness.level === "stale" || claim.freshness.level === "unknown") {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "quality",
|
||||
code: "stale-claim",
|
||||
path: claim.pagePath,
|
||||
message: `Claim ${claim.claimId ? `\`${claim.claimId}\`` : `\`${claim.text}\``} freshness needs review (${claim.freshness.reason}).`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const cluster of buildClaimContradictionClusters({ pages })) {
|
||||
for (const entry of cluster.entries) {
|
||||
issues.push({
|
||||
severity: "warning",
|
||||
category: "contradictions",
|
||||
code: "claim-conflict",
|
||||
path: entry.pagePath,
|
||||
message: `Claim cluster \`${cluster.label}\` has competing variants across ${cluster.entries.length} pages.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const [id, matches] of pagesById.entries()) {
|
||||
if (matches.length > 1) {
|
||||
for (const match of matches) {
|
||||
issues.push({
|
||||
severity: "error",
|
||||
category: "structure",
|
||||
code: "duplicate-id",
|
||||
path: match.relativePath,
|
||||
message: `Duplicate page id \`${id}\`.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
issues.push(...collectBrokenLinkIssues(pages));
|
||||
return issues.toSorted((left, right) => left.path.localeCompare(right.path));
|
||||
}
|
||||
|
||||
function buildIssuesByCategory(
|
||||
issues: MemoryWikiLintIssue[],
|
||||
): Record<MemoryWikiLintIssue["category"], MemoryWikiLintIssue[]> {
|
||||
return {
|
||||
structure: issues.filter((issue) => issue.category === "structure"),
|
||||
provenance: issues.filter((issue) => issue.category === "provenance"),
|
||||
links: issues.filter((issue) => issue.category === "links"),
|
||||
contradictions: issues.filter((issue) => issue.category === "contradictions"),
|
||||
"open-questions": issues.filter((issue) => issue.category === "open-questions"),
|
||||
quality: issues.filter((issue) => issue.category === "quality"),
|
||||
};
|
||||
}
|
||||
|
||||
function buildLintReportBody(issues: MemoryWikiLintIssue[]): string {
|
||||
if (issues.length === 0) {
|
||||
return "No issues found.";
|
||||
}
|
||||
|
||||
const errors = issues.filter((issue) => issue.severity === "error");
|
||||
const warnings = issues.filter((issue) => issue.severity === "warning");
|
||||
const byCategory = buildIssuesByCategory(issues);
|
||||
const lines = [`- Errors: ${errors.length}`, `- Warnings: ${warnings.length}`];
|
||||
|
||||
if (errors.length > 0) {
|
||||
lines.push("", "### Errors");
|
||||
for (const issue of errors) {
|
||||
lines.push(`- \`${issue.path}\`: ${issue.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (warnings.length > 0) {
|
||||
lines.push("", "### Warnings");
|
||||
for (const issue of warnings) {
|
||||
lines.push(`- \`${issue.path}\`: ${issue.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (byCategory.contradictions.length > 0) {
|
||||
lines.push("", "### Contradictions");
|
||||
for (const issue of byCategory.contradictions) {
|
||||
lines.push(`- \`${issue.path}\`: ${issue.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (byCategory["open-questions"].length > 0) {
|
||||
lines.push("", "### Open Questions");
|
||||
for (const issue of byCategory["open-questions"]) {
|
||||
lines.push(`- \`${issue.path}\`: ${issue.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (byCategory.provenance.length > 0 || byCategory.quality.length > 0) {
|
||||
lines.push("", "### Quality Follow-Up");
|
||||
for (const issue of [...byCategory.provenance, ...byCategory.quality]) {
|
||||
lines.push(`- \`${issue.path}\`: ${issue.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
async function writeLintReport(rootDir: string, issues: MemoryWikiLintIssue[]): Promise<string> {
|
||||
const reportPath = path.join(rootDir, "reports", "lint.md");
|
||||
const original = await fs.readFile(reportPath, "utf8").catch(() =>
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "report",
|
||||
id: "report.lint",
|
||||
title: "Lint Report",
|
||||
status: "active",
|
||||
},
|
||||
body: "# Lint Report\n",
|
||||
}),
|
||||
);
|
||||
const updated = replaceManagedMarkdownBlock({
|
||||
original,
|
||||
heading: "## Generated",
|
||||
startMarker: "<!-- openclaw:wiki:lint:start -->",
|
||||
endMarker: "<!-- openclaw:wiki:lint:end -->",
|
||||
body: buildLintReportBody(issues),
|
||||
});
|
||||
await fs.writeFile(reportPath, withTrailingNewline(updated), "utf8");
|
||||
return reportPath;
|
||||
}
|
||||
|
||||
export async function lintMemoryWikiVault(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
): Promise<LintMemoryWikiResult> {
|
||||
const compileResult = await compileMemoryWikiVault(config);
|
||||
const issues = collectPageIssues(compileResult.pages);
|
||||
const issuesByCategory = buildIssuesByCategory(issues);
|
||||
const reportPath = await writeLintReport(config.vault.path, issues);
|
||||
|
||||
await appendMemoryWikiLog(config.vault.path, {
|
||||
type: "lint",
|
||||
timestamp: new Date().toISOString(),
|
||||
details: {
|
||||
issueCount: issues.length,
|
||||
reportPath: path.relative(config.vault.path, reportPath),
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
vaultRoot: config.vault.path,
|
||||
issueCount: issues.length,
|
||||
issues,
|
||||
issuesByCategory,
|
||||
reportPath,
|
||||
};
|
||||
}
|
||||
17
openclaw/extensions/memory-wiki/src/log.ts
Normal file
17
openclaw/extensions/memory-wiki/src/log.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export type MemoryWikiLogEntry = {
|
||||
type: "init" | "ingest" | "compile" | "lint";
|
||||
timestamp: string;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export async function appendMemoryWikiLog(
|
||||
vaultRoot: string,
|
||||
entry: MemoryWikiLogEntry,
|
||||
): Promise<void> {
|
||||
const logPath = path.join(vaultRoot, ".openclaw-wiki", "log.jsonl");
|
||||
await fs.mkdir(path.dirname(logPath), { recursive: true });
|
||||
await fs.appendFile(logPath, `${JSON.stringify(entry)}\n`, "utf8");
|
||||
}
|
||||
42
openclaw/extensions/memory-wiki/src/markdown.test.ts
Normal file
42
openclaw/extensions/memory-wiki/src/markdown.test.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createWikiPageFilename, slugifyWikiSegment } from "./markdown.js";
|
||||
|
||||
describe("slugifyWikiSegment", () => {
|
||||
it("preserves Unicode letters and numbers in wiki slugs", () => {
|
||||
expect(slugifyWikiSegment("大语言模型概述")).toBe("大语言模型概述");
|
||||
expect(slugifyWikiSegment("LLM 架构分析")).toBe("llm-架构分析");
|
||||
expect(slugifyWikiSegment("Circuit Breaker 自動恢復")).toBe("circuit-breaker-自動恢復");
|
||||
});
|
||||
|
||||
it("keeps ASCII behavior unchanged", () => {
|
||||
expect(slugifyWikiSegment("hello world")).toBe("hello-world");
|
||||
expect(slugifyWikiSegment("")).toBe("page");
|
||||
});
|
||||
|
||||
it("retains combining marks so distinct titles do not collapse", () => {
|
||||
expect(slugifyWikiSegment("किताब")).toBe("किताब");
|
||||
expect(slugifyWikiSegment("कुतुब")).toBe("कुतुब");
|
||||
expect(slugifyWikiSegment("कीताब")).toBe("कीताब");
|
||||
});
|
||||
|
||||
it("caps long Unicode slugs to a safe filename byte length", () => {
|
||||
const title = "漢".repeat(90);
|
||||
const slug = slugifyWikiSegment(title);
|
||||
|
||||
expect(slug.endsWith(`-${createHash("sha1").update(title).digest("hex").slice(0, 12)}`)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(Buffer.byteLength(slug)).toBeLessThanOrEqual(240);
|
||||
expect(slugifyWikiSegment(title)).toBe(slug);
|
||||
});
|
||||
|
||||
it("caps composed wiki page filenames to a safe path-component length", () => {
|
||||
const stem = `bridge-${"漢".repeat(45)}-${"語".repeat(45)}`;
|
||||
const fileName = createWikiPageFilename(stem);
|
||||
|
||||
expect(fileName.endsWith(".md")).toBe(true);
|
||||
expect(Buffer.byteLength(fileName)).toBeLessThanOrEqual(255);
|
||||
expect(createWikiPageFilename(stem)).toBe(fileName);
|
||||
});
|
||||
});
|
||||
321
openclaw/extensions/memory-wiki/src/markdown.ts
Normal file
321
openclaw/extensions/memory-wiki/src/markdown.ts
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
normalizeSingleOrTrimmedStringList,
|
||||
} from "openclaw/plugin-sdk/text-runtime";
|
||||
import YAML from "yaml";
|
||||
|
||||
export const WIKI_PAGE_KINDS = ["entity", "concept", "source", "synthesis", "report"] as const;
|
||||
export const WIKI_RELATED_START_MARKER = "<!-- openclaw:wiki:related:start -->";
|
||||
export const WIKI_RELATED_END_MARKER = "<!-- openclaw:wiki:related:end -->";
|
||||
|
||||
export type WikiPageKind = (typeof WIKI_PAGE_KINDS)[number];
|
||||
|
||||
export type ParsedWikiMarkdown = {
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
};
|
||||
|
||||
export type WikiClaimEvidence = {
|
||||
sourceId?: string;
|
||||
path?: string;
|
||||
lines?: string;
|
||||
weight?: number;
|
||||
note?: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
export type WikiClaim = {
|
||||
id?: string;
|
||||
text: string;
|
||||
status?: string;
|
||||
confidence?: number;
|
||||
evidence: WikiClaimEvidence[];
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
export type WikiPageSummary = {
|
||||
absolutePath: string;
|
||||
relativePath: string;
|
||||
kind: WikiPageKind;
|
||||
title: string;
|
||||
id?: string;
|
||||
pageType?: string;
|
||||
sourceIds: string[];
|
||||
linkTargets: string[];
|
||||
claims: WikiClaim[];
|
||||
contradictions: string[];
|
||||
questions: string[];
|
||||
confidence?: number;
|
||||
sourceType?: string;
|
||||
provenanceMode?: string;
|
||||
sourcePath?: string;
|
||||
bridgeRelativePath?: string;
|
||||
bridgeWorkspaceDir?: string;
|
||||
unsafeLocalConfiguredPath?: string;
|
||||
unsafeLocalRelativePath?: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
const FRONTMATTER_PATTERN = /^---\n([\s\S]*?)\n---\n?/;
|
||||
const OBSIDIAN_LINK_PATTERN = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
|
||||
const MARKDOWN_LINK_PATTERN = /\[[^\]]+\]\(([^)]+)\)/g;
|
||||
const RELATED_BLOCK_PATTERN = new RegExp(
|
||||
`${WIKI_RELATED_START_MARKER}[\\s\\S]*?${WIKI_RELATED_END_MARKER}`,
|
||||
"g",
|
||||
);
|
||||
const MAX_WIKI_SEGMENT_BYTES = 240;
|
||||
const MAX_WIKI_FILENAME_COMPONENT_BYTES = 255;
|
||||
const WIKI_SEGMENT_HASH_BYTES = 12;
|
||||
|
||||
function truncateUtf8CodePointSafe(value: string, maxBytes: number): string {
|
||||
let result = "";
|
||||
let bytes = 0;
|
||||
for (const char of value) {
|
||||
const nextBytes = Buffer.byteLength(char);
|
||||
if (bytes + nextBytes > maxBytes) {
|
||||
break;
|
||||
}
|
||||
result += char;
|
||||
bytes += nextBytes;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function capWikiValueWithHash(raw: string, maxBytes: number, fallback: string): string {
|
||||
if (Buffer.byteLength(raw) <= maxBytes) {
|
||||
return raw;
|
||||
}
|
||||
const suffix = createHash("sha1").update(raw).digest("hex").slice(0, WIKI_SEGMENT_HASH_BYTES);
|
||||
const truncated = truncateUtf8CodePointSafe(
|
||||
raw,
|
||||
maxBytes - Buffer.byteLength(`-${suffix}`),
|
||||
).replace(/-+$/g, "");
|
||||
return `${truncated || fallback}-${suffix}`;
|
||||
}
|
||||
|
||||
export function slugifyWikiSegment(raw: string): string {
|
||||
const slug = normalizeLowercaseStringOrEmpty(raw)
|
||||
.replace(/[^\p{L}\p{N}\p{M}]+/gu, "-")
|
||||
.replace(/-+/g, "-")
|
||||
.replace(/^-+|-+$/g, "");
|
||||
if (!slug) {
|
||||
return "page";
|
||||
}
|
||||
return capWikiValueWithHash(slug, MAX_WIKI_SEGMENT_BYTES, "page");
|
||||
}
|
||||
|
||||
export function createWikiPageFilename(stem: string, extension = ".md"): string {
|
||||
const normalizedExtension = extension.startsWith(".") ? extension : `.${extension}`;
|
||||
const maxStemBytes = Math.max(
|
||||
1,
|
||||
MAX_WIKI_FILENAME_COMPONENT_BYTES - Buffer.byteLength(normalizedExtension),
|
||||
);
|
||||
return `${capWikiValueWithHash(stem, maxStemBytes, "page")}${normalizedExtension}`;
|
||||
}
|
||||
|
||||
export function parseWikiMarkdown(content: string): ParsedWikiMarkdown {
|
||||
const match = content.match(FRONTMATTER_PATTERN);
|
||||
if (!match) {
|
||||
return { frontmatter: {}, body: content };
|
||||
}
|
||||
const parsed = YAML.parse(match[1]) as unknown;
|
||||
return {
|
||||
frontmatter:
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {},
|
||||
body: content.slice(match[0].length),
|
||||
};
|
||||
}
|
||||
|
||||
export function renderWikiMarkdown(params: {
|
||||
frontmatter: Record<string, unknown>;
|
||||
body: string;
|
||||
}): string {
|
||||
const frontmatter = YAML.stringify(params.frontmatter).trimEnd();
|
||||
return `---\n${frontmatter}\n---\n\n${params.body.trimStart()}`;
|
||||
}
|
||||
|
||||
export function extractTitleFromMarkdown(body: string): string | undefined {
|
||||
const match = body.match(/^#\s+(.+?)\s*$/m);
|
||||
return normalizeOptionalString(match?.[1]);
|
||||
}
|
||||
|
||||
export function normalizeSourceIds(value: unknown): string[] {
|
||||
return normalizeSingleOrTrimmedStringList(value);
|
||||
}
|
||||
|
||||
function normalizeWikiClaimEvidence(value: unknown): WikiClaimEvidence | null {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const sourceId = normalizeOptionalString(record.sourceId);
|
||||
const evidencePath = normalizeOptionalString(record.path);
|
||||
const lines = normalizeOptionalString(record.lines);
|
||||
const note = normalizeOptionalString(record.note);
|
||||
const updatedAt = normalizeOptionalString(record.updatedAt);
|
||||
const weight =
|
||||
typeof record.weight === "number" && Number.isFinite(record.weight) ? record.weight : undefined;
|
||||
if (!sourceId && !evidencePath && !lines && !note && weight === undefined && !updatedAt) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
...(sourceId ? { sourceId } : {}),
|
||||
...(evidencePath ? { path: evidencePath } : {}),
|
||||
...(lines ? { lines } : {}),
|
||||
...(weight !== undefined ? { weight } : {}),
|
||||
...(note ? { note } : {}),
|
||||
...(updatedAt ? { updatedAt } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeWikiClaims(value: unknown): WikiClaim[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.flatMap((entry) => {
|
||||
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
||||
return [];
|
||||
}
|
||||
const record = entry as Record<string, unknown>;
|
||||
const text = normalizeOptionalString(record.text);
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
const evidence = Array.isArray(record.evidence)
|
||||
? record.evidence.flatMap((candidate) => {
|
||||
const normalized = normalizeWikiClaimEvidence(candidate);
|
||||
return normalized ? [normalized] : [];
|
||||
})
|
||||
: [];
|
||||
const confidence =
|
||||
typeof record.confidence === "number" && Number.isFinite(record.confidence)
|
||||
? record.confidence
|
||||
: undefined;
|
||||
return [
|
||||
{
|
||||
...(normalizeOptionalString(record.id) ? { id: normalizeOptionalString(record.id) } : {}),
|
||||
text,
|
||||
...(normalizeOptionalString(record.status)
|
||||
? { status: normalizeOptionalString(record.status) }
|
||||
: {}),
|
||||
...(confidence !== undefined ? { confidence } : {}),
|
||||
evidence,
|
||||
...(normalizeOptionalString(record.updatedAt)
|
||||
? { updatedAt: normalizeOptionalString(record.updatedAt) }
|
||||
: {}),
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export function extractWikiLinks(markdown: string): string[] {
|
||||
const searchable = markdown.replace(RELATED_BLOCK_PATTERN, "");
|
||||
const links: string[] = [];
|
||||
for (const match of searchable.matchAll(OBSIDIAN_LINK_PATTERN)) {
|
||||
const target = match[1]?.trim();
|
||||
if (target) {
|
||||
links.push(target);
|
||||
}
|
||||
}
|
||||
for (const match of searchable.matchAll(MARKDOWN_LINK_PATTERN)) {
|
||||
const rawTarget = match[1]?.trim();
|
||||
if (!rawTarget || rawTarget.startsWith("#") || /^[a-z]+:/i.test(rawTarget)) {
|
||||
continue;
|
||||
}
|
||||
const target = rawTarget.split("#")[0]?.split("?")[0]?.replace(/\\/g, "/").trim();
|
||||
if (target) {
|
||||
links.push(target);
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
export function formatWikiLink(params: {
|
||||
renderMode: "native" | "obsidian";
|
||||
relativePath: string;
|
||||
title: string;
|
||||
}): string {
|
||||
const withoutExtension = params.relativePath.replace(/\.md$/i, "");
|
||||
return params.renderMode === "obsidian"
|
||||
? `[[${withoutExtension}|${params.title}]]`
|
||||
: `[${params.title}](${params.relativePath})`;
|
||||
}
|
||||
|
||||
export function renderMarkdownFence(content: string, infoString = "text"): string {
|
||||
const fenceSize = Math.max(
|
||||
3,
|
||||
...Array.from(content.matchAll(/`+/g), (match) => match[0].length + 1),
|
||||
);
|
||||
const fence = "`".repeat(fenceSize);
|
||||
return `${fence}${infoString}\n${content}\n${fence}`;
|
||||
}
|
||||
|
||||
export function inferWikiPageKind(relativePath: string): WikiPageKind | null {
|
||||
const normalized = relativePath.split(path.sep).join("/");
|
||||
if (normalized.startsWith("entities/")) {
|
||||
return "entity";
|
||||
}
|
||||
if (normalized.startsWith("concepts/")) {
|
||||
return "concept";
|
||||
}
|
||||
if (normalized.startsWith("sources/")) {
|
||||
return "source";
|
||||
}
|
||||
if (normalized.startsWith("syntheses/")) {
|
||||
return "synthesis";
|
||||
}
|
||||
if (normalized.startsWith("reports/")) {
|
||||
return "report";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function toWikiPageSummary(params: {
|
||||
absolutePath: string;
|
||||
relativePath: string;
|
||||
raw: string;
|
||||
}): WikiPageSummary | null {
|
||||
const kind = inferWikiPageKind(params.relativePath);
|
||||
if (!kind) {
|
||||
return null;
|
||||
}
|
||||
const parsed = parseWikiMarkdown(params.raw);
|
||||
const title =
|
||||
(typeof parsed.frontmatter.title === "string" && parsed.frontmatter.title.trim()) ||
|
||||
extractTitleFromMarkdown(parsed.body) ||
|
||||
path.basename(params.relativePath, ".md");
|
||||
|
||||
return {
|
||||
absolutePath: params.absolutePath,
|
||||
relativePath: params.relativePath.split(path.sep).join("/"),
|
||||
kind,
|
||||
title,
|
||||
id: normalizeOptionalString(parsed.frontmatter.id),
|
||||
pageType: normalizeOptionalString(parsed.frontmatter.pageType),
|
||||
sourceIds: normalizeSourceIds(parsed.frontmatter.sourceIds),
|
||||
linkTargets: extractWikiLinks(params.raw),
|
||||
claims: normalizeWikiClaims(parsed.frontmatter.claims),
|
||||
contradictions: normalizeSingleOrTrimmedStringList(parsed.frontmatter.contradictions),
|
||||
questions: normalizeSingleOrTrimmedStringList(parsed.frontmatter.questions),
|
||||
confidence:
|
||||
typeof parsed.frontmatter.confidence === "number" &&
|
||||
Number.isFinite(parsed.frontmatter.confidence)
|
||||
? parsed.frontmatter.confidence
|
||||
: undefined,
|
||||
sourceType: normalizeOptionalString(parsed.frontmatter.sourceType),
|
||||
provenanceMode: normalizeOptionalString(parsed.frontmatter.provenanceMode),
|
||||
sourcePath: normalizeOptionalString(parsed.frontmatter.sourcePath),
|
||||
bridgeRelativePath: normalizeOptionalString(parsed.frontmatter.bridgeRelativePath),
|
||||
bridgeWorkspaceDir: normalizeOptionalString(parsed.frontmatter.bridgeWorkspaceDir),
|
||||
unsafeLocalConfiguredPath: normalizeOptionalString(
|
||||
parsed.frontmatter.unsafeLocalConfiguredPath,
|
||||
),
|
||||
unsafeLocalRelativePath: normalizeOptionalString(parsed.frontmatter.unsafeLocalRelativePath),
|
||||
updatedAt: normalizeOptionalString(parsed.frontmatter.updatedAt),
|
||||
};
|
||||
}
|
||||
91
openclaw/extensions/memory-wiki/src/memory-palace.test.ts
Normal file
91
openclaw/extensions/memory-wiki/src/memory-palace.test.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { renderWikiMarkdown } from "./markdown.js";
|
||||
import { listMemoryWikiPalace } from "./memory-palace.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("listMemoryWikiPalace", () => {
|
||||
it("groups wiki pages by kind and surfaces claims, questions, and contradictions", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-palace-",
|
||||
initialize: true,
|
||||
});
|
||||
|
||||
await fs.mkdir(path.join(rootDir, "syntheses"), { recursive: true });
|
||||
await fs.mkdir(path.join(rootDir, "entities"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "syntheses", "travel-system.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "synthesis",
|
||||
id: "synthesis.travel.system",
|
||||
title: "Travel system",
|
||||
claims: [
|
||||
{ text: "Mariano prefers direct receipts from airlines when possible." },
|
||||
{ text: "Travel admin friction keeps showing up across chats." },
|
||||
],
|
||||
questions: ["Should flight receipts be standardized into one process?"],
|
||||
contradictions: ["Old BA receipts guidance may now be stale."],
|
||||
updatedAt: "2026-04-10T12:00:00.000Z",
|
||||
},
|
||||
body: [
|
||||
"# Travel system",
|
||||
"",
|
||||
"This synthesis rolls up recurring travel admin patterns from imported chats.",
|
||||
"",
|
||||
].join("\n"),
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "mariano.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.mariano",
|
||||
title: "Mariano",
|
||||
claims: [{ text: "He prefers compact, inspectable systems." }],
|
||||
updatedAt: "2026-04-09T08:00:00.000Z",
|
||||
},
|
||||
body: ["# Mariano", "", "Primary operator profile page.", ""].join("\n"),
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await listMemoryWikiPalace(config);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
totalItems: 2,
|
||||
totalClaims: 3,
|
||||
totalQuestions: 1,
|
||||
totalContradictions: 1,
|
||||
});
|
||||
expect(result.clusters[0]).toMatchObject({
|
||||
key: "synthesis",
|
||||
label: "Syntheses",
|
||||
itemCount: 1,
|
||||
claimCount: 2,
|
||||
questionCount: 1,
|
||||
contradictionCount: 1,
|
||||
});
|
||||
expect(result.clusters[0]?.items[0]).toMatchObject({
|
||||
title: "Travel system",
|
||||
claims: [
|
||||
"Mariano prefers direct receipts from airlines when possible.",
|
||||
"Travel admin friction keeps showing up across chats.",
|
||||
],
|
||||
questions: ["Should flight receipts be standardized into one process?"],
|
||||
contradictions: ["Old BA receipts guidance may now be stale."],
|
||||
snippet: "This synthesis rolls up recurring travel admin patterns from imported chats.",
|
||||
});
|
||||
expect(result.clusters[1]).toMatchObject({
|
||||
key: "entity",
|
||||
label: "Entities",
|
||||
itemCount: 1,
|
||||
claimCount: 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
148
openclaw/extensions/memory-wiki/src/memory-palace.ts
Normal file
148
openclaw/extensions/memory-wiki/src/memory-palace.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { parseWikiMarkdown, type WikiPageKind } from "./markdown.js";
|
||||
import { readQueryableWikiPages } from "./query.js";
|
||||
|
||||
const PALACE_KIND_ORDER: WikiPageKind[] = ["synthesis", "entity", "concept", "source", "report"];
|
||||
const PRIMARY_PALACE_KINDS = new Set<WikiPageKind>(["synthesis", "entity", "concept"]);
|
||||
const PALACE_KIND_LABELS: Record<WikiPageKind, string> = {
|
||||
synthesis: "Syntheses",
|
||||
entity: "Entities",
|
||||
concept: "Concepts",
|
||||
source: "Sources",
|
||||
report: "Reports",
|
||||
};
|
||||
|
||||
export type MemoryWikiPalaceItem = {
|
||||
pagePath: string;
|
||||
title: string;
|
||||
kind: WikiPageKind;
|
||||
id?: string;
|
||||
updatedAt?: string;
|
||||
sourceType?: string;
|
||||
claimCount: number;
|
||||
questionCount: number;
|
||||
contradictionCount: number;
|
||||
claims: string[];
|
||||
questions: string[];
|
||||
contradictions: string[];
|
||||
snippet?: string;
|
||||
};
|
||||
|
||||
export type MemoryWikiPalaceCluster = {
|
||||
key: WikiPageKind;
|
||||
label: string;
|
||||
itemCount: number;
|
||||
claimCount: number;
|
||||
questionCount: number;
|
||||
contradictionCount: number;
|
||||
updatedAt?: string;
|
||||
items: MemoryWikiPalaceItem[];
|
||||
};
|
||||
|
||||
export type MemoryWikiPalaceStatus = {
|
||||
totalItems: number;
|
||||
totalClaims: number;
|
||||
totalQuestions: number;
|
||||
totalContradictions: number;
|
||||
clusters: MemoryWikiPalaceCluster[];
|
||||
};
|
||||
|
||||
function normalizeTimestamp(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
return trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function extractSnippet(body: string): string | undefined {
|
||||
for (const rawLine of body.split(/\r?\n/)) {
|
||||
const line = rawLine.trim();
|
||||
if (
|
||||
!line ||
|
||||
line.startsWith("#") ||
|
||||
line.startsWith("```") ||
|
||||
line.startsWith("<!--") ||
|
||||
line.startsWith("- ") ||
|
||||
line.startsWith("* ")
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
return line;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function comparePalaceItems(left: MemoryWikiPalaceItem, right: MemoryWikiPalaceItem): number {
|
||||
const leftKey = left.updatedAt ?? "";
|
||||
const rightKey = right.updatedAt ?? "";
|
||||
if (rightKey !== leftKey) {
|
||||
return rightKey.localeCompare(leftKey);
|
||||
}
|
||||
if (right.claimCount !== left.claimCount) {
|
||||
return right.claimCount - left.claimCount;
|
||||
}
|
||||
return left.title.localeCompare(right.title);
|
||||
}
|
||||
|
||||
export async function listMemoryWikiPalace(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
): Promise<MemoryWikiPalaceStatus> {
|
||||
const pages = await readQueryableWikiPages(config.vault.path);
|
||||
const items = pages
|
||||
.map((page) => {
|
||||
const parsed = parseWikiMarkdown(page.raw);
|
||||
return Object.assign(
|
||||
{ pagePath: page.relativePath, title: page.title, kind: page.kind },
|
||||
page.id ? { id: page.id } : {},
|
||||
normalizeTimestamp(page.updatedAt) ? { updatedAt: normalizeTimestamp(page.updatedAt) } : {},
|
||||
typeof page.sourceType === `string` && page.sourceType.trim().length > 0
|
||||
? { sourceType: page.sourceType.trim() }
|
||||
: {},
|
||||
{
|
||||
claimCount: page.claims.length,
|
||||
questionCount: page.questions.length,
|
||||
contradictionCount: page.contradictions.length,
|
||||
claims: page.claims.map((claim) => claim.text).slice(0, 3),
|
||||
questions: page.questions.slice(0, 3),
|
||||
contradictions: page.contradictions.slice(0, 3),
|
||||
},
|
||||
extractSnippet(parsed.body) ? { snippet: extractSnippet(parsed.body) } : {},
|
||||
) satisfies MemoryWikiPalaceItem;
|
||||
})
|
||||
.filter(
|
||||
(item) =>
|
||||
PRIMARY_PALACE_KINDS.has(item.kind) ||
|
||||
item.claimCount > 0 ||
|
||||
item.questionCount > 0 ||
|
||||
item.contradictionCount > 0,
|
||||
)
|
||||
.toSorted(comparePalaceItems);
|
||||
|
||||
const clusters = PALACE_KIND_ORDER.map((kind) => {
|
||||
const clusterItems = items.filter((item) => item.kind === kind);
|
||||
if (clusterItems.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return Object.assign(
|
||||
{
|
||||
key: kind,
|
||||
label: PALACE_KIND_LABELS[kind],
|
||||
itemCount: clusterItems.length,
|
||||
claimCount: clusterItems.reduce((sum, item) => sum + item.claimCount, 0),
|
||||
questionCount: clusterItems.reduce((sum, item) => sum + item.questionCount, 0),
|
||||
contradictionCount: clusterItems.reduce((sum, item) => sum + item.contradictionCount, 0),
|
||||
},
|
||||
clusterItems[0]?.updatedAt ? { updatedAt: clusterItems[0].updatedAt } : {},
|
||||
{ items: clusterItems },
|
||||
) satisfies MemoryWikiPalaceCluster;
|
||||
}).filter((entry): entry is MemoryWikiPalaceCluster => entry !== null);
|
||||
|
||||
return {
|
||||
totalItems: items.length,
|
||||
totalClaims: items.reduce((sum, item) => sum + item.claimCount, 0),
|
||||
totalQuestions: items.reduce((sum, item) => sum + item.questionCount, 0),
|
||||
totalContradictions: items.reduce((sum, item) => sum + item.contradictionCount, 0),
|
||||
clusters,
|
||||
};
|
||||
}
|
||||
58
openclaw/extensions/memory-wiki/src/obsidian.test.ts
Normal file
58
openclaw/extensions/memory-wiki/src/obsidian.test.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { resolveMemoryWikiConfig } from "./config.js";
|
||||
import { runObsidianDaily, runObsidianSearch } from "./obsidian.js";
|
||||
|
||||
describe("runObsidianSearch", () => {
|
||||
it("builds the official obsidian cli argv with the configured vault name", async () => {
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{
|
||||
obsidian: {
|
||||
enabled: true,
|
||||
useOfficialCli: true,
|
||||
vaultName: "OpenClaw Wiki",
|
||||
},
|
||||
},
|
||||
{ homedir: "/Users/tester" },
|
||||
);
|
||||
const calls: Array<{ command: string; argv: string[] }> = [];
|
||||
const execImpl = async (command: string, argv?: readonly string[] | null) => {
|
||||
calls.push({ command, argv: argv ? [...argv] : [] });
|
||||
return { stdout: "search output\n", stderr: "" };
|
||||
};
|
||||
const exec = execImpl as unknown as NonNullable<
|
||||
NonNullable<Parameters<typeof runObsidianSearch>[0]["deps"]>["exec"]
|
||||
>;
|
||||
|
||||
const result = await runObsidianSearch({
|
||||
config,
|
||||
query: "agent memory",
|
||||
deps: {
|
||||
exec,
|
||||
resolveCommand: async () => "/usr/local/bin/obsidian",
|
||||
},
|
||||
});
|
||||
|
||||
expect(calls).toEqual([
|
||||
{
|
||||
command: "/usr/local/bin/obsidian",
|
||||
argv: ["vault=OpenClaw Wiki", "search", "query=agent memory"],
|
||||
},
|
||||
]);
|
||||
expect(result.stdout).toBe("search output\n");
|
||||
});
|
||||
});
|
||||
|
||||
describe("runObsidianDaily", () => {
|
||||
it("fails cleanly when the obsidian cli is not installed", async () => {
|
||||
const config = resolveMemoryWikiConfig(undefined, { homedir: "/Users/tester" });
|
||||
|
||||
await expect(
|
||||
runObsidianDaily({
|
||||
config,
|
||||
deps: {
|
||||
resolveCommand: async () => null,
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("Obsidian CLI is not available on PATH.");
|
||||
});
|
||||
});
|
||||
145
openclaw/extensions/memory-wiki/src/obsidian.ts
Normal file
145
openclaw/extensions/memory-wiki/src/obsidian.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export type ObsidianCliProbe = {
|
||||
available: boolean;
|
||||
command: string | null;
|
||||
};
|
||||
|
||||
export type ObsidianCliResult = {
|
||||
command: string;
|
||||
argv: string[];
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
};
|
||||
|
||||
type ObsidianCliDeps = {
|
||||
exec?: typeof execFileAsync;
|
||||
resolveCommand?: (command: string) => Promise<string | null>;
|
||||
};
|
||||
|
||||
async function isExecutableFile(inputPath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(inputPath, process.platform === "win32" ? fsConstants.F_OK : fsConstants.X_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveCommandOnPath(command: string): Promise<string | null> {
|
||||
const pathValue = process.env.PATH ?? "";
|
||||
const pathEntries = pathValue.split(path.delimiter).filter(Boolean);
|
||||
const windowsExts =
|
||||
process.platform === "win32"
|
||||
? (process.env.PATHEXT?.split(";").filter(Boolean) ?? [".EXE", ".CMD", ".BAT"])
|
||||
: [""];
|
||||
|
||||
if (command.includes(path.sep)) {
|
||||
return (await isExecutableFile(command)) ? command : null;
|
||||
}
|
||||
|
||||
for (const dir of pathEntries) {
|
||||
for (const extension of windowsExts) {
|
||||
const candidate = path.join(dir, extension ? `${command}${extension}` : command);
|
||||
if (await isExecutableFile(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function buildVaultPrefix(config: ResolvedMemoryWikiConfig): string[] {
|
||||
return config.obsidian.vaultName ? [`vault=${config.obsidian.vaultName}`] : [];
|
||||
}
|
||||
|
||||
export async function probeObsidianCli(
|
||||
deps?: Pick<ObsidianCliDeps, "resolveCommand">,
|
||||
): Promise<ObsidianCliProbe> {
|
||||
const resolveCommand = deps?.resolveCommand ?? resolveCommandOnPath;
|
||||
const command = await resolveCommand("obsidian");
|
||||
return {
|
||||
available: command !== null,
|
||||
command,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runObsidianCli(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
subcommand: string;
|
||||
args?: string[];
|
||||
deps?: ObsidianCliDeps;
|
||||
}): Promise<ObsidianCliResult> {
|
||||
const resolveCommand = params.deps?.resolveCommand ?? resolveCommandOnPath;
|
||||
const exec = params.deps?.exec ?? execFileAsync;
|
||||
const probe = await probeObsidianCli({ resolveCommand });
|
||||
if (!probe.command) {
|
||||
throw new Error("Obsidian CLI is not available on PATH.");
|
||||
}
|
||||
const argv = [...buildVaultPrefix(params.config), params.subcommand, ...(params.args ?? [])];
|
||||
const { stdout, stderr } = await exec(probe.command, argv, { encoding: "utf8" });
|
||||
return {
|
||||
command: probe.command,
|
||||
argv,
|
||||
stdout,
|
||||
stderr,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runObsidianSearch(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
query: string;
|
||||
deps?: ObsidianCliDeps;
|
||||
}) {
|
||||
return await runObsidianCli({
|
||||
config: params.config,
|
||||
subcommand: "search",
|
||||
args: [`query=${params.query}`],
|
||||
deps: params.deps,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runObsidianOpen(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
vaultPath: string;
|
||||
deps?: ObsidianCliDeps;
|
||||
}) {
|
||||
return await runObsidianCli({
|
||||
config: params.config,
|
||||
subcommand: "open",
|
||||
args: [`path=${params.vaultPath}`],
|
||||
deps: params.deps,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runObsidianCommand(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
id: string;
|
||||
deps?: ObsidianCliDeps;
|
||||
}) {
|
||||
return await runObsidianCli({
|
||||
config: params.config,
|
||||
subcommand: "command",
|
||||
args: [`id=${params.id}`],
|
||||
deps: params.deps,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runObsidianDaily(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
deps?: ObsidianCliDeps;
|
||||
}) {
|
||||
return await runObsidianCli({
|
||||
config: params.config,
|
||||
subcommand: "daily",
|
||||
deps: params.deps,
|
||||
});
|
||||
}
|
||||
179
openclaw/extensions/memory-wiki/src/prompt-section.test.ts
Normal file
179
openclaw/extensions/memory-wiki/src/prompt-section.test.ts
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { resolveMemoryWikiConfig } from "./config.js";
|
||||
import { buildWikiPromptSection, createWikiPromptSectionBuilder } from "./prompt-section.js";
|
||||
|
||||
let suiteRoot = "";
|
||||
|
||||
beforeAll(async () => {
|
||||
suiteRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-prompt-suite-"));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (suiteRoot) {
|
||||
await fs.rm(suiteRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("buildWikiPromptSection", () => {
|
||||
it("prefers shared memory corpus guidance when memory tools are available", () => {
|
||||
const lines = buildWikiPromptSection({
|
||||
availableTools: new Set(["memory_search", "memory_get", "wiki_search", "wiki_get"]),
|
||||
});
|
||||
|
||||
expect(lines.join("\n")).toContain("`memory_search` with `corpus=all`");
|
||||
expect(lines.join("\n")).toContain("`memory_get` with `corpus=wiki` or `corpus=all`");
|
||||
expect(lines.join("\n")).toContain("wiki-specific ranking or provenance details");
|
||||
});
|
||||
|
||||
it("stays empty when no wiki or memory-adjacent tools are registered", () => {
|
||||
expect(buildWikiPromptSection({ availableTools: new Set(["web_search"]) })).toEqual([]);
|
||||
});
|
||||
|
||||
it("can append a compact compiled digest snapshot when enabled", async () => {
|
||||
const rootDir = path.join(suiteRoot, "digest-enabled");
|
||||
await fs.mkdir(path.join(rootDir, ".openclaw-wiki", "cache"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
claimCount: 8,
|
||||
contradictionClusters: [{ key: "claim.alpha.db" }],
|
||||
pages: [
|
||||
{
|
||||
title: "Alpha",
|
||||
kind: "entity",
|
||||
claimCount: 3,
|
||||
questions: ["Still active?"],
|
||||
contradictions: ["Conflicts with source.beta"],
|
||||
topClaims: [
|
||||
{
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
status: "supported",
|
||||
confidence: 0.91,
|
||||
freshnessLevel: "fresh",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
"utf8",
|
||||
);
|
||||
const builder = createWikiPromptSectionBuilder(
|
||||
resolveMemoryWikiConfig({
|
||||
vault: { path: rootDir },
|
||||
context: { includeCompiledDigestPrompt: true },
|
||||
}),
|
||||
);
|
||||
|
||||
const lines = builder({ availableTools: new Set(["web_search"]) });
|
||||
|
||||
expect(lines.join("\n")).toContain("## Compiled Wiki Snapshot");
|
||||
expect(lines.join("\n")).toContain(
|
||||
"Alpha: entity, 3 claims, 1 open questions, 1 contradiction notes",
|
||||
);
|
||||
expect(lines.join("\n")).toContain("Alpha uses PostgreSQL for production writes.");
|
||||
});
|
||||
|
||||
it("keeps the digest snapshot disabled by default", async () => {
|
||||
const rootDir = path.join(suiteRoot, "digest-disabled");
|
||||
await fs.mkdir(path.join(rootDir, ".openclaw-wiki", "cache"), { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"),
|
||||
JSON.stringify({
|
||||
claimCount: 1,
|
||||
pages: [{ title: "Alpha", kind: "entity", claimCount: 1, topClaims: [] }],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const builder = createWikiPromptSectionBuilder(
|
||||
resolveMemoryWikiConfig({
|
||||
vault: { path: rootDir },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(builder({ availableTools: new Set(["web_search"]) })).toEqual([]);
|
||||
});
|
||||
|
||||
it("stabilizes digest prompt ordering for prompt-cache-friendly output", async () => {
|
||||
const rootDir = path.join(suiteRoot, "digest-stable");
|
||||
const digestPath = path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json");
|
||||
await fs.mkdir(path.dirname(digestPath), { recursive: true });
|
||||
|
||||
const builder = createWikiPromptSectionBuilder(
|
||||
resolveMemoryWikiConfig({
|
||||
vault: { path: rootDir },
|
||||
context: { includeCompiledDigestPrompt: true },
|
||||
}),
|
||||
);
|
||||
|
||||
const firstDigest = {
|
||||
claimCount: 6,
|
||||
contradictionClusters: [{ key: "claim.alpha.db" }],
|
||||
pages: [
|
||||
{
|
||||
title: "Zulu",
|
||||
kind: "concept",
|
||||
claimCount: 2,
|
||||
questions: [],
|
||||
contradictions: [],
|
||||
topClaims: [
|
||||
{
|
||||
text: "Zulu fallback note.",
|
||||
confidence: 0.3,
|
||||
freshnessLevel: "stale",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: "Alpha",
|
||||
kind: "entity",
|
||||
claimCount: 4,
|
||||
questions: ["Still active?"],
|
||||
contradictions: ["Conflicts with source.beta"],
|
||||
topClaims: [
|
||||
{
|
||||
text: "Alpha was renamed in 2026.",
|
||||
confidence: 0.42,
|
||||
freshnessLevel: "aging",
|
||||
},
|
||||
{
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
confidence: 0.91,
|
||||
freshnessLevel: "fresh",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
const secondDigest = {
|
||||
...firstDigest,
|
||||
pages: [
|
||||
{
|
||||
...firstDigest.pages[1],
|
||||
topClaims: firstDigest.pages[1].topClaims.toReversed(),
|
||||
},
|
||||
firstDigest.pages[0],
|
||||
],
|
||||
};
|
||||
|
||||
await fs.writeFile(digestPath, JSON.stringify(firstDigest, null, 2), "utf8");
|
||||
const firstLines = builder({ availableTools: new Set(["web_search"]) });
|
||||
|
||||
await fs.writeFile(digestPath, JSON.stringify(secondDigest, null, 2), "utf8");
|
||||
const secondLines = builder({ availableTools: new Set(["web_search"]) });
|
||||
|
||||
expect(firstLines).toEqual(secondLines);
|
||||
expect(firstLines.join("\n")).toContain(
|
||||
"Alpha uses PostgreSQL for production writes. (confidence 0.91, freshness fresh)",
|
||||
);
|
||||
expect(firstLines.join("\n")).toContain(
|
||||
"Alpha was renamed in 2026. (confidence 0.42, freshness aging)",
|
||||
);
|
||||
});
|
||||
});
|
||||
235
openclaw/extensions/memory-wiki/src/prompt-section.ts
Normal file
235
openclaw/extensions/memory-wiki/src/prompt-section.ts
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { MemoryPromptSectionBuilder } from "openclaw/plugin-sdk/memory-host-core";
|
||||
import { resolveMemoryWikiConfig, type ResolvedMemoryWikiConfig } from "./config.js";
|
||||
|
||||
const AGENT_DIGEST_PATH = ".openclaw-wiki/cache/agent-digest.json";
|
||||
const DIGEST_MAX_PAGES = 4;
|
||||
const DIGEST_MAX_CLAIMS_PER_PAGE = 2;
|
||||
|
||||
type PromptDigestClaim = {
|
||||
text: string;
|
||||
status?: string;
|
||||
confidence?: number;
|
||||
freshnessLevel?: string;
|
||||
};
|
||||
|
||||
type PromptDigestPage = {
|
||||
title: string;
|
||||
kind: string;
|
||||
claimCount: number;
|
||||
questions?: string[];
|
||||
contradictions?: string[];
|
||||
topClaims?: PromptDigestClaim[];
|
||||
};
|
||||
|
||||
type PromptDigest = {
|
||||
pageCounts?: Record<string, number>;
|
||||
claimCount?: number;
|
||||
contradictionClusters?: Array<unknown>;
|
||||
pages?: PromptDigestPage[];
|
||||
};
|
||||
|
||||
function tryReadPromptDigest(config: ResolvedMemoryWikiConfig): PromptDigest | null {
|
||||
const digestPath = path.join(config.vault.path, AGENT_DIGEST_PATH);
|
||||
try {
|
||||
const raw = fs.readFileSync(digestPath, "utf8");
|
||||
const parsed = JSON.parse(raw) as PromptDigest;
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
return null;
|
||||
}
|
||||
return parsed;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function rankPromptDigestPage(page: PromptDigestPage): number {
|
||||
return (
|
||||
(page.contradictions?.length ?? 0) * 6 +
|
||||
(page.questions?.length ?? 0) * 4 +
|
||||
Math.min(page.claimCount ?? 0, 6) * 2 +
|
||||
Math.min(page.topClaims?.length ?? 0, 3)
|
||||
);
|
||||
}
|
||||
|
||||
function rankPromptClaimFreshness(level?: string): number {
|
||||
switch (level) {
|
||||
case "fresh":
|
||||
return 3;
|
||||
case "aging":
|
||||
return 2;
|
||||
case "stale":
|
||||
return 1;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function sortPromptClaims(claims: PromptDigestClaim[]): PromptDigestClaim[] {
|
||||
return [...claims].toSorted((left, right) => {
|
||||
const leftConfidence = typeof left.confidence === "number" ? left.confidence : -1;
|
||||
const rightConfidence = typeof right.confidence === "number" ? right.confidence : -1;
|
||||
if (leftConfidence !== rightConfidence) {
|
||||
return rightConfidence - leftConfidence;
|
||||
}
|
||||
const leftFreshness = rankPromptClaimFreshness(left.freshnessLevel);
|
||||
const rightFreshness = rankPromptClaimFreshness(right.freshnessLevel);
|
||||
if (leftFreshness !== rightFreshness) {
|
||||
return rightFreshness - leftFreshness;
|
||||
}
|
||||
return left.text.localeCompare(right.text);
|
||||
});
|
||||
}
|
||||
|
||||
function formatPromptClaim(claim: PromptDigestClaim): string {
|
||||
const qualifiers = [
|
||||
claim.status?.trim() ? `status ${claim.status.trim()}` : null,
|
||||
typeof claim.confidence === "number" ? `confidence ${claim.confidence.toFixed(2)}` : null,
|
||||
claim.freshnessLevel?.trim() ? `freshness ${claim.freshnessLevel.trim()}` : null,
|
||||
].filter(Boolean);
|
||||
if (qualifiers.length === 0) {
|
||||
return claim.text;
|
||||
}
|
||||
return `${claim.text} (${qualifiers.join(", ")})`;
|
||||
}
|
||||
|
||||
function buildDigestPromptSection(config: ResolvedMemoryWikiConfig): string[] {
|
||||
if (!config.context.includeCompiledDigestPrompt) {
|
||||
return [];
|
||||
}
|
||||
const digest = tryReadPromptDigest(config);
|
||||
if (!digest?.pages?.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const selectedPages = [...digest.pages]
|
||||
.filter(
|
||||
(page) =>
|
||||
(page.claimCount ?? 0) > 0 ||
|
||||
(page.questions?.length ?? 0) > 0 ||
|
||||
(page.contradictions?.length ?? 0) > 0,
|
||||
)
|
||||
.toSorted((left, right) => {
|
||||
const leftScore = rankPromptDigestPage(left);
|
||||
const rightScore = rankPromptDigestPage(right);
|
||||
if (leftScore !== rightScore) {
|
||||
return rightScore - leftScore;
|
||||
}
|
||||
return left.title.localeCompare(right.title);
|
||||
})
|
||||
.slice(0, DIGEST_MAX_PAGES);
|
||||
|
||||
if (selectedPages.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const lines = [
|
||||
"## Compiled Wiki Snapshot",
|
||||
`Compiled wiki currently tracks ${digest.claimCount ?? 0} claims across ${selectedPages.length} high-signal pages.`,
|
||||
];
|
||||
if (Array.isArray(digest.contradictionClusters)) {
|
||||
lines.push(`Contradiction clusters: ${digest.contradictionClusters.length}.`);
|
||||
}
|
||||
for (const page of selectedPages) {
|
||||
const details = [
|
||||
page.kind,
|
||||
`${page.claimCount} claims`,
|
||||
(page.questions?.length ?? 0) > 0 ? `${page.questions?.length} open questions` : null,
|
||||
(page.contradictions?.length ?? 0) > 0
|
||||
? `${page.contradictions?.length} contradiction notes`
|
||||
: null,
|
||||
].filter(Boolean);
|
||||
lines.push(`- ${page.title}: ${details.join(", ")}`);
|
||||
for (const claim of sortPromptClaims(page.topClaims ?? []).slice(
|
||||
0,
|
||||
DIGEST_MAX_CLAIMS_PER_PAGE,
|
||||
)) {
|
||||
lines.push(` - ${formatPromptClaim(claim)}`);
|
||||
}
|
||||
}
|
||||
lines.push("");
|
||||
return lines;
|
||||
}
|
||||
|
||||
function buildWikiToolGuidance(availableTools: Set<string>): string[] {
|
||||
const hasMemorySearch = availableTools.has("memory_search");
|
||||
const hasMemoryGet = availableTools.has("memory_get");
|
||||
const hasWikiSearch = availableTools.has("wiki_search");
|
||||
const hasWikiGet = availableTools.has("wiki_get");
|
||||
const hasWikiApply = availableTools.has("wiki_apply");
|
||||
const hasWikiLint = availableTools.has("wiki_lint");
|
||||
|
||||
if (
|
||||
!hasMemorySearch &&
|
||||
!hasMemoryGet &&
|
||||
!hasWikiSearch &&
|
||||
!hasWikiGet &&
|
||||
!hasWikiApply &&
|
||||
!hasWikiLint
|
||||
) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const lines = [
|
||||
"## Compiled Wiki",
|
||||
"Use the wiki when the answer depends on accumulated project knowledge, prior syntheses, entity pages, or source-backed notes that should survive beyond one conversation.",
|
||||
];
|
||||
|
||||
if (hasMemorySearch) {
|
||||
lines.push(
|
||||
"Prefer `memory_search` with `corpus=all` for one recall pass across durable memory and the compiled wiki when both are relevant.",
|
||||
);
|
||||
}
|
||||
if (hasMemoryGet) {
|
||||
lines.push(
|
||||
"Use `memory_get` with `corpus=wiki` or `corpus=all` when you already know the page path and want a small excerpt without leaving the shared memory tool flow.",
|
||||
);
|
||||
}
|
||||
|
||||
if (hasWikiSearch && hasWikiGet) {
|
||||
lines.push(
|
||||
"Workflow: `wiki_search` first, then `wiki_get` for the exact page or imported memory file you need. Use this when you want wiki-specific ranking or provenance details instead of the broader shared memory flow.",
|
||||
);
|
||||
} else if (hasWikiSearch) {
|
||||
lines.push(
|
||||
"Use `wiki_search` before answering from stored knowledge when you want wiki-specific ranking or provenance details.",
|
||||
);
|
||||
} else if (hasWikiGet) {
|
||||
lines.push(
|
||||
"Use `wiki_get` to inspect specific wiki pages or imported memory files by path/id.",
|
||||
);
|
||||
}
|
||||
|
||||
if (hasWikiApply) {
|
||||
lines.push(
|
||||
"Use `wiki_apply` for narrow synthesis filing and metadata repair instead of rewriting managed markdown blocks by hand.",
|
||||
);
|
||||
}
|
||||
if (hasWikiLint) {
|
||||
lines.push("After meaningful wiki updates, run `wiki_lint` before trusting the vault.");
|
||||
}
|
||||
lines.push("");
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function createWikiPromptSectionBuilder(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
): MemoryPromptSectionBuilder {
|
||||
return ({ availableTools }) => {
|
||||
const digestLines = buildDigestPromptSection(config);
|
||||
const toolGuidance = buildWikiToolGuidance(availableTools);
|
||||
if (digestLines.length === 0 && toolGuidance.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return [...toolGuidance, ...digestLines];
|
||||
};
|
||||
}
|
||||
|
||||
export const buildWikiPromptSection: MemoryPromptSectionBuilder = ({ availableTools }) =>
|
||||
createWikiPromptSectionBuilder(
|
||||
resolveMemoryWikiConfig({
|
||||
vault: { path: "" },
|
||||
context: { includeCompiledDigestPrompt: false },
|
||||
}),
|
||||
)({ availableTools });
|
||||
640
openclaw/extensions/memory-wiki/src/query.test.ts
Normal file
640
openclaw/extensions/memory-wiki/src/query.test.ts
Normal file
|
|
@ -0,0 +1,640 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
import { compileMemoryWikiVault } from "./compile.js";
|
||||
import type { MemoryWikiPluginConfig } from "./config.js";
|
||||
import { renderWikiMarkdown } from "./markdown.js";
|
||||
import { getMemoryWikiPage, searchMemoryWiki } from "./query.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { getActiveMemorySearchManagerMock, resolveDefaultAgentIdMock, resolveSessionAgentIdMock } =
|
||||
vi.hoisted(() => ({
|
||||
getActiveMemorySearchManagerMock: vi.fn(),
|
||||
resolveDefaultAgentIdMock: vi.fn(() => "main"),
|
||||
resolveSessionAgentIdMock: vi.fn(({ sessionKey }: { sessionKey?: string }) =>
|
||||
sessionKey === "agent:secondary:thread" ? "secondary" : "main",
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/memory-host-search", () => ({
|
||||
getActiveMemorySearchManager: getActiveMemorySearchManagerMock,
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/memory-host-core", () => ({
|
||||
resolveDefaultAgentId: resolveDefaultAgentIdMock,
|
||||
resolveSessionAgentId: resolveSessionAgentIdMock,
|
||||
}));
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
let suiteRoot = "";
|
||||
let caseIndex = 0;
|
||||
|
||||
beforeEach(() => {
|
||||
getActiveMemorySearchManagerMock.mockReset();
|
||||
getActiveMemorySearchManagerMock.mockResolvedValue({ manager: null, error: "unavailable" });
|
||||
resolveDefaultAgentIdMock.mockClear();
|
||||
resolveSessionAgentIdMock.mockClear();
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
suiteRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-query-suite-"));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (suiteRoot) {
|
||||
await fs.rm(suiteRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
async function createQueryVault(options?: {
|
||||
config?: MemoryWikiPluginConfig;
|
||||
initialize?: boolean;
|
||||
}) {
|
||||
return createVault({
|
||||
prefix: "memory-wiki-query-",
|
||||
rootDir: path.join(suiteRoot, `case-${caseIndex++}`),
|
||||
initialize: options?.initialize,
|
||||
config: options?.config,
|
||||
});
|
||||
}
|
||||
|
||||
function createAppConfig(): OpenClawConfig {
|
||||
return {
|
||||
agents: {
|
||||
list: [{ id: "main", default: true }],
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
}
|
||||
|
||||
function createMemoryManager(overrides?: {
|
||||
searchResults?: Array<{
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
snippet: string;
|
||||
source: "memory" | "sessions";
|
||||
citation?: string;
|
||||
}>;
|
||||
readResult?: { text: string; path: string };
|
||||
}) {
|
||||
return {
|
||||
search: vi.fn().mockResolvedValue(overrides?.searchResults ?? []),
|
||||
readFile: vi.fn().mockImplementation(async () => {
|
||||
if (!overrides?.readResult) {
|
||||
throw new Error("missing");
|
||||
}
|
||||
return overrides.readResult;
|
||||
}),
|
||||
status: vi.fn().mockReturnValue({ backend: "builtin", provider: "builtin" }),
|
||||
probeEmbeddingAvailability: vi.fn().mockResolvedValue({ ok: true }),
|
||||
probeVectorAvailability: vi.fn().mockResolvedValue(false),
|
||||
close: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
describe("searchMemoryWiki", () => {
|
||||
it("finds wiki pages by title and body", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha Source" },
|
||||
body: "# Alpha Source\n\nalpha body text\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const results = await searchMemoryWiki({ config, query: "alpha" });
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.corpus).toBe("wiki");
|
||||
expect(results[0]?.path).toBe("sources/alpha.md");
|
||||
expect(getActiveMemorySearchManagerMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("finds wiki pages by structured claim text and surfaces the claim as the snippet", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha",
|
||||
title: "Alpha",
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.postgres",
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
status: "supported",
|
||||
confidence: 0.91,
|
||||
evidence: [{ sourceId: "source.alpha", lines: "12-18" }],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha\n\nsummary without the query phrase\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const results = await searchMemoryWiki({ config, query: "postgresql" });
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]).toMatchObject({
|
||||
corpus: "wiki",
|
||||
path: "entities/alpha.md",
|
||||
snippet: "Alpha uses PostgreSQL for production writes.",
|
||||
});
|
||||
});
|
||||
|
||||
it("ranks fresh supported claims ahead of stale contested claims", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "alpha-fresh.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha.fresh",
|
||||
title: "Alpha Fresh",
|
||||
updatedAt: "2026-04-01T00:00:00.000Z",
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.db.fresh",
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
status: "supported",
|
||||
confidence: 0.91,
|
||||
evidence: [
|
||||
{
|
||||
sourceId: "source.alpha",
|
||||
lines: "4-7",
|
||||
updatedAt: "2026-04-01T00:00:00.000Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha Fresh\n\nsummary without the keyword\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "alpha-stale.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha.stale",
|
||||
title: "Alpha Stale",
|
||||
updatedAt: "2025-10-01T00:00:00.000Z",
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.db.stale",
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
status: "contested",
|
||||
confidence: 0.92,
|
||||
evidence: [
|
||||
{
|
||||
sourceId: "source.alpha.old",
|
||||
lines: "1-2",
|
||||
updatedAt: "2025-10-01T00:00:00.000Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha Stale\n\nsummary without the keyword\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const results = await searchMemoryWiki({ config, query: "postgresql" });
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results[0]?.path).toBe("entities/alpha-fresh.md");
|
||||
expect(results[1]?.path).toBe("entities/alpha-stale.md");
|
||||
});
|
||||
|
||||
it("surfaces bridge provenance for imported source pages", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "bridge-alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.bridge.alpha",
|
||||
title: "Bridge Alpha",
|
||||
sourceType: "memory-bridge",
|
||||
sourcePath: "/tmp/workspace/MEMORY.md",
|
||||
bridgeRelativePath: "MEMORY.md",
|
||||
bridgeWorkspaceDir: "/tmp/workspace",
|
||||
updatedAt: "2026-04-05T12:00:00.000Z",
|
||||
},
|
||||
body: "# Bridge Alpha\n\nalpha bridge body\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const results = await searchMemoryWiki({ config, query: "alpha" });
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]).toMatchObject({
|
||||
corpus: "wiki",
|
||||
sourceType: "memory-bridge",
|
||||
sourcePath: "/tmp/workspace/MEMORY.md",
|
||||
provenanceLabel: "bridge: MEMORY.md",
|
||||
updatedAt: "2026-04-05T12:00:00.000Z",
|
||||
});
|
||||
});
|
||||
|
||||
it("includes active memory results when shared search and all corpora are enabled", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
config: {
|
||||
search: { backend: "shared", corpus: "all" },
|
||||
},
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha Source" },
|
||||
body: "# Alpha Source\n\nalpha body text\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const manager = createMemoryManager({
|
||||
searchResults: [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 4,
|
||||
endLine: 8,
|
||||
score: 42,
|
||||
snippet: "alpha durable memory",
|
||||
source: "memory",
|
||||
citation: "MEMORY.md#L4-L8",
|
||||
},
|
||||
],
|
||||
});
|
||||
getActiveMemorySearchManagerMock.mockResolvedValue({ manager });
|
||||
|
||||
const results = await searchMemoryWiki({
|
||||
config,
|
||||
appConfig: createAppConfig(),
|
||||
query: "alpha",
|
||||
maxResults: 5,
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(2);
|
||||
expect(results.some((result) => result.corpus === "wiki")).toBe(true);
|
||||
expect(results.some((result) => result.corpus === "memory")).toBe(true);
|
||||
expect(manager.search).toHaveBeenCalledWith("alpha", { maxResults: 5 });
|
||||
expect(getActiveMemorySearchManagerMock).toHaveBeenCalledWith({
|
||||
cfg: createAppConfig(),
|
||||
agentId: "main",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the active session agent for shared memory search", async () => {
|
||||
const { config } = await createQueryVault({
|
||||
initialize: true,
|
||||
config: {
|
||||
search: { backend: "shared", corpus: "memory" },
|
||||
},
|
||||
});
|
||||
const manager = createMemoryManager({
|
||||
searchResults: [
|
||||
{
|
||||
path: "memory/2026-04-07.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
score: 1,
|
||||
snippet: "secondary agent memory",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
getActiveMemorySearchManagerMock.mockResolvedValue({ manager });
|
||||
|
||||
await searchMemoryWiki({
|
||||
config,
|
||||
appConfig: createAppConfig(),
|
||||
agentSessionKey: "agent:secondary:thread",
|
||||
query: "secondary",
|
||||
});
|
||||
|
||||
expect(resolveSessionAgentIdMock).toHaveBeenCalledWith({
|
||||
sessionKey: "agent:secondary:thread",
|
||||
config: createAppConfig(),
|
||||
});
|
||||
expect(getActiveMemorySearchManagerMock).toHaveBeenCalledWith({
|
||||
cfg: createAppConfig(),
|
||||
agentId: "secondary",
|
||||
});
|
||||
});
|
||||
|
||||
it("allows per-call corpus overrides without changing config defaults", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
config: {
|
||||
search: { backend: "shared", corpus: "wiki" },
|
||||
},
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha Source" },
|
||||
body: "# Alpha Source\n\nalpha body text\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const manager = createMemoryManager({
|
||||
searchResults: [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 10,
|
||||
endLine: 12,
|
||||
score: 99,
|
||||
snippet: "memory-only alpha",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
getActiveMemorySearchManagerMock.mockResolvedValue({ manager });
|
||||
|
||||
const memoryOnly = await searchMemoryWiki({
|
||||
config,
|
||||
appConfig: createAppConfig(),
|
||||
query: "alpha",
|
||||
searchCorpus: "memory",
|
||||
});
|
||||
|
||||
expect(memoryOnly).toHaveLength(1);
|
||||
expect(memoryOnly[0]?.corpus).toBe("memory");
|
||||
expect(manager.search).toHaveBeenCalledWith("alpha", { maxResults: 10 });
|
||||
});
|
||||
|
||||
it("keeps memory search disabled when the backend is local", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
config: {
|
||||
search: { backend: "local", corpus: "all" },
|
||||
},
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha Source" },
|
||||
body: "# Alpha Source\n\nalpha only wiki\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const manager = createMemoryManager({
|
||||
searchResults: [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 1,
|
||||
endLine: 2,
|
||||
score: 50,
|
||||
snippet: "alpha memory",
|
||||
source: "memory",
|
||||
},
|
||||
],
|
||||
});
|
||||
getActiveMemorySearchManagerMock.mockResolvedValue({ manager });
|
||||
|
||||
const results = await searchMemoryWiki({
|
||||
config,
|
||||
appConfig: createAppConfig(),
|
||||
query: "alpha",
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.corpus).toBe("wiki");
|
||||
expect(manager.search).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("getMemoryWikiPage", () => {
|
||||
it("reads wiki pages by relative path and slices line ranges", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha Source" },
|
||||
body: "# Alpha Source\n\nline one\nline two\nline three\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await getMemoryWikiPage({
|
||||
config,
|
||||
lookup: "sources/alpha.md",
|
||||
fromLine: 4,
|
||||
lineCount: 2,
|
||||
});
|
||||
|
||||
expect(result?.corpus).toBe("wiki");
|
||||
expect(result?.path).toBe("sources/alpha.md");
|
||||
expect(result?.content).toContain("line one");
|
||||
expect(result?.content).toContain("line two");
|
||||
expect(result?.content).not.toContain("line three");
|
||||
expect(result?.totalLines).toBe(7);
|
||||
expect(result?.truncated).toBe(true);
|
||||
});
|
||||
|
||||
it("resolves compiled claim ids back to the owning page", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.alpha",
|
||||
title: "Alpha",
|
||||
claims: [
|
||||
{
|
||||
id: "claim.alpha.db",
|
||||
text: "Alpha uses PostgreSQL for production writes.",
|
||||
status: "supported",
|
||||
evidence: [{ sourceId: "source.alpha", lines: "1-2" }],
|
||||
},
|
||||
],
|
||||
},
|
||||
body: "# Alpha\n\nline one\nline two\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await compileMemoryWikiVault(config);
|
||||
|
||||
const result = await getMemoryWikiPage({
|
||||
config,
|
||||
lookup: "claim.alpha.db",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
corpus: "wiki",
|
||||
path: "entities/alpha.md",
|
||||
title: "Alpha",
|
||||
id: "entity.alpha",
|
||||
});
|
||||
expect(result?.content).toContain("line one");
|
||||
});
|
||||
|
||||
it("returns provenance for imported wiki source pages", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "unsafe-alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.unsafe.alpha",
|
||||
title: "Unsafe Alpha",
|
||||
sourceType: "memory-unsafe-local",
|
||||
provenanceMode: "unsafe-local",
|
||||
sourcePath: "/tmp/private/alpha.md",
|
||||
unsafeLocalConfiguredPath: "/tmp/private",
|
||||
unsafeLocalRelativePath: "alpha.md",
|
||||
updatedAt: "2026-04-05T13:00:00.000Z",
|
||||
},
|
||||
body: "# Unsafe Alpha\n\nsecret alpha\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await getMemoryWikiPage({
|
||||
config,
|
||||
lookup: "sources/unsafe-alpha.md",
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
corpus: "wiki",
|
||||
path: "sources/unsafe-alpha.md",
|
||||
sourceType: "memory-unsafe-local",
|
||||
provenanceMode: "unsafe-local",
|
||||
sourcePath: "/tmp/private/alpha.md",
|
||||
provenanceLabel: "unsafe-local: alpha.md",
|
||||
updatedAt: "2026-04-05T13:00:00.000Z",
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to active memory reads when memory corpus is selected", async () => {
|
||||
const { config } = await createQueryVault({
|
||||
initialize: true,
|
||||
config: {
|
||||
search: { backend: "shared", corpus: "memory" },
|
||||
},
|
||||
});
|
||||
const manager = createMemoryManager({
|
||||
readResult: {
|
||||
path: "MEMORY.md",
|
||||
text: "durable alpha memory\nline two",
|
||||
},
|
||||
});
|
||||
getActiveMemorySearchManagerMock.mockResolvedValue({ manager });
|
||||
|
||||
const result = await getMemoryWikiPage({
|
||||
config,
|
||||
appConfig: createAppConfig(),
|
||||
lookup: "MEMORY.md",
|
||||
fromLine: 2,
|
||||
lineCount: 2,
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
corpus: "memory",
|
||||
path: "MEMORY.md",
|
||||
title: "MEMORY",
|
||||
kind: "memory",
|
||||
content: "durable alpha memory\nline two",
|
||||
fromLine: 2,
|
||||
lineCount: 2,
|
||||
});
|
||||
expect(manager.readFile).toHaveBeenCalledWith({
|
||||
relPath: "MEMORY.md",
|
||||
from: 2,
|
||||
lines: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the active session agent for shared memory reads", async () => {
|
||||
const { config } = await createQueryVault({
|
||||
initialize: true,
|
||||
config: {
|
||||
search: { backend: "shared", corpus: "memory" },
|
||||
},
|
||||
});
|
||||
const manager = createMemoryManager({
|
||||
readResult: {
|
||||
path: "MEMORY.md",
|
||||
text: "secondary memory line",
|
||||
},
|
||||
});
|
||||
getActiveMemorySearchManagerMock.mockResolvedValue({ manager });
|
||||
|
||||
const result = await getMemoryWikiPage({
|
||||
config,
|
||||
appConfig: createAppConfig(),
|
||||
agentSessionKey: "agent:secondary:thread",
|
||||
lookup: "MEMORY.md",
|
||||
});
|
||||
|
||||
expect(result?.corpus).toBe("memory");
|
||||
expect(resolveSessionAgentIdMock).toHaveBeenCalledWith({
|
||||
sessionKey: "agent:secondary:thread",
|
||||
config: createAppConfig(),
|
||||
});
|
||||
expect(getActiveMemorySearchManagerMock).toHaveBeenCalledWith({
|
||||
cfg: createAppConfig(),
|
||||
agentId: "secondary",
|
||||
});
|
||||
});
|
||||
|
||||
it("allows per-call get overrides to bypass wiki and force memory fallback", async () => {
|
||||
const { rootDir, config } = await createQueryVault({
|
||||
initialize: true,
|
||||
config: {
|
||||
search: { backend: "shared", corpus: "wiki" },
|
||||
},
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "MEMORY.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.memory.shadow", title: "Shadow Memory" },
|
||||
body: "# Shadow Memory\n\nwiki copy\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const manager = createMemoryManager({
|
||||
readResult: {
|
||||
path: "MEMORY.md",
|
||||
text: "forced memory read",
|
||||
},
|
||||
});
|
||||
getActiveMemorySearchManagerMock.mockResolvedValue({ manager });
|
||||
|
||||
const result = await getMemoryWikiPage({
|
||||
config,
|
||||
appConfig: createAppConfig(),
|
||||
lookup: "MEMORY.md",
|
||||
searchCorpus: "memory",
|
||||
});
|
||||
|
||||
expect(result?.corpus).toBe("memory");
|
||||
expect(result?.content).toBe("forced memory read");
|
||||
expect(manager.readFile).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
775
openclaw/extensions/memory-wiki/src/query.ts
Normal file
775
openclaw/extensions/memory-wiki/src/query.ts
Normal file
|
|
@ -0,0 +1,775 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { resolveDefaultAgentId, resolveSessionAgentId } from "openclaw/plugin-sdk/memory-host-core";
|
||||
import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-host-files";
|
||||
import { getActiveMemorySearchManager } from "openclaw/plugin-sdk/memory-host-search";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
import { assessClaimFreshness, isClaimContestedStatus } from "./claim-health.js";
|
||||
import type { ResolvedMemoryWikiConfig, WikiSearchBackend, WikiSearchCorpus } from "./config.js";
|
||||
import {
|
||||
parseWikiMarkdown,
|
||||
toWikiPageSummary,
|
||||
type WikiClaim,
|
||||
type WikiPageSummary,
|
||||
} from "./markdown.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
const QUERY_DIRS = ["entities", "concepts", "sources", "syntheses", "reports"] as const;
|
||||
const AGENT_DIGEST_PATH = ".openclaw-wiki/cache/agent-digest.json";
|
||||
const CLAIMS_DIGEST_PATH = ".openclaw-wiki/cache/claims.jsonl";
|
||||
|
||||
type QueryDigestPage = {
|
||||
id?: string;
|
||||
title: string;
|
||||
kind: WikiPageSummary["kind"];
|
||||
path: string;
|
||||
sourceIds: string[];
|
||||
questions: string[];
|
||||
contradictions: string[];
|
||||
};
|
||||
|
||||
type QueryDigestClaim = {
|
||||
id?: string;
|
||||
pageId?: string;
|
||||
pageTitle: string;
|
||||
pageKind: WikiPageSummary["kind"];
|
||||
pagePath: string;
|
||||
text: string;
|
||||
status?: string;
|
||||
confidence?: number;
|
||||
sourceIds?: string[];
|
||||
freshnessLevel?: string;
|
||||
lastTouchedAt?: string;
|
||||
};
|
||||
|
||||
type QueryDigestBundle = {
|
||||
pages: QueryDigestPage[];
|
||||
claims: QueryDigestClaim[];
|
||||
};
|
||||
|
||||
export type WikiSearchResult = {
|
||||
corpus: "wiki" | "memory";
|
||||
path: string;
|
||||
title: string;
|
||||
kind: WikiPageSummary["kind"] | "memory";
|
||||
score: number;
|
||||
snippet: string;
|
||||
id?: string;
|
||||
startLine?: number;
|
||||
endLine?: number;
|
||||
citation?: string;
|
||||
memorySource?: MemorySearchResult["source"];
|
||||
sourceType?: string;
|
||||
provenanceMode?: string;
|
||||
sourcePath?: string;
|
||||
provenanceLabel?: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
export type WikiGetResult = {
|
||||
corpus: "wiki" | "memory";
|
||||
path: string;
|
||||
title: string;
|
||||
kind: WikiPageSummary["kind"] | "memory";
|
||||
content: string;
|
||||
fromLine: number;
|
||||
lineCount: number;
|
||||
totalLines?: number;
|
||||
truncated?: boolean;
|
||||
id?: string;
|
||||
sourceType?: string;
|
||||
provenanceMode?: string;
|
||||
sourcePath?: string;
|
||||
provenanceLabel?: string;
|
||||
updatedAt?: string;
|
||||
};
|
||||
|
||||
export type QueryableWikiPage = WikiPageSummary & {
|
||||
raw: string;
|
||||
};
|
||||
|
||||
type QuerySearchOverrides = {
|
||||
searchBackend?: WikiSearchBackend;
|
||||
searchCorpus?: WikiSearchCorpus;
|
||||
};
|
||||
|
||||
async function listWikiMarkdownFiles(rootDir: string): Promise<string[]> {
|
||||
const files = (
|
||||
await Promise.all(
|
||||
QUERY_DIRS.map(async (relativeDir) => {
|
||||
const dirPath = path.join(rootDir, relativeDir);
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true }).catch(() => []);
|
||||
return entries
|
||||
.filter(
|
||||
(entry) => entry.isFile() && entry.name.endsWith(".md") && entry.name !== "index.md",
|
||||
)
|
||||
.map((entry) => path.join(relativeDir, entry.name));
|
||||
}),
|
||||
)
|
||||
).flat();
|
||||
return files.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export async function readQueryableWikiPages(rootDir: string): Promise<QueryableWikiPage[]> {
|
||||
const files = await listWikiMarkdownFiles(rootDir);
|
||||
return readQueryableWikiPagesByPaths(rootDir, files);
|
||||
}
|
||||
|
||||
async function readQueryableWikiPagesByPaths(
|
||||
rootDir: string,
|
||||
files: string[],
|
||||
): Promise<QueryableWikiPage[]> {
|
||||
const pages = await Promise.all(
|
||||
files.map(async (relativePath) => {
|
||||
const absolutePath = path.join(rootDir, relativePath);
|
||||
const raw = await fs.readFile(absolutePath, "utf8");
|
||||
const summary = toWikiPageSummary({ absolutePath, relativePath, raw });
|
||||
return summary ? { ...summary, raw } : null;
|
||||
}),
|
||||
);
|
||||
return pages.flatMap((page) => (page ? [page] : []));
|
||||
}
|
||||
|
||||
function parseClaimsDigest(raw: string): QueryDigestClaim[] {
|
||||
return raw.split(/\r?\n/).flatMap((line) => {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as QueryDigestClaim;
|
||||
if (!parsed || typeof parsed !== "object" || typeof parsed.pagePath !== "string") {
|
||||
return [];
|
||||
}
|
||||
return [parsed];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function readQueryDigestBundle(rootDir: string): Promise<QueryDigestBundle | null> {
|
||||
const [agentDigestRaw, claimsDigestRaw] = await Promise.all([
|
||||
fs.readFile(path.join(rootDir, AGENT_DIGEST_PATH), "utf8").catch(() => null),
|
||||
fs.readFile(path.join(rootDir, CLAIMS_DIGEST_PATH), "utf8").catch(() => null),
|
||||
]);
|
||||
if (!agentDigestRaw && !claimsDigestRaw) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const pages = (() => {
|
||||
if (!agentDigestRaw) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(agentDigestRaw) as { pages?: QueryDigestPage[] };
|
||||
return Array.isArray(parsed.pages) ? parsed.pages : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})();
|
||||
const claims = claimsDigestRaw ? parseClaimsDigest(claimsDigestRaw) : [];
|
||||
|
||||
if (pages.length === 0 && claims.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { pages, claims };
|
||||
}
|
||||
|
||||
function buildSnippet(raw: string, query: string): string {
|
||||
const queryLower = normalizeLowercaseStringOrEmpty(query);
|
||||
const matchingLine = raw
|
||||
.split(/\r?\n/)
|
||||
.find(
|
||||
(line) =>
|
||||
normalizeLowercaseStringOrEmpty(line).includes(queryLower) && line.trim().length > 0,
|
||||
);
|
||||
return (
|
||||
matchingLine?.trim() ||
|
||||
raw
|
||||
.split(/\r?\n/)
|
||||
.find((line) => line.trim().length > 0)
|
||||
?.trim() ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
function buildPageSearchText(page: QueryableWikiPage): string {
|
||||
return [
|
||||
page.title,
|
||||
page.relativePath,
|
||||
page.id ?? "",
|
||||
page.sourceIds.join(" "),
|
||||
page.questions.join(" "),
|
||||
page.contradictions.join(" "),
|
||||
page.claims.map((claim) => claim.text).join(" "),
|
||||
page.claims.map((claim) => claim.id ?? "").join(" "),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function buildDigestPageSearchText(page: QueryDigestPage, claims: QueryDigestClaim[]): string {
|
||||
return [
|
||||
page.title,
|
||||
page.path,
|
||||
page.id ?? "",
|
||||
page.sourceIds.join(" "),
|
||||
page.questions.join(" "),
|
||||
page.contradictions.join(" "),
|
||||
claims.map((claim) => claim.text).join(" "),
|
||||
claims.map((claim) => claim.id ?? "").join(" "),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function scoreDigestClaimMatch(claim: QueryDigestClaim, queryLower: string): number {
|
||||
let score = 0;
|
||||
if (normalizeLowercaseStringOrEmpty(claim.text).includes(queryLower)) {
|
||||
score += 25;
|
||||
}
|
||||
if (normalizeLowercaseStringOrEmpty(claim.id).includes(queryLower)) {
|
||||
score += 10;
|
||||
}
|
||||
if (typeof claim.confidence === "number") {
|
||||
score += Math.round(claim.confidence * 10);
|
||||
}
|
||||
switch (claim.freshnessLevel) {
|
||||
case "fresh":
|
||||
score += 8;
|
||||
break;
|
||||
case "aging":
|
||||
score += 4;
|
||||
break;
|
||||
case "stale":
|
||||
score -= 2;
|
||||
break;
|
||||
case "unknown":
|
||||
score -= 4;
|
||||
break;
|
||||
}
|
||||
score += isClaimContestedStatus(claim.status) ? -6 : 4;
|
||||
return score;
|
||||
}
|
||||
|
||||
function buildDigestCandidatePaths(params: {
|
||||
digest: QueryDigestBundle;
|
||||
query: string;
|
||||
maxResults: number;
|
||||
}): string[] {
|
||||
const queryLower = normalizeLowercaseStringOrEmpty(params.query);
|
||||
const claimsByPage = new Map<string, QueryDigestClaim[]>();
|
||||
for (const claim of params.digest.claims) {
|
||||
const current = claimsByPage.get(claim.pagePath) ?? [];
|
||||
current.push(claim);
|
||||
claimsByPage.set(claim.pagePath, current);
|
||||
}
|
||||
|
||||
return params.digest.pages
|
||||
.map((page) => {
|
||||
const claims = claimsByPage.get(page.path) ?? [];
|
||||
const metadataLower = normalizeLowercaseStringOrEmpty(
|
||||
buildDigestPageSearchText(page, claims),
|
||||
);
|
||||
if (!metadataLower.includes(queryLower)) {
|
||||
return { path: page.path, score: 0 };
|
||||
}
|
||||
let score = 1;
|
||||
const titleLower = normalizeLowercaseStringOrEmpty(page.title);
|
||||
const pathLower = normalizeLowercaseStringOrEmpty(page.path);
|
||||
const idLower = normalizeLowercaseStringOrEmpty(page.id);
|
||||
if (titleLower === queryLower) {
|
||||
score += 50;
|
||||
} else if (titleLower.includes(queryLower)) {
|
||||
score += 20;
|
||||
}
|
||||
if (pathLower.includes(queryLower)) {
|
||||
score += 10;
|
||||
}
|
||||
if (idLower.includes(queryLower)) {
|
||||
score += 20;
|
||||
}
|
||||
if (
|
||||
page.sourceIds.some((sourceId) =>
|
||||
normalizeLowercaseStringOrEmpty(sourceId).includes(queryLower),
|
||||
)
|
||||
) {
|
||||
score += 12;
|
||||
}
|
||||
const matchingClaims = claims
|
||||
.filter((claim) => {
|
||||
if (normalizeLowercaseStringOrEmpty(claim.text).includes(queryLower)) {
|
||||
return true;
|
||||
}
|
||||
return normalizeLowercaseStringOrEmpty(claim.id).includes(queryLower);
|
||||
})
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
scoreDigestClaimMatch(right, queryLower) - scoreDigestClaimMatch(left, queryLower),
|
||||
);
|
||||
if (matchingClaims.length > 0) {
|
||||
score += scoreDigestClaimMatch(matchingClaims[0], queryLower);
|
||||
score += Math.min(10, (matchingClaims.length - 1) * 2);
|
||||
}
|
||||
return { path: page.path, score };
|
||||
})
|
||||
.filter((candidate) => candidate.score > 0)
|
||||
.toSorted((left, right) => {
|
||||
if (left.score !== right.score) {
|
||||
return right.score - left.score;
|
||||
}
|
||||
return left.path.localeCompare(right.path);
|
||||
})
|
||||
.slice(0, Math.max(params.maxResults * 4, 20))
|
||||
.map((candidate) => candidate.path);
|
||||
}
|
||||
|
||||
function isClaimMatch(claim: WikiClaim, queryLower: string): boolean {
|
||||
if (normalizeLowercaseStringOrEmpty(claim.text).includes(queryLower)) {
|
||||
return true;
|
||||
}
|
||||
return normalizeLowercaseStringOrEmpty(claim.id).includes(queryLower);
|
||||
}
|
||||
|
||||
function rankClaimMatch(page: QueryableWikiPage, claim: WikiClaim, queryLower: string): number {
|
||||
let score = 0;
|
||||
if (normalizeLowercaseStringOrEmpty(claim.text).includes(queryLower)) {
|
||||
score += 25;
|
||||
}
|
||||
if (normalizeLowercaseStringOrEmpty(claim.id).includes(queryLower)) {
|
||||
score += 10;
|
||||
}
|
||||
if (typeof claim.confidence === "number") {
|
||||
score += Math.round(claim.confidence * 10);
|
||||
}
|
||||
const freshness = assessClaimFreshness({ page, claim });
|
||||
switch (freshness.level) {
|
||||
case "fresh":
|
||||
score += 8;
|
||||
break;
|
||||
case "aging":
|
||||
score += 4;
|
||||
break;
|
||||
case "stale":
|
||||
score -= 2;
|
||||
break;
|
||||
case "unknown":
|
||||
score -= 4;
|
||||
break;
|
||||
}
|
||||
score += isClaimContestedStatus(claim.status) ? -6 : 4;
|
||||
return score;
|
||||
}
|
||||
|
||||
function getMatchingClaims(page: QueryableWikiPage, queryLower: string): WikiClaim[] {
|
||||
return page.claims
|
||||
.filter((claim) => isClaimMatch(claim, queryLower))
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
rankClaimMatch(page, right, queryLower) - rankClaimMatch(page, left, queryLower),
|
||||
);
|
||||
}
|
||||
|
||||
function buildPageSnippet(page: QueryableWikiPage, query: string): string {
|
||||
const queryLower = normalizeLowercaseStringOrEmpty(query);
|
||||
const matchingClaim = getMatchingClaims(page, queryLower)[0];
|
||||
if (matchingClaim) {
|
||||
return matchingClaim.text;
|
||||
}
|
||||
return buildSnippet(page.raw, query);
|
||||
}
|
||||
|
||||
function scorePage(page: QueryableWikiPage, query: string): number {
|
||||
const queryLower = normalizeLowercaseStringOrEmpty(query);
|
||||
const titleLower = normalizeLowercaseStringOrEmpty(page.title);
|
||||
const pathLower = normalizeLowercaseStringOrEmpty(page.relativePath);
|
||||
const idLower = normalizeLowercaseStringOrEmpty(page.id);
|
||||
const metadataLower = normalizeLowercaseStringOrEmpty(buildPageSearchText(page));
|
||||
const rawLower = normalizeLowercaseStringOrEmpty(page.raw);
|
||||
if (
|
||||
!(
|
||||
titleLower.includes(queryLower) ||
|
||||
pathLower.includes(queryLower) ||
|
||||
idLower.includes(queryLower) ||
|
||||
metadataLower.includes(queryLower) ||
|
||||
rawLower.includes(queryLower)
|
||||
)
|
||||
) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let score = 1;
|
||||
if (titleLower === queryLower) {
|
||||
score += 50;
|
||||
} else if (titleLower.includes(queryLower)) {
|
||||
score += 20;
|
||||
}
|
||||
if (pathLower.includes(queryLower)) {
|
||||
score += 10;
|
||||
}
|
||||
if (idLower.includes(queryLower)) {
|
||||
score += 20;
|
||||
}
|
||||
if (
|
||||
page.sourceIds.some((sourceId) =>
|
||||
normalizeLowercaseStringOrEmpty(sourceId).includes(queryLower),
|
||||
)
|
||||
) {
|
||||
score += 12;
|
||||
}
|
||||
const matchingClaims = getMatchingClaims(page, queryLower);
|
||||
if (matchingClaims.length > 0) {
|
||||
score += rankClaimMatch(page, matchingClaims[0], queryLower);
|
||||
score += Math.min(10, (matchingClaims.length - 1) * 2);
|
||||
}
|
||||
const bodyOccurrences = rawLower.split(queryLower).length - 1;
|
||||
score += Math.min(10, bodyOccurrences);
|
||||
return score;
|
||||
}
|
||||
|
||||
function normalizeLookupKey(value: string): string {
|
||||
const normalized = value.trim().replace(/\\/g, "/");
|
||||
return normalized.endsWith(".md") ? normalized : normalized.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function buildLookupCandidates(lookup: string): string[] {
|
||||
const normalized = normalizeLookupKey(lookup);
|
||||
const withExtension = normalized.endsWith(".md") ? normalized : `${normalized}.md`;
|
||||
return [...new Set([normalized, withExtension])];
|
||||
}
|
||||
|
||||
function shouldSearchWiki(config: ResolvedMemoryWikiConfig): boolean {
|
||||
return config.search.corpus === "wiki" || config.search.corpus === "all";
|
||||
}
|
||||
|
||||
function shouldSearchSharedMemory(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
): boolean {
|
||||
return (
|
||||
config.search.backend === "shared" &&
|
||||
appConfig !== undefined &&
|
||||
(config.search.corpus === "memory" || config.search.corpus === "all")
|
||||
);
|
||||
}
|
||||
|
||||
function resolveActiveMemoryAgentId(params: {
|
||||
appConfig?: OpenClawConfig;
|
||||
agentId?: string;
|
||||
agentSessionKey?: string;
|
||||
}): string | null {
|
||||
if (!params.appConfig) {
|
||||
return null;
|
||||
}
|
||||
if (params.agentId?.trim()) {
|
||||
return params.agentId.trim();
|
||||
}
|
||||
if (params.agentSessionKey?.trim()) {
|
||||
return resolveSessionAgentId({
|
||||
sessionKey: params.agentSessionKey,
|
||||
config: params.appConfig,
|
||||
});
|
||||
}
|
||||
return resolveDefaultAgentId(params.appConfig);
|
||||
}
|
||||
|
||||
async function resolveActiveMemoryManager(params: {
|
||||
appConfig?: OpenClawConfig;
|
||||
agentId?: string;
|
||||
agentSessionKey?: string;
|
||||
}) {
|
||||
const agentId = resolveActiveMemoryAgentId(params);
|
||||
if (!params.appConfig || !agentId) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const { manager } = await getActiveMemorySearchManager({
|
||||
cfg: params.appConfig,
|
||||
agentId,
|
||||
});
|
||||
return manager;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function buildMemorySearchTitle(resultPath: string): string {
|
||||
const basename = path.basename(resultPath, path.extname(resultPath));
|
||||
return basename.length > 0 ? basename : resultPath;
|
||||
}
|
||||
|
||||
function applySearchOverrides(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
overrides?: QuerySearchOverrides,
|
||||
): ResolvedMemoryWikiConfig {
|
||||
if (!overrides?.searchBackend && !overrides?.searchCorpus) {
|
||||
return config;
|
||||
}
|
||||
return {
|
||||
...config,
|
||||
search: {
|
||||
backend: overrides.searchBackend ?? config.search.backend,
|
||||
corpus: overrides.searchCorpus ?? config.search.corpus,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildWikiProvenanceLabel(
|
||||
page: Pick<
|
||||
WikiPageSummary,
|
||||
| "sourceType"
|
||||
| "provenanceMode"
|
||||
| "bridgeRelativePath"
|
||||
| "unsafeLocalRelativePath"
|
||||
| "relativePath"
|
||||
>,
|
||||
): string | undefined {
|
||||
if (page.sourceType === "memory-bridge-events") {
|
||||
return `bridge events: ${page.bridgeRelativePath ?? page.relativePath}`;
|
||||
}
|
||||
if (page.sourceType === "memory-bridge") {
|
||||
return `bridge: ${page.bridgeRelativePath ?? page.relativePath}`;
|
||||
}
|
||||
if (page.provenanceMode === "unsafe-local" || page.sourceType === "memory-unsafe-local") {
|
||||
return `unsafe-local: ${page.unsafeLocalRelativePath ?? page.relativePath}`;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function toWikiSearchResult(page: QueryableWikiPage, query: string): WikiSearchResult {
|
||||
return {
|
||||
corpus: "wiki",
|
||||
path: page.relativePath,
|
||||
title: page.title,
|
||||
kind: page.kind,
|
||||
score: scorePage(page, query),
|
||||
snippet: buildPageSnippet(page, query),
|
||||
...(page.id ? { id: page.id } : {}),
|
||||
...(page.sourceType ? { sourceType: page.sourceType } : {}),
|
||||
...(page.provenanceMode ? { provenanceMode: page.provenanceMode } : {}),
|
||||
...(page.sourcePath ? { sourcePath: page.sourcePath } : {}),
|
||||
...(buildWikiProvenanceLabel(page) ? { provenanceLabel: buildWikiProvenanceLabel(page) } : {}),
|
||||
...(page.updatedAt ? { updatedAt: page.updatedAt } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function toMemoryWikiSearchResult(result: MemorySearchResult): WikiSearchResult {
|
||||
return {
|
||||
corpus: "memory",
|
||||
path: result.path,
|
||||
title: buildMemorySearchTitle(result.path),
|
||||
kind: "memory",
|
||||
score: result.score,
|
||||
snippet: result.snippet,
|
||||
startLine: result.startLine,
|
||||
endLine: result.endLine,
|
||||
memorySource: result.source,
|
||||
...(result.citation ? { citation: result.citation } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function searchWikiCorpus(params: {
|
||||
rootDir: string;
|
||||
query: string;
|
||||
maxResults: number;
|
||||
}): Promise<WikiSearchResult[]> {
|
||||
const digest = await readQueryDigestBundle(params.rootDir);
|
||||
const candidatePaths = digest
|
||||
? buildDigestCandidatePaths({
|
||||
digest,
|
||||
query: params.query,
|
||||
maxResults: params.maxResults,
|
||||
})
|
||||
: [];
|
||||
const seenPaths = new Set<string>();
|
||||
const candidatePages =
|
||||
candidatePaths.length > 0
|
||||
? await readQueryableWikiPagesByPaths(params.rootDir, candidatePaths)
|
||||
: await readQueryableWikiPages(params.rootDir);
|
||||
for (const page of candidatePages) {
|
||||
seenPaths.add(page.relativePath);
|
||||
}
|
||||
|
||||
const results = candidatePages
|
||||
.map((page) => toWikiSearchResult(page, params.query))
|
||||
.filter((page) => page.score > 0);
|
||||
if (candidatePaths.length === 0 || results.length >= params.maxResults) {
|
||||
return results;
|
||||
}
|
||||
|
||||
const remainingPaths = (await listWikiMarkdownFiles(params.rootDir)).filter(
|
||||
(relativePath) => !seenPaths.has(relativePath),
|
||||
);
|
||||
const remainingPages = await readQueryableWikiPagesByPaths(params.rootDir, remainingPaths);
|
||||
return [
|
||||
...results,
|
||||
...remainingPages
|
||||
.map((page) => toWikiSearchResult(page, params.query))
|
||||
.filter((page) => page.score > 0),
|
||||
];
|
||||
}
|
||||
|
||||
function resolveDigestClaimLookup(digest: QueryDigestBundle, lookup: string): string | null {
|
||||
const trimmed = lookup.trim();
|
||||
const claimId = trimmed.replace(/^claim:/i, "");
|
||||
const match = digest.claims.find((claim) => claim.id === claimId);
|
||||
return match?.pagePath ?? null;
|
||||
}
|
||||
|
||||
export function resolveQueryableWikiPageByLookup(
|
||||
pages: QueryableWikiPage[],
|
||||
lookup: string,
|
||||
): QueryableWikiPage | null {
|
||||
const key = normalizeLookupKey(lookup);
|
||||
const withExtension = key.endsWith(".md") ? key : `${key}.md`;
|
||||
return (
|
||||
pages.find((page) => page.relativePath === key) ??
|
||||
pages.find((page) => page.relativePath === withExtension) ??
|
||||
pages.find((page) => page.relativePath.replace(/\.md$/i, "") === key) ??
|
||||
pages.find((page) => path.basename(page.relativePath, ".md") === key) ??
|
||||
pages.find((page) => page.id === key) ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
export async function searchMemoryWiki(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
agentId?: string;
|
||||
agentSessionKey?: string;
|
||||
query: string;
|
||||
maxResults?: number;
|
||||
searchBackend?: WikiSearchBackend;
|
||||
searchCorpus?: WikiSearchCorpus;
|
||||
}): Promise<WikiSearchResult[]> {
|
||||
const effectiveConfig = applySearchOverrides(params.config, params);
|
||||
await initializeMemoryWikiVault(effectiveConfig);
|
||||
const maxResults = Math.max(1, params.maxResults ?? 10);
|
||||
|
||||
const wikiResults = shouldSearchWiki(effectiveConfig)
|
||||
? await searchWikiCorpus({
|
||||
rootDir: effectiveConfig.vault.path,
|
||||
query: params.query,
|
||||
maxResults,
|
||||
})
|
||||
: [];
|
||||
|
||||
const sharedMemoryManager = shouldSearchSharedMemory(effectiveConfig, params.appConfig)
|
||||
? await resolveActiveMemoryManager({
|
||||
appConfig: params.appConfig,
|
||||
agentId: params.agentId,
|
||||
agentSessionKey: params.agentSessionKey,
|
||||
})
|
||||
: null;
|
||||
const memoryResults = sharedMemoryManager
|
||||
? (await sharedMemoryManager.search(params.query, { maxResults })).map((result) =>
|
||||
toMemoryWikiSearchResult(result),
|
||||
)
|
||||
: [];
|
||||
|
||||
return [...wikiResults, ...memoryResults]
|
||||
.toSorted((left, right) => {
|
||||
if (left.score !== right.score) {
|
||||
return right.score - left.score;
|
||||
}
|
||||
return left.title.localeCompare(right.title);
|
||||
})
|
||||
.slice(0, maxResults);
|
||||
}
|
||||
|
||||
export async function getMemoryWikiPage(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
agentId?: string;
|
||||
agentSessionKey?: string;
|
||||
lookup: string;
|
||||
fromLine?: number;
|
||||
lineCount?: number;
|
||||
searchBackend?: WikiSearchBackend;
|
||||
searchCorpus?: WikiSearchCorpus;
|
||||
}): Promise<WikiGetResult | null> {
|
||||
const effectiveConfig = applySearchOverrides(params.config, params);
|
||||
await initializeMemoryWikiVault(effectiveConfig);
|
||||
const fromLine = Math.max(1, params.fromLine ?? 1);
|
||||
const lineCount = Math.max(1, params.lineCount ?? 200);
|
||||
|
||||
if (shouldSearchWiki(effectiveConfig)) {
|
||||
const digest = await readQueryDigestBundle(effectiveConfig.vault.path);
|
||||
const digestClaimPagePath = digest ? resolveDigestClaimLookup(digest, params.lookup) : null;
|
||||
const digestLookupPage = digestClaimPagePath
|
||||
? ((
|
||||
await readQueryableWikiPagesByPaths(effectiveConfig.vault.path, [digestClaimPagePath])
|
||||
)[0] ?? null)
|
||||
: null;
|
||||
const pages = digestLookupPage
|
||||
? [digestLookupPage]
|
||||
: await readQueryableWikiPages(effectiveConfig.vault.path);
|
||||
const page = digestLookupPage ?? resolveQueryableWikiPageByLookup(pages, params.lookup);
|
||||
if (page) {
|
||||
const parsed = parseWikiMarkdown(page.raw);
|
||||
const lines = parsed.body.split(/\r?\n/);
|
||||
const totalLines = lines.length;
|
||||
const slice = lines.slice(fromLine - 1, fromLine - 1 + lineCount).join("\n");
|
||||
const truncated = fromLine - 1 + lineCount < totalLines;
|
||||
|
||||
return {
|
||||
corpus: "wiki",
|
||||
path: page.relativePath,
|
||||
title: page.title,
|
||||
kind: page.kind,
|
||||
content: slice,
|
||||
fromLine,
|
||||
lineCount,
|
||||
totalLines,
|
||||
truncated,
|
||||
...(page.id ? { id: page.id } : {}),
|
||||
...(page.sourceType ? { sourceType: page.sourceType } : {}),
|
||||
...(page.provenanceMode ? { provenanceMode: page.provenanceMode } : {}),
|
||||
...(page.sourcePath ? { sourcePath: page.sourcePath } : {}),
|
||||
...(buildWikiProvenanceLabel(page)
|
||||
? { provenanceLabel: buildWikiProvenanceLabel(page) }
|
||||
: {}),
|
||||
...(page.updatedAt ? { updatedAt: page.updatedAt } : {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!shouldSearchSharedMemory(effectiveConfig, params.appConfig)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const manager = await resolveActiveMemoryManager({
|
||||
appConfig: params.appConfig,
|
||||
agentId: params.agentId,
|
||||
agentSessionKey: params.agentSessionKey,
|
||||
});
|
||||
if (!manager) {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (const relPath of buildLookupCandidates(params.lookup)) {
|
||||
try {
|
||||
const result = await manager.readFile({
|
||||
relPath,
|
||||
from: fromLine,
|
||||
lines: lineCount,
|
||||
});
|
||||
return {
|
||||
corpus: "memory",
|
||||
path: result.path,
|
||||
title: buildMemorySearchTitle(result.path),
|
||||
kind: "memory",
|
||||
content: result.text,
|
||||
fromLine,
|
||||
lineCount,
|
||||
};
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
61
openclaw/extensions/memory-wiki/src/source-page-shared.ts
Normal file
61
openclaw/extensions/memory-wiki/src/source-page-shared.ts
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathExists } from "./source-path-shared.js";
|
||||
import {
|
||||
setImportedSourceEntry,
|
||||
shouldSkipImportedSourceWrite,
|
||||
type MemoryWikiImportedSourceGroup,
|
||||
} from "./source-sync-state.js";
|
||||
|
||||
type ImportedSourceState = Parameters<typeof shouldSkipImportedSourceWrite>[0]["state"];
|
||||
|
||||
export async function writeImportedSourcePage(params: {
|
||||
vaultRoot: string;
|
||||
syncKey: string;
|
||||
sourcePath: string;
|
||||
sourceUpdatedAtMs: number;
|
||||
sourceSize: number;
|
||||
renderFingerprint: string;
|
||||
pagePath: string;
|
||||
group: MemoryWikiImportedSourceGroup;
|
||||
state: ImportedSourceState;
|
||||
buildRendered: (raw: string, updatedAt: string) => string;
|
||||
}): Promise<{ pagePath: string; changed: boolean; created: boolean }> {
|
||||
const pageAbsPath = path.join(params.vaultRoot, params.pagePath);
|
||||
const created = !(await pathExists(pageAbsPath));
|
||||
const updatedAt = new Date(params.sourceUpdatedAtMs).toISOString();
|
||||
const shouldSkip = await shouldSkipImportedSourceWrite({
|
||||
vaultRoot: params.vaultRoot,
|
||||
syncKey: params.syncKey,
|
||||
expectedPagePath: params.pagePath,
|
||||
expectedSourcePath: params.sourcePath,
|
||||
sourceUpdatedAtMs: params.sourceUpdatedAtMs,
|
||||
sourceSize: params.sourceSize,
|
||||
renderFingerprint: params.renderFingerprint,
|
||||
state: params.state,
|
||||
});
|
||||
if (shouldSkip) {
|
||||
return { pagePath: params.pagePath, changed: false, created };
|
||||
}
|
||||
|
||||
const raw = await fs.readFile(params.sourcePath, "utf8");
|
||||
const rendered = params.buildRendered(raw, updatedAt);
|
||||
const existing = await fs.readFile(pageAbsPath, "utf8").catch(() => "");
|
||||
if (existing !== rendered) {
|
||||
await fs.writeFile(pageAbsPath, rendered, "utf8");
|
||||
}
|
||||
|
||||
setImportedSourceEntry({
|
||||
syncKey: params.syncKey,
|
||||
state: params.state,
|
||||
entry: {
|
||||
group: params.group,
|
||||
pagePath: params.pagePath,
|
||||
sourcePath: params.sourcePath,
|
||||
sourceUpdatedAtMs: params.sourceUpdatedAtMs,
|
||||
sourceSize: params.sourceSize,
|
||||
renderFingerprint: params.renderFingerprint,
|
||||
},
|
||||
});
|
||||
return { pagePath: params.pagePath, changed: existing !== rendered, created };
|
||||
}
|
||||
19
openclaw/extensions/memory-wiki/src/source-path-shared.ts
Normal file
19
openclaw/extensions/memory-wiki/src/source-path-shared.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { lowercasePreservingWhitespace } from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
export async function pathExists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveArtifactKey(absolutePath: string): Promise<string> {
|
||||
const canonicalPath = await fs.realpath(absolutePath).catch(() => path.resolve(absolutePath));
|
||||
return process.platform === "win32"
|
||||
? lowercasePreservingWhitespace(canonicalPath)
|
||||
: canonicalPath;
|
||||
}
|
||||
123
openclaw/extensions/memory-wiki/src/source-sync-state.ts
Normal file
123
openclaw/extensions/memory-wiki/src/source-sync-state.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export type MemoryWikiImportedSourceGroup = "bridge" | "unsafe-local";
|
||||
|
||||
export type MemoryWikiImportedSourceStateEntry = {
|
||||
group: MemoryWikiImportedSourceGroup;
|
||||
pagePath: string;
|
||||
sourcePath: string;
|
||||
sourceUpdatedAtMs: number;
|
||||
sourceSize: number;
|
||||
renderFingerprint: string;
|
||||
};
|
||||
|
||||
type MemoryWikiImportedSourceState = {
|
||||
version: 1;
|
||||
entries: Record<string, MemoryWikiImportedSourceStateEntry>;
|
||||
};
|
||||
|
||||
const EMPTY_STATE: MemoryWikiImportedSourceState = {
|
||||
version: 1,
|
||||
entries: {},
|
||||
};
|
||||
|
||||
export function resolveMemoryWikiSourceSyncStatePath(vaultRoot: string): string {
|
||||
return path.join(vaultRoot, ".openclaw-wiki", "source-sync.json");
|
||||
}
|
||||
|
||||
export async function readMemoryWikiSourceSyncState(
|
||||
vaultRoot: string,
|
||||
): Promise<MemoryWikiImportedSourceState> {
|
||||
const statePath = resolveMemoryWikiSourceSyncStatePath(vaultRoot);
|
||||
const raw = await fs.readFile(statePath, "utf8").catch((err: unknown) => {
|
||||
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
|
||||
return "";
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
if (!raw.trim()) {
|
||||
return {
|
||||
version: EMPTY_STATE.version,
|
||||
entries: {},
|
||||
};
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as Partial<MemoryWikiImportedSourceState>;
|
||||
return {
|
||||
version: 1,
|
||||
entries: { ...parsed.entries },
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
version: EMPTY_STATE.version,
|
||||
entries: {},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeMemoryWikiSourceSyncState(
|
||||
vaultRoot: string,
|
||||
state: MemoryWikiImportedSourceState,
|
||||
): Promise<void> {
|
||||
const statePath = resolveMemoryWikiSourceSyncStatePath(vaultRoot);
|
||||
await fs.mkdir(path.dirname(statePath), { recursive: true });
|
||||
await fs.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
export async function shouldSkipImportedSourceWrite(params: {
|
||||
vaultRoot: string;
|
||||
syncKey: string;
|
||||
expectedPagePath: string;
|
||||
expectedSourcePath: string;
|
||||
sourceUpdatedAtMs: number;
|
||||
sourceSize: number;
|
||||
renderFingerprint: string;
|
||||
state: MemoryWikiImportedSourceState;
|
||||
}): Promise<boolean> {
|
||||
const entry = params.state.entries[params.syncKey];
|
||||
if (!entry) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
entry.pagePath !== params.expectedPagePath ||
|
||||
entry.sourcePath !== params.expectedSourcePath ||
|
||||
entry.sourceUpdatedAtMs !== params.sourceUpdatedAtMs ||
|
||||
entry.sourceSize !== params.sourceSize ||
|
||||
entry.renderFingerprint !== params.renderFingerprint
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const pagePath = path.join(params.vaultRoot, params.expectedPagePath);
|
||||
return await fs
|
||||
.access(pagePath)
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
}
|
||||
|
||||
export async function pruneImportedSourceEntries(params: {
|
||||
vaultRoot: string;
|
||||
group: MemoryWikiImportedSourceGroup;
|
||||
activeKeys: Set<string>;
|
||||
state: MemoryWikiImportedSourceState;
|
||||
}): Promise<number> {
|
||||
let removedCount = 0;
|
||||
for (const [syncKey, entry] of Object.entries(params.state.entries)) {
|
||||
if (entry.group !== params.group || params.activeKeys.has(syncKey)) {
|
||||
continue;
|
||||
}
|
||||
const pageAbsPath = path.join(params.vaultRoot, entry.pagePath);
|
||||
await fs.rm(pageAbsPath, { force: true }).catch(() => undefined);
|
||||
delete params.state.entries[syncKey];
|
||||
removedCount += 1;
|
||||
}
|
||||
return removedCount;
|
||||
}
|
||||
|
||||
export function setImportedSourceEntry(params: {
|
||||
syncKey: string;
|
||||
entry: MemoryWikiImportedSourceStateEntry;
|
||||
state: MemoryWikiImportedSourceState;
|
||||
}): void {
|
||||
params.state.entries[params.syncKey] = params.entry;
|
||||
}
|
||||
139
openclaw/extensions/memory-wiki/src/source-sync.test.ts
Normal file
139
openclaw/extensions/memory-wiki/src/source-sync.test.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { syncMemoryWikiImportedSources } from "./source-sync.js";
|
||||
|
||||
const { syncBridgeMock, syncUnsafeLocalMock, refreshIndexesMock } = vi.hoisted(() => ({
|
||||
syncBridgeMock: vi.fn(),
|
||||
syncUnsafeLocalMock: vi.fn(),
|
||||
refreshIndexesMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./bridge.js", () => ({
|
||||
syncMemoryWikiBridgeSources: syncBridgeMock,
|
||||
}));
|
||||
|
||||
vi.mock("./unsafe-local.js", () => ({
|
||||
syncMemoryWikiUnsafeLocalSources: syncUnsafeLocalMock,
|
||||
}));
|
||||
|
||||
vi.mock("./compile.js", () => ({
|
||||
refreshMemoryWikiIndexesAfterImport: refreshIndexesMock,
|
||||
}));
|
||||
|
||||
const bridgeResult = {
|
||||
importedCount: 1,
|
||||
updatedCount: 2,
|
||||
skippedCount: 3,
|
||||
removedCount: 4,
|
||||
artifactCount: 10,
|
||||
workspaces: 2,
|
||||
pagePaths: ["sources/alpha.md"],
|
||||
};
|
||||
|
||||
describe("syncMemoryWikiImportedSources", () => {
|
||||
beforeEach(() => {
|
||||
syncBridgeMock.mockReset();
|
||||
syncUnsafeLocalMock.mockReset();
|
||||
refreshIndexesMock.mockReset();
|
||||
syncBridgeMock.mockResolvedValue(bridgeResult);
|
||||
syncUnsafeLocalMock.mockResolvedValue({
|
||||
...bridgeResult,
|
||||
workspaces: 0,
|
||||
});
|
||||
refreshIndexesMock.mockResolvedValue({
|
||||
refreshed: true,
|
||||
reason: "import-changed",
|
||||
compile: { updatedFiles: ["index.md", "sources/index.md"] },
|
||||
});
|
||||
});
|
||||
|
||||
it("routes bridge mode through bridge sync and merges refresh results", async () => {
|
||||
const config = { vaultMode: "bridge" } as Parameters<
|
||||
typeof syncMemoryWikiImportedSources
|
||||
>[0]["config"];
|
||||
const appConfig = { agents: { list: [{ id: "main", default: true }] } } as Parameters<
|
||||
typeof syncMemoryWikiImportedSources
|
||||
>[0]["appConfig"];
|
||||
|
||||
const result = await syncMemoryWikiImportedSources({ config, appConfig });
|
||||
|
||||
expect(syncBridgeMock).toHaveBeenCalledWith({ config, appConfig });
|
||||
expect(syncUnsafeLocalMock).not.toHaveBeenCalled();
|
||||
expect(refreshIndexesMock).toHaveBeenCalledWith({
|
||||
config,
|
||||
syncResult: bridgeResult,
|
||||
});
|
||||
expect(result).toEqual({
|
||||
...bridgeResult,
|
||||
indexesRefreshed: true,
|
||||
indexRefreshReason: "import-changed",
|
||||
indexUpdatedFiles: ["index.md", "sources/index.md"],
|
||||
});
|
||||
});
|
||||
|
||||
it("routes unsafe-local mode through unsafe-local sync", async () => {
|
||||
const unsafeLocalResult = {
|
||||
...bridgeResult,
|
||||
importedCount: 2,
|
||||
workspaces: 0,
|
||||
pagePaths: ["sources/private.md"],
|
||||
};
|
||||
syncUnsafeLocalMock.mockResolvedValueOnce(unsafeLocalResult);
|
||||
refreshIndexesMock.mockResolvedValueOnce({
|
||||
refreshed: false,
|
||||
reason: "auto-compile-disabled",
|
||||
});
|
||||
const config = { vaultMode: "unsafe-local" } as Parameters<
|
||||
typeof syncMemoryWikiImportedSources
|
||||
>[0]["config"];
|
||||
|
||||
const result = await syncMemoryWikiImportedSources({ config });
|
||||
|
||||
expect(syncUnsafeLocalMock).toHaveBeenCalledWith(config);
|
||||
expect(syncBridgeMock).not.toHaveBeenCalled();
|
||||
expect(refreshIndexesMock).toHaveBeenCalledWith({
|
||||
config,
|
||||
syncResult: unsafeLocalResult,
|
||||
});
|
||||
expect(result).toEqual({
|
||||
...unsafeLocalResult,
|
||||
indexesRefreshed: false,
|
||||
indexRefreshReason: "auto-compile-disabled",
|
||||
indexUpdatedFiles: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a no-op sync result outside imported-source modes", async () => {
|
||||
const config = { vaultMode: "isolated" } as Parameters<
|
||||
typeof syncMemoryWikiImportedSources
|
||||
>[0]["config"];
|
||||
|
||||
const result = await syncMemoryWikiImportedSources({ config });
|
||||
|
||||
expect(syncBridgeMock).not.toHaveBeenCalled();
|
||||
expect(syncUnsafeLocalMock).not.toHaveBeenCalled();
|
||||
expect(refreshIndexesMock).toHaveBeenCalledWith({
|
||||
config,
|
||||
syncResult: {
|
||||
importedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
removedCount: 0,
|
||||
artifactCount: 0,
|
||||
workspaces: 0,
|
||||
pagePaths: [],
|
||||
},
|
||||
});
|
||||
expect(result).toEqual({
|
||||
importedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
removedCount: 0,
|
||||
artifactCount: 0,
|
||||
workspaces: 0,
|
||||
pagePaths: [],
|
||||
indexesRefreshed: true,
|
||||
indexRefreshReason: "import-changed",
|
||||
indexUpdatedFiles: ["index.md", "sources/index.md"],
|
||||
});
|
||||
});
|
||||
});
|
||||
46
openclaw/extensions/memory-wiki/src/source-sync.ts
Normal file
46
openclaw/extensions/memory-wiki/src/source-sync.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
import type { OpenClawConfig } from "../api.js";
|
||||
import { syncMemoryWikiBridgeSources, type BridgeMemoryWikiResult } from "./bridge.js";
|
||||
import {
|
||||
refreshMemoryWikiIndexesAfterImport,
|
||||
type RefreshMemoryWikiIndexesResult,
|
||||
} from "./compile.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { syncMemoryWikiUnsafeLocalSources } from "./unsafe-local.js";
|
||||
|
||||
export type MemoryWikiImportedSourceSyncResult = BridgeMemoryWikiResult & {
|
||||
indexesRefreshed: boolean;
|
||||
indexUpdatedFiles: string[];
|
||||
indexRefreshReason: RefreshMemoryWikiIndexesResult["reason"];
|
||||
};
|
||||
|
||||
export async function syncMemoryWikiImportedSources(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
appConfig?: OpenClawConfig;
|
||||
}): Promise<MemoryWikiImportedSourceSyncResult> {
|
||||
let syncResult: BridgeMemoryWikiResult;
|
||||
if (params.config.vaultMode === "bridge") {
|
||||
syncResult = await syncMemoryWikiBridgeSources(params);
|
||||
} else if (params.config.vaultMode === "unsafe-local") {
|
||||
syncResult = await syncMemoryWikiUnsafeLocalSources(params.config);
|
||||
} else {
|
||||
syncResult = {
|
||||
importedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
removedCount: 0,
|
||||
artifactCount: 0,
|
||||
workspaces: 0,
|
||||
pagePaths: [],
|
||||
};
|
||||
}
|
||||
const refreshResult = await refreshMemoryWikiIndexesAfterImport({
|
||||
config: params.config,
|
||||
syncResult,
|
||||
});
|
||||
return {
|
||||
...syncResult,
|
||||
indexesRefreshed: refreshResult.refreshed,
|
||||
indexUpdatedFiles: refreshResult.compile?.updatedFiles ?? [],
|
||||
indexRefreshReason: refreshResult.reason,
|
||||
};
|
||||
}
|
||||
248
openclaw/extensions/memory-wiki/src/status.test.ts
Normal file
248
openclaw/extensions/memory-wiki/src/status.test.ts
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
import { resolveMemoryWikiConfig } from "./config.js";
|
||||
import { renderWikiMarkdown } from "./markdown.js";
|
||||
import {
|
||||
buildMemoryWikiDoctorReport,
|
||||
renderMemoryWikiDoctor,
|
||||
renderMemoryWikiStatus,
|
||||
resolveMemoryWikiStatus,
|
||||
} from "./status.js";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
async function resolveBridgeMissingArtifactsStatus() {
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{
|
||||
vaultMode: "bridge",
|
||||
bridge: {
|
||||
enabled: true,
|
||||
readMemoryArtifacts: true,
|
||||
},
|
||||
},
|
||||
{ homedir: "/Users/tester" },
|
||||
);
|
||||
|
||||
return resolveMemoryWikiStatus(config, {
|
||||
appConfig: {
|
||||
agents: {
|
||||
list: [{ id: "main", default: true, workspace: "/tmp/workspace" }],
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
listPublicArtifacts: async () => [],
|
||||
pathExists: async () => true,
|
||||
resolveCommand: async () => null,
|
||||
});
|
||||
}
|
||||
|
||||
describe("resolveMemoryWikiStatus", () => {
|
||||
it("reports missing vault and missing requested obsidian cli", async () => {
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{
|
||||
vault: { path: "/tmp/wiki" },
|
||||
obsidian: { enabled: true, useOfficialCli: true },
|
||||
},
|
||||
{ homedir: "/Users/tester" },
|
||||
);
|
||||
|
||||
const status = await resolveMemoryWikiStatus(config, {
|
||||
pathExists: async () => false,
|
||||
resolveCommand: async () => null,
|
||||
});
|
||||
|
||||
expect(status.vaultExists).toBe(false);
|
||||
expect(status.obsidianCli.requested).toBe(true);
|
||||
expect(status.warnings.map((warning) => warning.code)).toEqual([
|
||||
"vault-missing",
|
||||
"obsidian-cli-missing",
|
||||
]);
|
||||
expect(status.sourceCounts).toEqual({
|
||||
native: 0,
|
||||
bridge: 0,
|
||||
bridgeEvents: 0,
|
||||
unsafeLocal: 0,
|
||||
other: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("warns when unsafe-local is selected without explicit private access", async () => {
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{
|
||||
vaultMode: "unsafe-local",
|
||||
},
|
||||
{ homedir: "/Users/tester" },
|
||||
);
|
||||
|
||||
const status = await resolveMemoryWikiStatus(config, {
|
||||
pathExists: async () => true,
|
||||
resolveCommand: async () => "/usr/local/bin/obsidian",
|
||||
});
|
||||
|
||||
expect(status.warnings.map((warning) => warning.code)).toContain("unsafe-local-disabled");
|
||||
});
|
||||
|
||||
it("warns when bridge mode has no exported memory artifacts", async () => {
|
||||
const status = await resolveBridgeMissingArtifactsStatus();
|
||||
|
||||
expect(status.bridgePublicArtifactCount).toBe(0);
|
||||
expect(status.warnings.map((warning) => warning.code)).toContain("bridge-artifacts-missing");
|
||||
});
|
||||
|
||||
it("counts source provenance from the vault", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-status-",
|
||||
initialize: true,
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "native.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.native", title: "Native Source" },
|
||||
body: "# Native Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "bridge.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.bridge",
|
||||
title: "Bridge Source",
|
||||
sourceType: "memory-bridge",
|
||||
},
|
||||
body: "# Bridge Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "events.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.events",
|
||||
title: "Event Source",
|
||||
sourceType: "memory-bridge-events",
|
||||
},
|
||||
body: "# Event Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "unsafe.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: "source.unsafe",
|
||||
title: "Unsafe Source",
|
||||
sourceType: "memory-unsafe-local",
|
||||
provenanceMode: "unsafe-local",
|
||||
},
|
||||
body: "# Unsafe Source\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const status = await resolveMemoryWikiStatus(config, {
|
||||
pathExists: async () => true,
|
||||
resolveCommand: async () => null,
|
||||
});
|
||||
|
||||
expect(status.pageCounts.source).toBe(4);
|
||||
expect(status.sourceCounts).toEqual({
|
||||
native: 1,
|
||||
bridge: 1,
|
||||
bridgeEvents: 1,
|
||||
unsafeLocal: 1,
|
||||
other: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderMemoryWikiStatus", () => {
|
||||
it("includes warnings in the text output", () => {
|
||||
const rendered = renderMemoryWikiStatus({
|
||||
vaultMode: "isolated",
|
||||
renderMode: "native",
|
||||
vaultPath: "/tmp/wiki",
|
||||
vaultExists: false,
|
||||
bridge: {
|
||||
enabled: false,
|
||||
readMemoryArtifacts: true,
|
||||
indexDreamReports: true,
|
||||
indexDailyNotes: true,
|
||||
indexMemoryRoot: true,
|
||||
followMemoryEvents: true,
|
||||
},
|
||||
bridgePublicArtifactCount: null,
|
||||
obsidianCli: {
|
||||
enabled: true,
|
||||
requested: true,
|
||||
available: false,
|
||||
command: null,
|
||||
},
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: false,
|
||||
pathCount: 0,
|
||||
},
|
||||
pageCounts: {
|
||||
source: 0,
|
||||
entity: 0,
|
||||
concept: 0,
|
||||
synthesis: 0,
|
||||
report: 0,
|
||||
},
|
||||
sourceCounts: {
|
||||
native: 0,
|
||||
bridge: 0,
|
||||
bridgeEvents: 0,
|
||||
unsafeLocal: 0,
|
||||
other: 0,
|
||||
},
|
||||
warnings: [{ code: "vault-missing", message: "Wiki vault has not been initialized yet." }],
|
||||
});
|
||||
|
||||
expect(rendered).toContain("Wiki vault mode: isolated");
|
||||
expect(rendered).toContain("Pages: 0 sources, 0 entities, 0 concepts, 0 syntheses, 0 reports");
|
||||
expect(rendered).toContain(
|
||||
"Source provenance: 0 native, 0 bridge, 0 bridge-events, 0 unsafe-local, 0 other",
|
||||
);
|
||||
expect(rendered).toContain("Warnings:");
|
||||
expect(rendered).toContain("Wiki vault has not been initialized yet.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("memory wiki doctor", () => {
|
||||
it("builds actionable fixes from status warnings", async () => {
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{
|
||||
vault: { path: "/tmp/wiki" },
|
||||
obsidian: { enabled: true, useOfficialCli: true },
|
||||
},
|
||||
{ homedir: "/Users/tester" },
|
||||
);
|
||||
|
||||
const status = await resolveMemoryWikiStatus(config, {
|
||||
pathExists: async () => false,
|
||||
resolveCommand: async () => null,
|
||||
});
|
||||
const report = buildMemoryWikiDoctorReport(status);
|
||||
const rendered = renderMemoryWikiDoctor(report);
|
||||
|
||||
expect(report.healthy).toBe(false);
|
||||
expect(report.warningCount).toBe(2);
|
||||
expect(report.fixes.map((fix) => fix.code)).toEqual(["vault-missing", "obsidian-cli-missing"]);
|
||||
expect(rendered).toContain("Suggested fixes:");
|
||||
expect(rendered).toContain("openclaw wiki init");
|
||||
});
|
||||
|
||||
it("suggests bridge fixes when no public artifacts are exported", async () => {
|
||||
const status = await resolveBridgeMissingArtifactsStatus();
|
||||
const report = buildMemoryWikiDoctorReport(status);
|
||||
|
||||
expect(report.fixes.map((fix) => fix.code)).toContain("bridge-artifacts-missing");
|
||||
expect(renderMemoryWikiDoctor(report)).toContain("exports public artifacts");
|
||||
});
|
||||
});
|
||||
339
openclaw/extensions/memory-wiki/src/status.ts
Normal file
339
openclaw/extensions/memory-wiki/src/status.ts
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { listActiveMemoryPublicArtifacts } from "openclaw/plugin-sdk/memory-host-core";
|
||||
import type { OpenClawConfig } from "../api.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { inferWikiPageKind, toWikiPageSummary, type WikiPageKind } from "./markdown.js";
|
||||
import { probeObsidianCli } from "./obsidian.js";
|
||||
|
||||
export type MemoryWikiStatusWarning = {
|
||||
code:
|
||||
| "vault-missing"
|
||||
| "obsidian-cli-missing"
|
||||
| "bridge-disabled"
|
||||
| "bridge-artifacts-missing"
|
||||
| "unsafe-local-disabled"
|
||||
| "unsafe-local-paths-missing"
|
||||
| "unsafe-local-without-mode";
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type MemoryWikiStatus = {
|
||||
vaultMode: ResolvedMemoryWikiConfig["vaultMode"];
|
||||
renderMode: ResolvedMemoryWikiConfig["vault"]["renderMode"];
|
||||
vaultPath: string;
|
||||
vaultExists: boolean;
|
||||
bridge: ResolvedMemoryWikiConfig["bridge"];
|
||||
bridgePublicArtifactCount: number | null;
|
||||
obsidianCli: {
|
||||
enabled: boolean;
|
||||
requested: boolean;
|
||||
available: boolean;
|
||||
command: string | null;
|
||||
};
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: boolean;
|
||||
pathCount: number;
|
||||
};
|
||||
pageCounts: Record<WikiPageKind, number>;
|
||||
sourceCounts: {
|
||||
native: number;
|
||||
bridge: number;
|
||||
bridgeEvents: number;
|
||||
unsafeLocal: number;
|
||||
other: number;
|
||||
};
|
||||
warnings: MemoryWikiStatusWarning[];
|
||||
};
|
||||
|
||||
export type MemoryWikiDoctorFix = {
|
||||
code: MemoryWikiStatusWarning["code"];
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type MemoryWikiDoctorReport = {
|
||||
healthy: boolean;
|
||||
warningCount: number;
|
||||
status: MemoryWikiStatus;
|
||||
fixes: MemoryWikiDoctorFix[];
|
||||
};
|
||||
|
||||
type ResolveMemoryWikiStatusDeps = {
|
||||
appConfig?: OpenClawConfig;
|
||||
pathExists?: (inputPath: string) => Promise<boolean>;
|
||||
listPublicArtifacts?: typeof listActiveMemoryPublicArtifacts;
|
||||
resolveCommand?: (command: string) => Promise<string | null>;
|
||||
};
|
||||
|
||||
async function pathExists(inputPath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(inputPath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function collectVaultCounts(vaultPath: string): Promise<{
|
||||
pageCounts: Record<WikiPageKind, number>;
|
||||
sourceCounts: MemoryWikiStatus["sourceCounts"];
|
||||
}> {
|
||||
const pageCounts: Record<WikiPageKind, number> = {
|
||||
entity: 0,
|
||||
concept: 0,
|
||||
source: 0,
|
||||
synthesis: 0,
|
||||
report: 0,
|
||||
};
|
||||
const sourceCounts: MemoryWikiStatus["sourceCounts"] = {
|
||||
native: 0,
|
||||
bridge: 0,
|
||||
bridgeEvents: 0,
|
||||
unsafeLocal: 0,
|
||||
other: 0,
|
||||
};
|
||||
const dirs = ["entities", "concepts", "sources", "syntheses", "reports"] as const;
|
||||
for (const dir of dirs) {
|
||||
const entries = await fs
|
||||
.readdir(path.join(vaultPath, dir), { withFileTypes: true })
|
||||
.catch(() => []);
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !entry.name.endsWith(".md") || entry.name === "index.md") {
|
||||
continue;
|
||||
}
|
||||
const kind = inferWikiPageKind(path.join(dir, entry.name));
|
||||
if (kind) {
|
||||
pageCounts[kind] += 1;
|
||||
}
|
||||
if (dir === "sources") {
|
||||
const absolutePath = path.join(vaultPath, dir, entry.name);
|
||||
const raw = await fs.readFile(absolutePath, "utf8").catch(() => null);
|
||||
if (!raw) {
|
||||
continue;
|
||||
}
|
||||
const page = toWikiPageSummary({
|
||||
absolutePath,
|
||||
relativePath: path.join(dir, entry.name),
|
||||
raw,
|
||||
});
|
||||
if (!page) {
|
||||
continue;
|
||||
}
|
||||
if (page.sourceType === "memory-bridge-events") {
|
||||
sourceCounts.bridgeEvents += 1;
|
||||
} else if (page.sourceType === "memory-bridge") {
|
||||
sourceCounts.bridge += 1;
|
||||
} else if (
|
||||
page.provenanceMode === "unsafe-local" ||
|
||||
page.sourceType === "memory-unsafe-local"
|
||||
) {
|
||||
sourceCounts.unsafeLocal += 1;
|
||||
} else if (!page.sourceType) {
|
||||
sourceCounts.native += 1;
|
||||
} else {
|
||||
sourceCounts.other += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return { pageCounts, sourceCounts };
|
||||
}
|
||||
|
||||
function buildWarnings(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
bridgePublicArtifactCount: number | null;
|
||||
vaultExists: boolean;
|
||||
obsidianCommand: string | null;
|
||||
}): MemoryWikiStatusWarning[] {
|
||||
const warnings: MemoryWikiStatusWarning[] = [];
|
||||
if (!params.vaultExists) {
|
||||
warnings.push({
|
||||
code: "vault-missing",
|
||||
message: "Wiki vault has not been initialized yet.",
|
||||
});
|
||||
}
|
||||
if (
|
||||
params.config.obsidian.enabled &&
|
||||
params.config.obsidian.useOfficialCli &&
|
||||
!params.obsidianCommand
|
||||
) {
|
||||
warnings.push({
|
||||
code: "obsidian-cli-missing",
|
||||
message: "Obsidian CLI is enabled in config but `obsidian` is not available on PATH.",
|
||||
});
|
||||
}
|
||||
if (params.config.vaultMode === "bridge" && !params.config.bridge.enabled) {
|
||||
warnings.push({
|
||||
code: "bridge-disabled",
|
||||
message: "vaultMode is `bridge` but bridge.enabled is false.",
|
||||
});
|
||||
}
|
||||
if (
|
||||
params.config.vaultMode === "bridge" &&
|
||||
params.config.bridge.enabled &&
|
||||
params.config.bridge.readMemoryArtifacts &&
|
||||
params.bridgePublicArtifactCount === 0
|
||||
) {
|
||||
warnings.push({
|
||||
code: "bridge-artifacts-missing",
|
||||
message:
|
||||
"Bridge mode is enabled but the active memory plugin is not exporting any public memory artifacts yet.",
|
||||
});
|
||||
}
|
||||
if (
|
||||
params.config.vaultMode === "unsafe-local" &&
|
||||
!params.config.unsafeLocal.allowPrivateMemoryCoreAccess
|
||||
) {
|
||||
warnings.push({
|
||||
code: "unsafe-local-disabled",
|
||||
message: "vaultMode is `unsafe-local` but private memory-core access is disabled.",
|
||||
});
|
||||
}
|
||||
if (
|
||||
params.config.vaultMode === "unsafe-local" &&
|
||||
params.config.unsafeLocal.allowPrivateMemoryCoreAccess &&
|
||||
params.config.unsafeLocal.paths.length === 0
|
||||
) {
|
||||
warnings.push({
|
||||
code: "unsafe-local-paths-missing",
|
||||
message: "unsafe-local access is enabled but no private paths are configured.",
|
||||
});
|
||||
}
|
||||
if (
|
||||
params.config.vaultMode !== "unsafe-local" &&
|
||||
params.config.unsafeLocal.allowPrivateMemoryCoreAccess
|
||||
) {
|
||||
warnings.push({
|
||||
code: "unsafe-local-without-mode",
|
||||
message: "Private memory-core access is enabled outside unsafe-local mode.",
|
||||
});
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
||||
export async function resolveMemoryWikiStatus(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
deps?: ResolveMemoryWikiStatusDeps,
|
||||
): Promise<MemoryWikiStatus> {
|
||||
const exists = deps?.pathExists ?? pathExists;
|
||||
const vaultExists = await exists(config.vault.path);
|
||||
const bridgePublicArtifactCount =
|
||||
deps?.appConfig && config.vaultMode === "bridge" && config.bridge.enabled
|
||||
? (
|
||||
await (deps.listPublicArtifacts ?? listActiveMemoryPublicArtifacts)({
|
||||
cfg: deps.appConfig,
|
||||
})
|
||||
).length
|
||||
: null;
|
||||
const obsidianProbe = await probeObsidianCli({ resolveCommand: deps?.resolveCommand });
|
||||
const counts = vaultExists
|
||||
? await collectVaultCounts(config.vault.path)
|
||||
: {
|
||||
pageCounts: {
|
||||
entity: 0,
|
||||
concept: 0,
|
||||
source: 0,
|
||||
synthesis: 0,
|
||||
report: 0,
|
||||
},
|
||||
sourceCounts: {
|
||||
native: 0,
|
||||
bridge: 0,
|
||||
bridgeEvents: 0,
|
||||
unsafeLocal: 0,
|
||||
other: 0,
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
vaultMode: config.vaultMode,
|
||||
renderMode: config.vault.renderMode,
|
||||
vaultPath: config.vault.path,
|
||||
vaultExists,
|
||||
bridge: config.bridge,
|
||||
bridgePublicArtifactCount,
|
||||
obsidianCli: {
|
||||
enabled: config.obsidian.enabled,
|
||||
requested: config.obsidian.enabled && config.obsidian.useOfficialCli,
|
||||
available: obsidianProbe.available,
|
||||
command: obsidianProbe.command,
|
||||
},
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: config.unsafeLocal.allowPrivateMemoryCoreAccess,
|
||||
pathCount: config.unsafeLocal.paths.length,
|
||||
},
|
||||
pageCounts: counts.pageCounts,
|
||||
sourceCounts: counts.sourceCounts,
|
||||
warnings: buildWarnings({
|
||||
config,
|
||||
bridgePublicArtifactCount,
|
||||
vaultExists,
|
||||
obsidianCommand: obsidianProbe.command,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildMemoryWikiDoctorReport(status: MemoryWikiStatus): MemoryWikiDoctorReport {
|
||||
const fixes = status.warnings.map((warning) => ({
|
||||
code: warning.code,
|
||||
message:
|
||||
warning.code === "vault-missing"
|
||||
? "Run `openclaw wiki init` to create the vault layout."
|
||||
: warning.code === "obsidian-cli-missing"
|
||||
? "Install the official Obsidian CLI or disable `obsidian.useOfficialCli`."
|
||||
: warning.code === "bridge-disabled"
|
||||
? "Enable `plugins.entries.memory-wiki.config.bridge.enabled` or switch vaultMode away from `bridge`."
|
||||
: warning.code === "bridge-artifacts-missing"
|
||||
? "Use a memory plugin that exports public artifacts, create/import memory artifacts first, or switch the wiki back to isolated mode."
|
||||
: warning.code === "unsafe-local-disabled"
|
||||
? "Enable `unsafeLocal.allowPrivateMemoryCoreAccess` or switch vaultMode away from `unsafe-local`."
|
||||
: warning.code === "unsafe-local-paths-missing"
|
||||
? "Add explicit `unsafeLocal.paths` entries before running unsafe-local imports."
|
||||
: "Disable private memory-core access unless you explicitly want unsafe-local mode.",
|
||||
}));
|
||||
return {
|
||||
healthy: status.warnings.length === 0,
|
||||
warningCount: status.warnings.length,
|
||||
status,
|
||||
fixes,
|
||||
};
|
||||
}
|
||||
|
||||
export function renderMemoryWikiStatus(status: MemoryWikiStatus): string {
|
||||
const lines = [
|
||||
`Wiki vault mode: ${status.vaultMode}`,
|
||||
`Vault: ${status.vaultExists ? "ready" : "missing"} (${status.vaultPath})`,
|
||||
`Render mode: ${status.renderMode}`,
|
||||
`Obsidian CLI: ${status.obsidianCli.available ? "available" : "missing"}${status.obsidianCli.requested ? " (requested)" : ""}`,
|
||||
`Bridge: ${status.bridge.enabled ? "enabled" : "disabled"}${typeof status.bridgePublicArtifactCount === "number" ? ` (${status.bridgePublicArtifactCount} exported artifact${status.bridgePublicArtifactCount === 1 ? "" : "s"})` : ""}`,
|
||||
`Unsafe local: ${status.unsafeLocal.allowPrivateMemoryCoreAccess ? `enabled (${status.unsafeLocal.pathCount} paths)` : "disabled"}`,
|
||||
`Pages: ${status.pageCounts.source} sources, ${status.pageCounts.entity} entities, ${status.pageCounts.concept} concepts, ${status.pageCounts.synthesis} syntheses, ${status.pageCounts.report} reports`,
|
||||
`Source provenance: ${status.sourceCounts.native} native, ${status.sourceCounts.bridge} bridge, ${status.sourceCounts.bridgeEvents} bridge-events, ${status.sourceCounts.unsafeLocal} unsafe-local, ${status.sourceCounts.other} other`,
|
||||
];
|
||||
|
||||
if (status.warnings.length > 0) {
|
||||
lines.push("", "Warnings:");
|
||||
for (const warning of status.warnings) {
|
||||
lines.push(`- ${warning.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function renderMemoryWikiDoctor(report: MemoryWikiDoctorReport): string {
|
||||
const lines = [
|
||||
report.healthy ? "Wiki doctor: healthy" : `Wiki doctor: ${report.warningCount} issue(s) found`,
|
||||
"",
|
||||
renderMemoryWikiStatus(report.status),
|
||||
];
|
||||
|
||||
if (report.fixes.length > 0) {
|
||||
lines.push("", "Suggested fixes:");
|
||||
for (const fix of report.fixes) {
|
||||
lines.push(`- ${fix.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
105
openclaw/extensions/memory-wiki/src/test-helpers.ts
Normal file
105
openclaw/extensions/memory-wiki/src/test-helpers.ts
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
||||
import { afterEach, vi } from "vitest";
|
||||
import { createTestPluginApi } from "../../../test/helpers/plugins/plugin-api.js";
|
||||
import type { OpenClawPluginApi } from "../api.js";
|
||||
import {
|
||||
resolveMemoryWikiConfig,
|
||||
type MemoryWikiPluginConfig,
|
||||
type ResolvedMemoryWikiConfig,
|
||||
} from "./config.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
const MEMORY_WIKI_TEST_HOME = "/Users/tester";
|
||||
|
||||
export type MemoryWikiTestVault = {
|
||||
rootDir: string;
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
};
|
||||
|
||||
export type MemoryWikiPluginApiHarness = {
|
||||
api: OpenClawPluginApi;
|
||||
registerCli: ReturnType<typeof vi.fn>;
|
||||
registerGatewayMethod: ReturnType<typeof vi.fn>;
|
||||
registerMemoryCorpusSupplement: ReturnType<typeof vi.fn>;
|
||||
registerMemoryPromptSupplement: ReturnType<typeof vi.fn>;
|
||||
registerTool: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
export function createMemoryWikiTestHarness() {
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
async function createTempDir(prefix: string): Promise<string> {
|
||||
const tempDir = await fs.mkdtemp(path.join(resolvePreferredOpenClawTmpDir(), prefix));
|
||||
tempDirs.push(tempDir);
|
||||
return tempDir;
|
||||
}
|
||||
|
||||
async function createVault(options?: {
|
||||
prefix?: string;
|
||||
rootDir?: string;
|
||||
config?: MemoryWikiPluginConfig;
|
||||
initialize?: boolean;
|
||||
}): Promise<MemoryWikiTestVault> {
|
||||
const rootDir =
|
||||
options?.rootDir ?? (await createTempDir(options?.prefix ?? "memory-wiki-test-"));
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{
|
||||
...options?.config,
|
||||
vault: {
|
||||
...options?.config?.vault,
|
||||
path: rootDir,
|
||||
},
|
||||
},
|
||||
{ homedir: MEMORY_WIKI_TEST_HOME },
|
||||
);
|
||||
|
||||
if (options?.initialize) {
|
||||
await initializeMemoryWikiVault(config);
|
||||
}
|
||||
|
||||
return { rootDir, config };
|
||||
}
|
||||
|
||||
function createPluginApi(): MemoryWikiPluginApiHarness {
|
||||
const registerCli = vi.fn();
|
||||
const registerGatewayMethod = vi.fn();
|
||||
const registerMemoryCorpusSupplement = vi.fn();
|
||||
const registerMemoryPromptSupplement = vi.fn();
|
||||
const registerTool = vi.fn();
|
||||
const api = createTestPluginApi({
|
||||
id: "memory-wiki",
|
||||
name: "Memory Wiki",
|
||||
source: "test",
|
||||
config: {},
|
||||
runtime: {} as OpenClawPluginApi["runtime"],
|
||||
registerCli,
|
||||
registerGatewayMethod,
|
||||
registerMemoryCorpusSupplement,
|
||||
registerMemoryPromptSupplement,
|
||||
registerTool,
|
||||
});
|
||||
|
||||
return {
|
||||
api,
|
||||
registerCli,
|
||||
registerGatewayMethod,
|
||||
registerMemoryCorpusSupplement,
|
||||
registerMemoryPromptSupplement,
|
||||
registerTool,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
createPluginApi,
|
||||
createTempDir,
|
||||
createVault,
|
||||
};
|
||||
}
|
||||
269
openclaw/extensions/memory-wiki/src/tool.ts
Normal file
269
openclaw/extensions/memory-wiki/src/tool.ts
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
import { Type } from "@sinclair/typebox";
|
||||
import type { AnyAgentTool, OpenClawConfig } from "../api.js";
|
||||
import { applyMemoryWikiMutation, normalizeMemoryWikiMutationInput } from "./apply.js";
|
||||
import {
|
||||
WIKI_SEARCH_BACKENDS,
|
||||
WIKI_SEARCH_CORPORA,
|
||||
type ResolvedMemoryWikiConfig,
|
||||
} from "./config.js";
|
||||
import { lintMemoryWikiVault } from "./lint.js";
|
||||
import { getMemoryWikiPage, searchMemoryWiki } from "./query.js";
|
||||
import { syncMemoryWikiImportedSources } from "./source-sync.js";
|
||||
import { renderMemoryWikiStatus, resolveMemoryWikiStatus } from "./status.js";
|
||||
|
||||
const WikiStatusSchema = Type.Object({}, { additionalProperties: false });
|
||||
const WikiLintSchema = Type.Object({}, { additionalProperties: false });
|
||||
const WikiSearchBackendSchema = Type.Union(
|
||||
WIKI_SEARCH_BACKENDS.map((value) => Type.Literal(value)),
|
||||
);
|
||||
const WikiSearchCorpusSchema = Type.Union(WIKI_SEARCH_CORPORA.map((value) => Type.Literal(value)));
|
||||
const WikiSearchSchema = Type.Object(
|
||||
{
|
||||
query: Type.String({ minLength: 1 }),
|
||||
maxResults: Type.Optional(Type.Number({ minimum: 1 })),
|
||||
backend: Type.Optional(WikiSearchBackendSchema),
|
||||
corpus: Type.Optional(WikiSearchCorpusSchema),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
const WikiGetSchema = Type.Object(
|
||||
{
|
||||
lookup: Type.String({ minLength: 1 }),
|
||||
fromLine: Type.Optional(Type.Number({ minimum: 1 })),
|
||||
lineCount: Type.Optional(Type.Number({ minimum: 1 })),
|
||||
backend: Type.Optional(WikiSearchBackendSchema),
|
||||
corpus: Type.Optional(WikiSearchCorpusSchema),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
const WikiClaimEvidenceSchema = Type.Object(
|
||||
{
|
||||
sourceId: Type.Optional(Type.String({ minLength: 1 })),
|
||||
path: Type.Optional(Type.String({ minLength: 1 })),
|
||||
lines: Type.Optional(Type.String({ minLength: 1 })),
|
||||
weight: Type.Optional(Type.Number({ minimum: 0 })),
|
||||
note: Type.Optional(Type.String({ minLength: 1 })),
|
||||
updatedAt: Type.Optional(Type.String({ minLength: 1 })),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
const WikiClaimSchema = Type.Object(
|
||||
{
|
||||
id: Type.Optional(Type.String({ minLength: 1 })),
|
||||
text: Type.String({ minLength: 1 }),
|
||||
status: Type.Optional(Type.String({ minLength: 1 })),
|
||||
confidence: Type.Optional(Type.Number({ minimum: 0, maximum: 1 })),
|
||||
evidence: Type.Optional(Type.Array(WikiClaimEvidenceSchema)),
|
||||
updatedAt: Type.Optional(Type.String({ minLength: 1 })),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
const WikiApplySchema = Type.Object(
|
||||
{
|
||||
op: Type.Union([Type.Literal("create_synthesis"), Type.Literal("update_metadata")]),
|
||||
title: Type.Optional(Type.String({ minLength: 1 })),
|
||||
body: Type.Optional(Type.String({ minLength: 1 })),
|
||||
lookup: Type.Optional(Type.String({ minLength: 1 })),
|
||||
sourceIds: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
|
||||
claims: Type.Optional(Type.Array(WikiClaimSchema)),
|
||||
contradictions: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
|
||||
questions: Type.Optional(Type.Array(Type.String({ minLength: 1 }))),
|
||||
confidence: Type.Optional(Type.Union([Type.Number({ minimum: 0, maximum: 1 }), Type.Null()])),
|
||||
status: Type.Optional(Type.String({ minLength: 1 })),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
async function syncImportedSourcesIfNeeded(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
) {
|
||||
await syncMemoryWikiImportedSources({ config, appConfig });
|
||||
}
|
||||
|
||||
type WikiToolMemoryContext = {
|
||||
agentId?: string;
|
||||
agentSessionKey?: string;
|
||||
};
|
||||
|
||||
export function createWikiStatusTool(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
): AnyAgentTool {
|
||||
return {
|
||||
name: "wiki_status",
|
||||
label: "Wiki Status",
|
||||
description:
|
||||
"Inspect the current memory wiki vault mode, health, and Obsidian CLI availability.",
|
||||
parameters: WikiStatusSchema,
|
||||
execute: async () => {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const status = await resolveMemoryWikiStatus(config, {
|
||||
appConfig,
|
||||
});
|
||||
return {
|
||||
content: [{ type: "text", text: renderMemoryWikiStatus(status) }],
|
||||
details: status,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWikiSearchTool(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
memoryContext: WikiToolMemoryContext = {},
|
||||
): AnyAgentTool {
|
||||
return {
|
||||
name: "wiki_search",
|
||||
label: "Wiki Search",
|
||||
description:
|
||||
"Search wiki pages and, when shared search is enabled, the active memory corpus by title, path, id, or body text.",
|
||||
parameters: WikiSearchSchema,
|
||||
execute: async (_toolCallId, rawParams) => {
|
||||
const params = rawParams as {
|
||||
query: string;
|
||||
maxResults?: number;
|
||||
backend?: ResolvedMemoryWikiConfig["search"]["backend"];
|
||||
corpus?: ResolvedMemoryWikiConfig["search"]["corpus"];
|
||||
};
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const results = await searchMemoryWiki({
|
||||
config,
|
||||
appConfig,
|
||||
agentId: memoryContext.agentId,
|
||||
agentSessionKey: memoryContext.agentSessionKey,
|
||||
query: params.query,
|
||||
maxResults: params.maxResults,
|
||||
...(params.backend ? { searchBackend: params.backend } : {}),
|
||||
...(params.corpus ? { searchCorpus: params.corpus } : {}),
|
||||
});
|
||||
const text =
|
||||
results.length === 0
|
||||
? "No wiki or memory results."
|
||||
: results
|
||||
.map(
|
||||
(result, index) =>
|
||||
`${index + 1}. ${result.title} (${result.corpus}/${result.kind})\nPath: ${result.path}${typeof result.startLine === "number" && typeof result.endLine === "number" ? `\nLines: ${result.startLine}-${result.endLine}` : ""}${result.provenanceLabel ? `\nProvenance: ${result.provenanceLabel}` : ""}\nSnippet: ${result.snippet}`,
|
||||
)
|
||||
.join("\n\n");
|
||||
return {
|
||||
content: [{ type: "text", text }],
|
||||
details: { results },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWikiLintTool(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
): AnyAgentTool {
|
||||
return {
|
||||
name: "wiki_lint",
|
||||
label: "Wiki Lint",
|
||||
description:
|
||||
"Lint the wiki vault and surface structural issues, provenance gaps, contradictions, and open questions.",
|
||||
parameters: WikiLintSchema,
|
||||
execute: async () => {
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const result = await lintMemoryWikiVault(config);
|
||||
const contradictions = result.issuesByCategory.contradictions.length;
|
||||
const openQuestions = result.issuesByCategory["open-questions"].length;
|
||||
const provenance = result.issuesByCategory.provenance.length;
|
||||
const errors = result.issues.filter((issue) => issue.severity === "error").length;
|
||||
const warnings = result.issues.filter((issue) => issue.severity === "warning").length;
|
||||
const summary =
|
||||
result.issueCount === 0
|
||||
? "No wiki lint issues."
|
||||
: [
|
||||
`Issues: ${result.issueCount} total (${errors} errors, ${warnings} warnings)`,
|
||||
`Contradictions: ${contradictions}`,
|
||||
`Open questions: ${openQuestions}`,
|
||||
`Provenance gaps: ${provenance}`,
|
||||
`Report: ${result.reportPath}`,
|
||||
].join("\n");
|
||||
return {
|
||||
content: [{ type: "text", text: summary }],
|
||||
details: result,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWikiApplyTool(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
): AnyAgentTool {
|
||||
return {
|
||||
name: "wiki_apply",
|
||||
label: "Wiki Apply",
|
||||
description:
|
||||
"Apply narrow wiki mutations for syntheses and page metadata without freeform markdown surgery.",
|
||||
parameters: WikiApplySchema,
|
||||
execute: async (_toolCallId, rawParams) => {
|
||||
const mutation = normalizeMemoryWikiMutationInput(rawParams);
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const result = await applyMemoryWikiMutation({ config, mutation });
|
||||
const action = result.changed ? "Updated" : "No changes for";
|
||||
const compileSummary =
|
||||
result.compile.updatedFiles.length > 0
|
||||
? `Refreshed ${result.compile.updatedFiles.length} index file${result.compile.updatedFiles.length === 1 ? "" : "s"}.`
|
||||
: "Indexes unchanged.";
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `${action} ${result.pagePath} via ${result.operation}. ${compileSummary}`,
|
||||
},
|
||||
],
|
||||
details: result,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWikiGetTool(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
appConfig?: OpenClawConfig,
|
||||
memoryContext: WikiToolMemoryContext = {},
|
||||
): AnyAgentTool {
|
||||
return {
|
||||
name: "wiki_get",
|
||||
label: "Wiki Get",
|
||||
description:
|
||||
"Read a wiki page by id or relative path, or fall back to the active memory corpus when shared search is enabled.",
|
||||
parameters: WikiGetSchema,
|
||||
execute: async (_toolCallId, rawParams) => {
|
||||
const params = rawParams as {
|
||||
lookup: string;
|
||||
fromLine?: number;
|
||||
lineCount?: number;
|
||||
backend?: ResolvedMemoryWikiConfig["search"]["backend"];
|
||||
corpus?: ResolvedMemoryWikiConfig["search"]["corpus"];
|
||||
};
|
||||
await syncImportedSourcesIfNeeded(config, appConfig);
|
||||
const result = await getMemoryWikiPage({
|
||||
config,
|
||||
appConfig,
|
||||
agentId: memoryContext.agentId,
|
||||
agentSessionKey: memoryContext.agentSessionKey,
|
||||
lookup: params.lookup,
|
||||
fromLine: params.fromLine,
|
||||
lineCount: params.lineCount,
|
||||
...(params.backend ? { searchBackend: params.backend } : {}),
|
||||
...(params.corpus ? { searchCorpus: params.corpus } : {}),
|
||||
});
|
||||
if (!result) {
|
||||
return {
|
||||
content: [{ type: "text", text: `Wiki page not found: ${params.lookup}` }],
|
||||
details: { found: false },
|
||||
};
|
||||
}
|
||||
return {
|
||||
content: [{ type: "text", text: result.content }],
|
||||
details: { found: true, ...result },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
132
openclaw/extensions/memory-wiki/src/unsafe-local.test.ts
Normal file
132
openclaw/extensions/memory-wiki/src/unsafe-local.test.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
import { syncMemoryWikiUnsafeLocalSources } from "./unsafe-local.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("syncMemoryWikiUnsafeLocalSources", () => {
|
||||
let fixtureRoot = "";
|
||||
let caseId = 0;
|
||||
|
||||
beforeAll(async () => {
|
||||
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-unsafe-suite-"));
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (!fixtureRoot) {
|
||||
return;
|
||||
}
|
||||
await fs.rm(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function nextCaseRoot(name: string): string {
|
||||
return path.join(fixtureRoot, `case-${caseId++}-${name}`);
|
||||
}
|
||||
|
||||
async function createPrivateDir(name: string): Promise<string> {
|
||||
const privateDir = nextCaseRoot(name);
|
||||
await fs.mkdir(privateDir, { recursive: true });
|
||||
return privateDir;
|
||||
}
|
||||
|
||||
it("imports explicit private paths and preserves unsafe-local provenance", async () => {
|
||||
const privateDir = await createPrivateDir("private");
|
||||
|
||||
await fs.mkdir(path.join(privateDir, "nested"), { recursive: true });
|
||||
await fs.writeFile(path.join(privateDir, "nested", "state.md"), "# internal state\n", "utf8");
|
||||
await fs.writeFile(path.join(privateDir, "nested", "cache.json"), '{"ok":true}\n', "utf8");
|
||||
await fs.writeFile(path.join(privateDir, "nested", "blob.bin"), "\u0000\u0001", "utf8");
|
||||
const directPath = path.join(privateDir, "events.log");
|
||||
await fs.writeFile(directPath, "private log\n", "utf8");
|
||||
|
||||
const { rootDir: vaultDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot("vault"),
|
||||
config: {
|
||||
vaultMode: "unsafe-local",
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: true,
|
||||
paths: [path.join(privateDir, "nested"), directPath],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const first = await syncMemoryWikiUnsafeLocalSources(config);
|
||||
|
||||
expect(first.artifactCount).toBe(3);
|
||||
expect(first.importedCount).toBe(3);
|
||||
expect(first.updatedCount).toBe(0);
|
||||
expect(first.skippedCount).toBe(0);
|
||||
expect(first.removedCount).toBe(0);
|
||||
|
||||
const page = await fs.readFile(path.join(vaultDir, first.pagePaths[0] ?? ""), "utf8");
|
||||
expect(page).toContain("sourceType: memory-unsafe-local");
|
||||
expect(page).toContain("provenanceMode: unsafe-local");
|
||||
|
||||
const second = await syncMemoryWikiUnsafeLocalSources(config);
|
||||
|
||||
expect(second.importedCount).toBe(0);
|
||||
expect(second.updatedCount).toBe(0);
|
||||
expect(second.skippedCount).toBe(3);
|
||||
expect(second.removedCount).toBe(0);
|
||||
});
|
||||
|
||||
it("prunes stale unsafe-local pages when configured files disappear", async () => {
|
||||
const privateDir = await createPrivateDir("private-prune");
|
||||
|
||||
const secretPath = path.join(privateDir, "secret.md");
|
||||
await fs.writeFile(secretPath, "# private\n", "utf8");
|
||||
|
||||
const { rootDir: vaultDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot("prune-vault"),
|
||||
config: {
|
||||
vaultMode: "unsafe-local",
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: true,
|
||||
paths: [secretPath],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const first = await syncMemoryWikiUnsafeLocalSources(config);
|
||||
const firstPagePath = first.pagePaths[0] ?? "";
|
||||
await expect(fs.stat(path.join(vaultDir, firstPagePath))).resolves.toBeTruthy();
|
||||
|
||||
await fs.rm(secretPath);
|
||||
const second = await syncMemoryWikiUnsafeLocalSources(config);
|
||||
|
||||
expect(second.artifactCount).toBe(0);
|
||||
expect(second.removedCount).toBe(1);
|
||||
await expect(fs.stat(path.join(vaultDir, firstPagePath))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
});
|
||||
|
||||
it("caps composed unsafe-local filenames to the filesystem component limit", async () => {
|
||||
const privateDir = await createPrivateDir(`${"漢".repeat(50)}-private`);
|
||||
const nestedDir = path.join(privateDir, `${"語".repeat(50)}-nested`);
|
||||
const secretPath = path.join(nestedDir, `${"録".repeat(50)}.md`);
|
||||
await fs.mkdir(nestedDir, { recursive: true });
|
||||
await fs.writeFile(secretPath, "# very private\n", "utf8");
|
||||
|
||||
const { rootDir: vaultDir, config } = await createVault({
|
||||
rootDir: nextCaseRoot("long-unsafe-vault"),
|
||||
config: {
|
||||
vaultMode: "unsafe-local",
|
||||
unsafeLocal: {
|
||||
allowPrivateMemoryCoreAccess: true,
|
||||
paths: [privateDir],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await syncMemoryWikiUnsafeLocalSources(config);
|
||||
const pagePath = result.pagePaths[0] ?? "";
|
||||
|
||||
expect(result.importedCount).toBe(1);
|
||||
expect(Buffer.byteLength(path.basename(pagePath))).toBeLessThanOrEqual(255);
|
||||
await expect(fs.stat(path.join(vaultDir, pagePath))).resolves.toBeTruthy();
|
||||
});
|
||||
});
|
||||
271
openclaw/extensions/memory-wiki/src/unsafe-local.ts
Normal file
271
openclaw/extensions/memory-wiki/src/unsafe-local.ts
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import type { BridgeMemoryWikiResult } from "./bridge.js";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
import {
|
||||
createWikiPageFilename,
|
||||
renderMarkdownFence,
|
||||
renderWikiMarkdown,
|
||||
slugifyWikiSegment,
|
||||
} from "./markdown.js";
|
||||
import { writeImportedSourcePage } from "./source-page-shared.js";
|
||||
import { resolveArtifactKey } from "./source-path-shared.js";
|
||||
import {
|
||||
pruneImportedSourceEntries,
|
||||
readMemoryWikiSourceSyncState,
|
||||
writeMemoryWikiSourceSyncState,
|
||||
} from "./source-sync-state.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
type UnsafeLocalArtifact = {
|
||||
syncKey: string;
|
||||
configuredPath: string;
|
||||
absolutePath: string;
|
||||
relativePath: string;
|
||||
};
|
||||
|
||||
const DIRECTORY_TEXT_EXTENSIONS = new Set([".json", ".jsonl", ".md", ".txt", ".yaml", ".yml"]);
|
||||
|
||||
function detectFenceLanguage(filePath: string): string {
|
||||
const ext = normalizeLowercaseStringOrEmpty(path.extname(filePath));
|
||||
if (ext === ".json" || ext === ".jsonl") {
|
||||
return "json";
|
||||
}
|
||||
if (ext === ".yaml" || ext === ".yml") {
|
||||
return "yaml";
|
||||
}
|
||||
if (ext === ".txt") {
|
||||
return "text";
|
||||
}
|
||||
return "markdown";
|
||||
}
|
||||
|
||||
async function listAllowedFilesRecursive(rootDir: string): Promise<string[]> {
|
||||
const entries = await fs.readdir(rootDir, { withFileTypes: true }).catch(() => []);
|
||||
const files: string[] = [];
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(rootDir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await listAllowedFilesRecursive(fullPath)));
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
entry.isFile() &&
|
||||
DIRECTORY_TEXT_EXTENSIONS.has(normalizeLowercaseStringOrEmpty(path.extname(entry.name)))
|
||||
) {
|
||||
files.push(fullPath);
|
||||
}
|
||||
}
|
||||
return files.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
async function collectUnsafeLocalArtifacts(
|
||||
configuredPaths: string[],
|
||||
): Promise<UnsafeLocalArtifact[]> {
|
||||
const artifacts: UnsafeLocalArtifact[] = [];
|
||||
for (const configuredPath of configuredPaths) {
|
||||
const absoluteConfiguredPath = path.resolve(configuredPath);
|
||||
const stat = await fs.stat(absoluteConfiguredPath).catch(() => null);
|
||||
if (!stat) {
|
||||
continue;
|
||||
}
|
||||
if (stat.isDirectory()) {
|
||||
const files = await listAllowedFilesRecursive(absoluteConfiguredPath);
|
||||
for (const absolutePath of files) {
|
||||
artifacts.push({
|
||||
syncKey: await resolveArtifactKey(absolutePath),
|
||||
configuredPath: absoluteConfiguredPath,
|
||||
absolutePath,
|
||||
relativePath: path.relative(absoluteConfiguredPath, absolutePath).replace(/\\/g, "/"),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (stat.isFile()) {
|
||||
artifacts.push({
|
||||
syncKey: await resolveArtifactKey(absoluteConfiguredPath),
|
||||
configuredPath: absoluteConfiguredPath,
|
||||
absolutePath: absoluteConfiguredPath,
|
||||
relativePath: path.basename(absoluteConfiguredPath),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const deduped = new Map<string, UnsafeLocalArtifact>();
|
||||
for (const artifact of artifacts) {
|
||||
deduped.set(artifact.syncKey, artifact);
|
||||
}
|
||||
return [...deduped.values()];
|
||||
}
|
||||
|
||||
function resolveUnsafeLocalPagePath(params: { configuredPath: string; absolutePath: string }): {
|
||||
pageId: string;
|
||||
pagePath: string;
|
||||
} {
|
||||
const configuredBaseSlug = slugifyWikiSegment(path.basename(params.configuredPath));
|
||||
const configuredHash = createHash("sha1")
|
||||
.update(path.resolve(params.configuredPath))
|
||||
.digest("hex")
|
||||
.slice(0, 8);
|
||||
const artifactBaseSlug = slugifyWikiSegment(path.basename(params.absolutePath));
|
||||
const artifactHash = createHash("sha1")
|
||||
.update(path.resolve(params.absolutePath))
|
||||
.digest("hex")
|
||||
.slice(0, 8);
|
||||
const pageSlug = `${configuredBaseSlug}-${configuredHash}-${artifactBaseSlug}-${artifactHash}`;
|
||||
return {
|
||||
pageId: `source.unsafe-local.${pageSlug}`,
|
||||
pagePath: path
|
||||
.join("sources", createWikiPageFilename(`unsafe-local-${pageSlug}`))
|
||||
.replace(/\\/g, "/"),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveUnsafeLocalTitle(artifact: UnsafeLocalArtifact): string {
|
||||
return `Unsafe Local Import: ${artifact.relativePath}`;
|
||||
}
|
||||
|
||||
async function writeUnsafeLocalSourcePage(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
artifact: UnsafeLocalArtifact;
|
||||
sourceUpdatedAtMs: number;
|
||||
sourceSize: number;
|
||||
state: Awaited<ReturnType<typeof readMemoryWikiSourceSyncState>>;
|
||||
}): Promise<{ pagePath: string; changed: boolean; created: boolean }> {
|
||||
const { pageId, pagePath } = resolveUnsafeLocalPagePath({
|
||||
configuredPath: params.artifact.configuredPath,
|
||||
absolutePath: params.artifact.absolutePath,
|
||||
});
|
||||
const title = resolveUnsafeLocalTitle(params.artifact);
|
||||
const renderFingerprint = createHash("sha1")
|
||||
.update(
|
||||
JSON.stringify({
|
||||
configuredPath: params.artifact.configuredPath,
|
||||
relativePath: params.artifact.relativePath,
|
||||
}),
|
||||
)
|
||||
.digest("hex");
|
||||
return writeImportedSourcePage({
|
||||
vaultRoot: params.config.vault.path,
|
||||
syncKey: params.artifact.syncKey,
|
||||
sourcePath: params.artifact.absolutePath,
|
||||
sourceUpdatedAtMs: params.sourceUpdatedAtMs,
|
||||
sourceSize: params.sourceSize,
|
||||
renderFingerprint,
|
||||
pagePath,
|
||||
group: "unsafe-local",
|
||||
state: params.state,
|
||||
buildRendered: (raw, updatedAt) =>
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "source",
|
||||
id: pageId,
|
||||
title,
|
||||
sourceType: "memory-unsafe-local",
|
||||
provenanceMode: "unsafe-local",
|
||||
sourcePath: params.artifact.absolutePath,
|
||||
unsafeLocalConfiguredPath: params.artifact.configuredPath,
|
||||
unsafeLocalRelativePath: params.artifact.relativePath,
|
||||
status: "active",
|
||||
updatedAt,
|
||||
},
|
||||
body: [
|
||||
`# ${title}`,
|
||||
"",
|
||||
"## Unsafe Local Source",
|
||||
`- Configured path: \`${params.artifact.configuredPath}\``,
|
||||
`- Relative path: \`${params.artifact.relativePath}\``,
|
||||
`- Updated: ${updatedAt}`,
|
||||
"",
|
||||
"## Content",
|
||||
renderMarkdownFence(raw, detectFenceLanguage(params.artifact.absolutePath)),
|
||||
"",
|
||||
"## Notes",
|
||||
"<!-- openclaw:human:start -->",
|
||||
"<!-- openclaw:human:end -->",
|
||||
"",
|
||||
].join("\n"),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function syncMemoryWikiUnsafeLocalSources(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
): Promise<BridgeMemoryWikiResult> {
|
||||
await initializeMemoryWikiVault(config);
|
||||
if (
|
||||
config.vaultMode !== "unsafe-local" ||
|
||||
!config.unsafeLocal.allowPrivateMemoryCoreAccess ||
|
||||
config.unsafeLocal.paths.length === 0
|
||||
) {
|
||||
return {
|
||||
importedCount: 0,
|
||||
updatedCount: 0,
|
||||
skippedCount: 0,
|
||||
removedCount: 0,
|
||||
artifactCount: 0,
|
||||
workspaces: 0,
|
||||
pagePaths: [],
|
||||
};
|
||||
}
|
||||
|
||||
const artifacts = await collectUnsafeLocalArtifacts(config.unsafeLocal.paths);
|
||||
const state = await readMemoryWikiSourceSyncState(config.vault.path);
|
||||
const activeKeys = new Set<string>();
|
||||
const results = await Promise.all(
|
||||
artifacts.map(async (artifact) => {
|
||||
const stats = await fs.stat(artifact.absolutePath);
|
||||
activeKeys.add(artifact.syncKey);
|
||||
return await writeUnsafeLocalSourcePage({
|
||||
config,
|
||||
artifact,
|
||||
sourceUpdatedAtMs: stats.mtimeMs,
|
||||
sourceSize: stats.size,
|
||||
state,
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
const removedCount = await pruneImportedSourceEntries({
|
||||
vaultRoot: config.vault.path,
|
||||
group: "unsafe-local",
|
||||
activeKeys,
|
||||
state,
|
||||
});
|
||||
await writeMemoryWikiSourceSyncState(config.vault.path, state);
|
||||
const importedCount = results.filter((result) => result.changed && result.created).length;
|
||||
const updatedCount = results.filter((result) => result.changed && !result.created).length;
|
||||
const skippedCount = results.filter((result) => !result.changed).length;
|
||||
const pagePaths = results
|
||||
.map((result) => result.pagePath)
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
|
||||
if (importedCount > 0 || updatedCount > 0 || removedCount > 0) {
|
||||
await appendMemoryWikiLog(config.vault.path, {
|
||||
type: "ingest",
|
||||
timestamp: new Date().toISOString(),
|
||||
details: {
|
||||
sourceType: "memory-unsafe-local",
|
||||
configuredPathCount: config.unsafeLocal.paths.length,
|
||||
artifactCount: artifacts.length,
|
||||
importedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
removedCount,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
importedCount,
|
||||
updatedCount,
|
||||
skippedCount,
|
||||
removedCount,
|
||||
artifactCount: artifacts.length,
|
||||
workspaces: 0,
|
||||
pagePaths,
|
||||
};
|
||||
}
|
||||
53
openclaw/extensions/memory-wiki/src/vault.test.ts
Normal file
53
openclaw/extensions/memory-wiki/src/vault.test.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createMemoryWikiTestHarness } from "./test-helpers.js";
|
||||
import { initializeMemoryWikiVault, WIKI_VAULT_DIRECTORIES } from "./vault.js";
|
||||
|
||||
const { createVault } = createMemoryWikiTestHarness();
|
||||
|
||||
describe("initializeMemoryWikiVault", () => {
|
||||
it("creates the wiki layout and seed files", async () => {
|
||||
const { rootDir, config } = await createVault({
|
||||
prefix: "memory-wiki-",
|
||||
config: {
|
||||
vault: {
|
||||
renderMode: "obsidian",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await initializeMemoryWikiVault(config, {
|
||||
nowMs: Date.UTC(2026, 3, 5, 12, 0, 0),
|
||||
});
|
||||
|
||||
expect(result.created).toBe(true);
|
||||
await Promise.all(
|
||||
WIKI_VAULT_DIRECTORIES.map(async (relativeDir) => {
|
||||
await expect(fs.stat(path.join(rootDir, relativeDir))).resolves.toBeTruthy();
|
||||
}),
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "AGENTS.md"), "utf8")).resolves.toContain(
|
||||
"Memory Wiki Agent Guide",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "WIKI.md"), "utf8")).resolves.toContain(
|
||||
"Render mode: `obsidian`",
|
||||
);
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, ".openclaw-wiki", "state.json"), "utf8"),
|
||||
).resolves.toContain('"renderMode": "obsidian"');
|
||||
});
|
||||
|
||||
it("is idempotent when the vault already exists", async () => {
|
||||
const { config } = await createVault({
|
||||
prefix: "memory-wiki-",
|
||||
});
|
||||
|
||||
await initializeMemoryWikiVault(config);
|
||||
const second = await initializeMemoryWikiVault(config);
|
||||
|
||||
expect(second.created).toBe(false);
|
||||
expect(second.createdDirectories).toHaveLength(0);
|
||||
expect(second.createdFiles).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
163
openclaw/extensions/memory-wiki/src/vault.ts
Normal file
163
openclaw/extensions/memory-wiki/src/vault.ts
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
replaceManagedMarkdownBlock,
|
||||
withTrailingNewline,
|
||||
} from "openclaw/plugin-sdk/memory-host-markdown";
|
||||
import type { ResolvedMemoryWikiConfig } from "./config.js";
|
||||
import { appendMemoryWikiLog } from "./log.js";
|
||||
|
||||
export const WIKI_VAULT_DIRECTORIES = [
|
||||
"entities",
|
||||
"concepts",
|
||||
"syntheses",
|
||||
"sources",
|
||||
"reports",
|
||||
"_attachments",
|
||||
"_views",
|
||||
".openclaw-wiki",
|
||||
".openclaw-wiki/locks",
|
||||
".openclaw-wiki/cache",
|
||||
] as const;
|
||||
|
||||
export type InitializeMemoryWikiVaultResult = {
|
||||
rootDir: string;
|
||||
created: boolean;
|
||||
createdDirectories: string[];
|
||||
createdFiles: string[];
|
||||
};
|
||||
|
||||
function buildIndexMarkdown(): string {
|
||||
return withTrailingNewline(
|
||||
replaceManagedMarkdownBlock({
|
||||
original: "# Wiki Index\n",
|
||||
heading: "## Generated",
|
||||
startMarker: "<!-- openclaw:wiki:index:start -->",
|
||||
endMarker: "<!-- openclaw:wiki:index:end -->",
|
||||
body: "- No compiled pages yet.",
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function buildAgentsMarkdown(): string {
|
||||
return withTrailingNewline(`\
|
||||
# Memory Wiki Agent Guide
|
||||
|
||||
- Treat generated blocks as plugin-owned.
|
||||
- Preserve human notes outside managed markers.
|
||||
- Prefer source-backed claims over wiki-to-wiki citation loops.
|
||||
- Prefer structured \`claims\` with evidence over burying key beliefs only in prose.
|
||||
- Use \`.openclaw-wiki/cache/agent-digest.json\` and \`claims.jsonl\` for machine reads; markdown pages are the human view.
|
||||
`);
|
||||
}
|
||||
|
||||
function buildWikiOverviewMarkdown(config: ResolvedMemoryWikiConfig): string {
|
||||
return withTrailingNewline(`\
|
||||
# Memory Wiki
|
||||
|
||||
This vault is maintained by the OpenClaw memory-wiki plugin.
|
||||
|
||||
- Vault mode: \`${config.vaultMode}\`
|
||||
- Render mode: \`${config.vault.renderMode}\`
|
||||
- Search corpus default: \`${config.search.corpus}\`
|
||||
|
||||
## Architecture
|
||||
- Raw sources remain the evidence layer.
|
||||
- Wiki pages are the human-readable synthesis layer.
|
||||
- \`.openclaw-wiki/cache/agent-digest.json\` is the agent-facing compiled digest.
|
||||
|
||||
## Notes
|
||||
<!-- openclaw:human:start -->
|
||||
<!-- openclaw:human:end -->
|
||||
`);
|
||||
}
|
||||
|
||||
async function pathExists(inputPath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(inputPath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function writeFileIfMissing(
|
||||
filePath: string,
|
||||
content: string,
|
||||
createdFiles: string[],
|
||||
): Promise<void> {
|
||||
if (await pathExists(filePath)) {
|
||||
return;
|
||||
}
|
||||
await fs.writeFile(filePath, content, "utf8");
|
||||
createdFiles.push(filePath);
|
||||
}
|
||||
|
||||
export async function initializeMemoryWikiVault(
|
||||
config: ResolvedMemoryWikiConfig,
|
||||
options?: { nowMs?: number },
|
||||
): Promise<InitializeMemoryWikiVaultResult> {
|
||||
const rootDir = config.vault.path;
|
||||
const createdDirectories: string[] = [];
|
||||
const createdFiles: string[] = [];
|
||||
|
||||
if (!(await pathExists(rootDir))) {
|
||||
createdDirectories.push(rootDir);
|
||||
}
|
||||
await fs.mkdir(rootDir, { recursive: true });
|
||||
|
||||
for (const relativeDir of WIKI_VAULT_DIRECTORIES) {
|
||||
const fullPath = path.join(rootDir, relativeDir);
|
||||
if (!(await pathExists(fullPath))) {
|
||||
createdDirectories.push(fullPath);
|
||||
}
|
||||
await fs.mkdir(fullPath, { recursive: true });
|
||||
}
|
||||
|
||||
await writeFileIfMissing(path.join(rootDir, "AGENTS.md"), buildAgentsMarkdown(), createdFiles);
|
||||
await writeFileIfMissing(
|
||||
path.join(rootDir, "WIKI.md"),
|
||||
buildWikiOverviewMarkdown(config),
|
||||
createdFiles,
|
||||
);
|
||||
await writeFileIfMissing(path.join(rootDir, "index.md"), buildIndexMarkdown(), createdFiles);
|
||||
await writeFileIfMissing(
|
||||
path.join(rootDir, "inbox.md"),
|
||||
withTrailingNewline("# Inbox\n\nDrop raw ideas, questions, and source links here.\n"),
|
||||
createdFiles,
|
||||
);
|
||||
await writeFileIfMissing(
|
||||
path.join(rootDir, ".openclaw-wiki", "state.json"),
|
||||
withTrailingNewline(
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
createdAt: new Date(options?.nowMs ?? Date.now()).toISOString(),
|
||||
renderMode: config.vault.renderMode,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
),
|
||||
createdFiles,
|
||||
);
|
||||
await writeFileIfMissing(path.join(rootDir, ".openclaw-wiki", "log.jsonl"), "", createdFiles);
|
||||
|
||||
if (createdDirectories.length > 0 || createdFiles.length > 0) {
|
||||
await appendMemoryWikiLog(rootDir, {
|
||||
type: "init",
|
||||
timestamp: new Date(options?.nowMs ?? Date.now()).toISOString(),
|
||||
details: {
|
||||
createdDirectories: createdDirectories.map((dir) => path.relative(rootDir, dir) || "."),
|
||||
createdFiles: createdFiles.map((file) => path.relative(rootDir, file)),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
rootDir,
|
||||
created: createdDirectories.length > 0 || createdFiles.length > 0,
|
||||
createdDirectories,
|
||||
createdFiles,
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue