hansjone/oclaw
1
0
Fork 0
mirror of https://github.com/hansjone/oclaw.git synced 2026-10-11 05:20:44 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

重构主控编排与运行时预热链路，统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。

Browse source 
同时收敛启动与运维脚本默认行为（含 wiki worker）、更新 Admin 可观测性与相关测试，降低首轮时延并提高运行稳定性。

Made-with: Cursor
This commit is contained in:
oliver 2026-04-26 08:34:33 +08:00
parent 4a23b715a2
commit dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
Download patch file Download diff file Expand all files Collapse all files

42
openclaw/extensions/duckduckgo/src/config.ts Normal file
View file

@ -0,0 +1,42 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
export const DEFAULT_DDG_SAFE_SEARCH = "moderate";
export type DdgSafeSearch = "strict" | "moderate" | "off";
type DdgPluginConfig = {
webSearch?: {
region?: string;
safeSearch?: string;
};
};
export function resolveDdgWebSearchConfig(
config?: OpenClawConfig,
): DdgPluginConfig["webSearch"] | undefined {
const pluginConfig = config?.plugins?.entries?.duckduckgo?.config as DdgPluginConfig | undefined;
const webSearch = pluginConfig?.webSearch;
if (webSearch && typeof webSearch === "object" && !Array.isArray(webSearch)) {
return webSearch;
}
return undefined;
}
export function resolveDdgRegion(config?: OpenClawConfig): string | undefined {
const region = resolveDdgWebSearchConfig(config)?.region;
if (typeof region !== "string") {
return undefined;
}
const trimmed = region.trim();
return trimmed || undefined;
}
export function resolveDdgSafeSearch(config?: OpenClawConfig): DdgSafeSearch {
const safeSearch = resolveDdgWebSearchConfig(config)?.safeSearch;
const normalized = normalizeLowercaseStringOrEmpty(safeSearch);
if (normalized === "strict" || normalized === "off") {
return normalized;
}
return DEFAULT_DDG_SAFE_SEARCH;
}

212
openclaw/extensions/duckduckgo/src/ddg-client.ts Normal file
View file

@ -0,0 +1,212 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import {
DEFAULT_CACHE_TTL_MINUTES,
DEFAULT_SEARCH_COUNT,
normalizeCacheKey,
readCache,
readResponseText,
resolveCacheTtlMs,
resolveSearchCount,
resolveSiteName,
resolveTimeoutSeconds,
withTrustedWebSearchEndpoint,
wrapWebContent,
writeCache,
} from "openclaw/plugin-sdk/provider-web-search";
import { resolveDdgRegion, resolveDdgSafeSearch, type DdgSafeSearch } from "./config.js";
const DDG_HTML_ENDPOINT = "https://html.duckduckgo.com/html";
const DEFAULT_TIMEOUT_SECONDS = 20;
const DDG_SAFE_SEARCH_PARAM: Record<DdgSafeSearch, string> = {
strict: "1",
moderate: "-1",
off: "-2",
};
const DDG_SEARCH_CACHE = new Map<
string,
{ value: Record<string, unknown>; insertedAt: number; expiresAt: number }
>();
type DuckDuckGoResult = {
title: string;
url: string;
snippet: string;
};
function decodeHtmlEntities(text: string): string {
return text
.replace(/&amp;/g, "&")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&#39;/g, "'")
.replace(/&#x27;/g, "'")
.replace(/&#x2F;/g, "/")
.replace(/&nbsp;/g, " ")
.replace(/&ndash;/g, "-")
.replace(/&mdash;/g, "--")
.replace(/&hellip;/g, "...")
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
.replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)));
}
function stripHtml(html: string): string {
return html
.replace(/<[^>]+>/g, " ")
.replace(/\s+/g, " ")
.trim();
}
function decodeDuckDuckGoUrl(rawUrl: string): string {
try {
const normalized = rawUrl.startsWith("//") ? `https:${rawUrl}` : rawUrl;
const parsed = new URL(normalized);
const uddg = parsed.searchParams.get("uddg");
if (uddg) {
return uddg;
}
} catch {
// Keep the original value when DuckDuckGo already returns a direct link.
}
return rawUrl;
}
function readHrefAttribute(tagAttributes: string): string {
return /\bhref="([^"]*)"/i.exec(tagAttributes)?.[1] ?? "";
}
function isBotChallenge(html: string): boolean {
if (/class="[^"]*\bresult__a\b[^"]*"/i.test(html)) {
return false;
}
return /g-recaptcha|are you a human|id="challenge-form"|name="challenge"/i.test(html);
}
function parseDuckDuckGoHtml(html: string): DuckDuckGoResult[] {
const results: DuckDuckGoResult[] = [];
const resultRegex = /<a\b(?=[^>]*\bclass="[^"]*\bresult__a\b[^"]*")([^>]*)>([\s\S]*?)<\/a>/gi;
const nextResultRegex = /<a\b(?=[^>]*\bclass="[^"]*\bresult__a\b[^"]*")[^>]*>/i;
const snippetRegex = /<a\b(?=[^>]*\bclass="[^"]*\bresult__snippet\b[^"]*")[^>]*>([\s\S]*?)<\/a>/i;
for (const match of html.matchAll(resultRegex)) {
const rawAttributes = match[1] ?? "";
const rawTitle = match[2] ?? "";
const rawUrl = readHrefAttribute(rawAttributes);
const matchEnd = (match.index ?? 0) + match[0].length;
const trailingHtml = html.slice(matchEnd);
const nextResultIndex = trailingHtml.search(nextResultRegex);
const scopedTrailingHtml =
nextResultIndex >= 0 ? trailingHtml.slice(0, nextResultIndex) : trailingHtml;
const rawSnippet = snippetRegex.exec(scopedTrailingHtml)?.[1] ?? "";
const title = decodeHtmlEntities(stripHtml(rawTitle));
const url = decodeDuckDuckGoUrl(decodeHtmlEntities(rawUrl));
const snippet = decodeHtmlEntities(stripHtml(rawSnippet));
if (title && url) {
results.push({ title, url, snippet });
}
}
return results;
}
export async function runDuckDuckGoSearch(params: {
config?: OpenClawConfig;
query: string;
count?: number;
region?: string;
safeSearch?: DdgSafeSearch;
timeoutSeconds?: number;
cacheTtlMinutes?: number;
}): Promise<Record<string, unknown>> {
const count = resolveSearchCount(params.count, DEFAULT_SEARCH_COUNT);
const region = params.region ?? resolveDdgRegion(params.config);
const safeSearch =
params.safeSearch === "strict" ||
params.safeSearch === "moderate" ||
params.safeSearch === "off"
? params.safeSearch
: resolveDdgSafeSearch(params.config);
const timeoutSeconds = resolveTimeoutSeconds(params.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS);
const cacheTtlMs = resolveCacheTtlMs(params.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES);
const cacheKey = normalizeCacheKey(
JSON.stringify({
provider: "duckduckgo",
query: params.query,
count,
region: region ?? "",
safeSearch,
}),
);
const cached = readCache(DDG_SEARCH_CACHE, cacheKey);
if (cached) {
return { ...cached.value, cached: true };
}
const url = new URL(DDG_HTML_ENDPOINT);
url.searchParams.set("q", params.query);
if (region) {
url.searchParams.set("kl", region);
}
url.searchParams.set("kp", DDG_SAFE_SEARCH_PARAM[safeSearch]);
const startedAt = Date.now();
const results = await withTrustedWebSearchEndpoint(
{
url: url.toString(),
timeoutSeconds,
init: {
method: "GET",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
},
},
},
async (response) => {
if (!response.ok) {
const detail = (await readResponseText(response, { maxBytes: 64_000 })).text;
throw new Error(
`DuckDuckGo search error (${response.status}): ${detail || response.statusText}`,
);
}
const html = await response.text();
if (isBotChallenge(html)) {
throw new Error("DuckDuckGo returned a bot-detection challenge.");
}
return parseDuckDuckGoHtml(html).slice(0, count);
},
);
const payload = {
query: params.query,
provider: "duckduckgo",
count: results.length,
tookMs: Date.now() - startedAt,
externalContent: {
untrusted: true,
source: "web_search",
provider: "duckduckgo",
wrapped: true,
},
results: results.map((result) => ({
title: wrapWebContent(result.title, "web_search"),
url: result.url,
snippet: result.snippet ? wrapWebContent(result.snippet, "web_search") : "",
siteName: resolveSiteName(result.url) || undefined,
})),
} satisfies Record<string, unknown>;
writeCache(DDG_SEARCH_CACHE, cacheKey, payload, cacheTtlMs);
return payload;
}
export const __testing = {
decodeDuckDuckGoUrl,
decodeHtmlEntities,
isBotChallenge,
parseDuckDuckGoHtml,
};

207
openclaw/extensions/duckduckgo/src/ddg-search-provider.test.ts Normal file
View file

@ -0,0 +1,207 @@
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { DEFAULT_DDG_SAFE_SEARCH, resolveDdgRegion, resolveDdgSafeSearch } from "./config.js";
const { runDuckDuckGoSearch } = vi.hoisted(() => ({
runDuckDuckGoSearch: vi.fn(async (params: Record<string, unknown>) => params),
}));
vi.mock("./ddg-client.js", () => ({
runDuckDuckGoSearch,
}));
describe("duckduckgo web search provider", () => {
let createDuckDuckGoWebSearchProvider: typeof import("./ddg-search-provider.js").createDuckDuckGoWebSearchProvider;
let ddgClientTesting: typeof import("./ddg-client.js").__testing;
beforeAll(async () => {
({ createDuckDuckGoWebSearchProvider } = await import("./ddg-search-provider.js"));
({ __testing: ddgClientTesting } =
await vi.importActual<typeof import("./ddg-client.js")>("./ddg-client.js"));
await import("../index.js");
});
beforeEach(() => {
runDuckDuckGoSearch.mockReset();
runDuckDuckGoSearch.mockImplementation(async (params: Record<string, unknown>) => params);
});
it("exposes keyless metadata and enables the plugin in config", () => {
const provider = createDuckDuckGoWebSearchProvider();
if (!provider.applySelectionConfig) {
throw new Error("Expected applySelectionConfig to be defined");
}
const applied = provider.applySelectionConfig({});
expect(provider.id).toBe("duckduckgo");
expect(provider.label).toBe("DuckDuckGo Search (experimental)");
expect(provider.requiresCredential).toBe(false);
expect(provider.credentialPath).toBe("");
expect(applied.plugins?.entries?.duckduckgo?.enabled).toBe(true);
});
it("maps generic tool arguments into DuckDuckGo search params", async () => {
const provider = createDuckDuckGoWebSearchProvider();
const tool = provider.createTool({
config: { test: true },
} as never);
if (!tool) {
throw new Error("Expected tool definition");
}
const result = await tool.execute({
query: "openclaw docs",
count: 4,
region: "us-en",
safeSearch: "off",
});
expect(runDuckDuckGoSearch).toHaveBeenCalledWith({
config: { test: true },
query: "openclaw docs",
count: 4,
region: "us-en",
safeSearch: "off",
});
expect(result).toEqual({
config: { test: true },
query: "openclaw docs",
count: 4,
region: "us-en",
safeSearch: "off",
});
});
it("reads region from plugin config and normalizes empty values away", () => {
expect(
resolveDdgRegion({
plugins: {
entries: {
duckduckgo: {
config: {
webSearch: {
region: "de-de",
},
},
},
},
},
} as never),
).toBe("de-de");
expect(
resolveDdgRegion({
plugins: {
entries: {
duckduckgo: {
config: {
webSearch: {
region: " ",
},
},
},
},
},
} as never),
).toBeUndefined();
});
it("defaults safeSearch to moderate and accepts strict and off", () => {
expect(resolveDdgSafeSearch(undefined)).toBe(DEFAULT_DDG_SAFE_SEARCH);
expect(
resolveDdgSafeSearch({
plugins: {
entries: {
duckduckgo: {
config: {
webSearch: {
safeSearch: "strict",
},
},
},
},
},
} as never),
).toBe("strict");
expect(
resolveDdgSafeSearch({
plugins: {
entries: {
duckduckgo: {
config: {
webSearch: {
safeSearch: "off",
},
},
},
},
},
} as never),
).toBe("off");
});
it("decodes direct and redirect urls plus common html entities", () => {
expect(
ddgClientTesting.decodeDuckDuckGoUrl(
"https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fsearch%3Fq%3Dclaw",
),
).toBe("https://example.com/search?q=claw");
expect(ddgClientTesting.decodeDuckDuckGoUrl("https://example.com")).toBe("https://example.com");
expect(ddgClientTesting.decodeHtmlEntities("Fish &amp; Chips&nbsp;&hellip; &#39;ok&#39;")).toBe(
"Fish & Chips ... 'ok'",
);
});
it("parses results when href appears before class", () => {
const html = `
<a href="https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com" class="result__a">
Example &amp; Co
</a>
<a class="result__snippet">Fast&nbsp;search &hellip; with details</a>
<a class="result__a" href="https://example.org/direct">Direct result</a>
<a class="result__snippet">Second snippet</a>
`;
expect(ddgClientTesting.parseDuckDuckGoHtml(html)).toEqual([
{
title: "Example & Co",
url: "https://example.com",
snippet: "Fast search ... with details",
},
{
title: "Direct result",
url: "https://example.org/direct",
snippet: "Second snippet",
},
]);
});
it("detects bot challenge pages without flagging ordinary result snippets", () => {
const challengeHtml = `
<html>
<body>
<form>
<h1>Are you a human?</h1>
<div class="g-recaptcha">captcha</div>
</form>
</body>
</html>
`;
const normalHtml = `
<a class="result__a" href="https://example.com/challenge">Coding Challenge</a>
<a class="result__snippet">A fun coding challenge for interview prep.</a>
`;
expect(ddgClientTesting.isBotChallenge(challengeHtml)).toBe(true);
expect(ddgClientTesting.parseDuckDuckGoHtml(challengeHtml)).toEqual([]);
expect(ddgClientTesting.isBotChallenge(normalHtml)).toBe(false);
expect(ddgClientTesting.parseDuckDuckGoHtml(normalHtml)).toEqual([
{
title: "Coding Challenge",
url: "https://example.com/challenge",
snippet: "A fun coding challenge for interview prep.",
},
]);
});
});

75
openclaw/extensions/duckduckgo/src/ddg-search-provider.ts Normal file
View file

@ -0,0 +1,75 @@
import { readNumberParam, readStringParam } from "openclaw/plugin-sdk/param-readers";
import {
createWebSearchProviderContractFields,
type WebSearchProviderPlugin,
} from "openclaw/plugin-sdk/provider-web-search-contract";
type DuckDuckGoClientModule = typeof import("./ddg-client.js");
let duckDuckGoClientModulePromise: Promise<DuckDuckGoClientModule> | undefined;
function loadDuckDuckGoClientModule(): Promise<DuckDuckGoClientModule> {
duckDuckGoClientModulePromise ??= import("./ddg-client.js");
return duckDuckGoClientModulePromise;
}
const DuckDuckGoSearchSchema = {
type: "object",
properties: {
query: { type: "string", description: "Search query string." },
count: {
type: "number",
description: "Number of results to return (1-10).",
minimum: 1,
maximum: 10,
},
region: {
type: "string",
description: "Optional DuckDuckGo region code such as us-en, uk-en, or de-de.",
},
safeSearch: {
type: "string",
description: "SafeSearch level: strict, moderate, or off.",
},
},
additionalProperties: false,
} satisfies Record<string, unknown>;
export function createDuckDuckGoWebSearchProvider(): WebSearchProviderPlugin {
return {
id: "duckduckgo",
label: "DuckDuckGo Search (experimental)",
hint: "Free web search fallback with no API key required",
requiresCredential: false,
envVars: [],
placeholder: "(no key needed)",
signupUrl: "https://duckduckgo.com/",
docsUrl: "https://docs.openclaw.ai/tools/web",
autoDetectOrder: 100,
credentialPath: "",
...createWebSearchProviderContractFields({
credentialPath: "",
searchCredential: { type: "scoped", scopeId: "duckduckgo" },
selectionPluginId: "duckduckgo",
}),
createTool: (ctx) => ({
description:
"Search the web using DuckDuckGo. Returns titles, URLs, and snippets with no API key required.",
parameters: DuckDuckGoSearchSchema,
execute: async (args) => {
const { runDuckDuckGoSearch } = await loadDuckDuckGoClientModule();
return await runDuckDuckGoSearch({
config: ctx.config,
query: readStringParam(args, "query", { required: true }),
count: readNumberParam(args, "count", { integer: true }),
region: readStringParam(args, "region"),
safeSearch: readStringParam(args, "safeSearch") as
| "strict"
| "moderate"
| "off"
| undefined,
});
},
}),
};
}