mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-17 06:11:01 +08:00
重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
236
openclaw/extensions/searxng/src/searxng-client.ts
Normal file
236
openclaw/extensions/searxng/src/searxng-client.ts
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import {
|
||||
DEFAULT_CACHE_TTL_MINUTES,
|
||||
DEFAULT_SEARCH_COUNT,
|
||||
normalizeCacheKey,
|
||||
readCache,
|
||||
readResponseText,
|
||||
resolveCacheTtlMs,
|
||||
resolveSearchCount,
|
||||
resolveSiteName,
|
||||
resolveTimeoutSeconds,
|
||||
withTrustedWebSearchEndpoint,
|
||||
wrapWebContent,
|
||||
writeCache,
|
||||
} from "openclaw/plugin-sdk/provider-web-search";
|
||||
import {
|
||||
assertHttpUrlTargetsPrivateNetwork,
|
||||
type LookupFn,
|
||||
} from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import {
|
||||
resolveSearxngBaseUrl,
|
||||
resolveSearxngCategories,
|
||||
resolveSearxngLanguage,
|
||||
} from "./config.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_SECONDS = 20;
|
||||
const MAX_RESPONSE_BYTES = 1_000_000;
|
||||
|
||||
const SEARXNG_SEARCH_CACHE = new Map<
|
||||
string,
|
||||
{ value: Record<string, unknown>; insertedAt: number; expiresAt: number }
|
||||
>();
|
||||
|
||||
type SearxngResult = {
|
||||
url: string;
|
||||
title: string;
|
||||
content?: string;
|
||||
};
|
||||
|
||||
type SearxngResponse = {
|
||||
results?: SearxngResult[];
|
||||
};
|
||||
|
||||
function normalizeSearxngResult(value: unknown): SearxngResult | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const candidate = value as { url?: unknown; title?: unknown; content?: unknown };
|
||||
if (typeof candidate.url !== "string" || typeof candidate.title !== "string") {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
url: candidate.url,
|
||||
title: candidate.title,
|
||||
content: typeof candidate.content === "string" ? candidate.content : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSearxngSearchUrl(params: {
|
||||
baseUrl: string;
|
||||
query: string;
|
||||
categories?: string;
|
||||
language?: string;
|
||||
}): string {
|
||||
const url = new URL(params.baseUrl);
|
||||
const pathname = url.pathname.endsWith("/") ? `${url.pathname}search` : `${url.pathname}/search`;
|
||||
url.pathname = pathname;
|
||||
url.search = "";
|
||||
url.searchParams.set("q", params.query);
|
||||
url.searchParams.set("format", "json");
|
||||
if (params.categories) {
|
||||
url.searchParams.set("categories", params.categories);
|
||||
}
|
||||
if (params.language) {
|
||||
url.searchParams.set("language", params.language);
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
async function validateSearxngBaseUrl(baseUrl: string, lookupFn?: LookupFn): Promise<void> {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(baseUrl);
|
||||
} catch {
|
||||
throw new Error("SearXNG base URL must be a valid http:// or https:// URL.");
|
||||
}
|
||||
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new Error("SearXNG base URL must use http:// or https://.");
|
||||
}
|
||||
|
||||
if (parsed.protocol === "http:") {
|
||||
await assertHttpUrlTargetsPrivateNetwork(parsed.toString(), {
|
||||
dangerouslyAllowPrivateNetwork: true,
|
||||
lookupFn,
|
||||
errorMessage:
|
||||
"SearXNG HTTP base URL must target a trusted private or loopback host. Use https:// for public hosts.",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function parseSearxngResponseText(text: string, count: number): SearxngResult[] {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(text) as SearxngResponse;
|
||||
} catch {
|
||||
throw new Error("SearXNG returned invalid JSON.");
|
||||
}
|
||||
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const response = parsed as SearxngResponse;
|
||||
const rawResults = Array.isArray(response.results) ? response.results : [];
|
||||
const results: SearxngResult[] = [];
|
||||
|
||||
for (const rawResult of rawResults) {
|
||||
const result = normalizeSearxngResult(rawResult);
|
||||
if (result) {
|
||||
results.push(result);
|
||||
}
|
||||
if (results.length >= count) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function runSearxngSearch(params: {
|
||||
config?: OpenClawConfig;
|
||||
query: string;
|
||||
count?: number;
|
||||
categories?: string;
|
||||
language?: string;
|
||||
baseUrl?: string;
|
||||
timeoutSeconds?: number;
|
||||
cacheTtlMinutes?: number;
|
||||
}): Promise<Record<string, unknown>> {
|
||||
const count = resolveSearchCount(params.count, DEFAULT_SEARCH_COUNT);
|
||||
const categories = params.categories ?? resolveSearxngCategories(params.config);
|
||||
const language = params.language ?? resolveSearxngLanguage(params.config);
|
||||
const baseUrl = params.baseUrl ?? resolveSearxngBaseUrl(params.config);
|
||||
const timeoutSeconds = resolveTimeoutSeconds(params.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS);
|
||||
const cacheTtlMs = resolveCacheTtlMs(params.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES);
|
||||
|
||||
if (!baseUrl) {
|
||||
throw new Error(
|
||||
"SearXNG base URL is not configured. Set SEARXNG_BASE_URL or configure plugins.entries.searxng.config.webSearch.baseUrl.",
|
||||
);
|
||||
}
|
||||
await validateSearxngBaseUrl(baseUrl);
|
||||
|
||||
const cacheKey = normalizeCacheKey(
|
||||
JSON.stringify({
|
||||
provider: "searxng",
|
||||
query: params.query,
|
||||
count,
|
||||
categories: categories ?? "",
|
||||
language: language ?? "",
|
||||
baseUrl,
|
||||
}),
|
||||
);
|
||||
const cached = readCache(SEARXNG_SEARCH_CACHE, cacheKey);
|
||||
if (cached) {
|
||||
return { ...cached.value, cached: true };
|
||||
}
|
||||
|
||||
const url = buildSearxngSearchUrl({
|
||||
baseUrl,
|
||||
query: params.query,
|
||||
categories,
|
||||
language,
|
||||
});
|
||||
|
||||
const startedAt = Date.now();
|
||||
const results = await withTrustedWebSearchEndpoint(
|
||||
{
|
||||
url,
|
||||
timeoutSeconds,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
},
|
||||
},
|
||||
},
|
||||
async (response) => {
|
||||
if (!response.ok) {
|
||||
const detail = (await readResponseText(response, { maxBytes: 64_000 })).text;
|
||||
throw new Error(
|
||||
`SearXNG search error (${response.status}): ${detail || response.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const body = await readResponseText(response, { maxBytes: MAX_RESPONSE_BYTES });
|
||||
if (body.truncated) {
|
||||
throw new Error("SearXNG response too large.");
|
||||
}
|
||||
return parseSearxngResponseText(body.text, count);
|
||||
},
|
||||
);
|
||||
|
||||
const payload = {
|
||||
query: params.query,
|
||||
provider: "searxng",
|
||||
count: results.length,
|
||||
tookMs: Date.now() - startedAt,
|
||||
externalContent: {
|
||||
untrusted: true,
|
||||
source: "web_search",
|
||||
provider: "searxng",
|
||||
wrapped: true,
|
||||
},
|
||||
results: results.map((result) => ({
|
||||
title: wrapWebContent(result.title, "web_search"),
|
||||
url: result.url,
|
||||
snippet: result.content ? wrapWebContent(result.content, "web_search") : "",
|
||||
siteName: resolveSiteName(result.url) || undefined,
|
||||
})),
|
||||
} satisfies Record<string, unknown>;
|
||||
|
||||
writeCache(SEARXNG_SEARCH_CACHE, cacheKey, payload, cacheTtlMs);
|
||||
return payload;
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
buildSearxngSearchUrl,
|
||||
normalizeSearxngResult,
|
||||
parseSearxngResponseText,
|
||||
validateSearxngBaseUrl,
|
||||
SEARXNG_SEARCH_CACHE,
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue