import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; import { DEFAULT_CACHE_TTL_MINUTES, DEFAULT_SEARCH_COUNT, normalizeCacheKey, readCache, readResponseText, resolveCacheTtlMs, resolveSearchCount, resolveSiteName, resolveTimeoutSeconds, withTrustedWebSearchEndpoint, wrapWebContent, writeCache, } from "openclaw/plugin-sdk/provider-web-search"; import { assertHttpUrlTargetsPrivateNetwork, type LookupFn, } from "openclaw/plugin-sdk/ssrf-runtime"; import { resolveSearxngBaseUrl, resolveSearxngCategories, resolveSearxngLanguage, } from "./config.js"; const DEFAULT_TIMEOUT_SECONDS = 20; const MAX_RESPONSE_BYTES = 1_000_000; const SEARXNG_SEARCH_CACHE = new Map< string, { value: Record; insertedAt: number; expiresAt: number } >(); type SearxngResult = { url: string; title: string; content?: string; }; type SearxngResponse = { results?: SearxngResult[]; }; function normalizeSearxngResult(value: unknown): SearxngResult | null { if (!value || typeof value !== "object") { return null; } const candidate = value as { url?: unknown; title?: unknown; content?: unknown }; if (typeof candidate.url !== "string" || typeof candidate.title !== "string") { return null; } return { url: candidate.url, title: candidate.title, content: typeof candidate.content === "string" ? candidate.content : undefined, }; } function buildSearxngSearchUrl(params: { baseUrl: string; query: string; categories?: string; language?: string; }): string { const url = new URL(params.baseUrl); const pathname = url.pathname.endsWith("/") ? `${url.pathname}search` : `${url.pathname}/search`; url.pathname = pathname; url.search = ""; url.searchParams.set("q", params.query); url.searchParams.set("format", "json"); if (params.categories) { url.searchParams.set("categories", params.categories); } if (params.language) { url.searchParams.set("language", params.language); } return url.toString(); } async function validateSearxngBaseUrl(baseUrl: string, lookupFn?: LookupFn): Promise { let parsed: URL; try { parsed = new URL(baseUrl); } catch { throw new Error("SearXNG base URL must be a valid http:// or https:// URL."); } if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { throw new Error("SearXNG base URL must use http:// or https://."); } if (parsed.protocol === "http:") { await assertHttpUrlTargetsPrivateNetwork(parsed.toString(), { dangerouslyAllowPrivateNetwork: true, lookupFn, errorMessage: "SearXNG HTTP base URL must target a trusted private or loopback host. Use https:// for public hosts.", }); } } function parseSearxngResponseText(text: string, count: number): SearxngResult[] { let parsed: unknown; try { parsed = JSON.parse(text) as SearxngResponse; } catch { throw new Error("SearXNG returned invalid JSON."); } if (!parsed || typeof parsed !== "object") { return []; } const response = parsed as SearxngResponse; const rawResults = Array.isArray(response.results) ? response.results : []; const results: SearxngResult[] = []; for (const rawResult of rawResults) { const result = normalizeSearxngResult(rawResult); if (result) { results.push(result); } if (results.length >= count) { break; } } return results; } export async function runSearxngSearch(params: { config?: OpenClawConfig; query: string; count?: number; categories?: string; language?: string; baseUrl?: string; timeoutSeconds?: number; cacheTtlMinutes?: number; }): Promise> { const count = resolveSearchCount(params.count, DEFAULT_SEARCH_COUNT); const categories = params.categories ?? resolveSearxngCategories(params.config); const language = params.language ?? resolveSearxngLanguage(params.config); const baseUrl = params.baseUrl ?? resolveSearxngBaseUrl(params.config); const timeoutSeconds = resolveTimeoutSeconds(params.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS); const cacheTtlMs = resolveCacheTtlMs(params.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES); if (!baseUrl) { throw new Error( "SearXNG base URL is not configured. Set SEARXNG_BASE_URL or configure plugins.entries.searxng.config.webSearch.baseUrl.", ); } await validateSearxngBaseUrl(baseUrl); const cacheKey = normalizeCacheKey( JSON.stringify({ provider: "searxng", query: params.query, count, categories: categories ?? "", language: language ?? "", baseUrl, }), ); const cached = readCache(SEARXNG_SEARCH_CACHE, cacheKey); if (cached) { return { ...cached.value, cached: true }; } const url = buildSearxngSearchUrl({ baseUrl, query: params.query, categories, language, }); const startedAt = Date.now(); const results = await withTrustedWebSearchEndpoint( { url, timeoutSeconds, init: { method: "GET", headers: { Accept: "application/json", }, }, }, async (response) => { if (!response.ok) { const detail = (await readResponseText(response, { maxBytes: 64_000 })).text; throw new Error( `SearXNG search error (${response.status}): ${detail || response.statusText}`, ); } const body = await readResponseText(response, { maxBytes: MAX_RESPONSE_BYTES }); if (body.truncated) { throw new Error("SearXNG response too large."); } return parseSearxngResponseText(body.text, count); }, ); const payload = { query: params.query, provider: "searxng", count: results.length, tookMs: Date.now() - startedAt, externalContent: { untrusted: true, source: "web_search", provider: "searxng", wrapped: true, }, results: results.map((result) => ({ title: wrapWebContent(result.title, "web_search"), url: result.url, snippet: result.content ? wrapWebContent(result.content, "web_search") : "", siteName: resolveSiteName(result.url) || undefined, })), } satisfies Record; writeCache(SEARXNG_SEARCH_CACHE, cacheKey, payload, cacheTtlMs); return payload; } export const __testing = { buildSearxngSearchUrl, normalizeSearxngResult, parseSearxngResponseText, validateSearxngBaseUrl, SEARXNG_SEARCH_CACHE, };