mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-09 01:50:44 +08:00
适配 Bailian WebParser 兼容接入并完善 sidecar 文档。
新增 WebParser 兼容工具与 MCP health/tools-sync 兼容模式,让 WebParser/sse 可注册到 agent 并暴露明确 schema;同时修复 mcp argv 环境变量展开并补齐微信 sidecar 纯本地化安装与文档说明。 Made-with: Cursor
This commit is contained in:
parent
329fb2239a
commit
df91434936
16 changed files with 640 additions and 56 deletions
|
|
@ -22,6 +22,7 @@ This project exposes local atomic capabilities as shared `public` tools for all
|
|||
- `set_env`
|
||||
- `list_processes`
|
||||
- `kill_process`
|
||||
- `bailian_webparser` (DashScope WebParser compatibility tool)
|
||||
|
||||
## Loading path
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ Notes:
|
|||
|
||||
- No global `openclaw` CLI installation is required.
|
||||
- Runtime dependencies are installed locally in sidecar workspace.
|
||||
- Login/account state is persisted under `data/channel_sidecar/oclaw-weixin/state/`.
|
||||
- The Weixin flow does not rely on `%USERPROFILE%\.openclaw`.
|
||||
|
||||
## Path C: Enable WhatsApp (experimental)
|
||||
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ powershell -ExecutionPolicy Bypass -File .\scripts\stop_gateway.ps1 -Force
|
|||
|
||||
### 1.1.4 安装官方微信插件(Personal WeChat)
|
||||
|
||||
> 这一步会安装/更新官方插件到:`%USERPROFILE%\.openclaw\extensions\openclaw-weixin\`
|
||||
> 这一步会安装/更新微信 sidecar 运行时到:`data/channel_sidecar/oclaw-weixin/`
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File .\runtime\operations\scripts\weixin_install.ps1
|
||||
|
|
@ -87,10 +87,10 @@ powershell -ExecutionPolicy Bypass -File .\runtime\operations\scripts\weixin_ins
|
|||
powershell -ExecutionPolicy Bypass -File .\runtime\operations\scripts\weixin_login.ps1
|
||||
```
|
||||
|
||||
按提示扫码完成绑定(会写入账号 ID / token 等状态到 `%USERPROFILE%\.openclaw\openclaw-weixin\`)。
|
||||
按提示扫码完成绑定(会写入账号 ID / token 等状态到 `data/channel_sidecar/oclaw-weixin/state/`)。
|
||||
|
||||
> 说明:`weixin_install.ps1` **不要求你全局安装 openclaw**(不需要 `npm install -g openclaw`)。
|
||||
> 脚本会在 `data/channel_sidecar/oclaw-weixin/` 下安装本地运行时依赖,并完成官方插件安装。
|
||||
> 脚本会在 `data/channel_sidecar/oclaw-weixin/` 下安装本地运行时依赖与官方插件包(`node_modules`),全流程不依赖 `%USERPROFILE%\.openclaw`。
|
||||
|
||||
### 1.1.6 启动微信 sidecar(原生模式)
|
||||
|
||||
|
|
@ -548,6 +548,7 @@ powershell -ExecutionPolicy Bypass -File .\scripts\weixin_install.ps1
|
|||
```
|
||||
|
||||
注意:当前仅保留官方插件单链路,直接执行 `weixin_install.ps1` 即可。
|
||||
且为纯 sidecar 方案:不会写入 `%USERPROFILE%\.openclaw\`。
|
||||
|
||||
安装目录:
|
||||
|
||||
|
|
@ -563,6 +564,7 @@ powershell -ExecutionPolicy Bypass -File .\scripts\weixin_login.ps1
|
|||
|
||||
- `data/channel_sidecar/oclaw-weixin/state/oclaw-weixin/accounts/*.json`
|
||||
- `data/channel_sidecar/oclaw-weixin/state/oclaw-weixin/accounts.json`
|
||||
- 不会写入 `%USERPROFILE%\.openclaw\openclaw-weixin\`
|
||||
|
||||
### 11.3 启动微信 sidecar
|
||||
|
||||
|
|
|
|||
|
|
@ -120,6 +120,12 @@ def _mcp_health_and_sync_one(store: SqliteStore, row: dict[str, Any]) -> dict[st
|
|||
}
|
||||
store.set_mcp_server_health(server_id=sid, status="error", detail=item["health"])
|
||||
return item
|
||||
if _is_bailian_webparser_remote(entry_command=cmd, entry_args=args):
|
||||
tools = _bailian_webparser_virtual_tools()
|
||||
store.replace_mcp_server_tools(server_id=sid, tools=tools)
|
||||
detail = {"synced_tools": len(tools), "compat_mode": "bailian_webparser"}
|
||||
store.set_mcp_server_health(server_id=sid, status="ok", detail=detail)
|
||||
return {"server_id": sid, "ok": True, "health": detail, "tools_synced": len(tools)}
|
||||
rt = McpProcessRuntime(
|
||||
build_mcp_process_command(cmd, args, store=store),
|
||||
timeout_s=float(row.get("timeout_s") or 30.0),
|
||||
|
|
@ -158,6 +164,32 @@ def _mcp_health_and_sync_one(store: SqliteStore, row: dict[str, Any]) -> dict[st
|
|||
rt.stop()
|
||||
|
||||
|
||||
def _is_bailian_webparser_remote(*, entry_command: str, entry_args: list[str]) -> bool:
|
||||
cmd = str(entry_command or "").strip().lower()
|
||||
if cmd not in {"npx", "npx.cmd", "node"}:
|
||||
return False
|
||||
joined = " ".join(str(x or "").strip().lower() for x in (entry_args or []))
|
||||
return "mcp-remote" in joined and "/api/v1/mcps/webparser/sse" in joined
|
||||
|
||||
|
||||
def _bailian_webparser_virtual_tools() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"tool_name": "bailian_webparser_parse",
|
||||
"description": "Parse webpage via DashScope WebParser compatibility mode. Requires `url` (http/https).",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {"type": "string", "description": "Target webpage URL (required). Example: https://example.com"},
|
||||
"timeout": {"type": "integer", "default": 35, "minimum": 8, "maximum": 90},
|
||||
},
|
||||
"required": ["url"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _http_get_json(url: str, *, timeout: float = 8.0) -> dict[str, Any]:
|
||||
req = urllib_request.Request(
|
||||
url,
|
||||
|
|
@ -3178,6 +3210,10 @@ def build_admin_router() -> APIRouter:
|
|||
args = [str(x) for x in (row.get("entry_args") or []) if str(x).strip()]
|
||||
if not cmd:
|
||||
return {"ok": False, "error_code": "mcp_entry_missing", "error": "entry_command_missing"}
|
||||
if _is_bailian_webparser_remote(entry_command=cmd, entry_args=args):
|
||||
detail = {"ok": True, "status": "ok", "compat_mode": "bailian_webparser", "tools_count": 1}
|
||||
store.set_mcp_server_health(server_id=server_id, status="ok", detail=detail)
|
||||
return {"ok": True, "response": detail}
|
||||
rt = McpProcessRuntime(
|
||||
build_mcp_process_command(cmd, args, store=store),
|
||||
timeout_s=float(row.get("timeout_s") or 30.0),
|
||||
|
|
@ -3225,6 +3261,15 @@ def build_admin_router() -> APIRouter:
|
|||
args = [str(x) for x in (row.get("entry_args") or []) if str(x).strip()]
|
||||
if not cmd:
|
||||
return {"ok": False, "error_code": "mcp_entry_missing", "error": "entry_command_missing"}
|
||||
if _is_bailian_webparser_remote(entry_command=cmd, entry_args=args):
|
||||
norm = _bailian_webparser_virtual_tools()
|
||||
store.replace_mcp_server_tools(server_id=server_id, tools=norm)
|
||||
store.set_mcp_server_health(
|
||||
server_id=server_id,
|
||||
status="ok",
|
||||
detail={"synced_tools": len(norm), "compat_mode": "bailian_webparser"},
|
||||
)
|
||||
return {"ok": True, "server_id": server_id, "tools": norm, "compat_mode": "bailian_webparser"}
|
||||
rt = McpProcessRuntime(
|
||||
build_mcp_process_command(cmd, args, store=store),
|
||||
timeout_s=float(row.get("timeout_s") or 30.0),
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ function Resolve-RepoRoot {
|
|||
$oclawRoot = Resolve-RepoRoot
|
||||
$sidecarRoot = Join-Path $oclawRoot "data\\channel_sidecar\\$ChannelId"
|
||||
$stateDir = Join-Path $sidecarRoot "state"
|
||||
$pluginRoot = Join-Path $env:USERPROFILE ".openclaw\\extensions\\openclaw-weixin"
|
||||
function Sync-WeixinBridgeRunners {
|
||||
$bridgeSrc = Join-Path $oclawRoot "runtime\\operations\\weixin_bridge"
|
||||
foreach ($name in @("official_runner.ts", "login.ts")) {
|
||||
|
|
@ -29,41 +28,29 @@ New-Item -ItemType Directory -Force -Path $sidecarRoot | Out-Null
|
|||
New-Item -ItemType Directory -Force -Path (Join-Path $sidecarRoot "logs") | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
||||
|
||||
function Ensure-OfficialPluginRuntimeDeps {
|
||||
if (-not (Test-Path (Join-Path $pluginRoot "package.json"))) {
|
||||
throw "official plugin root not found at $pluginRoot"
|
||||
}
|
||||
Push-Location $pluginRoot
|
||||
try {
|
||||
npm.cmd install openclaw@latest --no-save
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "npm install official plugin runtime deps failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
}
|
||||
|
||||
Push-Location $sidecarRoot
|
||||
try {
|
||||
$npmRegistry = ($env:OCLAW_NPM_REGISTRY).Trim()
|
||||
$npmRegistryArgs = @()
|
||||
if ($npmRegistry) {
|
||||
$npmRegistryArgs = @("--registry", $npmRegistry)
|
||||
}
|
||||
if (-not (Test-Path (Join-Path $sidecarRoot "package.json"))) {
|
||||
npm.cmd init -y | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "npm init failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
npx.cmd -y @tencent-weixin/openclaw-weixin-cli@latest install
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "openclaw-weixin-cli install failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
npm.cmd install openclaw@latest --save tsx@4.21.0 typescript@6.0.3
|
||||
# Pure sidecar mode:
|
||||
# - No ~/.openclaw usage
|
||||
# - Install the official plugin package into node_modules and point it to $stateDir via OPENCLAW_STATE_DIR at runtime
|
||||
npm.cmd install --no-audit --no-fund --save @tencent-weixin/openclaw-weixin@latest openclaw@latest tsx@4.21.0 typescript@6.0.3 @npmRegistryArgs
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "npm install bridge runtime deps failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
} finally {
|
||||
Pop-Location
|
||||
}
|
||||
Ensure-OfficialPluginRuntimeDeps
|
||||
Sync-WeixinBridgeRunners
|
||||
Write-Host "[ok] installed official openclaw-weixin plugin sidecar runtime into $sidecarRoot"
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
|
|||
Push-Location $sidecarRoot
|
||||
try {
|
||||
$env:OCLAW_STATE_DIR = $stateDir
|
||||
# Ensure the official plugin stores state under the sidecar.
|
||||
$env:OPENCLAW_STATE_DIR = $stateDir
|
||||
if (Test-Path (Join-Path $sidecarRoot "login.ts")) {
|
||||
npm.cmd exec -- tsx login.ts
|
||||
exit 0
|
||||
|
|
|
|||
|
|
@ -1,24 +1,76 @@
|
|||
import { spawn } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
function resolveLocalOpenclawBin(): string {
|
||||
// Prefer local openclaw installed in the sidecar runtime (no global CLI required).
|
||||
// Windows: node_modules/.bin/openclaw.cmd
|
||||
return path.join(process.cwd(), "node_modules", ".bin", process.platform === "win32" ? "openclaw.cmd" : "openclaw");
|
||||
const STATE_DIR = (process.env.OCLAW_STATE_DIR || path.resolve(process.cwd(), "state")).trim();
|
||||
|
||||
function resolvePluginRoot(): string {
|
||||
const configured = String(process.env.OCLAW_WEIXIN_PLUGIN_ROOT || "").trim();
|
||||
return configured || path.join(process.cwd(), "node_modules", "@tencent-weixin", "openclaw-weixin");
|
||||
}
|
||||
|
||||
function run(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const openclawBin = resolveLocalOpenclawBin();
|
||||
const child = spawn(openclawBin, ["channels", "login", "--channel", "openclaw-weixin"], {
|
||||
stdio: "inherit",
|
||||
shell: true,
|
||||
});
|
||||
child.on("error", reject);
|
||||
child.on("exit", (code) => resolve(code ?? 1));
|
||||
async function importFromPluginSrc(relativePath: string): Promise<any> {
|
||||
const pluginRoot = resolvePluginRoot();
|
||||
const srcRoot = path.join(pluginRoot, "src");
|
||||
const fullPath = path.join(srcRoot, relativePath);
|
||||
return import(pathToFileURL(fullPath).href);
|
||||
}
|
||||
|
||||
async function run(): Promise<void> {
|
||||
// Force the plugin's state-dir resolver away from ~/.openclaw
|
||||
process.env.OPENCLAW_STATE_DIR = STATE_DIR;
|
||||
|
||||
const loginQr = await importFromPluginSrc(path.join("auth", "login-qr.ts"));
|
||||
const accounts = await importFromPluginSrc(path.join("auth", "accounts.ts"));
|
||||
|
||||
const startWeixinLoginWithQr = loginQr.startWeixinLoginWithQr as ((opts: any) => Promise<any>) | undefined;
|
||||
const waitForWeixinLogin = loginQr.waitForWeixinLogin as ((opts: any) => Promise<any>) | undefined;
|
||||
const displayQRCode = loginQr.displayQRCode as ((qrcodeUrl: string) => Promise<void>) | undefined;
|
||||
const registerWeixinAccountId = accounts.registerWeixinAccountId as ((accountId: string) => void) | undefined;
|
||||
const saveWeixinAccount = accounts.saveWeixinAccount as ((accountId: string, update: any) => void) | undefined;
|
||||
|
||||
if (!startWeixinLoginWithQr || !waitForWeixinLogin || !displayQRCode) {
|
||||
throw new Error("weixin plugin login-qr module missing exports");
|
||||
}
|
||||
if (!registerWeixinAccountId || !saveWeixinAccount) {
|
||||
throw new Error("weixin plugin accounts module missing exports");
|
||||
}
|
||||
|
||||
const apiBaseUrl = "https://ilinkai.weixin.qq.com";
|
||||
const botType = process.env.OCLAW_WEIXIN_BOT_TYPE?.trim();
|
||||
|
||||
const start = await startWeixinLoginWithQr({
|
||||
apiBaseUrl,
|
||||
botType,
|
||||
});
|
||||
if (!start.qrcodeUrl) {
|
||||
throw new Error(start.message || "failed to start login");
|
||||
}
|
||||
|
||||
process.stdout.write(`${start.message}\n`);
|
||||
await displayQRCode(start.qrcodeUrl);
|
||||
|
||||
const result = await waitForWeixinLogin({
|
||||
sessionKey: start.sessionKey,
|
||||
apiBaseUrl,
|
||||
botType,
|
||||
});
|
||||
|
||||
process.stdout.write(`${result.message}\n`);
|
||||
if (!result.connected || !result.accountId || !result.botToken) {
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
registerWeixinAccountId(result.accountId);
|
||||
saveWeixinAccount(result.accountId, {
|
||||
token: result.botToken,
|
||||
baseUrl: result.baseUrl,
|
||||
userId: result.userId,
|
||||
});
|
||||
process.stdout.write(`saved account=${result.accountId} into ${STATE_DIR}\n`);
|
||||
}
|
||||
|
||||
void run().then((code) => {
|
||||
process.exitCode = code;
|
||||
void run().catch((err) => {
|
||||
process.stderr.write(`weixin login failed: ${String(err)}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
|
|
@ -89,13 +88,9 @@ function sleep(ms: number): Promise<void> {
|
|||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function homeOpenclawPath(...parts: string[]): string {
|
||||
return path.join(os.homedir(), ".openclaw", ...parts);
|
||||
}
|
||||
|
||||
function resolvePluginRoot(): string {
|
||||
const configured = String(process.env.OCLAW_WEIXIN_PLUGIN_ROOT || "").trim();
|
||||
return configured || homeOpenclawPath("extensions", "openclaw-weixin");
|
||||
return configured || path.join(process.cwd(), "node_modules", "@tencent-weixin", "openclaw-weixin");
|
||||
}
|
||||
|
||||
async function loadOfficialModules(): Promise<OfficialModules> {
|
||||
|
|
@ -130,19 +125,34 @@ async function loadOfficialModules(): Promise<OfficialModules> {
|
|||
return officialModulesPromise;
|
||||
}
|
||||
|
||||
function resolveAccount(): { accountId: string; token: string; cloudBaseUrl: string } {
|
||||
const ids = readJsonFile<string[]>(homeOpenclawPath("openclaw-weixin", "accounts.json")) || [];
|
||||
async function resolveAccount(): Promise<{ accountId: string; token: string; cloudBaseUrl: string }> {
|
||||
if (!String(process.env.OPENCLAW_STATE_DIR || "").trim()) {
|
||||
process.env.OPENCLAW_STATE_DIR = STATE_DIR;
|
||||
}
|
||||
const pluginRoot = resolvePluginRoot();
|
||||
const srcRoot = path.join(pluginRoot, "src");
|
||||
const importTs = async (relativePath: string): Promise<any> => {
|
||||
const fullPath = path.join(srcRoot, relativePath);
|
||||
return import(pathToFileURL(fullPath).href);
|
||||
};
|
||||
const accountsMod = await importTs(path.join("auth", "accounts.ts"));
|
||||
const listIds = accountsMod.listIndexedWeixinAccountIds as (() => string[]) | undefined;
|
||||
const loadAcc = accountsMod.loadWeixinAccount as ((accountId: string) => any) | undefined;
|
||||
if (!listIds || !loadAcc) {
|
||||
throw new Error("weixin plugin accounts module missing exports");
|
||||
}
|
||||
const ids = listIds() || [];
|
||||
const accountId = String(ids[0] || "").trim();
|
||||
if (!accountId) {
|
||||
throw new Error("no weixin account id found; run login first");
|
||||
}
|
||||
const account = readJsonFile<Json>(homeOpenclawPath("openclaw-weixin", "accounts", `${accountId}.json`)) || {};
|
||||
const token = String(account.token || "").trim();
|
||||
const data = loadAcc(accountId) || {};
|
||||
const token = String(data.token || "").trim();
|
||||
if (!token) {
|
||||
throw new Error(`missing token for account ${accountId}; run login again`);
|
||||
}
|
||||
const envCloud = String(process.env.OCLAW_WEIXIN_CLOUD_BASE_URL || "").trim();
|
||||
const cfgCloud = String(account.baseUrl || "").trim();
|
||||
const cfgCloud = String(data.baseUrl || "").trim();
|
||||
const cloudBaseUrl = (envCloud || cfgCloud || "https://ilinkai.weixin.qq.com").trim();
|
||||
return { accountId, token, cloudBaseUrl };
|
||||
}
|
||||
|
|
@ -424,7 +434,7 @@ async function main(): Promise<void> {
|
|||
state.user_context_tokens && typeof state.user_context_tokens === "object"
|
||||
? (state.user_context_tokens as TokenMap)
|
||||
: {};
|
||||
const { accountId, token, cloudBaseUrl } = resolveAccount();
|
||||
const { accountId, token, cloudBaseUrl } = await resolveAccount();
|
||||
const modules = await loadOfficialModules();
|
||||
modules.restoreContextTokens(accountId);
|
||||
log(`official runner started account=${accountId} cloud=${cloudBaseUrl} local=${LOCAL_BASE_URL}`);
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from oclaw.runtime.skills import SkillSpec, materialize_skills_from_tool_specs
|
|||
from oclaw.runtime.tools.base import ToolSpec
|
||||
from oclaw.runtime.tools.mcp.filesystem_argv import build_mcp_process_command
|
||||
from oclaw.runtime.tools.mcp.runtime import McpProcessRuntime
|
||||
from oclaw.runtime.tools.public.bailian_webparser_tool import bailian_webparser_tool
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -64,6 +65,14 @@ def materialize_mcp_tools_for_specialist(
|
|||
path_policy_tenant_id: str | None = None,
|
||||
path_policy_user_id: str | None = None,
|
||||
) -> list[ToolSpec]:
|
||||
def _is_bailian_webparser_remote_row(r: dict[str, Any]) -> bool:
|
||||
cmd2 = str(r.get("entry_command") or "").strip().lower()
|
||||
if cmd2 not in {"npx", "npx.cmd", "node"}:
|
||||
return False
|
||||
argv = [str(x or "").strip().lower() for x in (r.get("entry_args") or [])]
|
||||
joined = " ".join(argv)
|
||||
return "mcp-remote" in joined and "/api/v1/mcps/webparser/sse" in joined
|
||||
|
||||
sp = str(specialist or "").strip().lower()
|
||||
if sp == "manager":
|
||||
# Manager is a first-class binding role in admin UI/config.
|
||||
|
|
@ -125,9 +134,27 @@ def materialize_mcp_tools_for_specialist(
|
|||
except Exception:
|
||||
tools = []
|
||||
for t in tools:
|
||||
tname = str(t.get("tool_name") or "")
|
||||
if _is_bailian_webparser_remote_row(row) and tname == "bailian_webparser_parse":
|
||||
compat = bailian_webparser_tool()
|
||||
out.append(
|
||||
ToolSpec(
|
||||
name=f"mcp__{server_id}__{tname}",
|
||||
description=str(t.get("description") or compat.description),
|
||||
parameters=t.get("parameters") if isinstance(t.get("parameters"), dict) else compat.parameters,
|
||||
handler=compat.handler,
|
||||
tags=frozenset({"mcp", "plugin", "compat"}),
|
||||
version="v1",
|
||||
risk_level="high",
|
||||
timeout_s=float(row.get("timeout_s") or 30.0),
|
||||
required_permissions=frozenset(str(x) for x in (row.get("required_permissions") or [])),
|
||||
execution_mode="subprocess",
|
||||
)
|
||||
)
|
||||
continue
|
||||
spec = _McpBoundTool(
|
||||
server_id=server_id,
|
||||
tool_name=str(t.get("tool_name") or ""),
|
||||
tool_name=tname,
|
||||
description=str(t.get("description") or f"MCP tool {t.get('tool_name') or ''}"),
|
||||
parameters=t.get("parameters") if isinstance(t.get("parameters"), dict) else {},
|
||||
command=command,
|
||||
|
|
|
|||
|
|
@ -212,9 +212,17 @@ def build_mcp_process_command(
|
|||
path_policy_tenant_id: str | None = None,
|
||||
path_policy_user_id: str | None = None,
|
||||
) -> list[str]:
|
||||
"""``[cmd] + args`` after filesystem argv augmentation."""
|
||||
"""``[cmd] + args`` after env placeholder expansion + filesystem argv augmentation."""
|
||||
base = [str(cmd)]
|
||||
for x in args:
|
||||
s = str(x).strip()
|
||||
if not s:
|
||||
continue
|
||||
# Allow mcp entry args like: "Authorization: Bearer ${DASHSCOPE_API_KEY}".
|
||||
# Unknown vars are kept unchanged by expandvars.
|
||||
base.append(os.path.expandvars(s))
|
||||
return augment_filesystem_mcp_argv(
|
||||
[cmd] + [x for x in args if str(x).strip()],
|
||||
base,
|
||||
store=store,
|
||||
policy_session_id=policy_session_id,
|
||||
path_policy_tenant_id=path_policy_tenant_id,
|
||||
|
|
|
|||
281
runtime/tools/public/bailian_webparser_tool.py
Normal file
281
runtime/tools/public/bailian_webparser_tool.py
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from oclaw.runtime.tools.base import ToolSpec
|
||||
|
||||
_WEBPARSER_SSE_URL = "https://dashscope.aliyuncs.com/api/v1/mcps/WebParser/sse"
|
||||
_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
def _open_request(
|
||||
url: str,
|
||||
*,
|
||||
timeout_s: int,
|
||||
headers: dict[str, str],
|
||||
body: dict[str, Any] | None = None,
|
||||
):
|
||||
data = None
|
||||
method = "GET"
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
method = "POST"
|
||||
req = urllib.request.Request(str(url), data=data, headers=headers, method=method)
|
||||
return urllib.request.urlopen(req, timeout=max(5, min(int(timeout_s), 90)))
|
||||
|
||||
|
||||
def _read_sse_until(
|
||||
sse_resp,
|
||||
*,
|
||||
timeout_s: int,
|
||||
message_url: str | None = None,
|
||||
wait_id: int | None = None,
|
||||
auth_header: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
deadline = time.time() + max(3, int(timeout_s))
|
||||
event_name = ""
|
||||
while time.time() < deadline:
|
||||
raw = sse_resp.readline()
|
||||
if not raw:
|
||||
continue
|
||||
line = raw.decode("utf-8", errors="replace").strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[6:].strip()
|
||||
continue
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data = line[5:].strip()
|
||||
if event_name == "endpoint" and wait_id is None:
|
||||
return {"endpoint": data}
|
||||
try:
|
||||
obj = json.loads(data)
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
# Keep-alive ping from server; respond with empty result.
|
||||
if (
|
||||
message_url
|
||||
and isinstance(obj.get("id"), (int, str))
|
||||
and str(obj.get("method") or "").strip() == "ping"
|
||||
):
|
||||
try:
|
||||
_open_request(
|
||||
message_url,
|
||||
timeout_s=10,
|
||||
headers={
|
||||
"Authorization": auth_header,
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": _UA,
|
||||
},
|
||||
body={"jsonrpc": "2.0", "id": obj.get("id"), "result": {}},
|
||||
).close()
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
if wait_id is not None and obj.get("id") == wait_id:
|
||||
return obj
|
||||
return None
|
||||
|
||||
|
||||
def _call_webparser(url: str, *, api_key: str, timeout_s: int) -> dict[str, Any]:
|
||||
auth_header = f"Bearer {api_key}"
|
||||
sse_headers = {
|
||||
"Authorization": auth_header,
|
||||
"Accept": "text/event-stream",
|
||||
"User-Agent": _UA,
|
||||
}
|
||||
with _open_request(_WEBPARSER_SSE_URL, timeout_s=timeout_s, headers=sse_headers) as sse_resp:
|
||||
endpoint_evt = _read_sse_until(sse_resp, timeout_s=timeout_s)
|
||||
endpoint = str((endpoint_evt or {}).get("endpoint") or "").strip()
|
||||
if not endpoint or "/message?" not in endpoint:
|
||||
return {"ok": False, "error_code": "webparser_no_endpoint", "error": "webparser_no_endpoint"}
|
||||
message_url = urllib.parse.urljoin(_WEBPARSER_SSE_URL, endpoint)
|
||||
|
||||
def _post(payload: dict[str, Any], post_timeout: int = 30) -> None:
|
||||
with _open_request(
|
||||
message_url,
|
||||
timeout_s=post_timeout,
|
||||
headers={
|
||||
"Authorization": auth_header,
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": _UA,
|
||||
},
|
||||
body=payload,
|
||||
):
|
||||
return
|
||||
|
||||
_post(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "oclaw-bailian-webparser", "version": "0.1.0"},
|
||||
},
|
||||
}
|
||||
)
|
||||
init_res = _read_sse_until(
|
||||
sse_resp,
|
||||
timeout_s=timeout_s,
|
||||
message_url=message_url,
|
||||
wait_id=1,
|
||||
auth_header=auth_header,
|
||||
)
|
||||
if not init_res or not isinstance(init_res.get("result"), dict):
|
||||
return {"ok": False, "error_code": "webparser_initialize_failed", "error": "initialize_no_result"}
|
||||
|
||||
_post({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, post_timeout=12)
|
||||
_post({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}})
|
||||
list_res = _read_sse_until(
|
||||
sse_resp,
|
||||
timeout_s=timeout_s,
|
||||
message_url=message_url,
|
||||
wait_id=2,
|
||||
auth_header=auth_header,
|
||||
)
|
||||
tools = []
|
||||
try:
|
||||
tools = list((list_res or {}).get("result", {}).get("tools") or [])
|
||||
except Exception:
|
||||
tools = []
|
||||
if not tools:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "webparser_tools_empty",
|
||||
"error": "tools_list_empty",
|
||||
"initialize": init_res,
|
||||
"tools_list": list_res,
|
||||
}
|
||||
tool_name = str((tools[0] or {}).get("name") or "").strip()
|
||||
if not tool_name:
|
||||
return {"ok": False, "error_code": "webparser_tool_name_missing", "error": "tool_name_missing"}
|
||||
|
||||
candidate_args = (
|
||||
{"url": url},
|
||||
{"urls": [url]},
|
||||
{"query": url},
|
||||
)
|
||||
call_res: dict[str, Any] | None = None
|
||||
used_args: dict[str, Any] | None = None
|
||||
for args in candidate_args:
|
||||
_post({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": tool_name, "arguments": args}})
|
||||
call_res = _read_sse_until(
|
||||
sse_resp,
|
||||
timeout_s=timeout_s,
|
||||
message_url=message_url,
|
||||
wait_id=3,
|
||||
auth_header=auth_header,
|
||||
)
|
||||
if call_res and not isinstance(call_res.get("error"), dict):
|
||||
used_args = args
|
||||
break
|
||||
if not call_res:
|
||||
return {"ok": False, "error_code": "webparser_call_timeout", "error": "call_timeout"}
|
||||
if isinstance(call_res.get("error"), dict):
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "webparser_call_failed",
|
||||
"error": str((call_res.get("error") or {}).get("message") or "webparser_call_failed"),
|
||||
"rpc_error": call_res.get("error"),
|
||||
"tool_name": tool_name,
|
||||
"arguments": used_args or {"url": url},
|
||||
}
|
||||
result = call_res.get("result")
|
||||
text_parts: list[str] = []
|
||||
if isinstance(result, dict):
|
||||
for it in list(result.get("content") or []):
|
||||
if isinstance(it, dict) and str(it.get("type") or "") == "text":
|
||||
txt = str(it.get("text") or "").strip()
|
||||
if txt:
|
||||
text_parts.append(txt)
|
||||
return {
|
||||
"ok": True,
|
||||
"tool_name": tool_name,
|
||||
"arguments": used_args or {"url": url},
|
||||
"text": "\n\n".join(text_parts).strip(),
|
||||
"result": result,
|
||||
}
|
||||
|
||||
|
||||
def bailian_webparser_tool() -> ToolSpec:
|
||||
def _handler(args: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = args if isinstance(args, dict) else {}
|
||||
url = str(payload.get("url") or "").strip()
|
||||
if not url:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "url_required",
|
||||
"error": 'url_required: provide {"url":"https://example.com/article"}',
|
||||
}
|
||||
timeout_s = max(8, min(int(payload.get("timeout") or 35), 90))
|
||||
api_key = str(payload.get("api_key") or os.getenv("DASHSCOPE_API_KEY") or "").strip()
|
||||
if not api_key:
|
||||
return {"ok": False, "error_code": "api_key_missing", "error": "DASHSCOPE_API_KEY required"}
|
||||
try:
|
||||
p = urllib.parse.urlparse(url)
|
||||
except Exception:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "invalid_url",
|
||||
"error": 'invalid_url: provide full http/https URL, e.g. {"url":"https://example.com/article"}',
|
||||
}
|
||||
if p.scheme not in {"http", "https"}:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "unsupported_scheme",
|
||||
"error": "unsupported_scheme: url must start with http:// or https://",
|
||||
}
|
||||
try:
|
||||
out = _call_webparser(url, api_key=api_key, timeout_s=timeout_s)
|
||||
return out
|
||||
except Exception as exc:
|
||||
return {"ok": False, "error_code": "webparser_runtime_failed", "error": f"{type(exc).__name__}: {exc}"}
|
||||
|
||||
return ToolSpec(
|
||||
name="bailian_webparser",
|
||||
description="Parse webpage content via DashScope WebParser (SSE-compatible fallback path). `url` is required.",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "Target webpage URL to parse (required). Example: https://example.com/article",
|
||||
},
|
||||
"timeout": {
|
||||
"type": "integer",
|
||||
"default": 35,
|
||||
"minimum": 8,
|
||||
"maximum": 90,
|
||||
"description": "Total timeout seconds [8..90].",
|
||||
},
|
||||
"api_key": {
|
||||
"type": "string",
|
||||
"description": "Optional DashScope API key override; defaults to DASHSCOPE_API_KEY.",
|
||||
},
|
||||
},
|
||||
"required": ["url"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=_handler,
|
||||
tags=frozenset({"public", "web", "parser", "read"}),
|
||||
risk_level="low",
|
||||
read_only=True,
|
||||
timeout_s=95.0,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["bailian_webparser_tool"]
|
||||
|
||||
81
tests/test_bailian_webparser_tool.py
Normal file
81
tests/test_bailian_webparser_tool.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from oclaw.runtime.tools.public.bailian_webparser_tool import bailian_webparser_tool
|
||||
|
||||
|
||||
class _SseResp:
|
||||
def __init__(self, lines: list[str]):
|
||||
self._lines = [x.encode("utf-8") for x in lines]
|
||||
|
||||
def readline(self) -> bytes:
|
||||
if not self._lines:
|
||||
return b""
|
||||
return self._lines.pop(0)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _PostResp:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
def test_bailian_webparser_sse_flow(monkeypatch) -> None:
|
||||
sse_lines = [
|
||||
"event:endpoint\n",
|
||||
"data:/api/v1/mcps/WebParser/message?sessionId=abc\n",
|
||||
"\n",
|
||||
"event:message\n",
|
||||
'data:{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2024-11-05","capabilities":{"tools":{"listChanged":true}},"serverInfo":{"name":"x","version":"1"}}}\n',
|
||||
"\n",
|
||||
"event:message\n",
|
||||
'data:{"jsonrpc":"2.0","id":2,"result":{"tools":[{"name":"webparser","description":"d","inputSchema":{"type":"object"}}]}}\n',
|
||||
"\n",
|
||||
"event:message\n",
|
||||
'data:{"jsonrpc":"2.0","id":3,"result":{"content":[{"type":"text","text":"parsed ok"}]}}\n',
|
||||
"\n",
|
||||
]
|
||||
|
||||
def _fake_open(req, timeout=0): # noqa: ARG001
|
||||
method = str(getattr(req, "method", "") or "GET").upper()
|
||||
url = str(getattr(req, "full_url", "") or "")
|
||||
if method == "GET" and url.endswith("/WebParser/sse"):
|
||||
return _SseResp(sse_lines)
|
||||
if method == "POST" and "/message?sessionId=abc" in url:
|
||||
# Ensure request is valid JSON-RPC object.
|
||||
raw = getattr(req, "data", b"") or b""
|
||||
_ = json.loads(raw.decode("utf-8"))
|
||||
return _PostResp()
|
||||
raise AssertionError(f"unexpected request method={method} url={url}")
|
||||
|
||||
monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-test")
|
||||
monkeypatch.setattr("urllib.request.urlopen", _fake_open)
|
||||
|
||||
out = bailian_webparser_tool().handler({"url": "https://example.com/x"})
|
||||
assert out.get("ok") is True, out
|
||||
assert out.get("tool_name") == "webparser"
|
||||
assert "parsed ok" in str(out.get("text") or "")
|
||||
|
||||
|
||||
def test_bailian_webparser_requires_key(monkeypatch) -> None:
|
||||
monkeypatch.delenv("DASHSCOPE_API_KEY", raising=False)
|
||||
out = bailian_webparser_tool().handler({"url": "https://example.com/x"})
|
||||
assert out.get("ok") is False
|
||||
assert out.get("error_code") == "api_key_missing"
|
||||
|
||||
|
||||
def test_bailian_webparser_missing_url_hint() -> None:
|
||||
out = bailian_webparser_tool().handler({})
|
||||
assert out.get("ok") is False
|
||||
assert out.get("error_code") == "url_required"
|
||||
assert "https://example.com/article" in str(out.get("error") or "")
|
||||
|
||||
|
|
@ -197,3 +197,18 @@ def test_run_command_does_not_follow_cd_state(tmp_path: Path, monkeypatch) -> No
|
|||
# If run_command follows cd state this would be "subdir"; default should be data/workspace.
|
||||
assert str(out_run.get("cwd") or "").replace("\\", "/").rstrip("/").endswith("/data/workspace")
|
||||
|
||||
|
||||
def test_run_command_reads_db_toggle_without_ops_db_env(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("OPS_WORKSPACE_ROOT", str(tmp_path))
|
||||
monkeypatch.delenv("OPS_ASSISTANT_DB_PATH", raising=False)
|
||||
monkeypatch.setenv("AIA_ENABLE_RUN_COMMAND", "0")
|
||||
|
||||
db_file = tmp_path / "ops.sqlite"
|
||||
from oclaw.platform.persistence.sqlite_store import SqliteStore
|
||||
|
||||
SqliteStore(str(db_file)).set_setting("AIA_ENABLE_RUN_COMMAND", "1")
|
||||
monkeypatch.setattr("oclaw.platform.config.paths.db_path", lambda: str(db_file))
|
||||
|
||||
out = LocalAdapter().run_command(command="echo hi", timeout=10)
|
||||
assert out.get("ok") is True, out
|
||||
|
||||
|
|
|
|||
|
|
@ -166,10 +166,42 @@ class McpAdapterTests(unittest.TestCase):
|
|||
self.assertIn("GOOGLE_CALENDAR_MCP_TOKEN_PATH", keys)
|
||||
self.assertIn("GITHUB_PERSONAL_ACCESS_TOKEN", keys)
|
||||
self.assertIn("CONTEXT7_API_KEY", keys)
|
||||
self.assertIn("DASHSCOPE_API_KEY", keys)
|
||||
finally:
|
||||
if old is not None:
|
||||
os.environ["OPS_MCP_ENV_ALLOWLIST"] = old
|
||||
|
||||
def test_materialize_bailian_webparser_compat_tool(self) -> None:
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as td:
|
||||
store = SqliteStore(str(Path(td) / "ops.sqlite"))
|
||||
store.upsert_mcp_server(
|
||||
server_id="webparser-compat",
|
||||
source_type="npm",
|
||||
source_ref="mcp-remote",
|
||||
entry_command="npx",
|
||||
entry_args=[
|
||||
"-y",
|
||||
"mcp-remote",
|
||||
"https://dashscope.aliyuncs.com/api/v1/mcps/WebParser/sse",
|
||||
"--header",
|
||||
"Authorization: Bearer ${DASHSCOPE_API_KEY}",
|
||||
],
|
||||
enabled=True,
|
||||
)
|
||||
store.replace_mcp_server_tools(
|
||||
server_id="webparser-compat",
|
||||
tools=[
|
||||
{
|
||||
"tool_name": "bailian_webparser_parse",
|
||||
"description": "compat",
|
||||
"parameters": {"type": "object", "properties": {"url": {"type": "string"}}},
|
||||
}
|
||||
],
|
||||
)
|
||||
specs = materialize_mcp_tools(store)
|
||||
names = {x.name for x in specs}
|
||||
self.assertIn("mcp__webparser-compat__bailian_webparser_parse", names)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -140,6 +140,33 @@ class McpAdminApiTests(unittest.TestCase):
|
|||
tools = sync.json().get("tools") or []
|
||||
self.assertTrue(any(str(t.get("tool_name") or "") == "ping" for t in tools))
|
||||
|
||||
def test_healthcheck_and_tools_sync_bailian_webparser_compat(self) -> None:
|
||||
store = SqliteStore(db_path())
|
||||
store.upsert_mcp_server(
|
||||
server_id="webparser-compat",
|
||||
source_type="npm",
|
||||
source_ref="mcp-remote",
|
||||
entry_command="npx",
|
||||
entry_args=[
|
||||
"-y",
|
||||
"mcp-remote",
|
||||
"https://dashscope.aliyuncs.com/api/v1/mcps/WebParser/sse",
|
||||
"--header",
|
||||
"Authorization: Bearer ${DASHSCOPE_API_KEY}",
|
||||
],
|
||||
enabled=True,
|
||||
)
|
||||
health = self.client.post("/admin/api/mcp/healthcheck", json={"server_id": "webparser-compat"}, headers=self._headers())
|
||||
self.assertEqual(health.status_code, 200)
|
||||
self.assertTrue(health.json().get("ok"), health.json())
|
||||
self.assertEqual(str((health.json().get("response") or {}).get("compat_mode") or ""), "bailian_webparser")
|
||||
|
||||
sync = self.client.post("/admin/api/mcp/tools/sync", json={"server_id": "webparser-compat"}, headers=self._headers())
|
||||
self.assertEqual(sync.status_code, 200)
|
||||
self.assertTrue(sync.json().get("ok"), sync.json())
|
||||
tools = sync.json().get("tools") or []
|
||||
self.assertTrue(any(str(t.get("tool_name") or "") == "bailian_webparser_parse" for t in tools))
|
||||
|
||||
def test_reinstall_from_saved_manifest(self) -> None:
|
||||
script = self._write_mcp_server()
|
||||
store = SqliteStore(db_path())
|
||||
|
|
|
|||
|
|
@ -38,6 +38,18 @@ class McpFilesystemArgvTests(unittest.TestCase):
|
|||
out = build_mcp_process_command("echo", ["a"], store=None)
|
||||
self.assertEqual(out, ["echo", "a"])
|
||||
|
||||
def test_build_mcp_process_command_expands_env_placeholders(self) -> None:
|
||||
with mock.patch.dict(os.environ, {"DASHSCOPE_API_KEY": "sk-test-123"}, clear=False):
|
||||
out = build_mcp_process_command(
|
||||
"npx",
|
||||
["-y", "mcp-remote", "https://example.com/sse", "--header", "Authorization: Bearer ${DASHSCOPE_API_KEY}"],
|
||||
store=None,
|
||||
)
|
||||
self.assertEqual(
|
||||
out,
|
||||
["npx", "-y", "mcp-remote", "https://example.com/sse", "--header", "Authorization: Bearer sk-test-123"],
|
||||
)
|
||||
|
||||
def test_policy_session_only_that_users_db_roots(self) -> None:
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as td:
|
||||
dbf = Path(td) / "ops.sqlite"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue