mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-13 04:40:44 +08:00
适配 Bailian WebParser 兼容接入并完善 sidecar 文档。
新增 WebParser 兼容工具与 MCP health/tools-sync 兼容模式，让 WebParser/sse 可注册到 agent 并暴露明确 schema；同时修复 mcp argv 环境变量展开并补齐微信 sidecar 纯本地化安装与文档说明。 Made-with: Cursor
This commit is contained in:
parent
329fb2239a
commit
df91434936
16 changed files with 640 additions and 56 deletions
|
|
@ -10,6 +10,7 @@ from oclaw.runtime.skills import SkillSpec, materialize_skills_from_tool_specs
|
|||
from oclaw.runtime.tools.base import ToolSpec
|
||||
from oclaw.runtime.tools.mcp.filesystem_argv import build_mcp_process_command
|
||||
from oclaw.runtime.tools.mcp.runtime import McpProcessRuntime
|
||||
from oclaw.runtime.tools.public.bailian_webparser_tool import bailian_webparser_tool
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -64,6 +65,14 @@ def materialize_mcp_tools_for_specialist(
|
|||
path_policy_tenant_id: str | None = None,
|
||||
path_policy_user_id: str | None = None,
|
||||
) -> list[ToolSpec]:
|
||||
def _is_bailian_webparser_remote_row(r: dict[str, Any]) -> bool:
|
||||
cmd2 = str(r.get("entry_command") or "").strip().lower()
|
||||
if cmd2 not in {"npx", "npx.cmd", "node"}:
|
||||
return False
|
||||
argv = [str(x or "").strip().lower() for x in (r.get("entry_args") or [])]
|
||||
joined = " ".join(argv)
|
||||
return "mcp-remote" in joined and "/api/v1/mcps/webparser/sse" in joined
|
||||
|
||||
sp = str(specialist or "").strip().lower()
|
||||
if sp == "manager":
|
||||
# Manager is a first-class binding role in admin UI/config.
|
||||
|
|
@ -125,9 +134,27 @@ def materialize_mcp_tools_for_specialist(
|
|||
except Exception:
|
||||
tools = []
|
||||
for t in tools:
|
||||
tname = str(t.get("tool_name") or "")
|
||||
if _is_bailian_webparser_remote_row(row) and tname == "bailian_webparser_parse":
|
||||
compat = bailian_webparser_tool()
|
||||
out.append(
|
||||
ToolSpec(
|
||||
name=f"mcp__{server_id}__{tname}",
|
||||
description=str(t.get("description") or compat.description),
|
||||
parameters=t.get("parameters") if isinstance(t.get("parameters"), dict) else compat.parameters,
|
||||
handler=compat.handler,
|
||||
tags=frozenset({"mcp", "plugin", "compat"}),
|
||||
version="v1",
|
||||
risk_level="high",
|
||||
timeout_s=float(row.get("timeout_s") or 30.0),
|
||||
required_permissions=frozenset(str(x) for x in (row.get("required_permissions") or [])),
|
||||
execution_mode="subprocess",
|
||||
)
|
||||
)
|
||||
continue
|
||||
spec = _McpBoundTool(
|
||||
server_id=server_id,
|
||||
tool_name=str(t.get("tool_name") or ""),
|
||||
tool_name=tname,
|
||||
description=str(t.get("description") or f"MCP tool {t.get('tool_name') or ''}"),
|
||||
parameters=t.get("parameters") if isinstance(t.get("parameters"), dict) else {},
|
||||
command=command,
|
||||
|
|
|
|||
|
|
@ -212,9 +212,17 @@ def build_mcp_process_command(
|
|||
path_policy_tenant_id: str | None = None,
|
||||
path_policy_user_id: str | None = None,
|
||||
) -> list[str]:
|
||||
"""``[cmd] + args`` after filesystem argv augmentation."""
|
||||
"""``[cmd] + args`` after env placeholder expansion + filesystem argv augmentation."""
|
||||
base = [str(cmd)]
|
||||
for x in args:
|
||||
s = str(x).strip()
|
||||
if not s:
|
||||
continue
|
||||
# Allow mcp entry args like: "Authorization: Bearer ${DASHSCOPE_API_KEY}".
|
||||
# Unknown vars are kept unchanged by expandvars.
|
||||
base.append(os.path.expandvars(s))
|
||||
return augment_filesystem_mcp_argv(
|
||||
[cmd] + [x for x in args if str(x).strip()],
|
||||
base,
|
||||
store=store,
|
||||
policy_session_id=policy_session_id,
|
||||
path_policy_tenant_id=path_policy_tenant_id,
|
||||
|
|
|
|||
281
runtime/tools/public/bailian_webparser_tool.py
Normal file
281
runtime/tools/public/bailian_webparser_tool.py
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from oclaw.runtime.tools.base import ToolSpec
|
||||
|
||||
_WEBPARSER_SSE_URL = "https://dashscope.aliyuncs.com/api/v1/mcps/WebParser/sse"
|
||||
_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
def _open_request(
|
||||
url: str,
|
||||
*,
|
||||
timeout_s: int,
|
||||
headers: dict[str, str],
|
||||
body: dict[str, Any] | None = None,
|
||||
):
|
||||
data = None
|
||||
method = "GET"
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||
method = "POST"
|
||||
req = urllib.request.Request(str(url), data=data, headers=headers, method=method)
|
||||
return urllib.request.urlopen(req, timeout=max(5, min(int(timeout_s), 90)))
|
||||
|
||||
|
||||
def _read_sse_until(
|
||||
sse_resp,
|
||||
*,
|
||||
timeout_s: int,
|
||||
message_url: str | None = None,
|
||||
wait_id: int | None = None,
|
||||
auth_header: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
deadline = time.time() + max(3, int(timeout_s))
|
||||
event_name = ""
|
||||
while time.time() < deadline:
|
||||
raw = sse_resp.readline()
|
||||
if not raw:
|
||||
continue
|
||||
line = raw.decode("utf-8", errors="replace").strip()
|
||||
if not line:
|
||||
continue
|
||||
if line.startswith("event:"):
|
||||
event_name = line[6:].strip()
|
||||
continue
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
data = line[5:].strip()
|
||||
if event_name == "endpoint" and wait_id is None:
|
||||
return {"endpoint": data}
|
||||
try:
|
||||
obj = json.loads(data)
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
# Keep-alive ping from server; respond with empty result.
|
||||
if (
|
||||
message_url
|
||||
and isinstance(obj.get("id"), (int, str))
|
||||
and str(obj.get("method") or "").strip() == "ping"
|
||||
):
|
||||
try:
|
||||
_open_request(
|
||||
message_url,
|
||||
timeout_s=10,
|
||||
headers={
|
||||
"Authorization": auth_header,
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": _UA,
|
||||
},
|
||||
body={"jsonrpc": "2.0", "id": obj.get("id"), "result": {}},
|
||||
).close()
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
if wait_id is not None and obj.get("id") == wait_id:
|
||||
return obj
|
||||
return None
|
||||
|
||||
|
||||
def _call_webparser(url: str, *, api_key: str, timeout_s: int) -> dict[str, Any]:
|
||||
auth_header = f"Bearer {api_key}"
|
||||
sse_headers = {
|
||||
"Authorization": auth_header,
|
||||
"Accept": "text/event-stream",
|
||||
"User-Agent": _UA,
|
||||
}
|
||||
with _open_request(_WEBPARSER_SSE_URL, timeout_s=timeout_s, headers=sse_headers) as sse_resp:
|
||||
endpoint_evt = _read_sse_until(sse_resp, timeout_s=timeout_s)
|
||||
endpoint = str((endpoint_evt or {}).get("endpoint") or "").strip()
|
||||
if not endpoint or "/message?" not in endpoint:
|
||||
return {"ok": False, "error_code": "webparser_no_endpoint", "error": "webparser_no_endpoint"}
|
||||
message_url = urllib.parse.urljoin(_WEBPARSER_SSE_URL, endpoint)
|
||||
|
||||
def _post(payload: dict[str, Any], post_timeout: int = 30) -> None:
|
||||
with _open_request(
|
||||
message_url,
|
||||
timeout_s=post_timeout,
|
||||
headers={
|
||||
"Authorization": auth_header,
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": _UA,
|
||||
},
|
||||
body=payload,
|
||||
):
|
||||
return
|
||||
|
||||
_post(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {},
|
||||
"clientInfo": {"name": "oclaw-bailian-webparser", "version": "0.1.0"},
|
||||
},
|
||||
}
|
||||
)
|
||||
init_res = _read_sse_until(
|
||||
sse_resp,
|
||||
timeout_s=timeout_s,
|
||||
message_url=message_url,
|
||||
wait_id=1,
|
||||
auth_header=auth_header,
|
||||
)
|
||||
if not init_res or not isinstance(init_res.get("result"), dict):
|
||||
return {"ok": False, "error_code": "webparser_initialize_failed", "error": "initialize_no_result"}
|
||||
|
||||
_post({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, post_timeout=12)
|
||||
_post({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}})
|
||||
list_res = _read_sse_until(
|
||||
sse_resp,
|
||||
timeout_s=timeout_s,
|
||||
message_url=message_url,
|
||||
wait_id=2,
|
||||
auth_header=auth_header,
|
||||
)
|
||||
tools = []
|
||||
try:
|
||||
tools = list((list_res or {}).get("result", {}).get("tools") or [])
|
||||
except Exception:
|
||||
tools = []
|
||||
if not tools:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "webparser_tools_empty",
|
||||
"error": "tools_list_empty",
|
||||
"initialize": init_res,
|
||||
"tools_list": list_res,
|
||||
}
|
||||
tool_name = str((tools[0] or {}).get("name") or "").strip()
|
||||
if not tool_name:
|
||||
return {"ok": False, "error_code": "webparser_tool_name_missing", "error": "tool_name_missing"}
|
||||
|
||||
candidate_args = (
|
||||
{"url": url},
|
||||
{"urls": [url]},
|
||||
{"query": url},
|
||||
)
|
||||
call_res: dict[str, Any] | None = None
|
||||
used_args: dict[str, Any] | None = None
|
||||
for args in candidate_args:
|
||||
_post({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": tool_name, "arguments": args}})
|
||||
call_res = _read_sse_until(
|
||||
sse_resp,
|
||||
timeout_s=timeout_s,
|
||||
message_url=message_url,
|
||||
wait_id=3,
|
||||
auth_header=auth_header,
|
||||
)
|
||||
if call_res and not isinstance(call_res.get("error"), dict):
|
||||
used_args = args
|
||||
break
|
||||
if not call_res:
|
||||
return {"ok": False, "error_code": "webparser_call_timeout", "error": "call_timeout"}
|
||||
if isinstance(call_res.get("error"), dict):
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "webparser_call_failed",
|
||||
"error": str((call_res.get("error") or {}).get("message") or "webparser_call_failed"),
|
||||
"rpc_error": call_res.get("error"),
|
||||
"tool_name": tool_name,
|
||||
"arguments": used_args or {"url": url},
|
||||
}
|
||||
result = call_res.get("result")
|
||||
text_parts: list[str] = []
|
||||
if isinstance(result, dict):
|
||||
for it in list(result.get("content") or []):
|
||||
if isinstance(it, dict) and str(it.get("type") or "") == "text":
|
||||
txt = str(it.get("text") or "").strip()
|
||||
if txt:
|
||||
text_parts.append(txt)
|
||||
return {
|
||||
"ok": True,
|
||||
"tool_name": tool_name,
|
||||
"arguments": used_args or {"url": url},
|
||||
"text": "\n\n".join(text_parts).strip(),
|
||||
"result": result,
|
||||
}
|
||||
|
||||
|
||||
def bailian_webparser_tool() -> ToolSpec:
|
||||
def _handler(args: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = args if isinstance(args, dict) else {}
|
||||
url = str(payload.get("url") or "").strip()
|
||||
if not url:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "url_required",
|
||||
"error": 'url_required: provide {"url":"https://example.com/article"}',
|
||||
}
|
||||
timeout_s = max(8, min(int(payload.get("timeout") or 35), 90))
|
||||
api_key = str(payload.get("api_key") or os.getenv("DASHSCOPE_API_KEY") or "").strip()
|
||||
if not api_key:
|
||||
return {"ok": False, "error_code": "api_key_missing", "error": "DASHSCOPE_API_KEY required"}
|
||||
try:
|
||||
p = urllib.parse.urlparse(url)
|
||||
except Exception:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "invalid_url",
|
||||
"error": 'invalid_url: provide full http/https URL, e.g. {"url":"https://example.com/article"}',
|
||||
}
|
||||
if p.scheme not in {"http", "https"}:
|
||||
return {
|
||||
"ok": False,
|
||||
"error_code": "unsupported_scheme",
|
||||
"error": "unsupported_scheme: url must start with http:// or https://",
|
||||
}
|
||||
try:
|
||||
out = _call_webparser(url, api_key=api_key, timeout_s=timeout_s)
|
||||
return out
|
||||
except Exception as exc:
|
||||
return {"ok": False, "error_code": "webparser_runtime_failed", "error": f"{type(exc).__name__}: {exc}"}
|
||||
|
||||
return ToolSpec(
|
||||
name="bailian_webparser",
|
||||
description="Parse webpage content via DashScope WebParser (SSE-compatible fallback path). `url` is required.",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "Target webpage URL to parse (required). Example: https://example.com/article",
|
||||
},
|
||||
"timeout": {
|
||||
"type": "integer",
|
||||
"default": 35,
|
||||
"minimum": 8,
|
||||
"maximum": 90,
|
||||
"description": "Total timeout seconds [8..90].",
|
||||
},
|
||||
"api_key": {
|
||||
"type": "string",
|
||||
"description": "Optional DashScope API key override; defaults to DASHSCOPE_API_KEY.",
|
||||
},
|
||||
},
|
||||
"required": ["url"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=_handler,
|
||||
tags=frozenset({"public", "web", "parser", "read"}),
|
||||
risk_level="low",
|
||||
read_only=True,
|
||||
timeout_s=95.0,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["bailian_webparser_tool"]
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue