mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-09 00:40:45 +08:00
oliver
605751b6a6
Prefer report tools without hard-hiding CLI, keep soft CLI budgets, store compact+result_ref for the model, and clear frozen wire after MCP sync. Co-authored-by: Cursor <cursoragent@cursor.com>
1697 lines
70 KiB
Python
1697 lines
70 KiB
Python
from __future__ import annotations
|
||
|
||
"""Agent 工具执行模块。
|
||
|
||
本模块把“工具执行（校验/并发/落库/回写）”从 `Agent.run_turn` 中下沉出来，
|
||
以便被单 Agent 与编排器（manager/specialist）复用。
|
||
"""
|
||
|
||
import json
|
||
import logging
|
||
import time
|
||
import os
|
||
import threading
|
||
from pathlib import Path
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from concurrent.futures import TimeoutError as FuturesTimeoutError
|
||
from dataclasses import dataclass
|
||
from typing import Any, Callable, Optional
|
||
|
||
from svc.persistence.sqlite_store import SqliteStore
|
||
from runtime.tools.base import ToolRegistry
|
||
from svc.llm.chat_models import LLMToolCall
|
||
from runtime.tools.tool_validation import format_invalid_arguments_error, validate_tool_arguments
|
||
from runtime.chat.media_redact import ingest_embedded_image_blobs_as_refs
|
||
from runtime.tools.path_guard import (
|
||
workspace_path_access_scope,
|
||
workspace_write_namespace_scope,
|
||
)
|
||
from runtime.chat.tool_invocation_context import tool_workspace_lane_scope
|
||
|
||
logger = logging.getLogger(__name__)
|
||
_tool_exec_log = logging.getLogger("oclaw.tool_exec")
|
||
|
||
_TOOL_ERROR_MAP = {
|
||
"tool_timeout_or_failed": "tool_timeout_or_failed",
|
||
}
|
||
_SQL_REPLAY_COMPACT_TOOL_NAMES = {
|
||
"query_tabular_attachment",
|
||
"run_tabular_sql",
|
||
"analyze_tabular_attachment_full_scan",
|
||
}
|
||
_TABULAR_QUERY_TOOL_NAMES = {
|
||
"query_tabular_attachment",
|
||
"analyze_tabular_attachment_full_scan",
|
||
}
|
||
_TEXT_QUERY_TOOL_NAMES = {
|
||
"query_text_attachment",
|
||
}
|
||
_IMAGE_QUERY_TOOL_NAMES = {
|
||
"query_image_attachment",
|
||
}
|
||
_VIDEO_QUERY_TOOL_NAMES = {
|
||
"query_video_attachment",
|
||
}
|
||
|
||
|
||
def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]:
|
||
"""Extract renderable attachments from tool results for durable chat history."""
|
||
if not isinstance(result, dict):
|
||
return []
|
||
out: list[dict[str, Any]] = []
|
||
root_deliverable = result.get("deliverable") is True
|
||
|
||
def _with_deliverable(item: dict[str, Any], src: dict[str, Any] | None = None) -> dict[str, Any]:
|
||
if root_deliverable or (isinstance(src, dict) and src.get("deliverable") is True):
|
||
item["deliverable"] = True
|
||
return item
|
||
|
||
aid = str(result.get("attachment_id") or "").strip()
|
||
root_mime = str(result.get("mime") or "").strip()
|
||
if aid:
|
||
ref_type = _ref_type_for_mime(root_mime)
|
||
out.append(
|
||
_with_deliverable(
|
||
{
|
||
"type": ref_type,
|
||
"attachment_id": aid,
|
||
"name": str(result.get("name") or "generated-image"),
|
||
"mime": root_mime or "application/octet-stream",
|
||
"bytes": result.get("bytes"),
|
||
"width": result.get("width"),
|
||
"height": result.get("height"),
|
||
},
|
||
result,
|
||
)
|
||
)
|
||
refs = result.get("attachments")
|
||
if isinstance(refs, list):
|
||
for r in refs:
|
||
if not isinstance(r, dict):
|
||
continue
|
||
p_uri = str(r.get("pointer_uri") or "").strip()
|
||
if p_uri:
|
||
out.append(
|
||
{
|
||
"type": "relay_pointer",
|
||
"pointer_uri": p_uri,
|
||
"rel_path": str(r.get("rel_path") or ""),
|
||
"mime": str(r.get("mime_type") or r.get("mime") or ""),
|
||
"bytes": r.get("bytes"),
|
||
"sha256": str(r.get("sha256") or ""),
|
||
"name": str(r.get("name") or ""),
|
||
}
|
||
)
|
||
continue
|
||
r_aid = str(r.get("attachment_id") or "").strip()
|
||
if r_aid:
|
||
r_typ = str(r.get("type") or "").strip().lower()
|
||
r_mime = str(r.get("mime_type") or r.get("mime") or "").strip()
|
||
if r_typ not in {"image_ref", "video_ref", "text_ref", "binary_ref"}:
|
||
r_typ = _ref_type_for_mime(r_mime)
|
||
out.append(
|
||
_with_deliverable(
|
||
{
|
||
"type": r_typ,
|
||
"attachment_id": r_aid,
|
||
"name": str(r.get("name") or "generated-image"),
|
||
"mime": r_mime or "application/octet-stream",
|
||
"bytes": r.get("bytes"),
|
||
"width": r.get("width"),
|
||
"height": r.get("height"),
|
||
},
|
||
r,
|
||
)
|
||
)
|
||
inner = result.get("result")
|
||
if isinstance(inner, dict):
|
||
content = inner.get("content")
|
||
if isinstance(content, list):
|
||
for item in content:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
typ = str(item.get("type") or "").strip().lower()
|
||
if typ in {"image_ref", "video_ref", "text_ref", "binary_ref"}:
|
||
a_id = str(item.get("attachment_id") or "").strip()
|
||
if a_id:
|
||
out.append(
|
||
_with_deliverable(
|
||
{
|
||
"type": typ,
|
||
"attachment_id": a_id,
|
||
"mime": str(item.get("mime_type") or item.get("mime") or "application/octet-stream"),
|
||
"name": str(item.get("name") or "tool-attachment"),
|
||
"bytes": item.get("bytes"),
|
||
"width": item.get("width"),
|
||
"height": item.get("height"),
|
||
},
|
||
item,
|
||
)
|
||
)
|
||
elif typ == "image_url":
|
||
src = str(item.get("url") or item.get("image_url") or "").strip()
|
||
if src:
|
||
out.append({"type": "image_url", "url": src, "name": str(item.get("name") or "tool-image")})
|
||
uniq: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for a in out:
|
||
k = str(
|
||
a.get("attachment_id")
|
||
or a.get("pointer_uri")
|
||
or a.get("url")
|
||
or ""
|
||
).strip()
|
||
if not k or k in seen:
|
||
continue
|
||
seen.add(k)
|
||
uniq.append(a)
|
||
return uniq
|
||
|
||
|
||
def _message_has_tabular_ref(raw_attachments: Any) -> bool:
|
||
if raw_attachments is None:
|
||
return False
|
||
obj = raw_attachments
|
||
if isinstance(raw_attachments, str):
|
||
s = str(raw_attachments or "").strip()
|
||
if not s:
|
||
return False
|
||
try:
|
||
obj = json.loads(s)
|
||
except Exception:
|
||
return False
|
||
if isinstance(obj, dict):
|
||
items = [obj]
|
||
elif isinstance(obj, list):
|
||
items = obj
|
||
else:
|
||
return False
|
||
for it in items:
|
||
if not isinstance(it, dict):
|
||
continue
|
||
if str(it.get("type") or "").strip().lower() == "tabular_ref":
|
||
return True
|
||
return False
|
||
|
||
|
||
def _message_has_text_ref(raw_attachments: Any) -> bool:
|
||
if raw_attachments is None:
|
||
return False
|
||
obj = raw_attachments
|
||
if isinstance(raw_attachments, str):
|
||
s = str(raw_attachments or "").strip()
|
||
if not s:
|
||
return False
|
||
try:
|
||
obj = json.loads(s)
|
||
except Exception:
|
||
return False
|
||
if isinstance(obj, dict):
|
||
items = [obj]
|
||
elif isinstance(obj, list):
|
||
items = obj
|
||
else:
|
||
return False
|
||
for it in items:
|
||
if not isinstance(it, dict):
|
||
continue
|
||
if str(it.get("type") or "").strip().lower() == "text_ref":
|
||
return True
|
||
return False
|
||
|
||
|
||
def _message_has_image_ref(raw_attachments: Any) -> bool:
|
||
if raw_attachments is None:
|
||
return False
|
||
obj = raw_attachments
|
||
if isinstance(raw_attachments, str):
|
||
s = str(raw_attachments or "").strip()
|
||
if not s:
|
||
return False
|
||
try:
|
||
obj = json.loads(s)
|
||
except Exception:
|
||
return False
|
||
if isinstance(obj, dict):
|
||
items = [obj]
|
||
elif isinstance(obj, list):
|
||
items = obj
|
||
else:
|
||
return False
|
||
for it in items:
|
||
if not isinstance(it, dict):
|
||
continue
|
||
t = str(it.get("type") or "").strip().lower()
|
||
if t in {"image_ref", "image", "input_image"}:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _message_has_video_ref(raw_attachments: Any) -> bool:
|
||
if raw_attachments is None:
|
||
return False
|
||
obj = raw_attachments
|
||
if isinstance(raw_attachments, str):
|
||
s = str(raw_attachments or "").strip()
|
||
if not s:
|
||
return False
|
||
try:
|
||
obj = json.loads(s)
|
||
except Exception:
|
||
return False
|
||
if isinstance(obj, dict):
|
||
items = [obj]
|
||
elif isinstance(obj, list):
|
||
items = obj
|
||
else:
|
||
return False
|
||
for it in items:
|
||
if not isinstance(it, dict):
|
||
continue
|
||
t = str(it.get("type") or "").strip().lower()
|
||
if t == "video_ref":
|
||
return True
|
||
return False
|
||
|
||
|
||
def _session_has_tabular_ref(store: Any, session_id: str, *, limit: int = 300) -> bool:
|
||
try:
|
||
rows = store.get_messages(session_id=session_id, limit=max(1, int(limit)))
|
||
except Exception:
|
||
return False
|
||
for m in rows or []:
|
||
if _message_has_tabular_ref(getattr(m, "attachments", None)):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _session_has_text_ref(store: Any, session_id: str, *, limit: int = 300) -> bool:
|
||
try:
|
||
rows = store.get_messages(session_id=session_id, limit=max(1, int(limit)))
|
||
except Exception:
|
||
return False
|
||
for m in rows or []:
|
||
if _message_has_text_ref(getattr(m, "attachments", None)):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _session_has_image_ref(store: Any, session_id: str, *, limit: int = 300) -> bool:
|
||
try:
|
||
rows = store.get_messages(session_id=session_id, limit=max(1, int(limit)))
|
||
except Exception:
|
||
return False
|
||
for m in rows or []:
|
||
if _message_has_image_ref(getattr(m, "attachments", None)):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _session_has_video_ref(store: Any, session_id: str, *, limit: int = 300) -> bool:
|
||
try:
|
||
rows = store.get_messages(session_id=session_id, limit=max(1, int(limit)))
|
||
except Exception:
|
||
return False
|
||
for m in rows or []:
|
||
if _message_has_video_ref(getattr(m, "attachments", None)):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _parse_event_payload(raw: Any) -> dict[str, Any]:
|
||
if isinstance(raw, dict):
|
||
return dict(raw)
|
||
if isinstance(raw, str) and raw.strip():
|
||
try:
|
||
data = json.loads(raw)
|
||
return data if isinstance(data, dict) else {}
|
||
except Exception:
|
||
return {}
|
||
return {}
|
||
|
||
|
||
def _load_turn_failed_signatures(store: Any, *, session_id: str, turn_uuid: str) -> set[str]:
|
||
"""Signatures that already failed earlier in this turn (blind-retry guard)."""
|
||
tu = str(turn_uuid or "").strip()
|
||
sid = str(session_id or "").strip()
|
||
out: set[str] = set()
|
||
if not tu or not sid:
|
||
return out
|
||
try:
|
||
rows = store.get_messages(session_id=sid, limit=500)
|
||
except Exception:
|
||
return out
|
||
for m in rows or []:
|
||
if str(getattr(m, "role", "") or "").strip().lower() != "tool":
|
||
continue
|
||
if str(getattr(m, "turn_uuid", "") or "").strip() != tu:
|
||
continue
|
||
ep = _parse_event_payload(getattr(m, "event_payload", None))
|
||
sig = str(ep.get("tool_signature") or "").strip()
|
||
if not sig:
|
||
continue
|
||
if ep.get("ok") is False:
|
||
out.add(sig)
|
||
continue
|
||
# Fallback: inspect tool body when older rows lack ok in event_payload.
|
||
try:
|
||
payload = json.loads(str(getattr(m, "content", "") or "") or "{}")
|
||
except Exception:
|
||
payload = {}
|
||
if isinstance(payload, dict) and payload.get("ok") is False:
|
||
out.add(sig)
|
||
return out
|
||
|
||
|
||
def _result_is_retry_forbidden(payload: dict[str, Any] | None) -> bool:
|
||
if not isinstance(payload, dict):
|
||
return False
|
||
if payload.get("retry_forbidden") is True:
|
||
return True
|
||
code = str(payload.get("error_code") or "").strip().lower()
|
||
if code == "insufficient_scope":
|
||
return True
|
||
fc = str(payload.get("failure_class") or "").strip().lower()
|
||
if fc == "auth" and "scope" in f"{payload.get('error') or ''} {code}".lower():
|
||
return True
|
||
return False
|
||
|
||
|
||
def _load_turn_retry_forbidden_tools(store: Any, *, session_id: str, turn_uuid: str) -> set[str]:
|
||
"""Tool names that returned retry_forbidden / insufficient_scope earlier in this turn."""
|
||
tu = str(turn_uuid or "").strip()
|
||
sid = str(session_id or "").strip()
|
||
out: set[str] = set()
|
||
if not tu or not sid:
|
||
return out
|
||
try:
|
||
rows = store.get_messages(session_id=sid, limit=500)
|
||
except Exception:
|
||
return out
|
||
for m in rows or []:
|
||
if str(getattr(m, "role", "") or "").strip().lower() != "tool":
|
||
continue
|
||
if str(getattr(m, "turn_uuid", "") or "").strip() != tu:
|
||
continue
|
||
ep = _parse_event_payload(getattr(m, "event_payload", None))
|
||
name = str(ep.get("tool_name") or "").strip()
|
||
if ep.get("retry_forbidden") is True and name:
|
||
out.add(name)
|
||
continue
|
||
try:
|
||
payload = json.loads(str(getattr(m, "content", "") or "") or "{}")
|
||
except Exception:
|
||
payload = {}
|
||
if isinstance(payload, dict) and _result_is_retry_forbidden(payload):
|
||
raw_tc = getattr(m, "tool_calls", None)
|
||
if isinstance(raw_tc, str):
|
||
try:
|
||
raw_tc = json.loads(raw_tc)
|
||
except Exception:
|
||
raw_tc = None
|
||
if not name and isinstance(raw_tc, dict):
|
||
name = str(raw_tc.get("name") or "").strip()
|
||
if name:
|
||
out.add(name)
|
||
return out
|
||
|
||
|
||
def normalize_tool_result(result: Any) -> dict[str, Any]:
|
||
if isinstance(result, dict):
|
||
out = dict(result)
|
||
if "ok" in out:
|
||
out["ok"] = bool(out.get("ok"))
|
||
else:
|
||
# Backward compatibility: many lightweight tools return payload-only dicts.
|
||
# Treat those as success unless they explicitly carry error semantics.
|
||
has_error = bool(str(out.get("error_code") or "").strip() or str(out.get("error") or "").strip())
|
||
out["ok"] = not has_error
|
||
else:
|
||
out = {"ok": False, "error": "tool_result_not_dict", "data": result}
|
||
if not out["ok"]:
|
||
raw_ec = str(out.get("error_code") or "").strip()
|
||
raw_err = str(out.get("error") or "").strip()
|
||
if not raw_ec:
|
||
out["error_code"] = _TOOL_ERROR_MAP.get(raw_err, "tool_failed")
|
||
from runtime.tools.tool_error_hints import stamp_tool_failure_class
|
||
|
||
out = stamp_tool_failure_class(out)
|
||
return out
|
||
|
||
|
||
def tool_llm_message_max_chars() -> int:
|
||
"""Cap for tool payloads fed back into the *model* mid-context.
|
||
|
||
Default 0 = unlimited for the active turn (large UME dumps stay usable
|
||
until the turn ends). Persistence uses :func:`tool_persist_max_chars`.
|
||
"""
|
||
raw = str(os.getenv("AIA_TOOL_LLM_MESSAGE_MAX_CHARS") or "").strip()
|
||
if raw.isdigit():
|
||
n = int(raw)
|
||
if n == 0:
|
||
return 0
|
||
return max(4096, min(n, 500_000))
|
||
return 0
|
||
|
||
|
||
def tool_persist_max_chars(store: Any | None = None) -> int:
|
||
"""Cap for rewriting ``role=tool`` chat_message rows after a turn finishes.
|
||
|
||
Multi-day field sessions rarely need multi-MB tool JSON in history; default
|
||
24_000 keeps SQLite growth down. Set ``AIA_TOOL_PERSIST_MAX_CHARS=0`` (or
|
||
the same DB setting) to disable post-turn compaction.
|
||
"""
|
||
raw = ""
|
||
if store is not None:
|
||
try:
|
||
raw = str(store.get_setting("AIA_TOOL_PERSIST_MAX_CHARS") or "").strip()
|
||
except Exception:
|
||
raw = ""
|
||
if not raw:
|
||
raw = str(os.getenv("AIA_TOOL_PERSIST_MAX_CHARS") or "").strip()
|
||
# Fall back to the older LLM-message setting when operators already tuned it.
|
||
if not raw and store is not None:
|
||
try:
|
||
raw = str(store.get_setting("AIA_TOOL_LLM_MESSAGE_MAX_CHARS") or "").strip()
|
||
except Exception:
|
||
raw = ""
|
||
if not raw:
|
||
raw = str(os.getenv("AIA_TOOL_LLM_MESSAGE_MAX_CHARS") or "").strip()
|
||
if raw.isdigit():
|
||
n = int(raw)
|
||
if n == 0:
|
||
return 0
|
||
return max(4096, min(n, 500_000))
|
||
return 24_000
|
||
|
||
|
||
def tool_history_summary_after_calls() -> int:
|
||
raw = str(os.getenv("AIA_TOOL_HISTORY_SUMMARY_AFTER_CALLS") or "").strip()
|
||
if raw.isdigit():
|
||
return max(0, min(int(raw), 200))
|
||
# Default: when the same tool is called >= 3 times in one turn, keep history compact.
|
||
return 3
|
||
|
||
|
||
def _json_blob_size(obj: Any) -> int:
|
||
try:
|
||
return len(json.dumps(obj, ensure_ascii=False, default=str))
|
||
except Exception:
|
||
return len(repr(obj))
|
||
|
||
|
||
def _estimate_observed_rows(result: dict[str, Any]) -> int:
|
||
if not isinstance(result, dict):
|
||
return 0
|
||
try:
|
||
rr = result.get("rows_returned")
|
||
if isinstance(rr, (int, float)):
|
||
return max(0, int(rr))
|
||
except Exception:
|
||
pass
|
||
rows = result.get("rows")
|
||
if isinstance(rows, list):
|
||
return max(0, len(rows))
|
||
nested = result.get("result")
|
||
if isinstance(nested, dict):
|
||
nrows = nested.get("rows")
|
||
if isinstance(nrows, list):
|
||
return max(0, len(nrows))
|
||
return 0
|
||
|
||
|
||
def _deep_truncate_for_llm(obj: Any, *, max_str: int, max_list: int) -> Any:
|
||
if isinstance(obj, dict):
|
||
return {str(k): _deep_truncate_for_llm(v, max_str=max_str, max_list=max_list) for k, v in obj.items()}
|
||
if isinstance(obj, list):
|
||
items = obj
|
||
omitted = 0
|
||
if len(items) > max_list:
|
||
omitted = len(items) - max_list
|
||
items = items[:max_list]
|
||
out: list[Any] = [_deep_truncate_for_llm(x, max_str=max_str, max_list=max_list) for x in items]
|
||
if omitted:
|
||
out.append(f"…({omitted} more list items omitted)")
|
||
return out
|
||
if isinstance(obj, str) and len(obj) > max_str:
|
||
return obj[:max_str] + "\n...<truncated>"
|
||
return obj
|
||
|
||
|
||
def partition_tool_use_batches(
|
||
tool_uses: list[LLMToolCall],
|
||
registry: ToolRegistry,
|
||
) -> list[list[LLMToolCall]]:
|
||
"""Split tool uses into ordered batches (cc-mini ``Engine.submit`` scheduling).
|
||
|
||
Consecutive tools whose ``ToolSpec.is_read_only()`` is true are merged into one batch
|
||
and may run in parallel when the batch length is greater than one. Any other tool
|
||
starts a new batch (typically length 1), which runs sequentially relative to other
|
||
batches and uses a single worker within the batch.
|
||
"""
|
||
batches: list[tuple[bool, list[LLMToolCall]]] = []
|
||
for tc in tool_uses:
|
||
spec = registry.get(tc.name)
|
||
is_concurrent = bool(spec and spec.is_read_only())
|
||
if batches and batches[-1][0] == is_concurrent and is_concurrent:
|
||
batches[-1][1].append(tc)
|
||
else:
|
||
batches.append((is_concurrent, [tc]))
|
||
return [chunk for _, chunk in batches]
|
||
|
||
|
||
def truncate_tool_result_for_llm_messages(result: dict[str, Any], *, max_chars: int | None = None) -> dict[str, Any]:
|
||
"""Return a copy safe to put in ``role=tool`` ``content`` so the next LLM request stays under provider limits."""
|
||
cap = tool_llm_message_max_chars() if max_chars is None else max(0, min(int(max_chars), 500_000))
|
||
if cap == 0:
|
||
return result if isinstance(result, dict) else {"ok": False, "error": "tool_result_not_dict", "data": result}
|
||
if not isinstance(result, dict):
|
||
return {"ok": False, "error": "tool_result_not_dict", "payload_type": type(result).__name__}
|
||
if _json_blob_size(result) <= cap:
|
||
return result
|
||
orig_files_n = len(result["files"]) if isinstance(result.get("files"), list) else 0
|
||
pairs = (
|
||
(12_000, 800),
|
||
(8000, 500),
|
||
(4000, 300),
|
||
(2000, 200),
|
||
(1200, 120),
|
||
(800, 80),
|
||
(500, 50),
|
||
(400, 40),
|
||
)
|
||
for max_str, max_list in pairs:
|
||
slim = _deep_truncate_for_llm(result, max_str=max_str, max_list=max_list)
|
||
if not isinstance(slim, dict):
|
||
slim = {"ok": bool(result.get("ok")), "payload": slim}
|
||
if _json_blob_size(slim) <= cap:
|
||
slim = dict(slim)
|
||
slim["_truncated_for_llm"] = True
|
||
if orig_files_n and isinstance(slim.get("files"), list):
|
||
kept = sum(1 for x in slim["files"] if isinstance(x, str))
|
||
if kept < orig_files_n:
|
||
slim["files_total"] = orig_files_n
|
||
slim["files_omitted"] = orig_files_n - kept
|
||
return slim
|
||
return {
|
||
"ok": bool(result.get("ok")),
|
||
"_truncated_for_llm": True,
|
||
"hint": (
|
||
"Tool output exceeded model message size limits. "
|
||
"Narrow the glob, lower max_results, or list a subdirectory. / "
|
||
"工具输出超过模型单条消息限制，请缩小列举范围或降低 max_results。"
|
||
),
|
||
}
|
||
|
||
|
||
_INTERNAL_TOOL_SPECIALISTS = frozenset({"", "oclaw"})
|
||
|
||
|
||
def _resolve_creator_specialist(ctx: ToolExecutionContext) -> str:
|
||
"""Specialist actively running this turn (for schedule_create inheritance)."""
|
||
from runtime.types import normalize_requested_specialist
|
||
|
||
md = ctx.inbound_metadata if isinstance(ctx.inbound_metadata, dict) else {}
|
||
for raw in (
|
||
str(ctx.workspace_lane_role or "").strip(),
|
||
str(ctx.specialist or "").strip(),
|
||
str(md.get("selected_specialist") or "").strip(),
|
||
str(md.get("manager_selected_specialist") or "").strip(),
|
||
):
|
||
if not raw or raw.lower() in _INTERNAL_TOOL_SPECIALISTS:
|
||
continue
|
||
return normalize_requested_specialist(raw)
|
||
for raw in (
|
||
str(ctx.workspace_lane_role or "").strip(),
|
||
str(md.get("selected_specialist") or "").strip(),
|
||
str(md.get("manager_selected_specialist") or "").strip(),
|
||
):
|
||
if raw:
|
||
return normalize_requested_specialist(raw)
|
||
return "generalist"
|
||
|
||
|
||
def _truthy_flag(raw: Any) -> bool | None:
|
||
"""Return True/False when the flag is explicit; None when unset."""
|
||
if raw is None:
|
||
return None
|
||
if isinstance(raw, bool):
|
||
return raw
|
||
text = str(raw).strip().lower()
|
||
if not text:
|
||
return None
|
||
if text in {"1", "true", "yes", "on"}:
|
||
return True
|
||
if text in {"0", "false", "no", "off"}:
|
||
return False
|
||
return None
|
||
|
||
|
||
def _maybe_auto_mark_xlsx_deliverable(
|
||
*,
|
||
tool_name: str,
|
||
tool_args: dict[str, Any],
|
||
result: dict[str, Any],
|
||
inbound_metadata: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
"""On WhatsApp/WeChat, auto-mark successful write_xlsx as deliverable unless explicitly false."""
|
||
if not isinstance(result, dict) or result.get("ok") is False:
|
||
return result
|
||
if str(tool_name or "").strip() != "write_xlsx":
|
||
return result
|
||
channel = str(inbound_metadata.get("channel") or "").strip().lower()
|
||
if channel not in {"whatsapp", "wechat", "wecom"}:
|
||
return result
|
||
explicit = _truthy_flag(tool_args.get("deliverable")) if isinstance(tool_args, dict) else None
|
||
if explicit is False:
|
||
return result
|
||
if result.get("deliverable") is True:
|
||
return result
|
||
out = dict(result)
|
||
out["deliverable"] = True
|
||
out["auto_deliverable"] = True
|
||
out["hint"] = (
|
||
"Excel auto-marked deliverable for channel outbound — file will attach on WhatsApp/WeChat."
|
||
)
|
||
return out
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ToolExecutionConfig:
|
||
max_workers: int = 8
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ToolExecutionContext:
|
||
store: SqliteStore
|
||
tools: ToolRegistry
|
||
session_id: str
|
||
lang: str = "zh"
|
||
user_text: str = ""
|
||
inbound_metadata: dict[str, Any] | None = None
|
||
specialist: str = ""
|
||
task_kind: str = ""
|
||
policy_engine: Any | None = None
|
||
trace_id: str | None = None
|
||
parent_span_id: str | None = None
|
||
#: When ``session_id`` is a specialist temp chat row (no ``ui_session_owner``), use the user's UI session for ``extra_roots`` / allowlist.
|
||
workspace_owner_session_id: str | None = None
|
||
#: If ``get_ui_session_owner`` fails, load allowlist for this (tenant, user) from the HTTP/gateway request (``metadata``).
|
||
path_policy_tenant_id: str | None = None
|
||
path_policy_user_id: str | None = None
|
||
workspace_dir: str | None = None
|
||
turn_uuid: str | None = None
|
||
#: Binding role for private ``skill_auto_install`` paths (``_workspace/<role>/``, sibling of ``public/``).
|
||
workspace_lane_role: str | None = None
|
||
|
||
|
||
class ToolExecutor:
|
||
"""执行一组 tool uses，并把结果写回 store。"""
|
||
|
||
def __init__(self, *, config: ToolExecutionConfig | None = None):
|
||
self.config = config or ToolExecutionConfig()
|
||
|
||
def _execute_tool(self, ctx: ToolExecutionContext, tc: LLMToolCall) -> tuple[dict[str, Any], int]:
|
||
t0 = time.perf_counter()
|
||
|
||
tool = ctx.tools.get(tc.name)
|
||
if not tool:
|
||
from runtime.tools.tool_error_hints import format_unregistered_tool_error
|
||
|
||
available = [t.name for t in ctx.tools.list()] if hasattr(ctx.tools, "list") else []
|
||
return format_unregistered_tool_error(
|
||
str(tc.name or ""),
|
||
available,
|
||
lang=str(ctx.lang or "zh"),
|
||
), int((time.perf_counter() - t0) * 1000)
|
||
|
||
from runtime.tools.context_inject import enrich_tool_arguments
|
||
from runtime.tools.tool_validation import filter_arguments_to_schema
|
||
|
||
raw_args = tc.arguments if isinstance(tc.arguments, dict) else {}
|
||
tool_args = enrich_tool_arguments(
|
||
store=ctx.store,
|
||
session_id=ctx.session_id,
|
||
tool_name=str(tc.name or ""),
|
||
arguments=raw_args,
|
||
path_policy_tenant_id=ctx.path_policy_tenant_id,
|
||
path_policy_user_id=ctx.path_policy_user_id,
|
||
)
|
||
# Default specialist context: if the model did not explicitly provide a specialist for
|
||
# schedule tools, inherit from the current executor specialist.
|
||
#
|
||
# This makes scheduled jobs created by ops/generalist reflect the creator specialist by default,
|
||
# which is important for "created-by specialist" semantics and for later scheduled runs.
|
||
try:
|
||
tname = str(tc.name or "")
|
||
if tname in {"schedule_create", "schedule_delete"}:
|
||
md = ctx.inbound_metadata if isinstance(ctx.inbound_metadata, dict) else {}
|
||
raw_block = md.get("raw") if isinstance(md.get("raw"), dict) else {}
|
||
push_name = str(
|
||
md.get("push_name") or md.get("pushName") or raw_block.get("pushName") or ""
|
||
).strip()
|
||
external_uid = str(
|
||
md.get("external_user_id")
|
||
or md.get("sender_jid")
|
||
or raw_block.get("participant")
|
||
or raw_block.get("sender")
|
||
or ""
|
||
).strip()
|
||
if tname == "schedule_create":
|
||
cur_spec = _resolve_creator_specialist(ctx)
|
||
if cur_spec and not str(tool_args.get("specialist") or "").strip() and not str(
|
||
tool_args.get("selected_specialist") or ""
|
||
).strip():
|
||
tool_args["selected_specialist"] = cur_spec
|
||
if external_uid and not str(tool_args.get("creator_external_user_id") or "").strip():
|
||
tool_args["creator_external_user_id"] = external_uid
|
||
if push_name and not str(tool_args.get("creator_push_name") or "").strip():
|
||
tool_args["creator_push_name"] = push_name
|
||
bot_jid_raw = str(md.get("bot_jid") or raw_block.get("botJid") or "").strip()
|
||
bot_lid_raw = str(md.get("bot_lid") or md.get("botLid") or raw_block.get("botLid") or "").strip()
|
||
if bot_jid_raw and not str(tool_args.get("whatsapp_bot_jid") or "").strip():
|
||
tool_args["whatsapp_bot_jid"] = bot_jid_raw
|
||
if bot_lid_raw and not str(tool_args.get("whatsapp_bot_lid") or "").strip():
|
||
tool_args["whatsapp_bot_lid"] = bot_lid_raw
|
||
raw_mentions = md.get("mentioned_jids") or md.get("mentionedJids") or md.get("mentions") or []
|
||
mention_list = raw_mentions if isinstance(raw_mentions, list) else []
|
||
bot_jid = str(md.get("bot_jid") or "").strip().lower()
|
||
bot_lid = str(md.get("bot_lid") or "").strip().lower()
|
||
cleaned: list[str] = []
|
||
seen: set[str] = set()
|
||
for m in mention_list:
|
||
jid = str(m or "").strip()
|
||
if not jid:
|
||
continue
|
||
low = jid.lower()
|
||
if (bot_jid and low == bot_jid) or (bot_lid and low == bot_lid):
|
||
continue
|
||
if low in seen:
|
||
continue
|
||
seen.add(low)
|
||
cleaned.append(jid)
|
||
if cleaned:
|
||
tool_args["whatsapp_mention_jids"] = cleaned
|
||
elif tool_args.get("whatsapp_mention_jids") is not None:
|
||
from runtime.scheduler.whatsapp_mentions import normalize_whatsapp_mention_jids
|
||
|
||
tool_args["whatsapp_mention_jids"] = normalize_whatsapp_mention_jids(
|
||
tool_args.get("whatsapp_mention_jids")
|
||
)
|
||
if tool_args.get("whatsapp_mention_names") is None:
|
||
from runtime.scheduler.whatsapp_mentions import extract_whatsapp_mention_names
|
||
|
||
user_text = str(ctx.user_text or "")
|
||
names = extract_whatsapp_mention_names(user_text)
|
||
bot_names = {
|
||
str(x or "").strip().lower()
|
||
for x in (
|
||
md.get("bot_push_name"),
|
||
"oliver",
|
||
)
|
||
if str(x or "").strip()
|
||
}
|
||
filtered_names = [n for n in names if n.lower() not in bot_names]
|
||
if filtered_names:
|
||
tool_args["whatsapp_mention_names"] = filtered_names
|
||
elif md.get("mention_names"):
|
||
names_md = md.get("mention_names")
|
||
if isinstance(names_md, list):
|
||
tool_args["whatsapp_mention_names"] = [
|
||
str(x or "").strip() for x in names_md if str(x or "").strip()
|
||
]
|
||
else:
|
||
if external_uid and not str(tool_args.get("actor_external_user_id") or "").strip():
|
||
tool_args["actor_external_user_id"] = external_uid
|
||
except Exception:
|
||
pass
|
||
tool_args = filter_arguments_to_schema(tool.parameters, tool_args)
|
||
|
||
ok, v_err = validate_tool_arguments(tool.parameters, tool_args)
|
||
if not ok:
|
||
intent = None
|
||
try:
|
||
md = ctx.inbound_metadata if isinstance(ctx.inbound_metadata, dict) else {}
|
||
intent = str(md.get("ops_short_intent") or "").strip() or None
|
||
except Exception:
|
||
intent = None
|
||
return format_invalid_arguments_error(
|
||
tool.parameters or {},
|
||
str(v_err or "invalid"),
|
||
lang=str(ctx.lang or "zh"),
|
||
tool_name=str(tc.name or tool.name or ""),
|
||
intent=intent,
|
||
), int((time.perf_counter() - t0) * 1000)
|
||
|
||
try:
|
||
timeout_s = getattr(tool, "timeout_s", None)
|
||
# Default timeout for plugin tools if not specified.
|
||
if timeout_s is None and "plugin" in getattr(tool, "tags", frozenset()):
|
||
timeout_s = 30.0
|
||
|
||
def _call() -> Any:
|
||
ws_ns = ""
|
||
raw_ws = str(ctx.workspace_dir or "").strip()
|
||
if raw_ws:
|
||
try:
|
||
wp = Path(raw_ws)
|
||
ws_ns = str(wp.name or wp.stem or "").strip()
|
||
except Exception:
|
||
ws_ns = ""
|
||
with workspace_path_access_scope(
|
||
ctx.store,
|
||
ctx.session_id,
|
||
owner_fallback_session_id=ctx.workspace_owner_session_id,
|
||
allowlist_tenant_id=ctx.path_policy_tenant_id,
|
||
allowlist_user_id=ctx.path_policy_user_id,
|
||
), workspace_write_namespace_scope(ws_ns), tool_workspace_lane_scope(
|
||
workspace_owner_session_id=ctx.workspace_owner_session_id,
|
||
session_id=ctx.session_id,
|
||
workspace_lane_role=ctx.workspace_lane_role,
|
||
):
|
||
netx_lang_token = None
|
||
if str(tc.name or "").startswith("netx_"):
|
||
from runtime.tools.experts.network_ops.netx_tools import NETX_TOOL_LANG
|
||
|
||
netx_lang_token = NETX_TOOL_LANG.set(str(ctx.lang or "zh"))
|
||
try:
|
||
return tool.handler(tool_args)
|
||
finally:
|
||
if netx_lang_token is not None:
|
||
from runtime.tools.experts.network_ops.netx_tools import NETX_TOOL_LANG
|
||
|
||
NETX_TOOL_LANG.reset(netx_lang_token)
|
||
|
||
if isinstance(timeout_s, (int, float)) and float(timeout_s) > 0:
|
||
ex = ThreadPoolExecutor(max_workers=1)
|
||
fut = ex.submit(_call)
|
||
try:
|
||
result = fut.result(timeout=float(timeout_s))
|
||
except FuturesTimeoutError as e:
|
||
try:
|
||
fut.cancel()
|
||
except Exception:
|
||
pass
|
||
try:
|
||
ex.shutdown(wait=False, cancel_futures=True)
|
||
except Exception:
|
||
ex.shutdown(wait=False)
|
||
return {
|
||
"ok": False,
|
||
"error_code": "tool_timeout_or_failed",
|
||
"error": "tool_timeout_or_failed",
|
||
"detail": f"{type(e).__name__}: {e}",
|
||
"timeout_s": float(timeout_s),
|
||
"tool": str(tc.name or ""),
|
||
"hint": (
|
||
"Tool wall-clock timeout. For mcp__netx__execManagedNe raise read_timeout_sec "
|
||
"and/or reduce commands; do not blind-retry the same call."
|
||
if str(ctx.lang or "").startswith("en")
|
||
else "工具墙钟超时。对 mcp__netx__execManagedNe 请提高 read_timeout_sec 或减少命令条数，禁止相同参数盲重试。"
|
||
),
|
||
}, int((time.perf_counter() - t0) * 1000)
|
||
except Exception as e:
|
||
try:
|
||
ex.shutdown(wait=False, cancel_futures=True)
|
||
except Exception:
|
||
ex.shutdown(wait=False)
|
||
return {
|
||
"ok": False,
|
||
"error_code": "tool_timeout_or_failed",
|
||
"error": "tool_timeout_or_failed",
|
||
"detail": f"{type(e).__name__}: {e}",
|
||
"tool": str(tc.name or ""),
|
||
}, int((time.perf_counter() - t0) * 1000)
|
||
else:
|
||
try:
|
||
ex.shutdown(wait=False, cancel_futures=True)
|
||
except Exception:
|
||
ex.shutdown(wait=False)
|
||
else:
|
||
result = _call()
|
||
out = normalize_tool_result(result)
|
||
out = _maybe_auto_mark_xlsx_deliverable(
|
||
tool_name=str(tc.name or ""),
|
||
tool_args=tool_args if isinstance(tool_args, dict) else {},
|
||
result=out,
|
||
inbound_metadata=ctx.inbound_metadata if isinstance(ctx.inbound_metadata, dict) else {},
|
||
)
|
||
dur_ms = int((time.perf_counter() - t0) * 1000)
|
||
try:
|
||
_tool_exec_log.info(
|
||
"tool_exec name=%s ok=%s dur_ms=%s session=%s specialist=%s tool_call_id=%s error_code=%s",
|
||
str(tc.name or ""),
|
||
str(bool(out.get("ok"))),
|
||
str(dur_ms),
|
||
str(ctx.session_id or ""),
|
||
str(ctx.specialist or ""),
|
||
str(getattr(tc, "id", "") or ""),
|
||
str(out.get("error_code") or ""),
|
||
)
|
||
except Exception:
|
||
pass
|
||
return out, dur_ms
|
||
except Exception as e:
|
||
if ctx.lang.startswith("en"):
|
||
err = {"ok": False, "error_code": "tool_execution_error", "error": f"Tool execution error: {type(e).__name__}: {e}"}
|
||
else:
|
||
err = {"ok": False, "error_code": "tool_execution_error", "error": f"工具执行异常: {type(e).__name__}: {e}"}
|
||
out = normalize_tool_result(err)
|
||
dur_ms = int((time.perf_counter() - t0) * 1000)
|
||
try:
|
||
_tool_exec_log.info(
|
||
"tool_exec name=%s ok=%s dur_ms=%s session=%s specialist=%s tool_call_id=%s error_code=%s",
|
||
str(tc.name or ""),
|
||
str(bool(out.get("ok"))),
|
||
str(dur_ms),
|
||
str(ctx.session_id or ""),
|
||
str(ctx.specialist or ""),
|
||
str(getattr(tc, "id", "") or ""),
|
||
str(out.get("error_code") or ""),
|
||
)
|
||
except Exception:
|
||
pass
|
||
return out, dur_ms
|
||
|
||
@staticmethod
|
||
def _json_dumps_safe(obj: Any) -> str:
|
||
try:
|
||
return json.dumps(obj, ensure_ascii=False, default=str)
|
||
except (TypeError, ValueError):
|
||
return json.dumps({"ok": False, "error": "tool result is not JSON-serializable"}, ensure_ascii=False)
|
||
|
||
def execute_tool_uses(
|
||
self,
|
||
*,
|
||
ctx: ToolExecutionContext,
|
||
assistant_msg_id: int,
|
||
tool_uses: list[LLMToolCall],
|
||
on_tool_ui: Optional[Callable[[str, dict[str, Any]], None]] = None,
|
||
should_stop: Optional[Callable[[], bool]] = None,
|
||
signature_budget: int = 2,
|
||
) -> tuple[list[dict[str, Any]], dict[str, tuple[dict[str, Any], int]]]:
|
||
"""执行并回写 tool messages。
|
||
|
||
Returns:
|
||
- tool_messages: 用于写入对话 history 的 `role=tool` 消息 payload 列表（与 tool_uses 顺序一致）
|
||
- results_by_id: tool_call_id -> (result_dict, duration_ms)
|
||
"""
|
||
|
||
def _check_stop() -> None:
|
||
if should_stop and should_stop():
|
||
raise RuntimeError("generation interrupted by user")
|
||
|
||
def _trace(event_type: str, payload: dict[str, Any]) -> None:
|
||
if not ctx.trace_id:
|
||
return
|
||
try:
|
||
from runtime.orchestration.trace import new_span_id
|
||
|
||
ctx.store.add_trace_event(
|
||
session_id=ctx.session_id,
|
||
trace_id=str(ctx.trace_id),
|
||
span_id=new_span_id(),
|
||
parent_span_id=ctx.parent_span_id,
|
||
event_type=str(event_type),
|
||
payload=dict(payload or {}),
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
def _load_turn_tool_stats() -> tuple[dict[str, int], dict[str, int]]:
|
||
counts: dict[str, int] = {}
|
||
observed_rows: dict[str, int] = {}
|
||
if not str(ctx.turn_uuid or "").strip():
|
||
return counts, observed_rows
|
||
try:
|
||
rows = ctx.store.get_messages(session_id=ctx.session_id, limit=500)
|
||
except Exception:
|
||
return counts, observed_rows
|
||
for m in rows or []:
|
||
if str(getattr(m, "role", "") or "") != "tool":
|
||
continue
|
||
if str(getattr(m, "turn_uuid", "") or "") != str(ctx.turn_uuid or ""):
|
||
continue
|
||
raw_tc = getattr(m, "tool_calls", None)
|
||
name = ""
|
||
if isinstance(raw_tc, str):
|
||
try:
|
||
parsed = json.loads(raw_tc)
|
||
except Exception:
|
||
parsed = None
|
||
else:
|
||
parsed = raw_tc
|
||
if isinstance(parsed, dict):
|
||
name = str(parsed.get("name") or "").strip()
|
||
if not name:
|
||
continue
|
||
counts[name] = int(counts.get(name, 0)) + 1
|
||
try:
|
||
raw_content = str(getattr(m, "content", "") or "")
|
||
payload = json.loads(raw_content) if raw_content else {}
|
||
except Exception:
|
||
payload = {}
|
||
if isinstance(payload, dict):
|
||
observed_rows[name] = int(observed_rows.get(name, 0)) + int(
|
||
_estimate_observed_rows(payload)
|
||
or payload.get("_tool_observed_rows_this_call")
|
||
or 0
|
||
)
|
||
return counts, observed_rows
|
||
|
||
_check_stop()
|
||
if not tool_uses:
|
||
return [], {}
|
||
turn_tool_name_counts, turn_tool_observed_rows = _load_turn_tool_stats()
|
||
local_turn_tool_name_counts: dict[str, int] = {}
|
||
local_turn_tool_observed_rows: dict[str, int] = {}
|
||
planned_sql_name_counts: dict[str, int] = {}
|
||
for tc in tool_uses:
|
||
name = str(tc.name or "")
|
||
if name in _SQL_REPLAY_COMPACT_TOOL_NAMES:
|
||
planned_sql_name_counts[name] = int(planned_sql_name_counts.get(name, 0)) + 1
|
||
compact_sql_names = {
|
||
name for name, cnt in planned_sql_name_counts.items() if int(cnt) >= int(tool_history_summary_after_calls())
|
||
}
|
||
has_tabular_ref_in_session = _session_has_tabular_ref(ctx.store, ctx.session_id)
|
||
has_text_ref_in_session = _session_has_text_ref(ctx.store, ctx.session_id)
|
||
has_image_ref_in_session = _session_has_image_ref(ctx.store, ctx.session_id)
|
||
has_video_ref_in_session = _session_has_video_ref(ctx.store, ctx.session_id)
|
||
failed_signatures = _load_turn_failed_signatures(
|
||
ctx.store,
|
||
session_id=ctx.session_id,
|
||
turn_uuid=str(ctx.turn_uuid or ""),
|
||
)
|
||
retry_forbidden_tools = _load_turn_retry_forbidden_tools(
|
||
ctx.store,
|
||
session_id=ctx.session_id,
|
||
turn_uuid=str(ctx.turn_uuid or ""),
|
||
)
|
||
from runtime.chat.exec_managed_ne_guard import (
|
||
budget_block_payload,
|
||
exec_managed_ne_fail_budget,
|
||
exec_managed_ne_single_budget,
|
||
is_batch_exec_args,
|
||
is_exec_managed_ne_tool,
|
||
load_turn_exec_managed_ne_stats,
|
||
)
|
||
from runtime.application.gateway.ops_short_intent import ops_short_intent_cli_soft_budgets
|
||
|
||
prior_single_exec, prior_batch_exec, prior_exec_fails = load_turn_exec_managed_ne_stats(
|
||
ctx.store,
|
||
session_id=ctx.session_id,
|
||
turn_uuid=str(ctx.turn_uuid or ""),
|
||
)
|
||
single_exec_budget = exec_managed_ne_single_budget()
|
||
fail_exec_budget = exec_managed_ne_fail_budget()
|
||
batch_exec_budget = 0 # 0 = unlimited (global default)
|
||
try:
|
||
md_budget = ctx.inbound_metadata if isinstance(ctx.inbound_metadata, dict) else {}
|
||
soft = ops_short_intent_cli_soft_budgets(str(md_budget.get("ops_short_intent") or "").strip() or None)
|
||
if soft:
|
||
single_exec_budget = int(soft.get("single") or single_exec_budget)
|
||
fail_exec_budget = int(soft.get("fail") or fail_exec_budget)
|
||
batch_exec_budget = int(soft.get("batch") or 0)
|
||
except Exception:
|
||
pass
|
||
local_single_exec = 0
|
||
local_batch_exec = 0
|
||
local_exec_fails = 0
|
||
|
||
results_by_id: dict[str, tuple[dict[str, Any], int]] = {}
|
||
runnable_tool_uses: list[LLMToolCall] = []
|
||
dedupe_alias_to_source: dict[str, str] = {}
|
||
first_tool_call_id_by_signature: dict[str, str] = {}
|
||
sig_seen: dict[str, int] = {}
|
||
budget = max(1, min(int(signature_budget or 2), 8))
|
||
for tc in tool_uses:
|
||
tool_name = str(tc.name or "")
|
||
if tool_name in retry_forbidden_tools:
|
||
results_by_id[tc.id] = (
|
||
{
|
||
"ok": False,
|
||
"error_code": "retry_forbidden_blocked",
|
||
"failure_class": "retry_guard",
|
||
"retry_forbidden": True,
|
||
"error": f"tool blocked for remainder of turn after non-retryable failure: {tc.name}",
|
||
"hint": (
|
||
"This tool already failed with a non-retryable error (e.g. insufficient_scope). "
|
||
"Switch to fallback tools (aggregateUmeAlarms / queryUmeAlarmsRaw / "
|
||
"ume_alarm_xlsx_report) or ask an admin — do not call it again this turn."
|
||
),
|
||
},
|
||
0,
|
||
)
|
||
_trace(
|
||
"retry_forbidden_blocked",
|
||
{"tool_name": tc.name},
|
||
)
|
||
continue
|
||
if tc.name in _TABULAR_QUERY_TOOL_NAMES and not has_tabular_ref_in_session:
|
||
results_by_id[tc.id] = (
|
||
{
|
||
"ok": False,
|
||
"error_code": "tabular_ref_missing",
|
||
"error": "tabular_ref_missing",
|
||
"hint": "No tabular_ref attachment found in this session. Query tools require table_id from tabular_ref.",
|
||
},
|
||
0,
|
||
)
|
||
_trace(
|
||
"tabular_query_guard",
|
||
{
|
||
"tool_name": tc.name,
|
||
"blocked": True,
|
||
"reason": "tabular_ref_missing",
|
||
},
|
||
)
|
||
continue
|
||
if tc.name in _TEXT_QUERY_TOOL_NAMES and not has_text_ref_in_session:
|
||
results_by_id[tc.id] = (
|
||
{
|
||
"ok": False,
|
||
"error_code": "text_ref_missing",
|
||
"error": "text_ref_missing",
|
||
"hint": "No text_ref attachment found in this session. Query tools require text_id from text_ref.",
|
||
},
|
||
0,
|
||
)
|
||
_trace(
|
||
"text_query_guard",
|
||
{
|
||
"tool_name": tc.name,
|
||
"blocked": True,
|
||
"reason": "text_ref_missing",
|
||
},
|
||
)
|
||
continue
|
||
if tc.name in _IMAGE_QUERY_TOOL_NAMES and not has_image_ref_in_session:
|
||
results_by_id[tc.id] = (
|
||
{
|
||
"ok": False,
|
||
"error_code": "image_ref_missing",
|
||
"error": "image_ref_missing",
|
||
"hint": "No image_ref attachment found in this session. Query tools require attachment_id from image_ref.",
|
||
},
|
||
0,
|
||
)
|
||
_trace(
|
||
"image_query_guard",
|
||
{
|
||
"tool_name": tc.name,
|
||
"blocked": True,
|
||
"reason": "image_ref_missing",
|
||
},
|
||
)
|
||
continue
|
||
if tc.name in _VIDEO_QUERY_TOOL_NAMES and not has_video_ref_in_session:
|
||
results_by_id[tc.id] = (
|
||
{
|
||
"ok": False,
|
||
"error_code": "video_ref_missing",
|
||
"error": "video_ref_missing",
|
||
"hint": "No video_ref attachment found in this session. Query tools require attachment_id from video_ref.",
|
||
},
|
||
0,
|
||
)
|
||
_trace(
|
||
"video_query_guard",
|
||
{
|
||
"tool_name": tc.name,
|
||
"blocked": True,
|
||
"reason": "video_ref_missing",
|
||
},
|
||
)
|
||
continue
|
||
sig = f"{tc.name}:{self._json_dumps_safe(dict(tc.arguments or {}))}"
|
||
if sig in failed_signatures:
|
||
results_by_id[tc.id] = (
|
||
{
|
||
"ok": False,
|
||
"error_code": "identical_retry_blocked",
|
||
"failure_class": "retry_guard",
|
||
"error": f"identical failed call already ran this turn: {tc.name}",
|
||
"hint": (
|
||
"Same tool+arguments already failed earlier in this turn. "
|
||
"Change arguments, raise timeouts, or switch tools — do not blind-retry."
|
||
),
|
||
},
|
||
0,
|
||
)
|
||
_trace(
|
||
"identical_retry_blocked",
|
||
{
|
||
"tool_name": tc.name,
|
||
"signature": sig[:300],
|
||
},
|
||
)
|
||
continue
|
||
if is_exec_managed_ne_tool(tool_name):
|
||
batchish = is_batch_exec_args(dict(tc.arguments or {}))
|
||
single_used = int(prior_single_exec) + int(local_single_exec)
|
||
batch_used = int(prior_batch_exec) + int(local_batch_exec)
|
||
fail_used = int(prior_exec_fails) + int(local_exec_fails)
|
||
if batchish and int(batch_exec_budget) > 0 and batch_used >= int(batch_exec_budget):
|
||
results_by_id[tc.id] = (
|
||
budget_block_payload(
|
||
reason="batch_budget",
|
||
lang=str(ctx.lang or "en"),
|
||
single_used=single_used,
|
||
single_budget=single_exec_budget,
|
||
fail_used=fail_used,
|
||
fail_budget=fail_exec_budget,
|
||
batch_used=batch_used,
|
||
batch_budget=batch_exec_budget,
|
||
),
|
||
0,
|
||
)
|
||
_trace(
|
||
"cli_batch_budget_exceeded",
|
||
{
|
||
"tool_name": tc.name,
|
||
"batch_used": batch_used,
|
||
"batch_budget": batch_exec_budget,
|
||
},
|
||
)
|
||
continue
|
||
if (not batchish) and fail_used >= int(fail_exec_budget):
|
||
results_by_id[tc.id] = (
|
||
budget_block_payload(
|
||
reason="fail_budget",
|
||
lang=str(ctx.lang or "en"),
|
||
single_used=single_used,
|
||
single_budget=single_exec_budget,
|
||
fail_used=fail_used,
|
||
fail_budget=fail_exec_budget,
|
||
batch_used=batch_used,
|
||
batch_budget=batch_exec_budget,
|
||
),
|
||
0,
|
||
)
|
||
_trace(
|
||
"cli_fail_budget_exceeded",
|
||
{"tool_name": tc.name, "fail_used": fail_used, "fail_budget": fail_exec_budget},
|
||
)
|
||
continue
|
||
if (not batchish) and single_used >= int(single_exec_budget):
|
||
results_by_id[tc.id] = (
|
||
budget_block_payload(
|
||
reason="call_budget",
|
||
lang=str(ctx.lang or "en"),
|
||
single_used=single_used,
|
||
single_budget=single_exec_budget,
|
||
fail_used=fail_used,
|
||
fail_budget=fail_exec_budget,
|
||
batch_used=batch_used,
|
||
batch_budget=batch_exec_budget,
|
||
),
|
||
0,
|
||
)
|
||
_trace(
|
||
"cli_call_budget_exceeded",
|
||
{
|
||
"tool_name": tc.name,
|
||
"single_used": single_used,
|
||
"single_budget": single_exec_budget,
|
||
},
|
||
)
|
||
continue
|
||
count = int(sig_seen.get(sig, 0))
|
||
name_low = str(tc.name or "").strip().lower()
|
||
listish = name_low.endswith(
|
||
(
|
||
"listclitargets",
|
||
"listmanagedne",
|
||
"queryumeneinventory",
|
||
"queryumealarms",
|
||
"queryumealarmsraw",
|
||
"aggregateumealarms",
|
||
"aggregateumealarmsraw",
|
||
"runumediagnostics",
|
||
"sqlqueryume",
|
||
"findtopologypaths",
|
||
)
|
||
)
|
||
if count >= budget:
|
||
results_by_id[tc.id] = (
|
||
{
|
||
"ok": False,
|
||
"error_code": "tool_loop_guard",
|
||
"error": f"tool loop guard triggered for signature: {tc.name}",
|
||
"hint": (
|
||
"Identical list/query/aggregate call already ran this turn; reuse prior rows "
|
||
"or change filters — do not re-query identically."
|
||
if listish
|
||
else "Identical tool call already ran this turn; change arguments or continue without retry."
|
||
),
|
||
},
|
||
0,
|
||
)
|
||
_trace(
|
||
"tool_loop_guard",
|
||
{
|
||
"tool_name": tc.name,
|
||
"signature": sig[:300],
|
||
"budget": budget,
|
||
},
|
||
)
|
||
continue
|
||
sig_seen[sig] = count + 1
|
||
source_tool_call_id = str(first_tool_call_id_by_signature.get(sig) or "").strip()
|
||
if source_tool_call_id:
|
||
dedupe_alias_to_source[str(tc.id or "")] = source_tool_call_id
|
||
_trace(
|
||
"tool_cache_hit_same_round",
|
||
{
|
||
"tool_name": tc.name,
|
||
"tool_call_id": str(tc.id or ""),
|
||
"source_tool_call_id": source_tool_call_id,
|
||
},
|
||
)
|
||
continue
|
||
first_tool_call_id_by_signature[sig] = str(tc.id or "")
|
||
if is_exec_managed_ne_tool(tool_name):
|
||
if is_batch_exec_args(dict(tc.arguments or {})):
|
||
local_batch_exec += 1
|
||
else:
|
||
local_single_exec += 1
|
||
runnable_tool_uses.append(tc)
|
||
|
||
for batch in partition_tool_use_batches(runnable_tool_uses, ctx.tools):
|
||
_check_stop()
|
||
_trace(
|
||
"tool_batch_started",
|
||
{
|
||
"batch_size": len(batch),
|
||
"tool_names": [str(getattr(x, "name", "") or "") for x in batch],
|
||
},
|
||
)
|
||
if on_tool_ui:
|
||
for tc in batch:
|
||
try:
|
||
on_tool_ui(
|
||
"tool_use_call",
|
||
{
|
||
"phase": "call",
|
||
"tool_name": str(tc.name or ""),
|
||
"tool_call_id": str(tc.id or ""),
|
||
"arguments": dict(tc.arguments or {}),
|
||
},
|
||
)
|
||
except Exception:
|
||
pass
|
||
if len(batch) > 1:
|
||
workers = min(int(self.config.max_workers), len(batch))
|
||
with ThreadPoolExecutor(max_workers=workers) as ex:
|
||
fut_to_tc = {ex.submit(self._execute_tool, ctx, tc): tc for tc in batch}
|
||
for fut in as_completed(fut_to_tc):
|
||
tc = fut_to_tc[fut]
|
||
results_by_id[tc.id] = fut.result()
|
||
else:
|
||
for tc in batch:
|
||
results_by_id[tc.id] = self._execute_tool(ctx, tc)
|
||
_trace(
|
||
"tool_batch_finished",
|
||
{
|
||
"batch_size": len(batch),
|
||
"tool_names": [str(getattr(x, "name", "") or "") for x in batch],
|
||
},
|
||
)
|
||
|
||
for tool_call_id, source_tool_call_id in dedupe_alias_to_source.items():
|
||
if source_tool_call_id in results_by_id:
|
||
results_by_id[tool_call_id] = results_by_id[source_tool_call_id]
|
||
|
||
tool_messages: list[dict[str, Any]] = []
|
||
for tc in tool_uses:
|
||
_check_stop()
|
||
_trace(
|
||
"tool_called",
|
||
{
|
||
"tool_name": tc.name,
|
||
"arguments": tc.arguments,
|
||
"arguments_bytes": _json_blob_size(tc.arguments),
|
||
},
|
||
)
|
||
result, duration_ms = results_by_id[tc.id]
|
||
result = normalize_tool_result(result)
|
||
if isinstance(result, dict) and result.get("ok") is False:
|
||
failed_signatures.add(f"{tc.name}:{self._json_dumps_safe(dict(tc.arguments or {}))}")
|
||
if is_exec_managed_ne_tool(str(tc.name or "")):
|
||
# Count blocked budget responses too so fail-budget can engage same turn.
|
||
if str(result.get("error_code") or "") not in {
|
||
"cli_call_budget_exceeded",
|
||
"cli_fail_budget_exceeded",
|
||
"cli_batch_budget_exceeded",
|
||
"identical_retry_blocked",
|
||
"retry_forbidden_blocked",
|
||
"tool_loop_guard",
|
||
}:
|
||
local_exec_fails += 1
|
||
if isinstance(result, dict) and _result_is_retry_forbidden(result):
|
||
retry_forbidden_tools.add(str(tc.name or ""))
|
||
persisted_result, ingested_refs = ingest_embedded_image_blobs_as_refs(
|
||
result,
|
||
filename_prefix=f"{str(tc.name or 'tool')}-{str(tc.id or '')}",
|
||
)
|
||
logger.info(
|
||
"tool_runtime tool session=%s name=%s duration_ms=%d ok=%s",
|
||
ctx.session_id[:12],
|
||
tc.name,
|
||
duration_ms,
|
||
result.get("ok") if isinstance(result, dict) else None,
|
||
)
|
||
t_db1 = time.perf_counter()
|
||
ctx.store.add_tool_log(
|
||
session_id=ctx.session_id,
|
||
tool_name=tc.name,
|
||
args=tc.arguments,
|
||
result=persisted_result,
|
||
specialist=ctx.specialist,
|
||
duration_ms=duration_ms,
|
||
)
|
||
tool_log_write_ms = int((time.perf_counter() - t_db1) * 1000)
|
||
# Persist full blob when large; model context gets compact + result_ref.
|
||
t_trunc = time.perf_counter()
|
||
observed_rows_this_call = int(_estimate_observed_rows(result))
|
||
result_for_llm = dict(persisted_result or {})
|
||
result_ref: str | None = None
|
||
try:
|
||
from runtime.chat.tool_result_store import attach_result_ref, save_tool_result_blob
|
||
|
||
result_ref = save_tool_result_blob(
|
||
session_id=str(ctx.session_id or ""),
|
||
tool_call_id=str(tc.id or ""),
|
||
result=persisted_result if isinstance(persisted_result, dict) else result_for_llm,
|
||
)
|
||
except Exception:
|
||
result_ref = None
|
||
if tc.name in _SQL_REPLAY_COMPACT_TOOL_NAMES:
|
||
current = int(local_turn_tool_name_counts.get(tc.name, 0))
|
||
current_rows = int(local_turn_tool_observed_rows.get(tc.name, 0))
|
||
local_turn_tool_name_counts[tc.name] = current + 1
|
||
local_turn_tool_observed_rows[tc.name] = current_rows + observed_rows_this_call
|
||
if tc.name in compact_sql_names:
|
||
cumulative_rows = int(turn_tool_observed_rows.get(tc.name, 0)) + int(local_turn_tool_observed_rows.get(tc.name, 0))
|
||
result_for_llm["_history_compacted"] = True
|
||
result_for_llm["_history_compact_reason"] = "repeated_tool_calls_in_turn"
|
||
result_for_llm["_tool_observed_rows_this_call"] = int(observed_rows_this_call)
|
||
result_for_llm["_tool_observed_rows_cumulative_in_turn"] = int(cumulative_rows)
|
||
result_for_llm["audit_note"] = "Result compacted for history replay safety."
|
||
# Compact oversized payloads for the model wire; full body remains in blob/tool_log.
|
||
try:
|
||
capped = truncate_tool_result_for_llm_messages(result_for_llm)
|
||
if isinstance(capped, dict) and capped.get("_truncated_for_llm"):
|
||
result_for_llm = capped
|
||
if not result_ref:
|
||
from runtime.chat.tool_result_store import save_tool_result_blob
|
||
|
||
result_ref = save_tool_result_blob(
|
||
session_id=str(ctx.session_id or ""),
|
||
tool_call_id=str(tc.id or ""),
|
||
result=persisted_result if isinstance(persisted_result, dict) else dict(result_for_llm),
|
||
force=True,
|
||
)
|
||
if result_ref:
|
||
from runtime.chat.tool_result_store import attach_result_ref
|
||
|
||
result_for_llm = attach_result_ref(result_for_llm, result_ref=result_ref)
|
||
except Exception:
|
||
if result_ref and isinstance(result_for_llm, dict):
|
||
result_for_llm["result_ref"] = result_ref
|
||
trunc_ms = int((time.perf_counter() - t_trunc) * 1000)
|
||
tool_content = self._json_dumps_safe(result_for_llm)
|
||
t_db2 = time.perf_counter()
|
||
tool_sig = f"{tc.name}:{self._json_dumps_safe(dict(tc.arguments or {}))}"
|
||
exec_ne_mode = ""
|
||
if is_exec_managed_ne_tool(str(tc.name or "")):
|
||
exec_ne_mode = "batch" if is_batch_exec_args(dict(tc.arguments or {})) else "single"
|
||
msg_row = ctx.store.add_message(
|
||
session_id=ctx.session_id,
|
||
role="tool",
|
||
content=tool_content,
|
||
tool_calls={"tool_call_id": tc.id, "name": tc.name, "assistant_message_id": assistant_msg_id},
|
||
attachments=(_merge_attachments(_attachments_from_tool_result(persisted_result), ingested_refs) or None),
|
||
turn_uuid=ctx.turn_uuid,
|
||
event_type="tool_result",
|
||
event_payload={
|
||
"tool_name": tc.name,
|
||
"observed_rows": int(observed_rows_this_call),
|
||
"ok": bool(result.get("ok")) if isinstance(result, dict) else False,
|
||
"tool_signature": tool_sig[:800],
|
||
"failure_class": str((result or {}).get("failure_class") or "") if isinstance(result, dict) else "",
|
||
"retry_forbidden": bool(
|
||
isinstance(result, dict) and _result_is_retry_forbidden(result)
|
||
),
|
||
**({"exec_ne_mode": exec_ne_mode} if exec_ne_mode else {}),
|
||
**({"result_ref": result_ref} if result_ref else {}),
|
||
},
|
||
)
|
||
try:
|
||
owner = ctx.store.get_ui_session_owner(session_id=ctx.session_id) or {}
|
||
tid = str(owner.get("tenant_id") or "").strip()
|
||
uid = str(owner.get("user_id") or "").strip()
|
||
atts = msg_row.attachments
|
||
if tid and uid and atts:
|
||
raw = json.loads(atts) if isinstance(atts, str) else atts
|
||
items = raw if isinstance(raw, list) else ([raw] if isinstance(raw, dict) else [])
|
||
for a in items:
|
||
if not isinstance(a, dict):
|
||
continue
|
||
aid = str(a.get("attachment_id") or "").strip().lower()
|
||
if aid:
|
||
ctx.store.link_attachment_acl(
|
||
tenant_id=tid,
|
||
user_id=uid,
|
||
session_id=ctx.session_id,
|
||
attachment_id=aid,
|
||
source=f"tool:{str(tc.name or '')}",
|
||
)
|
||
except Exception:
|
||
pass
|
||
tool_msg_write_ms = int((time.perf_counter() - t_db2) * 1000)
|
||
tool_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_content, "name": tc.name})
|
||
_trace(
|
||
"tool_result",
|
||
{
|
||
"tool_name": tc.name,
|
||
"duration_ms": duration_ms,
|
||
"ok": bool(result.get("ok")) if isinstance(result, dict) else None,
|
||
"error_code": str(result.get("error_code") or "") if isinstance(result, dict) else "",
|
||
"result_bytes": _json_blob_size(result),
|
||
"result_for_llm_bytes": len(tool_content or ""),
|
||
"tool_log_write_ms": tool_log_write_ms,
|
||
"tool_message_write_ms": tool_msg_write_ms,
|
||
"truncate_ms": trunc_ms,
|
||
"active_threads": int(threading.active_count()),
|
||
},
|
||
)
|
||
if on_tool_ui:
|
||
truncated_for_llm = bool(
|
||
isinstance(result_for_llm, dict) and result_for_llm.get("_truncated_for_llm")
|
||
)
|
||
payload = {
|
||
"name": tc.name,
|
||
"result": result,
|
||
"llm_wire": {
|
||
"truncated_for_llm": truncated_for_llm,
|
||
"max_chars": int(tool_llm_message_max_chars()),
|
||
"result_bytes": int(_json_blob_size(result)),
|
||
"result_for_llm_bytes": int(len(tool_content or "")),
|
||
"truncate_ms": int(trunc_ms),
|
||
},
|
||
}
|
||
on_tool_ui("tool_use_result", payload)
|
||
return tool_messages, results_by_id
|
||
|
||
|
||
def compact_turn_tool_messages_for_storage(
|
||
*,
|
||
store: Any,
|
||
session_id: str,
|
||
turn_uuid: str | None,
|
||
) -> dict[str, int]:
|
||
"""Compact persisted tool messages after turn completion.
|
||
|
||
This intentionally runs *after* the active turn so current-round model
|
||
context is not affected by truncation/compaction.
|
||
"""
|
||
tid = str(turn_uuid or "").strip()
|
||
if not tid:
|
||
return {"scanned": 0, "updated": 0}
|
||
persist_cap = tool_persist_max_chars(store)
|
||
if persist_cap <= 0:
|
||
return {"scanned": 0, "updated": 0, "skipped": 1}
|
||
try:
|
||
rows = store.get_messages(session_id=session_id, limit=800)
|
||
except Exception:
|
||
return {"scanned": 0, "updated": 0}
|
||
scanned = 0
|
||
updated = 0
|
||
for m in rows or []:
|
||
if str(getattr(m, "role", "") or "") != "tool":
|
||
continue
|
||
if str(getattr(m, "turn_uuid", "") or "") != tid:
|
||
continue
|
||
scanned += 1
|
||
raw = str(getattr(m, "content", "") or "")
|
||
if not raw.strip():
|
||
continue
|
||
try:
|
||
obj = json.loads(raw)
|
||
except Exception:
|
||
continue
|
||
if not isinstance(obj, dict):
|
||
continue
|
||
compacted = truncate_tool_result_for_llm_messages(obj, max_chars=persist_cap)
|
||
if compacted == obj:
|
||
continue
|
||
try:
|
||
store.update_message_content(
|
||
session_id=session_id,
|
||
message_id=int(getattr(m, "id", 0) or 0),
|
||
content=json.dumps(compacted, ensure_ascii=False, default=str),
|
||
event_payload=getattr(m, "event_payload", None),
|
||
)
|
||
updated += 1
|
||
except Exception:
|
||
continue
|
||
return {"scanned": int(scanned), "updated": int(updated), "persist_cap": int(persist_cap)}
|
||
|
||
__all__ = [
|
||
"ToolExecutionConfig",
|
||
"ToolExecutionContext",
|
||
"ToolExecutor",
|
||
"normalize_tool_result",
|
||
"partition_tool_use_batches",
|
||
"tool_llm_message_max_chars",
|
||
"tool_persist_max_chars",
|
||
"truncate_tool_result_for_llm_messages",
|
||
"compact_turn_tool_messages_for_storage",
|
||
]
|
||
|
||
|
||
def _merge_attachments(*parts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
out: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for part in parts:
|
||
for a in part or []:
|
||
if not isinstance(a, dict):
|
||
continue
|
||
k = str(a.get("attachment_id") or a.get("pointer_uri") or a.get("url") or "").strip()
|
||
if not k or k in seen:
|
||
continue
|
||
seen.add(k)
|
||
out.append(a)
|
||
return out
|
||
|
||
|
||
def _ref_type_for_mime(mime: str) -> str:
|
||
m = str(mime or "").strip().lower()
|
||
if m.startswith("image/"):
|
||
return "image_ref"
|
||
if m.startswith("video/"):
|
||
return "video_ref"
|
||
if m.startswith("text/"):
|
||
return "text_ref"
|
||
return "binary_ref"
|