初始化:独立 oclaw 仓库首提交

- 在 oclaw/ 下重新初始化 Git 仓库
- 补齐子仓库 .gitignore,避免提交本地运行态数据(_local、node_modules、logs 等)
- 提交当前工程代码与配置

Made-with: Cursor
This commit is contained in:
oliver 2026-04-24 22:31:22 +08:00
commit ba3836f00f
579 changed files with 83112 additions and 0 deletions

0
platform/__init__.py Normal file
View file

View file

View file

@ -0,0 +1,21 @@
"""访问密码解析(环境变量 + 数据库);Streamlit secrets 在认证模块中处理。"""
from __future__ import annotations
import os
from oclaw.platform.persistence.sqlite_store import SqliteStore
def load_expected_password(store: SqliteStore, *, extra_candidate: str | None = None) -> str | None:
pwd = (os.getenv("AIA_ASSISTANT_PASSWORD") or "").strip()
if not pwd:
pwd = (os.getenv("OPS_ASSISTANT_PASSWORD") or "").strip()
if not pwd and extra_candidate:
pwd = extra_candidate.strip()
if not pwd:
pwd = (store.get_secret("auth_password") or "").strip()
return pwd if pwd else None
__all__ = ["load_expected_password"]

194
platform/config/paths.py Normal file
View file

@ -0,0 +1,194 @@
"""仓库与数据文件路径解析(与框架无关,供 ``db`` / UI 等共用)。"""
from __future__ import annotations
import os
import shutil
import sqlite3
import sys
import time
import warnings
from pathlib import Path
def _project_root() -> Path:
if getattr(sys, "frozen", False):
return Path(sys.executable).resolve().parent
return Path(__file__).resolve().parents[3]
PROJECT_ROOT = _project_root()
_DATA_MIGRATION_DONE = False
def _canonical_data_root() -> Path:
return (PROJECT_ROOT / "oclaw" / "data").resolve()
def _legacy_platform_data_root() -> Path:
return (PROJECT_ROOT / "src" / "platform" / "data").resolve()
def _legacy_root_data_root() -> Path:
return (PROJECT_ROOT / "data").resolve()
def _backup_keep_count() -> int:
raw = (os.getenv("AIA_ASSISTANT_PREMERGE_BACKUP_KEEP") or os.getenv("OPS_ASSISTANT_PREMERGE_BACKUP_KEEP") or "").strip()
if not raw:
return 3
try:
val = int(raw)
except Exception:
return 3
return max(0, val)
def _cleanup_pre_merge_backups() -> None:
root = _canonical_data_root()
if not root.exists():
return
keep = _backup_keep_count()
backups = sorted([p for p in root.glob("_pre_merge_sqlite_*") if p.is_dir()], key=lambda p: p.name, reverse=True)
for old in backups[keep:]:
try:
shutil.rmtree(old)
except Exception as exc:
warnings.warn(f"自动清理旧备份失败(已跳过): {old} ({exc})", RuntimeWarning, stacklevel=1)
def attachments_dir() -> Path:
p = (_canonical_data_root() / "attachments").resolve()
p.mkdir(parents=True, exist_ok=True)
return p
def _sqlite_sidecars(main: Path) -> list[Path]:
return [main, Path(str(main) + "-wal"), Path(str(main) + "-shm")]
def _sqlite_chat_session_count(db: Path) -> int | None:
if not db.exists():
return None
try:
conn = sqlite3.connect(f"file:{db.as_posix()}?mode=ro", uri=True, timeout=3.0)
try:
r = conn.execute("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='chat_session'").fetchone()
if not r or int(r[0] or 0) == 0:
return 0
row = conn.execute("SELECT COUNT(*) FROM chat_session").fetchone()
return int(row[0] or 0) if row else 0
finally:
conn.close()
except Exception:
return None
def _checkpoint_wal_best_effort(main: Path) -> None:
try:
conn = sqlite3.connect(str(main), timeout=5.0)
try:
conn.execute("PRAGMA wal_checkpoint(TRUNCATE);")
finally:
conn.close()
except Exception:
pass
def _copy_sqlite_bundle(src_main: Path, dst_main: Path) -> None:
dst_main.parent.mkdir(parents=True, exist_ok=True)
_checkpoint_wal_best_effort(src_main)
for s in _sqlite_sidecars(src_main):
if not s.exists():
continue
d = dst_main.parent / s.name
if s.resolve() == d.resolve():
continue
shutil.copy2(s, d)
def _backup_sqlite_bundle_copy(main: Path, dest_dir: Path) -> None:
dest_dir.mkdir(parents=True, exist_ok=True)
_checkpoint_wal_best_effort(main)
for s in _sqlite_sidecars(main):
if s.exists():
shutil.copy2(s, dest_dir / s.name)
def _merge_attachment_tree(src: Path, dst: Path) -> int:
if not src.is_dir():
return 0
dst.mkdir(parents=True, exist_ok=True)
copied = 0
for f in src.rglob("*"):
if not f.is_file():
continue
rel = f.relative_to(src)
target = dst / rel
if target.exists():
continue
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(f, target)
copied += 1
return copied
def _run_default_data_migration(canonical_db: Path) -> None:
legacy_root_data = _legacy_root_data_root()
legacy_root_db = legacy_root_data / "ai_ops.sqlite"
if legacy_root_db.exists() and not canonical_db.exists():
_copy_sqlite_bundle(legacy_root_db, canonical_db)
_merge_attachment_tree(legacy_root_data / "attachments", attachments_dir())
legacy_platform_data = _legacy_platform_data_root()
legacy_db = legacy_platform_data / "ai_ops.sqlite"
if legacy_db.exists():
leg_n = _sqlite_chat_session_count(legacy_db)
if leg_n is not None:
if not canonical_db.exists():
_copy_sqlite_bundle(legacy_db, canonical_db)
else:
can_n = _sqlite_chat_session_count(canonical_db)
can_n = 0 if can_n is None else can_n
force = str(os.getenv("AIA_LEGACY_DB_FORCE_PREMERGE") or os.getenv("OPS_LEGACY_DB_FORCE_PREMERGE") or "").strip().lower() in (
"1",
"true",
"yes",
"on",
)
if force and leg_n > can_n:
ts = time.strftime("%Y%m%d_%H%M%S")
bak = _canonical_data_root() / f"_pre_merge_sqlite_{ts}"
try:
_checkpoint_wal_best_effort(canonical_db)
_backup_sqlite_bundle_copy(canonical_db, bak)
_copy_sqlite_bundle(legacy_db, canonical_db)
except (OSError, PermissionError, shutil.Error) as exc:
warnings.warn(
"未能用旧库覆盖当前主库(可能被其它进程占用)。"
f"已备份当前库到: {bak}。可关闭占用后重启,或手动合并。原因: {exc}",
RuntimeWarning,
stacklevel=1,
)
_merge_attachment_tree(legacy_platform_data / "attachments", attachments_dir())
_cleanup_pre_merge_backups()
def db_path() -> str:
global _DATA_MIGRATION_DONE
p = os.getenv("AIA_ASSISTANT_DB_PATH") or os.getenv("OPS_ASSISTANT_DB_PATH") or "oclaw/data/ai_ops.sqlite"
path = Path(p)
if not path.is_absolute():
path = (PROJECT_ROOT / path).resolve()
use_default = not (os.getenv("AIA_ASSISTANT_DB_PATH") or os.getenv("OPS_ASSISTANT_DB_PATH"))
if use_default and not _DATA_MIGRATION_DONE:
_DATA_MIGRATION_DONE = True
try:
_run_default_data_migration(path)
except Exception as exc:
warnings.warn(f"默认数据目录迁移未完全成功(可稍后重试或检查权限): {exc}", RuntimeWarning, stacklevel=1)
return str(path)
__all__ = ["PROJECT_ROOT", "attachments_dir", "db_path"]

View file

@ -0,0 +1,77 @@
from __future__ import annotations
import hashlib
import os
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class EmbeddingResult:
model: str
vector: list[float]
class EmbeddingClient:
def embed(self, text: str) -> EmbeddingResult:
raise NotImplementedError
class OpenAIEmbeddingClient(EmbeddingClient):
def __init__(self, *, model: str | None = None, api_key: str | None = None, base_url: str | None = None):
self.model = (model or os.getenv("OPENAI_EMBEDDING_MODEL") or "text-embedding-3-small").strip()
self.api_key = (api_key or os.getenv("OPENAI_API_KEY") or "").strip()
self.base_url = (base_url or os.getenv("OPENAI_BASE_URL") or "").strip() or None
if not self.api_key:
raise RuntimeError("OPENAI_API_KEY is required for embeddings")
from openai import OpenAI
kwargs: dict[str, Any] = {"api_key": self.api_key}
if self.base_url:
kwargs["base_url"] = self.base_url
self._client = OpenAI(**kwargs)
def embed(self, text: str) -> EmbeddingResult:
t = (text or "").strip()
if not t:
return EmbeddingResult(model=self.model, vector=[0.0] * 8)
resp = self._client.embeddings.create(model=self.model, input=t)
vec = list(resp.data[0].embedding)
return EmbeddingResult(model=self.model, vector=[float(x) for x in vec])
class HashEmbeddingClient(EmbeddingClient):
"""Offline fallback: deterministic small vector (NOT semantic)."""
def __init__(self, *, dim: int = 32, model: str = "hash-embed-32"):
self.dim = int(dim)
self.model = model
def embed(self, text: str) -> EmbeddingResult:
t = (text or "").encode("utf-8", errors="ignore")
h = hashlib.sha256(t).digest()
vec = []
for i in range(self.dim):
b = h[i % len(h)]
vec.append((float(b) / 255.0) * 2.0 - 1.0)
return EmbeddingResult(model=self.model, vector=vec)
def build_default_embedding_client() -> EmbeddingClient:
mode = (os.getenv("AIA_RAG_EMBEDDING_MODE") or "").strip().lower()
if mode in ("hash", "offline"):
return HashEmbeddingClient()
try:
return OpenAIEmbeddingClient()
except Exception:
return HashEmbeddingClient()
__all__ = [
"EmbeddingClient",
"EmbeddingResult",
"OpenAIEmbeddingClient",
"HashEmbeddingClient",
"build_default_embedding_client",
]

View file

View file

@ -0,0 +1,333 @@
from __future__ import annotations
import base64
import hashlib
import json
import os
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional
from oclaw.platform.config.paths import attachments_dir
_META_SUFFIX: Final[str] = ".meta.json"
def _utc_ts() -> int:
return int(time.time())
def _safe_rel_name(name: str) -> str:
# keep it simple: drop path parts, trim, and avoid empty
raw = (name or "").replace("\\", "/").split("/")[-1].strip()
return raw or "image"
def _ext_from_mime(mime: str | None) -> str:
m = (mime or "").strip().lower()
if m == "image/png":
return ".png"
if m in ("image/jpg", "image/jpeg"):
return ".jpg"
if m == "image/webp":
return ".webp"
if m == "image/gif":
return ".gif"
return ""
def _guess_mime_from_ext(ext: str) -> str:
e = (ext or "").lower().lstrip(".")
if e == "png":
return "image/png"
if e in ("jpg", "jpeg"):
return "image/jpeg"
if e == "webp":
return "image/webp"
if e == "gif":
return "image/gif"
return "application/octet-stream"
@dataclass(frozen=True)
class AttachmentMeta:
attachment_id: str
name: str
mime: str
bytes: int
created_at: int
last_access: int
width: int | None = None
height: int | None = None
def to_dict(self) -> dict[str, Any]:
d: dict[str, Any] = {
"attachment_id": self.attachment_id,
"name": self.name,
"mime": self.mime,
"bytes": int(self.bytes),
"created_at": int(self.created_at),
"last_access": int(self.last_access),
}
if self.width is not None:
d["width"] = int(self.width)
if self.height is not None:
d["height"] = int(self.height)
return d
@staticmethod
def from_dict(d: dict[str, Any]) -> "AttachmentMeta":
return AttachmentMeta(
attachment_id=str(d.get("attachment_id") or ""),
name=str(d.get("name") or ""),
mime=str(d.get("mime") or "application/octet-stream"),
bytes=int(d.get("bytes") or 0),
created_at=int(d.get("created_at") or 0),
last_access=int(d.get("last_access") or 0),
width=int(d["width"]) if d.get("width") is not None else None,
height=int(d["height"]) if d.get("height") is not None else None,
)
class AttachmentAssetStore:
"""Disk-backed attachment store (project-local by default)."""
def __init__(self, root_dir: str | Path | None = None):
self.root = Path(root_dir) if root_dir is not None else attachments_dir()
self.root.mkdir(parents=True, exist_ok=True)
def _data_path(self, attachment_id: str, *, ext: str) -> Path:
# bucket to avoid huge single dir
aid = (attachment_id or "").strip()
p1, p2 = (aid[:2] or "xx"), (aid[2:4] or "yy")
d = self.root / p1 / p2
d.mkdir(parents=True, exist_ok=True)
return d / f"{aid}{ext}"
def _meta_path(self, attachment_id: str) -> Path:
aid = (attachment_id or "").strip()
p1, p2 = (aid[:2] or "xx"), (aid[2:4] or "yy")
d = self.root / p1 / p2
d.mkdir(parents=True, exist_ok=True)
return d / f"{aid}{_META_SUFFIX}"
def save_bytes(
self,
data: bytes,
*,
filename: str,
mime: str | None,
width: int | None = None,
height: int | None = None,
) -> AttachmentMeta:
blob = bytes(data or b"")
h = hashlib.sha256(blob).hexdigest()
ext = _ext_from_mime(mime) or (("." + filename.split(".")[-1].lower()) if "." in filename else "")
ext = ext if ext in (".png", ".jpg", ".jpeg", ".webp", ".gif") else _ext_from_mime(mime) or ""
data_path = self._data_path(h, ext=ext)
meta_path = self._meta_path(h)
now = _utc_ts()
if not data_path.exists():
tmp = str(data_path) + ".tmp"
with open(tmp, "wb") as f:
f.write(blob)
os.replace(tmp, data_path)
meta = AttachmentMeta(
attachment_id=h,
name=_safe_rel_name(filename),
mime=(mime or _guess_mime_from_ext(ext)).strip() or _guess_mime_from_ext(ext),
bytes=len(blob),
created_at=now,
last_access=now,
width=width,
height=height,
)
# best-effort: keep existing created_at if present
if meta_path.exists():
try:
prev = json.loads(meta_path.read_text("utf-8"))
if isinstance(prev, dict) and prev.get("created_at"):
meta = AttachmentMeta(
**{**meta.to_dict(), "created_at": int(prev.get("created_at") or now)} # type: ignore[arg-type]
)
except Exception:
pass
meta_path.write_text(json.dumps(meta.to_dict(), ensure_ascii=False), encoding="utf-8")
return meta
def get_meta(self, attachment_id: str) -> Optional[AttachmentMeta]:
mp = self._meta_path(attachment_id)
if not mp.exists():
return None
try:
obj = json.loads(mp.read_text("utf-8"))
if not isinstance(obj, dict):
return None
return AttachmentMeta.from_dict(obj)
except Exception:
return None
def touch(self, attachment_id: str) -> None:
mp = self._meta_path(attachment_id)
if not mp.exists():
return
try:
obj = json.loads(mp.read_text("utf-8"))
if not isinstance(obj, dict):
return
obj["last_access"] = _utc_ts()
mp.write_text(json.dumps(obj, ensure_ascii=False), encoding="utf-8")
except Exception:
return
def load_bytes(self, attachment_id: str) -> tuple[bytes, Optional[AttachmentMeta]]:
meta = self.get_meta(attachment_id)
# try find data file by scanning common extensions
exts = (".png", ".jpg", ".jpeg", ".webp", ".gif", "")
data_path = None
for ext in exts:
p = self._data_path(attachment_id, ext=ext)
if p.exists():
data_path = p
break
if data_path is None:
return b"", meta
try:
blob = data_path.read_bytes()
except Exception:
return b"", meta
self.touch(attachment_id)
return blob, meta
def get_local_path(self, attachment_id: str) -> Path | None:
exts = (".png", ".jpg", ".jpeg", ".webp", ".gif", "")
for ext in exts:
p = self._data_path(attachment_id, ext=ext)
if p.exists():
self.touch(attachment_id)
return p
return None
def inspect_asset(self, attachment_id: str) -> dict[str, Any]:
meta = self.get_meta(attachment_id)
p = self.get_local_path(attachment_id)
if meta is None and p is None:
return {"state": "missing", "attachment_id": attachment_id}
if meta is not None and p is None:
return {
"state": "meta_exists_blob_missing",
"attachment_id": attachment_id,
"meta": meta.to_dict(),
}
return {
"state": "ok",
"attachment_id": attachment_id,
"meta": meta.to_dict() if meta else None,
"path": str(p) if p is not None else "",
}
def get_render_cache_key(self, attachment_id: str, *, mime_hint: str | None = None) -> tuple[str, str, int, int]:
meta = self.get_meta(attachment_id)
return (
str(attachment_id or "").strip(),
str(mime_hint or (meta.mime if meta else "") or "image/jpeg"),
int(meta.bytes if meta else 0),
int(meta.created_at if meta else 0),
)
def gc(self, *, ttl_days: int = 7) -> dict[str, Any]:
ttl_sec = max(1, int(ttl_days)) * 86400
now = _utc_ts()
removed = 0
kept = 0
if not self.root.exists():
return {"ok": True, "removed": 0, "kept": 0}
for mp in self.root.rglob(f"*{_META_SUFFIX}"):
try:
obj = json.loads(mp.read_text("utf-8"))
if not isinstance(obj, dict):
continue
last_access = int(obj.get("last_access") or obj.get("created_at") or 0)
aid = str(obj.get("attachment_id") or mp.name.replace(_META_SUFFIX, ""))
except Exception:
continue
if last_access and (now - last_access) <= ttl_sec:
kept += 1
continue
# delete meta + possible data files
try:
mp.unlink(missing_ok=True) # py3.8+; on 3.14 ok
except Exception:
pass
for ext in (".png", ".jpg", ".jpeg", ".webp", ".gif", ""):
try:
self._data_path(aid, ext=ext).unlink(missing_ok=True)
except Exception:
pass
removed += 1
return {"ok": True, "removed": removed, "kept": kept}
_LAST_GC_TS: int = 0
def gc_if_due(*, ttl_days: int = 7, min_interval_sec: int = 3600, root_dir: str | Path | None = None) -> None:
"""Best-effort throttled GC to be called in UI loop."""
global _LAST_GC_TS
now = _utc_ts()
if _LAST_GC_TS and (now - _LAST_GC_TS) < max(10, int(min_interval_sec)):
return
_LAST_GC_TS = now
try:
AttachmentAssetStore(root_dir).gc(ttl_days=ttl_days)
except Exception:
return
def image_ref_to_data_url(att: dict[str, Any], *, root_dir: str | Path | None = None) -> tuple[str, str]:
"""Helper for UI: convert image_ref to (mime, data_url_base64_payload)."""
aid = str(att.get("attachment_id") or "").strip()
if not aid:
return "image/jpeg", ""
store = AttachmentAssetStore(root_dir)
blob, meta = store.load_bytes(aid)
mime = str(att.get("mime") or (meta.mime if meta else "image/jpeg") or "image/jpeg")
if not blob:
return mime, ""
b64 = base64.b64encode(blob).decode("ascii")
return mime, b64
def attachment_id_to_data_url(
attachment_id: str,
*,
mime: str | None = None,
root_dir: str | Path | None = None,
) -> str:
"""Convert stored attachment bytes into data URL form."""
aid = str(attachment_id or "").strip()
if not aid:
return ""
store = AttachmentAssetStore(root_dir)
blob, meta = store.load_bytes(aid)
if not blob:
return ""
resolved_mime = str(mime or (meta.mime if meta else "") or "image/jpeg")
b64 = base64.b64encode(blob).decode("ascii")
return f"data:{resolved_mime};base64,{b64}"
__all__ = [
"AttachmentAssetStore",
"AttachmentMeta",
"attachment_id_to_data_url",
"gc_if_due",
"image_ref_to_data_url",
]

View file

@ -0,0 +1,479 @@
from __future__ import annotations
"""将上传文件解析为附件字典(不依赖 Streamlit)。"""
import io
import json
import os
import re
import zipfile
from functools import lru_cache
from pathlib import Path
from pathlib import PurePosixPath
from typing import Any
import pandas as pd
from docx import Document
from PIL import Image
from PyPDF2 import PdfReader
from oclaw.platform.config.paths import PROJECT_ROOT
from oclaw.platform.files.attachment_assets import AttachmentAssetStore
from oclaw.platform.files.tabular_attachment_store import save_dataframe, save_workbook
def _sniff_image_mime(data: bytes) -> str | None:
if len(data) < 12:
return None
if data[:8] == b"\x89PNG\r\n\x1a\n":
return "image/png"
if data[:3] == b"\xff\xd8\xff":
return "image/jpeg"
if data[:6] in (b"GIF87a", b"GIF89a"):
return "image/gif"
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
return "image/webp"
return None
MAX_TEXT_CHARS = 12_000
EXCEL_PREVIEW_ROWS = 80
LARGE_TABLE_PREVIEW_ROWS = 20
MAX_ZIP_DEPTH = 2
MAX_ZIP_FILE_COUNT = 200
MAX_ZIP_ENTRY_BYTES = 10 * 1024 * 1024
MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES = 50 * 1024 * 1024
DEFAULT_TABULAR_ROWS_READ = 5000
DEFAULT_TABULAR_COLUMNS = 200
DEFAULT_TABULAR_CELL_CHARS = 500
DEFAULT_TABULAR_TOOL_MODE_ENABLED = True
DEFAULT_TABULAR_TOOL_MODE_MIN_ROWS = 20_000
DEFAULT_TABULAR_TOOL_MODE_MAX_BYTES = 30 * 1024 * 1024
DEFAULT_LARGE_TABLE_PREVIEW_ROWS = LARGE_TABLE_PREVIEW_ROWS
MAX_ZIP_MEMBER_NAME_LENGTH = 240
MAX_EXCEL_ZIP_FILE_COUNT = 500
MAX_EXCEL_ZIP_ENTRY_BYTES = 20 * 1024 * 1024
MAX_EXCEL_ZIP_TOTAL_UNCOMPRESSED_BYTES = 120 * 1024 * 1024
DEFAULT_MAX_EXCEL_SHEETS = 50
def _truncate_text(text: str, limit: int = MAX_TEXT_CHARS) -> str:
t = str(text or "")
if len(t) <= limit:
return t
return t[:limit] + f"\n\n...[truncated to {limit} chars]"
def _decode_text_bytes(data: bytes) -> str:
try:
return data.decode("utf-8")
except UnicodeDecodeError:
try:
return data.decode("gbk")
except Exception:
return "Error decoding text file (unknown encoding)"
def _clean_html_to_text(raw: str) -> str:
text = re.sub(r"(?is)<script[^>]*>.*?</script>", " ", str(raw or ""))
text = re.sub(r"(?is)<style[^>]*>.*?</style>", " ", text)
text = re.sub(r"(?is)<[^>]+>", " ", text)
text = re.sub(r"[ \t\r\f\v]+", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def _safe_int(raw: Any, default: int, *, min_value: int = 1, max_value: int = 2_000_000) -> int:
try:
value = int(raw)
except Exception:
return default
if value < min_value:
return default
return min(value, max_value)
@lru_cache(maxsize=1)
def _attachments_limits() -> dict[str, Any]:
cfg_path_raw = str(os.getenv("AIA_OCLAW_CONFIG_PATH") or "").strip()
cfg_path = Path(cfg_path_raw).expanduser() if cfg_path_raw else (Path(PROJECT_ROOT) / "oclaw" / "oclaw.json").resolve()
if not cfg_path.is_absolute():
cfg_path = (Path(PROJECT_ROOT) / cfg_path).resolve()
tabular_cfg: dict[str, Any] = {}
try:
obj = json.loads(cfg_path.read_text(encoding="utf-8"))
tabular_cfg = (
(((obj or {}).get("plugins") or {}).get("entries") or {}).get("memory-wiki", {}).get("auto", {}).get("attachments", {}).get("tabular", {}) or {}
)
except Exception:
tabular_cfg = {}
return {
"rows_read": _safe_int(tabular_cfg.get("max_rows_read"), DEFAULT_TABULAR_ROWS_READ),
"columns": _safe_int(tabular_cfg.get("max_columns"), DEFAULT_TABULAR_COLUMNS),
"cell_chars": _safe_int(tabular_cfg.get("max_cell_chars"), DEFAULT_TABULAR_CELL_CHARS),
"max_excel_sheets": _safe_int(tabular_cfg.get("max_excel_sheets"), DEFAULT_MAX_EXCEL_SHEETS, max_value=500),
"large_table_preview_rows": _safe_int(
tabular_cfg.get("large_table_preview_rows"), DEFAULT_LARGE_TABLE_PREVIEW_ROWS, max_value=500
),
"tool_mode_enabled": bool(tabular_cfg.get("tool_mode_enabled", DEFAULT_TABULAR_TOOL_MODE_ENABLED)),
"tool_mode_min_rows": _safe_int(tabular_cfg.get("tool_mode_min_rows"), DEFAULT_TABULAR_TOOL_MODE_MIN_ROWS),
"tool_mode_max_bytes": _safe_int(
tabular_cfg.get("tool_mode_max_bytes"), DEFAULT_TABULAR_TOOL_MODE_MAX_BYTES, max_value=500 * 1024 * 1024
),
}
def clear_attachment_limits_cache() -> None:
_attachments_limits.cache_clear()
def _clip_tabular_cells(df: pd.DataFrame, *, max_cell_chars: int) -> tuple[pd.DataFrame, bool]:
clipped_any = False
def _clip_cell(value: Any) -> Any:
nonlocal clipped_any
if value is None:
return value
s = str(value)
if len(s) <= max_cell_chars:
return value
clipped_any = True
return s[:max_cell_chars] + "...[cell-truncated]"
out = df.copy()
for col in out.columns:
out[col] = out[col].map(_clip_cell)
return out, clipped_any
def _dataframe_summary_text(
df: pd.DataFrame,
*,
sampled: bool = False,
clipped_columns: bool = False,
clipped_cells: bool = False,
preview_rows: int = EXCEL_PREVIEW_ROWS,
) -> str:
rows = int(len(df.index))
cols = int(len(df.columns))
header = [str(x) for x in list(df.columns)]
dtypes = [f"{str(k)}:{str(v)}" for k, v in df.dtypes.items()]
show_rows = max(1, int(preview_rows or EXCEL_PREVIEW_ROWS))
sample = df.head(show_rows).to_string(index=False)
body = (
f"# Table Summary\n"
f"- rows: {rows}\n"
f"- cols: {cols}\n"
f"- sampled: {'yes' if sampled else 'no'}\n"
f"- clipped_columns: {'yes' if clipped_columns else 'no'}\n"
f"- clipped_cells: {'yes' if clipped_cells else 'no'}\n"
f"- columns: {', '.join(header)}\n"
f"- dtypes: {', '.join(dtypes)}\n\n"
f"## Preview (first {min(rows, show_rows)} rows)\n"
f"{sample}"
)
return _truncate_text(body)
def _pdf_summary_text(reader: PdfReader) -> str:
pages = list(getattr(reader, "pages", []) or [])
chunks: list[str] = [f"# PDF Summary\n- pages: {len(pages)}\n"]
for idx, page in enumerate(pages, start=1):
txt = ""
try:
txt = str(page.extract_text() or "").strip()
except Exception:
txt = ""
if not txt:
txt = "(empty page text)"
chunks.append(f"## Page {idx}\n{txt}")
return _truncate_text("\n\n".join(chunks))
def _is_safe_zip_member_name(name: str) -> bool:
n = str(name or "").replace("\\", "/").strip()
if not n:
return False
if len(n) > MAX_ZIP_MEMBER_NAME_LENGTH:
return False
if any(ord(ch) < 32 for ch in n):
return False
p = PurePosixPath(n)
if p.is_absolute():
return False
if any(part in ("..", "") for part in p.parts):
return False
return True
def _validate_excel_zip_payload(data: bytes) -> str | None:
try:
with zipfile.ZipFile(io.BytesIO(data)) as zf:
infos = [x for x in zf.infolist() if not x.is_dir()]
if len(infos) > MAX_EXCEL_ZIP_FILE_COUNT:
return f"excel zip has too many entries ({len(infos)} > {MAX_EXCEL_ZIP_FILE_COUNT})"
total_uncompressed = 0
for info in infos:
if not _is_safe_zip_member_name(info.filename):
return f"excel zip contains unsafe path: {info.filename}"
fsize = int(info.file_size or 0)
total_uncompressed += fsize
if fsize > MAX_EXCEL_ZIP_ENTRY_BYTES:
return f"excel zip entry too large: {info.filename} ({fsize} > {MAX_EXCEL_ZIP_ENTRY_BYTES})"
if total_uncompressed > MAX_EXCEL_ZIP_TOTAL_UNCOMPRESSED_BYTES:
return (
"excel zip total uncompressed size too large "
f"({total_uncompressed} > {MAX_EXCEL_ZIP_TOTAL_UNCOMPRESSED_BYTES})"
)
return None
except Exception:
return "invalid excel zip payload"
def process_zip(zip_data: bytes, *, _depth: int = 0) -> list[dict[str, Any]]:
if _depth > MAX_ZIP_DEPTH:
return [
{
"type": "text",
"name": "zip-error",
"content": f"ZIP nesting too deep (>{MAX_ZIP_DEPTH})",
}
]
attachments: list[dict[str, Any]] = []
try:
with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
infos = [x for x in zf.infolist() if not x.is_dir()]
if len(infos) > MAX_ZIP_FILE_COUNT:
raise ValueError(f"ZIP has too many files ({len(infos)} > {MAX_ZIP_FILE_COUNT})")
total_uncompressed = 0
for info in infos:
if not _is_safe_zip_member_name(info.filename):
raise ValueError(f"ZIP contains unsafe path: {info.filename}")
total_uncompressed += int(info.file_size or 0)
if int(info.file_size or 0) > MAX_ZIP_ENTRY_BYTES:
raise ValueError(
f"ZIP entry too large: {info.filename} ({int(info.file_size or 0)} > {MAX_ZIP_ENTRY_BYTES})"
)
if total_uncompressed > MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES:
raise ValueError(
"ZIP total uncompressed size too large "
f"({total_uncompressed} > {MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES})"
)
with zf.open(info) as f:
data = f.read()
att = process_file_data(info.filename, data, _zip_depth=_depth + 1)
if att:
attachments.extend(att)
except Exception as e:
attachments.append(
{
"type": "text",
"name": "zip-error",
"content": f"Error parsing zip: {e}",
}
)
return attachments
def process_file_data(name: str, data: bytes, *, _zip_depth: int = 0) -> list[dict[str, Any]]:
ext = name.split(".")[-1].lower() if "." in name else ""
attachments: list[dict[str, Any]] = []
if ext in ("png", "jpg", "jpeg", "webp", "gif", "jfif", "pjpeg", "bmp", "tif", "tiff"):
if ext == "png":
mime = "image/png"
elif ext in ("jpg", "jpeg", "jfif", "pjpeg"):
mime = "image/jpeg"
elif ext == "webp":
mime = "image/webp"
elif ext == "gif":
mime = "image/gif"
elif ext == "bmp":
mime = "image/bmp"
elif ext in ("tif", "tiff"):
mime = "image/tiff"
width = None
height = None
try:
with Image.open(io.BytesIO(data)) as im:
width, height = im.size
except Exception:
pass
meta = AttachmentAssetStore().save_bytes(data, filename=name, mime=mime, width=width, height=height)
attachments.append(
{
"type": "image_ref",
"name": meta.name,
"mime": meta.mime,
"attachment_id": meta.attachment_id,
"bytes": meta.bytes,
"width": meta.width,
"height": meta.height,
}
)
return attachments
sniff_mime = _sniff_image_mime(data)
if sniff_mime:
width = None
height = None
try:
with Image.open(io.BytesIO(data)) as im:
width, height = im.size
except Exception:
pass
meta = AttachmentAssetStore().save_bytes(data, filename=name, mime=sniff_mime, width=width, height=height)
attachments.append(
{
"type": "image_ref",
"name": meta.name,
"mime": meta.mime,
"attachment_id": meta.attachment_id,
"bytes": meta.bytes,
"width": meta.width,
"height": meta.height,
}
)
return attachments
if ext in ("xlsx", "xls", "csv"):
try:
if ext == "xlsx":
zip_err = _validate_excel_zip_payload(data)
if zip_err:
attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing excel/csv: {zip_err}")})
return attachments
limits = _attachments_limits()
rows_read = int(limits.get("rows_read") or DEFAULT_TABULAR_ROWS_READ)
max_columns = int(limits.get("columns") or DEFAULT_TABULAR_COLUMNS)
max_cell_chars = int(limits.get("cell_chars") or DEFAULT_TABULAR_CELL_CHARS)
max_excel_sheets = int(limits.get("max_excel_sheets") or DEFAULT_MAX_EXCEL_SHEETS)
large_table_preview_rows = int(
limits.get("large_table_preview_rows") or DEFAULT_LARGE_TABLE_PREVIEW_ROWS
)
tool_mode_enabled = bool(limits.get("tool_mode_enabled", DEFAULT_TABULAR_TOOL_MODE_ENABLED))
tool_mode_min_rows = int(limits.get("tool_mode_min_rows") or DEFAULT_TABULAR_TOOL_MODE_MIN_ROWS)
tool_mode_max_bytes = int(limits.get("tool_mode_max_bytes") or DEFAULT_TABULAR_TOOL_MODE_MAX_BYTES)
if ext == "csv":
df = pd.read_csv(io.BytesIO(data), nrows=rows_read)
else:
df = pd.read_excel(io.BytesIO(data), nrows=rows_read)
sampled = int(len(df.index)) >= rows_read
estimated_rows = 0
if ext == "csv":
estimated_rows = max(0, int(data.count(b"\n")) - 1)
else:
estimated_rows = int(len(df.index))
clipped_columns = int(len(df.columns)) > max_columns
if clipped_columns:
df = df.iloc[:, :max_columns]
df, clipped_cells = _clip_tabular_cells(df, max_cell_chars=max_cell_chars)
tool_mode = bool(
tool_mode_enabled
and int(len(data or b"")) <= tool_mode_max_bytes
and (estimated_rows >= tool_mode_min_rows or (ext in ("xlsx", "xls") and sampled))
)
attachments.append(
{
"type": "text",
"name": name,
"content": _dataframe_summary_text(
df,
sampled=sampled,
clipped_columns=clipped_columns,
clipped_cells=clipped_cells,
preview_rows=(large_table_preview_rows if tool_mode else EXCEL_PREVIEW_ROWS),
)
+ (
(
f"\n\n# Large Table Mode\n- estimated_rows: {estimated_rows}\n- query_tool: query_tabular_attachment\n"
"- note: use the `table_id` from the accompanying `tabular_ref` attachment to query full data."
)
if tool_mode
else ""
),
}
)
if tool_mode:
workbook_sheets: dict[str, pd.DataFrame]
clipped_sheets = False
if ext == "csv":
full_df = pd.read_csv(io.BytesIO(data), dtype=str)
if int(len(full_df.columns)) > max_columns:
full_df = full_df.iloc[:, :max_columns]
full_df, _ = _clip_tabular_cells(full_df, max_cell_chars=max_cell_chars)
workbook_sheets = {"Sheet1": full_df}
else:
excel_obj = pd.read_excel(io.BytesIO(data), dtype=str, sheet_name=None)
workbook_sheets = {}
sheet_items = list((excel_obj or {}).items())
if len(sheet_items) > max_excel_sheets:
clipped_sheets = True
sheet_items = sheet_items[:max_excel_sheets]
for sheet_name, sheet_df in sheet_items:
sdf = sheet_df
if int(len(sdf.columns)) > max_columns:
sdf = sdf.iloc[:, :max_columns]
sdf, _ = _clip_tabular_cells(sdf, max_cell_chars=max_cell_chars)
workbook_sheets[str(sheet_name or f"Sheet{len(workbook_sheets)+1}")] = sdf
mime = "text/csv" if ext == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
meta = AttachmentAssetStore().save_bytes(data, filename=name, mime=mime)
table_meta = (
save_dataframe(attachment_id=str(meta.attachment_id), name=name, df=workbook_sheets.get("Sheet1", pd.DataFrame()))
if ext == "csv"
else save_workbook(attachment_id=str(meta.attachment_id), name=name, sheets=workbook_sheets)
)
attachments.append(
{
"type": "tabular_ref",
"name": str(name),
"attachment_id": str(meta.attachment_id),
"table_id": str(table_meta.get("table_id") or ""),
"rows": int(table_meta.get("rows") or 0),
"cols": int(table_meta.get("cols") or 0),
"columns": list(table_meta.get("columns") or []),
"sheets": list(table_meta.get("sheets") or []),
}
)
if clipped_sheets:
attachments.append(
{
"type": "text",
"name": f"{name}.sheet-limit",
"content": (
f"Excel workbook contains too many sheets; only the first {max_excel_sheets} sheets were stored "
"for tool-mode querying. Adjust max_excel_sheets if full coverage is required."
),
}
)
except Exception as e:
attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing excel/csv: {e}")})
elif ext == "docx":
try:
doc = Document(io.BytesIO(data))
text = "\n".join([p.text for p in doc.paragraphs])
attachments.append({"type": "text", "name": name, "content": _truncate_text(text)})
except Exception as e:
attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing docx: {e}")})
elif ext == "pdf":
try:
reader = PdfReader(io.BytesIO(data))
attachments.append({"type": "text", "name": name, "content": _pdf_summary_text(reader)})
except Exception as e:
attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing pdf: {e}")})
elif ext in ("txt", "py", "js", "html", "css", "md", "json", "yaml", "yml", "sh", "log"):
text = _decode_text_bytes(data)
if ext == "html":
text = _clean_html_to_text(text)
attachments.append({"type": "text", "name": name, "content": _truncate_text(text)})
elif ext == "zip":
attachments.extend(process_zip(data, _depth=_zip_depth))
return attachments
__all__ = ["process_zip", "process_file_data", "clear_attachment_limits_cache"]

View file

@ -0,0 +1,66 @@
from __future__ import annotations
import json
from typing import Any
from oclaw.platform.persistence.sqlite_store import ChatMessage, SqliteStore
def export_session_json(store: SqliteStore, session_id: str, message_limit: int = 50_000) -> str:
title = ""
s = store.get_session(session_id)
if s:
title = s.title
msgs = store.get_messages(session_id=session_id, limit=message_limit)
payload: dict[str, Any] = {
"session_id": session_id,
"title": title,
"messages": [_message_to_dict(m) for m in msgs],
}
return json.dumps(payload, ensure_ascii=False, indent=2)
def export_session_markdown(store: SqliteStore, session_id: str, message_limit: int = 50_000) -> str:
s = store.get_session(session_id)
title = (s.title if s else "") or session_id[:12]
lines = [f"# {title}", "", f"Session: `{session_id}`", ""]
for m in store.get_messages(session_id=session_id, limit=message_limit):
lines.extend(_message_to_markdown_lines(m))
lines.append("")
return "\n".join(lines).rstrip() + "\n"
def _message_to_dict(m: ChatMessage) -> dict[str, Any]:
return {
"id": m.id,
"role": m.role,
"content": m.content,
"tool_calls": m.tool_calls,
"attachments": m.attachments,
"timestamp": m.timestamp,
}
def _message_to_markdown_lines(m: ChatMessage) -> list[str]:
role = m.role.upper()
ts = m.timestamp or ""
head = f"## {role} · `{m.id}` · {ts}"
body = m.content or ""
if m.role == "tool" and m.tool_calls:
head = f"## TOOL · `{m.id}` · {ts}"
lines = [head, ""]
if body.strip():
lines.append(body)
lines.append("")
if m.tool_calls and m.role != "tool":
lines.append("```json")
lines.append(m.tool_calls)
lines.append("```")
elif m.tool_calls and m.role == "tool":
lines.append(f"_meta_: `{m.tool_calls}`")
if m.attachments:
lines.append("_attachments_: see JSON export for full data.")
return lines
__all__ = ["export_session_json", "export_session_markdown"]

View file

@ -0,0 +1,659 @@
from __future__ import annotations
import hashlib
import json
import os
import re
import sqlite3
import subprocess
import time
from pathlib import Path
from typing import Any
import pandas as pd
from oclaw.platform.config.paths import PROJECT_ROOT
from oclaw.platform.config.paths import attachments_dir
def _tabular_root() -> Path:
p = (attachments_dir() / "tabular").resolve()
p.mkdir(parents=True, exist_ok=True)
return p
def _table_id(attachment_id: str, name: str) -> str:
raw = f"{attachment_id}:{name}".encode("utf-8", errors="ignore")
return hashlib.sha256(raw).hexdigest()
def _db_path(table_id: str) -> Path:
return _tabular_root() / f"{table_id}.sqlite"
def _meta_path(table_id: str) -> Path:
return _tabular_root() / f"{table_id}.meta.json"
def _normalize_columns(raw_cols: list[Any]) -> tuple[list[str], list[dict[str, Any]]]:
out: list[str] = []
mapping: list[dict[str, Any]] = []
seen: dict[str, int] = {}
for idx, raw in enumerate(raw_cols):
original = "" if raw is None else str(raw)
base = original.strip()
if not base or base.lower().startswith("unnamed:"):
base = f"col_{idx + 1}"
base = re.sub(r"\s+", "_", base)
base = re.sub(r"\W+", "_", base, flags=re.UNICODE).strip("_")
if not base:
base = f"col_{idx + 1}"
n = int(seen.get(base, 0)) + 1
seen[base] = n
name = base if n == 1 else f"{base}__{n}"
out.append(name)
mapping.append({"index": idx, "original": original, "normalized": name})
return out, mapping
def _prepare_writable_df(df: pd.DataFrame) -> tuple[pd.DataFrame, list[str], list[dict[str, Any]]]:
raw_cols = [x for x in list(df.columns)]
cols, col_map = _normalize_columns(raw_cols)
writable = df.copy()
writable.columns = cols
for c in cols:
writable[c] = writable[c].map(lambda x: "" if x is None else str(x))
return writable, cols, col_map
def _safe_sheet_table_name(sheet_name: str, idx: int) -> str:
base = re.sub(r"[^0-9a-zA-Z_]+", "_", str(sheet_name or "").strip()).strip("_").lower()
if not base:
base = f"sheet_{idx + 1}"
return f"sheet_{idx + 1}_{base}"
def _mcp_sqlite_command() -> list[str]:
raw = str(os.getenv("AIA_MCP_SQLITE_COMMAND") or "").strip()
if not raw:
return []
return [x for x in raw.split(" ") if x]
def _tabular_use_mcp() -> bool:
raw = str(os.getenv("AIA_TABULAR_USE_MCP") or "").strip().lower()
return raw in {"1", "true", "yes", "on"}
def _tabular_sql_timeout_ms() -> int:
def _clamp_timeout(raw: Any, default: int = 8_000) -> int:
try:
n = int(raw)
except Exception:
return default
return max(100, min(n, 120_000))
raw = str(os.getenv("AIA_TABULAR_SQL_TIMEOUT_MS") or "").strip()
if raw:
return _clamp_timeout(raw)
cfg_path_raw = str(os.getenv("AIA_OCLAW_CONFIG_PATH") or "").strip()
cfg_path = Path(cfg_path_raw).expanduser() if cfg_path_raw else (Path(PROJECT_ROOT) / "oclaw" / "oclaw.json").resolve()
if not cfg_path.is_absolute():
cfg_path = (Path(PROJECT_ROOT) / cfg_path).resolve()
try:
obj = json.loads(cfg_path.read_text(encoding="utf-8"))
except Exception:
return 8_000
tabular_cfg = (
(((obj or {}).get("plugins") or {}).get("entries") or {}).get("memory-wiki", {}).get("auto", {}).get("attachments", {}).get("tabular", {}) or {}
)
if not isinstance(tabular_cfg, dict):
return 8_000
return _clamp_timeout(tabular_cfg.get("sql_timeout_ms"), 8_000)
def _extract_rows_from_mcp_result(res: dict[str, Any]) -> list[dict[str, Any]] | None:
if not isinstance(res, dict) or not bool(res.get("ok")):
return None
data = res.get("data") if isinstance(res.get("data"), dict) else res.get("result")
if isinstance(data, dict):
direct_rows = data.get("rows")
if isinstance(direct_rows, list):
out: list[dict[str, Any]] = []
for r in direct_rows:
if isinstance(r, dict):
out.append({str(k): r[k] for k in r.keys()})
return out
content = data.get("content")
if isinstance(content, list):
for it in content:
if not isinstance(it, dict):
continue
txt = str(it.get("text") or "").strip()
if not txt:
continue
try:
obj = json.loads(txt)
except Exception:
continue
if isinstance(obj, dict) and isinstance(obj.get("rows"), list):
rows = obj.get("rows")
out: list[dict[str, Any]] = []
for r in rows:
if isinstance(r, dict):
out.append({str(k): r[k] for k in r.keys()})
return out
return None
def _run_select_via_mcp(*, db_path: Path, sql: str, params: list[Any]) -> list[dict[str, Any]] | None:
if not _tabular_use_mcp():
return None
cmd = _mcp_sqlite_command()
if not cmd:
return None
def _call_once(tool_name: str, arguments: dict[str, Any]) -> dict[str, Any]:
try:
p = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
)
except Exception:
return {"ok": False}
try:
if p.stdin is None or p.stdout is None:
return {"ok": False}
init_req = {
"jsonrpc": "2.0",
"id": 1,
"method": "initialize",
"params": {
"protocolVersion": "2024-11-05",
"capabilities": {},
"clientInfo": {"name": "oclaw-tabular", "version": "0.1.0"},
},
}
p.stdin.write(json.dumps(init_req, ensure_ascii=False) + "\n")
p.stdin.flush()
_ = p.stdout.readline()
p.stdin.write(json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, ensure_ascii=False) + "\n")
p.stdin.flush()
call_req = {
"jsonrpc": "2.0",
"id": 2,
"method": "tools/call",
"params": {"name": str(tool_name), "arguments": arguments},
}
p.stdin.write(json.dumps(call_req, ensure_ascii=False) + "\n")
p.stdin.flush()
line = p.stdout.readline()
if not line:
return {"ok": False}
obj = json.loads(line)
if not isinstance(obj, dict):
return {"ok": False}
if isinstance(obj.get("error"), dict):
return {"ok": False}
return {"ok": True, "result": obj.get("result")}
except Exception:
return {"ok": False}
finally:
try:
p.terminate()
except Exception:
pass
# Probe several common sqlite MCP tool names/argument shapes.
attempts: list[tuple[str, dict[str, Any]]] = [
("query", {"database": str(db_path), "sql": sql, "params": params}),
("query", {"db_path": str(db_path), "sql": sql, "params": params}),
("execute_sql", {"database": str(db_path), "sql": sql, "params": params}),
("read_query", {"database": str(db_path), "query": sql, "params": params}),
("sqlite_query", {"database": str(db_path), "sql": sql, "params": params}),
]
for tool_name, args in attempts:
res = _call_once(tool_name=tool_name, arguments=args)
rows = _extract_rows_from_mcp_result(res if isinstance(res, dict) else {})
if rows is not None:
return rows
return None
def _run_select_sql(
*,
db_path: Path,
sql: str,
params: list[Any],
timeout_ms: int | None = None,
) -> tuple[list[dict[str, Any]], str]:
rows = _run_select_via_mcp(db_path=db_path, sql=sql, params=params)
if rows is not None:
return rows, "mcp_sqlite"
out_rows: list[dict[str, Any]] = []
timeout_ms_eff = max(100, int(timeout_ms or _tabular_sql_timeout_ms()))
deadline = time.perf_counter() + (float(timeout_ms_eff) / 1000.0)
timed_out = {"hit": False}
with sqlite3.connect(str(db_path)) as conn:
conn.row_factory = sqlite3.Row
def _progress_cb() -> int:
if time.perf_counter() > deadline:
timed_out["hit"] = True
return 1
return 0
conn.set_progress_handler(_progress_cb, 1000)
try:
for r in conn.execute(sql, params).fetchall():
out_rows.append({k: r[k] for k in r.keys()})
except sqlite3.OperationalError as exc:
if timed_out["hit"] and "interrupted" in str(exc).lower():
raise TimeoutError(f"tabular_sql_timeout_ms={timeout_ms_eff}") from exc
raise
finally:
conn.set_progress_handler(None, 0)
return out_rows, "builtin_sqlite"
def save_dataframe(*, attachment_id: str, name: str, df: pd.DataFrame) -> dict[str, Any]:
return save_workbook(attachment_id=attachment_id, name=name, sheets={"Sheet1": df})
def save_workbook(*, attachment_id: str, name: str, sheets: dict[str, pd.DataFrame]) -> dict[str, Any]:
tid = _table_id(attachment_id=attachment_id, name=name)
db = _db_path(tid)
meta = _meta_path(tid)
sheet_rows: list[dict[str, Any]] = []
default_table = "rows_data"
default_columns: list[str] = []
default_rows = 0
default_cols = 0
with sqlite3.connect(str(db)) as conn:
for idx, (sheet_name, df) in enumerate((sheets or {}).items()):
writable, cols, col_map = _prepare_writable_df(df)
table_name = _safe_sheet_table_name(str(sheet_name or f"Sheet{idx + 1}"), idx)
writable.to_sql(table_name, conn, if_exists="replace", index=False)
if idx == 0:
# backward compatibility: old queries use rows_data
writable.to_sql("rows_data", conn, if_exists="replace", index=False)
default_table = table_name
default_columns = list(cols)
default_rows = int(len(writable.index))
default_cols = int(len(cols))
sheet_rows.append(
{
"sheet_name": str(sheet_name or f"Sheet{idx + 1}"),
"table_name": table_name,
"rows": int(len(writable.index)),
"cols": int(len(cols)),
"columns": list(cols),
"column_map": col_map,
}
)
payload = {
"table_id": tid,
"attachment_id": str(attachment_id or ""),
"name": str(name or ""),
"rows": int(default_rows),
"cols": int(default_cols),
"columns": default_columns,
"default_table": default_table,
"sheets": sheet_rows,
"db_path": str(db),
}
meta.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
return payload
def get_table_meta(table_id: str) -> dict[str, Any] | None:
p = _meta_path(str(table_id or "").strip())
if not p.exists():
return None
try:
obj = json.loads(p.read_text(encoding="utf-8"))
return obj if isinstance(obj, dict) else None
except Exception:
return None
def _resolve_sheet_table(meta: dict[str, Any], sheet: str | None) -> str:
default_table = str(meta.get("default_table") or "rows_data")
if not sheet:
return default_table
target = str(sheet).strip()
if not target:
return default_table
sheets = meta.get("sheets") if isinstance(meta.get("sheets"), list) else []
for row in sheets:
if not isinstance(row, dict):
continue
if str(row.get("sheet_name") or "") == target or str(row.get("table_name") or "") == target:
return str(row.get("table_name") or default_table)
return ""
def _columns_for_sheet(meta: dict[str, Any], table_name: str) -> list[str]:
sheets = meta.get("sheets") if isinstance(meta.get("sheets"), list) else []
for row in sheets:
if not isinstance(row, dict):
continue
if str(row.get("table_name") or "") == str(table_name):
cols = row.get("columns")
if isinstance(cols, list):
return [str(x) for x in cols]
cols = meta.get("columns")
if isinstance(cols, list):
return [str(x) for x in cols]
return []
def query_table(
*,
table_id: str,
columns: list[str] | None = None,
limit: int = 50,
offset: int = 0,
where_contains: dict[str, str] | None = None,
sheet: str | None = None,
) -> dict[str, Any]:
meta = get_table_meta(table_id)
if not isinstance(meta, dict):
return {"ok": False, "error": "table_not_found"}
db = Path(str(meta.get("db_path") or ""))
if not db.exists():
return {"ok": False, "error": "table_store_missing"}
table_name = _resolve_sheet_table(meta, sheet)
if not table_name:
return {"ok": False, "error": "sheet_not_found"}
known_cols = _columns_for_sheet(meta, table_name)
use_cols = [c for c in (columns or known_cols) if c in known_cols]
if not use_cols:
use_cols = known_cols[: min(10, len(known_cols))]
lim = max(1, min(int(limit or 50), 200))
off = max(0, int(offset or 0))
select_sql = ", ".join([f'"{c}"' for c in use_cols]) if use_cols else "*"
where_sql = ""
params: list[Any] = []
wc = where_contains if isinstance(where_contains, dict) else {}
wc_col = str(wc.get("column") or "").strip()
wc_kw = str(wc.get("keyword") or "").strip()
if wc_col and wc_kw and wc_col in known_cols:
where_sql = f' WHERE "{wc_col}" LIKE ? '
params.append(f"%{wc_kw}%")
sql = f'SELECT {select_sql} FROM "{table_name}"{where_sql} LIMIT ? OFFSET ?'
params.extend([lim, off])
out_rows: list[dict[str, Any]] = []
total = 0
out_rows, engine = _run_select_sql(db_path=db, sql=sql, params=params)
with sqlite3.connect(str(db)) as conn:
total = int(conn.execute(f'SELECT COUNT(1) AS n FROM "{table_name}"').fetchone()[0] or 0)
return {
"ok": True,
"table_id": str(table_id),
"name": str(meta.get("name") or ""),
"sheet": str(sheet or ""),
"table_name": table_name,
"rows_total": total,
"columns": use_cols,
"rows": out_rows,
"limit": lim,
"offset": off,
"engine": engine,
}
def aggregate_table(
*,
table_id: str,
metric: str,
target_column: str | None = None,
group_by: str | None = None,
where_contains: dict[str, str] | None = None,
top_n: int = 20,
sheet: str | None = None,
) -> dict[str, Any]:
meta = get_table_meta(table_id)
if not isinstance(meta, dict):
return {"ok": False, "error": "table_not_found"}
db = Path(str(meta.get("db_path") or ""))
if not db.exists():
return {"ok": False, "error": "table_store_missing"}
table_name = _resolve_sheet_table(meta, sheet)
if not table_name:
return {"ok": False, "error": "sheet_not_found"}
known_cols = _columns_for_sheet(meta, table_name)
mt = str(metric or "").strip().lower()
if mt not in {"count", "sum", "avg"}:
return {"ok": False, "error": "invalid_metric"}
grp = str(group_by or "").strip()
if grp and grp not in known_cols:
return {"ok": False, "error": "invalid_group_by"}
tgt = str(target_column or "").strip()
if mt in {"sum", "avg"} and tgt not in known_cols:
return {"ok": False, "error": "target_column_required"}
where_sql = ""
params: list[Any] = []
wc = where_contains if isinstance(where_contains, dict) else {}
wc_col = str(wc.get("column") or "").strip()
wc_kw = str(wc.get("keyword") or "").strip()
if wc_col and wc_kw and wc_col in known_cols:
where_sql = f' WHERE "{wc_col}" LIKE ? '
params.append(f"%{wc_kw}%")
lim = max(1, min(int(top_n or 20), 200))
if mt == "count":
metric_sql = "COUNT(1)"
else:
metric_sql = f'{"SUM" if mt == "sum" else "AVG"}(CAST(NULLIF("{tgt}", \'\') AS REAL))'
if grp:
sql = (
f'SELECT "{grp}" AS group_key, {metric_sql} AS metric_value '
f'FROM "{table_name}"{where_sql} GROUP BY "{grp}" '
"ORDER BY metric_value DESC LIMIT ?"
)
params.append(lim)
raw_rows, engine = _run_select_sql(db_path=db, sql=sql, params=params)
rows: list[dict[str, Any]] = []
for r in raw_rows:
rows.append(
{
"group": r.get("group_key"),
"value": float(r.get("metric_value") or 0.0),
}
)
return {
"ok": True,
"table_id": str(table_id),
"sheet": str(sheet or ""),
"table_name": table_name,
"metric": mt,
"target_column": tgt if mt in {"sum", "avg"} else None,
"group_by": grp,
"rows": rows,
"top_n": lim,
"engine": engine,
}
sql = f'SELECT {metric_sql} AS metric_value FROM "{table_name}"{where_sql} '
raw_rows, engine = _run_select_sql(db_path=db, sql=sql, params=params)
row = raw_rows[0] if raw_rows else {}
return {
"ok": True,
"table_id": str(table_id),
"sheet": str(sheet or ""),
"table_name": table_name,
"metric": mt,
"target_column": tgt if mt in {"sum", "avg"} else None,
"group_by": "",
"value": float((row.get("metric_value") if isinstance(row, dict) else 0.0) or 0.0),
"engine": engine,
}
def run_table_sql(
*,
table_id: str,
sql: str,
limit: int = 200,
sheet: str | None = None,
) -> dict[str, Any]:
meta = get_table_meta(table_id)
if not isinstance(meta, dict):
return {"ok": False, "error": "table_not_found"}
db = Path(str(meta.get("db_path") or ""))
if not db.exists():
return {"ok": False, "error": "table_store_missing"}
table_name = _resolve_sheet_table(meta, sheet)
if not table_name:
return {"ok": False, "error": "sheet_not_found"}
raw = str(sql or "").strip()
if not raw:
return {"ok": False, "error": "sql_required"}
lowered = raw.lower()
deny = ("insert ", "update ", "delete ", "drop ", "alter ", "attach ", "detach ", "pragma ", "create ")
if any(k in lowered for k in deny):
return {"ok": False, "error": "sql_not_readonly"}
if ";" in raw:
return {"ok": False, "error": "sql_multiple_statements_forbidden"}
if not (lowered.startswith("select ") or lowered.startswith("with ")):
return {"ok": False, "error": "sql_must_start_with_select_or_with"}
lim = max(1, min(int(limit or 200), 500))
auto_limited = " limit " not in lowered
sql_targeted = re.sub(r"(?i)\brows_data\b", f'"{table_name}"', raw)
safe_sql = sql_targeted if not auto_limited else f"{sql_targeted}\nLIMIT {lim}"
timeout_ms = _tabular_sql_timeout_ms()
try:
rows, engine = _run_select_sql(db_path=db, sql=safe_sql, params=[], timeout_ms=timeout_ms)
except TimeoutError:
return {
"ok": False,
"error": "sql_timeout",
"table_id": str(table_id),
"name": str(meta.get("name") or ""),
"sheet": str(sheet or ""),
"table_name": table_name,
"input_sql": raw,
"executed_sql": safe_sql,
"sql_guard": {
"readonly_enforced": True,
"multi_statement_forbidden": True,
"auto_limit_applied": bool(auto_limited),
"result_row_cap": lim,
"timeout_ms": int(timeout_ms),
"timeout_hit": True,
},
}
out_rows = rows[:lim]
return {
"ok": True,
"table_id": str(table_id),
"name": str(meta.get("name") or ""),
"sheet": str(sheet or ""),
"table_name": table_name,
"input_sql": raw,
"executed_sql": safe_sql,
"sql_guard": {
"readonly_enforced": True,
"multi_statement_forbidden": True,
"auto_limit_applied": bool(auto_limited),
"result_row_cap": lim,
"timeout_ms": int(timeout_ms),
"timeout_hit": False,
},
"rows": out_rows,
"rows_returned": int(len(out_rows)),
"limit": lim,
"engine": engine,
}
def analyze_table_full_scan(
*,
table_id: str,
columns: list[str] | None = None,
sheet: str | None = None,
top_values_limit: int = 3,
) -> dict[str, Any]:
meta = get_table_meta(table_id)
if not isinstance(meta, dict):
return {"ok": False, "error": "table_not_found"}
db = Path(str(meta.get("db_path") or ""))
if not db.exists():
return {"ok": False, "error": "table_store_missing"}
table_name = _resolve_sheet_table(meta, sheet)
if not table_name:
return {"ok": False, "error": "sheet_not_found"}
known_cols = _columns_for_sheet(meta, table_name)
if not known_cols:
return {"ok": False, "error": "table_columns_missing"}
use_cols = [c for c in (columns or known_cols) if c in known_cols]
if not use_cols:
return {"ok": False, "error": "columns_not_found"}
top_lim = max(0, min(int(top_values_limit or 0), 10))
started = time.perf_counter()
with sqlite3.connect(str(db)) as conn:
total_rows = int(conn.execute(f'SELECT COUNT(1) FROM "{table_name}"').fetchone()[0] or 0)
col_stats: list[dict[str, Any]] = []
for col in use_cols:
row = conn.execute(
(
f'SELECT '
f'SUM(CASE WHEN NULLIF(TRIM("{col}"), \'\') IS NOT NULL THEN 1 ELSE 0 END) AS non_empty_rows, '
f'COUNT(DISTINCT NULLIF(TRIM("{col}"), \'\')) AS distinct_values '
f'FROM "{table_name}"'
)
).fetchone()
non_empty_rows = int((row[0] if row else 0) or 0)
distinct_values = int((row[1] if row else 0) or 0)
item: dict[str, Any] = {
"column": col,
"non_empty_rows": non_empty_rows,
"empty_rows": max(0, int(total_rows - non_empty_rows)),
"distinct_values": distinct_values,
"coverage_ratio": (float(non_empty_rows) / float(total_rows)) if total_rows > 0 else 0.0,
}
if top_lim > 0:
tops = conn.execute(
(
f'SELECT "{col}" AS val, COUNT(1) AS n '
f'FROM "{table_name}" '
f'WHERE NULLIF(TRIM("{col}"), \'\') IS NOT NULL '
f'GROUP BY "{col}" ORDER BY n DESC LIMIT ?'
),
[top_lim],
).fetchall()
item["top_values"] = [{"value": str(r[0]), "count": int(r[1] or 0)} for r in tops]
col_stats.append(item)
elapsed_ms = int((time.perf_counter() - started) * 1000)
return {
"ok": True,
"table_id": str(table_id),
"name": str(meta.get("name") or ""),
"sheet": str(sheet or ""),
"table_name": table_name,
"rows_scanned": int(total_rows),
"columns_analyzed": list(use_cols),
"column_stats": col_stats,
"scan_audit": {
"full_scan": True,
"rows_scanned": int(total_rows),
"columns_count": int(len(use_cols)),
"elapsed_ms": int(elapsed_ms),
},
"engine": "builtin_sqlite_fullscan",
}
__all__ = [
"save_dataframe",
"save_workbook",
"get_table_meta",
"query_table",
"aggregate_table",
"run_table_sql",
"analyze_table_full_scan",
]

View file

@ -0,0 +1,85 @@
"""Locate and optionally launch the ``vendor/cc-mini`` git submodule.
Upstream lives at https://github.com/e10nMa2k/cc-mini — keep it as a submodule
rather than copying sources into this tree. At the time of integration the
submodule repository did not ship a root ``LICENSE`` file; treat compliance as
upstream-owned until one is published.
"""
from __future__ import annotations
import os
import subprocess
import sys
from pathlib import Path
from typing import Any
from oclaw.platform.config.paths import PROJECT_ROOT
def cc_mini_root() -> Path:
"""Path to the ``cc-mini`` checkout (``<repo>/vendor/cc-mini``)."""
return (PROJECT_ROOT / "vendor" / "cc-mini").resolve()
def cc_mini_src_dir() -> Path:
"""Directory that must be on ``PYTHONPATH`` for ``python -m tui.app``."""
return (cc_mini_root() / "src").resolve()
def cc_mini_available() -> bool:
"""True when the submodule tree looks present (``src/core/engine.py``)."""
return (cc_mini_src_dir() / "core" / "engine.py").is_file()
def cc_mini_pythonpath_env(base: dict[str, str] | None = None) -> dict[str, str]:
"""Return env with ``vendor/cc-mini/src`` prepended to ``PYTHONPATH``."""
env = dict(base or os.environ)
src = str(cc_mini_src_dir())
prev = (env.get("PYTHONPATH") or "").strip()
env["PYTHONPATH"] = src + (os.pathsep + prev if prev else "")
return env
def prepend_cc_mini_src_to_sys_path() -> bool:
"""Insert ``vendor/cc-mini/src`` at the front of ``sys.path`` if available.
Prefer subprocess + :func:`cc_mini_pythonpath_env` when isolation matters;
importing ``core`` / ``features`` from cc-mini can clash with similarly
named packages elsewhere on ``sys.path``.
"""
src = cc_mini_src_dir()
if not cc_mini_available():
return False
s = str(src)
if s not in sys.path:
sys.path.insert(0, s)
return True
def run_cc_mini_cli(
argv: list[str] | None = None,
*,
cwd: Path | None = None,
env: dict[str, str] | None = None,
**kwargs: Any,
) -> subprocess.CompletedProcess[str]:
"""Run cc-mini's interactive entry (``python -m tui.app``) like the ``cc-mini`` console script.
Requires ``cc_mini_available()``; raises ``FileNotFoundError`` if the checkout is missing.
"""
root = cc_mini_root()
if not cc_mini_available():
raise FileNotFoundError(
f"cc-mini not found under {root}; run: git submodule update --init vendor/cc-mini"
)
merged = cc_mini_pythonpath_env(env if env is not None else os.environ)
cmd = [sys.executable, "-m", "tui.app", *(argv or [])]
return subprocess.run(
cmd,
cwd=str(cwd or root),
env=merged,
check=False,
text=True,
**kwargs,
)

View file

@ -0,0 +1,64 @@
from __future__ import annotations
from oclaw.platform.persistence.sqlite_store import SqliteStore
class WeComClient:
"""WeCom AI bot (websocket / bot_api only)."""
def __init__(self, store: SqliteStore):
self.store = store
def _bot_cfg(self) -> tuple[str, str]:
bot_id = str(self.store.get_setting("wecom_bot_id") or "").strip()
bot_secret = str(self.store.get_secret("wecom_bot_secret") or self.store.get_setting("wecom_bot_secret") or "").strip()
if bot_id and bot_secret:
return bot_id, bot_secret
if bot_id:
row = self.store.find_user_by_channel_account(channel="wecom", account_id=bot_id)
if row:
tid = str(row["tenant_id"])
uid = str(row["user_id"])
sk = f"wecom:bot_secret:{tid}:{uid}:{bot_id}"
sec = str(self.store.get_secret(sk) or "").strip()
if sec:
return bot_id, sec
try:
tenants = self.store.list_tenants(limit=1)
tenant_id = str((tenants[0] or {}).get("id") or "") if tenants else ""
if not tenant_id:
return bot_id, bot_secret
admin = self.store.get_user_by_username(tenant_id=tenant_id, username="administrator")
user_id = str((admin or {}).get("id") or "")
if not user_id:
return bot_id, bot_secret
accounts = self.store.list_user_channel_accounts(
tenant_id=tenant_id, user_id=user_id, channel="wecom", include_inactive=False
)
if not accounts:
return bot_id, bot_secret
aid = str(accounts[0].get("account_id") or "").strip()
if not aid:
return bot_id, bot_secret
scoped_key = f"wecom:bot_secret:{tenant_id}:{user_id}:{aid}"
sec = str(self.store.get_secret(scoped_key) or "").strip()
if aid and sec:
return aid, sec
except Exception:
return bot_id, bot_secret
return bot_id, bot_secret
def mode_for_account(self, account_id: str) -> str: # noqa: ARG002
return "bot_api"
def mode(self) -> str:
return "bot_api"
def get_bot_credentials(self) -> tuple[str, str]:
bot_id, bot_secret = self._bot_cfg()
if not bot_id or not bot_secret:
raise RuntimeError("missing wecom bot config: wecom_bot_id / wecom_bot_secret")
return bot_id, bot_secret
__all__ = ["WeComClient"]

View file

@ -0,0 +1,27 @@
MIT License
Copyright (c) 2025 Peter Steinberger (OpenClaw)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
---
The Python modules `tool_call_id.py` and `replay_policy.py` in this directory
implement replay / tool-call-id handling inspired by OpenClaw's
`src/agents/tool-call-id.ts` and `src/agents/transcript-policy.ts`.

1
platform/llm/__init__.py Normal file
View file

@ -0,0 +1 @@
# oclaw.platform.llm package

View file

@ -0,0 +1,69 @@
from __future__ import annotations
"""Compatibility re-export layer.
The codebase imports `ChatModel/OpenAIChatModel/...` from this module.
Implementations live under `src.platform.llm.transports.*`.
"""
import os
from oclaw.platform.llm.tool_schema import shrink_openai_tools_payload_for_api
from oclaw.platform.llm.tool_schema import openai_tools_json_byte_length as _openai_tools_json_byte_length
from oclaw.platform.llm.transports.base import ChatModel, LLMResponse, LLMToolCall, normalize_image_b64_payload
from oclaw.platform.llm.transports.google_gemini_sse import GoogleGeminiChatModel
from oclaw.platform.llm.transports.openai_chat_completions import (
OpenAIChatModel,
_likely_gemini_openai_compat_base_url,
_model_id_suggests_gemini,
)
from oclaw.platform.llm.transports.simple import RuleBasedChatModel, StaticTextChatModel
# Backward-compat alias used by `src/chat/agent.py`.
_normalize_image_b64_payload = normalize_image_b64_payload
def gemini_openai_compat_client(model: ChatModel) -> bool:
if not isinstance(model, OpenAIChatModel):
return False
return _likely_gemini_openai_compat_base_url(getattr(model, "base_url", None)) or _model_id_suggests_gemini(
getattr(model, "model", None)
)
def build_default_model() -> ChatModel:
mode = (os.getenv("AIA_ASSISTANT_MODE") or "").strip().lower()
if mode == "rule":
return RuleBasedChatModel()
if mode == "ollama":
try:
from oclaw.agents.factory import DEFAULT_OLLAMA_BASE_URL, DEFAULT_OLLAMA_MODEL
return OpenAIChatModel(model=DEFAULT_OLLAMA_MODEL, api_key="ollama", base_url=DEFAULT_OLLAMA_BASE_URL)
except Exception:
return RuleBasedChatModel()
if mode == "openai" or mode == "":
try:
return OpenAIChatModel()
except Exception:
return RuleBasedChatModel()
# Provider selection is normally handled via LLM profiles in `src/agents/factory.py`.
return RuleBasedChatModel()
__all__ = [
"ChatModel",
"LLMResponse",
"LLMToolCall",
"OpenAIChatModel",
"GoogleGeminiChatModel",
"RuleBasedChatModel",
"StaticTextChatModel",
"build_default_model",
"gemini_openai_compat_client",
"normalize_image_b64_payload",
"_normalize_image_b64_payload",
"shrink_openai_tools_payload_for_api",
"_openai_tools_json_byte_length",
]

View file

@ -0,0 +1,685 @@
from __future__ import annotations
import base64
import io
import os
import time
from typing import Any
import httpx
from PIL import Image
from oclaw.prompts import render_prompt
def _summarize_data_url(url: str) -> dict[str, Any]:
"""
Summarize data URLs without logging raw base64 payload.
"""
s = str(url or "")
if not s.startswith("data:"):
return {"kind": "url", "prefix": s[:30]}
mime = "image/jpeg"
try:
# data:{mime};base64,{b64}
head = s.split(";", 1)[0] # data:image/xxx
if head.startswith("data:"):
mime = head[len("data:") :] or mime
except Exception:
pass
b64_part = s.split(",", 1)[1] if "," in s else ""
has_b64 = ";base64," in s
return {"kind": "data_url", "mime": mime, "has_base64": has_b64, "b64_len": len(b64_part)}
def _summarize_dashscope_messages(messages: list[dict[str, Any]], *, max_items: int = 3) -> dict[str, Any]:
"""
Debug-only summary of messages[0].content structure.
"""
if not messages or not isinstance(messages, list):
return {"content_items": []}
first = messages[0] if messages else {}
if not isinstance(first, dict):
return {"content_items": []}
content = first.get("content")
if not isinstance(content, list):
return {"content_items": []}
out: list[dict[str, Any]] = []
for it in content[:max_items]:
if not isinstance(it, dict):
out.append({"item_type": "non_dict"})
continue
item: dict[str, Any] = {"keys": sorted([str(k) for k in it.keys()])}
if "type" in it:
item["type"] = it.get("type")
if "image" in it:
item["image"] = _summarize_data_url(str(it.get("image") or ""))
if "image_url" in it:
iu = it.get("image_url")
if isinstance(iu, dict) and "url" in iu:
item["image_url"] = _summarize_data_url(str(iu.get("url") or ""))
else:
item["image_url"] = {"kind": "image_url_non_dict"}
if "text" in it:
item["text_len"] = len(str(it.get("text") or ""))
out.append(item)
return {"content_items": out}
def _join_url(base: str, path: str) -> str:
b = (base or "").rstrip("/")
p = (path or "").lstrip("/")
return f"{b}/{p}"
def _is_data_url(s: str) -> bool:
return s.startswith("data:") and ";base64," in s
def _compress_data_url_image(
data_url: str,
*,
max_side: int = 1600,
max_bytes: int = 2 * 1024 * 1024,
jpeg_quality: int = 82,
) -> str:
"""Best-effort compress image data URLs to reduce transport failures."""
if not _is_data_url(data_url):
return data_url
try:
header, b64 = data_url.split(";base64,", 1)
mime = header.replace("data:", "", 1).strip() or "image/jpeg"
raw = base64.b64decode(b64.encode("ascii"))
if len(raw) <= max_bytes:
return data_url
with Image.open(io.BytesIO(raw)) as im:
im = im.convert("RGB")
w, h = im.size
longest = max(w, h)
if longest > max_side:
scale = max_side / float(longest)
nw, nh = max(1, int(w * scale)), max(1, int(h * scale))
im = im.resize((nw, nh), Image.Resampling.LANCZOS)
out = io.BytesIO()
# Use JPEG for broad compatibility and smaller payload.
im.save(out, format="JPEG", quality=max(30, min(95, int(jpeg_quality))), optimize=True)
blob = out.getvalue()
if not blob:
return data_url
b64_new = base64.b64encode(blob).decode("ascii")
return f"data:{'image/jpeg' if mime.startswith('image/') else mime};base64,{b64_new}"
except Exception:
return data_url
def _post_with_retry(
client: httpx.Client,
*,
url: str,
headers: dict[str, str],
payload: dict[str, Any],
retries: int = 3,
backoff_sec: float = 0.8,
) -> httpx.Response:
last_exc: Exception | None = None
for attempt in range(1, max(1, retries) + 1):
try:
return client.post(url, headers=headers, json=payload)
except (httpx.ReadError, httpx.ConnectError, httpx.TimeoutException) as e:
last_exc = e
if attempt >= retries:
break
time.sleep(backoff_sec * (2 ** (attempt - 1)))
assert last_exc is not None
raise last_exc
def _extract_text_and_images(resp_json: dict[str, Any]) -> tuple[str, list[str]]:
text_parts: list[str] = []
images: list[str] = []
choices = resp_json.get("choices")
if isinstance(choices, list) and choices:
msg = choices[0].get("message") if isinstance(choices[0], dict) else None
if isinstance(msg, dict):
c = msg.get("content")
if isinstance(c, str):
text_parts.append(c)
elif isinstance(c, list):
for it in c:
if not isinstance(it, dict):
continue
if isinstance(it.get("text"), str):
text_parts.append(str(it.get("text")))
elif isinstance(it.get("image"), str):
images.append(str(it.get("image")))
elif isinstance(it.get("image_url"), str):
images.append(str(it.get("image_url")))
elif isinstance(it.get("image_url"), dict) and isinstance(it["image_url"].get("url"), str):
images.append(str(it["image_url"]["url"]))
elif isinstance(it.get("b64_json"), str):
images.append(f"data:image/png;base64,{it['b64_json']}")
# fallback for image APIs that return data[0].b64_json/url
if not images:
data = resp_json.get("data")
if isinstance(data, list):
for it in data:
if not isinstance(it, dict):
continue
if isinstance(it.get("url"), str):
images.append(str(it.get("url")))
if isinstance(it.get("b64_json"), str):
images.append(f"data:image/png;base64,{it['b64_json']}")
return "\n".join([x for x in text_parts if x]).strip(), images
def _extract_from_dashscope_response(resp: Any) -> tuple[str, list[str], str]:
"""Parse dashscope SDK response object (or dict-like) into text/images."""
text_parts: list[str] = []
images: list[str] = []
def _as_dict(x: Any) -> dict[str, Any]:
if isinstance(x, dict):
return x
if hasattr(x, "__dict__"):
try:
return dict(vars(x))
except Exception:
return {}
return {}
if isinstance(resp, dict):
obj = resp
else:
obj = _as_dict(resp)
if not obj and hasattr(resp, "to_dict"):
try:
obj = resp.to_dict()
except Exception:
obj = {}
status_code = int(obj.get("status_code") or getattr(resp, "status_code", 0) or 0)
output = obj.get("output") if isinstance(obj.get("output"), dict) else _as_dict(getattr(resp, "output", {}))
choices = output.get("choices") if isinstance(output, dict) else None
if isinstance(choices, list) and choices:
msg = choices[0].get("message") if isinstance(choices[0], dict) else _as_dict(getattr(choices[0], "message", {}))
content = msg.get("content")
if isinstance(content, list):
for it in content:
# DashScope SDK sometimes returns content items as objects, not plain dicts.
# Convert best-effort so we don't silently drop images.
it_d: dict[str, Any]
if isinstance(it, dict):
it_d = it
else:
it_d = _as_dict(it)
if not it_d:
continue
if isinstance(it_d.get("text"), str):
text_parts.append(str(it_d["text"]))
# image can be str/dict/list depending on SDK variant
img_val = it_d.get("image")
if isinstance(img_val, str):
images.append(img_val)
elif isinstance(img_val, dict):
if isinstance(img_val.get("url"), str):
images.append(str(img_val["url"]))
elif isinstance(img_val, list):
for elem in img_val:
if isinstance(elem, str):
images.append(elem)
elif isinstance(elem, dict) and isinstance(elem.get("url"), str):
images.append(str(elem["url"]))
if isinstance(it_d.get("image_url"), str):
images.append(str(it_d["image_url"]))
elif isinstance(it_d.get("image_url"), dict) and isinstance(it_d["image_url"].get("url"), str):
images.append(str(it_d["image_url"]["url"]))
elif isinstance(it_d.get("image_url"), list):
for elem in it_d["image_url"]:
if isinstance(elem, str):
images.append(elem)
elif isinstance(elem, dict) and isinstance(elem.get("url"), str):
images.append(str(elem["url"]))
b64_val = it_d.get("b64_json")
if isinstance(b64_val, str):
images.append(f"data:image/png;base64,{b64_val}")
elif isinstance(b64_val, list):
for elem in b64_val:
if isinstance(elem, str):
images.append(f"data:image/png;base64,{elem}")
# fallback for variants returning output.data[{url|b64_json}]
if not images and isinstance(output, dict):
data = output.get("data")
if isinstance(data, list):
for it in data:
it_d: dict[str, Any]
if isinstance(it, dict):
it_d = it
else:
it_d = _as_dict(it)
if not it_d:
continue
url_val = it_d.get("url")
if isinstance(url_val, str):
images.append(url_val)
elif isinstance(url_val, list):
for elem in url_val:
if isinstance(elem, str):
images.append(elem)
b64_val = it_d.get("b64_json")
if isinstance(b64_val, str):
images.append(f"data:image/png;base64,{b64_val}")
elif isinstance(b64_val, list):
for elem in b64_val:
if isinstance(elem, str):
images.append(f"data:image/png;base64,{elem}")
return "\n".join([x for x in text_parts if x]).strip(), images, str(status_code)
def _dashscope_messages(images: list[str], prompt: str) -> list[dict[str, Any]]:
prompt_text = str(prompt or "").strip() or render_prompt("image/default_edit_prompt.zh.md", strict=True)
content: list[dict[str, Any]] = [{"image": img} for img in images]
content.append({"text": prompt_text})
return [{"role": "user", "content": content}]
def _http_content_blocks(images: list[str], prompt: str, *, typed: bool) -> list[dict[str, Any]]:
prompt_text = str(prompt or "").strip() or render_prompt("image/default_edit_prompt.zh.md", strict=True)
if not typed:
blocks: list[dict[str, Any]] = [{"image": img} for img in images]
blocks.append({"text": prompt_text})
return blocks
# DashScope compatible-mode error is very explicit about `content[*].type` missing.
# To align with DashScope's simple schema keys ({"image": ...}, {"text": ...}),
# we keep those keys and only add the `type` discriminator.
blocks_typed: list[dict[str, Any]] = [{"type": "image", "image": img} for img in images]
blocks_typed.append({"type": "text", "text": prompt_text})
return blocks_typed
def _responses_input_messages(images: list[str], prompt: str) -> list[dict[str, Any]]:
prompt_text = str(prompt or "").strip() or render_prompt("image/default_edit_prompt.zh.md", strict=True)
content: list[dict[str, Any]] = []
for img in images:
s = str(img or "").strip()
if not s:
continue
if s.startswith("data:") or s.startswith("http://") or s.startswith("https://"):
content.append({"type": "input_image", "image_url": s})
if prompt_text:
content.append({"type": "input_text", "text": prompt_text})
return [{"role": "user", "content": content}]
def _dashscope_typed_messages(images: list[str], prompt: str) -> list[dict[str, Any]]:
"""
DashScope compatible-mode often requires a typed schema, where each content item has `type`.
Example:
{"type":"image_url","image_url":{"url":"..."}}, {"type":"text","text":"..."}
"""
typed_blocks = _http_content_blocks(images, prompt, typed=True)
return [{"role": "user", "content": typed_blocks}]
def _send_via_dashscope(
*,
images: list[str],
prompt: str,
model_name: str,
api_key: str | None = None,
base_http_api_url: str | None = None,
force_typed_schema: bool = False,
) -> dict[str, Any]:
try:
import dashscope # type: ignore
from dashscope import MultiModalConversation # type: ignore
except Exception as e:
return {"ok": False, "error": f"dashscope package unavailable: {type(e).__name__}: {e}"}
resolved_key = (api_key or os.getenv("DASHSCOPE_API_KEY") or "").strip()
if not resolved_key:
return {"ok": False, "error": "missing DASHSCOPE_API_KEY"}
ds_base = (base_http_api_url or os.getenv("DASHSCOPE_BASE_HTTP_API_URL") or "").strip()
if ds_base:
dashscope.base_http_api_url = ds_base
call_kwargs_base: dict[str, Any] = {
"api_key": resolved_key,
"model": model_name,
"stream": False,
}
# Optional params compatible with user's sample
if os.getenv("DASHSCOPE_IMAGE_N"):
try:
call_kwargs_base["n"] = int(os.getenv("DASHSCOPE_IMAGE_N") or "1")
except Exception:
pass
if os.getenv("DASHSCOPE_IMAGE_WATERMARK"):
call_kwargs_base["watermark"] = os.getenv("DASHSCOPE_IMAGE_WATERMARK", "false").lower() in ("1", "true", "yes")
if os.getenv("DASHSCOPE_IMAGE_NEGATIVE_PROMPT"):
call_kwargs_base["negative_prompt"] = os.getenv("DASHSCOPE_IMAGE_NEGATIVE_PROMPT")
if os.getenv("DASHSCOPE_IMAGE_PROMPT_EXTEND"):
call_kwargs_base["prompt_extend"] = os.getenv("DASHSCOPE_IMAGE_PROMPT_EXTEND", "true").lower() in ("1", "true", "yes")
if os.getenv("DASHSCOPE_IMAGE_SIZE"):
call_kwargs_base["size"] = os.getenv("DASHSCOPE_IMAGE_SIZE")
def _call_with_retry(messages: list[dict[str, Any]]) -> tuple[Any | None, Exception | None]:
retries = max(1, int(os.getenv("AIA_IMAGE_RETRIES", "3") or "3"))
backoff_sec = float(os.getenv("AIA_IMAGE_RETRY_BACKOFF_SEC", "0.8") or "0.8")
last_exc: Exception | None = None
resp_obj: Any | None = None
for attempt in range(1, retries + 1):
try:
kwargs = dict(call_kwargs_base)
kwargs["messages"] = messages
resp_obj = MultiModalConversation.call(**kwargs)
return resp_obj, None
except Exception as e:
last_exc = e
if attempt >= retries:
break
time.sleep(backoff_sec * (2 ** (attempt - 1)))
return None, last_exc
# DashScope SDK path:
# Compatible-mode may require typed content schema (`content[*].type`),
# so if we detect compatible-mode, prefer typed schema directly.
typed_msgs = _dashscope_typed_messages(images, prompt)
simple_msgs = _dashscope_messages(images, prompt)
first_msgs = typed_msgs if force_typed_schema else simple_msgs
second_msgs = simple_msgs if force_typed_schema else typed_msgs
first_schema = "typed" if first_msgs is typed_msgs else "simple"
second_schema = "simple" if first_schema == "typed" else "typed"
debug_first = _summarize_dashscope_messages(first_msgs)
debug_second = _summarize_dashscope_messages(second_msgs)
used_schema = first_schema
used_debug = debug_first
resp, err = _call_with_retry(first_msgs)
if resp is None:
# If the first schema fails without a response, try the other schema once.
resp2, err2 = _call_with_retry(second_msgs)
if resp2 is None:
assert err2 is not None
return {
"ok": False,
"error": (
f"dashscope call failed: {type(err2).__name__}: {err2} "
f"(debug={{first_schema={first_schema},second_schema={second_schema}}})"
)[:900],
}
resp = resp2
used_schema = second_schema
used_debug = debug_second
text, out_images, status = _extract_from_dashscope_response(resp)
try:
status_code = int(status)
except Exception:
status_code = 0
if status_code and status_code != 200:
# DashScope may return HTTP 429 (rate limit) as a non-exception status.
# _call_with_retry only retries on exceptions, so handle 429 explicitly here.
if status_code == 429:
status_retries = max(1, int(os.getenv("AIA_IMAGE_STATUS_RETRIES", "4") or "4"))
status_backoff_sec = float(os.getenv("AIA_IMAGE_STATUS_RETRY_BACKOFF_SEC", "5.0") or "5.0")
for attempt in range(1, status_retries):
time.sleep(status_backoff_sec * (2 ** (attempt - 1)))
resp_retry, _err_retry = _call_with_retry(simple_msgs)
if resp_retry is None:
continue
text_r, out_images_r, status_r = _extract_from_dashscope_response(resp_retry)
try:
status_r_code = int(status_r)
except Exception:
status_r_code = 0
if status_r_code == 200 or not status_r_code:
return {
"ok": True,
"text": text_r,
"images": out_images_r,
"backend_shape": "dashscope",
}
if status_r_code == 400:
# If we hit schema issues after rate limit, fall back to typed once.
typed_msgs = _dashscope_typed_messages(images, prompt)
resp_t, _err_t = _call_with_retry(typed_msgs)
if resp_t is not None:
text_t, out_images_t, status_t = _extract_from_dashscope_response(resp_t)
try:
status_t_code = int(status_t)
except Exception:
status_t_code = 0
if status_t_code == 200 or not status_t_code:
return {
"ok": True,
"text": text_t,
"images": out_images_t,
"backend_shape": "dashscope",
}
# continue loop for more 429 tries
code = getattr(resp, "code", "") or ""
msg = getattr(resp, "message", "") or ""
return {
"ok": False,
"error": (
f"dashscope status={status_code} code={code} debug={{used_schema={used_schema},used_debug={used_debug}}} "
f"message={msg[:400]} (after 429 retries)"
),
"backend_shape": "dashscope",
}
# Compatible-mode may reject "simple" schema with HTTP 400 and a message like:
# messages[0].content[1].type missing required parameter.
# To make this robust, retry typed schema whenever we see 400.
if status_code == 400:
resp2, _err2 = _call_with_retry(typed_msgs)
if resp2 is not None:
text2, out_images2, status2 = _extract_from_dashscope_response(resp2)
try:
status2_code = int(status2)
except Exception:
status2_code = 0
if status2_code == 200 or not status2_code:
return {
"ok": True,
"text": text2,
"images": out_images2,
"backend_shape": "dashscope",
}
code = getattr(resp, "code", "") or ""
msg = getattr(resp, "message", "") or ""
return {
"ok": False,
"error": (
f"dashscope status={status_code} code={code} debug={{used_schema={used_schema},used_debug={used_debug}}} "
f"message={msg[:400]}"
),
"backend_shape": "dashscope",
}
return {
"ok": True,
"text": text,
"images": out_images,
"backend_shape": "dashscope",
"debug_used_schema": used_schema,
"debug_used_debug": used_debug,
}
def send_image_messages(
*,
images: list[str],
prompt: str,
model: str | None = None,
timeout_sec: int = 60,
api_key: str | None = None,
base_url: str | None = None,
dashscope_api_key: str | None = None,
dashscope_base_http_api_url: str | None = None,
) -> dict[str, Any]:
"""Send multimodal messages payload with image/text content blocks.
Preferred payload: {"messages":[{"role":"user","content":[{"image":"..."},{"text":"..."}]}]}
Auto-compat: try multi-image first, then fallback to single `image` field style.
"""
model_name = (
model
or os.getenv("AIA_IMAGE_MODEL")
or os.getenv("DASHSCOPE_IMAGE_MODEL")
or os.getenv("OPENAI_MODEL")
or "qwen-image-2.0-pro"
).strip()
resolved_base_url = (base_url or os.getenv("AIA_IMAGE_BASE_URL") or os.getenv("OPENAI_BASE_URL") or "https://api.openai.com/v1").strip()
resolved_api_key = (api_key or os.getenv("AIA_IMAGE_API_KEY") or os.getenv("OPENAI_API_KEY") or "").strip()
resolved_ds_key = (dashscope_api_key or os.getenv("DASHSCOPE_API_KEY") or "").strip()
resolved_ds_base = (dashscope_base_http_api_url or os.getenv("DASHSCOPE_BASE_HTTP_API_URL") or "").strip()
endpoint = (os.getenv("AIA_IMAGE_CHAT_ENDPOINT") or "/chat/completions").strip()
url = _join_url(resolved_base_url, endpoint)
if not images:
return {"ok": False, "error": "at least one image input is required"}
raw_selected = [str(x).strip() for x in images if str(x).strip()][:3]
selected: list[str] = []
input_kind: list[str] = []
for img in raw_selected:
if _is_data_url(img):
selected.append(_compress_data_url_image(img))
input_kind.append("data_url")
continue
if img.startswith("http://") or img.startswith("https://"):
# Keep URL as-is when caller provides URL input.
selected.append(img)
input_kind.append("url")
continue
if not selected:
return {"ok": False, "error": "no usable image input (expected URL or data URL)"}
if resolved_ds_key:
# SpecialistAgent already normalizes DashScope SDK calls to native `/api/v1`.
# For the SDK path, prefer the official simple schema for both single and
# multi-image requests: {"image": ...}, {"text": ...}.
#
# This avoids a split where multi-image succeeds but single-image still
# trips over `messages[0].content[*].type` handling.
ds = _send_via_dashscope(
images=selected,
prompt=prompt,
model_name=model_name,
api_key=resolved_ds_key,
base_http_api_url=resolved_ds_base,
force_typed_schema=False,
)
ds["input_kind"] = input_kind if input_kind else ["data_url"]
# When DashScope credentials are present, always return the DashScope path result.
# Do not silently fall through to the OpenAI-compatible HTTP path, because that can
# mask the real SDK error with a secondary `content[*].type` error from another backend path.
return ds
if not resolved_api_key:
return {"ok": False, "error": "missing api key (AIA_IMAGE_API_KEY/OPENAI_API_KEY or DASHSCOPE_API_KEY)"}
prefer_typed_http = "compatible-mode" in resolved_base_url.lower()
content_multi = _http_content_blocks(selected, prompt, typed=prefer_typed_http)
content_multi_fallback = _http_content_blocks(selected, prompt, typed=not prefer_typed_http)
headers = {
"Authorization": f"Bearer {resolved_api_key}",
"Content-Type": "application/json",
}
payload_multi = {
"model": model_name,
"messages": [{"role": "user", "content": content_multi}],
}
payload_single = {
"model": model_name,
"messages": [{"role": "user", "content": content_multi_fallback[-2:] if len(content_multi_fallback) >= 2 else content_multi_fallback}],
}
responses_payload = {
"model": model_name,
"input": {"messages": _responses_input_messages(selected, prompt)},
}
with httpx.Client(timeout=float(timeout_sec)) as client:
# try multi-image payload first
try:
r = _post_with_retry(client, url=url, headers=headers, payload=payload_multi)
except Exception as e:
return {"ok": False, "error": f"http request failed: {type(e).__name__}: {e}", "backend_shape": "multi"}
if r.status_code >= 400:
# fallback single-image shape for strict backends
try:
r2 = _post_with_retry(client, url=url, headers=headers, payload=payload_single)
except Exception as e:
return {
"ok": False,
"error": f"http fallback request failed: {type(e).__name__}: {e}",
"backend_shape": "single-fallback-failed",
}
if r2.status_code >= 400:
body2 = str(r2.text or "")
if ("input.messages" in body2) or ("Input should be 'user'" in body2):
# OpenAI-compatible Responses schema fallback.
try:
r3 = _post_with_retry(
client,
url=_join_url(resolved_base_url, "/responses"),
headers=headers,
payload=responses_payload,
)
if r3.status_code < 400:
obj3 = r3.json()
text3, out_images3 = _extract_text_and_images(obj3 if isinstance(obj3, dict) else {})
return {
"ok": True,
"text": text3,
"images": out_images3,
"backend_shape": "responses-fallback",
"input_kind": input_kind if input_kind else ["data_url"],
}
except Exception:
pass
return {
"ok": False,
"error": f"http {r2.status_code}: {r2.text[:500]}",
"backend_shape": "single-fallback-failed",
}
try:
obj2 = r2.json()
except Exception:
return {"ok": False, "error": f"non-json response: {r2.text[:500]}", "backend_shape": "single"}
text, out_images = _extract_text_and_images(obj2 if isinstance(obj2, dict) else {})
return {
"ok": True,
"text": text,
"images": out_images,
"backend_shape": "single",
"input_kind": input_kind[-1:] if input_kind else ["data_url"],
}
try:
obj = r.json()
except Exception:
return {"ok": False, "error": f"non-json response: {r.text[:500]}", "backend_shape": "multi"}
text, out_images = _extract_text_and_images(obj if isinstance(obj, dict) else {})
return {
"ok": True,
"text": text,
"images": out_images,
"backend_shape": "multi",
"input_kind": input_kind if input_kind else ["data_url"],
}
__all__ = ["send_image_messages"]

View file

@ -0,0 +1,91 @@
"""Transcript replay policy for OpenAI-compatible Chat Completions.
Inspired by OpenClaw (MIT) `src/agents/transcript-policy.ts` defaults for
`openai-completions`: enable strict tool-call-id sanitization so proxies and
multi-provider histories do not break on id format / length.
See OPENCLAW_MIT_LICENSE.txt in this package.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any
from oclaw.platform.llm.tool_call_id import DEFAULT_MAX_OPENAI_TOOL_CALL_ID_LEN
@dataclass(frozen=True)
class ReplayPolicy:
"""Controls pre-request normalization of chat messages for tool-use flow."""
enabled: bool
sanitize_tool_call_ids: bool
repair_tool_pairing: bool
max_tool_call_id_len: int
def _env_bool(name: str, default: bool) -> bool:
raw = str(os.getenv(name) or "").strip().lower()
if raw in ("0", "false", "no", "off"):
return False
if raw in ("1", "true", "yes", "on"):
return True
return default
def resolve_replay_policy(
base_url: str | None,
model_id: str | None,
*,
llm_profile_mode: str | None = None,
) -> ReplayPolicy:
"""Resolve replay policy for OpenAI-compatible chat completions.
When enabled (default): sanitize assistant/tool ids before every request.
This matches oclaw's strict OpenAI-compatible replay defaults.
"""
del base_url, model_id, llm_profile_mode # reserved for provider-specific overrides
if not _env_bool("AIA_REPLAY_POLICY_ENABLED", True):
return ReplayPolicy(
enabled=False,
sanitize_tool_call_ids=False,
repair_tool_pairing=False,
max_tool_call_id_len=DEFAULT_MAX_OPENAI_TOOL_CALL_ID_LEN,
)
max_len = DEFAULT_MAX_OPENAI_TOOL_CALL_ID_LEN
raw_ml = str(os.getenv("AIA_TOOL_CALL_ID_MAX_LEN") or "").strip()
if raw_ml.isdigit():
max_len = max(8, min(int(raw_ml), 128))
repair = _env_bool("AIA_REPLAY_REPAIR_TOOL_PAIRING", True)
return ReplayPolicy(
enabled=True,
sanitize_tool_call_ids=True,
repair_tool_pairing=repair,
max_tool_call_id_len=max_len,
)
def apply_replay_policy_to_messages(
messages: list[dict[str, Any]],
policy: ReplayPolicy,
) -> list[dict[str, Any]]:
"""Apply repair + id sanitization in order."""
if not policy.enabled or not messages:
return messages
from oclaw.platform.llm.tool_call_id import (
repair_orphan_tool_messages,
rewrite_openai_chat_messages_tool_ids,
)
out = messages
if policy.repair_tool_pairing:
out = repair_orphan_tool_messages(out)
if policy.sanitize_tool_call_ids:
out = rewrite_openai_chat_messages_tool_ids(out, max_len=policy.max_tool_call_id_len)
return out

View file

@ -0,0 +1,160 @@
"""OpenAI-compatible tool_call_id / tool result id rewriting.
Inspired by OpenClaw (MIT) `src/agents/tool-call-id.ts`: sanitize ids to strict
alphanumeric form and keep assistant.tool_calls[].id paired with role=tool
tool_call_id in encounter order.
See OPENCLAW_MIT_LICENSE.txt in this package.
"""
from __future__ import annotations
import hashlib
import re
from collections import deque
from typing import Any, Literal
ToolCallIdMode = Literal["strict", "strict9"]
# OpenAI Chat Completions: tool_calls[].id commonly capped ~40 chars on some gateways.
DEFAULT_MAX_OPENAI_TOOL_CALL_ID_LEN = 40
def _short_hash(text: str, length: int = 8) -> str:
return hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()[:length]
def sanitize_tool_call_id(raw: str | None, *, mode: ToolCallIdMode = "strict") -> str:
"""Strip to [a-zA-Z0-9] only; empty input gets a deterministic placeholder."""
if not raw or not isinstance(raw, str):
return "sanitizedtoolid" if mode == "strict" else "defaultid"
s = raw.strip()
if not s:
return "sanitizedtoolid" if mode == "strict" else "defaultid"
alnum = re.sub(r"[^a-zA-Z0-9]", "", s)
if mode == "strict9":
if len(alnum) >= 9:
return alnum[:9]
if alnum:
return _short_hash(alnum, 9)
return _short_hash("sanitized", 9)
return alnum if alnum else "sanitizedtoolid"
def _make_unique(seed: str, used: set[str], *, max_len: int) -> str:
base = sanitize_tool_call_id(seed, mode="strict")[:max_len]
if base and base not in used:
used.add(base)
return base
h = _short_hash(seed, 8)
for i in range(1000):
cand = f"{base[: max(0, max_len - len(h) - 1)]}{h}"[:max_len]
if i > 0:
suf = f"x{i}"
cand = f"{base[: max(0, max_len - len(suf) - len(h))]}{suf}{h}"[:max_len]
if cand not in used:
used.add(cand)
return cand
cand = _short_hash(f"{seed}:{id(used)}", max_len)[:max_len]
used.add(cand)
return cand
def rewrite_openai_chat_messages_tool_ids(
messages: list[dict[str, Any]],
*,
max_len: int = DEFAULT_MAX_OPENAI_TOOL_CALL_ID_LEN,
) -> list[dict[str, Any]]:
"""Rewrite assistant tool_calls ids and matching tool tool_call_id/call_id.
Uses a per-raw-id queue (oclaw-style) so repeated raw ids map to distinct
new ids in order, and tool results consume the queue FIFO.
"""
if not messages:
return messages
pending: dict[str, deque[str]] = {}
used: set[str] = set()
def _alloc_for_raw(raw: str) -> str:
raw = str(raw or "").strip()
seed = raw if raw else "empty"
return _make_unique(seed, used, max_len=max_len)
out: list[dict[str, Any]] = []
for m in messages:
if not isinstance(m, dict):
continue
role = str(m.get("role") or "")
if role == "assistant" and m.get("tool_calls"):
mm = dict(m)
tcs = mm.get("tool_calls")
if isinstance(tcs, list):
new_tcs: list[dict[str, Any]] = []
for tc in tcs:
if not isinstance(tc, dict):
new_tcs.append(tc)
continue
raw_id = str(tc.get("id") or "").strip()
new_id = _alloc_for_raw(raw_id)
q = pending.setdefault(raw_id, deque())
q.append(new_id)
new_tcs.append({**tc, "id": new_id})
mm["tool_calls"] = new_tcs
out.append(mm)
continue
if role == "tool":
mm = dict(m)
raw_ref = (
str(mm.get("tool_call_id") or mm.get("call_id") or "").strip()
)
new_ref: str | None = None
if raw_ref and raw_ref in pending and pending[raw_ref]:
new_ref = pending[raw_ref].popleft()
elif raw_ref:
# Orphan or ordering mismatch: allocate fresh id (best-effort pairing)
new_ref = _alloc_for_raw(raw_ref)
if new_ref:
mm["tool_call_id"] = new_ref
# Keep call_id in sync when both present (some gateways expect call_id)
if "call_id" in mm:
mm["call_id"] = new_ref
else:
mm.pop("tool_call_id", None)
mm.pop("call_id", None)
out.append(mm)
continue
out.append(dict(m) if isinstance(m, dict) else m)
return out
def repair_orphan_tool_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Drop tool_call_id/call_id from tool rows that reference no prior assistant tool_calls id."""
allowed: set[str] = set()
for m in messages:
if not isinstance(m, dict):
continue
if str(m.get("role") or "") != "assistant":
continue
tcs = m.get("tool_calls")
if not isinstance(tcs, list):
continue
for tc in tcs:
if isinstance(tc, dict) and tc.get("id"):
allowed.add(str(tc.get("id")))
out: list[dict[str, Any]] = []
for m in messages:
if not isinstance(m, dict):
out.append(m)
continue
if str(m.get("role") or "") != "tool":
out.append(m)
continue
mm = dict(m)
tid = str(mm.get("tool_call_id") or mm.get("call_id") or "").strip()
if tid and tid not in allowed:
mm.pop("tool_call_id", None)
mm.pop("call_id", None)
out.append(mm)
return out

103
platform/llm/tool_schema.py Normal file
View file

@ -0,0 +1,103 @@
from __future__ import annotations
import json
import logging
import os
from typing import Any
logger = logging.getLogger(__name__)
_MIN_TOOL_PARAMETERS: dict[str, Any] = {"type": "object", "additionalProperties": True}
def openai_tools_json_byte_length(tools: list[dict[str, Any]]) -> int:
try:
return len(json.dumps(tools, ensure_ascii=False, default=str))
except Exception:
return 0
def shrink_openai_tools_payload_for_api(tools: list[dict[str, Any]], *, max_json_bytes: int) -> list[dict[str, Any]]:
"""Shrink OpenAI ``tools`` JSON for strict OpenAI-compatible gateways."""
if not tools or max_json_bytes <= 0:
return tools
if openai_tools_json_byte_length(tools) <= max_json_bytes:
return tools
flat: list[dict[str, Any]] = []
for t in tools:
if not isinstance(t, dict) or str(t.get("type") or "") != "function":
continue
fn = t.get("function")
if not isinstance(fn, dict):
continue
name = str(fn.get("name") or "").strip()
if not name:
continue
desc = str(fn.get("description") or "")
flat.append({"type": "function", "function": {"name": name, "description": desc, "parameters": dict(_MIN_TOOL_PARAMETERS)}})
if not flat:
return tools
def _serialized_with_desc_cap(cap: int) -> str:
trimmed: list[dict[str, Any]] = []
for x in flat:
fn0 = x["function"]
trimmed.append(
{
"type": "function",
"function": {
"name": fn0["name"],
"description": (fn0.get("description") or "")[: max(0, cap)],
"parameters": fn0["parameters"],
},
}
)
return json.dumps(trimmed, ensure_ascii=False, default=str)
lo, hi = 0, min(8000, max_json_bytes)
best_cap = 0
while lo <= hi:
mid = (lo + hi) // 2
s = _serialized_with_desc_cap(mid)
if len(s) <= max_json_bytes:
best_cap = mid
lo = mid + 1
else:
hi = mid - 1
out_obj: list[dict[str, Any]] = json.loads(_serialized_with_desc_cap(best_cap))
blob = json.dumps(out_obj, ensure_ascii=False, default=str)
dropped = 0
while len(out_obj) > 8 and len(blob) > max_json_bytes:
out_obj.pop()
dropped += 1
blob = json.dumps(out_obj, ensure_ascii=False, default=str)
logger.warning(
"Shrunk OpenAI tools[] for API size limits: count=%d max_json_bytes=%d desc_cap=%d dropped=%d final_bytes=%d",
len(out_obj),
max_json_bytes,
best_cap,
dropped,
len(blob),
)
return out_obj
def default_max_openai_tools_json_bytes(base_url: str | None) -> int | None:
raw = str(os.getenv("AIA_OPENAI_TOOLS_MAX_JSON_CHARS") or "").strip()
if raw.isdigit():
return max(2000, int(raw))
u = (base_url or "").lower()
if "dashscope.aliyuncs.com" in u:
return 28000
if str(os.getenv("AIA_SHRINK_OPENAI_TOOLS") or "").strip().lower() in ("1", "true", "yes", "on"):
mx = str(os.getenv("AIA_SHRINK_OPENAI_TOOLS_MAX_JSON") or "28000").strip()
return max(2000, int(mx)) if mx.isdigit() else 28000
return None
__all__ = [
"default_max_openai_tools_json_bytes",
"openai_tools_json_byte_length",
"shrink_openai_tools_payload_for_api",
]

View file

@ -0,0 +1,779 @@
"""Graduated OpenAI ``tools[]`` shaping + usage-aware omission / per-tool admin policies.
Uses ``tool_log`` aggregates, ``app_setting`` for admin config / per-tool levels / penalty state.
Wire payload differs from local :class:`~src.tools.base.ToolSpec`; validation still uses full schemas.
"""
from __future__ import annotations
import json
import logging
import os
from datetime import datetime, timedelta, timezone
from typing import Any
logger = logging.getLogger(__name__)
SETTINGS_KEY_PENALTY_STATE = "mcp_tool_wire_penalty_state"
SETTINGS_KEY_TOOL_POLICIES = "mcp_tool_wire_tool_policies"
SETTINGS_KEY_ADMIN_CONFIG = "mcp_tool_wire_admin_config"
# Role-scoped (per expert) overrides. Shape:
# - policies: { "<role>": { "mcp__server__tool": <int level> } }
# - penalty: { "<role>": { "mcp__server__tool": <penalty state dict> } }
SETTINGS_KEY_PENALTY_STATE_BY_ROLE = "mcp_tool_wire_penalty_state_by_role"
SETTINGS_KEY_TOOL_POLICIES_BY_ROLE = "mcp_tool_wire_tool_policies_by_role"
SETTINGS_KEY_ROLE_MODE_BY_ROLE = "mcp_tool_wire_role_mode_by_role"
_MIN_PARAMETERS: dict[str, Any] = {"type": "object", "additionalProperties": True}
def _env_prefixed(name_suffix: str, default: str = "") -> str:
raw = str(os.getenv(f"AIA_{name_suffix}") or "").strip()
if raw:
return raw
raw = str(os.getenv(f"OPS_{name_suffix}") or "").strip()
if raw:
return raw
return str(default).strip()
def wire_policy_enabled(base_url: str | None) -> bool:
raw = _env_prefixed("MCP_WIRE_USAGE_POLICY", "").lower()
if raw in ("1", "true", "yes", "on"):
return True
if raw in ("0", "false", "no", "off"):
return False
u = (base_url or "").lower()
return "dashscope.aliyuncs.com" in u
def penalty_disabled() -> bool:
return _env_prefixed("MCP_WIRE_PENALTY_DISABLE", "").lower() in ("1", "true", "yes", "on")
def _admin_defaults_from_env() -> dict[str, Any]:
return {
"wire_policy": "inherit",
"top_n_full": int(_env_prefixed("MCP_WIRE_TOP_N_FULL", "20") or "20"),
"stale_hours": float(_env_prefixed("MCP_WIRE_STALE_HOURS", "3") or "3"),
"penalty_minutes": float(_env_prefixed("MCP_WIRE_PENALTY_MINUTES", "30") or "30"),
"medium_rank_start": int(_env_prefixed("MCP_WIRE_MEDIUM_RANK_START", "21") or "21"),
"medium_rank_end": int(_env_prefixed("MCP_WIRE_MEDIUM_RANK_END", "50") or "50"),
"medium_desc_chars": int(_env_prefixed("MCP_WIRE_MEDIUM_DESC_CHARS", "520") or "520"),
"minimal_desc_cap": int(_env_prefixed("MCP_WIRE_MINIMAL_DESC_CAP", "80") or "80"),
"penalty_disable": False,
}
def load_merged_admin_config(store: Any) -> dict[str, Any]:
out = _admin_defaults_from_env()
try:
raw = store.get_setting(SETTINGS_KEY_ADMIN_CONFIG)
if not raw:
return out
d = json.loads(raw) if isinstance(raw, str) else raw
if isinstance(d, dict):
for k, v in d.items():
if k in out or k == "wire_policy":
out[k] = v
except Exception:
pass
pd = out.get("penalty_disable")
if isinstance(pd, str):
out["penalty_disable"] = pd.strip().lower() in ("1", "true", "yes", "on")
return out
def load_tool_policies_dict(store: Any) -> dict[str, int]:
try:
raw = store.get_setting(SETTINGS_KEY_TOOL_POLICIES) or "{}"
d = json.loads(raw) if isinstance(raw, str) else raw
except Exception:
return {}
if not isinstance(d, dict):
return {}
out: dict[str, int] = {}
for k, v in d.items():
nm = str(k or "").strip()
if not nm.startswith("mcp__"):
continue
try:
n = int(v)
except (TypeError, ValueError):
continue
if n == 9999:
out[nm] = 9999
elif n <= 0:
out[nm] = 0
else:
out[nm] = min(n, 9998)
return out
def load_tool_policies_dict_for_role(store: Any, *, role: str | None) -> dict[str, int]:
"""Role-scoped policies; falls back to global if absent/unparseable."""
r = str(role or "").strip().lower()
if not r:
return load_tool_policies_dict(store)
try:
raw = store.get_setting(SETTINGS_KEY_TOOL_POLICIES_BY_ROLE) or "{}"
outer = json.loads(raw) if isinstance(raw, str) else raw
except Exception:
outer = {}
if not isinstance(outer, dict):
return load_tool_policies_dict(store)
inner = outer.get(r)
if not isinstance(inner, dict):
return load_tool_policies_dict(store)
# Reuse same coercion rules as global loader.
out: dict[str, int] = {}
for k, v in inner.items():
nm = str(k or "").strip()
if not nm.startswith("mcp__"):
continue
try:
n = int(v)
except (TypeError, ValueError):
continue
if n == 9999:
out[nm] = 9999
elif n <= 0:
out[nm] = 0
else:
out[nm] = min(n, 9998)
return out
def load_penalty_state_for_role(store: Any, *, role: str | None) -> dict[str, Any]:
"""Role-scoped penalty state; falls back to global if absent/unparseable."""
r = str(role or "").strip().lower()
if not r:
raw_pen = store.get_setting(SETTINGS_KEY_PENALTY_STATE) or "{}"
try:
pen = json.loads(raw_pen) if isinstance(raw_pen, str) else {}
except Exception:
pen = {}
if not isinstance(pen, dict):
pen = {}
return migrate_legacy_penalty_store(pen)
try:
raw = store.get_setting(SETTINGS_KEY_PENALTY_STATE_BY_ROLE) or "{}"
outer = json.loads(raw) if isinstance(raw, str) else raw
except Exception:
outer = {}
if not isinstance(outer, dict):
return load_penalty_state_for_role(store, role=None)
inner = outer.get(r)
if not isinstance(inner, dict):
return load_penalty_state_for_role(store, role=None)
return migrate_legacy_penalty_store(inner)
def _persist_penalty_state_for_role(store: Any, *, role: str | None, penalty_state: dict[str, Any]) -> None:
r = str(role or "").strip().lower()
clean: dict[str, Any] = {}
for k, v in (penalty_state or {}).items():
if v is None:
continue
if isinstance(v, dict):
clean[str(k)] = v
if not r:
store.set_setting(SETTINGS_KEY_PENALTY_STATE, json.dumps(clean, ensure_ascii=False))
return
try:
raw = store.get_setting(SETTINGS_KEY_PENALTY_STATE_BY_ROLE) or "{}"
outer = json.loads(raw) if isinstance(raw, str) else raw
except Exception:
outer = {}
if not isinstance(outer, dict):
outer = {}
outer[r] = clean
store.set_setting(SETTINGS_KEY_PENALTY_STATE_BY_ROLE, json.dumps(outer, ensure_ascii=False))
def wire_graduation_effective(base_url: str | None, admin: dict[str, Any]) -> bool:
mode = str(admin.get("wire_policy") or "inherit").strip().lower()
if mode in ("always", "on", "true", "1", "yes"):
return True
if mode in ("never", "off", "false", "0", "no"):
return False
return wire_policy_enabled(base_url)
def penalty_effective_disabled(admin: dict[str, Any]) -> bool:
if bool(admin.get("penalty_disable")):
return True
return penalty_disabled()
def migrate_legacy_penalty_store(raw: dict[str, Any]) -> dict[str, Any]:
"""Older rows used ``omit_until`` without ``phase``; add ``kind`` for global tier."""
out: dict[str, Any] = {}
for k, v in (raw or {}).items():
if not isinstance(v, dict):
continue
v2 = dict(v)
if str(v2.get("phase") or "") in ("active", "done"):
if "kind" not in v2:
v2["kind"] = "d"
out[str(k)] = v2
continue
ou_s = str(v2.get("omit_until") or "")
ws = str(v2.get("wave_ts") or "")
ou = _parse_iso(ou_s)
if ou_s and ou:
if _utc_now() < ou:
v2 = {"phase": "active", "omit_until": ou_s, "wave_ts": ws, "kind": v2.get("kind") or "d"}
else:
v2 = {"phase": "done", "wave_ts": ws, "kind": v2.get("kind") or "d"}
elif ws:
v2 = {"phase": "done", "wave_ts": ws, "kind": v2.get("kind") or "d"}
else:
continue
out[str(k)] = v2
return out
def _parse_iso(ts: str) -> datetime | None:
s = str(ts or "").strip()
if not s:
return None
try:
if s.endswith("Z"):
s = s[:-1] + "+00:00"
return datetime.fromisoformat(s)
except Exception:
return None
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
def _wire_json_size(tools: list[dict[str, Any]]) -> int:
try:
return len(json.dumps(tools, ensure_ascii=False, default=str))
except Exception:
return 0
def _strip_nested_descriptions(obj: Any) -> Any:
if isinstance(obj, dict):
out: dict[str, Any] = {}
for k, v in obj.items():
if str(k) == "description":
continue
out[str(k)] = _strip_nested_descriptions(v)
return out
if isinstance(obj, list):
return [_strip_nested_descriptions(x) for x in obj]
return obj
def _tier_medium(fn: dict[str, Any], *, cfg: dict[str, Any]) -> dict[str, Any]:
mid_desc_len = int(cfg.get("medium_desc_chars") or 520)
mid_desc_len = max(80, min(mid_desc_len, 4000))
desc = str(fn.get("description") or "")[:mid_desc_len]
params = fn.get("parameters")
if isinstance(params, dict):
params_out = dict(_strip_nested_descriptions(params))
else:
params_out = dict(_MIN_PARAMETERS)
return {"name": fn.get("name"), "description": desc, "parameters": params_out}
def _tier_minimal(fn: dict[str, Any], *, desc_cap: int) -> dict[str, Any]:
desc = str(fn.get("description") or "")[: max(0, desc_cap)]
return {"name": fn.get("name"), "description": desc, "parameters": dict(_MIN_PARAMETERS)}
def _clone_tool_entry(t: dict[str, Any]) -> dict[str, Any]:
try:
return json.loads(json.dumps(t, ensure_ascii=False, default=str))
except Exception:
return dict(t)
def _is_builtin_tool_name(name: str) -> bool:
return not str(name).startswith("mcp__")
def _normalize_role_mode(v: Any) -> str:
s = str(v or "").strip().lower()
if s in {"unrestricted", "forbidden", "restricted"}:
return s
return "restricted"
def load_role_mode_for_role(store: Any, *, role: str | None) -> str:
r = str(role or "").strip().lower()
if not r:
return "restricted"
try:
raw = store.get_setting(SETTINGS_KEY_ROLE_MODE_BY_ROLE) or "{}"
obj = json.loads(raw) if isinstance(raw, str) else raw
except Exception:
obj = {}
if not isinstance(obj, dict):
return "restricted"
return _normalize_role_mode(obj.get(r))
def _is_stale_hours(last_ts_iso: str | None, stale_hours: float) -> bool:
if not last_ts_iso:
return False
dt = _parse_iso(last_ts_iso)
if dt is None:
return False
return (_utc_now() - dt).total_seconds() > stale_hours * 3600.0
def _idle_minutes_exceeded(last_ts_iso: str | None, idle_minutes: float) -> bool:
if not last_ts_iso:
return False
dt = _parse_iso(last_ts_iso)
if dt is None:
return False
return (_utc_now() - dt).total_seconds() > idle_minutes * 60.0
def _wave_penalty_omit(
name: str,
last_ts: str | None,
idle_hit: bool,
penalty_minutes: float,
st_nm: Any,
out_penalty_updates: dict[str, Any],
*,
kind: str,
) -> bool:
pw = last_ts or ""
mins = max(1.0, min(float(penalty_minutes), 24 * 60))
if not idle_hit:
return False
prev = out_penalty_updates.get(name)
if isinstance(prev, dict) and str(prev.get("kind") or "") != str(kind):
out_penalty_updates.pop(name, None)
st_nm = None
else:
st_nm = prev
omit = False
if isinstance(st_nm, dict) and str(st_nm.get("phase") or "") == "active":
ou = _parse_iso(str(st_nm.get("omit_until") or ""))
if ou and _utc_now() < ou:
omit = True
else:
out_penalty_updates[name] = {"phase": "done", "wave_ts": pw, "kind": str(kind)}
elif isinstance(st_nm, dict) and str(st_nm.get("phase") or "") == "done" and str(st_nm.get("wave_ts") or "") == pw:
pass
else:
until = (_utc_now() + timedelta(minutes=mins)).isoformat()
out_penalty_updates[name] = {"phase": "active", "omit_until": until, "wave_ts": pw, "kind": str(kind)}
omit = True
return omit
def filter_permanent_ban_mcp_tools(tools: list[dict[str, Any]], policies: dict[str, int]) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for entry in tools:
if not isinstance(entry, dict) or str(entry.get("type") or "") != "function":
out.append(entry)
continue
fn = entry.get("function")
if not isinstance(fn, dict):
out.append(entry)
continue
name = str(fn.get("name") or "").strip()
if not name or _is_builtin_tool_name(name):
out.append(entry)
continue
if policies.get(name) == 9999:
continue
out.append(entry)
return out
def prepare_openai_tools_for_llm_api(
tools: list[dict[str, Any]],
*,
base_url: str | None,
max_json_bytes: int | None,
store: Any | None = None,
role: str | None = None,
) -> list[dict[str, Any]]:
"""Build wire ``tools[]``: optional graduated policy + byte cap."""
def _fallback_shrink(raw: list[dict[str, Any]], cap: int) -> list[dict[str, Any]]:
from oclaw.platform.llm.tool_schema import shrink_openai_tools_payload_for_api
return shrink_openai_tools_payload_for_api(raw, max_json_bytes=cap)
if not tools:
return tools
policies: dict[str, int] = {}
admin: dict[str, Any] = _admin_defaults_from_env()
role_mode = "restricted"
try:
if store is None:
from oclaw.platform.config.paths import db_path
from oclaw.platform.persistence.sqlite_store import SqliteStore
store = SqliteStore(db_path())
# Role-scoped policies if configured; otherwise fall back to global.
policies = load_tool_policies_dict_for_role(store, role=str(role or "").strip().lower() or None)
admin = load_merged_admin_config(store)
role_mode = load_role_mode_for_role(store, role=str(role or "").strip().lower() or None)
except Exception as exc:
logger.warning("tool_wire_policy: load admin/policies (%s)", exc)
def _filter_mcp_out(raw: list[dict[str, Any]]) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for entry in raw:
if not isinstance(entry, dict) or str(entry.get("type") or "") != "function":
out.append(entry)
continue
fn = entry.get("function")
if not isinstance(fn, dict):
out.append(entry)
continue
nm = str(fn.get("name") or "").strip()
if nm.startswith("mcp__"):
continue
out.append(entry)
return out
tools_pass1 = filter_permanent_ban_mcp_tools(tools, policies)
if role_mode == "forbidden":
out = _filter_mcp_out(tools)
if max_json_bytes is not None and max_json_bytes > 0:
return _fallback_shrink(out, max_json_bytes)
return out
if role_mode == "unrestricted":
if max_json_bytes is not None and max_json_bytes > 0:
return _fallback_shrink(tools_pass1, max_json_bytes)
return tools_pass1
if not wire_graduation_effective(base_url, admin):
if max_json_bytes is not None and max_json_bytes > 0:
return _fallback_shrink(tools_pass1, max_json_bytes)
return tools_pass1
top_n = int(admin.get("top_n_full") or 20)
top_n = max(3, min(top_n, 80))
stale_h = float(admin.get("stale_hours") or 3)
stale_h = max(0.25, min(stale_h, 720.0))
penalty_min_global = float(admin.get("penalty_minutes") or 30)
penalty_min_global = max(1.0, min(penalty_min_global, 24 * 60))
medium_floor = int(admin.get("medium_rank_start") or 21)
medium_ceiling = int(admin.get("medium_rank_end") or 50)
if medium_floor > medium_ceiling:
medium_floor, medium_ceiling = medium_ceiling, medium_floor
usage_map: dict[str, dict[str, Any]] = {}
penalty_state: dict[str, Any] = {}
try:
usage_map = store.list_mcp_tool_aggregate_usage()
penalty_state = load_penalty_state_for_role(store, role=str(role or "").strip().lower() or None)
except Exception as exc:
logger.warning("tool_wire_policy: usage store unavailable (%s); fallback shrink only", exc)
if max_json_bytes is not None and max_json_bytes > 0:
return _fallback_shrink(tools_pass1, max_json_bytes)
return tools_pass1
ranked = sorted(
usage_map.items(),
key=lambda kv: (-int(kv[1].get("count") or 0), str(kv[1].get("last_ts") or "")),
)
rank_by_name = {nm: idx + 1 for idx, (nm, _) in enumerate(ranked)}
top_full = {nm for nm, _ in ranked[:top_n]}
out_penalty_updates: dict[str, Any] = dict(penalty_state)
omit_set: set[str] = set()
for nm in list(out_penalty_updates.keys()):
meta = usage_map.get(nm)
cur_ts = str(meta.get("last_ts") or "") if meta else ""
st = out_penalty_updates.get(nm)
if not isinstance(st, dict):
continue
w_ts = str(st.get("wave_ts") or "")
if w_ts and cur_ts and w_ts != cur_ts:
out_penalty_updates.pop(nm, None)
pen_glob = penalty_effective_disabled(admin)
built: list[dict[str, Any]] = []
for entry in tools_pass1:
if not isinstance(entry, dict) or str(entry.get("type") or "") != "function":
built.append(entry)
continue
fn = entry.get("function")
if not isinstance(fn, dict):
built.append(entry)
continue
name = str(fn.get("name") or "").strip()
if not name:
built.append(entry)
continue
if _is_builtin_tool_name(name):
built.append(_clone_tool_entry(entry))
continue
usage = usage_map.get(name) or {}
last_ts = str(usage.get("last_ts") or "") or None
rk = rank_by_name.get(name)
policy_lv = policies.get(name)
st_nm = out_penalty_updates.get(name)
# policy_lv semantics (punishment only):
# - None: inherit global graduation + idle penalty.
# - 0: exempt this tool from idle penalty (but still participates in graduation tiers).
# - 1..9998: per-tool idle/penalty tuning (N * 10 minutes) + still participates in graduation tiers.
# - 9999: permanent ban (filtered earlier in filter_permanent_ban_mcp_tools).
omit = False
if policy_lv == 0:
omit = False
elif policy_lv is not None and policy_lv >= 1:
idle_m = float(policy_lv) * 10.0
pen_m = float(policy_lv) * 10.0
idle_hit = _idle_minutes_exceeded(last_ts, idle_m)
kind = f"L{int(policy_lv)}"
if not pen_glob:
omit = _wave_penalty_omit(
name, last_ts, idle_hit, pen_m, st_nm, out_penalty_updates, kind=kind
)
else:
idle_hit = _is_stale_hours(last_ts, stale_h)
exempt = name in top_full
if not pen_glob and idle_hit and not exempt:
omit = _wave_penalty_omit(
name,
last_ts,
idle_hit,
penalty_min_global,
st_nm,
out_penalty_updates,
kind="d",
)
if omit:
omit_set.add(name)
continue
tier = "minimal"
if name in top_full:
tier = "full"
elif rk is not None and medium_floor <= rk <= medium_ceiling:
tier = "medium"
else:
tier = "minimal"
if tier == "full":
built.append(_clone_tool_entry(entry))
elif tier == "medium":
mf = _tier_medium(fn, cfg=admin)
built.append({"type": "function", "function": mf})
else:
cap0 = int(admin.get("minimal_desc_cap") or 80)
cap0 = max(0, min(cap0, 2000))
mf = _tier_minimal(fn, desc_cap=cap0)
built.append({"type": "function", "function": mf})
try:
_persist_penalty_state_for_role(
store,
role=str(role or "").strip().lower() or None,
penalty_state=out_penalty_updates,
)
except Exception as exc:
logger.warning("tool_wire_policy: persist penalty state failed: %s", exc)
if omit_set:
logger.info(
"tool_wire_policy: omitted %d MCP tools during penalty window: %s",
len(omit_set),
sorted(omit_set)[:12],
)
if max_json_bytes is not None and max_json_bytes > 0 and _wire_json_size(built) > max_json_bytes:
return _squeeze_to_budget(built, max_json_bytes, admin=admin)
return built
def _squeeze_to_budget(
built: list[dict[str, Any]], max_json_bytes: int, *, admin: dict[str, Any]
) -> list[dict[str, Any]]:
work = json.loads(json.dumps(built, ensure_ascii=False, default=str))
def sz() -> int:
return _wire_json_size(work)
if sz() <= max_json_bytes:
return work
for i, ent in enumerate(work):
fn = ent.get("function") if isinstance(ent.get("function"), dict) else {}
ps = fn.get("parameters")
if isinstance(ps, dict) and len(json.dumps(ps, ensure_ascii=False)) > 400:
nm = str(fn.get("name") or "")
if nm.startswith("mcp__"):
cap = int(admin.get("minimal_desc_cap") or 80)
work[i] = {
"type": "function",
"function": _tier_minimal(fn, desc_cap=max(0, cap)),
}
if sz() <= max_json_bytes:
return work
for cap in (60, 40, 24, 12, 8, 4, 0):
for i, ent in enumerate(work):
fn = ent.get("function") if isinstance(ent.get("function"), dict) else {}
nm = str(fn.get("name") or "")
if nm.startswith("mcp__"):
work[i] = {"type": "function", "function": _tier_minimal(fn, desc_cap=cap)}
if sz() <= max_json_bytes:
return work
idxs = [i for i, e in enumerate(work) if str((e.get("function") or {}).get("name") or "").startswith("mcp__")]
for i in reversed(idxs):
if sz() <= max_json_bytes:
break
work.pop(i)
if sz() > max_json_bytes:
from oclaw.platform.llm.tool_schema import shrink_openai_tools_payload_for_api
work = shrink_openai_tools_payload_for_api(work, max_json_bytes=max_json_bytes)
return work
def penalty_row_status(
*,
wire_name: str,
policy_level: int | None,
penalty: dict[str, Any] | None,
last_ts: str | None,
) -> dict[str, Any]:
"""Human-readable row for Admin UI."""
if policy_level is None:
return {
"phase": "inherit",
"omit_until": None,
"unblock_hint": "未配置:走全局线侧与闲置惩罚(与显式写入 0 不同)",
"kind": None,
}
if policy_level == 9999:
return {
"phase": "permanent_ban",
"omit_until": None,
"unblock_hint": "永久不上送(需将策略改为非 9999)",
"kind": None,
}
if policy_level == 0:
return {
"phase": "exempt",
"omit_until": None,
"unblock_hint": "策略 0:不参与闲置惩罚(仍可能受线侧分层/压缩影响)",
"kind": None,
}
st = penalty.get(wire_name) if isinstance(penalty, dict) else None
if not isinstance(st, dict):
return {"phase": "none", "omit_until": None, "unblock_hint": "-", "kind": None}
ph = str(st.get("phase") or "")
ou = str(st.get("omit_until") or "")
kd = str(st.get("kind") or "")
if ph == "active" and ou:
dt = _parse_iso(ou)
hint = f"惩罚中,预计解封(omit_until): {ou}"
if dt and _utc_now() >= dt:
hint = "惩罚窗口应已结束;下次请求会写入 done"
return {"phase": "active", "omit_until": ou, "unblock_hint": hint, "kind": kd}
if ph == "done":
return {
"phase": "done",
"omit_until": None,
"unblock_hint": "本轮已服完刑;再次闲置达到阈值后会重新惩罚",
"kind": kd,
}
return {"phase": ph or "unknown", "omit_until": ou or None, "unblock_hint": "-", "kind": kd or None}
def build_tool_wire_snapshot(store: Any, *, role: str | None = None) -> dict[str, Any]:
"""Aggregate MCP install list + usage + policies for Admin GET."""
from oclaw.tools.mcp.registry import McpRegistry
admin = load_merged_admin_config(store)
role_norm = str(role or "").strip().lower() or None
role_mode = load_role_mode_for_role(store, role=role_norm)
policies = load_tool_policies_dict_for_role(store, role=role_norm)
pen = load_penalty_state_for_role(store, role=role_norm)
usage = store.list_mcp_tool_aggregate_usage()
servers = McpRegistry(store).list_servers(enabled_only=False)
tools_out: list[dict[str, Any]] = []
for s in servers:
sid = str(s.get("server_id") or "").strip()
if not sid:
continue
for t in store.list_mcp_server_tools(server_id=sid):
short = str(t.get("tool_name") or "").strip()
if not short:
continue
wire = f"mcp__{sid}__{short}"
agg = usage.get(wire) or {}
in_db = wire in policies
lv = int(policies[wire]) if in_db else None
st = penalty_row_status(
wire_name=wire,
policy_level=lv,
penalty=pen,
last_ts=str(agg.get("last_ts") or "") or None,
)
tools_out.append(
{
"server_id": sid,
"mcp_tool_name": short,
"wire_name": wire,
"policy_level": lv,
"policy_in_db": in_db,
"count": int(agg.get("count") or 0),
"last_ts": str(agg.get("last_ts") or ""),
"penalty": st,
"raw_penalty": pen.get(wire) if isinstance(pen.get(wire), dict) else None,
}
)
tools_out.sort(key=lambda x: (x["server_id"], x["wire_name"]))
return {
"ok": True,
"role": role_norm or "",
"role_mode": role_mode,
"config": admin,
"policies": policies,
"penalty_state": pen,
"tools": tools_out,
}
__all__ = [
"SETTINGS_KEY_PENALTY_STATE",
"SETTINGS_KEY_TOOL_POLICIES",
"SETTINGS_KEY_ADMIN_CONFIG",
"SETTINGS_KEY_PENALTY_STATE_BY_ROLE",
"SETTINGS_KEY_TOOL_POLICIES_BY_ROLE",
"SETTINGS_KEY_ROLE_MODE_BY_ROLE",
"build_tool_wire_snapshot",
"filter_permanent_ban_mcp_tools",
"load_merged_admin_config",
"load_tool_policies_dict",
"load_tool_policies_dict_for_role",
"load_penalty_state_for_role",
"load_role_mode_for_role",
"migrate_legacy_penalty_store",
"penalty_row_status",
"prepare_openai_tools_for_llm_api",
"wire_policy_enabled",
]

View file

@ -0,0 +1 @@
# oclaw.platform.llm.transports package

View file

@ -0,0 +1,204 @@
from __future__ import annotations
import json
import logging
import os
import uuid
from typing import Any, Optional
from collections.abc import Callable, Iterable
from oclaw.platform.llm.transports.base import ChatModel, LLMResponse, LLMToolCall
logger = logging.getLogger(__name__)
def _as_dict(obj: Any) -> dict[str, Any] | None:
if obj is None:
return None
if isinstance(obj, dict):
return obj
if hasattr(obj, "model_dump"):
try:
d = obj.model_dump(mode="python")
return d if isinstance(d, dict) else None
except Exception:
return None
return None
def parse_anthropic_stream_events(
events: Iterable[Any],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> tuple[str, list[LLMToolCall]]:
parts: list[str] = []
tool_by_id: dict[str, dict[str, Any]] = {}
current_block: dict[str, Any] | None = None
for ev in events:
d = _as_dict(ev) or {}
typ = str(d.get("type") or "")
if typ in ("message_start", "message_delta", "message_stop"):
continue
if typ == "content_block_start":
cb = d.get("content_block") if isinstance(d.get("content_block"), dict) else {}
current_block = cb
# tool_use blocks carry name/id/input in start
if str(cb.get("type") or "") == "tool_use":
tid = str(cb.get("id") or "") or f"call_{uuid.uuid4().hex}"
tool_by_id[tid] = {
"id": tid,
"name": str(cb.get("name") or ""),
"input_json": cb.get("input") if isinstance(cb.get("input"), dict) else {},
"_input_buf": "",
}
continue
if typ == "content_block_delta":
delta = d.get("delta") if isinstance(d.get("delta"), dict) else {}
dt = str(delta.get("type") or "")
if dt == "text_delta":
t = str(delta.get("text") or "")
if t:
parts.append(t)
if on_token:
on_token(t)
continue
if dt in ("input_json_delta", "tool_use_delta"):
partial = delta.get("partial_json") or delta.get("partial") or ""
s = str(partial or "")
# attach to last tool_use block if any
# Anthropics uses content block index; we just buffer by most recent tool_use id.
if current_block and str(current_block.get("type") or "") == "tool_use":
tid = str(current_block.get("id") or "")
if tid and tid in tool_by_id:
tool_by_id[tid]["_input_buf"] = str(tool_by_id[tid].get("_input_buf") or "") + s
continue
continue
if typ == "content_block_stop":
current_block = None
continue
tool_calls: list[LLMToolCall] = []
for tid, slot in tool_by_id.items():
args = slot.get("input_json") if isinstance(slot.get("input_json"), dict) else {}
buf = str(slot.get("_input_buf") or "").strip()
if buf:
try:
parsed = json.loads(buf)
if isinstance(parsed, dict):
args = parsed
else:
args = {"_raw": parsed}
except Exception:
# keep best-effort
pass
tool_calls.append(LLMToolCall(id=tid, name=str(slot.get("name") or ""), arguments=args, thought_signature=None))
return "".join(parts), tool_calls
class AnthropicMessagesModel(ChatModel):
def __init__(self, *, model: str | None = None, api_key: str | None = None, base_url: str | None = None):
self.model = (model or os.getenv("ANTHROPIC_MODEL") or "claude-3-5-sonnet-latest").strip()
self.api_key = (api_key or os.getenv("ANTHROPIC_API_KEY") or "").strip()
self.base_url = (base_url or os.getenv("ANTHROPIC_BASE_URL") or "").strip() or None
if not self.api_key:
raise RuntimeError("未设置 ANTHROPIC_API_KEY,无法使用 Anthropic 模型")
try:
import anthropic
except Exception as exc:
raise RuntimeError("未安装 anthropic 依赖,请先 pip install -r requirements.txt") from exc
kw: dict[str, Any] = {"api_key": self.api_key}
if self.base_url:
kw["base_url"] = self.base_url
self._client = anthropic.Anthropic(**kw)
@staticmethod
def _to_anthropic_messages(messages: list[dict[str, Any]]) -> tuple[str, list[dict[str, Any]]]:
system_chunks: list[str] = []
out: list[dict[str, Any]] = []
for m in messages or []:
if not isinstance(m, dict):
continue
role = str(m.get("role") or "")
content = m.get("content")
if role == "system":
if isinstance(content, str) and content.strip():
system_chunks.append(content)
continue
if role in ("user", "assistant"):
if isinstance(content, str):
out.append({"role": role, "content": content})
else:
out.append({"role": role, "content": str(content)})
continue
if role == "tool":
# Anthropic expects tool results as user content blocks; we stringify.
name = str(m.get("name") or m.get("tool_name") or m.get("toolName") or "tool").strip()
out.append({"role": "user", "content": f"[tool_result:{name}] {str(content)}"})
continue
return "\n\n".join(system_chunks).strip(), out
@staticmethod
def _to_anthropic_tools(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for t in tools or []:
if not isinstance(t, dict) or str(t.get("type") or "") != "function":
continue
fn = t.get("function")
if not isinstance(fn, dict):
continue
name = str(fn.get("name") or "").strip()
if not name:
continue
out.append(
{
"name": name,
"description": str(fn.get("description") or ""),
"input_schema": fn.get("parameters") if isinstance(fn.get("parameters"), dict) else {"type": "object"},
}
)
return out
def chat(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> LLMResponse:
system, msgs = self._to_anthropic_messages(messages)
atools = self._to_anthropic_tools(tools)
try:
stream = self._client.messages.stream(
model=self.model,
system=system or None,
messages=msgs,
tools=atools or None,
max_tokens=2048,
)
text, tool_calls = parse_anthropic_stream_events(stream, on_token=on_token)
return LLMResponse(content=text, tool_calls=tool_calls)
except Exception as exc:
logger.info("anthropic stream failed; fallback to non-stream (%s)", exc)
resp = self._client.messages.create(
model=self.model,
system=system or None,
messages=msgs,
tools=atools or None,
max_tokens=2048,
)
d = _as_dict(resp) or {}
content = d.get("content")
text = ""
if isinstance(content, list):
for blk in content:
if isinstance(blk, dict) and blk.get("type") == "text":
text += str(blk.get("text") or "")
if on_token and text:
on_token(text)
return LLMResponse(content=text, tool_calls=[])
__all__ = ["AnthropicMessagesModel", "parse_anthropic_stream_events"]

View file

@ -0,0 +1,63 @@
from __future__ import annotations
import base64
import json
from dataclasses import dataclass
from typing import Any, Optional
from collections.abc import Callable
@dataclass(frozen=True)
class LLMToolCall:
id: str
name: str
arguments: dict[str, Any]
# Gemini OpenAI-compat and native transports may require preserving a signature across tool loops.
thought_signature: str | None = None
@dataclass(frozen=True)
class LLMResponse:
content: str
tool_calls: list[LLMToolCall]
reasoning_content: str = ""
class ChatModel:
def chat(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> LLMResponse:
raise NotImplementedError
def coerce_thought_signature_for_storage(v: Any) -> str | None:
if v is None:
return None
if isinstance(v, str):
return v
if isinstance(v, (bytes, bytearray)):
try:
return bytes(v).decode("utf-8")
except UnicodeDecodeError:
return base64.b64encode(bytes(v)).decode("ascii")
if isinstance(v, (dict, list)):
return json.dumps(v, separators=(",", ":"), ensure_ascii=False)
return str(v)
def normalize_image_b64_payload(raw: Any) -> str:
if raw is None:
return ""
if isinstance(raw, (bytes, bytearray)):
return base64.b64encode(bytes(raw)).decode("ascii")
s = str(raw).strip()
if not s:
return ""
if s.startswith("data:") and ";base64," in s:
s = s.split(";base64,", 1)[-1].strip()
return "".join(s.split())

View file

@ -0,0 +1,265 @@
from __future__ import annotations
import json
import os
import uuid
from typing import Any, Optional
from collections.abc import Callable
from oclaw.platform.llm.transports.base import ChatModel, LLMResponse, LLMToolCall
class GoogleGeminiChatModel(ChatModel):
"""Native Google Gemini streaming transport (SSE), modeled after OpenClaw's google transport."""
def __init__(self, *, model: str, api_key: str, base_url: str | None = None):
self.model = (model or "").strip() or "gemini-2.5-pro"
self.api_key = (api_key or "").strip()
self.base_url = (base_url or "").strip() or "https://generativelanguage.googleapis.com/v1beta"
if not self.api_key:
raise RuntimeError("未设置 GOOGLE_API_KEY,无法使用 Gemini 原生接口")
def _tool_decls_from_openai_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
decls: list[dict[str, Any]] = []
for t in tools or []:
if not isinstance(t, dict) or str(t.get("type") or "") != "function":
continue
fn = t.get("function")
if not isinstance(fn, dict):
continue
name = str(fn.get("name") or "").strip()
if not name:
continue
decls.append(
{
"name": name,
"description": str(fn.get("description") or ""),
# OpenClaw uses `parametersJsonSchema`
"parametersJsonSchema": fn.get("parameters") if isinstance(fn.get("parameters"), dict) else {"type": "object"},
}
)
return decls
@staticmethod
def _thinking_config_from_env() -> dict[str, Any] | None:
enabled = str(os.getenv("AIA_GEMINI_THINKING") or "").strip().lower()
if enabled in ("0", "false", "no", "off"):
return {"includeThoughts": False}
if enabled in ("1", "true", "yes", "on"):
cfg: dict[str, Any] = {"includeThoughts": True}
lvl = str(os.getenv("AIA_GEMINI_THINKING_LEVEL") or "").strip()
if lvl:
cfg["thinkingLevel"] = lvl
bud = str(os.getenv("AIA_GEMINI_THINKING_BUDGET") or "").strip()
if bud.isdigit():
cfg["thinkingBudget"] = int(bud)
return cfg
return None
@staticmethod
def _parse_tool_result_payload(raw: Any) -> Any:
if raw is None:
return {}
if isinstance(raw, (dict, list)):
return raw
s = str(raw or "").strip()
if not s:
return {}
try:
return json.loads(s)
except Exception:
return {"raw": s}
@staticmethod
def _openai_messages_to_gemini_contents(messages: list[dict[str, Any]]) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]:
system_parts: list[dict[str, Any]] = []
contents: list[dict[str, Any]] = []
def _append(role: str, parts: list[dict[str, Any]]) -> None:
if not parts:
return
contents.append({"role": role, "parts": parts})
for m in messages or []:
if not isinstance(m, dict):
continue
role = str(m.get("role") or "")
content = m.get("content")
if role == "system":
if isinstance(content, str) and content.strip():
system_parts.append({"text": content})
continue
if role == "user":
parts: list[dict[str, Any]] = []
if isinstance(content, str):
if content.strip():
parts.append({"text": content})
elif isinstance(content, list):
for item in content:
if not isinstance(item, dict):
continue
if item.get("type") == "text" and str(item.get("text") or "").strip():
parts.append({"text": str(item.get("text") or "")})
_append("user", parts)
continue
if role == "assistant":
parts: list[dict[str, Any]] = []
if isinstance(content, str) and content.strip():
parts.append({"text": content})
_append("model", parts)
continue
if role == "tool":
name = str(m.get("name") or "").strip() or str(m.get("tool_name") or m.get("toolName") or "").strip()
payload = GoogleGeminiChatModel._parse_tool_result_payload(content)
_append("user", [{"functionResponse": {"name": name or "tool", "response": payload}}])
continue
system_instruction = {"parts": system_parts} if system_parts else None
return system_instruction, contents
def _build_url(self) -> str:
base = self._normalize_base_url_for_native_sse(self.base_url).rstrip("/")
model_path = self.model
if not model_path.startswith("models/") and not model_path.startswith("tunedModels/"):
model_path = f"models/{model_path}"
# Prefer header auth (OpenClaw-style); avoid forcing key into query string.
return f"{base}/{model_path}:streamGenerateContent?alt=sse"
@staticmethod
def _normalize_base_url_for_native_sse(base_url: str) -> str:
"""Normalize unified gateway base_url for Gemini native SSE.
Many unified gateways expose OpenAI-compat under `/v1/*`, but Gemini native endpoints
are usually implemented under `/v1beta/*` semantics.
"""
raw = str(base_url or "").strip()
if not raw:
return raw
# Strip trailing slashes first.
while raw.endswith("/"):
raw = raw[:-1]
# Common unified gateway layout: https://host/v1 -> https://host/v1beta
if raw.lower().endswith("/v1"):
return raw[:-3] + "/v1beta"
# Some gateways keep an `/openai` suffix for compat endpoints.
if raw.lower().endswith("/openai"):
raw = raw[:-7]
if raw.lower().endswith("/v1"):
return raw[:-3] + "/v1beta"
return raw
return raw
def _auth_headers(self) -> dict[str, str]:
k = str(self.api_key or "").strip()
if not k:
return {}
# If user stored a bearer token in the profile secret.
if k.lower().startswith("bearer "):
return {"authorization": k}
# Gemini API supports x-goog-api-key; unified gateways often accept this too.
return {"x-goog-api-key": k}
def chat(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> LLMResponse:
system_instruction, contents = self._openai_messages_to_gemini_contents(messages)
decls = self._tool_decls_from_openai_tools(tools)
req: dict[str, Any] = {"contents": contents}
if system_instruction is not None:
req["systemInstruction"] = system_instruction
if decls:
req["tools"] = [{"functionDeclarations": decls}]
thinking_cfg = self._thinking_config_from_env()
if thinking_cfg is not None:
req["generationConfig"] = {"thinkingConfig": thinking_cfg}
url = self._build_url()
try:
import httpx
except Exception as exc:
raise RuntimeError("missing dependency: httpx") from exc
thinking_parts: list[str] = []
thinking_signature: str | None = None
out_text: list[str] = []
tool_calls: list[LLMToolCall] = []
def _stream_once(stream_url: str) -> None:
with httpx.Client(timeout=httpx.Timeout(60.0, connect=10.0)) as client:
with client.stream(
"POST",
stream_url,
json=req,
headers={
"Content-Type": "application/json",
"accept": "text/event-stream",
**self._auth_headers(),
},
) as r:
r.raise_for_status()
for line in r.iter_lines():
if not line or not str(line).startswith("data:"):
continue
data = str(line)[len("data:") :].strip()
if not data or data == "[DONE]":
continue
try:
chunk = json.loads(data)
except Exception:
continue
cands = chunk.get("candidates") if isinstance(chunk, dict) else None
if not isinstance(cands, list) or not cands:
continue
c0 = cands[0] if isinstance(cands[0], dict) else {}
content0 = c0.get("content") if isinstance(c0.get("content"), dict) else {}
parts = content0.get("parts") if isinstance(content0.get("parts"), list) else []
for p in parts:
if not isinstance(p, dict):
continue
if p.get("thought") is True:
txt = str(p.get("text") or "")
if txt:
thinking_parts.append(txt)
sig = p.get("thoughtSignature")
if isinstance(sig, str) and sig:
thinking_signature = sig
continue
fc = p.get("functionCall") if isinstance(p.get("functionCall"), dict) else None
if fc is not None:
name = str(fc.get("name") or "").strip()
args = fc.get("args") if isinstance(fc.get("args"), dict) else {}
tid = str(fc.get("id") or "") or f"call_{uuid.uuid4().hex}"
tool_calls.append(
LLMToolCall(id=tid, name=name, arguments=args, thought_signature=thinking_signature)
)
continue
txt = str(p.get("text") or "")
if txt:
out_text.append(txt)
if on_token:
on_token(txt)
try:
_stream_once(url)
except Exception as exc:
# If gateway returns 500 on /v1* SSE path, try /v1beta* fallback once.
msg = str(exc or "").lower()
if "500" in msg and "/v1/" in str(url).lower():
alt = str(url).replace("/v1/", "/v1beta/")
_stream_once(alt)
else:
raise
return LLMResponse(
content="".join(out_text),
tool_calls=tool_calls,
reasoning_content="".join(thinking_parts).strip(),
)
__all__ = ["GoogleGeminiChatModel"]

View file

@ -0,0 +1,309 @@
from __future__ import annotations
import json
import logging
import os
import re
import uuid
from typing import Any, Optional
from collections.abc import Callable
from oclaw.platform.llm.transports.base import ChatModel, LLMResponse, LLMToolCall, normalize_image_b64_payload, coerce_thought_signature_for_storage
logger = logging.getLogger(__name__)
def _likely_gemini_openai_compat_base_url(url: str | None) -> bool:
if not url:
return False
u = url.lower()
return "generativelanguage.googleapis.com" in u or "aiplatform.googleapis.com" in u
def _model_id_suggests_gemini(model: str | None) -> bool:
return "gemini" in (model or "").lower()
def _find_thought_signature_in_obj(o: Any) -> str | None:
if isinstance(o, dict):
for k in ("thought_signature", "thoughtSignature"):
v = o.get(k)
if v is not None:
coerced = coerce_thought_signature_for_storage(v)
if coerced is not None:
return coerced
for v in o.values():
found = _find_thought_signature_in_obj(v)
if found is not None:
return found
elif isinstance(o, list):
for v in o:
found = _find_thought_signature_in_obj(v)
if found is not None:
return found
return None
def _extract_thought_signature_from_message_tool_call(tc: Any) -> str | None:
if hasattr(tc, "model_dump"):
try:
d = tc.model_dump(mode="python")
ec = d.get("extra_content")
if isinstance(ec, dict):
g = ec.get("google")
if isinstance(g, dict) and "thought_signature" in g:
return coerce_thought_signature_for_storage(g.get("thought_signature"))
except Exception:
pass
ec = getattr(tc, "extra_content", None)
if isinstance(ec, dict):
g = ec.get("google")
if isinstance(g, dict) and "thought_signature" in g:
return coerce_thought_signature_for_storage(g.get("thought_signature"))
return _extract_thought_signature_from_tool_delta(tc)
def _extract_thought_signature_from_tool_delta(tc: Any) -> str | None:
for attr in ("thought_signature", "thoughtSignature"):
v = getattr(tc, attr, None)
if v is not None:
return coerce_thought_signature_for_storage(v)
extra = getattr(tc, "model_extra", None)
if isinstance(extra, dict):
for k in ("thought_signature", "thoughtSignature"):
ev = extra.get(k)
if ev is not None:
return coerce_thought_signature_for_storage(ev)
pyd_ex = getattr(tc, "__pydantic_extra__", None)
if isinstance(pyd_ex, dict):
for k in ("thought_signature", "thoughtSignature"):
ev = pyd_ex.get(k)
if ev is not None:
return coerce_thought_signature_for_storage(ev)
if hasattr(tc, "model_dump"):
try:
found = _find_thought_signature_in_obj(tc.model_dump(mode="python"))
if found is not None:
return found
except Exception:
pass
return None
class OpenAIChatModel(ChatModel):
def __init__(
self,
model: str | None = None,
api_key: str | None = None,
base_url: str | None = None,
):
self.model = model or os.getenv("OPENAI_MODEL") or "gpt-4o-mini"
self.api_key = api_key or os.getenv("OPENAI_API_KEY")
self.base_url = base_url or os.getenv("OPENAI_BASE_URL")
if not self.api_key:
raise RuntimeError("未设置 OPENAI_API_KEY,无法使用 OpenAI 模型")
try:
from openai import OpenAI
except Exception as e:
raise RuntimeError("未安装 openai 依赖,请先 pip install -r requirements.txt") from e
client_kwargs: dict[str, Any] = {"api_key": self.api_key}
if self.base_url:
client_kwargs["base_url"] = self.base_url
self._client = OpenAI(**client_kwargs)
self._skip_tools: bool = False
def _create_chat_completion(self, norm_msgs: list[dict[str, Any]], tools: list[dict[str, Any]], *, stream: bool):
# replay_policy/tool_wire_policy live in sibling modules; import lazily to avoid cycles.
try:
from oclaw.platform.llm.replay_policy import apply_replay_policy_to_messages, resolve_replay_policy
policy = resolve_replay_policy(self.base_url, self.model)
if policy.enabled:
norm_msgs = apply_replay_policy_to_messages(norm_msgs, policy)
except Exception as exc:
logger.warning("replay_policy apply failed (%s); continuing without rewrite", exc)
use_tools = bool(tools)
cleaned_msgs: list[dict[str, Any]] = []
for m in norm_msgs or []:
if not isinstance(m, dict):
continue
if str(m.get("role") or "") == "tool":
mm = dict(m)
for k in ("tool_call_id", "call_id"):
if k in mm:
v = mm.get(k)
try:
sv = str(v).strip() if v is not None else ""
except Exception:
sv = ""
if not sv:
mm.pop(k, None)
else:
mm[k] = sv
cleaned_msgs.append(mm)
else:
cleaned_msgs.append(m)
kwargs: dict[str, Any] = {"model": self.model, "messages": cleaned_msgs, "stream": stream}
if use_tools:
try:
from oclaw.platform.config.paths import db_path
from oclaw.platform.persistence.sqlite_store import SqliteStore
from oclaw.tools.exposure_plan import build_llm_tools_plan
plan = build_llm_tools_plan(
store=SqliteStore(db_path()),
role="",
base_url=self.base_url,
max_json_bytes=_default_max_openai_tools_json_bytes(self.base_url),
include_mcp=False,
preview_internal=False,
raw_openai_tools_override=tools,
)
kwargs["tools"] = plan.tools_wired
except Exception:
kwargs["tools"] = tools
return self._client.chat.completions.create(**kwargs)
def _llm_response_from_completion(self, completion: Any, *, on_token: Optional[Callable[[str], None]]) -> LLMResponse:
msg = completion.choices[0].message
reasoning_parts = getattr(msg, "reasoning_content", None) or ""
reasoning_text = str(reasoning_parts).strip() if reasoning_parts else ""
content = msg.content or ""
if on_token and content:
on_token(content)
tool_calls: list[LLMToolCall] = []
if msg.tool_calls:
for tc in msg.tool_calls:
tid = str(getattr(tc, "id", "") or "")
fn = getattr(tc, "function", None)
name = str(fn.name) if fn and getattr(fn, "name", None) else ""
args_raw = str(fn.arguments) if fn and getattr(fn, "arguments", None) is not None else "{}"
if not name and not tid:
continue
if not tid:
tid = f"call_{uuid.uuid4().hex}"
try:
args = json.loads(args_raw)
except json.JSONDecodeError:
args = {"_raw": args_raw}
sig = _extract_thought_signature_from_message_tool_call(tc)
if sig is None and fn is not None:
sig = _extract_thought_signature_from_tool_delta(fn)
tool_calls.append(LLMToolCall(id=tid, name=name, arguments=args, thought_signature=sig))
return LLMResponse(content=content, tool_calls=tool_calls, reasoning_content=reasoning_text)
def chat(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> LLMResponse:
# Normalize input_image to image_url.
norm_msgs = []
for m in messages:
if m.get("role") == "user" and isinstance(m.get("content"), list):
norm_content = []
for item in m["content"]:
if isinstance(item, dict) and item.get("type") == "input_image":
mime = str(item.get("mime") or "image/jpeg").strip() or "image/jpeg"
b64 = normalize_image_b64_payload(item.get("image_base64") or item.get("data"))
if not b64:
continue
norm_content.append({"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}})
else:
norm_content.append(item)
norm_msgs.append({"role": m["role"], "content": norm_content})
else:
norm_msgs.append(m)
try:
stream = self._create_chat_completion(norm_msgs, tools, stream=True)
except Exception as exc:
logger.info("Streaming chat completion failed; fallback to non-stream (%s)", exc)
completion = self._create_chat_completion(norm_msgs, tools, stream=False)
return self._llm_response_from_completion(completion, on_token=on_token)
content_parts: list[str] = []
tool_acc: dict[int, dict[str, Any]] = {}
reasoning_parts: list[str] = []
def _consume_stream(stream_obj: Any) -> None:
nonlocal content_parts, tool_acc, reasoning_parts
for chunk in stream_obj:
if not chunk.choices:
continue
delta = chunk.choices[0].delta
rc = getattr(delta, "reasoning_content", None) or ""
if rc:
reasoning_parts.append(rc)
if delta.tool_calls:
for tc in delta.tool_calls:
idx = int(tc.index)
slot = tool_acc.setdefault(idx, {"id": None, "name": None, "arguments": "", "thought_signature": None})
if tc.id:
slot["id"] = tc.id
if tc.function:
if tc.function.name:
slot["name"] = tc.function.name
if tc.function.arguments:
slot["arguments"] = f"{slot['arguments']}{tc.function.arguments}"
sig = _extract_thought_signature_from_message_tool_call(tc)
if sig is None:
sig = _extract_thought_signature_from_tool_delta(tc)
if sig is None and tc.function is not None:
sig = _extract_thought_signature_from_tool_delta(tc.function)
if sig is not None:
slot["thought_signature"] = sig
if delta.content:
content_parts.append(delta.content)
if on_token:
on_token(delta.content)
try:
_consume_stream(stream)
except Exception:
raise
content = "".join(content_parts)
reasoning_text = "".join(reasoning_parts).strip()
tool_calls: list[LLMToolCall] = []
for idx in sorted(tool_acc.keys()):
slot = tool_acc[idx]
tid = slot.get("id") or ""
name = slot.get("name") or ""
args_raw = slot.get("arguments") or "{}"
if not name and not tid:
continue
if not tid:
tid = f"call_{uuid.uuid4().hex}"
try:
args = json.loads(args_raw)
except json.JSONDecodeError:
args = {"_raw": args_raw}
ts_raw = slot.get("thought_signature")
tsig = coerce_thought_signature_for_storage(ts_raw) if ts_raw is not None else None
tool_calls.append(LLMToolCall(id=str(tid), name=str(name), arguments=args, thought_signature=tsig))
return LLMResponse(content=content, tool_calls=tool_calls, reasoning_content=reasoning_text)
__all__ = ["OpenAIChatModel", "_likely_gemini_openai_compat_base_url", "_model_id_suggests_gemini"]
def _default_max_openai_tools_json_bytes(base_url: str | None) -> int | None:
u = str(base_url or "").strip().lower()
if not u:
return None
# Conservative caps for some OpenAI-compatible gateways.
if "dashscope" in u or "aliyuncs" in u:
return 28_000
return None

View file

@ -0,0 +1,236 @@
from __future__ import annotations
import json
import logging
import os
import uuid
from typing import Any, Optional
from collections.abc import Callable, Iterable
from oclaw.platform.llm.transports.base import ChatModel, LLMResponse, LLMToolCall, normalize_image_b64_payload
logger = logging.getLogger(__name__)
def _as_dict(obj: Any) -> dict[str, Any] | None:
if obj is None:
return None
if isinstance(obj, dict):
return obj
if hasattr(obj, "model_dump"):
try:
d = obj.model_dump(mode="python")
return d if isinstance(d, dict) else None
except Exception:
return None
return None
def _collect_tool_calls_from_response_dict(resp: dict[str, Any]) -> list[LLMToolCall]:
out: list[LLMToolCall] = []
items = resp.get("output")
if not isinstance(items, list):
return out
for it in items:
d = _as_dict(it)
if not d:
continue
if str(d.get("type") or "") in ("function_call", "tool_call"):
call_id = str(d.get("call_id") or d.get("id") or "") or f"call_{uuid.uuid4().hex}"
name = str(d.get("name") or "") or str((d.get("function") or {}).get("name") or "")
args = d.get("arguments") or d.get("input") or {}
if isinstance(args, str):
try:
args = json.loads(args)
except Exception:
args = {"_raw": args}
if not isinstance(args, dict):
args = {"_raw": args}
out.append(LLMToolCall(id=call_id, name=name, arguments=args, thought_signature=None))
return out
def parse_openai_responses_stream_events(
events: Iterable[Any],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> tuple[str, list[LLMToolCall], dict[str, Any] | None]:
"""Pure stream parser (offline-testable).
Returns: (assembled_text, tool_calls, final_response_dict?)
"""
parts: list[str] = []
final_resp: dict[str, Any] | None = None
for ev in events:
d = _as_dict(ev) or {}
typ = str(d.get("type") or "")
# Text deltas
if typ in ("response.output_text.delta", "response.output_text"):
delta = d.get("delta")
if delta is None:
delta = d.get("text")
s = str(delta or "")
if s:
parts.append(s)
if on_token:
on_token(s)
continue
# Sometimes the SDK emits generic `response.delta` with nested segments.
if typ == "response.delta":
delta = d.get("delta")
if isinstance(delta, dict):
txt = delta.get("output_text")
if isinstance(txt, str) and txt:
parts.append(txt)
if on_token:
on_token(txt)
continue
# Terminal response object
if typ in ("response.completed", "response.complete", "response.done"):
resp = d.get("response") or d.get("data") or d.get("result")
final_resp = _as_dict(resp) or final_resp
continue
if typ == "response.created":
continue
# Some SDK versions yield the response object directly (no explicit event type).
if d.get("output") is not None and d.get("id") is not None:
final_resp = d
text = "".join(parts)
tool_calls = _collect_tool_calls_from_response_dict(final_resp) if final_resp else []
return text, tool_calls, final_resp
class OpenAIResponsesModel(ChatModel):
"""OpenAI Responses API transport (OpenAI-compatible gateways may implement this surface)."""
def __init__(self, *, model: str | None = None, api_key: str | None = None, base_url: str | None = None):
self.model = (model or os.getenv("OPENAI_MODEL") or "gpt-4o-mini").strip()
self.api_key = (api_key or os.getenv("OPENAI_API_KEY") or "").strip()
self.base_url = (base_url or os.getenv("OPENAI_BASE_URL") or "").strip() or None
if not self.api_key:
raise RuntimeError("未设置 OPENAI_API_KEY,无法使用 OpenAI Responses")
try:
from openai import OpenAI
except Exception as e:
raise RuntimeError("未安装 openai 依赖,请先 pip install -r requirements.txt") from e
kw: dict[str, Any] = {"api_key": self.api_key}
if self.base_url:
kw["base_url"] = self.base_url
self._client = OpenAI(**kw)
@staticmethod
def _normalize_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""
Normalize to the strictest OpenAI-compatible `responses` shape:
- input.messages[*].role MUST be "user"
- input.messages[*].content MUST be a list (content blocks)
- text blocks use {"type":"text","text":...}
- image blocks use {"type":"image_url","image_url":{"url":"..."}} or {"type":"input_image","image_url":"..."} depending on gateway;
we prefer the common "image_url" block here.
"""
out: list[dict[str, Any]] = []
for m in messages or []:
if not isinstance(m, dict):
continue
role = str(m.get("role") or "user").strip().lower() or "user"
content = m.get("content")
norm_content: list[dict[str, Any]] = []
if isinstance(content, list):
for item in content:
if isinstance(item, dict) and item.get("type") == "input_image":
mime = str(item.get("mime") or "image/jpeg").strip() or "image/jpeg"
b64 = normalize_image_b64_payload(item.get("image_base64") or item.get("data"))
if not b64:
continue
norm_content.append({"type": "image_url", "image_url": {"url": f"data:{mime};base64,{b64}"}})
continue
if isinstance(item, dict) and item.get("type") in ("input_text", "text"):
t = str(item.get("text") or "").strip()
if t:
norm_content.append({"type": "text", "text": t})
continue
if isinstance(item, dict):
# Best-effort coerce unknown blocks into valid text blocks.
s = str(item.get("text") or "").strip()
if s:
norm_content.append({"type": "text", "text": s})
continue
s = str(item or "").strip()
if s:
norm_content.append({"type": "text", "text": s})
elif isinstance(content, str):
txt = content.strip()
if txt:
norm_content.append({"type": "text", "text": txt})
elif content is not None:
s = str(content).strip()
if s:
norm_content.append({"type": "text", "text": s})
if not norm_content:
# Keep shape valid even for empty messages.
continue
# Enforce role=user and preserve other roles via prefix tag.
if role != "user":
prefix = "assistant" if role == "assistant" else ("system" if role == "system" else role)
norm_content.insert(0, {"type": "text", "text": f"[{prefix}]"})
out.append({"role": "user", "content": norm_content})
return out
def chat(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> LLMResponse:
norm = self._normalize_messages(messages)
# OpenAI-compatible gateways differ: some require input={"messages":[...]} with role=user only.
stream_errors: list[str] = []
stream_variants: list[dict[str, Any]] = [
{"model": self.model, "input": {"messages": norm}, "tools": tools or None, "stream": True},
{"model": self.model, "input": norm, "tools": tools or None, "stream": True},
]
try:
for payload in stream_variants:
try:
stream = self._client.responses.create(**payload)
text, tool_calls, final_resp = parse_openai_responses_stream_events(stream, on_token=on_token)
if (not text.strip()) and final_resp:
ot = final_resp.get("output_text")
if isinstance(ot, str) and ot.strip():
text = ot
if on_token:
on_token(text)
return LLMResponse(content=text, tool_calls=tool_calls)
except Exception as exc:
stream_errors.append(str(exc))
continue
raise RuntimeError("; ".join(stream_errors) or "responses_stream_all_variants_failed")
except Exception as exc:
logger.info("responses stream failed; fallback to non-stream (%s)", exc)
nonstream_errors: list[str] = []
for payload in (
{"model": self.model, "input": {"messages": norm}, "tools": tools or None},
{"model": self.model, "input": norm, "tools": tools or None},
):
try:
resp = self._client.responses.create(**payload)
d = _as_dict(resp) or {}
text = str(d.get("output_text") or "")
tool_calls = _collect_tool_calls_from_response_dict(d)
if on_token and text:
on_token(text)
return LLMResponse(content=text, tool_calls=tool_calls)
except Exception as e2:
nonstream_errors.append(str(e2))
continue
raise RuntimeError(
"openai_responses_request_failed: "
+ " | ".join([str(exc)] + nonstream_errors[-2:])
) from exc
__all__ = ["OpenAIResponsesModel", "parse_openai_responses_stream_events"]

View file

@ -0,0 +1,144 @@
from __future__ import annotations
import json
import re
from typing import Any, Optional
from collections.abc import Callable
from oclaw.platform.llm.transports.base import ChatModel, LLMResponse, LLMToolCall
class StaticTextChatModel(ChatModel):
def __init__(self, text: str):
self._text = text or ""
def chat(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> LLMResponse:
if on_token and self._text:
on_token(self._text)
return LLMResponse(content=self._text, tool_calls=[])
class RuleBasedChatModel(ChatModel):
_IP_RE = re.compile(
r"(?P<ip>(?:\d{1,3}\.){3}\d{1,3})|(?P<ipv6>(?:[0-9a-fA-F]{0,4}:){2,}[0-9a-fA-F]{0,4})"
)
def chat(
self,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]],
*,
on_token: Optional[Callable[[str], None]] = None,
) -> LLMResponse:
def make_call(name: str, arguments: dict[str, Any]) -> LLMResponse:
return LLMResponse(
content="",
tool_calls=[LLMToolCall(id=f"rb_{name}_1", name=name, arguments=arguments)],
)
def emit(resp: LLMResponse) -> LLMResponse:
if on_token and (resp.content or "").strip():
on_token(resp.content)
return resp
if messages:
last = messages[-1]
if last.get("role") == "tool":
raw = str(last.get("content") or "")
try:
data = json.loads(raw)
except json.JSONDecodeError:
data = {"ok": True, "raw": raw}
if isinstance(data, dict) and data.get("ok") is False:
return emit(LLMResponse(content=f"工具执行失败:{data.get('error')}", tool_calls=[]))
if isinstance(data, dict) and "route" in data:
r = data.get("route") or {}
return emit(
LLMResponse(
content=f"路由查询结果:目的 {data.get('destination')} 命中 {r.get('prefix')},下一跳 {r.get('next_hop')},出接口 {r.get('out_if')}。",
tool_calls=[],
)
)
if isinstance(data, dict) and "hops" in data:
hops = data.get("hops") or []
return emit(
LLMResponse(
content=f"路径计算结果:{data.get('src')} → {data.get('dst')} 共 {data.get('hop_count')} 跳,路径为:{' -> '.join(map(str, hops))}",
tool_calls=[],
)
)
if isinstance(data, dict) and "diff" in data:
diff = data.get("diff") or ""
if not diff:
return emit(LLMResponse(content="配置对比结果:两段配置一致。", tool_calls=[]))
return emit(LLMResponse(content=f"配置对比结果:\n\n```diff\n{diff}\n```", tool_calls=[]))
if isinstance(data, dict) and "level_count" in data:
lc = data.get("level_count") or {}
return emit(LLMResponse(content=f"日志分析结果:统计 {lc}。建议优先查看 ERROR 样例与日志尾部定位根因。", tool_calls=[]))
if isinstance(data, dict) and "reachable" in data:
reach = bool(data.get("reachable"))
avg = data.get("avg_ms")
if reach:
tail = f"平均时延 {avg}ms" if isinstance(avg, int) else "时延解析失败"
return emit(LLMResponse(content=f"设备状态:可达,{tail}。", tool_calls=[]))
return emit(LLMResponse(content="设备状态:不可达。建议检查 ACL/路由/DNS/防火墙策略,并确认目标在线。", tool_calls=[]))
return emit(LLMResponse(content=f"工具执行结果:{json.dumps(data, ensure_ascii=False)}", tool_calls=[]))
last_user = ""
for m in reversed(messages):
if m.get("role") == "user":
content = m.get("content")
if isinstance(content, list):
texts = [item["text"] for item in content if isinstance(item, dict) and item.get("type") == "text"]
last_user = "\n".join(texts)
else:
last_user = str(content or "")
break
m = self._IP_RE.search(last_user)
if "路由" in last_user and m:
ip = m.group("ip") or m.group("ipv6")
return make_call("query_route", {"destination": ip})
if ("路径" in last_user or "链路" in last_user) and ("到" in last_user or "->" in last_user):
m2 = re.search(r"([A-Za-z0-9_-]+)\s*(?:到|->)\s*([A-Za-z0-9_-]+)", last_user)
if m2:
src, dst = m2.group(1), m2.group(2)
return make_call("get_path", {"src": src, "dst": dst})
if "对比" in last_user or "diff" in last_user.lower():
if "left:" in last_user.lower() and "right:" in last_user.lower():
left = last_user.split("left:", 1)[1].split("right:", 1)[0].strip()
right = last_user.split("right:", 1)[1].strip()
return make_call("config_diff", {"left": left, "right": right})
if "日志" in last_user and ("分析" in last_user or "统计" in last_user):
return make_call("log_analysis", {"log": last_user})
if ("设备" in last_user or "主机" in last_user) and ("状态" in last_user or "可达" in last_user or "ping" in last_user.lower()) and m:
ip = m.group("ip") or m.group("ipv6")
return make_call("device_status", {"host": ip})
return emit(
LLMResponse(
content="我可以进行路由查询、路径计算、配置对比、设备状态检查、日志分析。请直接描述需求并提供必要输入。",
tool_calls=[],
)
)
__all__ = ["StaticTextChatModel", "RuleBasedChatModel"]

View file

File diff suppressed because it is too large Load diff