mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-16 03:10:47 +08:00
将附件/多模态查询与编辑工具从 generalist 迁移到 public。
同时移除 network_ops 的转发壳文件,统一工具入口并更新对应测试引用,降低重复维护成本。 Made-with: Cursor
This commit is contained in:
parent
5d67303d22
commit
1c71a29ec1
10 changed files with 84 additions and 126 deletions
|
|
@ -1,6 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from oclaw.runtime.tools.experts.generalist.image_query import query_image_attachment_tool
|
|
||||||
|
|
||||||
__all__ = ["query_image_attachment_tool"]
|
|
||||||
|
|
||||||
|
|
@ -1,10 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from oclaw.runtime.tools.experts.generalist.tabular_query import (
|
|
||||||
analyze_tabular_attachment_full_scan_tool,
|
|
||||||
query_tabular_attachment_tool,
|
|
||||||
run_tabular_sql_tool,
|
|
||||||
)
|
|
||||||
|
|
||||||
__all__ = ["query_tabular_attachment_tool", "run_tabular_sql_tool", "analyze_tabular_attachment_full_scan_tool"]
|
|
||||||
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from oclaw.runtime.tools.experts.generalist.text_query import query_text_attachment_tool
|
|
||||||
|
|
||||||
__all__ = ["query_text_attachment_tool"]
|
|
||||||
|
|
||||||
|
|
@ -1,6 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from oclaw.runtime.tools.experts.generalist.video_query import query_video_attachment_tool
|
|
||||||
|
|
||||||
__all__ = ["query_video_attachment_tool"]
|
|
||||||
|
|
||||||
|
|
@ -47,13 +47,11 @@ def image_edit_tool() -> ToolSpec:
|
||||||
client_kwargs["base_url"] = base_url
|
client_kwargs["base_url"] = base_url
|
||||||
client = OpenAI(**client_kwargs)
|
client = OpenAI(**client_kwargs)
|
||||||
|
|
||||||
# OpenAI SDK expects a file-like object for edits.
|
|
||||||
img_file = io.BytesIO(blob)
|
img_file = io.BytesIO(blob)
|
||||||
img_file.name = "input.png" # type: ignore[attr-defined]
|
img_file.name = "input.png" # type: ignore[attr-defined]
|
||||||
|
|
||||||
b64_out: str | None = None
|
b64_out: str | None = None
|
||||||
try:
|
try:
|
||||||
# Preferred: image edit endpoint (if supported by the gateway/model).
|
|
||||||
resp = client.images.edit( # type: ignore[attr-defined]
|
resp = client.images.edit( # type: ignore[attr-defined]
|
||||||
model=model,
|
model=model,
|
||||||
image=img_file,
|
image=img_file,
|
||||||
|
|
@ -63,7 +61,6 @@ def image_edit_tool() -> ToolSpec:
|
||||||
data0 = resp.data[0] if getattr(resp, "data", None) else None
|
data0 = resp.data[0] if getattr(resp, "data", None) else None
|
||||||
b64_out = getattr(data0, "b64_json", None) if data0 is not None else None
|
b64_out = getattr(data0, "b64_json", None) if data0 is not None else None
|
||||||
except Exception:
|
except Exception:
|
||||||
# Fallback: generate a new image from prompt (still returns an image, but not true edit).
|
|
||||||
try:
|
try:
|
||||||
resp = client.images.generate( # type: ignore[attr-defined]
|
resp = client.images.generate( # type: ignore[attr-defined]
|
||||||
model=model,
|
model=model,
|
||||||
|
|
@ -126,4 +123,3 @@ def image_edit_tool() -> ToolSpec:
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["image_edit_tool"]
|
__all__ = ["image_edit_tool"]
|
||||||
|
|
||||||
|
|
@ -72,4 +72,3 @@ def query_image_attachment_tool() -> ToolSpec:
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["query_image_attachment_tool"]
|
__all__ = ["query_image_attachment_tool"]
|
||||||
|
|
||||||
|
|
@ -38,4 +38,3 @@ def query_text_attachment_tool() -> ToolSpec:
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["query_text_attachment_tool"]
|
__all__ = ["query_text_attachment_tool"]
|
||||||
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
import io
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
@ -144,7 +144,6 @@ def query_video_attachment_tool() -> ToolSpec:
|
||||||
if p is None:
|
if p is None:
|
||||||
return {"ok": False, "error": "attachment_not_found"}
|
return {"ok": False, "error": "attachment_not_found"}
|
||||||
|
|
||||||
# Basic metadata (no heavy deps). Prefer ffprobe if present.
|
|
||||||
if task == "meta":
|
if task == "meta":
|
||||||
fp = _ffprobe_json(p)
|
fp = _ffprobe_json(p)
|
||||||
norm = _normalized_video_meta(fp)
|
norm = _normalized_video_meta(fp)
|
||||||
|
|
@ -163,93 +162,88 @@ def query_video_attachment_tool() -> ToolSpec:
|
||||||
"note": "Use task=transcript to extract audio transcript (requires ffmpeg + OpenAI key).",
|
"note": "Use task=transcript to extract audio transcript (requires ffmpeg + OpenAI key).",
|
||||||
}
|
}
|
||||||
|
|
||||||
# transcript: extract audio then transcribe via OpenAI, store as text chunks.
|
if not _ffmpeg_exists():
|
||||||
if task == "transcript":
|
return {
|
||||||
if not _ffmpeg_exists():
|
"ok": False,
|
||||||
|
"error": "ffmpeg_missing",
|
||||||
|
"hint": "Install ffmpeg (ffmpeg/ffprobe on PATH) to enable transcript extraction.",
|
||||||
|
}
|
||||||
|
api_key = str(os.getenv("OPENAI_API_KEY") or "").strip()
|
||||||
|
if not api_key:
|
||||||
|
return {"ok": False, "error": "OPENAI_API_KEY_missing"}
|
||||||
|
|
||||||
|
try:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
wav = Path(td) / "audio.wav"
|
||||||
|
subprocess.run(
|
||||||
|
["ffmpeg", "-y", "-i", str(p), "-vn", "-ac", "1", "-ar", "16000", str(wav)],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
if not wav.exists() or wav.stat().st_size <= 0:
|
||||||
|
return {"ok": False, "error": "audio_extract_failed"}
|
||||||
|
try:
|
||||||
|
from openai import OpenAI
|
||||||
|
except Exception as e:
|
||||||
|
return {"ok": False, "error": f"openai_package_missing: {type(e).__name__}: {e}"}
|
||||||
|
|
||||||
|
base_url = str(os.getenv("OPENAI_BASE_URL") or "").strip()
|
||||||
|
client_kwargs: dict[str, Any] = {"api_key": api_key}
|
||||||
|
if base_url:
|
||||||
|
client_kwargs["base_url"] = base_url
|
||||||
|
client = OpenAI(**client_kwargs)
|
||||||
|
|
||||||
|
model = str(args.get("model") or os.getenv("OPENAI_AUDIO_TRANSCRIPTION_MODEL") or OPENAI_DEFAULT_AUDIO_TRANSCRIPTION_MODEL).strip()
|
||||||
|
prompt = str(args.get("prompt") or "").strip()
|
||||||
|
wav_bytes = wav.read_bytes()
|
||||||
|
f = io.BytesIO(wav_bytes)
|
||||||
|
f.name = "audio.wav" # type: ignore[attr-defined]
|
||||||
|
try:
|
||||||
|
resp = client.audio.transcriptions.create( # type: ignore[attr-defined]
|
||||||
|
model=model,
|
||||||
|
file=f,
|
||||||
|
**({"prompt": prompt} if prompt else {}),
|
||||||
|
)
|
||||||
|
text = str(getattr(resp, "text", "") or "")
|
||||||
|
except Exception as e:
|
||||||
|
return {"ok": False, "error": f"transcription_failed: {type(e).__name__}: {e}"}
|
||||||
|
|
||||||
|
if not text.strip():
|
||||||
|
return {"ok": False, "error": "empty_transcript"}
|
||||||
|
|
||||||
|
name = str(meta.name if meta else p.name)
|
||||||
|
text_name = f"{name}.transcript.txt"
|
||||||
|
cfg_chunk_size, cfg_chunk_overlap = _video_transcript_chunk_defaults()
|
||||||
|
chunk_size = int(args.get("chunk_size") or cfg_chunk_size)
|
||||||
|
chunk_overlap = int(args.get("chunk_overlap") or cfg_chunk_overlap)
|
||||||
|
text_meta = save_text_document(
|
||||||
|
attachment_id=str(attachment_id),
|
||||||
|
name=text_name,
|
||||||
|
text=text,
|
||||||
|
source_kind="video_transcript",
|
||||||
|
chunk_size=chunk_size,
|
||||||
|
chunk_overlap=chunk_overlap,
|
||||||
|
)
|
||||||
|
preview = text[:1200]
|
||||||
|
note = (
|
||||||
|
"Use query_text_attachment(text_id=...) to retrieve exact evidence with offsets."
|
||||||
|
if lang.startswith("en")
|
||||||
|
else "后续请用 query_text_attachment(text_id=...) 按需检索证据(支持 offset/top_k/关键词)。"
|
||||||
|
)
|
||||||
return {
|
return {
|
||||||
"ok": False,
|
"ok": True,
|
||||||
"error": "ffmpeg_missing",
|
"task": "transcript",
|
||||||
"hint": "Install ffmpeg (ffmpeg/ffprobe on PATH) to enable transcript extraction.",
|
"attachment_id": attachment_id,
|
||||||
|
"name": text_name,
|
||||||
|
"text_id": str(text_meta.get("text_id") or ""),
|
||||||
|
"chars": int(text_meta.get("chars") or 0),
|
||||||
|
"chunks": int(text_meta.get("chunks") or 0),
|
||||||
|
"preview": preview,
|
||||||
|
"note": note,
|
||||||
}
|
}
|
||||||
api_key = str(os.getenv("OPENAI_API_KEY") or "").strip()
|
except Exception as e:
|
||||||
if not api_key:
|
return {"ok": False, "error": f"transcript_failed: {type(e).__name__}: {e}"}
|
||||||
return {"ok": False, "error": "OPENAI_API_KEY_missing"}
|
|
||||||
|
|
||||||
try:
|
|
||||||
with tempfile.TemporaryDirectory() as td:
|
|
||||||
wav = Path(td) / "audio.wav"
|
|
||||||
subprocess.run(
|
|
||||||
["ffmpeg", "-y", "-i", str(p), "-vn", "-ac", "1", "-ar", "16000", str(wav)],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
timeout=60,
|
|
||||||
)
|
|
||||||
if not wav.exists() or wav.stat().st_size <= 0:
|
|
||||||
return {"ok": False, "error": "audio_extract_failed"}
|
|
||||||
try:
|
|
||||||
from openai import OpenAI
|
|
||||||
except Exception as e:
|
|
||||||
return {"ok": False, "error": f"openai_package_missing: {type(e).__name__}: {e}"}
|
|
||||||
|
|
||||||
base_url = str(os.getenv("OPENAI_BASE_URL") or "").strip()
|
|
||||||
client_kwargs: dict[str, Any] = {"api_key": api_key}
|
|
||||||
if base_url:
|
|
||||||
client_kwargs["base_url"] = base_url
|
|
||||||
client = OpenAI(**client_kwargs)
|
|
||||||
|
|
||||||
model = str(args.get("model") or os.getenv("OPENAI_AUDIO_TRANSCRIPTION_MODEL") or OPENAI_DEFAULT_AUDIO_TRANSCRIPTION_MODEL).strip()
|
|
||||||
prompt = str(args.get("prompt") or "").strip()
|
|
||||||
# OpenAI SDK expects a file-like object with a name.
|
|
||||||
wav_bytes = wav.read_bytes()
|
|
||||||
f = io.BytesIO(wav_bytes)
|
|
||||||
f.name = "audio.wav" # type: ignore[attr-defined]
|
|
||||||
# Best-effort: different gateways may accept different param names; keep it minimal.
|
|
||||||
try:
|
|
||||||
resp = client.audio.transcriptions.create( # type: ignore[attr-defined]
|
|
||||||
model=model,
|
|
||||||
file=f,
|
|
||||||
**({"prompt": prompt} if prompt else {}),
|
|
||||||
)
|
|
||||||
text = str(getattr(resp, "text", "") or "")
|
|
||||||
except Exception as e:
|
|
||||||
return {"ok": False, "error": f"transcription_failed: {type(e).__name__}: {e}"}
|
|
||||||
|
|
||||||
if not text.strip():
|
|
||||||
return {"ok": False, "error": "empty_transcript"}
|
|
||||||
|
|
||||||
# Persist transcript as a long text document so the model can query evidence by text_id.
|
|
||||||
name = str(meta.name if meta else p.name)
|
|
||||||
text_name = f"{name}.transcript.txt"
|
|
||||||
cfg_chunk_size, cfg_chunk_overlap = _video_transcript_chunk_defaults()
|
|
||||||
chunk_size = int(args.get("chunk_size") or cfg_chunk_size)
|
|
||||||
chunk_overlap = int(args.get("chunk_overlap") or cfg_chunk_overlap)
|
|
||||||
text_meta = save_text_document(
|
|
||||||
attachment_id=str(attachment_id),
|
|
||||||
name=text_name,
|
|
||||||
text=text,
|
|
||||||
source_kind="video_transcript",
|
|
||||||
chunk_size=chunk_size,
|
|
||||||
chunk_overlap=chunk_overlap,
|
|
||||||
)
|
|
||||||
preview = text[:1200]
|
|
||||||
note = (
|
|
||||||
"Use query_text_attachment(text_id=...) to retrieve exact evidence with offsets."
|
|
||||||
if lang.startswith("en")
|
|
||||||
else "后续请用 query_text_attachment(text_id=...) 按需检索证据(支持 offset/top_k/关键词)。"
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
"ok": True,
|
|
||||||
"task": "transcript",
|
|
||||||
"attachment_id": attachment_id,
|
|
||||||
"name": text_name,
|
|
||||||
"text_id": str(text_meta.get("text_id") or ""),
|
|
||||||
"chars": int(text_meta.get("chars") or 0),
|
|
||||||
"chunks": int(text_meta.get("chunks") or 0),
|
|
||||||
"preview": preview,
|
|
||||||
"note": note,
|
|
||||||
}
|
|
||||||
except Exception as e:
|
|
||||||
return {"ok": False, "error": f"transcript_failed: {type(e).__name__}: {e}"}
|
|
||||||
|
|
||||||
return ToolSpec(
|
return ToolSpec(
|
||||||
name="query_video_attachment",
|
name="query_video_attachment",
|
||||||
|
|
@ -275,4 +269,3 @@ def query_video_attachment_tool() -> ToolSpec:
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["query_video_attachment_tool"]
|
__all__ = ["query_video_attachment_tool"]
|
||||||
|
|
||||||
|
|
@ -36,7 +36,7 @@ def query_tabular_attachment_tool() -> ToolSpec:
|
||||||
columns=cols,
|
columns=cols,
|
||||||
limit=int(args.get("limit") or 50),
|
limit=int(args.get("limit") or 50),
|
||||||
offset=int(args.get("offset") or 0),
|
offset=int(args.get("offset") or 0),
|
||||||
where_contains=where_contains, # {"column":"...", "keyword":"..."}
|
where_contains=where_contains,
|
||||||
sheet=sheet,
|
sheet=sheet,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -147,4 +147,3 @@ def analyze_tabular_attachment_full_scan_tool() -> ToolSpec:
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["query_tabular_attachment_tool", "run_tabular_sql_tool", "analyze_tabular_attachment_full_scan_tool"]
|
__all__ = ["query_tabular_attachment_tool", "run_tabular_sql_tool", "analyze_tabular_attachment_full_scan_tool"]
|
||||||
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import oclaw.runtime.tools.experts.generalist.video_query as vq
|
import oclaw.runtime.tools.public.query_video_attachment_tool as vq
|
||||||
|
|
||||||
|
|
||||||
def test_normalized_video_meta_extracts_top_level_fields() -> None:
|
def test_normalized_video_meta_extracts_top_level_fields() -> None:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue