hansjone/oclaw
1
0
Fork 0
mirror of https://github.com/hansjone/oclaw.git synced 2026-10-10 17:23:25 +08:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

实现附件处理链路的统一引用化与可检索增强，避免大文件/多模态内容直接撑爆上下文并提升工具可用性。

Browse source 
本次补齐 text/image/video/archive 的标准化处理、会话级工具守卫、回放压缩、配置与文档对齐，并修复表格与流式输出相关体验问题。

Made-with: Cursor
This commit is contained in:
oliver 2026-04-26 15:25:50 +08:00
parent 9d2900db02
commit 37a2ef35f4
27 changed files with 3227 additions and 278 deletions
Download patch file Download diff file Expand all files Collapse all files

30
platform/__init__.py
View file

@ -0,0 +1,30 @@
from __future__ import annotations
import importlib.util
import sysconfig
from pathlib import Path
def _load_stdlib_platform() -> object | None:
stdlib_dir = Path(sysconfig.get_paths().get("stdlib") or "")
platform_py = (stdlib_dir / "platform.py").resolve()
if not platform_py.exists():
return None
spec = importlib.util.spec_from_file_location("_stdlib_platform", str(platform_py))
if spec is None or spec.loader is None:
return None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
_stdlib = _load_stdlib_platform()
if _stdlib is not None:
for _name in dir(_stdlib):
if _name in {"__name__", "__package__", "__loader__", "__spec__", "__file__", "__cached__"}:
continue
globals()[_name] = getattr(_stdlib, _name)
__all__ = list(getattr(_stdlib, "__all__", []))
else:
__all__ = []

202
platform/files/archive_processor.py Normal file
View file

@ -0,0 +1,202 @@
from __future__ import annotations
import gzip
import io
import tarfile
import zipfile
from pathlib import PurePosixPath
from typing import Any, Callable
MAX_ARCHIVE_DEPTH = 2
MAX_ARCHIVE_FILE_COUNT = 200
MAX_ARCHIVE_ENTRY_BYTES = 10 * 1024 * 1024
MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES = 50 * 1024 * 1024
MAX_ARCHIVE_MEMBER_NAME_LENGTH = 240
def is_safe_archive_member_name(name: str) -> bool:
n = str(name or "").replace("\\", "/").strip()
if not n:
return False
if len(n) > MAX_ARCHIVE_MEMBER_NAME_LENGTH:
return False
if any(ord(ch) < 32 for ch in n):
return False
p = PurePosixPath(n)
if p.is_absolute():
return False
if any(part in ("..", "") for part in p.parts):
return False
return True
def detect_archive_kind(name: str) -> str | None:
n = str(name or "").strip().lower()
if n.endswith(".tar.gz") or n.endswith(".tgz"):
return "tgz"
if n.endswith(".zip"):
return "zip"
if n.endswith(".tar"):
return "tar"
if n.endswith(".gz"):
return "gz"
if n.endswith(".rar"):
return "rar"
if n.endswith(".7z"):
return "7z"
return None
def sniff_archive_kind(data: bytes) -> str | None:
b = bytes(data or b"")
if len(b) >= 4 and b[:4] == b"PK\x03\x04":
return "zip"
if len(b) >= 2 and b[:2] == b"\x1f\x8b":
return "gz"
if len(b) >= 6 and b[:6] == b"Rar!\x1a\x07":
return "rar"
if len(b) >= 6 and b[:6] == b"7z\xbc\xaf\x27\x1c":
return "7z"
if len(b) >= 262 and b[257:262] == b"ustar":
return "tar"
return None
def process_archive(
*,
archive_name: str,
archive_data: bytes,
process_member: Callable[[str, bytes, int], list[dict[str, Any]]],
depth: int = 0,
limits: dict[str, int] | None = None,
) -> list[dict[str, Any]]:
lim = limits or {}
max_depth = int(lim.get("max_depth") or MAX_ARCHIVE_DEPTH)
max_file_count = int(lim.get("max_file_count") or MAX_ARCHIVE_FILE_COUNT)
max_entry_bytes = int(lim.get("max_entry_bytes") or MAX_ARCHIVE_ENTRY_BYTES)
max_total_uncompressed_bytes = int(
lim.get("max_total_uncompressed_bytes") or MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES
)
kind_from_sig = sniff_archive_kind(archive_data)
kind_from_name = detect_archive_kind(archive_name)
# For .tgz/.tar.gz, signature looks like gzip; keep richer name hint.
if kind_from_name == "tgz" and kind_from_sig == "gz":
kind = "tgz"
else:
kind = kind_from_sig or kind_from_name
if kind is None:
return []
if depth > max_depth:
return [{"type": "text", "name": "zip-error", "content": f"ZIP nesting too deep (>{max_depth})"}]
out: list[dict[str, Any]] = []
def _err(msg: str, *, error_code: str) -> list[dict[str, Any]]:
base = {
"type": "text",
"name": ("zip-error" if kind == "zip" else "archive-error"),
"error_code": str(error_code or "archive_error"),
"content": (
f"[ArchiveError]\n- error_code: {error_code}\n- format: {kind}\n- detail: {msg}"
),
}
return [base]
try:
if kind in {"rar", "7z"}:
raise ValueError(
f"unsupported archive format: {kind}; currently supported formats are zip/tar/tgz/gz"
)
if kind == "zip":
with zipfile.ZipFile(io.BytesIO(archive_data)) as zf:
infos = [x for x in zf.infolist() if not x.is_dir()]
if len(infos) > max_file_count:
raise ValueError(f"ZIP has too many files ({len(infos)} > {max_file_count})")
total_uncompressed = 0
for info in infos:
if not is_safe_archive_member_name(info.filename):
raise ValueError(f"ZIP contains unsafe path: {info.filename}")
fsize = int(info.file_size or 0)
total_uncompressed += fsize
if fsize > max_entry_bytes:
raise ValueError(f"ZIP entry too large: {info.filename} ({fsize} > {max_entry_bytes})")
if total_uncompressed > max_total_uncompressed_bytes:
raise ValueError(
f"ZIP total uncompressed size too large ({total_uncompressed} > {max_total_uncompressed_bytes})"
)
with zf.open(info) as f:
out.extend(process_member(info.filename, f.read(), depth + 1))
return out
if kind in {"tar", "tgz"}:
mode = "r:gz" if kind == "tgz" else "r:"
with tarfile.open(fileobj=io.BytesIO(archive_data), mode=mode) as tf:
all_members = list(tf.getmembers())
for m in all_members:
if m.issym() or m.islnk():
raise ValueError(f"TAR contains link entry: {m.name}")
if m.ischr() or m.isblk() or m.isfifo():
raise ValueError(f"TAR contains device/fifo entry: {m.name}")
members = [m for m in all_members if m.isfile()]
if len(members) > max_file_count:
raise ValueError(f"TAR has too many files ({len(members)} > {max_file_count})")
total_uncompressed = 0
for m in members:
if not is_safe_archive_member_name(m.name):
raise ValueError(f"TAR contains unsafe path: {m.name}")
fsize = int(m.size or 0)
total_uncompressed += fsize
if fsize > max_entry_bytes:
raise ValueError(f"TAR entry too large: {m.name} ({fsize} > {max_entry_bytes})")
if total_uncompressed > max_total_uncompressed_bytes:
raise ValueError(
f"TAR total uncompressed size too large ({total_uncompressed} > {max_total_uncompressed_bytes})"
)
ext = tf.extractfile(m)
if ext is None:
continue
out.extend(process_member(m.name, ext.read(), depth + 1))
return out
if kind == "gz":
raw = gzip.decompress(archive_data)
if len(raw) > max_entry_bytes:
raise ValueError(f"GZ entry too large ({len(raw)} > {max_entry_bytes})")
name = str(archive_name or "file.gz")
inner_name = name[:-3] if name.lower().endswith(".gz") else f"{name}.out"
out.extend(process_member(inner_name, raw, depth + 1))
return out
except Exception as e:
s = str(e).lower()
code = "archive_parse_failed"
if "unsupported archive format" in s:
code = "archive_unsupported_format"
elif "unsafe path" in s:
code = "archive_path_traversal"
elif "nesting too deep" in s:
code = "archive_max_depth_exceeded"
elif "too many files" in s:
code = "archive_max_file_count_exceeded"
elif "entry too large" in s:
code = "archive_max_entry_bytes_exceeded"
elif "total uncompressed size too large" in s:
code = "archive_max_total_uncompressed_bytes_exceeded"
elif "link entry" in s:
code = "archive_link_entry_forbidden"
elif "device/fifo entry" in s:
code = "archive_special_entry_forbidden"
return _err(str(e), error_code=code)
return out
__all__ = [
"MAX_ARCHIVE_DEPTH",
"MAX_ARCHIVE_ENTRY_BYTES",
"MAX_ARCHIVE_FILE_COUNT",
"MAX_ARCHIVE_MEMBER_NAME_LENGTH",
"MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES",
"detect_archive_kind",
"sniff_archive_kind",
"is_safe_archive_member_name",
"process_archive",
]

431
platform/files/file_attachments.py
View file

@ -6,10 +6,10 @@ import io
import json
import os
import re
import subprocess
import zipfile
from functools import lru_cache
from pathlib import Path
from pathlib import PurePosixPath
from typing import Any
import pandas as pd
@ -18,8 +18,24 @@ from PIL import Image
from PyPDF2 import PdfReader
from oclaw.platform.config.paths import PROJECT_ROOT
from oclaw.platform.files.archive_processor import (
MAX_ARCHIVE_DEPTH,
MAX_ARCHIVE_ENTRY_BYTES,
MAX_ARCHIVE_FILE_COUNT,
MAX_ARCHIVE_MEMBER_NAME_LENGTH,
MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES,
detect_archive_kind,
is_safe_archive_member_name,
process_archive,
)
from oclaw.platform.files.attachment_assets import AttachmentAssetStore
from oclaw.platform.files.tabular_attachment_store import save_dataframe, save_workbook
from oclaw.platform.files.text_attachment_store import (
DEFAULT_TEXT_CHUNK_OVERLAP,
DEFAULT_TEXT_CHUNK_SIZE,
DEFAULT_TEXT_INLINE_MAX_CHARS,
save_text_document,
)
def _sniff_image_mime(data: bytes) -> str | None:
@ -36,25 +52,101 @@ def _sniff_image_mime(data: bytes) -> str | None:
return None
def _ffprobe_exists() -> bool:
try:
p = subprocess.run(["ffprobe", "-version"], capture_output=True, text=True, timeout=3)
return p.returncode == 0
except Exception:
return False
def _extract_video_meta_from_ffprobe(obj: dict[str, Any]) -> dict[str, Any]:
fmt = obj.get("format") if isinstance(obj.get("format"), dict) else {}
streams = obj.get("streams") if isinstance(obj.get("streams"), list) else []
duration_sec = None
if isinstance(fmt, dict) and fmt.get("duration") is not None:
try:
duration_sec = float(fmt.get("duration"))
except Exception:
duration_sec = None
width = None
height = None
fps = None
for s in streams:
if not isinstance(s, dict):
continue
if str(s.get("codec_type") or "") != "video":
continue
if s.get("width") is not None and s.get("height") is not None:
try:
width = int(s.get("width") or 0) or None
height = int(s.get("height") or 0) or None
except Exception:
width = width
height = height
fr = str(s.get("avg_frame_rate") or s.get("r_frame_rate") or "").strip()
if fr and fr != "0/0" and "/" in fr:
try:
a, b = fr.split("/", 1)
fa = float(a)
fb = float(b)
if fb:
fps = fa / fb
except Exception:
fps = fps
break
out: dict[str, Any] = {}
if duration_sec is not None:
out["duration_sec"] = duration_sec
if width is not None:
out["width"] = width
if height is not None:
out["height"] = height
if fps is not None:
out["fps"] = fps
return out
def _ffprobe_video_meta(path: Path) -> dict[str, Any]:
if not _ffprobe_exists():
return {}
try:
p = subprocess.run(
["ffprobe", "-v", "error", "-print_format", "json", "-show_format", "-show_streams", str(path)],
capture_output=True,
text=True,
timeout=8,
)
if p.returncode != 0:
return {}
obj = json.loads(p.stdout or "{}")
if not isinstance(obj, dict):
return {}
return _extract_video_meta_from_ffprobe(obj)
except Exception:
return {}
MAX_TEXT_CHARS = 12_000
EXCEL_PREVIEW_ROWS = 80
LARGE_TABLE_PREVIEW_ROWS = 20
MAX_ZIP_DEPTH = 2
MAX_ZIP_FILE_COUNT = 200
MAX_ZIP_ENTRY_BYTES = 10 * 1024 * 1024
MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES = 50 * 1024 * 1024
MAX_ZIP_DEPTH = MAX_ARCHIVE_DEPTH
MAX_ZIP_FILE_COUNT = MAX_ARCHIVE_FILE_COUNT
MAX_ZIP_ENTRY_BYTES = MAX_ARCHIVE_ENTRY_BYTES
MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES = MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES
DEFAULT_TABULAR_ROWS_READ = 5000
DEFAULT_TABULAR_COLUMNS = 200
DEFAULT_TABULAR_CELL_CHARS = 500
DEFAULT_TABULAR_TOOL_MODE_ENABLED = True
DEFAULT_TABULAR_TOOL_MODE_MIN_ROWS = 20_000
DEFAULT_TABULAR_TOOL_MODE_MIN_ROWS = 5_000
DEFAULT_TABULAR_TOOL_MODE_MAX_BYTES = 30 * 1024 * 1024
DEFAULT_LARGE_TABLE_PREVIEW_ROWS = LARGE_TABLE_PREVIEW_ROWS
MAX_ZIP_MEMBER_NAME_LENGTH = 240
MAX_ZIP_MEMBER_NAME_LENGTH = MAX_ARCHIVE_MEMBER_NAME_LENGTH
MAX_EXCEL_ZIP_FILE_COUNT = 500
MAX_EXCEL_ZIP_ENTRY_BYTES = 20 * 1024 * 1024
MAX_EXCEL_ZIP_TOTAL_UNCOMPRESSED_BYTES = 120 * 1024 * 1024
DEFAULT_MAX_EXCEL_SHEETS = 50
DEFAULT_TEXT_QUERY_TOP_K = 5
def _truncate_text(text: str, limit: int = MAX_TEXT_CHARS) -> str:
@ -64,6 +156,24 @@ def _truncate_text(text: str, limit: int = MAX_TEXT_CHARS) -> str:
return t[:limit] + f"\n\n...[truncated to {limit} chars]"
def _text_ref_summary(*, name: str, chars: int, chunks: int, source_kind: str, preview: str, top_k: int) -> str:
return _truncate_text(
(
f"# Text Document Summary\n"
f"- name: {name}\n"
f"- source_kind: {source_kind}\n"
f"- chars: {chars}\n"
f"- chunks: {chunks}\n"
f"- mode: text_ref\n"
f"- query_tool: query_text_attachment\n"
f"- note: use `text_id` from accompanying `text_ref` attachment; recommended top_k <= {top_k}\n\n"
f"## Preview (first {min(len(preview), 1500)} chars)\n"
f"{preview[:1500]}"
),
limit=MAX_TEXT_CHARS,
)
def _decode_text_bytes(data: bytes) -> str:
try:
return data.decode("utf-8")
@ -120,6 +230,26 @@ def _attachments_limits() -> dict[str, Any]:
"tool_mode_max_bytes": _safe_int(
tabular_cfg.get("tool_mode_max_bytes"), DEFAULT_TABULAR_TOOL_MODE_MAX_BYTES, max_value=500 * 1024 * 1024
),
"text_inline_max_chars": _safe_int(
tabular_cfg.get("text_inline_max_chars"), DEFAULT_TEXT_INLINE_MAX_CHARS, max_value=200_000
),
"text_chunk_size": _safe_int(tabular_cfg.get("text_chunk_size"), DEFAULT_TEXT_CHUNK_SIZE, max_value=8_000),
"text_chunk_overlap": _safe_int(
tabular_cfg.get("text_chunk_overlap"), DEFAULT_TEXT_CHUNK_OVERLAP, max_value=4_000
),
"text_query_top_k": _safe_int(tabular_cfg.get("text_query_top_k"), DEFAULT_TEXT_QUERY_TOP_K, max_value=50),
"archive_max_depth": _safe_int(tabular_cfg.get("archive_max_depth"), MAX_ARCHIVE_DEPTH, max_value=10),
"archive_max_file_count": _safe_int(
tabular_cfg.get("archive_max_file_count"), MAX_ARCHIVE_FILE_COUNT, max_value=20_000
),
"archive_max_entry_bytes": _safe_int(
tabular_cfg.get("archive_max_entry_bytes"), MAX_ARCHIVE_ENTRY_BYTES, max_value=2_000_000_000
),
"archive_max_total_uncompressed_bytes": _safe_int(
tabular_cfg.get("archive_max_total_uncompressed_bytes"),
MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES,
max_value=5_000_000_000,
),
}
@ -160,6 +290,11 @@ def _dataframe_summary_text(
dtypes = [f"{str(k)}:{str(v)}" for k, v in df.dtypes.items()]
show_rows = max(1, int(preview_rows or EXCEL_PREVIEW_ROWS))
sample = df.head(show_rows).to_string(index=False)
rows_scope_line = (
f"## Preview (first {min(rows, show_rows)} rows)"
if rows > show_rows
else f"## Full table included ({rows} rows)"
)
body = (
f"# Table Summary\n"
f"- rows: {rows}\n"
@ -169,7 +304,7 @@ def _dataframe_summary_text(
f"- clipped_cells: {'yes' if clipped_cells else 'no'}\n"
f"- columns: {', '.join(header)}\n"
f"- dtypes: {', '.join(dtypes)}\n\n"
f"## Preview (first {min(rows, show_rows)} rows)\n"
f"{rows_scope_line}\n"
f"{sample}"
)
return _truncate_text(body)
@ -191,19 +326,7 @@ def _pdf_summary_text(reader: PdfReader) -> str:
def _is_safe_zip_member_name(name: str) -> bool:
n = str(name or "").replace("\\", "/").strip()
if not n:
return False
if len(n) > MAX_ZIP_MEMBER_NAME_LENGTH:
return False
if any(ord(ch) < 32 for ch in n):
return False
p = PurePosixPath(n)
if p.is_absolute():
return False
if any(part in ("..", "") for part in p.parts):
return False
return True
return is_safe_archive_member_name(name)
def _validate_excel_zip_payload(data: bytes) -> str | None:
@ -231,48 +354,21 @@ def _validate_excel_zip_payload(data: bytes) -> str | None:
def process_zip(zip_data: bytes, *, _depth: int = 0) -> list[dict[str, Any]]:
if _depth > MAX_ZIP_DEPTH:
return [
{
"type": "text",
"name": "zip-error",
"content": f"ZIP nesting too deep (>{MAX_ZIP_DEPTH})",
}
]
attachments: list[dict[str, Any]] = []
try:
with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
infos = [x for x in zf.infolist() if not x.is_dir()]
if len(infos) > MAX_ZIP_FILE_COUNT:
raise ValueError(f"ZIP has too many files ({len(infos)} > {MAX_ZIP_FILE_COUNT})")
total_uncompressed = 0
for info in infos:
if not _is_safe_zip_member_name(info.filename):
raise ValueError(f"ZIP contains unsafe path: {info.filename}")
total_uncompressed += int(info.file_size or 0)
if int(info.file_size or 0) > MAX_ZIP_ENTRY_BYTES:
raise ValueError(
f"ZIP entry too large: {info.filename} ({int(info.file_size or 0)} > {MAX_ZIP_ENTRY_BYTES})"
)
if total_uncompressed > MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES:
raise ValueError(
"ZIP total uncompressed size too large "
f"({total_uncompressed} > {MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES})"
)
with zf.open(info) as f:
data = f.read()
att = process_file_data(info.filename, data, _zip_depth=_depth + 1)
if att:
attachments.extend(att)
except Exception as e:
attachments.append(
{
"type": "text",
"name": "zip-error",
"content": f"Error parsing zip: {e}",
}
)
return attachments
limits = _attachments_limits()
return process_archive(
archive_name="bundle.zip",
archive_data=zip_data,
process_member=lambda n, b, d: process_file_data(n, b, _zip_depth=d),
depth=_depth,
limits={
"max_depth": int(limits.get("archive_max_depth") or MAX_ARCHIVE_DEPTH),
"max_file_count": int(limits.get("archive_max_file_count") or MAX_ARCHIVE_FILE_COUNT),
"max_entry_bytes": int(limits.get("archive_max_entry_bytes") or MAX_ARCHIVE_ENTRY_BYTES),
"max_total_uncompressed_bytes": int(
limits.get("archive_max_total_uncompressed_bytes") or MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES
),
},
)
def process_file_data(name: str, data: bytes, *, _zip_depth: int = 0) -> list[dict[str, Any]]:
@ -336,6 +432,67 @@ def process_file_data(name: str, data: bytes, *, _zip_depth: int = 0) -> list[di
)
return attachments
if ext in ("mp4", "mov", "mkv", "webm", "avi", "m4v"):
mime = "video/mp4"
if ext == "webm":
mime = "video/webm"
elif ext == "mov":
mime = "video/quicktime"
elif ext == "mkv":
mime = "video/x-matroska"
elif ext == "avi":
mime = "video/x-msvideo"
meta = AttachmentAssetStore().save_bytes(data, filename=name, mime=mime)
probe_meta: dict[str, Any] = {}
try:
p = AttachmentAssetStore().get_local_path(str(meta.attachment_id))
if p is not None:
probe_meta = _ffprobe_video_meta(p)
except Exception:
probe_meta = {}
attachments.append(
{
"type": "video_ref",
"name": meta.name,
"mime": meta.mime,
"attachment_id": meta.attachment_id,
"bytes": meta.bytes,
**probe_meta,
}
)
attachments.append(
{
"type": "text",
"name": name,
"content": _truncate_text(
"".join(
[
"# Video Attachment\n",
f"- name: {meta.name}\n",
f"- mime: {meta.mime}\n",
f"- bytes: {meta.bytes}\n",
(
f"- duration_sec: {probe_meta.get('duration_sec')}\n"
if probe_meta.get("duration_sec") is not None
else ""
),
(
f"- size: {probe_meta.get('width')}x{probe_meta.get('height')}\n"
if probe_meta.get("width") and probe_meta.get("height")
else ""
),
(f"- fps: {probe_meta.get('fps')}\n" if probe_meta.get("fps") is not None else ""),
"- mode: video_ref\n",
"- query_tool: query_video_attachment\n",
"- note: use `attachment_id` from accompanying `video_ref` attachment.\n",
]
),
limit=MAX_TEXT_CHARS,
),
}
)
return attachments
if ext in ("xlsx", "xls", "csv"):
try:
if ext == "xlsx":
@ -453,14 +610,97 @@ def process_file_data(name: str, data: bytes, *, _zip_depth: int = 0) -> list[di
try:
doc = Document(io.BytesIO(data))
text = "\n".join([p.text for p in doc.paragraphs])
attachments.append({"type": "text", "name": name, "content": _truncate_text(text)})
limits = _attachments_limits()
inline_cap = int(limits.get("text_inline_max_chars") or DEFAULT_TEXT_INLINE_MAX_CHARS)
if len(str(text or "")) <= inline_cap:
attachments.append({"type": "text", "name": name, "content": _truncate_text(text, limit=inline_cap)})
else:
meta = AttachmentAssetStore().save_bytes(
data,
filename=name,
mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
)
text_meta = save_text_document(
attachment_id=str(meta.attachment_id),
name=name,
text=str(text or ""),
source_kind="docx",
chunk_size=int(limits.get("text_chunk_size") or DEFAULT_TEXT_CHUNK_SIZE),
chunk_overlap=int(limits.get("text_chunk_overlap") or DEFAULT_TEXT_CHUNK_OVERLAP),
)
top_k = int(limits.get("text_query_top_k") or DEFAULT_TEXT_QUERY_TOP_K)
attachments.append(
{
"type": "text",
"name": name,
"content": _text_ref_summary(
name=name,
chars=int(text_meta.get("chars") or 0),
chunks=int(text_meta.get("chunks") or 0),
source_kind="docx",
preview=str(text or ""),
top_k=top_k,
),
}
)
attachments.append(
{
"type": "text_ref",
"name": str(name),
"attachment_id": str(meta.attachment_id),
"text_id": str(text_meta.get("text_id") or ""),
"chars": int(text_meta.get("chars") or 0),
"chunks": int(text_meta.get("chunks") or 0),
"source_kind": "docx",
}
)
except Exception as e:
attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing docx: {e}")})
elif ext == "pdf":
try:
reader = PdfReader(io.BytesIO(data))
attachments.append({"type": "text", "name": name, "content": _pdf_summary_text(reader)})
text = _pdf_summary_text(reader)
limits = _attachments_limits()
inline_cap = int(limits.get("text_inline_max_chars") or DEFAULT_TEXT_INLINE_MAX_CHARS)
if len(str(text or "")) <= inline_cap:
attachments.append({"type": "text", "name": name, "content": _truncate_text(text, limit=inline_cap)})
else:
meta = AttachmentAssetStore().save_bytes(data, filename=name, mime="application/pdf")
text_meta = save_text_document(
attachment_id=str(meta.attachment_id),
name=name,
text=str(text or ""),
source_kind="pdf",
chunk_size=int(limits.get("text_chunk_size") or DEFAULT_TEXT_CHUNK_SIZE),
chunk_overlap=int(limits.get("text_chunk_overlap") or DEFAULT_TEXT_CHUNK_OVERLAP),
)
top_k = int(limits.get("text_query_top_k") or DEFAULT_TEXT_QUERY_TOP_K)
attachments.append(
{
"type": "text",
"name": name,
"content": _text_ref_summary(
name=name,
chars=int(text_meta.get("chars") or 0),
chunks=int(text_meta.get("chunks") or 0),
source_kind="pdf",
preview=str(text or ""),
top_k=top_k,
),
}
)
attachments.append(
{
"type": "text_ref",
"name": str(name),
"attachment_id": str(meta.attachment_id),
"text_id": str(text_meta.get("text_id") or ""),
"chars": int(text_meta.get("chars") or 0),
"chunks": int(text_meta.get("chunks") or 0),
"source_kind": "pdf",
}
)
except Exception as e:
attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing pdf: {e}")})
@ -468,10 +708,65 @@ def process_file_data(name: str, data: bytes, *, _zip_depth: int = 0) -> list[di
text = _decode_text_bytes(data)
if ext == "html":
text = _clean_html_to_text(text)
attachments.append({"type": "text", "name": name, "content": _truncate_text(text)})
limits = _attachments_limits()
inline_cap = int(limits.get("text_inline_max_chars") or DEFAULT_TEXT_INLINE_MAX_CHARS)
if len(str(text or "")) <= inline_cap:
attachments.append({"type": "text", "name": name, "content": _truncate_text(text, limit=inline_cap)})
else:
meta = AttachmentAssetStore().save_bytes(data, filename=name, mime="text/plain")
text_meta = save_text_document(
attachment_id=str(meta.attachment_id),
name=name,
text=str(text or ""),
source_kind=ext or "text",
chunk_size=int(limits.get("text_chunk_size") or DEFAULT_TEXT_CHUNK_SIZE),
chunk_overlap=int(limits.get("text_chunk_overlap") or DEFAULT_TEXT_CHUNK_OVERLAP),
)
top_k = int(limits.get("text_query_top_k") or DEFAULT_TEXT_QUERY_TOP_K)
attachments.append(
{
"type": "text",
"name": name,
"content": _text_ref_summary(
name=name,
chars=int(text_meta.get("chars") or 0),
chunks=int(text_meta.get("chunks") or 0),
source_kind=str(text_meta.get("source_kind") or ext or "text"),
preview=str(text or ""),
top_k=top_k,
),
}
)
attachments.append(
{
"type": "text_ref",
"name": str(name),
"attachment_id": str(meta.attachment_id),
"text_id": str(text_meta.get("text_id") or ""),
"chars": int(text_meta.get("chars") or 0),
"chunks": int(text_meta.get("chunks") or 0),
"source_kind": str(text_meta.get("source_kind") or ext or "text"),
}
)
elif ext == "zip":
attachments.extend(process_zip(data, _depth=_zip_depth))
elif detect_archive_kind(name):
limits = _attachments_limits()
attachments.extend(
process_archive(
archive_name=name,
archive_data=data,
process_member=lambda n, b, d: process_file_data(n, b, _zip_depth=d),
depth=_zip_depth,
limits={
"max_depth": int(limits.get("archive_max_depth") or MAX_ARCHIVE_DEPTH),
"max_file_count": int(limits.get("archive_max_file_count") or MAX_ARCHIVE_FILE_COUNT),
"max_entry_bytes": int(limits.get("archive_max_entry_bytes") or MAX_ARCHIVE_ENTRY_BYTES),
"max_total_uncompressed_bytes": int(
limits.get("archive_max_total_uncompressed_bytes") or MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES
),
},
)
)
return attachments

83
platform/files/tabular_attachment_store.py
View file

@ -35,6 +35,11 @@ def _meta_path(table_id: str) -> Path:
return _tabular_root() / f"{table_id}.meta.json"
def _is_valid_table_id(table_id: str) -> bool:
tid = str(table_id or "").strip().lower()
return bool(re.fullmatch(r"[0-9a-f]{64}", tid))
def _normalize_columns(raw_cols: list[Any]) -> tuple[list[str], list[dict[str, Any]]]:
out: list[str] = []
mapping: list[dict[str, Any]] = []
@ -311,7 +316,10 @@ def save_workbook(*, attachment_id: str, name: str, sheets: dict[str, pd.DataFra
def get_table_meta(table_id: str) -> dict[str, Any] | None:
p = _meta_path(str(table_id or "").strip())
tid = str(table_id or "").strip()
if not _is_valid_table_id(tid):
return None
p = _meta_path(tid)
if not p.exists():
return None
try:
@ -352,6 +360,29 @@ def _columns_for_sheet(meta: dict[str, Any], table_name: str) -> list[str]:
return []
def _sheet_hint_payload(meta: dict[str, Any], *, selected_sheet: str | None, table_name: str) -> dict[str, Any] | None:
sheets = meta.get("sheets") if isinstance(meta.get("sheets"), list) else []
valid_rows = [x for x in sheets if isinstance(x, dict)]
if len(valid_rows) <= 1:
return None
if str(selected_sheet or "").strip():
return None
names = [str((x or {}).get("sheet_name") or "") for x in valid_rows]
names = [x for x in names if x]
if len(names) <= 1:
return None
default_sheet_name = ""
for row in valid_rows:
if str(row.get("table_name") or "") == str(table_name):
default_sheet_name = str(row.get("sheet_name") or "")
break
return {
"available_sheets": names[:20],
"default_sheet": default_sheet_name,
"note": "Workbook contains multiple sheets; pass `sheet` to target a specific sheet.",
}
def query_table(
*,
table_id: str,
@ -361,6 +392,12 @@ def query_table(
where_contains: dict[str, str] | None = None,
sheet: str | None = None,
) -> dict[str, Any]:
if not _is_valid_table_id(table_id):
return {
"ok": False,
"error": "table_id_invalid_format",
"hint": "Use table_id from tabular_ref attachment (64-char hex id).",
}
meta = get_table_meta(table_id)
if not isinstance(meta, dict):
return {"ok": False, "error": "table_not_found"}
@ -392,6 +429,7 @@ def query_table(
out_rows, engine = _run_select_sql(db_path=db, sql=sql, params=params)
with sqlite3.connect(str(db)) as conn:
total = int(conn.execute(f'SELECT COUNT(1) AS n FROM "{table_name}"').fetchone()[0] or 0)
hint = _sheet_hint_payload(meta, selected_sheet=sheet, table_name=table_name)
return {
"ok": True,
"table_id": str(table_id),
@ -404,6 +442,7 @@ def query_table(
"limit": lim,
"offset": off,
"engine": engine,
**({"sheet_hint": hint} if isinstance(hint, dict) else {}),
}
@ -417,6 +456,12 @@ def aggregate_table(
top_n: int = 20,
sheet: str | None = None,
) -> dict[str, Any]:
if not _is_valid_table_id(table_id):
return {
"ok": False,
"error": "table_id_invalid_format",
"hint": "Use table_id from tabular_ref attachment (64-char hex id).",
}
meta = get_table_meta(table_id)
if not isinstance(meta, dict):
return {"ok": False, "error": "table_not_found"}
@ -465,7 +510,7 @@ def aggregate_table(
"value": float(r.get("metric_value") or 0.0),
}
)
return {
out = {
"ok": True,
"table_id": str(table_id),
"sheet": str(sheet or ""),
@ -477,10 +522,14 @@ def aggregate_table(
"top_n": lim,
"engine": engine,
}
hint = _sheet_hint_payload(meta, selected_sheet=sheet, table_name=table_name)
if isinstance(hint, dict):
out["sheet_hint"] = hint
return out
sql = f'SELECT {metric_sql} AS metric_value FROM "{table_name}"{where_sql} '
raw_rows, engine = _run_select_sql(db_path=db, sql=sql, params=params)
row = raw_rows[0] if raw_rows else {}
return {
out = {
"ok": True,
"table_id": str(table_id),
"sheet": str(sheet or ""),
@ -491,6 +540,10 @@ def aggregate_table(
"value": float((row.get("metric_value") if isinstance(row, dict) else 0.0) or 0.0),
"engine": engine,
}
hint = _sheet_hint_payload(meta, selected_sheet=sheet, table_name=table_name)
if isinstance(hint, dict):
out["sheet_hint"] = hint
return out
def run_table_sql(
@ -500,6 +553,12 @@ def run_table_sql(
limit: int = 200,
sheet: str | None = None,
) -> dict[str, Any]:
if not _is_valid_table_id(table_id):
return {
"ok": False,
"error": "table_id_invalid_format",
"hint": "Use table_id from tabular_ref attachment (64-char hex id).",
}
meta = get_table_meta(table_id)
if not isinstance(meta, dict):
return {"ok": False, "error": "table_not_found"}
@ -547,7 +606,7 @@ def run_table_sql(
},
}
out_rows = rows[:lim]
return {
out = {
"ok": True,
"table_id": str(table_id),
"name": str(meta.get("name") or ""),
@ -568,6 +627,10 @@ def run_table_sql(
"limit": lim,
"engine": engine,
}
hint = _sheet_hint_payload(meta, selected_sheet=sheet, table_name=table_name)
if isinstance(hint, dict):
out["sheet_hint"] = hint
return out
def analyze_table_full_scan(
@ -577,6 +640,12 @@ def analyze_table_full_scan(
sheet: str | None = None,
top_values_limit: int = 3,
) -> dict[str, Any]:
if not _is_valid_table_id(table_id):
return {
"ok": False,
"error": "table_id_invalid_format",
"hint": "Use table_id from tabular_ref attachment (64-char hex id).",
}
meta = get_table_meta(table_id)
if not isinstance(meta, dict):
return {"ok": False, "error": "table_not_found"}
@ -628,7 +697,7 @@ def analyze_table_full_scan(
item["top_values"] = [{"value": str(r[0]), "count": int(r[1] or 0)} for r in tops]
col_stats.append(item)
elapsed_ms = int((time.perf_counter() - started) * 1000)
return {
out = {
"ok": True,
"table_id": str(table_id),
"name": str(meta.get("name") or ""),
@ -645,6 +714,10 @@ def analyze_table_full_scan(
},
"engine": "builtin_sqlite_fullscan",
}
hint = _sheet_hint_payload(meta, selected_sheet=sheet, table_name=table_name)
if isinstance(hint, dict):
out["sheet_hint"] = hint
return out
__all__ = [

197
platform/files/text_attachment_store.py Normal file
View file

@ -0,0 +1,197 @@
from __future__ import annotations
import hashlib
import json
import re
import sqlite3
from pathlib import Path
from typing import Any
from oclaw.platform.config.paths import attachments_dir
DEFAULT_TEXT_INLINE_MAX_CHARS = 12_000
DEFAULT_TEXT_CHUNK_SIZE = 1_600
DEFAULT_TEXT_CHUNK_OVERLAP = 200
def _text_root() -> Path:
p = (attachments_dir() / "textual").resolve()
p.mkdir(parents=True, exist_ok=True)
return p
def _text_id(attachment_id: str, name: str) -> str:
raw = f"{attachment_id}:{name}".encode("utf-8", errors="ignore")
return hashlib.sha256(raw).hexdigest()
def _db_path(text_id: str) -> Path:
return _text_root() / f"{text_id}.sqlite"
def _meta_path(text_id: str) -> Path:
return _text_root() / f"{text_id}.meta.json"
def _is_valid_text_id(text_id: str) -> bool:
tid = str(text_id or "").strip().lower()
return bool(re.fullmatch(r"[0-9a-f]{64}", tid))
def _chunk_text(*, text: str, chunk_size: int, overlap: int) -> list[dict[str, Any]]:
body = str(text or "")
if not body:
return []
size = max(200, min(int(chunk_size or DEFAULT_TEXT_CHUNK_SIZE), 8_000))
ov = max(0, min(int(overlap or DEFAULT_TEXT_CHUNK_OVERLAP), size - 1))
chunks: list[dict[str, Any]] = []
i = 0
idx = 0
n = len(body)
while i < n:
end = min(n, i + size)
chunk = body[i:end]
chunks.append({"chunk_index": idx, "start_char": i, "end_char": end, "content": chunk})
if end >= n:
break
i = end - ov
idx += 1
return chunks
def save_text_document(
*,
attachment_id: str,
name: str,
text: str,
source_kind: str,
chunk_size: int = DEFAULT_TEXT_CHUNK_SIZE,
chunk_overlap: int = DEFAULT_TEXT_CHUNK_OVERLAP,
) -> dict[str, Any]:
tid = _text_id(attachment_id=attachment_id, name=name)
db = _db_path(tid)
meta = _meta_path(tid)
chunks = _chunk_text(text=str(text or ""), chunk_size=chunk_size, overlap=chunk_overlap)
with sqlite3.connect(str(db)) as conn:
conn.execute("DROP TABLE IF EXISTS text_chunks")
conn.execute(
(
"CREATE TABLE text_chunks ("
"chunk_index INTEGER NOT NULL, "
"start_char INTEGER NOT NULL, "
"end_char INTEGER NOT NULL, "
"content TEXT NOT NULL)"
)
)
for c in chunks:
conn.execute(
"INSERT INTO text_chunks(chunk_index,start_char,end_char,content) VALUES(?,?,?,?)",
[
int(c.get("chunk_index") or 0),
int(c.get("start_char") or 0),
int(c.get("end_char") or 0),
str(c.get("content") or ""),
],
)
conn.commit()
payload = {
"text_id": tid,
"attachment_id": str(attachment_id or ""),
"name": str(name or ""),
"source_kind": str(source_kind or "text"),
"chars": int(len(str(text or ""))),
"chunks": int(len(chunks)),
"chunk_size": int(chunk_size),
"chunk_overlap": int(chunk_overlap),
"db_path": str(db),
}
meta.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
return payload
def get_text_meta(text_id: str) -> dict[str, Any] | None:
tid = str(text_id or "").strip()
if not _is_valid_text_id(tid):
return None
p = _meta_path(tid)
if not p.exists():
return None
try:
obj = json.loads(p.read_text(encoding="utf-8"))
return obj if isinstance(obj, dict) else None
except Exception:
return None
def query_text_document(
*,
text_id: str,
query: str | None = None,
top_k: int = 5,
offset: int = 0,
) -> dict[str, Any]:
tid = str(text_id or "").strip()
if not _is_valid_text_id(tid):
return {"ok": False, "error": "text_id_invalid_format"}
meta = get_text_meta(tid)
if not isinstance(meta, dict):
return {"ok": False, "error": "text_not_found"}
db = Path(str(meta.get("db_path") or ""))
if not db.exists():
return {"ok": False, "error": "text_store_missing"}
lim = max(1, min(int(top_k or 5), 50))
off = max(0, int(offset or 0))
q = str(query or "").strip()
rows: list[dict[str, Any]] = []
with sqlite3.connect(str(db)) as conn:
conn.row_factory = sqlite3.Row
total = int(conn.execute("SELECT COUNT(1) AS n FROM text_chunks").fetchone()[0] or 0)
if q:
q_like = f"%{q.lower()}%"
sql = (
"SELECT chunk_index,start_char,end_char,content, "
"(LENGTH(LOWER(content)) - LENGTH(REPLACE(LOWER(content), LOWER(?), ''))) / MAX(1, LENGTH(?)) AS score "
"FROM text_chunks WHERE LOWER(content) LIKE ? "
"ORDER BY score DESC, chunk_index ASC LIMIT ? OFFSET ?"
)
out = conn.execute(sql, [q, q, q_like, lim, off]).fetchall()
else:
out = conn.execute(
"SELECT chunk_index,start_char,end_char,content, 0 AS score FROM text_chunks ORDER BY chunk_index ASC LIMIT ? OFFSET ?",
[lim, off],
).fetchall()
for r in out:
rows.append(
{
"chunk_index": int(r["chunk_index"] or 0),
"start_char": int(r["start_char"] or 0),
"end_char": int(r["end_char"] or 0),
"content": str(r["content"] or ""),
"score": float(r["score"] or 0.0),
}
)
return {
"ok": True,
"text_id": tid,
"name": str(meta.get("name") or ""),
"source_kind": str(meta.get("source_kind") or "text"),
"chars": int(meta.get("chars") or 0),
"chunks_total": int(meta.get("chunks") or 0),
"rows_total": int(total),
"query": q,
"rows": rows,
"top_k": lim,
"offset": off,
}
__all__ = [
"DEFAULT_TEXT_INLINE_MAX_CHARS",
"DEFAULT_TEXT_CHUNK_SIZE",
"DEFAULT_TEXT_CHUNK_OVERLAP",
"save_text_document",
"get_text_meta",
"query_text_document",
]