from __future__ import annotations
import io
import json
import gzip
import tarfile
import zipfile
from pathlib import Path
import pandas as pd
import svc.files.file_attachments as fa
from svc.files.file_attachments import process_file_data
from svc.files.tabular_attachment_store import (
aggregate_table,
analyze_table_full_scan,
query_table,
run_table_sql,
save_workbook,
)
from svc.files.text_attachment_store import query_text_document
def _zip_bytes(files: dict[str, bytes]) -> bytes:
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
for name, data in files.items():
zf.writestr(name, data)
return buf.getvalue()
def _tar_bytes(files: dict[str, bytes], *, gz: bool = False) -> bytes:
buf = io.BytesIO()
mode = "w:gz" if gz else "w"
with tarfile.open(fileobj=buf, mode=mode) as tf:
for name, data in files.items():
ti = tarfile.TarInfo(name=name)
ti.size = len(data)
tf.addfile(ti, io.BytesIO(data))
return buf.getvalue()
def test_csv_is_summarized_not_full_dump() -> None:
payload = (
"c1,c2,c3\n"
"1,2,3\n"
"4,5,6\n"
).encode("utf-8")
out = process_file_data("table.csv", payload)
assert out and out[0]["type"] == "text"
content = str(out[0].get("content") or "")
assert "# Table Summary" in content
assert "rows: 2" in content
assert "cols: 3" in content
assert "## Full table included (2 rows)" in content
def test_long_txt_emits_text_ref_and_can_query() -> None:
payload = ("A" * 15000 + "\nneedle\n" + "B" * 2000).encode("utf-8")
out = process_file_data("notes.txt", payload)
text_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "text_ref"]
assert text_refs
text_id = str(text_refs[0].get("text_id") or "")
assert text_id
got = query_text_document(text_id=text_id, query="needle", top_k=3, offset=0)
assert bool(got.get("ok"))
rows = got.get("rows") or []
assert rows
assert any("needle" in str(x.get("content") or "") for x in rows)
def test_video_upload_emits_video_ref() -> None:
# Minimal MP4 header-like bytes; parser should not crash and should still store as video_ref.
payload = b"\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom"
out = process_file_data("clip.mp4", payload)
vrefs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "video_ref"]
assert vrefs
vr = vrefs[0]
assert str(vr.get("attachment_id") or "")
assert str(vr.get("mime") or "").startswith("video/")
def test_zip_file_count_limit_returns_error_attachment() -> None:
files = {f"f{i}.txt": b"x" for i in range(205)}
out = process_file_data("bulk.zip", _zip_bytes(files))
assert out
first = out[0]
assert str(first.get("name") or "") == "zip-error"
assert "too many files" in str(first.get("content") or "").lower()
def test_tar_path_traversal_is_blocked() -> None:
payload = _tar_bytes({"../evil.txt": b"x"})
out = process_file_data("unsafe.tar", payload)
assert out
first = out[0]
assert str(first.get("name") or "") == "archive-error"
assert "unsafe path" in str(first.get("content") or "").lower()
def test_tgz_is_parsed_and_members_processed() -> None:
payload = _tar_bytes({"ok.txt": b"hello"}, gz=True)
out = process_file_data("bundle.tgz", payload)
assert out
assert any(str(x.get("type") or "") == "text" and "hello" in str(x.get("content") or "") for x in out)
def test_tar_link_entry_is_explicitly_rejected() -> None:
buf = io.BytesIO()
with tarfile.open(fileobj=buf, mode="w") as tf:
ti = tarfile.TarInfo(name="target.txt")
data = b"ok"
ti.size = len(data)
tf.addfile(ti, io.BytesIO(data))
lnk = tarfile.TarInfo(name="sym")
lnk.type = tarfile.SYMTYPE
lnk.linkname = "target.txt"
tf.addfile(lnk)
out = process_file_data("has-link.tar", buf.getvalue())
assert out
first = out[0]
assert str(first.get("name") or "") == "archive-error"
assert str(first.get("error_code") or "") == "archive_link_entry_forbidden"
def test_gz_single_file_is_decompressed_and_processed() -> None:
payload = gzip.compress(b"hello-gz")
out = process_file_data("note.txt.gz", payload)
assert out
assert any(str(x.get("type") or "") == "text" and "hello-gz" in str(x.get("content") or "") for x in out)
def test_rar_returns_explicit_unsupported_error() -> None:
out = process_file_data("bundle.rar", b"not-a-real-rar")
assert out
first = out[0]
assert str(first.get("name") or "") == "archive-error"
assert "unsupported archive format" in str(first.get("content") or "").lower()
def test_7z_returns_explicit_unsupported_error() -> None:
out = process_file_data("bundle.7z", b"not-a-real-7z")
assert out
first = out[0]
assert str(first.get("name") or "") == "archive-error"
assert "unsupported archive format" in str(first.get("content") or "").lower()
assert str(first.get("error_code") or "") == "archive_unsupported_format"
def test_archive_signature_detection_overrides_misleading_extension() -> None:
payload = _zip_bytes({"ok.txt": b"sig-ok"})
out = process_file_data("misleading.tar", payload)
assert out
assert any(str(x.get("type") or "") == "text" and "sig-ok" in str(x.get("content") or "") for x in out)
def test_archive_limits_can_be_overridden_by_config(tmp_path: Path, monkeypatch) -> None:
cfg = {
"plugins": {
"entries": {
"memory-wiki": {
"auto": {
"attachments": {
"tabular": {
"archive_max_depth": 1,
"archive_max_file_count": 2,
"archive_max_entry_bytes": 20,
"archive_max_total_uncompressed_bytes": 30,
}
}
}
}
}
}
}
cfg_path = tmp_path / "oclaw.json"
cfg_path.write_text(json.dumps(cfg), encoding="utf-8")
monkeypatch.setenv("AIA_OCLAW_CONFIG_PATH", str(cfg_path))
fa._attachments_limits.cache_clear()
try:
payload = _zip_bytes({"a.txt": b"x" * 50})
out = process_file_data("limited.zip", payload)
finally:
fa._attachments_limits.cache_clear()
assert out
first = out[0]
assert "entry too large" in str(first.get("content") or "").lower()
def test_nested_zip_depth_limit_is_enforced() -> None:
z4 = _zip_bytes({"deep.txt": b"hello"})
z3 = _zip_bytes({"z4.zip": z4})
z2 = _zip_bytes({"z3.zip": z3})
z1 = _zip_bytes({"z2.zip": z2})
out = process_file_data("z1.zip", z1)
assert any("nesting too deep" in str(x.get("content") or "").lower() for x in out)
def test_html_is_cleaned_before_attachment_text() -> None:
html = b"
Title
Hello
"
out = process_file_data("page.html", html)
content = str((out[0] if out else {}).get("content") or "")
assert "Title" in content
assert "Hello" in content
assert "alert(1)" not in content
assert "" not in content
def test_pdf_summary_contains_page_markers() -> None:
class _Page:
def __init__(self, text: str) -> None:
self._text = text
def extract_text(self) -> str:
return self._text
class _Reader:
def __init__(self, _stream) -> None:
self.pages = [_Page("P1"), _Page("P2")]
old = fa.PdfReader
try:
fa.PdfReader = _Reader # type: ignore[assignment]
out = process_file_data("doc.pdf", b"%PDF")
finally:
fa.PdfReader = old # type: ignore[assignment]
content = str((out[0] if out else {}).get("content") or "")
assert "# PDF Summary" in content
assert "pages: 2" in content
assert "## Page 1" in content
assert "## Page 2" in content
def test_zip_unsafe_member_path_is_blocked() -> None:
payload = _zip_bytes({"../evil.txt": b"x"})
out = process_file_data("unsafe.zip", payload)
assert out
first = out[0]
assert str(first.get("name") or "") == "zip-error"
assert "unsafe path" in str(first.get("content") or "").lower()
def test_csv_marks_sampled_when_row_limit_hit() -> None:
rows = ["c1,c2"] + [f"{i},{i+1}" for i in range(0, 6000)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("big.csv", payload)
content = str((out[0] if out else {}).get("content") or "")
assert "sampled: yes" in content
def test_zip_member_name_too_long_is_blocked() -> None:
payload = _zip_bytes({("a" * 300) + ".txt": b"x"})
out = process_file_data("unsafe.zip", payload)
assert out
first = out[0]
assert str(first.get("name") or "") == "zip-error"
assert "unsafe path" in str(first.get("content") or "").lower()
def test_csv_marks_clipped_columns_when_width_limit_hit() -> None:
columns = [f"c{i}" for i in range(0, 260)]
header = ",".join(columns)
row = ",".join(["1"] * len(columns))
payload = (header + "\n" + row + "\n").encode("utf-8")
out = process_file_data("wide.csv", payload)
content = str((out[0] if out else {}).get("content") or "")
assert "clipped_columns: yes" in content
assert "cols: 200" in content
def test_csv_marks_clipped_cells_when_cell_content_too_long() -> None:
long_cell = "x" * 1200
payload = f"c1,c2\n{long_cell},ok\n".encode("utf-8")
out = process_file_data("long-cell.csv", payload)
content = str((out[0] if out else {}).get("content") or "")
assert "clipped_cells: yes" in content
assert "...[cell-truncated]" in content
def test_tabular_limits_can_be_overridden_by_config(tmp_path: Path, monkeypatch) -> None:
cfg = {
"plugins": {
"entries": {
"memory-wiki": {
"auto": {
"attachments": {
"tabular": {
"max_rows_read": 2,
"max_columns": 2,
"max_cell_chars": 10,
"large_table_preview_rows": 1,
"tool_mode_enabled": True,
"tool_mode_min_rows": 3,
"tool_mode_max_bytes": 1024 * 1024,
}
}
}
}
}
}
}
cfg_path = tmp_path / "oclaw.json"
cfg_path.write_text(json.dumps(cfg), encoding="utf-8")
monkeypatch.setenv("AIA_OCLAW_CONFIG_PATH", str(cfg_path))
fa._attachments_limits.cache_clear()
try:
payload = "c1,c2,c3\n" + ("x" * 40) + ",b,c\n1,2,3\n4,5,6\n"
out = process_file_data("configured.csv", payload.encode("utf-8"))
finally:
fa._attachments_limits.cache_clear()
content = str((out[0] if out else {}).get("content") or "")
assert "sampled: yes" in content
assert "clipped_columns: yes" in content
assert "clipped_cells: yes" in content
assert "Preview (first 1 rows)" in content
def test_large_csv_emits_tabular_ref_and_can_query() -> None:
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25050)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("large.csv", payload)
assert out
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
table_id = str(tab_refs[0].get("table_id") or "")
assert table_id
got = query_table(
table_id=table_id,
columns=["c1", "c2"],
limit=5,
offset=0,
where_contains={"column": "c2", "keyword": "row-12"},
)
assert bool(got.get("ok"))
assert str(got.get("table_id") or "") == table_id
assert str(got.get("engine") or "") in {"builtin_sqlite", "mcp_sqlite"}
def test_medium_csv_enters_tool_mode_with_default_threshold(tmp_path: Path, monkeypatch) -> None:
"""Isolate from repo ``oclaw.json`` (may raise ``tool_mode_min_rows`` above this test's row count)."""
cfg = {
"plugins": {
"entries": {
"memory-wiki": {
"auto": {
"attachments": {
"tabular": {
"tool_mode_enabled": True,
"tool_mode_min_rows": 5000,
"tool_mode_max_bytes": 30 * 1024 * 1024,
}
}
}
}
}
}
}
cfg_path = tmp_path / "oclaw.json"
cfg_path.write_text(json.dumps(cfg), encoding="utf-8")
monkeypatch.setenv("AIA_OCLAW_CONFIG_PATH", str(cfg_path))
fa._attachments_limits.cache_clear()
try:
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 6000)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("medium-default.csv", payload)
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
finally:
fa._attachments_limits.cache_clear()
def test_large_csv_can_aggregate_grouped_sum() -> None:
rows = ["dept,amount"] + [f"{'A' if i % 2 == 0 else 'B'},{i}" for i in range(0, 25010)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("large2.csv", payload)
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
table_id = str(tab_refs[0].get("table_id") or "")
got = aggregate_table(
table_id=table_id,
metric="sum",
target_column="amount",
group_by="dept",
top_n=5,
)
assert bool(got.get("ok"))
rows_out = got.get("rows") or []
assert isinstance(rows_out, list) and len(rows_out) >= 2
groups = {str(x.get("group") or "") for x in rows_out}
assert "A" in groups and "B" in groups
assert str(got.get("engine") or "") in {"builtin_sqlite", "mcp_sqlite"}
def test_query_falls_back_to_builtin_when_mcp_unavailable(monkeypatch) -> None:
monkeypatch.setenv("AIA_MCP_SQLITE_COMMAND", "nonexistent_mcp_sqlite_binary")
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25020)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("fallback.csv", payload)
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
table_id = str(tab_refs[0].get("table_id") or "")
got = query_table(table_id=table_id, columns=["c1"], limit=3, offset=0)
assert bool(got.get("ok"))
assert str(got.get("engine") or "") == "builtin_sqlite"
def test_run_tabular_sql_select_works_and_blocks_mutation() -> None:
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25020)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("sql.csv", payload)
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
table_id = str(tab_refs[0].get("table_id") or "")
ok = run_table_sql(table_id=table_id, sql='SELECT "c1","c2" FROM rows_data WHERE "c2" LIKE \'%row-12%\'', limit=10)
assert bool(ok.get("ok"))
assert int(ok.get("rows_returned") or 0) >= 1
assert "SELECT" in str(ok.get("executed_sql") or "")
guard = ok.get("sql_guard") or {}
assert bool(guard.get("readonly_enforced"))
assert bool(guard.get("auto_limit_applied"))
bad = run_table_sql(table_id=table_id, sql="DROP TABLE rows_data", limit=10)
assert not bool(bad.get("ok"))
assert str(bad.get("error") or "") == "sql_not_readonly"
def test_run_tabular_sql_timeout_returns_guard(monkeypatch) -> None:
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25020)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("sql-timeout.csv", payload)
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
table_id = str(tab_refs[0].get("table_id") or "")
monkeypatch.setenv("AIA_TABULAR_SQL_TIMEOUT_MS", "100")
slow_sql = (
"WITH RECURSIVE t(n) AS ("
"SELECT 1 UNION ALL SELECT n+1 FROM t WHERE n < 5000000"
") SELECT SUM(n) FROM t"
)
got = run_table_sql(table_id=table_id, sql=slow_sql, limit=200)
assert not bool(got.get("ok"))
assert str(got.get("error") or "") == "sql_timeout"
guard = got.get("sql_guard") or {}
assert bool(guard.get("timeout_hit"))
assert int(guard.get("timeout_ms") or 0) == 100
def test_run_tabular_sql_timeout_reads_oclaw_config(tmp_path: Path, monkeypatch) -> None:
cfg = {
"plugins": {
"entries": {
"memory-wiki": {
"auto": {
"attachments": {
"tabular": {
"sql_timeout_ms": 222,
}
}
}
}
}
}
}
cfg_path = tmp_path / "oclaw.json"
cfg_path.write_text(json.dumps(cfg), encoding="utf-8")
monkeypatch.setenv("AIA_OCLAW_CONFIG_PATH", str(cfg_path))
monkeypatch.delenv("AIA_TABULAR_SQL_TIMEOUT_MS", raising=False)
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25020)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("sql-timeout-config.csv", payload)
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
table_id = str(tab_refs[0].get("table_id") or "")
got = run_table_sql(table_id=table_id, sql='SELECT "c1" FROM rows_data WHERE "c2" LIKE \'%row-12%\'', limit=10)
assert bool(got.get("ok"))
guard = got.get("sql_guard") or {}
assert int(guard.get("timeout_ms") or 0) == 222
def test_save_workbook_normalizes_duplicate_and_blank_headers() -> None:
df = pd.DataFrame([["1", "2", "3"]], columns=["", "重复", "重复"])
meta = save_workbook(attachment_id="aid1", name="dup.xlsx", sheets={"Data": df})
cols = list(meta.get("columns") or [])
assert cols[0].startswith("col_")
assert "重复" in cols
assert "重复__2" in cols
def test_query_can_target_specific_excel_sheet() -> None:
s1 = pd.DataFrame([["a", "1"]], columns=["k", "v"])
s2 = pd.DataFrame([["b", "2"]], columns=["k", "v"])
meta = save_workbook(attachment_id="aid2", name="multi.xlsx", sheets={"S1": s1, "S2": s2})
tid = str(meta.get("table_id") or "")
got = query_table(table_id=tid, sheet="S2", columns=["k", "v"], limit=5, offset=0)
assert bool(got.get("ok"))
rows = got.get("rows") or []
assert rows and str(rows[0].get("k") or "") == "b"
def test_multi_sheet_query_without_sheet_returns_sheet_hint() -> None:
s1 = pd.DataFrame([["a", "1"]], columns=["k", "v"])
s2 = pd.DataFrame([["b", "2"]], columns=["k", "v"])
meta = save_workbook(attachment_id="aid3", name="multi2.xlsx", sheets={"Main": s1, "Backup": s2})
tid = str(meta.get("table_id") or "")
got = query_table(table_id=tid, columns=["k", "v"], limit=5, offset=0)
assert bool(got.get("ok"))
hint = got.get("sheet_hint") or {}
assert isinstance(hint, dict)
names = hint.get("available_sheets") or []
assert "Main" in names and "Backup" in names
assert str(hint.get("default_sheet") or "") == "Main"
def test_full_scan_analyzes_all_rows_and_returns_audit() -> None:
rows = ["dept,score"] + [f"{'A' if i % 2 == 0 else 'B'},{i % 5}" for i in range(0, 25025)]
payload = ("\n".join(rows)).encode("utf-8")
out = process_file_data("fullscan.csv", payload)
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
table_id = str(tab_refs[0].get("table_id") or "")
got = analyze_table_full_scan(table_id=table_id, columns=["dept", "score"], top_values_limit=2)
assert bool(got.get("ok"))
assert int(got.get("rows_scanned") or 0) >= 25025
audit = got.get("scan_audit") or {}
assert bool(audit.get("full_scan"))
assert int(audit.get("rows_scanned") or 0) >= 25025
stats = got.get("column_stats") or []
assert isinstance(stats, list) and len(stats) == 2
dept = [x for x in stats if str(x.get("column") or "") == "dept"]
assert dept
tops = dept[0].get("top_values") or []
assert isinstance(tops, list) and len(tops) <= 2
def test_tabular_tools_reject_non_table_id_filename() -> None:
bad_id = "日志.xlsx"
q = query_table(table_id=bad_id, columns=["a"], limit=5, offset=0)
assert not bool(q.get("ok"))
assert str(q.get("error") or "") == "table_id_invalid_format"
ag = aggregate_table(table_id=bad_id, metric="count")
assert not bool(ag.get("ok"))
assert str(ag.get("error") or "") == "table_id_invalid_format"
sql = run_table_sql(table_id=bad_id, sql="SELECT 1", limit=5)
assert not bool(sql.get("ok"))
assert str(sql.get("error") or "") == "table_id_invalid_format"
fs = analyze_table_full_scan(table_id=bad_id, columns=["a"], top_values_limit=1)
assert not bool(fs.get("ok"))
assert str(fs.get("error") or "") == "table_id_invalid_format"
def test_xlsx_zip_safety_blocks_unsafe_paths() -> None:
payload = _zip_bytes({"../xl/workbook.xml": b"x"})
out = process_file_data("bad.xlsx", payload)
assert out
first = out[0]
assert str(first.get("type") or "") == "text"
assert "unsafe path" in str(first.get("content") or "").lower()
def test_excel_sheet_count_cap_applies_in_tool_mode(tmp_path: Path, monkeypatch) -> None:
cfg = {
"plugins": {
"entries": {
"memory-wiki": {
"auto": {
"attachments": {
"tabular": {
"max_rows_read": 200,
"max_columns": 50,
"max_cell_chars": 200,
"max_excel_sheets": 1,
"large_table_preview_rows": 20,
"tool_mode_enabled": True,
"tool_mode_min_rows": 1,
"tool_mode_max_bytes": 10 * 1024 * 1024,
}
}
}
}
}
}
}
cfg_path = tmp_path / "oclaw.json"
cfg_path.write_text(json.dumps(cfg), encoding="utf-8")
monkeypatch.setenv("AIA_OCLAW_CONFIG_PATH", str(cfg_path))
fa._attachments_limits.cache_clear()
try:
buf = io.BytesIO()
with pd.ExcelWriter(buf) as writer:
pd.DataFrame({"a": ["1", "2"], "b": ["3", "4"]}).to_excel(writer, index=False, sheet_name="S1")
pd.DataFrame({"a": ["5", "6"], "b": ["7", "8"]}).to_excel(writer, index=False, sheet_name="S2")
out = process_file_data("multi.xlsx", buf.getvalue())
finally:
fa._attachments_limits.cache_clear()
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
assert tab_refs
sheets = list(tab_refs[0].get("sheets") or [])
assert len(sheets) == 1
notes = [x for x in out if isinstance(x, dict) and str(x.get("name") or "").endswith(".sheet-limit")]
assert notes
def test_attachment_asset_store_mp4_load_bytes_roundtrip(tmp_path: Path) -> None:
"""Regression: ``load_bytes`` must resolve ``.mp4`` on-disk blobs (not only image extensions)."""
from svc.files.attachment_assets import AttachmentAssetStore
ast = AttachmentAssetStore(str(tmp_path))
payload = b"\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom" + b"vid" * 400
meta = ast.save_bytes(payload, filename="clip.mp4", mime="video/mp4")
blob, meta2 = ast.load_bytes(meta.attachment_id)
assert blob == payload
assert meta2 is not None
assert meta2.bytes == len(payload)