mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-09 03:13:19 +08:00
oliver
ba3836f00f
- 在 oclaw/ 下重新初始化 Git 仓库 - 补齐子仓库 .gitignore，避免提交本地运行态数据（_local、node_modules、logs 等） - 提交当前工程代码与配置 Made-with: Cursor
398 lines
15 KiB
Python
398 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
import oclaw.platform.files.file_attachments as fa
|
|
from oclaw.platform.files.file_attachments import process_file_data
|
|
from oclaw.platform.files.tabular_attachment_store import (
|
|
aggregate_table,
|
|
analyze_table_full_scan,
|
|
query_table,
|
|
run_table_sql,
|
|
save_workbook,
|
|
)
|
|
|
|
|
|
def _zip_bytes(files: dict[str, bytes]) -> bytes:
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
|
for name, data in files.items():
|
|
zf.writestr(name, data)
|
|
return buf.getvalue()
|
|
|
|
|
|
def test_csv_is_summarized_not_full_dump() -> None:
|
|
payload = (
|
|
"c1,c2,c3\n"
|
|
"1,2,3\n"
|
|
"4,5,6\n"
|
|
).encode("utf-8")
|
|
out = process_file_data("table.csv", payload)
|
|
assert out and out[0]["type"] == "text"
|
|
content = str(out[0].get("content") or "")
|
|
assert "# Table Summary" in content
|
|
assert "rows: 2" in content
|
|
assert "cols: 3" in content
|
|
|
|
|
|
def test_zip_file_count_limit_returns_error_attachment() -> None:
|
|
files = {f"f{i}.txt": b"x" for i in range(205)}
|
|
out = process_file_data("bulk.zip", _zip_bytes(files))
|
|
assert out
|
|
first = out[0]
|
|
assert str(first.get("name") or "") == "zip-error"
|
|
assert "too many files" in str(first.get("content") or "").lower()
|
|
|
|
|
|
def test_nested_zip_depth_limit_is_enforced() -> None:
|
|
z4 = _zip_bytes({"deep.txt": b"hello"})
|
|
z3 = _zip_bytes({"z4.zip": z4})
|
|
z2 = _zip_bytes({"z3.zip": z3})
|
|
z1 = _zip_bytes({"z2.zip": z2})
|
|
out = process_file_data("z1.zip", z1)
|
|
assert any("nesting too deep" in str(x.get("content") or "").lower() for x in out)
|
|
|
|
|
|
def test_html_is_cleaned_before_attachment_text() -> None:
|
|
html = b"<html><head><style>.x{display:none}</style></head><body><h1>Title</h1><script>alert(1)</script><p>Hello</p></body></html>"
|
|
out = process_file_data("page.html", html)
|
|
content = str((out[0] if out else {}).get("content") or "")
|
|
assert "Title" in content
|
|
assert "Hello" in content
|
|
assert "alert(1)" not in content
|
|
assert "<h1>" not in content
|
|
|
|
|
|
def test_pdf_summary_contains_page_markers() -> None:
|
|
class _Page:
|
|
def __init__(self, text: str) -> None:
|
|
self._text = text
|
|
|
|
def extract_text(self) -> str:
|
|
return self._text
|
|
|
|
class _Reader:
|
|
def __init__(self, _stream) -> None:
|
|
self.pages = [_Page("P1"), _Page("P2")]
|
|
|
|
old = fa.PdfReader
|
|
try:
|
|
fa.PdfReader = _Reader # type: ignore[assignment]
|
|
out = process_file_data("doc.pdf", b"%PDF")
|
|
finally:
|
|
fa.PdfReader = old # type: ignore[assignment]
|
|
content = str((out[0] if out else {}).get("content") or "")
|
|
assert "# PDF Summary" in content
|
|
assert "pages: 2" in content
|
|
assert "## Page 1" in content
|
|
assert "## Page 2" in content
|
|
|
|
|
|
def test_zip_unsafe_member_path_is_blocked() -> None:
|
|
payload = _zip_bytes({"../evil.txt": b"x"})
|
|
out = process_file_data("unsafe.zip", payload)
|
|
assert out
|
|
first = out[0]
|
|
assert str(first.get("name") or "") == "zip-error"
|
|
assert "unsafe path" in str(first.get("content") or "").lower()
|
|
|
|
|
|
def test_csv_marks_sampled_when_row_limit_hit() -> None:
|
|
rows = ["c1,c2"] + [f"{i},{i+1}" for i in range(0, 6000)]
|
|
payload = ("\n".join(rows)).encode("utf-8")
|
|
out = process_file_data("big.csv", payload)
|
|
content = str((out[0] if out else {}).get("content") or "")
|
|
assert "sampled: yes" in content
|
|
|
|
|
|
def test_zip_member_name_too_long_is_blocked() -> None:
|
|
payload = _zip_bytes({("a" * 300) + ".txt": b"x"})
|
|
out = process_file_data("unsafe.zip", payload)
|
|
assert out
|
|
first = out[0]
|
|
assert str(first.get("name") or "") == "zip-error"
|
|
assert "unsafe path" in str(first.get("content") or "").lower()
|
|
|
|
|
|
def test_csv_marks_clipped_columns_when_width_limit_hit() -> None:
|
|
columns = [f"c{i}" for i in range(0, 260)]
|
|
header = ",".join(columns)
|
|
row = ",".join(["1"] * len(columns))
|
|
payload = (header + "\n" + row + "\n").encode("utf-8")
|
|
out = process_file_data("wide.csv", payload)
|
|
content = str((out[0] if out else {}).get("content") or "")
|
|
assert "clipped_columns: yes" in content
|
|
assert "cols: 200" in content
|
|
|
|
|
|
def test_csv_marks_clipped_cells_when_cell_content_too_long() -> None:
|
|
long_cell = "x" * 1200
|
|
payload = f"c1,c2\n{long_cell},ok\n".encode("utf-8")
|
|
out = process_file_data("long-cell.csv", payload)
|
|
content = str((out[0] if out else {}).get("content") or "")
|
|
assert "clipped_cells: yes" in content
|
|
assert "...[cell-truncated]" in content
|
|
|
|
|
|
def test_tabular_limits_can_be_overridden_by_config(tmp_path: Path, monkeypatch) -> None:
|
|
cfg = {
|
|
"plugins": {
|
|
"entries": {
|
|
"memory-wiki": {
|
|
"auto": {
|
|
"attachments": {
|
|
"tabular": {
|
|
"max_rows_read": 2,
|
|
"max_columns": 2,
|
|
"max_cell_chars": 10,
|
|
"large_table_preview_rows": 1,
|
|
"tool_mode_enabled": True,
|
|
"tool_mode_min_rows": 3,
|
|
"tool_mode_max_bytes": 1024 * 1024,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
cfg_path = tmp_path / "oclaw.json"
|
|
cfg_path.write_text(json.dumps(cfg), encoding="utf-8")
|
|
monkeypatch.setenv("AIA_OCLAW_CONFIG_PATH", str(cfg_path))
|
|
fa._attachments_limits.cache_clear()
|
|
try:
|
|
payload = "c1,c2,c3\n" + ("x" * 40) + ",b,c\n1,2,3\n4,5,6\n"
|
|
out = process_file_data("configured.csv", payload.encode("utf-8"))
|
|
finally:
|
|
fa._attachments_limits.cache_clear()
|
|
content = str((out[0] if out else {}).get("content") or "")
|
|
assert "sampled: yes" in content
|
|
assert "clipped_columns: yes" in content
|
|
assert "clipped_cells: yes" in content
|
|
assert "Preview (first 1 rows)" in content
|
|
|
|
|
|
def test_large_csv_emits_tabular_ref_and_can_query() -> None:
|
|
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25050)]
|
|
payload = ("\n".join(rows)).encode("utf-8")
|
|
out = process_file_data("large.csv", payload)
|
|
assert out
|
|
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
|
|
assert tab_refs
|
|
table_id = str(tab_refs[0].get("table_id") or "")
|
|
assert table_id
|
|
got = query_table(
|
|
table_id=table_id,
|
|
columns=["c1", "c2"],
|
|
limit=5,
|
|
offset=0,
|
|
where_contains={"column": "c2", "keyword": "row-12"},
|
|
)
|
|
assert bool(got.get("ok"))
|
|
assert str(got.get("table_id") or "") == table_id
|
|
assert str(got.get("engine") or "") in {"builtin_sqlite", "mcp_sqlite"}
|
|
|
|
|
|
def test_large_csv_can_aggregate_grouped_sum() -> None:
|
|
rows = ["dept,amount"] + [f"{'A' if i % 2 == 0 else 'B'},{i}" for i in range(0, 25010)]
|
|
payload = ("\n".join(rows)).encode("utf-8")
|
|
out = process_file_data("large2.csv", payload)
|
|
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
|
|
assert tab_refs
|
|
table_id = str(tab_refs[0].get("table_id") or "")
|
|
got = aggregate_table(
|
|
table_id=table_id,
|
|
metric="sum",
|
|
target_column="amount",
|
|
group_by="dept",
|
|
top_n=5,
|
|
)
|
|
assert bool(got.get("ok"))
|
|
rows_out = got.get("rows") or []
|
|
assert isinstance(rows_out, list) and len(rows_out) >= 2
|
|
groups = {str(x.get("group") or "") for x in rows_out}
|
|
assert "A" in groups and "B" in groups
|
|
assert str(got.get("engine") or "") in {"builtin_sqlite", "mcp_sqlite"}
|
|
|
|
|
|
def test_query_falls_back_to_builtin_when_mcp_unavailable(monkeypatch) -> None:
|
|
monkeypatch.setenv("AIA_MCP_SQLITE_COMMAND", "nonexistent_mcp_sqlite_binary")
|
|
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25020)]
|
|
payload = ("\n".join(rows)).encode("utf-8")
|
|
out = process_file_data("fallback.csv", payload)
|
|
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
|
|
assert tab_refs
|
|
table_id = str(tab_refs[0].get("table_id") or "")
|
|
got = query_table(table_id=table_id, columns=["c1"], limit=3, offset=0)
|
|
assert bool(got.get("ok"))
|
|
assert str(got.get("engine") or "") == "builtin_sqlite"
|
|
|
|
|
|
def test_run_tabular_sql_select_works_and_blocks_mutation() -> None:
|
|
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25020)]
|
|
payload = ("\n".join(rows)).encode("utf-8")
|
|
out = process_file_data("sql.csv", payload)
|
|
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
|
|
assert tab_refs
|
|
table_id = str(tab_refs[0].get("table_id") or "")
|
|
ok = run_table_sql(table_id=table_id, sql='SELECT "c1","c2" FROM rows_data WHERE "c2" LIKE \'%row-12%\'', limit=10)
|
|
assert bool(ok.get("ok"))
|
|
assert int(ok.get("rows_returned") or 0) >= 1
|
|
assert "SELECT" in str(ok.get("executed_sql") or "")
|
|
guard = ok.get("sql_guard") or {}
|
|
assert bool(guard.get("readonly_enforced"))
|
|
assert bool(guard.get("auto_limit_applied"))
|
|
bad = run_table_sql(table_id=table_id, sql="DROP TABLE rows_data", limit=10)
|
|
assert not bool(bad.get("ok"))
|
|
assert str(bad.get("error") or "") == "sql_not_readonly"
|
|
|
|
|
|
def test_run_tabular_sql_timeout_returns_guard(monkeypatch) -> None:
|
|
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25020)]
|
|
payload = ("\n".join(rows)).encode("utf-8")
|
|
out = process_file_data("sql-timeout.csv", payload)
|
|
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
|
|
assert tab_refs
|
|
table_id = str(tab_refs[0].get("table_id") or "")
|
|
monkeypatch.setenv("AIA_TABULAR_SQL_TIMEOUT_MS", "100")
|
|
slow_sql = (
|
|
"WITH RECURSIVE t(n) AS ("
|
|
"SELECT 1 UNION ALL SELECT n+1 FROM t WHERE n < 5000000"
|
|
") SELECT SUM(n) FROM t"
|
|
)
|
|
got = run_table_sql(table_id=table_id, sql=slow_sql, limit=200)
|
|
assert not bool(got.get("ok"))
|
|
assert str(got.get("error") or "") == "sql_timeout"
|
|
guard = got.get("sql_guard") or {}
|
|
assert bool(guard.get("timeout_hit"))
|
|
assert int(guard.get("timeout_ms") or 0) == 100
|
|
|
|
|
|
def test_run_tabular_sql_timeout_reads_oclaw_config(tmp_path: Path, monkeypatch) -> None:
|
|
cfg = {
|
|
"plugins": {
|
|
"entries": {
|
|
"memory-wiki": {
|
|
"auto": {
|
|
"attachments": {
|
|
"tabular": {
|
|
"sql_timeout_ms": 222,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
cfg_path = tmp_path / "oclaw.json"
|
|
cfg_path.write_text(json.dumps(cfg), encoding="utf-8")
|
|
monkeypatch.setenv("AIA_OCLAW_CONFIG_PATH", str(cfg_path))
|
|
monkeypatch.delenv("AIA_TABULAR_SQL_TIMEOUT_MS", raising=False)
|
|
rows = ["c1,c2"] + [f"{i},row-{i}" for i in range(0, 25020)]
|
|
payload = ("\n".join(rows)).encode("utf-8")
|
|
out = process_file_data("sql-timeout-config.csv", payload)
|
|
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
|
|
assert tab_refs
|
|
table_id = str(tab_refs[0].get("table_id") or "")
|
|
got = run_table_sql(table_id=table_id, sql='SELECT "c1" FROM rows_data WHERE "c2" LIKE \'%row-12%\'', limit=10)
|
|
assert bool(got.get("ok"))
|
|
guard = got.get("sql_guard") or {}
|
|
assert int(guard.get("timeout_ms") or 0) == 222
|
|
|
|
|
|
def test_save_workbook_normalizes_duplicate_and_blank_headers() -> None:
|
|
df = pd.DataFrame([["1", "2", "3"]], columns=["", "重复", "重复"])
|
|
meta = save_workbook(attachment_id="aid1", name="dup.xlsx", sheets={"Data": df})
|
|
cols = list(meta.get("columns") or [])
|
|
assert cols[0].startswith("col_")
|
|
assert "重复" in cols
|
|
assert "重复__2" in cols
|
|
|
|
|
|
def test_query_can_target_specific_excel_sheet() -> None:
|
|
s1 = pd.DataFrame([["a", "1"]], columns=["k", "v"])
|
|
s2 = pd.DataFrame([["b", "2"]], columns=["k", "v"])
|
|
meta = save_workbook(attachment_id="aid2", name="multi.xlsx", sheets={"S1": s1, "S2": s2})
|
|
tid = str(meta.get("table_id") or "")
|
|
got = query_table(table_id=tid, sheet="S2", columns=["k", "v"], limit=5, offset=0)
|
|
assert bool(got.get("ok"))
|
|
rows = got.get("rows") or []
|
|
assert rows and str(rows[0].get("k") or "") == "b"
|
|
|
|
|
|
def test_full_scan_analyzes_all_rows_and_returns_audit() -> None:
|
|
rows = ["dept,score"] + [f"{'A' if i % 2 == 0 else 'B'},{i % 5}" for i in range(0, 25025)]
|
|
payload = ("\n".join(rows)).encode("utf-8")
|
|
out = process_file_data("fullscan.csv", payload)
|
|
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
|
|
assert tab_refs
|
|
table_id = str(tab_refs[0].get("table_id") or "")
|
|
got = analyze_table_full_scan(table_id=table_id, columns=["dept", "score"], top_values_limit=2)
|
|
assert bool(got.get("ok"))
|
|
assert int(got.get("rows_scanned") or 0) >= 25025
|
|
audit = got.get("scan_audit") or {}
|
|
assert bool(audit.get("full_scan"))
|
|
assert int(audit.get("rows_scanned") or 0) >= 25025
|
|
stats = got.get("column_stats") or []
|
|
assert isinstance(stats, list) and len(stats) == 2
|
|
dept = [x for x in stats if str(x.get("column") or "") == "dept"]
|
|
assert dept
|
|
tops = dept[0].get("top_values") or []
|
|
assert isinstance(tops, list) and len(tops) <= 2
|
|
|
|
|
|
def test_xlsx_zip_safety_blocks_unsafe_paths() -> None:
|
|
payload = _zip_bytes({"../xl/workbook.xml": b"x"})
|
|
out = process_file_data("bad.xlsx", payload)
|
|
assert out
|
|
first = out[0]
|
|
assert str(first.get("type") or "") == "text"
|
|
assert "unsafe path" in str(first.get("content") or "").lower()
|
|
|
|
|
|
def test_excel_sheet_count_cap_applies_in_tool_mode(tmp_path: Path, monkeypatch) -> None:
|
|
cfg = {
|
|
"plugins": {
|
|
"entries": {
|
|
"memory-wiki": {
|
|
"auto": {
|
|
"attachments": {
|
|
"tabular": {
|
|
"max_rows_read": 200,
|
|
"max_columns": 50,
|
|
"max_cell_chars": 200,
|
|
"max_excel_sheets": 1,
|
|
"large_table_preview_rows": 20,
|
|
"tool_mode_enabled": True,
|
|
"tool_mode_min_rows": 1,
|
|
"tool_mode_max_bytes": 10 * 1024 * 1024,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
cfg_path = tmp_path / "oclaw.json"
|
|
cfg_path.write_text(json.dumps(cfg), encoding="utf-8")
|
|
monkeypatch.setenv("AIA_OCLAW_CONFIG_PATH", str(cfg_path))
|
|
fa._attachments_limits.cache_clear()
|
|
try:
|
|
buf = io.BytesIO()
|
|
with pd.ExcelWriter(buf) as writer:
|
|
pd.DataFrame({"a": ["1", "2"], "b": ["3", "4"]}).to_excel(writer, index=False, sheet_name="S1")
|
|
pd.DataFrame({"a": ["5", "6"], "b": ["7", "8"]}).to_excel(writer, index=False, sheet_name="S2")
|
|
out = process_file_data("multi.xlsx", buf.getvalue())
|
|
finally:
|
|
fa._attachments_limits.cache_clear()
|
|
tab_refs = [x for x in out if isinstance(x, dict) and str(x.get("type") or "") == "tabular_ref"]
|
|
assert tab_refs
|
|
sheets = list(tab_refs[0].get("sheets") or [])
|
|
assert len(sheets) == 1
|
|
notes = [x for x in out if isinstance(x, dict) and str(x.get("name") or "").endswith(".sheet-limit")]
|
|
assert notes
|
|
|