from __future__ import annotations """将上传文件解析为附件字典（不依赖 Streamlit）。""" import io import json import os import re import subprocess import zipfile from functools import lru_cache from pathlib import Path from typing import Any import pandas as pd from docx import Document from PIL import Image from PyPDF2 import PdfReader from svc.config.paths import PROJECT_ROOT from svc.files.archive_processor import ( MAX_ARCHIVE_DEPTH, MAX_ARCHIVE_ENTRY_BYTES, MAX_ARCHIVE_FILE_COUNT, MAX_ARCHIVE_MEMBER_NAME_LENGTH, MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES, detect_archive_kind, is_safe_archive_member_name, process_archive, ) from svc.files.attachment_assets import AttachmentAssetStore from svc.files.tabular_attachment_store import save_dataframe, save_workbook from svc.files.text_attachment_store import ( DEFAULT_TEXT_CHUNK_OVERLAP, DEFAULT_TEXT_CHUNK_SIZE, DEFAULT_TEXT_INLINE_MAX_CHARS, save_text_document, ) def _sniff_image_mime(data: bytes) -> str | None: if len(data) < 12: return None if data[:8] == b"\x89PNG\r\n\x1a\n": return "image/png" if data[:3] == b"\xff\xd8\xff": return "image/jpeg" if data[:6] in (b"GIF87a", b"GIF89a"): return "image/gif" if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP": return "image/webp" return None def _ffprobe_exists() -> bool: try: p = subprocess.run(["ffprobe", "-version"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3) return p.returncode == 0 except Exception: return False def _extract_video_meta_from_ffprobe(obj: dict[str, Any]) -> dict[str, Any]: fmt = obj.get("format") if isinstance(obj.get("format"), dict) else {} streams = obj.get("streams") if isinstance(obj.get("streams"), list) else [] duration_sec = None if isinstance(fmt, dict) and fmt.get("duration") is not None: try: duration_sec = float(fmt.get("duration")) except Exception: duration_sec = None width = None height = None fps = None for s in streams: if not isinstance(s, dict): continue if str(s.get("codec_type") or "") != "video": continue if s.get("width") is not None and s.get("height") is not None: try: width = int(s.get("width") or 0) or None height = int(s.get("height") or 0) or None except Exception: width = width height = height fr = str(s.get("avg_frame_rate") or s.get("r_frame_rate") or "").strip() if fr and fr != "0/0" and "/" in fr: try: a, b = fr.split("/", 1) fa = float(a) fb = float(b) if fb: fps = fa / fb except Exception: fps = fps break out: dict[str, Any] = {} if duration_sec is not None: out["duration_sec"] = duration_sec if width is not None: out["width"] = width if height is not None: out["height"] = height if fps is not None: out["fps"] = fps return out def _ffprobe_video_meta(path: Path) -> dict[str, Any]: if not _ffprobe_exists(): return {} try: p = subprocess.run( ["ffprobe", "-v", "error", "-print_format", "json", "-show_format", "-show_streams", str(path)], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=8, ) if p.returncode != 0: return {} obj = json.loads(p.stdout or "{}") if not isinstance(obj, dict): return {} return _extract_video_meta_from_ffprobe(obj) except Exception: return {} MAX_TEXT_CHARS = 12_000 EXCEL_PREVIEW_ROWS = 80 LARGE_TABLE_PREVIEW_ROWS = 20 MAX_ZIP_DEPTH = MAX_ARCHIVE_DEPTH MAX_ZIP_FILE_COUNT = MAX_ARCHIVE_FILE_COUNT MAX_ZIP_ENTRY_BYTES = MAX_ARCHIVE_ENTRY_BYTES MAX_ZIP_TOTAL_UNCOMPRESSED_BYTES = MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES DEFAULT_TABULAR_ROWS_READ = 5000 DEFAULT_TABULAR_COLUMNS = 200 DEFAULT_TABULAR_CELL_CHARS = 500 DEFAULT_TABULAR_TOOL_MODE_ENABLED = True DEFAULT_TABULAR_TOOL_MODE_MIN_ROWS = 5_000 DEFAULT_TABULAR_TOOL_MODE_MAX_BYTES = 30 * 1024 * 1024 DEFAULT_LARGE_TABLE_PREVIEW_ROWS = LARGE_TABLE_PREVIEW_ROWS MAX_ZIP_MEMBER_NAME_LENGTH = MAX_ARCHIVE_MEMBER_NAME_LENGTH MAX_EXCEL_ZIP_FILE_COUNT = 500 MAX_EXCEL_ZIP_ENTRY_BYTES = 20 * 1024 * 1024 MAX_EXCEL_ZIP_TOTAL_UNCOMPRESSED_BYTES = 120 * 1024 * 1024 DEFAULT_MAX_EXCEL_SHEETS = 50 DEFAULT_TEXT_QUERY_TOP_K = 5 def _truncate_text(text: str, limit: int = MAX_TEXT_CHARS) -> str: t = str(text or "") if len(t) <= limit: return t return t[:limit] + f"\n\n...[truncated to {limit} chars]" def _text_ref_summary(*, name: str, chars: int, chunks: int, source_kind: str, preview: str, top_k: int) -> str: return _truncate_text( ( f"# Text Document Summary\n" f"- name: {name}\n" f"- source_kind: {source_kind}\n" f"- chars: {chars}\n" f"- chunks: {chunks}\n" f"- mode: text_ref\n" f"- query_tool: query_text_attachment\n" f"- note: use `text_id` from accompanying `text_ref` attachment; recommended top_k <= {top_k}\n\n" f"## Preview (first {min(len(preview), 1500)} chars)\n" f"{preview[:1500]}" ), limit=MAX_TEXT_CHARS, ) def _decode_text_bytes(data: bytes) -> str: try: return data.decode("utf-8") except UnicodeDecodeError: try: return data.decode("gbk") except Exception: return "Error decoding text file (unknown encoding)" def _clean_html_to_text(raw: str) -> str: text = re.sub(r"(?is)]*>.*?", " ", str(raw or "")) text = re.sub(r"(?is)]*>.*?", " ", text) text = re.sub(r"(?is)<[^>]+>", " ", text) text = re.sub(r"[ \t\r\f\v]+", " ", text) text = re.sub(r"\n{3,}", "\n\n", text) return text.strip() def _safe_int(raw: Any, default: int, *, min_value: int = 1, max_value: int = 2_000_000) -> int: try: value = int(raw) except Exception: return default if value < min_value: return default return min(value, max_value) @lru_cache(maxsize=1) def _attachments_limits() -> dict[str, Any]: cfg_path_raw = str(os.getenv("AIA_OCLAW_CONFIG_PATH") or "").strip() cfg_path = Path(cfg_path_raw).expanduser() if cfg_path_raw else (Path(PROJECT_ROOT) / "oclaw.json").resolve() if not cfg_path.is_absolute(): cfg_path = (Path(PROJECT_ROOT) / cfg_path).resolve() tabular_cfg: dict[str, Any] = {} try: obj = json.loads(cfg_path.read_text(encoding="utf-8")) tabular_cfg = ( (((obj or {}).get("plugins") or {}).get("entries") or {}).get("memory-wiki", {}).get("auto", {}).get("attachments", {}).get("tabular", {}) or {} ) except Exception: tabular_cfg = {} return { "rows_read": _safe_int(tabular_cfg.get("max_rows_read"), DEFAULT_TABULAR_ROWS_READ), "columns": _safe_int(tabular_cfg.get("max_columns"), DEFAULT_TABULAR_COLUMNS), "cell_chars": _safe_int(tabular_cfg.get("max_cell_chars"), DEFAULT_TABULAR_CELL_CHARS), "max_excel_sheets": _safe_int(tabular_cfg.get("max_excel_sheets"), DEFAULT_MAX_EXCEL_SHEETS, max_value=500), "large_table_preview_rows": _safe_int( tabular_cfg.get("large_table_preview_rows"), DEFAULT_LARGE_TABLE_PREVIEW_ROWS, max_value=500 ), "tool_mode_enabled": bool(tabular_cfg.get("tool_mode_enabled", DEFAULT_TABULAR_TOOL_MODE_ENABLED)), "tool_mode_min_rows": _safe_int(tabular_cfg.get("tool_mode_min_rows"), DEFAULT_TABULAR_TOOL_MODE_MIN_ROWS), "tool_mode_max_bytes": _safe_int( tabular_cfg.get("tool_mode_max_bytes"), DEFAULT_TABULAR_TOOL_MODE_MAX_BYTES, max_value=500 * 1024 * 1024 ), "text_inline_max_chars": _safe_int( tabular_cfg.get("text_inline_max_chars"), DEFAULT_TEXT_INLINE_MAX_CHARS, max_value=200_000 ), "text_chunk_size": _safe_int(tabular_cfg.get("text_chunk_size"), DEFAULT_TEXT_CHUNK_SIZE, max_value=8_000), "text_chunk_overlap": _safe_int( tabular_cfg.get("text_chunk_overlap"), DEFAULT_TEXT_CHUNK_OVERLAP, max_value=4_000 ), "text_query_top_k": _safe_int(tabular_cfg.get("text_query_top_k"), DEFAULT_TEXT_QUERY_TOP_K, max_value=50), "archive_max_depth": _safe_int(tabular_cfg.get("archive_max_depth"), MAX_ARCHIVE_DEPTH, max_value=10), "archive_max_file_count": _safe_int( tabular_cfg.get("archive_max_file_count"), MAX_ARCHIVE_FILE_COUNT, max_value=20_000 ), "archive_max_entry_bytes": _safe_int( tabular_cfg.get("archive_max_entry_bytes"), MAX_ARCHIVE_ENTRY_BYTES, max_value=2_000_000_000 ), "archive_max_total_uncompressed_bytes": _safe_int( tabular_cfg.get("archive_max_total_uncompressed_bytes"), MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES, max_value=5_000_000_000, ), } def clear_attachment_limits_cache() -> None: _attachments_limits.cache_clear() def _clip_tabular_cells(df: pd.DataFrame, *, max_cell_chars: int) -> tuple[pd.DataFrame, bool]: clipped_any = False def _clip_cell(value: Any) -> Any: nonlocal clipped_any if value is None: return value s = str(value) if len(s) <= max_cell_chars: return value clipped_any = True return s[:max_cell_chars] + "...[cell-truncated]" out = df.copy() for col in out.columns: out[col] = out[col].map(_clip_cell) return out, clipped_any def _dataframe_summary_text( df: pd.DataFrame, *, sampled: bool = False, clipped_columns: bool = False, clipped_cells: bool = False, preview_rows: int = EXCEL_PREVIEW_ROWS, ) -> str: rows = int(len(df.index)) cols = int(len(df.columns)) header = [str(x) for x in list(df.columns)] dtypes = [f"{str(k)}:{str(v)}" for k, v in df.dtypes.items()] show_rows = max(1, int(preview_rows or EXCEL_PREVIEW_ROWS)) sample = df.head(show_rows).to_string(index=False) rows_scope_line = ( f"## Preview (first {min(rows, show_rows)} rows)" if rows > show_rows else f"## Full table included ({rows} rows)" ) body = ( f"# Table Summary\n" f"- rows: {rows}\n" f"- cols: {cols}\n" f"- sampled: {'yes' if sampled else 'no'}\n" f"- clipped_columns: {'yes' if clipped_columns else 'no'}\n" f"- clipped_cells: {'yes' if clipped_cells else 'no'}\n" f"- columns: {', '.join(header)}\n" f"- dtypes: {', '.join(dtypes)}\n\n" f"{rows_scope_line}\n" f"{sample}" ) return _truncate_text(body) def _pdf_summary_text(reader: PdfReader) -> str: pages = list(getattr(reader, "pages", []) or []) chunks: list[str] = [f"# PDF Summary\n- pages: {len(pages)}\n"] for idx, page in enumerate(pages, start=1): txt = "" try: txt = str(page.extract_text() or "").strip() except Exception: txt = "" if not txt: txt = "(empty page text)" chunks.append(f"## Page {idx}\n{txt}") return _truncate_text("\n\n".join(chunks)) def _is_safe_zip_member_name(name: str) -> bool: return is_safe_archive_member_name(name) def _validate_excel_zip_payload(data: bytes) -> str | None: try: with zipfile.ZipFile(io.BytesIO(data)) as zf: infos = [x for x in zf.infolist() if not x.is_dir()] if len(infos) > MAX_EXCEL_ZIP_FILE_COUNT: return f"excel zip has too many entries ({len(infos)} > {MAX_EXCEL_ZIP_FILE_COUNT})" total_uncompressed = 0 for info in infos: if not _is_safe_zip_member_name(info.filename): return f"excel zip contains unsafe path: {info.filename}" fsize = int(info.file_size or 0) total_uncompressed += fsize if fsize > MAX_EXCEL_ZIP_ENTRY_BYTES: return f"excel zip entry too large: {info.filename} ({fsize} > {MAX_EXCEL_ZIP_ENTRY_BYTES})" if total_uncompressed > MAX_EXCEL_ZIP_TOTAL_UNCOMPRESSED_BYTES: return ( "excel zip total uncompressed size too large " f"({total_uncompressed} > {MAX_EXCEL_ZIP_TOTAL_UNCOMPRESSED_BYTES})" ) return None except Exception: return "invalid excel zip payload" def process_zip(zip_data: bytes, *, _depth: int = 0) -> list[dict[str, Any]]: limits = _attachments_limits() return process_archive( archive_name="bundle.zip", archive_data=zip_data, process_member=lambda n, b, d: process_file_data(n, b, _zip_depth=d), depth=_depth, limits={ "max_depth": int(limits.get("archive_max_depth") or MAX_ARCHIVE_DEPTH), "max_file_count": int(limits.get("archive_max_file_count") or MAX_ARCHIVE_FILE_COUNT), "max_entry_bytes": int(limits.get("archive_max_entry_bytes") or MAX_ARCHIVE_ENTRY_BYTES), "max_total_uncompressed_bytes": int( limits.get("archive_max_total_uncompressed_bytes") or MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES ), }, ) def expand_attachment_ref( att: dict[str, Any], *, data: bytes | None = None, name: str | None = None, ) -> list[dict[str, Any]]: """Expand ``binary_ref`` (or raw bytes) into parsed attachment dicts for the agent.""" if not isinstance(att, dict): return [] t = str(att.get("type") or "").strip().lower() if t and t not in {"binary_ref"}: return [att] aid = str(att.get("attachment_id") or att.get("attachmentId") or "").strip().lower() meta = None if data is None and aid: blob, meta = AttachmentAssetStore().load_bytes(aid) if not blob: return [att] if t == "binary_ref" else [] data = blob name = name or str(getattr(meta, "name", "") or "file") if data is None: return [att] if t == "binary_ref" else [] fname = str(name or att.get("name") or getattr(meta, "name", "") or "file") got = process_file_data(fname, data) if got: return got if aid: if meta is None: meta = AttachmentAssetStore().get_meta(aid) return [ { "type": "binary_ref", "attachment_id": aid, "name": fname, "mime": str(att.get("mime") or getattr(meta, "mime", "") or "application/octet-stream"), "bytes": att.get("bytes") if att.get("bytes") is not None else getattr(meta, "bytes", None), } ] return [] def process_file_data(name: str, data: bytes, *, _zip_depth: int = 0) -> list[dict[str, Any]]: ext = name.split(".")[-1].lower() if "." in name else "" attachments: list[dict[str, Any]] = [] if ext in ("png", "jpg", "jpeg", "webp", "gif", "jfif", "pjpeg", "bmp", "tif", "tiff"): if ext == "png": mime = "image/png" elif ext in ("jpg", "jpeg", "jfif", "pjpeg"): mime = "image/jpeg" elif ext == "webp": mime = "image/webp" elif ext == "gif": mime = "image/gif" elif ext == "bmp": mime = "image/bmp" elif ext in ("tif", "tiff"): mime = "image/tiff" width = None height = None try: with Image.open(io.BytesIO(data)) as im: width, height = im.size except Exception: pass meta = AttachmentAssetStore().save_bytes(data, filename=name, mime=mime, width=width, height=height) attachments.append( { "type": "image_ref", "name": meta.name, "mime": meta.mime, "attachment_id": meta.attachment_id, "bytes": meta.bytes, "width": meta.width, "height": meta.height, } ) return attachments sniff_mime = _sniff_image_mime(data) if sniff_mime: width = None height = None try: with Image.open(io.BytesIO(data)) as im: width, height = im.size except Exception: pass meta = AttachmentAssetStore().save_bytes(data, filename=name, mime=sniff_mime, width=width, height=height) attachments.append( { "type": "image_ref", "name": meta.name, "mime": meta.mime, "attachment_id": meta.attachment_id, "bytes": meta.bytes, "width": meta.width, "height": meta.height, } ) return attachments if ext in ("mp4", "mov", "mkv", "webm", "avi", "m4v"): mime = "video/mp4" if ext == "webm": mime = "video/webm" elif ext == "mov": mime = "video/quicktime" elif ext == "mkv": mime = "video/x-matroska" elif ext == "avi": mime = "video/x-msvideo" meta = AttachmentAssetStore().save_bytes(data, filename=name, mime=mime) probe_meta: dict[str, Any] = {} try: p = AttachmentAssetStore().get_local_path(str(meta.attachment_id)) if p is not None: probe_meta = _ffprobe_video_meta(p) except Exception: probe_meta = {} attachments.append( { "type": "video_ref", "name": meta.name, "mime": meta.mime, "attachment_id": meta.attachment_id, "bytes": meta.bytes, **probe_meta, } ) attachments.append( { "type": "text", "name": name, "content": _truncate_text( "".join( [ "# Video Attachment\n", f"- name: {meta.name}\n", f"- mime: {meta.mime}\n", f"- bytes: {meta.bytes}\n", ( f"- duration_sec: {probe_meta.get('duration_sec')}\n" if probe_meta.get("duration_sec") is not None else "" ), ( f"- size: {probe_meta.get('width')}x{probe_meta.get('height')}\n" if probe_meta.get("width") and probe_meta.get("height") else "" ), (f"- fps: {probe_meta.get('fps')}\n" if probe_meta.get("fps") is not None else ""), "- mode: video_ref\n", "- query_tool: query_video_attachment\n", "- note: use `attachment_id` from accompanying `video_ref` attachment.\n", ] ), limit=MAX_TEXT_CHARS, ), } ) return attachments if ext in ("xlsx", "xls", "csv"): try: if ext == "xlsx": zip_err = _validate_excel_zip_payload(data) if zip_err: attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing excel/csv: {zip_err}")}) return attachments limits = _attachments_limits() rows_read = int(limits.get("rows_read") or DEFAULT_TABULAR_ROWS_READ) max_columns = int(limits.get("columns") or DEFAULT_TABULAR_COLUMNS) max_cell_chars = int(limits.get("cell_chars") or DEFAULT_TABULAR_CELL_CHARS) max_excel_sheets = int(limits.get("max_excel_sheets") or DEFAULT_MAX_EXCEL_SHEETS) large_table_preview_rows = int( limits.get("large_table_preview_rows") or DEFAULT_LARGE_TABLE_PREVIEW_ROWS ) tool_mode_enabled = bool(limits.get("tool_mode_enabled", DEFAULT_TABULAR_TOOL_MODE_ENABLED)) tool_mode_min_rows = int(limits.get("tool_mode_min_rows") or DEFAULT_TABULAR_TOOL_MODE_MIN_ROWS) tool_mode_max_bytes = int(limits.get("tool_mode_max_bytes") or DEFAULT_TABULAR_TOOL_MODE_MAX_BYTES) if ext == "csv": df = pd.read_csv(io.BytesIO(data), nrows=rows_read) else: df = pd.read_excel(io.BytesIO(data), nrows=rows_read) sampled = int(len(df.index)) >= rows_read estimated_rows = 0 if ext == "csv": estimated_rows = max(0, int(data.count(b"\n")) - 1) else: estimated_rows = int(len(df.index)) clipped_columns = int(len(df.columns)) > max_columns if clipped_columns: df = df.iloc[:, :max_columns] df, clipped_cells = _clip_tabular_cells(df, max_cell_chars=max_cell_chars) tool_mode = bool( tool_mode_enabled and int(len(data or b"")) <= tool_mode_max_bytes and (estimated_rows >= tool_mode_min_rows or (ext in ("xlsx", "xls") and sampled)) ) attachments.append( { "type": "text", "name": name, "content": _dataframe_summary_text( df, sampled=sampled, clipped_columns=clipped_columns, clipped_cells=clipped_cells, preview_rows=(large_table_preview_rows if tool_mode else EXCEL_PREVIEW_ROWS), ) + ( ( f"\n\n# Large Table Mode\n- estimated_rows: {estimated_rows}\n- query_tool: query_tabular_attachment\n" "- note: use the `table_id` from the accompanying `tabular_ref` attachment to query full data." ) if tool_mode else "" ), } ) if tool_mode: workbook_sheets: dict[str, pd.DataFrame] clipped_sheets = False if ext == "csv": full_df = pd.read_csv(io.BytesIO(data), dtype=str) if int(len(full_df.columns)) > max_columns: full_df = full_df.iloc[:, :max_columns] full_df, _ = _clip_tabular_cells(full_df, max_cell_chars=max_cell_chars) workbook_sheets = {"Sheet1": full_df} else: excel_obj = pd.read_excel(io.BytesIO(data), dtype=str, sheet_name=None) workbook_sheets = {} sheet_items = list((excel_obj or {}).items()) if len(sheet_items) > max_excel_sheets: clipped_sheets = True sheet_items = sheet_items[:max_excel_sheets] for sheet_name, sheet_df in sheet_items: sdf = sheet_df if int(len(sdf.columns)) > max_columns: sdf = sdf.iloc[:, :max_columns] sdf, _ = _clip_tabular_cells(sdf, max_cell_chars=max_cell_chars) workbook_sheets[str(sheet_name or f"Sheet{len(workbook_sheets)+1}")] = sdf mime = "text/csv" if ext == "csv" else "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" meta = AttachmentAssetStore().save_bytes(data, filename=name, mime=mime) table_meta = ( save_dataframe(attachment_id=str(meta.attachment_id), name=name, df=workbook_sheets.get("Sheet1", pd.DataFrame())) if ext == "csv" else save_workbook(attachment_id=str(meta.attachment_id), name=name, sheets=workbook_sheets) ) attachments.append( { "type": "tabular_ref", "name": str(name), "attachment_id": str(meta.attachment_id), "table_id": str(table_meta.get("table_id") or ""), "rows": int(table_meta.get("rows") or 0), "cols": int(table_meta.get("cols") or 0), "columns": list(table_meta.get("columns") or []), "sheets": list(table_meta.get("sheets") or []), } ) if clipped_sheets: attachments.append( { "type": "text", "name": f"{name}.sheet-limit", "content": ( f"Excel workbook contains too many sheets; only the first {max_excel_sheets} sheets were stored " "for tool-mode querying. Adjust max_excel_sheets if full coverage is required." ), } ) except Exception as e: attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing excel/csv: {e}")}) elif ext == "docx": try: doc = Document(io.BytesIO(data)) text = "\n".join([p.text for p in doc.paragraphs]) limits = _attachments_limits() inline_cap = int(limits.get("text_inline_max_chars") or DEFAULT_TEXT_INLINE_MAX_CHARS) if len(str(text or "")) <= inline_cap: attachments.append({"type": "text", "name": name, "content": _truncate_text(text, limit=inline_cap)}) else: meta = AttachmentAssetStore().save_bytes( data, filename=name, mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document", ) text_meta = save_text_document( attachment_id=str(meta.attachment_id), name=name, text=str(text or ""), source_kind="docx", chunk_size=int(limits.get("text_chunk_size") or DEFAULT_TEXT_CHUNK_SIZE), chunk_overlap=int(limits.get("text_chunk_overlap") or DEFAULT_TEXT_CHUNK_OVERLAP), ) top_k = int(limits.get("text_query_top_k") or DEFAULT_TEXT_QUERY_TOP_K) attachments.append( { "type": "text", "name": name, "content": _text_ref_summary( name=name, chars=int(text_meta.get("chars") or 0), chunks=int(text_meta.get("chunks") or 0), source_kind="docx", preview=str(text or ""), top_k=top_k, ), } ) attachments.append( { "type": "text_ref", "name": str(name), "attachment_id": str(meta.attachment_id), "text_id": str(text_meta.get("text_id") or ""), "chars": int(text_meta.get("chars") or 0), "chunks": int(text_meta.get("chunks") or 0), "source_kind": "docx", } ) except Exception as e: attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing docx: {e}")}) elif ext == "pdf": try: reader = PdfReader(io.BytesIO(data)) text = _pdf_summary_text(reader) limits = _attachments_limits() inline_cap = int(limits.get("text_inline_max_chars") or DEFAULT_TEXT_INLINE_MAX_CHARS) if len(str(text or "")) <= inline_cap: attachments.append({"type": "text", "name": name, "content": _truncate_text(text, limit=inline_cap)}) else: meta = AttachmentAssetStore().save_bytes(data, filename=name, mime="application/pdf") text_meta = save_text_document( attachment_id=str(meta.attachment_id), name=name, text=str(text or ""), source_kind="pdf", chunk_size=int(limits.get("text_chunk_size") or DEFAULT_TEXT_CHUNK_SIZE), chunk_overlap=int(limits.get("text_chunk_overlap") or DEFAULT_TEXT_CHUNK_OVERLAP), ) top_k = int(limits.get("text_query_top_k") or DEFAULT_TEXT_QUERY_TOP_K) attachments.append( { "type": "text", "name": name, "content": _text_ref_summary( name=name, chars=int(text_meta.get("chars") or 0), chunks=int(text_meta.get("chunks") or 0), source_kind="pdf", preview=str(text or ""), top_k=top_k, ), } ) attachments.append( { "type": "text_ref", "name": str(name), "attachment_id": str(meta.attachment_id), "text_id": str(text_meta.get("text_id") or ""), "chars": int(text_meta.get("chars") or 0), "chunks": int(text_meta.get("chunks") or 0), "source_kind": "pdf", } ) except Exception as e: attachments.append({"type": "text", "name": name, "content": _truncate_text(f"Error parsing pdf: {e}")}) elif ext in ("txt", "py", "js", "html", "css", "md", "json", "yaml", "yml", "sh", "log"): text = _decode_text_bytes(data) if ext == "html": text = _clean_html_to_text(text) limits = _attachments_limits() inline_cap = int(limits.get("text_inline_max_chars") or DEFAULT_TEXT_INLINE_MAX_CHARS) if len(str(text or "")) <= inline_cap: attachments.append({"type": "text", "name": name, "content": _truncate_text(text, limit=inline_cap)}) else: meta = AttachmentAssetStore().save_bytes(data, filename=name, mime="text/plain") text_meta = save_text_document( attachment_id=str(meta.attachment_id), name=name, text=str(text or ""), source_kind=ext or "text", chunk_size=int(limits.get("text_chunk_size") or DEFAULT_TEXT_CHUNK_SIZE), chunk_overlap=int(limits.get("text_chunk_overlap") or DEFAULT_TEXT_CHUNK_OVERLAP), ) top_k = int(limits.get("text_query_top_k") or DEFAULT_TEXT_QUERY_TOP_K) attachments.append( { "type": "text", "name": name, "content": _text_ref_summary( name=name, chars=int(text_meta.get("chars") or 0), chunks=int(text_meta.get("chunks") or 0), source_kind=str(text_meta.get("source_kind") or ext or "text"), preview=str(text or ""), top_k=top_k, ), } ) attachments.append( { "type": "text_ref", "name": str(name), "attachment_id": str(meta.attachment_id), "text_id": str(text_meta.get("text_id") or ""), "chars": int(text_meta.get("chars") or 0), "chunks": int(text_meta.get("chunks") or 0), "source_kind": str(text_meta.get("source_kind") or ext or "text"), } ) elif detect_archive_kind(name): limits = _attachments_limits() attachments.extend( process_archive( archive_name=name, archive_data=data, process_member=lambda n, b, d: process_file_data(n, b, _zip_depth=d), depth=_zip_depth, limits={ "max_depth": int(limits.get("archive_max_depth") or MAX_ARCHIVE_DEPTH), "max_file_count": int(limits.get("archive_max_file_count") or MAX_ARCHIVE_FILE_COUNT), "max_entry_bytes": int(limits.get("archive_max_entry_bytes") or MAX_ARCHIVE_ENTRY_BYTES), "max_total_uncompressed_bytes": int( limits.get("archive_max_total_uncompressed_bytes") or MAX_ARCHIVE_TOTAL_UNCOMPRESSED_BYTES ), }, ) ) return attachments __all__ = ["process_zip", "process_file_data", "expand_attachment_ref", "clear_attachment_limits_cache"]