"""Filesystem spool for biz_state collect: CLI/raw + parsed records before DB flush.""" from __future__ import annotations import json import logging import re import shutil from dataclasses import dataclass, field from pathlib import Path from typing import Any from ..config import settings _log = logging.getLogger("netx.biz_state.spool") _SAFE_RE = re.compile(r"[^A-Za-z0-9._-]+") def spool_root() -> Path: root = Path(str(getattr(settings, "biz_state_spool_dir", None) or "data/biz_state_spool")) root.mkdir(parents=True, exist_ok=True) return root.resolve() def batch_spool_dir(batch_id: str) -> Path: bid = _SAFE_RE.sub("_", str(batch_id or "").strip())[:64] or "unknown" path = spool_root() / bid path.mkdir(parents=True, exist_ok=True) return path def clear_batch_spool(batch_id: str) -> None: bid = _SAFE_RE.sub("_", str(batch_id or "").strip())[:64] if not bid: return path = (spool_root() / bid).resolve() root = spool_root() if not str(path).startswith(str(root)) or path == root: return if path.is_dir(): shutil.rmtree(path, ignore_errors=True) def _cmd_paths(batch_id: str, cmd_id: str) -> tuple[Path, Path, Path]: base = batch_spool_dir(batch_id) cid = _SAFE_RE.sub("_", str(cmd_id or "").strip())[:64] or "cmd" return base / f"{cid}.raw.txt", base / f"{cid}.meta.json", base / f"{cid}.records.jsonl" def write_raw_text(batch_id: str, cmd_id: str, text: str) -> str: """Write CLI output; return path relative to spool root (posix).""" raw_path, _, _ = _cmd_paths(batch_id, cmd_id) data = str(text or "").encode("utf-8", errors="replace") raw_path.write_bytes(data) rel = raw_path.resolve().relative_to(spool_root()) return str(rel).replace("\\", "/") def count_file_lines(rel_path: str) -> int: """Count lines in a spool raw file (full file, not DB-truncated).""" if not rel_path: return 0 path = (spool_root() / str(rel_path)).resolve() if not str(path).startswith(str(spool_root())) or not path.is_file(): return 0 n = 0 with path.open("rb") as fh: for _ in fh: n += 1 return n def count_text_lines(text: str | None) -> int: s = text or "" if not s: return 0 return s.count("\n") + (0 if s.endswith("\n") else 1) def write_records(batch_id: str, cmd_id: str, records: list[dict[str, Any]]) -> str: """Write parsed records as JSONL; return relative path.""" _, _, rec_path = _cmd_paths(batch_id, cmd_id) with rec_path.open("w", encoding="utf-8", errors="replace") as fh: for rec in records or []: fh.write(json.dumps(rec, ensure_ascii=False, default=str)) fh.write("\n") rel = rec_path.resolve().relative_to(spool_root()) return str(rel).replace("\\", "/") def write_meta(batch_id: str, cmd_id: str, meta: dict[str, Any]) -> str: _, meta_path, _ = _cmd_paths(batch_id, cmd_id) meta_path.write_text( json.dumps(meta, ensure_ascii=False, default=str), encoding="utf-8", errors="replace", ) rel = meta_path.resolve().relative_to(spool_root()) return str(rel).replace("\\", "/") def read_raw_text(rel_path: str, *, max_bytes: int = 0) -> str: if not rel_path: return "" path = (spool_root() / str(rel_path)).resolve() if not str(path).startswith(str(spool_root())) or not path.is_file(): return "" data = path.read_bytes() cap = int(max_bytes or 0) if cap > 0 and len(data) > cap: text = data[:cap].decode("utf-8", errors="replace") return text + f"\n...[truncated {cap} bytes cap]\n" return data.decode("utf-8", errors="replace") def read_records(rel_path: str) -> list[dict[str, Any]]: if not rel_path: return [] path = (spool_root() / str(rel_path)).resolve() if not str(path).startswith(str(spool_root())) or not path.is_file(): return [] out: list[dict[str, Any]] = [] with path.open("r", encoding="utf-8", errors="replace") as fh: for line in fh: line = line.strip() if not line: continue try: rec = json.loads(line) except json.JSONDecodeError: continue if isinstance(rec, dict): out.append(rec) return out @dataclass class SpooledCommand: """One command (primary or aux) collected on disk, awaiting DB flush.""" id: str batch_id: str task_item_id: str = "" profile_id: str = "" parser_id: str = "" metric_id: str = "" raw_command: str = "" params_json: dict[str, Any] = field(default_factory=dict) parse_status: str = "" message: str = "" raw_rel_path: str = "" records_rel_path: str = "" row_count: int = 0 # Full CLI line count (before DB raw_text truncate). raw_line_count: int = 0 # Device-declared total when present (e.g. BGP "Total number of routes"). declared_total: int = 0 # True when raw_text stored in DB was truncated by raw_max_bytes. raw_truncated: bool = False # "" | "metric" | "lldp" persist_kind: str = "" def to_meta(self) -> dict[str, Any]: return { "id": self.id, "batch_id": self.batch_id, "task_item_id": self.task_item_id, "profile_id": self.profile_id, "parser_id": self.parser_id, "metric_id": self.metric_id, "raw_command": self.raw_command, "params_json": dict(self.params_json or {}), "parse_status": self.parse_status, "message": self.message, "raw_rel_path": self.raw_rel_path, "records_rel_path": self.records_rel_path, "row_count": self.row_count, "raw_line_count": self.raw_line_count, "declared_total": self.declared_total, "raw_truncated": self.raw_truncated, "persist_kind": self.persist_kind, } def persist_every_cmds() -> int: return max(1, int(getattr(settings, "biz_state_persist_every_cmds", 8) or 8)) def raw_max_bytes() -> int: return max(0, int(getattr(settings, "biz_state_raw_max_bytes", 8 * 1024 * 1024) or 0))