"""Collection quality of a compare side: is a missing row a business loss or a data problem? Compare results are only as good as the two batches behind them. This module reads *receipts* (``BizStateBatchCommand``: counts and status, never ``raw_text``) to tell apart: - ``not_collected`` – a side never ran a command (or the slice) for the sheet - ``suspect_parse`` – the device printed lines but the parser produced no rows - ``no_data`` – both sides ran fine and are really empty - ``ok`` – ordinary comparison """ from __future__ import annotations from typing import Any, Mapping, Sequence from sqlalchemy.orm import Session from ..models import BizStateBatchCommand from .compare_rules import scalar_text # More than this many CLI lines with zero parsed rows is not "an empty table". SUSPECT_RAW_LINES = 5 DATA_STATES = ("ok", "no_data", "suspect_parse", "not_collected") UNJUDGED_STATES = ("no_data", "not_collected") def _has_receipts(db: Session, batch_id: str) -> bool: return bool(batch_id) and db.query(BizStateBatchCommand.id).filter( BizStateBatchCommand.batch_id == batch_id).limit(1).first() is not None def side_receipts(db: Session, batch_id: str, metric_id: str) -> list[dict[str, Any]]: if not batch_id or not metric_id: return [] rows = ( db.query( BizStateBatchCommand.profile_id, BizStateBatchCommand.parse_status, BizStateBatchCommand.row_count, BizStateBatchCommand.raw_line_count, BizStateBatchCommand.declared_total, ) .filter(BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id == metric_id) .all() ) return [ { "profile_id": str(r[0] or ""), "parse_status": str(r[1] or ""), "row_count": int(r[2] or 0), "raw_line_count": int(r[3] or 0), "declared_total": int(r[4] or 0), } for r in rows ] def collect_quality(receipts: Sequence[Mapping[str, Any]]) -> dict[str, Any]: statuses: dict[str, int] = {} for r in receipts: key = str(r.get("parse_status") or "unknown") statuses[key] = statuses.get(key, 0) + 1 return { "commands": len(receipts), "parse_status": statuses, "row_count": sum(int(r.get("row_count") or 0) for r in receipts), "raw_line_count": sum(int(r.get("raw_line_count") or 0) for r in receipts), "declared_total": sum(int(r.get("declared_total") or 0) for r in receipts), "profile_ids": sorted({str(r.get("profile_id") or "") for r in receipts if r.get("profile_id")}), } def sheet_slice(sheet: Mapping[str, Any]) -> dict[str, str]: """``{field: value}`` the sheet pins with top-level ``eq`` row filters.""" out: dict[str, str] = {} for f in sheet.get("row_filters") or []: if not isinstance(f, Mapping) or "any" in f or "all" in f: continue if str(f.get("op") or "eq").strip().lower() not in ("eq", "==", "ci_eq"): continue field = str(f.get("field") or "").strip() value = scalar_text(f.get("value")).strip().lower() if field and value: out[field] = value return out def covers_slice( receipts: Sequence[Mapping[str, Any]], wanted: Mapping[str, str], *, profile_slice: Any = None, ) -> bool: """True when at least one successful command could have produced ``wanted``. A command without a declared slice (or unknown profile) covers everything. """ if profile_slice is None: from .profiles import get_profile def profile_slice(pid: str) -> dict[str, str]: # noqa: F811 prof = get_profile(pid) if pid else None return dict(prof.slice) if prof is not None and prof.slice else {} for r in receipts: if str(r.get("parse_status") or "") in ("failed", "skipped_custom"): continue declared = profile_slice(str(r.get("profile_id") or "")) if all( f not in declared or str(declared[f]).strip().lower() == v for f, v in wanted.items() ): return True return False def classify_sheet( *, before: Sequence[Mapping[str, Any]] | None, after: Sequence[Mapping[str, Any]] | None, before_rows: int, after_rows: int, wanted_slice: Mapping[str, str] | None = None, ) -> tuple[str, list[str]]: """``(data_state, reasons)`` for one sheet; reasons name the offending side. ``None`` receipts mean the batch carries no receipts at all (imported or legacy data): that side cannot be judged and is trusted as-is. """ wanted = dict(wanted_slice or {}) reasons: list[str] = [] for side, receipts in (("before", before), ("after", after)): if receipts is None: continue if not receipts: reasons.append(f"{side}:no_command") elif wanted and not covers_slice(receipts, wanted): reasons.append(f"{side}:slice_not_collected") if reasons: return "not_collected", reasons for side, receipts in (("before", before), ("after", after)): if receipts is None: continue q = collect_quality(receipts) if q["row_count"] == 0 and q["raw_line_count"] > SUSPECT_RAW_LINES: reasons.append(f"{side}:parsed_0_of_{q['raw_line_count']}_lines") elif q["commands"] and q["parse_status"].get("failed", 0) == q["commands"]: reasons.append(f"{side}:all_commands_failed") if reasons: return "suspect_parse", reasons if before_rows == 0 and after_rows == 0: return "no_data", ["both_sides_empty"] return "ok", [] def assess_sheet( db: Session, sheet: Mapping[str, Any], *, before_batch_id: str, after_batch_id: str, before_rows: int, after_rows: int, ) -> dict[str, Any]: """Quality block stored on a sheet summary.""" metric_id = str(sheet.get("metric_id") or "") before = side_receipts(db, before_batch_id, metric_id) if _has_receipts(db, before_batch_id) else None after = side_receipts(db, after_batch_id, metric_id) if _has_receipts(db, after_batch_id) else None state, reasons = classify_sheet( before=before, after=after, before_rows=before_rows, after_rows=after_rows, wanted_slice=sheet_slice(sheet), ) return { "data_state": state, "data_state_reasons": reasons, "collect_quality": { "before": collect_quality(before or []), "after": collect_quality(after or []), "unknown_sides": [n for n, r in (("before", before), ("after", after)) if r is None], }, } def command_coverage( db: Session, *, before_batch_id: str, after_batch_id: str, ) -> dict[str, Any]: """Collected "metric{slice}" labels per side and their difference. Profile ids differ per vendor (``zte.x`` vs ``huawei.x``), so the comparison is on what a command *produces*: its metric and slice. """ from .profiles import get_profile def labels(batch_id: str) -> set[str]: if not batch_id: return set() rows = db.query(BizStateBatchCommand.metric_id, BizStateBatchCommand.profile_id).filter( BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id != "", ).distinct().all() out: set[str] = set() for metric_id, profile_id in rows: prof = get_profile(str(profile_id or "")) if profile_id else None sl = dict(prof.slice) if prof is not None and prof.slice else {} suffix = ",".join(f"{k}={v}" for k, v in sorted(sl.items())) out.add(f"{metric_id}{{{suffix}}}" if suffix else str(metric_id)) return out b, a = labels(before_batch_id), labels(after_batch_id) return {"before": sorted(b), "after": sorted(a), "only_before": sorted(b - a), "only_after": sorted(a - b)}