mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 04:20:45 +08:00
Store compare diffs in row table with server-side paging per sheet.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
33264d7b76
commit
499a5fec2e
8 changed files with 487 additions and 69 deletions
|
|
@ -3,6 +3,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import io
|
import io
|
||||||
|
import json
|
||||||
import zipfile
|
import zipfile
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
@ -12,6 +13,7 @@ from fastapi import HTTPException
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ..models import (
|
from ..models import (
|
||||||
|
BizCompareDiff,
|
||||||
BizCompareJob,
|
BizCompareJob,
|
||||||
BizCompareRun,
|
BizCompareRun,
|
||||||
BizCompareTemplate,
|
BizCompareTemplate,
|
||||||
|
|
@ -29,6 +31,129 @@ def _utcnow() -> datetime:
|
||||||
return utcnow_naive()
|
return utcnow_naive()
|
||||||
|
|
||||||
|
|
||||||
|
_DIFF_CHUNK = 2000
|
||||||
|
_SEARCH_TEXT_MAX = 4000
|
||||||
|
|
||||||
|
|
||||||
|
def _diff_search_text(d: dict[str, Any]) -> str:
|
||||||
|
parts = [str(d.get("kind") or "")]
|
||||||
|
for key in ("key", "before", "after", "mapped_before", "changes"):
|
||||||
|
val = d.get(key)
|
||||||
|
if val:
|
||||||
|
try:
|
||||||
|
parts.append(json.dumps(val, ensure_ascii=False, default=str, separators=(",", ":")))
|
||||||
|
except Exception:
|
||||||
|
parts.append(str(val))
|
||||||
|
return " ".join(parts)[:_SEARCH_TEXT_MAX]
|
||||||
|
|
||||||
|
|
||||||
|
def _top_changed_fields(diffs: list[dict[str, Any]], *, limit: int = 8) -> list[dict[str, Any]]:
|
||||||
|
field_counts: dict[str, int] = {}
|
||||||
|
for d in diffs:
|
||||||
|
if str(d.get("kind") or "") != "changed":
|
||||||
|
continue
|
||||||
|
for fname in d.get("changes") or {}:
|
||||||
|
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
|
||||||
|
return sorted(
|
||||||
|
[{"field": k, "count": v} for k, v in field_counts.items()],
|
||||||
|
key=lambda x: (-int(x["count"]), str(x["field"])),
|
||||||
|
)[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _persist_sheet_diffs(
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
run_id: str,
|
||||||
|
metric_id: str,
|
||||||
|
diffs: list[dict[str, Any]],
|
||||||
|
) -> None:
|
||||||
|
"""Bulk-insert diff rows; avoids embedding million-row arrays in summary_json."""
|
||||||
|
buf: list[dict[str, Any]] = []
|
||||||
|
for i, d in enumerate(diffs):
|
||||||
|
buf.append(
|
||||||
|
{
|
||||||
|
"id": uuid4().hex,
|
||||||
|
"run_id": run_id,
|
||||||
|
"metric_id": metric_id,
|
||||||
|
"seq": i,
|
||||||
|
"kind": str(d.get("kind") or ""),
|
||||||
|
"key_json": dict(d.get("key") or {}),
|
||||||
|
"before_json": dict(d.get("before") or {}),
|
||||||
|
"after_json": dict(d.get("after") or {}),
|
||||||
|
"mapped_before_json": dict(d.get("mapped_before") or {}),
|
||||||
|
"changes_json": dict(d.get("changes") or {}),
|
||||||
|
"search_text": _diff_search_text(d),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if len(buf) >= _DIFF_CHUNK:
|
||||||
|
db.bulk_insert_mappings(BizCompareDiff, buf)
|
||||||
|
buf.clear()
|
||||||
|
if buf:
|
||||||
|
db.bulk_insert_mappings(BizCompareDiff, buf)
|
||||||
|
|
||||||
|
|
||||||
|
def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"kind": r.kind,
|
||||||
|
"key": r.key_json or {},
|
||||||
|
"before": r.before_json or {},
|
||||||
|
"after": r.after_json or {},
|
||||||
|
"mapped_before": r.mapped_before_json or {},
|
||||||
|
"changes": r.changes_json or {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _run_has_diff_rows(db: Session, run_id: str) -> bool:
|
||||||
|
return (
|
||||||
|
db.query(BizCompareDiff.id).filter(BizCompareDiff.run_id == run_id).limit(1).first()
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_inline_diffs(
|
||||||
|
diffs: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
kind: str,
|
||||||
|
kw: str,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
kind_n = (kind or "diff").strip().lower()
|
||||||
|
kw_n = (kw or "").strip().lower()
|
||||||
|
out: list[dict[str, Any]] = []
|
||||||
|
for d in diffs:
|
||||||
|
dk = str(d.get("kind") or "")
|
||||||
|
if kind_n == "diff":
|
||||||
|
if dk == "unchanged":
|
||||||
|
continue
|
||||||
|
elif kind_n != "all" and dk != kind_n:
|
||||||
|
continue
|
||||||
|
if kw_n:
|
||||||
|
blob = _diff_search_text(d).lower()
|
||||||
|
if kw_n not in blob:
|
||||||
|
continue
|
||||||
|
out.append(d)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: Any) -> list[dict[str, Any]]:
|
||||||
|
sheets = list(summary.get("sheets") or [])
|
||||||
|
if sheets:
|
||||||
|
return sheets
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"metric_id": run.metric_id,
|
||||||
|
"key_fields": list((tpl.key_fields if tpl else None) or []),
|
||||||
|
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
|
||||||
|
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
|
||||||
|
"mode": "fields",
|
||||||
|
"summary": {
|
||||||
|
k: summary.get(k, 0)
|
||||||
|
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count")
|
||||||
|
},
|
||||||
|
"diffs": list(run.diffs_json or []),
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def _str_list(raw: Any) -> list[str]:
|
def _str_list(raw: Any) -> list[str]:
|
||||||
return [str(x).strip() for x in (raw or []) if str(x).strip()]
|
return [str(x).strip() for x in (raw or []) if str(x).strip()]
|
||||||
|
|
||||||
|
|
@ -624,6 +749,13 @@ def delete_job(db: Session, job_id: str) -> None:
|
||||||
j = db.get(BizCompareJob, job_id)
|
j = db.get(BizCompareJob, job_id)
|
||||||
if not j:
|
if not j:
|
||||||
raise HTTPException(status_code=404, detail="job_not_found")
|
raise HTTPException(status_code=404, detail="job_not_found")
|
||||||
|
run_ids = [
|
||||||
|
rid for (rid,) in db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id).all()
|
||||||
|
]
|
||||||
|
if run_ids:
|
||||||
|
db.query(BizCompareDiff).filter(BizCompareDiff.run_id.in_(run_ids)).delete(
|
||||||
|
synchronize_session=False
|
||||||
|
)
|
||||||
db.query(BizCompareRun).filter(BizCompareRun.job_id == job_id).delete()
|
db.query(BizCompareRun).filter(BizCompareRun.job_id == job_id).delete()
|
||||||
db.delete(j)
|
db.delete(j)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
@ -725,9 +857,24 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
|
||||||
mapping_by_metric[one["metric_id"]] = one["mapping_stats"]
|
mapping_by_metric[one["metric_id"]] = one["mapping_stats"]
|
||||||
|
|
||||||
first = sheet_results[0]
|
first = sheet_results[0]
|
||||||
|
field_counts: dict[str, int] = {}
|
||||||
|
for s in sheet_results:
|
||||||
|
for d in list(s.get("diffs") or []):
|
||||||
|
if str(d.get("kind") or "") != "changed":
|
||||||
|
continue
|
||||||
|
for fname in d.get("changes") or {}:
|
||||||
|
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
|
||||||
|
top_fields = sorted(
|
||||||
|
[{"field": k, "count": v} for k, v in field_counts.items()],
|
||||||
|
key=lambda x: (-int(x["count"]), str(x["field"])),
|
||||||
|
)[:8]
|
||||||
|
|
||||||
|
run_id = uuid4().hex
|
||||||
|
# Persist counts/meta only — diffs go to biz_compare_diff rows
|
||||||
summary_payload = {
|
summary_payload = {
|
||||||
**agg,
|
**agg,
|
||||||
"sheet_count": len(sheet_results),
|
"sheet_count": len(sheet_results),
|
||||||
|
"top_changed_fields": top_fields,
|
||||||
"sheets": [
|
"sheets": [
|
||||||
{
|
{
|
||||||
"metric_id": s["metric_id"],
|
"metric_id": s["metric_id"],
|
||||||
|
|
@ -736,14 +883,13 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
|
||||||
"compare_fields": s["compare_fields"],
|
"compare_fields": s["compare_fields"],
|
||||||
"mode": s["mode"],
|
"mode": s["mode"],
|
||||||
"summary": s["summary"],
|
"summary": s["summary"],
|
||||||
"diffs": s["diffs"],
|
|
||||||
}
|
}
|
||||||
for s in sheet_results
|
for s in sheet_results
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
run = BizCompareRun(
|
run = BizCompareRun(
|
||||||
id=uuid4().hex,
|
id=run_id,
|
||||||
job_id=j.id,
|
job_id=j.id,
|
||||||
template_id=tpl.id,
|
template_id=tpl.id,
|
||||||
mapping_id=j.mapping_id,
|
mapping_id=j.mapping_id,
|
||||||
|
|
@ -752,12 +898,20 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
|
||||||
metric_id=first["metric_id"],
|
metric_id=first["metric_id"],
|
||||||
status="success",
|
status="success",
|
||||||
summary_json=summary_payload,
|
summary_json=summary_payload,
|
||||||
diffs_json=first["diffs"],
|
diffs_json=[],
|
||||||
mapping_stats_json=mapping_by_metric,
|
mapping_stats_json=mapping_by_metric,
|
||||||
message="",
|
message="",
|
||||||
created_at=_utcnow(),
|
created_at=_utcnow(),
|
||||||
)
|
)
|
||||||
db.add(run)
|
db.add(run)
|
||||||
|
db.flush()
|
||||||
|
for s in sheet_results:
|
||||||
|
_persist_sheet_diffs(
|
||||||
|
db,
|
||||||
|
run_id=run_id,
|
||||||
|
metric_id=str(s["metric_id"]),
|
||||||
|
diffs=list(s.get("diffs") or []),
|
||||||
|
)
|
||||||
j.updated_at = _utcnow()
|
j.updated_at = _utcnow()
|
||||||
if j.mode == "manual":
|
if j.mode == "manual":
|
||||||
j.after_batch_id = after_batch_id
|
j.after_batch_id = after_batch_id
|
||||||
|
|
@ -815,7 +969,6 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
||||||
pass_rate = round((unchanged / matched) * 100, 1) if matched else (100.0 if total == 0 else 0.0)
|
pass_rate = round((unchanged / matched) * 100, 1) if matched else (100.0 if total == 0 else 0.0)
|
||||||
diff_rate = round((diff_count / total) * 100, 1) if total else 0.0
|
diff_rate = round((diff_count / total) * 100, 1) if total else 0.0
|
||||||
|
|
||||||
field_counts: dict[str, int] = {}
|
|
||||||
sheet_cards: list[dict[str, Any]] = []
|
sheet_cards: list[dict[str, Any]] = []
|
||||||
for sh in sheets:
|
for sh in sheets:
|
||||||
ss = dict(sh.get("summary") or {})
|
ss = dict(sh.get("summary") or {})
|
||||||
|
|
@ -839,16 +992,21 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
||||||
"pass_rate": round((su / sm) * 100, 1) if sm else (100.0 if st == 0 else 0.0),
|
"pass_rate": round((su / sm) * 100, 1) if sm else (100.0 if st == 0 else 0.0),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
for d in list(sh.get("diffs") or []):
|
|
||||||
if str(d.get("kind") or "") != "changed":
|
|
||||||
continue
|
|
||||||
for fname in (d.get("changes") or {}):
|
|
||||||
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
|
|
||||||
|
|
||||||
top_fields = sorted(
|
top_fields = list(summary.get("top_changed_fields") or [])
|
||||||
[{"field": k, "count": v} for k, v in field_counts.items()],
|
if not top_fields:
|
||||||
key=lambda x: (-int(x["count"]), str(x["field"])),
|
# Legacy runs that still embed diffs in summary_json
|
||||||
)[:8]
|
field_counts: dict[str, int] = {}
|
||||||
|
for sh in sheets:
|
||||||
|
for d in list(sh.get("diffs") or []):
|
||||||
|
if str(d.get("kind") or "") != "changed":
|
||||||
|
continue
|
||||||
|
for fname in d.get("changes") or {}:
|
||||||
|
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
|
||||||
|
top_fields = sorted(
|
||||||
|
[{"field": k, "count": v} for k, v in field_counts.items()],
|
||||||
|
key=lambda x: (-int(x["count"]), str(x["field"])),
|
||||||
|
)[:8]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"added": added,
|
"added": added,
|
||||||
|
|
@ -875,24 +1033,21 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
raise HTTPException(status_code=404, detail="run_not_found")
|
raise HTTPException(status_code=404, detail="run_not_found")
|
||||||
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
|
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
|
||||||
summary = dict(r.summary_json or {})
|
summary = dict(r.summary_json or {})
|
||||||
sheets = list(summary.get("sheets") or [])
|
raw_sheets = _sheet_meta_from_summary(summary, r, tpl)
|
||||||
if not sheets:
|
# Never return full diffs in run detail (million-row safe)
|
||||||
# Legacy single-metric run
|
sheets = [
|
||||||
sheets = [
|
{
|
||||||
{
|
"metric_id": sh.get("metric_id") or "",
|
||||||
"metric_id": r.metric_id,
|
"key_fields": list(sh.get("key_fields") or []),
|
||||||
"key_fields": list((tpl.key_fields if tpl else None) or []),
|
"iface_fields": list(sh.get("iface_fields") or []),
|
||||||
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
|
"compare_fields": list(sh.get("compare_fields") or []),
|
||||||
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
|
"mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
|
||||||
"mode": "fields",
|
"summary": dict(sh.get("summary") or {}),
|
||||||
"summary": {
|
}
|
||||||
k: summary.get(k, 0)
|
for sh in raw_sheets
|
||||||
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count")
|
]
|
||||||
},
|
enriched = _enrich_summary(summary, raw_sheets)
|
||||||
"diffs": list(r.diffs_json or []),
|
stored = "rows" if _run_has_diff_rows(db, run_id) else "inline"
|
||||||
}
|
|
||||||
]
|
|
||||||
enriched = _enrich_summary(summary, sheets)
|
|
||||||
return {
|
return {
|
||||||
"id": r.id,
|
"id": r.id,
|
||||||
"job_id": r.job_id,
|
"job_id": r.job_id,
|
||||||
|
|
@ -904,7 +1059,8 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
"status": r.status,
|
"status": r.status,
|
||||||
"summary": enriched,
|
"summary": enriched,
|
||||||
"sheets": sheets,
|
"sheets": sheets,
|
||||||
"diffs": list(r.diffs_json or []),
|
"diffs": [],
|
||||||
|
"diffs_stored": stored,
|
||||||
"mapping_stats": r.mapping_stats_json or {},
|
"mapping_stats": r.mapping_stats_json or {},
|
||||||
"message": r.message,
|
"message": r.message,
|
||||||
"created_at": r.created_at.isoformat() + "Z" if r.created_at else None,
|
"created_at": r.created_at.isoformat() + "Z" if r.created_at else None,
|
||||||
|
|
@ -912,6 +1068,110 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def list_run_diffs(
|
||||||
|
db: Session,
|
||||||
|
run_id: str,
|
||||||
|
*,
|
||||||
|
metric_id: str = "",
|
||||||
|
kind: str = "diff",
|
||||||
|
kw: str = "",
|
||||||
|
page: int = 1,
|
||||||
|
page_size: int = 100,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
r = db.get(BizCompareRun, run_id)
|
||||||
|
if not r:
|
||||||
|
raise HTTPException(status_code=404, detail="run_not_found")
|
||||||
|
page_n = max(1, int(page or 1))
|
||||||
|
size_n = max(1, min(500, int(page_size or 100)))
|
||||||
|
kind_n = (kind or "diff").strip().lower()
|
||||||
|
kw_n = (kw or "").strip()
|
||||||
|
|
||||||
|
summary = dict(r.summary_json or {})
|
||||||
|
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
|
||||||
|
sheets = _sheet_meta_from_summary(summary, r, tpl)
|
||||||
|
mid = (metric_id or "").strip() or (sheets[0].get("metric_id") if sheets else r.metric_id) or ""
|
||||||
|
|
||||||
|
if _run_has_diff_rows(db, run_id):
|
||||||
|
q = db.query(BizCompareDiff).filter(
|
||||||
|
BizCompareDiff.run_id == run_id,
|
||||||
|
BizCompareDiff.metric_id == mid,
|
||||||
|
)
|
||||||
|
if kind_n == "diff":
|
||||||
|
q = q.filter(BizCompareDiff.kind.in_(("added", "removed", "changed")))
|
||||||
|
elif kind_n != "all":
|
||||||
|
q = q.filter(BizCompareDiff.kind == kind_n)
|
||||||
|
if kw_n:
|
||||||
|
q = q.filter(BizCompareDiff.search_text.ilike(f"%{kw_n}%"))
|
||||||
|
total = q.count()
|
||||||
|
rows = (
|
||||||
|
q.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc())
|
||||||
|
.offset((page_n - 1) * size_n)
|
||||||
|
.limit(size_n)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"page": page_n,
|
||||||
|
"page_size": size_n,
|
||||||
|
"metric_id": mid,
|
||||||
|
"items": [_diff_row_out(x) for x in rows],
|
||||||
|
}
|
||||||
|
|
||||||
|
# Legacy: diffs embedded in summary_json / diffs_json
|
||||||
|
sheet = next((s for s in sheets if str(s.get("metric_id") or "") == mid), None)
|
||||||
|
if sheet is None and sheets:
|
||||||
|
sheet = sheets[0]
|
||||||
|
mid = str(sheet.get("metric_id") or mid)
|
||||||
|
inline = list((sheet or {}).get("diffs") or [])
|
||||||
|
if not inline and mid == r.metric_id:
|
||||||
|
inline = list(r.diffs_json or [])
|
||||||
|
filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n)
|
||||||
|
total = len(filtered)
|
||||||
|
start = (page_n - 1) * size_n
|
||||||
|
page_items = filtered[start : start + size_n]
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"page": page_n,
|
||||||
|
"page_size": size_n,
|
||||||
|
"metric_id": mid,
|
||||||
|
"items": page_items,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str, Any]]:
|
||||||
|
"""Load all diffs for one sheet (export). Prefer row table; fall back to inline."""
|
||||||
|
if _run_has_diff_rows(db, run_id):
|
||||||
|
out: list[dict[str, Any]] = []
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
rows = (
|
||||||
|
db.query(BizCompareDiff)
|
||||||
|
.filter(BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id)
|
||||||
|
.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc())
|
||||||
|
.offset(offset)
|
||||||
|
.limit(_DIFF_CHUNK)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
out.extend(_diff_row_out(x) for x in rows)
|
||||||
|
offset += len(rows)
|
||||||
|
if len(rows) < _DIFF_CHUNK:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
r = db.get(BizCompareRun, run_id)
|
||||||
|
if not r:
|
||||||
|
return []
|
||||||
|
summary = dict(r.summary_json or {})
|
||||||
|
sheets = list(summary.get("sheets") or [])
|
||||||
|
for sh in sheets:
|
||||||
|
if str(sh.get("metric_id") or "") == metric_id:
|
||||||
|
return list(sh.get("diffs") or [])
|
||||||
|
if metric_id == r.metric_id:
|
||||||
|
return list(r.diffs_json or [])
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
def export_run_zip(db: Session, run_id: str) -> bytes:
|
def export_run_zip(db: Session, run_id: str) -> bytes:
|
||||||
detail = get_run(db, run_id)
|
detail = get_run(db, run_id)
|
||||||
buf = io.BytesIO()
|
buf = io.BytesIO()
|
||||||
|
|
@ -941,8 +1201,11 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
|
||||||
for sheet in list(detail.get("sheets") or []):
|
for sheet in list(detail.get("sheets") or []):
|
||||||
mid = str(sheet.get("metric_id") or "sheet")
|
mid = str(sheet.get("metric_id") or "sheet")
|
||||||
safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "sheet"
|
safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "sheet"
|
||||||
zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet))
|
sheet_full = {
|
||||||
# summary table
|
**sheet,
|
||||||
|
"diffs": _iter_sheet_diffs(db, run_id, mid),
|
||||||
|
}
|
||||||
|
zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet_full))
|
||||||
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"]
|
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"]
|
||||||
for card in list(s.get("sheet_cards") or []):
|
for card in list(s.get("sheet_cards") or []):
|
||||||
sum_lines.append(
|
sum_lines.append(
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,9 @@ def apply_biz_state_schema(conn: Connection) -> None:
|
||||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)",
|
"CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)",
|
||||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)",
|
"CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)",
|
||||||
"CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)",
|
"CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)",
|
||||||
|
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_id ON biz_compare_diff (run_id)",
|
||||||
|
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_kind ON biz_compare_diff (run_id, metric_id, kind)",
|
||||||
|
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_seq ON biz_compare_diff (run_id, metric_id, seq)",
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
_run_sql(conn, sql)
|
_run_sql(conn, sql)
|
||||||
|
|
|
||||||
|
|
@ -371,6 +371,27 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||||
return cmp_svc.get_run(db, run_id)
|
return cmp_svc.get_run(db, run_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/compare/runs/{run_id}/diffs")
|
||||||
|
def api_list_run_diffs(
|
||||||
|
run_id: str,
|
||||||
|
metric_id: str = "",
|
||||||
|
kind: str = "diff",
|
||||||
|
kw: str = "",
|
||||||
|
page: int = 1,
|
||||||
|
page_size: int = 100,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return cmp_svc.list_run_diffs(
|
||||||
|
db,
|
||||||
|
run_id,
|
||||||
|
metric_id=metric_id,
|
||||||
|
kind=kind,
|
||||||
|
kw=kw,
|
||||||
|
page=page,
|
||||||
|
page_size=page_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/compare/runs/{run_id}/export")
|
@router.get("/compare/runs/{run_id}/export")
|
||||||
def api_export_run(run_id: str, db: Session = Depends(get_db)) -> StreamingResponse:
|
def api_export_run(run_id: str, db: Session = Depends(get_db)) -> StreamingResponse:
|
||||||
data = cmp_svc.export_run_zip(db, run_id)
|
data = cmp_svc.export_run_zip(db, run_id)
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ from .managed_ne import (
|
||||||
UmeCliOverride,
|
UmeCliOverride,
|
||||||
)
|
)
|
||||||
from .biz_state import (
|
from .biz_state import (
|
||||||
|
BizCompareDiff,
|
||||||
BizCompareJob,
|
BizCompareJob,
|
||||||
BizCompareRun,
|
BizCompareRun,
|
||||||
BizCompareTemplate,
|
BizCompareTemplate,
|
||||||
|
|
@ -142,4 +143,5 @@ __all__ = [
|
||||||
"BizPortMappingRow",
|
"BizPortMappingRow",
|
||||||
"BizCompareJob",
|
"BizCompareJob",
|
||||||
"BizCompareRun",
|
"BizCompareRun",
|
||||||
|
"BizCompareDiff",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text, UniqueConstraint
|
from sqlalchemy import Boolean, DateTime, Index, Integer, String, Text, UniqueConstraint
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from ..db import Base
|
from ..db import Base
|
||||||
|
|
@ -256,3 +256,25 @@ class BizCompareRun(Base):
|
||||||
mapping_stats_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
mapping_stats_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||||
message: Mapped[str] = mapped_column(String(1024), default="")
|
message: Mapped[str] = mapped_column(String(1024), default="")
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive, index=True)
|
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive, index=True)
|
||||||
|
|
||||||
|
|
||||||
|
class BizCompareDiff(Base):
|
||||||
|
"""Per-row compare result — supports million-scale sheets with paged queries."""
|
||||||
|
|
||||||
|
__tablename__ = "biz_compare_diff"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_biz_compare_diff_run_metric_kind", "run_id", "metric_id", "kind"),
|
||||||
|
Index("ix_biz_compare_diff_run_metric_seq", "run_id", "metric_id", "seq"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex)
|
||||||
|
run_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
||||||
|
metric_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
||||||
|
seq: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
kind: Mapped[str] = mapped_column(String(16), default="", index=True)
|
||||||
|
key_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||||
|
before_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||||
|
after_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||||
|
mapped_before_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||||
|
changes_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||||
|
search_text: Mapped[str] = mapped_column(Text, default="")
|
||||||
|
|
|
||||||
|
|
@ -125,5 +125,32 @@ class CompareEngineTests(unittest.TestCase):
|
||||||
self.assertEqual(out["diffs"][0]["kind"], "unchanged")
|
self.assertEqual(out["diffs"][0]["kind"], "unchanged")
|
||||||
|
|
||||||
|
|
||||||
|
class CompareDiffPagingTests(unittest.TestCase):
|
||||||
|
def test_filter_inline_diffs_kind_and_kw(self) -> None:
|
||||||
|
from netx_api.biz_state.compare_service import _filter_inline_diffs
|
||||||
|
|
||||||
|
diffs = [
|
||||||
|
{"kind": "added", "key": {"p": "1"}, "before": {}, "after": {"p": "1"}, "changes": {}},
|
||||||
|
{"kind": "removed", "key": {"p": "2"}, "before": {"p": "2"}, "after": {}, "changes": {}},
|
||||||
|
{"kind": "unchanged", "key": {"p": "3"}, "before": {"p": "3"}, "after": {"p": "3"}, "changes": {}},
|
||||||
|
{
|
||||||
|
"kind": "changed",
|
||||||
|
"key": {"p": "4"},
|
||||||
|
"before": {"x": "a"},
|
||||||
|
"after": {"x": "b"},
|
||||||
|
"changes": {"x": {"before": "a", "after": "b"}},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
only_diff = _filter_inline_diffs(diffs, kind="diff", kw="")
|
||||||
|
self.assertEqual({d["kind"] for d in only_diff}, {"added", "removed", "changed"})
|
||||||
|
only_added = _filter_inline_diffs(diffs, kind="added", kw="")
|
||||||
|
self.assertEqual(len(only_added), 1)
|
||||||
|
hit = _filter_inline_diffs(diffs, kind="all", kw='"p":"4"')
|
||||||
|
self.assertEqual(len(hit), 1)
|
||||||
|
self.assertEqual(hit[0]["kind"], "changed")
|
||||||
|
hit2 = _filter_inline_diffs(diffs, kind="all", kw="changed")
|
||||||
|
self.assertTrue(any(d["kind"] == "changed" for d in hit2))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import { Button, Input, Modal } from "@heroui/react";
|
import { Button, Input, Modal } from "@heroui/react";
|
||||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||||
|
import { ListPager } from "../../components/ListPager";
|
||||||
import { AppModalShell } from "../../components/ui/AppModalShell";
|
import { AppModalShell } from "../../components/ui/AppModalShell";
|
||||||
import { FieldSelect } from "../../components/ui/FieldSelect";
|
import { FieldSelect } from "../../components/ui/FieldSelect";
|
||||||
import { useDebouncedValue } from "../../hooks/useDebouncedValue";
|
import { useDebouncedValue } from "../../hooks/useDebouncedValue";
|
||||||
|
|
@ -16,6 +17,7 @@ import {
|
||||||
bizCompareListJobs,
|
bizCompareListJobs,
|
||||||
bizCompareListMappings,
|
bizCompareListMappings,
|
||||||
bizCompareListMetrics,
|
bizCompareListMetrics,
|
||||||
|
bizCompareListRunDiffs,
|
||||||
bizCompareListRuns,
|
bizCompareListRuns,
|
||||||
bizCompareListTemplates,
|
bizCompareListTemplates,
|
||||||
bizCompareRunJob,
|
bizCompareRunJob,
|
||||||
|
|
@ -197,10 +199,15 @@ export function BizComparePage() {
|
||||||
const [runDetail, setRunDetail] = useState<any>(null);
|
const [runDetail, setRunDetail] = useState<any>(null);
|
||||||
const [resultSheetId, setResultSheetId] = useState("");
|
const [resultSheetId, setResultSheetId] = useState("");
|
||||||
|
|
||||||
// result filters
|
// result filters (server-paged)
|
||||||
const [kindFilter, setKindFilter] = useState<KindFilter>("diff");
|
const [kindFilter, setKindFilter] = useState<KindFilter>("diff");
|
||||||
const [resultKw, setResultKw] = useState("");
|
const [resultKw, setResultKw] = useState("");
|
||||||
const debouncedResultKw = useDebouncedValue(resultKw, 200);
|
const debouncedResultKw = useDebouncedValue(resultKw, 300);
|
||||||
|
const [resultPage, setResultPage] = useState(1);
|
||||||
|
const [resultPageSize, setResultPageSize] = useState(100);
|
||||||
|
const [resultTotal, setResultTotal] = useState(0);
|
||||||
|
const [pagedDiffs, setPagedDiffs] = useState<DiffRow[]>([]);
|
||||||
|
const [diffsLoading, setDiffsLoading] = useState(false);
|
||||||
const boardRef = useRef<HTMLDivElement | null>(null);
|
const boardRef = useRef<HTMLDivElement | null>(null);
|
||||||
const [boardFs, setBoardFs] = useState(false);
|
const [boardFs, setBoardFs] = useState(false);
|
||||||
|
|
||||||
|
|
@ -316,38 +323,71 @@ export function BizComparePage() {
|
||||||
[runSheets, resultSheetId],
|
[runSheets, resultSheetId],
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Reset page when sheet / filter / page size changes
|
||||||
|
useEffect(() => {
|
||||||
|
setResultPage(1);
|
||||||
|
}, [resultSheetId, kindFilter, debouncedResultKw, resultPageSize, runDetail?.id]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const runId = String(runDetail?.id || "");
|
||||||
|
const mid = resultSheetId || activeRunSheet?.metric_id || "";
|
||||||
|
if (!runId || !mid || jobDetailTab !== "result") {
|
||||||
|
setPagedDiffs([]);
|
||||||
|
setResultTotal(0);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let cancelled = false;
|
||||||
|
void (async () => {
|
||||||
|
setDiffsLoading(true);
|
||||||
|
try {
|
||||||
|
const res = await bizCompareListRunDiffs({
|
||||||
|
runId,
|
||||||
|
metricId: mid,
|
||||||
|
kind: kindFilter,
|
||||||
|
kw: debouncedResultKw.trim(),
|
||||||
|
page: resultPage,
|
||||||
|
pageSize: resultPageSize,
|
||||||
|
});
|
||||||
|
if (cancelled) return;
|
||||||
|
setPagedDiffs((res.items || []) as DiffRow[]);
|
||||||
|
setResultTotal(Number(res.total || 0));
|
||||||
|
const pages = Math.max(
|
||||||
|
1,
|
||||||
|
Math.ceil(Number(res.total || 0) / Number(res.page_size || resultPageSize)),
|
||||||
|
);
|
||||||
|
if (resultPage > pages) setResultPage(pages);
|
||||||
|
} catch (e) {
|
||||||
|
if (!cancelled) {
|
||||||
|
setPagedDiffs([]);
|
||||||
|
setResultTotal(0);
|
||||||
|
showError(formatErr(e));
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (!cancelled) setDiffsLoading(false);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [
|
||||||
|
runDetail?.id,
|
||||||
|
resultSheetId,
|
||||||
|
activeRunSheet?.metric_id,
|
||||||
|
kindFilter,
|
||||||
|
debouncedResultKw,
|
||||||
|
resultPage,
|
||||||
|
resultPageSize,
|
||||||
|
jobDetailTab,
|
||||||
|
showError,
|
||||||
|
]);
|
||||||
|
|
||||||
const resultColumns = useMemo(() => {
|
const resultColumns = useMemo(() => {
|
||||||
const keys =
|
const keys = activeRunSheet?.key_fields?.length
|
||||||
activeRunSheet?.key_fields?.length
|
? activeRunSheet.key_fields
|
||||||
? activeRunSheet.key_fields
|
: Object.keys((pagedDiffs[0]?.key as Record<string, unknown>) || {});
|
||||||
: Object.keys((activeRunSheet?.diffs?.[0]?.key as any) || {});
|
|
||||||
const compare = (activeRunSheet?.compare_fields || []).filter((f) => !keys.includes(f));
|
const compare = (activeRunSheet?.compare_fields || []).filter((f) => !keys.includes(f));
|
||||||
return { keys, compare, presence: !(activeRunSheet?.compare_fields || []).length };
|
return { keys, compare, presence: !(activeRunSheet?.compare_fields || []).length };
|
||||||
}, [activeRunSheet]);
|
}, [activeRunSheet, pagedDiffs]);
|
||||||
|
|
||||||
const filteredDiffs = useMemo(() => {
|
|
||||||
const diffs = (activeRunSheet?.diffs || []) as DiffRow[];
|
|
||||||
const kw = debouncedResultKw.trim().toLowerCase();
|
|
||||||
return diffs.filter((d) => {
|
|
||||||
if (kindFilter === "diff") {
|
|
||||||
if (d.kind === "unchanged") return false;
|
|
||||||
} else if (kindFilter !== "all" && d.kind !== kindFilter) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!kw) return true;
|
|
||||||
const blob = [
|
|
||||||
d.kind,
|
|
||||||
...Object.values(d.key || {}),
|
|
||||||
...Object.values(d.before || {}),
|
|
||||||
...Object.values(d.after || {}),
|
|
||||||
JSON.stringify(d.changes || {}),
|
|
||||||
]
|
|
||||||
.map(cellText)
|
|
||||||
.join(" ")
|
|
||||||
.toLowerCase();
|
|
||||||
return blob.includes(kw);
|
|
||||||
});
|
|
||||||
}, [activeRunSheet, kindFilter, debouncedResultKw]);
|
|
||||||
|
|
||||||
const kindLabel = (kind: string) => {
|
const kindLabel = (kind: string) => {
|
||||||
if (kind === "added") return t("bizCompare.kindAddedShort");
|
if (kind === "added") return t("bizCompare.kindAddedShort");
|
||||||
|
|
@ -358,6 +398,7 @@ export function BizComparePage() {
|
||||||
};
|
};
|
||||||
|
|
||||||
const summary = runDetail?.summary || {};
|
const summary = runDetail?.summary || {};
|
||||||
|
const resultPages = Math.max(1, Math.ceil(resultTotal / Math.max(1, resultPageSize)));
|
||||||
const sheetCards = useMemo(() => {
|
const sheetCards = useMemo(() => {
|
||||||
const raw = (summary.sheet_cards || []) as Array<{
|
const raw = (summary.sheet_cards || []) as Array<{
|
||||||
metric_id: string;
|
metric_id: string;
|
||||||
|
|
@ -1484,7 +1525,9 @@ export function BizComparePage() {
|
||||||
onChange={(e) => setResultKw(e.target.value)}
|
onChange={(e) => setResultKw(e.target.value)}
|
||||||
/>
|
/>
|
||||||
<span className="muted bs-sheet-count">
|
<span className="muted bs-sheet-count">
|
||||||
{filteredDiffs.length}/{(activeRunSheet?.diffs || []).length}
|
{diffsLoading
|
||||||
|
? "…"
|
||||||
|
: `${pagedDiffs.length}/${resultTotal}`}
|
||||||
{resultColumns.presence ? ` · ${t("bizCompare.presenceShort")}` : ""}
|
{resultColumns.presence ? ` · ${t("bizCompare.presenceShort")}` : ""}
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -1503,7 +1546,7 @@ export function BizComparePage() {
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{filteredDiffs.slice(0, 2000).map((d, i) => {
|
{pagedDiffs.map((d, i) => {
|
||||||
const pre = (d.mapped_before || d.before || {}) as Record<
|
const pre = (d.mapped_before || d.before || {}) as Record<
|
||||||
string,
|
string,
|
||||||
unknown
|
unknown
|
||||||
|
|
@ -1569,7 +1612,7 @@ export function BizComparePage() {
|
||||||
</tr>
|
</tr>
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
{runDetail && !filteredDiffs.length ? (
|
{runDetail && !diffsLoading && !pagedDiffs.length ? (
|
||||||
<tr>
|
<tr>
|
||||||
<td
|
<td
|
||||||
colSpan={
|
colSpan={
|
||||||
|
|
@ -1585,6 +1628,20 @@ export function BizComparePage() {
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<ListPager
|
||||||
|
page={resultPage}
|
||||||
|
pages={resultPages}
|
||||||
|
total={resultTotal}
|
||||||
|
pageSize={resultPageSize}
|
||||||
|
pageSizeOptions={[50, 100, 200, 500]}
|
||||||
|
onPageChange={setResultPage}
|
||||||
|
onPageSizeChange={(n) => {
|
||||||
|
setResultPageSize(n);
|
||||||
|
setResultPage(1);
|
||||||
|
}}
|
||||||
|
disabled={diffsLoading || !runDetail}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
|
|
|
||||||
|
|
@ -1912,6 +1912,29 @@ export const bizCompareListRuns = (jobId: string, limit = 20) =>
|
||||||
export const bizCompareGetRun = (runId: string) =>
|
export const bizCompareGetRun = (runId: string) =>
|
||||||
apiGet<Record<string, unknown>>(`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`);
|
apiGet<Record<string, unknown>>(`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`);
|
||||||
|
|
||||||
|
export const bizCompareListRunDiffs = (params: {
|
||||||
|
runId: string;
|
||||||
|
metricId?: string;
|
||||||
|
kind?: string;
|
||||||
|
kw?: string;
|
||||||
|
page?: number;
|
||||||
|
pageSize?: number;
|
||||||
|
}) => {
|
||||||
|
const p = new URLSearchParams();
|
||||||
|
if (params.metricId) p.set("metric_id", params.metricId);
|
||||||
|
if (params.kind) p.set("kind", params.kind);
|
||||||
|
if (params.kw) p.set("kw", params.kw);
|
||||||
|
p.set("page", String(Math.max(1, Number(params.page || 1))));
|
||||||
|
p.set("page_size", String(Math.max(1, Math.min(500, Number(params.pageSize || 100)))));
|
||||||
|
return apiGet<{
|
||||||
|
total: number;
|
||||||
|
page: number;
|
||||||
|
page_size: number;
|
||||||
|
metric_id: string;
|
||||||
|
items: Record<string, unknown>[];
|
||||||
|
}>(`/v1/biz-state/compare/runs/${encodeURIComponent(params.runId)}/diffs?${p.toString()}`);
|
||||||
|
};
|
||||||
|
|
||||||
export const bizCompareDownloadRun = async (runId: string): Promise<void> => {
|
export const bizCompareDownloadRun = async (runId: string): Promise<void> => {
|
||||||
const path = `/v1/biz-state/compare/runs/${encodeURIComponent(runId)}/export`;
|
const path = `/v1/biz-state/compare/runs/${encodeURIComponent(runId)}/export`;
|
||||||
const res = await fetch(path, { method: "GET", credentials: fetchCreds, headers: authHeaders() });
|
const res = await fetch(path, { method: "GET", credentials: fetchCreds, headers: authHeaders() });
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue