mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 03:10:46 +08:00
Store compare diffs in row table with server-side paging per sheet.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
33264d7b76
commit
499a5fec2e
8 changed files with 487 additions and 69 deletions
|
|
@ -3,6 +3,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
|
@ -12,6 +13,7 @@ from fastapi import HTTPException
|
|||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
BizCompareDiff,
|
||||
BizCompareJob,
|
||||
BizCompareRun,
|
||||
BizCompareTemplate,
|
||||
|
|
@ -29,6 +31,129 @@ def _utcnow() -> datetime:
|
|||
return utcnow_naive()
|
||||
|
||||
|
||||
_DIFF_CHUNK = 2000
|
||||
_SEARCH_TEXT_MAX = 4000
|
||||
|
||||
|
||||
def _diff_search_text(d: dict[str, Any]) -> str:
|
||||
parts = [str(d.get("kind") or "")]
|
||||
for key in ("key", "before", "after", "mapped_before", "changes"):
|
||||
val = d.get(key)
|
||||
if val:
|
||||
try:
|
||||
parts.append(json.dumps(val, ensure_ascii=False, default=str, separators=(",", ":")))
|
||||
except Exception:
|
||||
parts.append(str(val))
|
||||
return " ".join(parts)[:_SEARCH_TEXT_MAX]
|
||||
|
||||
|
||||
def _top_changed_fields(diffs: list[dict[str, Any]], *, limit: int = 8) -> list[dict[str, Any]]:
|
||||
field_counts: dict[str, int] = {}
|
||||
for d in diffs:
|
||||
if str(d.get("kind") or "") != "changed":
|
||||
continue
|
||||
for fname in d.get("changes") or {}:
|
||||
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
|
||||
return sorted(
|
||||
[{"field": k, "count": v} for k, v in field_counts.items()],
|
||||
key=lambda x: (-int(x["count"]), str(x["field"])),
|
||||
)[:limit]
|
||||
|
||||
|
||||
def _persist_sheet_diffs(
|
||||
db: Session,
|
||||
*,
|
||||
run_id: str,
|
||||
metric_id: str,
|
||||
diffs: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Bulk-insert diff rows; avoids embedding million-row arrays in summary_json."""
|
||||
buf: list[dict[str, Any]] = []
|
||||
for i, d in enumerate(diffs):
|
||||
buf.append(
|
||||
{
|
||||
"id": uuid4().hex,
|
||||
"run_id": run_id,
|
||||
"metric_id": metric_id,
|
||||
"seq": i,
|
||||
"kind": str(d.get("kind") or ""),
|
||||
"key_json": dict(d.get("key") or {}),
|
||||
"before_json": dict(d.get("before") or {}),
|
||||
"after_json": dict(d.get("after") or {}),
|
||||
"mapped_before_json": dict(d.get("mapped_before") or {}),
|
||||
"changes_json": dict(d.get("changes") or {}),
|
||||
"search_text": _diff_search_text(d),
|
||||
}
|
||||
)
|
||||
if len(buf) >= _DIFF_CHUNK:
|
||||
db.bulk_insert_mappings(BizCompareDiff, buf)
|
||||
buf.clear()
|
||||
if buf:
|
||||
db.bulk_insert_mappings(BizCompareDiff, buf)
|
||||
|
||||
|
||||
def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]:
|
||||
return {
|
||||
"kind": r.kind,
|
||||
"key": r.key_json or {},
|
||||
"before": r.before_json or {},
|
||||
"after": r.after_json or {},
|
||||
"mapped_before": r.mapped_before_json or {},
|
||||
"changes": r.changes_json or {},
|
||||
}
|
||||
|
||||
|
||||
def _run_has_diff_rows(db: Session, run_id: str) -> bool:
|
||||
return (
|
||||
db.query(BizCompareDiff.id).filter(BizCompareDiff.run_id == run_id).limit(1).first()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def _filter_inline_diffs(
|
||||
diffs: list[dict[str, Any]],
|
||||
*,
|
||||
kind: str,
|
||||
kw: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
kind_n = (kind or "diff").strip().lower()
|
||||
kw_n = (kw or "").strip().lower()
|
||||
out: list[dict[str, Any]] = []
|
||||
for d in diffs:
|
||||
dk = str(d.get("kind") or "")
|
||||
if kind_n == "diff":
|
||||
if dk == "unchanged":
|
||||
continue
|
||||
elif kind_n != "all" and dk != kind_n:
|
||||
continue
|
||||
if kw_n:
|
||||
blob = _diff_search_text(d).lower()
|
||||
if kw_n not in blob:
|
||||
continue
|
||||
out.append(d)
|
||||
return out
|
||||
|
||||
|
||||
def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: Any) -> list[dict[str, Any]]:
|
||||
sheets = list(summary.get("sheets") or [])
|
||||
if sheets:
|
||||
return sheets
|
||||
return [
|
||||
{
|
||||
"metric_id": run.metric_id,
|
||||
"key_fields": list((tpl.key_fields if tpl else None) or []),
|
||||
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
|
||||
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
|
||||
"mode": "fields",
|
||||
"summary": {
|
||||
k: summary.get(k, 0)
|
||||
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count")
|
||||
},
|
||||
"diffs": list(run.diffs_json or []),
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _str_list(raw: Any) -> list[str]:
|
||||
return [str(x).strip() for x in (raw or []) if str(x).strip()]
|
||||
|
||||
|
|
@ -624,6 +749,13 @@ def delete_job(db: Session, job_id: str) -> None:
|
|||
j = db.get(BizCompareJob, job_id)
|
||||
if not j:
|
||||
raise HTTPException(status_code=404, detail="job_not_found")
|
||||
run_ids = [
|
||||
rid for (rid,) in db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id).all()
|
||||
]
|
||||
if run_ids:
|
||||
db.query(BizCompareDiff).filter(BizCompareDiff.run_id.in_(run_ids)).delete(
|
||||
synchronize_session=False
|
||||
)
|
||||
db.query(BizCompareRun).filter(BizCompareRun.job_id == job_id).delete()
|
||||
db.delete(j)
|
||||
db.commit()
|
||||
|
|
@ -725,9 +857,24 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
|
|||
mapping_by_metric[one["metric_id"]] = one["mapping_stats"]
|
||||
|
||||
first = sheet_results[0]
|
||||
field_counts: dict[str, int] = {}
|
||||
for s in sheet_results:
|
||||
for d in list(s.get("diffs") or []):
|
||||
if str(d.get("kind") or "") != "changed":
|
||||
continue
|
||||
for fname in d.get("changes") or {}:
|
||||
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
|
||||
top_fields = sorted(
|
||||
[{"field": k, "count": v} for k, v in field_counts.items()],
|
||||
key=lambda x: (-int(x["count"]), str(x["field"])),
|
||||
)[:8]
|
||||
|
||||
run_id = uuid4().hex
|
||||
# Persist counts/meta only — diffs go to biz_compare_diff rows
|
||||
summary_payload = {
|
||||
**agg,
|
||||
"sheet_count": len(sheet_results),
|
||||
"top_changed_fields": top_fields,
|
||||
"sheets": [
|
||||
{
|
||||
"metric_id": s["metric_id"],
|
||||
|
|
@ -736,14 +883,13 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
|
|||
"compare_fields": s["compare_fields"],
|
||||
"mode": s["mode"],
|
||||
"summary": s["summary"],
|
||||
"diffs": s["diffs"],
|
||||
}
|
||||
for s in sheet_results
|
||||
],
|
||||
}
|
||||
|
||||
run = BizCompareRun(
|
||||
id=uuid4().hex,
|
||||
id=run_id,
|
||||
job_id=j.id,
|
||||
template_id=tpl.id,
|
||||
mapping_id=j.mapping_id,
|
||||
|
|
@ -752,12 +898,20 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
|
|||
metric_id=first["metric_id"],
|
||||
status="success",
|
||||
summary_json=summary_payload,
|
||||
diffs_json=first["diffs"],
|
||||
diffs_json=[],
|
||||
mapping_stats_json=mapping_by_metric,
|
||||
message="",
|
||||
created_at=_utcnow(),
|
||||
)
|
||||
db.add(run)
|
||||
db.flush()
|
||||
for s in sheet_results:
|
||||
_persist_sheet_diffs(
|
||||
db,
|
||||
run_id=run_id,
|
||||
metric_id=str(s["metric_id"]),
|
||||
diffs=list(s.get("diffs") or []),
|
||||
)
|
||||
j.updated_at = _utcnow()
|
||||
if j.mode == "manual":
|
||||
j.after_batch_id = after_batch_id
|
||||
|
|
@ -815,7 +969,6 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
|||
pass_rate = round((unchanged / matched) * 100, 1) if matched else (100.0 if total == 0 else 0.0)
|
||||
diff_rate = round((diff_count / total) * 100, 1) if total else 0.0
|
||||
|
||||
field_counts: dict[str, int] = {}
|
||||
sheet_cards: list[dict[str, Any]] = []
|
||||
for sh in sheets:
|
||||
ss = dict(sh.get("summary") or {})
|
||||
|
|
@ -839,16 +992,21 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
|||
"pass_rate": round((su / sm) * 100, 1) if sm else (100.0 if st == 0 else 0.0),
|
||||
}
|
||||
)
|
||||
for d in list(sh.get("diffs") or []):
|
||||
if str(d.get("kind") or "") != "changed":
|
||||
continue
|
||||
for fname in (d.get("changes") or {}):
|
||||
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
|
||||
|
||||
top_fields = sorted(
|
||||
[{"field": k, "count": v} for k, v in field_counts.items()],
|
||||
key=lambda x: (-int(x["count"]), str(x["field"])),
|
||||
)[:8]
|
||||
top_fields = list(summary.get("top_changed_fields") or [])
|
||||
if not top_fields:
|
||||
# Legacy runs that still embed diffs in summary_json
|
||||
field_counts: dict[str, int] = {}
|
||||
for sh in sheets:
|
||||
for d in list(sh.get("diffs") or []):
|
||||
if str(d.get("kind") or "") != "changed":
|
||||
continue
|
||||
for fname in d.get("changes") or {}:
|
||||
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
|
||||
top_fields = sorted(
|
||||
[{"field": k, "count": v} for k, v in field_counts.items()],
|
||||
key=lambda x: (-int(x["count"]), str(x["field"])),
|
||||
)[:8]
|
||||
|
||||
return {
|
||||
"added": added,
|
||||
|
|
@ -875,24 +1033,21 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
|
|||
raise HTTPException(status_code=404, detail="run_not_found")
|
||||
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
|
||||
summary = dict(r.summary_json or {})
|
||||
sheets = list(summary.get("sheets") or [])
|
||||
if not sheets:
|
||||
# Legacy single-metric run
|
||||
sheets = [
|
||||
{
|
||||
"metric_id": r.metric_id,
|
||||
"key_fields": list((tpl.key_fields if tpl else None) or []),
|
||||
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
|
||||
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
|
||||
"mode": "fields",
|
||||
"summary": {
|
||||
k: summary.get(k, 0)
|
||||
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count")
|
||||
},
|
||||
"diffs": list(r.diffs_json or []),
|
||||
}
|
||||
]
|
||||
enriched = _enrich_summary(summary, sheets)
|
||||
raw_sheets = _sheet_meta_from_summary(summary, r, tpl)
|
||||
# Never return full diffs in run detail (million-row safe)
|
||||
sheets = [
|
||||
{
|
||||
"metric_id": sh.get("metric_id") or "",
|
||||
"key_fields": list(sh.get("key_fields") or []),
|
||||
"iface_fields": list(sh.get("iface_fields") or []),
|
||||
"compare_fields": list(sh.get("compare_fields") or []),
|
||||
"mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
|
||||
"summary": dict(sh.get("summary") or {}),
|
||||
}
|
||||
for sh in raw_sheets
|
||||
]
|
||||
enriched = _enrich_summary(summary, raw_sheets)
|
||||
stored = "rows" if _run_has_diff_rows(db, run_id) else "inline"
|
||||
return {
|
||||
"id": r.id,
|
||||
"job_id": r.job_id,
|
||||
|
|
@ -904,7 +1059,8 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
|
|||
"status": r.status,
|
||||
"summary": enriched,
|
||||
"sheets": sheets,
|
||||
"diffs": list(r.diffs_json or []),
|
||||
"diffs": [],
|
||||
"diffs_stored": stored,
|
||||
"mapping_stats": r.mapping_stats_json or {},
|
||||
"message": r.message,
|
||||
"created_at": r.created_at.isoformat() + "Z" if r.created_at else None,
|
||||
|
|
@ -912,6 +1068,110 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
|
|||
}
|
||||
|
||||
|
||||
def list_run_diffs(
|
||||
db: Session,
|
||||
run_id: str,
|
||||
*,
|
||||
metric_id: str = "",
|
||||
kind: str = "diff",
|
||||
kw: str = "",
|
||||
page: int = 1,
|
||||
page_size: int = 100,
|
||||
) -> dict[str, Any]:
|
||||
r = db.get(BizCompareRun, run_id)
|
||||
if not r:
|
||||
raise HTTPException(status_code=404, detail="run_not_found")
|
||||
page_n = max(1, int(page or 1))
|
||||
size_n = max(1, min(500, int(page_size or 100)))
|
||||
kind_n = (kind or "diff").strip().lower()
|
||||
kw_n = (kw or "").strip()
|
||||
|
||||
summary = dict(r.summary_json or {})
|
||||
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
|
||||
sheets = _sheet_meta_from_summary(summary, r, tpl)
|
||||
mid = (metric_id or "").strip() or (sheets[0].get("metric_id") if sheets else r.metric_id) or ""
|
||||
|
||||
if _run_has_diff_rows(db, run_id):
|
||||
q = db.query(BizCompareDiff).filter(
|
||||
BizCompareDiff.run_id == run_id,
|
||||
BizCompareDiff.metric_id == mid,
|
||||
)
|
||||
if kind_n == "diff":
|
||||
q = q.filter(BizCompareDiff.kind.in_(("added", "removed", "changed")))
|
||||
elif kind_n != "all":
|
||||
q = q.filter(BizCompareDiff.kind == kind_n)
|
||||
if kw_n:
|
||||
q = q.filter(BizCompareDiff.search_text.ilike(f"%{kw_n}%"))
|
||||
total = q.count()
|
||||
rows = (
|
||||
q.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc())
|
||||
.offset((page_n - 1) * size_n)
|
||||
.limit(size_n)
|
||||
.all()
|
||||
)
|
||||
return {
|
||||
"total": total,
|
||||
"page": page_n,
|
||||
"page_size": size_n,
|
||||
"metric_id": mid,
|
||||
"items": [_diff_row_out(x) for x in rows],
|
||||
}
|
||||
|
||||
# Legacy: diffs embedded in summary_json / diffs_json
|
||||
sheet = next((s for s in sheets if str(s.get("metric_id") or "") == mid), None)
|
||||
if sheet is None and sheets:
|
||||
sheet = sheets[0]
|
||||
mid = str(sheet.get("metric_id") or mid)
|
||||
inline = list((sheet or {}).get("diffs") or [])
|
||||
if not inline and mid == r.metric_id:
|
||||
inline = list(r.diffs_json or [])
|
||||
filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n)
|
||||
total = len(filtered)
|
||||
start = (page_n - 1) * size_n
|
||||
page_items = filtered[start : start + size_n]
|
||||
return {
|
||||
"total": total,
|
||||
"page": page_n,
|
||||
"page_size": size_n,
|
||||
"metric_id": mid,
|
||||
"items": page_items,
|
||||
}
|
||||
|
||||
|
||||
def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str, Any]]:
|
||||
"""Load all diffs for one sheet (export). Prefer row table; fall back to inline."""
|
||||
if _run_has_diff_rows(db, run_id):
|
||||
out: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
while True:
|
||||
rows = (
|
||||
db.query(BizCompareDiff)
|
||||
.filter(BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id)
|
||||
.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc())
|
||||
.offset(offset)
|
||||
.limit(_DIFF_CHUNK)
|
||||
.all()
|
||||
)
|
||||
if not rows:
|
||||
break
|
||||
out.extend(_diff_row_out(x) for x in rows)
|
||||
offset += len(rows)
|
||||
if len(rows) < _DIFF_CHUNK:
|
||||
break
|
||||
return out
|
||||
r = db.get(BizCompareRun, run_id)
|
||||
if not r:
|
||||
return []
|
||||
summary = dict(r.summary_json or {})
|
||||
sheets = list(summary.get("sheets") or [])
|
||||
for sh in sheets:
|
||||
if str(sh.get("metric_id") or "") == metric_id:
|
||||
return list(sh.get("diffs") or [])
|
||||
if metric_id == r.metric_id:
|
||||
return list(r.diffs_json or [])
|
||||
return []
|
||||
|
||||
|
||||
def export_run_zip(db: Session, run_id: str) -> bytes:
|
||||
detail = get_run(db, run_id)
|
||||
buf = io.BytesIO()
|
||||
|
|
@ -941,8 +1201,11 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
|
|||
for sheet in list(detail.get("sheets") or []):
|
||||
mid = str(sheet.get("metric_id") or "sheet")
|
||||
safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "sheet"
|
||||
zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet))
|
||||
# summary table
|
||||
sheet_full = {
|
||||
**sheet,
|
||||
"diffs": _iter_sheet_diffs(db, run_id, mid),
|
||||
}
|
||||
zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet_full))
|
||||
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"]
|
||||
for card in list(s.get("sheet_cards") or []):
|
||||
sum_lines.append(
|
||||
|
|
|
|||
|
|
@ -24,6 +24,9 @@ def apply_biz_state_schema(conn: Connection) -> None:
|
|||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_id ON biz_compare_diff (run_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_kind ON biz_compare_diff (run_id, metric_id, kind)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_seq ON biz_compare_diff (run_id, metric_id, seq)",
|
||||
):
|
||||
try:
|
||||
_run_sql(conn, sql)
|
||||
|
|
|
|||
|
|
@ -371,6 +371,27 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
|||
return cmp_svc.get_run(db, run_id)
|
||||
|
||||
|
||||
@router.get("/compare/runs/{run_id}/diffs")
|
||||
def api_list_run_diffs(
|
||||
run_id: str,
|
||||
metric_id: str = "",
|
||||
kind: str = "diff",
|
||||
kw: str = "",
|
||||
page: int = 1,
|
||||
page_size: int = 100,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
return cmp_svc.list_run_diffs(
|
||||
db,
|
||||
run_id,
|
||||
metric_id=metric_id,
|
||||
kind=kind,
|
||||
kw=kw,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/compare/runs/{run_id}/export")
|
||||
def api_export_run(run_id: str, db: Session = Depends(get_db)) -> StreamingResponse:
|
||||
data = cmp_svc.export_run_zip(db, run_id)
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from .managed_ne import (
|
|||
UmeCliOverride,
|
||||
)
|
||||
from .biz_state import (
|
||||
BizCompareDiff,
|
||||
BizCompareJob,
|
||||
BizCompareRun,
|
||||
BizCompareTemplate,
|
||||
|
|
@ -142,4 +143,5 @@ __all__ = [
|
|||
"BizPortMappingRow",
|
||||
"BizCompareJob",
|
||||
"BizCompareRun",
|
||||
"BizCompareDiff",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
|||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import Boolean, DateTime, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ..db import Base
|
||||
|
|
@ -256,3 +256,25 @@ class BizCompareRun(Base):
|
|||
mapping_stats_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
message: Mapped[str] = mapped_column(String(1024), default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive, index=True)
|
||||
|
||||
|
||||
class BizCompareDiff(Base):
|
||||
"""Per-row compare result — supports million-scale sheets with paged queries."""
|
||||
|
||||
__tablename__ = "biz_compare_diff"
|
||||
__table_args__ = (
|
||||
Index("ix_biz_compare_diff_run_metric_kind", "run_id", "metric_id", "kind"),
|
||||
Index("ix_biz_compare_diff_run_metric_seq", "run_id", "metric_id", "seq"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex)
|
||||
run_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
||||
metric_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
||||
seq: Mapped[int] = mapped_column(Integer, default=0)
|
||||
kind: Mapped[str] = mapped_column(String(16), default="", index=True)
|
||||
key_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
before_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
after_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
mapped_before_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
changes_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
search_text: Mapped[str] = mapped_column(Text, default="")
|
||||
|
|
|
|||
|
|
@ -125,5 +125,32 @@ class CompareEngineTests(unittest.TestCase):
|
|||
self.assertEqual(out["diffs"][0]["kind"], "unchanged")
|
||||
|
||||
|
||||
class CompareDiffPagingTests(unittest.TestCase):
|
||||
def test_filter_inline_diffs_kind_and_kw(self) -> None:
|
||||
from netx_api.biz_state.compare_service import _filter_inline_diffs
|
||||
|
||||
diffs = [
|
||||
{"kind": "added", "key": {"p": "1"}, "before": {}, "after": {"p": "1"}, "changes": {}},
|
||||
{"kind": "removed", "key": {"p": "2"}, "before": {"p": "2"}, "after": {}, "changes": {}},
|
||||
{"kind": "unchanged", "key": {"p": "3"}, "before": {"p": "3"}, "after": {"p": "3"}, "changes": {}},
|
||||
{
|
||||
"kind": "changed",
|
||||
"key": {"p": "4"},
|
||||
"before": {"x": "a"},
|
||||
"after": {"x": "b"},
|
||||
"changes": {"x": {"before": "a", "after": "b"}},
|
||||
},
|
||||
]
|
||||
only_diff = _filter_inline_diffs(diffs, kind="diff", kw="")
|
||||
self.assertEqual({d["kind"] for d in only_diff}, {"added", "removed", "changed"})
|
||||
only_added = _filter_inline_diffs(diffs, kind="added", kw="")
|
||||
self.assertEqual(len(only_added), 1)
|
||||
hit = _filter_inline_diffs(diffs, kind="all", kw='"p":"4"')
|
||||
self.assertEqual(len(hit), 1)
|
||||
self.assertEqual(hit[0]["kind"], "changed")
|
||||
hit2 = _filter_inline_diffs(diffs, kind="all", kw="changed")
|
||||
self.assertTrue(any(d["kind"] == "changed" for d in hit2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { Button, Input, Modal } from "@heroui/react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { ListPager } from "../../components/ListPager";
|
||||
import { AppModalShell } from "../../components/ui/AppModalShell";
|
||||
import { FieldSelect } from "../../components/ui/FieldSelect";
|
||||
import { useDebouncedValue } from "../../hooks/useDebouncedValue";
|
||||
|
|
@ -16,6 +17,7 @@ import {
|
|||
bizCompareListJobs,
|
||||
bizCompareListMappings,
|
||||
bizCompareListMetrics,
|
||||
bizCompareListRunDiffs,
|
||||
bizCompareListRuns,
|
||||
bizCompareListTemplates,
|
||||
bizCompareRunJob,
|
||||
|
|
@ -197,10 +199,15 @@ export function BizComparePage() {
|
|||
const [runDetail, setRunDetail] = useState<any>(null);
|
||||
const [resultSheetId, setResultSheetId] = useState("");
|
||||
|
||||
// result filters
|
||||
// result filters (server-paged)
|
||||
const [kindFilter, setKindFilter] = useState<KindFilter>("diff");
|
||||
const [resultKw, setResultKw] = useState("");
|
||||
const debouncedResultKw = useDebouncedValue(resultKw, 200);
|
||||
const debouncedResultKw = useDebouncedValue(resultKw, 300);
|
||||
const [resultPage, setResultPage] = useState(1);
|
||||
const [resultPageSize, setResultPageSize] = useState(100);
|
||||
const [resultTotal, setResultTotal] = useState(0);
|
||||
const [pagedDiffs, setPagedDiffs] = useState<DiffRow[]>([]);
|
||||
const [diffsLoading, setDiffsLoading] = useState(false);
|
||||
const boardRef = useRef<HTMLDivElement | null>(null);
|
||||
const [boardFs, setBoardFs] = useState(false);
|
||||
|
||||
|
|
@ -316,38 +323,71 @@ export function BizComparePage() {
|
|||
[runSheets, resultSheetId],
|
||||
);
|
||||
|
||||
// Reset page when sheet / filter / page size changes
|
||||
useEffect(() => {
|
||||
setResultPage(1);
|
||||
}, [resultSheetId, kindFilter, debouncedResultKw, resultPageSize, runDetail?.id]);
|
||||
|
||||
useEffect(() => {
|
||||
const runId = String(runDetail?.id || "");
|
||||
const mid = resultSheetId || activeRunSheet?.metric_id || "";
|
||||
if (!runId || !mid || jobDetailTab !== "result") {
|
||||
setPagedDiffs([]);
|
||||
setResultTotal(0);
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
void (async () => {
|
||||
setDiffsLoading(true);
|
||||
try {
|
||||
const res = await bizCompareListRunDiffs({
|
||||
runId,
|
||||
metricId: mid,
|
||||
kind: kindFilter,
|
||||
kw: debouncedResultKw.trim(),
|
||||
page: resultPage,
|
||||
pageSize: resultPageSize,
|
||||
});
|
||||
if (cancelled) return;
|
||||
setPagedDiffs((res.items || []) as DiffRow[]);
|
||||
setResultTotal(Number(res.total || 0));
|
||||
const pages = Math.max(
|
||||
1,
|
||||
Math.ceil(Number(res.total || 0) / Number(res.page_size || resultPageSize)),
|
||||
);
|
||||
if (resultPage > pages) setResultPage(pages);
|
||||
} catch (e) {
|
||||
if (!cancelled) {
|
||||
setPagedDiffs([]);
|
||||
setResultTotal(0);
|
||||
showError(formatErr(e));
|
||||
}
|
||||
} finally {
|
||||
if (!cancelled) setDiffsLoading(false);
|
||||
}
|
||||
})();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [
|
||||
runDetail?.id,
|
||||
resultSheetId,
|
||||
activeRunSheet?.metric_id,
|
||||
kindFilter,
|
||||
debouncedResultKw,
|
||||
resultPage,
|
||||
resultPageSize,
|
||||
jobDetailTab,
|
||||
showError,
|
||||
]);
|
||||
|
||||
const resultColumns = useMemo(() => {
|
||||
const keys =
|
||||
activeRunSheet?.key_fields?.length
|
||||
? activeRunSheet.key_fields
|
||||
: Object.keys((activeRunSheet?.diffs?.[0]?.key as any) || {});
|
||||
const keys = activeRunSheet?.key_fields?.length
|
||||
? activeRunSheet.key_fields
|
||||
: Object.keys((pagedDiffs[0]?.key as Record<string, unknown>) || {});
|
||||
const compare = (activeRunSheet?.compare_fields || []).filter((f) => !keys.includes(f));
|
||||
return { keys, compare, presence: !(activeRunSheet?.compare_fields || []).length };
|
||||
}, [activeRunSheet]);
|
||||
|
||||
const filteredDiffs = useMemo(() => {
|
||||
const diffs = (activeRunSheet?.diffs || []) as DiffRow[];
|
||||
const kw = debouncedResultKw.trim().toLowerCase();
|
||||
return diffs.filter((d) => {
|
||||
if (kindFilter === "diff") {
|
||||
if (d.kind === "unchanged") return false;
|
||||
} else if (kindFilter !== "all" && d.kind !== kindFilter) {
|
||||
return false;
|
||||
}
|
||||
if (!kw) return true;
|
||||
const blob = [
|
||||
d.kind,
|
||||
...Object.values(d.key || {}),
|
||||
...Object.values(d.before || {}),
|
||||
...Object.values(d.after || {}),
|
||||
JSON.stringify(d.changes || {}),
|
||||
]
|
||||
.map(cellText)
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
return blob.includes(kw);
|
||||
});
|
||||
}, [activeRunSheet, kindFilter, debouncedResultKw]);
|
||||
}, [activeRunSheet, pagedDiffs]);
|
||||
|
||||
const kindLabel = (kind: string) => {
|
||||
if (kind === "added") return t("bizCompare.kindAddedShort");
|
||||
|
|
@ -358,6 +398,7 @@ export function BizComparePage() {
|
|||
};
|
||||
|
||||
const summary = runDetail?.summary || {};
|
||||
const resultPages = Math.max(1, Math.ceil(resultTotal / Math.max(1, resultPageSize)));
|
||||
const sheetCards = useMemo(() => {
|
||||
const raw = (summary.sheet_cards || []) as Array<{
|
||||
metric_id: string;
|
||||
|
|
@ -1484,7 +1525,9 @@ export function BizComparePage() {
|
|||
onChange={(e) => setResultKw(e.target.value)}
|
||||
/>
|
||||
<span className="muted bs-sheet-count">
|
||||
{filteredDiffs.length}/{(activeRunSheet?.diffs || []).length}
|
||||
{diffsLoading
|
||||
? "…"
|
||||
: `${pagedDiffs.length}/${resultTotal}`}
|
||||
{resultColumns.presence ? ` · ${t("bizCompare.presenceShort")}` : ""}
|
||||
</span>
|
||||
</div>
|
||||
|
|
@ -1503,7 +1546,7 @@ export function BizComparePage() {
|
|||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filteredDiffs.slice(0, 2000).map((d, i) => {
|
||||
{pagedDiffs.map((d, i) => {
|
||||
const pre = (d.mapped_before || d.before || {}) as Record<
|
||||
string,
|
||||
unknown
|
||||
|
|
@ -1569,7 +1612,7 @@ export function BizComparePage() {
|
|||
</tr>
|
||||
);
|
||||
})}
|
||||
{runDetail && !filteredDiffs.length ? (
|
||||
{runDetail && !diffsLoading && !pagedDiffs.length ? (
|
||||
<tr>
|
||||
<td
|
||||
colSpan={
|
||||
|
|
@ -1585,6 +1628,20 @@ export function BizComparePage() {
|
|||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<ListPager
|
||||
page={resultPage}
|
||||
pages={resultPages}
|
||||
total={resultTotal}
|
||||
pageSize={resultPageSize}
|
||||
pageSizeOptions={[50, 100, 200, 500]}
|
||||
onPageChange={setResultPage}
|
||||
onPageSizeChange={(n) => {
|
||||
setResultPageSize(n);
|
||||
setResultPage(1);
|
||||
}}
|
||||
disabled={diffsLoading || !runDetail}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
|
|
|
|||
|
|
@ -1912,6 +1912,29 @@ export const bizCompareListRuns = (jobId: string, limit = 20) =>
|
|||
export const bizCompareGetRun = (runId: string) =>
|
||||
apiGet<Record<string, unknown>>(`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`);
|
||||
|
||||
export const bizCompareListRunDiffs = (params: {
|
||||
runId: string;
|
||||
metricId?: string;
|
||||
kind?: string;
|
||||
kw?: string;
|
||||
page?: number;
|
||||
pageSize?: number;
|
||||
}) => {
|
||||
const p = new URLSearchParams();
|
||||
if (params.metricId) p.set("metric_id", params.metricId);
|
||||
if (params.kind) p.set("kind", params.kind);
|
||||
if (params.kw) p.set("kw", params.kw);
|
||||
p.set("page", String(Math.max(1, Number(params.page || 1))));
|
||||
p.set("page_size", String(Math.max(1, Math.min(500, Number(params.pageSize || 100)))));
|
||||
return apiGet<{
|
||||
total: number;
|
||||
page: number;
|
||||
page_size: number;
|
||||
metric_id: string;
|
||||
items: Record<string, unknown>[];
|
||||
}>(`/v1/biz-state/compare/runs/${encodeURIComponent(params.runId)}/diffs?${p.toString()}`);
|
||||
};
|
||||
|
||||
export const bizCompareDownloadRun = async (runId: string): Promise<void> => {
|
||||
const path = `/v1/biz-state/compare/runs/${encodeURIComponent(runId)}/export`;
|
||||
const res = await fetch(path, { method: "GET", credentials: fetchCreds, headers: authHeaders() });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue