Store compare diffs in row table with server-side paging per sheet.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-17 19:59:06 +08:00
parent 33264d7b76
commit 499a5fec2e
8 changed files with 487 additions and 69 deletions

View file

@ -3,6 +3,7 @@
from __future__ import annotations from __future__ import annotations
import io import io
import json
import zipfile import zipfile
from datetime import datetime from datetime import datetime
from typing import Any from typing import Any
@ -12,6 +13,7 @@ from fastapi import HTTPException
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from ..models import ( from ..models import (
BizCompareDiff,
BizCompareJob, BizCompareJob,
BizCompareRun, BizCompareRun,
BizCompareTemplate, BizCompareTemplate,
@ -29,6 +31,129 @@ def _utcnow() -> datetime:
return utcnow_naive() return utcnow_naive()
_DIFF_CHUNK = 2000
_SEARCH_TEXT_MAX = 4000
def _diff_search_text(d: dict[str, Any]) -> str:
parts = [str(d.get("kind") or "")]
for key in ("key", "before", "after", "mapped_before", "changes"):
val = d.get(key)
if val:
try:
parts.append(json.dumps(val, ensure_ascii=False, default=str, separators=(",", ":")))
except Exception:
parts.append(str(val))
return " ".join(parts)[:_SEARCH_TEXT_MAX]
def _top_changed_fields(diffs: list[dict[str, Any]], *, limit: int = 8) -> list[dict[str, Any]]:
field_counts: dict[str, int] = {}
for d in diffs:
if str(d.get("kind") or "") != "changed":
continue
for fname in d.get("changes") or {}:
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
return sorted(
[{"field": k, "count": v} for k, v in field_counts.items()],
key=lambda x: (-int(x["count"]), str(x["field"])),
)[:limit]
def _persist_sheet_diffs(
db: Session,
*,
run_id: str,
metric_id: str,
diffs: list[dict[str, Any]],
) -> None:
"""Bulk-insert diff rows; avoids embedding million-row arrays in summary_json."""
buf: list[dict[str, Any]] = []
for i, d in enumerate(diffs):
buf.append(
{
"id": uuid4().hex,
"run_id": run_id,
"metric_id": metric_id,
"seq": i,
"kind": str(d.get("kind") or ""),
"key_json": dict(d.get("key") or {}),
"before_json": dict(d.get("before") or {}),
"after_json": dict(d.get("after") or {}),
"mapped_before_json": dict(d.get("mapped_before") or {}),
"changes_json": dict(d.get("changes") or {}),
"search_text": _diff_search_text(d),
}
)
if len(buf) >= _DIFF_CHUNK:
db.bulk_insert_mappings(BizCompareDiff, buf)
buf.clear()
if buf:
db.bulk_insert_mappings(BizCompareDiff, buf)
def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]:
return {
"kind": r.kind,
"key": r.key_json or {},
"before": r.before_json or {},
"after": r.after_json or {},
"mapped_before": r.mapped_before_json or {},
"changes": r.changes_json or {},
}
def _run_has_diff_rows(db: Session, run_id: str) -> bool:
return (
db.query(BizCompareDiff.id).filter(BizCompareDiff.run_id == run_id).limit(1).first()
is not None
)
def _filter_inline_diffs(
diffs: list[dict[str, Any]],
*,
kind: str,
kw: str,
) -> list[dict[str, Any]]:
kind_n = (kind or "diff").strip().lower()
kw_n = (kw or "").strip().lower()
out: list[dict[str, Any]] = []
for d in diffs:
dk = str(d.get("kind") or "")
if kind_n == "diff":
if dk == "unchanged":
continue
elif kind_n != "all" and dk != kind_n:
continue
if kw_n:
blob = _diff_search_text(d).lower()
if kw_n not in blob:
continue
out.append(d)
return out
def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: Any) -> list[dict[str, Any]]:
sheets = list(summary.get("sheets") or [])
if sheets:
return sheets
return [
{
"metric_id": run.metric_id,
"key_fields": list((tpl.key_fields if tpl else None) or []),
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
"mode": "fields",
"summary": {
k: summary.get(k, 0)
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count")
},
"diffs": list(run.diffs_json or []),
}
]
def _str_list(raw: Any) -> list[str]: def _str_list(raw: Any) -> list[str]:
return [str(x).strip() for x in (raw or []) if str(x).strip()] return [str(x).strip() for x in (raw or []) if str(x).strip()]
@ -624,6 +749,13 @@ def delete_job(db: Session, job_id: str) -> None:
j = db.get(BizCompareJob, job_id) j = db.get(BizCompareJob, job_id)
if not j: if not j:
raise HTTPException(status_code=404, detail="job_not_found") raise HTTPException(status_code=404, detail="job_not_found")
run_ids = [
rid for (rid,) in db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id).all()
]
if run_ids:
db.query(BizCompareDiff).filter(BizCompareDiff.run_id.in_(run_ids)).delete(
synchronize_session=False
)
db.query(BizCompareRun).filter(BizCompareRun.job_id == job_id).delete() db.query(BizCompareRun).filter(BizCompareRun.job_id == job_id).delete()
db.delete(j) db.delete(j)
db.commit() db.commit()
@ -725,9 +857,24 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
mapping_by_metric[one["metric_id"]] = one["mapping_stats"] mapping_by_metric[one["metric_id"]] = one["mapping_stats"]
first = sheet_results[0] first = sheet_results[0]
field_counts: dict[str, int] = {}
for s in sheet_results:
for d in list(s.get("diffs") or []):
if str(d.get("kind") or "") != "changed":
continue
for fname in d.get("changes") or {}:
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
top_fields = sorted(
[{"field": k, "count": v} for k, v in field_counts.items()],
key=lambda x: (-int(x["count"]), str(x["field"])),
)[:8]
run_id = uuid4().hex
# Persist counts/meta only — diffs go to biz_compare_diff rows
summary_payload = { summary_payload = {
**agg, **agg,
"sheet_count": len(sheet_results), "sheet_count": len(sheet_results),
"top_changed_fields": top_fields,
"sheets": [ "sheets": [
{ {
"metric_id": s["metric_id"], "metric_id": s["metric_id"],
@ -736,14 +883,13 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
"compare_fields": s["compare_fields"], "compare_fields": s["compare_fields"],
"mode": s["mode"], "mode": s["mode"],
"summary": s["summary"], "summary": s["summary"],
"diffs": s["diffs"],
} }
for s in sheet_results for s in sheet_results
], ],
} }
run = BizCompareRun( run = BizCompareRun(
id=uuid4().hex, id=run_id,
job_id=j.id, job_id=j.id,
template_id=tpl.id, template_id=tpl.id,
mapping_id=j.mapping_id, mapping_id=j.mapping_id,
@ -752,12 +898,20 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
metric_id=first["metric_id"], metric_id=first["metric_id"],
status="success", status="success",
summary_json=summary_payload, summary_json=summary_payload,
diffs_json=first["diffs"], diffs_json=[],
mapping_stats_json=mapping_by_metric, mapping_stats_json=mapping_by_metric,
message="", message="",
created_at=_utcnow(), created_at=_utcnow(),
) )
db.add(run) db.add(run)
db.flush()
for s in sheet_results:
_persist_sheet_diffs(
db,
run_id=run_id,
metric_id=str(s["metric_id"]),
diffs=list(s.get("diffs") or []),
)
j.updated_at = _utcnow() j.updated_at = _utcnow()
if j.mode == "manual": if j.mode == "manual":
j.after_batch_id = after_batch_id j.after_batch_id = after_batch_id
@ -815,7 +969,6 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
pass_rate = round((unchanged / matched) * 100, 1) if matched else (100.0 if total == 0 else 0.0) pass_rate = round((unchanged / matched) * 100, 1) if matched else (100.0 if total == 0 else 0.0)
diff_rate = round((diff_count / total) * 100, 1) if total else 0.0 diff_rate = round((diff_count / total) * 100, 1) if total else 0.0
field_counts: dict[str, int] = {}
sheet_cards: list[dict[str, Any]] = [] sheet_cards: list[dict[str, Any]] = []
for sh in sheets: for sh in sheets:
ss = dict(sh.get("summary") or {}) ss = dict(sh.get("summary") or {})
@ -839,16 +992,21 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
"pass_rate": round((su / sm) * 100, 1) if sm else (100.0 if st == 0 else 0.0), "pass_rate": round((su / sm) * 100, 1) if sm else (100.0 if st == 0 else 0.0),
} }
) )
for d in list(sh.get("diffs") or []):
if str(d.get("kind") or "") != "changed":
continue
for fname in (d.get("changes") or {}):
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
top_fields = sorted( top_fields = list(summary.get("top_changed_fields") or [])
[{"field": k, "count": v} for k, v in field_counts.items()], if not top_fields:
key=lambda x: (-int(x["count"]), str(x["field"])), # Legacy runs that still embed diffs in summary_json
)[:8] field_counts: dict[str, int] = {}
for sh in sheets:
for d in list(sh.get("diffs") or []):
if str(d.get("kind") or "") != "changed":
continue
for fname in d.get("changes") or {}:
field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1
top_fields = sorted(
[{"field": k, "count": v} for k, v in field_counts.items()],
key=lambda x: (-int(x["count"]), str(x["field"])),
)[:8]
return { return {
"added": added, "added": added,
@ -875,24 +1033,21 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
raise HTTPException(status_code=404, detail="run_not_found") raise HTTPException(status_code=404, detail="run_not_found")
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
summary = dict(r.summary_json or {}) summary = dict(r.summary_json or {})
sheets = list(summary.get("sheets") or []) raw_sheets = _sheet_meta_from_summary(summary, r, tpl)
if not sheets: # Never return full diffs in run detail (million-row safe)
# Legacy single-metric run sheets = [
sheets = [ {
{ "metric_id": sh.get("metric_id") or "",
"metric_id": r.metric_id, "key_fields": list(sh.get("key_fields") or []),
"key_fields": list((tpl.key_fields if tpl else None) or []), "iface_fields": list(sh.get("iface_fields") or []),
"iface_fields": list((tpl.iface_fields if tpl else None) or []), "compare_fields": list(sh.get("compare_fields") or []),
"compare_fields": list((tpl.compare_fields if tpl else None) or []), "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
"mode": "fields", "summary": dict(sh.get("summary") or {}),
"summary": { }
k: summary.get(k, 0) for sh in raw_sheets
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count") ]
}, enriched = _enrich_summary(summary, raw_sheets)
"diffs": list(r.diffs_json or []), stored = "rows" if _run_has_diff_rows(db, run_id) else "inline"
}
]
enriched = _enrich_summary(summary, sheets)
return { return {
"id": r.id, "id": r.id,
"job_id": r.job_id, "job_id": r.job_id,
@ -904,7 +1059,8 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
"status": r.status, "status": r.status,
"summary": enriched, "summary": enriched,
"sheets": sheets, "sheets": sheets,
"diffs": list(r.diffs_json or []), "diffs": [],
"diffs_stored": stored,
"mapping_stats": r.mapping_stats_json or {}, "mapping_stats": r.mapping_stats_json or {},
"message": r.message, "message": r.message,
"created_at": r.created_at.isoformat() + "Z" if r.created_at else None, "created_at": r.created_at.isoformat() + "Z" if r.created_at else None,
@ -912,6 +1068,110 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
} }
def list_run_diffs(
db: Session,
run_id: str,
*,
metric_id: str = "",
kind: str = "diff",
kw: str = "",
page: int = 1,
page_size: int = 100,
) -> dict[str, Any]:
r = db.get(BizCompareRun, run_id)
if not r:
raise HTTPException(status_code=404, detail="run_not_found")
page_n = max(1, int(page or 1))
size_n = max(1, min(500, int(page_size or 100)))
kind_n = (kind or "diff").strip().lower()
kw_n = (kw or "").strip()
summary = dict(r.summary_json or {})
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
sheets = _sheet_meta_from_summary(summary, r, tpl)
mid = (metric_id or "").strip() or (sheets[0].get("metric_id") if sheets else r.metric_id) or ""
if _run_has_diff_rows(db, run_id):
q = db.query(BizCompareDiff).filter(
BizCompareDiff.run_id == run_id,
BizCompareDiff.metric_id == mid,
)
if kind_n == "diff":
q = q.filter(BizCompareDiff.kind.in_(("added", "removed", "changed")))
elif kind_n != "all":
q = q.filter(BizCompareDiff.kind == kind_n)
if kw_n:
q = q.filter(BizCompareDiff.search_text.ilike(f"%{kw_n}%"))
total = q.count()
rows = (
q.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc())
.offset((page_n - 1) * size_n)
.limit(size_n)
.all()
)
return {
"total": total,
"page": page_n,
"page_size": size_n,
"metric_id": mid,
"items": [_diff_row_out(x) for x in rows],
}
# Legacy: diffs embedded in summary_json / diffs_json
sheet = next((s for s in sheets if str(s.get("metric_id") or "") == mid), None)
if sheet is None and sheets:
sheet = sheets[0]
mid = str(sheet.get("metric_id") or mid)
inline = list((sheet or {}).get("diffs") or [])
if not inline and mid == r.metric_id:
inline = list(r.diffs_json or [])
filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n)
total = len(filtered)
start = (page_n - 1) * size_n
page_items = filtered[start : start + size_n]
return {
"total": total,
"page": page_n,
"page_size": size_n,
"metric_id": mid,
"items": page_items,
}
def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str, Any]]:
"""Load all diffs for one sheet (export). Prefer row table; fall back to inline."""
if _run_has_diff_rows(db, run_id):
out: list[dict[str, Any]] = []
offset = 0
while True:
rows = (
db.query(BizCompareDiff)
.filter(BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id)
.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc())
.offset(offset)
.limit(_DIFF_CHUNK)
.all()
)
if not rows:
break
out.extend(_diff_row_out(x) for x in rows)
offset += len(rows)
if len(rows) < _DIFF_CHUNK:
break
return out
r = db.get(BizCompareRun, run_id)
if not r:
return []
summary = dict(r.summary_json or {})
sheets = list(summary.get("sheets") or [])
for sh in sheets:
if str(sh.get("metric_id") or "") == metric_id:
return list(sh.get("diffs") or [])
if metric_id == r.metric_id:
return list(r.diffs_json or [])
return []
def export_run_zip(db: Session, run_id: str) -> bytes: def export_run_zip(db: Session, run_id: str) -> bytes:
detail = get_run(db, run_id) detail = get_run(db, run_id)
buf = io.BytesIO() buf = io.BytesIO()
@ -941,8 +1201,11 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
for sheet in list(detail.get("sheets") or []): for sheet in list(detail.get("sheets") or []):
mid = str(sheet.get("metric_id") or "sheet") mid = str(sheet.get("metric_id") or "sheet")
safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "sheet" safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "sheet"
zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet)) sheet_full = {
# summary table **sheet,
"diffs": _iter_sheet_diffs(db, run_id, mid),
}
zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet_full))
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"] sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"]
for card in list(s.get("sheet_cards") or []): for card in list(s.get("sheet_cards") or []):
sum_lines.append( sum_lines.append(

View file

@ -24,6 +24,9 @@ def apply_biz_state_schema(conn: Connection) -> None:
"CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)",
"CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)",
"CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)",
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_id ON biz_compare_diff (run_id)",
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_kind ON biz_compare_diff (run_id, metric_id, kind)",
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_seq ON biz_compare_diff (run_id, metric_id, seq)",
): ):
try: try:
_run_sql(conn, sql) _run_sql(conn, sql)

View file

@ -371,6 +371,27 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
return cmp_svc.get_run(db, run_id) return cmp_svc.get_run(db, run_id)
@router.get("/compare/runs/{run_id}/diffs")
def api_list_run_diffs(
run_id: str,
metric_id: str = "",
kind: str = "diff",
kw: str = "",
page: int = 1,
page_size: int = 100,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return cmp_svc.list_run_diffs(
db,
run_id,
metric_id=metric_id,
kind=kind,
kw=kw,
page=page,
page_size=page_size,
)
@router.get("/compare/runs/{run_id}/export") @router.get("/compare/runs/{run_id}/export")
def api_export_run(run_id: str, db: Session = Depends(get_db)) -> StreamingResponse: def api_export_run(run_id: str, db: Session = Depends(get_db)) -> StreamingResponse:
data = cmp_svc.export_run_zip(db, run_id) data = cmp_svc.export_run_zip(db, run_id)

View file

@ -25,6 +25,7 @@ from .managed_ne import (
UmeCliOverride, UmeCliOverride,
) )
from .biz_state import ( from .biz_state import (
BizCompareDiff,
BizCompareJob, BizCompareJob,
BizCompareRun, BizCompareRun,
BizCompareTemplate, BizCompareTemplate,
@ -142,4 +143,5 @@ __all__ = [
"BizPortMappingRow", "BizPortMappingRow",
"BizCompareJob", "BizCompareJob",
"BizCompareRun", "BizCompareRun",
"BizCompareDiff",
] ]

View file

@ -5,7 +5,7 @@ from __future__ import annotations
from datetime import datetime from datetime import datetime
from uuid import uuid4 from uuid import uuid4
from sqlalchemy import Boolean, DateTime, Integer, String, Text, UniqueConstraint from sqlalchemy import Boolean, DateTime, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.orm import Mapped, mapped_column
from ..db import Base from ..db import Base
@ -256,3 +256,25 @@ class BizCompareRun(Base):
mapping_stats_json: Mapped[dict] = mapped_column(_JsonType, default=dict) mapping_stats_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
message: Mapped[str] = mapped_column(String(1024), default="") message: Mapped[str] = mapped_column(String(1024), default="")
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive, index=True) created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive, index=True)
class BizCompareDiff(Base):
"""Per-row compare result — supports million-scale sheets with paged queries."""
__tablename__ = "biz_compare_diff"
__table_args__ = (
Index("ix_biz_compare_diff_run_metric_kind", "run_id", "metric_id", "kind"),
Index("ix_biz_compare_diff_run_metric_seq", "run_id", "metric_id", "seq"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex)
run_id: Mapped[str] = mapped_column(String(64), default="", index=True)
metric_id: Mapped[str] = mapped_column(String(64), default="", index=True)
seq: Mapped[int] = mapped_column(Integer, default=0)
kind: Mapped[str] = mapped_column(String(16), default="", index=True)
key_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
before_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
after_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
mapped_before_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
changes_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
search_text: Mapped[str] = mapped_column(Text, default="")

View file

@ -125,5 +125,32 @@ class CompareEngineTests(unittest.TestCase):
self.assertEqual(out["diffs"][0]["kind"], "unchanged") self.assertEqual(out["diffs"][0]["kind"], "unchanged")
class CompareDiffPagingTests(unittest.TestCase):
def test_filter_inline_diffs_kind_and_kw(self) -> None:
from netx_api.biz_state.compare_service import _filter_inline_diffs
diffs = [
{"kind": "added", "key": {"p": "1"}, "before": {}, "after": {"p": "1"}, "changes": {}},
{"kind": "removed", "key": {"p": "2"}, "before": {"p": "2"}, "after": {}, "changes": {}},
{"kind": "unchanged", "key": {"p": "3"}, "before": {"p": "3"}, "after": {"p": "3"}, "changes": {}},
{
"kind": "changed",
"key": {"p": "4"},
"before": {"x": "a"},
"after": {"x": "b"},
"changes": {"x": {"before": "a", "after": "b"}},
},
]
only_diff = _filter_inline_diffs(diffs, kind="diff", kw="")
self.assertEqual({d["kind"] for d in only_diff}, {"added", "removed", "changed"})
only_added = _filter_inline_diffs(diffs, kind="added", kw="")
self.assertEqual(len(only_added), 1)
hit = _filter_inline_diffs(diffs, kind="all", kw='"p":"4"')
self.assertEqual(len(hit), 1)
self.assertEqual(hit[0]["kind"], "changed")
hit2 = _filter_inline_diffs(diffs, kind="all", kw="changed")
self.assertTrue(any(d["kind"] == "changed" for d in hit2))
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

View file

@ -1,5 +1,6 @@
import { Button, Input, Modal } from "@heroui/react"; import { Button, Input, Modal } from "@heroui/react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { ListPager } from "../../components/ListPager";
import { AppModalShell } from "../../components/ui/AppModalShell"; import { AppModalShell } from "../../components/ui/AppModalShell";
import { FieldSelect } from "../../components/ui/FieldSelect"; import { FieldSelect } from "../../components/ui/FieldSelect";
import { useDebouncedValue } from "../../hooks/useDebouncedValue"; import { useDebouncedValue } from "../../hooks/useDebouncedValue";
@ -16,6 +17,7 @@ import {
bizCompareListJobs, bizCompareListJobs,
bizCompareListMappings, bizCompareListMappings,
bizCompareListMetrics, bizCompareListMetrics,
bizCompareListRunDiffs,
bizCompareListRuns, bizCompareListRuns,
bizCompareListTemplates, bizCompareListTemplates,
bizCompareRunJob, bizCompareRunJob,
@ -197,10 +199,15 @@ export function BizComparePage() {
const [runDetail, setRunDetail] = useState<any>(null); const [runDetail, setRunDetail] = useState<any>(null);
const [resultSheetId, setResultSheetId] = useState(""); const [resultSheetId, setResultSheetId] = useState("");
// result filters // result filters (server-paged)
const [kindFilter, setKindFilter] = useState<KindFilter>("diff"); const [kindFilter, setKindFilter] = useState<KindFilter>("diff");
const [resultKw, setResultKw] = useState(""); const [resultKw, setResultKw] = useState("");
const debouncedResultKw = useDebouncedValue(resultKw, 200); const debouncedResultKw = useDebouncedValue(resultKw, 300);
const [resultPage, setResultPage] = useState(1);
const [resultPageSize, setResultPageSize] = useState(100);
const [resultTotal, setResultTotal] = useState(0);
const [pagedDiffs, setPagedDiffs] = useState<DiffRow[]>([]);
const [diffsLoading, setDiffsLoading] = useState(false);
const boardRef = useRef<HTMLDivElement | null>(null); const boardRef = useRef<HTMLDivElement | null>(null);
const [boardFs, setBoardFs] = useState(false); const [boardFs, setBoardFs] = useState(false);
@ -316,38 +323,71 @@ export function BizComparePage() {
[runSheets, resultSheetId], [runSheets, resultSheetId],
); );
// Reset page when sheet / filter / page size changes
useEffect(() => {
setResultPage(1);
}, [resultSheetId, kindFilter, debouncedResultKw, resultPageSize, runDetail?.id]);
useEffect(() => {
const runId = String(runDetail?.id || "");
const mid = resultSheetId || activeRunSheet?.metric_id || "";
if (!runId || !mid || jobDetailTab !== "result") {
setPagedDiffs([]);
setResultTotal(0);
return;
}
let cancelled = false;
void (async () => {
setDiffsLoading(true);
try {
const res = await bizCompareListRunDiffs({
runId,
metricId: mid,
kind: kindFilter,
kw: debouncedResultKw.trim(),
page: resultPage,
pageSize: resultPageSize,
});
if (cancelled) return;
setPagedDiffs((res.items || []) as DiffRow[]);
setResultTotal(Number(res.total || 0));
const pages = Math.max(
1,
Math.ceil(Number(res.total || 0) / Number(res.page_size || resultPageSize)),
);
if (resultPage > pages) setResultPage(pages);
} catch (e) {
if (!cancelled) {
setPagedDiffs([]);
setResultTotal(0);
showError(formatErr(e));
}
} finally {
if (!cancelled) setDiffsLoading(false);
}
})();
return () => {
cancelled = true;
};
}, [
runDetail?.id,
resultSheetId,
activeRunSheet?.metric_id,
kindFilter,
debouncedResultKw,
resultPage,
resultPageSize,
jobDetailTab,
showError,
]);
const resultColumns = useMemo(() => { const resultColumns = useMemo(() => {
const keys = const keys = activeRunSheet?.key_fields?.length
activeRunSheet?.key_fields?.length ? activeRunSheet.key_fields
? activeRunSheet.key_fields : Object.keys((pagedDiffs[0]?.key as Record<string, unknown>) || {});
: Object.keys((activeRunSheet?.diffs?.[0]?.key as any) || {});
const compare = (activeRunSheet?.compare_fields || []).filter((f) => !keys.includes(f)); const compare = (activeRunSheet?.compare_fields || []).filter((f) => !keys.includes(f));
return { keys, compare, presence: !(activeRunSheet?.compare_fields || []).length }; return { keys, compare, presence: !(activeRunSheet?.compare_fields || []).length };
}, [activeRunSheet]); }, [activeRunSheet, pagedDiffs]);
const filteredDiffs = useMemo(() => {
const diffs = (activeRunSheet?.diffs || []) as DiffRow[];
const kw = debouncedResultKw.trim().toLowerCase();
return diffs.filter((d) => {
if (kindFilter === "diff") {
if (d.kind === "unchanged") return false;
} else if (kindFilter !== "all" && d.kind !== kindFilter) {
return false;
}
if (!kw) return true;
const blob = [
d.kind,
...Object.values(d.key || {}),
...Object.values(d.before || {}),
...Object.values(d.after || {}),
JSON.stringify(d.changes || {}),
]
.map(cellText)
.join(" ")
.toLowerCase();
return blob.includes(kw);
});
}, [activeRunSheet, kindFilter, debouncedResultKw]);
const kindLabel = (kind: string) => { const kindLabel = (kind: string) => {
if (kind === "added") return t("bizCompare.kindAddedShort"); if (kind === "added") return t("bizCompare.kindAddedShort");
@ -358,6 +398,7 @@ export function BizComparePage() {
}; };
const summary = runDetail?.summary || {}; const summary = runDetail?.summary || {};
const resultPages = Math.max(1, Math.ceil(resultTotal / Math.max(1, resultPageSize)));
const sheetCards = useMemo(() => { const sheetCards = useMemo(() => {
const raw = (summary.sheet_cards || []) as Array<{ const raw = (summary.sheet_cards || []) as Array<{
metric_id: string; metric_id: string;
@ -1484,7 +1525,9 @@ export function BizComparePage() {
onChange={(e) => setResultKw(e.target.value)} onChange={(e) => setResultKw(e.target.value)}
/> />
<span className="muted bs-sheet-count"> <span className="muted bs-sheet-count">
{filteredDiffs.length}/{(activeRunSheet?.diffs || []).length} {diffsLoading
? "…"
: `${pagedDiffs.length}/${resultTotal}`}
{resultColumns.presence ? ` · ${t("bizCompare.presenceShort")}` : ""} {resultColumns.presence ? ` · ${t("bizCompare.presenceShort")}` : ""}
</span> </span>
</div> </div>
@ -1503,7 +1546,7 @@ export function BizComparePage() {
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{filteredDiffs.slice(0, 2000).map((d, i) => { {pagedDiffs.map((d, i) => {
const pre = (d.mapped_before || d.before || {}) as Record< const pre = (d.mapped_before || d.before || {}) as Record<
string, string,
unknown unknown
@ -1569,7 +1612,7 @@ export function BizComparePage() {
</tr> </tr>
); );
})} })}
{runDetail && !filteredDiffs.length ? ( {runDetail && !diffsLoading && !pagedDiffs.length ? (
<tr> <tr>
<td <td
colSpan={ colSpan={
@ -1585,6 +1628,20 @@ export function BizComparePage() {
</tbody> </tbody>
</table> </table>
</div> </div>
<ListPager
page={resultPage}
pages={resultPages}
total={resultTotal}
pageSize={resultPageSize}
pageSizeOptions={[50, 100, 200, 500]}
onPageChange={setResultPage}
onPageSizeChange={(n) => {
setResultPageSize(n);
setResultPage(1);
}}
disabled={diffsLoading || !runDetail}
/>
</div> </div>
</div> </div>
) : ( ) : (

View file

@ -1912,6 +1912,29 @@ export const bizCompareListRuns = (jobId: string, limit = 20) =>
export const bizCompareGetRun = (runId: string) => export const bizCompareGetRun = (runId: string) =>
apiGet<Record<string, unknown>>(`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`); apiGet<Record<string, unknown>>(`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`);
export const bizCompareListRunDiffs = (params: {
runId: string;
metricId?: string;
kind?: string;
kw?: string;
page?: number;
pageSize?: number;
}) => {
const p = new URLSearchParams();
if (params.metricId) p.set("metric_id", params.metricId);
if (params.kind) p.set("kind", params.kind);
if (params.kw) p.set("kw", params.kw);
p.set("page", String(Math.max(1, Number(params.page || 1))));
p.set("page_size", String(Math.max(1, Math.min(500, Number(params.pageSize || 100)))));
return apiGet<{
total: number;
page: number;
page_size: number;
metric_id: string;
items: Record<string, unknown>[];
}>(`/v1/biz-state/compare/runs/${encodeURIComponent(params.runId)}/diffs?${p.toString()}`);
};
export const bizCompareDownloadRun = async (runId: string): Promise<void> => { export const bizCompareDownloadRun = async (runId: string): Promise<void> => {
const path = `/v1/biz-state/compare/runs/${encodeURIComponent(runId)}/export`; const path = `/v1/biz-state/compare/runs/${encodeURIComponent(runId)}/export`;
const res = await fetch(path, { method: "GET", credentials: fetchCreds, headers: authHeaders() }); const res = await fetch(path, { method: "GET", credentials: fetchCreds, headers: authHeaders() });