mirror of
https://github.com/hansjone/netx.git
synced 2026-10-19 19:30:50 +08:00
Compare commits
No commits in common. "9adf9f27648b78badbc450e835765cab3d712ff1" and "227959b9a231114b3010da0e8042aeeb652e78fa" have entirely different histories.
9adf9f2764
...
227959b9a2
5 changed files with 94 additions and 306 deletions
|
|
@ -9,11 +9,10 @@ import threading
|
||||||
import time
|
import time
|
||||||
import zipfile
|
import zipfile
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Callable
|
from typing import Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from fastapi import HTTPException
|
from fastapi import HTTPException
|
||||||
from sqlalchemy import and_, or_
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ..models import (
|
from ..models import (
|
||||||
|
|
@ -430,10 +429,6 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A
|
||||||
sheets = list(summary.get("sheets") or [])
|
sheets = list(summary.get("sheets") or [])
|
||||||
if sheets:
|
if sheets:
|
||||||
return sheets
|
return sheets
|
||||||
# Running / empty: do NOT invent a fake first-metric sheet (was showing
|
|
||||||
# isis 100% with 0→0 while BGP was still loading).
|
|
||||||
if str(run.status or "") in ("running", "queued"):
|
|
||||||
return []
|
|
||||||
return [
|
return [
|
||||||
{
|
{
|
||||||
"metric_id": run.metric_id,
|
"metric_id": run.metric_id,
|
||||||
|
|
@ -443,7 +438,6 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A
|
||||||
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
|
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
|
||||||
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
|
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
|
||||||
"mode": "fields",
|
"mode": "fields",
|
||||||
"status": "done",
|
|
||||||
"summary": {
|
"summary": {
|
||||||
k: summary.get(k, 0)
|
k: summary.get(k, 0)
|
||||||
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count")
|
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count")
|
||||||
|
|
@ -453,33 +447,6 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _pending_sheet_meta(sheet: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
"""Placeholder meta so the UI lists all check items while a run is in flight."""
|
|
||||||
key_fields = list(sheet.get("key_fields") or [])
|
|
||||||
compare_fields = list(sheet.get("compare_fields") or [])
|
|
||||||
return {
|
|
||||||
"sheet_id": sheet_key(sheet),
|
|
||||||
"title": sheet_title(sheet),
|
|
||||||
"metric_id": sheet.get("metric_id") or "",
|
|
||||||
"key_fields": key_fields,
|
|
||||||
"iface_fields": list(sheet.get("iface_fields") or []),
|
|
||||||
"compare_fields": compare_fields,
|
|
||||||
"display_fields": list(sheet.get("display_fields") or []),
|
|
||||||
"field_rules": list(sheet.get("field_rules") or []),
|
|
||||||
"ignore_port_changes": sheet.get("ignore_port_changes"),
|
|
||||||
"mode": "presence" if not compare_fields else "fields",
|
|
||||||
"status": "pending",
|
|
||||||
"summary": {
|
|
||||||
"added": 0,
|
|
||||||
"removed": 0,
|
|
||||||
"changed": 0,
|
|
||||||
"unchanged": 0,
|
|
||||||
"before_count": 0,
|
|
||||||
"after_count": 0,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _str_list(raw: Any) -> list[str]:
|
def _str_list(raw: Any) -> list[str]:
|
||||||
return [str(x).strip() for x in (raw or []) if str(x).strip()]
|
return [str(x).strip() for x in (raw or []) if str(x).strip()]
|
||||||
|
|
||||||
|
|
@ -1380,29 +1347,16 @@ def _port_map_dict(db: Session, mapping_id: str) -> dict[str, str]:
|
||||||
return {str(r.before_if): str(r.after_if) for r in rows if r.before_if and r.after_if}
|
return {str(r.before_if): str(r.after_if) for r in rows if r.before_if and r.after_if}
|
||||||
|
|
||||||
|
|
||||||
def _load_metric_rows(
|
def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dict[str, Any]]:
|
||||||
db: Session,
|
"""Load metric rows with streaming fetch to avoid holding the full ORM set."""
|
||||||
*,
|
|
||||||
batch_id: str,
|
|
||||||
metric_id: str,
|
|
||||||
on_chunk: Callable[[int], None] | None = None,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
"""Load metric rows in keyset chunks (stable on million-row sheets).
|
|
||||||
|
|
||||||
Avoids ORM ``yield_per``/``unique()`` clash and OFFSET degradation on BGP-sized tables.
|
|
||||||
``on_chunk(loaded_count)`` is called after each chunk for progress UI.
|
|
||||||
"""
|
|
||||||
if metric_id == "lldp_neighbor":
|
if metric_id == "lldp_neighbor":
|
||||||
|
q = (
|
||||||
|
db.query(BizStateLldpNeighbor)
|
||||||
|
.filter(BizStateLldpNeighbor.batch_id == batch_id)
|
||||||
|
.execution_options(stream_results=True, yield_per=_LOAD_YIELD_PER)
|
||||||
|
)
|
||||||
out: list[dict[str, Any]] = []
|
out: list[dict[str, Any]] = []
|
||||||
last_id = ""
|
for n in q:
|
||||||
while True:
|
|
||||||
q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id)
|
|
||||||
if last_id:
|
|
||||||
q = q.filter(BizStateLldpNeighbor.id > last_id)
|
|
||||||
chunk = q.order_by(BizStateLldpNeighbor.id.asc()).limit(_LOAD_YIELD_PER).all()
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
for n in chunk:
|
|
||||||
out.append(
|
out.append(
|
||||||
{
|
{
|
||||||
"local_if": n.local_if,
|
"local_if": n.local_if,
|
||||||
|
|
@ -1423,41 +1377,21 @@ def _load_metric_rows(
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
db.expunge(n)
|
db.expunge(n)
|
||||||
last_id = str(chunk[-1].id)
|
|
||||||
if on_chunk:
|
|
||||||
on_chunk(len(out))
|
|
||||||
if len(chunk) < _LOAD_YIELD_PER:
|
|
||||||
break
|
|
||||||
return out
|
return out
|
||||||
# Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …)
|
# Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …)
|
||||||
from ..models import BizStateMetricRow
|
from ..models import BizStateMetricRow
|
||||||
|
|
||||||
out: list[dict[str, Any]] = []
|
q = (
|
||||||
last_seq = -1
|
db.query(BizStateMetricRow)
|
||||||
last_id = ""
|
.filter(
|
||||||
while True:
|
|
||||||
q = db.query(BizStateMetricRow).filter(
|
|
||||||
BizStateMetricRow.batch_id == batch_id,
|
BizStateMetricRow.batch_id == batch_id,
|
||||||
BizStateMetricRow.metric_id == metric_id,
|
BizStateMetricRow.metric_id == metric_id,
|
||||||
)
|
)
|
||||||
if last_id:
|
.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc())
|
||||||
q = q.filter(
|
.execution_options(stream_results=True, yield_per=_LOAD_YIELD_PER)
|
||||||
or_(
|
|
||||||
BizStateMetricRow.seq > last_seq,
|
|
||||||
and_(
|
|
||||||
BizStateMetricRow.seq == last_seq,
|
|
||||||
BizStateMetricRow.id > last_id,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
)
|
out = []
|
||||||
chunk = (
|
for r in q:
|
||||||
q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc())
|
|
||||||
.limit(_LOAD_YIELD_PER)
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
for r in chunk:
|
|
||||||
# Raw rows only — filtering belongs to the compare sheet template
|
# Raw rows only — filtering belongs to the compare sheet template
|
||||||
# (``row_filters``), not metric-specific branches here.
|
# (``row_filters``), not metric-specific branches here.
|
||||||
# ``_netx`` is collector provenance (stripped before field compare).
|
# ``_netx`` is collector provenance (stripped before field compare).
|
||||||
|
|
@ -1477,12 +1411,6 @@ def _load_metric_rows(
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
db.expunge(r)
|
db.expunge(r)
|
||||||
last_seq = int(chunk[-1].seq or 0)
|
|
||||||
last_id = str(chunk[-1].id)
|
|
||||||
if on_chunk:
|
|
||||||
on_chunk(len(out))
|
|
||||||
if len(chunk) < _LOAD_YIELD_PER:
|
|
||||||
break
|
|
||||||
if out:
|
if out:
|
||||||
return out
|
return out
|
||||||
# Known metric with zero rows is OK; unknown metric still errors
|
# Known metric with zero rows is OK; unknown metric still errors
|
||||||
|
|
@ -1681,7 +1609,6 @@ def _run_sheet(
|
||||||
port_map: dict[str, str],
|
port_map: dict[str, str],
|
||||||
iface_normalize_rules: list[dict[str, str]] | None = None,
|
iface_normalize_rules: list[dict[str, str]] | None = None,
|
||||||
store_unchanged: str = "auto",
|
store_unchanged: str = "auto",
|
||||||
on_load_progress: Callable[[str, int], None] | None = None,
|
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
key_fields = list(sheet.get("key_fields") or [])
|
key_fields = list(sheet.get("key_fields") or [])
|
||||||
iface_fields = list(sheet.get("iface_fields") or [])
|
iface_fields = list(sheet.get("iface_fields") or [])
|
||||||
|
|
@ -1702,22 +1629,8 @@ def _run_sheet(
|
||||||
ignore_ports = sheet.get("ignore_port_changes")
|
ignore_ports = sheet.get("ignore_port_changes")
|
||||||
if ignore_ports is not None:
|
if ignore_ports is not None:
|
||||||
ignore_ports = bool(ignore_ports)
|
ignore_ports = bool(ignore_ports)
|
||||||
mid = sheet["metric_id"]
|
before_raw = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"])
|
||||||
|
after_raw = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"])
|
||||||
def _before_chunk(n: int) -> None:
|
|
||||||
if on_load_progress:
|
|
||||||
on_load_progress("before", n)
|
|
||||||
|
|
||||||
def _after_chunk(n: int) -> None:
|
|
||||||
if on_load_progress:
|
|
||||||
on_load_progress("after", n)
|
|
||||||
|
|
||||||
before_raw = _load_metric_rows(
|
|
||||||
db, batch_id=before_batch_id, metric_id=mid, on_chunk=_before_chunk
|
|
||||||
)
|
|
||||||
after_raw = _load_metric_rows(
|
|
||||||
db, batch_id=after_batch_id, metric_id=mid, on_chunk=_after_chunk
|
|
||||||
)
|
|
||||||
before_rows = apply_row_filters(before_raw, row_filters)
|
before_rows = apply_row_filters(before_raw, row_filters)
|
||||||
after_rows = apply_row_filters(after_raw, row_filters)
|
after_rows = apply_row_filters(after_raw, row_filters)
|
||||||
policy = resolve_unchanged_policy(
|
policy = resolve_unchanged_policy(
|
||||||
|
|
@ -1754,7 +1667,6 @@ def _run_sheet(
|
||||||
"field_rules": field_rules,
|
"field_rules": field_rules,
|
||||||
"ignore_port_changes": ignore_ports,
|
"ignore_port_changes": ignore_ports,
|
||||||
"mode": mode,
|
"mode": mode,
|
||||||
"status": "done",
|
|
||||||
"summary": summary,
|
"summary": summary,
|
||||||
"diffs": result["diffs"],
|
"diffs": result["diffs"],
|
||||||
"mapping_stats": result["mapping_stats"],
|
"mapping_stats": result["mapping_stats"],
|
||||||
|
|
@ -1803,7 +1715,6 @@ def _create_running_run(
|
||||||
sheets_cfg: list[dict[str, Any]],
|
sheets_cfg: list[dict[str, Any]],
|
||||||
) -> BizCompareRun:
|
) -> BizCompareRun:
|
||||||
first_metric = str(sheets_cfg[0].get("metric_id") or "")
|
first_metric = str(sheets_cfg[0].get("metric_id") or "")
|
||||||
pending_sheets = [_pending_sheet_meta(s) for s in sheets_cfg]
|
|
||||||
run = BizCompareRun(
|
run = BizCompareRun(
|
||||||
id=uuid4().hex,
|
id=uuid4().hex,
|
||||||
job_id=job.id,
|
job_id=job.id,
|
||||||
|
|
@ -1830,7 +1741,7 @@ def _create_running_run(
|
||||||
"duplicate": 0,
|
"duplicate": 0,
|
||||||
"before_count": 0,
|
"before_count": 0,
|
||||||
"after_count": 0,
|
"after_count": 0,
|
||||||
"sheets": pending_sheets,
|
"sheets": [],
|
||||||
},
|
},
|
||||||
diffs_json=[],
|
diffs_json=[],
|
||||||
mapping_stats_json={},
|
mapping_stats_json={},
|
||||||
|
|
@ -1911,6 +1822,7 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
store_mode = normalize_store_unchanged(getattr(j, "store_unchanged", None))
|
store_mode = normalize_store_unchanged(getattr(j, "store_unchanged", None))
|
||||||
pmap = _port_map_dict(db, run.mapping_id)
|
pmap = _port_map_dict(db, run.mapping_id)
|
||||||
norm_rules = template_iface_normalize(tpl)
|
norm_rules = template_iface_normalize(tpl)
|
||||||
|
sheet_metas: list[dict[str, Any]] = []
|
||||||
unchanged_listed_total = 0
|
unchanged_listed_total = 0
|
||||||
unchanged_truncated_any = False
|
unchanged_truncated_any = False
|
||||||
unchanged_compact_any = False
|
unchanged_compact_any = False
|
||||||
|
|
@ -1927,51 +1839,8 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
field_counts: dict[str, int] = {}
|
field_counts: dict[str, int] = {}
|
||||||
total = len(sheets_cfg)
|
total = len(sheets_cfg)
|
||||||
|
|
||||||
# Prefer seeded pending sheets from create; rebuild if missing
|
|
||||||
sheet_metas: list[dict[str, Any]] = list((run.summary_json or {}).get("sheets") or [])
|
|
||||||
if len(sheet_metas) != total:
|
|
||||||
sheet_metas = [_pending_sheet_meta(s) for s in sheets_cfg]
|
|
||||||
|
|
||||||
def _publish_sheets() -> None:
|
|
||||||
prev = dict(run.summary_json or {})
|
|
||||||
prev["sheets"] = list(sheet_metas)
|
|
||||||
prev.update({k: agg[k] for k in agg})
|
|
||||||
run.summary_json = prev
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for idx, sheet in enumerate(sheets_cfg, start=1):
|
for idx, sheet in enumerate(sheets_cfg, start=1):
|
||||||
sid = sheet_key(sheet)
|
|
||||||
# Mark current sheet running in the sidebar list
|
|
||||||
for meta in sheet_metas:
|
|
||||||
if sheet_key(meta) == sid:
|
|
||||||
meta["status"] = "running"
|
|
||||||
break
|
|
||||||
_publish_sheets()
|
|
||||||
|
|
||||||
_load_pub = {"t": 0.0, "n": -1}
|
|
||||||
|
|
||||||
def _on_load(side: str, n: int, _idx: int = idx, _sheet: dict = sheet) -> None:
|
|
||||||
now = time.monotonic()
|
|
||||||
# Avoid committing every 5k on million-row BGP loads
|
|
||||||
if n - _load_pub["n"] < 25_000 and now - _load_pub["t"] < 2.0:
|
|
||||||
return
|
|
||||||
_load_pub["t"] = now
|
|
||||||
_load_pub["n"] = n
|
|
||||||
_set_run_progress(
|
|
||||||
db,
|
|
||||||
run,
|
|
||||||
phase="loading",
|
|
||||||
sheet_index=_idx,
|
|
||||||
sheet_total=total,
|
|
||||||
sheet=_sheet,
|
|
||||||
started_mono=started_mono,
|
|
||||||
extra={
|
|
||||||
"load_side": side,
|
|
||||||
"rows_loaded": n,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
_set_run_progress(
|
_set_run_progress(
|
||||||
db,
|
db,
|
||||||
run,
|
run,
|
||||||
|
|
@ -1989,7 +1858,6 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
port_map=pmap,
|
port_map=pmap,
|
||||||
iface_normalize_rules=norm_rules,
|
iface_normalize_rules=norm_rules,
|
||||||
store_unchanged=store_mode,
|
store_unchanged=store_mode,
|
||||||
on_load_progress=_on_load,
|
|
||||||
)
|
)
|
||||||
s = one["summary"]
|
s = one["summary"]
|
||||||
listed = int(s.get("unchanged_listed") or 0)
|
listed = int(s.get("unchanged_listed") or 0)
|
||||||
|
|
@ -2001,7 +1869,7 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
_set_run_progress(
|
_set_run_progress(
|
||||||
db,
|
db,
|
||||||
run,
|
run,
|
||||||
phase="comparing",
|
phase="persisting",
|
||||||
sheet_index=idx,
|
sheet_index=idx,
|
||||||
sheet_total=total,
|
sheet_total=total,
|
||||||
sheet=sheet,
|
sheet=sheet,
|
||||||
|
|
@ -2062,7 +1930,8 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
for k in agg:
|
for k in agg:
|
||||||
agg[k] += int(s.get(k) or 0)
|
agg[k] += int(s.get(k) or 0)
|
||||||
mapping_by_metric[mid] = one["mapping_stats"]
|
mapping_by_metric[mid] = one["mapping_stats"]
|
||||||
done_meta = {
|
sheet_metas.append(
|
||||||
|
{
|
||||||
"sheet_id": one.get("sheet_id") or one["metric_id"],
|
"sheet_id": one.get("sheet_id") or one["metric_id"],
|
||||||
"title": one.get("title") or one.get("sheet_id") or one["metric_id"],
|
"title": one.get("title") or one.get("sheet_id") or one["metric_id"],
|
||||||
"metric_id": one["metric_id"],
|
"metric_id": one["metric_id"],
|
||||||
|
|
@ -2073,18 +1942,9 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
"field_rules": one.get("field_rules") or [],
|
"field_rules": one.get("field_rules") or [],
|
||||||
"ignore_port_changes": one.get("ignore_port_changes"),
|
"ignore_port_changes": one.get("ignore_port_changes"),
|
||||||
"mode": one["mode"],
|
"mode": one["mode"],
|
||||||
"status": "done",
|
|
||||||
"summary": one["summary"],
|
"summary": one["summary"],
|
||||||
}
|
}
|
||||||
replaced = False
|
)
|
||||||
for i, meta in enumerate(sheet_metas):
|
|
||||||
if sheet_key(meta) == mid:
|
|
||||||
sheet_metas[i] = done_meta
|
|
||||||
replaced = True
|
|
||||||
break
|
|
||||||
if not replaced:
|
|
||||||
sheet_metas.append(done_meta)
|
|
||||||
_publish_sheets()
|
|
||||||
# Drop heavy diffs before next sheet
|
# Drop heavy diffs before next sheet
|
||||||
one.clear()
|
one.clear()
|
||||||
diffs.clear()
|
diffs.clear()
|
||||||
|
|
@ -2307,15 +2167,12 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
||||||
st = sa + sr + sc + su
|
st = sa + sr + sc + su
|
||||||
sf = sr + sc
|
sf = sr + sc
|
||||||
sj = sf + su
|
sj = sf + su
|
||||||
status = str(sh.get("status") or "done")
|
|
||||||
pending = status in ("pending", "running", "queued")
|
|
||||||
sheet_cards.append(
|
sheet_cards.append(
|
||||||
{
|
{
|
||||||
"sheet_id": sheet_key(sh),
|
"sheet_id": sheet_key(sh),
|
||||||
"title": sheet_title(sh),
|
"title": sheet_title(sh),
|
||||||
"metric_id": sh.get("metric_id") or "",
|
"metric_id": sh.get("metric_id") or "",
|
||||||
"mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
|
"mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
|
||||||
"status": status,
|
|
||||||
"added": sa,
|
"added": sa,
|
||||||
"removed": sr,
|
"removed": sr,
|
||||||
"changed": sc,
|
"changed": sc,
|
||||||
|
|
@ -2325,10 +2182,7 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
||||||
"fail_count": sf,
|
"fail_count": sf,
|
||||||
"success_count": su,
|
"success_count": su,
|
||||||
"diff_count": sf,
|
"diff_count": sf,
|
||||||
# Pending sheets must not look like "100% pass"
|
"pass_rate": round((su / sj) * 100, 1) if sj else (100.0 if st == 0 else 0.0),
|
||||||
"pass_rate": None
|
|
||||||
if pending
|
|
||||||
else (round((su / sj) * 100, 1) if sj else (100.0 if st == 0 else 0.0)),
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -2347,9 +2201,6 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
||||||
key=lambda x: (-int(x["count"]), str(x["field"])),
|
key=lambda x: (-int(x["count"]), str(x["field"])),
|
||||||
)[:8]
|
)[:8]
|
||||||
|
|
||||||
any_pending = any(
|
|
||||||
str(sh.get("status") or "") in ("pending", "running", "queued") for sh in sheets
|
|
||||||
)
|
|
||||||
return {
|
return {
|
||||||
"added": added,
|
"added": added,
|
||||||
"removed": removed,
|
"removed": removed,
|
||||||
|
|
@ -2363,9 +2214,9 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
||||||
"fail_count": fail_count,
|
"fail_count": fail_count,
|
||||||
"success_count": success_count,
|
"success_count": success_count,
|
||||||
"diff_count": diff_count,
|
"diff_count": diff_count,
|
||||||
"pass_rate": None if any_pending else pass_rate,
|
"pass_rate": pass_rate,
|
||||||
"diff_rate": diff_rate,
|
"diff_rate": diff_rate,
|
||||||
"ok": False if any_pending else fail_count == 0,
|
"ok": fail_count == 0,
|
||||||
"sheet_cards": sheet_cards,
|
"sheet_cards": sheet_cards,
|
||||||
"top_changed_fields": top_fields,
|
"top_changed_fields": top_fields,
|
||||||
"duration_ms": int(summary.get("duration_ms") or 0),
|
"duration_ms": int(summary.get("duration_ms") or 0),
|
||||||
|
|
@ -2397,7 +2248,6 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
"display_fields": list(sh.get("display_fields") or []),
|
"display_fields": list(sh.get("display_fields") or []),
|
||||||
"field_rules": list(sh.get("field_rules") or []),
|
"field_rules": list(sh.get("field_rules") or []),
|
||||||
"mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
|
"mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
|
||||||
"status": str(sh.get("status") or "done"),
|
|
||||||
"summary": dict(sh.get("summary") or {}),
|
"summary": dict(sh.get("summary") or {}),
|
||||||
}
|
}
|
||||||
for sh in raw_sheets
|
for sh in raw_sheets
|
||||||
|
|
|
||||||
|
|
@ -500,8 +500,6 @@ const en = {
|
||||||
runStatusFailed: "Failed",
|
runStatusFailed: "Failed",
|
||||||
runStatusDoneSec: "Done {{s}}s",
|
runStatusDoneSec: "Done {{s}}s",
|
||||||
runProgress: "{{phase}} · sheet {{i}}/{{n}} · {{sheet}} · {{s}}s elapsed",
|
runProgress: "{{phase}} · sheet {{i}}/{{n}} · {{sheet}} · {{s}}s elapsed",
|
||||||
runRowsLoaded: "Loaded {{side}} {{n}} rows",
|
|
||||||
sheetPending: "Pending",
|
|
||||||
ranWithDuration: "Compare finished ({{s}}s)",
|
ranWithDuration: "Compare finished ({{s}}s)",
|
||||||
unchangedNotStored: "Success rows were counted but not stored. Set “Store success rows” to sample and re-run for spot-check.",
|
unchangedNotStored: "Success rows were counted but not stored. Set “Store success rows” to sample and re-run for spot-check.",
|
||||||
unchangedSampleHint: "{{total}} success rows total; browsing a sample of {{listed}} (hydrated from source tables for cutover spot-check). Pass rate uses all {{total}}.",
|
unchangedSampleHint: "{{total}} success rows total; browsing a sample of {{listed}} (hydrated from source tables for cutover spot-check). Pass rate uses all {{total}}.",
|
||||||
|
|
|
||||||
|
|
@ -499,8 +499,6 @@ const zh = {
|
||||||
runStatusFailed: "失败",
|
runStatusFailed: "失败",
|
||||||
runStatusDoneSec: "完成 {{s}}s",
|
runStatusDoneSec: "完成 {{s}}s",
|
||||||
runProgress: "{{phase}} · 表 {{i}}/{{n}} · {{sheet}} · 已用 {{s}}s",
|
runProgress: "{{phase}} · 表 {{i}}/{{n}} · {{sheet}} · 已用 {{s}}s",
|
||||||
runRowsLoaded: "已加载 {{side}} {{n}} 行",
|
|
||||||
sheetPending: "等待中",
|
|
||||||
ranWithDuration: "比对完成(耗时 {{s}} 秒)",
|
ranWithDuration: "比对完成(耗时 {{s}} 秒)",
|
||||||
unchangedNotStored: "成功行仅统计数量未落库。可在任务配置将「成功行保存」改为抽样后重新比对(抽查用)。",
|
unchangedNotStored: "成功行仅统计数量未落库。可在任务配置将「成功行保存」改为抽样后重新比对(抽查用)。",
|
||||||
unchangedSampleHint: "成功共 {{total}} 条,明细抽样 {{listed}} 条(从原表补全显示,割接抽查用)。通过率按全部 {{total}} 计。",
|
unchangedSampleHint: "成功共 {{total}} 条,明细抽样 {{listed}} 条(从原表补全显示,割接抽查用)。通过率按全部 {{total}} 计。",
|
||||||
|
|
|
||||||
|
|
@ -12152,15 +12152,6 @@ body:has(.bs-cmp-board:fullscreen) {
|
||||||
box-shadow: 0 0 0 2px rgba(245, 158, 11, 0.22);
|
box-shadow: 0 0 0 2px rgba(245, 158, 11, 0.22);
|
||||||
}
|
}
|
||||||
|
|
||||||
.bs-cmp-nav__item.is-pending .bs-cmp-nav__dot {
|
|
||||||
background: #64748b;
|
|
||||||
box-shadow: 0 0 0 2px rgba(100, 116, 139, 0.25);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bs-cmp-nav__item.is-pending .bs-cmp-nav__name {
|
|
||||||
opacity: 0.75;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bs-cmp-nav__name {
|
.bs-cmp-nav__name {
|
||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
|
|
|
||||||
|
|
@ -1198,8 +1198,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
Number(c.success_count ?? c.unchanged ?? 0);
|
Number(c.success_count ?? c.unchanged ?? 0);
|
||||||
|
|
||||||
const sheetPassRateOf = (c: {
|
const sheetPassRateOf = (c: {
|
||||||
pass_rate?: number | null;
|
pass_rate?: number;
|
||||||
status?: string;
|
|
||||||
fail_count?: number;
|
fail_count?: number;
|
||||||
diff_count?: number;
|
diff_count?: number;
|
||||||
removed?: number;
|
removed?: number;
|
||||||
|
|
@ -1207,10 +1206,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
success_count?: number;
|
success_count?: number;
|
||||||
unchanged?: number;
|
unchanged?: number;
|
||||||
added?: number;
|
added?: number;
|
||||||
}): number | null => {
|
}) => {
|
||||||
const st = String(c.status || "");
|
|
||||||
if (st === "pending" || st === "running" || st === "queued") return null;
|
|
||||||
if (c.pass_rate === null) return null;
|
|
||||||
// Always recompute from fail/success so added never skews pass rate
|
// Always recompute from fail/success so added never skews pass rate
|
||||||
const fail = sheetFailOf(c);
|
const fail = sheetFailOf(c);
|
||||||
const ok = sheetSuccessOf(c);
|
const ok = sheetSuccessOf(c);
|
||||||
|
|
@ -1226,7 +1222,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
title?: string;
|
title?: string;
|
||||||
metric_id: string;
|
metric_id: string;
|
||||||
mode?: string;
|
mode?: string;
|
||||||
status?: string;
|
|
||||||
added?: number;
|
added?: number;
|
||||||
removed?: number;
|
removed?: number;
|
||||||
changed?: number;
|
changed?: number;
|
||||||
|
|
@ -1236,13 +1231,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
fail_count?: number;
|
fail_count?: number;
|
||||||
success_count?: number;
|
success_count?: number;
|
||||||
diff_count?: number;
|
diff_count?: number;
|
||||||
pass_rate?: number | null;
|
pass_rate?: number;
|
||||||
}>;
|
}>;
|
||||||
// Failures first; keep pending/running at the end so completed fails stay visible
|
// Failures first so ops can scan quickly when many sheets
|
||||||
return [...raw].sort((a, b) => {
|
return [...raw].sort((a, b) => {
|
||||||
const pa = ["pending", "running", "queued"].includes(String(a.status || "")) ? 1 : 0;
|
|
||||||
const pb = ["pending", "running", "queued"].includes(String(b.status || "")) ? 1 : 0;
|
|
||||||
if (pa !== pb) return pa - pb;
|
|
||||||
const da = sheetFailOf(a);
|
const da = sheetFailOf(a);
|
||||||
const db = sheetFailOf(b);
|
const db = sheetFailOf(b);
|
||||||
if (da !== db) return db - da;
|
if (da !== db) return db - da;
|
||||||
|
|
@ -1253,9 +1245,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
const activeFail = sheetFailOf(activeSheetCard || {});
|
const activeFail = sheetFailOf(activeSheetCard || {});
|
||||||
const activeSuccess = sheetSuccessOf(activeSheetCard || {});
|
const activeSuccess = sheetSuccessOf(activeSheetCard || {});
|
||||||
const activePassRate = sheetPassRateOf(activeSheetCard || {});
|
const activePassRate = sheetPassRateOf(activeSheetCard || {});
|
||||||
const activeSheetPending = ["pending", "running", "queued"].includes(
|
|
||||||
String(activeSheetCard?.status || ""),
|
|
||||||
);
|
|
||||||
const activeAdded = Number(activeSheetCard?.added || 0);
|
const activeAdded = Number(activeSheetCard?.added || 0);
|
||||||
const showFailCol =
|
const showFailCol =
|
||||||
kindFilter === "diff" || kindFilter === "all" || kindFilter === "added";
|
kindFilter === "diff" || kindFilter === "all" || kindFilter === "added";
|
||||||
|
|
@ -1879,8 +1868,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
before_count?: number;
|
before_count?: number;
|
||||||
after_count?: number;
|
after_count?: number;
|
||||||
diff_rows?: number;
|
diff_rows?: number;
|
||||||
load_side?: string;
|
|
||||||
rows_loaded?: number;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Poll active compare runs so the modal can be closed and reopened safely.
|
// Poll active compare runs so the modal can be closed and reopened safely.
|
||||||
|
|
@ -3033,14 +3020,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
s: String(Math.round(Number(runProgress.elapsed_ms || 0) / 1000)),
|
s: String(Math.round(Number(runProgress.elapsed_ms || 0) / 1000)),
|
||||||
})}
|
})}
|
||||||
</span>
|
</span>
|
||||||
{Number(runProgress.rows_loaded || 0) > 0 ? (
|
|
||||||
<span className="muted">
|
|
||||||
{t("bizCompare.runRowsLoaded", {
|
|
||||||
side: String(runProgress.load_side || "—"),
|
|
||||||
n: String(runProgress.rows_loaded || 0),
|
|
||||||
})}
|
|
||||||
</span>
|
|
||||||
) : null}
|
|
||||||
{runDetail.message ? (
|
{runDetail.message ? (
|
||||||
<span className="muted bs-cmp-progress__msg">{String(runDetail.message)}</span>
|
<span className="muted bs-cmp-progress__msg">{String(runDetail.message)}</span>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
@ -3119,14 +3098,11 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
const unchanged = Number(s.summary?.unchanged || 0);
|
const unchanged = Number(s.summary?.unchanged || 0);
|
||||||
const fail = removed + changed;
|
const fail = removed + changed;
|
||||||
const judged = fail + unchanged;
|
const judged = fail + unchanged;
|
||||||
const st = String((s as any).status || "done");
|
|
||||||
const pending = ["pending", "running", "queued"].includes(st);
|
|
||||||
return {
|
return {
|
||||||
sheet_id: sheetIdentity(s),
|
sheet_id: sheetIdentity(s),
|
||||||
title: sheetLabel(s),
|
title: sheetLabel(s),
|
||||||
metric_id: s.metric_id,
|
metric_id: s.metric_id,
|
||||||
mode: s.mode,
|
mode: s.mode,
|
||||||
status: st,
|
|
||||||
added: s.summary?.added,
|
added: s.summary?.added,
|
||||||
removed,
|
removed,
|
||||||
changed,
|
changed,
|
||||||
|
|
@ -3136,9 +3112,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
fail_count: fail,
|
fail_count: fail,
|
||||||
success_count: unchanged,
|
success_count: unchanged,
|
||||||
diff_count: fail,
|
diff_count: fail,
|
||||||
pass_rate: pending
|
pass_rate: judged
|
||||||
? null
|
|
||||||
: judged
|
|
||||||
? Math.round((unchanged / judged) * 1000) / 10
|
? Math.round((unchanged / judged) * 1000) / 10
|
||||||
: 100,
|
: 100,
|
||||||
};
|
};
|
||||||
|
|
@ -3150,9 +3124,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
const id = sheetIdentity(c);
|
const id = sheetIdentity(c);
|
||||||
const label = sheetLabel(c);
|
const label = sheetLabel(c);
|
||||||
const active = resultSheetId === id;
|
const active = resultSheetId === id;
|
||||||
const pending = ["pending", "running", "queued"].includes(
|
|
||||||
String((c as any).status || ""),
|
|
||||||
);
|
|
||||||
return (
|
return (
|
||||||
<button
|
<button
|
||||||
key={id}
|
key={id}
|
||||||
|
|
@ -3160,14 +3131,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
role="tab"
|
role="tab"
|
||||||
aria-selected={active}
|
aria-selected={active}
|
||||||
className={`bs-cmp-nav__item${active ? " is-active" : ""}${
|
className={`bs-cmp-nav__item${active ? " is-active" : ""}${
|
||||||
pending ? " is-pending" : fail > 0 ? " has-diff" : " is-clean"
|
fail > 0 ? " has-diff" : " is-clean"
|
||||||
}`}
|
}`}
|
||||||
onClick={() => setResultSheetId(id)}
|
onClick={() => setResultSheetId(id)}
|
||||||
title={
|
title={`${label} · ${t("bizCompare.kindFail")} ${fail} · ${t("bizCompare.kindSuccess")} ${ok} · ${t("bizCompare.passRateShort")} ${rate}%`}
|
||||||
pending
|
|
||||||
? `${label} · ${t("bizCompare.sheetPending")}`
|
|
||||||
: `${label} · ${t("bizCompare.kindFail")} ${fail} · ${t("bizCompare.kindSuccess")} ${ok} · ${t("bizCompare.passRateShort")} ${rate}%`
|
|
||||||
}
|
|
||||||
>
|
>
|
||||||
<span className="bs-cmp-nav__dot" aria-hidden />
|
<span className="bs-cmp-nav__dot" aria-hidden />
|
||||||
<span className="bs-cmp-nav__name">{label}</span>
|
<span className="bs-cmp-nav__name">{label}</span>
|
||||||
|
|
@ -3177,13 +3144,13 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
fail > 0 ? " is-hot" : ""
|
fail > 0 ? " is-hot" : ""
|
||||||
}`}
|
}`}
|
||||||
>
|
>
|
||||||
{pending ? "—" : fail}
|
{fail}
|
||||||
</span>
|
</span>
|
||||||
<span className="bs-cmp-nav__num bs-cmp-nav__num--ok is-hot">
|
<span className="bs-cmp-nav__num bs-cmp-nav__num--ok is-hot">
|
||||||
{pending ? "—" : ok}
|
{ok}
|
||||||
</span>
|
</span>
|
||||||
<span className="bs-cmp-nav__num bs-cmp-nav__num--rate">
|
<span className="bs-cmp-nav__num bs-cmp-nav__num--rate">
|
||||||
{pending || rate === null ? "…" : `${rate}%`}
|
{rate}%
|
||||||
</span>
|
</span>
|
||||||
</span>
|
</span>
|
||||||
</button>
|
</button>
|
||||||
|
|
@ -3194,13 +3161,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
|
|
||||||
<div className="bs-cmp-main">
|
<div className="bs-cmp-main">
|
||||||
<div
|
<div
|
||||||
className={`bs-cmp-strip${
|
className={`bs-cmp-strip${activeFail > 0 ? " is-warn" : " is-ok"}`}
|
||||||
activeSheetPending
|
|
||||||
? " is-pending"
|
|
||||||
: activeFail > 0
|
|
||||||
? " is-warn"
|
|
||||||
: " is-ok"
|
|
||||||
}`}
|
|
||||||
>
|
>
|
||||||
<div className="bs-cmp-strip__sheet">
|
<div className="bs-cmp-strip__sheet">
|
||||||
<span className="bs-cmp-strip__sheet-tag">
|
<span className="bs-cmp-strip__sheet-tag">
|
||||||
|
|
@ -3213,28 +3174,18 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
{sheetLabel(activeSheetCard)}
|
{sheetLabel(activeSheetCard)}
|
||||||
</strong>
|
</strong>
|
||||||
<span className="muted bs-cmp-strip__sheet-mode">
|
<span className="muted bs-cmp-strip__sheet-mode">
|
||||||
{activeSheetPending
|
{activeFail > 0 ? t("bizCompare.kindFail") : t("bizCompare.kindPass")}
|
||||||
? t("bizCompare.sheetPending")
|
|
||||||
: activeFail > 0
|
|
||||||
? t("bizCompare.kindFail")
|
|
||||||
: t("bizCompare.kindPass")}
|
|
||||||
{activeSheetCard?.mode === "presence"
|
{activeSheetCard?.mode === "presence"
|
||||||
? ` · ${t("bizCompare.presenceShort")}`
|
? ` · ${t("bizCompare.presenceShort")}`
|
||||||
: ""}
|
: ""}
|
||||||
{activeSheetPending
|
{` · ${activeSheetCard?.before_count ?? 0}→${activeSheetCard?.after_count ?? 0}`}
|
||||||
? ""
|
|
||||||
: ` · ${activeSheetCard?.before_count ?? 0}→${activeSheetCard?.after_count ?? 0}`}
|
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div className="bs-cmp-strip__pass" title={t("bizCompare.passRate")}>
|
<div className="bs-cmp-strip__pass" title={t("bizCompare.passRate")}>
|
||||||
<span className="bs-cmp-strip__pass-label">
|
<span className="bs-cmp-strip__pass-label">
|
||||||
{t("bizCompare.passRate")}
|
{t("bizCompare.passRate")}
|
||||||
</span>
|
</span>
|
||||||
<span className="bs-cmp-strip__pass-value">
|
<span className="bs-cmp-strip__pass-value">{activePassRate}%</span>
|
||||||
{activeSheetPending || activePassRate === null
|
|
||||||
? "…"
|
|
||||||
: `${activePassRate}%`}
|
|
||||||
</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div className="bs-cmp-strip__kinds" role="group">
|
<div className="bs-cmp-strip__kinds" role="group">
|
||||||
{(
|
{(
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue