mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 00:50:46 +08:00
Add compare batch cancel and restart recovery so stuck runs unblock.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
9adf9f2764
commit
958942c5b6
8 changed files with 532 additions and 50 deletions
|
|
@ -159,6 +159,17 @@ def run_api_startup() -> None:
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
_log.exception("startup: biz_state collect recovery failed")
|
_log.exception("startup: biz_state collect recovery failed")
|
||||||
|
try:
|
||||||
|
from .biz_state.compare_service import recover_interrupted_compares_on_startup
|
||||||
|
|
||||||
|
cmp_rec = recover_interrupted_compares_on_startup(db)
|
||||||
|
if cmp_rec.get("runs"):
|
||||||
|
_log.info(
|
||||||
|
"startup: cancelled %s interrupted biz compare run(s)",
|
||||||
|
cmp_rec.get("runs"),
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
_log.exception("startup: biz compare recovery failed")
|
||||||
try:
|
try:
|
||||||
from .port_traffic_migrate import backfill_port_traffic_series
|
from .port_traffic_migrate import backfill_port_traffic_series
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1868,13 +1868,17 @@ def _set_run_progress(
|
||||||
progress.update(extra)
|
progress.update(extra)
|
||||||
prev["progress"] = progress
|
prev["progress"] = progress
|
||||||
run.summary_json = prev
|
run.summary_json = prev
|
||||||
run.status = "running"
|
# Re-read status from DB — cancel may have been committed by another session
|
||||||
title = progress["sheet_title"] or progress["sheet_id"] or ""
|
# (UI cancel / startup recovery) while this worker still holds a stale "running".
|
||||||
run.message = (
|
db.expire(run, ["status", "message"])
|
||||||
f"{phase} {sheet_index}/{sheet_total}"
|
if str(run.status or "") != "cancelled":
|
||||||
+ (f" · {title}" if title else "")
|
run.status = "running"
|
||||||
+ f" · {elapsed_ms // 1000}s"
|
title = progress["sheet_title"] or progress["sheet_id"] or ""
|
||||||
)[:1024]
|
run.message = (
|
||||||
|
f"{phase} {sheet_index}/{sheet_total}"
|
||||||
|
+ (f" · {title}" if title else "")
|
||||||
|
+ f" · {elapsed_ms // 1000}s"
|
||||||
|
)[:1024]
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1937,10 +1941,18 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
prev["sheets"] = list(sheet_metas)
|
prev["sheets"] = list(sheet_metas)
|
||||||
prev.update({k: agg[k] for k in agg})
|
prev.update({k: agg[k] for k in agg})
|
||||||
run.summary_json = prev
|
run.summary_json = prev
|
||||||
|
db.expire(run, ["status"])
|
||||||
|
# Never resurrect cancelled while publishing incremental sheet metas
|
||||||
|
if str(run.status or "") == "cancelled":
|
||||||
|
db.commit()
|
||||||
|
return
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
for idx, sheet in enumerate(sheets_cfg, start=1):
|
for idx, sheet in enumerate(sheets_cfg, start=1):
|
||||||
|
if _run_is_cancelled(db, run_id):
|
||||||
|
run = db.get(BizCompareRun, run_id) or run
|
||||||
|
return get_run(db, run.id)
|
||||||
sid = sheet_key(sheet)
|
sid = sheet_key(sheet)
|
||||||
# Mark current sheet running in the sidebar list
|
# Mark current sheet running in the sidebar list
|
||||||
for meta in sheet_metas:
|
for meta in sheet_metas:
|
||||||
|
|
@ -2091,6 +2103,13 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
fail_diffs.clear()
|
fail_diffs.clear()
|
||||||
ok_diffs.clear()
|
ok_diffs.clear()
|
||||||
db.commit()
|
db.commit()
|
||||||
|
if _run_is_cancelled(db, run_id):
|
||||||
|
run = db.get(BizCompareRun, run_id) or run
|
||||||
|
return get_run(db, run.id)
|
||||||
|
|
||||||
|
if _run_is_cancelled(db, run_id):
|
||||||
|
run = db.get(BizCompareRun, run_id) or run
|
||||||
|
return get_run(db, run.id)
|
||||||
|
|
||||||
duration_ms = int((time.monotonic() - started_mono) * 1000)
|
duration_ms = int((time.monotonic() - started_mono) * 1000)
|
||||||
top_fields = sorted(
|
top_fields = sorted(
|
||||||
|
|
@ -2117,6 +2136,9 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
},
|
},
|
||||||
"sheets": sheet_metas,
|
"sheets": sheet_metas,
|
||||||
}
|
}
|
||||||
|
run = db.get(BizCompareRun, run_id) or run
|
||||||
|
if str(run.status or "") == "cancelled":
|
||||||
|
return get_run(db, run.id)
|
||||||
run.status = "success"
|
run.status = "success"
|
||||||
run.summary_json = summary_payload
|
run.summary_json = summary_payload
|
||||||
run.mapping_stats_json = mapping_by_metric
|
run.mapping_stats_json = mapping_by_metric
|
||||||
|
|
@ -2129,34 +2151,123 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
db.commit()
|
db.commit()
|
||||||
return get_run(db, run.id)
|
return get_run(db, run.id)
|
||||||
except HTTPException as exc:
|
except HTTPException as exc:
|
||||||
run.status = "failed"
|
run = db.get(BizCompareRun, run_id) or run
|
||||||
run.message = str(getattr(exc, "detail", "") or exc)[:1024]
|
if str(run.status or "") != "cancelled":
|
||||||
prev = dict(run.summary_json or {})
|
run.status = "failed"
|
||||||
prog = dict(prev.get("progress") or {})
|
run.message = str(getattr(exc, "detail", "") or exc)[:1024]
|
||||||
prog["phase"] = "failed"
|
prev = dict(run.summary_json or {})
|
||||||
prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000)
|
prog = dict(prev.get("progress") or {})
|
||||||
prev["progress"] = prog
|
prog["phase"] = "failed"
|
||||||
prev["duration_ms"] = prog["elapsed_ms"]
|
prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000)
|
||||||
run.summary_json = prev
|
prev["progress"] = prog
|
||||||
db.commit()
|
prev["duration_ms"] = prog["elapsed_ms"]
|
||||||
|
run.summary_json = prev
|
||||||
|
db.commit()
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
_log.exception("compare run failed run=%s job=%s", run_id, j.id)
|
_log.exception("compare run failed run=%s job=%s", run_id, j.id)
|
||||||
run.status = "failed"
|
run = db.get(BizCompareRun, run_id) or run
|
||||||
run.message = str(exc)[:1024]
|
if str(run.status or "") != "cancelled":
|
||||||
prev = dict(run.summary_json or {})
|
run.status = "failed"
|
||||||
prog = dict(prev.get("progress") or {})
|
run.message = str(exc)[:1024]
|
||||||
prog["phase"] = "failed"
|
prev = dict(run.summary_json or {})
|
||||||
prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000)
|
prog = dict(prev.get("progress") or {})
|
||||||
prev["progress"] = prog
|
prog["phase"] = "failed"
|
||||||
prev["duration_ms"] = prog["elapsed_ms"]
|
prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000)
|
||||||
run.summary_json = prev
|
prev["progress"] = prog
|
||||||
db.commit()
|
prev["duration_ms"] = prog["elapsed_ms"]
|
||||||
|
run.summary_json = prev
|
||||||
|
db.commit()
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
_INTERRUPT_MARK = "interrupted_by_restart"
|
||||||
|
|
||||||
|
|
||||||
|
def _job_has_active_run(db: Session, job_id: str) -> BizCompareRun | None:
|
||||||
|
return (
|
||||||
|
db.query(BizCompareRun)
|
||||||
|
.filter(
|
||||||
|
BizCompareRun.job_id == str(job_id or ""),
|
||||||
|
BizCompareRun.status.in_(("running", "queued")),
|
||||||
|
)
|
||||||
|
.order_by(BizCompareRun.created_at.desc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_is_cancelled(db: Session, run_id: str) -> bool:
|
||||||
|
"""Re-read status so user/startup cancel is visible to the worker thread."""
|
||||||
|
db.expire_all()
|
||||||
|
r = db.get(BizCompareRun, run_id)
|
||||||
|
return bool(r and str(r.status or "") == "cancelled")
|
||||||
|
|
||||||
|
|
||||||
|
def recover_interrupted_compares_on_startup(db: Session) -> dict[str, Any]:
|
||||||
|
"""Mark orphaned running/queued compare runs as cancelled after process restart.
|
||||||
|
|
||||||
|
In-memory job locks die with the process; without this, the UI stays on
|
||||||
|
「比对中」and blocks a new run.
|
||||||
|
"""
|
||||||
|
now = _utcnow()
|
||||||
|
rows = (
|
||||||
|
db.query(BizCompareRun)
|
||||||
|
.filter(BizCompareRun.status.in_(("running", "queued")))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
n = 0
|
||||||
|
for r in rows:
|
||||||
|
r.status = "cancelled"
|
||||||
|
msg = str(r.message or "").strip()
|
||||||
|
if _INTERRUPT_MARK not in msg:
|
||||||
|
r.message = f"{msg} | {_INTERRUPT_MARK}".strip(" |")[:1024]
|
||||||
|
prev = dict(r.summary_json or {})
|
||||||
|
prog = dict(prev.get("progress") or {})
|
||||||
|
prog["phase"] = "cancelled"
|
||||||
|
prog["elapsed_ms"] = int(prog.get("elapsed_ms") or 0)
|
||||||
|
prev["progress"] = prog
|
||||||
|
# Mark in-flight sheet placeholders so UI does not show fake pass
|
||||||
|
sheets = list(prev.get("sheets") or [])
|
||||||
|
for sh in sheets:
|
||||||
|
st = str(sh.get("status") or "")
|
||||||
|
if st in ("pending", "running", "queued"):
|
||||||
|
sh["status"] = "cancelled"
|
||||||
|
prev["sheets"] = sheets
|
||||||
|
r.summary_json = prev
|
||||||
|
n += 1
|
||||||
|
if n:
|
||||||
|
db.commit()
|
||||||
|
_log.info("startup: cancelled %s interrupted compare run(s)", n)
|
||||||
|
return {"runs": n, "at": now.isoformat() + "Z"}
|
||||||
|
|
||||||
|
|
||||||
|
def cancel_compare_run(db: Session, run_id: str) -> dict[str, Any]:
|
||||||
|
"""Cancel a running/queued compare so a new run can start."""
|
||||||
|
r = db.get(BizCompareRun, run_id)
|
||||||
|
if not r:
|
||||||
|
raise HTTPException(status_code=404, detail="run_not_found")
|
||||||
|
st = str(r.status or "")
|
||||||
|
if st not in ("running", "queued"):
|
||||||
|
return get_run(db, run_id)
|
||||||
|
r.status = "cancelled"
|
||||||
|
msg = str(r.message or "").strip()
|
||||||
|
r.message = f"{msg} | cancelled_by_user".strip(" |")[:1024]
|
||||||
|
prev = dict(r.summary_json or {})
|
||||||
|
prog = dict(prev.get("progress") or {})
|
||||||
|
prog["phase"] = "cancelled"
|
||||||
|
prev["progress"] = prog
|
||||||
|
for sh in list(prev.get("sheets") or []):
|
||||||
|
if str(sh.get("status") or "") in ("pending", "running", "queued"):
|
||||||
|
sh["status"] = "cancelled"
|
||||||
|
r.summary_json = prev
|
||||||
|
db.commit()
|
||||||
|
return get_run(db, run_id)
|
||||||
|
|
||||||
|
|
||||||
def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]:
|
def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]:
|
||||||
"""Synchronous compare (auto-compare / tests). Blocks the caller until done."""
|
"""Synchronous compare (auto-compare / tests). Blocks the caller until done."""
|
||||||
|
if _job_has_active_run(db, job_id):
|
||||||
|
raise HTTPException(status_code=409, detail="compare_already_running")
|
||||||
lock = _job_compare_lock(job_id)
|
lock = _job_compare_lock(job_id)
|
||||||
if not lock.acquire(blocking=False):
|
if not lock.acquire(blocking=False):
|
||||||
raise HTTPException(status_code=409, detail="compare_already_running")
|
raise HTTPException(status_code=409, detail="compare_already_running")
|
||||||
|
|
@ -2183,6 +2294,8 @@ def enqueue_compare(
|
||||||
Returns immediately so the HTTP worker / UI stay responsive. Poll
|
Returns immediately so the HTTP worker / UI stay responsive. Poll
|
||||||
``GET /compare/runs/{id}`` for progress (``summary.progress``).
|
``GET /compare/runs/{id}`` for progress (``summary.progress``).
|
||||||
"""
|
"""
|
||||||
|
if _job_has_active_run(db, job_id):
|
||||||
|
raise HTTPException(status_code=409, detail="compare_already_running")
|
||||||
lock = _job_compare_lock(job_id)
|
lock = _job_compare_lock(job_id)
|
||||||
if not lock.acquire(blocking=False):
|
if not lock.acquire(blocking=False):
|
||||||
raise HTTPException(status_code=409, detail="compare_already_running")
|
raise HTTPException(status_code=409, detail="compare_already_running")
|
||||||
|
|
|
||||||
|
|
@ -746,6 +746,12 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||||
return cmp_svc.get_run(db, run_id)
|
return cmp_svc.get_run(db, run_id)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/compare/runs/{run_id}/cancel")
|
||||||
|
def api_cancel_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||||
|
"""Cancel a stuck/running compare so a new run can start."""
|
||||||
|
return cmp_svc.cancel_compare_run(db, run_id)
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/compare/runs/{run_id}")
|
@router.delete("/compare/runs/{run_id}")
|
||||||
def api_delete_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
def api_delete_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||||
return cmp_svc.delete_run(db, run_id)
|
return cmp_svc.delete_run(db, run_id)
|
||||||
|
|
|
||||||
98
tests/test_biz_state_compare_recovery.py
Normal file
98
tests/test_biz_state_compare_recovery.py
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
"""Startup recovery for interrupted biz-state compare runs."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from netx_api.biz_state.compare_service import (
|
||||||
|
cancel_compare_run,
|
||||||
|
recover_interrupted_compares_on_startup,
|
||||||
|
)
|
||||||
|
from netx_api.db import Base
|
||||||
|
from netx_api.models import BizCompareJob, BizCompareRun
|
||||||
|
|
||||||
|
|
||||||
|
class BizStateCompareRecoveryTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
|
||||||
|
TestingSession = sessionmaker(
|
||||||
|
bind=engine, autoflush=False, autocommit=False, expire_on_commit=False
|
||||||
|
)
|
||||||
|
Base.metadata.create_all(bind=engine)
|
||||||
|
self.db = TestingSession()
|
||||||
|
self.job = BizCompareJob(
|
||||||
|
id="j1",
|
||||||
|
name="cutover",
|
||||||
|
template_id="tpl1",
|
||||||
|
status="active",
|
||||||
|
)
|
||||||
|
self.db.add(self.job)
|
||||||
|
self.running = BizCompareRun(
|
||||||
|
id="r_run",
|
||||||
|
job_id="j1",
|
||||||
|
status="running",
|
||||||
|
message="loading 1/2 · BGP",
|
||||||
|
summary_json={
|
||||||
|
"progress": {"phase": "loading", "sheet_index": 1, "sheet_total": 2},
|
||||||
|
"sheets": [
|
||||||
|
{"sheet_id": "bgp", "status": "running"},
|
||||||
|
{"sheet_id": "isis", "status": "pending"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.queued = BizCompareRun(
|
||||||
|
id="r_q",
|
||||||
|
job_id="j1",
|
||||||
|
status="queued",
|
||||||
|
message="queued",
|
||||||
|
summary_json={"sheets": [{"sheet_id": "bgp", "status": "pending"}]},
|
||||||
|
)
|
||||||
|
self.ok = BizCompareRun(
|
||||||
|
id="r_ok",
|
||||||
|
job_id="j1",
|
||||||
|
status="success",
|
||||||
|
message="done",
|
||||||
|
summary_json={"added": 0, "removed": 0, "changed": 0},
|
||||||
|
)
|
||||||
|
self.db.add_all([self.running, self.queued, self.ok])
|
||||||
|
self.db.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.db.close()
|
||||||
|
|
||||||
|
def test_startup_cancels_running_and_queued(self) -> None:
|
||||||
|
out = recover_interrupted_compares_on_startup(self.db)
|
||||||
|
self.assertEqual(out["runs"], 2)
|
||||||
|
|
||||||
|
self.db.refresh(self.running)
|
||||||
|
self.db.refresh(self.queued)
|
||||||
|
self.db.refresh(self.ok)
|
||||||
|
self.assertEqual(self.running.status, "cancelled")
|
||||||
|
self.assertIn("interrupted_by_restart", self.running.message or "")
|
||||||
|
self.assertEqual(
|
||||||
|
(self.running.summary_json or {}).get("progress", {}).get("phase"),
|
||||||
|
"cancelled",
|
||||||
|
)
|
||||||
|
sheets = list((self.running.summary_json or {}).get("sheets") or [])
|
||||||
|
self.assertEqual(sheets[0].get("status"), "cancelled")
|
||||||
|
self.assertEqual(sheets[1].get("status"), "cancelled")
|
||||||
|
self.assertEqual(self.queued.status, "cancelled")
|
||||||
|
self.assertEqual(self.ok.status, "success")
|
||||||
|
|
||||||
|
def test_cancel_compare_run_user(self) -> None:
|
||||||
|
out = cancel_compare_run(self.db, "r_run")
|
||||||
|
self.assertEqual(out["status"], "cancelled")
|
||||||
|
self.db.refresh(self.running)
|
||||||
|
self.assertEqual(self.running.status, "cancelled")
|
||||||
|
self.assertIn("cancelled_by_user", self.running.message or "")
|
||||||
|
|
||||||
|
def test_cancel_idempotent_on_success(self) -> None:
|
||||||
|
out = cancel_compare_run(self.db, "r_ok")
|
||||||
|
self.assertEqual(out["status"], "success")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|
@ -520,10 +520,20 @@ const en = {
|
||||||
edit: "Edit",
|
edit: "Edit",
|
||||||
delete: "Delete",
|
delete: "Delete",
|
||||||
tabConfig: "Job config",
|
tabConfig: "Job config",
|
||||||
|
tabRuns: "Compare batches",
|
||||||
tabResult: "Result",
|
tabResult: "Result",
|
||||||
runs: "Run history",
|
runs: "Run history",
|
||||||
|
runsHint: "Each “Run now” creates a batch. Stuck “Running” after restart is auto-cancelled; you can also cancel manually and re-run.",
|
||||||
noRuns: "No runs yet — run a compare first",
|
noRuns: "No runs yet — run a compare first",
|
||||||
result: "Result",
|
result: "Result",
|
||||||
|
viewResult: "View result",
|
||||||
|
cancelRun: "Cancel",
|
||||||
|
confirmCancelRun: "Cancel this compare batch? You can start a new run afterward.",
|
||||||
|
runCancelled: "Compare cancelled",
|
||||||
|
runStatusCancelled: "Cancelled",
|
||||||
|
compareAlreadyRunning: "A compare is already running — cancel it under Compare batches or wait",
|
||||||
|
runBatchSummary: "Fail {{fail}} · Pass {{ok}} · Added {{added}}",
|
||||||
|
colTime: "Time",
|
||||||
pickBatchRun: "Select compare run",
|
pickBatchRun: "Select compare run",
|
||||||
pickRun: "Select run…",
|
pickRun: "Select run…",
|
||||||
runCount: "{{n}} runs",
|
runCount: "{{n}} runs",
|
||||||
|
|
|
||||||
|
|
@ -519,10 +519,20 @@ const zh = {
|
||||||
edit: "编辑",
|
edit: "编辑",
|
||||||
delete: "删除",
|
delete: "删除",
|
||||||
tabConfig: "任务配置",
|
tabConfig: "任务配置",
|
||||||
|
tabRuns: "对比批次",
|
||||||
tabResult: "比对结果",
|
tabResult: "比对结果",
|
||||||
runs: "历史比对",
|
runs: "历史比对",
|
||||||
|
runsHint: "每次「立即比对」生成一条批次。重启后卡住的「比对中」会自动标为已取消;也可手动终止后重跑。",
|
||||||
noRuns: "尚无比对记录,请先执行比对",
|
noRuns: "尚无比对记录,请先执行比对",
|
||||||
result: "比对结果",
|
result: "比对结果",
|
||||||
|
viewResult: "查看结果",
|
||||||
|
cancelRun: "终止",
|
||||||
|
confirmCancelRun: "确定终止该比对批次?终止后可重新发起比对。",
|
||||||
|
runCancelled: "比对已取消",
|
||||||
|
runStatusCancelled: "已取消",
|
||||||
|
compareAlreadyRunning: "已有比对在进行中,请先在「对比批次」中终止或等待完成",
|
||||||
|
runBatchSummary: "失败 {{fail}} · 成功 {{ok}} · 新增 {{added}}",
|
||||||
|
colTime: "时间",
|
||||||
pickBatchRun: "选择比对记录",
|
pickBatchRun: "选择比对记录",
|
||||||
pickRun: "选择比对记录…",
|
pickRun: "选择比对记录…",
|
||||||
runCount: "{{n}} 次",
|
runCount: "{{n}} 次",
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import { useDebouncedValue } from "../../hooks/useDebouncedValue";
|
||||||
import { useToast } from "../../hooks/useToast";
|
import { useToast } from "../../hooks/useToast";
|
||||||
import { useI18n } from "../../i18n";
|
import { useI18n } from "../../i18n";
|
||||||
import {
|
import {
|
||||||
|
bizCompareCancelRun,
|
||||||
bizCompareCreateJob,
|
bizCompareCreateJob,
|
||||||
bizCompareCreateMapping,
|
bizCompareCreateMapping,
|
||||||
bizCompareCreateTemplate,
|
bizCompareCreateTemplate,
|
||||||
|
|
@ -35,7 +36,7 @@ import { cutoverCachedGet, cutoverCachedGetSWR, invalidateCutoverCache } from ".
|
||||||
import { jobChipColor, NmStatusChip } from "./nmChips";
|
import { jobChipColor, NmStatusChip } from "./nmChips";
|
||||||
|
|
||||||
type PageTab = "templates" | "jobs";
|
type PageTab = "templates" | "jobs";
|
||||||
type JobDetailTab = "config" | "result";
|
type JobDetailTab = "config" | "runs" | "result";
|
||||||
type KindFilter = "diff" | "all" | "added" | "removed" | "changed" | "unchanged";
|
type KindFilter = "diff" | "all" | "added" | "removed" | "changed" | "unchanged";
|
||||||
type CreateJobStep = 0 | 1 | 2 | 3;
|
type CreateJobStep = 0 | 1 | 2 | 3;
|
||||||
const CREATE_JOB_STEPS = 4;
|
const CREATE_JOB_STEPS = 4;
|
||||||
|
|
@ -1802,12 +1803,14 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
const job = jobs.find((x) => x.id === id);
|
const job = jobs.find((x) => x.id === id);
|
||||||
if (job) resetJobForm(job);
|
if (job) resetJobForm(job);
|
||||||
try {
|
try {
|
||||||
const r = await bizCompareListRuns(id);
|
const r = await bizCompareListRuns(id, 50);
|
||||||
setRuns(r.items || []);
|
const items = r.items || [];
|
||||||
if ((r.items || []).length) {
|
setRuns(items);
|
||||||
const latest = await bizCompareGetRun(String((r.items as any[])[0].id));
|
if (items.length) {
|
||||||
|
const latest = await bizCompareGetRun(String((items as any[])[0].id));
|
||||||
setRunDetail(latest);
|
setRunDetail(latest);
|
||||||
setJobDetailTab("result");
|
// Land on batch list so stuck/cancelled runs are visible and actionable
|
||||||
|
setJobDetailTab("runs");
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
showError(formatErr(e));
|
showError(formatErr(e));
|
||||||
|
|
@ -1843,32 +1846,42 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const runStatus = String(runDetail?.status || "");
|
||||||
|
const runIsActive = runStatus === "running" || runStatus === "queued";
|
||||||
|
const jobHasActiveRun = runs.some((r) => {
|
||||||
|
const st = String((r as any).status || "");
|
||||||
|
return st === "running" || st === "queued";
|
||||||
|
});
|
||||||
|
|
||||||
const runNow = async () => {
|
const runNow = async () => {
|
||||||
if (!jobId) return;
|
if (!jobId) return;
|
||||||
if (!enabledJobSheetCount) {
|
if (!enabledJobSheetCount) {
|
||||||
showError(t("bizCompare.needSheets"));
|
showError(t("bizCompare.needSheets"));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (jobHasActiveRun) {
|
||||||
|
showError(t("bizCompare.compareAlreadyRunning"));
|
||||||
|
setJobDetailTab("runs");
|
||||||
|
return;
|
||||||
|
}
|
||||||
setBusy(true);
|
setBusy(true);
|
||||||
try {
|
try {
|
||||||
await bizCompareUpdateJob(jobId, jobConfigBody());
|
await bizCompareUpdateJob(jobId, jobConfigBody());
|
||||||
// Async enqueue — returns immediately with status=running; poll below.
|
// Async enqueue — returns immediately with status=running; poll below.
|
||||||
const run = await bizCompareRunJob(jobId);
|
const run = await bizCompareRunJob(jobId);
|
||||||
setRunDetail(run);
|
setRunDetail(run);
|
||||||
setJobDetailTab("result");
|
setJobDetailTab("runs");
|
||||||
showOk(t("bizCompare.runStarted"));
|
showOk(t("bizCompare.runStarted"));
|
||||||
const r = await bizCompareListRuns(jobId);
|
const r = await bizCompareListRuns(jobId, 50);
|
||||||
setRuns(r.items || []);
|
setRuns(r.items || []);
|
||||||
await refresh({ force: true });
|
await refresh({ force: true });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
showError(formatErr(e));
|
showError(formatErr(e));
|
||||||
|
setJobDetailTab("runs");
|
||||||
} finally {
|
} finally {
|
||||||
setBusy(false);
|
setBusy(false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const runStatus = String(runDetail?.status || "");
|
|
||||||
const runIsActive = runStatus === "running" || runStatus === "queued";
|
|
||||||
const runProgress = (runDetail?.summary?.progress || {}) as {
|
const runProgress = (runDetail?.summary?.progress || {}) as {
|
||||||
phase?: string;
|
phase?: string;
|
||||||
sheet_index?: number;
|
sheet_index?: number;
|
||||||
|
|
@ -1906,6 +1919,9 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
} else if (!notified && st === "failed") {
|
} else if (!notified && st === "failed") {
|
||||||
notified = true;
|
notified = true;
|
||||||
showError(String(d.message || t("bizCompare.runFailed")));
|
showError(String(d.message || t("bizCompare.runFailed")));
|
||||||
|
} else if (!notified && st === "cancelled") {
|
||||||
|
notified = true;
|
||||||
|
showError(String(d.message || t("bizCompare.runCancelled")));
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (!cancelled) showError(formatErr(e));
|
if (!cancelled) showError(formatErr(e));
|
||||||
|
|
@ -1919,6 +1935,31 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
};
|
};
|
||||||
}, [runDetail?.id, runIsActive, jobId, showOk, showError, t]);
|
}, [runDetail?.id, runIsActive, jobId, showOk, showError, t]);
|
||||||
|
|
||||||
|
// Keep the runs list fresh while any batch on this job is active
|
||||||
|
useEffect(() => {
|
||||||
|
if (!jobId || !jobHasActiveRun || jobDetailTab !== "runs") return;
|
||||||
|
let cancelled = false;
|
||||||
|
const tick = async () => {
|
||||||
|
try {
|
||||||
|
const r = await bizCompareListRuns(jobId, 50);
|
||||||
|
if (!cancelled) setRuns(r.items || []);
|
||||||
|
const curId = String(runDetail?.id || "");
|
||||||
|
if (curId) {
|
||||||
|
const d = await bizCompareGetRun(curId);
|
||||||
|
if (!cancelled) setRunDetail(d);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* ignore list poll errors */
|
||||||
|
}
|
||||||
|
};
|
||||||
|
void tick();
|
||||||
|
const id = window.setInterval(() => void tick(), 2500);
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
window.clearInterval(id);
|
||||||
|
};
|
||||||
|
}, [jobId, jobHasActiveRun, jobDetailTab, runDetail?.id]);
|
||||||
|
|
||||||
const loadRun = async (runId: string) => {
|
const loadRun = async (runId: string) => {
|
||||||
try {
|
try {
|
||||||
const d = await bizCompareGetRun(runId);
|
const d = await bizCompareGetRun(runId);
|
||||||
|
|
@ -1931,6 +1972,25 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const cancelRun = async (runId: string) => {
|
||||||
|
if (!runId) return;
|
||||||
|
if (!window.confirm(t("bizCompare.confirmCancelRun"))) return;
|
||||||
|
setBusy(true);
|
||||||
|
try {
|
||||||
|
const d = await bizCompareCancelRun(runId);
|
||||||
|
if (String(runDetail?.id || "") === runId) setRunDetail(d);
|
||||||
|
if (jobId) {
|
||||||
|
const r = await bizCompareListRuns(jobId, 50);
|
||||||
|
setRuns(r.items || []);
|
||||||
|
}
|
||||||
|
showOk(t("bizCompare.runCancelled"));
|
||||||
|
} catch (e) {
|
||||||
|
showError(formatErr(e));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const removeRun = async (runId: string) => {
|
const removeRun = async (runId: string) => {
|
||||||
if (!runId) return;
|
if (!runId) return;
|
||||||
if (!window.confirm(t("bizCompare.confirmDeleteRun"))) return;
|
if (!window.confirm(t("bizCompare.confirmDeleteRun"))) return;
|
||||||
|
|
@ -1940,19 +2000,16 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
await bizCompareDeleteRun(runId);
|
await bizCompareDeleteRun(runId);
|
||||||
let nextRuns: typeof runs = [];
|
let nextRuns: typeof runs = [];
|
||||||
if (jobId) {
|
if (jobId) {
|
||||||
const r = await bizCompareListRuns(jobId);
|
const r = await bizCompareListRuns(jobId, 50);
|
||||||
nextRuns = r.items || [];
|
nextRuns = r.items || [];
|
||||||
} else {
|
} else {
|
||||||
nextRuns = (runs || []).filter((r) => String(r.id) !== runId);
|
nextRuns = (runs || []).filter((r) => String(r.id) !== runId);
|
||||||
}
|
}
|
||||||
setRuns(nextRuns);
|
setRuns(nextRuns);
|
||||||
if (wasCurrent) {
|
if (wasCurrent) {
|
||||||
if (nextRuns.length) {
|
setRunDetail(null);
|
||||||
await loadRun(String(nextRuns[0].id));
|
setResultSheetId("");
|
||||||
} else {
|
setJobDetailTab("runs");
|
||||||
setRunDetail(null);
|
|
||||||
setResultSheetId("");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
showOk(t("bizCompare.runDeleted"));
|
showOk(t("bizCompare.runDeleted"));
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|
@ -2864,6 +2921,18 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
>
|
>
|
||||||
{t("bizCompare.tabConfig")}
|
{t("bizCompare.tabConfig")}
|
||||||
</Button>
|
</Button>
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
variant={jobDetailTab === "runs" ? "primary" : "secondary"}
|
||||||
|
className={jobDetailTab === "runs" ? "is-active" : undefined}
|
||||||
|
onPress={() => {
|
||||||
|
setJobDetailTab("runs");
|
||||||
|
if (jobId) void bizCompareListRuns(jobId, 50).then((r) => setRuns(r.items || []));
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{t("bizCompare.tabRuns")}
|
||||||
|
{runs.length ? ` (${runs.length})` : ""}
|
||||||
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
size="sm"
|
size="sm"
|
||||||
variant={jobDetailTab === "result" ? "primary" : "secondary"}
|
variant={jobDetailTab === "result" ? "primary" : "secondary"}
|
||||||
|
|
@ -2876,6 +2945,156 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
|
|
||||||
{jobDetailTab === "config" ? (
|
{jobDetailTab === "config" ? (
|
||||||
renderJobForm()
|
renderJobForm()
|
||||||
|
) : jobDetailTab === "runs" ? (
|
||||||
|
<div className="bs-cmp-runs">
|
||||||
|
<div className="btn-row" style={{ marginBottom: 8, justifyContent: "space-between" }}>
|
||||||
|
<p className="muted bm-hint" style={{ margin: 0 }}>
|
||||||
|
{t("bizCompare.runsHint")}
|
||||||
|
</p>
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
variant="primary"
|
||||||
|
isDisabled={busy || jobHasActiveRun}
|
||||||
|
onPress={() => void runNow()}
|
||||||
|
>
|
||||||
|
{jobHasActiveRun ? t("bizCompare.runStatusRunning") : t("bizCompare.runNow")}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
{!runs.length ? (
|
||||||
|
<div className="pt-list-empty">{t("bizCompare.noRuns")}</div>
|
||||||
|
) : (
|
||||||
|
<div className="table-wrap">
|
||||||
|
<table className="data-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>{t("bizCompare.colStatus")}</th>
|
||||||
|
<th>{t("bizCompare.colTime")}</th>
|
||||||
|
<th>{t("bizCompare.sidesTitle")}</th>
|
||||||
|
<th>{t("bizCompare.colResult")}</th>
|
||||||
|
<th>{t("bizCompare.colActions")}</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{runs.map((r) => {
|
||||||
|
const st = String((r as any).status || "");
|
||||||
|
const active = st === "running" || st === "queued";
|
||||||
|
const selected = String(runDetail?.id || "") === String(r.id);
|
||||||
|
const before = enrichSide(
|
||||||
|
(r as any).before,
|
||||||
|
beforeTaskId,
|
||||||
|
tasks,
|
||||||
|
beforeBatches,
|
||||||
|
);
|
||||||
|
const after = enrichSide(
|
||||||
|
(r as any).after,
|
||||||
|
afterTaskId || beforeTaskId,
|
||||||
|
tasks,
|
||||||
|
afterBatches.length ? afterBatches : beforeBatches,
|
||||||
|
);
|
||||||
|
const when = formatSystemTime((r as any).created_at) || "—";
|
||||||
|
const sum = ((r as any).summary || {}) as Record<string, number>;
|
||||||
|
const fail =
|
||||||
|
Number(sum.removed || 0) + Number(sum.changed || 0);
|
||||||
|
const ok = Number(sum.unchanged || 0);
|
||||||
|
const durMs = Number(sum.duration_ms || 0);
|
||||||
|
const prog = ((r as any).progress || {}) as {
|
||||||
|
phase?: string;
|
||||||
|
sheet_index?: number;
|
||||||
|
sheet_total?: number;
|
||||||
|
sheet_title?: string;
|
||||||
|
rows_loaded?: number;
|
||||||
|
load_side?: string;
|
||||||
|
};
|
||||||
|
const stLabel =
|
||||||
|
st === "running" || st === "queued"
|
||||||
|
? t("bizCompare.runStatusRunning")
|
||||||
|
: st === "failed"
|
||||||
|
? t("bizCompare.runStatusFailed")
|
||||||
|
: st === "cancelled"
|
||||||
|
? t("bizCompare.runStatusCancelled")
|
||||||
|
: t("bizCompare.runStatusDoneSec", {
|
||||||
|
s: String(durMs > 0 ? Math.round(durMs / 1000) : 0),
|
||||||
|
});
|
||||||
|
return (
|
||||||
|
<tr key={String(r.id)} className={selected ? "is-selected" : undefined}>
|
||||||
|
<td>
|
||||||
|
<NmStatusChip color={jobChipColor(st)}>{stLabel}</NmStatusChip>
|
||||||
|
{active && prog.phase ? (
|
||||||
|
<div className="muted" style={{ fontSize: "0.75rem", marginTop: 4 }}>
|
||||||
|
{prog.phase}
|
||||||
|
{prog.sheet_total
|
||||||
|
? ` · ${prog.sheet_index || 0}/${prog.sheet_total}`
|
||||||
|
: ""}
|
||||||
|
{prog.sheet_title ? ` · ${prog.sheet_title}` : ""}
|
||||||
|
{Number(prog.rows_loaded || 0) > 0
|
||||||
|
? ` · ${prog.load_side || ""} ${prog.rows_loaded}`
|
||||||
|
: ""}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{(r as any).message ? (
|
||||||
|
<div
|
||||||
|
className="muted"
|
||||||
|
style={{ fontSize: "0.75rem", marginTop: 2 }}
|
||||||
|
title={String((r as any).message)}
|
||||||
|
>
|
||||||
|
{String((r as any).message).slice(0, 80)}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</td>
|
||||||
|
<td>{when}</td>
|
||||||
|
<td>
|
||||||
|
{sideDeviceName(before)} {sideCollectTime(before)}
|
||||||
|
{" → "}
|
||||||
|
{sideDeviceName(after)} {sideCollectTime(after)}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{active
|
||||||
|
? "…"
|
||||||
|
: t("bizCompare.runBatchSummary", {
|
||||||
|
fail: String(fail),
|
||||||
|
ok: String(ok),
|
||||||
|
added: String(Number(sum.added || 0)),
|
||||||
|
})}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<div className="btn-row">
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
variant="secondary"
|
||||||
|
isDisabled={busy || active}
|
||||||
|
onPress={() => void loadRun(String(r.id))}
|
||||||
|
>
|
||||||
|
{t("bizCompare.viewResult")}
|
||||||
|
</Button>
|
||||||
|
{active ? (
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
variant="danger"
|
||||||
|
isDisabled={busy}
|
||||||
|
onPress={() => void cancelRun(String(r.id))}
|
||||||
|
>
|
||||||
|
{t("bizCompare.cancelRun")}
|
||||||
|
</Button>
|
||||||
|
) : (
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
variant="danger"
|
||||||
|
isDisabled={busy}
|
||||||
|
onPress={() => void removeRun(String(r.id))}
|
||||||
|
>
|
||||||
|
{t("bizCompare.deleteRun")}
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<div
|
<div
|
||||||
ref={boardRef}
|
ref={boardRef}
|
||||||
|
|
@ -3536,10 +3755,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
<Button
|
<Button
|
||||||
size="sm"
|
size="sm"
|
||||||
variant="primary"
|
variant="primary"
|
||||||
isDisabled={busy || runIsActive}
|
isDisabled={busy || jobHasActiveRun}
|
||||||
onPress={() => void runNow()}
|
onPress={() => void runNow()}
|
||||||
>
|
>
|
||||||
{runIsActive ? t("bizCompare.runStatusRunning") : t("bizCompare.runNow")}
|
{jobHasActiveRun ? t("bizCompare.runStatusRunning") : t("bizCompare.runNow")}
|
||||||
</Button>
|
</Button>
|
||||||
{jobId ? (
|
{jobId ? (
|
||||||
<Button size="sm" variant="danger" isDisabled={busy} onPress={() => void removeJob(jobId)}>
|
<Button size="sm" variant="danger" isDisabled={busy} onPress={() => void removeJob(jobId)}>
|
||||||
|
|
@ -3547,12 +3766,21 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
</Button>
|
</Button>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
|
) : jobDetailTab === "runs" ? (
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
variant="primary"
|
||||||
|
isDisabled={busy || jobHasActiveRun}
|
||||||
|
onPress={() => void runNow()}
|
||||||
|
>
|
||||||
|
{jobHasActiveRun ? t("bizCompare.runStatusRunning") : t("bizCompare.runNow")}
|
||||||
|
</Button>
|
||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
<Button
|
<Button
|
||||||
size="sm"
|
size="sm"
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
isDisabled={busy || !runDetail?.id}
|
isDisabled={busy || runIsActive || !runDetail?.id}
|
||||||
onPress={() => void downloadRunTables()}
|
onPress={() => void downloadRunTables()}
|
||||||
>
|
>
|
||||||
{t("bizCompare.exportTables")}
|
{t("bizCompare.exportTables")}
|
||||||
|
|
@ -3568,10 +3796,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
|
||||||
<Button
|
<Button
|
||||||
size="sm"
|
size="sm"
|
||||||
variant="primary"
|
variant="primary"
|
||||||
isDisabled={busy || runIsActive}
|
isDisabled={busy || jobHasActiveRun}
|
||||||
onPress={() => void runNow()}
|
onPress={() => void runNow()}
|
||||||
>
|
>
|
||||||
{runIsActive ? t("bizCompare.runStatusRunning") : t("bizCompare.runNow")}
|
{jobHasActiveRun ? t("bizCompare.runStatusRunning") : t("bizCompare.runNow")}
|
||||||
</Button>
|
</Button>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
|
||||||
|
|
@ -2154,6 +2154,12 @@ export const bizCompareListRuns = (jobId: string, limit = 20) =>
|
||||||
export const bizCompareGetRun = (runId: string) =>
|
export const bizCompareGetRun = (runId: string) =>
|
||||||
apiGet<Record<string, unknown>>(`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`);
|
apiGet<Record<string, unknown>>(`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`);
|
||||||
|
|
||||||
|
export const bizCompareCancelRun = (runId: string) =>
|
||||||
|
apiPost<Record<string, unknown>>(
|
||||||
|
`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}/cancel`,
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
|
||||||
export const bizCompareDeleteRun = (runId: string) =>
|
export const bizCompareDeleteRun = (runId: string) =>
|
||||||
apiDelete<{ ok: boolean; job_id?: string; run_id?: string }>(
|
apiDelete<{ ok: boolean; job_id?: string; run_id?: string }>(
|
||||||
`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`,
|
`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue