hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-11 15:30:45 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions

fix: enforce business continuity and live cutover monitoring

Browse source
This commit is contained in:
oliver 2026-10-11 07:33:17 +08:00
parent 355bcfc4c0
commit fe4c1db763
11 changed files with 769 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

104
netx_api/biz_migration/auto_monitor.py Normal file
View file

@ -0,0 +1,104 @@
"""Coalesced collect-completion events for live cutover evaluation.
One worker per process bounds concurrency. Project row locks in run_evaluate
serialize persistence across processes. No device collection runs here.
"""
from __future__ import annotations
import hashlib
import json
import logging
import threading
from sqlalchemy.orm import Session
from ..models import BizMigrationBatch, BizMigrationProject, BizMigrationRun, BizMonitorTemplate, BizStateBatch
from . import service
_log = logging.getLogger(__name__)
_lock = threading.Lock()
_pending: set[str] = set()
_worker_running = False
def try_auto_monitor_for_task(db: Session, task_id: str) -> int:
count = 0
# Bindings are JSON for multiple sampling intervals; match in Python after
# limiting to projects with an active operation batch.
projects = db.query(BizMigrationProject).filter(
BizMigrationProject.status != "done",
BizMigrationProject.id.in_(db.query(BizMigrationBatch.project_id).filter(
BizMigrationBatch.status == "active")),
).all()
for candidate in projects:
tids = service._all_hf_task_ids(candidate, "old") + service._all_hf_task_ids(candidate, "new")
if task_id not in tids or not candidate.old_baseline_batch_id or not candidate.new_baseline_batch_id:
continue
# Lock before deduplication so two worker processes cannot save the same
# snapshot. run_evaluate holds the same lock through the transaction.
proj = db.query(BizMigrationProject).filter(
BizMigrationProject.id == candidate.id).with_for_update().populate_existing().first()
batch = db.query(BizMigrationBatch).filter(
BizMigrationBatch.project_id == proj.id, BizMigrationBatch.status == "active"
).order_by(BizMigrationBatch.started_at.desc()).first()
if not batch or proj.status == "done":
db.rollback()
continue
mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None
if not mt:
db.rollback()
_log.error("cutover monitor template missing project=%s", proj.id)
continue
sheets, overrides, defaults, norms = service.resolve_evaluate_sheets(db, mt)
samples = []
for tid in tids:
sample = db.query(BizStateBatch).filter(
BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued"))
).order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first()
samples.append([tid, sample.id if sample else "", sample.status if sample else ""])
payload = [samples, proj.old_baseline_batch_id, proj.new_baseline_batch_id,
batch.expect_set_json, service._port_map_dict(db, proj.mapping_id),
sheets, overrides, defaults, norms, service.resolve_collect_metric_ids(db, proj)]
fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
last = db.query(BizMigrationRun).filter(BizMigrationRun.batch_id == batch.id).order_by(
BizMigrationRun.created_at.desc()).first()
if last and (last.summary_json or {}).get("auto_fingerprint") == fingerprint:
db.rollback()
continue
try:
result = service.run_evaluate(db, batch_id=batch.id, purpose="auto", _commit=False)
run = db.get(BizMigrationRun, result["id"])
run.summary_json = {**run.summary_json, "auto_fingerprint": fingerprint}
db.commit()
count += 1
except Exception:
db.rollback()
_log.exception("cutover monitor evaluate failed project=%s", proj.id)
return count
def schedule_auto_monitor_for_task(task_id: str) -> None:
global _worker_running
with _lock:
_pending.add(str(task_id))
if _worker_running:
return
_worker_running = True
def work() -> None:
global _worker_running
from ..db import SessionLocal
while True:
with _lock:
if not _pending:
_worker_running = False
return
tid = _pending.pop()
try:
with SessionLocal() as db:
try_auto_monitor_for_task(db, tid)
except Exception:
_log.exception("cutover monitor worker failed task=%s", tid)
threading.Thread(target=work, name="biz-cutover-monitor", daemon=True).start()

132
netx_api/biz_migration/evaluate.py
View file

@ -5,6 +5,9 @@ from __future__ import annotations
from typing import Any
from ..biz_state.compare_engine import apply_port_map, compare_rows
from ..biz_state.compare_rules import (
effective_compare_fields, field_rule_map, scalar_text, values_equal,
)
PORT_METRIC_ID = "interface_brief"
PORT_STATUS_FIELDS = ("admin", "phy", "prot")
@ -22,7 +25,7 @@ def normalize_expect_key(raw: str) -> str:
Accepts legacy ``a|b`` multi-field keys and already-normalized ``\\x1f`` keys.
"""
s = str(raw or "").strip()
s = scalar_text(raw).strip()
if not s:
return ""
if KEY_SEP in s:
@ -65,13 +68,13 @@ def classify_status(row: dict[str, Any] | None, sheet_override: dict[str, Any] |
return "none"
down_vals = {str(x).lower() for x in (ov.get("down_values") or ["down"])}
up_vals = {str(x).lower() for x in (ov.get("up_values") or ["up"])}
vals = [str(row.get(f) or "").strip().lower() for f in fields]
vals = [scalar_text(row.get(f)).strip().lower() for f in fields]
vals = [v for v in vals if v]
if not vals:
return "none"
if any(v in down_vals for v in vals):
return "down"
if vals and all(v in up_vals for v in vals):
if len(vals) == len(fields) and all(v in up_vals for v in vals):
return "up"
return "other"
@ -219,7 +222,7 @@ def _field_tokens_from_row(row: dict[str, Any] | None, fields: set[str]) -> set[
return set()
out: set[str] = set()
for f in fields:
v = str(row.get(f) or "").strip()
v = scalar_text(row.get(f)).strip()
if v:
out.add(field_token(f, v))
return out
@ -245,8 +248,8 @@ def _side_tokens(
def _norm_list_values(raw: Any) -> list[str]:
if isinstance(raw, list):
return [str(x).strip().lower() for x in raw if str(x).strip()]
s = str(raw or "").strip()
return [scalar_text(x).strip().lower() for x in raw if scalar_text(x).strip()]
s = scalar_text(raw).strip()
if not s:
return []
if "," in s:
@ -266,18 +269,18 @@ def _eval_field_op(
f = str(field or "").strip()
if not f:
return False
raw = "" if not row else str(row.get(f) or "").strip()
raw = "" if not row else scalar_text(row.get(f)).strip()
val = raw.lower()
o = str(op or "eq").strip().lower()
if o in ("changed", "diff"):
base = "" if not base_row else str(base_row.get(f) or "").strip().lower()
base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower()
if not row:
return False
return base != val
if o in ("unchanged", "same"):
if not row:
return False
base = "" if not base_row else str(base_row.get(f) or "").strip().lower()
base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower()
return base == val
if o == "empty":
return not val
@ -568,7 +571,7 @@ def _match_field_token_rules(
if not field or not allowed:
continue
row = old_row if side == "old" else new_row if side == "new" else None
val = str((row or {}).get(field) or "").strip().lower()
val = scalar_text((row or {}).get(field)).strip().lower()
if val not in allowed:
return False
return True
@ -858,6 +861,14 @@ def _port_identity(key_fields: list[str], iface_fields: list[str]) -> bool:
return all(str(f) in iface_set for f in key_fields)
def _iface_selected(value: str, ports: set[str]) -> bool:
"""Set lookup plus parent interface prefixes; independent of scope size."""
if value in ports:
return True
parts = value.split(".")
return any(".".join(parts[:i]) in ports for i in range(len(parts) - 1, 0, -1))
def _unmapped_same_iface_anomaly(
*,
old_kind: str,
@ -892,6 +903,8 @@ def _remap_key_str(
return key_str
if not port_map:
return key_str
if key_fields and not set(key_fields).intersection(iface_fields):
return key_str
if (not key_fields or len(key_fields) == 1) and (
key_str in port_map
or any(
@ -928,6 +941,8 @@ def _reverse_remap_key_str(
return key_str
if not rev_map:
return key_str
if key_fields and not set(key_fields).intersection(iface_fields):
return key_str
if (not key_fields or len(key_fields) == 1) and (
key_str in rev_map
or any(
@ -1009,7 +1024,7 @@ def row_match_key(
data = strip_netx(row)
if iface_fields and rules:
data = apply_iface_normalize(dict(data), iface_fields=iface_fields, rules=rules)
return KEY_SEP.join(str(data.get(k) or "").strip() for k in key_fields)
return KEY_SEP.join(scalar_text(data.get(k)).strip() for k in key_fields)
def index_raw_by_match_key(
@ -1033,7 +1048,7 @@ def display_key_from_raw(raw: dict[str, Any] | None, key_fields: list[str]) -> s
if not raw or not key_fields:
return ""
data = strip_netx(raw)
return "|".join(str(data.get(k) or "").strip() for k in key_fields)
return "|".join(scalar_text(data.get(k)).strip() for k in key_fields)
def iface_lineage(
@ -1053,7 +1068,7 @@ def iface_lineage(
pmap = port_map or {}
out: list[dict[str, Any]] = []
for f in iface_fields:
raw_v = str(data.get(f) or "").strip()
raw_v = scalar_text(data.get(f)).strip()
norm_v = normalize_iface_name(raw_v, rules) if rules else raw_v
mapped = False
map_to = ""
@ -1092,6 +1107,8 @@ def evaluate_metric_dual(
out_of_expect: str = "strict",
sheet_id: str = "",
iface_normalize_rules: list[dict[str, str]] | None = None,
previous_expect: dict[str, set[str]] | None = None,
allow_mapping_scope: bool = True,
) -> dict[str, Any]:
"""Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge.
@ -1108,6 +1125,9 @@ def evaluate_metric_dual(
expect_keys = expect_keys_for_metric(
expect, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid
)
previous_keys = expect_keys_for_metric(
previous_expect or {}, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid
)
ov = sheet_override or {}
if ov.get("skip_dual"):
return {
@ -1178,6 +1198,7 @@ def evaluate_metric_dual(
port_map=None,
field_rules=field_rules,
iface_normalize_rules=None, # already applied
include_unchanged=True,
)
old_idx = build_diff_index_from_compare(old_cmp)
@ -1186,18 +1207,8 @@ def evaluate_metric_dual(
if new_baseline_rows is not None:
new_before = list(new_baseline_rows)
new_baseline_mode = "provided"
# Explicit empty batch is as unusable as "no baseline" for presence semantics.
if not new_before and (old_baseline_rows or new_current_rows):
if port_map and iface_fields:
new_before = [
apply_port_map(r, iface_fields=iface_fields, port_map=port_map)
for r in old_baseline_rows
]
new_baseline_mode = "port_mapped"
new_baseline_missing = False
else:
new_baseline_missing = True
new_baseline_mode = "empty"
# A successfully collected empty baseline is valid on a replacement NE.
# Collection completeness is checked by the service, not inferred from rows.
elif port_map and iface_fields:
new_before = [
apply_port_map(r, iface_fields=iface_fields, port_map=port_map)
@ -1218,12 +1229,22 @@ def evaluate_metric_dual(
compare_fields=compare_fields,
port_map=None,
field_rules=field_rules,
include_unchanged=True,
)
new_idx = build_diff_index_from_compare(new_cmp)
rev_map = {v: k for k, v in port_map.items()}
map_scoped = _map_defines_iface_expect(metric_id, iface_fields, port_map)
if map_scoped:
# Mapping describes identity; an explicit batch selection defines scope.
# Port selection also covers business rows attached to those interfaces.
for scope, keys in ((expect, expect_keys), (previous_expect or {}, previous_keys)):
ports = scope.get("_ports") or set()
if ports and not keys and iface_fields:
for ks, diff in old_idx.items():
row = diff.get("before") or diff.get("after") or {}
if any(_iface_selected(scalar_text(row.get(f)), ports) for f in iface_fields):
keys.add(ks)
if allow_mapping_scope and map_scoped and not any(expect.values()):
scoped: set[str] = set()
for ks in old_idx:
if _key_in_port_map(
@ -1253,6 +1274,8 @@ def evaluate_metric_dual(
for ok in sorted(expect_keys):
_add(ok)
for ok in sorted(previous_keys):
_add(ok)
for ks in sorted(old_idx):
_add(ks)
for ks in sorted(new_idx):
@ -1270,6 +1293,10 @@ def evaluate_metric_dual(
progress_total = 0
anomaly = 0
anomaly_in_expect = 0
rules_by_field = field_rule_map(field_rules)
integrity_fields = effective_compare_fields(compare_fields, field_rules)
# Status transitions are interpreted by the monitor's correction rules.
integrity_fields = [f for f in integrity_fields if f not in (ov.get("status_fields") or [])]
for old_ks in canon_keys:
new_ks = _remap_key_str(
@ -1279,11 +1306,12 @@ def evaluate_metric_dual(
port_map=port_map,
)
in_exp = old_ks in expect_keys
previous = old_ks in previous_keys and not in_exp
if in_exp:
progress_total += 1
od = old_idx.get(old_ks)
nd = new_idx.get(new_ks) or new_idx.get(old_ks)
nd = new_idx.get(new_ks)
old_kind = str((od or {}).get("kind") or "")
new_kind = str((nd or {}).get("kind") or "")
@ -1301,11 +1329,7 @@ def evaluate_metric_dual(
old_st = classify_status(old_cur, ov)
new_st = classify_status(new_cur, ov)
# Mapped iface rows are the expect set. Unmapped rows stay out of the
# migration compare, except same-name ports where old went down and the
# new device's same port is up — that is anomaly, not a successful cutover.
if map_scoped and not in_exp:
if not (
if map_scoped and not (in_exp or previous) and (
_port_identity(key_fields, iface_fields)
and _unmapped_same_iface_anomaly(
old_kind=old_kind,
@ -1314,26 +1338,62 @@ def evaluate_metric_dual(
new_status=new_st,
)
):
continue
verdict, color, rule_hit = "anomaly", "red", "unmapped_same_iface"
else:
verdict, color, rule_hit = dual_verdict_ex(
old_kind=old_kind or "",
new_kind=new_kind or "",
in_expect=in_exp,
in_expect=in_exp or previous,
window_active=window_active,
acceptance=acceptance,
old_status=old_st,
new_status=new_st,
success_patterns=success_patterns or None,
anomaly_patterns=anomaly_patterns or None,
out_of_expect=out_of_expect,
# Business drift outside this batch must remain visible.
out_of_expect="strict",
old_row=old_cur or None,
new_row=new_cur or None,
old_base=old_base or None,
new_base=new_base or None,
sheet_override=ov,
)
integrity_changes: list[dict[str, Any]] = []
if in_exp or previous:
old_departed = old_kind == "removed" or old_st == "down"
new_healthy = bool(new_cur) and (
new_st == "up" if ov.get("status_fields") else True
)
if old_departed and not new_healthy:
verdict, color, rule_hit = "lost", "red", "service_not_received"
elif verdict == "migrated":
if not new_healthy:
verdict, color, rule_hit = "anomaly", "red", "target_unhealthy"
elif not old_base:
verdict, color, rule_hit = "anomaly", "red", "old_service_not_in_baseline"
else:
reference = apply_port_map(old_base, iface_fields=iface_fields, port_map=port_map)
integrity_changes = [
{"field": f, "before": reference.get(f), "after": new_cur.get(f)}
for f in integrity_fields
if not values_equal(reference.get(f), new_cur.get(f), rule=rules_by_field.get(f))
]
if integrity_changes:
verdict, color, rule_hit = "anomaly", "red", "service_content_changed"
elif not old_departed and not success_patterns:
verdict, color, rule_hit = (
("unfinished", "red", "old_service_still_present") if acceptance
else ("migrating", "yellow", "old_service_still_present")
)
if previous:
if verdict == "migrated":
verdict, rule_hit = "migrated_previous", "previous_batch_received"
elif color != "red":
verdict, color, rule_hit = "unfinished_previous", "red", "previous_batch_not_received"
# Retain all drift outside scope, but avoid filling the board with stable
# unmapped inventory when a port map defines the legacy migration scope.
if map_scoped and not in_exp and verdict == "not_involved":
continue
if in_exp and verdict == "migrated" and color == "green":
progress_ok += 1
if color == "red":
@ -1411,7 +1471,9 @@ def evaluate_metric_dual(
"verdict": verdict,
"color": color,
"rule_hit": rule_hit,
"integrity_changes": integrity_changes,
"in_expect": in_exp,
"previous_batch": previous,
"old_kind": old_kind,
"new_kind": new_kind,
"old": old_disp,

10
netx_api/biz_migration/monitor_templates.py
View file

@ -8,7 +8,8 @@ from uuid import uuid4
from fastapi import HTTPException
from sqlalchemy.orm import Session
from ..models import BizCompareTemplate, BizMonitorTemplate
from ..models import BizCompareTemplate, BizMigrationProject, BizMonitorTemplate
from .validation import validate_monitor_rules
from ..timeutil import utcnow_naive
from ..biz_state import compare_service as cmp_svc
from ..biz_state.iface_normalize import default_zte_iface_normalize_rules
@ -456,6 +457,7 @@ def create_monitor_template(db: Session, body: dict[str, Any]) -> dict[str, Any]
overrides = body.get("sheet_overrides")
if not isinstance(overrides, list):
overrides = []
validate_monitor_rules(defaults, overrides)
row = BizMonitorTemplate(
id=uuid4().hex,
name=name[:256],
@ -479,6 +481,10 @@ def update_monitor_template(db: Session, template_id: str, body: dict[str, Any])
row = db.get(BizMonitorTemplate, template_id)
if not row:
raise HTTPException(status_code=404, detail="monitor_template_not_found")
validate_monitor_rules(body.get("defaults", row.defaults_json or {}),
body.get("sheet_overrides", row.sheet_overrides_json or []))
if body.get("compare_template_id") and not db.get(BizCompareTemplate, body["compare_template_id"]):
raise HTTPException(status_code=404, detail="compare_template_not_found")
if "name" in body and body["name"] is not None:
name = str(body["name"] or "").strip()
if not name:
@ -515,5 +521,7 @@ def delete_monitor_template(db: Session, template_id: str) -> None:
row = db.get(BizMonitorTemplate, template_id)
if not row:
raise HTTPException(status_code=404, detail="monitor_template_not_found")
if db.query(BizMigrationProject.id).filter(BizMigrationProject.monitor_template_id == template_id).first():
raise HTTPException(status_code=409, detail="monitor_template_in_use")
db.delete(row)
db.commit()

255
netx_api/biz_migration/service.py
View file

@ -13,7 +13,6 @@ from ..biz_state.compare_rules import apply_row_filters
from ..biz_state.compare_service import (
_load_metric_rows,
_port_map_dict,
batch_metric_collect_ok,
sheet_key,
sheet_title,
template_metrics,
@ -28,6 +27,7 @@ from ..models import (
BizMonitorTemplate,
BizPortMapping,
BizStateBatch,
BizStateBatchCommand,
BizStateTask,
)
from ..timeutil import utcnow_naive
@ -149,7 +149,7 @@ def hf_task_id(proj: BizMigrationProject, side: str) -> str:
def current_task_id(proj: BizMigrationProject, side: str) -> str:
"""Evaluate current batches: prefer HF, fall back to portrait."""
return hf_task_id(proj, side) or portrait_task_id(proj, side)
return hf_task_id(proj, side)
def _metric_interval_map(proj: BizMigrationProject) -> dict[str, int]:
@ -268,15 +268,54 @@ def _current_batch_for_metric(
# Only honor pin when it belongs to this metric's HF task (multi-interval safe)
if pinned and (not tid or str(pinned.task_id or "") == tid):
# Pinned partial / failed metric → treat as missing (avoid false red)
if not batch_metric_collect_ok(db, pinned.id, metric_id):
if not _metric_sample_ok(db, pinned, metric_id):
return None
return pinned
batch = _latest_success_batch(db, tid) if tid else None
if batch and not batch_metric_collect_ok(db, batch.id, metric_id):
batch = (
db.query(BizStateBatch)
.filter(BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued")))
.order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first()
) if tid else None
if batch and not _metric_sample_ok(db, batch, metric_id):
return None
return batch
def _metric_sample_ok(db: Session, batch: BizStateBatch, metric_id: str) -> bool:
"""An empty result requires a successful command receipt, just like a full one.
Batch success alone cannot prove this metric was collected. Unmatched and
skipped parsers are uncertainty, never evidence that a business disappeared.
"""
if batch.status not in ("success", "partial"):
return False
statuses = [str(s or "").lower() for (s,) in db.query(BizStateBatchCommand.parse_status)
.filter(BizStateBatchCommand.batch_id == batch.id,
BizStateBatchCommand.metric_id == metric_id).all()]
return bool(statuses) and all(s in ("ok", "ok_aux") for s in statuses)
def _sample_timing(proj: BizMigrationProject, mid: str, old: BizStateBatch,
new: BizStateBatch, *, check_age: bool) -> dict[str, Any]:
interval = max(60, _metric_interval_map(proj).get(mid) or proj.hf_interval_sec or 60)
old_at, new_at = old.ended_at or old.started_at, new.ended_at or new.started_at
age_limit, skew_limit = interval * 3, interval * 2
reasons: list[str] = []
if not old_at or not new_at:
reasons.append("sample_time_missing")
skew = None
else:
skew = abs((old_at - new_at).total_seconds())
if skew > skew_limit:
reasons.append("sample_skew")
if check_age and max((utcnow_naive() - old_at).total_seconds(),
(utcnow_naive() - new_at).total_seconds()) > age_limit:
reasons.append("sample_stale")
return {"old_collected_at": _dt_iso(old_at), "new_collected_at": _dt_iso(new_at),
"age_limit_sec": age_limit, "skew_limit_sec": skew_limit,
"skew_sec": skew, "sample_issues": reasons}
def find_portrait_task_for_ne(db: Session, *, source: str, ne_id: str) -> BizStateTask | None:
"""Best portrait (non-HF) biz_state task for an NE — for baseline binding."""
src = str(source or "managed").strip().lower() or "managed"
@ -655,6 +694,14 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st
p = db.get(BizMigrationProject, project_id)
if not p:
raise HTTPException(status_code=404, detail="project_not_found")
# Validate incoming baseline ownership before mutating any project field.
for side in ("old", "new"):
bid = str(body.get(f"{side}_baseline_batch_id", getattr(p, f"{side}_baseline_batch_id")) or "").strip()
tid = str(body.get(f"{side}_task_id", getattr(p, f"{side}_task_id")) or "").strip()
if bid:
sample = db.get(BizStateBatch, bid)
if not sample or sample.task_id != tid or sample.status not in ("success", "partial"):
raise HTTPException(status_code=400, detail=f"{side}_baseline_invalid")
if "name" in body and body["name"] is not None:
p.name = str(body["name"]).strip() or p.name
if "note" in body and body["note"] is not None:
@ -768,11 +815,13 @@ def create_batch(db: Session, project_id: str, body: dict[str, Any]) -> dict[str
raise HTTPException(status_code=404, detail="project_not_found")
label = str(body.get("batch_label") or "").strip() or "batch"
expect = body.get("expect_set") if isinstance(body.get("expect_set"), dict) else {}
status = str(body.get("status") or "pending").strip()
_validate_batch_status(db, project_id, "", status)
b = BizMigrationBatch(
id=uuid4().hex,
project_id=project_id,
batch_label=label,
status=str(body.get("status") or "pending").strip() or "pending",
status=status,
expect_set_json=dict(expect or {}),
note=str(body.get("note") or "")[:500],
)
@ -819,17 +868,35 @@ def run_evaluate(
new_batch_id: str = "",
acceptance: bool = False,
purpose: str = "manual",
_commit: bool = True,
) -> dict[str, Any]:
mb = db.get(BizMigrationBatch, batch_id)
if not mb:
raise HTTPException(status_code=404, detail="batch_not_found")
proj = db.get(BizMigrationProject, mb.project_id)
# PostgreSQL serializes evaluate/finish for this project across API workers.
proj = db.query(BizMigrationProject).filter(BizMigrationProject.id == mb.project_id).with_for_update().populate_existing().first()
if not proj:
raise HTTPException(status_code=404, detail="project_not_found")
db.refresh(mb)
if acceptance and mb.status not in ("active", "review"):
raise HTTPException(status_code=409, detail="monitor_window_closed")
if purpose == "auto" and (mb.status != "active" or proj.status == "done"):
raise HTTPException(status_code=409, detail="monitor_window_closed")
if not proj.old_baseline_batch_id:
raise HTTPException(status_code=400, detail="old_baseline_required")
if not proj.new_baseline_batch_id:
raise HTTPException(status_code=400, detail="new_baseline_required")
for side in ("old", "new"):
baseline = db.get(BizStateBatch, getattr(proj, f"{side}_baseline_batch_id"))
if not baseline or baseline.task_id != portrait_task_id(proj, side):
raise HTTPException(status_code=400, detail=f"{side}_baseline_task_mismatch")
if baseline.status not in ("success", "partial"):
raise HTTPException(status_code=400, detail=f"{side}_baseline_not_complete")
pin = (old_batch_id if side == "old" else new_batch_id).strip()
if pin:
pinned = db.get(BizStateBatch, pin)
if not pinned or pinned.task_id not in _all_hf_task_ids(proj, side):
raise HTTPException(status_code=400, detail=f"{side}_current_batch_task_mismatch")
old_batch = db.get(BizStateBatch, old_batch_id.strip()) if old_batch_id.strip() else None
new_batch = db.get(BizStateBatch, new_batch_id.strip()) if new_batch_id.strip() else None
@ -854,11 +921,41 @@ def run_evaluate(
pin_new = new_batch_id.strip()
port_map = _port_map_dict(db, proj.mapping_id)
if proj.mapping_id and not db.get(BizPortMapping, proj.mapping_id):
raise HTTPException(status_code=400, detail="mapping_missing")
if len(set(port_map.values())) != len(port_map):
raise HTTPException(status_code=400, detail="mapping_target_ambiguous")
expect = parse_expect_set(mb.expect_set_json if isinstance(mb.expect_set_json, dict) else {})
previous_expect: dict[str, set[str]] = {}
for prior in db.query(BizMigrationBatch).filter(
BizMigrationBatch.project_id == proj.id, BizMigrationBatch.id != mb.id,
BizMigrationBatch.status.in_(("review", "done")),
BizMigrationBatch.created_at <= mb.created_at,
).all():
for mid, keys in parse_expect_set(prior.expect_set_json).items():
previous_expect.setdefault(mid, set()).update(keys)
# Final acceptance: window closed → unfinished expect = red
window_active = (mb.status == "active") and (not acceptance)
mt = resolve_project_monitor_template(db, proj)
# Evaluations never seed or commit default configuration inside the lock.
mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None
if not mt:
raise HTTPException(status_code=400, detail="monitor_template_missing")
if mt.compare_template_id and not db.get(BizCompareTemplate, mt.compare_template_id):
raise HTTPException(status_code=400, detail="compare_template_missing")
sheets, sheet_overrides, defaults, iface_norm = resolve_evaluate_sheets(db, mt)
from .validation import validate_monitor_rules
validate_monitor_rules(defaults, sheet_overrides)
from ..biz_state.compare_validation import validate_template_body
validate_template_body({"metrics": sheets, "iface_normalize_rules": iface_norm})
known_sheets = {sheet_key(s) for s in sheets}
known_metrics = {str(s.get("metric_id") or "") for s in sheets}
for override in sheet_overrides:
if (override.get("sheet_id") and override["sheet_id"] not in known_sheets) or (
not override.get("sheet_id") and override.get("metric_id") not in known_metrics
):
raise HTTPException(status_code=400, detail="monitor_override_sheet_missing")
out_of_expect = str((defaults or {}).get("out_of_expect") or "strict").strip().lower()
collect_ids = set(resolve_collect_metric_ids(db, proj))
@ -867,6 +964,16 @@ def run_evaluate(
metric_batches: dict[str, dict[str, str]] = {}
seq = 0
verdict_counts: dict[str, int] = {}
rows_cache: dict[tuple[str, str], list[dict[str, Any]]] = {}
command_cache: dict[str, dict[str, Any]] = {}
scope_ids = {str(s.get("metric_id") or "") for s in sheets} | {sheet_key(s) for s in sheets} | {"_ports"}
uncovered_scope = sorted(k for k, values in expect.items() if values and k not in scope_ids)
def metric_rows(bid: str, mid: str) -> list[dict[str, Any]]:
key = (bid, mid)
if key not in rows_cache:
rows_cache[key] = _load_metric_rows(db, batch_id=bid, metric_id=mid)
return rows_cache[key]
for sheet in sheets:
mid = str(sheet.get("metric_id") or "").strip()
@ -880,6 +987,9 @@ def run_evaluate(
field_rules = list(sheet.get("field_rules") or [])
sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid)
if sheet_ov.get("skip_dual"):
sheet_cards.append({"metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet),
"progress_ok": 0, "progress_total": 0, "anomaly": 0,
"collect_skipped": True})
continue
# Metric not in HF collect set → skip dual eval (avoid false red)
@ -927,21 +1037,31 @@ def run_evaluate(
continue
old_cur_mid = old_cur_batch.id
new_cur_mid = new_cur_batch.id
timing = _sample_timing(proj, mid, old_cur_batch, new_cur_batch,
check_age=acceptance or not (pin_old and pin_new))
baseline_missing = [side for side in ("old", "new") if not _metric_sample_ok(
db, db.get(BizStateBatch, getattr(proj, f"{side}_baseline_batch_id")), mid)]
if timing["sample_issues"] or baseline_missing:
sheet_cards.append({"metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet),
"progress_ok": 0, "progress_total": 0, "anomaly": 0,
"current_missing": True, "collect_incomplete": True,
"baseline_missing_sides": baseline_missing, **timing})
continue
old_base = apply_row_filters(
_load_metric_rows(db, batch_id=proj.old_baseline_batch_id, metric_id=mid),
metric_rows(proj.old_baseline_batch_id, mid),
row_filters,
)
old_now = apply_row_filters(
_load_metric_rows(db, batch_id=old_cur_mid, metric_id=mid),
metric_rows(old_cur_mid, mid),
row_filters,
)
new_base_rows = apply_row_filters(
_load_metric_rows(db, batch_id=proj.new_baseline_batch_id, metric_id=mid),
metric_rows(proj.new_baseline_batch_id, mid),
row_filters,
)
new_now = apply_row_filters(
_load_metric_rows(db, batch_id=new_cur_mid, metric_id=mid),
metric_rows(new_cur_mid, mid),
row_filters,
)
@ -963,7 +1083,19 @@ def run_evaluate(
sheet_override=sheet_ov,
out_of_expect=out_of_expect,
iface_normalize_rules=iface_norm,
previous_expect=previous_expect,
allow_mapping_scope=False,
)
# A target snapshot taken before the source's disappearance cannot prove
# loss. Wait for the next target receipt; final acceptance stays blocked.
awaiting = 0
if (new_cur_batch.ended_at or new_cur_batch.started_at) < old_cur_batch.started_at:
for row in one["rows"]:
if row["in_expect"] and row["rule_hit"] == "service_not_received":
row.update(verdict="awaiting_peer", color="gray", rule_hit="target_sample_predates_source")
one["anomaly"] -= 1
one["anomaly_in_expect"] -= 1
awaiting += 1
old_tid = _hf_task_id_for_metric(proj, "old", mid) or current_task_id(proj, "old")
new_tid = _hf_task_id_for_metric(proj, "new", mid) or current_task_id(proj, "new")
old_task_brief = _task_brief(db, old_tid)
@ -973,6 +1105,7 @@ def run_evaluate(
"new_batch_id": new_cur_mid,
"old_task_id": old_tid,
"new_task_id": new_tid,
**timing,
}
sheet_cards.append(
{
@ -993,6 +1126,8 @@ def run_evaluate(
"duplicate_key_list": list(one.get("duplicate_key_list") or []),
"duplicate_keys_before": int(one.get("duplicate_keys_before") or 0),
"duplicate_keys_after": int(one.get("duplicate_keys_after") or 0),
**timing,
"awaiting_peer": awaiting,
}
)
for r in one["rows"]:
@ -1003,6 +1138,7 @@ def run_evaluate(
new_batch_id=new_cur_mid,
old_task=old_task_brief,
new_task=new_task_brief,
command_cache=command_cache,
)
r["seq"] = seq
r["sheet_id"] = sid
@ -1052,6 +1188,13 @@ def run_evaluate(
if c.get("collect_skipped")
],
"verdict_counts": verdict_counts,
"anomaly_outside_expect": sum(int(c.get("anomaly") or 0) - int(c.get("anomaly_in_expect") or 0) for c in active_cards),
"duplicate_keys": sum(int(c.get("duplicate_keys_before") or 0) + int(c.get("duplicate_keys_after") or 0) for c in active_cards),
"uncovered_scope": uncovered_scope,
"coverage_complete": bool(sheet_cards) and not uncovered_scope and not any(c.get("collect_skipped") or c.get("current_missing") or c.get("awaiting_peer") or c.get("duplicate_keys_before") or c.get("duplicate_keys_after") for c in sheet_cards),
"config_snapshot": {"sheets": sheets, "sheet_overrides": sheet_overrides,
"defaults": defaults, "iface_normalize": iface_norm,
"port_map": port_map, "expect_set": mb.expect_set_json},
"window_active": window_active,
"expect_ports": sorted(expect.get("_ports") or ()),
"old_current_task_id": current_task_id(proj, "old"),
@ -1098,6 +1241,8 @@ def run_evaluate(
"old_status": r.get("old_status"),
"new_status": r.get("new_status"),
"rule_hit": r.get("rule_hit") or "",
"integrity_changes": r.get("integrity_changes") or [],
"previous_batch": bool(r.get("previous_batch")),
"sheet_id": r.get("sheet_id") or "",
"evidence": r.get("evidence") or {},
},
@ -1109,8 +1254,10 @@ def run_evaluate(
search_text=search[:2000],
)
)
db.commit()
db.refresh(run)
db.flush()
if _commit:
db.commit()
db.refresh(run)
return run_to_dict(db, run)
@ -1147,7 +1294,12 @@ def _command_brief(db: Session, command_id: str) -> dict[str, Any]:
cid = str(command_id or "").strip()
if not cid:
return {}
c = db.get(BizStateBatchCommand, cid)
# Select metadata only: raw CLI output can be several MB per command.
c = db.query(BizStateBatchCommand.id, BizStateBatchCommand.raw_command,
BizStateBatchCommand.parse_status, BizStateBatchCommand.row_count,
BizStateBatchCommand.profile_id, BizStateBatchCommand.parser_id,
BizStateBatchCommand.metric_id, BizStateBatchCommand.message).filter(
BizStateBatchCommand.id == cid).first()
if not c:
return {"command_id": cid}
return {
@ -1170,6 +1322,7 @@ def _enrich_row_evidence(
new_batch_id: str,
old_task: dict[str, Any] | None,
new_task: dict[str, Any] | None,
command_cache: dict[str, dict[str, Any]] | None = None,
) -> None:
"""Attach device / collect / show-command onto evaluate evidence (in-place)."""
ev = dict(row.get("evidence") or {})
@ -1179,7 +1332,11 @@ def _enrich_row_evidence(
):
side_ev = dict(ev.get(side) or {})
netx = dict(side_ev.get("netx") or {})
cmd = _command_brief(db, str(netx.get("batch_command_id") or ""))
cid = str(netx.get("batch_command_id") or "")
cache = command_cache if command_cache is not None else {}
if cid not in cache:
cache[cid] = _command_brief(db, cid)
cmd = cache[cid]
side_ev["device"] = {
"ne_id": str(brief.get("ne_id") or netx.get("ne_id") or ""),
"ne_name": str(brief.get("ne_name") or ""),
@ -1221,6 +1378,8 @@ def diff_to_dict(d: BizMigrationDiff) -> dict[str, Any]:
"old_status": kj.get("old_status") or "",
"new_status": kj.get("new_status") or "",
"rule_hit": kj.get("rule_hit") or "",
"integrity_changes": kj.get("integrity_changes") or [],
"previous_batch": bool(kj.get("previous_batch")),
"evidence": dict(kj.get("evidence") or {}),
"old_kind": d.old_kind,
"new_kind": d.new_kind,
@ -1922,26 +2081,30 @@ def _persist_red_tickets_from_run(
)
.all()
)
by_id: dict[tuple[str, str], BizMigrationRedTicket] = {}
by_id: dict[tuple[str, str, str], BizMigrationRedTicket] = {}
for t in existing:
detail = t.detail_json if isinstance(t.detail_json, dict) else {}
mk = str(getattr(t, "match_key_str", "") or "").strip()
if not mk:
detail = t.detail_json if isinstance(t.detail_json, dict) else {}
mk = str(detail.get("match_old_key") or t.key_str or "").strip()
if mk:
by_id[(str(t.metric_id or ""), mk)] = t
by_id[(str(t.metric_id or ""), str(detail.get("sheet_id") or t.metric_id or ""),
str(detail.get("match_old_key") or mk))] = t
created_or_updated: list[BizMigrationRedTicket] = []
seen: set[tuple[str, str]] = set()
seen: set[tuple[str, str, str]] = set()
for d in diffs:
kj = d.key_json if isinstance(d.key_json, dict) else {}
mid = str(d.metric_id or "")
match_key = str(
kj.get("match_old_key") or kj.get("key_str") or kj.get("old_key") or ""
).strip()[:256]
).strip()
if not match_key:
match_key = str(kj.get("key_str") or "")[:256]
ident = (mid, match_key)
ident = (mid, str(kj.get("sheet_id") or mid), match_key)
from hashlib import sha256
stored_key = match_key if len(match_key) <= 256 else "sha256:" + sha256(match_key.encode()).hexdigest()
detail = {
"old_kind": d.old_kind,
"new_kind": d.new_kind,
@ -1952,6 +2115,8 @@ def _persist_red_tickets_from_run(
"new_key": kj.get("new_key") or kj.get("new_key_str") or "",
"match_old_key": kj.get("match_old_key") or match_key,
"match_new_key": kj.get("match_new_key") or "",
"sheet_id": kj.get("sheet_id") or mid,
"integrity_changes": kj.get("integrity_changes") or [],
"evidence": dict(kj.get("evidence") or {}),
}
prev = by_id.get(ident)
@ -1960,7 +2125,7 @@ def _persist_red_tickets_from_run(
prev.run_id = run_id
prev.key_str = str(kj.get("key_str") or "")[:256]
prev.new_key_str = str(kj.get("new_key_str") or "")[:256]
prev.match_key_str = match_key
prev.match_key_str = stored_key
prev.verdict = d.verdict
prev.color = d.color or "red"
prev.old_status = str(kj.get("old_status") or "")[:64]
@ -1980,7 +2145,7 @@ def _persist_red_tickets_from_run(
metric_id=mid,
key_str=str(kj.get("key_str") or "")[:256],
new_key_str=str(kj.get("new_key_str") or "")[:256],
match_key_str=match_key,
match_key_str=stored_key,
verdict=d.verdict,
color=d.color or "red",
old_status=str(kj.get("old_status") or "")[:64],
@ -2014,14 +2179,9 @@ def finish_batch(
detail=f"batch_not_finishable:{mb.status}",
)
# Close window first so acceptance uses non-active rules
if mb.status == "active":
mb.ended_at = utcnow_naive()
mb.status = "done" if mark_done else "review"
mb.updated_at = utcnow_naive()
db.commit()
run = run_evaluate(db, batch_id=batch_id, acceptance=True, purpose="acceptance")
# acceptance=True closes the logical window. Persist batch closure only after
# evaluation succeeds, so an API/collection failure keeps the batch operable.
run = run_evaluate(db, batch_id=batch_id, acceptance=True, purpose="acceptance", _commit=False)
summary = dict(run.get("summary") or {})
progress = dict(summary.get("progress") or {})
anomaly = int(summary.get("anomaly") or 0)
@ -2036,6 +2196,10 @@ def finish_batch(
fail_reasons.append("new_baseline_missing")
if current_missing:
fail_reasons.append("current_missing")
if not summary.get("coverage_complete", False):
fail_reasons.append("coverage_incomplete")
if summary.get("duplicate_keys"):
fail_reasons.append("duplicate_keys")
if anomaly:
fail_reasons.append("anomaly")
if total > 0 and ok < total:
@ -2061,6 +2225,9 @@ def finish_batch(
mb = db.get(BizMigrationBatch, batch_id)
if not mb:
raise HTTPException(status_code=404, detail="batch_not_found")
if mb.status == "active":
mb.ended_at = utcnow_naive()
mb.status = "done" if mark_done else "review"
mb.accept_run_id = str(run.get("id") or "")
mb.accept_status = "passed" if passed else "failed"
mb.accept_summary_json = accept_summary
@ -2086,10 +2253,22 @@ def finish_batch(
}
def _validate_batch_status(db: Session, project_id: str, batch_id: str, status: str) -> None:
if status not in ("pending", "active", "review", "done"):
raise HTTPException(status_code=400, detail="batch_status_invalid")
if status == "active":
db.query(BizMigrationProject).filter(BizMigrationProject.id == project_id).with_for_update().first()
if db.query(BizMigrationBatch.id).filter(BizMigrationBatch.project_id == project_id,
BizMigrationBatch.id != batch_id, BizMigrationBatch.status == "active").first():
raise HTTPException(status_code=409, detail="another_batch_active")
def patch_batch(db: Session, batch_id: str, body: dict[str, Any]) -> dict[str, Any]:
b = db.get(BizMigrationBatch, batch_id)
if not b:
raise HTTPException(status_code=404, detail="batch_not_found")
if body.get("status") is not None:
_validate_batch_status(db, b.project_id, b.id, str(body["status"]).strip())
if "batch_label" in body and body["batch_label"] is not None:
b.batch_label = str(body["batch_label"]).strip() or b.batch_label
if "note" in body and body["note"] is not None:
@ -2135,6 +2314,8 @@ def list_run_diffs(
run_id: str,
*,
metric_id: str = "",
sheet_id: str = "",
only_expect: bool = False,
verdict: str = "",
color: str = "",
kw: str = "",
@ -2146,19 +2327,23 @@ def list_run_diffs(
q = db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id == run_id)
if metric_id:
q = q.filter(BizMigrationDiff.metric_id == metric_id)
if sheet_id:
q = q.filter(BizMigrationDiff.key_json["sheet_id"].as_string() == sheet_id)
if only_expect:
q = q.filter(BizMigrationDiff.in_expect.is_(True))
if verdict:
q = q.filter(BizMigrationDiff.verdict == verdict)
if color:
q = q.filter(BizMigrationDiff.color == color)
if kw.strip():
like = f"%{kw.strip()}%"
q = q.filter(BizMigrationDiff.search_text.ilike(like))
literal = kw.strip().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
q = q.filter(BizMigrationDiff.search_text.ilike(f"%{literal}%", escape="\\"))
total = q.count()
rows = q.order_by(BizMigrationDiff.seq.asc()).offset(max(0, offset)).limit(min(500, max(1, limit))).all()
return {
"total": total,
"offset": offset,
"limit": limit,
"offset": max(0, offset),
"limit": min(500, max(1, limit)),
"items": [diff_to_dict(d) for d in rows],
}

70
netx_api/biz_migration/validation.py Normal file
View file

@ -0,0 +1,70 @@
"""Validate monitor correction rules before they can affect live verdicts."""
from typing import Any
from fastapi import HTTPException
from .evaluate import _groups_for_side, _normalize_cond_type
def validate_monitor_rules(defaults: Any, overrides: Any) -> None:
def fail(path: str) -> None:
raise HTTPException(status_code=400, detail=f"monitor_rule_invalid:{path}")
if not isinstance(defaults, dict) or not isinstance(overrides, list):
fail("defaults/sheet_overrides")
if defaults.get("out_of_expect", "strict") not in ("strict", "warn", "ignore"):
fail("defaults.out_of_expect")
seen = set()
for i, ov in enumerate(overrides):
path = f"sheet_overrides[{i}]"
if not isinstance(ov, dict) or not (ov.get("sheet_id") or ov.get("metric_id")):
fail(path)
ident = ov.get("sheet_id") or ov.get("metric_id")
if ident in seen:
fail(path + ".duplicate")
seen.add(ident)
for key in ("status_fields", "up_values", "down_values"):
if key in ov and (not isinstance(ov[key], list) or any(not str(v).strip() for v in ov[key])):
fail(path + "." + key)
for key in ("success", "anomaly"):
patterns = ov.get(key, [])
if not isinstance(patterns, list):
fail(path + "." + key)
for j, pat in enumerate(patterns):
pp = f"{path}.{key}[{j}]"
if not isinstance(pat, dict):
fail(pp)
for side in ("old", "new"):
for attr in (side, side + "_conds", side + "_groups"):
if attr in pat and not isinstance(pat[attr], list):
fail(pp + "." + attr)
if any(isinstance(g, list) and not g for g in pat.get(side + "_groups", [])):
fail(pp + "." + side + "_groups.empty_group")
groups = _groups_for_side(pat, side)
if key == "success" and not groups:
fail(pp + "." + side + ".required")
for group in groups:
for cond in group:
cp = pp + "." + side
if isinstance(cond, str):
if cond not in ("added", "removed", "changed", "unchanged", "up", "down", "other", "none") and not (cond.startswith("field:") and cond.count(":") >= 2):
fail(cp + ".token")
elif isinstance(cond, dict):
typ = _normalize_cond_type(cond)
if typ not in ("presence", "status", "value"):
fail(cp + ".type")
val = cond.get("value", cond.get("kind", cond.get("status")))
if typ == "presence" and val not in ("added", "removed", "changed", "unchanged"):
fail(cp + ".presence")
if typ == "status" and val not in ("up", "down", "other", "none"):
fail(cp + ".status")
if typ == "value":
op = cond.get("op", "eq")
if op not in ("eq", "ne", "in", "not_in", "empty", "not_empty", "changed", "unchanged", "same", "diff"):
fail(cp + ".op")
if op in ("eq", "ne", "in", "not_in") and (val is None or val == "" or val == []):
fail(cp + ".value")
else:
fail(cp + ".condition")
if key == "anomaly" and not any(_groups_for_side(pat, s) for s in ("old", "new")):
fail(pp + ".empty")

4
netx_api/biz_migration_router.py
View file

@ -285,6 +285,8 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)):
def api_list_diffs(
run_id: str,
metric_id: str = "",
sheet_id: str = "",
only_expect: bool = False,
verdict: str = "",
color: str = "",
kw: str = "",
@ -296,6 +298,8 @@ def api_list_diffs(
db,
run_id,
metric_id=metric_id,
sheet_id=sheet_id,
only_expect=only_expect,
verdict=verdict,
color=color,
kw=kw,

8
netx_api/biz_state/collect_runner.py
View file

@ -1720,6 +1720,14 @@ def _finalize_batch_status(
batch.message = ""
status = str(batch.status or "")
db.commit()
# Failed/partial receipts also update cutover coverage; never keep an old
# success looking healthy after the latest collection has failed.
try:
from ..biz_migration.auto_monitor import schedule_auto_monitor_for_task
schedule_auto_monitor_for_task(task_id)
except Exception:
_log.exception("cutover monitor schedule failed task=%s", task_id)
# Only full success triggers auto compare; never block the collect thread.
if status == "success" and not stopped:
try: