Optimize cutover compare/monitor per the 2026-10-12 plan.

Correctness (A): normalize port-map names with the template rules (snapshot v2,
v1 replays unchanged); protect mappings/templates that are referenced; stop
dropping edited mapping text; accept pulled-but-not-shutdown ports as migrated;
validate baseline ownership on project create; one canonical key form; fix
built-in template tolerances and upgrade old packaged values in place.

Cross-vendor identity (B): single identity pipeline for every path, per-side and
per-vendor interface rules, value maps, filters after normalization, mapping
pre-flight for compare and cutover projects, SQL eligibility fixes.

Data quality (V): no_data / suspect_parse / not_collected sheet states, command
slice coverage, directional compare modes, value-sorted duplicate pairing,
before-side raw names in results and CSV.

Robustness (H): bounded auto-run history, background evaluate/finish with
polling, manual route tables out of the acceptance gate, baseline row cache,
scoped duplicate-key failures, merged HF tasks with a per-NE session cap,
same-device cutover, explicit scope_by_port_map, read-only GETs, resolvable red
tickets.

Frontend: pre-flight report, mapping table view, template editor support for the
new rules, data-state banners. Docs: canonical field contract.

Not included: Huawei status parsers (no real samples), SQL-path route compare.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

netx_api/biz_migration/auto_monitor.py

@@ -44,6 +44,11 @@ def try_auto_monitor_for_task(db: Session, task_id: str) -> int:
         if not batch or proj.status == "done":
             db.rollback()
             continue
+        if db.query(BizMigrationRun.id).filter(
+            BizMigrationRun.project_id == proj.id, BizMigrationRun.status == "running"
+        ).first():
+            db.rollback()  # a manual / acceptance evaluation owns the project right now
+            continue
         mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None
         if not mt:
             db.rollback()
@@ -60,7 +65,8 @@ def try_auto_monitor_for_task(db: Session, task_id: str) -> int:
                    batch.expect_set_json, service._port_map_dict(db, proj.mapping_id),
                    sheets, overrides, defaults, norms, service.resolve_collect_metric_ids(db, proj)]
         fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
-        last = db.query(BizMigrationRun).filter(BizMigrationRun.batch_id == batch.id).order_by(
+        last = db.query(BizMigrationRun).filter(
+            BizMigrationRun.batch_id == batch.id, BizMigrationRun.status == "success").order_by(
             BizMigrationRun.created_at.desc()).first()
         if last and (last.summary_json or {}).get("auto_fingerprint") == fingerprint:
             db.rollback()

netx_api/biz_migration/evaluate.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import Any
 
-from ..biz_state.compare_engine import apply_port_map, compare_rows
+from ..biz_state.compare_engine import apply_port_map, compare_rows, key_tuple
 from ..biz_state.compare_rules import (
     effective_compare_fields, field_rule_map, scalar_text, values_equal,
 )
@@ -756,10 +756,10 @@ def build_diff_index_from_compare(result: dict[str, Any]) -> dict[str, dict[str,
             continue
         key = d.get("key")
         if isinstance(key, dict):
-            key_list = [str(v) for v in key.values()]
+            key_list = [scalar_text(v).strip() for v in key.values()]
             ks = _key_str(key_list)
         elif isinstance(key, (list, tuple)):
-            key_list = [str(x) for x in key]
+            key_list = [scalar_text(x).strip() for x in key]
             ks = _key_str(key_list)
         else:
             ks = str(key or "")
@@ -801,17 +801,18 @@ def expect_keys_for_metric(
     return keys
 
 
-def _map_defines_iface_expect(metric_id: str, iface_fields: list[str], port_map: dict[str, str]) -> bool:
-    """Iface sheets: a non-empty port map is the expect scope.
+def _map_defines_iface_expect(
+    iface_fields: list[str], port_map: dict[str, str], scope_by_port_map: bool | None = None,
+) -> bool:
+    """Whether a non-empty port map is this sheet's expect scope.
 
-    BGP / PW keep the batch expect picker even if a column is marked iface.
+    The sheet override decides (``scope_by_port_map``); the built-in presets set it to
+    false for BGP / pseudowire sheets, which keep the batch expect picker. Unset means
+    "an interface sheet with a map is scoped by the map".
     """
     if not port_map or not iface_fields:
         return False
-    mid = str(metric_id or "").strip().lower()
-    if "bgp" in mid or "pw" in mid or "pseudowire" in mid:
-        return False
-    return True
+    return True if scope_by_port_map is None else bool(scope_by_port_map)
 
 
 def _iface_values(key_str: str, *, key_fields: list[str], iface_fields: list[str]) -> list[str]:
@@ -1022,11 +1023,14 @@ def row_match_key(
     key_fields: list[str],
     iface_fields: list[str],
     rules: list[dict[str, str]] | None,
+    normalizer: Any = None,
 ) -> str:
     from ..biz_state.iface_normalize import apply_iface_normalize
 
     data = strip_netx(row)
-    if iface_fields and rules:
+    if normalizer is not None:
+        data = normalizer(dict(data))
+    elif iface_fields and rules:
         data = apply_iface_normalize(dict(data), iface_fields=iface_fields, rules=rules)
     return KEY_SEP.join(scalar_text(data.get(k)).strip() for k in key_fields)
 
@@ -1037,11 +1041,14 @@ def index_raw_by_match_key(
     key_fields: list[str],
     iface_fields: list[str],
     rules: list[dict[str, str]] | None,
+    normalizer: Any = None,
 ) -> dict[str, dict[str, Any]]:
     """First raw row per internal match key (normalized iface, raw elsewhere)."""
     idx: dict[str, dict[str, Any]] = {}
     for raw in raw_rows:
-        ks = row_match_key(raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules)
+        ks = row_match_key(
+            raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules, normalizer=normalizer,
+        )
         if ks and ks not in idx:
             idx[ks] = raw
     return idx
@@ -1062,6 +1069,7 @@ def iface_lineage(
     rules: list[dict[str, str]] | None,
     port_map: dict[str, str] | None,
     apply_map: bool,
+    namer: Any = None,
 ) -> list[dict[str, Any]]:
     """Per iface field: raw → normalized → (optional) mapped."""
     from ..biz_state.iface_normalize import normalize_iface_name, resolve_mapped_iface
@@ -1073,7 +1081,7 @@ def iface_lineage(
     out: list[dict[str, Any]] = []
     for f in iface_fields:
         raw_v = scalar_text(data.get(f)).strip()
-        norm_v = normalize_iface_name(raw_v, rules) if rules else raw_v
+        norm_v = namer(raw_v) if namer is not None else (normalize_iface_name(raw_v, rules) if rules else raw_v)
         mapped = False
         map_to = ""
         if apply_map and pmap and norm_v:
@@ -1113,6 +1121,7 @@ def evaluate_metric_dual(
     iface_normalize_rules: list[dict[str, str]] | None = None,
     previous_expect: dict[str, set[str]] | None = None,
     allow_mapping_scope: bool = True,
+    pipeline: Any = None,
 ) -> dict[str, Any]:
     """Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge.
 
@@ -1120,10 +1129,12 @@ def evaluate_metric_dual(
     Board / AI fields ``old_key`` / ``new_key`` and ``old`` / ``new`` stay **raw**
     device values (A/B), never the post-map BB form.
     """
-    from ..biz_state.iface_normalize import (
-        apply_iface_normalize_rows,
-        normalize_iface_rules,
-        normalize_iface_name,
+    from ..biz_state.identity_pipeline import legacy_pipeline
+
+    # Old device = "before" side, new device = "after" side of the identity pipeline.
+    pipe = pipeline or legacy_pipeline(
+        key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields,
+        field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules,
     )
 
     sid = str(sheet_id or "").strip() or str(metric_id or "").strip()
@@ -1159,48 +1170,44 @@ def evaluate_metric_dual(
         [dict(r) for r in new_baseline_rows] if new_baseline_rows is not None else None
     )
 
-    norm_rules = normalize_iface_rules(iface_normalize_rules)
     def normalized_key(key: str) -> str:
         parts = key.split(KEY_SEP) if KEY_SEP in key else ([key] if len(key_fields) == 1 else key.split("|"))
         if len(parts) != len(key_fields):
             return key
-        return KEY_SEP.join(normalize_iface_name(p, norm_rules) if f in iface_fields else p
+        return KEY_SEP.join(pipe.normalize_name(p, "before") if f in iface_fields else p
                             for p, f in zip(parts, key_fields))
 
+    def norm_before(row: dict[str, Any]) -> dict[str, Any]:
+        return pipe.normalize(row, "before")
+
+    def norm_after(row: dict[str, Any]) -> dict[str, Any]:
+        return pipe.normalize(row, "after")
+
     expect_keys = {normalized_key(key) for key in expect_keys}
     previous_keys = {normalized_key(key) for key in previous_keys}
-    old_baseline_rows = apply_iface_normalize_rows(
-        [strip_netx(r) for r in raw_old_base], iface_fields=iface_fields, rules=norm_rules
-    )
-    old_current_rows = apply_iface_normalize_rows(
-        [strip_netx(r) for r in raw_old_cur], iface_fields=iface_fields, rules=norm_rules
-    )
-    new_current_rows = apply_iface_normalize_rows(
-        [strip_netx(r) for r in raw_new_cur], iface_fields=iface_fields, rules=norm_rules
-    )
+    old_baseline_rows = [norm_before(strip_netx(r)) for r in raw_old_base]
+    old_current_rows = [norm_before(strip_netx(r)) for r in raw_old_cur]
+    new_current_rows = [norm_after(strip_netx(r)) for r in raw_new_cur]
     if raw_new_base is not None:
-        new_baseline_rows = apply_iface_normalize_rows(
-            [strip_netx(r) for r in raw_new_base],
-            iface_fields=iface_fields,
-            rules=norm_rules,
-        )
+        new_baseline_rows = [norm_after(strip_netx(r)) for r in raw_new_base]
     else:
         new_baseline_rows = None
 
     old_raw_cur_idx = index_raw_by_match_key(
-        raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
+        raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before
     )
     old_raw_base_idx = index_raw_by_match_key(
-        raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
+        raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before
     )
     new_raw_cur_idx = index_raw_by_match_key(
-        raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
+        raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_after
     )
     new_raw_base_idx = index_raw_by_match_key(
         raw_new_base or [],
         key_fields=key_fields,
         iface_fields=iface_fields,
-        rules=norm_rules,
+        rules=None,
+        normalizer=norm_after,
     )
 
     old_cmp = compare_rows(
@@ -1213,6 +1220,7 @@ def evaluate_metric_dual(
         field_rules=field_rules,
         iface_normalize_rules=None,  # already applied
         include_unchanged=True,
+        duplicate_pairing=pipe.duplicate_pairing,
     )
     old_idx = build_diff_index_from_compare(old_cmp)
 
@@ -1244,15 +1252,16 @@ def evaluate_metric_dual(
         port_map=None,
         field_rules=field_rules,
         include_unchanged=True,
+        duplicate_pairing=pipe.duplicate_pairing,
     )
     new_idx = build_diff_index_from_compare(new_cmp)
 
     rev_map = {v: k for k, v in port_map.items()}
-    map_scoped = _map_defines_iface_expect(metric_id, iface_fields, port_map)
+    map_scoped = _map_defines_iface_expect(iface_fields, port_map, ov.get("scope_by_port_map"))
     # Mapping describes identity; an explicit batch selection defines scope.
     # Port selection also covers business rows attached to those interfaces.
     for scope, keys in ((expect, expect_keys), (previous_expect or {}, previous_keys)):
-        ports = {normalize_iface_name(p, norm_rules) for p in scope.get("_ports") or set()}
+        ports = {pipe.normalize_name(p, "before") for p in scope.get("_ports") or set()}
         if ports and not keys and iface_fields:
             for ks, diff in old_idx.items():
                 row = diff.get("before") or diff.get("after") or {}
@@ -1278,6 +1287,21 @@ def evaluate_metric_dual(
                 scoped.add(canon)
         expect_keys = scoped
 
+    # Duplicate keys only block when they sit inside this wave's scope.
+    from collections import Counter
+
+    def _dup_keys(rows: list[dict[str, Any]]) -> set[str]:
+        return {KEY_SEP.join(k) for k, n in Counter(key_tuple(r, key_fields) for r in rows).items() if n > 1}
+
+    in_scope = expect_keys | previous_keys
+    new_dups = _dup_keys(new_current_rows) | _dup_keys(new_baseline_rows or [])
+    flagged = (
+        _dup_keys(old_baseline_rows) | _dup_keys(old_current_rows) | new_dups
+        | {_reverse_remap_key_str(k, key_fields=key_fields, iface_fields=iface_fields, rev_map=rev_map)
+           for k in new_dups}
+    )
+    duplicate_in_expect = sorted(flagged & in_scope)[:64]
+
     canon_keys: list[str] = []
     seen: set[str] = set()
 
@@ -1447,9 +1471,10 @@ def evaluate_metric_dual(
                 "iface": iface_lineage(
                     raw_old,
                     iface_fields=iface_fields,
-                    rules=norm_rules,
+                    rules=None,
                     port_map=port_map,
                     apply_map=True,
+                    namer=lambda v: pipe.normalize_name(v, "before"),
                 ),
             },
             "new": {
@@ -1457,9 +1482,10 @@ def evaluate_metric_dual(
                 "iface": iface_lineage(
                     raw_new,
                     iface_fields=iface_fields,
-                    rules=norm_rules,
+                    rules=None,
                     port_map=None,
                     apply_map=False,
+                    namer=lambda v: pipe.normalize_name(v, "after"),
                 ),
             },
             "port_map": {
@@ -1522,6 +1548,7 @@ def evaluate_metric_dual(
 
     return {
         "metric_id": metric_id,
+        "effective_scope_by_port_map": map_scoped,
         "old_summary": old_cmp.get("summary") or {},
         "new_summary": new_cmp.get("summary") or {},
         "progress_ok": progress_ok,
@@ -1536,6 +1563,7 @@ def evaluate_metric_dual(
         + int((new_cmp.get("summary") or {}).get("duplicate_keys_before") or 0),
         "duplicate_keys_after": int((old_cmp.get("summary") or {}).get("duplicate_keys_after") or 0)
         + int((new_cmp.get("summary") or {}).get("duplicate_keys_after") or 0),
+        "duplicate_in_expect": duplicate_in_expect,
         "duplicate_key_list": list(
             dict.fromkeys(
                 list((old_cmp.get("summary") or {}).get("duplicate_key_list") or [])

netx_api/biz_migration/monitor_templates.py

@@ -61,6 +61,18 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None
     return out
 
 
+def _has_legacy_port_and_group(override: dict[str, Any]) -> bool:
+    """True when a success rule's old side demands admin down AND phy down."""
+    legacy = _port_old_down_groups(legacy_and=True)[0]
+    for rule in override.get("success") or []:
+        if not isinstance(rule, dict):
+            continue
+        for group in rule.get("old_groups") or []:
+            if group == legacy:
+                return True
+    return False
+
+
 def validate_sheet_overrides(
     db: Session,
     *,
@@ -94,6 +106,9 @@ def validate_sheet_overrides(
     for ov in overrides:
         sid = str(ov.get("sheet_id") or "").strip()
         mid = str(ov.get("metric_id") or "").strip()
+        if _has_legacy_port_and_group(ov):
+            warnings.append({"sheet_id": sid, "metric_id": mid,
+                             "reason": "port_success_legacy_and_semantics"})
         if sid:
             if sid not in sheet_ids:
                 warnings.append(
@@ -228,9 +243,29 @@ def _success_stateful(
     }
 
 
-def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[str, Any]:
+_PORT_ADMIN_DOWN = {"type": "value", "field": "admin", "op": "in", "value": ["down"]}
+_PORT_PHY_DOWN = {"type": "value", "field": "phy", "op": "in", "value": ["down"]}
+
+
+def _port_old_down_groups(*, legacy_and: bool = False) -> list[list[dict[str, Any]]]:
+    """Old-side "port is out of service" groups (OR of groups, AND within one).
+
+    A fibre pulled without ``shutdown`` leaves admin up / phy down, so either
+    signal alone counts. ``legacy_and`` is the pre-A4 form requiring both.
+    """
+    if legacy_and:
+        return [[dict(_PORT_ADMIN_DOWN), dict(_PORT_PHY_DOWN)]]
+    return [[dict(_PORT_PHY_DOWN)], [dict(_PORT_ADMIN_DOWN)]]
+
+
+def preset_override_for_metric(
+    metric_id: str, *, sheet_id: str = "", legacy_port_and: bool = False,
+) -> dict[str, Any]:
     """Default dual-verdict overlay for a compare-template sheet.
 
+    ``legacy_port_and`` rebuilds the pre-A4 port preset (old side must be
+    admin down AND phy down) so stored copies can be recognised and upgraded.
+
     ``metric_id`` is the collected source table; ``sheet_id`` identifies a
     filtered split (e.g. bgp_peer.vpnv4). Presets key off the source metric.
     """
@@ -244,6 +279,8 @@ def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[st
     look = mid.split(".", 1)[0] if "." in mid else mid
 
     def _out(body: dict[str, Any]) -> dict[str, Any]:
+        if _scope_by_map_default_off(look):
+            body["scope_by_port_map"] = False
         body["metric_id"] = mid or look
         if sid and sid != (mid or look):
             body["sheet_id"] = sid
@@ -261,12 +298,7 @@ def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[st
                 "up_values": ["up"],
                 "success": [
                     _success_stateful(
-                        old_down_groups=[
-                            [
-                                {"type": "value", "field": "admin", "op": "in", "value": ["down"]},
-                                {"type": "value", "field": "phy", "op": "in", "value": ["down"]},
-                            ]
-                        ],
+                        old_down_groups=_port_old_down_groups(legacy_and=legacy_port_and),
                         new_up_conds=up,
                     )
                 ],
@@ -369,6 +401,8 @@ DEFAULT_MONITOR_TEMPLATE_NAME = "默认割接监控"
 def ensure_default_monitor_templates(db: Session) -> None:
     """Seed once when monitor-template table is empty — one template on the full status compare."""
     if db.query(BizMonitorTemplate.id).limit(1).first():
+        upgrade_legacy_port_overrides(db)
+        fill_scope_by_port_map(db)
         return
     cmp_svc.ensure_default_templates(db)
     zte = (
@@ -403,9 +437,68 @@ def ensure_default_monitor_templates(db: Session) -> None:
     db.commit()
 
 
-def default_port_monitor_template_id(db: Session) -> str:
-    """Ensure seeds exist and return the default monitor template id."""
-    ensure_default_monitor_templates(db)
+def _scope_by_map_default_off(name: str) -> bool:
+    """BGP / pseudowire sheets keep the batch expect picker; a port map does not scope them."""
+    low = str(name or "").lower()
+    return "bgp" in low or "pw" in low or "pseudowire" in low
+
+
+def fill_scope_by_port_map(db: Session) -> int:
+    """Make the old name-based default explicit on stored overrides (one-off, idempotent)."""
+    changed = 0
+    for row in db.query(BizMonitorTemplate).all():
+        out = []
+        touched = False
+        for ov in list(row.sheet_overrides_json or []):
+            if (isinstance(ov, dict) and "scope_by_port_map" not in ov
+                    and _scope_by_map_default_off(f"{ov.get('metric_id') or ''} {ov.get('sheet_id') or ''}")):
+                ov = {**ov, "scope_by_port_map": False}
+                touched = True
+            out.append(ov)
+        if touched:
+            row.sheet_overrides_json = out
+            row.updated_at = _utcnow()
+            changed += 1
+    if changed:
+        db.commit()
+    return changed
+
+
+def upgrade_legacy_port_overrides(db: Session) -> int:
+    """Rewrite stored port overrides that are exactly the pre-A4 preset (AND semantics).
+
+    Field-by-field equality only: anything a user customised is left alone and
+    surfaces ``port_success_legacy_and_semantics`` in ``override_warnings``.
+    """
+    changed = 0
+    for row in db.query(BizMonitorTemplate).all():
+        overrides = list(row.sheet_overrides_json or [])
+        out: list[Any] = []
+        touched = False
+        for ov in overrides:
+            if isinstance(ov, dict):
+                mid = str(ov.get("metric_id") or "").strip()
+                sid = str(ov.get("sheet_id") or "").strip()
+                if (mid or sid).split(".", 1)[0] == PORT_METRIC_ID and ov == preset_override_for_metric(
+                    mid, sheet_id=sid, legacy_port_and=True,
+                ):
+                    out.append(preset_override_for_metric(mid, sheet_id=sid))
+                    touched = True
+                    continue
+            out.append(ov)
+        if touched:
+            row.sheet_overrides_json = out
+            row.updated_at = _utcnow()
+            changed += 1
+    if changed:
+        db.commit()
+    return changed
+
+
+def default_port_monitor_template_id(db: Session, *, seed: bool = True) -> str:
+    """Return the default monitor template id, seeding the built-ins first unless ``seed=False``."""
+    if seed:
+        ensure_default_monitor_templates(db)
     for name in (
         DEFAULT_MONITOR_TEMPLATE_NAME,
         "ZTE 状态割接监控",

netx_api/biz_migration/validation.py

@@ -23,6 +23,8 @@ def validate_monitor_rules(defaults: Any, overrides: Any) -> None:
         if ident in seen:
             fail(path + ".duplicate")
         seen.add(ident)
+        if ov.get("scope_by_port_map") is not None and not isinstance(ov["scope_by_port_map"], bool):
+            fail(path + ".scope_by_port_map")
         for key in ("status_fields", "up_values", "down_values"):
             if key in ov and (not isinstance(ov[key], list) or any(not str(v).strip() for v in ov[key])):
                 fail(path + "." + key)