hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-14 19:50:53 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 4

Optimize cutover compare/monitor per the 2026-10-12 plan.

Browse source 
Correctness (A): normalize port-map names with the template rules (snapshot v2,
v1 replays unchanged); protect mappings/templates that are referenced; stop
dropping edited mapping text; accept pulled-but-not-shutdown ports as migrated;
validate baseline ownership on project create; one canonical key form; fix
built-in template tolerances and upgrade old packaged values in place.

Cross-vendor identity (B): single identity pipeline for every path, per-side and
per-vendor interface rules, value maps, filters after normalization, mapping
pre-flight for compare and cutover projects, SQL eligibility fixes.

Data quality (V): no_data / suspect_parse / not_collected sheet states, command
slice coverage, directional compare modes, value-sorted duplicate pairing,
before-side raw names in results and CSV.

Robustness (H): bounded auto-run history, background evaluate/finish with
polling, manual route tables out of the acceptance gate, baseline row cache,
scoped duplicate-key failures, merged HF tasks with a per-NE session cap,
same-device cutover, explicit scope_by_port_map, read-only GETs, resolvable red
tickets.

Frontend: pre-flight report, mapping table view, template editor support for the
new rules, data-state banners. Docs: canonical field contract.

Not included: Huawei status parsers (no real samples), SQL-path route compare.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
oliver 2026-10-12 11:51:55 +08:00
parent fff7d63b9b
commit 315539e0fd
59 changed files with 6483 additions and 487 deletions
Download patch file Download diff file Expand all files Collapse all files

8
netx_api/app_startup.py
View file

@ -170,6 +170,14 @@ def run_api_startup() -> None:
)
except Exception:
_log.exception("startup: biz compare recovery failed")
try:
from .biz_migration.service import recover_interrupted_evaluations_on_startup
n_eval = recover_interrupted_evaluations_on_startup(db)
if n_eval:
_log.info("startup: cancelled %s interrupted cutover evaluation(s)", n_eval)
except Exception:
_log.exception("startup: cutover evaluation recovery failed")
try:
from .port_traffic_migrate import backfill_port_traffic_series

8
netx_api/biz_migration/auto_monitor.py
View file

@ -44,6 +44,11 @@ def try_auto_monitor_for_task(db: Session, task_id: str) -> int:
if not batch or proj.status == "done":
db.rollback()
continue
if db.query(BizMigrationRun.id).filter(
BizMigrationRun.project_id == proj.id, BizMigrationRun.status == "running"
).first():
db.rollback() # a manual / acceptance evaluation owns the project right now
continue
mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None
if not mt:
db.rollback()
@ -60,7 +65,8 @@ def try_auto_monitor_for_task(db: Session, task_id: str) -> int:
batch.expect_set_json, service._port_map_dict(db, proj.mapping_id),
sheets, overrides, defaults, norms, service.resolve_collect_metric_ids(db, proj)]
fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
last = db.query(BizMigrationRun).filter(BizMigrationRun.batch_id == batch.id).order_by(
last = db.query(BizMigrationRun).filter(
BizMigrationRun.batch_id == batch.id, BizMigrationRun.status == "success").order_by(
BizMigrationRun.created_at.desc()).first()
if last and (last.summary_json or {}).get("auto_fingerprint") == fingerprint:
db.rollback()

110
netx_api/biz_migration/evaluate.py
View file

@ -4,7 +4,7 @@ from __future__ import annotations
from typing import Any
from ..biz_state.compare_engine import apply_port_map, compare_rows
from ..biz_state.compare_engine import apply_port_map, compare_rows, key_tuple
from ..biz_state.compare_rules import (
effective_compare_fields, field_rule_map, scalar_text, values_equal,
)
@ -756,10 +756,10 @@ def build_diff_index_from_compare(result: dict[str, Any]) -> dict[str, dict[str,
continue
key = d.get("key")
if isinstance(key, dict):
key_list = [str(v) for v in key.values()]
key_list = [scalar_text(v).strip() for v in key.values()]
ks = _key_str(key_list)
elif isinstance(key, (list, tuple)):
key_list = [str(x) for x in key]
key_list = [scalar_text(x).strip() for x in key]
ks = _key_str(key_list)
else:
ks = str(key or "")
@ -801,17 +801,18 @@ def expect_keys_for_metric(
return keys
def _map_defines_iface_expect(metric_id: str, iface_fields: list[str], port_map: dict[str, str]) -> bool:
"""Iface sheets: a non-empty port map is the expect scope.
def _map_defines_iface_expect(
iface_fields: list[str], port_map: dict[str, str], scope_by_port_map: bool | None = None,
) -> bool:
"""Whether a non-empty port map is this sheet's expect scope.
BGP / PW keep the batch expect picker even if a column is marked iface.
The sheet override decides (``scope_by_port_map``); the built-in presets set it to
false for BGP / pseudowire sheets, which keep the batch expect picker. Unset means
"an interface sheet with a map is scoped by the map".
"""
if not port_map or not iface_fields:
return False
mid = str(metric_id or "").strip().lower()
if "bgp" in mid or "pw" in mid or "pseudowire" in mid:
return False
return True
return True if scope_by_port_map is None else bool(scope_by_port_map)
def _iface_values(key_str: str, *, key_fields: list[str], iface_fields: list[str]) -> list[str]:
@ -1022,11 +1023,14 @@ def row_match_key(
key_fields: list[str],
iface_fields: list[str],
rules: list[dict[str, str]] | None,
normalizer: Any = None,
) -> str:
from ..biz_state.iface_normalize import apply_iface_normalize
data = strip_netx(row)
if iface_fields and rules:
if normalizer is not None:
data = normalizer(dict(data))
elif iface_fields and rules:
data = apply_iface_normalize(dict(data), iface_fields=iface_fields, rules=rules)
return KEY_SEP.join(scalar_text(data.get(k)).strip() for k in key_fields)
@ -1037,11 +1041,14 @@ def index_raw_by_match_key(
key_fields: list[str],
iface_fields: list[str],
rules: list[dict[str, str]] | None,
normalizer: Any = None,
) -> dict[str, dict[str, Any]]:
"""First raw row per internal match key (normalized iface, raw elsewhere)."""
idx: dict[str, dict[str, Any]] = {}
for raw in raw_rows:
ks = row_match_key(raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules)
ks = row_match_key(
raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules, normalizer=normalizer,
)
if ks and ks not in idx:
idx[ks] = raw
return idx
@ -1062,6 +1069,7 @@ def iface_lineage(
rules: list[dict[str, str]] | None,
port_map: dict[str, str] | None,
apply_map: bool,
namer: Any = None,
) -> list[dict[str, Any]]:
"""Per iface field: raw → normalized → (optional) mapped."""
from ..biz_state.iface_normalize import normalize_iface_name, resolve_mapped_iface
@ -1073,7 +1081,7 @@ def iface_lineage(
out: list[dict[str, Any]] = []
for f in iface_fields:
raw_v = scalar_text(data.get(f)).strip()
norm_v = normalize_iface_name(raw_v, rules) if rules else raw_v
norm_v = namer(raw_v) if namer is not None else (normalize_iface_name(raw_v, rules) if rules else raw_v)
mapped = False
map_to = ""
if apply_map and pmap and norm_v:
@ -1113,6 +1121,7 @@ def evaluate_metric_dual(
iface_normalize_rules: list[dict[str, str]] | None = None,
previous_expect: dict[str, set[str]] | None = None,
allow_mapping_scope: bool = True,
pipeline: Any = None,
) -> dict[str, Any]:
"""Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge.
@ -1120,10 +1129,12 @@ def evaluate_metric_dual(
Board / AI fields ``old_key`` / ``new_key`` and ``old`` / ``new`` stay **raw**
device values (A/B), never the post-map BB form.
"""
from ..biz_state.iface_normalize import (
apply_iface_normalize_rows,
normalize_iface_rules,
normalize_iface_name,
from ..biz_state.identity_pipeline import legacy_pipeline
# Old device = "before" side, new device = "after" side of the identity pipeline.
pipe = pipeline or legacy_pipeline(
key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields,
field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules,
)
sid = str(sheet_id or "").strip() or str(metric_id or "").strip()
@ -1159,48 +1170,44 @@ def evaluate_metric_dual(
[dict(r) for r in new_baseline_rows] if new_baseline_rows is not None else None
)
norm_rules = normalize_iface_rules(iface_normalize_rules)
def normalized_key(key: str) -> str:
parts = key.split(KEY_SEP) if KEY_SEP in key else ([key] if len(key_fields) == 1 else key.split("|"))
if len(parts) != len(key_fields):
return key
return KEY_SEP.join(normalize_iface_name(p, norm_rules) if f in iface_fields else p
return KEY_SEP.join(pipe.normalize_name(p, "before") if f in iface_fields else p
for p, f in zip(parts, key_fields))
def norm_before(row: dict[str, Any]) -> dict[str, Any]:
return pipe.normalize(row, "before")
def norm_after(row: dict[str, Any]) -> dict[str, Any]:
return pipe.normalize(row, "after")
expect_keys = {normalized_key(key) for key in expect_keys}
previous_keys = {normalized_key(key) for key in previous_keys}
old_baseline_rows = apply_iface_normalize_rows(
[strip_netx(r) for r in raw_old_base], iface_fields=iface_fields, rules=norm_rules
)
old_current_rows = apply_iface_normalize_rows(
[strip_netx(r) for r in raw_old_cur], iface_fields=iface_fields, rules=norm_rules
)
new_current_rows = apply_iface_normalize_rows(
[strip_netx(r) for r in raw_new_cur], iface_fields=iface_fields, rules=norm_rules
)
old_baseline_rows = [norm_before(strip_netx(r)) for r in raw_old_base]
old_current_rows = [norm_before(strip_netx(r)) for r in raw_old_cur]
new_current_rows = [norm_after(strip_netx(r)) for r in raw_new_cur]
if raw_new_base is not None:
new_baseline_rows = apply_iface_normalize_rows(
[strip_netx(r) for r in raw_new_base],
iface_fields=iface_fields,
rules=norm_rules,
)
new_baseline_rows = [norm_after(strip_netx(r)) for r in raw_new_base]
else:
new_baseline_rows = None
old_raw_cur_idx = index_raw_by_match_key(
raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before
)
old_raw_base_idx = index_raw_by_match_key(
raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before
)
new_raw_cur_idx = index_raw_by_match_key(
raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_after
)
new_raw_base_idx = index_raw_by_match_key(
raw_new_base or [],
key_fields=key_fields,
iface_fields=iface_fields,
rules=norm_rules,
rules=None,
normalizer=norm_after,
)
old_cmp = compare_rows(
@ -1213,6 +1220,7 @@ def evaluate_metric_dual(
field_rules=field_rules,
iface_normalize_rules=None, # already applied
include_unchanged=True,
duplicate_pairing=pipe.duplicate_pairing,
)
old_idx = build_diff_index_from_compare(old_cmp)
@ -1244,15 +1252,16 @@ def evaluate_metric_dual(
port_map=None,
field_rules=field_rules,
include_unchanged=True,
duplicate_pairing=pipe.duplicate_pairing,
)
new_idx = build_diff_index_from_compare(new_cmp)
rev_map = {v: k for k, v in port_map.items()}
map_scoped = _map_defines_iface_expect(metric_id, iface_fields, port_map)
map_scoped = _map_defines_iface_expect(iface_fields, port_map, ov.get("scope_by_port_map"))
# Mapping describes identity; an explicit batch selection defines scope.
# Port selection also covers business rows attached to those interfaces.
for scope, keys in ((expect, expect_keys), (previous_expect or {}, previous_keys)):
ports = {normalize_iface_name(p, norm_rules) for p in scope.get("_ports") or set()}
ports = {pipe.normalize_name(p, "before") for p in scope.get("_ports") or set()}
if ports and not keys and iface_fields:
for ks, diff in old_idx.items():
row = diff.get("before") or diff.get("after") or {}
@ -1278,6 +1287,21 @@ def evaluate_metric_dual(
scoped.add(canon)
expect_keys = scoped
# Duplicate keys only block when they sit inside this wave's scope.
from collections import Counter
def _dup_keys(rows: list[dict[str, Any]]) -> set[str]:
return {KEY_SEP.join(k) for k, n in Counter(key_tuple(r, key_fields) for r in rows).items() if n > 1}
in_scope = expect_keys | previous_keys
new_dups = _dup_keys(new_current_rows) | _dup_keys(new_baseline_rows or [])
flagged = (
_dup_keys(old_baseline_rows) | _dup_keys(old_current_rows) | new_dups
| {_reverse_remap_key_str(k, key_fields=key_fields, iface_fields=iface_fields, rev_map=rev_map)
for k in new_dups}
)
duplicate_in_expect = sorted(flagged & in_scope)[:64]
canon_keys: list[str] = []
seen: set[str] = set()
@ -1447,9 +1471,10 @@ def evaluate_metric_dual(
"iface": iface_lineage(
raw_old,
iface_fields=iface_fields,
rules=norm_rules,
rules=None,
port_map=port_map,
apply_map=True,
namer=lambda v: pipe.normalize_name(v, "before"),
),
},
"new": {
@ -1457,9 +1482,10 @@ def evaluate_metric_dual(
"iface": iface_lineage(
raw_new,
iface_fields=iface_fields,
rules=norm_rules,
rules=None,
port_map=None,
apply_map=False,
namer=lambda v: pipe.normalize_name(v, "after"),
),
},
"port_map": {
@ -1522,6 +1548,7 @@ def evaluate_metric_dual(
return {
"metric_id": metric_id,
"effective_scope_by_port_map": map_scoped,
"old_summary": old_cmp.get("summary") or {},
"new_summary": new_cmp.get("summary") or {},
"progress_ok": progress_ok,
@ -1536,6 +1563,7 @@ def evaluate_metric_dual(
+ int((new_cmp.get("summary") or {}).get("duplicate_keys_before") or 0),
"duplicate_keys_after": int((old_cmp.get("summary") or {}).get("duplicate_keys_after") or 0)
+ int((new_cmp.get("summary") or {}).get("duplicate_keys_after") or 0),
"duplicate_in_expect": duplicate_in_expect,
"duplicate_key_list": list(
dict.fromkeys(
list((old_cmp.get("summary") or {}).get("duplicate_key_list") or [])

113
netx_api/biz_migration/monitor_templates.py
View file

@ -61,6 +61,18 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None
return out
def _has_legacy_port_and_group(override: dict[str, Any]) -> bool:
"""True when a success rule's old side demands admin down AND phy down."""
legacy = _port_old_down_groups(legacy_and=True)[0]
for rule in override.get("success") or []:
if not isinstance(rule, dict):
continue
for group in rule.get("old_groups") or []:
if group == legacy:
return True
return False
def validate_sheet_overrides(
db: Session,
*,
@ -94,6 +106,9 @@ def validate_sheet_overrides(
for ov in overrides:
sid = str(ov.get("sheet_id") or "").strip()
mid = str(ov.get("metric_id") or "").strip()
if _has_legacy_port_and_group(ov):
warnings.append({"sheet_id": sid, "metric_id": mid,
"reason": "port_success_legacy_and_semantics"})
if sid:
if sid not in sheet_ids:
warnings.append(
@ -228,9 +243,29 @@ def _success_stateful(
}
def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[str, Any]:
_PORT_ADMIN_DOWN = {"type": "value", "field": "admin", "op": "in", "value": ["down"]}
_PORT_PHY_DOWN = {"type": "value", "field": "phy", "op": "in", "value": ["down"]}
def _port_old_down_groups(*, legacy_and: bool = False) -> list[list[dict[str, Any]]]:
"""Old-side "port is out of service" groups (OR of groups, AND within one).
A fibre pulled without ``shutdown`` leaves admin up / phy down, so either
signal alone counts. ``legacy_and`` is the pre-A4 form requiring both.
"""
if legacy_and:
return [[dict(_PORT_ADMIN_DOWN), dict(_PORT_PHY_DOWN)]]
return [[dict(_PORT_PHY_DOWN)], [dict(_PORT_ADMIN_DOWN)]]
def preset_override_for_metric(
metric_id: str, *, sheet_id: str = "", legacy_port_and: bool = False,
) -> dict[str, Any]:
"""Default dual-verdict overlay for a compare-template sheet.
``legacy_port_and`` rebuilds the pre-A4 port preset (old side must be
admin down AND phy down) so stored copies can be recognised and upgraded.
``metric_id`` is the collected source table; ``sheet_id`` identifies a
filtered split (e.g. bgp_peer.vpnv4). Presets key off the source metric.
"""
@ -244,6 +279,8 @@ def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[st
look = mid.split(".", 1)[0] if "." in mid else mid
def _out(body: dict[str, Any]) -> dict[str, Any]:
if _scope_by_map_default_off(look):
body["scope_by_port_map"] = False
body["metric_id"] = mid or look
if sid and sid != (mid or look):
body["sheet_id"] = sid
@ -261,12 +298,7 @@ def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[st
"up_values": ["up"],
"success": [
_success_stateful(
old_down_groups=[
[
{"type": "value", "field": "admin", "op": "in", "value": ["down"]},
{"type": "value", "field": "phy", "op": "in", "value": ["down"]},
]
],
old_down_groups=_port_old_down_groups(legacy_and=legacy_port_and),
new_up_conds=up,
)
],
@ -369,6 +401,8 @@ DEFAULT_MONITOR_TEMPLATE_NAME = "默认割接监控"
def ensure_default_monitor_templates(db: Session) -> None:
"""Seed once when monitor-template table is empty — one template on the full status compare."""
if db.query(BizMonitorTemplate.id).limit(1).first():
upgrade_legacy_port_overrides(db)
fill_scope_by_port_map(db)
return
cmp_svc.ensure_default_templates(db)
zte = (
@ -403,9 +437,68 @@ def ensure_default_monitor_templates(db: Session) -> None:
db.commit()
def default_port_monitor_template_id(db: Session) -> str:
"""Ensure seeds exist and return the default monitor template id."""
ensure_default_monitor_templates(db)
def _scope_by_map_default_off(name: str) -> bool:
"""BGP / pseudowire sheets keep the batch expect picker; a port map does not scope them."""
low = str(name or "").lower()
return "bgp" in low or "pw" in low or "pseudowire" in low
def fill_scope_by_port_map(db: Session) -> int:
"""Make the old name-based default explicit on stored overrides (one-off, idempotent)."""
changed = 0
for row in db.query(BizMonitorTemplate).all():
out = []
touched = False
for ov in list(row.sheet_overrides_json or []):
if (isinstance(ov, dict) and "scope_by_port_map" not in ov
and _scope_by_map_default_off(f"{ov.get('metric_id') or ''} {ov.get('sheet_id') or ''}")):
ov = {**ov, "scope_by_port_map": False}
touched = True
out.append(ov)
if touched:
row.sheet_overrides_json = out
row.updated_at = _utcnow()
changed += 1
if changed:
db.commit()
return changed
def upgrade_legacy_port_overrides(db: Session) -> int:
"""Rewrite stored port overrides that are exactly the pre-A4 preset (AND semantics).
Field-by-field equality only: anything a user customised is left alone and
surfaces ``port_success_legacy_and_semantics`` in ``override_warnings``.
"""
changed = 0
for row in db.query(BizMonitorTemplate).all():
overrides = list(row.sheet_overrides_json or [])
out: list[Any] = []
touched = False
for ov in overrides:
if isinstance(ov, dict):
mid = str(ov.get("metric_id") or "").strip()
sid = str(ov.get("sheet_id") or "").strip()
if (mid or sid).split(".", 1)[0] == PORT_METRIC_ID and ov == preset_override_for_metric(
mid, sheet_id=sid, legacy_port_and=True,
):
out.append(preset_override_for_metric(mid, sheet_id=sid))
touched = True
continue
out.append(ov)
if touched:
row.sheet_overrides_json = out
row.updated_at = _utcnow()
changed += 1
if changed:
db.commit()
return changed
def default_port_monitor_template_id(db: Session, *, seed: bool = True) -> str:
"""Return the default monitor template id, seeding the built-ins first unless ``seed=False``."""
if seed:
ensure_default_monitor_templates(db)
for name in (
DEFAULT_MONITOR_TEMPLATE_NAME,
"ZTE 状态割接监控",

715
netx_api/biz_migration/service.py
View file

File diff suppressed because it is too large Load diff

2
netx_api/biz_migration/validation.py
View file

@ -23,6 +23,8 @@ def validate_monitor_rules(defaults: Any, overrides: Any) -> None:
if ident in seen:
fail(path + ".duplicate")
seen.add(ident)
if ov.get("scope_by_port_map") is not None and not isinstance(ov["scope_by_port_map"], bool):
fail(path + ".scope_by_port_map")
for key in ("status_fields", "up_values", "down_values"):
if key in ov and (not isinstance(ov[key], list) or any(not str(v).strip() for v in ov[key])):
fail(path + "." + key)

20
netx_api/biz_migration_router.py
View file

@ -151,6 +151,12 @@ def api_baseline_ports(project_id: str, db: Session = Depends(get_db)):
return svc.list_baseline_ports(db, project_id)
@router.get("/projects/{project_id}/preflight")
@router.post("/projects/{project_id}/preflight")
def api_project_preflight(project_id: str, db: Session = Depends(get_db)):
return svc.preflight_project(db, project_id)
@router.get("/projects/{project_id}/baseline-expect")
def api_baseline_expect(project_id: str, db: Session = Depends(get_db)):
"""Multi-metric baseline keys for expect-set picking (from monitor template sheets)."""
@ -222,8 +228,13 @@ def api_finish_batch(
mark_done: bool = False,
db: Session = Depends(get_db),
):
"""本批完成：终验 + 小结 + 红单（允许带红继续）."""
return svc.finish_batch(db, batch_id, mark_done=mark_done)
"""本批完成：终验 + 小结 + 红单（允许带红继续）. Runs in the background; poll the run."""
return svc.enqueue_finish(db, batch_id, mark_done=mark_done)
@router.post("/projects/{project_id}/red-tickets/resolve-resolvable")
def api_resolve_resolvable(project_id: str, db: Session = Depends(get_db)):
return svc.resolve_resolvable_red_tickets(db, project_id)
@router.get("/projects/{project_id}/red-tickets")
@ -258,9 +269,10 @@ def api_patch_batch(batch_id: str, body: BatchPatchIn, db: Session = Depends(get
@router.post("/batches/{batch_id}/evaluate")
def api_evaluate(batch_id: str, body: EvaluateIn | None = None, db: Session = Depends(get_db)):
payload = body.model_dump() if body else {}
return svc.run_evaluate(
# Returns the ``running`` run at once; poll GET /runs/{id} until ``status`` settles.
return svc.enqueue_evaluate(
db,
batch_id=batch_id,
batch_id,
old_batch_id=str(payload.get("old_batch_id") or ""),
new_batch_id=str(payload.get("new_batch_id") or ""),
)

41
netx_api/biz_state/claim.py
View file

@ -2,6 +2,8 @@
from __future__ import annotations
from collections import defaultdict
import logging
from datetime import datetime, timedelta
from typing import Any
@ -25,6 +27,11 @@ def max_concurrent_tasks() -> int:
return max(1, int(getattr(settings, "biz_state_max_concurrent_tasks", 16) or 16))
def max_sessions_per_ne() -> int:
"""Concurrent SSH collections allowed per network element (>= 1)."""
return max(1, int(getattr(settings, "biz_state_max_sessions_per_ne", 2) or 2))
def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]:
"""Create a queued batch and mark task collect_running. Does not open SSH.
@ -135,15 +142,16 @@ def enqueue_collect_group(group_id: str, *, manual: bool = False,
BizStateTask.id).with_for_update().populate_existing().all()
result: dict[str, Any] = {"ok": False, "queued": False, "task_id": requested_task_id,
"collect_group_id": group_id}
if len(tasks) != 2 or len({t.ne_id for t in tasks}) != 2:
# One task = a single device used as both sides (same-device cutover); two tasks = a pair.
if len(tasks) not in (1, 2) or len({t.ne_id for t in tasks}) != len(tasks):
return {**result, "reason": "collect_pair_invalid"}
if max_concurrent_tasks() < 2:
if max_concurrent_tasks() < len(tasks):
return {**result, "reason": "collect_pair_capacity"}
from ..cli_budget import clamp_cli_workers
worker_cap = clamp_cli_workers(int(getattr(settings, "biz_state_worker_collect_threads", None)
or getattr(settings, "biz_state_dispatch_workers", 8) or 8))
if worker_cap < 2:
if worker_cap < len(tasks):
return {**result, "reason": "collect_pair_capacity"}
if any(t.collect_running for t in tasks):
return {**result, "reason": "collect_pair_busy"}
@ -230,13 +238,12 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
if slots <= 0:
return []
busy_nes = {
str(r[0] or "").strip()
for r in db.query(BizStateBatch.ne_id)
.filter(BizStateBatch.status == "running")
.all()
if str(r[0] or "").strip()
}
# Running collections per NE; an NE accepts new ones until it reaches its session cap.
per_ne_cap = max_sessions_per_ne()
ne_running: dict[str, int] = defaultdict(int)
for (running_ne,) in db.query(BizStateBatch.ne_id).filter(BizStateBatch.status == "running").all():
if str(running_ne or "").strip():
ne_running[str(running_ne).strip()] += 1
# Manual route rounds yield to fresh monitoring, but must eventually
# run even when a busy device always has regular metrics queued.
@ -266,14 +273,19 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
peers = db.query(BizStateBatch).filter(
BizStateBatch.collect_round_id == batch.collect_round_id,
).with_for_update(skip_locked=pg).all()
if len(peers) != 2 or any(p.status != "queued" for p in peers):
expected = db.query(BizStateTask.id).filter(
BizStateTask.collect_group_id == batch.collect_group_id).count() if batch.collect_group_id else 2
if len(peers) != max(1, expected) or any(p.status != "queued" for p in peers):
continue
group = peers
visited.update(p.id for p in group)
if len(group) > slots - len(claimed_rows):
continue
nes = {str(p.ne_id or "").strip() for p in group}
if nes & busy_nes:
wanted: dict[str, int] = defaultdict(int)
for p in group:
if str(p.ne_id or "").strip():
wanted[str(p.ne_id).strip()] += 1
if any(ne_running[ne] + n > per_ne_cap for ne, n in wanted.items()):
continue
now = _utcnow()
for member in group:
@ -285,7 +297,8 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
if task:
task.last_collect_started_at = now
claimed_rows.append(member)
busy_nes.update(nes - {""})
for ne, n in wanted.items():
ne_running[ne] += n
if not claimed_rows:
db.rollback()

103
netx_api/biz_state/compare_engine.py
View file

@ -93,8 +93,17 @@ def apply_port_map(
return out
def key_tuple(row: Mapping[str, Any], fields: Sequence[str]) -> tuple[str, ...]:
"""The one canonical match-key form: ``scalar_text(...).strip()`` per field.
Every place that builds, stores or looks up a key string must go through
this (``None`` → ``""``, ``True`` → ``"true"``, surrounding blanks trimmed).
"""
return tuple(scalar_text(row.get(k)).strip() for k in fields)
def row_key(row: dict[str, Any], key_fields: list[str]) -> tuple[str, ...]:
return tuple(scalar_text(row.get(k)).strip() for k in key_fields)
return key_tuple(row, key_fields)
def mapping_stats(
@ -109,8 +118,8 @@ def mapping_stats(
``hit_before``: map key appears as a normalized before iface, or as the
parent of a before subinterface (so main-port-only maps still validate).
Map keys are matched against **normalized** iface values (same pipeline as
compare). Enter mapping keys in normalized form.
Map keys are matched against **normalized** iface values; callers pass a
port map already normalized with the template iface rules.
"""
before_ifaces: set[str] = set()
after_ifaces: set[str] = set()
@ -156,7 +165,7 @@ def mapping_stats(
"miss_after": miss_after,
"unused_before_keys": unused,
"ok": not miss_before and not miss_after,
"hint": "map keys must match normalized iface names (post iface_normalize)",
"hint": "map keys/values are normalized with the template iface rules before matching",
}
@ -174,6 +183,10 @@ def compare_rows(
include_unchanged: bool = False,
unchanged_limit: int | None = None,
compact_unchanged: bool = False,
pipeline: Any = None,
match_keys_override: Sequence[str] | None = None,
prepared: tuple[list, list] | None = None,
duplicate_pairing: str = "ordered",
) -> dict[str, Any]:
"""Return summary + diffs list.
@ -200,6 +213,11 @@ def compare_rows(
- ``True``: force drop iface from match key when a non-empty candidate exists.
- ``False``: never drop iface from match key.
``pipeline`` (``identity_pipeline.Pipeline``) owns normalization, filtering,
port mapping and key building; when omitted a legacy pipeline is derived
from the loose arguments (callers have already filtered the rows).
``match_keys_override`` pins the match key (re-running a stored result).
``summary.duplicate`` is always 0. ``duplicate_keys_*`` count match keys
that appear more than once on a side (diagnostic only).
@ -208,32 +226,40 @@ def compare_rows(
"""
if not key_fields:
raise ValueError("key_fields required")
pmap = dict(port_map or {})
rules = field_rule_map(field_rules)
norm_rules = normalize_iface_rules(iface_normalize_rules)
iface_list = [str(f) for f in (iface_fields or []) if str(f).strip()]
if pipeline is None:
from .identity_pipeline import legacy_pipeline
pipeline = legacy_pipeline(
key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields,
field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules,
ignore_port_changes=ignore_port_changes, duplicate_pairing=duplicate_pairing,
)
pmap = dict(pipeline.port_map)
rules = field_rule_map(field_rules if field_rules is not None else pipeline.field_rules)
iface_list = list(pipeline.iface_fields)
iface_set = set(iface_list)
before_norm = apply_iface_normalize_rows(
before_rows, iface_fields=iface_list, rules=norm_rules
) if norm_rules and iface_list else before_rows
after_norm = apply_iface_normalize_rows(
after_rows, iface_fields=iface_list, rules=norm_rules
) if norm_rules and iface_list else after_rows
if prepared is not None:
before_pairs, after_pairs = prepared
else:
before_pairs = pipeline.prepare(before_rows, "before")
after_pairs = pipeline.prepare(after_rows, "after")
before_rows = [orig for orig, _ in before_pairs]
after_rows = [orig for orig, _ in after_pairs]
before_norm = [norm for _, norm in before_pairs]
after_norm = [norm for _, norm in after_pairs]
ignore_ports = False
# No map → optionally ignore port renames by dropping iface from match key.
if not pmap and iface_set:
candidate = [k for k in key_fields if k not in iface_set]
if not candidate:
match_keys = list(key_fields)
elif ignore_port_changes is True:
match_keys = candidate
ignore_ports = True
elif ignore_port_changes is False:
match_keys = list(key_fields)
else:
# Auto heuristic (legacy default)
if match_keys_override is not None:
match_keys = [str(k) for k in match_keys_override]
ignore_ports = match_keys != list(key_fields)
else:
# No map → optionally ignore port renames by dropping iface from match key.
policy = pipeline.ignore_port_changes
unique = None
if not pmap and iface_set and policy is None:
candidate = [k for k in key_fields if k not in iface_set]
def unique_keys(rows: list[dict[str, Any]]) -> bool:
seen: set[tuple[str, ...]] = set()
for row in rows:
@ -243,23 +269,20 @@ def compare_rows(
seen.add(key)
return True
if unique_keys(before_norm) and unique_keys(after_norm):
match_keys = candidate
ignore_ports = True
else:
match_keys = list(key_fields)
else:
match_keys = list(key_fields)
unique = bool(candidate) and unique_keys(before_norm) and unique_keys(after_norm)
match_keys = pipeline.effective_match_keys(unique_without_iface=unique)
ignore_ports = match_keys != list(key_fields)
before_groups: dict[tuple[str, ...], list[tuple[dict[str, Any], dict[str, Any]]]] = (
defaultdict(list)
)
after_groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list)
for orig, norm in zip(before_rows, before_norm):
mapped = apply_port_map(norm, iface_fields=iface_list, port_map=pmap) if iface_list else norm
before_groups[row_key(mapped, match_keys)].append((orig, mapped))
for r in after_norm:
after_groups[row_key(r, match_keys)].append(r)
mapped, k = pipeline.key_of(norm, "before", match_keys)
before_groups[k].append((orig, mapped))
for norm in after_norm:
after_groups[pipeline.key_of(norm, "after", match_keys)[1]].append(norm)
sort_pairs = pipeline.duplicate_pairing == "sorted"
diffs: list[dict[str, Any]] = []
added = removed = changed = unchanged = 0
@ -271,7 +294,7 @@ def compare_rows(
sample = _StratifiedSample(limit_n) if include_unchanged and limit_n is not None else None
def _key_obj(row: dict[str, Any]) -> dict[str, Any]:
return {f: row.get(f, "") for f in key_fields}
return dict(zip(key_fields, key_tuple(row, key_fields)))
def _row_id(row: dict[str, Any] | None) -> str:
if not isinstance(row, dict):
@ -325,6 +348,10 @@ def compare_rows(
multi_before_keys.append(k)
if len(a_list) > 1:
multi_after_keys.append(k)
if sort_pairs and (len(b_list) > 1 or len(a_list) > 1):
# Same-key rows pair by value, not by CLI output order (ECMP, multipath).
b_list = sorted(b_list, key=lambda pair: pipeline.pair_sort_key(pair[1]))
a_list = sorted(a_list, key=pipeline.pair_sort_key)
n = max(len(b_list), len(a_list))
for i in range(n):
if i >= len(b_list):

194
netx_api/biz_state/compare_preflight.py Normal file
View file

@ -0,0 +1,194 @@
"""Pre-flight check of a port mapping against the two batches it will be applied to.
The same function backs the compare "validate mapping" button and the cutover
project pre-flight, so both modes judge mapping coverage the same way. It answers
before anyone runs a comparison:
- which *before* interfaces are mapped (exact / by parent), unmapped but present on
the *after* side by the same name, or **suspect** (not mapped, not on *after*)
- which map keys / targets match nothing, which targets receive several sources
- whether each sheet's two sides were actually collected (``data_state``)
"""
from __future__ import annotations
from typing import Any, Callable, Mapping, Sequence
from sqlalchemy.orm import Session
from .compare_quality import assess_sheet, command_coverage, collect_quality, side_receipts
from .identity_pipeline import build_pipeline
from .iface_normalize import normalize_port_map, resolve_mapped_iface
LIST_CAP = 200
# Million-row tables are never loaded just to look at interface names.
HEAVY_METRICS = frozenset({"bgp_route", "ip_route", "ipv6_route", "l2vpn_mac", "evpn_mac", "interface_detail"})
RowsLoader = Callable[[str, Mapping[str, Any]], list[dict[str, Any]]]
def _parents(name: str) -> list[str]:
parts = name.split(".")
return [".".join(parts[:i]) for i in range(len(parts) - 1, 0, -1)]
def _classify(name: str, pmap: Mapping[str, str], after: set[str]) -> str:
if name in pmap:
return "mapped_exact"
if any(p in pmap for p in _parents(name)):
return "mapped_by_parent"
if name in after:
return "identity_same_name"
return "suspect_unmapped"
def _present(name: str, names: set[str]) -> bool:
"""``name`` exists, or is the parent of an existing sub-interface."""
if name in names:
return True
prefix = name + "."
return any(n.startswith(prefix) for n in names)
def _top_level(names: Sequence[str]) -> list[str]:
"""Drop sub-interfaces whose parent is already in the list (noise)."""
pool = set(names)
return sorted(n for n in pool if not any(p in pool for p in _parents(n)))
def sheet_mapping_coverage(
pipe: Any,
before_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]],
after_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]],
) -> dict[str, Any]:
"""Classify every *before* interface of a sheet against the map and the *after* side.
``before_pairs`` / ``after_pairs`` are ``pipeline.prepare`` outputs. The result
carries two private sets (``_after_ifaces``, ``_used_keys``) for aggregation.
"""
b_if = {str(n.get(f) or "").strip() for _, n in before_pairs for f in pipe.iface_fields}
a_if = {str(n.get(f) or "").strip() for _, n in after_pairs for f in pipe.iface_fields}
b_if.discard("")
a_if.discard("")
buckets: dict[str, list[str]] = {
"mapped_exact": [], "mapped_by_parent": [], "identity_same_name": [], "suspect_unmapped": [],
}
for name in sorted(b_if):
buckets[_classify(name, pipe.port_map, a_if)].append(name)
used = {k for name in b_if for k in pipe.port_map if name == k or name.startswith(k + ".")}
missing_targets = sorted({
resolve_mapped_iface(n, pipe.port_map) for n in buckets["mapped_exact"] + buckets["mapped_by_parent"]
if not _present(resolve_mapped_iface(n, pipe.port_map), a_if)
})
suspects = _top_level(buckets["suspect_unmapped"])
return {
"counts": {k: len(v) for k, v in buckets.items()},
"mapped_exact": buckets["mapped_exact"][:LIST_CAP],
"mapped_by_parent": buckets["mapped_by_parent"][:LIST_CAP],
"identity_same_name": buckets["identity_same_name"][:LIST_CAP],
"suspect_unmapped": suspects[:LIST_CAP],
"suspect_unmapped_total": len(suspects),
"mapped_target_missing_on_after": missing_targets[:LIST_CAP],
"_after_ifaces": a_if,
"_used_keys": used,
}
def preflight(
db: Session,
*,
sheets: Sequence[Mapping[str, Any]],
iface_rules: Sequence[Mapping[str, Any]] | None,
raw_port_map: Mapping[str, str] | None,
before_batch_id: str,
after_batch_id: str,
before_vendor: str = "",
after_vendor: str = "",
load_rows: RowsLoader,
) -> dict[str, Any]:
"""Build the pre-flight report. ``load_rows(side, sheet)`` returns raw rows."""
raw_map = dict(raw_port_map or {})
out_sheets: list[dict[str, Any]] = []
used_keys: set[str] = set()
after_all: set[str] = set()
totals = {"mapped": 0, "suspect": 0, "same_name": 0}
risk_reasons: list[str] = []
full_pmap: dict[str, str] = {}
conflicts: list[dict[str, str]] = []
for sheet in sheets:
sid = str(sheet.get("sheet_id") or sheet.get("metric_id") or "")
mid = str(sheet.get("metric_id") or "")
title = str(sheet.get("title") or sid)
pipe = build_pipeline(
sheet, iface_rules=iface_rules, port_map=raw_map,
before_vendor=before_vendor, after_vendor=after_vendor,
)
if not full_pmap and pipe.port_map:
full_pmap = dict(pipe.port_map)
heavy = mid in HEAVY_METRICS
card: dict[str, Any] = {"sheet_id": sid, "metric_id": mid, "title": title}
if heavy:
b_n = collect_quality(side_receipts(db, before_batch_id, mid))["row_count"]
a_n = collect_quality(side_receipts(db, after_batch_id, mid))["row_count"]
card.update(skipped="heavy_table", before_count=b_n, after_count=a_n)
else:
b_pairs = pipe.prepare(load_rows("before", sheet), "before")
a_pairs = pipe.prepare(load_rows("after", sheet), "after")
card.update(before_count=len(b_pairs), after_count=len(a_pairs))
if pipe.iface_fields:
cov = sheet_mapping_coverage(pipe, b_pairs, a_pairs)
after_all |= cov.pop("_after_ifaces")
used_keys |= cov.pop("_used_keys")
card.update(cov)
totals["mapped"] += cov["counts"]["mapped_exact"] + cov["counts"]["mapped_by_parent"]
totals["same_name"] += cov["counts"]["identity_same_name"]
totals["suspect"] += cov["suspect_unmapped_total"]
card.update(assess_sheet(
db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id,
before_rows=int(card.get("before_count") or 0), after_rows=int(card.get("after_count") or 0),
))
if card["data_state"] in ("not_collected", "suspect_parse"):
risk_reasons.append(f"{sid}:{card['data_state']}")
out_sheets.append(card)
if raw_map and not full_pmap:
full_pmap, conflicts = normalize_port_map(raw_map, iface_rules)
elif raw_map:
_, conflicts = normalize_port_map(raw_map, iface_rules)
sources: dict[str, list[str]] = {}
for k, v in full_pmap.items():
sources.setdefault(v, []).append(k)
n_to_1 = [{"target": t, "sources": sorted(src)} for t, src in sorted(sources.items()) if len(src) > 1]
unused = sorted(k for k in full_pmap if k not in used_keys)
targets_missing = sorted(v for v in set(full_pmap.values()) if after_all and not _present(v, after_all))
judged = totals["mapped"] + totals["suspect"]
coverage = {
"mapped": totals["mapped"], "suspect_unmapped": totals["suspect"],
"identity_same_name": totals["same_name"],
"rate": round(totals["mapped"] / judged * 100, 1) if judged else None,
}
mapping = {
"pairs": len(full_pmap), "conflicts": conflicts[:LIST_CAP],
"unused_map_keys": unused[:LIST_CAP], "unused_map_keys_total": len(unused),
"map_targets_missing_on_after": targets_missing[:LIST_CAP],
"map_targets_missing_on_after_total": len(targets_missing),
"n_to_1_targets": n_to_1[:LIST_CAP],
}
if totals["suspect"]:
risk_reasons.append("suspect_unmapped")
if unused:
risk_reasons.append("unused_map_keys")
if targets_missing:
risk_reasons.append("map_targets_missing_on_after")
if conflicts:
risk_reasons.append("map_conflicts")
return {
"verdict": "risk" if risk_reasons else "ready",
"risk_reasons": sorted(set(risk_reasons)),
"coverage": coverage,
"mapping": mapping,
"sheets": out_sheets,
"command_coverage": command_coverage(db, before_batch_id=before_batch_id, after_batch_id=after_batch_id),
}

198
netx_api/biz_state/compare_quality.py Normal file
View file

@ -0,0 +1,198 @@
"""Collection quality of a compare side: is a missing row a business loss or a data problem?
Compare results are only as good as the two batches behind them. This module
reads *receipts* (``BizStateBatchCommand``: counts and status, never ``raw_text``)
to tell apart:
- ``not_collected`` – a side never ran a command (or the slice) for the sheet
- ``suspect_parse`` – the device printed lines but the parser produced no rows
- ``no_data`` – both sides ran fine and are really empty
- ``ok`` – ordinary comparison
"""
from __future__ import annotations
from typing import Any, Mapping, Sequence
from sqlalchemy.orm import Session
from ..models import BizStateBatchCommand
from .compare_rules import scalar_text
# More than this many CLI lines with zero parsed rows is not "an empty table".
SUSPECT_RAW_LINES = 5
DATA_STATES = ("ok", "no_data", "suspect_parse", "not_collected")
UNJUDGED_STATES = ("no_data", "not_collected")
def _has_receipts(db: Session, batch_id: str) -> bool:
return bool(batch_id) and db.query(BizStateBatchCommand.id).filter(
BizStateBatchCommand.batch_id == batch_id).limit(1).first() is not None
def side_receipts(db: Session, batch_id: str, metric_id: str) -> list[dict[str, Any]]:
if not batch_id or not metric_id:
return []
rows = (
db.query(
BizStateBatchCommand.profile_id, BizStateBatchCommand.parse_status,
BizStateBatchCommand.row_count, BizStateBatchCommand.raw_line_count,
BizStateBatchCommand.declared_total,
)
.filter(BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id == metric_id)
.all()
)
return [
{
"profile_id": str(r[0] or ""), "parse_status": str(r[1] or ""),
"row_count": int(r[2] or 0), "raw_line_count": int(r[3] or 0),
"declared_total": int(r[4] or 0),
}
for r in rows
]
def collect_quality(receipts: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
statuses: dict[str, int] = {}
for r in receipts:
key = str(r.get("parse_status") or "unknown")
statuses[key] = statuses.get(key, 0) + 1
return {
"commands": len(receipts),
"parse_status": statuses,
"row_count": sum(int(r.get("row_count") or 0) for r in receipts),
"raw_line_count": sum(int(r.get("raw_line_count") or 0) for r in receipts),
"declared_total": sum(int(r.get("declared_total") or 0) for r in receipts),
"profile_ids": sorted({str(r.get("profile_id") or "") for r in receipts if r.get("profile_id")}),
}
def sheet_slice(sheet: Mapping[str, Any]) -> dict[str, str]:
"""``{field: value}`` the sheet pins with top-level ``eq`` row filters."""
out: dict[str, str] = {}
for f in sheet.get("row_filters") or []:
if not isinstance(f, Mapping) or "any" in f or "all" in f:
continue
if str(f.get("op") or "eq").strip().lower() not in ("eq", "==", "ci_eq"):
continue
field = str(f.get("field") or "").strip()
value = scalar_text(f.get("value")).strip().lower()
if field and value:
out[field] = value
return out
def covers_slice(
receipts: Sequence[Mapping[str, Any]], wanted: Mapping[str, str], *, profile_slice: Any = None,
) -> bool:
"""True when at least one successful command could have produced ``wanted``.
A command without a declared slice (or unknown profile) covers everything.
"""
if profile_slice is None:
from .profiles import get_profile
def profile_slice(pid: str) -> dict[str, str]: # noqa: F811
prof = get_profile(pid) if pid else None
return dict(prof.slice) if prof is not None and prof.slice else {}
for r in receipts:
if str(r.get("parse_status") or "") in ("failed", "skipped_custom"):
continue
declared = profile_slice(str(r.get("profile_id") or ""))
if all(
f not in declared or str(declared[f]).strip().lower() == v
for f, v in wanted.items()
):
return True
return False
def classify_sheet(
*,
before: Sequence[Mapping[str, Any]] | None,
after: Sequence[Mapping[str, Any]] | None,
before_rows: int,
after_rows: int,
wanted_slice: Mapping[str, str] | None = None,
) -> tuple[str, list[str]]:
"""``(data_state, reasons)`` for one sheet; reasons name the offending side.
``None`` receipts mean the batch carries no receipts at all (imported or
legacy data): that side cannot be judged and is trusted as-is.
"""
wanted = dict(wanted_slice or {})
reasons: list[str] = []
for side, receipts in (("before", before), ("after", after)):
if receipts is None:
continue
if not receipts:
reasons.append(f"{side}:no_command")
elif wanted and not covers_slice(receipts, wanted):
reasons.append(f"{side}:slice_not_collected")
if reasons:
return "not_collected", reasons
for side, receipts in (("before", before), ("after", after)):
if receipts is None:
continue
q = collect_quality(receipts)
if q["row_count"] == 0 and q["raw_line_count"] > SUSPECT_RAW_LINES:
reasons.append(f"{side}:parsed_0_of_{q['raw_line_count']}_lines")
elif q["commands"] and q["parse_status"].get("failed", 0) == q["commands"]:
reasons.append(f"{side}:all_commands_failed")
if reasons:
return "suspect_parse", reasons
if before_rows == 0 and after_rows == 0:
return "no_data", ["both_sides_empty"]
return "ok", []
def assess_sheet(
db: Session, sheet: Mapping[str, Any], *, before_batch_id: str, after_batch_id: str,
before_rows: int, after_rows: int,
) -> dict[str, Any]:
"""Quality block stored on a sheet summary."""
metric_id = str(sheet.get("metric_id") or "")
before = side_receipts(db, before_batch_id, metric_id) if _has_receipts(db, before_batch_id) else None
after = side_receipts(db, after_batch_id, metric_id) if _has_receipts(db, after_batch_id) else None
state, reasons = classify_sheet(
before=before, after=after, before_rows=before_rows, after_rows=after_rows,
wanted_slice=sheet_slice(sheet),
)
return {
"data_state": state,
"data_state_reasons": reasons,
"collect_quality": {
"before": collect_quality(before or []), "after": collect_quality(after or []),
"unknown_sides": [n for n, r in (("before", before), ("after", after)) if r is None],
},
}
def command_coverage(
db: Session, *, before_batch_id: str, after_batch_id: str,
) -> dict[str, Any]:
"""Collected "metric{slice}" labels per side and their difference.
Profile ids differ per vendor (``zte.x`` vs ``huawei.x``), so the comparison
is on what a command *produces*: its metric and slice.
"""
from .profiles import get_profile
def labels(batch_id: str) -> set[str]:
if not batch_id:
return set()
rows = db.query(BizStateBatchCommand.metric_id, BizStateBatchCommand.profile_id).filter(
BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id != "",
).distinct().all()
out: set[str] = set()
for metric_id, profile_id in rows:
prof = get_profile(str(profile_id or "")) if profile_id else None
sl = dict(prof.slice) if prof is not None and prof.slice else {}
suffix = ",".join(f"{k}={v}" for k, v in sorted(sl.items()))
out.add(f"{metric_id}{{{suffix}}}" if suffix else str(metric_id))
return out
b, a = labels(before_batch_id), labels(after_batch_id)
return {"before": sorted(b), "after": sorted(a),
"only_before": sorted(b - a), "only_after": sorted(a - b)}

50
netx_api/biz_state/compare_rules.py
View file

@ -137,6 +137,8 @@ def field_rule_map(rules: Sequence[Mapping[str, Any]] | None) -> dict[str, dict[
name = str(raw.get("field") or "").strip()
if not name:
continue
if str(raw.get("side") or "").strip():
continue # side-scoped rules feed the identity pipeline, not the compare
out[name] = dict(raw)
return out
@ -210,6 +212,50 @@ def _parse_float(text: str) -> float | None:
return None
DIRECTIONAL_MODES = ("not_worse", "numeric_min", "numeric_max")
def _rank_index(value: str, rank: Sequence[Any]) -> int | None:
wanted = value.strip().lower()
for i, item in enumerate(rank or []):
if scalar_text(item).strip().lower() == wanted:
return i
return None
def _tolerance(rule: Mapping[str, Any]) -> float:
try:
return float(rule.get("tolerance") or 0)
except (TypeError, ValueError):
return 0.0
def _directional_verdict(bv: str, av: str, rule: Mapping[str, Any]) -> tuple[bool, str]:
"""``(equal_or_better, reason)`` for not_worse / numeric_min / numeric_max."""
mode = str(rule.get("compare") or "").strip().lower()
if mode == "not_worse":
rank = rule.get("rank") or []
bi, ai = _rank_index(bv, rank), _rank_index(av, rank)
if bi is None or ai is None:
return bv == av, "neq"
return ai <= bi, (f"worse: {bv}→{av}" if ai > bi else "")
bn, an = _parse_float(bv), _parse_float(av)
if bn is None or an is None:
return bv == av, "neq"
tol = _tolerance(rule)
if mode == "numeric_min":
floor = bn * (1 - tol / 100.0)
if an >= floor:
return True, ""
drop = (bn - an) / abs(bn) * 100.0 if bn else 0.0
return False, f"drop {drop:.0f}% > {tol:g}%"
ceil = bn * (1 + tol / 100.0)
if an <= ceil:
return True, ""
rise = (an - bn) / abs(bn) * 100.0 if bn else 0.0
return False, f"rise {rise:.0f}% > {tol:g}%"
def explain_diff(
before: Any,
after: Any,
@ -222,6 +268,8 @@ def explain_diff(
bv = normalize_value(before, norm)
av = normalize_value(after, norm)
mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
if mode in DIRECTIONAL_MODES:
return _directional_verdict(bv, av, rule)[1]
if mode in ("percent", "pct", "rel"):
bn = _parse_float(bv)
an = _parse_float(av)
@ -262,6 +310,8 @@ def values_equal(
mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
if mode in ("ignore", "skip", "off"):
return True
if mode in DIRECTIONAL_MODES:
return _directional_verdict(bv, av, rule)[0]
if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"):
bn = _parse_float(bv)
an = _parse_float(av)

522
netx_api/biz_state/compare_service.py
View file

@ -22,6 +22,8 @@ from ..models import (
BizCompareJob,
BizCompareRun,
BizCompareTemplate,
BizMigrationProject,
BizMonitorTemplate,
BizPortMapping,
BizPortMappingRow,
BizStateBatch,
@ -34,6 +36,7 @@ from .compare_engine import apply_port_map, compare_rows, mapping_stats, row_key
from .compare_rules import (
ROW_FILTER_PRESETS,
apply_row_filters,
scalar_text,
arp_dynamic_row_filters,
effective_compare_fields,
effective_display_fields,
@ -43,12 +46,19 @@ from .iface_normalize import (
apply_iface_normalize_rows,
default_zte_iface_normalize_rules,
normalize_iface_rules,
normalize_port_map,
)
from .compare_quality import UNJUDGED_STATES, assess_sheet
from .identity_pipeline import Pipeline, build_pipeline, legacy_pipeline
from .profiles import metric_field_map
from .compare_validation import validate_template_body
_log = logging.getLogger("netx.biz_state.compare")
# v1: port-map keys/values must be written post-normalization.
# v2: port-map keys/values are normalized with the template iface rules (A1).
CONFIG_SNAPSHOT_VERSION = 2
# Per-job mutex so auto + manual run_compare do not dual-write the same job.
_auto_compare_locks: dict[str, threading.Lock] = {}
_auto_compare_locks_guard = threading.Lock()
@ -426,11 +436,25 @@ def _metric_rows_by_ids(db: Session, ids: list[str]) -> dict[str, dict[str, Any]
return out
def _attach_raw_key_before(items: list[dict[str, Any]], key_fields: list[str]) -> list[dict[str, Any]]:
"""``raw_key_before``: the before-side key as printed, only where it differs from the match key."""
for it in items:
before = it.get("before")
if not before or it.get("raw_key_before"):
continue
shown = it.get("key") or {}
raw = {f: scalar_text(before.get(f)).strip() for f in key_fields}
if any(raw[f] != scalar_text(shown.get(f)).strip() for f in key_fields if f in shown):
it["raw_key_before"] = raw
return items
def _hydrate_diff_rows(
db: Session, items: list[dict[str, Any]], *,
run: BizCompareRun | None = None, sheet: dict[str, Any] | None = None,
) -> list[dict[str, Any]]:
"""Fill empty before/after from metric tables when row_ids are present."""
key_fields = list((sheet or {}).get("key_fields") or [])
need: list[str] = []
for it in items:
if not it.get("before") and it.get("before_row_id"):
@ -438,27 +462,19 @@ def _hydrate_diff_rows(
if not it.get("after") and it.get("after_row_id"):
need.append(str(it["after_row_id"]))
if not need:
return items
return _attach_raw_key_before(items, key_fields)
by_id = _metric_rows_by_ids(db, need)
pmap, norm_rules = _run_transform_config(db, run) if run else ({}, [])
iface_fields = list((sheet or {}).get("iface_fields") or [])
pipeline = _run_pipeline(db, run, sheet or {})
for it in items:
brid = str(it.get("before_row_id") or "")
arid = str(it.get("after_row_id") or "")
if not it.get("before") and brid and brid in by_id:
it["before"] = by_id[brid]
if not it.get("after") and arid and arid in by_id:
it["after"] = apply_iface_normalize_rows(
[by_id[arid]], iface_fields=iface_fields, rules=norm_rules,
)[0]
it["after"] = pipeline.normalize(by_id[arid], "after")
if not it.get("mapped_before") and it.get("before"):
normalized = apply_iface_normalize_rows(
[it["before"]], iface_fields=iface_fields, rules=norm_rules,
)[0]
it["mapped_before"] = apply_port_map(
normalized, iface_fields=iface_fields, port_map=pmap,
)
return items
it["mapped_before"] = pipeline.transform(it["before"], "before")
return _attach_raw_key_before(items, key_fields)
def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]:
@ -653,18 +669,41 @@ def _normalize_row_filters(raw: Any) -> list[dict[str, Any]]:
return out
# Optional sheet-level switches that must survive normalization.
_SHEET_OPTION_VALUES: dict[str, tuple[str, ...]] = {
"unmapped_port_policy": ("by_name", "ignore_port"),
"duplicate_pairing": ("sorted", "ordered"),
}
def _sheet_options(raw: dict[str, Any]) -> dict[str, Any]:
out: dict[str, Any] = {}
for key, allowed in _SHEET_OPTION_VALUES.items():
val = str(raw.get(key) or "").strip().lower()
if val in allowed:
out[key] = val
if raw.get("builtin_vendor_canonical") is False:
out["builtin_vendor_canonical"] = False
if raw.get("scope_by_port_map") is not None:
out["scope_by_port_map"] = bool(raw.get("scope_by_port_map"))
return out
def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]:
if not isinstance(raw, list):
return []
out: list[dict[str, Any]] = []
seen: set[str] = set()
seen: set[tuple[str, str]] = set()
for item in raw:
if not isinstance(item, dict):
continue
name = str(item.get("field") or "").strip()
if not name or name in seen:
side = str(item.get("side") or "").strip().lower()
side = side if side in ("before", "after") else ""
# A field may carry one rule per side (e.g. a different value_map each).
if not name or (name, side) in seen:
continue
seen.add(name)
seen.add((name, side))
rule: dict[str, Any] = {"field": name}
compare = str(item.get("compare") or "").strip().lower()
if compare:
@ -680,6 +719,16 @@ def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]:
rule["tolerance"] = float(item.get("tolerance"))
except (TypeError, ValueError):
pass
vmap = item.get("value_map")
if isinstance(vmap, dict) and vmap:
rule["value_map"] = {str(k): str(v) for k, v in vmap.items()}
if side:
rule["side"] = side
if isinstance(item.get("apply_to_key"), bool):
rule["apply_to_key"] = item["apply_to_key"]
rank = item.get("rank")
if isinstance(rank, list) and rank:
rule["rank"] = [str(x) for x in rank]
# Drop empty rules (only field name)
if len(rule) > 1:
out.append(rule)
@ -709,6 +758,7 @@ def _sheet_def(
row_filters: list[dict[str, Any]] | None = None,
field_rules: list[dict[str, Any]] | None = None,
ignore_port_changes: bool | None = None,
options: dict[str, Any] | None = None,
) -> dict[str, Any]:
mid = str(metric_id or "").strip()
sid = str(sheet_id or "").strip() or mid
@ -749,6 +799,7 @@ def _sheet_def(
}
if ignore_port_changes is not None:
sheet["ignore_port_changes"] = bool(ignore_port_changes)
sheet.update(options or {})
return sheet
@ -946,6 +997,49 @@ def _builtin_status_needs_packaged_upgrade(existing: list[dict[str, Any]]) -> bo
return len(packaged_keys - have_keys) >= 3
# A7: (metric_id, kind, field, old packaged value, new value). A stored sheet is
# only touched when its value still equals the *old* packaged one, so anything
# an operator edited survives the upgrade.
_INTERFACE_DETAIL_OLD_COMPARE = [
"admin", "input_bps", "output_bps", "port_status", "in_util", "out_util",
"ip_mtu", "mtu", "mpls_mtu", "ipv6_mtu", "port_media", "negotiation",
]
_BUILTIN_STATUS_VALUE_UPGRADES: tuple[tuple[str, str, str, Any, Any], ...] = (
("interface_detail", "rule", "output_bps",
{"compare": "numeric", "tolerance": 5}, {"compare": "percent", "tolerance": 5}),
("interface_detail", "list", "compare_fields", _INTERFACE_DETAIL_OLD_COMPARE,
[f for f in _INTERFACE_DETAIL_OLD_COMPARE if f not in ("input_bps", "output_bps", "in_util", "out_util")]),
("optical_brief", "rule", "rx_power",
{"compare": "percent", "tolerance": 5}, {"compare": "numeric", "tolerance": 2}),
("l2vpn_mac", "list", "iface_fields", [], ["ac_port"]),
# ARP: filter on the canonical ``entry_type`` (every vendor parser emits it); the
# age-timer clause stays only as a fallback for rows parsed before entry_type existed.
("arp", "list", "row_filters", [{"op": "age_timer", "field": "age", "value": ""}], arp_dynamic_row_filters()),
)
def _upgrade_builtin_status_values(sheet: dict[str, Any]) -> tuple[dict[str, Any], bool]:
"""Apply ``_BUILTIN_STATUS_VALUE_UPGRADES`` to one stored sheet."""
mid = str(sheet.get("metric_id") or "")
out = deepcopy(sheet)
changed = False
for metric_id, kind, field, old, new in _BUILTIN_STATUS_VALUE_UPGRADES:
if mid != metric_id:
continue
if kind == "list":
if list(out.get(field) or []) == old:
out[field] = list(new)
changed = True
continue
for rule in out.get("field_rules") or []:
if isinstance(rule, dict) and rule.get("field") == field and all(
rule.get(k) == v for k, v in old.items()
):
rule.update(new)
changed = True
return out, changed
def _default_zte_config_sheets() -> list[dict[str, Any]]:
"""Config-intent metrics for cutover / intent-vs-intent compare."""
return [
@ -1009,6 +1103,7 @@ def _normalize_sheet(raw: Any) -> dict[str, Any] | None:
row_filters=_normalize_row_filters(raw.get("row_filters")),
field_rules=rules,
ignore_port_changes=ignore_ports,
options=_sheet_options(raw),
)
@ -1252,6 +1347,8 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate:
if not cur.get("field_rules") and src and src.get("field_rules"):
cur["field_rules"] = list(src["field_rules"])
changed = True
cur, value_changed = _upgrade_builtin_status_values(cur)
changed = changed or value_changed
upgraded.append(_normalize_sheet(cur) or cur)
have_mids = {str(s.get("metric_id") or "") for s in upgraded}
for s in sheets:
@ -1389,7 +1486,16 @@ def list_templates(db: Session) -> list[dict[str, Any]]:
upgrade_builtin_split_sheets(db)
rows = db.query(BizCompareTemplate).order_by(BizCompareTemplate.name.asc()).all()
usage = dict(db.query(BizCompareJob.template_id, func.count(BizCompareJob.id)).group_by(BizCompareJob.template_id).all())
return [{**_template_out(t), "job_count": usage.get(t.id, 0)} for t in rows]
monitor_usage = dict(
db.query(BizMonitorTemplate.compare_template_id, func.count(BizMonitorTemplate.id))
.filter(BizMonitorTemplate.compare_template_id != "")
.group_by(BizMonitorTemplate.compare_template_id).all()
)
return [
{**_template_out(t), "job_count": usage.get(t.id, 0),
"monitor_template_count": monitor_usage.get(t.id, 0)}
for t in rows
]
def list_metric_schemas() -> list[dict[str, Any]]:
@ -1522,8 +1628,19 @@ def delete_template(db: Session, template_id: str) -> None:
t = db.get(BizCompareTemplate, template_id)
if not t:
raise HTTPException(status_code=404, detail="template_not_found")
if db.query(BizCompareJob.id).filter(BizCompareJob.template_id == template_id).first():
raise HTTPException(status_code=409, detail="template_in_use")
jobs = [n or i for i, n in db.query(BizCompareJob.id, BizCompareJob.name)
.filter(BizCompareJob.template_id == template_id).all()]
monitors = [n or i for i, n in db.query(BizMonitorTemplate.id, BizMonitorTemplate.name)
.filter(BizMonitorTemplate.compare_template_id == template_id).all()]
if jobs or monitors:
raise HTTPException(status_code=409, detail={
"code": "template_in_use",
"message": "template_in_use: " + "; ".join(filter(None, [
f"jobs={', '.join(jobs)}" if jobs else "",
f"monitor_templates={', '.join(monitors)}" if monitors else "",
])),
"jobs": jobs, "monitor_templates": monitors,
})
db.delete(t)
db.commit()
@ -1544,9 +1661,30 @@ def _mapping_out(db: Session, m: BizPortMapping) -> dict[str, Any]:
}
def _mapping_references(db: Session, mapping_id: str) -> tuple[list[str], list[str]]:
"""(compare job names, cutover project names) that reference this mapping."""
jobs = [n or i for i, n in db.query(BizCompareJob.id, BizCompareJob.name)
.filter(BizCompareJob.mapping_id == mapping_id).all()]
projects = [n or i for i, n in db.query(BizMigrationProject.id, BizMigrationProject.name)
.filter(BizMigrationProject.mapping_id == mapping_id).all()]
return jobs, projects
def list_mappings(db: Session) -> list[dict[str, Any]]:
rows = db.query(BizPortMapping).order_by(BizPortMapping.name.asc()).all()
return [_mapping_out(db, m) for m in rows]
job_counts = dict(
db.query(BizCompareJob.mapping_id, func.count(BizCompareJob.id))
.filter(BizCompareJob.mapping_id != "").group_by(BizCompareJob.mapping_id).all()
)
project_counts = dict(
db.query(BizMigrationProject.mapping_id, func.count(BizMigrationProject.id))
.filter(BizMigrationProject.mapping_id != "").group_by(BizMigrationProject.mapping_id).all()
)
return [
{**_mapping_out(db, m), "job_count": job_counts.get(m.id, 0),
"project_count": project_counts.get(m.id, 0)}
for m in rows
]
def create_mapping(db: Session, body: dict[str, Any]) -> dict[str, Any]:
@ -1604,6 +1742,16 @@ def delete_mapping(db: Session, mapping_id: str) -> None:
m = db.get(BizPortMapping, mapping_id)
if not m:
raise HTTPException(status_code=404, detail="mapping_not_found")
jobs, projects = _mapping_references(db, mapping_id)
if jobs or projects:
raise HTTPException(status_code=409, detail={
"code": "mapping_in_use",
"message": "mapping_in_use: " + "; ".join(filter(None, [
f"jobs={', '.join(jobs)}" if jobs else "",
f"projects={', '.join(projects)}" if projects else "",
])),
"jobs": jobs, "projects": projects,
})
db.query(BizPortMappingRow).filter(BizPortMappingRow.mapping_id == mapping_id).delete()
db.delete(m)
db.commit()
@ -1822,11 +1970,11 @@ def validate_mapping(
sheets = template_metrics(tpl)
if not sheets:
raise HTTPException(status_code=400, detail="template_has_no_metrics")
pmap = _port_map_dict(db, mapping_id)
# Validate against first sheet that has iface fields (or first sheet)
target = next((s for s in sheets if s.get("iface_fields")), sheets[0])
iface_fields = list(target.get("iface_fields") or [])
norm = template_iface_normalize(tpl)
pmap, _ = normalize_port_map(_port_map_dict(db, mapping_id), norm)
before = apply_iface_normalize_rows(
_load_metric_rows(db, batch_id=before_batch_id, metric_id=target["metric_id"]),
iface_fields=iface_fields,
@ -1845,6 +1993,66 @@ def validate_mapping(
)
def _mapping_rows_to_dict(rows: list[Any]) -> tuple[dict[str, str], list[dict[str, str]]]:
"""Editor rows → raw map; a repeated before port is reported, first one wins."""
out: dict[str, str] = {}
dups: list[dict[str, str]] = []
for r in rows or []:
raw = r if isinstance(r, dict) else dict(r)
before = str(raw.get("before_if") or "").strip()
after = str(raw.get("after_if") or "").strip()
if not before or not after:
continue
if before in out:
dups.append({"before_if": before, "kept_to": out[before], "dropped_to": after})
continue
out[before] = after
return out, dups
def preflight_compare(db: Session, body: dict[str, Any]) -> dict[str, Any]:
"""Mapping coverage + data-quality report for a template and two batches."""
from .compare_preflight import preflight
ensure_default_templates(db)
template_id = str(body.get("template_id") or "")
tpl = db.get(BizCompareTemplate, template_id) if template_id else ensure_default_cutover_template(db)
if not tpl:
raise HTTPException(status_code=404, detail="template_not_found")
before_id = str(body.get("before_batch_id") or "")
after_id = str(body.get("after_batch_id") or "")
if not db.get(BizStateBatch, before_id) or not db.get(BizStateBatch, after_id):
raise HTTPException(status_code=404, detail="batch_not_found")
sheets = _filter_enabled_sheets(template_metrics(tpl), _str_list(body.get("enabled_sheet_ids")))
if not sheets:
raise HTTPException(status_code=400, detail="no_enabled_sheets")
dups: list[dict[str, str]] = []
if body.get("mapping_rows") is not None:
raw_map, dups = _mapping_rows_to_dict(list(body.get("mapping_rows") or []))
else:
raw_map = _port_map_dict(db, str(body.get("mapping_id") or ""))
cache: dict[tuple[str, str], list[dict[str, Any]]] = {}
def load(side: str, sheet: dict[str, Any]) -> list[dict[str, Any]]:
batch_id = before_id if side == "before" else after_id
key = (batch_id, str(sheet["metric_id"]))
if key not in cache:
cache[key] = _load_metric_rows(db, batch_id=batch_id, metric_id=key[1])
return cache[key]
report = preflight(
db, sheets=sheets, iface_rules=template_iface_normalize(tpl), raw_port_map=raw_map,
before_batch_id=before_id, after_batch_id=after_id,
before_vendor=_batch_vendor_key(db, before_id), after_vendor=_batch_vendor_key(db, after_id),
load_rows=load,
)
if dups:
report["mapping"]["duplicate_before"] = dups
report["risk_reasons"] = sorted({*report["risk_reasons"], "duplicate_before"})
report["verdict"] = "risk"
return report
def _job_out(j: BizCompareJob) -> dict[str, Any]:
return {
"id": j.id,
@ -2040,14 +2248,23 @@ def _sheet_result_envelope(
}
def _sheet_quality(
db: Session, sheet: dict[str, Any], before_batch_id: str, after_batch_id: str,
summary: dict[str, Any],
) -> dict[str, Any]:
return assess_sheet(
db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id,
before_rows=int(summary.get("before_count") or 0), after_rows=int(summary.get("after_count") or 0),
)
def _run_sheet(
db: Session,
*,
sheet: dict[str, Any],
before_batch_id: str,
after_batch_id: str,
port_map: dict[str, str],
iface_normalize_rules: list[dict[str, str]] | None = None,
pipeline: Pipeline,
store_unchanged: str = "auto",
on_load_progress: Callable[..., None] | None = None,
) -> dict[str, Any]:
@ -2083,12 +2300,7 @@ def _run_sheet(
# PostgreSQL path: pushdown-safe sheets join in-DB (BGP-scale).
from .compare_sql import SqlCompareSkip, run_sql_sheet_compare, sql_compare_skip_reason
skip_reason = sql_compare_skip_reason(
db,
sheet,
port_map=port_map,
iface_normalize_rules=iface_normalize_rules,
)
skip_reason = sql_compare_skip_reason(db, sheet, pipeline=pipeline)
if not skip_reason:
try:
result = run_sql_sheet_compare(
@ -2098,6 +2310,7 @@ def _run_sheet(
after_batch_id=after_batch_id,
store_unchanged=store_unchanged,
on_progress=_emit_load,
duplicate_pairing=pipeline.duplicate_pairing,
)
summary = dict(result["summary"])
# Engine already sets raw counts / policy; keep keys stable
@ -2107,6 +2320,7 @@ def _run_sheet(
before_n=int(summary.get("before_count") or 0),
after_n=int(summary.get("after_count") or 0),
)
summary.update(_sheet_quality(db, sheet, before_batch_id, after_batch_id, summary))
return _sheet_result_envelope(
sheet,
key_fields=key_fields,
@ -2161,39 +2375,38 @@ def _run_sheet(
def _after_chunk(n: int) -> None:
_emit_load("after", n, engine="python", note=skip_reason or "python", phase="loading")
# Only filters that read untransformed fields may run in SQL on the raw rows;
# the pipeline applies every filter (post-normalize for v2) in ``prepare``.
before_raw = _load_metric_rows(
db,
batch_id=before_batch_id,
metric_id=mid,
on_chunk=_before_chunk,
row_filters=row_filters,
row_filters=pipeline.pushdown_filters("before"),
)
after_raw = _load_metric_rows(
db,
batch_id=after_batch_id,
metric_id=mid,
on_chunk=_after_chunk,
row_filters=row_filters,
row_filters=pipeline.pushdown_filters("after"),
)
# Filters already applied in load when pushdown-safe; keep apply for safety
before_rows = apply_row_filters(before_raw, row_filters) if row_filters else before_raw
after_rows = apply_row_filters(after_raw, row_filters) if row_filters else after_raw
before_pairs = pipeline.prepare(before_raw, "before")
after_pairs = pipeline.prepare(after_raw, "after")
policy = resolve_unchanged_policy(
store_unchanged, before_n=len(before_rows), after_n=len(after_rows)
store_unchanged, before_n=len(before_pairs), after_n=len(after_pairs)
)
result = compare_rows(
before_rows=before_rows,
after_rows=after_rows,
before_rows=[], after_rows=[],
key_fields=key_fields,
iface_fields=iface_fields,
compare_fields=compare_fields,
port_map=port_map,
field_rules=field_rules,
iface_normalize_rules=iface_normalize_rules,
ignore_port_changes=ignore_ports,
include_unchanged=bool(policy["include"]),
unchanged_limit=policy.get("limit"),
compact_unchanged=bool(policy.get("compact")),
pipeline=pipeline,
prepared=(before_pairs, after_pairs),
)
summary = dict(result["summary"])
summary["before_raw_count"] = len(before_raw)
@ -2201,6 +2414,14 @@ def _run_sheet(
summary["row_filters"] = len(row_filters)
summary["unchanged_policy"] = policy
summary.setdefault("engine", "python")
if pipeline.port_map and pipeline.iface_fields:
from .compare_preflight import sheet_mapping_coverage
coverage = sheet_mapping_coverage(pipeline, before_pairs, after_pairs)
coverage.pop("_after_ifaces", None)
coverage.pop("_used_keys", None)
summary["mapping_coverage"] = coverage
summary.update(_sheet_quality(db, sheet, before_batch_id, after_batch_id, summary))
return _sheet_result_envelope(
sheet,
key_fields=key_fields,
@ -2227,6 +2448,8 @@ def _validate_compare_job(
tpl = db.get(BizCompareTemplate, j.template_id)
if not tpl:
raise HTTPException(status_code=404, detail="template_not_found")
if j.mapping_id and not db.get(BizPortMapping, j.mapping_id):
raise HTTPException(status_code=400, detail="mapping_missing")
before_batch_id = str(j.before_batch_id or "")
after_batch_id = str(force_after_batch_id or "").strip() or _resolve_after_batch(db, j)
if not before_batch_id or not after_batch_id:
@ -2272,15 +2495,31 @@ def _create_running_run(
before_batch_id: str,
after_batch_id: str,
sheets_cfg: list[dict[str, Any]],
purpose: str = "manual",
) -> BizCompareRun:
first_metric = str(sheets_cfg[0].get("metric_id") or "")
pending_sheets = [_pending_sheet_meta(s) for s in sheets_cfg]
mapping = db.get(BizPortMapping, job.mapping_id) if job.mapping_id else None
vendors = {
"before": _batch_vendor_key(db, before_batch_id),
"after": _batch_vendor_key(db, after_batch_id),
}
raw_port_map = _port_map_dict(db, str(job.mapping_id or ""))
iface_rules = template_iface_normalize(tpl)
identity = {
sheet_key(sh): build_pipeline(
sh, iface_rules=iface_rules, port_map=raw_port_map,
before_vendor=vendors["before"], after_vendor=vendors["after"],
).describe()
for sh in sheets_cfg
}
config_snapshot = deepcopy({
"version": 1,
"version": CONFIG_SNAPSHOT_VERSION,
"template": _template_out(tpl),
"sheets": sheets_cfg,
"port_map": _port_map_dict(db, str(job.mapping_id or "")),
"port_map": raw_port_map,
"vendors": vendors,
"identity": identity,
"store_unchanged": normalize_store_unchanged(getattr(job, "store_unchanged", None)),
"job_name": job.name or "",
"before_task_id": job.before_task_id or "",
@ -2297,6 +2536,7 @@ def _create_running_run(
metric_id=first_metric,
status="running",
summary_json={
"purpose": purpose,
"config_snapshot": config_snapshot,
"progress": {
"phase": "queued",
@ -2329,7 +2569,7 @@ def _create_running_run(
def _run_config_snapshot(run: BizCompareRun) -> dict[str, Any] | None:
snapshot = (run.summary_json or {}).get("config_snapshot")
if isinstance(snapshot, dict) and snapshot.get("version") == 1:
if isinstance(snapshot, dict) and snapshot.get("version") in (1, 2):
return snapshot
return None
@ -2339,14 +2579,66 @@ def _run_transform_config(
) -> tuple[dict[str, str], list[dict[str, str]]]:
snapshot = _run_config_snapshot(run)
if snapshot is not None:
return dict(snapshot.get("port_map") or {}), normalize_iface_rules(
rules = normalize_iface_rules(
(snapshot.get("template") or {}).get("iface_normalize_rules") or [],
)
pmap = dict(snapshot.get("port_map") or {})
if int(snapshot.get("version") or 1) >= 2:
# v2 runs: map keys/values are written as the device prints them.
pmap, _ = normalize_port_map(pmap, rules)
return pmap, rules
# Legacy runs did not record these settings; retain their fallback behavior.
tpl = db.get(BizCompareTemplate, run.template_id) if run.template_id else None
return _port_map_dict(db, str(run.mapping_id or "")), template_iface_normalize(tpl)
def _batch_vendor_key(db: Session, batch_id: str) -> str:
"""Vendor key (``huawei``/``zte``/…) for a batch, '' when unknown."""
from ..topology_lldp import resolve_vendor_key
batch = db.get(BizStateBatch, batch_id) if batch_id else None
if batch is None:
return ""
task = db.get(BizStateTask, batch.task_id) if batch.task_id else None
key = resolve_vendor_key(
batch.vendor or (task.vendor if task else ""), (task.device_type if task else "") or "",
)
return "" if key == "generic" else key
def _run_pipeline(db: Session, run: BizCompareRun | None, sheet: dict[str, Any]) -> Pipeline:
"""The identity pipeline a run used (or would use) for one sheet.
Snapshot v2 runs get the full pipeline; v1 and snapshot-less runs get the
legacy one so stored results replay exactly as they were produced.
"""
sid = sheet_key(sheet)
snapshot = _run_config_snapshot(run) if run is not None else None
if snapshot is None:
pmap, rules = _run_transform_config(db, run) if run is not None else ({}, [])
return legacy_pipeline(
key_fields=sheet.get("key_fields") or [], iface_fields=sheet.get("iface_fields") or [],
compare_fields=sheet.get("compare_fields") or [], field_rules=sheet.get("field_rules"),
port_map=pmap, iface_normalize_rules=rules, row_filters=sheet.get("row_filters"),
ignore_port_changes=sheet.get("ignore_port_changes"),
)
authoritative = next(
(sh for sh in snapshot.get("sheets") or [] if isinstance(sh, dict) and sheet_key(sh) == sid), None,
)
cfg = authoritative or sheet
version = int(snapshot.get("version") or 1)
vendors = snapshot.get("vendors") or {}
return build_pipeline(
cfg,
iface_rules=(snapshot.get("template") or {}).get("iface_normalize_rules") or [],
port_map=dict(snapshot.get("port_map") or {}),
before_vendor=str(vendors.get("before") or ""),
after_vendor=str(vendors.get("after") or ""),
legacy=version < 2,
normalize_map=version >= 2,
)
def _set_run_progress(
db: Session,
run: BizCompareRun,
@ -2458,7 +2750,6 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
store_mode = normalize_store_unchanged(
snapshot.get("store_unchanged") if snapshot is not None else getattr(j, "store_unchanged", None),
)
pmap, norm_rules = _run_transform_config(db, run)
unchanged_listed_total = 0
unchanged_truncated_any = False
unchanged_compact_any = False
@ -2563,8 +2854,7 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
sheet=sheet,
before_batch_id=before_batch_id,
after_batch_id=after_batch_id,
port_map=pmap,
iface_normalize_rules=norm_rules,
pipeline=_run_pipeline(db, run, sheet),
store_unchanged=store_mode,
on_load_progress=_on_load,
)
@ -2755,6 +3045,7 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
if snapshot is not None:
summary_payload["config_snapshot"] = snapshot
run = db.get(BizCompareRun, run_id) or run
summary_payload["purpose"] = (run.summary_json or {}).get("purpose") or "manual"
if str(run.status or "") == "cancelled":
return get_run(db, run.id)
run.status = "success"
@ -2882,7 +3173,9 @@ def cancel_compare_run(db: Session, run_id: str) -> dict[str, Any]:
return get_run(db, run_id)
def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]:
def run_compare(
db: Session, job_id: str, *, force_after_batch_id: str = "", purpose: str = "manual",
) -> dict[str, Any]:
"""Synchronous compare (auto-compare / tests). Blocks the caller until done."""
if _job_has_active_run(db, job_id):
raise HTTPException(status_code=409, detail="compare_already_running")
@ -2898,6 +3191,7 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
before_batch_id=ctx["before_batch_id"],
after_batch_id=ctx["after_batch_id"],
sheets_cfg=ctx["sheets_cfg"],
purpose=purpose,
)
return _execute_compare_into_run(db, run.id)
finally:
@ -2972,7 +3266,9 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) ->
)
# Non-key columns already ordered Key→Compare→Display by effective_display_fields
extra = [f for f in display if f not in key_set]
headers = ["kind", *keys]
# Interface keys also get the name as the device printed it (before side).
raw_cols = [k for k in keys if k in set(sheet.get("iface_fields") or [])]
headers = ["kind", *keys, *[f"{k}__raw_before" for k in raw_cols]]
for f in extra:
if f in compare_set:
headers.append(f"{f}__pre")
@ -2988,6 +3284,9 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) ->
row = [kind]
for k in keys:
row.append(key.get(k, pre.get(k, post.get(k, ""))))
raw_before = dict(d.get("raw_key_before") or {})
for k in raw_cols:
row.append(raw_before.get(k, ""))
for f in extra:
if f in compare_set:
if kind == "added":
@ -3012,6 +3311,18 @@ def _sheet_csv(sheet: dict[str, Any]) -> str:
return "\ufeff" + "\n".join(_sheet_csv_lines(sheet, sheet.get("diffs") or [])) + "\n"
def _task_ok(sheets: list[dict[str, Any]], fail_count: int, any_pending: bool) -> bool | None:
"""True/False once judged; None when data quality leaves the verdict open."""
if any_pending:
return False
states = [str((sh.get("summary") or {}).get("data_state") or "ok") for sh in sheets]
if any(st in ("suspect_parse", "not_collected") for st in states):
return None
if states and all(st == "no_data" for st in states):
return None
return fail_count == 0
def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> dict[str, Any]:
added = int(summary.get("added") or 0)
removed = int(summary.get("removed") or 0)
@ -3020,16 +3331,28 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
before_count = int(summary.get("before_count") or 0)
after_count = int(summary.get("after_count") or 0)
total = added + removed + changed + unchanged
def _data_state(sh: dict[str, Any]) -> str:
return str((sh.get("summary") or {}).get("data_state") or "ok")
# Sheets whose side was never collected cannot be judged: their "missing"
# rows are a collection gap, not a business loss, so they stay out of the totals.
dropped = [sh for sh in sheets if _data_state(sh) == "not_collected"
and str(sh.get("status") or "done") not in ("pending", "running", "queued")]
drop_fail = sum(int((sh.get("summary") or {}).get("removed") or 0)
+ int((sh.get("summary") or {}).get("changed") or 0) for sh in dropped)
drop_ok = sum(int((sh.get("summary") or {}).get("unchanged") or 0) for sh in dropped)
# Compare verdict: fail = missing + mismatch; success = match; added is special
fail_count = removed + changed
success_count = unchanged
fail_count = removed + changed - drop_fail
success_count = unchanged - drop_ok
judged = fail_count + success_count
matched = changed + unchanged
diff_count = fail_count
# Nothing compared at all is "not judged" (None), never a 100% pass.
pass_rate = (
compute_pass_rate(success=success_count, fail=fail_count, empty_as=100.0 if total == 0 else 0.0)
compute_pass_rate(success=success_count, fail=fail_count)
if judged
else (100.0 if total == 0 else 0.0)
else (None if total == 0 else 0.0)
)
diff_rate = round((fail_count / judged) * 100, 1) if judged else 0.0
@ -3045,8 +3368,12 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
sj = sf + su
status = str(sh.get("status") or "done")
pending = status in ("pending", "running", "queued")
data_state = str(ss.get("data_state") or "ok")
sheet_cards.append(
{
"data_state": data_state,
"data_state_reasons": list(ss.get("data_state_reasons") or []),
"collect_quality": ss.get("collect_quality") or {},
"sheet_id": sheet_key(sh),
"title": sheet_title(sh),
"metric_id": sh.get("metric_id") or "",
@ -3063,12 +3390,8 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
"diff_count": sf,
# Pending sheets must not look like "100% pass"
"pass_rate": None
if pending
else (
compute_pass_rate(success=su, fail=sf, empty_as=100.0 if st == 0 else 0.0)
if sj
else (100.0 if st == 0 else 0.0)
),
if pending or data_state in UNJUDGED_STATES
else (compute_pass_rate(success=su, fail=sf) if sj else (None if st == 0 else 0.0)),
}
)
@ -3105,7 +3428,9 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
"diff_count": diff_count,
"pass_rate": None if any_pending else pass_rate,
"diff_rate": diff_rate,
"ok": False if any_pending else fail_count == 0,
"ok": _task_ok(sheets, fail_count, any_pending),
"unjudged_sheets": sum(1 for sh in sheets if _data_state(sh) in UNJUDGED_STATES),
"suspect_sheets": sum(1 for sh in sheets if _data_state(sh) == "suspect_parse"),
"sheet_cards": sheet_cards,
"top_changed_fields": top_fields,
"duration_ms": int(summary.get("duration_ms") or 0),
@ -3449,21 +3774,25 @@ def _load_metric_rows_for_search(
def _load_search_key_groups(
db: Session, *, batch_id: str, metric_id: str, keys: set[tuple[str, ...]],
match_keys: list[str], iface_fields: list[str], row_filters: list[dict[str, Any]],
norm_rules: list[dict[str, str]], port_map: dict[str, str],
match_keys: list[str], pipeline: Pipeline, side: str,
) -> tuple[list[dict[str, Any]], set[tuple[str, ...]]]:
"""Complete candidate key groups so a one-sided search cannot change verdicts.
Interface transforms run in Python. Other identity fields narrow the SQL
query. If the load budget is exceeded, omit entire groups rather than compare
partial groups and manufacture additions/deletions.
Identity rules (normalize / value map / port map) run in the pipeline, in
Python. Key fields they never touch narrow the SQL query. If the load budget
is exceeded, omit entire groups rather than compare partial groups and
manufacture additions/deletions.
"""
from ..models import BizStateMetricRow
model = BizStateLldpNeighbor if metric_id == "lldp_neighbor" else BizStateMetricRow
invariant = [f for f in match_keys if f not in iface_fields]
# Narrowing compares raw SQL text, so only fields no rule rewrites may be used,
# and only when key tuples line up 1:1 with ``match_keys`` (no mapped/unmapped marker).
invariant = [] if pipeline.ignores_ports_by_policy() else [
f for f in match_keys if f not in pipeline._transformed # noqa: SLF001
]
positions = [match_keys.index(f) for f in invariant]
coarse = sorted({tuple(k[i] for i in positions) for k in keys})
coarse = sorted({tuple(k[i] for i in positions) for k in keys}) if invariant else [()]
out: list[dict[str, Any]] = []
complete: set[tuple[str, ...]] = set()
# Bound memory even when a candidate identifies a very large duplicate group.
@ -3471,7 +3800,7 @@ def _load_search_key_groups(
for start in range(0, len(coarse), 100):
group = coarse[start:start + 100]
group_set = set(group)
target = {k for k in keys if tuple(k[i] for i in positions) in group_set}
target = {k for k in keys if tuple(k[i] for i in positions) in group_set} if invariant else set(keys)
q = db.query(model).filter(model.batch_id == batch_id)
if metric_id == "lldp_neighbor":
q = q.order_by(model.id.asc())
@ -3497,13 +3826,10 @@ def _load_search_key_groups(
)
row = {**data, "_netx": {"row_id": r.id}}
db.expunge(r)
if row_filters and not all(row_matches_filter(row, f) for f in row_filters):
norm = pipeline.normalize(row, side)
if not pipeline.accepts(row, norm, side):
continue
normalized = apply_iface_normalize_rows(
[row], iface_fields=iface_fields, rules=norm_rules,
)[0]
mapped = apply_port_map(normalized, iface_fields=iface_fields, port_map=port_map)
if row_key(mapped, match_keys) in target:
if pipeline.key_of(norm, side, match_keys)[1] in target:
out.append(row)
return out, complete
@ -3566,13 +3892,15 @@ def _live_search_sheet_diffs(
ignore_ports = sheet.get("ignore_port_changes")
if ignore_ports is not None:
ignore_ports = bool(ignore_ports)
pmap, norm_rules = _run_transform_config(db, run)
pipeline = _run_pipeline(db, run, sheet)
if pipeline.legacy and not pipeline.row_filters and row_filters:
pipeline.row_filters = [dict(f) for f in row_filters if isinstance(f, dict)]
before_rows, trunc_b = _load_metric_rows_for_search(
db,
batch_id=str(run.before_batch_id or ""),
metric_id=mid_src,
row_filters=row_filters,
row_filters=pipeline.pushdown_filters("before"),
key_fields=key_fields,
kw=free_kw,
field_q=merged_q,
@ -3581,49 +3909,46 @@ def _live_search_sheet_diffs(
db,
batch_id=str(run.after_batch_id or ""),
metric_id=mid_src,
row_filters=row_filters,
row_filters=pipeline.pushdown_filters("after"),
key_fields=key_fields,
kw=free_kw,
field_q=merged_q,
)
match_keys = list((sheet.get("summary") or {}).get("match_key_fields") or key_fields)
if ignore_ports is True and not pmap:
if ignore_ports is True and not pipeline.port_map:
match_keys = [f for f in key_fields if f not in iface_fields] or key_fields
keys: set[tuple[str, ...]] = set()
for rows, mapping in ((before_rows, pmap), (after_rows, {})):
for row in apply_iface_normalize_rows(rows, iface_fields=iface_fields, rules=norm_rules):
keys.add(row_key(apply_port_map(row, iface_fields=iface_fields, port_map=mapping), match_keys))
for rows, side in ((before_rows, "before"), (after_rows, "after")):
for raw, norm in pipeline.prepare(rows, side):
keys.add(pipeline.key_of(norm, side, match_keys)[1])
before_rows, complete_b = _load_search_key_groups(
db, batch_id=str(run.before_batch_id or ""), metric_id=mid_src, keys=keys,
match_keys=match_keys, iface_fields=iface_fields, row_filters=row_filters,
norm_rules=norm_rules, port_map=pmap,
match_keys=match_keys, pipeline=pipeline, side="before",
)
after_rows, complete_a = _load_search_key_groups(
db, batch_id=str(run.after_batch_id or ""), metric_id=mid_src, keys=keys,
match_keys=match_keys, iface_fields=iface_fields, row_filters=row_filters,
norm_rules=norm_rules, port_map={},
match_keys=match_keys, pipeline=pipeline, side="after",
)
complete = complete_b & complete_a
def keep_complete(rows: list[dict[str, Any]], mapping: dict[str, str]) -> list[dict[str, Any]]:
normalized = apply_iface_normalize_rows(rows, iface_fields=iface_fields, rules=norm_rules)
return [orig for orig, norm in zip(rows, normalized) if row_key(
apply_port_map(norm, iface_fields=iface_fields, port_map=mapping), match_keys,
) in complete]
before_rows = keep_complete(before_rows, pmap)
after_rows = keep_complete(after_rows, {})
def keep_complete(rows: list[dict[str, Any]], side: str) -> list[dict[str, Any]]:
return [raw for raw, norm in pipeline.prepare(rows, side)
if pipeline.key_of(norm, side, match_keys)[1] in complete]
before_rows = keep_complete(before_rows, "before")
after_rows = keep_complete(after_rows, "after")
result = compare_rows(
before_rows=before_rows,
after_rows=after_rows,
key_fields=key_fields,
iface_fields=iface_fields,
compare_fields=compare_fields,
port_map=pmap,
field_rules=field_rules,
iface_normalize_rules=norm_rules,
ignore_port_changes=(match_keys != key_fields) if not pmap else False,
include_unchanged=True,
unchanged_limit=None,
compact_unchanged=False,
pipeline=pipeline,
match_keys_override=match_keys,
)
kind_n = (kind or "diff").strip().lower()
filtered = [
@ -3636,7 +3961,7 @@ def _live_search_sheet_diffs(
filtered = filtered[:_LIVE_SEARCH_RESULT_CAP]
total = len(filtered)
start = (page - 1) * page_size
page_items = filtered[start : start + page_size]
page_items = _attach_raw_key_before(filtered[start : start + page_size], key_fields)
return {
"total": total,
"page": page,
@ -3844,7 +4169,7 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
csv_file.write(b"\xef\xbb\xbf")
for line in _sheet_csv_lines(sheet, _iter_sheet_diffs(db, run_id, sid)):
csv_file.write((line + "\n").encode("utf-8"))
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"]
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate,data_state"]
for card in list(s.get("sheet_cards") or []):
sum_lines.append(
",".join(
@ -3860,6 +4185,7 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
card.get("unchanged"),
card.get("diff_count"),
card.get("pass_rate"),
card.get("data_state") or "ok",
)
)
)
@ -3927,7 +4253,7 @@ def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int:
if not j.before_batch_id:
continue
try:
run_compare(db, j.id, force_after_batch_id=batch_id)
run_compare(db, j.id, force_after_batch_id=batch_id, purpose="auto")
n += 1
except HTTPException as exc:
if int(getattr(exc, "status_code", 0) or 0) == 409:

52
netx_api/biz_state/compare_sql.py
View file

@ -14,6 +14,7 @@ from uuid import uuid4
from sqlalchemy import text
from sqlalchemy.orm import Session
from .compare_engine import key_tuple
from .compare_rules import effective_compare_fields, field_rule_map, NUMBER_PATTERN, scalar_text
_log = logging.getLogger("netx.biz_state.compare_sql")
@ -154,16 +155,23 @@ def sql_compare_skip_reason(
*,
port_map: Mapping[str, str] | None = None,
iface_normalize_rules: Sequence[Mapping[str, str]] | None = None,
pipeline: Any = None,
) -> str:
"""Empty string when SQL is allowed; otherwise a short reason for progress/logs.
Simple ``row_filters`` (eq/in/contains/…) used for BGP sheet splits are fine —
they do **not** force Python by themselves.
A port map only blocks SQL when the sheet actually reads an interface field
in its key or compare columns; a BGP route table with no interface fields is
unaffected by it. ``pipeline`` (v2 identity rules) adds its own blockers.
"""
if not _dialect_is_postgres(db):
return "not_postgres"
if port_map:
return "port_map"
if pipeline is not None:
port_map = pipeline.port_map
if iface_normalize_rules is None and (pipeline.before.has_iface_rules or pipeline.after.has_iface_rules):
iface_normalize_rules = [{"from": "*", "to": "*"}]
key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()]
if not key_fields:
return "no_key_fields"
@ -171,12 +179,24 @@ def sql_compare_skip_reason(
return "unsafe_key_field"
iface_fields = [str(f).strip() for f in (sheet.get("iface_fields") or []) if str(f).strip()]
iface_set = set(iface_fields)
if port_map and iface_set:
early_cmp = effective_compare_fields(
list(sheet.get("compare_fields") or []), list(sheet.get("field_rules") or []),
)
if iface_set & (set(key_fields) | {str(f).strip() for f in early_cmp}):
return "port_map"
ignore_ports = sheet.get("ignore_port_changes")
if ignore_ports is True:
return "ignore_port_changes"
if ignore_ports is None and iface_set and any(k in iface_set for k in key_fields):
return "auto_ignore_ports"
field_rules = list(sheet.get("field_rules") or [])
if any(
isinstance(r, dict) and str(r.get("compare") or "").strip().lower() in
("not_worse", "numeric_min", "numeric_max")
for r in field_rules
):
return "directional_compare"
if not _field_rules_sql_compatible(field_rules):
return "field_rules"
compare_fields = effective_compare_fields(
@ -186,8 +206,14 @@ def sql_compare_skip_reason(
if any(not _FIELD_RE.match(str(f).strip()) for f in compare_fields if str(f).strip()):
return "unsafe_compare_field"
used = set(key_fields) | {str(f).strip() for f in compare_fields if str(f).strip()}
if port_map and (used & iface_set):
return "port_map"
if iface_normalize_rules and (used & iface_set):
return "iface_normalize"
if pipeline is not None:
blocked = pipeline.sql_blockers(used)
if blocked:
return blocked
if not _filters_sql_compatible(list(sheet.get("row_filters") or [])):
return "row_filters"
if not str(sheet.get("metric_id") or "").strip():
@ -371,7 +397,7 @@ def _changed_predicate(
def _key_obj_from_data(data: dict[str, Any] | None, key_fields: list[str]) -> dict[str, Any]:
row = data if isinstance(data, dict) else {}
return {f: row.get(f, "") for f in key_fields}
return dict(zip(key_fields, key_tuple(row, key_fields)))
def _changes_from_rows(
@ -399,6 +425,22 @@ def _changes_from_rows(
return out
def dup_pairing_order_sql(compare_fields: Sequence[str], rules: Mapping[str, Mapping[str, Any]]) -> str:
"""``ORDER BY`` prefix that pairs same-key rows by value (mirrors ``Pipeline.pair_sort_key``).
Byte-order collation matches Python's code-point string order, so both
engines choose the same pairs. Returns '' (or a trailing-comma list).
"""
parts = []
for f in compare_fields:
name = str(f).strip()
if not name:
continue
mode = str((rules.get(name) or {}).get("normalize") or "strip").strip().lower() or "strip"
parts.append(f'({_norm_expr("data_json", name, mode)}) COLLATE "C" ASC')
return ", ".join(parts) + ", " if parts else ""
def run_sql_sheet_compare(
db: Session,
*,
@ -407,6 +449,7 @@ def run_sql_sheet_compare(
after_batch_id: str,
store_unchanged: str = "auto",
on_progress: Callable[..., None] | None = None,
duplicate_pairing: str = "ordered",
) -> dict[str, Any]:
"""Compare one sheet via PostgreSQL TEMP tables + FULL OUTER JOIN.
@ -443,6 +486,7 @@ def run_sql_sheet_compare(
filter_sql, filter_params = compile_row_filters_sql(row_filters)
rk = _rk_sql(key_fields)
dup_order = dup_pairing_order_sql(compare_fields, rules) if duplicate_pairing == "sorted" else ""
tag = uuid4().hex[:8]
tb = f"_netx_cmp_b_{tag}"
ta = f"_netx_cmp_a_{tag}"
@ -500,7 +544,7 @@ def run_sql_sheet_compare(
data_json AS data,
row_number() OVER (
PARTITION BY ({rk})
ORDER BY seq ASC, id ASC
ORDER BY {dup_order}seq ASC, id ASC
) AS dup_rn
FROM biz_state_metric_row
WHERE batch_id = :bid

46
netx_api/biz_state/compare_validation.py
View file

@ -8,7 +8,10 @@ from typing import Any
from fastapi import HTTPException
FILTER_OPS = {"eq", "==", "ci_eq", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "regex", "age_timer"}
COMPARE_MODES = {"", "eq", "ignore", "skip", "off", "numeric", "number", "int", "float", "percent", "pct", "rel"}
COMPARE_MODES = {"", "eq", "ignore", "skip", "off", "numeric", "number", "int", "float", "percent", "pct", "rel",
"not_worse", "numeric_min", "numeric_max"}
SIDES = {"", "both", "before", "after"}
MAX_VALUE_MAP = 500
NORMALIZE_MODES = {"", "none", "strip", "lower", "upper", "mac", "empty_as_blank"}
@ -34,6 +37,8 @@ def validate_filters(value: Any, path: str, depth: int = 0) -> None:
here = f"{path}[{i}]"
if not isinstance(node, dict) or not node:
invalid(here, "filter_required")
if depth == 0 and str(node.get("side") or "").strip().lower() not in SIDES:
invalid(here, "unknown_filter_side")
groups = [key for key in ("any", "all") if key in node]
if groups:
if len(groups) != 1 or any(node.get(key) is not None for key in ("field", "op", "value")):
@ -73,6 +78,12 @@ def validate_sheet(sheet: Any, path: str, *, partial: bool = False) -> None:
for key in ("key_fields", "iface_fields", "compare_fields", "display_fields", "ignore_fields"):
if key in sheet or (key == "key_fields" and not partial):
_strings(sheet.get(key), f"{path}.{key}", required=key == "key_fields")
for opt, allowed in (("unmapped_port_policy", {"by_name", "ignore_port"}),
("duplicate_pairing", {"sorted", "ordered"})):
if str(sheet.get(opt) or "").strip() and str(sheet[opt]).strip().lower() not in allowed:
invalid(path, f"unknown_{opt}")
if sheet.get("builtin_vendor_canonical") is not None and not isinstance(sheet["builtin_vendor_canonical"], bool):
invalid(path, "boolean_builtin_vendor_canonical_required")
if "ignore_port_changes" in sheet and sheet["ignore_port_changes"] is not None and not isinstance(sheet["ignore_port_changes"], bool):
invalid(path, "boolean_port_policy_required")
if "row_filters" in sheet:
@ -87,9 +98,30 @@ def validate_sheet(sheet: Any, path: str, *, partial: bool = False) -> None:
if not isinstance(rule, dict) or not isinstance(rule.get("field"), str) or not rule["field"].strip():
invalid(here, "rule_field_required")
field = rule["field"].strip()
if field in seen:
rule_side = str(rule.get("side") or "").strip().lower()
if rule_side not in SIDES:
invalid(here, "unknown_rule_side")
marker = (field, "" if rule_side == "both" else rule_side)
if marker in seen:
invalid(here, "duplicate_field_rule")
seen.add(field)
seen.add(marker)
vmap = rule.get("value_map")
if vmap is not None:
if not isinstance(vmap, dict) or any(
not isinstance(k, str) or not isinstance(v, str) for k, v in vmap.items()
):
invalid(here, "value_map_string_pairs_required")
if len(vmap) > MAX_VALUE_MAP:
invalid(here, "value_map_too_large")
if rule.get("apply_to_key") is not None and not isinstance(rule["apply_to_key"], bool):
invalid(here, "boolean_apply_to_key_required")
mode = str(rule.get("compare") or "").strip().lower()
if mode == "not_worse":
rank = rule.get("rank")
if not isinstance(rank, list) or len(rank) < 2 or any(not isinstance(x, str) or not x.strip() for x in rank):
invalid(here, "rank_list_required")
elif rule.get("rank") is not None:
invalid(here, "rank_only_for_not_worse")
if str(rule.get("compare") or "").strip().lower() not in COMPARE_MODES:
invalid(here, "unknown_compare_mode")
if str(rule.get("normalize") or "").strip().lower() not in NORMALIZE_MODES:
@ -124,7 +156,13 @@ def validate_template_body(body: dict[str, Any], *, partial: bool = False) -> No
path = f"iface_normalize_rules[{i}]"
if not isinstance(rule, dict) or any(not isinstance(rule.get(k), str) or not rule[k].strip() for k in ("from", "to")):
invalid(path, "alias_pair_required")
key = rule["from"].strip().lower()
rside = str(rule.get("side") or "").strip().lower()
if rside not in SIDES:
invalid(path, "unknown_alias_side")
if rule.get("vendor") is not None and not isinstance(rule["vendor"], str):
invalid(path, "alias_vendor_string_required")
key = (rule["from"].strip().lower(), "" if rside == "both" else rside,
str(rule.get("vendor") or "").strip().lower())
if key in seen:
invalid(path, "duplicate_alias")
seen.add(key)

36
netx_api/biz_state/data/default_zte_status_template.json
View file

@ -131,9 +131,25 @@
],
"row_filters": [
{
"op": "age_timer",
"field": "age",
"value": ""
"any": [
{
"field": "entry_type",
"op": "eq",
"value": "dynamic"
},
{
"all": [
{
"field": "entry_type",
"op": "empty"
},
{
"field": "age",
"op": "age_timer"
}
]
}
]
}
],
"field_rules": [
@ -415,11 +431,7 @@
],
"compare_fields": [
"admin",
"input_bps",
"output_bps",
"port_status",
"in_util",
"out_util",
"ip_mtu",
"mtu",
"mpls_mtu",
@ -454,7 +466,7 @@
},
{
"field": "output_bps",
"compare": "numeric",
"compare": "percent",
"tolerance": 5
},
{
@ -593,8 +605,8 @@
"field_rules": [
{
"field": "rx_power",
"compare": "percent",
"tolerance": 5
"compare": "numeric",
"tolerance": 2
}
]
},
@ -668,7 +680,9 @@
"vpn_sid",
"neighbor_sid"
],
"iface_fields": [],
"iface_fields": [
"ac_port"
],
"compare_fields": [],
"display_fields": [
"mac",

454
netx_api/biz_state/identity_pipeline.py Normal file
View file

@ -0,0 +1,454 @@
"""One identity pipeline for every compare path.
A row's *match key* must come out identical whether it is computed while a run
executes, while its results are searched / hydrated / exported, while a mapping
is validated, or while a cutover monitor evaluates the same sheet. All of those
paths therefore build a :class:`Pipeline` and call the same methods instead of
re-implementing "normalize → map → key".
Stages for one row on one side (``before`` or ``after``)::
raw row ──normalize──▶ iface aliases (+vendor canonical) ▶ key normalize ▶ value_map
──filter────▶ ``row_filters`` (v2: on the normalized row; legacy: on the raw row)
──map───────▶ port map (before side only)
──key───────▶ ``key_tuple`` over the match key fields
``legacy=True`` reproduces the pre-pipeline behaviour (snapshot v1 runs): the
template alias rules are the only normalization, keys are never normalized,
there is no value map and the filters see the raw row.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Mapping, Sequence
from .compare_rules import (
normalize_value,
row_matches_filter,
scalar_text,
)
from .iface_normalize import (
normalize_iface_name,
normalize_iface_rules,
normalize_port_map,
resolve_mapped_iface,
rules_for_side,
vendor_canonical_rules,
)
SIDES = ("before", "after")
DIRECTIONAL_MODES = {"not_worse", "numeric_min", "numeric_max"}
UNMAPPED_POLICIES = ("by_name", "ignore_port")
DUPLICATE_PAIRING = ("sorted", "ordered")
def filter_fields(filt: Mapping[str, Any] | None) -> set[str]:
"""Every field name a (possibly nested) row filter reads."""
out: set[str] = set()
if not isinstance(filt, Mapping):
return out
for group in ("any", "all"):
if group in filt:
for kid in filt.get(group) or []:
out |= filter_fields(kid)
return out
name = str(filt.get("field") or "").strip()
if name:
out.add(name)
return out
def _vendor_key(vendor: str = "", device_type: str = "") -> str:
from ..topology_lldp import resolve_vendor_key
key = resolve_vendor_key(vendor, device_type)
return "" if key == "generic" else key
@dataclass(eq=False)
class SideSpec:
side: str
vendor: str = ""
canonical: list[dict[str, str]] = field(default_factory=list)
alias_rules: list[dict[str, str]] = field(default_factory=list)
# field -> {lowercased source value: canonical value}
value_maps: dict[str, dict[str, str]] = field(default_factory=dict)
# key field -> normalize mode (only fields whose rule asks for one)
key_norms: dict[str, str] = field(default_factory=dict)
strip_inner_space: bool = True
def iface(self, name: Any) -> str:
text = str(name or "").strip()
if not text:
return ""
if self.strip_inner_space:
text = "".join(text.split())
if self.canonical:
text = normalize_iface_name(text, self.canonical)
if self.alias_rules:
text = normalize_iface_name(text, self.alias_rules)
return text
@property
def has_iface_rules(self) -> bool:
return bool(self.canonical or self.alias_rules)
def describe(self) -> dict[str, Any]:
"""JSON-able record of the rules in force (stored in the run snapshot)."""
return {
"vendor": self.vendor,
"canonical": list(self.canonical),
"alias_rules": list(self.alias_rules),
"value_maps": {k: dict(v) for k, v in self.value_maps.items()},
"key_norms": dict(self.key_norms),
}
@dataclass(eq=False)
class Pipeline:
key_fields: list[str]
iface_fields: list[str]
compare_fields: list[str]
field_rules: list[dict[str, Any]]
row_filters: list[dict[str, Any]]
port_map: dict[str, str]
before: SideSpec
after: SideSpec
legacy: bool = False
ignore_port_changes: bool | None = None
unmapped_port_policy: str = "by_name"
duplicate_pairing: str = "ordered"
_iface_set: set[str] = field(default_factory=set, repr=False)
_transformed: set[str] = field(default_factory=set, repr=False)
_targets: set[str] = field(default_factory=set, repr=False)
def __post_init__(self) -> None:
self._iface_set = set(self.iface_fields)
self._transformed = set(self._iface_set)
for spec in (self.before, self.after):
self._transformed |= set(spec.value_maps) | set(spec.key_norms)
self._targets = set(self.port_map.values())
# -- side helpers ----------------------------------------------------
def spec(self, side: str) -> SideSpec:
return self.before if side == "before" else self.after
# -- stage 1: normalize ----------------------------------------------
def normalize(self, row: Mapping[str, Any], side: str) -> dict[str, Any]:
out = dict(row)
spec = self.spec(side)
for f in self.iface_fields:
val = scalar_text(out.get(f)).strip()
if val:
out[f] = spec.iface(val)
if self.legacy:
return out
for f, mode in spec.key_norms.items():
if f in out:
out[f] = normalize_value(out[f], mode)
for f, vmap in spec.value_maps.items():
if f not in out:
continue
raw = scalar_text(out[f]).strip()
hit = vmap.get(raw.lower())
if hit is not None:
out[f] = hit
return out
def normalize_name(self, name: Any, side: str) -> str:
"""Interface-name normalization only (port-map keys / values, selectors)."""
return self.spec(side).iface(name)
# -- stage 2: filter -------------------------------------------------
def filters_for(self, side: str) -> list[dict[str, Any]]:
out = []
for f in self.row_filters:
fs = str(f.get("side") or "").strip().lower()
if not fs or fs == "both" or fs == side:
out.append(f)
return out
def keep(self, row: Mapping[str, Any], side: str) -> bool:
"""Evaluate row filters. v2 expects the *normalized* row, legacy the raw one."""
return all(row_matches_filter(row, f) for f in self.filters_for(side))
def accepts(self, raw: Mapping[str, Any], norm: Mapping[str, Any], side: str) -> bool:
"""Row-filter verdict: legacy reads the raw row, v2 the normalized one."""
return self.keep(raw if self.legacy else norm, side)
def pushdown_filters(self, side: str) -> list[dict[str, Any]]:
"""Filters that read only untransformed fields, so they may run in SQL on raw rows."""
if self.legacy:
return list(self.row_filters)
return [
f for f in self.row_filters
if not str(f.get("side") or "").strip()
and not (filter_fields(f) & self._transformed)
]
def prepare(self, rows: Sequence[Mapping[str, Any]], side: str) -> list[tuple[dict[str, Any], dict[str, Any]]]:
"""``[(original row, normalized row)]`` after filtering, in input order."""
out: list[tuple[dict[str, Any], dict[str, Any]]] = []
has_filters = bool(self.filters_for(side))
for raw in rows:
if not isinstance(raw, dict):
continue
norm = self.normalize(raw, side)
if has_filters and not self.accepts(raw, norm, side):
continue
out.append((raw, norm))
return out
# -- stage 3: port map -----------------------------------------------
def apply_port_map(self, norm_row: Mapping[str, Any]) -> dict[str, Any]:
out = dict(norm_row)
if not self.port_map:
return out
for f in self.iface_fields:
val = scalar_text(out.get(f)).strip()
if val:
out[f] = resolve_mapped_iface(val, self.port_map)
return out
def is_mapped(self, norm_row: Mapping[str, Any], side: str) -> bool:
"""True when every interface value of the row is covered by the port map."""
if not self.iface_fields:
return True
values = [scalar_text(norm_row.get(f)).strip() for f in self.iface_fields]
values = [v for v in values if v]
if not values:
return False
for v in values:
if side == "before":
if resolve_mapped_iface(v, self.port_map) == v and v not in self.port_map:
return False
else:
if not self._after_value_is_target(v):
return False
return True
def _after_value_is_target(self, value: str) -> bool:
if value in self._targets:
return True
parts = value.split(".")
return any(".".join(parts[:i]) in self._targets for i in range(len(parts) - 1, 0, -1))
def transform(self, row: Mapping[str, Any], side: str) -> dict[str, Any]:
norm = self.normalize(row, side)
return self.apply_port_map(norm) if side == "before" else norm
# -- stage 4: key ----------------------------------------------------
def effective_match_keys(self, *, unique_without_iface: bool | None = None) -> list[str]:
"""Match-key fields once the port policy is resolved.
With a port map the full key is used (the map aligns the names). Without
one, ``ignore_port_changes`` may drop the interface fields; ``None`` means
"auto", resolved by the caller via ``unique_without_iface``.
"""
keys = list(self.key_fields)
if self.port_map or not self._iface_set:
return keys
candidate = [k for k in keys if k not in self._iface_set]
if not candidate:
return keys
policy = self.ignore_port_changes
if policy is True:
return candidate
if policy is False:
return keys
return candidate if unique_without_iface else keys
def ignores_ports_by_policy(self) -> bool:
return bool(self.port_map) and self.unmapped_port_policy == "ignore_port" and bool(self._iface_set)
def key_of(
self, norm_row: Mapping[str, Any], side: str, match_keys: Sequence[str],
) -> tuple[dict[str, Any], tuple[str, ...]]:
"""``(row after port map, match key)`` for a *normalized* row."""
row = self.apply_port_map(norm_row) if side == "before" else dict(norm_row)
if self.ignores_ports_by_policy():
if not self.is_mapped(norm_row, side):
reduced = [k for k in match_keys if k not in self._iface_set] or list(match_keys)
return row, ("U",) + tuple(scalar_text(row.get(k)).strip() for k in reduced)
return row, ("M",) + tuple(scalar_text(row.get(k)).strip() for k in match_keys)
return row, tuple(scalar_text(row.get(k)).strip() for k in match_keys)
def match_key(
self, row: Mapping[str, Any], side: str, match_keys: Sequence[str] | None = None,
) -> tuple[str, ...]:
"""Match key of a *raw* row (normalize → port map → key)."""
keys = list(match_keys) if match_keys is not None else self.effective_match_keys()
return self.key_of(self.normalize(row, side), side, keys)[1]
# -- duplicate pairing -----------------------------------------------
def pair_sort_key(self, norm_row: Mapping[str, Any]) -> tuple[str, ...]:
rules = {str(r.get("field")): r for r in self.field_rules if isinstance(r, dict)}
out = []
for f in self.compare_fields:
mode = str((rules.get(f) or {}).get("normalize") or "strip")
out.append(normalize_value(norm_row.get(f), mode))
return tuple(out)
# -- SQL eligibility -------------------------------------------------
def sql_blockers(self, used_fields: set[str]) -> str:
"""Reason the SQL engine must not run this sheet ('' when it may)."""
if self.legacy:
return ""
for spec in (self.before, self.after):
if set(spec.value_maps) & used_fields:
return "value_map"
if set(spec.key_norms) & set(self.key_fields):
return "key_normalize"
if used_fields & self._iface_set and (self.before.has_iface_rules or self.after.has_iface_rules):
return "iface_normalize"
if any(str(f.get("side") or "").strip() for f in self.row_filters):
return "row_filters_side"
if any(filter_fields(f) & self._transformed for f in self.row_filters):
return "row_filters_post_normalize"
if any(
str(r.get("compare") or "").strip().lower() in DIRECTIONAL_MODES
for r in self.field_rules if isinstance(r, dict)
):
return "directional_compare"
return ""
def describe(self) -> dict[str, Any]:
return {
"legacy": self.legacy,
"before": self.before.describe(),
"after": self.after.describe(),
"unmapped_port_policy": self.unmapped_port_policy,
"duplicate_pairing": self.duplicate_pairing,
}
def _field_rule_specs(
field_rules: Sequence[Mapping[str, Any]], key_fields: Sequence[str], side: str,
) -> tuple[dict[str, dict[str, str]], dict[str, str]]:
value_maps: dict[str, dict[str, str]] = {}
key_norms: dict[str, str] = {}
key_set = set(key_fields)
for rule in field_rules or []:
if not isinstance(rule, Mapping):
continue
name = str(rule.get("field") or "").strip()
if not name:
continue
rs = str(rule.get("side") or "").strip().lower()
if rs and rs != "both" and rs != side:
continue
apply_to_key = rule.get("apply_to_key")
apply_to_key = (name in key_set) if apply_to_key is None else bool(apply_to_key)
is_key = name in key_set
if is_key and not apply_to_key:
continue
mode = str(rule.get("normalize") or "").strip().lower()
vmap = rule.get("value_map")
if is_key and mode and mode not in ("none", "strip"):
key_norms[name] = mode
if isinstance(vmap, Mapping) and vmap:
pre = mode if (mode and mode not in ("none", "strip")) else "strip"
built: dict[str, str] = {}
for src, dst in vmap.items():
built[normalize_value(src, pre).lower()] = scalar_text(dst).strip()
value_maps[name] = built
return value_maps, key_norms
def build_pipeline(
sheet: Mapping[str, Any],
*,
iface_rules: Sequence[Mapping[str, Any]] | None,
port_map: Mapping[str, str] | None,
before_vendor: str = "",
after_vendor: str = "",
legacy: bool = False,
normalize_map: bool = True,
) -> Pipeline:
"""Build the pipeline for one sheet.
``port_map`` is the map as the user wrote it. Unless ``normalize_map`` is
false (legacy runs, whose maps were written post-normalization) its keys are
normalized with the *before* side rules and its values with the *after* side
rules.
"""
key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()]
iface_fields = [str(k).strip() for k in (sheet.get("iface_fields") or []) if str(k).strip()]
field_rules = [dict(r) for r in (sheet.get("field_rules") or []) if isinstance(r, Mapping)]
from .compare_rules import effective_compare_fields
compare_fields = effective_compare_fields(list(sheet.get("compare_fields") or []), field_rules)
canonical_on = sheet.get("builtin_vendor_canonical", True) is not False and not legacy
all_rules = normalize_iface_rules(list(iface_rules or []))
specs: dict[str, SideSpec] = {}
for side, vendor_raw in (("before", before_vendor), ("after", after_vendor)):
vendor = _vendor_key(vendor_raw) if vendor_raw else ""
if legacy:
alias = normalize_iface_rules([{"from": r["from"], "to": r["to"]} for r in all_rules])
value_maps, key_norms = {}, {}
else:
alias = rules_for_side(all_rules, side, vendor)
value_maps, key_norms = _field_rule_specs(field_rules, key_fields, side)
specs[side] = SideSpec(
side=side, vendor=vendor,
canonical=vendor_canonical_rules(vendor) if canonical_on else [],
alias_rules=alias, value_maps=value_maps, key_norms=key_norms,
strip_inner_space=not legacy,
)
pmap = dict(port_map or {})
if pmap and normalize_map and not legacy:
pmap, _ = normalize_port_map(
pmap, None,
key_normalizer=specs["before"].iface, value_normalizer=specs["after"].iface,
)
ignore_ports = sheet.get("ignore_port_changes")
policy = str(sheet.get("unmapped_port_policy") or "by_name").strip().lower()
pairing = str(sheet.get("duplicate_pairing") or ("ordered" if legacy else "sorted")).strip().lower()
return Pipeline(
key_fields=key_fields,
iface_fields=iface_fields,
compare_fields=compare_fields,
field_rules=field_rules,
row_filters=[dict(f) for f in (sheet.get("row_filters") or []) if isinstance(f, Mapping)],
port_map=pmap,
before=specs["before"],
after=specs["after"],
legacy=legacy,
ignore_port_changes=None if ignore_ports is None else bool(ignore_ports),
unmapped_port_policy=policy if policy in UNMAPPED_POLICIES else "by_name",
duplicate_pairing="ordered" if legacy else (pairing if pairing in DUPLICATE_PAIRING else "sorted"),
)
def legacy_pipeline(
*,
key_fields: Sequence[str],
iface_fields: Sequence[str],
compare_fields: Sequence[str],
field_rules: Sequence[Mapping[str, Any]] | None = None,
port_map: Mapping[str, str] | None = None,
iface_normalize_rules: Sequence[Mapping[str, Any]] | None = None,
ignore_port_changes: bool | None = None,
row_filters: Sequence[Mapping[str, Any]] | None = None,
duplicate_pairing: str = "ordered",
) -> Pipeline:
"""Pipeline with the pre-v2 semantics, for snapshot-v1 runs and bare callers."""
alias = normalize_iface_rules(
[{"from": r.get("from"), "to": r.get("to")} for r in (iface_normalize_rules or [])]
)
return Pipeline(
key_fields=[str(k) for k in key_fields],
iface_fields=[str(f) for f in iface_fields if str(f).strip()],
compare_fields=[str(f) for f in compare_fields],
field_rules=[dict(r) for r in (field_rules or []) if isinstance(r, Mapping)],
row_filters=[dict(f) for f in (row_filters or []) if isinstance(f, Mapping)],
port_map=dict(port_map or {}),
before=SideSpec(side="before", alias_rules=list(alias), strip_inner_space=False),
after=SideSpec(side="after", alias_rules=list(alias), strip_inner_space=False),
legacy=True,
ignore_port_changes=ignore_port_changes,
duplicate_pairing=duplicate_pairing if duplicate_pairing in DUPLICATE_PAIRING else "ordered",
)

100
netx_api/biz_state/iface_normalize.py
View file

@ -39,7 +39,14 @@ def normalize_iface_rules(raw: Any) -> list[dict[str, str]]:
fr = str(item.get("from") or item.get("src") or "").strip()
to = str(item.get("to") or item.get("dst") or "").strip()
if fr and to:
out.append({"from": fr, "to": to})
rule = {"from": fr, "to": to}
side = str(item.get("side") or "").strip().lower()
if side in ("before", "after"):
rule["side"] = side
vendor = str(item.get("vendor") or "").strip().lower()
if vendor:
rule["vendor"] = vendor
out.append(rule)
# Longest prefix first so XXVGE wins over XGE / GE.
out.sort(key=lambda r: len(r["from"]), reverse=True)
return out
@ -107,6 +114,47 @@ def apply_iface_normalize_rows(
]
def normalize_port_map(
port_map: Mapping[str, str] | None,
rules: Sequence[Mapping[str, str]] | None,
value_rules: Sequence[Mapping[str, str]] | None = None,
*,
key_normalizer: Any = None,
value_normalizer: Any = None,
) -> tuple[dict[str, str], list[dict[str, str]]]:
"""Normalize port-map keys and values with the same rules as iface fields.
``value_rules`` (default: ``rules``) normalizes the after-side values, for
per-side rules. ``key_normalizer`` / ``value_normalizer`` are callables that
replace the rule-based normalization (used by the identity pipeline).
Returns ``(normalized_map, conflicts)``. A conflict is two raw keys that
normalize to the same key but point at different (normalized) targets; the
first one wins in ``normalized_map``.
"""
out: dict[str, str] = {}
origin: dict[str, str] = {}
conflicts: list[dict[str, str]] = []
for raw_k, raw_v in (port_map or {}).items():
k = key_normalizer(raw_k) if key_normalizer else normalize_iface_name(raw_k, rules)
v = (
value_normalizer(raw_v) if value_normalizer
else normalize_iface_name(raw_v, rules if value_rules is None else value_rules)
)
if not k or not v:
continue
if k in out:
if out[k] != v:
conflicts.append({
"before_if": k, "kept_from": origin[k], "kept_to": out[k],
"dropped_from": str(raw_k), "dropped_to": v,
})
continue
out[k] = v
origin[k] = str(raw_k)
return out, conflicts
def resolve_mapped_iface(name: str, port_map: Mapping[str, str] | None) -> str:
"""Exact map hit, else longest mapped parent + remaining suffix (QinQ-safe).
@ -142,3 +190,53 @@ def default_zte_iface_normalize_rules() -> list[dict[str, str]]:
{"from": "SG", "to": "smartgroup"},
]
)
# ---------------------------------------------------------------------------
# Built-in per-vendor canonical names (same-vendor synonyms only; cross-vendor
# numbering is the port map's job). Applied before template alias rules.
# ---------------------------------------------------------------------------
VENDOR_CANONICAL: dict[str, list[dict[str, str]]] = {
"huawei": [
{"from": "XGigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"},
{"from": "100GE", "to": "100GE"}, {"from": "40GE", "to": "40GE"}, {"from": "25GE", "to": "25GE"},
{"from": "10GE", "to": "10GE"}, {"from": "Eth-Trunk", "to": "Eth-Trunk"},
],
"cisco": [
{"from": "GigabitEthernet", "to": "Gi"}, {"from": "TenGigabitEthernet", "to": "Te"},
{"from": "TenGigE", "to": "Te"}, {"from": "HundredGigE", "to": "Hu"}, {"from": "FortyGigE", "to": "Fo"},
{"from": "TwentyFiveGigE", "to": "Twe"}, {"from": "FastEthernet", "to": "Fa"},
{"from": "Bundle-Ether", "to": "BE"}, {"from": "Loopback", "to": "Lo"},
],
"h3c": [
{"from": "Ten-GigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"},
{"from": "FortyGigE", "to": "FGE"}, {"from": "HundredGigE", "to": "HGE"},
{"from": "Bridge-Aggregation", "to": "BAGG"},
],
"zte": [
{"from": "xxvgei", "to": "xxvgei"}, {"from": "xgei", "to": "xgei"}, {"from": "cgei", "to": "cgei"},
{"from": "gei", "to": "gei"}, {"from": "smartgroup", "to": "smartgroup"},
],
}
def vendor_canonical_rules(vendor: str) -> list[dict[str, str]]:
return normalize_iface_rules(VENDOR_CANONICAL.get(str(vendor or "").strip().lower(), []))
def rules_for_side(
rules: Sequence[Mapping[str, str]] | None, side: str, vendor: str = "",
) -> list[dict[str, str]]:
"""Template alias rules that apply to ``side`` (``before``/``after``) and ``vendor``."""
vendor_n = str(vendor or "").strip().lower()
out: list[dict[str, str]] = []
for r in rules or []:
rs = str(r.get("side") or "").strip().lower()
if rs and rs != side:
continue
rv = str(r.get("vendor") or "").strip().lower()
if rv and rv != vendor_n:
continue
out.append({"from": str(r.get("from") or ""), "to": str(r.get("to") or "")})
return normalize_iface_rules(out)

39
netx_api/biz_state/profiles.py
View file

@ -85,6 +85,9 @@ class ParseProfile:
enrich_joins: list[EnrichJoin] = field(default_factory=list)
# light: default shared SSH lane; heavy: dedicated long-timeout connection
collect_lane: str = "light"
# Which slice of the metric this command produces, e.g. {"afi": "ipv4"}.
# Empty = the command covers the whole metric (or its slicing is unknown).
slice: dict[str, str] = field(default_factory=dict)
_LLDP_FIELDS: list[FieldDef] = [
@ -1651,6 +1654,11 @@ _HEAVY_LANE_METRICS = frozenset(
)
def metric_lane(metric_id: str) -> str:
"""``heavy`` for huge CLI dumps (own SSH connection), else ``light``."""
return "heavy" if str(metric_id or "").strip() in _HEAVY_LANE_METRICS else "light"
def _apply_collect_lanes(profiles: list[ParseProfile]) -> list[ParseProfile]:
for p in profiles:
if str(p.metric_id or "").strip() in _HEAVY_LANE_METRICS:
@ -1752,12 +1760,41 @@ def _validate_profiles(profiles: list[ParseProfile]) -> None:
)
# profile_id -> slice. Compare sheets filter on these fields (``afi``/``af``/``direction``),
# so a missing command is detected as "slice not collected" instead of "business lost".
_PROFILE_SLICES: dict[str, dict[str, str]] = {
**{f"zte.bgp_{afi}_summary": {"afi": afi} for afi in ("vpnv4", "ipv4", "vpnv6", "ipv6", "evpn", "vpls")},
"zte.bgp_vpnv4_vrf_summary": {"afi": "vpnv4"},
"zte.bgp_vpnv6_vrf_summary": {"afi": "vpnv6"},
"zte.vrrp_ipv4": {"af": "ipv4"},
"zte.vrrp_ipv6": {"af": "ipv6"},
**{
f"zte.bgp_{afi}_neighbor_{direction}": {"afi": afi, "direction": direction}
for afi in ("vpnv4", "vpnv6", "ipv4", "ipv6")
for direction in ("in", "out")
},
**{
f"zte.bgp_{afi}_vrf_neighbor_{direction}": {"afi": afi, "direction": direction}
for afi in ("vpnv4", "vpnv6")
for direction in ("in", "out")
},
}
def _apply_profile_slices(profiles: list[ParseProfile]) -> list[ParseProfile]:
for p in profiles:
declared = _PROFILE_SLICES.get(p.profile_id)
if declared and not p.slice:
p.slice = dict(declared)
return profiles
def _finalize_profiles(profiles: list[ParseProfile]) -> list[ParseProfile]:
"""Normalize discover→aux, validate, apply collect lanes."""
for p in profiles:
_normalize_discover_aux(p)
_validate_profiles(profiles)
return _apply_collect_lanes(profiles)
return _apply_profile_slices(_apply_collect_lanes(profiles))
def all_profiles() -> list[ParseProfile]:

100
netx_api/biz_state/retention.py
View file

@ -6,13 +6,17 @@ from collections import defaultdict
from datetime import datetime, timedelta
from typing import Any
from sqlalchemy import or_, select
from sqlalchemy import func, or_, select
from sqlalchemy.orm import Session
from ..models import (
BizCompareDiff,
BizCompareJob,
BizCompareRun,
BizMigrationBatch,
BizMigrationDiff,
BizMigrationProject,
BizMigrationRedTicket,
BizMigrationRun,
BizStateBatch,
BizStateBatchCommand,
@ -76,9 +80,103 @@ def protected_batch_map(
add(before_id, before_reason)
add(after_id, after_reason)
# Every batch a *kept* acceptance/manual run (or the newest auto run of a wave)
# compared, not only the first binding. Older auto runs are bounded by
# ``purge_auto_runs`` and keep protecting just their primary pair above.
for run_batches in _kept_run_metric_batches(db):
for bid in run_batches:
add(bid, "migration_run_metric")
return dict(out)
def _kept_run_metric_batches(db: Session, *, limit: int = 500) -> list[set[str]]:
newest_auto = (
db.query(func.max(BizMigrationRun.created_at).label("at"), BizMigrationRun.batch_id.label("bid"))
.filter(BizMigrationRun.purpose == "auto").group_by(BizMigrationRun.batch_id).subquery()
)
latest_auto_ids = [
r[0] for r in db.query(BizMigrationRun.id).join(
newest_auto,
(BizMigrationRun.batch_id == newest_auto.c.bid) & (BizMigrationRun.created_at == newest_auto.c.at),
).limit(limit).all()
]
kept_ids = [
r[0] for r in db.query(BizMigrationRun.id).filter(BizMigrationRun.purpose != "auto")
.order_by(BizMigrationRun.created_at.desc()).limit(limit).all()
] + latest_auto_ids
out: list[set[str]] = []
for start in range(0, len(kept_ids), 100):
for (summary,) in db.query(BizMigrationRun.summary_json).filter(
BizMigrationRun.id.in_(kept_ids[start:start + 100])
):
batches: set[str] = set()
for item in ((summary or {}).get("metric_batches") or {}).values():
if isinstance(item, dict):
batches.update(str(item.get(k) or "") for k in ("old_batch_id", "new_batch_id"))
batches.discard("")
if batches:
out.append(batches)
return out
def purge_auto_runs(db: Session, *, migration_keep: int = 300, compare_keep: int = 50) -> dict[str, int]:
"""Bound auto-generated run history.
Cutover monitoring writes a run (+ one diff row per object) every minute and
auto-compare writes one per new sample. Acceptance / manual runs and runs a
red ticket points at are kept forever; only the newest ``keep`` auto runs of
each wave / job survive.
"""
dropped = {"migration_runs": 0, "compare_runs": 0}
ticket_runs = {r[0] for r in db.query(BizMigrationRedTicket.run_id).filter(
BizMigrationRedTicket.run_id != "").distinct()}
accepted = {r[0] for r in db.query(BizMigrationBatch.accept_run_id).filter(
BizMigrationBatch.accept_run_id != "")}
keep_forever = ticket_runs | accepted
def surplus(model, group_col, extra_filter, keep: int) -> list[str]:
over = [g for g, n in db.query(group_col, func.count(model.id)).filter(*extra_filter)
.group_by(group_col).all() if n > keep]
ids: list[str] = []
for g in over:
rows = db.query(model.id).filter(group_col == g, *extra_filter).order_by(
model.created_at.desc(), model.id.desc()).offset(keep).all()
ids.extend(r[0] for r in rows if r[0] not in keep_forever)
return ids
mig_ids = surplus(BizMigrationRun, BizMigrationRun.batch_id, [BizMigrationRun.purpose == "auto"],
max(1, int(migration_keep)))
for start in range(0, len(mig_ids), 500):
chunk = mig_ids[start:start + 500]
db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id.in_(chunk)).delete(synchronize_session=False)
db.query(BizMigrationRun).filter(BizMigrationRun.id.in_(chunk)).delete(synchronize_session=False)
dropped["migration_runs"] = len(mig_ids)
# Auto-mode jobs only. A run's origin lives in summary_json["purpose"]; unmarked runs
# are manual and stay. Only the surplus beyond ``compare_keep`` is inspected.
cmp_ids: list[str] = []
keep_n = max(1, int(compare_keep))
for (job_id,) in db.query(BizCompareJob.id).filter(BizCompareJob.mode == "auto").all():
surplus_ids = [r[0] for r in db.query(BizCompareRun.id).filter(
BizCompareRun.job_id == job_id, BizCompareRun.status.notin_(("queued", "running")),
).order_by(BizCompareRun.created_at.desc(), BizCompareRun.id.desc()).offset(keep_n).all()]
for start in range(0, len(surplus_ids), 200):
for rid, summary in db.query(BizCompareRun.id, BizCompareRun.summary_json).filter(
BizCompareRun.id.in_(surplus_ids[start:start + 200])
):
if (summary or {}).get("purpose") == "auto":
cmp_ids.append(rid)
for start in range(0, len(cmp_ids), 500):
chunk = cmp_ids[start:start + 500]
db.query(BizCompareDiff).filter(BizCompareDiff.run_id.in_(chunk)).delete(synchronize_session=False)
db.query(BizCompareRun).filter(BizCompareRun.id.in_(chunk)).delete(synchronize_session=False)
dropped["compare_runs"] = len(cmp_ids)
if mig_ids or cmp_ids:
db.commit()
return dropped
def protected_batch_ids(db: Session, *, task_id: str = "") -> set[str]:
return set(protected_batch_map(db, task_id=task_id).keys())

24
netx_api/biz_state_router.py
View file

@ -549,6 +549,12 @@ class FieldRuleIn(BaseModel):
normalize: str = ""
ignore: bool = False
tolerance: float | None = None
# Cross-vendor identity rules (see identity_pipeline).
value_map: dict[str, str] | None = None
side: str = ""
apply_to_key: bool | None = None
# Order for compare=not_worse: earlier entries are better.
rank: list[str] | None = None
class RowFilterIn(BaseModel):
@ -575,6 +581,9 @@ class TemplateMetricIn(BaseModel):
row_filters: list[dict[str, Any]] = Field(default_factory=list)
field_rules: list[FieldRuleIn] = Field(default_factory=list)
ignore_port_changes: bool | None = None
unmapped_port_policy: str = ""
duplicate_pairing: str = ""
builtin_vendor_canonical: bool | None = None
class TemplateIn(BaseModel):
@ -623,6 +632,16 @@ class MappingIn(BaseModel):
rows: list[MappingRowIn] = Field(default_factory=list)
class PreflightIn(BaseModel):
template_id: str = ""
enabled_sheet_ids: list[str] = Field(default_factory=list)
mapping_id: str = ""
# Unsaved editor rows win over ``mapping_id`` when given.
mapping_rows: list[MappingRowIn] | None = None
before_batch_id: str
after_batch_id: str
class ValidateMappingIn(BaseModel):
mapping_id: str
before_batch_id: str
@ -711,6 +730,11 @@ def api_validate_mapping(body: ValidateMappingIn, db: Session = Depends(get_db))
)
@router.post("/compare/preflight")
def api_compare_preflight(body: PreflightIn, db: Session = Depends(get_db)) -> dict[str, Any]:
return cmp_svc.preflight_compare(db, body.model_dump())
@router.get("/compare/jobs")
def api_list_jobs(db: Session = Depends(get_db)) -> dict[str, Any]:
return {"items": cmp_svc.list_jobs(db)}

9
netx_api/biz_state_scheduler.py
View file

@ -47,6 +47,15 @@ def _maybe_purge_retention() -> None:
from .biz_state.retention import purge_all_tasks
info = purge_all_tasks(db)
from .biz_state.retention import purge_auto_runs
runs = purge_auto_runs(
db,
migration_keep=int(getattr(settings, "biz_migration_auto_run_keep", 300) or 300),
compare_keep=int(getattr(settings, "biz_compare_auto_run_keep", 50) or 50),
)
if any(runs.values()):
_log.info("biz_state periodic run purge %s", runs)
if info.get("dropped"):
_log.info(
"biz_state periodic retention dropped=%s tasks=%s",

7
netx_api/config.py
View file

@ -137,6 +137,13 @@ class Settings(BaseSettings):
biz_state_dedicated_workers: bool = True
# Global ceiling for simultaneous running batches (across all workers).
biz_state_max_concurrent_tasks: int = 16
# Auto-generated run history kept per cutover wave / auto compare job (acceptance + manual runs are never purged).
biz_migration_auto_run_keep: int = 300
biz_compare_auto_run_keep: int = 50
# Rows of immutable baseline batches cached across cutover evaluations (0 disables).
biz_migration_baseline_cache_rows: int = 500000
# Concurrent SSH collections per network element (0 = unlimited).
biz_state_max_sessions_per_ne: int = 2
biz_state_worker_collect_threads: int = 8
biz_state_persist_workers: int = 4
# TextFSM / normalize workers (off SSH collect threads). Default 8.