hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-09 04:20:45 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 2

Allow compare templates to split one collected metric into filtered sheets.

Browse source 
BGP/ISIS (and any other metric) can share a metric_id with distinct sheet_id + row_filters; drop conflicting bgp state_or_pfx and keep state/pfx_rcd.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-19 01:46:03 +08:00
parent 88b7558a48
commit a65c9fc420
17 changed files with 745 additions and 230 deletions
Download patch file Download diff file Expand all files Collapse all files

54
netx_api/biz_migration/evaluate.py
View file

@ -79,9 +79,10 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]:
if not isinstance(it, dict):
continue
mid = str(it.get("metric_id") or "").strip()
if not mid:
sid = str(it.get("sheet_id") or "").strip() or mid
if not sid:
continue
bucket = out.setdefault(mid, set())
bucket = out.setdefault(sid, set())
if it.get("key") is not None:
k = it.get("key")
if isinstance(k, (list, tuple)):
@ -747,10 +748,20 @@ def expect_keys_for_metric(
*,
metric_id: str,
iface_fields: list[str],
sheet_id: str = "",
) -> set[str]:
keys = set(expect.get(metric_id) or ())
"""Resolve expect keys for a compare sheet.
Prefer ``sheet_id`` (split tables); fall back to ``metric_id`` for legacy
unsplit sheets / expect items that only carry metric_id.
"""
sid = str(sheet_id or "").strip()
mid = str(metric_id or "").strip()
keys = set(expect.get(sid) or ()) if sid else set()
if not keys and mid:
keys = set(expect.get(mid) or ())
# ports: shorthand is only for interface_brief, not every sheet with iface fields
if metric_id == PORT_METRIC_ID and expect.get("_ports"):
if mid == PORT_METRIC_ID and expect.get("_ports"):
keys |= set(expect["_ports"])
return keys
@ -879,15 +890,34 @@ def _current_row(diff: dict[str, Any] | None) -> dict[str, Any]:
return dict(diff.get("after") or {})
def override_for_sheet(
sheet_overrides: list[dict[str, Any]] | None,
*,
sheet_id: str,
metric_id: str,
) -> dict[str, Any]:
"""Match a monitor override. sheet_id wins; a legacy metric-only override applies to every split."""
sid = str(sheet_id or "").strip()
mid = str(metric_id or "").strip()
legacy: dict[str, Any] | None = None
for ov in sheet_overrides or []:
if not isinstance(ov, dict):
continue
ov_sid = str(ov.get("sheet_id") or "").strip()
ov_mid = str(ov.get("metric_id") or "").strip()
if ov_sid and ov_sid == sid:
return ov
if not ov_sid and ov_mid and ov_mid in (sid, mid) and legacy is None:
legacy = ov
return legacy or {}
def override_for_metric(
sheet_overrides: list[dict[str, Any]] | None,
metric_id: str,
) -> dict[str, Any]:
mid = str(metric_id or "").strip()
for ov in sheet_overrides or []:
if isinstance(ov, dict) and str(ov.get("metric_id") or "").strip() == mid:
return ov
return {}
return override_for_sheet(sheet_overrides, sheet_id=mid, metric_id=mid)
def evaluate_metric_dual(
@ -907,9 +937,13 @@ def evaluate_metric_dual(
field_rules: list[dict[str, Any]] | None = None,
sheet_override: dict[str, Any] | None = None,
out_of_expect: str = "strict",
sheet_id: str = "",
) -> dict[str, Any]:
"""Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge."""
expect_keys = expect_keys_for_metric(expect, metric_id=metric_id, iface_fields=iface_fields)
sid = str(sheet_id or "").strip() or str(metric_id or "").strip()
expect_keys = expect_keys_for_metric(
expect, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid
)
ov = sheet_override or {}
if ov.get("skip_dual"):
return {
@ -1155,6 +1189,8 @@ def evaluate_metric_dual(
def port_sheet_def() -> dict[str, Any]:
"""Default sheet for port-status cutover monitor (interface_brief only)."""
return {
"sheet_id": PORT_METRIC_ID,
"title": PORT_METRIC_ID,
"metric_id": PORT_METRIC_ID,
"key_fields": ["interface"],
"iface_fields": ["interface"],

211
netx_api/biz_migration/monitor_templates.py
View file

@ -127,68 +127,89 @@ def _success_stateful(
}
def preset_override_for_metric(metric_id: str) -> dict[str, Any]:
"""Default dual-verdict overlay for a compare-template sheet."""
def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[str, Any]:
"""Default dual-verdict overlay for a compare-template sheet.
``metric_id`` is the collected source table; ``sheet_id`` identifies a
filtered split (e.g. bgp_peer.vpnv4). Presets key off the source metric.
"""
mid = str(metric_id or "").strip()
if mid == PORT_METRIC_ID:
sid = str(sheet_id or "").strip()
# Callers may pass only a split id like "bgp_peer.vpnv4"
if not mid and sid:
mid = sid.split(".", 1)[0]
if not sid:
sid = mid
look = mid.split(".", 1)[0] if "." in mid else mid
def _out(body: dict[str, Any]) -> dict[str, Any]:
body["metric_id"] = mid or look
if sid and sid != (mid or look):
body["sheet_id"] = sid
return body
if look == PORT_METRIC_ID:
up = [
{"type": "value", "field": "admin", "op": "in", "value": ["up"]},
{"type": "value", "field": "phy", "op": "in", "value": ["up"]},
]
return {
"metric_id": mid,
"status_fields": list(PORT_STATUS_FIELDS),
"down_values": ["down"],
"up_values": ["up"],
"success": [
_success_stateful(
old_down_groups=[
[
{"type": "value", "field": "admin", "op": "in", "value": ["down"]},
{"type": "value", "field": "phy", "op": "in", "value": ["down"]},
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state("admin", ["down"]),
}
if mid == "bgp_peer":
return _out(
{
"status_fields": list(PORT_STATUS_FIELDS),
"down_values": ["down"],
"up_values": ["up"],
"success": [
_success_stateful(
old_down_groups=[
[
{"type": "value", "field": "admin", "op": "in", "value": ["down"]},
{"type": "value", "field": "phy", "op": "in", "value": ["down"]},
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state("admin", ["down"]),
}
)
if look == "bgp_peer":
up = [{"type": "value", "field": "state", "op": "eq", "value": "established"}]
return {
"metric_id": mid,
"status_fields": ["state"],
"down_values": ["idle", "active", "connect", "down"],
"up_values": ["established"],
"success": [
_success_stateful(
old_down_groups=[
[
{
"type": "value",
"field": "state",
"op": "in",
"value": ["idle", "active", "connect", "down"],
}
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state(
"state", ["idle", "active", "connect", "down"]
),
}
if mid in ("arp", "nd6_cache", "lldp_neighbor"):
return {
"metric_id": mid,
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
if "isis" in mid or "ospf" in mid or "adjacency" in mid:
return _out(
{
"status_fields": ["state"],
"down_values": ["idle", "active", "connect", "down"],
"up_values": ["established"],
"success": [
_success_stateful(
old_down_groups=[
[
{
"type": "value",
"field": "state",
"op": "in",
"value": ["idle", "active", "connect", "down"],
}
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state(
"state", ["idle", "active", "connect", "down"]
),
}
)
if look in ("arp", "nd6_cache", "lldp_neighbor"):
return _out(
{
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
)
if "isis" in look or "ospf" in look or "adjacency" in look:
up = [
{
"type": "value",
@ -197,45 +218,48 @@ def preset_override_for_metric(metric_id: str) -> dict[str, Any]:
"value": ["up", "full", "2way"],
}
]
return {
"metric_id": mid,
"status_fields": ["state"],
"down_values": ["down", "init", "idle"],
"up_values": ["up", "full", "2way"],
"success": [
_success_stateful(
old_down_groups=[
[
{
"type": "value",
"field": "state",
"op": "in",
"value": ["down", "init", "idle"],
}
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state("state", ["down", "init", "idle"]),
}
if "route" in mid or "vrf" in mid:
return {
"metric_id": mid,
return _out(
{
"status_fields": ["state"],
"down_values": ["down", "init", "idle"],
"up_values": ["up", "full", "2way"],
"success": [
_success_stateful(
old_down_groups=[
[
{
"type": "value",
"field": "state",
"op": "in",
"value": ["down", "init", "idle"],
}
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state("state", ["down", "init", "idle"]),
}
)
if "route" in look or "vrf" in look:
return _out(
{
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
)
return _out(
{
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
return {
"metric_id": mid,
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
)
def ensure_default_monitor_templates(db: Session) -> None:
@ -264,7 +288,14 @@ def ensure_default_monitor_templates(db: Session) -> None:
]
if zte:
zte_sheets = cmp_svc.template_metrics(zte)
zte_overrides = [preset_override_for_metric(str(s.get("metric_id") or "")) for s in zte_sheets if s.get("metric_id")]
zte_overrides = [
preset_override_for_metric(
str(s.get("metric_id") or ""),
sheet_id=str(s.get("sheet_id") or s.get("metric_id") or ""),
)
for s in zte_sheets
if s.get("metric_id")
]
seeds.append(
BizMonitorTemplate(
id=uuid4().hex,

28
netx_api/biz_migration/service.py
View file

@ -9,7 +9,7 @@ from fastapi import HTTPException
from sqlalchemy.orm import Session
from ..biz_state.compare_rules import apply_row_filters
from ..biz_state.compare_service import _load_metric_rows, _port_map_dict, template_metrics
from ..biz_state.compare_service import _load_metric_rows, _port_map_dict, sheet_key, sheet_title, template_metrics
from ..models import (
BizCompareTemplate,
BizMigrationBatch,
@ -27,7 +27,7 @@ from . import monitor_templates as mon_tpl
from .evaluate import (
PORT_METRIC_ID,
evaluate_metric_dual,
override_for_metric,
override_for_sheet,
parse_expect_set,
port_sheet_def,
)
@ -124,7 +124,12 @@ def resolve_collect_metric_ids(db: Session, proj: BizMigrationProject) -> list[s
if collect:
return collect
sheets, _, _ = resolve_evaluate_sheets(db, mt)
return [str(s.get("metric_id") or "").strip() for s in sheets if str(s.get("metric_id") or "").strip()]
seen: list[str] = []
for s in sheets:
mid = str(s.get("metric_id") or "").strip()
if mid and mid not in seen:
seen.append(mid)
return seen
def project_to_dict(db: Session, p: BizMigrationProject) -> dict[str, Any]:
@ -372,6 +377,7 @@ def run_evaluate(
for sheet in sheets:
mid = str(sheet.get("metric_id") or "").strip()
sid = sheet_key(sheet)
key_fields = list(sheet.get("key_fields") or [])
if not mid or not key_fields:
continue
@ -379,7 +385,7 @@ def run_evaluate(
compare_fields = list(sheet.get("compare_fields") or [])
row_filters = list(sheet.get("row_filters") or [])
field_rules = list(sheet.get("field_rules") or [])
sheet_ov = override_for_metric(sheet_overrides, mid)
sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid)
if sheet_ov.get("skip_dual"):
continue
@ -402,6 +408,7 @@ def run_evaluate(
one = evaluate_metric_dual(
metric_id=mid,
sheet_id=sid,
key_fields=key_fields,
iface_fields=iface_fields,
compare_fields=compare_fields,
@ -420,7 +427,8 @@ def run_evaluate(
sheet_cards.append(
{
"metric_id": mid,
"title": mid,
"sheet_id": sid,
"title": sheet_title(sheet),
"progress_ok": one["progress_ok"],
"progress_total": one["progress_total"],
"anomaly": one["anomaly"],
@ -433,6 +441,7 @@ def run_evaluate(
)
for r in one["rows"]:
r["seq"] = seq
r["sheet_id"] = sid
seq += 1
all_rows.append(r)
v = str(r.get("verdict") or "")
@ -460,7 +469,7 @@ def run_evaluate(
"anomaly": sum(c["anomaly"] for c in sheet_cards),
"new_baseline_missing": any(bool(c.get("new_baseline_missing")) for c in sheet_cards),
"missing_metrics": [
str(c.get("metric_id") or "")
str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "")
for c in sheet_cards
if c.get("new_baseline_missing")
],
@ -499,6 +508,7 @@ def run_evaluate(
"old_status": r.get("old_status"),
"new_status": r.get("new_status"),
"rule_hit": r.get("rule_hit") or "",
"sheet_id": r.get("sheet_id") or "",
},
old_kind=str(r.get("old_kind") or ""),
new_kind=str(r.get("new_kind") or ""),
@ -545,6 +555,7 @@ def diff_to_dict(d: BizMigrationDiff) -> dict[str, Any]:
return {
"id": d.id,
"metric_id": d.metric_id,
"sheet_id": kj.get("sheet_id") or d.metric_id or "",
"seq": d.seq,
"verdict": d.verdict,
"color": d.color,
@ -610,10 +621,11 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any]
out_sheets: list[dict[str, Any]] = []
for sheet in sheets:
mid = str(sheet.get("metric_id") or "").strip()
sid = sheet_key(sheet)
key_fields = [str(k) for k in (sheet.get("key_fields") or []) if str(k).strip()]
if not mid or not key_fields:
continue
sheet_ov = override_for_metric(sheet_overrides, mid)
sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid)
if sheet_ov.get("skip_dual"):
continue
iface_fields = [str(k) for k in (sheet.get("iface_fields") or []) if str(k).strip()]
@ -643,6 +655,8 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any]
out_sheets.append(
{
"metric_id": mid,
"sheet_id": sid,
"title": sheet_title(sheet),
"key_fields": key_fields,
"iface_fields": iface_fields,
"items": items,

3
netx_api/biz_state/compare_rules.py
View file

@ -44,6 +44,9 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool:
if op in ("not_in", "nin"):
opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()}
return raw.lower() not in opts
if op == "contains":
needle = str(expect or "").strip().lower()
return bool(needle) and needle in raw.lower()
if op == "empty":
return not raw
if op in ("not_empty", "nonempty"):

191
netx_api/biz_state/compare_service.py
View file

@ -225,6 +225,8 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A
return [
{
"metric_id": run.metric_id,
"sheet_id": run.metric_id,
"title": run.metric_id,
"key_fields": list((tpl.key_fields if tpl else None) or []),
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
@ -285,10 +287,23 @@ def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]:
return out
def sheet_key(sheet: dict[str, Any] | None) -> str:
"""Unique compare-item id. Falls back to metric_id so old sheets stay valid."""
data = sheet or {}
return str(data.get("sheet_id") or data.get("metric_id") or "").strip()
def sheet_title(sheet: dict[str, Any] | None) -> str:
data = sheet or {}
return str(data.get("title") or "").strip() or sheet_key(data)
def _sheet_def(
*,
metric_id: str,
key_fields: list[str],
sheet_id: str | None = None,
title: str | None = None,
iface_fields: list[str] | None = None,
compare_fields: list[str] | None = None,
display_fields: list[str] | None = None,
@ -296,6 +311,8 @@ def _sheet_def(
field_rules: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
mid = str(metric_id or "").strip()
sid = str(sheet_id or "").strip() or mid
ttl = str(title or "").strip() or sid
keys = _str_list(key_fields)
ifaces = _str_list(iface_fields)
# compare_fields empty → presence-only mode (intentional)
@ -320,6 +337,8 @@ def _sheet_def(
display_fields=_str_list(display_fields),
)
sheet: dict[str, Any] = {
"sheet_id": sid,
"title": ttl,
"metric_id": mid,
"key_fields": keys,
"iface_fields": ifaces,
@ -389,13 +408,78 @@ def _default_sheet_for_metric(metric_id: str, *, compare_roles: tuple[str, ...]
)
def _sheets_split_by_field(
metric_id: str,
field: str,
slices: tuple[tuple[str, str, str], ...],
*,
op: str = "eq",
compare_roles: tuple[str, ...] = ("state",),
) -> list[dict[str, Any]]:
"""One collected metric → many compare sheets, each a row_filter slice.
``slices`` is ``(sheet_id, title, filter_value)``. Any metric can be split
this way (BGP afi, ISIS af, …); the engine does not special-case names.
"""
out: list[dict[str, Any]] = []
for sid, title, value in slices:
base = _default_sheet_for_metric(metric_id, compare_roles=compare_roles)
out.append(
_sheet_def(
metric_id=metric_id,
sheet_id=sid,
title=title,
key_fields=list(base.get("key_fields") or []),
iface_fields=list(base.get("iface_fields") or []),
compare_fields=list(base.get("compare_fields") or []),
display_fields=list(base.get("display_fields") or []),
row_filters=[{"field": field, "op": op, "value": value}],
field_rules=list(base.get("field_rules") or []),
)
)
return out
def _bgp_afi_sheets() -> list[dict[str, Any]]:
return _sheets_split_by_field(
"bgp_peer",
"afi",
(
("bgp_peer.ipv4", "BGP IPv4", "ipv4"),
("bgp_peer.ipv6", "BGP IPv6", "ipv6"),
("bgp_peer.vpnv4", "BGP VPNv4", "vpnv4"),
("bgp_peer.vpnv6", "BGP VPNv6", "vpnv6"),
),
op="eq",
)
def _isis_af_sheets() -> list[dict[str, Any]]:
return _sheets_split_by_field(
"isis_adjacency",
"af",
(
("isis_adjacency.ipv4", "ISIS IPv4", "IPv4"),
("isis_adjacency.ipv6", "ISIS IPv6", "IPv6"),
),
op="contains",
)
def _builtin_source_splits() -> dict[str, list[dict[str, Any]]]:
return {
"bgp_peer": _bgp_afi_sheets(),
"isis_adjacency": _isis_af_sheets(),
}
def _default_zte_status_sheets() -> list[dict[str, Any]]:
return [
_default_sheet_for_metric("isis_adjacency", compare_roles=("state",)),
*_isis_af_sheets(),
_default_sheet_for_metric("interface_brief", compare_roles=("state",)),
_default_sheet_for_metric("arp", compare_roles=("state",)),
_default_sheet_for_metric("nd6_cache", compare_roles=("state",)),
_default_sheet_for_metric("bgp_peer", compare_roles=("state",)),
*_bgp_afi_sheets(),
]
@ -421,6 +505,8 @@ def _normalize_sheet(raw: Any) -> dict[str, Any] | None:
disp_arg = None
return _sheet_def(
metric_id=mid,
sheet_id=str(raw.get("sheet_id") or "").strip() or mid,
title=str(raw.get("title") or "").strip() or None,
key_fields=keys,
iface_fields=_str_list(raw.get("iface_fields")),
compare_fields=_str_list(raw.get("compare_fields")),
@ -458,10 +544,10 @@ def template_metrics(t: BizCompareTemplate) -> list[dict[str, Any]]:
sheet = _normalize_sheet(item)
if not sheet:
continue
mid = sheet["metric_id"]
if mid in seen:
sid = sheet_key(sheet)
if sid in seen:
continue
seen.add(mid)
seen.add(sid)
out.append(sheet)
if out:
return out
@ -494,10 +580,10 @@ def _parse_metrics_body(body: dict[str, Any]) -> list[dict[str, Any]]:
sheet = _normalize_sheet(raw)
if not sheet:
continue
mid = sheet["metric_id"]
if mid in seen:
raise HTTPException(status_code=400, detail=f"duplicate_metric:{mid}")
seen.add(mid)
sid = sheet_key(sheet)
if sid in seen:
raise HTTPException(status_code=400, detail=f"duplicate_sheet:{sid}")
seen.add(sid)
sheets.append(sheet)
if not sheets:
raise HTTPException(status_code=400, detail="metrics_required")
@ -524,6 +610,8 @@ def _parse_metrics_body(body: dict[str, Any]) -> list[dict[str, Any]]:
return [
_sheet_def(
metric_id=mid,
sheet_id=str(body.get("sheet_id") or "").strip() or mid,
title=str(body.get("title") or "").strip() or None,
key_fields=keys,
iface_fields=_str_list(body.get("iface_fields")),
compare_fields=compare,
@ -541,7 +629,7 @@ def _template_out(t: BizCompareTemplate) -> dict[str, Any]:
"id": t.id,
"name": t.name,
"metrics": sheets,
"metric_ids": [s["metric_id"] for s in sheets],
"metric_ids": list(dict.fromkeys(s["metric_id"] for s in sheets if s.get("metric_id"))),
# legacy mirrors (first sheet)
"metric_id": (first or {}).get("metric_id") or t.metric_id or "",
"key_fields": list((first or {}).get("key_fields") or t.key_fields or []),
@ -698,8 +786,44 @@ def ensure_default_templates(db: Session) -> None:
ensure_default_zte_status_template(db)
def upgrade_builtin_split_sheets(db: Session) -> None:
"""Split unfiltered whole-table sheets on the built-in ZTE template only.
A sheet is replaced when its id is still the source metric and it has no
row filters. Custom templates and already-split sheets are left alone.
"""
row = (
db.query(BizCompareTemplate)
.filter(BizCompareTemplate.name == "ZTE status default")
.one_or_none()
)
if not row:
return
existing = template_metrics(row)
splits = _builtin_source_splits()
out: list[dict[str, Any]] = []
changed = False
replaced: set[str] = set()
for s in existing:
mid = str(s.get("metric_id") or "")
if mid in splits and sheet_key(s) == mid and not list(s.get("row_filters") or []):
if mid not in replaced:
out.extend(splits[mid])
replaced.add(mid)
changed = True
continue
out.append(s)
if not changed or not out:
return
_apply_sheets_to_row(row, out)
row.note = "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP, address-family sheets)"
row.updated_at = _utcnow()
db.commit()
def list_templates(db: Session) -> list[dict[str, Any]]:
ensure_default_templates(db)
upgrade_builtin_split_sheets(db)
rows = db.query(BizCompareTemplate).order_by(BizCompareTemplate.name.asc()).all()
return [_template_out(t) for t in rows]
@ -1143,6 +1267,8 @@ def _run_sheet(
summary["after_raw_count"] = len(after_raw)
summary["row_filters"] = len(row_filters)
return {
"sheet_id": sheet_key(sheet),
"title": sheet_title(sheet),
"metric_id": sheet["metric_id"],
"key_fields": key_fields,
"iface_fields": iface_fields,
@ -1198,7 +1324,7 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
s = one["summary"]
for k in agg:
agg[k] += int(s.get(k) or 0)
mapping_by_metric[one["metric_id"]] = one["mapping_stats"]
mapping_by_metric[sheet_key(one)] = one["mapping_stats"]
first = sheet_results[0]
field_counts: dict[str, int] = {}
@ -1221,6 +1347,8 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
"top_changed_fields": top_fields,
"sheets": [
{
"sheet_id": s.get("sheet_id") or s["metric_id"],
"title": s.get("title") or s.get("sheet_id") or s["metric_id"],
"metric_id": s["metric_id"],
"key_fields": s["key_fields"],
"iface_fields": s["iface_fields"],
@ -1255,7 +1383,7 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
_persist_sheet_diffs(
db,
run_id=run_id,
metric_id=str(s["metric_id"]),
metric_id=sheet_key(s),
diffs=list(s.get("diffs") or []),
)
j.updated_at = _utcnow()
@ -1352,6 +1480,8 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
sj = sf + su
sheet_cards.append(
{
"sheet_id": sheet_key(sh),
"title": sheet_title(sh),
"metric_id": sh.get("metric_id") or "",
"mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
"added": sa,
@ -1413,6 +1543,8 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
# Never return full diffs in run detail (million-row safe)
sheets = [
{
"sheet_id": sh.get("sheet_id") or sh.get("metric_id") or "",
"title": sh.get("title") or sh.get("sheet_id") or sh.get("metric_id") or "",
"metric_id": sh.get("metric_id") or "",
"key_fields": list(sh.get("key_fields") or []),
"iface_fields": list(sh.get("iface_fields") or []),
@ -1459,6 +1591,20 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
}
def _lookup_sheet(sheets: list[dict[str, Any]], key: str) -> dict[str, Any] | None:
"""Find a result sheet by sheet_id, or by metric_id when that source is unique."""
k = str(key or "").strip()
if not k:
return sheets[0] if sheets else None
for s in sheets:
if sheet_key(s) == k:
return s
hits = [s for s in sheets if str(s.get("metric_id") or "") == k]
if len(hits) == 1:
return hits[0]
return None
def list_run_diffs(
db: Session,
run_id: str,
@ -1480,7 +1626,9 @@ def list_run_diffs(
summary = dict(r.summary_json or {})
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
sheets = _sheet_meta_from_summary(summary, r, tpl)
mid = (metric_id or "").strip() or (sheets[0].get("metric_id") if sheets else r.metric_id) or ""
asked = (metric_id or "").strip()
sheet = _lookup_sheet(sheets, asked) if asked else (sheets[0] if sheets else None)
mid = sheet_key(sheet) if sheet else (asked or str(r.metric_id or ""))
if _run_has_diff_rows(db, run_id):
q = db.query(BizCompareDiff).filter(
@ -1509,10 +1657,9 @@ def list_run_diffs(
}
# Legacy: diffs embedded in summary_json / diffs_json
sheet = next((s for s in sheets if str(s.get("metric_id") or "") == mid), None)
if sheet is None and sheets:
sheet = sheets[0]
mid = str(sheet.get("metric_id") or mid)
mid = sheet_key(sheet)
inline = list((sheet or {}).get("diffs") or [])
if not inline and mid == r.metric_id:
inline = list(r.diffs_json or [])
@ -1556,7 +1703,9 @@ def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str
summary = dict(r.summary_json or {})
sheets = list(summary.get("sheets") or [])
for sh in sheets:
if str(sh.get("metric_id") or "") == metric_id:
if sheet_key(sh) == metric_id or (
str(sh.get("metric_id") or "") == metric_id and sheet_key(sh) == metric_id
):
return list(sh.get("diffs") or [])
if metric_id == r.metric_id:
return list(r.diffs_json or [])
@ -1584,17 +1733,17 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
]
for card in list(s.get("sheet_cards") or []):
manifest.append(
f"- {card.get('metric_id')}: diff={card.get('diff_count')} "
f"- {card.get('title') or card.get('sheet_id') or card.get('metric_id')}: diff={card.get('diff_count')} "
f"pass={card.get('pass_rate')}% "
f"+{card.get('added')}/-{card.get('removed')}/~{card.get('changed')}/= {card.get('unchanged')}"
)
zf.writestr("manifest.txt", "\n".join(manifest) + "\n")
for sheet in list(detail.get("sheets") or []):
mid = str(sheet.get("metric_id") or "sheet")
safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "sheet"
sid = sheet_key(sheet) or "sheet"
safe = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in sid)[:80] or "sheet"
sheet_full = {
**sheet,
"diffs": _iter_sheet_diffs(db, run_id, mid),
"diffs": _iter_sheet_diffs(db, run_id, sid),
}
zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet_full))
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"]
@ -1603,7 +1752,7 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
",".join(
_csv_cell(x)
for x in (
card.get("metric_id"),
card.get("title") or card.get("sheet_id") or card.get("metric_id"),
card.get("mode"),
card.get("before_count"),
card.get("after_count"),

1
netx_api/biz_state/parsers/zte/bgp_peer.py
View file

@ -76,7 +76,6 @@ def normalize_bgp_peer(
"up_down": m.group("up")[:32],
"state": state[:64],
"pfx_rcd": pfx[:32],
"state_or_pfx": state_raw[:64],
}
)
return out

1
netx_api/biz_state/profiles.py
View file

@ -298,7 +298,6 @@ _BGP_PEER_FIELDS: list[FieldDef] = [
FieldDef("as_num", length=16, role="state", display_name="AS"),
FieldDef("state", length=64, role="state", display_name="State"),
FieldDef("pfx_rcd", length=32, role="state", display_name="PfxRcd"),
FieldDef("state_or_pfx", length=64, role="meta", display_name="State/PfxRcd"),
FieldDef("ver", length=8, role="meta", display_name="Ver"),
FieldDef("msg_rcvd", length=32, role="meta", display_name="MsgRcvd"),
FieldDef("msg_send", length=32, role="meta", display_name="MsgSend"),

2
netx_api/models/biz_migration.py
View file

@ -44,7 +44,7 @@ class BizMigrationBatch(Base):
batch_label: Mapped[str] = mapped_column(String(128), default="")
# pending | active | review | done
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
# {"ports": ["gei-..."], "items": [{"metric_id":"bgp_peer","key":"..."}]}
# {"ports": ["gei-..."], "items": [{"metric_id":"bgp_peer","sheet_id":"bgp_peer.vpnv4","key":"..."}]}
expect_set_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)

7
netx_api/models/biz_state.py
View file

@ -205,8 +205,11 @@ class BizStateCommandOverride(Base):
class BizCompareTemplate(Base):
"""Compare template: one or more metric sheets with key/iface/compare roles.
``metrics_json`` is the source of truth (list of sheet defs). Legacy
``metric_id`` / ``key_fields`` / … mirror the first sheet for older rows.
``metrics_json`` is the source of truth (list of sheet defs). Each sheet has
``metric_id`` (collected source table) and optional ``sheet_id`` / ``title`` /
``row_filters`` so one source can be split into multiple compare items
(e.g. bgp_peer → vpnv4/vpnv6). Legacy ``metric_id`` / ``key_fields`` / …
mirror the first sheet for older rows.
Empty ``compare_fields`` on a sheet = presence-only (entry set match).
"""