hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-09 03:10:46 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 2

Allow compare templates to split one collected metric into filtered sheets.

Browse source 
BGP/ISIS (and any other metric) can share a metric_id with distinct sheet_id + row_filters; drop conflicting bgp state_or_pfx and keep state/pfx_rcd.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-19 01:46:03 +08:00
parent 88b7558a48
commit a65c9fc420
17 changed files with 745 additions and 230 deletions
Download patch file Download diff file Expand all files Collapse all files

54
netx_api/biz_migration/evaluate.py
View file

@ -79,9 +79,10 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]:
if not isinstance(it, dict):
continue
mid = str(it.get("metric_id") or "").strip()
if not mid:
sid = str(it.get("sheet_id") or "").strip() or mid
if not sid:
continue
bucket = out.setdefault(mid, set())
bucket = out.setdefault(sid, set())
if it.get("key") is not None:
k = it.get("key")
if isinstance(k, (list, tuple)):
@ -747,10 +748,20 @@ def expect_keys_for_metric(
*,
metric_id: str,
iface_fields: list[str],
sheet_id: str = "",
) -> set[str]:
keys = set(expect.get(metric_id) or ())
"""Resolve expect keys for a compare sheet.
Prefer ``sheet_id`` (split tables); fall back to ``metric_id`` for legacy
unsplit sheets / expect items that only carry metric_id.
"""
sid = str(sheet_id or "").strip()
mid = str(metric_id or "").strip()
keys = set(expect.get(sid) or ()) if sid else set()
if not keys and mid:
keys = set(expect.get(mid) or ())
# ports: shorthand is only for interface_brief, not every sheet with iface fields
if metric_id == PORT_METRIC_ID and expect.get("_ports"):
if mid == PORT_METRIC_ID and expect.get("_ports"):
keys |= set(expect["_ports"])
return keys
@ -879,15 +890,34 @@ def _current_row(diff: dict[str, Any] | None) -> dict[str, Any]:
return dict(diff.get("after") or {})
def override_for_sheet(
sheet_overrides: list[dict[str, Any]] | None,
*,
sheet_id: str,
metric_id: str,
) -> dict[str, Any]:
"""Match a monitor override. sheet_id wins; a legacy metric-only override applies to every split."""
sid = str(sheet_id or "").strip()
mid = str(metric_id or "").strip()
legacy: dict[str, Any] | None = None
for ov in sheet_overrides or []:
if not isinstance(ov, dict):
continue
ov_sid = str(ov.get("sheet_id") or "").strip()
ov_mid = str(ov.get("metric_id") or "").strip()
if ov_sid and ov_sid == sid:
return ov
if not ov_sid and ov_mid and ov_mid in (sid, mid) and legacy is None:
legacy = ov
return legacy or {}
def override_for_metric(
sheet_overrides: list[dict[str, Any]] | None,
metric_id: str,
) -> dict[str, Any]:
mid = str(metric_id or "").strip()
for ov in sheet_overrides or []:
if isinstance(ov, dict) and str(ov.get("metric_id") or "").strip() == mid:
return ov
return {}
return override_for_sheet(sheet_overrides, sheet_id=mid, metric_id=mid)
def evaluate_metric_dual(
@ -907,9 +937,13 @@ def evaluate_metric_dual(
field_rules: list[dict[str, Any]] | None = None,
sheet_override: dict[str, Any] | None = None,
out_of_expect: str = "strict",
sheet_id: str = "",
) -> dict[str, Any]:
"""Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge."""
expect_keys = expect_keys_for_metric(expect, metric_id=metric_id, iface_fields=iface_fields)
sid = str(sheet_id or "").strip() or str(metric_id or "").strip()
expect_keys = expect_keys_for_metric(
expect, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid
)
ov = sheet_override or {}
if ov.get("skip_dual"):
return {
@ -1155,6 +1189,8 @@ def evaluate_metric_dual(
def port_sheet_def() -> dict[str, Any]:
"""Default sheet for port-status cutover monitor (interface_brief only)."""
return {
"sheet_id": PORT_METRIC_ID,
"title": PORT_METRIC_ID,
"metric_id": PORT_METRIC_ID,
"key_fields": ["interface"],
"iface_fields": ["interface"],

211
netx_api/biz_migration/monitor_templates.py
View file

@ -127,68 +127,89 @@ def _success_stateful(
}
def preset_override_for_metric(metric_id: str) -> dict[str, Any]:
"""Default dual-verdict overlay for a compare-template sheet."""
def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[str, Any]:
"""Default dual-verdict overlay for a compare-template sheet.
``metric_id`` is the collected source table; ``sheet_id`` identifies a
filtered split (e.g. bgp_peer.vpnv4). Presets key off the source metric.
"""
mid = str(metric_id or "").strip()
if mid == PORT_METRIC_ID:
sid = str(sheet_id or "").strip()
# Callers may pass only a split id like "bgp_peer.vpnv4"
if not mid and sid:
mid = sid.split(".", 1)[0]
if not sid:
sid = mid
look = mid.split(".", 1)[0] if "." in mid else mid
def _out(body: dict[str, Any]) -> dict[str, Any]:
body["metric_id"] = mid or look
if sid and sid != (mid or look):
body["sheet_id"] = sid
return body
if look == PORT_METRIC_ID:
up = [
{"type": "value", "field": "admin", "op": "in", "value": ["up"]},
{"type": "value", "field": "phy", "op": "in", "value": ["up"]},
]
return {
"metric_id": mid,
"status_fields": list(PORT_STATUS_FIELDS),
"down_values": ["down"],
"up_values": ["up"],
"success": [
_success_stateful(
old_down_groups=[
[
{"type": "value", "field": "admin", "op": "in", "value": ["down"]},
{"type": "value", "field": "phy", "op": "in", "value": ["down"]},
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state("admin", ["down"]),
}
if mid == "bgp_peer":
return _out(
{
"status_fields": list(PORT_STATUS_FIELDS),
"down_values": ["down"],
"up_values": ["up"],
"success": [
_success_stateful(
old_down_groups=[
[
{"type": "value", "field": "admin", "op": "in", "value": ["down"]},
{"type": "value", "field": "phy", "op": "in", "value": ["down"]},
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state("admin", ["down"]),
}
)
if look == "bgp_peer":
up = [{"type": "value", "field": "state", "op": "eq", "value": "established"}]
return {
"metric_id": mid,
"status_fields": ["state"],
"down_values": ["idle", "active", "connect", "down"],
"up_values": ["established"],
"success": [
_success_stateful(
old_down_groups=[
[
{
"type": "value",
"field": "state",
"op": "in",
"value": ["idle", "active", "connect", "down"],
}
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state(
"state", ["idle", "active", "connect", "down"]
),
}
if mid in ("arp", "nd6_cache", "lldp_neighbor"):
return {
"metric_id": mid,
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
if "isis" in mid or "ospf" in mid or "adjacency" in mid:
return _out(
{
"status_fields": ["state"],
"down_values": ["idle", "active", "connect", "down"],
"up_values": ["established"],
"success": [
_success_stateful(
old_down_groups=[
[
{
"type": "value",
"field": "state",
"op": "in",
"value": ["idle", "active", "connect", "down"],
}
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state(
"state", ["idle", "active", "connect", "down"]
),
}
)
if look in ("arp", "nd6_cache", "lldp_neighbor"):
return _out(
{
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
)
if "isis" in look or "ospf" in look or "adjacency" in look:
up = [
{
"type": "value",
@ -197,45 +218,48 @@ def preset_override_for_metric(metric_id: str) -> dict[str, Any]:
"value": ["up", "full", "2way"],
}
]
return {
"metric_id": mid,
"status_fields": ["state"],
"down_values": ["down", "init", "idle"],
"up_values": ["up", "full", "2way"],
"success": [
_success_stateful(
old_down_groups=[
[
{
"type": "value",
"field": "state",
"op": "in",
"value": ["down", "init", "idle"],
}
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state("state", ["down", "init", "idle"]),
}
if "route" in mid or "vrf" in mid:
return {
"metric_id": mid,
return _out(
{
"status_fields": ["state"],
"down_values": ["down", "init", "idle"],
"up_values": ["up", "full", "2way"],
"success": [
_success_stateful(
old_down_groups=[
[
{
"type": "value",
"field": "state",
"op": "in",
"value": ["down", "init", "idle"],
}
]
],
new_up_conds=up,
)
],
"anomaly": _default_anomaly_for_state("state", ["down", "init", "idle"]),
}
)
if "route" in look or "vrf" in look:
return _out(
{
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
)
return _out(
{
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
return {
"metric_id": mid,
"status_fields": [],
"down_values": [],
"up_values": [],
"success": [_success_presence_migrate()],
"anomaly": _default_anomaly_presence_only(),
}
)
def ensure_default_monitor_templates(db: Session) -> None:
@ -264,7 +288,14 @@ def ensure_default_monitor_templates(db: Session) -> None:
]
if zte:
zte_sheets = cmp_svc.template_metrics(zte)
zte_overrides = [preset_override_for_metric(str(s.get("metric_id") or "")) for s in zte_sheets if s.get("metric_id")]
zte_overrides = [
preset_override_for_metric(
str(s.get("metric_id") or ""),
sheet_id=str(s.get("sheet_id") or s.get("metric_id") or ""),
)
for s in zte_sheets
if s.get("metric_id")
]
seeds.append(
BizMonitorTemplate(
id=uuid4().hex,

28
netx_api/biz_migration/service.py
View file

@ -9,7 +9,7 @@ from fastapi import HTTPException
from sqlalchemy.orm import Session
from ..biz_state.compare_rules import apply_row_filters
from ..biz_state.compare_service import _load_metric_rows, _port_map_dict, template_metrics
from ..biz_state.compare_service import _load_metric_rows, _port_map_dict, sheet_key, sheet_title, template_metrics
from ..models import (
BizCompareTemplate,
BizMigrationBatch,
@ -27,7 +27,7 @@ from . import monitor_templates as mon_tpl
from .evaluate import (
PORT_METRIC_ID,
evaluate_metric_dual,
override_for_metric,
override_for_sheet,
parse_expect_set,
port_sheet_def,
)
@ -124,7 +124,12 @@ def resolve_collect_metric_ids(db: Session, proj: BizMigrationProject) -> list[s
if collect:
return collect
sheets, _, _ = resolve_evaluate_sheets(db, mt)
return [str(s.get("metric_id") or "").strip() for s in sheets if str(s.get("metric_id") or "").strip()]
seen: list[str] = []
for s in sheets:
mid = str(s.get("metric_id") or "").strip()
if mid and mid not in seen:
seen.append(mid)
return seen
def project_to_dict(db: Session, p: BizMigrationProject) -> dict[str, Any]:
@ -372,6 +377,7 @@ def run_evaluate(
for sheet in sheets:
mid = str(sheet.get("metric_id") or "").strip()
sid = sheet_key(sheet)
key_fields = list(sheet.get("key_fields") or [])
if not mid or not key_fields:
continue
@ -379,7 +385,7 @@ def run_evaluate(
compare_fields = list(sheet.get("compare_fields") or [])
row_filters = list(sheet.get("row_filters") or [])
field_rules = list(sheet.get("field_rules") or [])
sheet_ov = override_for_metric(sheet_overrides, mid)
sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid)
if sheet_ov.get("skip_dual"):
continue
@ -402,6 +408,7 @@ def run_evaluate(
one = evaluate_metric_dual(
metric_id=mid,
sheet_id=sid,
key_fields=key_fields,
iface_fields=iface_fields,
compare_fields=compare_fields,
@ -420,7 +427,8 @@ def run_evaluate(
sheet_cards.append(
{
"metric_id": mid,
"title": mid,
"sheet_id": sid,
"title": sheet_title(sheet),
"progress_ok": one["progress_ok"],
"progress_total": one["progress_total"],
"anomaly": one["anomaly"],
@ -433,6 +441,7 @@ def run_evaluate(
)
for r in one["rows"]:
r["seq"] = seq
r["sheet_id"] = sid
seq += 1
all_rows.append(r)
v = str(r.get("verdict") or "")
@ -460,7 +469,7 @@ def run_evaluate(
"anomaly": sum(c["anomaly"] for c in sheet_cards),
"new_baseline_missing": any(bool(c.get("new_baseline_missing")) for c in sheet_cards),
"missing_metrics": [
str(c.get("metric_id") or "")
str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "")
for c in sheet_cards
if c.get("new_baseline_missing")
],
@ -499,6 +508,7 @@ def run_evaluate(
"old_status": r.get("old_status"),
"new_status": r.get("new_status"),
"rule_hit": r.get("rule_hit") or "",
"sheet_id": r.get("sheet_id") or "",
},
old_kind=str(r.get("old_kind") or ""),
new_kind=str(r.get("new_kind") or ""),
@ -545,6 +555,7 @@ def diff_to_dict(d: BizMigrationDiff) -> dict[str, Any]:
return {
"id": d.id,
"metric_id": d.metric_id,
"sheet_id": kj.get("sheet_id") or d.metric_id or "",
"seq": d.seq,
"verdict": d.verdict,
"color": d.color,
@ -610,10 +621,11 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any]
out_sheets: list[dict[str, Any]] = []
for sheet in sheets:
mid = str(sheet.get("metric_id") or "").strip()
sid = sheet_key(sheet)
key_fields = [str(k) for k in (sheet.get("key_fields") or []) if str(k).strip()]
if not mid or not key_fields:
continue
sheet_ov = override_for_metric(sheet_overrides, mid)
sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid)
if sheet_ov.get("skip_dual"):
continue
iface_fields = [str(k) for k in (sheet.get("iface_fields") or []) if str(k).strip()]
@ -643,6 +655,8 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any]
out_sheets.append(
{
"metric_id": mid,
"sheet_id": sid,
"title": sheet_title(sheet),
"key_fields": key_fields,
"iface_fields": iface_fields,
"items": items,