Multi-metric compare templates with presence mode and batch workbook results.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-17 17:47:51 +08:00
parent 14f5870e19
commit ea6e0041c1
11 changed files with 1060 additions and 453 deletions

View file

@ -109,6 +109,8 @@ def compare_rows(
} }
) )
continue continue
# Empty compare_fields = presence-only: keyed rows that exist on both
# sides are unchanged (no value checks).
field_changes: dict[str, dict[str, Any]] = {} field_changes: dict[str, dict[str, Any]] = {}
for f in compare_fields: for f in compare_fields:
bv = mapped.get(f, "") bv = mapped.get(f, "")

View file

@ -27,107 +27,266 @@ def _utcnow() -> datetime:
return utcnow_naive() return utcnow_naive()
def _default_lldp_template_fields() -> dict[str, list[str]]: def _str_list(raw: Any) -> list[str]:
fields = metric_field_map().get("lldp_neighbor") or [] return [str(x).strip() for x in (raw or []) if str(x).strip()]
keys = [f.name for f in fields if f.is_key]
ifaces = [f.name for f in fields if f.is_interface]
compare = [f.name for f in fields if f.name not in keys or f.name in ifaces] def _sheet_def(
# Prefer comparing identity+meta that aren't pure key-only if listed *,
compare = [f.name for f in fields if f.role in ("identity", "state", "meta") or f.is_key] metric_id: str,
# Deduplicate while keeping order key_fields: list[str],
seen: set[str] = set() iface_fields: list[str] | None = None,
cmp_out: list[str] = [] compare_fields: list[str] | None = None,
for n in compare: ) -> dict[str, Any]:
if n not in seen: mid = str(metric_id or "").strip()
seen.add(n) keys = _str_list(key_fields)
cmp_out.append(n) ifaces = _str_list(iface_fields)
# compare_fields empty → presence-only mode (intentional)
compare = _str_list(compare_fields) if compare_fields is not None else []
# Keys are identity only; strip them from compare so UI/engine stay clear
key_set = set(keys)
compare = [f for f in compare if f not in key_set]
return { return {
"key_fields": keys or ["local_if", "remote_sys", "remote_if"], "metric_id": mid,
"iface_fields": ifaces or ["local_if"], "key_fields": keys,
"compare_fields": cmp_out or ["remote_sys", "remote_if", "remote_ip", "protocol"], "iface_fields": ifaces,
"compare_fields": compare,
} }
def ensure_default_lldp_template(db: Session) -> BizCompareTemplate: def _default_lldp_sheet() -> dict[str, Any]:
fields = metric_field_map().get("lldp_neighbor") or []
keys = [f.name for f in fields if f.is_key] or ["local_if", "remote_sys", "remote_if"]
ifaces = [f.name for f in fields if f.is_interface] or ["local_if"]
# Value checks: non-key state/meta (e.g. remote_ip / protocol)
compare = [f.name for f in fields if not f.is_key and f.role in ("state", "meta", "identity")]
if not compare:
compare = [n for n in ("remote_ip", "protocol") if n not in keys]
return _sheet_def(
metric_id="lldp_neighbor",
key_fields=keys,
iface_fields=ifaces,
compare_fields=compare,
)
def _default_vrf_sheet() -> dict[str, Any]:
fields = metric_field_map().get("vrf_route_summary") or []
keys = [f.name for f in fields if f.is_key] or ["vrf", "source"]
ifaces = [f.name for f in fields if f.is_interface]
compare = [f.name for f in fields if not f.is_key and f.role in ("state", "meta", "identity")]
if not compare:
compare = [n for n in ("networks",) if n not in keys]
return _sheet_def(
metric_id="vrf_route_summary",
key_fields=keys,
iface_fields=ifaces,
compare_fields=compare,
)
def _normalize_sheet(raw: Any) -> dict[str, Any] | None:
if not isinstance(raw, dict):
return None
mid = str(raw.get("metric_id") or "").strip()
keys = _str_list(raw.get("key_fields"))
if not mid or not keys:
return None
return _sheet_def(
metric_id=mid,
key_fields=keys,
iface_fields=_str_list(raw.get("iface_fields")),
compare_fields=_str_list(raw.get("compare_fields")),
)
def _legacy_sheets(t: BizCompareTemplate) -> list[dict[str, Any]]:
mid = str(t.metric_id or "").strip()
keys = _str_list(t.key_fields)
if not mid or not keys:
return []
ignore = set(_str_list(t.ignore_fields))
compare = [f for f in _str_list(t.compare_fields) if f not in ignore]
return [
_sheet_def(
metric_id=mid,
key_fields=keys,
iface_fields=_str_list(t.iface_fields),
compare_fields=compare,
)
]
def template_metrics(t: BizCompareTemplate) -> list[dict[str, Any]]:
"""Resolved metric sheets for a template (metrics_json or legacy single)."""
raw = list(t.metrics_json or [])
out: list[dict[str, Any]] = []
seen: set[str] = set()
for item in raw:
sheet = _normalize_sheet(item)
if not sheet:
continue
mid = sheet["metric_id"]
if mid in seen:
continue
seen.add(mid)
out.append(sheet)
if out:
return out
return _legacy_sheets(t)
def _apply_sheets_to_row(t: BizCompareTemplate, sheets: list[dict[str, Any]]) -> None:
t.metrics_json = sheets
first = sheets[0] if sheets else None
if first:
t.metric_id = first["metric_id"]
t.key_fields = list(first["key_fields"])
t.iface_fields = list(first["iface_fields"])
t.compare_fields = list(first["compare_fields"])
t.ignore_fields = []
else:
t.metric_id = ""
t.key_fields = []
t.iface_fields = []
t.compare_fields = []
t.ignore_fields = []
def _parse_metrics_body(body: dict[str, Any]) -> list[dict[str, Any]]:
"""Accept ``metrics`` list or legacy single-metric fields."""
if "metrics" in body and body.get("metrics") is not None:
sheets: list[dict[str, Any]] = []
seen: set[str] = set()
for raw in list(body.get("metrics") or []):
sheet = _normalize_sheet(raw)
if not sheet:
continue
mid = sheet["metric_id"]
if mid in seen:
raise HTTPException(status_code=400, detail=f"duplicate_metric:{mid}")
seen.add(mid)
sheets.append(sheet)
if not sheets:
raise HTTPException(status_code=400, detail="metrics_required")
return sheets
mid = str(body.get("metric_id") or "").strip()
keys = _str_list(body.get("key_fields"))
if not mid:
raise HTTPException(status_code=400, detail="metric_id_required")
if not keys:
raise HTTPException(status_code=400, detail="key_fields_required")
ignore = set(_str_list(body.get("ignore_fields")))
compare = [f for f in _str_list(body.get("compare_fields")) if f not in ignore]
return [
_sheet_def(
metric_id=mid,
key_fields=keys,
iface_fields=_str_list(body.get("iface_fields")),
compare_fields=compare,
)
]
def _template_out(t: BizCompareTemplate) -> dict[str, Any]:
sheets = template_metrics(t)
first = sheets[0] if sheets else None
return {
"id": t.id,
"name": t.name,
"metrics": sheets,
"metric_ids": [s["metric_id"] for s in sheets],
# legacy mirrors (first sheet)
"metric_id": (first or {}).get("metric_id") or t.metric_id or "",
"key_fields": list((first or {}).get("key_fields") or t.key_fields or []),
"iface_fields": list((first or {}).get("iface_fields") or t.iface_fields or []),
"compare_fields": list((first or {}).get("compare_fields") or t.compare_fields or []),
"ignore_fields": [],
"note": t.note,
"updated_at": t.updated_at.isoformat() + "Z" if t.updated_at else None,
}
def ensure_default_cutover_template(db: Session) -> BizCompareTemplate:
row = ( row = (
db.query(BizCompareTemplate) db.query(BizCompareTemplate)
.filter(BizCompareTemplate.metric_id == "lldp_neighbor", BizCompareTemplate.name == "LLDP default") .filter(BizCompareTemplate.name == "Cutover default")
.one_or_none() .one_or_none()
) )
if row: if row:
# Upgrade legacy single-sheet cutover if needed
sheets = template_metrics(row)
if len(sheets) < 2:
_apply_sheets_to_row(row, [_default_lldp_sheet(), _default_vrf_sheet()])
row.note = "Built-in multi-metric cutover template (LLDP + VRF)"
row.updated_at = _utcnow()
db.commit()
db.refresh(row)
return row return row
defs = _default_lldp_template_fields() sheets = [_default_lldp_sheet(), _default_vrf_sheet()]
row = BizCompareTemplate( row = BizCompareTemplate(
id=uuid4().hex, id=uuid4().hex,
name="LLDP default", name="Cutover default",
metric_id="lldp_neighbor", note="Built-in multi-metric cutover template (LLDP + VRF)",
key_fields=defs["key_fields"],
iface_fields=defs["iface_fields"],
compare_fields=defs["compare_fields"],
ignore_fields=[],
note="Built-in template for LLDP neighbor cutover compare",
created_at=_utcnow(), created_at=_utcnow(),
updated_at=_utcnow(), updated_at=_utcnow(),
) )
_apply_sheets_to_row(row, sheets)
db.add(row) db.add(row)
db.commit() db.commit()
db.refresh(row) db.refresh(row)
return row return row
def _template_out(t: BizCompareTemplate) -> dict[str, Any]: def ensure_default_lldp_template(db: Session) -> BizCompareTemplate:
return { row = (
"id": t.id, db.query(BizCompareTemplate)
"name": t.name, .filter(BizCompareTemplate.name == "LLDP default")
"metric_id": t.metric_id, .one_or_none()
"key_fields": list(t.key_fields or []), )
"iface_fields": list(t.iface_fields or []), if row:
"compare_fields": list(t.compare_fields or []), if not template_metrics(row):
"ignore_fields": list(t.ignore_fields or []), _apply_sheets_to_row(row, [_default_lldp_sheet()])
"note": t.note, row.updated_at = _utcnow()
"updated_at": t.updated_at.isoformat() + "Z" if t.updated_at else None, db.commit()
} db.refresh(row)
return row
row = BizCompareTemplate(
def list_templates(db: Session) -> list[dict[str, Any]]: id=uuid4().hex,
ensure_default_templates(db) name="LLDP default",
rows = db.query(BizCompareTemplate).order_by(BizCompareTemplate.name.asc()).all() note="Built-in template for LLDP neighbor cutover compare",
return [_template_out(t) for t in rows] created_at=_utcnow(),
updated_at=_utcnow(),
)
_apply_sheets_to_row(row, [_default_lldp_sheet()])
db.add(row)
db.commit()
db.refresh(row)
return row
def ensure_default_vrf_template(db: Session) -> BizCompareTemplate: def ensure_default_vrf_template(db: Session) -> BizCompareTemplate:
row = ( row = (
db.query(BizCompareTemplate) db.query(BizCompareTemplate)
.filter( .filter(BizCompareTemplate.name == "VRF route summary default")
BizCompareTemplate.metric_id == "vrf_route_summary",
BizCompareTemplate.name == "VRF route summary default",
)
.one_or_none() .one_or_none()
) )
if row: if row:
if not template_metrics(row):
_apply_sheets_to_row(row, [_default_vrf_sheet()])
row.updated_at = _utcnow()
db.commit()
db.refresh(row)
return row return row
fields = metric_field_map().get("vrf_route_summary") or []
keys = [f.name for f in fields if f.is_key] or ["vrf", "source"]
ifaces = [f.name for f in fields if f.is_interface]
compare = [f.name for f in fields if f.role in ("state", "identity", "meta") or f.is_key]
seen: set[str] = set()
cmp_out: list[str] = []
for n in compare:
if n not in seen:
seen.add(n)
cmp_out.append(n)
row = BizCompareTemplate( row = BizCompareTemplate(
id=uuid4().hex, id=uuid4().hex,
name="VRF route summary default", name="VRF route summary default",
metric_id="vrf_route_summary",
key_fields=keys,
iface_fields=ifaces,
compare_fields=cmp_out or ["networks"],
ignore_fields=[],
note="Built-in template for per-VRF route summary cutover compare", note="Built-in template for per-VRF route summary cutover compare",
created_at=_utcnow(), created_at=_utcnow(),
updated_at=_utcnow(), updated_at=_utcnow(),
) )
_apply_sheets_to_row(row, [_default_vrf_sheet()])
db.add(row) db.add(row)
db.commit() db.commit()
db.refresh(row) db.refresh(row)
@ -135,10 +294,17 @@ def ensure_default_vrf_template(db: Session) -> BizCompareTemplate:
def ensure_default_templates(db: Session) -> None: def ensure_default_templates(db: Session) -> None:
ensure_default_cutover_template(db)
ensure_default_lldp_template(db) ensure_default_lldp_template(db)
ensure_default_vrf_template(db) ensure_default_vrf_template(db)
def list_templates(db: Session) -> list[dict[str, Any]]:
ensure_default_templates(db)
rows = db.query(BizCompareTemplate).order_by(BizCompareTemplate.name.asc()).all()
return [_template_out(t) for t in rows]
def list_metric_schemas() -> list[dict[str, Any]]: def list_metric_schemas() -> list[dict[str, Any]]:
"""Field catalog for template editors (key / iface / compare pickers).""" """Field catalog for template editors (key / iface / compare pickers)."""
out: list[dict[str, Any]] = [] out: list[dict[str, Any]] = []
@ -165,24 +331,15 @@ def list_metric_schemas() -> list[dict[str, Any]]:
def create_template(db: Session, body: dict[str, Any]) -> dict[str, Any]: def create_template(db: Session, body: dict[str, Any]) -> dict[str, Any]:
metric_id = str(body.get("metric_id") or "").strip() sheets = _parse_metrics_body(body)
if not metric_id:
raise HTTPException(status_code=400, detail="metric_id_required")
key_fields = [str(x) for x in (body.get("key_fields") or []) if str(x).strip()]
if not key_fields:
raise HTTPException(status_code=400, detail="key_fields_required")
t = BizCompareTemplate( t = BizCompareTemplate(
id=uuid4().hex, id=uuid4().hex,
name=str(body.get("name") or metric_id)[:256], name=str(body.get("name") or sheets[0]["metric_id"])[:256],
metric_id=metric_id,
key_fields=key_fields,
iface_fields=[str(x) for x in (body.get("iface_fields") or []) if str(x).strip()],
compare_fields=[str(x) for x in (body.get("compare_fields") or []) if str(x).strip()],
ignore_fields=[str(x) for x in (body.get("ignore_fields") or []) if str(x).strip()],
note=str(body.get("note") or "")[:512], note=str(body.get("note") or "")[:512],
created_at=_utcnow(), created_at=_utcnow(),
updated_at=_utcnow(), updated_at=_utcnow(),
) )
_apply_sheets_to_row(t, sheets)
db.add(t) db.add(t)
db.commit() db.commit()
return _template_out(t) return _template_out(t)
@ -194,23 +351,36 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict
raise HTTPException(status_code=404, detail="template_not_found") raise HTTPException(status_code=404, detail="template_not_found")
if "name" in body: if "name" in body:
t.name = str(body.get("name") or "")[:256] t.name = str(body.get("name") or "")[:256]
if "metric_id" in body and body.get("metric_id") is not None:
mid = str(body.get("metric_id") or "").strip()
if mid:
t.metric_id = mid
if "key_fields" in body:
keys = [str(x) for x in (body.get("key_fields") or []) if str(x).strip()]
if not keys:
raise HTTPException(status_code=400, detail="key_fields_required")
t.key_fields = keys
if "iface_fields" in body:
t.iface_fields = [str(x) for x in (body.get("iface_fields") or []) if str(x).strip()]
if "compare_fields" in body:
t.compare_fields = [str(x) for x in (body.get("compare_fields") or []) if str(x).strip()]
if "ignore_fields" in body:
t.ignore_fields = [str(x) for x in (body.get("ignore_fields") or []) if str(x).strip()]
if "note" in body: if "note" in body:
t.note = str(body.get("note") or "")[:512] t.note = str(body.get("note") or "")[:512]
if any(k in body for k in ("metrics", "metric_id", "key_fields", "iface_fields", "compare_fields", "ignore_fields")):
# Prefer explicit metrics; otherwise merge into current sheets from legacy keys
if "metrics" in body and body.get("metrics") is not None:
sheets = _parse_metrics_body(body)
else:
# Patch first sheet (or create) from legacy fields
sheets = list(template_metrics(t))
if not sheets:
sheets = _parse_metrics_body(body)
else:
first = dict(sheets[0])
if "metric_id" in body and body.get("metric_id") is not None:
mid = str(body.get("metric_id") or "").strip()
if mid:
first["metric_id"] = mid
if "key_fields" in body:
keys = _str_list(body.get("key_fields"))
if not keys:
raise HTTPException(status_code=400, detail="key_fields_required")
first["key_fields"] = keys
if "iface_fields" in body:
first["iface_fields"] = _str_list(body.get("iface_fields"))
if "compare_fields" in body or "ignore_fields" in body:
ignore = set(_str_list(body.get("ignore_fields"))) if "ignore_fields" in body else set()
compare = _str_list(body.get("compare_fields")) if "compare_fields" in body else list(first.get("compare_fields") or [])
first["compare_fields"] = [f for f in compare if f not in ignore]
sheets[0] = _normalize_sheet(first) or first
_apply_sheets_to_row(t, sheets)
t.updated_at = _utcnow() t.updated_at = _utcnow()
db.commit() db.commit()
return _template_out(t) return _template_out(t)
@ -353,16 +523,21 @@ def validate_mapping(
template_id: str = "", template_id: str = "",
) -> dict[str, Any]: ) -> dict[str, Any]:
ensure_default_templates(db) ensure_default_templates(db)
tpl = db.get(BizCompareTemplate, template_id) if template_id else ensure_default_lldp_template(db) tpl = db.get(BizCompareTemplate, template_id) if template_id else ensure_default_cutover_template(db)
if not tpl: if not tpl:
raise HTTPException(status_code=404, detail="template_not_found") raise HTTPException(status_code=404, detail="template_not_found")
before = _load_metric_rows(db, batch_id=before_batch_id, metric_id=tpl.metric_id) sheets = template_metrics(tpl)
after = _load_metric_rows(db, batch_id=after_batch_id, metric_id=tpl.metric_id) if not sheets:
raise HTTPException(status_code=400, detail="template_has_no_metrics")
pmap = _port_map_dict(db, mapping_id) pmap = _port_map_dict(db, mapping_id)
# Validate against first sheet that has iface fields (or first sheet)
target = next((s for s in sheets if s.get("iface_fields")), sheets[0])
before = _load_metric_rows(db, batch_id=before_batch_id, metric_id=target["metric_id"])
after = _load_metric_rows(db, batch_id=after_batch_id, metric_id=target["metric_id"])
return mapping_stats( return mapping_stats(
before_rows=before, before_rows=before,
after_rows=after, after_rows=after,
iface_fields=list(tpl.iface_fields or []), iface_fields=list(target.get("iface_fields") or []),
port_map=pmap, port_map=pmap,
) )
@ -393,7 +568,7 @@ def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]:
ensure_default_templates(db) ensure_default_templates(db)
template_id = str(body.get("template_id") or "").strip() template_id = str(body.get("template_id") or "").strip()
if not template_id: if not template_id:
tpl = ensure_default_lldp_template(db) tpl = ensure_default_cutover_template(db)
template_id = tpl.id template_id = tpl.id
else: else:
if not db.get(BizCompareTemplate, template_id): if not db.get(BizCompareTemplate, template_id):
@ -470,6 +645,40 @@ def _resolve_after_batch(db: Session, job: BizCompareJob) -> str:
return str(latest.id) if latest else "" return str(latest.id) if latest else ""
def _run_sheet(
db: Session,
*,
sheet: dict[str, Any],
before_batch_id: str,
after_batch_id: str,
port_map: dict[str, str],
) -> dict[str, Any]:
key_fields = list(sheet.get("key_fields") or [])
iface_fields = list(sheet.get("iface_fields") or [])
compare_fields = list(sheet.get("compare_fields") or [])
mode = "presence" if not compare_fields else "fields"
before_rows = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"])
after_rows = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"])
result = compare_rows(
before_rows=before_rows,
after_rows=after_rows,
key_fields=key_fields,
iface_fields=iface_fields,
compare_fields=compare_fields,
port_map=port_map,
)
return {
"metric_id": sheet["metric_id"],
"key_fields": key_fields,
"iface_fields": iface_fields,
"compare_fields": compare_fields,
"mode": mode,
"summary": result["summary"],
"diffs": result["diffs"],
"mapping_stats": result["mapping_stats"],
}
def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]: def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]:
j = db.get(BizCompareJob, job_id) j = db.get(BizCompareJob, job_id)
if not j: if not j:
@ -484,26 +693,52 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
if not db.get(BizStateBatch, before_batch_id) or not db.get(BizStateBatch, after_batch_id): if not db.get(BizStateBatch, before_batch_id) or not db.get(BizStateBatch, after_batch_id):
raise HTTPException(status_code=404, detail="batch_not_found") raise HTTPException(status_code=404, detail="batch_not_found")
key_fields = list(tpl.key_fields or []) sheets_cfg = template_metrics(tpl)
iface_fields = list(tpl.iface_fields or []) if not sheets_cfg:
compare_fields = list(tpl.compare_fields or []) raise HTTPException(status_code=400, detail="template_has_no_metrics")
ignore = set(str(x) for x in (tpl.ignore_fields or []))
compare_fields = [f for f in compare_fields if f not in ignore]
if not compare_fields:
compare_fields = [f for f in key_fields if f not in ignore]
before_rows = _load_metric_rows(db, batch_id=before_batch_id, metric_id=tpl.metric_id)
after_rows = _load_metric_rows(db, batch_id=after_batch_id, metric_id=tpl.metric_id)
pmap = _port_map_dict(db, j.mapping_id) pmap = _port_map_dict(db, j.mapping_id)
sheet_results: list[dict[str, Any]] = []
agg = {
"before_count": 0,
"after_count": 0,
"added": 0,
"removed": 0,
"changed": 0,
"unchanged": 0,
}
mapping_by_metric: dict[str, Any] = {}
for sheet in sheets_cfg:
one = _run_sheet(
db,
sheet=sheet,
before_batch_id=before_batch_id,
after_batch_id=after_batch_id,
port_map=pmap,
)
sheet_results.append(one)
s = one["summary"]
for k in agg:
agg[k] += int(s.get(k) or 0)
mapping_by_metric[one["metric_id"]] = one["mapping_stats"]
result = compare_rows( first = sheet_results[0]
before_rows=before_rows, summary_payload = {
after_rows=after_rows, **agg,
key_fields=key_fields, "sheet_count": len(sheet_results),
iface_fields=iface_fields, "sheets": [
compare_fields=compare_fields, {
port_map=pmap, "metric_id": s["metric_id"],
) "key_fields": s["key_fields"],
"iface_fields": s["iface_fields"],
"compare_fields": s["compare_fields"],
"mode": s["mode"],
"summary": s["summary"],
"diffs": s["diffs"],
}
for s in sheet_results
],
}
run = BizCompareRun( run = BizCompareRun(
id=uuid4().hex, id=uuid4().hex,
@ -512,11 +747,11 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
mapping_id=j.mapping_id, mapping_id=j.mapping_id,
before_batch_id=before_batch_id, before_batch_id=before_batch_id,
after_batch_id=after_batch_id, after_batch_id=after_batch_id,
metric_id=tpl.metric_id, metric_id=first["metric_id"],
status="success", status="success",
summary_json=result["summary"], summary_json=summary_payload,
diffs_json=result["diffs"], diffs_json=first["diffs"],
mapping_stats_json=result["mapping_stats"], mapping_stats_json=mapping_by_metric,
message="", message="",
created_at=_utcnow(), created_at=_utcnow(),
) )
@ -533,6 +768,21 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
if not r: if not r:
raise HTTPException(status_code=404, detail="run_not_found") raise HTTPException(status_code=404, detail="run_not_found")
tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None
summary = dict(r.summary_json or {})
sheets = list(summary.get("sheets") or [])
if not sheets:
# Legacy single-metric run
sheets = [
{
"metric_id": r.metric_id,
"key_fields": list((tpl.key_fields if tpl else None) or []),
"iface_fields": list((tpl.iface_fields if tpl else None) or []),
"compare_fields": list((tpl.compare_fields if tpl else None) or []),
"mode": "fields",
"summary": {k: summary.get(k, 0) for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count")},
"diffs": list(r.diffs_json or []),
}
]
return { return {
"id": r.id, "id": r.id,
"job_id": r.job_id, "job_id": r.job_id,
@ -542,8 +792,12 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
"after_batch_id": r.after_batch_id, "after_batch_id": r.after_batch_id,
"metric_id": r.metric_id, "metric_id": r.metric_id,
"status": r.status, "status": r.status,
"summary": r.summary_json or {}, "summary": {
"diffs": r.diffs_json or [], k: summary.get(k, 0)
for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count", "sheet_count")
},
"sheets": sheets,
"diffs": list(r.diffs_json or []),
"mapping_stats": r.mapping_stats_json or {}, "mapping_stats": r.mapping_stats_json or {},
"message": r.message, "message": r.message,
"created_at": r.created_at.isoformat() + "Z" if r.created_at else None, "created_at": r.created_at.isoformat() + "Z" if r.created_at else None,
@ -565,7 +819,10 @@ def list_runs(db: Session, job_id: str, *, limit: int = 20) -> list[dict[str, An
"before_batch_id": r.before_batch_id, "before_batch_id": r.before_batch_id,
"after_batch_id": r.after_batch_id, "after_batch_id": r.after_batch_id,
"status": r.status, "status": r.status,
"summary": r.summary_json or {}, "summary": {
k: (r.summary_json or {}).get(k, 0)
for k in ("added", "removed", "changed", "unchanged", "sheet_count")
},
"created_at": r.created_at.isoformat() + "Z" if r.created_at else None, "created_at": r.created_at.isoformat() + "Z" if r.created_at else None,
} }
for r in rows for r in rows

View file

@ -16,6 +16,7 @@ def apply_biz_state_schema(conn: Connection) -> None:
# Tables are defined on ORM Base; create_all covers new installs. # Tables are defined on ORM Base; create_all covers new installs.
# Keep lightweight indexes that older DBs might miss. # Keep lightweight indexes that older DBs might miss.
for sql in ( for sql in (
"ALTER TABLE biz_compare_template ADD COLUMN IF NOT EXISTS metrics_json JSON DEFAULT '[]'",
"CREATE INDEX IF NOT EXISTS ix_biz_state_task_status ON biz_state_task (status)", "CREATE INDEX IF NOT EXISTS ix_biz_state_task_status ON biz_state_task (status)",
"CREATE INDEX IF NOT EXISTS ix_biz_state_batch_task_id ON biz_state_batch (task_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_task_id ON biz_state_batch (task_id)",
"CREATE INDEX IF NOT EXISTS ix_biz_state_batch_command_batch_id ON biz_state_batch_command (batch_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_command_batch_id ON biz_state_batch_command (batch_id)",

View file

@ -212,24 +212,35 @@ def api_export_batch(batch_id: str, db: Session = Depends(get_db)) -> StreamingR
from .biz_state import compare_service as cmp_svc # noqa: E402 from .biz_state import compare_service as cmp_svc # noqa: E402
class TemplateMetricIn(BaseModel):
metric_id: str
key_fields: list[str] = Field(default_factory=list)
iface_fields: list[str] = Field(default_factory=list)
compare_fields: list[str] = Field(default_factory=list)
class TemplateIn(BaseModel): class TemplateIn(BaseModel):
name: str = "" name: str = ""
note: str = ""
# Preferred: multi-metric sheets
metrics: list[TemplateMetricIn] | None = None
# Legacy single-metric fields (still accepted)
metric_id: str = "lldp_neighbor" metric_id: str = "lldp_neighbor"
key_fields: list[str] = Field(default_factory=list) key_fields: list[str] = Field(default_factory=list)
iface_fields: list[str] = Field(default_factory=list) iface_fields: list[str] = Field(default_factory=list)
compare_fields: list[str] = Field(default_factory=list) compare_fields: list[str] = Field(default_factory=list)
ignore_fields: list[str] = Field(default_factory=list) ignore_fields: list[str] = Field(default_factory=list)
note: str = ""
class TemplatePatchIn(BaseModel): class TemplatePatchIn(BaseModel):
name: str | None = None name: str | None = None
note: str | None = None
metrics: list[TemplateMetricIn] | None = None
metric_id: str | None = None metric_id: str | None = None
key_fields: list[str] | None = None key_fields: list[str] | None = None
iface_fields: list[str] | None = None iface_fields: list[str] | None = None
compare_fields: list[str] | None = None compare_fields: list[str] | None = None
ignore_fields: list[str] | None = None ignore_fields: list[str] | None = None
note: str | None = None
class MappingRowIn(BaseModel): class MappingRowIn(BaseModel):

View file

@ -171,7 +171,12 @@ class BizStateCommandOverride(Base):
class BizCompareTemplate(Base): class BizCompareTemplate(Base):
"""Global compare template: which fields are keys / interfaces / compared.""" """Compare template: one or more metric sheets with key/iface/compare roles.
``metrics_json`` is the source of truth (list of sheet defs). Legacy
``metric_id`` / ``key_fields`` / … mirror the first sheet for older rows.
Empty ``compare_fields`` on a sheet = presence-only (entry set match).
"""
__tablename__ = "biz_compare_template" __tablename__ = "biz_compare_template"
@ -182,6 +187,7 @@ class BizCompareTemplate(Base):
iface_fields: Mapped[list] = mapped_column(_JsonType, default=list) iface_fields: Mapped[list] = mapped_column(_JsonType, default=list)
compare_fields: Mapped[list] = mapped_column(_JsonType, default=list) compare_fields: Mapped[list] = mapped_column(_JsonType, default=list)
ignore_fields: Mapped[list] = mapped_column(_JsonType, default=list) ignore_fields: Mapped[list] = mapped_column(_JsonType, default=list)
metrics_json: Mapped[list] = mapped_column(_JsonType, default=list)
note: Mapped[str] = mapped_column(String(512), default="") note: Mapped[str] = mapped_column(String(512), default="")
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive)
updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive)

View file

@ -62,6 +62,30 @@ class CompareEngineTests(unittest.TestCase):
self.assertIn("missing", stats["miss_before"]) self.assertIn("missing", stats["miss_before"])
self.assertFalse(stats["ok"]) self.assertFalse(stats["ok"])
def test_presence_only_empty_compare(self) -> None:
"""Empty compare_fields → only entry set matters; value diffs ignored."""
before = [
{"local_if": "gei-0/1", "remote_sys": "A", "remote_if": "x1", "remote_ip": "1.1.1.1"},
{"local_if": "gei-0/2", "remote_sys": "B", "remote_if": "y1", "remote_ip": "2.2.2.2"},
]
after = [
{"local_if": "gei-0/1", "remote_sys": "A", "remote_if": "x1", "remote_ip": "9.9.9.9"},
{"local_if": "gei-0/3", "remote_sys": "C", "remote_if": "z1", "remote_ip": ""},
]
out = compare_rows(
before_rows=before,
after_rows=after,
key_fields=["local_if", "remote_sys", "remote_if"],
iface_fields=["local_if"],
compare_fields=[],
port_map={},
)
s = out["summary"]
self.assertEqual(s["changed"], 0)
self.assertEqual(s["unchanged"], 1)
self.assertEqual(s["removed"], 1)
self.assertEqual(s["added"], 1)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

View file

@ -131,8 +131,8 @@ src/
- API:`/v1/biz-state/profiles`、`/discover`、`/tasks*`、`/batches*`、`/batches/{id}/export`、`/compare/*` - API:`/v1/biz-state/profiles`、`/discover`、`/tasks*`、`/batches*`、`/batches/{id}/export`、`/compare/*`
- **ParseProfile**:命令模板 + TextFSM + 回调 + schema;LLDP 与拓扑 **共享解析**(`lldp_shared`),业务流程写批次表,拓扑写 Fabric - **ParseProfile**:命令模板 + TextFSM + 回调 + schema;LLDP 与拓扑 **共享解析**(`lldp_shared`),业务流程写批次表,拓扑写 Fabric
- Phase1:LLDP 邻居快照;建任务默认启用对应厂商 LLDP profile;支持自定义只采不解析行 - Phase1:LLDP 邻居快照;建任务默认启用对应厂商 LLDP profile;支持自定义只采不解析行
- Phase2:比对模板(可选 metric / key / 接口 / 比对字段)、端口映射、CompareJob 列表与详情、`pre/post` 结果表(高亮 + 筛选)、`/network/tasks/biz-compare` - Phase2:比对模板(多监控表 / Key·接口·比对字段;空比对=仅条目)、端口映射、CompareJob 列表与详情、先选批次再按 Excel sheet 看 pre/post、`/network/tasks/biz-compare`
- API 补充:`GET /v1/biz-state/compare/metrics`(模板字段目录) - API 补充:`GET /v1/biz-state/compare/metrics`;模板 `metrics[]` 多表结构
- Phase3:VRF 发现→人选关联→带参采集(`*.route_vrf_summary`);`PUT …/items/{id}/bindings` - Phase3:VRF 发现→人选关联→带参采集(`*.route_vrf_summary`);`PUT …/items/{id}/bindings`
- 调度:`NETX_BIZ_STATE_SCHEDULER_ENABLED`(默认开),tick `NETX_BIZ_STATE_SCHEDULER_TICK_SEC` - 调度:`NETX_BIZ_STATE_SCHEDULER_ENABLED`(默认开),tick `NETX_BIZ_STATE_SCHEDULER_TICK_SEC`
- 前端:`/network/tasks/biz-state`(勾选监控项 / 发现 VRF / 批次 / 导出 zip) - 前端:`/network/tasks/biz-state`(勾选监控项 / 发现 VRF / 批次 / 导出 zip)

View file

@ -238,7 +238,7 @@ const en = {
confirmDeleteTemplate: "Delete this compare template?", confirmDeleteTemplate: "Delete this compare template?",
emptyTemplates: "No templates yet. Create one or use built-ins.", emptyTemplates: "No templates yet. Create one or use built-ins.",
templateHint: templateHint:
"Field roles: Key aligns rows; interface fields use port mapping; compare fields show pre/post; ignore skips diff.", "A template holds multiple metric sheets. Key: row identity (no separate compare). Interface: port mapping. Compare: value checks after key match. Leave compare empty for presence-only (entry set match).",
keyFields: "Key", keyFields: "Key",
ifaceFields: "Interface", ifaceFields: "Interface",
compareFields: "Compare", compareFields: "Compare",
@ -246,9 +246,20 @@ const en = {
field: "Field", field: "Field",
note: "Note", note: "Note",
keyRequired: "Select at least one key field", keyRequired: "Select at least one key field",
keyIsIdentity: "Key aligns rows; not used as a value-compare field",
metricsRequired: "Add at least one metric sheet",
noMetricFields: "No fields for this metric", noMetricFields: "No fields for this metric",
templateFilterPh: "Filter template / metric", templateFilterPh: "Filter template / metric",
jobFilterPh: "Filter job / template / status", jobFilterPh: "Filter job / template / status",
addMetric: "Add metric",
pickMetric: "Pick metric…",
addSheet: "Add to template",
removeSheet: "Remove sheet",
colSheets: "Sheets",
sheetsUnit: "sheets",
presenceShort: "presence",
modeFields: "Mode: compare selected field values",
modePresence: "Mode: presence-only (entry set match)",
mapping: "Port mapping", mapping: "Port mapping",
newMapping: "New mapping…", newMapping: "New mapping…",
mapName: "Mapping name", mapName: "Mapping name",
@ -284,7 +295,8 @@ const en = {
runs: "Run history", runs: "Run history",
noRuns: "No runs yet — run a compare first", noRuns: "No runs yet — run a compare first",
result: "Result", result: "Result",
pickRun: "Pick a run below or click Run now", pickBatchRun: "Select compare batch",
pickRun: "Select batch…",
resultEmpty: "No matching diff rows", resultEmpty: "No matching diff rows",
resultFilterPh: "Filter key / values…", resultFilterPh: "Filter key / values…",
kindAll: "All kinds", kindAll: "All kinds",

View file

@ -237,7 +237,8 @@ const zh = {
templateDeleted: "模板已删除", templateDeleted: "模板已删除",
confirmDeleteTemplate: "删除该比对模板?", confirmDeleteTemplate: "删除该比对模板?",
emptyTemplates: "暂无模板。可新建或使用系统默认。", emptyTemplates: "暂无模板。可新建或使用系统默认。",
templateHint: "勾选字段角色:Key 用于对齐行;接口字段参与端口映射;比对字段左右对照;忽略字段不参与 diff。", templateHint:
"一个模板可包含多张监控表。Key:对齐前后条目(选中后即为行身份,无需再勾比对);接口:参与端口映射;比对:勾选具体参数做值检查。若不勾任何比对字段,则只校验条目是否一致(存在性)。",
keyFields: "Key", keyFields: "Key",
ifaceFields: "接口", ifaceFields: "接口",
compareFields: "比对", compareFields: "比对",
@ -245,9 +246,20 @@ const zh = {
field: "字段", field: "字段",
note: "备注", note: "备注",
keyRequired: "至少选择一个 Key 字段", keyRequired: "至少选择一个 Key 字段",
keyIsIdentity: "Key 用于对齐行,不作为值比对字段",
metricsRequired: "请至少添加一张监控表",
noMetricFields: "该 metric 无字段定义", noMetricFields: "该 metric 无字段定义",
templateFilterPh: "筛选模板名 / metric", templateFilterPh: "筛选模板名 / metric",
jobFilterPh: "筛选任务名 / 模板 / 状态", jobFilterPh: "筛选任务名 / 模板 / 状态",
addMetric: "添加监控项",
pickMetric: "选择监控项…",
addSheet: "加入模板",
removeSheet: "移出本表",
colSheets: "监控表",
sheetsUnit: "张表",
presenceShort: "仅条目",
modeFields: "模式:按比对字段检查值",
modePresence: "模式:仅校验前后条目一致(不比具体值)",
mapping: "端口映射", mapping: "端口映射",
newMapping: "新建映射…", newMapping: "新建映射…",
mapName: "映射名称", mapName: "映射名称",
@ -283,7 +295,8 @@ const zh = {
runs: "历史 Run", runs: "历史 Run",
noRuns: "尚无比对记录,请先执行比对", noRuns: "尚无比对记录,请先执行比对",
result: "比对结果", result: "比对结果",
pickRun: "选择底部 Run 或点「立即比对」", pickBatchRun: "选择比对批次",
pickRun: "选择批次…",
resultEmpty: "无匹配差异行", resultEmpty: "无匹配差异行",
resultFilterPh: "筛选 key / 字段值…", resultFilterPh: "筛选 key / 字段值…",
kindAll: "全部类型", kindAll: "全部类型",

View file

@ -9917,6 +9917,28 @@ html.login-page--paused .login-page__flare {
margin-bottom: 10px; margin-bottom: 10px;
} }
.nm-page-panel .nm-config-modal__tabs button.is-active,
.bs-cmp-job-body .nm-config-modal__tabs button.is-active {
outline: 2px solid rgba(37, 99, 235, 0.55);
outline-offset: 1px;
box-shadow: 0 0 0 1px rgba(37, 99, 235, 0.25);
font-weight: 600;
}
.bs-cmp-metric-chips {
display: flex;
flex-wrap: wrap;
gap: 4px;
}
.bs-cmp-metric-chip {
display: inline-block;
padding: 2px 6px;
border-radius: 4px;
background: rgba(148, 163, 184, 0.14);
font-size: 12px;
}
/* biz_state batch workbook (Excel-like sheets) */ /* biz_state batch workbook (Excel-like sheets) */
.bs-workbook-modal .bs-workbook-body { .bs-workbook-modal .bs-workbook-body {
min-height: min(62vh, 640px); min-height: min(62vh, 640px);

File diff suppressed because it is too large Load diff