mirror of
https://github.com/hansjone/netx.git
synced 2026-10-15 01:00:53 +08:00
Optimize cutover compare/monitor per the 2026-10-12 plan.
Correctness (A): normalize port-map names with the template rules (snapshot v2, v1 replays unchanged); protect mappings/templates that are referenced; stop dropping edited mapping text; accept pulled-but-not-shutdown ports as migrated; validate baseline ownership on project create; one canonical key form; fix built-in template tolerances and upgrade old packaged values in place. Cross-vendor identity (B): single identity pipeline for every path, per-side and per-vendor interface rules, value maps, filters after normalization, mapping pre-flight for compare and cutover projects, SQL eligibility fixes. Data quality (V): no_data / suspect_parse / not_collected sheet states, command slice coverage, directional compare modes, value-sorted duplicate pairing, before-side raw names in results and CSV. Robustness (H): bounded auto-run history, background evaluate/finish with polling, manual route tables out of the acceptance gate, baseline row cache, scoped duplicate-key failures, merged HF tasks with a per-NE session cap, same-device cutover, explicit scope_by_port_map, read-only GETs, resolvable red tickets. Frontend: pre-flight report, mapping table view, template editor support for the new rules, data-state banners. Docs: canonical field contract. Not included: Huawei status parsers (no real samples), SQL-path route compare. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fff7d63b9b
commit
315539e0fd
59 changed files with 6483 additions and 487 deletions
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
|
@ -25,6 +27,11 @@ def max_concurrent_tasks() -> int:
|
|||
return max(1, int(getattr(settings, "biz_state_max_concurrent_tasks", 16) or 16))
|
||||
|
||||
|
||||
def max_sessions_per_ne() -> int:
|
||||
"""Concurrent SSH collections allowed per network element (>= 1)."""
|
||||
return max(1, int(getattr(settings, "biz_state_max_sessions_per_ne", 2) or 2))
|
||||
|
||||
|
||||
def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]:
|
||||
"""Create a queued batch and mark task collect_running. Does not open SSH.
|
||||
|
||||
|
|
@ -135,15 +142,16 @@ def enqueue_collect_group(group_id: str, *, manual: bool = False,
|
|||
BizStateTask.id).with_for_update().populate_existing().all()
|
||||
result: dict[str, Any] = {"ok": False, "queued": False, "task_id": requested_task_id,
|
||||
"collect_group_id": group_id}
|
||||
if len(tasks) != 2 or len({t.ne_id for t in tasks}) != 2:
|
||||
# One task = a single device used as both sides (same-device cutover); two tasks = a pair.
|
||||
if len(tasks) not in (1, 2) or len({t.ne_id for t in tasks}) != len(tasks):
|
||||
return {**result, "reason": "collect_pair_invalid"}
|
||||
if max_concurrent_tasks() < 2:
|
||||
if max_concurrent_tasks() < len(tasks):
|
||||
return {**result, "reason": "collect_pair_capacity"}
|
||||
from ..cli_budget import clamp_cli_workers
|
||||
|
||||
worker_cap = clamp_cli_workers(int(getattr(settings, "biz_state_worker_collect_threads", None)
|
||||
or getattr(settings, "biz_state_dispatch_workers", 8) or 8))
|
||||
if worker_cap < 2:
|
||||
if worker_cap < len(tasks):
|
||||
return {**result, "reason": "collect_pair_capacity"}
|
||||
if any(t.collect_running for t in tasks):
|
||||
return {**result, "reason": "collect_pair_busy"}
|
||||
|
|
@ -230,13 +238,12 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
|
|||
if slots <= 0:
|
||||
return []
|
||||
|
||||
busy_nes = {
|
||||
str(r[0] or "").strip()
|
||||
for r in db.query(BizStateBatch.ne_id)
|
||||
.filter(BizStateBatch.status == "running")
|
||||
.all()
|
||||
if str(r[0] or "").strip()
|
||||
}
|
||||
# Running collections per NE; an NE accepts new ones until it reaches its session cap.
|
||||
per_ne_cap = max_sessions_per_ne()
|
||||
ne_running: dict[str, int] = defaultdict(int)
|
||||
for (running_ne,) in db.query(BizStateBatch.ne_id).filter(BizStateBatch.status == "running").all():
|
||||
if str(running_ne or "").strip():
|
||||
ne_running[str(running_ne).strip()] += 1
|
||||
|
||||
# Manual route rounds yield to fresh monitoring, but must eventually
|
||||
# run even when a busy device always has regular metrics queued.
|
||||
|
|
@ -266,14 +273,19 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
|
|||
peers = db.query(BizStateBatch).filter(
|
||||
BizStateBatch.collect_round_id == batch.collect_round_id,
|
||||
).with_for_update(skip_locked=pg).all()
|
||||
if len(peers) != 2 or any(p.status != "queued" for p in peers):
|
||||
expected = db.query(BizStateTask.id).filter(
|
||||
BizStateTask.collect_group_id == batch.collect_group_id).count() if batch.collect_group_id else 2
|
||||
if len(peers) != max(1, expected) or any(p.status != "queued" for p in peers):
|
||||
continue
|
||||
group = peers
|
||||
visited.update(p.id for p in group)
|
||||
if len(group) > slots - len(claimed_rows):
|
||||
continue
|
||||
nes = {str(p.ne_id or "").strip() for p in group}
|
||||
if nes & busy_nes:
|
||||
wanted: dict[str, int] = defaultdict(int)
|
||||
for p in group:
|
||||
if str(p.ne_id or "").strip():
|
||||
wanted[str(p.ne_id).strip()] += 1
|
||||
if any(ne_running[ne] + n > per_ne_cap for ne, n in wanted.items()):
|
||||
continue
|
||||
now = _utcnow()
|
||||
for member in group:
|
||||
|
|
@ -285,7 +297,8 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
|
|||
if task:
|
||||
task.last_collect_started_at = now
|
||||
claimed_rows.append(member)
|
||||
busy_nes.update(nes - {""})
|
||||
for ne, n in wanted.items():
|
||||
ne_running[ne] += n
|
||||
|
||||
if not claimed_rows:
|
||||
db.rollback()
|
||||
|
|
|
|||
|
|
@ -93,8 +93,17 @@ def apply_port_map(
|
|||
return out
|
||||
|
||||
|
||||
def key_tuple(row: Mapping[str, Any], fields: Sequence[str]) -> tuple[str, ...]:
|
||||
"""The one canonical match-key form: ``scalar_text(...).strip()`` per field.
|
||||
|
||||
Every place that builds, stores or looks up a key string must go through
|
||||
this (``None`` → ``""``, ``True`` → ``"true"``, surrounding blanks trimmed).
|
||||
"""
|
||||
return tuple(scalar_text(row.get(k)).strip() for k in fields)
|
||||
|
||||
|
||||
def row_key(row: dict[str, Any], key_fields: list[str]) -> tuple[str, ...]:
|
||||
return tuple(scalar_text(row.get(k)).strip() for k in key_fields)
|
||||
return key_tuple(row, key_fields)
|
||||
|
||||
|
||||
def mapping_stats(
|
||||
|
|
@ -109,8 +118,8 @@ def mapping_stats(
|
|||
``hit_before``: map key appears as a normalized before iface, or as the
|
||||
parent of a before subinterface (so main-port-only maps still validate).
|
||||
|
||||
Map keys are matched against **normalized** iface values (same pipeline as
|
||||
compare). Enter mapping keys in normalized form.
|
||||
Map keys are matched against **normalized** iface values; callers pass a
|
||||
port map already normalized with the template iface rules.
|
||||
"""
|
||||
before_ifaces: set[str] = set()
|
||||
after_ifaces: set[str] = set()
|
||||
|
|
@ -156,7 +165,7 @@ def mapping_stats(
|
|||
"miss_after": miss_after,
|
||||
"unused_before_keys": unused,
|
||||
"ok": not miss_before and not miss_after,
|
||||
"hint": "map keys must match normalized iface names (post iface_normalize)",
|
||||
"hint": "map keys/values are normalized with the template iface rules before matching",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -174,6 +183,10 @@ def compare_rows(
|
|||
include_unchanged: bool = False,
|
||||
unchanged_limit: int | None = None,
|
||||
compact_unchanged: bool = False,
|
||||
pipeline: Any = None,
|
||||
match_keys_override: Sequence[str] | None = None,
|
||||
prepared: tuple[list, list] | None = None,
|
||||
duplicate_pairing: str = "ordered",
|
||||
) -> dict[str, Any]:
|
||||
"""Return summary + diffs list.
|
||||
|
||||
|
|
@ -200,6 +213,11 @@ def compare_rows(
|
|||
- ``True``: force drop iface from match key when a non-empty candidate exists.
|
||||
- ``False``: never drop iface from match key.
|
||||
|
||||
``pipeline`` (``identity_pipeline.Pipeline``) owns normalization, filtering,
|
||||
port mapping and key building; when omitted a legacy pipeline is derived
|
||||
from the loose arguments (callers have already filtered the rows).
|
||||
``match_keys_override`` pins the match key (re-running a stored result).
|
||||
|
||||
``summary.duplicate`` is always 0. ``duplicate_keys_*`` count match keys
|
||||
that appear more than once on a side (diagnostic only).
|
||||
|
||||
|
|
@ -208,32 +226,40 @@ def compare_rows(
|
|||
"""
|
||||
if not key_fields:
|
||||
raise ValueError("key_fields required")
|
||||
pmap = dict(port_map or {})
|
||||
rules = field_rule_map(field_rules)
|
||||
norm_rules = normalize_iface_rules(iface_normalize_rules)
|
||||
iface_list = [str(f) for f in (iface_fields or []) if str(f).strip()]
|
||||
if pipeline is None:
|
||||
from .identity_pipeline import legacy_pipeline
|
||||
|
||||
pipeline = legacy_pipeline(
|
||||
key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields,
|
||||
field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules,
|
||||
ignore_port_changes=ignore_port_changes, duplicate_pairing=duplicate_pairing,
|
||||
)
|
||||
pmap = dict(pipeline.port_map)
|
||||
rules = field_rule_map(field_rules if field_rules is not None else pipeline.field_rules)
|
||||
iface_list = list(pipeline.iface_fields)
|
||||
iface_set = set(iface_list)
|
||||
|
||||
before_norm = apply_iface_normalize_rows(
|
||||
before_rows, iface_fields=iface_list, rules=norm_rules
|
||||
) if norm_rules and iface_list else before_rows
|
||||
after_norm = apply_iface_normalize_rows(
|
||||
after_rows, iface_fields=iface_list, rules=norm_rules
|
||||
) if norm_rules and iface_list else after_rows
|
||||
if prepared is not None:
|
||||
before_pairs, after_pairs = prepared
|
||||
else:
|
||||
before_pairs = pipeline.prepare(before_rows, "before")
|
||||
after_pairs = pipeline.prepare(after_rows, "after")
|
||||
before_rows = [orig for orig, _ in before_pairs]
|
||||
after_rows = [orig for orig, _ in after_pairs]
|
||||
before_norm = [norm for _, norm in before_pairs]
|
||||
after_norm = [norm for _, norm in after_pairs]
|
||||
|
||||
ignore_ports = False
|
||||
# No map → optionally ignore port renames by dropping iface from match key.
|
||||
if not pmap and iface_set:
|
||||
candidate = [k for k in key_fields if k not in iface_set]
|
||||
if not candidate:
|
||||
match_keys = list(key_fields)
|
||||
elif ignore_port_changes is True:
|
||||
match_keys = candidate
|
||||
ignore_ports = True
|
||||
elif ignore_port_changes is False:
|
||||
match_keys = list(key_fields)
|
||||
else:
|
||||
# Auto heuristic (legacy default)
|
||||
if match_keys_override is not None:
|
||||
match_keys = [str(k) for k in match_keys_override]
|
||||
ignore_ports = match_keys != list(key_fields)
|
||||
else:
|
||||
# No map → optionally ignore port renames by dropping iface from match key.
|
||||
policy = pipeline.ignore_port_changes
|
||||
unique = None
|
||||
if not pmap and iface_set and policy is None:
|
||||
candidate = [k for k in key_fields if k not in iface_set]
|
||||
|
||||
def unique_keys(rows: list[dict[str, Any]]) -> bool:
|
||||
seen: set[tuple[str, ...]] = set()
|
||||
for row in rows:
|
||||
|
|
@ -243,23 +269,20 @@ def compare_rows(
|
|||
seen.add(key)
|
||||
return True
|
||||
|
||||
if unique_keys(before_norm) and unique_keys(after_norm):
|
||||
match_keys = candidate
|
||||
ignore_ports = True
|
||||
else:
|
||||
match_keys = list(key_fields)
|
||||
else:
|
||||
match_keys = list(key_fields)
|
||||
unique = bool(candidate) and unique_keys(before_norm) and unique_keys(after_norm)
|
||||
match_keys = pipeline.effective_match_keys(unique_without_iface=unique)
|
||||
ignore_ports = match_keys != list(key_fields)
|
||||
|
||||
before_groups: dict[tuple[str, ...], list[tuple[dict[str, Any], dict[str, Any]]]] = (
|
||||
defaultdict(list)
|
||||
)
|
||||
after_groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list)
|
||||
for orig, norm in zip(before_rows, before_norm):
|
||||
mapped = apply_port_map(norm, iface_fields=iface_list, port_map=pmap) if iface_list else norm
|
||||
before_groups[row_key(mapped, match_keys)].append((orig, mapped))
|
||||
for r in after_norm:
|
||||
after_groups[row_key(r, match_keys)].append(r)
|
||||
mapped, k = pipeline.key_of(norm, "before", match_keys)
|
||||
before_groups[k].append((orig, mapped))
|
||||
for norm in after_norm:
|
||||
after_groups[pipeline.key_of(norm, "after", match_keys)[1]].append(norm)
|
||||
sort_pairs = pipeline.duplicate_pairing == "sorted"
|
||||
|
||||
diffs: list[dict[str, Any]] = []
|
||||
added = removed = changed = unchanged = 0
|
||||
|
|
@ -271,7 +294,7 @@ def compare_rows(
|
|||
sample = _StratifiedSample(limit_n) if include_unchanged and limit_n is not None else None
|
||||
|
||||
def _key_obj(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {f: row.get(f, "") for f in key_fields}
|
||||
return dict(zip(key_fields, key_tuple(row, key_fields)))
|
||||
|
||||
def _row_id(row: dict[str, Any] | None) -> str:
|
||||
if not isinstance(row, dict):
|
||||
|
|
@ -325,6 +348,10 @@ def compare_rows(
|
|||
multi_before_keys.append(k)
|
||||
if len(a_list) > 1:
|
||||
multi_after_keys.append(k)
|
||||
if sort_pairs and (len(b_list) > 1 or len(a_list) > 1):
|
||||
# Same-key rows pair by value, not by CLI output order (ECMP, multipath).
|
||||
b_list = sorted(b_list, key=lambda pair: pipeline.pair_sort_key(pair[1]))
|
||||
a_list = sorted(a_list, key=pipeline.pair_sort_key)
|
||||
n = max(len(b_list), len(a_list))
|
||||
for i in range(n):
|
||||
if i >= len(b_list):
|
||||
|
|
|
|||
194
netx_api/biz_state/compare_preflight.py
Normal file
194
netx_api/biz_state/compare_preflight.py
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
"""Pre-flight check of a port mapping against the two batches it will be applied to.
|
||||
|
||||
The same function backs the compare "validate mapping" button and the cutover
|
||||
project pre-flight, so both modes judge mapping coverage the same way. It answers
|
||||
before anyone runs a comparison:
|
||||
|
||||
- which *before* interfaces are mapped (exact / by parent), unmapped but present on
|
||||
the *after* side by the same name, or **suspect** (not mapped, not on *after*)
|
||||
- which map keys / targets match nothing, which targets receive several sources
|
||||
- whether each sheet's two sides were actually collected (``data_state``)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Mapping, Sequence
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .compare_quality import assess_sheet, command_coverage, collect_quality, side_receipts
|
||||
from .identity_pipeline import build_pipeline
|
||||
from .iface_normalize import normalize_port_map, resolve_mapped_iface
|
||||
|
||||
LIST_CAP = 200
|
||||
# Million-row tables are never loaded just to look at interface names.
|
||||
HEAVY_METRICS = frozenset({"bgp_route", "ip_route", "ipv6_route", "l2vpn_mac", "evpn_mac", "interface_detail"})
|
||||
|
||||
RowsLoader = Callable[[str, Mapping[str, Any]], list[dict[str, Any]]]
|
||||
|
||||
|
||||
def _parents(name: str) -> list[str]:
|
||||
parts = name.split(".")
|
||||
return [".".join(parts[:i]) for i in range(len(parts) - 1, 0, -1)]
|
||||
|
||||
|
||||
def _classify(name: str, pmap: Mapping[str, str], after: set[str]) -> str:
|
||||
if name in pmap:
|
||||
return "mapped_exact"
|
||||
if any(p in pmap for p in _parents(name)):
|
||||
return "mapped_by_parent"
|
||||
if name in after:
|
||||
return "identity_same_name"
|
||||
return "suspect_unmapped"
|
||||
|
||||
|
||||
def _present(name: str, names: set[str]) -> bool:
|
||||
"""``name`` exists, or is the parent of an existing sub-interface."""
|
||||
if name in names:
|
||||
return True
|
||||
prefix = name + "."
|
||||
return any(n.startswith(prefix) for n in names)
|
||||
|
||||
|
||||
def _top_level(names: Sequence[str]) -> list[str]:
|
||||
"""Drop sub-interfaces whose parent is already in the list (noise)."""
|
||||
pool = set(names)
|
||||
return sorted(n for n in pool if not any(p in pool for p in _parents(n)))
|
||||
|
||||
|
||||
def sheet_mapping_coverage(
|
||||
pipe: Any,
|
||||
before_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]],
|
||||
after_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]],
|
||||
) -> dict[str, Any]:
|
||||
"""Classify every *before* interface of a sheet against the map and the *after* side.
|
||||
|
||||
``before_pairs`` / ``after_pairs`` are ``pipeline.prepare`` outputs. The result
|
||||
carries two private sets (``_after_ifaces``, ``_used_keys``) for aggregation.
|
||||
"""
|
||||
b_if = {str(n.get(f) or "").strip() for _, n in before_pairs for f in pipe.iface_fields}
|
||||
a_if = {str(n.get(f) or "").strip() for _, n in after_pairs for f in pipe.iface_fields}
|
||||
b_if.discard("")
|
||||
a_if.discard("")
|
||||
buckets: dict[str, list[str]] = {
|
||||
"mapped_exact": [], "mapped_by_parent": [], "identity_same_name": [], "suspect_unmapped": [],
|
||||
}
|
||||
for name in sorted(b_if):
|
||||
buckets[_classify(name, pipe.port_map, a_if)].append(name)
|
||||
used = {k for name in b_if for k in pipe.port_map if name == k or name.startswith(k + ".")}
|
||||
missing_targets = sorted({
|
||||
resolve_mapped_iface(n, pipe.port_map) for n in buckets["mapped_exact"] + buckets["mapped_by_parent"]
|
||||
if not _present(resolve_mapped_iface(n, pipe.port_map), a_if)
|
||||
})
|
||||
suspects = _top_level(buckets["suspect_unmapped"])
|
||||
return {
|
||||
"counts": {k: len(v) for k, v in buckets.items()},
|
||||
"mapped_exact": buckets["mapped_exact"][:LIST_CAP],
|
||||
"mapped_by_parent": buckets["mapped_by_parent"][:LIST_CAP],
|
||||
"identity_same_name": buckets["identity_same_name"][:LIST_CAP],
|
||||
"suspect_unmapped": suspects[:LIST_CAP],
|
||||
"suspect_unmapped_total": len(suspects),
|
||||
"mapped_target_missing_on_after": missing_targets[:LIST_CAP],
|
||||
"_after_ifaces": a_if,
|
||||
"_used_keys": used,
|
||||
}
|
||||
|
||||
|
||||
def preflight(
|
||||
db: Session,
|
||||
*,
|
||||
sheets: Sequence[Mapping[str, Any]],
|
||||
iface_rules: Sequence[Mapping[str, Any]] | None,
|
||||
raw_port_map: Mapping[str, str] | None,
|
||||
before_batch_id: str,
|
||||
after_batch_id: str,
|
||||
before_vendor: str = "",
|
||||
after_vendor: str = "",
|
||||
load_rows: RowsLoader,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the pre-flight report. ``load_rows(side, sheet)`` returns raw rows."""
|
||||
raw_map = dict(raw_port_map or {})
|
||||
out_sheets: list[dict[str, Any]] = []
|
||||
used_keys: set[str] = set()
|
||||
after_all: set[str] = set()
|
||||
totals = {"mapped": 0, "suspect": 0, "same_name": 0}
|
||||
risk_reasons: list[str] = []
|
||||
full_pmap: dict[str, str] = {}
|
||||
conflicts: list[dict[str, str]] = []
|
||||
|
||||
for sheet in sheets:
|
||||
sid = str(sheet.get("sheet_id") or sheet.get("metric_id") or "")
|
||||
mid = str(sheet.get("metric_id") or "")
|
||||
title = str(sheet.get("title") or sid)
|
||||
pipe = build_pipeline(
|
||||
sheet, iface_rules=iface_rules, port_map=raw_map,
|
||||
before_vendor=before_vendor, after_vendor=after_vendor,
|
||||
)
|
||||
if not full_pmap and pipe.port_map:
|
||||
full_pmap = dict(pipe.port_map)
|
||||
heavy = mid in HEAVY_METRICS
|
||||
card: dict[str, Any] = {"sheet_id": sid, "metric_id": mid, "title": title}
|
||||
if heavy:
|
||||
b_n = collect_quality(side_receipts(db, before_batch_id, mid))["row_count"]
|
||||
a_n = collect_quality(side_receipts(db, after_batch_id, mid))["row_count"]
|
||||
card.update(skipped="heavy_table", before_count=b_n, after_count=a_n)
|
||||
else:
|
||||
b_pairs = pipe.prepare(load_rows("before", sheet), "before")
|
||||
a_pairs = pipe.prepare(load_rows("after", sheet), "after")
|
||||
card.update(before_count=len(b_pairs), after_count=len(a_pairs))
|
||||
if pipe.iface_fields:
|
||||
cov = sheet_mapping_coverage(pipe, b_pairs, a_pairs)
|
||||
after_all |= cov.pop("_after_ifaces")
|
||||
used_keys |= cov.pop("_used_keys")
|
||||
card.update(cov)
|
||||
totals["mapped"] += cov["counts"]["mapped_exact"] + cov["counts"]["mapped_by_parent"]
|
||||
totals["same_name"] += cov["counts"]["identity_same_name"]
|
||||
totals["suspect"] += cov["suspect_unmapped_total"]
|
||||
card.update(assess_sheet(
|
||||
db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id,
|
||||
before_rows=int(card.get("before_count") or 0), after_rows=int(card.get("after_count") or 0),
|
||||
))
|
||||
if card["data_state"] in ("not_collected", "suspect_parse"):
|
||||
risk_reasons.append(f"{sid}:{card['data_state']}")
|
||||
out_sheets.append(card)
|
||||
|
||||
if raw_map and not full_pmap:
|
||||
full_pmap, conflicts = normalize_port_map(raw_map, iface_rules)
|
||||
elif raw_map:
|
||||
_, conflicts = normalize_port_map(raw_map, iface_rules)
|
||||
sources: dict[str, list[str]] = {}
|
||||
for k, v in full_pmap.items():
|
||||
sources.setdefault(v, []).append(k)
|
||||
n_to_1 = [{"target": t, "sources": sorted(src)} for t, src in sorted(sources.items()) if len(src) > 1]
|
||||
unused = sorted(k for k in full_pmap if k not in used_keys)
|
||||
targets_missing = sorted(v for v in set(full_pmap.values()) if after_all and not _present(v, after_all))
|
||||
|
||||
judged = totals["mapped"] + totals["suspect"]
|
||||
coverage = {
|
||||
"mapped": totals["mapped"], "suspect_unmapped": totals["suspect"],
|
||||
"identity_same_name": totals["same_name"],
|
||||
"rate": round(totals["mapped"] / judged * 100, 1) if judged else None,
|
||||
}
|
||||
mapping = {
|
||||
"pairs": len(full_pmap), "conflicts": conflicts[:LIST_CAP],
|
||||
"unused_map_keys": unused[:LIST_CAP], "unused_map_keys_total": len(unused),
|
||||
"map_targets_missing_on_after": targets_missing[:LIST_CAP],
|
||||
"map_targets_missing_on_after_total": len(targets_missing),
|
||||
"n_to_1_targets": n_to_1[:LIST_CAP],
|
||||
}
|
||||
if totals["suspect"]:
|
||||
risk_reasons.append("suspect_unmapped")
|
||||
if unused:
|
||||
risk_reasons.append("unused_map_keys")
|
||||
if targets_missing:
|
||||
risk_reasons.append("map_targets_missing_on_after")
|
||||
if conflicts:
|
||||
risk_reasons.append("map_conflicts")
|
||||
return {
|
||||
"verdict": "risk" if risk_reasons else "ready",
|
||||
"risk_reasons": sorted(set(risk_reasons)),
|
||||
"coverage": coverage,
|
||||
"mapping": mapping,
|
||||
"sheets": out_sheets,
|
||||
"command_coverage": command_coverage(db, before_batch_id=before_batch_id, after_batch_id=after_batch_id),
|
||||
}
|
||||
198
netx_api/biz_state/compare_quality.py
Normal file
198
netx_api/biz_state/compare_quality.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
"""Collection quality of a compare side: is a missing row a business loss or a data problem?
|
||||
|
||||
Compare results are only as good as the two batches behind them. This module
|
||||
reads *receipts* (``BizStateBatchCommand``: counts and status, never ``raw_text``)
|
||||
to tell apart:
|
||||
|
||||
- ``not_collected`` – a side never ran a command (or the slice) for the sheet
|
||||
- ``suspect_parse`` – the device printed lines but the parser produced no rows
|
||||
- ``no_data`` – both sides ran fine and are really empty
|
||||
- ``ok`` – ordinary comparison
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import BizStateBatchCommand
|
||||
from .compare_rules import scalar_text
|
||||
|
||||
# More than this many CLI lines with zero parsed rows is not "an empty table".
|
||||
SUSPECT_RAW_LINES = 5
|
||||
DATA_STATES = ("ok", "no_data", "suspect_parse", "not_collected")
|
||||
UNJUDGED_STATES = ("no_data", "not_collected")
|
||||
|
||||
|
||||
def _has_receipts(db: Session, batch_id: str) -> bool:
|
||||
return bool(batch_id) and db.query(BizStateBatchCommand.id).filter(
|
||||
BizStateBatchCommand.batch_id == batch_id).limit(1).first() is not None
|
||||
|
||||
|
||||
def side_receipts(db: Session, batch_id: str, metric_id: str) -> list[dict[str, Any]]:
|
||||
if not batch_id or not metric_id:
|
||||
return []
|
||||
rows = (
|
||||
db.query(
|
||||
BizStateBatchCommand.profile_id, BizStateBatchCommand.parse_status,
|
||||
BizStateBatchCommand.row_count, BizStateBatchCommand.raw_line_count,
|
||||
BizStateBatchCommand.declared_total,
|
||||
)
|
||||
.filter(BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id == metric_id)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"profile_id": str(r[0] or ""), "parse_status": str(r[1] or ""),
|
||||
"row_count": int(r[2] or 0), "raw_line_count": int(r[3] or 0),
|
||||
"declared_total": int(r[4] or 0),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def collect_quality(receipts: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
statuses: dict[str, int] = {}
|
||||
for r in receipts:
|
||||
key = str(r.get("parse_status") or "unknown")
|
||||
statuses[key] = statuses.get(key, 0) + 1
|
||||
return {
|
||||
"commands": len(receipts),
|
||||
"parse_status": statuses,
|
||||
"row_count": sum(int(r.get("row_count") or 0) for r in receipts),
|
||||
"raw_line_count": sum(int(r.get("raw_line_count") or 0) for r in receipts),
|
||||
"declared_total": sum(int(r.get("declared_total") or 0) for r in receipts),
|
||||
"profile_ids": sorted({str(r.get("profile_id") or "") for r in receipts if r.get("profile_id")}),
|
||||
}
|
||||
|
||||
|
||||
def sheet_slice(sheet: Mapping[str, Any]) -> dict[str, str]:
|
||||
"""``{field: value}`` the sheet pins with top-level ``eq`` row filters."""
|
||||
out: dict[str, str] = {}
|
||||
for f in sheet.get("row_filters") or []:
|
||||
if not isinstance(f, Mapping) or "any" in f or "all" in f:
|
||||
continue
|
||||
if str(f.get("op") or "eq").strip().lower() not in ("eq", "==", "ci_eq"):
|
||||
continue
|
||||
field = str(f.get("field") or "").strip()
|
||||
value = scalar_text(f.get("value")).strip().lower()
|
||||
if field and value:
|
||||
out[field] = value
|
||||
return out
|
||||
|
||||
|
||||
def covers_slice(
|
||||
receipts: Sequence[Mapping[str, Any]], wanted: Mapping[str, str], *, profile_slice: Any = None,
|
||||
) -> bool:
|
||||
"""True when at least one successful command could have produced ``wanted``.
|
||||
|
||||
A command without a declared slice (or unknown profile) covers everything.
|
||||
"""
|
||||
if profile_slice is None:
|
||||
from .profiles import get_profile
|
||||
|
||||
def profile_slice(pid: str) -> dict[str, str]: # noqa: F811
|
||||
prof = get_profile(pid) if pid else None
|
||||
return dict(prof.slice) if prof is not None and prof.slice else {}
|
||||
|
||||
for r in receipts:
|
||||
if str(r.get("parse_status") or "") in ("failed", "skipped_custom"):
|
||||
continue
|
||||
declared = profile_slice(str(r.get("profile_id") or ""))
|
||||
if all(
|
||||
f not in declared or str(declared[f]).strip().lower() == v
|
||||
for f, v in wanted.items()
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def classify_sheet(
|
||||
*,
|
||||
before: Sequence[Mapping[str, Any]] | None,
|
||||
after: Sequence[Mapping[str, Any]] | None,
|
||||
before_rows: int,
|
||||
after_rows: int,
|
||||
wanted_slice: Mapping[str, str] | None = None,
|
||||
) -> tuple[str, list[str]]:
|
||||
"""``(data_state, reasons)`` for one sheet; reasons name the offending side.
|
||||
|
||||
``None`` receipts mean the batch carries no receipts at all (imported or
|
||||
legacy data): that side cannot be judged and is trusted as-is.
|
||||
"""
|
||||
wanted = dict(wanted_slice or {})
|
||||
reasons: list[str] = []
|
||||
for side, receipts in (("before", before), ("after", after)):
|
||||
if receipts is None:
|
||||
continue
|
||||
if not receipts:
|
||||
reasons.append(f"{side}:no_command")
|
||||
elif wanted and not covers_slice(receipts, wanted):
|
||||
reasons.append(f"{side}:slice_not_collected")
|
||||
if reasons:
|
||||
return "not_collected", reasons
|
||||
for side, receipts in (("before", before), ("after", after)):
|
||||
if receipts is None:
|
||||
continue
|
||||
q = collect_quality(receipts)
|
||||
if q["row_count"] == 0 and q["raw_line_count"] > SUSPECT_RAW_LINES:
|
||||
reasons.append(f"{side}:parsed_0_of_{q['raw_line_count']}_lines")
|
||||
elif q["commands"] and q["parse_status"].get("failed", 0) == q["commands"]:
|
||||
reasons.append(f"{side}:all_commands_failed")
|
||||
if reasons:
|
||||
return "suspect_parse", reasons
|
||||
if before_rows == 0 and after_rows == 0:
|
||||
return "no_data", ["both_sides_empty"]
|
||||
return "ok", []
|
||||
|
||||
|
||||
def assess_sheet(
|
||||
db: Session, sheet: Mapping[str, Any], *, before_batch_id: str, after_batch_id: str,
|
||||
before_rows: int, after_rows: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Quality block stored on a sheet summary."""
|
||||
metric_id = str(sheet.get("metric_id") or "")
|
||||
before = side_receipts(db, before_batch_id, metric_id) if _has_receipts(db, before_batch_id) else None
|
||||
after = side_receipts(db, after_batch_id, metric_id) if _has_receipts(db, after_batch_id) else None
|
||||
state, reasons = classify_sheet(
|
||||
before=before, after=after, before_rows=before_rows, after_rows=after_rows,
|
||||
wanted_slice=sheet_slice(sheet),
|
||||
)
|
||||
return {
|
||||
"data_state": state,
|
||||
"data_state_reasons": reasons,
|
||||
"collect_quality": {
|
||||
"before": collect_quality(before or []), "after": collect_quality(after or []),
|
||||
"unknown_sides": [n for n, r in (("before", before), ("after", after)) if r is None],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def command_coverage(
|
||||
db: Session, *, before_batch_id: str, after_batch_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Collected "metric{slice}" labels per side and their difference.
|
||||
|
||||
Profile ids differ per vendor (``zte.x`` vs ``huawei.x``), so the comparison
|
||||
is on what a command *produces*: its metric and slice.
|
||||
"""
|
||||
from .profiles import get_profile
|
||||
|
||||
def labels(batch_id: str) -> set[str]:
|
||||
if not batch_id:
|
||||
return set()
|
||||
rows = db.query(BizStateBatchCommand.metric_id, BizStateBatchCommand.profile_id).filter(
|
||||
BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id != "",
|
||||
).distinct().all()
|
||||
out: set[str] = set()
|
||||
for metric_id, profile_id in rows:
|
||||
prof = get_profile(str(profile_id or "")) if profile_id else None
|
||||
sl = dict(prof.slice) if prof is not None and prof.slice else {}
|
||||
suffix = ",".join(f"{k}={v}" for k, v in sorted(sl.items()))
|
||||
out.add(f"{metric_id}{{{suffix}}}" if suffix else str(metric_id))
|
||||
return out
|
||||
|
||||
b, a = labels(before_batch_id), labels(after_batch_id)
|
||||
return {"before": sorted(b), "after": sorted(a),
|
||||
"only_before": sorted(b - a), "only_after": sorted(a - b)}
|
||||
|
|
@ -137,6 +137,8 @@ def field_rule_map(rules: Sequence[Mapping[str, Any]] | None) -> dict[str, dict[
|
|||
name = str(raw.get("field") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
if str(raw.get("side") or "").strip():
|
||||
continue # side-scoped rules feed the identity pipeline, not the compare
|
||||
out[name] = dict(raw)
|
||||
return out
|
||||
|
||||
|
|
@ -210,6 +212,50 @@ def _parse_float(text: str) -> float | None:
|
|||
return None
|
||||
|
||||
|
||||
DIRECTIONAL_MODES = ("not_worse", "numeric_min", "numeric_max")
|
||||
|
||||
|
||||
def _rank_index(value: str, rank: Sequence[Any]) -> int | None:
|
||||
wanted = value.strip().lower()
|
||||
for i, item in enumerate(rank or []):
|
||||
if scalar_text(item).strip().lower() == wanted:
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
def _tolerance(rule: Mapping[str, Any]) -> float:
|
||||
try:
|
||||
return float(rule.get("tolerance") or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _directional_verdict(bv: str, av: str, rule: Mapping[str, Any]) -> tuple[bool, str]:
|
||||
"""``(equal_or_better, reason)`` for not_worse / numeric_min / numeric_max."""
|
||||
mode = str(rule.get("compare") or "").strip().lower()
|
||||
if mode == "not_worse":
|
||||
rank = rule.get("rank") or []
|
||||
bi, ai = _rank_index(bv, rank), _rank_index(av, rank)
|
||||
if bi is None or ai is None:
|
||||
return bv == av, "neq"
|
||||
return ai <= bi, (f"worse: {bv}→{av}" if ai > bi else "")
|
||||
bn, an = _parse_float(bv), _parse_float(av)
|
||||
if bn is None or an is None:
|
||||
return bv == av, "neq"
|
||||
tol = _tolerance(rule)
|
||||
if mode == "numeric_min":
|
||||
floor = bn * (1 - tol / 100.0)
|
||||
if an >= floor:
|
||||
return True, ""
|
||||
drop = (bn - an) / abs(bn) * 100.0 if bn else 0.0
|
||||
return False, f"drop {drop:.0f}% > {tol:g}%"
|
||||
ceil = bn * (1 + tol / 100.0)
|
||||
if an <= ceil:
|
||||
return True, ""
|
||||
rise = (an - bn) / abs(bn) * 100.0 if bn else 0.0
|
||||
return False, f"rise {rise:.0f}% > {tol:g}%"
|
||||
|
||||
|
||||
def explain_diff(
|
||||
before: Any,
|
||||
after: Any,
|
||||
|
|
@ -222,6 +268,8 @@ def explain_diff(
|
|||
bv = normalize_value(before, norm)
|
||||
av = normalize_value(after, norm)
|
||||
mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
|
||||
if mode in DIRECTIONAL_MODES:
|
||||
return _directional_verdict(bv, av, rule)[1]
|
||||
if mode in ("percent", "pct", "rel"):
|
||||
bn = _parse_float(bv)
|
||||
an = _parse_float(av)
|
||||
|
|
@ -262,6 +310,8 @@ def values_equal(
|
|||
mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
|
||||
if mode in ("ignore", "skip", "off"):
|
||||
return True
|
||||
if mode in DIRECTIONAL_MODES:
|
||||
return _directional_verdict(bv, av, rule)[0]
|
||||
if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"):
|
||||
bn = _parse_float(bv)
|
||||
an = _parse_float(av)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ from ..models import (
|
|||
BizCompareJob,
|
||||
BizCompareRun,
|
||||
BizCompareTemplate,
|
||||
BizMigrationProject,
|
||||
BizMonitorTemplate,
|
||||
BizPortMapping,
|
||||
BizPortMappingRow,
|
||||
BizStateBatch,
|
||||
|
|
@ -34,6 +36,7 @@ from .compare_engine import apply_port_map, compare_rows, mapping_stats, row_key
|
|||
from .compare_rules import (
|
||||
ROW_FILTER_PRESETS,
|
||||
apply_row_filters,
|
||||
scalar_text,
|
||||
arp_dynamic_row_filters,
|
||||
effective_compare_fields,
|
||||
effective_display_fields,
|
||||
|
|
@ -43,12 +46,19 @@ from .iface_normalize import (
|
|||
apply_iface_normalize_rows,
|
||||
default_zte_iface_normalize_rules,
|
||||
normalize_iface_rules,
|
||||
normalize_port_map,
|
||||
)
|
||||
from .compare_quality import UNJUDGED_STATES, assess_sheet
|
||||
from .identity_pipeline import Pipeline, build_pipeline, legacy_pipeline
|
||||
from .profiles import metric_field_map
|
||||
from .compare_validation import validate_template_body
|
||||
|
||||
_log = logging.getLogger("netx.biz_state.compare")
|
||||
|
||||
# v1: port-map keys/values must be written post-normalization.
|
||||
# v2: port-map keys/values are normalized with the template iface rules (A1).
|
||||
CONFIG_SNAPSHOT_VERSION = 2
|
||||
|
||||
# Per-job mutex so auto + manual run_compare do not dual-write the same job.
|
||||
_auto_compare_locks: dict[str, threading.Lock] = {}
|
||||
_auto_compare_locks_guard = threading.Lock()
|
||||
|
|
@ -426,11 +436,25 @@ def _metric_rows_by_ids(db: Session, ids: list[str]) -> dict[str, dict[str, Any]
|
|||
return out
|
||||
|
||||
|
||||
def _attach_raw_key_before(items: list[dict[str, Any]], key_fields: list[str]) -> list[dict[str, Any]]:
|
||||
"""``raw_key_before``: the before-side key as printed, only where it differs from the match key."""
|
||||
for it in items:
|
||||
before = it.get("before")
|
||||
if not before or it.get("raw_key_before"):
|
||||
continue
|
||||
shown = it.get("key") or {}
|
||||
raw = {f: scalar_text(before.get(f)).strip() for f in key_fields}
|
||||
if any(raw[f] != scalar_text(shown.get(f)).strip() for f in key_fields if f in shown):
|
||||
it["raw_key_before"] = raw
|
||||
return items
|
||||
|
||||
|
||||
def _hydrate_diff_rows(
|
||||
db: Session, items: list[dict[str, Any]], *,
|
||||
run: BizCompareRun | None = None, sheet: dict[str, Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Fill empty before/after from metric tables when row_ids are present."""
|
||||
key_fields = list((sheet or {}).get("key_fields") or [])
|
||||
need: list[str] = []
|
||||
for it in items:
|
||||
if not it.get("before") and it.get("before_row_id"):
|
||||
|
|
@ -438,27 +462,19 @@ def _hydrate_diff_rows(
|
|||
if not it.get("after") and it.get("after_row_id"):
|
||||
need.append(str(it["after_row_id"]))
|
||||
if not need:
|
||||
return items
|
||||
return _attach_raw_key_before(items, key_fields)
|
||||
by_id = _metric_rows_by_ids(db, need)
|
||||
pmap, norm_rules = _run_transform_config(db, run) if run else ({}, [])
|
||||
iface_fields = list((sheet or {}).get("iface_fields") or [])
|
||||
pipeline = _run_pipeline(db, run, sheet or {})
|
||||
for it in items:
|
||||
brid = str(it.get("before_row_id") or "")
|
||||
arid = str(it.get("after_row_id") or "")
|
||||
if not it.get("before") and brid and brid in by_id:
|
||||
it["before"] = by_id[brid]
|
||||
if not it.get("after") and arid and arid in by_id:
|
||||
it["after"] = apply_iface_normalize_rows(
|
||||
[by_id[arid]], iface_fields=iface_fields, rules=norm_rules,
|
||||
)[0]
|
||||
it["after"] = pipeline.normalize(by_id[arid], "after")
|
||||
if not it.get("mapped_before") and it.get("before"):
|
||||
normalized = apply_iface_normalize_rows(
|
||||
[it["before"]], iface_fields=iface_fields, rules=norm_rules,
|
||||
)[0]
|
||||
it["mapped_before"] = apply_port_map(
|
||||
normalized, iface_fields=iface_fields, port_map=pmap,
|
||||
)
|
||||
return items
|
||||
it["mapped_before"] = pipeline.transform(it["before"], "before")
|
||||
return _attach_raw_key_before(items, key_fields)
|
||||
|
||||
|
||||
def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]:
|
||||
|
|
@ -653,18 +669,41 @@ def _normalize_row_filters(raw: Any) -> list[dict[str, Any]]:
|
|||
return out
|
||||
|
||||
|
||||
# Optional sheet-level switches that must survive normalization.
|
||||
_SHEET_OPTION_VALUES: dict[str, tuple[str, ...]] = {
|
||||
"unmapped_port_policy": ("by_name", "ignore_port"),
|
||||
"duplicate_pairing": ("sorted", "ordered"),
|
||||
}
|
||||
|
||||
|
||||
def _sheet_options(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {}
|
||||
for key, allowed in _SHEET_OPTION_VALUES.items():
|
||||
val = str(raw.get(key) or "").strip().lower()
|
||||
if val in allowed:
|
||||
out[key] = val
|
||||
if raw.get("builtin_vendor_canonical") is False:
|
||||
out["builtin_vendor_canonical"] = False
|
||||
if raw.get("scope_by_port_map") is not None:
|
||||
out["scope_by_port_map"] = bool(raw.get("scope_by_port_map"))
|
||||
return out
|
||||
|
||||
|
||||
def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = str(item.get("field") or "").strip()
|
||||
if not name or name in seen:
|
||||
side = str(item.get("side") or "").strip().lower()
|
||||
side = side if side in ("before", "after") else ""
|
||||
# A field may carry one rule per side (e.g. a different value_map each).
|
||||
if not name or (name, side) in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
seen.add((name, side))
|
||||
rule: dict[str, Any] = {"field": name}
|
||||
compare = str(item.get("compare") or "").strip().lower()
|
||||
if compare:
|
||||
|
|
@ -680,6 +719,16 @@ def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]:
|
|||
rule["tolerance"] = float(item.get("tolerance"))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
vmap = item.get("value_map")
|
||||
if isinstance(vmap, dict) and vmap:
|
||||
rule["value_map"] = {str(k): str(v) for k, v in vmap.items()}
|
||||
if side:
|
||||
rule["side"] = side
|
||||
if isinstance(item.get("apply_to_key"), bool):
|
||||
rule["apply_to_key"] = item["apply_to_key"]
|
||||
rank = item.get("rank")
|
||||
if isinstance(rank, list) and rank:
|
||||
rule["rank"] = [str(x) for x in rank]
|
||||
# Drop empty rules (only field name)
|
||||
if len(rule) > 1:
|
||||
out.append(rule)
|
||||
|
|
@ -709,6 +758,7 @@ def _sheet_def(
|
|||
row_filters: list[dict[str, Any]] | None = None,
|
||||
field_rules: list[dict[str, Any]] | None = None,
|
||||
ignore_port_changes: bool | None = None,
|
||||
options: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
mid = str(metric_id or "").strip()
|
||||
sid = str(sheet_id or "").strip() or mid
|
||||
|
|
@ -749,6 +799,7 @@ def _sheet_def(
|
|||
}
|
||||
if ignore_port_changes is not None:
|
||||
sheet["ignore_port_changes"] = bool(ignore_port_changes)
|
||||
sheet.update(options or {})
|
||||
return sheet
|
||||
|
||||
|
||||
|
|
@ -946,6 +997,49 @@ def _builtin_status_needs_packaged_upgrade(existing: list[dict[str, Any]]) -> bo
|
|||
return len(packaged_keys - have_keys) >= 3
|
||||
|
||||
|
||||
# A7: (metric_id, kind, field, old packaged value, new value). A stored sheet is
|
||||
# only touched when its value still equals the *old* packaged one, so anything
|
||||
# an operator edited survives the upgrade.
|
||||
_INTERFACE_DETAIL_OLD_COMPARE = [
|
||||
"admin", "input_bps", "output_bps", "port_status", "in_util", "out_util",
|
||||
"ip_mtu", "mtu", "mpls_mtu", "ipv6_mtu", "port_media", "negotiation",
|
||||
]
|
||||
_BUILTIN_STATUS_VALUE_UPGRADES: tuple[tuple[str, str, str, Any, Any], ...] = (
|
||||
("interface_detail", "rule", "output_bps",
|
||||
{"compare": "numeric", "tolerance": 5}, {"compare": "percent", "tolerance": 5}),
|
||||
("interface_detail", "list", "compare_fields", _INTERFACE_DETAIL_OLD_COMPARE,
|
||||
[f for f in _INTERFACE_DETAIL_OLD_COMPARE if f not in ("input_bps", "output_bps", "in_util", "out_util")]),
|
||||
("optical_brief", "rule", "rx_power",
|
||||
{"compare": "percent", "tolerance": 5}, {"compare": "numeric", "tolerance": 2}),
|
||||
("l2vpn_mac", "list", "iface_fields", [], ["ac_port"]),
|
||||
# ARP: filter on the canonical ``entry_type`` (every vendor parser emits it); the
|
||||
# age-timer clause stays only as a fallback for rows parsed before entry_type existed.
|
||||
("arp", "list", "row_filters", [{"op": "age_timer", "field": "age", "value": ""}], arp_dynamic_row_filters()),
|
||||
)
|
||||
|
||||
|
||||
def _upgrade_builtin_status_values(sheet: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
"""Apply ``_BUILTIN_STATUS_VALUE_UPGRADES`` to one stored sheet."""
|
||||
mid = str(sheet.get("metric_id") or "")
|
||||
out = deepcopy(sheet)
|
||||
changed = False
|
||||
for metric_id, kind, field, old, new in _BUILTIN_STATUS_VALUE_UPGRADES:
|
||||
if mid != metric_id:
|
||||
continue
|
||||
if kind == "list":
|
||||
if list(out.get(field) or []) == old:
|
||||
out[field] = list(new)
|
||||
changed = True
|
||||
continue
|
||||
for rule in out.get("field_rules") or []:
|
||||
if isinstance(rule, dict) and rule.get("field") == field and all(
|
||||
rule.get(k) == v for k, v in old.items()
|
||||
):
|
||||
rule.update(new)
|
||||
changed = True
|
||||
return out, changed
|
||||
|
||||
|
||||
def _default_zte_config_sheets() -> list[dict[str, Any]]:
|
||||
"""Config-intent metrics for cutover / intent-vs-intent compare."""
|
||||
return [
|
||||
|
|
@ -1009,6 +1103,7 @@ def _normalize_sheet(raw: Any) -> dict[str, Any] | None:
|
|||
row_filters=_normalize_row_filters(raw.get("row_filters")),
|
||||
field_rules=rules,
|
||||
ignore_port_changes=ignore_ports,
|
||||
options=_sheet_options(raw),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1252,6 +1347,8 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate:
|
|||
if not cur.get("field_rules") and src and src.get("field_rules"):
|
||||
cur["field_rules"] = list(src["field_rules"])
|
||||
changed = True
|
||||
cur, value_changed = _upgrade_builtin_status_values(cur)
|
||||
changed = changed or value_changed
|
||||
upgraded.append(_normalize_sheet(cur) or cur)
|
||||
have_mids = {str(s.get("metric_id") or "") for s in upgraded}
|
||||
for s in sheets:
|
||||
|
|
@ -1389,7 +1486,16 @@ def list_templates(db: Session) -> list[dict[str, Any]]:
|
|||
upgrade_builtin_split_sheets(db)
|
||||
rows = db.query(BizCompareTemplate).order_by(BizCompareTemplate.name.asc()).all()
|
||||
usage = dict(db.query(BizCompareJob.template_id, func.count(BizCompareJob.id)).group_by(BizCompareJob.template_id).all())
|
||||
return [{**_template_out(t), "job_count": usage.get(t.id, 0)} for t in rows]
|
||||
monitor_usage = dict(
|
||||
db.query(BizMonitorTemplate.compare_template_id, func.count(BizMonitorTemplate.id))
|
||||
.filter(BizMonitorTemplate.compare_template_id != "")
|
||||
.group_by(BizMonitorTemplate.compare_template_id).all()
|
||||
)
|
||||
return [
|
||||
{**_template_out(t), "job_count": usage.get(t.id, 0),
|
||||
"monitor_template_count": monitor_usage.get(t.id, 0)}
|
||||
for t in rows
|
||||
]
|
||||
|
||||
|
||||
def list_metric_schemas() -> list[dict[str, Any]]:
|
||||
|
|
@ -1522,8 +1628,19 @@ def delete_template(db: Session, template_id: str) -> None:
|
|||
t = db.get(BizCompareTemplate, template_id)
|
||||
if not t:
|
||||
raise HTTPException(status_code=404, detail="template_not_found")
|
||||
if db.query(BizCompareJob.id).filter(BizCompareJob.template_id == template_id).first():
|
||||
raise HTTPException(status_code=409, detail="template_in_use")
|
||||
jobs = [n or i for i, n in db.query(BizCompareJob.id, BizCompareJob.name)
|
||||
.filter(BizCompareJob.template_id == template_id).all()]
|
||||
monitors = [n or i for i, n in db.query(BizMonitorTemplate.id, BizMonitorTemplate.name)
|
||||
.filter(BizMonitorTemplate.compare_template_id == template_id).all()]
|
||||
if jobs or monitors:
|
||||
raise HTTPException(status_code=409, detail={
|
||||
"code": "template_in_use",
|
||||
"message": "template_in_use: " + "; ".join(filter(None, [
|
||||
f"jobs={', '.join(jobs)}" if jobs else "",
|
||||
f"monitor_templates={', '.join(monitors)}" if monitors else "",
|
||||
])),
|
||||
"jobs": jobs, "monitor_templates": monitors,
|
||||
})
|
||||
db.delete(t)
|
||||
db.commit()
|
||||
|
||||
|
|
@ -1544,9 +1661,30 @@ def _mapping_out(db: Session, m: BizPortMapping) -> dict[str, Any]:
|
|||
}
|
||||
|
||||
|
||||
def _mapping_references(db: Session, mapping_id: str) -> tuple[list[str], list[str]]:
|
||||
"""(compare job names, cutover project names) that reference this mapping."""
|
||||
jobs = [n or i for i, n in db.query(BizCompareJob.id, BizCompareJob.name)
|
||||
.filter(BizCompareJob.mapping_id == mapping_id).all()]
|
||||
projects = [n or i for i, n in db.query(BizMigrationProject.id, BizMigrationProject.name)
|
||||
.filter(BizMigrationProject.mapping_id == mapping_id).all()]
|
||||
return jobs, projects
|
||||
|
||||
|
||||
def list_mappings(db: Session) -> list[dict[str, Any]]:
|
||||
rows = db.query(BizPortMapping).order_by(BizPortMapping.name.asc()).all()
|
||||
return [_mapping_out(db, m) for m in rows]
|
||||
job_counts = dict(
|
||||
db.query(BizCompareJob.mapping_id, func.count(BizCompareJob.id))
|
||||
.filter(BizCompareJob.mapping_id != "").group_by(BizCompareJob.mapping_id).all()
|
||||
)
|
||||
project_counts = dict(
|
||||
db.query(BizMigrationProject.mapping_id, func.count(BizMigrationProject.id))
|
||||
.filter(BizMigrationProject.mapping_id != "").group_by(BizMigrationProject.mapping_id).all()
|
||||
)
|
||||
return [
|
||||
{**_mapping_out(db, m), "job_count": job_counts.get(m.id, 0),
|
||||
"project_count": project_counts.get(m.id, 0)}
|
||||
for m in rows
|
||||
]
|
||||
|
||||
|
||||
def create_mapping(db: Session, body: dict[str, Any]) -> dict[str, Any]:
|
||||
|
|
@ -1604,6 +1742,16 @@ def delete_mapping(db: Session, mapping_id: str) -> None:
|
|||
m = db.get(BizPortMapping, mapping_id)
|
||||
if not m:
|
||||
raise HTTPException(status_code=404, detail="mapping_not_found")
|
||||
jobs, projects = _mapping_references(db, mapping_id)
|
||||
if jobs or projects:
|
||||
raise HTTPException(status_code=409, detail={
|
||||
"code": "mapping_in_use",
|
||||
"message": "mapping_in_use: " + "; ".join(filter(None, [
|
||||
f"jobs={', '.join(jobs)}" if jobs else "",
|
||||
f"projects={', '.join(projects)}" if projects else "",
|
||||
])),
|
||||
"jobs": jobs, "projects": projects,
|
||||
})
|
||||
db.query(BizPortMappingRow).filter(BizPortMappingRow.mapping_id == mapping_id).delete()
|
||||
db.delete(m)
|
||||
db.commit()
|
||||
|
|
@ -1822,11 +1970,11 @@ def validate_mapping(
|
|||
sheets = template_metrics(tpl)
|
||||
if not sheets:
|
||||
raise HTTPException(status_code=400, detail="template_has_no_metrics")
|
||||
pmap = _port_map_dict(db, mapping_id)
|
||||
# Validate against first sheet that has iface fields (or first sheet)
|
||||
target = next((s for s in sheets if s.get("iface_fields")), sheets[0])
|
||||
iface_fields = list(target.get("iface_fields") or [])
|
||||
norm = template_iface_normalize(tpl)
|
||||
pmap, _ = normalize_port_map(_port_map_dict(db, mapping_id), norm)
|
||||
before = apply_iface_normalize_rows(
|
||||
_load_metric_rows(db, batch_id=before_batch_id, metric_id=target["metric_id"]),
|
||||
iface_fields=iface_fields,
|
||||
|
|
@ -1845,6 +1993,66 @@ def validate_mapping(
|
|||
)
|
||||
|
||||
|
||||
def _mapping_rows_to_dict(rows: list[Any]) -> tuple[dict[str, str], list[dict[str, str]]]:
|
||||
"""Editor rows → raw map; a repeated before port is reported, first one wins."""
|
||||
out: dict[str, str] = {}
|
||||
dups: list[dict[str, str]] = []
|
||||
for r in rows or []:
|
||||
raw = r if isinstance(r, dict) else dict(r)
|
||||
before = str(raw.get("before_if") or "").strip()
|
||||
after = str(raw.get("after_if") or "").strip()
|
||||
if not before or not after:
|
||||
continue
|
||||
if before in out:
|
||||
dups.append({"before_if": before, "kept_to": out[before], "dropped_to": after})
|
||||
continue
|
||||
out[before] = after
|
||||
return out, dups
|
||||
|
||||
|
||||
def preflight_compare(db: Session, body: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Mapping coverage + data-quality report for a template and two batches."""
|
||||
from .compare_preflight import preflight
|
||||
|
||||
ensure_default_templates(db)
|
||||
template_id = str(body.get("template_id") or "")
|
||||
tpl = db.get(BizCompareTemplate, template_id) if template_id else ensure_default_cutover_template(db)
|
||||
if not tpl:
|
||||
raise HTTPException(status_code=404, detail="template_not_found")
|
||||
before_id = str(body.get("before_batch_id") or "")
|
||||
after_id = str(body.get("after_batch_id") or "")
|
||||
if not db.get(BizStateBatch, before_id) or not db.get(BizStateBatch, after_id):
|
||||
raise HTTPException(status_code=404, detail="batch_not_found")
|
||||
sheets = _filter_enabled_sheets(template_metrics(tpl), _str_list(body.get("enabled_sheet_ids")))
|
||||
if not sheets:
|
||||
raise HTTPException(status_code=400, detail="no_enabled_sheets")
|
||||
dups: list[dict[str, str]] = []
|
||||
if body.get("mapping_rows") is not None:
|
||||
raw_map, dups = _mapping_rows_to_dict(list(body.get("mapping_rows") or []))
|
||||
else:
|
||||
raw_map = _port_map_dict(db, str(body.get("mapping_id") or ""))
|
||||
cache: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
||||
|
||||
def load(side: str, sheet: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
batch_id = before_id if side == "before" else after_id
|
||||
key = (batch_id, str(sheet["metric_id"]))
|
||||
if key not in cache:
|
||||
cache[key] = _load_metric_rows(db, batch_id=batch_id, metric_id=key[1])
|
||||
return cache[key]
|
||||
|
||||
report = preflight(
|
||||
db, sheets=sheets, iface_rules=template_iface_normalize(tpl), raw_port_map=raw_map,
|
||||
before_batch_id=before_id, after_batch_id=after_id,
|
||||
before_vendor=_batch_vendor_key(db, before_id), after_vendor=_batch_vendor_key(db, after_id),
|
||||
load_rows=load,
|
||||
)
|
||||
if dups:
|
||||
report["mapping"]["duplicate_before"] = dups
|
||||
report["risk_reasons"] = sorted({*report["risk_reasons"], "duplicate_before"})
|
||||
report["verdict"] = "risk"
|
||||
return report
|
||||
|
||||
|
||||
def _job_out(j: BizCompareJob) -> dict[str, Any]:
|
||||
return {
|
||||
"id": j.id,
|
||||
|
|
@ -2040,14 +2248,23 @@ def _sheet_result_envelope(
|
|||
}
|
||||
|
||||
|
||||
def _sheet_quality(
|
||||
db: Session, sheet: dict[str, Any], before_batch_id: str, after_batch_id: str,
|
||||
summary: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return assess_sheet(
|
||||
db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id,
|
||||
before_rows=int(summary.get("before_count") or 0), after_rows=int(summary.get("after_count") or 0),
|
||||
)
|
||||
|
||||
|
||||
def _run_sheet(
|
||||
db: Session,
|
||||
*,
|
||||
sheet: dict[str, Any],
|
||||
before_batch_id: str,
|
||||
after_batch_id: str,
|
||||
port_map: dict[str, str],
|
||||
iface_normalize_rules: list[dict[str, str]] | None = None,
|
||||
pipeline: Pipeline,
|
||||
store_unchanged: str = "auto",
|
||||
on_load_progress: Callable[..., None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
|
|
@ -2083,12 +2300,7 @@ def _run_sheet(
|
|||
# PostgreSQL path: pushdown-safe sheets join in-DB (BGP-scale).
|
||||
from .compare_sql import SqlCompareSkip, run_sql_sheet_compare, sql_compare_skip_reason
|
||||
|
||||
skip_reason = sql_compare_skip_reason(
|
||||
db,
|
||||
sheet,
|
||||
port_map=port_map,
|
||||
iface_normalize_rules=iface_normalize_rules,
|
||||
)
|
||||
skip_reason = sql_compare_skip_reason(db, sheet, pipeline=pipeline)
|
||||
if not skip_reason:
|
||||
try:
|
||||
result = run_sql_sheet_compare(
|
||||
|
|
@ -2098,6 +2310,7 @@ def _run_sheet(
|
|||
after_batch_id=after_batch_id,
|
||||
store_unchanged=store_unchanged,
|
||||
on_progress=_emit_load,
|
||||
duplicate_pairing=pipeline.duplicate_pairing,
|
||||
)
|
||||
summary = dict(result["summary"])
|
||||
# Engine already sets raw counts / policy; keep keys stable
|
||||
|
|
@ -2107,6 +2320,7 @@ def _run_sheet(
|
|||
before_n=int(summary.get("before_count") or 0),
|
||||
after_n=int(summary.get("after_count") or 0),
|
||||
)
|
||||
summary.update(_sheet_quality(db, sheet, before_batch_id, after_batch_id, summary))
|
||||
return _sheet_result_envelope(
|
||||
sheet,
|
||||
key_fields=key_fields,
|
||||
|
|
@ -2161,39 +2375,38 @@ def _run_sheet(
|
|||
def _after_chunk(n: int) -> None:
|
||||
_emit_load("after", n, engine="python", note=skip_reason or "python", phase="loading")
|
||||
|
||||
# Only filters that read untransformed fields may run in SQL on the raw rows;
|
||||
# the pipeline applies every filter (post-normalize for v2) in ``prepare``.
|
||||
before_raw = _load_metric_rows(
|
||||
db,
|
||||
batch_id=before_batch_id,
|
||||
metric_id=mid,
|
||||
on_chunk=_before_chunk,
|
||||
row_filters=row_filters,
|
||||
row_filters=pipeline.pushdown_filters("before"),
|
||||
)
|
||||
after_raw = _load_metric_rows(
|
||||
db,
|
||||
batch_id=after_batch_id,
|
||||
metric_id=mid,
|
||||
on_chunk=_after_chunk,
|
||||
row_filters=row_filters,
|
||||
row_filters=pipeline.pushdown_filters("after"),
|
||||
)
|
||||
# Filters already applied in load when pushdown-safe; keep apply for safety
|
||||
before_rows = apply_row_filters(before_raw, row_filters) if row_filters else before_raw
|
||||
after_rows = apply_row_filters(after_raw, row_filters) if row_filters else after_raw
|
||||
before_pairs = pipeline.prepare(before_raw, "before")
|
||||
after_pairs = pipeline.prepare(after_raw, "after")
|
||||
policy = resolve_unchanged_policy(
|
||||
store_unchanged, before_n=len(before_rows), after_n=len(after_rows)
|
||||
store_unchanged, before_n=len(before_pairs), after_n=len(after_pairs)
|
||||
)
|
||||
result = compare_rows(
|
||||
before_rows=before_rows,
|
||||
after_rows=after_rows,
|
||||
before_rows=[], after_rows=[],
|
||||
key_fields=key_fields,
|
||||
iface_fields=iface_fields,
|
||||
compare_fields=compare_fields,
|
||||
port_map=port_map,
|
||||
field_rules=field_rules,
|
||||
iface_normalize_rules=iface_normalize_rules,
|
||||
ignore_port_changes=ignore_ports,
|
||||
include_unchanged=bool(policy["include"]),
|
||||
unchanged_limit=policy.get("limit"),
|
||||
compact_unchanged=bool(policy.get("compact")),
|
||||
pipeline=pipeline,
|
||||
prepared=(before_pairs, after_pairs),
|
||||
)
|
||||
summary = dict(result["summary"])
|
||||
summary["before_raw_count"] = len(before_raw)
|
||||
|
|
@ -2201,6 +2414,14 @@ def _run_sheet(
|
|||
summary["row_filters"] = len(row_filters)
|
||||
summary["unchanged_policy"] = policy
|
||||
summary.setdefault("engine", "python")
|
||||
if pipeline.port_map and pipeline.iface_fields:
|
||||
from .compare_preflight import sheet_mapping_coverage
|
||||
|
||||
coverage = sheet_mapping_coverage(pipeline, before_pairs, after_pairs)
|
||||
coverage.pop("_after_ifaces", None)
|
||||
coverage.pop("_used_keys", None)
|
||||
summary["mapping_coverage"] = coverage
|
||||
summary.update(_sheet_quality(db, sheet, before_batch_id, after_batch_id, summary))
|
||||
return _sheet_result_envelope(
|
||||
sheet,
|
||||
key_fields=key_fields,
|
||||
|
|
@ -2227,6 +2448,8 @@ def _validate_compare_job(
|
|||
tpl = db.get(BizCompareTemplate, j.template_id)
|
||||
if not tpl:
|
||||
raise HTTPException(status_code=404, detail="template_not_found")
|
||||
if j.mapping_id and not db.get(BizPortMapping, j.mapping_id):
|
||||
raise HTTPException(status_code=400, detail="mapping_missing")
|
||||
before_batch_id = str(j.before_batch_id or "")
|
||||
after_batch_id = str(force_after_batch_id or "").strip() or _resolve_after_batch(db, j)
|
||||
if not before_batch_id or not after_batch_id:
|
||||
|
|
@ -2272,15 +2495,31 @@ def _create_running_run(
|
|||
before_batch_id: str,
|
||||
after_batch_id: str,
|
||||
sheets_cfg: list[dict[str, Any]],
|
||||
purpose: str = "manual",
|
||||
) -> BizCompareRun:
|
||||
first_metric = str(sheets_cfg[0].get("metric_id") or "")
|
||||
pending_sheets = [_pending_sheet_meta(s) for s in sheets_cfg]
|
||||
mapping = db.get(BizPortMapping, job.mapping_id) if job.mapping_id else None
|
||||
vendors = {
|
||||
"before": _batch_vendor_key(db, before_batch_id),
|
||||
"after": _batch_vendor_key(db, after_batch_id),
|
||||
}
|
||||
raw_port_map = _port_map_dict(db, str(job.mapping_id or ""))
|
||||
iface_rules = template_iface_normalize(tpl)
|
||||
identity = {
|
||||
sheet_key(sh): build_pipeline(
|
||||
sh, iface_rules=iface_rules, port_map=raw_port_map,
|
||||
before_vendor=vendors["before"], after_vendor=vendors["after"],
|
||||
).describe()
|
||||
for sh in sheets_cfg
|
||||
}
|
||||
config_snapshot = deepcopy({
|
||||
"version": 1,
|
||||
"version": CONFIG_SNAPSHOT_VERSION,
|
||||
"template": _template_out(tpl),
|
||||
"sheets": sheets_cfg,
|
||||
"port_map": _port_map_dict(db, str(job.mapping_id or "")),
|
||||
"port_map": raw_port_map,
|
||||
"vendors": vendors,
|
||||
"identity": identity,
|
||||
"store_unchanged": normalize_store_unchanged(getattr(job, "store_unchanged", None)),
|
||||
"job_name": job.name or "",
|
||||
"before_task_id": job.before_task_id or "",
|
||||
|
|
@ -2297,6 +2536,7 @@ def _create_running_run(
|
|||
metric_id=first_metric,
|
||||
status="running",
|
||||
summary_json={
|
||||
"purpose": purpose,
|
||||
"config_snapshot": config_snapshot,
|
||||
"progress": {
|
||||
"phase": "queued",
|
||||
|
|
@ -2329,7 +2569,7 @@ def _create_running_run(
|
|||
|
||||
def _run_config_snapshot(run: BizCompareRun) -> dict[str, Any] | None:
|
||||
snapshot = (run.summary_json or {}).get("config_snapshot")
|
||||
if isinstance(snapshot, dict) and snapshot.get("version") == 1:
|
||||
if isinstance(snapshot, dict) and snapshot.get("version") in (1, 2):
|
||||
return snapshot
|
||||
return None
|
||||
|
||||
|
|
@ -2339,14 +2579,66 @@ def _run_transform_config(
|
|||
) -> tuple[dict[str, str], list[dict[str, str]]]:
|
||||
snapshot = _run_config_snapshot(run)
|
||||
if snapshot is not None:
|
||||
return dict(snapshot.get("port_map") or {}), normalize_iface_rules(
|
||||
rules = normalize_iface_rules(
|
||||
(snapshot.get("template") or {}).get("iface_normalize_rules") or [],
|
||||
)
|
||||
pmap = dict(snapshot.get("port_map") or {})
|
||||
if int(snapshot.get("version") or 1) >= 2:
|
||||
# v2 runs: map keys/values are written as the device prints them.
|
||||
pmap, _ = normalize_port_map(pmap, rules)
|
||||
return pmap, rules
|
||||
# Legacy runs did not record these settings; retain their fallback behavior.
|
||||
tpl = db.get(BizCompareTemplate, run.template_id) if run.template_id else None
|
||||
return _port_map_dict(db, str(run.mapping_id or "")), template_iface_normalize(tpl)
|
||||
|
||||
|
||||
def _batch_vendor_key(db: Session, batch_id: str) -> str:
|
||||
"""Vendor key (``huawei``/``zte``/…) for a batch, '' when unknown."""
|
||||
from ..topology_lldp import resolve_vendor_key
|
||||
|
||||
batch = db.get(BizStateBatch, batch_id) if batch_id else None
|
||||
if batch is None:
|
||||
return ""
|
||||
task = db.get(BizStateTask, batch.task_id) if batch.task_id else None
|
||||
key = resolve_vendor_key(
|
||||
batch.vendor or (task.vendor if task else ""), (task.device_type if task else "") or "",
|
||||
)
|
||||
return "" if key == "generic" else key
|
||||
|
||||
|
||||
def _run_pipeline(db: Session, run: BizCompareRun | None, sheet: dict[str, Any]) -> Pipeline:
|
||||
"""The identity pipeline a run used (or would use) for one sheet.
|
||||
|
||||
Snapshot v2 runs get the full pipeline; v1 and snapshot-less runs get the
|
||||
legacy one so stored results replay exactly as they were produced.
|
||||
"""
|
||||
sid = sheet_key(sheet)
|
||||
snapshot = _run_config_snapshot(run) if run is not None else None
|
||||
if snapshot is None:
|
||||
pmap, rules = _run_transform_config(db, run) if run is not None else ({}, [])
|
||||
return legacy_pipeline(
|
||||
key_fields=sheet.get("key_fields") or [], iface_fields=sheet.get("iface_fields") or [],
|
||||
compare_fields=sheet.get("compare_fields") or [], field_rules=sheet.get("field_rules"),
|
||||
port_map=pmap, iface_normalize_rules=rules, row_filters=sheet.get("row_filters"),
|
||||
ignore_port_changes=sheet.get("ignore_port_changes"),
|
||||
)
|
||||
authoritative = next(
|
||||
(sh for sh in snapshot.get("sheets") or [] if isinstance(sh, dict) and sheet_key(sh) == sid), None,
|
||||
)
|
||||
cfg = authoritative or sheet
|
||||
version = int(snapshot.get("version") or 1)
|
||||
vendors = snapshot.get("vendors") or {}
|
||||
return build_pipeline(
|
||||
cfg,
|
||||
iface_rules=(snapshot.get("template") or {}).get("iface_normalize_rules") or [],
|
||||
port_map=dict(snapshot.get("port_map") or {}),
|
||||
before_vendor=str(vendors.get("before") or ""),
|
||||
after_vendor=str(vendors.get("after") or ""),
|
||||
legacy=version < 2,
|
||||
normalize_map=version >= 2,
|
||||
)
|
||||
|
||||
|
||||
def _set_run_progress(
|
||||
db: Session,
|
||||
run: BizCompareRun,
|
||||
|
|
@ -2458,7 +2750,6 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
|||
store_mode = normalize_store_unchanged(
|
||||
snapshot.get("store_unchanged") if snapshot is not None else getattr(j, "store_unchanged", None),
|
||||
)
|
||||
pmap, norm_rules = _run_transform_config(db, run)
|
||||
unchanged_listed_total = 0
|
||||
unchanged_truncated_any = False
|
||||
unchanged_compact_any = False
|
||||
|
|
@ -2563,8 +2854,7 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
|||
sheet=sheet,
|
||||
before_batch_id=before_batch_id,
|
||||
after_batch_id=after_batch_id,
|
||||
port_map=pmap,
|
||||
iface_normalize_rules=norm_rules,
|
||||
pipeline=_run_pipeline(db, run, sheet),
|
||||
store_unchanged=store_mode,
|
||||
on_load_progress=_on_load,
|
||||
)
|
||||
|
|
@ -2755,6 +3045,7 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]:
|
|||
if snapshot is not None:
|
||||
summary_payload["config_snapshot"] = snapshot
|
||||
run = db.get(BizCompareRun, run_id) or run
|
||||
summary_payload["purpose"] = (run.summary_json or {}).get("purpose") or "manual"
|
||||
if str(run.status or "") == "cancelled":
|
||||
return get_run(db, run.id)
|
||||
run.status = "success"
|
||||
|
|
@ -2882,7 +3173,9 @@ def cancel_compare_run(db: Session, run_id: str) -> dict[str, Any]:
|
|||
return get_run(db, run_id)
|
||||
|
||||
|
||||
def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]:
|
||||
def run_compare(
|
||||
db: Session, job_id: str, *, force_after_batch_id: str = "", purpose: str = "manual",
|
||||
) -> dict[str, Any]:
|
||||
"""Synchronous compare (auto-compare / tests). Blocks the caller until done."""
|
||||
if _job_has_active_run(db, job_id):
|
||||
raise HTTPException(status_code=409, detail="compare_already_running")
|
||||
|
|
@ -2898,6 +3191,7 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
|
|||
before_batch_id=ctx["before_batch_id"],
|
||||
after_batch_id=ctx["after_batch_id"],
|
||||
sheets_cfg=ctx["sheets_cfg"],
|
||||
purpose=purpose,
|
||||
)
|
||||
return _execute_compare_into_run(db, run.id)
|
||||
finally:
|
||||
|
|
@ -2972,7 +3266,9 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) ->
|
|||
)
|
||||
# Non-key columns already ordered Key→Compare→Display by effective_display_fields
|
||||
extra = [f for f in display if f not in key_set]
|
||||
headers = ["kind", *keys]
|
||||
# Interface keys also get the name as the device printed it (before side).
|
||||
raw_cols = [k for k in keys if k in set(sheet.get("iface_fields") or [])]
|
||||
headers = ["kind", *keys, *[f"{k}__raw_before" for k in raw_cols]]
|
||||
for f in extra:
|
||||
if f in compare_set:
|
||||
headers.append(f"{f}__pre")
|
||||
|
|
@ -2988,6 +3284,9 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) ->
|
|||
row = [kind]
|
||||
for k in keys:
|
||||
row.append(key.get(k, pre.get(k, post.get(k, ""))))
|
||||
raw_before = dict(d.get("raw_key_before") or {})
|
||||
for k in raw_cols:
|
||||
row.append(raw_before.get(k, ""))
|
||||
for f in extra:
|
||||
if f in compare_set:
|
||||
if kind == "added":
|
||||
|
|
@ -3012,6 +3311,18 @@ def _sheet_csv(sheet: dict[str, Any]) -> str:
|
|||
return "\ufeff" + "\n".join(_sheet_csv_lines(sheet, sheet.get("diffs") or [])) + "\n"
|
||||
|
||||
|
||||
def _task_ok(sheets: list[dict[str, Any]], fail_count: int, any_pending: bool) -> bool | None:
|
||||
"""True/False once judged; None when data quality leaves the verdict open."""
|
||||
if any_pending:
|
||||
return False
|
||||
states = [str((sh.get("summary") or {}).get("data_state") or "ok") for sh in sheets]
|
||||
if any(st in ("suspect_parse", "not_collected") for st in states):
|
||||
return None
|
||||
if states and all(st == "no_data" for st in states):
|
||||
return None
|
||||
return fail_count == 0
|
||||
|
||||
|
||||
def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
added = int(summary.get("added") or 0)
|
||||
removed = int(summary.get("removed") or 0)
|
||||
|
|
@ -3020,16 +3331,28 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
|||
before_count = int(summary.get("before_count") or 0)
|
||||
after_count = int(summary.get("after_count") or 0)
|
||||
total = added + removed + changed + unchanged
|
||||
|
||||
def _data_state(sh: dict[str, Any]) -> str:
|
||||
return str((sh.get("summary") or {}).get("data_state") or "ok")
|
||||
|
||||
# Sheets whose side was never collected cannot be judged: their "missing"
|
||||
# rows are a collection gap, not a business loss, so they stay out of the totals.
|
||||
dropped = [sh for sh in sheets if _data_state(sh) == "not_collected"
|
||||
and str(sh.get("status") or "done") not in ("pending", "running", "queued")]
|
||||
drop_fail = sum(int((sh.get("summary") or {}).get("removed") or 0)
|
||||
+ int((sh.get("summary") or {}).get("changed") or 0) for sh in dropped)
|
||||
drop_ok = sum(int((sh.get("summary") or {}).get("unchanged") or 0) for sh in dropped)
|
||||
# Compare verdict: fail = missing + mismatch; success = match; added is special
|
||||
fail_count = removed + changed
|
||||
success_count = unchanged
|
||||
fail_count = removed + changed - drop_fail
|
||||
success_count = unchanged - drop_ok
|
||||
judged = fail_count + success_count
|
||||
matched = changed + unchanged
|
||||
diff_count = fail_count
|
||||
# Nothing compared at all is "not judged" (None), never a 100% pass.
|
||||
pass_rate = (
|
||||
compute_pass_rate(success=success_count, fail=fail_count, empty_as=100.0 if total == 0 else 0.0)
|
||||
compute_pass_rate(success=success_count, fail=fail_count)
|
||||
if judged
|
||||
else (100.0 if total == 0 else 0.0)
|
||||
else (None if total == 0 else 0.0)
|
||||
)
|
||||
diff_rate = round((fail_count / judged) * 100, 1) if judged else 0.0
|
||||
|
||||
|
|
@ -3045,8 +3368,12 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
|||
sj = sf + su
|
||||
status = str(sh.get("status") or "done")
|
||||
pending = status in ("pending", "running", "queued")
|
||||
data_state = str(ss.get("data_state") or "ok")
|
||||
sheet_cards.append(
|
||||
{
|
||||
"data_state": data_state,
|
||||
"data_state_reasons": list(ss.get("data_state_reasons") or []),
|
||||
"collect_quality": ss.get("collect_quality") or {},
|
||||
"sheet_id": sheet_key(sh),
|
||||
"title": sheet_title(sh),
|
||||
"metric_id": sh.get("metric_id") or "",
|
||||
|
|
@ -3063,12 +3390,8 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
|||
"diff_count": sf,
|
||||
# Pending sheets must not look like "100% pass"
|
||||
"pass_rate": None
|
||||
if pending
|
||||
else (
|
||||
compute_pass_rate(success=su, fail=sf, empty_as=100.0 if st == 0 else 0.0)
|
||||
if sj
|
||||
else (100.0 if st == 0 else 0.0)
|
||||
),
|
||||
if pending or data_state in UNJUDGED_STATES
|
||||
else (compute_pass_rate(success=su, fail=sf) if sj else (None if st == 0 else 0.0)),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -3105,7 +3428,9 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di
|
|||
"diff_count": diff_count,
|
||||
"pass_rate": None if any_pending else pass_rate,
|
||||
"diff_rate": diff_rate,
|
||||
"ok": False if any_pending else fail_count == 0,
|
||||
"ok": _task_ok(sheets, fail_count, any_pending),
|
||||
"unjudged_sheets": sum(1 for sh in sheets if _data_state(sh) in UNJUDGED_STATES),
|
||||
"suspect_sheets": sum(1 for sh in sheets if _data_state(sh) == "suspect_parse"),
|
||||
"sheet_cards": sheet_cards,
|
||||
"top_changed_fields": top_fields,
|
||||
"duration_ms": int(summary.get("duration_ms") or 0),
|
||||
|
|
@ -3449,21 +3774,25 @@ def _load_metric_rows_for_search(
|
|||
|
||||
def _load_search_key_groups(
|
||||
db: Session, *, batch_id: str, metric_id: str, keys: set[tuple[str, ...]],
|
||||
match_keys: list[str], iface_fields: list[str], row_filters: list[dict[str, Any]],
|
||||
norm_rules: list[dict[str, str]], port_map: dict[str, str],
|
||||
match_keys: list[str], pipeline: Pipeline, side: str,
|
||||
) -> tuple[list[dict[str, Any]], set[tuple[str, ...]]]:
|
||||
"""Complete candidate key groups so a one-sided search cannot change verdicts.
|
||||
|
||||
Interface transforms run in Python. Other identity fields narrow the SQL
|
||||
query. If the load budget is exceeded, omit entire groups rather than compare
|
||||
partial groups and manufacture additions/deletions.
|
||||
Identity rules (normalize / value map / port map) run in the pipeline, in
|
||||
Python. Key fields they never touch narrow the SQL query. If the load budget
|
||||
is exceeded, omit entire groups rather than compare partial groups and
|
||||
manufacture additions/deletions.
|
||||
"""
|
||||
from ..models import BizStateMetricRow
|
||||
|
||||
model = BizStateLldpNeighbor if metric_id == "lldp_neighbor" else BizStateMetricRow
|
||||
invariant = [f for f in match_keys if f not in iface_fields]
|
||||
# Narrowing compares raw SQL text, so only fields no rule rewrites may be used,
|
||||
# and only when key tuples line up 1:1 with ``match_keys`` (no mapped/unmapped marker).
|
||||
invariant = [] if pipeline.ignores_ports_by_policy() else [
|
||||
f for f in match_keys if f not in pipeline._transformed # noqa: SLF001
|
||||
]
|
||||
positions = [match_keys.index(f) for f in invariant]
|
||||
coarse = sorted({tuple(k[i] for i in positions) for k in keys})
|
||||
coarse = sorted({tuple(k[i] for i in positions) for k in keys}) if invariant else [()]
|
||||
out: list[dict[str, Any]] = []
|
||||
complete: set[tuple[str, ...]] = set()
|
||||
# Bound memory even when a candidate identifies a very large duplicate group.
|
||||
|
|
@ -3471,7 +3800,7 @@ def _load_search_key_groups(
|
|||
for start in range(0, len(coarse), 100):
|
||||
group = coarse[start:start + 100]
|
||||
group_set = set(group)
|
||||
target = {k for k in keys if tuple(k[i] for i in positions) in group_set}
|
||||
target = {k for k in keys if tuple(k[i] for i in positions) in group_set} if invariant else set(keys)
|
||||
q = db.query(model).filter(model.batch_id == batch_id)
|
||||
if metric_id == "lldp_neighbor":
|
||||
q = q.order_by(model.id.asc())
|
||||
|
|
@ -3497,13 +3826,10 @@ def _load_search_key_groups(
|
|||
)
|
||||
row = {**data, "_netx": {"row_id": r.id}}
|
||||
db.expunge(r)
|
||||
if row_filters and not all(row_matches_filter(row, f) for f in row_filters):
|
||||
norm = pipeline.normalize(row, side)
|
||||
if not pipeline.accepts(row, norm, side):
|
||||
continue
|
||||
normalized = apply_iface_normalize_rows(
|
||||
[row], iface_fields=iface_fields, rules=norm_rules,
|
||||
)[0]
|
||||
mapped = apply_port_map(normalized, iface_fields=iface_fields, port_map=port_map)
|
||||
if row_key(mapped, match_keys) in target:
|
||||
if pipeline.key_of(norm, side, match_keys)[1] in target:
|
||||
out.append(row)
|
||||
return out, complete
|
||||
|
||||
|
|
@ -3566,13 +3892,15 @@ def _live_search_sheet_diffs(
|
|||
ignore_ports = sheet.get("ignore_port_changes")
|
||||
if ignore_ports is not None:
|
||||
ignore_ports = bool(ignore_ports)
|
||||
pmap, norm_rules = _run_transform_config(db, run)
|
||||
pipeline = _run_pipeline(db, run, sheet)
|
||||
if pipeline.legacy and not pipeline.row_filters and row_filters:
|
||||
pipeline.row_filters = [dict(f) for f in row_filters if isinstance(f, dict)]
|
||||
|
||||
before_rows, trunc_b = _load_metric_rows_for_search(
|
||||
db,
|
||||
batch_id=str(run.before_batch_id or ""),
|
||||
metric_id=mid_src,
|
||||
row_filters=row_filters,
|
||||
row_filters=pipeline.pushdown_filters("before"),
|
||||
key_fields=key_fields,
|
||||
kw=free_kw,
|
||||
field_q=merged_q,
|
||||
|
|
@ -3581,49 +3909,46 @@ def _live_search_sheet_diffs(
|
|||
db,
|
||||
batch_id=str(run.after_batch_id or ""),
|
||||
metric_id=mid_src,
|
||||
row_filters=row_filters,
|
||||
row_filters=pipeline.pushdown_filters("after"),
|
||||
key_fields=key_fields,
|
||||
kw=free_kw,
|
||||
field_q=merged_q,
|
||||
)
|
||||
match_keys = list((sheet.get("summary") or {}).get("match_key_fields") or key_fields)
|
||||
if ignore_ports is True and not pmap:
|
||||
if ignore_ports is True and not pipeline.port_map:
|
||||
match_keys = [f for f in key_fields if f not in iface_fields] or key_fields
|
||||
keys: set[tuple[str, ...]] = set()
|
||||
for rows, mapping in ((before_rows, pmap), (after_rows, {})):
|
||||
for row in apply_iface_normalize_rows(rows, iface_fields=iface_fields, rules=norm_rules):
|
||||
keys.add(row_key(apply_port_map(row, iface_fields=iface_fields, port_map=mapping), match_keys))
|
||||
for rows, side in ((before_rows, "before"), (after_rows, "after")):
|
||||
for raw, norm in pipeline.prepare(rows, side):
|
||||
keys.add(pipeline.key_of(norm, side, match_keys)[1])
|
||||
before_rows, complete_b = _load_search_key_groups(
|
||||
db, batch_id=str(run.before_batch_id or ""), metric_id=mid_src, keys=keys,
|
||||
match_keys=match_keys, iface_fields=iface_fields, row_filters=row_filters,
|
||||
norm_rules=norm_rules, port_map=pmap,
|
||||
match_keys=match_keys, pipeline=pipeline, side="before",
|
||||
)
|
||||
after_rows, complete_a = _load_search_key_groups(
|
||||
db, batch_id=str(run.after_batch_id or ""), metric_id=mid_src, keys=keys,
|
||||
match_keys=match_keys, iface_fields=iface_fields, row_filters=row_filters,
|
||||
norm_rules=norm_rules, port_map={},
|
||||
match_keys=match_keys, pipeline=pipeline, side="after",
|
||||
)
|
||||
complete = complete_b & complete_a
|
||||
def keep_complete(rows: list[dict[str, Any]], mapping: dict[str, str]) -> list[dict[str, Any]]:
|
||||
normalized = apply_iface_normalize_rows(rows, iface_fields=iface_fields, rules=norm_rules)
|
||||
return [orig for orig, norm in zip(rows, normalized) if row_key(
|
||||
apply_port_map(norm, iface_fields=iface_fields, port_map=mapping), match_keys,
|
||||
) in complete]
|
||||
before_rows = keep_complete(before_rows, pmap)
|
||||
after_rows = keep_complete(after_rows, {})
|
||||
|
||||
def keep_complete(rows: list[dict[str, Any]], side: str) -> list[dict[str, Any]]:
|
||||
return [raw for raw, norm in pipeline.prepare(rows, side)
|
||||
if pipeline.key_of(norm, side, match_keys)[1] in complete]
|
||||
|
||||
before_rows = keep_complete(before_rows, "before")
|
||||
after_rows = keep_complete(after_rows, "after")
|
||||
result = compare_rows(
|
||||
before_rows=before_rows,
|
||||
after_rows=after_rows,
|
||||
key_fields=key_fields,
|
||||
iface_fields=iface_fields,
|
||||
compare_fields=compare_fields,
|
||||
port_map=pmap,
|
||||
field_rules=field_rules,
|
||||
iface_normalize_rules=norm_rules,
|
||||
ignore_port_changes=(match_keys != key_fields) if not pmap else False,
|
||||
include_unchanged=True,
|
||||
unchanged_limit=None,
|
||||
compact_unchanged=False,
|
||||
pipeline=pipeline,
|
||||
match_keys_override=match_keys,
|
||||
)
|
||||
kind_n = (kind or "diff").strip().lower()
|
||||
filtered = [
|
||||
|
|
@ -3636,7 +3961,7 @@ def _live_search_sheet_diffs(
|
|||
filtered = filtered[:_LIVE_SEARCH_RESULT_CAP]
|
||||
total = len(filtered)
|
||||
start = (page - 1) * page_size
|
||||
page_items = filtered[start : start + page_size]
|
||||
page_items = _attach_raw_key_before(filtered[start : start + page_size], key_fields)
|
||||
return {
|
||||
"total": total,
|
||||
"page": page,
|
||||
|
|
@ -3844,7 +4169,7 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
|
|||
csv_file.write(b"\xef\xbb\xbf")
|
||||
for line in _sheet_csv_lines(sheet, _iter_sheet_diffs(db, run_id, sid)):
|
||||
csv_file.write((line + "\n").encode("utf-8"))
|
||||
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"]
|
||||
sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate,data_state"]
|
||||
for card in list(s.get("sheet_cards") or []):
|
||||
sum_lines.append(
|
||||
",".join(
|
||||
|
|
@ -3860,6 +4185,7 @@ def export_run_zip(db: Session, run_id: str) -> bytes:
|
|||
card.get("unchanged"),
|
||||
card.get("diff_count"),
|
||||
card.get("pass_rate"),
|
||||
card.get("data_state") or "ok",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
@ -3927,7 +4253,7 @@ def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int:
|
|||
if not j.before_batch_id:
|
||||
continue
|
||||
try:
|
||||
run_compare(db, j.id, force_after_batch_id=batch_id)
|
||||
run_compare(db, j.id, force_after_batch_id=batch_id, purpose="auto")
|
||||
n += 1
|
||||
except HTTPException as exc:
|
||||
if int(getattr(exc, "status_code", 0) or 0) == 409:
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from uuid import uuid4
|
|||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .compare_engine import key_tuple
|
||||
from .compare_rules import effective_compare_fields, field_rule_map, NUMBER_PATTERN, scalar_text
|
||||
|
||||
_log = logging.getLogger("netx.biz_state.compare_sql")
|
||||
|
|
@ -154,16 +155,23 @@ def sql_compare_skip_reason(
|
|||
*,
|
||||
port_map: Mapping[str, str] | None = None,
|
||||
iface_normalize_rules: Sequence[Mapping[str, str]] | None = None,
|
||||
pipeline: Any = None,
|
||||
) -> str:
|
||||
"""Empty string when SQL is allowed; otherwise a short reason for progress/logs.
|
||||
|
||||
Simple ``row_filters`` (eq/in/contains/…) used for BGP sheet splits are fine —
|
||||
they do **not** force Python by themselves.
|
||||
|
||||
A port map only blocks SQL when the sheet actually reads an interface field
|
||||
in its key or compare columns; a BGP route table with no interface fields is
|
||||
unaffected by it. ``pipeline`` (v2 identity rules) adds its own blockers.
|
||||
"""
|
||||
if not _dialect_is_postgres(db):
|
||||
return "not_postgres"
|
||||
if port_map:
|
||||
return "port_map"
|
||||
if pipeline is not None:
|
||||
port_map = pipeline.port_map
|
||||
if iface_normalize_rules is None and (pipeline.before.has_iface_rules or pipeline.after.has_iface_rules):
|
||||
iface_normalize_rules = [{"from": "*", "to": "*"}]
|
||||
key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()]
|
||||
if not key_fields:
|
||||
return "no_key_fields"
|
||||
|
|
@ -171,12 +179,24 @@ def sql_compare_skip_reason(
|
|||
return "unsafe_key_field"
|
||||
iface_fields = [str(f).strip() for f in (sheet.get("iface_fields") or []) if str(f).strip()]
|
||||
iface_set = set(iface_fields)
|
||||
if port_map and iface_set:
|
||||
early_cmp = effective_compare_fields(
|
||||
list(sheet.get("compare_fields") or []), list(sheet.get("field_rules") or []),
|
||||
)
|
||||
if iface_set & (set(key_fields) | {str(f).strip() for f in early_cmp}):
|
||||
return "port_map"
|
||||
ignore_ports = sheet.get("ignore_port_changes")
|
||||
if ignore_ports is True:
|
||||
return "ignore_port_changes"
|
||||
if ignore_ports is None and iface_set and any(k in iface_set for k in key_fields):
|
||||
return "auto_ignore_ports"
|
||||
field_rules = list(sheet.get("field_rules") or [])
|
||||
if any(
|
||||
isinstance(r, dict) and str(r.get("compare") or "").strip().lower() in
|
||||
("not_worse", "numeric_min", "numeric_max")
|
||||
for r in field_rules
|
||||
):
|
||||
return "directional_compare"
|
||||
if not _field_rules_sql_compatible(field_rules):
|
||||
return "field_rules"
|
||||
compare_fields = effective_compare_fields(
|
||||
|
|
@ -186,8 +206,14 @@ def sql_compare_skip_reason(
|
|||
if any(not _FIELD_RE.match(str(f).strip()) for f in compare_fields if str(f).strip()):
|
||||
return "unsafe_compare_field"
|
||||
used = set(key_fields) | {str(f).strip() for f in compare_fields if str(f).strip()}
|
||||
if port_map and (used & iface_set):
|
||||
return "port_map"
|
||||
if iface_normalize_rules and (used & iface_set):
|
||||
return "iface_normalize"
|
||||
if pipeline is not None:
|
||||
blocked = pipeline.sql_blockers(used)
|
||||
if blocked:
|
||||
return blocked
|
||||
if not _filters_sql_compatible(list(sheet.get("row_filters") or [])):
|
||||
return "row_filters"
|
||||
if not str(sheet.get("metric_id") or "").strip():
|
||||
|
|
@ -371,7 +397,7 @@ def _changed_predicate(
|
|||
|
||||
def _key_obj_from_data(data: dict[str, Any] | None, key_fields: list[str]) -> dict[str, Any]:
|
||||
row = data if isinstance(data, dict) else {}
|
||||
return {f: row.get(f, "") for f in key_fields}
|
||||
return dict(zip(key_fields, key_tuple(row, key_fields)))
|
||||
|
||||
|
||||
def _changes_from_rows(
|
||||
|
|
@ -399,6 +425,22 @@ def _changes_from_rows(
|
|||
return out
|
||||
|
||||
|
||||
def dup_pairing_order_sql(compare_fields: Sequence[str], rules: Mapping[str, Mapping[str, Any]]) -> str:
|
||||
"""``ORDER BY`` prefix that pairs same-key rows by value (mirrors ``Pipeline.pair_sort_key``).
|
||||
|
||||
Byte-order collation matches Python's code-point string order, so both
|
||||
engines choose the same pairs. Returns '' (or a trailing-comma list).
|
||||
"""
|
||||
parts = []
|
||||
for f in compare_fields:
|
||||
name = str(f).strip()
|
||||
if not name:
|
||||
continue
|
||||
mode = str((rules.get(name) or {}).get("normalize") or "strip").strip().lower() or "strip"
|
||||
parts.append(f'({_norm_expr("data_json", name, mode)}) COLLATE "C" ASC')
|
||||
return ", ".join(parts) + ", " if parts else ""
|
||||
|
||||
|
||||
def run_sql_sheet_compare(
|
||||
db: Session,
|
||||
*,
|
||||
|
|
@ -407,6 +449,7 @@ def run_sql_sheet_compare(
|
|||
after_batch_id: str,
|
||||
store_unchanged: str = "auto",
|
||||
on_progress: Callable[..., None] | None = None,
|
||||
duplicate_pairing: str = "ordered",
|
||||
) -> dict[str, Any]:
|
||||
"""Compare one sheet via PostgreSQL TEMP tables + FULL OUTER JOIN.
|
||||
|
||||
|
|
@ -443,6 +486,7 @@ def run_sql_sheet_compare(
|
|||
|
||||
filter_sql, filter_params = compile_row_filters_sql(row_filters)
|
||||
rk = _rk_sql(key_fields)
|
||||
dup_order = dup_pairing_order_sql(compare_fields, rules) if duplicate_pairing == "sorted" else ""
|
||||
tag = uuid4().hex[:8]
|
||||
tb = f"_netx_cmp_b_{tag}"
|
||||
ta = f"_netx_cmp_a_{tag}"
|
||||
|
|
@ -500,7 +544,7 @@ def run_sql_sheet_compare(
|
|||
data_json AS data,
|
||||
row_number() OVER (
|
||||
PARTITION BY ({rk})
|
||||
ORDER BY seq ASC, id ASC
|
||||
ORDER BY {dup_order}seq ASC, id ASC
|
||||
) AS dup_rn
|
||||
FROM biz_state_metric_row
|
||||
WHERE batch_id = :bid
|
||||
|
|
|
|||
|
|
@ -8,7 +8,10 @@ from typing import Any
|
|||
from fastapi import HTTPException
|
||||
|
||||
FILTER_OPS = {"eq", "==", "ci_eq", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "regex", "age_timer"}
|
||||
COMPARE_MODES = {"", "eq", "ignore", "skip", "off", "numeric", "number", "int", "float", "percent", "pct", "rel"}
|
||||
COMPARE_MODES = {"", "eq", "ignore", "skip", "off", "numeric", "number", "int", "float", "percent", "pct", "rel",
|
||||
"not_worse", "numeric_min", "numeric_max"}
|
||||
SIDES = {"", "both", "before", "after"}
|
||||
MAX_VALUE_MAP = 500
|
||||
NORMALIZE_MODES = {"", "none", "strip", "lower", "upper", "mac", "empty_as_blank"}
|
||||
|
||||
|
||||
|
|
@ -34,6 +37,8 @@ def validate_filters(value: Any, path: str, depth: int = 0) -> None:
|
|||
here = f"{path}[{i}]"
|
||||
if not isinstance(node, dict) or not node:
|
||||
invalid(here, "filter_required")
|
||||
if depth == 0 and str(node.get("side") or "").strip().lower() not in SIDES:
|
||||
invalid(here, "unknown_filter_side")
|
||||
groups = [key for key in ("any", "all") if key in node]
|
||||
if groups:
|
||||
if len(groups) != 1 or any(node.get(key) is not None for key in ("field", "op", "value")):
|
||||
|
|
@ -73,6 +78,12 @@ def validate_sheet(sheet: Any, path: str, *, partial: bool = False) -> None:
|
|||
for key in ("key_fields", "iface_fields", "compare_fields", "display_fields", "ignore_fields"):
|
||||
if key in sheet or (key == "key_fields" and not partial):
|
||||
_strings(sheet.get(key), f"{path}.{key}", required=key == "key_fields")
|
||||
for opt, allowed in (("unmapped_port_policy", {"by_name", "ignore_port"}),
|
||||
("duplicate_pairing", {"sorted", "ordered"})):
|
||||
if str(sheet.get(opt) or "").strip() and str(sheet[opt]).strip().lower() not in allowed:
|
||||
invalid(path, f"unknown_{opt}")
|
||||
if sheet.get("builtin_vendor_canonical") is not None and not isinstance(sheet["builtin_vendor_canonical"], bool):
|
||||
invalid(path, "boolean_builtin_vendor_canonical_required")
|
||||
if "ignore_port_changes" in sheet and sheet["ignore_port_changes"] is not None and not isinstance(sheet["ignore_port_changes"], bool):
|
||||
invalid(path, "boolean_port_policy_required")
|
||||
if "row_filters" in sheet:
|
||||
|
|
@ -87,9 +98,30 @@ def validate_sheet(sheet: Any, path: str, *, partial: bool = False) -> None:
|
|||
if not isinstance(rule, dict) or not isinstance(rule.get("field"), str) or not rule["field"].strip():
|
||||
invalid(here, "rule_field_required")
|
||||
field = rule["field"].strip()
|
||||
if field in seen:
|
||||
rule_side = str(rule.get("side") or "").strip().lower()
|
||||
if rule_side not in SIDES:
|
||||
invalid(here, "unknown_rule_side")
|
||||
marker = (field, "" if rule_side == "both" else rule_side)
|
||||
if marker in seen:
|
||||
invalid(here, "duplicate_field_rule")
|
||||
seen.add(field)
|
||||
seen.add(marker)
|
||||
vmap = rule.get("value_map")
|
||||
if vmap is not None:
|
||||
if not isinstance(vmap, dict) or any(
|
||||
not isinstance(k, str) or not isinstance(v, str) for k, v in vmap.items()
|
||||
):
|
||||
invalid(here, "value_map_string_pairs_required")
|
||||
if len(vmap) > MAX_VALUE_MAP:
|
||||
invalid(here, "value_map_too_large")
|
||||
if rule.get("apply_to_key") is not None and not isinstance(rule["apply_to_key"], bool):
|
||||
invalid(here, "boolean_apply_to_key_required")
|
||||
mode = str(rule.get("compare") or "").strip().lower()
|
||||
if mode == "not_worse":
|
||||
rank = rule.get("rank")
|
||||
if not isinstance(rank, list) or len(rank) < 2 or any(not isinstance(x, str) or not x.strip() for x in rank):
|
||||
invalid(here, "rank_list_required")
|
||||
elif rule.get("rank") is not None:
|
||||
invalid(here, "rank_only_for_not_worse")
|
||||
if str(rule.get("compare") or "").strip().lower() not in COMPARE_MODES:
|
||||
invalid(here, "unknown_compare_mode")
|
||||
if str(rule.get("normalize") or "").strip().lower() not in NORMALIZE_MODES:
|
||||
|
|
@ -124,7 +156,13 @@ def validate_template_body(body: dict[str, Any], *, partial: bool = False) -> No
|
|||
path = f"iface_normalize_rules[{i}]"
|
||||
if not isinstance(rule, dict) or any(not isinstance(rule.get(k), str) or not rule[k].strip() for k in ("from", "to")):
|
||||
invalid(path, "alias_pair_required")
|
||||
key = rule["from"].strip().lower()
|
||||
rside = str(rule.get("side") or "").strip().lower()
|
||||
if rside not in SIDES:
|
||||
invalid(path, "unknown_alias_side")
|
||||
if rule.get("vendor") is not None and not isinstance(rule["vendor"], str):
|
||||
invalid(path, "alias_vendor_string_required")
|
||||
key = (rule["from"].strip().lower(), "" if rside == "both" else rside,
|
||||
str(rule.get("vendor") or "").strip().lower())
|
||||
if key in seen:
|
||||
invalid(path, "duplicate_alias")
|
||||
seen.add(key)
|
||||
|
|
|
|||
|
|
@ -131,9 +131,25 @@
|
|||
],
|
||||
"row_filters": [
|
||||
{
|
||||
"op": "age_timer",
|
||||
"field": "age",
|
||||
"value": ""
|
||||
"any": [
|
||||
{
|
||||
"field": "entry_type",
|
||||
"op": "eq",
|
||||
"value": "dynamic"
|
||||
},
|
||||
{
|
||||
"all": [
|
||||
{
|
||||
"field": "entry_type",
|
||||
"op": "empty"
|
||||
},
|
||||
{
|
||||
"field": "age",
|
||||
"op": "age_timer"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"field_rules": [
|
||||
|
|
@ -415,11 +431,7 @@
|
|||
],
|
||||
"compare_fields": [
|
||||
"admin",
|
||||
"input_bps",
|
||||
"output_bps",
|
||||
"port_status",
|
||||
"in_util",
|
||||
"out_util",
|
||||
"ip_mtu",
|
||||
"mtu",
|
||||
"mpls_mtu",
|
||||
|
|
@ -454,7 +466,7 @@
|
|||
},
|
||||
{
|
||||
"field": "output_bps",
|
||||
"compare": "numeric",
|
||||
"compare": "percent",
|
||||
"tolerance": 5
|
||||
},
|
||||
{
|
||||
|
|
@ -593,8 +605,8 @@
|
|||
"field_rules": [
|
||||
{
|
||||
"field": "rx_power",
|
||||
"compare": "percent",
|
||||
"tolerance": 5
|
||||
"compare": "numeric",
|
||||
"tolerance": 2
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -668,7 +680,9 @@
|
|||
"vpn_sid",
|
||||
"neighbor_sid"
|
||||
],
|
||||
"iface_fields": [],
|
||||
"iface_fields": [
|
||||
"ac_port"
|
||||
],
|
||||
"compare_fields": [],
|
||||
"display_fields": [
|
||||
"mac",
|
||||
|
|
|
|||
454
netx_api/biz_state/identity_pipeline.py
Normal file
454
netx_api/biz_state/identity_pipeline.py
Normal file
|
|
@ -0,0 +1,454 @@
|
|||
"""One identity pipeline for every compare path.
|
||||
|
||||
A row's *match key* must come out identical whether it is computed while a run
|
||||
executes, while its results are searched / hydrated / exported, while a mapping
|
||||
is validated, or while a cutover monitor evaluates the same sheet. All of those
|
||||
paths therefore build a :class:`Pipeline` and call the same methods instead of
|
||||
re-implementing "normalize → map → key".
|
||||
|
||||
Stages for one row on one side (``before`` or ``after``)::
|
||||
|
||||
raw row ──normalize──▶ iface aliases (+vendor canonical) ▶ key normalize ▶ value_map
|
||||
──filter────▶ ``row_filters`` (v2: on the normalized row; legacy: on the raw row)
|
||||
──map───────▶ port map (before side only)
|
||||
──key───────▶ ``key_tuple`` over the match key fields
|
||||
|
||||
``legacy=True`` reproduces the pre-pipeline behaviour (snapshot v1 runs): the
|
||||
template alias rules are the only normalization, keys are never normalized,
|
||||
there is no value map and the filters see the raw row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from .compare_rules import (
|
||||
normalize_value,
|
||||
row_matches_filter,
|
||||
scalar_text,
|
||||
)
|
||||
from .iface_normalize import (
|
||||
normalize_iface_name,
|
||||
normalize_iface_rules,
|
||||
normalize_port_map,
|
||||
resolve_mapped_iface,
|
||||
rules_for_side,
|
||||
vendor_canonical_rules,
|
||||
)
|
||||
|
||||
SIDES = ("before", "after")
|
||||
DIRECTIONAL_MODES = {"not_worse", "numeric_min", "numeric_max"}
|
||||
UNMAPPED_POLICIES = ("by_name", "ignore_port")
|
||||
DUPLICATE_PAIRING = ("sorted", "ordered")
|
||||
|
||||
|
||||
def filter_fields(filt: Mapping[str, Any] | None) -> set[str]:
|
||||
"""Every field name a (possibly nested) row filter reads."""
|
||||
out: set[str] = set()
|
||||
if not isinstance(filt, Mapping):
|
||||
return out
|
||||
for group in ("any", "all"):
|
||||
if group in filt:
|
||||
for kid in filt.get(group) or []:
|
||||
out |= filter_fields(kid)
|
||||
return out
|
||||
name = str(filt.get("field") or "").strip()
|
||||
if name:
|
||||
out.add(name)
|
||||
return out
|
||||
|
||||
|
||||
def _vendor_key(vendor: str = "", device_type: str = "") -> str:
|
||||
from ..topology_lldp import resolve_vendor_key
|
||||
|
||||
key = resolve_vendor_key(vendor, device_type)
|
||||
return "" if key == "generic" else key
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class SideSpec:
|
||||
side: str
|
||||
vendor: str = ""
|
||||
canonical: list[dict[str, str]] = field(default_factory=list)
|
||||
alias_rules: list[dict[str, str]] = field(default_factory=list)
|
||||
# field -> {lowercased source value: canonical value}
|
||||
value_maps: dict[str, dict[str, str]] = field(default_factory=dict)
|
||||
# key field -> normalize mode (only fields whose rule asks for one)
|
||||
key_norms: dict[str, str] = field(default_factory=dict)
|
||||
strip_inner_space: bool = True
|
||||
|
||||
def iface(self, name: Any) -> str:
|
||||
text = str(name or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
if self.strip_inner_space:
|
||||
text = "".join(text.split())
|
||||
if self.canonical:
|
||||
text = normalize_iface_name(text, self.canonical)
|
||||
if self.alias_rules:
|
||||
text = normalize_iface_name(text, self.alias_rules)
|
||||
return text
|
||||
|
||||
@property
|
||||
def has_iface_rules(self) -> bool:
|
||||
return bool(self.canonical or self.alias_rules)
|
||||
|
||||
def describe(self) -> dict[str, Any]:
|
||||
"""JSON-able record of the rules in force (stored in the run snapshot)."""
|
||||
return {
|
||||
"vendor": self.vendor,
|
||||
"canonical": list(self.canonical),
|
||||
"alias_rules": list(self.alias_rules),
|
||||
"value_maps": {k: dict(v) for k, v in self.value_maps.items()},
|
||||
"key_norms": dict(self.key_norms),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Pipeline:
|
||||
key_fields: list[str]
|
||||
iface_fields: list[str]
|
||||
compare_fields: list[str]
|
||||
field_rules: list[dict[str, Any]]
|
||||
row_filters: list[dict[str, Any]]
|
||||
port_map: dict[str, str]
|
||||
before: SideSpec
|
||||
after: SideSpec
|
||||
legacy: bool = False
|
||||
ignore_port_changes: bool | None = None
|
||||
unmapped_port_policy: str = "by_name"
|
||||
duplicate_pairing: str = "ordered"
|
||||
_iface_set: set[str] = field(default_factory=set, repr=False)
|
||||
_transformed: set[str] = field(default_factory=set, repr=False)
|
||||
_targets: set[str] = field(default_factory=set, repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._iface_set = set(self.iface_fields)
|
||||
self._transformed = set(self._iface_set)
|
||||
for spec in (self.before, self.after):
|
||||
self._transformed |= set(spec.value_maps) | set(spec.key_norms)
|
||||
self._targets = set(self.port_map.values())
|
||||
|
||||
# -- side helpers ----------------------------------------------------
|
||||
def spec(self, side: str) -> SideSpec:
|
||||
return self.before if side == "before" else self.after
|
||||
|
||||
# -- stage 1: normalize ----------------------------------------------
|
||||
def normalize(self, row: Mapping[str, Any], side: str) -> dict[str, Any]:
|
||||
out = dict(row)
|
||||
spec = self.spec(side)
|
||||
for f in self.iface_fields:
|
||||
val = scalar_text(out.get(f)).strip()
|
||||
if val:
|
||||
out[f] = spec.iface(val)
|
||||
if self.legacy:
|
||||
return out
|
||||
for f, mode in spec.key_norms.items():
|
||||
if f in out:
|
||||
out[f] = normalize_value(out[f], mode)
|
||||
for f, vmap in spec.value_maps.items():
|
||||
if f not in out:
|
||||
continue
|
||||
raw = scalar_text(out[f]).strip()
|
||||
hit = vmap.get(raw.lower())
|
||||
if hit is not None:
|
||||
out[f] = hit
|
||||
return out
|
||||
|
||||
def normalize_name(self, name: Any, side: str) -> str:
|
||||
"""Interface-name normalization only (port-map keys / values, selectors)."""
|
||||
return self.spec(side).iface(name)
|
||||
|
||||
# -- stage 2: filter -------------------------------------------------
|
||||
def filters_for(self, side: str) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for f in self.row_filters:
|
||||
fs = str(f.get("side") or "").strip().lower()
|
||||
if not fs or fs == "both" or fs == side:
|
||||
out.append(f)
|
||||
return out
|
||||
|
||||
def keep(self, row: Mapping[str, Any], side: str) -> bool:
|
||||
"""Evaluate row filters. v2 expects the *normalized* row, legacy the raw one."""
|
||||
return all(row_matches_filter(row, f) for f in self.filters_for(side))
|
||||
|
||||
def accepts(self, raw: Mapping[str, Any], norm: Mapping[str, Any], side: str) -> bool:
|
||||
"""Row-filter verdict: legacy reads the raw row, v2 the normalized one."""
|
||||
return self.keep(raw if self.legacy else norm, side)
|
||||
|
||||
def pushdown_filters(self, side: str) -> list[dict[str, Any]]:
|
||||
"""Filters that read only untransformed fields, so they may run in SQL on raw rows."""
|
||||
if self.legacy:
|
||||
return list(self.row_filters)
|
||||
return [
|
||||
f for f in self.row_filters
|
||||
if not str(f.get("side") or "").strip()
|
||||
and not (filter_fields(f) & self._transformed)
|
||||
]
|
||||
|
||||
def prepare(self, rows: Sequence[Mapping[str, Any]], side: str) -> list[tuple[dict[str, Any], dict[str, Any]]]:
|
||||
"""``[(original row, normalized row)]`` after filtering, in input order."""
|
||||
out: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||
has_filters = bool(self.filters_for(side))
|
||||
for raw in rows:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
norm = self.normalize(raw, side)
|
||||
if has_filters and not self.accepts(raw, norm, side):
|
||||
continue
|
||||
out.append((raw, norm))
|
||||
return out
|
||||
|
||||
# -- stage 3: port map -----------------------------------------------
|
||||
def apply_port_map(self, norm_row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
out = dict(norm_row)
|
||||
if not self.port_map:
|
||||
return out
|
||||
for f in self.iface_fields:
|
||||
val = scalar_text(out.get(f)).strip()
|
||||
if val:
|
||||
out[f] = resolve_mapped_iface(val, self.port_map)
|
||||
return out
|
||||
|
||||
def is_mapped(self, norm_row: Mapping[str, Any], side: str) -> bool:
|
||||
"""True when every interface value of the row is covered by the port map."""
|
||||
if not self.iface_fields:
|
||||
return True
|
||||
values = [scalar_text(norm_row.get(f)).strip() for f in self.iface_fields]
|
||||
values = [v for v in values if v]
|
||||
if not values:
|
||||
return False
|
||||
for v in values:
|
||||
if side == "before":
|
||||
if resolve_mapped_iface(v, self.port_map) == v and v not in self.port_map:
|
||||
return False
|
||||
else:
|
||||
if not self._after_value_is_target(v):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _after_value_is_target(self, value: str) -> bool:
|
||||
if value in self._targets:
|
||||
return True
|
||||
parts = value.split(".")
|
||||
return any(".".join(parts[:i]) in self._targets for i in range(len(parts) - 1, 0, -1))
|
||||
|
||||
def transform(self, row: Mapping[str, Any], side: str) -> dict[str, Any]:
|
||||
norm = self.normalize(row, side)
|
||||
return self.apply_port_map(norm) if side == "before" else norm
|
||||
|
||||
# -- stage 4: key ----------------------------------------------------
|
||||
def effective_match_keys(self, *, unique_without_iface: bool | None = None) -> list[str]:
|
||||
"""Match-key fields once the port policy is resolved.
|
||||
|
||||
With a port map the full key is used (the map aligns the names). Without
|
||||
one, ``ignore_port_changes`` may drop the interface fields; ``None`` means
|
||||
"auto", resolved by the caller via ``unique_without_iface``.
|
||||
"""
|
||||
keys = list(self.key_fields)
|
||||
if self.port_map or not self._iface_set:
|
||||
return keys
|
||||
candidate = [k for k in keys if k not in self._iface_set]
|
||||
if not candidate:
|
||||
return keys
|
||||
policy = self.ignore_port_changes
|
||||
if policy is True:
|
||||
return candidate
|
||||
if policy is False:
|
||||
return keys
|
||||
return candidate if unique_without_iface else keys
|
||||
|
||||
def ignores_ports_by_policy(self) -> bool:
|
||||
return bool(self.port_map) and self.unmapped_port_policy == "ignore_port" and bool(self._iface_set)
|
||||
|
||||
def key_of(
|
||||
self, norm_row: Mapping[str, Any], side: str, match_keys: Sequence[str],
|
||||
) -> tuple[dict[str, Any], tuple[str, ...]]:
|
||||
"""``(row after port map, match key)`` for a *normalized* row."""
|
||||
row = self.apply_port_map(norm_row) if side == "before" else dict(norm_row)
|
||||
if self.ignores_ports_by_policy():
|
||||
if not self.is_mapped(norm_row, side):
|
||||
reduced = [k for k in match_keys if k not in self._iface_set] or list(match_keys)
|
||||
return row, ("U",) + tuple(scalar_text(row.get(k)).strip() for k in reduced)
|
||||
return row, ("M",) + tuple(scalar_text(row.get(k)).strip() for k in match_keys)
|
||||
return row, tuple(scalar_text(row.get(k)).strip() for k in match_keys)
|
||||
|
||||
def match_key(
|
||||
self, row: Mapping[str, Any], side: str, match_keys: Sequence[str] | None = None,
|
||||
) -> tuple[str, ...]:
|
||||
"""Match key of a *raw* row (normalize → port map → key)."""
|
||||
keys = list(match_keys) if match_keys is not None else self.effective_match_keys()
|
||||
return self.key_of(self.normalize(row, side), side, keys)[1]
|
||||
|
||||
# -- duplicate pairing -----------------------------------------------
|
||||
def pair_sort_key(self, norm_row: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
rules = {str(r.get("field")): r for r in self.field_rules if isinstance(r, dict)}
|
||||
out = []
|
||||
for f in self.compare_fields:
|
||||
mode = str((rules.get(f) or {}).get("normalize") or "strip")
|
||||
out.append(normalize_value(norm_row.get(f), mode))
|
||||
return tuple(out)
|
||||
|
||||
# -- SQL eligibility -------------------------------------------------
|
||||
def sql_blockers(self, used_fields: set[str]) -> str:
|
||||
"""Reason the SQL engine must not run this sheet ('' when it may)."""
|
||||
if self.legacy:
|
||||
return ""
|
||||
for spec in (self.before, self.after):
|
||||
if set(spec.value_maps) & used_fields:
|
||||
return "value_map"
|
||||
if set(spec.key_norms) & set(self.key_fields):
|
||||
return "key_normalize"
|
||||
if used_fields & self._iface_set and (self.before.has_iface_rules or self.after.has_iface_rules):
|
||||
return "iface_normalize"
|
||||
if any(str(f.get("side") or "").strip() for f in self.row_filters):
|
||||
return "row_filters_side"
|
||||
if any(filter_fields(f) & self._transformed for f in self.row_filters):
|
||||
return "row_filters_post_normalize"
|
||||
if any(
|
||||
str(r.get("compare") or "").strip().lower() in DIRECTIONAL_MODES
|
||||
for r in self.field_rules if isinstance(r, dict)
|
||||
):
|
||||
return "directional_compare"
|
||||
return ""
|
||||
|
||||
def describe(self) -> dict[str, Any]:
|
||||
return {
|
||||
"legacy": self.legacy,
|
||||
"before": self.before.describe(),
|
||||
"after": self.after.describe(),
|
||||
"unmapped_port_policy": self.unmapped_port_policy,
|
||||
"duplicate_pairing": self.duplicate_pairing,
|
||||
}
|
||||
|
||||
|
||||
def _field_rule_specs(
|
||||
field_rules: Sequence[Mapping[str, Any]], key_fields: Sequence[str], side: str,
|
||||
) -> tuple[dict[str, dict[str, str]], dict[str, str]]:
|
||||
value_maps: dict[str, dict[str, str]] = {}
|
||||
key_norms: dict[str, str] = {}
|
||||
key_set = set(key_fields)
|
||||
for rule in field_rules or []:
|
||||
if not isinstance(rule, Mapping):
|
||||
continue
|
||||
name = str(rule.get("field") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
rs = str(rule.get("side") or "").strip().lower()
|
||||
if rs and rs != "both" and rs != side:
|
||||
continue
|
||||
apply_to_key = rule.get("apply_to_key")
|
||||
apply_to_key = (name in key_set) if apply_to_key is None else bool(apply_to_key)
|
||||
is_key = name in key_set
|
||||
if is_key and not apply_to_key:
|
||||
continue
|
||||
mode = str(rule.get("normalize") or "").strip().lower()
|
||||
vmap = rule.get("value_map")
|
||||
if is_key and mode and mode not in ("none", "strip"):
|
||||
key_norms[name] = mode
|
||||
if isinstance(vmap, Mapping) and vmap:
|
||||
pre = mode if (mode and mode not in ("none", "strip")) else "strip"
|
||||
built: dict[str, str] = {}
|
||||
for src, dst in vmap.items():
|
||||
built[normalize_value(src, pre).lower()] = scalar_text(dst).strip()
|
||||
value_maps[name] = built
|
||||
return value_maps, key_norms
|
||||
|
||||
|
||||
def build_pipeline(
|
||||
sheet: Mapping[str, Any],
|
||||
*,
|
||||
iface_rules: Sequence[Mapping[str, Any]] | None,
|
||||
port_map: Mapping[str, str] | None,
|
||||
before_vendor: str = "",
|
||||
after_vendor: str = "",
|
||||
legacy: bool = False,
|
||||
normalize_map: bool = True,
|
||||
) -> Pipeline:
|
||||
"""Build the pipeline for one sheet.
|
||||
|
||||
``port_map`` is the map as the user wrote it. Unless ``normalize_map`` is
|
||||
false (legacy runs, whose maps were written post-normalization) its keys are
|
||||
normalized with the *before* side rules and its values with the *after* side
|
||||
rules.
|
||||
"""
|
||||
key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()]
|
||||
iface_fields = [str(k).strip() for k in (sheet.get("iface_fields") or []) if str(k).strip()]
|
||||
field_rules = [dict(r) for r in (sheet.get("field_rules") or []) if isinstance(r, Mapping)]
|
||||
from .compare_rules import effective_compare_fields
|
||||
|
||||
compare_fields = effective_compare_fields(list(sheet.get("compare_fields") or []), field_rules)
|
||||
canonical_on = sheet.get("builtin_vendor_canonical", True) is not False and not legacy
|
||||
all_rules = normalize_iface_rules(list(iface_rules or []))
|
||||
|
||||
specs: dict[str, SideSpec] = {}
|
||||
for side, vendor_raw in (("before", before_vendor), ("after", after_vendor)):
|
||||
vendor = _vendor_key(vendor_raw) if vendor_raw else ""
|
||||
if legacy:
|
||||
alias = normalize_iface_rules([{"from": r["from"], "to": r["to"]} for r in all_rules])
|
||||
value_maps, key_norms = {}, {}
|
||||
else:
|
||||
alias = rules_for_side(all_rules, side, vendor)
|
||||
value_maps, key_norms = _field_rule_specs(field_rules, key_fields, side)
|
||||
specs[side] = SideSpec(
|
||||
side=side, vendor=vendor,
|
||||
canonical=vendor_canonical_rules(vendor) if canonical_on else [],
|
||||
alias_rules=alias, value_maps=value_maps, key_norms=key_norms,
|
||||
strip_inner_space=not legacy,
|
||||
)
|
||||
|
||||
pmap = dict(port_map or {})
|
||||
if pmap and normalize_map and not legacy:
|
||||
pmap, _ = normalize_port_map(
|
||||
pmap, None,
|
||||
key_normalizer=specs["before"].iface, value_normalizer=specs["after"].iface,
|
||||
)
|
||||
ignore_ports = sheet.get("ignore_port_changes")
|
||||
policy = str(sheet.get("unmapped_port_policy") or "by_name").strip().lower()
|
||||
pairing = str(sheet.get("duplicate_pairing") or ("ordered" if legacy else "sorted")).strip().lower()
|
||||
return Pipeline(
|
||||
key_fields=key_fields,
|
||||
iface_fields=iface_fields,
|
||||
compare_fields=compare_fields,
|
||||
field_rules=field_rules,
|
||||
row_filters=[dict(f) for f in (sheet.get("row_filters") or []) if isinstance(f, Mapping)],
|
||||
port_map=pmap,
|
||||
before=specs["before"],
|
||||
after=specs["after"],
|
||||
legacy=legacy,
|
||||
ignore_port_changes=None if ignore_ports is None else bool(ignore_ports),
|
||||
unmapped_port_policy=policy if policy in UNMAPPED_POLICIES else "by_name",
|
||||
duplicate_pairing="ordered" if legacy else (pairing if pairing in DUPLICATE_PAIRING else "sorted"),
|
||||
)
|
||||
|
||||
|
||||
def legacy_pipeline(
|
||||
*,
|
||||
key_fields: Sequence[str],
|
||||
iface_fields: Sequence[str],
|
||||
compare_fields: Sequence[str],
|
||||
field_rules: Sequence[Mapping[str, Any]] | None = None,
|
||||
port_map: Mapping[str, str] | None = None,
|
||||
iface_normalize_rules: Sequence[Mapping[str, Any]] | None = None,
|
||||
ignore_port_changes: bool | None = None,
|
||||
row_filters: Sequence[Mapping[str, Any]] | None = None,
|
||||
duplicate_pairing: str = "ordered",
|
||||
) -> Pipeline:
|
||||
"""Pipeline with the pre-v2 semantics, for snapshot-v1 runs and bare callers."""
|
||||
alias = normalize_iface_rules(
|
||||
[{"from": r.get("from"), "to": r.get("to")} for r in (iface_normalize_rules or [])]
|
||||
)
|
||||
return Pipeline(
|
||||
key_fields=[str(k) for k in key_fields],
|
||||
iface_fields=[str(f) for f in iface_fields if str(f).strip()],
|
||||
compare_fields=[str(f) for f in compare_fields],
|
||||
field_rules=[dict(r) for r in (field_rules or []) if isinstance(r, Mapping)],
|
||||
row_filters=[dict(f) for f in (row_filters or []) if isinstance(f, Mapping)],
|
||||
port_map=dict(port_map or {}),
|
||||
before=SideSpec(side="before", alias_rules=list(alias), strip_inner_space=False),
|
||||
after=SideSpec(side="after", alias_rules=list(alias), strip_inner_space=False),
|
||||
legacy=True,
|
||||
ignore_port_changes=ignore_port_changes,
|
||||
duplicate_pairing=duplicate_pairing if duplicate_pairing in DUPLICATE_PAIRING else "ordered",
|
||||
)
|
||||
|
|
@ -39,7 +39,14 @@ def normalize_iface_rules(raw: Any) -> list[dict[str, str]]:
|
|||
fr = str(item.get("from") or item.get("src") or "").strip()
|
||||
to = str(item.get("to") or item.get("dst") or "").strip()
|
||||
if fr and to:
|
||||
out.append({"from": fr, "to": to})
|
||||
rule = {"from": fr, "to": to}
|
||||
side = str(item.get("side") or "").strip().lower()
|
||||
if side in ("before", "after"):
|
||||
rule["side"] = side
|
||||
vendor = str(item.get("vendor") or "").strip().lower()
|
||||
if vendor:
|
||||
rule["vendor"] = vendor
|
||||
out.append(rule)
|
||||
# Longest prefix first so XXVGE wins over XGE / GE.
|
||||
out.sort(key=lambda r: len(r["from"]), reverse=True)
|
||||
return out
|
||||
|
|
@ -107,6 +114,47 @@ def apply_iface_normalize_rows(
|
|||
]
|
||||
|
||||
|
||||
def normalize_port_map(
|
||||
port_map: Mapping[str, str] | None,
|
||||
rules: Sequence[Mapping[str, str]] | None,
|
||||
value_rules: Sequence[Mapping[str, str]] | None = None,
|
||||
*,
|
||||
key_normalizer: Any = None,
|
||||
value_normalizer: Any = None,
|
||||
) -> tuple[dict[str, str], list[dict[str, str]]]:
|
||||
"""Normalize port-map keys and values with the same rules as iface fields.
|
||||
|
||||
``value_rules`` (default: ``rules``) normalizes the after-side values, for
|
||||
per-side rules. ``key_normalizer`` / ``value_normalizer`` are callables that
|
||||
replace the rule-based normalization (used by the identity pipeline).
|
||||
|
||||
Returns ``(normalized_map, conflicts)``. A conflict is two raw keys that
|
||||
normalize to the same key but point at different (normalized) targets; the
|
||||
first one wins in ``normalized_map``.
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
origin: dict[str, str] = {}
|
||||
conflicts: list[dict[str, str]] = []
|
||||
for raw_k, raw_v in (port_map or {}).items():
|
||||
k = key_normalizer(raw_k) if key_normalizer else normalize_iface_name(raw_k, rules)
|
||||
v = (
|
||||
value_normalizer(raw_v) if value_normalizer
|
||||
else normalize_iface_name(raw_v, rules if value_rules is None else value_rules)
|
||||
)
|
||||
if not k or not v:
|
||||
continue
|
||||
if k in out:
|
||||
if out[k] != v:
|
||||
conflicts.append({
|
||||
"before_if": k, "kept_from": origin[k], "kept_to": out[k],
|
||||
"dropped_from": str(raw_k), "dropped_to": v,
|
||||
})
|
||||
continue
|
||||
out[k] = v
|
||||
origin[k] = str(raw_k)
|
||||
return out, conflicts
|
||||
|
||||
|
||||
def resolve_mapped_iface(name: str, port_map: Mapping[str, str] | None) -> str:
|
||||
"""Exact map hit, else longest mapped parent + remaining suffix (QinQ-safe).
|
||||
|
||||
|
|
@ -142,3 +190,53 @@ def default_zte_iface_normalize_rules() -> list[dict[str, str]]:
|
|||
{"from": "SG", "to": "smartgroup"},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in per-vendor canonical names (same-vendor synonyms only; cross-vendor
|
||||
# numbering is the port map's job). Applied before template alias rules.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VENDOR_CANONICAL: dict[str, list[dict[str, str]]] = {
|
||||
"huawei": [
|
||||
{"from": "XGigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"},
|
||||
{"from": "100GE", "to": "100GE"}, {"from": "40GE", "to": "40GE"}, {"from": "25GE", "to": "25GE"},
|
||||
{"from": "10GE", "to": "10GE"}, {"from": "Eth-Trunk", "to": "Eth-Trunk"},
|
||||
],
|
||||
"cisco": [
|
||||
{"from": "GigabitEthernet", "to": "Gi"}, {"from": "TenGigabitEthernet", "to": "Te"},
|
||||
{"from": "TenGigE", "to": "Te"}, {"from": "HundredGigE", "to": "Hu"}, {"from": "FortyGigE", "to": "Fo"},
|
||||
{"from": "TwentyFiveGigE", "to": "Twe"}, {"from": "FastEthernet", "to": "Fa"},
|
||||
{"from": "Bundle-Ether", "to": "BE"}, {"from": "Loopback", "to": "Lo"},
|
||||
],
|
||||
"h3c": [
|
||||
{"from": "Ten-GigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"},
|
||||
{"from": "FortyGigE", "to": "FGE"}, {"from": "HundredGigE", "to": "HGE"},
|
||||
{"from": "Bridge-Aggregation", "to": "BAGG"},
|
||||
],
|
||||
"zte": [
|
||||
{"from": "xxvgei", "to": "xxvgei"}, {"from": "xgei", "to": "xgei"}, {"from": "cgei", "to": "cgei"},
|
||||
{"from": "gei", "to": "gei"}, {"from": "smartgroup", "to": "smartgroup"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def vendor_canonical_rules(vendor: str) -> list[dict[str, str]]:
|
||||
return normalize_iface_rules(VENDOR_CANONICAL.get(str(vendor or "").strip().lower(), []))
|
||||
|
||||
|
||||
def rules_for_side(
|
||||
rules: Sequence[Mapping[str, str]] | None, side: str, vendor: str = "",
|
||||
) -> list[dict[str, str]]:
|
||||
"""Template alias rules that apply to ``side`` (``before``/``after``) and ``vendor``."""
|
||||
vendor_n = str(vendor or "").strip().lower()
|
||||
out: list[dict[str, str]] = []
|
||||
for r in rules or []:
|
||||
rs = str(r.get("side") or "").strip().lower()
|
||||
if rs and rs != side:
|
||||
continue
|
||||
rv = str(r.get("vendor") or "").strip().lower()
|
||||
if rv and rv != vendor_n:
|
||||
continue
|
||||
out.append({"from": str(r.get("from") or ""), "to": str(r.get("to") or "")})
|
||||
return normalize_iface_rules(out)
|
||||
|
|
|
|||
|
|
@ -85,6 +85,9 @@ class ParseProfile:
|
|||
enrich_joins: list[EnrichJoin] = field(default_factory=list)
|
||||
# light: default shared SSH lane; heavy: dedicated long-timeout connection
|
||||
collect_lane: str = "light"
|
||||
# Which slice of the metric this command produces, e.g. {"afi": "ipv4"}.
|
||||
# Empty = the command covers the whole metric (or its slicing is unknown).
|
||||
slice: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
_LLDP_FIELDS: list[FieldDef] = [
|
||||
|
|
@ -1651,6 +1654,11 @@ _HEAVY_LANE_METRICS = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def metric_lane(metric_id: str) -> str:
|
||||
"""``heavy`` for huge CLI dumps (own SSH connection), else ``light``."""
|
||||
return "heavy" if str(metric_id or "").strip() in _HEAVY_LANE_METRICS else "light"
|
||||
|
||||
|
||||
def _apply_collect_lanes(profiles: list[ParseProfile]) -> list[ParseProfile]:
|
||||
for p in profiles:
|
||||
if str(p.metric_id or "").strip() in _HEAVY_LANE_METRICS:
|
||||
|
|
@ -1752,12 +1760,41 @@ def _validate_profiles(profiles: list[ParseProfile]) -> None:
|
|||
)
|
||||
|
||||
|
||||
# profile_id -> slice. Compare sheets filter on these fields (``afi``/``af``/``direction``),
|
||||
# so a missing command is detected as "slice not collected" instead of "business lost".
|
||||
_PROFILE_SLICES: dict[str, dict[str, str]] = {
|
||||
**{f"zte.bgp_{afi}_summary": {"afi": afi} for afi in ("vpnv4", "ipv4", "vpnv6", "ipv6", "evpn", "vpls")},
|
||||
"zte.bgp_vpnv4_vrf_summary": {"afi": "vpnv4"},
|
||||
"zte.bgp_vpnv6_vrf_summary": {"afi": "vpnv6"},
|
||||
"zte.vrrp_ipv4": {"af": "ipv4"},
|
||||
"zte.vrrp_ipv6": {"af": "ipv6"},
|
||||
**{
|
||||
f"zte.bgp_{afi}_neighbor_{direction}": {"afi": afi, "direction": direction}
|
||||
for afi in ("vpnv4", "vpnv6", "ipv4", "ipv6")
|
||||
for direction in ("in", "out")
|
||||
},
|
||||
**{
|
||||
f"zte.bgp_{afi}_vrf_neighbor_{direction}": {"afi": afi, "direction": direction}
|
||||
for afi in ("vpnv4", "vpnv6")
|
||||
for direction in ("in", "out")
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _apply_profile_slices(profiles: list[ParseProfile]) -> list[ParseProfile]:
|
||||
for p in profiles:
|
||||
declared = _PROFILE_SLICES.get(p.profile_id)
|
||||
if declared and not p.slice:
|
||||
p.slice = dict(declared)
|
||||
return profiles
|
||||
|
||||
|
||||
def _finalize_profiles(profiles: list[ParseProfile]) -> list[ParseProfile]:
|
||||
"""Normalize discover→aux, validate, apply collect lanes."""
|
||||
for p in profiles:
|
||||
_normalize_discover_aux(p)
|
||||
_validate_profiles(profiles)
|
||||
return _apply_collect_lanes(profiles)
|
||||
return _apply_profile_slices(_apply_collect_lanes(profiles))
|
||||
|
||||
|
||||
def all_profiles() -> list[ParseProfile]:
|
||||
|
|
|
|||
|
|
@ -6,13 +6,17 @@ from collections import defaultdict
|
|||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
BizCompareDiff,
|
||||
BizCompareJob,
|
||||
BizCompareRun,
|
||||
BizMigrationBatch,
|
||||
BizMigrationDiff,
|
||||
BizMigrationProject,
|
||||
BizMigrationRedTicket,
|
||||
BizMigrationRun,
|
||||
BizStateBatch,
|
||||
BizStateBatchCommand,
|
||||
|
|
@ -76,9 +80,103 @@ def protected_batch_map(
|
|||
add(before_id, before_reason)
|
||||
add(after_id, after_reason)
|
||||
|
||||
# Every batch a *kept* acceptance/manual run (or the newest auto run of a wave)
|
||||
# compared, not only the first binding. Older auto runs are bounded by
|
||||
# ``purge_auto_runs`` and keep protecting just their primary pair above.
|
||||
for run_batches in _kept_run_metric_batches(db):
|
||||
for bid in run_batches:
|
||||
add(bid, "migration_run_metric")
|
||||
|
||||
return dict(out)
|
||||
|
||||
|
||||
def _kept_run_metric_batches(db: Session, *, limit: int = 500) -> list[set[str]]:
|
||||
newest_auto = (
|
||||
db.query(func.max(BizMigrationRun.created_at).label("at"), BizMigrationRun.batch_id.label("bid"))
|
||||
.filter(BizMigrationRun.purpose == "auto").group_by(BizMigrationRun.batch_id).subquery()
|
||||
)
|
||||
latest_auto_ids = [
|
||||
r[0] for r in db.query(BizMigrationRun.id).join(
|
||||
newest_auto,
|
||||
(BizMigrationRun.batch_id == newest_auto.c.bid) & (BizMigrationRun.created_at == newest_auto.c.at),
|
||||
).limit(limit).all()
|
||||
]
|
||||
kept_ids = [
|
||||
r[0] for r in db.query(BizMigrationRun.id).filter(BizMigrationRun.purpose != "auto")
|
||||
.order_by(BizMigrationRun.created_at.desc()).limit(limit).all()
|
||||
] + latest_auto_ids
|
||||
out: list[set[str]] = []
|
||||
for start in range(0, len(kept_ids), 100):
|
||||
for (summary,) in db.query(BizMigrationRun.summary_json).filter(
|
||||
BizMigrationRun.id.in_(kept_ids[start:start + 100])
|
||||
):
|
||||
batches: set[str] = set()
|
||||
for item in ((summary or {}).get("metric_batches") or {}).values():
|
||||
if isinstance(item, dict):
|
||||
batches.update(str(item.get(k) or "") for k in ("old_batch_id", "new_batch_id"))
|
||||
batches.discard("")
|
||||
if batches:
|
||||
out.append(batches)
|
||||
return out
|
||||
|
||||
|
||||
def purge_auto_runs(db: Session, *, migration_keep: int = 300, compare_keep: int = 50) -> dict[str, int]:
|
||||
"""Bound auto-generated run history.
|
||||
|
||||
Cutover monitoring writes a run (+ one diff row per object) every minute and
|
||||
auto-compare writes one per new sample. Acceptance / manual runs and runs a
|
||||
red ticket points at are kept forever; only the newest ``keep`` auto runs of
|
||||
each wave / job survive.
|
||||
"""
|
||||
dropped = {"migration_runs": 0, "compare_runs": 0}
|
||||
ticket_runs = {r[0] for r in db.query(BizMigrationRedTicket.run_id).filter(
|
||||
BizMigrationRedTicket.run_id != "").distinct()}
|
||||
accepted = {r[0] for r in db.query(BizMigrationBatch.accept_run_id).filter(
|
||||
BizMigrationBatch.accept_run_id != "")}
|
||||
keep_forever = ticket_runs | accepted
|
||||
|
||||
def surplus(model, group_col, extra_filter, keep: int) -> list[str]:
|
||||
over = [g for g, n in db.query(group_col, func.count(model.id)).filter(*extra_filter)
|
||||
.group_by(group_col).all() if n > keep]
|
||||
ids: list[str] = []
|
||||
for g in over:
|
||||
rows = db.query(model.id).filter(group_col == g, *extra_filter).order_by(
|
||||
model.created_at.desc(), model.id.desc()).offset(keep).all()
|
||||
ids.extend(r[0] for r in rows if r[0] not in keep_forever)
|
||||
return ids
|
||||
|
||||
mig_ids = surplus(BizMigrationRun, BizMigrationRun.batch_id, [BizMigrationRun.purpose == "auto"],
|
||||
max(1, int(migration_keep)))
|
||||
for start in range(0, len(mig_ids), 500):
|
||||
chunk = mig_ids[start:start + 500]
|
||||
db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id.in_(chunk)).delete(synchronize_session=False)
|
||||
db.query(BizMigrationRun).filter(BizMigrationRun.id.in_(chunk)).delete(synchronize_session=False)
|
||||
dropped["migration_runs"] = len(mig_ids)
|
||||
|
||||
# Auto-mode jobs only. A run's origin lives in summary_json["purpose"]; unmarked runs
|
||||
# are manual and stay. Only the surplus beyond ``compare_keep`` is inspected.
|
||||
cmp_ids: list[str] = []
|
||||
keep_n = max(1, int(compare_keep))
|
||||
for (job_id,) in db.query(BizCompareJob.id).filter(BizCompareJob.mode == "auto").all():
|
||||
surplus_ids = [r[0] for r in db.query(BizCompareRun.id).filter(
|
||||
BizCompareRun.job_id == job_id, BizCompareRun.status.notin_(("queued", "running")),
|
||||
).order_by(BizCompareRun.created_at.desc(), BizCompareRun.id.desc()).offset(keep_n).all()]
|
||||
for start in range(0, len(surplus_ids), 200):
|
||||
for rid, summary in db.query(BizCompareRun.id, BizCompareRun.summary_json).filter(
|
||||
BizCompareRun.id.in_(surplus_ids[start:start + 200])
|
||||
):
|
||||
if (summary or {}).get("purpose") == "auto":
|
||||
cmp_ids.append(rid)
|
||||
for start in range(0, len(cmp_ids), 500):
|
||||
chunk = cmp_ids[start:start + 500]
|
||||
db.query(BizCompareDiff).filter(BizCompareDiff.run_id.in_(chunk)).delete(synchronize_session=False)
|
||||
db.query(BizCompareRun).filter(BizCompareRun.id.in_(chunk)).delete(synchronize_session=False)
|
||||
dropped["compare_runs"] = len(cmp_ids)
|
||||
if mig_ids or cmp_ids:
|
||||
db.commit()
|
||||
return dropped
|
||||
|
||||
|
||||
def protected_batch_ids(db: Session, *, task_id: str = "") -> set[str]:
|
||||
return set(protected_batch_map(db, task_id=task_id).keys())
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue