"""Fabric edge upsert, missing/purge lifecycle, absorb, and merge.""" from __future__ import annotations import re from datetime import datetime, timedelta from typing import Any from uuid import uuid4 from fastapi import HTTPException from sqlalchemy import and_, func, or_, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from .cli_resolve import get_default_profile, infer_device_type_vendor from .device_types import LLDP_DISCOVERED_NE_SOURCE, WEBCRT_NE_SOURCE, is_placeholder_ne_source from .models import ( ManagedNE, TopoFabricEdge, TopoFabricNode, TopoFabricStats, TopoViewEdgeStyle, TopoViewNode, UmeInventoryNE, ) from .topology_common import ( PAGE_DEFAULT, PAGE_MAX, VIEW_GRAPH_EDGE_HARD_CAP, _ADV_NS_FABRIC_MANAGED, _ADV_NS_FABRIC_UME, _EDGE_STATUS_MISSING, _EDGE_STATUS_MISSING_COMPAT, _MISS_PURGE_AFTER_CYCLES, _advisory_xact_lock, _clear_miss_attrs, _empty_to_none, _is_deadlock_error, _is_postgres, _norm_host, _normalize_edge_status, _purge_edge_if_due, _set_edge_missing, _sleep_deadlock_backoff, _utcnow, _edge_attrs, ) from .topology_lldp import NeighborHit, normalize_ifname from .topology_schemas import ( FabricEdgeOut, FabricNeighborhoodOut, FabricNodeOut, FabricSummaryOut, ) from .topology_fabric_nodes import ( _edge_out, _node_out, _nodes_by_ids, _normalize_endpoints, refresh_fabric_stats, ) from .topology_fabric_peers import _fabric_match_score, _is_inventory_node def upsert_fabric_edge( db: Session, *, a_node_id: str, b_node_id: str, a_port: str, b_port: str, source: str = "lldp", layer: str = "physical", now: datetime | None = None, ) -> tuple[TopoFabricEdge | None, str]: """Return (edge, action) where action is added|updated|kept_manual|skipped_*. Self-loops are skipped (``(None, \"skipped_self_loop\")``) so LLDP discovery can ignore a device advertising itself without aborting the rest of the scan. Manual edge APIs should treat that action as a client error. ``source`` may be ``lldp`` | ``manual`` | ``ume``. Provenance is tracked in ``attrs.sources`` (union). Primary ``source`` prefers manual > ume > lldp (UME is authority when present; LLDP is the no-UME discovery path). LLDP against an existing compatible edge upgrades bare UME ports to media- prefixed names. When no compatible edge exists, LLDP may still create a new edge even if both ends are UME inventory NEs (UME dump may omit the link). """ from .ume_port_normalize import prefer_richer_ifname now = now or _utcnow() a, b, ap, bp = _normalize_endpoints(a_node_id, b_node_id, a_port, b_port) if a == b: return None, "skipped_self_loop" layer_v = str(layer or "physical").strip() or "physical" src = str(source or "lldp").strip().lower() or "lldp" if src == "stale": src = "lldp" if src not in {"lldp", "manual", "ume"}: raise HTTPException(status_code=400, detail="invalid_edge_source") row = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == layer_v, TopoFabricEdge.a_node_id == a, TopoFabricEdge.b_node_id == b, TopoFabricEdge.a_port == ap, TopoFabricEdge.b_port == bp, ) .one_or_none() ) if row is None: # Compatible ports (bare UME ↔ LLDP media) — merge, don't duplicate. row = find_fabric_edge_compatible( db, a_node_id=a, b_node_id=b, a_port=ap, b_port=bp, layer=layer_v ) if row is not None: new_ap = prefer_richer_ifname(row.a_port, ap) new_bp = prefer_richer_ifname(row.b_port, bp) if new_ap != normalize_ifname(row.a_port) or new_bp != normalize_ifname(row.b_port): row.a_port = new_ap[:128] row.b_port = new_bp[:128] else: try: with db.begin_nested(): row = TopoFabricEdge( id=uuid4().hex, layer=layer_v, a_node_id=a, b_node_id=b, a_port=ap, b_port=bp, source=src, status="active", attrs={"sources": [src]}, discovered_at=now if src in {"lldp", "ume"} else None, last_seen_at=now, created_at=now, updated_at=now, ) db.add(row) db.flush() return row, "added" except IntegrityError: row = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == layer_v, TopoFabricEdge.a_node_id == a, TopoFabricEdge.b_node_id == b, TopoFabricEdge.a_port == ap, TopoFabricEdge.b_port == bp, ) .one_or_none() ) if row is None: raise # Manual edges keep primary source=manual; still record other sources. attrs = _clear_miss_attrs(_edge_attrs(row)) sources = _sources_from_attrs(attrs, fallback=row.source or src) sources.add(src) attrs["sources"] = sorted(sources) # Enrich ports when a richer compatible name arrives (typically LLDP). if src in {"lldp", "ume"}: new_ap = prefer_richer_ifname(row.a_port, ap) new_bp = prefer_richer_ifname(row.b_port, bp) if new_ap != normalize_ifname(row.a_port) or new_bp != normalize_ifname(row.b_port): row.a_port = new_ap[:128] row.b_port = new_bp[:128] if (row.source or "") == "manual" and src != "manual": row.attrs = attrs row.status = "active" row.last_seen_at = now row.updated_at = now return row, "kept_manual" row.source = _primary_source(sources) row.status = "active" row.attrs = attrs if src in {"lldp", "ume"}: row.discovered_at = row.discovered_at or now row.last_seen_at = now row.updated_at = now return row, "updated" def _sources_from_attrs(attrs: dict[str, Any], *, fallback: str = "") -> set[str]: raw = attrs.get("sources") if isinstance(attrs, dict) else None out: set[str] = set() if isinstance(raw, (list, tuple, set)): for item in raw: s = str(item or "").strip().lower() if s in {"lldp", "manual", "ume"}: out.add(s) fb = str(fallback or "").strip().lower() if fb in {"lldp", "manual", "ume"}: out.add(fb) elif fb == "stale": out.add("lldp") return out def _primary_source(sources: set[str]) -> str: """Paint / authority order: manual > ume > lldp.""" if "manual" in sources: return "manual" if "ume" in sources: return "ume" if "lldp" in sources: return "lldp" return "lldp" def find_fabric_edge_compatible( db: Session, *, a_node_id: str, b_node_id: str, a_port: str, b_port: str, layer: str = "physical", ) -> TopoFabricEdge | None: """Exact match first, then same endpoints with compatible port names.""" from .ume_port_normalize import port_keys_compatible a, b, ap, bp = _normalize_endpoints(a_node_id, b_node_id, a_port, b_port) layer_v = str(layer or "physical").strip() or "physical" exact = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == layer_v, TopoFabricEdge.a_node_id == a, TopoFabricEdge.b_node_id == b, TopoFabricEdge.a_port == ap, TopoFabricEdge.b_port == bp, ) .one_or_none() ) if exact is not None: return exact candidates = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == layer_v, TopoFabricEdge.a_node_id == a, TopoFabricEdge.b_node_id == b, ) .all() ) for e in candidates: if port_keys_compatible(e.a_port, ap) and port_keys_compatible(e.b_port, bp): return e return None def _absorb_fabric_node(db: Session, canon: TopoFabricNode, dupe: TopoFabricNode) -> None: """Retarget edges/view placements from dupe onto canon, then delete dupe.""" if canon is None or dupe is None or canon.id == dupe.id: return if db.get(TopoFabricNode, dupe.id) is None: return _retarget_fabric_edges(db, from_id=dupe.id, to_id=canon.id) vnodes = db.query(TopoViewNode).filter(TopoViewNode.fabric_node_id == dupe.id).all() for vn in vnodes: exists = ( db.query(TopoViewNode) .filter( TopoViewNode.view_id == vn.view_id, TopoViewNode.fabric_node_id == canon.id, ) .one_or_none() ) if exists is not None: db.delete(vn) else: vn.fabric_node_id = canon.id vn.updated_at = _utcnow() db.delete(dupe) def _prefer_fabric_canon( db: Session, a: TopoFabricNode, b: TopoFabricNode ) -> tuple[TopoFabricNode, TopoFabricNode]: """Return (canon, dupe) preferring higher inventory score, then older row.""" sa = _fabric_match_score(db, a) sb = _fabric_match_score(db, b) if sa != sb: return (a, b) if sa > sb else (b, a) ta = a.created_at or a.updated_at tb = b.created_at or b.updated_at if ta and tb and ta != tb: return (a, b) if ta <= tb else (b, a) return (a, b) if a.id <= b.id else (b, a) def _mark_replaced_port_peers( db: Session, *, self_id: str, local_port: str, peer_id: str, new_edge_id: str, layer: str = "physical", now: datetime | None = None, ) -> list[str]: """Same local port now peers with a different NE → mark old edges missing (cutover). If the previous peer is the same hostname (duplicate fabric rows for one device), absorb the weaker node instead of marking the link missing. Returns ids of edges touched by this replacement (skip re-bump in same job). """ now = now or _utcnow() lp = normalize_ifname(local_port) if not self_id or not peer_id or not lp: return [] layer_v = str(layer or "physical").strip() or "physical" new_peer = db.get(TopoFabricNode, peer_id) new_name = _norm_host(new_peer.name if new_peer is not None else "") candidates = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == layer_v, TopoFabricEdge.id != new_edge_id, TopoFabricEdge.source != "manual", or_(TopoFabricEdge.a_node_id == self_id, TopoFabricEdge.b_node_id == self_id), ) .all() ) handled: list[str] = [] from .ume_port_normalize import port_keys_compatible for e in candidates: if e.a_node_id == self_id: e_local, e_peer = e.a_port or "", e.b_node_id else: e_local, e_peer = e.b_port or "", e.a_node_id if not port_keys_compatible(e_local, lp): continue if e_peer == peer_id: continue old_peer = db.get(TopoFabricNode, e_peer) old_name = _norm_host(old_peer.name if old_peer is not None else "") # Same System Name under two fabric nodes → collapse, keep one link. if ( new_peer is not None and old_peer is not None and new_name and old_name and new_name == old_name ): canon, dupe = _prefer_fabric_canon(db, new_peer, old_peer) _absorb_fabric_node(db, canon, dupe) # Survivor edge on this port should stay active (retarget may have merged). survivor = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == layer_v, or_( and_( TopoFabricEdge.a_node_id == self_id, TopoFabricEdge.b_node_id == canon.id, ), and_( TopoFabricEdge.b_node_id == self_id, TopoFabricEdge.a_node_id == canon.id, ), ), ) .all() ) for se in survivor: se_local = se.a_port if se.a_node_id == self_id else se.b_port if not port_keys_compatible(se_local or "", lp): continue se.status = "active" se.attrs = _clear_miss_attrs(_edge_attrs(se)) se.last_seen_at = now se.updated_at = now handled.append(se.id) handled.append(e.id) continue _set_edge_missing(e, now, replaced_by_edge_id=new_edge_id) handled.append(e.id) return handled def _apply_missing_and_purge( db: Session, *, scanned_ok: set[str], touched_edge_ids: set[str], now: datetime | None = None, ) -> tuple[int, int]: """Rule A: endpoint scanned with valid LLDP evidence but edge absent → missing. ``scanned_ok`` must only contain nodes that actually logged in and produced trustworthy LLDP output (not login failures / stub parsers / empty CLI). Dual-source (ume+lldp): drop ``lldp`` provenance but keep the edge active under UME. Pure UME edges are left untouched. Pure LLDP edges may be miss-marked. Manual edges are excluded by the query filter. Returns (newly_marked_missing, purged). """ now = now or _utcnow() if not scanned_ok: return 0, 0 edges = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == "physical", TopoFabricEdge.source != "manual", or_( TopoFabricEdge.a_node_id.in_(list(scanned_ok)), TopoFabricEdge.b_node_id.in_(list(scanned_ok)), ), ) .all() ) newly_marked = 0 purged = 0 for e in edges: if e.id in touched_edge_ids: continue if e.a_node_id not in scanned_ok and e.b_node_id not in scanned_ok: continue attrs = _edge_attrs(e) sources = _sources_from_attrs(attrs, fallback=e.source or "") has_ume = "ume" in sources or str(e.source or "").strip().lower() == "ume" if has_ume: # UME still claims the link — strip stale LLDP mark only. if "lldp" in sources: sources.discard("lldp") attrs["sources"] = sorted(sources) e.attrs = attrs if str(e.source or "").strip().lower() != "manual": e.source = _primary_source(sources) e.status = "active" e.updated_at = now continue if _set_edge_missing(e, now): newly_marked += 1 if _purge_edge_if_due(db, e): purged += 1 return newly_marked, purged # --------------------------------------------------------------------------- # LLDP discovery → fabric # --------------------------------------------------------------------------- def _retarget_fabric_edges(db: Session, *, from_id: str, to_id: str) -> None: """Move edges from from_id onto to_id; drop duplicates / self-loops.""" if not from_id or not to_id or from_id == to_id: return edges = ( db.query(TopoFabricEdge) .filter(or_(TopoFabricEdge.a_node_id == from_id, TopoFabricEdge.b_node_id == from_id)) .all() ) for e in edges: a = to_id if e.a_node_id == from_id else e.a_node_id b = to_id if e.b_node_id == from_id else e.b_node_id if a == b: db.query(TopoViewEdgeStyle).filter(TopoViewEdgeStyle.fabric_edge_id == e.id).delete( synchronize_session=False ) db.delete(e) continue na, nb, ap, bp = _normalize_endpoints(a, b, e.a_port or "", e.b_port or "") clash = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.id != e.id, TopoFabricEdge.layer == (e.layer or "physical"), TopoFabricEdge.a_node_id == na, TopoFabricEdge.b_node_id == nb, TopoFabricEdge.a_port == ap, TopoFabricEdge.b_port == bp, ) .one_or_none() ) if clash is not None: # Keep the surviving edge fresher. if (e.last_seen_at or e.updated_at) and ( not clash.last_seen_at or (e.last_seen_at and clash.last_seen_at and e.last_seen_at > clash.last_seen_at) ): clash.source = e.source or clash.source clash.status = e.status or clash.status clash.last_seen_at = e.last_seen_at or clash.last_seen_at clash.discovered_at = e.discovered_at or clash.discovered_at clash.updated_at = _utcnow() db.query(TopoViewEdgeStyle).filter(TopoViewEdgeStyle.fabric_edge_id == e.id).delete( synchronize_session=False ) db.delete(e) continue e.a_node_id = na e.b_node_id = nb e.a_port = ap e.b_port = bp e.updated_at = _utcnow() def merge_duplicate_fabric_nodes(db: Session) -> dict[str, int]: """Collapse duplicate fabric nodes (same managed/ume/name/ip) onto inventory canonicals.""" nodes = db.query(TopoFabricNode).order_by(TopoFabricNode.created_at.asc()).all() merged = 0 placeholders_removed = 0 # 1) Same managed_ne_id / ume_ne_id (constraint may be missing on old DBs). by_managed: dict[str, list[TopoFabricNode]] = {} by_ume: dict[str, list[TopoFabricNode]] = {} for n in nodes: mid = str(n.managed_ne_id or "").strip() uid = str(n.ume_ne_id or "").strip() if mid: by_managed.setdefault(mid, []).append(n) if uid: by_ume.setdefault(uid, []).append(n) def _absorb(canon: TopoFabricNode, dupes: list[TopoFabricNode]) -> None: nonlocal merged for d in dupes: if d.id == canon.id: continue if db.get(TopoFabricNode, d.id) is None: continue _absorb_fabric_node(db, canon, d) merged += 1 seen_absorb: set[str] = set() for group in list(by_managed.values()) + list(by_ume.values()): alive = [n for n in group if n.id not in seen_absorb and db.get(TopoFabricNode, n.id) is not None] if len(alive) < 2: continue canon = next((n for n in alive if _is_inventory_node(n)), alive[0]) _absorb(canon, alive) for n in alive: seen_absorb.add(n.id) # 2) Orphans (no inventory ids) that collide with inventory node by name/ip. db.flush() nodes = db.query(TopoFabricNode).all() inventory = [n for n in nodes if _is_inventory_node(n)] orphans = [n for n in nodes if not _is_inventory_node(n)] inv_by_name: dict[str, TopoFabricNode] = {} inv_by_ip: dict[str, TopoFabricNode] = {} for n in sorted(inventory, key=lambda x: _fabric_match_score(db, x), reverse=True): nk = _norm_host(n.name or "") if nk and nk not in inv_by_name: inv_by_name[nk] = n ip = str(n.ip or "").strip() if ip and ip not in inv_by_ip: inv_by_ip[ip] = n for o in orphans: canon = None ip = str(o.ip or "").strip() nk = _norm_host(o.name or "") if ip and ip in inv_by_ip: canon = inv_by_ip[ip] elif nk and nk in inv_by_name: canon = inv_by_name[nk] if canon is None: continue _absorb(canon, [o]) # 2b) LLDP placeholders (score=2) → real inventory (score>=3) by hostname / seen mgmt IP. # Placeholders have managed_ne_id so they are NOT orphans; absorb + drop empty ManagedNE. db.flush() nodes = db.query(TopoFabricNode).all() reals = [n for n in nodes if _fabric_match_score(db, n) >= 3] placeholders = [n for n in nodes if _fabric_match_score(db, n) == 2] real_by_name: dict[str, TopoFabricNode] = {} real_by_ip: dict[str, TopoFabricNode] = {} for n in sorted(reals, key=lambda x: _fabric_match_score(db, x), reverse=True): nk = _norm_host(n.name or "") if nk and nk not in real_by_name: real_by_name[nk] = n ip = str(n.ip or "").strip() if ip and ip not in real_by_ip: real_by_ip[ip] = n for p in placeholders: if db.get(TopoFabricNode, p.id) is None: continue canon = None nk = _norm_host(p.name or "") ip = str(p.ip or "").strip() seen_ip = "" mid = str(p.managed_ne_id or "").strip() ph_ne = db.get(ManagedNE, mid) if mid else None if ph_ne is not None: seen_ip = str(ph_ne.source_ref or "").strip() if ip and ip in real_by_ip: canon = real_by_ip[ip] elif seen_ip and seen_ip in real_by_ip: canon = real_by_ip[seen_ip] elif nk and nk in real_by_name: canon = real_by_name[nk] if canon is None or canon.id == p.id: continue _absorb(canon, [p]) db.flush() # Drop placeholder ManagedNE if nothing else references it. if ph_ne is not None and is_placeholder_ne_source(ph_ne.source): still = ( db.query(TopoFabricNode) .filter(TopoFabricNode.managed_ne_id == ph_ne.id) .count() ) if still == 0: db.delete(ph_ne) placeholders_removed += 1 # 3) WebCRT session hosts sharing an IP with a real inventory fabric node. db.flush() nodes = db.query(TopoFabricNode).all() by_ip: dict[str, list[TopoFabricNode]] = {} for n in nodes: ip = str(n.ip or "").strip() if ip: by_ip.setdefault(ip, []).append(n) for group in by_ip.values(): if len(group) < 2: continue real = [n for n in group if _fabric_match_score(db, n) >= 3] webcrtish = [n for n in group if _fabric_match_score(db, n) == 1] if not real or not webcrtish: continue canon = max(real, key=lambda n: _fabric_match_score(db, n)) _absorb(canon, webcrtish) if merged or placeholders_removed: db.commit() refresh_fabric_stats(db) return {"merged": merged, "placeholders_removed": placeholders_removed} def delete_fabric_edges(db: Session, edge_ids: list[str]) -> dict[str, int]: """Hard-delete fabric edges and any per-view edge styles.""" ids = [str(x).strip() for x in (edge_ids or []) if str(x).strip()] # De-dupe while preserving order. seen: set[str] = set() ordered: list[str] = [] for i in ids: if i in seen: continue seen.add(i) ordered.append(i) if not ordered: raise HTTPException(status_code=400, detail="edge_ids_required") rows = db.query(TopoFabricEdge).filter(TopoFabricEdge.id.in_(ordered)).all() found_ids = [str(r.id) for r in rows] if not found_ids: raise HTTPException(status_code=404, detail="fabric_edge_not_found") db.query(TopoViewEdgeStyle).filter( TopoViewEdgeStyle.fabric_edge_id.in_(found_ids) ).delete(synchronize_session=False) db.query(TopoFabricEdge).filter(TopoFabricEdge.id.in_(found_ids)).delete( synchronize_session=False ) db.commit() try: refresh_fabric_stats(db) except Exception: # noqa: BLE001 pass return {"deleted": len(found_ids)} def delete_fabric_edge(db: Session, edge_id: str) -> dict[str, int]: return delete_fabric_edges(db, [edge_id])