Add UME TopoNodes/TopologicalLinks daily sync into local tables.

Phase-1 docks both topology APIs without writing Fabric; expose a 24h topology_auto_sync task and domain=topology manual sync for later LLDP alignment.

Co-authored-by: Cursor <cursoragent@cursor.com>

netx_api/ume_sync_topology.py

@@ -0,0 +1,263 @@
+"""UME TopoNodes + TopologicalLinks sync (phase-1: local tables only)."""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from .models import UmeSyncJob, UmeTopoLink, UmeTopoNode
+from .ume_client import UMEClient
+from .ume_raw import dumps_ume_raw
+from .ume_sync_common import _pick, _s, _utc_now_naive
+from .ume_sync_pull import _build_sync_job
+
+_sync_log = logging.getLogger("netx.ume.sync")
+
+_UUID_RE = re.compile(
+    r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+)
+_ME_BRACE_RE = re.compile(
+    r"ME\{([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\}",
+    re.IGNORECASE,
+)
+_PTP_RE = re.compile(r"PTP=\{([^}]*)\}", re.IGNORECASE)
+
+
+def extract_me_uuid(text: str) -> str:
+    """Extract managed-element uuid from TP ref or TOPO_NODE_ME* name."""
+    s = str(text or "").strip()
+    if not s:
+        return ""
+    m = _ME_BRACE_RE.search(s)
+    if m:
+        return m.group(1)
+    # TOPO_NODE_ME<uuid> (no braces) or trailing uuid
+    low = s.upper()
+    if "TOPO_NODE_ME" in low:
+        idx = low.find("TOPO_NODE_ME")
+        rest = s[idx + len("TOPO_NODE_ME") :]
+        m2 = _UUID_RE.search(rest)
+        if m2:
+            return m2.group(0)
+    m3 = _UUID_RE.search(s)
+    return m3.group(0) if m3 else ""
+
+
+def extract_ptp(text: str) -> str:
+    s = str(text or "").strip()
+    if not s:
+        return ""
+    m = _PTP_RE.search(s)
+    return (m.group(1) if m else "")[:256]
+
+
+def _first_tp_ref(value: Any) -> str:
+    if isinstance(value, list):
+        for item in value:
+            s = _s(item)
+            if s:
+                return s
+        return ""
+    return _s(value)
+
+
+def _as_optional_int(value: Any) -> int | None:
+    if value is None or value == "":
+        return None
+    try:
+        return int(value)
+    except Exception:
+        return None
+
+
+def _link_id_from_row(row: dict[str, Any]) -> str:
+    lid = _s(_pick(row, "linkId", "link-id", "link_id", "id"))
+    if lid:
+        return lid[:128]
+    name = _s(_pick(row, "name"))
+    if not name:
+        return ""
+    digest = hashlib.sha1(name.encode("utf-8", errors="replace")).hexdigest()[:40]
+    return f"name:{digest}"[:128]
+
+
+def _node_id_from_row(row: dict[str, Any]) -> str:
+    nid = _s(_pick(row, "nodeId", "node-id", "node_id", "id"))
+    if nid:
+        return nid[:128]
+    name = _s(_pick(row, "name"))
+    if not name:
+        return ""
+    digest = hashlib.sha1(name.encode("utf-8", errors="replace")).hexdigest()[:40]
+    return f"name:{digest}"[:128]
+
+
+def _ume_ne_id_for_topo_node(*, node_type: str, name: str) -> str:
+    nt = str(node_type or "").strip().upper()
+    if nt != "TOPO_NODE_ME":
+        return ""
+    return extract_me_uuid(name)
+
+
+def sync_topology_full(db: Session, client: UMEClient, *, trigger_mode: str = "manual") -> UmeSyncJob:
+    """Pull TopoNodes + TopologicalLinks into local tables (no Fabric apply)."""
+    job = _build_sync_job("topology", trigger_mode)
+    db.add(job)
+    db.flush()
+    db.commit()
+    _sync_log.info("topology sync job %s committed as running (trigger=%s)", getattr(job, "id", "?"), trigger_mode)
+
+    pulled = inserted = updated = 0
+    nodes_pulled = nodes_ins = nodes_upd = nodes_del = 0
+    links_pulled = links_ins = links_upd = links_del = 0
+    nodes_skip = links_skip = 0
+
+    try:
+        now = _utc_now_naive()
+
+        node_rows, node_diag = client.get_topo_nodes()
+        nodes_pulled = len(node_rows)
+        seen_nodes: set[str] = set()
+        for row in node_rows:
+            if not isinstance(row, dict):
+                nodes_skip += 1
+                continue
+            node_id = _node_id_from_row(row)
+            if not node_id:
+                nodes_skip += 1
+                continue
+            seen_nodes.add(node_id)
+            name = _s(_pick(row, "name"))
+            node_type = _s(_pick(row, "nodeType", "node-type", "node_type"))
+            existing = db.get(UmeTopoNode, node_id)
+            if existing is None:
+                existing = UmeTopoNode(node_id=node_id, first_seen_at=now)
+                db.add(existing)
+                nodes_ins += 1
+            else:
+                nodes_upd += 1
+            existing.name = name[:512]
+            existing.node_type = node_type[:64]
+            existing.user_label = _s(_pick(row, "userLabel", "user-label", "user_label"))[:512]
+            existing.owner = _s(_pick(row, "owner"))[:64]
+            existing.parent_node = _s(_pick(row, "parentNode", "parent-node", "parent_node"))[:512]
+            existing.x_pos = _as_optional_int(_pick(row, "xPos", "x-pos", "x_pos"))
+            existing.y_pos = _as_optional_int(_pick(row, "yPos", "y-pos", "y_pos"))
+            existing.ume_ne_id = _ume_ne_id_for_topo_node(node_type=node_type, name=name)[:128]
+            existing.last_seen_at = now
+            existing.raw_json = dumps_ume_raw(row)
+
+        db.flush()
+        if seen_nodes:
+            nodes_del = int(
+                db.query(UmeTopoNode)
+                .filter(~UmeTopoNode.node_id.in_(list(seen_nodes)))
+                .delete(synchronize_session=False)
+            )
+        else:
+            # Successful empty snapshot → clear local table.
+            nodes_del = int(db.query(UmeTopoNode).delete(synchronize_session=False))
+
+        link_rows, link_diag = client.get_topological_links()
+        links_pulled = len(link_rows)
+        seen_links: set[str] = set()
+        for row in link_rows:
+            if not isinstance(row, dict):
+                links_skip += 1
+                continue
+            link_id = _link_id_from_row(row)
+            if not link_id:
+                links_skip += 1
+                continue
+            seen_links.add(link_id)
+            a_ref = _first_tp_ref(_pick(row, "aEndTpRefList", "a-end-tp-ref-list", "aEndTpRef"))
+            z_ref = _first_tp_ref(_pick(row, "zEndTpRefList", "z-end-tp-ref-list", "zEndTpRef"))
+            existing = db.get(UmeTopoLink, link_id)
+            if existing is None:
+                existing = UmeTopoLink(link_id=link_id, first_seen_at=now)
+                db.add(existing)
+                links_ins += 1
+            else:
+                links_upd += 1
+            existing.name = _s(_pick(row, "name"))[:1024]
+            existing.user_label = _s(_pick(row, "userLabel", "user-label", "user_label"))
+            existing.owner = _s(_pick(row, "owner"))[:64]
+            existing.direction = _s(_pick(row, "direction"))[:32]
+            existing.layer_rate = _as_optional_int(_pick(row, "layerRate", "layer-rate", "layer_rate"))
+            existing.connection_status = _s(
+                _pick(row, "connection-status", "connectionStatus", "connection_status")
+            )[:64]
+            existing.a_end_tp_ref = a_ref
+            existing.z_end_tp_ref = z_ref
+            existing.a_ume_ne_id = extract_me_uuid(a_ref)[:128]
+            existing.z_ume_ne_id = extract_me_uuid(z_ref)[:128]
+            existing.a_ptp = extract_ptp(a_ref)
+            existing.z_ptp = extract_ptp(z_ref)
+            existing.last_seen_at = now
+            existing.raw_json = dumps_ume_raw(row)
+
+        db.flush()
+        if seen_links:
+            links_del = int(
+                db.query(UmeTopoLink)
+                .filter(~UmeTopoLink.link_id.in_(list(seen_links)))
+                .delete(synchronize_session=False)
+            )
+        else:
+            links_del = int(db.query(UmeTopoLink).delete(synchronize_session=False))
+
+        pulled = nodes_pulled + links_pulled
+        inserted = nodes_ins + links_ins
+        updated = nodes_upd + links_upd
+
+        job.details_json = json.dumps(
+            {
+                "nodes": {
+                    "pulled": nodes_pulled,
+                    "inserted": nodes_ins,
+                    "updated": nodes_upd,
+                    "deleted": nodes_del,
+                    "skipped": nodes_skip,
+                    "latency_ms": int(getattr(node_diag, "latency_ms", 0) or 0),
+                },
+                "links": {
+                    "pulled": links_pulled,
+                    "inserted": links_ins,
+                    "updated": links_upd,
+                    "deleted": links_del,
+                    "skipped": links_skip,
+                    "latency_ms": int(getattr(link_diag, "latency_ms", 0) or 0),
+                },
+                "deleted_topo_nodes": nodes_del,
+                "deleted_topo_links": links_del,
+            },
+            ensure_ascii=False,
+        )
+        job.status = "done"
+        _sync_log.info(
+            "topology sync done nodes=%s/%s/%s links=%s/%s/%s deleted_n=%s deleted_l=%s",
+            nodes_pulled,
+            nodes_ins,
+            nodes_upd,
+            links_pulled,
+            links_ins,
+            links_upd,
+            nodes_del,
+            links_del,
+        )
+    except Exception as exc:
+        job.status = "failed"
+        job.error_message = str(exc)[:1024]
+        _sync_log.exception("topology sync failed: %s", exc)
+    finally:
+        job.pulled_count = int(pulled)
+        job.inserted_count = int(inserted)
+        job.updated_count = int(updated)
+        job.ended_at = _utc_now_naive()
+        db.commit()
+        db.refresh(job)
+    return job