Heal UME dock-to-Fabric gaps: partial jobs, startup apply, and one-click UI.

Co-authored-by: Cursor <cursoragent@cursor.com>

netx_api/ume_runtime.py

@@ -326,6 +326,22 @@ def start_api_sideband_threads() -> None:
                         try:
                             fail_stale_topology_running_jobs(db)
                             client = ume_support._ume_client()
+                            # Empty inventory makes Fabric naming/IP weak — pull inventory first.
+                            try:
+                                from sqlalchemy import func
+
+                                from .models import UmeInventoryNE
+
+                                inv_n = int(db.query(func.count(UmeInventoryNE.ne_id)).scalar() or 0)
+                                if inv_n <= 0:
+                                    _schedule_log.info(
+                                        "topology_auto_sync: inventory empty — syncing inventory first"
+                                    )
+                                    sync_inventory_full(db, client, trigger_mode="schedule")
+                            except Exception:
+                                _schedule_log.exception(
+                                    "topology_auto_sync: pre-inventory sync failed (continuing topology)"
+                                )
                             sync_topology_full(db, client, trigger_mode="schedule")
                             _schedule_log.info("topology_auto_sync: sync finished ok")
                             ume_support._set_runtime_task(
@@ -445,5 +461,37 @@ def start_api_sideband_threads() -> None:
             last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
         )
 
+    # Dock already has topology but Fabric/World empty (or last apply partial): heal without waiting 24h.
+    try:
+
+        def _startup_ume_fabric_apply() -> None:
+            time.sleep(3)
+            db = SessionLocal()
+            try:
+                from .ume_topology_apply import apply_ume_dock_to_fabric_if_needed
+
+                out = apply_ume_dock_to_fabric_if_needed(db, reason="startup")
+                if out:
+                    _schedule_log.info(
+                        "startup: ume dock→fabric apply ok nodes_ensured=%s",
+                        (out.get("fabric_apply") or {}).get("nodes_ensured"),
+                    )
+                else:
+                    _schedule_log.info("startup: ume dock→fabric apply skipped (no gap)")
+            except Exception as exc:
+                _schedule_log.exception("startup: ume dock→fabric apply failed: %s", exc)
+            finally:
+                db.close()
+
+        t_apply = threading.Thread(
+            target=_startup_ume_fabric_apply,
+            name="ume-fabric-apply-startup",
+            daemon=True,
+        )
+        t_apply.start()
+        _schedule_log.info("started thread %s alive=%s", t_apply.name, t_apply.is_alive())
+    except Exception as exc:
+        _schedule_log.exception("startup: ume fabric apply thread init failed: %s", exc)
+
 
 

netx_api/ume_support.py

@@ -394,7 +394,7 @@ def _last_finished_job_ended_at(db: Session, domain: str) -> datetime | None:
         .filter(
             UmeSyncJob.domain == domain,
             UmeSyncJob.ended_at.isnot(None),
-            UmeSyncJob.status.in_(("done", "failed")),
+            UmeSyncJob.status.in_(("done", "failed", "partial")),
         )
         .order_by(UmeSyncJob.ended_at.desc())
         .limit(50)

netx_api/ume_sync_router.py

@@ -204,6 +204,16 @@ def ume_sync_status(
         items.append(item)
         if item["domain"] and item["domain"] not in latest_by_domain:
             latest_by_domain[item["domain"]] = item
+
+    topology_fabric: dict[str, Any] = {}
+    try:
+        from .ume_topology_apply import ume_topology_apply_gap
+
+        topology_fabric = ume_topology_apply_gap(db)
+    except Exception:
+        _log.exception("ume sync status: topology_fabric gap failed")
+        topology_fabric = {"needs_apply": False, "error": "gap_unavailable"}
+
     return {
         "total": total,
         "page": page,
@@ -212,6 +222,7 @@ def ume_sync_status(
         "latest_by_domain": latest_by_domain,
         "runtime_tasks": _list_runtime_tasks(),
         "alarm_subscription": get_subscription_status(),
+        "topology_fabric": topology_fabric,
     }
 
 

netx_api/ume_sync_topology.py

@@ -530,6 +530,8 @@ def sync_topology_full(db: Session, client: UMEClient, *, trigger_mode: str = "m
                 work.commit()
 
                 apply_stats: dict[str, Any] = {}
+                apply_ok = False
+                apply_err = ""
                 try:
                     from .ume_topology_apply import apply_ume_topology_to_fabric
                     from .ume_topology_world import ensure_ume_world_and_sbn_folders
@@ -540,17 +542,24 @@ def sync_topology_full(db: Session, client: UMEClient, *, trigger_mode: str = "m
                         apply_stats = apply_ume_topology_to_fabric(apply_db)
                         world_stats = ensure_ume_world_and_sbn_folders(apply_db)
                         apply_stats = {**apply_stats, "world": world_stats}
+                        apply_ok = True
                     finally:
                         try:
                             apply_db.close()
                         except Exception:
                             pass
-                except Exception:
+                except Exception as apply_exc:
+                    apply_err = str(apply_exc)[:500]
+                    apply_stats = {"error": apply_err, "ok": False}
                     _sync_log.exception(
                         "topology sync job %s: fabric apply failed (dock sync kept)",
                         job_id,
                     )
 
+                if not apply_ok:
+                    status = "partial"
+                    err = f"dock_ok_fabric_apply_failed:{apply_err}"[:1024]
+
                 pulled = nodes_pulled + links_pulled
                 inserted = nodes_ins + links_ins
                 updated = nodes_upd + links_upd
@@ -577,12 +586,14 @@ def sync_topology_full(db: Session, client: UMEClient, *, trigger_mode: str = "m
                         "deleted_topo_nodes": nodes_del,
                         "deleted_topo_links": links_del,
                         "fabric_apply": apply_stats,
+                        "fabric_apply_ok": apply_ok,
                     },
                     ensure_ascii=False,
                 )
                 _sync_log.info(
-                    "topology sync done job=%s nodes=%s/%s/%s links=%s/%s/%s deleted_n=%s deleted_l=%s",
+                    "topology sync done job=%s status=%s nodes=%s/%s/%s links=%s/%s/%s deleted_n=%s deleted_l=%s apply_ok=%s",
                     job_id,
+                    status,
                     nodes_pulled,
                     nodes_ins,
                     nodes_upd,
@@ -591,6 +602,7 @@ def sync_topology_full(db: Session, client: UMEClient, *, trigger_mode: str = "m
                     links_upd,
                     nodes_del,
                     links_del,
+                    apply_ok,
                 )
             finally:
                 try:

netx_api/ume_topology_apply.py

@@ -248,6 +248,81 @@ def apply_ume_topology_to_fabric(db: Session) -> dict[str, Any]:
     return stats
 
 
+def ume_topology_apply_gap(db: Session) -> dict[str, Any]:
+    """Detect dock→Fabric/World gap for startup heal and UI hints."""
+    from sqlalchemy import func
+
+    from .models import TopoFabricNode, TopoFolder, UmeSyncJob, UmeTopoNode
+    from .ume_topology_world import get_world_container_folder
+
+    dock_me = int(
+        db.query(func.count(UmeTopoNode.node_id))
+        .filter(UmeTopoNode.node_type == "TOPO_NODE_ME")
+        .scalar()
+        or 0
+    )
+    fabric_ume = int(
+        db.query(func.count(TopoFabricNode.id))
+        .filter(TopoFabricNode.ume_ne_id.isnot(None), TopoFabricNode.ume_ne_id != "")
+        .scalar()
+        or 0
+    )
+    world = get_world_container_folder(db)
+    world_exists = world is not None
+    latest = (
+        db.query(UmeSyncJob)
+        .filter(UmeSyncJob.domain == "topology", UmeSyncJob.ended_at.isnot(None))
+        .order_by(UmeSyncJob.ended_at.desc())
+        .first()
+    )
+    latest_status = str(getattr(latest, "status", "") or "")
+    latest_error = str(getattr(latest, "error_message", "") or "")
+    partial = latest_status == "partial" or latest_error.startswith("dock_ok_fabric_apply_failed")
+    needs_apply = bool(dock_me > 0 and (fabric_ume == 0 or not world_exists or partial))
+    return {
+        "dock_me_count": dock_me,
+        "fabric_ume_count": fabric_ume,
+        "world_exists": world_exists,
+        "world_folder_id": str(getattr(world, "id", "") or ""),
+        "latest_topology_status": latest_status,
+        "latest_topology_error": latest_error[:240],
+        "partial_apply": partial,
+        "needs_apply": needs_apply,
+    }
+
+
+def apply_ume_dock_to_fabric_if_needed(
+    db: Session, *, reason: str = "manual", force: bool = False
+) -> dict[str, Any] | None:
+    """Apply dock tables into Fabric/World when gap detected (or force)."""
+    from .ume_topology_world import ensure_ume_world_and_sbn_folders
+
+    gap = ume_topology_apply_gap(db)
+    if not force and not gap.get("needs_apply"):
+        return None
+    _log.info(
+        "ume dock→fabric apply start reason=%s force=%s gap=%s",
+        reason,
+        force,
+        {k: gap.get(k) for k in ("dock_me_count", "fabric_ume_count", "world_exists", "partial_apply")},
+    )
+    apply_stats = apply_ume_topology_to_fabric(db)
+    world_stats = ensure_ume_world_and_sbn_folders(db)
+    out = {
+        "ok": True,
+        "reason": reason,
+        "before": gap,
+        "fabric_apply": apply_stats,
+        "world": world_stats,
+    }
+    _log.info(
+        "ume dock→fabric apply done reason=%s nodes_ensured=%s",
+        reason,
+        (apply_stats or {}).get("nodes_ensured"),
+    )
+    return out
+
+
 def _clear_and_keep(attrs: dict[str, Any]) -> dict[str, Any]:
     from .topology_common import _clear_miss_attrs