hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-09 02:00:46 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 2

Defer canvas edge deletes to Save and cascade orphan Fabric on NE detach.

Browse source 
Queue fabric edge deletes until Save with undo-safe pending state; hard-delete topology/LLDP placeholders from the canvas; purge fully orphaned fabric nodes after inventory detach, with a periodic reconcile GC.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-08-05 22:43:03 +08:00
parent 2aafb72676
commit 6e2c8517cf
16 changed files with 732 additions and 166 deletions
Download patch file Download diff file Expand all files Collapse all files

7
netx_api/app_shutdown.py
View file

@ -38,6 +38,13 @@ def shutdown_runtime(*, reason: str = "lifespan") -> None:
except Exception: # noqa: BLE001
_log.exception("stop_port_traffic_scheduler failed")
try:
from .fabric_reconcile_scheduler import stop_fabric_reconcile_scheduler
stop_fabric_reconcile_scheduler()
except Exception: # noqa: BLE001
_log.exception("stop_fabric_reconcile_scheduler failed")
try:
import netx_api.ume_support as ume_support

6
netx_api/config.py
View file

@ -86,6 +86,10 @@ class Settings(BaseSettings):
# Reclaim hung discover jobs (updated_at / created_at older than these).
lldp_collect_stale_run_sec: int = 7200
lldp_collect_pending_stale_sec: int = 300
# Fabric inventory lifecycle: detach dangling refs + purge fully orphaned nodes.
fabric_reconcile_scheduler_enabled: bool = True
# Default 6h — light full-table GC; deletes still purge orphans immediately on detach.
fabric_reconcile_interval_sec: int = 21600
# Port traffic monitoring (CLI rate bit/s samples)
port_traffic_scheduler_enabled: bool = True
port_traffic_scheduler_tick_sec: int = 15
@ -141,7 +145,7 @@ class Settings(BaseSettings):
alembic_upgrade_on_start: bool = True
# Optional dedicated SQLAlchemy URL for /v1/sql/* (read-only DB role recommended).
sql_readonly_database_url: str = ""
# When true (default), API also runs config_sync / lldp / port_traffic schedulers.
# When true (default), API also runs config_sync / lldp / port_traffic / fabric_reconcile schedulers.
# Production split: set false and run `python -m netx_api.worker` beside the API.
run_inline_schedulers: bool = True
# Worker→API heartbeat file (used when run_inline_schedulers=false).

99
netx_api/fabric_reconcile_scheduler.py Normal file
View file

@ -0,0 +1,99 @@
"""Periodic Fabric inventory reconcile: dangling detach + fully-orphaned GC."""
from __future__ import annotations
import logging
import threading
import time
from typing import Any
from .config import settings
from .db import SessionLocal
_log = logging.getLogger("netx.fabric_reconcile.scheduler")
_stop = threading.Event()
_thread: threading.Thread | None = None
_last_run_mono: float = 0.0
_last_stats: dict[str, Any] | None = None
def run_fabric_reconcile_once() -> dict[str, Any]:
"""Detach dangling managed/UME refs and purge fully orphaned fabric nodes."""
global _last_run_mono, _last_stats
from .topology_inventory_lifecycle import reconcile_dangling_fabric_links
db = SessionLocal()
try:
stats = reconcile_dangling_fabric_links(db, sweep_orphans=True)
db.commit()
_last_run_mono = time.monotonic()
_last_stats = dict(stats)
purged = int(stats.get("purged_orphans") or 0)
dangling = int(stats.get("dangling_managed_refs") or 0) + int(
stats.get("dangling_ume_refs") or 0
)
if purged or dangling:
_log.info(
"fabric reconcile done dangling_m=%s dangling_u=%s purged_orphans=%s edges_deleted=%s",
stats.get("dangling_managed_refs"),
stats.get("dangling_ume_refs"),
purged,
stats.get("edges_deleted"),
)
else:
_log.debug("fabric reconcile idle")
return stats
except Exception:
db.rollback()
raise
finally:
db.close()
def _loop() -> None:
interval = max(300, int(getattr(settings, "fabric_reconcile_interval_sec", 21600) or 21600))
# Short first delay so API/worker boot is not blocked by a full-table scan.
startup_delay = min(120, interval)
_log.info(
"fabric reconcile scheduler started interval=%ss startup_delay=%ss",
interval,
startup_delay,
)
_stop.wait(startup_delay)
while not _stop.is_set():
try:
if bool(getattr(settings, "fabric_reconcile_scheduler_enabled", True)):
run_fabric_reconcile_once()
except Exception:
_log.exception("fabric reconcile tick failed")
_stop.wait(interval)
_log.info("fabric reconcile scheduler stopped")
def start_fabric_reconcile_scheduler() -> None:
global _thread
if not bool(getattr(settings, "fabric_reconcile_scheduler_enabled", True)):
_log.info("fabric reconcile scheduler disabled by settings")
return
if _thread and _thread.is_alive():
return
_stop.clear()
_thread = threading.Thread(target=_loop, name="fabric-reconcile-scheduler", daemon=True)
_thread.start()
def stop_fabric_reconcile_scheduler() -> None:
_stop.set()
def fabric_reconcile_scheduler_status() -> dict[str, Any]:
now = time.monotonic()
return {
"running": bool(_thread is not None and _thread.is_alive()),
"last_run_age_sec": (now - _last_run_mono) if _last_run_mono else None,
"last_stats": dict(_last_stats) if _last_stats else None,
"interval_sec": max(
300, int(getattr(settings, "fabric_reconcile_interval_sec", 21600) or 21600)
),
"enabled": bool(getattr(settings, "fabric_reconcile_scheduler_enabled", True)),
}

11
netx_api/metrics_router.py
View file

@ -110,12 +110,19 @@ def _prom_lines(metrics: dict[str, Any]) -> str:
("config_sync", "netx_config_sync_scheduler_running"),
("lldp_collect", "netx_lldp_collect_scheduler_running"),
("port_traffic", "netx_port_traffic_scheduler_running"),
("fabric_reconcile", "netx_fabric_reconcile_scheduler_running"),
):
block = sched.get(name) or metrics.get(name) or {}
if "running" in block:
lines.append(f'{key} {1 if block.get("running") else 0}')
if block.get("last_tick_age_sec") is not None:
lines.append(f"netx_{name}_tick_age_seconds {block['last_tick_age_sec']}")
age_key = "last_run_age_sec" if name == "fabric_reconcile" else "last_tick_age_sec"
if block.get(age_key) is not None:
metric_age = (
"netx_fabric_reconcile_run_age_seconds"
if name == "fabric_reconcile"
else f"netx_{name}_tick_age_seconds"
)
lines.append(f"{metric_age} {block[age_key]}")
pt = metrics.get("port_traffic") or {}
if pt.get("last_purge_age_sec") is not None:
lines.append(f'netx_port_traffic_purge_age_seconds {pt["last_purge_age_sec"]}')

6
netx_api/scheduler_heartbeat.py
View file

@ -61,6 +61,12 @@ def local_device_scheduler_status(*, role: str = "unknown") -> dict[str, Any]:
out["port_traffic"] = port_traffic_scheduler_status()
except Exception: # noqa: BLE001
out["port_traffic"] = {"running": False, "error": "unavailable"}
try:
from .fabric_reconcile_scheduler import fabric_reconcile_scheduler_status
out["fabric_reconcile"] = fabric_reconcile_scheduler_status()
except Exception: # noqa: BLE001
out["fabric_reconcile"] = {"running": False, "error": "unavailable"}
return out

204
netx_api/topology_inventory_lifecycle.py
View file

@ -115,39 +115,105 @@ def _strip_membership_ids(
def detach_fabric_from_managed(db: Session, managed_ne_ids: list[str]) -> dict[str, int]:
"""Clear fabric.managed_ne_id for deleted ManagedNEs; keep nodes/placements/edges."""
"""Clear fabric.managed_ne_id for deleted ManagedNEs; purge nodes that become fully orphaned."""
ids = _norm_ids(managed_ne_ids)
if not ids:
return {"detached_nodes": 0, "membership_views": 0}
return {
"detached_nodes": 0,
"membership_views": 0,
"purged_orphans": 0,
"edges_deleted": 0,
"placements_deleted": 0,
}
now = _utcnow()
rows = (
db.query(TopoFabricNode)
.filter(TopoFabricNode.managed_ne_id.in_(ids))
.all()
)
fabric_ids = [str(r.id) for r in rows]
for row in rows:
row.managed_ne_id = None
row.updated_at = now
views = _strip_membership_ids(db, managed_ne_ids=set(ids))
return {"detached_nodes": len(rows), "membership_views": views}
db.flush()
purged = purge_fully_orphaned_fabric_nodes(db, fabric_ids)
return {
"detached_nodes": len(rows),
"membership_views": views,
"purged_orphans": int(purged.get("deleted") or 0),
"edges_deleted": int(purged.get("edges_deleted") or 0),
"placements_deleted": int(purged.get("placements_deleted") or 0),
}
def detach_fabric_from_ume(db: Session, ume_ne_ids: list[str]) -> dict[str, int]:
"""Clear fabric.ume_ne_id for deleted UME inventory rows; keep topology traces."""
"""Clear fabric.ume_ne_id for deleted UME inventory rows; purge nodes that become fully orphaned."""
ids = _norm_ids(ume_ne_ids)
if not ids:
return {"detached_nodes": 0, "membership_views": 0}
return {
"detached_nodes": 0,
"membership_views": 0,
"purged_orphans": 0,
"edges_deleted": 0,
"placements_deleted": 0,
}
now = _utcnow()
rows = db.query(TopoFabricNode).filter(TopoFabricNode.ume_ne_id.in_(ids)).all()
fabric_ids = [str(r.id) for r in rows]
for row in rows:
row.ume_ne_id = None
row.updated_at = now
views = _strip_membership_ids(db, ume_ne_ids=set(ids))
return {"detached_nodes": len(rows), "membership_views": views}
db.flush()
purged = purge_fully_orphaned_fabric_nodes(db, fabric_ids)
return {
"detached_nodes": len(rows),
"membership_views": views,
"purged_orphans": int(purged.get("deleted") or 0),
"edges_deleted": int(purged.get("edges_deleted") or 0),
"placements_deleted": int(purged.get("placements_deleted") or 0),
}
def reconcile_dangling_fabric_links(db: Session) -> dict[str, int]:
"""Unbind fabric ids that no longer exist in managed_ne / ume_inventory_ne."""
def purge_fully_orphaned_fabric_nodes(
db: Session,
fabric_node_ids: list[str] | None = None,
) -> dict[str, int]:
"""Hard-delete fabric nodes with no managed_ne_id and no ume_ne_id (cascades edges/placements).
When ``fabric_node_ids`` is None, sweeps the whole fabric table (GC).
Caller commits.
"""
if fabric_node_ids is None:
rows = [
n
for n in db.query(TopoFabricNode).all()
if not str(n.managed_ne_id or "").strip() and not str(n.ume_ne_id or "").strip()
]
else:
ids = _norm_ids(fabric_node_ids)
if not ids:
return {"deleted": 0, "edges_deleted": 0, "placements_deleted": 0}
candidates = db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(ids)).all()
rows = [
n
for n in candidates
if not str(n.managed_ne_id or "").strip() and not str(n.ume_ne_id or "").strip()
]
return _delete_fabric_node_rows(db, rows)
def reconcile_dangling_fabric_links(
db: Session,
*,
sweep_orphans: bool = True,
) -> dict[str, int]:
"""Unbind fabric ids that no longer exist in managed_ne / ume_inventory_ne.
Detach paths purge nodes that become fully orphaned. When ``sweep_orphans`` is True
(default), also GC any remaining fully orphaned fabric nodes.
"""
managed_alive = {str(x[0]) for x in db.query(ManagedNE.id).all() if str(x[0] or "").strip()}
ume_alive = {
str(x[0]) for x in db.query(UmeInventoryNE.ne_id).all() if str(x[0] or "").strip()
@ -163,6 +229,21 @@ def reconcile_dangling_fabric_links(db: Session) -> dict[str, int]:
dangling_u.append(uid)
m_stats = detach_fabric_from_managed(db, dangling_m)
u_stats = detach_fabric_from_ume(db, dangling_u)
gc = (
purge_fully_orphaned_fabric_nodes(db, None)
if sweep_orphans
else {"deleted": 0, "edges_deleted": 0, "placements_deleted": 0}
)
purged = (
int(m_stats.get("purged_orphans") or 0)
+ int(u_stats.get("purged_orphans") or 0)
+ int(gc.get("deleted") or 0)
)
edges_deleted = (
int(m_stats.get("edges_deleted") or 0)
+ int(u_stats.get("edges_deleted") or 0)
+ int(gc.get("edges_deleted") or 0)
)
return {
"dangling_managed_refs": len(set(dangling_m)),
"dangling_ume_refs": len(set(dangling_u)),
@ -170,6 +251,9 @@ def reconcile_dangling_fabric_links(db: Session) -> dict[str, int]:
"detached_ume_nodes": int(u_stats.get("detached_nodes") or 0),
"membership_views": int(m_stats.get("membership_views") or 0)
+ int(u_stats.get("membership_views") or 0),
"purged_orphans": purged,
"edges_deleted": edges_deleted,
"gc_orphans": int(gc.get("deleted") or 0),
}
@ -218,23 +302,11 @@ def _strip_fabric_ids_from_membership(db: Session, fabric_ids: set[str]) -> int:
return touched
def delete_fabric_nodes(db: Session, fabric_node_ids: list[str]) -> dict[str, int]:
"""Hard-delete fabric nodes (placements + edges). Does not touch managed/UME inventory."""
ids = _norm_ids(fabric_node_ids)
def _delete_fabric_node_rows(db: Session, rows: list[TopoFabricNode]) -> dict[str, int]:
"""Delete fabric rows + incident edges + placements. Caller commits."""
ids = [str(r.id) for r in rows]
if not ids:
raise HTTPException(status_code=400, detail="fabric_node_ids_required")
rows = db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(ids)).all()
found = {str(r.id): r for r in rows}
missing = [i for i in ids if i not in found]
if missing:
raise HTTPException(status_code=404, detail=f"fabric_node_not_found:{missing[0]}")
blocked = [i for i, r in found.items() if not fabric_node_is_deletable(db, r)]
if blocked:
raise HTTPException(
status_code=400,
detail=f"fabric_node_not_deletable:{blocked[0]}",
)
return {"deleted": 0, "edges_deleted": 0, "placements_deleted": 0}
edge_ids = [
str(e.id)
for e in db.query(TopoFabricEdge)
@ -258,6 +330,84 @@ def delete_fabric_nodes(db: Session, fabric_node_ids: list[str]) -> dict[str, in
_strip_fabric_ids_from_membership(db, set(ids))
for r in rows:
db.delete(r)
return {
"deleted": len(rows),
"edges_deleted": len(edge_ids),
"placements_deleted": int(placements or 0),
}
def delete_fabric_nodes(db: Session, fabric_node_ids: list[str]) -> dict[str, int]:
"""Hard-delete fabric nodes (placements + edges). Does not touch managed/UME inventory."""
ids = _norm_ids(fabric_node_ids)
if not ids:
raise HTTPException(status_code=400, detail="fabric_node_ids_required")
rows = db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(ids)).all()
found = {str(r.id): r for r in rows}
missing = [i for i in ids if i not in found]
if missing:
raise HTTPException(status_code=404, detail=f"fabric_node_not_found:{missing[0]}")
blocked = [i for i, r in found.items() if not fabric_node_is_deletable(db, r)]
if blocked:
raise HTTPException(
status_code=400,
detail=f"fabric_node_not_deletable:{blocked[0]}",
)
out = _delete_fabric_node_rows(db, rows)
db.commit()
try:
from .topology_service import refresh_fabric_stats
refresh_fabric_stats(db)
except Exception: # noqa: BLE001
pass
return out
def purge_placeholder_fabric_nodes(db: Session, fabric_node_ids: list[str]) -> dict[str, int]:
"""Hard-delete LLDP/topology placeholders: ManagedNE + fabric node + edges + placements.
Rejects real inventory (manual / ume_sync), WebCRT sessions, and UME-only fabric rows.
"""
ids = _norm_ids(fabric_node_ids)
if not ids:
raise HTTPException(status_code=400, detail="fabric_node_ids_required")
rows = db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(ids)).all()
found = {str(r.id): r for r in rows}
missing = [i for i in ids if i not in found]
if missing:
raise HTTPException(status_code=404, detail=f"fabric_node_not_found:{missing[0]}")
managed_rows: list[ManagedNE] = []
managed_ids: set[str] = set()
for fid, fab in found.items():
mid = str(fab.managed_ne_id or "").strip()
uid = str(fab.ume_ne_id or "").strip()
if not mid:
if uid:
raise HTTPException(
status_code=400,
detail=f"fabric_node_not_placeholder:{fid}",
)
# Orphan fabric — allow purge (same as fabric delete).
continue
mrow = db.get(ManagedNE, mid)
if mrow is None:
continue
if not is_placeholder_ne_source(mrow.source):
raise HTTPException(
status_code=400,
detail=f"fabric_node_not_placeholder:{fid}",
)
if mid not in managed_ids:
managed_ids.add(mid)
managed_rows.append(mrow)
out = _delete_fabric_node_rows(db, rows)
for mrow in managed_rows:
db.delete(mrow)
membership_views = _strip_membership_ids(db, managed_ne_ids=managed_ids)
db.commit()
try:
from .topology_service import refresh_fabric_stats
@ -266,9 +416,9 @@ def delete_fabric_nodes(db: Session, fabric_node_ids: list[str]) -> dict[str, in
except Exception: # noqa: BLE001
pass
return {
"deleted": len(rows),
"edges_deleted": len(edge_ids),
"placements_deleted": int(placements or 0),
**out,
"managed_deleted": len(managed_rows),
"membership_views": membership_views,
}

31
netx_api/topology_router.py
View file

@ -205,20 +205,30 @@ def api_fabric_discover(
@router.post("/fabric/cleanup-duplicates")
def api_fabric_cleanup_duplicates(db: Session = Depends(get_db)) -> dict[str, Any]:
"""Merge duplicate fabric nodes (same managed/ume/name/ip) and retarget edges."""
from .topology_inventory_lifecycle import reconcile_dangling_fabric_links
from .topology_inventory_lifecycle import (
purge_fully_orphaned_fabric_nodes,
reconcile_dangling_fabric_links,
)
link_stats = reconcile_dangling_fabric_links(db)
# Detach dangling first but defer full orphan GC until after absorb/merge.
link_stats = reconcile_dangling_fabric_links(db, sweep_orphans=False)
result = merge_duplicate_fabric_nodes(db)
gc = purge_fully_orphaned_fabric_nodes(db, None)
db.commit()
return {"ok": True, **result, "reconcile_links": link_stats}
return {
"ok": True,
**result,
"reconcile_links": link_stats,
"purged_orphans": gc,
}
@router.post("/fabric/reconcile-links")
def api_fabric_reconcile_links(db: Session = Depends(get_db)) -> dict[str, Any]:
"""Detach fabric refs whose managed/UME inventory rows no longer exist."""
"""Detach fabric refs whose managed/UME inventory rows no longer exist; GC full orphans."""
from .topology_inventory_lifecycle import reconcile_dangling_fabric_links
stats = reconcile_dangling_fabric_links(db)
stats = reconcile_dangling_fabric_links(db, sweep_orphans=True)
db.commit()
return {"ok": True, **stats}
@ -467,6 +477,17 @@ def api_delete_fabric_nodes(
return delete_fabric_nodes(db, body.fabric_node_ids)
@router.post("/fabric/nodes/purge-placeholders")
def api_purge_placeholder_fabric_nodes(
body: FabricNodesDeleteRequest,
db: Session = Depends(get_db),
) -> dict[str, Any]:
"""Hard-delete topology/LLDP placeholder ManagedNE + fabric node + incident edges."""
from .topology_inventory_lifecycle import purge_placeholder_fabric_nodes
return purge_placeholder_fabric_nodes(db, body.fabric_node_ids)
@router.post("/slices/generate")
def api_generate_slices(
body: SliceGenerateRequest,

8
netx_api/topology_schemas.py
View file

@ -522,6 +522,14 @@ class FabricNodesDeleteOut(BaseModel):
placements_deleted: int = 0
class FabricPlaceholderPurgeOut(BaseModel):
deleted: int = 0
edges_deleted: int = 0
placements_deleted: int = 0
managed_deleted: int = 0
membership_views: int = 0
class SliceGenerateRequest(BaseModel):
folder_id: str
template: str = Field(description="core_only | core_agg | agg_access")

2
netx_api/ume_runtime.py
View file

@ -49,6 +49,7 @@ _schedule_log = logging.getLogger("netx.ume.schedule")
def start_device_schedulers() -> None:
"""Start device-facing periodic collectors (safe to call once per process)."""
from .config_sync_scheduler import start_config_sync_scheduler
from .fabric_reconcile_scheduler import start_fabric_reconcile_scheduler
from .lldp_collect_scheduler import start_lldp_collect_scheduler
from .port_traffic_scheduler import start_port_traffic_scheduler
from .scheduler_heartbeat import start_scheduler_heartbeat_publisher
@ -56,6 +57,7 @@ def start_device_schedulers() -> None:
start_config_sync_scheduler()
start_lldp_collect_scheduler()
start_port_traffic_scheduler()
start_fabric_reconcile_scheduler()
# Publish status so API /metrics can see collectors when run in a split worker.
role = "api_inline" if bool(getattr(settings, "run_inline_schedulers", True)) else "worker"
start_scheduler_heartbeat_publisher(role=role)

6
netx_api/worker.py
View file

@ -4,7 +4,7 @@ Optional when ``NETX_RUN_INLINE_SCHEDULERS=false`` (API does not start collector
python -m netx_api.worker
Starts: config_sync, lldp_collect, port_traffic tick loops.
Starts: config_sync, lldp_collect, port_traffic, fabric_reconcile tick loops.
UME WS / keepalive remain in the API process (token + alarm coordination).
By default the API runs collectors inline — no separate worker needed.
"""
@ -45,7 +45,9 @@ def main() -> None:
log_runtime_budget(role="worker")
except Exception:
_log.exception("worker runtime budget log failed")
_log.info("netx worker schedulers started (config_sync, lldp_collect, port_traffic)")
_log.info(
"netx worker schedulers started (config_sync, lldp_collect, port_traffic, fabric_reconcile)"
)
while not stop.is_set():
time.sleep(1.0)