hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-09 03:10:46 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 2

Add topology classify inventory, fabric lifecycle, and broader collect targets.

Browse source 
Treat Fabric as an inventory sheet for role/region tagging and slices; detach fabric links on managed/UME delete instead of cascading map deletes. Extend collection to all managed and UME NEs, show NE source in management, and refine canvas vendor icon tones.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-08-02 11:09:58 +08:00
parent 90dff56e48
commit a5a3a4f156
30 changed files with 5986 additions and 495 deletions
Download patch file Download diff file Expand all files Collapse all files

4
netx_api/collection_schemas.py
View file

@ -8,13 +8,15 @@ from pydantic import BaseModel, Field
class CollectionJobCreate(BaseModel):
title: str = ""
commands: str = Field(min_length=1)
ne_ids: list[str] = Field(min_length=1)
ne_ids: list[str] = Field(default_factory=list)
ume_ne_ids: list[str] = Field(default_factory=list)
class CollectionRunOut(BaseModel):
id: str
job_id: str
ne_id: str
ne_source: str = "managed"
ne_name: str
ne_ip: str
status: str

128
netx_api/collection_service.py
View file

@ -13,7 +13,7 @@ from fastapi import HTTPException
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from .models import ManagedNE, NeCollectionJob, NeCollectionRun
from .models import ManagedNE, NeCollectionJob, NeCollectionRun, UmeCliOverride, UmeInventoryNE
from .collection_job_state import (
finalize_collection_job,
reconcile_stale_collection_job,
@ -100,6 +100,7 @@ def run_to_out(row: NeCollectionRun) -> CollectionRunOut:
id=str(row.id),
job_id=str(row.job_id),
ne_id=str(row.ne_id),
ne_source=str(getattr(row, "ne_source", None) or "managed"),
ne_name=str(row.ne_name or ""),
ne_ip=str(row.ne_ip or ""),
status=str(row.status or "pending"),
@ -118,11 +119,17 @@ def list_eligible_ne(
page_size: int = 200,
keyword: str = "",
) -> dict[str, Any]:
stmt = db.query(ManagedNE).filter(ManagedNE.connect_status == "pass")
from .device_types import WEBCRT_NE_SOURCE
page = max(1, int(page or 1))
page_size = max(1, min(500, int(page_size or 200)))
kw = str(keyword or "").strip()
if kw:
like = f"%{kw}%"
stmt = stmt.filter(
like = f"%{kw}%" if kw else ""
# Managed inventory (any connect_status); exclude WebCRT session hosts.
m_stmt = db.query(ManagedNE).filter(ManagedNE.source != WEBCRT_NE_SOURCE)
if like:
m_stmt = m_stmt.filter(
or_(
ManagedNE.name.ilike(like),
ManagedNE.ip_address.ilike(like),
@ -130,16 +137,10 @@ def list_eligible_ne(
ManagedNE.device_type.ilike(like),
)
)
total = int(stmt.count())
rows = (
stmt.order_by(ManagedNE.name.asc())
.offset((page - 1) * page_size)
.limit(page_size)
.all()
)
items = [
managed_items = [
{
"id": str(x.id),
"source": "managed",
"name": str(x.name or ""),
"vendor": str(x.vendor or ""),
"device_type": str(x.device_type or ""),
@ -147,36 +148,94 @@ def list_eligible_ne(
"connect_status": str(x.connect_status or ""),
"connect_tested_at": x.connect_tested_at.isoformat() if x.connect_tested_at else None,
}
for x in rows
for x in m_stmt.order_by(ManagedNE.name.asc()).all()
]
return {"total": total, "page": page, "page_size": page_size, "items": items}
u_stmt = db.query(UmeInventoryNE)
if like:
u_stmt = u_stmt.filter(
or_(
UmeInventoryNE.ne_id.ilike(like),
UmeInventoryNE.ne_name.ilike(like),
UmeInventoryNE.user_label.ilike(like),
UmeInventoryNE.host_name.ilike(like),
UmeInventoryNE.ip_address.ilike(like),
UmeInventoryNE.vendor.ilike(like),
UmeInventoryNE.ne_type.ilike(like),
)
)
ume_rows = u_stmt.order_by(UmeInventoryNE.host_name.asc(), UmeInventoryNE.ne_name.asc()).all()
overrides = {
str(o.ume_ne_id): o
for o in db.query(UmeCliOverride).filter(
UmeCliOverride.ume_ne_id.in_([str(x.ne_id) for x in ume_rows] or ["__none__"])
).all()
}
ume_items = []
for x in ume_rows:
uid = str(x.ne_id)
ov = overrides.get(uid)
ume_items.append(
{
"id": uid,
"source": "ume",
"name": str(x.host_name or x.user_label or x.ne_name or x.ip_address or uid),
"vendor": str(x.vendor or "ZTE"),
"device_type": str(x.ne_type or ""),
"ip_address": str(x.ip_address or ""),
"connect_status": str(ov.connect_status or "unknown") if ov else "unknown",
"connect_tested_at": ov.connect_tested_at.isoformat()
if ov and ov.connect_tested_at
else None,
}
)
items = managed_items + ume_items
items.sort(key=lambda r: (str(r.get("name") or "").lower(), str(r.get("id") or "")))
total = len(items)
start = (page - 1) * page_size
page_items = items[start : start + page_size]
return {"total": total, "page": page, "page_size": page_size, "items": page_items}
def create_collection(db: Session, body: CollectionJobCreate) -> CollectionJobOut:
commands = _parse_commands(body.commands)
if not commands:
raise HTTPException(status_code=400, detail="commands_empty")
ne_ids = [str(x).strip() for x in body.ne_ids if str(x).strip()]
if not ne_ids:
ne_ids = [str(x).strip() for x in (body.ne_ids or []) if str(x).strip()]
ume_ids = [str(x).strip() for x in (body.ume_ne_ids or []) if str(x).strip()]
if not ne_ids and not ume_ids:
raise HTTPException(status_code=400, detail="ne_ids_required")
ne_rows: list[ManagedNE] = []
missing: list[str] = []
not_pass: list[str] = []
targets: list[tuple[str, str, str, str]] = [] # source, id, name, ip
missing_m: list[str] = []
for ne_id in ne_ids:
row = db.get(ManagedNE, ne_id)
if not row:
missing.append(ne_id)
missing_m.append(ne_id)
continue
if str(row.connect_status or "") != "pass":
not_pass.append(ne_id)
targets.append(
(
"managed",
str(row.id),
str(row.name or row.ip_address or ""),
str(row.ip_address or ""),
)
)
if missing_m:
raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing_m[:5])}")
missing_u: list[str] = []
for uid in ume_ids:
inv = db.get(UmeInventoryNE, uid)
if not inv:
missing_u.append(uid)
continue
ne_rows.append(row)
if missing:
raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing[:5])}")
if not_pass:
raise HTTPException(status_code=400, detail=f"ne_connect_not_pass: {','.join(not_pass[:5])}")
if not ne_rows:
name = str(inv.host_name or inv.user_label or inv.ne_name or inv.ip_address or uid)
targets.append(("ume", uid, name, str(inv.ip_address or "")))
if missing_u:
raise HTTPException(status_code=404, detail=f"ume_ne_not_found: {','.join(missing_u[:5])}")
if not targets:
raise HTTPException(status_code=400, detail="no_eligible_ne")
now = _now()
@ -184,7 +243,7 @@ def create_collection(db: Session, body: CollectionJobCreate) -> CollectionJobOu
title=str(body.title or "").strip() or f"collect-{now.strftime('%Y%m%d-%H%M%S')}",
commands="\n".join(commands),
status="pending",
ne_count=len(ne_rows),
ne_count=len(targets),
created_at=now,
started_at=None,
last_run_at=None,
@ -192,12 +251,13 @@ def create_collection(db: Session, body: CollectionJobCreate) -> CollectionJobOu
db.add(job)
db.flush()
for ne in ne_rows:
for source, tid, name, ip in targets:
run = NeCollectionRun(
job_id=str(job.id),
ne_id=str(ne.id),
ne_name=str(ne.name or ne.ip_address or ""),
ne_ip=str(ne.ip_address or ""),
ne_id=tid,
ne_source=source,
ne_name=name,
ne_ip=ip,
status="pending",
)
db.add(run)

12
netx_api/main.py
View file

@ -820,15 +820,25 @@ def on_startup() -> None:
ensure_port_traffic_series_schema(conn)
ensure_topology_schema(conn)
conn.exec_driver_sql(
"ALTER TABLE ne_collection_run ADD COLUMN IF NOT EXISTS ne_source VARCHAR(16) DEFAULT 'managed'"
)
try:
conn.exec_driver_sql(
"ALTER TABLE ne_collection_run ALTER COLUMN ne_id TYPE VARCHAR(128)"
)
except Exception:
pass
except Exception:
_schedule_log.exception("startup: auth/port_traffic/topology schema migration failed")
_reset_runtime_pause_flags()
_fail_stale_running_sync_jobs_on_startup()
try:
from .topology_service import reclaim_stale_discover_jobs
from .topology_service import bootstrap_topology_tree, reclaim_stale_discover_jobs
db_topo = SessionLocal()
try:
bootstrap_topology_tree(db_topo)
closed = reclaim_stale_discover_jobs(db_topo, force_all_open=True)
if closed:
_schedule_log.warning(

60
netx_api/models.py
View file

@ -383,7 +383,10 @@ class NeCollectionRun(Base):
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex)
job_id: Mapped[str] = mapped_column(String(64), index=True)
ne_id: Mapped[str] = mapped_column(String(64), index=True)
# managed_ne.id or ume_inventory_ne.ne_id
ne_id: Mapped[str] = mapped_column(String(128), index=True)
# managed | ume
ne_source: Mapped[str] = mapped_column(String(16), default="managed", index=True)
ne_name: Mapped[str] = mapped_column(String(256), default="")
ne_ip: Mapped[str] = mapped_column(String(128), default="")
status: Mapped[str] = mapped_column(String(32), default="pending", index=True)
@ -409,12 +412,39 @@ class TopoFabricNode(Base):
ip: Mapped[str] = mapped_column(String(128), default="", index=True)
vendor: Mapped[str] = mapped_column(String(64), default="")
device_type: Mapped[str] = mapped_column(String(64), default="")
# Classify tags (regex rules / manual). role: core|aggregation|access|unknown|""
role: Mapped[str] = mapped_column(String(32), default="", index=True)
region_folder_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# rule | manual | ""
role_source: Mapped[str] = mapped_column(String(16), default="")
region_source: Mapped[str] = mapped_column(String(16), default="")
attrs: Mapped[dict] = mapped_column(_JsonType, default=dict)
last_seen_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, index=True)
class TopoClassifyRule(Base):
"""Regex rules to assign fabric role or region from NE name/IP."""
__tablename__ = "topo_classify_rule"
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex)
# role | region
scope: Mapped[str] = mapped_column(String(16), default="role", index=True)
name: Mapped[str] = mapped_column(String(256), default="")
pattern: Mapped[str] = mapped_column(String(512), default="")
# name | ip | name_ip
match_field: Mapped[str] = mapped_column(String(32), default="name")
priority: Mapped[int] = mapped_column(Integer, default=100, index=True)
enabled: Mapped[bool] = mapped_column(Boolean, default=True, index=True)
# role: {role}; region: {folder_id} or {region_name_from_group}
payload: Mapped[dict] = mapped_column(_JsonType, default=dict)
remark: Mapped[str] = mapped_column(String(512), default="")
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, index=True)
class TopoFabricEdge(Base):
"""Global fabric link. Target scale ~1M; layer reserved for future BGP/tunnel/l2vpn."""
@ -448,15 +478,39 @@ class TopoFabricEdge(Base):
updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
class TopoFolder(Base):
"""Grouping node for topology tree (root / region). Not a canvas."""
__tablename__ = "topo_folder"
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex)
parent_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# root | region
kind: Mapped[str] = mapped_column(String(32), default="region", index=True)
name: Mapped[str] = mapped_column(String(256), default="", index=True)
sort_order: Mapped[int] = mapped_column(Integer, default=0)
is_system: Mapped[bool] = mapped_column(Boolean, default=False)
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, index=True)
class TopoView(Base):
"""Named topology view (presentation); replaces legacy topology_map."""
"""Topology canvas under a site/region (physical or custom; flat siblings)."""
__tablename__ = "topo_view"
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex)
folder_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# Legacy nesting column; unused (always null after vendor-model migration).
parent_view_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# physical | custom
kind: Mapped[str] = mapped_column(String(32), default="custom", index=True)
# Optional filter preset label (legacy core|aggregation|access); not a tree level.
role: Mapped[str] = mapped_column(String(32), default="core", index=True)
name: Mapped[str] = mapped_column(String(256), default="", index=True)
remark: Mapped[str] = mapped_column(String(1024), default="")
# { node_ids?: [], layer?: "physical", status?: "active", keyword?: "" }
sort_order: Mapped[int] = mapped_column(Integer, default=0)
# { layer?, status?, membership?: {...} }
filter: Mapped[dict] = mapped_column(_JsonType, default=dict)
viewport: Mapped[dict] = mapped_column(_JsonType, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)

20
netx_api/ne_collect_runner.py
View file

@ -12,11 +12,13 @@ from typing import Any
from .collection_job_state import finalize_collection_job, sync_job_progress
from .config import settings
from .db import SessionLocal
from .models import ManagedNE, NeCollectionJob, NeCollectionRun
from fastapi import HTTPException
from .cli_resolve import resolve_cli_target
from .models import NeCollectionJob, NeCollectionRun
from .ne_collection_paths import clear_run_output_files, run_output_dir
from .ne_crypto import CredentialCryptoError
from .ne_netmiko import send_show_command
from .ne_service import get_device_credentials
from .ne_session_factory import close_netmiko_connection, open_netmiko_connection
_log = logging.getLogger("netx.ne.collect")
@ -159,15 +161,16 @@ def _run_single(job_id: str, run_id: str, commands: list[str]) -> None:
run = db.get(NeCollectionRun, run_id)
if not run:
return
ne = db.get(ManagedNE, str(run.ne_id))
if not ne:
_update_run(run_id, status="fail", message="managed_ne_not_found", ended_at=datetime.now())
return
if _job_is_paused(job_id):
_update_run(run_id, status="cancelled", message="paused", ended_at=datetime.now())
return
try:
creds = get_device_credentials(ne)
source = str(getattr(run, "ne_source", None) or "managed").strip().lower()
tid = str(run.ne_id or "").strip()
if source == "ume":
creds, _device = resolve_cli_target(db, ume_ne_id=tid)
else:
creds, _device = resolve_cli_target(db, managed_ne_id=tid)
output = _collect_with_timeout(creds, commands)
finished_at = datetime.now()
name_part = _safe_filename_part(str(run.ne_name or creds.get("name") or "ne"))
@ -190,6 +193,9 @@ def _run_single(job_id: str, run_id: str, commands: list[str]) -> None:
)
except CredentialCryptoError as exc:
_update_run(run_id, status="fail", message=str(exc)[:1020], ended_at=datetime.now())
except HTTPException as exc:
detail = str(exc.detail if exc.detail is not None else exc)[:1020]
_update_run(run_id, status="fail", message=detail, ended_at=datetime.now())
except Exception as exc:
_log.exception("collection failed run=%s", run_id)
_update_run(run_id, status="fail", message=_format_run_error(exc), ended_at=datetime.now())

3
netx_api/ne_schemas.py
View file

@ -95,6 +95,9 @@ class ManagedNeOut(BaseModel):
connect_tested_at: datetime | None
tags: str
remark: str
# Provenance: "" | ume_sync | webcrt | lldp | …
source: str = ""
source_ref: str = ""
hop_enabled: bool = False
hop_vendor: str = "zte"
hop_host: str = ""

33
netx_api/ne_service.py
View file

@ -220,6 +220,8 @@ def row_to_out(row: ManagedNE) -> ManagedNeOut:
connect_tested_at=row.connect_tested_at,
tags=str(row.tags or ""),
remark=str(row.remark or ""),
source=str(row.source or ""),
source_ref=str(row.source_ref or ""),
hop_enabled=bool(row.hop_enabled),
hop_vendor=str(row.hop_vendor or "zte"),
hop_host=str(row.hop_host or ""),
@ -641,9 +643,12 @@ def batch_apply_account(db: Session, ids: list[str], account: BatchAccountConfig
def delete_managed_ne(db: Session, ne_id: str) -> dict[str, bool]:
from .topology_inventory_lifecycle import detach_fabric_from_managed
row = db.get(ManagedNE, ne_id)
if not row:
raise HTTPException(status_code=404, detail="managed_ne_not_found")
detach_fabric_from_managed(db, [str(row.id)])
db.delete(row)
db.commit()
return {"ok": True}
@ -724,6 +729,8 @@ def get_ids_by_tag(db: Session, tag: str | None) -> list[str]:
def batch_delete_managed_ne(db: Session, ids: list[str]) -> dict[str, Any]:
from .topology_inventory_lifecycle import detach_fabric_from_managed
ne_ids = [str(x).strip() for x in ids if str(x).strip()]
if not ne_ids:
raise HTTPException(status_code=400, detail="ids_required")
@ -732,6 +739,7 @@ def batch_delete_managed_ne(db: Session, ids: list[str]) -> dict[str, Any]:
missing = [x for x in ne_ids if x not in found_ids]
if missing:
raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing[:5])}")
detach_fabric_from_managed(db, [str(r.id) for r in rows])
for row in rows:
db.delete(row)
db.commit()
@ -779,12 +787,19 @@ def sync_ume_inventory_to_managed_ne(db: Session) -> UmeManagedSyncResult:
existing.source = UME_SYNC_SOURCE
existing.source_ref = source_ref
existing.updated_at = now
deleted = 0
for row in db.query(ManagedNE).filter(ManagedNE.source == UME_SYNC_SOURCE).all():
ref = str(row.source_ref or "").strip()
if not ref or ref not in inventory_ids:
from .topology_inventory_lifecycle import detach_fabric_from_managed
stale = [
row
for row in db.query(ManagedNE).filter(ManagedNE.source == UME_SYNC_SOURCE).all()
if (not str(row.source_ref or "").strip())
or str(row.source_ref or "").strip() not in inventory_ids
]
if stale:
detach_fabric_from_managed(db, [str(r.id) for r in stale])
for row in stale:
db.delete(row)
deleted += 1
deleted = len(stale)
db.commit()
return UmeManagedSyncResult(
inserted=inserted,
@ -795,10 +810,14 @@ def sync_ume_inventory_to_managed_ne(db: Session) -> UmeManagedSyncResult:
def delete_ume_synced_managed_ne(db: Session) -> UmeManagedDeleteResult:
from .topology_inventory_lifecycle import detach_fabric_from_managed
rows = db.query(ManagedNE).filter(ManagedNE.source == UME_SYNC_SOURCE).all()
deleted = len(rows)
for row in rows:
db.delete(row)
if rows:
detach_fabric_from_managed(db, [str(r.id) for r in rows])
for row in rows:
db.delete(row)
db.commit()
return UmeManagedDeleteResult(deleted=deleted)

927
netx_api/topology_classify.py Normal file
View file

@ -0,0 +1,927 @@
"""Regex-based fabric role/region classification + slice map generation."""
from __future__ import annotations
import re
from datetime import datetime
from typing import Any
from uuid import uuid4
from fastapi import HTTPException
from sqlalchemy.orm import Session
from .models import TopoClassifyRule, TopoFabricEdge, TopoFabricNode, TopoFolder, TopoView, TopoViewNode
from .topology_membership import (
VIEW_KIND_CUSTOM,
VIEW_ROLE_ACCESS,
VIEW_ROLE_AGGREGATION,
VIEW_ROLE_CORE,
VIEW_ROLES,
merge_filter_with_membership,
normalize_view_role,
)
from .topology_schemas import (
ClassifyApplyOut,
ClassifyPreviewOut,
ClassifyRuleCreate,
ClassifyRuleOut,
ClassifyRuleUpdate,
FabricNodeOut,
FabricNodesBulkTagOut,
FabricNodesBulkTagRequest,
FabricNodesMatchOut,
FabricNodesMatchRequest,
FabricNodeTagPatch,
SliceGenerateOut,
SliceGenerateRequest,
SliceMapPlan,
TopologyFolderCreate,
TopologyViewCreate,
)
_MAX_PATTERN_LEN = 512
_ROLE_VALUES = VIEW_ROLES | {"unknown"}
_MATCH_FIELDS = frozenset({"name", "ip", "name_ip"})
_SCOPES = frozenset({"role", "region"})
_SLICE_TEMPLATES = frozenset({"core_only", "core_agg", "agg_access"})
def _utcnow() -> datetime:
return datetime.utcnow()
def _compile_pattern(pattern: str) -> re.Pattern[str]:
p = str(pattern or "").strip()
if not p:
raise HTTPException(status_code=400, detail="pattern_required")
if len(p) > _MAX_PATTERN_LEN:
raise HTTPException(status_code=400, detail="pattern_too_long")
try:
return re.compile(p, re.IGNORECASE)
except re.error as exc:
raise HTTPException(status_code=400, detail=f"invalid_pattern:{exc}") from exc
def _match_text(node: TopoFabricNode, match_field: str) -> str:
name = str(node.name or "")
ip = str(node.ip or "")
mf = str(match_field or "name").strip().lower()
if mf == "ip":
return ip
if mf == "name_ip":
return f"{name} {ip}".strip()
return name
def _rule_out(row: TopoClassifyRule) -> ClassifyRuleOut:
return ClassifyRuleOut(
id=row.id,
scope=str(row.scope or "role"),
name=str(row.name or ""),
pattern=str(row.pattern or ""),
match_field=str(row.match_field or "name"),
priority=int(row.priority or 100),
enabled=bool(row.enabled),
payload=dict(row.payload or {}),
remark=str(row.remark or ""),
created_at=row.created_at,
updated_at=row.updated_at,
)
def _validate_payload(scope: str, payload: dict[str, Any]) -> dict[str, Any]:
out = dict(payload or {})
if scope == "role":
role = normalize_view_role(str(out.get("role") or ""))
if str(out.get("role") or "").strip().lower() not in VIEW_ROLES:
raise HTTPException(status_code=400, detail="role_payload_invalid")
return {"role": role}
if "folder_id" in out and str(out.get("folder_id") or "").strip():
return {"folder_id": str(out["folder_id"]).strip()}
if "region_name_from_group" in out:
try:
g = int(out.get("region_name_from_group"))
except (TypeError, ValueError) as exc:
raise HTTPException(status_code=400, detail="region_group_invalid") from exc
if g < 1:
raise HTTPException(status_code=400, detail="region_group_invalid")
return {"region_name_from_group": g}
raise HTTPException(status_code=400, detail="region_payload_invalid")
def list_rules(db: Session, *, scope: str = "") -> list[ClassifyRuleOut]:
q = db.query(TopoClassifyRule)
if scope.strip():
q = q.filter(TopoClassifyRule.scope == scope.strip().lower())
rows = q.order_by(
TopoClassifyRule.scope.asc(),
TopoClassifyRule.priority.asc(),
TopoClassifyRule.name.asc(),
).all()
return [_rule_out(r) for r in rows]
def create_rule(db: Session, body: ClassifyRuleCreate) -> ClassifyRuleOut:
scope = str(body.scope or "").strip().lower()
if scope not in _SCOPES:
raise HTTPException(status_code=400, detail="scope_invalid")
match_field = str(body.match_field or "name").strip().lower()
if match_field not in _MATCH_FIELDS:
raise HTTPException(status_code=400, detail="match_field_invalid")
_compile_pattern(body.pattern)
payload = _validate_payload(scope, dict(body.payload or {}))
if scope == "region" and payload.get("folder_id"):
folder = db.get(TopoFolder, payload["folder_id"])
if folder is None or str(folder.kind or "") != "region":
raise HTTPException(status_code=400, detail="folder_not_found")
now = _utcnow()
row = TopoClassifyRule(
id=uuid4().hex,
scope=scope,
name=str(body.name or "").strip()[:256] or f"{scope}-rule",
pattern=str(body.pattern or "").strip()[:_MAX_PATTERN_LEN],
match_field=match_field,
priority=int(body.priority if body.priority is not None else 100),
enabled=bool(body.enabled if body.enabled is not None else True),
payload=payload,
remark=str(body.remark or "")[:512],
created_at=now,
updated_at=now,
)
db.add(row)
db.commit()
db.refresh(row)
return _rule_out(row)
def update_rule(db: Session, rule_id: str, body: ClassifyRuleUpdate) -> ClassifyRuleOut:
row = db.get(TopoClassifyRule, rule_id)
if row is None:
raise HTTPException(status_code=404, detail="rule_not_found")
if body.name is not None:
row.name = str(body.name or "").strip()[:256]
if body.pattern is not None:
_compile_pattern(body.pattern)
row.pattern = str(body.pattern or "").strip()[:_MAX_PATTERN_LEN]
if body.match_field is not None:
mf = str(body.match_field or "name").strip().lower()
if mf not in _MATCH_FIELDS:
raise HTTPException(status_code=400, detail="match_field_invalid")
row.match_field = mf
if body.priority is not None:
row.priority = int(body.priority)
if body.enabled is not None:
row.enabled = bool(body.enabled)
if body.remark is not None:
row.remark = str(body.remark or "")[:512]
if body.payload is not None:
row.payload = _validate_payload(str(row.scope or "role"), dict(body.payload or {}))
if str(row.scope) == "region" and row.payload.get("folder_id"):
folder = db.get(TopoFolder, row.payload["folder_id"])
if folder is None or str(folder.kind or "") != "region":
raise HTTPException(status_code=400, detail="folder_not_found")
row.updated_at = _utcnow()
db.commit()
db.refresh(row)
return _rule_out(row)
def delete_rule(db: Session, rule_id: str) -> dict[str, Any]:
row = db.get(TopoClassifyRule, rule_id)
if row is None:
raise HTTPException(status_code=404, detail="rule_not_found")
db.delete(row)
db.commit()
return {"ok": True, "id": rule_id, "deleted": True}
def _enabled_rules(db: Session, scope: str) -> list[tuple[TopoClassifyRule, re.Pattern[str]]]:
rows = (
db.query(TopoClassifyRule)
.filter(TopoClassifyRule.scope == scope, TopoClassifyRule.enabled.is_(True))
.order_by(TopoClassifyRule.priority.asc(), TopoClassifyRule.name.asc())
.all()
)
out: list[tuple[TopoClassifyRule, re.Pattern[str]]] = []
for r in rows:
try:
out.append((r, _compile_pattern(r.pattern)))
except HTTPException:
continue
return out
def _ensure_region_by_name(db: Session, name: str) -> TopoFolder:
from .topology_service import bootstrap_topology_tree, create_folder
name = str(name or "").strip()[:256]
if not name:
raise HTTPException(status_code=400, detail="region_name_empty")
existing = (
db.query(TopoFolder)
.filter(TopoFolder.kind == "region", TopoFolder.name == name)
.first()
)
if existing is not None:
return existing
bootstrap_topology_tree(db)
created = create_folder(db, TopologyFolderCreate(name=name, kind="region"))
folder = db.get(TopoFolder, created.id)
assert folder is not None
return folder
def _resolve_role_hit(
node: TopoFabricNode, rules: list[tuple[TopoClassifyRule, re.Pattern[str]]]
) -> tuple[str | None, str | None, bool]:
"""Return (role, rule_id, multi_hit)."""
hits: list[tuple[str, str]] = []
for rule, cre in rules:
text = _match_text(node, rule.match_field)
if not text:
continue
if cre.search(text):
role = normalize_view_role(str((rule.payload or {}).get("role") or ""))
hits.append((role, rule.id))
if not hits:
return None, None, False
return hits[0][0], hits[0][1], len(hits) > 1
def _resolve_region_hit(
db: Session,
node: TopoFabricNode,
rules: list[tuple[TopoClassifyRule, re.Pattern[str]]],
*,
create_missing: bool,
) -> tuple[str | None, str | None, bool]:
hits: list[tuple[str, str]] = []
for rule, cre in rules:
text = _match_text(node, rule.match_field)
if not text:
continue
m = cre.search(text)
if not m:
continue
payload = dict(rule.payload or {})
folder_id = str(payload.get("folder_id") or "").strip()
if folder_id:
hits.append((folder_id, rule.id))
continue
g = int(payload.get("region_name_from_group") or 0)
try:
region_name = m.group(g)
except IndexError:
continue
region_name = str(region_name or "").strip()
if not region_name:
continue
if create_missing:
folder = _ensure_region_by_name(db, region_name)
hits.append((folder.id, rule.id))
else:
existing = (
db.query(TopoFolder)
.filter(TopoFolder.kind == "region", TopoFolder.name == region_name)
.first()
)
hits.append((existing.id if existing else f"new:{region_name}", rule.id))
if not hits:
return None, None, False
return hits[0][0], hits[0][1], len(hits) > 1
def preview_classify(db: Session, *, sample_limit: int = 20) -> ClassifyPreviewOut:
role_rules = _enabled_rules(db, "role")
region_rules = _enabled_rules(db, "region")
nodes = db.query(TopoFabricNode).order_by(TopoFabricNode.name.asc()).all()
role_matched = role_unmatched = role_conflict = 0
region_matched = region_unmatched = region_conflict = 0
role_samples: list[dict[str, Any]] = []
region_samples: list[dict[str, Any]] = []
unmatched_samples: list[dict[str, Any]] = []
for n in nodes:
role, _rid, multi_r = _resolve_role_hit(n, role_rules)
if role is None:
role_unmatched += 1
else:
role_matched += 1
if multi_r:
role_conflict += 1
if len(role_samples) < sample_limit:
role_samples.append(
{
"fabric_node_id": n.id,
"name": n.name,
"ip": n.ip,
"role": role,
"multi_hit": multi_r,
}
)
region_id, _rrid, multi_reg = _resolve_region_hit(
db, n, region_rules, create_missing=False
)
if region_id is None:
region_unmatched += 1
else:
region_matched += 1
if multi_reg:
region_conflict += 1
if len(region_samples) < sample_limit:
region_samples.append(
{
"fabric_node_id": n.id,
"name": n.name,
"ip": n.ip,
"region": region_id,
"multi_hit": multi_reg,
}
)
if role is None and region_id is None and len(unmatched_samples) < sample_limit:
unmatched_samples.append(
{"fabric_node_id": n.id, "name": n.name, "ip": n.ip, "vendor": n.vendor}
)
return ClassifyPreviewOut(
total_nodes=len(nodes),
role_matched=role_matched,
role_unmatched=role_unmatched,
role_conflicts=role_conflict,
region_matched=region_matched,
region_unmatched=region_unmatched,
region_conflicts=region_conflict,
role_samples=role_samples,
region_samples=region_samples,
unmatched_samples=unmatched_samples,
)
def apply_classify(
db: Session,
*,
skip_manual: bool = True,
fill_empty_only: bool = False,
) -> ClassifyApplyOut:
role_rules = _enabled_rules(db, "role")
region_rules = _enabled_rules(db, "region")
nodes = db.query(TopoFabricNode).all()
role_updated = region_updated = skipped_manual = 0
for n in nodes:
role, _, _ = _resolve_role_hit(n, role_rules)
if role is not None:
if skip_manual and str(n.role_source or "") == "manual":
skipped_manual += 1
elif fill_empty_only and str(n.role or "").strip():
pass
else:
n.role = role
n.role_source = "rule"
n.updated_at = _utcnow()
role_updated += 1
elif not str(n.role or "").strip() and str(n.role_source or "") != "manual":
n.role = "unknown"
n.role_source = "rule"
n.updated_at = _utcnow()
region_id, _, _ = _resolve_region_hit(db, n, region_rules, create_missing=True)
if region_id is not None and not str(region_id).startswith("new:"):
if skip_manual and str(n.region_source or "") == "manual":
skipped_manual += 1
elif fill_empty_only and str(n.region_folder_id or "").strip():
pass
else:
n.region_folder_id = region_id
n.region_source = "rule"
n.updated_at = _utcnow()
region_updated += 1
db.commit()
return ClassifyApplyOut(
role_updated=role_updated,
region_updated=region_updated,
skipped_manual=skipped_manual,
total_nodes=len(nodes),
)
def list_unmatched(
db: Session,
*,
kind: str = "any",
page: int = 1,
page_size: int = 50,
) -> dict[str, Any]:
from sqlalchemy import or_
q = db.query(TopoFabricNode)
k = str(kind or "any").strip().lower()
role_miss = or_(TopoFabricNode.role == "", TopoFabricNode.role == "unknown")
region_miss = or_(
TopoFabricNode.region_folder_id.is_(None),
TopoFabricNode.region_folder_id == "",
)
if k == "role":
q = q.filter(role_miss)
elif k == "region":
q = q.filter(region_miss)
else:
q = q.filter(or_(role_miss, region_miss))
total = q.count()
rows = (
q.order_by(TopoFabricNode.name.asc())
.offset(max(0, (page - 1) * page_size))
.limit(page_size)
.all()
)
from .topology_service import _node_out
return {
"total": total,
"page": page,
"page_size": page_size,
"items": [_node_out(n).model_dump() for n in rows],
}
def patch_fabric_node_tags(
db: Session, fabric_node_id: str, body: FabricNodeTagPatch
) -> FabricNodeOut:
from .topology_service import _node_out
n = db.get(TopoFabricNode, fabric_node_id)
if n is None:
raise HTTPException(status_code=404, detail="fabric_node_not_found")
if body.role is not None:
role = str(body.role or "").strip().lower()
if role and role not in _ROLE_VALUES:
raise HTTPException(status_code=400, detail="role_invalid")
n.role = role
n.role_source = "manual"
if body.region_folder_id is not None:
fid = str(body.region_folder_id or "").strip()
if fid:
folder = db.get(TopoFolder, fid)
if folder is None or str(folder.kind or "") != "region":
raise HTTPException(status_code=400, detail="folder_not_found")
n.region_folder_id = fid
else:
n.region_folder_id = None
n.region_source = "manual"
n.updated_at = _utcnow()
db.commit()
db.refresh(n)
return _node_out(n)
def _iter_regex_matches(
db: Session, *, pattern: str, match_field: str
) -> list[TopoFabricNode]:
cre = _compile_pattern(pattern)
mf = str(match_field or "name").strip().lower()
if mf not in _MATCH_FIELDS:
raise HTTPException(status_code=400, detail="match_field_invalid")
out: list[TopoFabricNode] = []
for n in db.query(TopoFabricNode).order_by(TopoFabricNode.name.asc()).all():
text = _match_text(n, mf)
if text and cre.search(text):
out.append(n)
return out
def match_fabric_nodes(db: Session, body: FabricNodesMatchRequest) -> FabricNodesMatchOut:
"""Ephemeral regex lookup — does not persist rules."""
from .topology_inventory_lifecycle import fabric_link_status
matched = _iter_regex_matches(
db, pattern=body.pattern, match_field=body.match_field
)
limit = max(1, min(200, int(body.sample_limit or 50)))
samples = [
{
"fabric_node_id": n.id,
"name": n.name,
"ip": n.ip,
"role": n.role or "",
"region_folder_id": n.region_folder_id or "",
"link_status": fabric_link_status(n),
}
for n in matched[:limit]
]
return FabricNodesMatchOut(
pattern=str(body.pattern or "").strip(),
match_field=str(body.match_field or "name"),
total_matched=len(matched),
samples=samples,
fabric_node_ids=[n.id for n in matched],
)
def bulk_tag_fabric_nodes(
db: Session, body: FabricNodesBulkTagRequest
) -> FabricNodesBulkTagOut:
"""Assign role/region after user confirms a regex or explicit selection."""
if body.role is None and body.region_folder_id is None:
raise HTTPException(status_code=400, detail="role_or_region_required")
role_v: str | None = None
if body.role is not None:
role_v = str(body.role or "").strip().lower()
if role_v and role_v not in _ROLE_VALUES:
raise HTTPException(status_code=400, detail="role_invalid")
region_v: str | None = None
if body.region_folder_id is not None:
region_v = str(body.region_folder_id or "").strip()
if region_v:
folder = db.get(TopoFolder, region_v)
if folder is None or str(folder.kind or "") != "region":
raise HTTPException(status_code=400, detail="folder_not_found")
else:
region_v = ""
ids = [str(x).strip() for x in (body.fabric_node_ids or []) if str(x).strip()]
if str(body.pattern or "").strip():
matched_nodes = _iter_regex_matches(
db, pattern=body.pattern, match_field=body.match_field
)
elif ids:
matched_nodes = (
db.query(TopoFabricNode)
.filter(TopoFabricNode.id.in_(ids))
.order_by(TopoFabricNode.name.asc())
.all()
)
else:
raise HTTPException(status_code=400, detail="ids_or_pattern_required")
samples = [
{
"fabric_node_id": n.id,
"name": n.name,
"ip": n.ip,
"role": n.role or "",
"region_folder_id": n.region_folder_id or "",
}
for n in matched_nodes[:50]
]
if body.dry_run:
return FabricNodesBulkTagOut(
dry_run=True,
matched=len(matched_nodes),
updated=0,
role=role_v,
region_folder_id=region_v,
samples=samples,
)
now = _utcnow()
updated = 0
for n in matched_nodes:
if role_v is not None:
n.role = role_v
n.role_source = "manual"
if region_v is not None:
n.region_folder_id = region_v or None
n.region_source = "manual"
n.updated_at = now
updated += 1
db.commit()
return FabricNodesBulkTagOut(
dry_run=False,
matched=len(matched_nodes),
updated=updated,
role=role_v,
region_folder_id=region_v,
samples=samples,
)
def apply_classify_empty_only(db: Session) -> ClassifyApplyOut:
"""Incremental classify for newly synced nodes (fill empty tags only)."""
return apply_classify(db, skip_manual=True, fill_empty_only=True)
# --- Slice generation -------------------------------------------------------
def _active_neighbors(db: Session, seed_ids: set[str], *, hops: int = 1) -> set[str]:
if not seed_ids or hops <= 0:
return set()
frontier = set(seed_ids)
found: set[str] = set()
for _ in range(hops):
if not frontier:
break
rows = (
db.query(TopoFabricEdge)
.filter(
TopoFabricEdge.layer == "physical",
TopoFabricEdge.status == "active",
(TopoFabricEdge.a_node_id.in_(frontier) | TopoFabricEdge.b_node_id.in_(frontier)),
)
.all()
)
nxt: set[str] = set()
for e in rows:
for a, b in ((e.a_node_id, e.b_node_id), (e.b_node_id, e.a_node_id)):
if a in frontier and b not in seed_ids and b not in found:
nxt.add(str(b))
found |= nxt
frontier = nxt
return found
def _connected_components(db: Session, node_ids: list[str]) -> list[list[str]]:
ids = [str(x) for x in node_ids if str(x)]
if not ids:
return []
id_set = set(ids)
adj: dict[str, set[str]] = {i: set() for i in ids}
rows = (
db.query(TopoFabricEdge)
.filter(
TopoFabricEdge.layer == "physical",
TopoFabricEdge.status == "active",
TopoFabricEdge.a_node_id.in_(ids),
TopoFabricEdge.b_node_id.in_(ids),
)
.all()
)
for e in rows:
a, b = str(e.a_node_id), str(e.b_node_id)
if a in id_set and b in id_set:
adj[a].add(b)
adj[b].add(a)
seen: set[str] = set()
comps: list[list[str]] = []
for nid in ids:
if nid in seen:
continue
stack = [nid]
seen.add(nid)
comp: list[str] = []
while stack:
cur = stack.pop()
comp.append(cur)
for nb in adj.get(cur, ()):
if nb not in seen:
seen.add(nb)
stack.append(nb)
comps.append(sorted(comp))
return comps
def _nodes_in_region(db: Session, folder_id: str, *, role: str = "") -> list[TopoFabricNode]:
q = db.query(TopoFabricNode).filter(TopoFabricNode.region_folder_id == folder_id)
if role:
q = q.filter(TopoFabricNode.role == role)
return q.order_by(TopoFabricNode.name.asc()).all()
def preview_slices(db: Session, body: SliceGenerateRequest) -> SliceGenerateOut:
folder = db.get(TopoFolder, body.folder_id)
if folder is None or str(folder.kind or "") != "region":
raise HTTPException(status_code=400, detail="folder_not_found")
template = str(body.template or "").strip().lower()
if template not in _SLICE_TEMPLATES:
raise HTTPException(status_code=400, detail="template_invalid")
max_nodes = max(1, min(2000, int(body.max_nodes or 300)))
plans: list[SliceMapPlan] = []
overlap_ids: set[str] = set()
seen_in_maps: dict[str, int] = {}
def _track(ids: list[str]) -> None:
for i in ids:
seen_in_maps[i] = seen_in_maps.get(i, 0) + 1
if template == "core_only":
cores = _nodes_in_region(db, folder.id, role=VIEW_ROLE_CORE)
comps = _connected_components(db, [n.id for n in cores]) or [
[n.id] for n in cores
]
for idx, comp in enumerate(comps, start=1):
if len(comp) > max_nodes:
raise HTTPException(
status_code=400,
detail=f"slice_exceeds_max_nodes:{len(comp)}>{max_nodes}",
)
name = f"Core-{idx}" if len(comps) > 1 else "Core"
plans.append(
SliceMapPlan(
name=name,
role=VIEW_ROLE_CORE,
seed_fabric_node_ids=comp,
member_fabric_node_ids=comp,
node_count=len(comp),
)
)
_track(comp)
elif template == "core_agg":
cores = _nodes_in_region(db, folder.id, role=VIEW_ROLE_CORE)
comps = _connected_components(db, [n.id for n in cores]) or [
[n.id] for n in cores
]
for idx, comp in enumerate(comps, start=1):
peers = _active_neighbors(db, set(comp), hops=1)
agg_ids = [
p
for p in peers
if (fn := db.get(TopoFabricNode, p)) is not None
and str(fn.role or "") == VIEW_ROLE_AGGREGATION
and str(fn.region_folder_id or "") == folder.id
]
members = sorted(set(comp) | set(agg_ids))
if len(members) > max_nodes:
raise HTTPException(
status_code=400,
detail=f"slice_exceeds_max_nodes:{len(members)}>{max_nodes}",
)
name = f"CoreAgg-{idx}" if len(comps) > 1 else "Core+Agg"
plans.append(
SliceMapPlan(
name=name,
role=VIEW_ROLE_CORE,
seed_fabric_node_ids=comp,
member_fabric_node_ids=members,
node_count=len(members),
)
)
_track(members)
else: # agg_access
aggs = _nodes_in_region(db, folder.id, role=VIEW_ROLE_AGGREGATION)
comps = _connected_components(db, [n.id for n in aggs]) or [
[n.id] for n in aggs
]
for idx, comp in enumerate(comps, start=1):
peers = _active_neighbors(db, set(comp), hops=1)
acc_ids = [
p
for p in peers
if (fn := db.get(TopoFabricNode, p)) is not None
and str(fn.role or "") == VIEW_ROLE_ACCESS
and str(fn.region_folder_id or "") == folder.id
]
members = sorted(set(comp) | set(acc_ids))
if len(members) > max_nodes:
raise HTTPException(
status_code=400,
detail=f"slice_exceeds_max_nodes:{len(members)}>{max_nodes}",
)
name = f"AggAccess-{idx}" if len(comps) > 1 else "Agg+Access"
plans.append(
SliceMapPlan(
name=name,
role=VIEW_ROLE_AGGREGATION,
seed_fabric_node_ids=comp,
member_fabric_node_ids=members,
node_count=len(members),
)
)
_track(members)
overlap_ids = {nid for nid, cnt in seen_in_maps.items() if cnt > 1}
return SliceGenerateOut(
folder_id=folder.id,
template=template,
dry_run=True,
maps=plans,
map_count=len(plans),
overlap_node_count=len(overlap_ids),
created_view_ids=[],
)
def generate_slices(db: Session, body: SliceGenerateRequest) -> SliceGenerateOut:
from .topology_service import create_view, _place_fabric_ids_on_view
preview = preview_slices(db, body)
if body.dry_run:
return preview
created: list[str] = []
for plan in preview.maps:
view = create_view(
db,
TopologyViewCreate(
name=plan.name,
folder_id=body.folder_id,
kind=VIEW_KIND_CUSTOM,
role=plan.role,
remark=f"slice:{body.template}",
),
)
mem = {
"mode": "hybrid",
"seed_fabric_node_ids": list(plan.member_fabric_node_ids),
"expand_hops": 0,
"max_nodes": int(body.max_nodes or 300),
"frozen": True,
"managed_ne_ids": [],
"tags_any": [],
"vendors": [],
"device_types": [],
"keyword": "",
}
row = db.get(TopoView, view.id)
assert row is not None
row.filter = merge_filter_with_membership(
dict(row.filter or {}), role=normalize_view_role(plan.role), membership=mem
)
_place_fabric_ids_on_view(db, row, list(plan.member_fabric_node_ids), existing=set())
row.updated_at = _utcnow()
db.commit()
created.append(view.id)
# Optionally seed physical overview with cores only
if body.seed_physical_cores:
from .topology_service import ensure_region_physical_view
phys = ensure_region_physical_view(db, body.folder_id, commit=True)
cores = [n.id for n in _nodes_in_region(db, body.folder_id, role=VIEW_ROLE_CORE)]
existing = {
vn.fabric_node_id
for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == phys.id).all()
}
to_add = [c for c in cores if c not in existing][: int(body.max_nodes or 300)]
if to_add:
_place_fabric_ids_on_view(db, phys, to_add, existing=existing)
mem = merge_filter_with_membership(
dict(phys.filter or {}),
role=VIEW_ROLE_CORE,
kind="physical",
membership={
**dict((phys.filter or {}).get("membership") or {}),
"frozen": True,
"max_nodes": int(body.max_nodes or 500),
},
)
phys.filter = mem
phys.updated_at = _utcnow()
db.commit()
return SliceGenerateOut(
folder_id=body.folder_id,
template=str(body.template),
dry_run=False,
maps=preview.maps,
map_count=len(preview.maps),
overlap_node_count=preview.overlap_node_count,
created_view_ids=created,
)
def search_fabric_nodes_with_views(
db: Session,
*,
keyword: str = "",
page: int = 1,
page_size: int = 50,
) -> dict[str, Any]:
from .topology_service import _node_out
q = db.query(TopoFabricNode)
kw = str(keyword or "").strip()
if kw:
like = f"%{kw}%"
q = q.filter(
(TopoFabricNode.name.ilike(like))
| (TopoFabricNode.ip.ilike(like))
| (TopoFabricNode.vendor.ilike(like))
)
total = q.count()
rows = (
q.order_by(TopoFabricNode.name.asc())
.offset(max(0, (page - 1) * page_size))
.limit(page_size)
.all()
)
node_ids = [n.id for n in rows]
placements: dict[str, list[dict[str, Any]]] = {nid: [] for nid in node_ids}
if node_ids:
vnodes = (
db.query(TopoViewNode, TopoView, TopoFolder)
.join(TopoView, TopoView.id == TopoViewNode.view_id)
.outerjoin(TopoFolder, TopoFolder.id == TopoView.folder_id)
.filter(TopoViewNode.fabric_node_id.in_(node_ids))
.all()
)
for vn, view, folder in vnodes:
placements.setdefault(vn.fabric_node_id, []).append(
{
"view_id": view.id,
"view_name": view.name,
"folder_id": view.folder_id or "",
"folder_name": (folder.name if folder else "") or "",
"kind": view.kind or "custom",
}
)
items = []
for n in rows:
d = _node_out(n).model_dump()
d["views"] = placements.get(n.id, [])
items.append(d)
return {"total": total, "page": page, "page_size": page_size, "items": items}

193
netx_api/topology_inventory_lifecycle.py Normal file
View file

@ -0,0 +1,193 @@
"""Detach fabric ↔ inventory links without deleting topology placements/edges."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from sqlalchemy.orm import Session
from sqlalchemy.orm.attributes import flag_modified
from .device_types import LLDP_DISCOVERED_NE_SOURCE, WEBCRT_NE_SOURCE
from .models import ManagedNE, TopoFabricNode, TopoView, UmeInventoryNE
UME_SYNC_SOURCE = "ume_sync"
def _utcnow() -> datetime:
return datetime.utcnow()
def _norm_ids(ids: list[str] | None) -> list[str]:
out: list[str] = []
seen: set[str] = set()
for raw in ids or []:
s = str(raw or "").strip()
if not s or s in seen:
continue
seen.add(s)
out.append(s)
return out
def fabric_link_status(n: TopoFabricNode) -> str:
mid = str(n.managed_ne_id or "").strip()
uid = str(n.ume_ne_id or "").strip()
if mid and uid:
return "both"
if mid:
return "managed"
if uid:
return "ume"
return "orphaned"
def managed_source_label(source: str | None) -> str:
src = str(source or "").strip().lower()
if src == UME_SYNC_SOURCE:
return "ume_sync"
if src == LLDP_DISCOVERED_NE_SOURCE or src == "lldp":
return "lldp"
if src == WEBCRT_NE_SOURCE or src == "webcrt":
return "webcrt"
if not src:
return "manual"
return src
def _strip_membership_ids(
db: Session,
*,
managed_ne_ids: set[str] | None = None,
ume_ne_ids: set[str] | None = None,
) -> int:
"""Remove deleted inventory ids from view membership JSON. Returns views touched."""
mids = managed_ne_ids or set()
uids = ume_ne_ids or set()
if not mids and not uids:
return 0
touched = 0
for view in db.query(TopoView).all():
filt = dict(view.filter or {})
mem = filt.get("membership")
if not isinstance(mem, dict):
continue
changed = False
if mids and isinstance(mem.get("managed_ne_ids"), list):
next_m = [str(x).strip() for x in mem["managed_ne_ids"] if str(x).strip() not in mids]
if next_m != [str(x).strip() for x in mem["managed_ne_ids"] if str(x).strip()]:
mem["managed_ne_ids"] = next_m
changed = True
if uids and isinstance(mem.get("ume_ne_ids"), list):
next_u = [str(x).strip() for x in mem["ume_ne_ids"] if str(x).strip() not in uids]
if next_u != [str(x).strip() for x in mem["ume_ne_ids"] if str(x).strip()]:
mem["ume_ne_ids"] = next_u
changed = True
if not changed:
continue
filt["membership"] = mem
view.filter = filt
flag_modified(view, "filter")
view.updated_at = _utcnow()
touched += 1
return touched
def detach_fabric_from_managed(db: Session, managed_ne_ids: list[str]) -> dict[str, int]:
"""Clear fabric.managed_ne_id for deleted ManagedNEs; keep nodes/placements/edges."""
ids = _norm_ids(managed_ne_ids)
if not ids:
return {"detached_nodes": 0, "membership_views": 0}
now = _utcnow()
rows = (
db.query(TopoFabricNode)
.filter(TopoFabricNode.managed_ne_id.in_(ids))
.all()
)
for row in rows:
row.managed_ne_id = None
row.updated_at = now
views = _strip_membership_ids(db, managed_ne_ids=set(ids))
return {"detached_nodes": len(rows), "membership_views": views}
def detach_fabric_from_ume(db: Session, ume_ne_ids: list[str]) -> dict[str, int]:
"""Clear fabric.ume_ne_id for deleted UME inventory rows; keep topology traces."""
ids = _norm_ids(ume_ne_ids)
if not ids:
return {"detached_nodes": 0, "membership_views": 0}
now = _utcnow()
rows = db.query(TopoFabricNode).filter(TopoFabricNode.ume_ne_id.in_(ids)).all()
for row in rows:
row.ume_ne_id = None
row.updated_at = now
views = _strip_membership_ids(db, ume_ne_ids=set(ids))
return {"detached_nodes": len(rows), "membership_views": views}
def reconcile_dangling_fabric_links(db: Session) -> dict[str, int]:
"""Unbind fabric ids that no longer exist in managed_ne / ume_inventory_ne."""
managed_alive = {str(x[0]) for x in db.query(ManagedNE.id).all() if str(x[0] or "").strip()}
ume_alive = {
str(x[0]) for x in db.query(UmeInventoryNE.ne_id).all() if str(x[0] or "").strip()
}
dangling_m: list[str] = []
dangling_u: list[str] = []
for n in db.query(TopoFabricNode).all():
mid = str(n.managed_ne_id or "").strip()
uid = str(n.ume_ne_id or "").strip()
if mid and mid not in managed_alive:
dangling_m.append(mid)
if uid and uid not in ume_alive:
dangling_u.append(uid)
m_stats = detach_fabric_from_managed(db, dangling_m)
u_stats = detach_fabric_from_ume(db, dangling_u)
return {
"dangling_managed_refs": len(set(dangling_m)),
"dangling_ume_refs": len(set(dangling_u)),
"detached_managed_nodes": int(m_stats.get("detached_nodes") or 0),
"detached_ume_nodes": int(u_stats.get("detached_nodes") or 0),
"membership_views": int(m_stats.get("membership_views") or 0)
+ int(u_stats.get("membership_views") or 0),
}
def enrich_fabric_node_dicts(
db: Session, items: list[dict[str, Any]]
) -> list[dict[str, Any]]:
"""Add link_status / alive flags / managed_source onto FabricNodeOut dicts."""
if not items:
return items
mids = [str(x.get("managed_ne_id") or "").strip() for x in items]
uids = [str(x.get("ume_ne_id") or "").strip() for x in items]
mids_f = [x for x in mids if x]
uids_f = [x for x in uids if x]
managed_by_id: dict[str, ManagedNE] = {}
if mids_f:
for row in db.query(ManagedNE).filter(ManagedNE.id.in_(mids_f)).all():
managed_by_id[str(row.id)] = row
ume_alive: set[str] = set()
if uids_f:
ume_alive = {
str(x[0])
for x in db.query(UmeInventoryNE.ne_id)
.filter(UmeInventoryNE.ne_id.in_(uids_f))
.all()
if str(x[0] or "").strip()
}
for item in items:
mid = str(item.get("managed_ne_id") or "").strip()
uid = str(item.get("ume_ne_id") or "").strip()
if mid and uid:
item["link_status"] = "both"
elif mid:
item["link_status"] = "managed"
elif uid:
item["link_status"] = "ume"
else:
item["link_status"] = "orphaned"
mrow = managed_by_id.get(mid) if mid else None
item["managed_alive"] = bool(mrow)
item["ume_alive"] = bool(uid and uid in ume_alive)
item["managed_source"] = managed_source_label(mrow.source) if mrow else ""
return items

141
netx_api/topology_membership.py Normal file
View file

@ -0,0 +1,141 @@
"""View membership boundaries — keep leaf canvases from sucking in the whole fabric."""
from __future__ import annotations
from typing import Any
VIEW_ROLE_CORE = "core"
VIEW_ROLE_AGGREGATION = "aggregation"
VIEW_ROLE_ACCESS = "access"
VIEW_ROLES = frozenset({VIEW_ROLE_CORE, VIEW_ROLE_AGGREGATION, VIEW_ROLE_ACCESS})
VIEW_KIND_PHYSICAL = "physical"
VIEW_KIND_CUSTOM = "custom"
VIEW_KINDS = frozenset({VIEW_KIND_PHYSICAL, VIEW_KIND_CUSTOM})
ROLE_DEFAULT_MAX_NODES = {
VIEW_ROLE_CORE: 80,
VIEW_ROLE_AGGREGATION: 200,
VIEW_ROLE_ACCESS: 300,
}
KIND_DEFAULT_MAX_NODES = {
VIEW_KIND_PHYSICAL: 500,
VIEW_KIND_CUSTOM: 300,
}
MEMBERSHIP_HARD_CAP = 2000
DEFAULT_EXPAND_HOPS = 1
MAX_EXPAND_HOPS = 3
def normalize_view_role(role: str | None) -> str:
r = str(role or "").strip().lower()
if r in VIEW_ROLES:
return r
return VIEW_ROLE_CORE
def normalize_view_kind(kind: str | None) -> str:
k = str(kind or "").strip().lower()
if k in VIEW_KINDS:
return k
return VIEW_KIND_CUSTOM
def default_max_nodes_for_role(role: str) -> int:
return int(ROLE_DEFAULT_MAX_NODES.get(normalize_view_role(role), 300))
def default_max_nodes_for_kind(kind: str) -> int:
return int(KIND_DEFAULT_MAX_NODES.get(normalize_view_kind(kind), 300))
def default_membership(
role: str = VIEW_ROLE_CORE, *, kind: str = VIEW_KIND_CUSTOM
) -> dict[str, Any]:
k = normalize_view_kind(kind)
max_nodes = (
default_max_nodes_for_kind(k)
if k == VIEW_KIND_PHYSICAL
else default_max_nodes_for_role(role)
)
return {
"mode": "hybrid",
"managed_ne_ids": [],
"tags_any": [],
"vendors": [],
"device_types": [],
"keyword": "",
"seed_fabric_node_ids": [],
"expand_hops": DEFAULT_EXPAND_HOPS,
"max_nodes": max_nodes,
"frozen": False,
}
def parse_membership(
filt: dict[str, Any] | None,
*,
role: str = VIEW_ROLE_CORE,
kind: str = VIEW_KIND_CUSTOM,
) -> dict[str, Any]:
base = default_membership(role, kind=kind)
raw = dict(filt or {})
mem = raw.get("membership")
if not isinstance(mem, dict):
return base
out = dict(base)
out["mode"] = str(mem.get("mode") or base["mode"]).strip() or "hybrid"
out["managed_ne_ids"] = [
str(x).strip() for x in (mem.get("managed_ne_ids") or []) if str(x).strip()
]
out["tags_any"] = [str(x).strip() for x in (mem.get("tags_any") or []) if str(x).strip()]
out["vendors"] = [str(x).strip() for x in (mem.get("vendors") or []) if str(x).strip()]
out["device_types"] = [
str(x).strip() for x in (mem.get("device_types") or []) if str(x).strip()
]
out["keyword"] = str(mem.get("keyword") or "").strip()
out["seed_fabric_node_ids"] = [
str(x).strip() for x in (mem.get("seed_fabric_node_ids") or []) if str(x).strip()
]
try:
hops = int(mem.get("expand_hops", DEFAULT_EXPAND_HOPS))
except (TypeError, ValueError):
hops = DEFAULT_EXPAND_HOPS
out["expand_hops"] = max(0, min(MAX_EXPAND_HOPS, hops))
try:
mx = int(mem.get("max_nodes", base["max_nodes"]))
except (TypeError, ValueError):
mx = int(base["max_nodes"])
out["max_nodes"] = max(1, min(MEMBERSHIP_HARD_CAP, mx))
out["frozen"] = bool(mem.get("frozen", False))
return out
def merge_filter_with_membership(
filt: dict[str, Any] | None,
*,
role: str,
kind: str = VIEW_KIND_CUSTOM,
membership: dict[str, Any] | None = None,
) -> dict[str, Any]:
out = dict(filt or {})
layer = str(out.get("layer") or "physical").strip() or "physical"
out["layer"] = layer
mem = (
membership
if membership is not None
else parse_membership(out, role=role, kind=kind)
)
out["membership"] = mem
return out
def has_hard_scope(mem: dict[str, Any]) -> bool:
return bool(
mem.get("managed_ne_ids")
or mem.get("tags_any")
or mem.get("vendors")
or mem.get("device_types")
or mem.get("keyword")
)

79
netx_api/topology_migrate.py
View file

@ -36,6 +36,42 @@ def ensure_topology_schema(conn: Connection) -> None:
"ALTER TABLE topo_discover_job ADD COLUMN IF NOT EXISTS trigger_mode VARCHAR(32) DEFAULT 'manual'",
"ALTER TABLE lldp_collect_policy ADD COLUMN IF NOT EXISTS history_keep INTEGER DEFAULT 30",
"ALTER TABLE lldp_collect_policy ADD COLUMN IF NOT EXISTS interval_hours INTEGER DEFAULT 24",
"ALTER TABLE topo_view ADD COLUMN IF NOT EXISTS folder_id VARCHAR(64)",
"ALTER TABLE topo_view ADD COLUMN IF NOT EXISTS parent_view_id VARCHAR(64)",
"ALTER TABLE topo_view ADD COLUMN IF NOT EXISTS kind VARCHAR(32) DEFAULT 'custom'",
"ALTER TABLE topo_view ADD COLUMN IF NOT EXISTS role VARCHAR(32) DEFAULT 'core'",
"ALTER TABLE topo_view ADD COLUMN IF NOT EXISTS sort_order INTEGER DEFAULT 0",
"ALTER TABLE topo_fabric_node ADD COLUMN IF NOT EXISTS role VARCHAR(32) DEFAULT ''",
"ALTER TABLE topo_fabric_node ADD COLUMN IF NOT EXISTS region_folder_id VARCHAR(64)",
"ALTER TABLE topo_fabric_node ADD COLUMN IF NOT EXISTS role_source VARCHAR(16) DEFAULT ''",
"ALTER TABLE topo_fabric_node ADD COLUMN IF NOT EXISTS region_source VARCHAR(16) DEFAULT ''",
"""
CREATE TABLE IF NOT EXISTS topo_folder (
id VARCHAR(64) PRIMARY KEY,
parent_id VARCHAR(64),
kind VARCHAR(32) DEFAULT 'region',
name VARCHAR(256) DEFAULT '',
sort_order INTEGER DEFAULT 0,
is_system BOOLEAN DEFAULT FALSE,
created_at TIMESTAMP,
updated_at TIMESTAMP
)
""",
"""
CREATE TABLE IF NOT EXISTS topo_classify_rule (
id VARCHAR(64) PRIMARY KEY,
scope VARCHAR(16) DEFAULT 'role',
name VARCHAR(256) DEFAULT '',
pattern VARCHAR(512) DEFAULT '',
match_field VARCHAR(32) DEFAULT 'name',
priority INTEGER DEFAULT 100,
enabled BOOLEAN DEFAULT TRUE,
payload JSONB DEFAULT '{}',
remark VARCHAR(512) DEFAULT '',
created_at TIMESTAMP,
updated_at TIMESTAMP
)
""",
]
)
elif dialect.startswith("sqlite"):
@ -45,6 +81,42 @@ def ensure_topology_schema(conn: Connection) -> None:
"ALTER TABLE topo_discover_job ADD COLUMN trigger_mode VARCHAR(32) DEFAULT 'manual'",
"ALTER TABLE lldp_collect_policy ADD COLUMN history_keep INTEGER DEFAULT 30",
"ALTER TABLE lldp_collect_policy ADD COLUMN interval_hours INTEGER DEFAULT 24",
"ALTER TABLE topo_view ADD COLUMN folder_id VARCHAR(64)",
"ALTER TABLE topo_view ADD COLUMN parent_view_id VARCHAR(64)",
"ALTER TABLE topo_view ADD COLUMN kind VARCHAR(32) DEFAULT 'custom'",
"ALTER TABLE topo_view ADD COLUMN role VARCHAR(32) DEFAULT 'core'",
"ALTER TABLE topo_view ADD COLUMN sort_order INTEGER DEFAULT 0",
"ALTER TABLE topo_fabric_node ADD COLUMN role VARCHAR(32) DEFAULT ''",
"ALTER TABLE topo_fabric_node ADD COLUMN region_folder_id VARCHAR(64)",
"ALTER TABLE topo_fabric_node ADD COLUMN role_source VARCHAR(16) DEFAULT ''",
"ALTER TABLE topo_fabric_node ADD COLUMN region_source VARCHAR(16) DEFAULT ''",
"""
CREATE TABLE IF NOT EXISTS topo_folder (
id VARCHAR(64) PRIMARY KEY,
parent_id VARCHAR(64),
kind VARCHAR(32) DEFAULT 'region',
name VARCHAR(256) DEFAULT '',
sort_order INTEGER DEFAULT 0,
is_system BOOLEAN DEFAULT 0,
created_at DATETIME,
updated_at DATETIME
)
""",
"""
CREATE TABLE IF NOT EXISTS topo_classify_rule (
id VARCHAR(64) PRIMARY KEY,
scope VARCHAR(16) DEFAULT 'role',
name VARCHAR(256) DEFAULT '',
pattern VARCHAR(512) DEFAULT '',
match_field VARCHAR(32) DEFAULT 'name',
priority INTEGER DEFAULT 100,
enabled INTEGER DEFAULT 1,
payload TEXT DEFAULT '{}',
remark VARCHAR(512) DEFAULT '',
created_at DATETIME,
updated_at DATETIME
)
""",
]
)
for sql in alter_stmts:
@ -85,6 +157,13 @@ def ensure_topology_schema(conn: Connection) -> None:
"CREATE INDEX IF NOT EXISTS ix_topo_fabric_edge_layer_seen ON topo_fabric_edge (layer, last_seen_at)",
"CREATE INDEX IF NOT EXISTS ix_topo_fabric_edge_active ON topo_fabric_edge (layer) WHERE status = 'active'",
"CREATE INDEX IF NOT EXISTS ix_topo_view_node_view ON topo_view_node (view_id)",
"CREATE INDEX IF NOT EXISTS ix_topo_view_folder ON topo_view (folder_id)",
"CREATE INDEX IF NOT EXISTS ix_topo_view_parent ON topo_view (parent_view_id)",
"CREATE INDEX IF NOT EXISTS ix_topo_view_kind ON topo_view (kind)",
"CREATE INDEX IF NOT EXISTS ix_topo_folder_parent ON topo_folder (parent_id)",
"CREATE INDEX IF NOT EXISTS ix_topo_fabric_node_role ON topo_fabric_node (role)",
"CREATE INDEX IF NOT EXISTS ix_topo_fabric_node_region ON topo_fabric_node (region_folder_id)",
"CREATE INDEX IF NOT EXISTS ix_topo_classify_rule_scope ON topo_classify_rule (scope, enabled, priority)",
"CREATE UNIQUE INDEX IF NOT EXISTS uq_topo_fabric_node_managed_nn ON topo_fabric_node (managed_ne_id) WHERE managed_ne_id IS NOT NULL",
"CREATE UNIQUE INDEX IF NOT EXISTS uq_topo_fabric_node_ume_nn ON topo_fabric_node (ume_ne_id) WHERE ume_ne_id IS NOT NULL",
"CREATE INDEX IF NOT EXISTS ix_topo_discover_job_trigger ON topo_discover_job (trigger_mode)",

244
netx_api/topology_router.py
View file

@ -1,4 +1,4 @@
"""Topology HTTP routes — fabric + views (final model)."""
"""Topology HTTP routes — fabric + folder tree + leaf views."""
from __future__ import annotations
@ -8,22 +8,49 @@ from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from .db import get_db
from .topology_classify import (
apply_classify,
bulk_tag_fabric_nodes,
create_rule,
delete_rule,
generate_slices,
list_rules,
list_unmatched,
match_fabric_nodes,
patch_fabric_node_tags,
preview_classify,
search_fabric_nodes_with_views,
update_rule,
)
from .topology_schemas import (
ClassifyRuleCreate,
ClassifyRuleUpdate,
FabricDiscoverRequest,
FabricManualEdgeIn,
FabricNodesBulkTagRequest,
FabricNodesMatchRequest,
FabricNodeTagPatch,
SliceGenerateRequest,
TopologyFolderCreate,
TopologyFolderUpdate,
TopologyViewCreate,
TopologyViewUpdate,
ViewEdgeStylePatch,
ViewNodesAdd,
ViewPopulateRequest,
ViewPositionsPatch,
)
from .topology_service import (
add_nodes_to_view,
bootstrap_topology_tree,
create_folder,
create_view,
delete_folder,
delete_view,
get_discover_job,
get_fabric_neighborhood,
get_fabric_summary,
get_topology_tree,
get_view_graph,
list_fabric_edges,
list_fabric_nodes,
@ -31,10 +58,12 @@ from .topology_service import (
merge_duplicate_fabric_nodes,
patch_view_edge_style,
patch_view_positions,
populate_view,
project_fabric_neighbors_to_view,
refresh_fabric_stats,
remove_view_nodes,
start_discover_job,
update_folder,
update_view,
upsert_fabric_edge,
)
@ -53,11 +82,27 @@ def api_fabric_summary(db: Session = Depends(get_db)) -> dict[str, Any]:
@router.get("/fabric/nodes")
def api_fabric_nodes(
keyword: str = "",
role: str = "",
region_folder_id: str = "",
unmatched: str = Query(default="", description="any | role | region"),
link_status: str = Query(
default="",
description="linked | orphaned | managed | ume | both",
),
page: int = Query(default=1, ge=1),
page_size: int = Query(default=100, ge=1, le=2000),
db: Session = Depends(get_db),
) -> dict[str, Any]:
return list_fabric_nodes(db, keyword=keyword, page=page, page_size=page_size)
return list_fabric_nodes(
db,
keyword=keyword,
role=role,
region_folder_id=region_folder_id,
unmatched=unmatched,
link_status=link_status,
page=page,
page_size=page_size,
)
@router.get("/fabric/edges")
@ -108,15 +153,19 @@ def api_fabric_manual_edge(
)
db.commit()
refresh_fabric_stats(db)
return {"ok": True, "action": action, "edge": {
"id": edge.id,
"a_node_id": edge.a_node_id,
"b_node_id": edge.b_node_id,
"a_port": edge.a_port,
"b_port": edge.b_port,
"source": edge.source,
"status": edge.status,
}}
return {
"ok": True,
"action": action,
"edge": {
"id": edge.id,
"a_node_id": edge.a_node_id,
"b_node_id": edge.b_node_id,
"a_port": edge.a_port,
"b_port": edge.b_port,
"source": edge.source,
"status": edge.status,
},
}
@router.post("/fabric/discover")
@ -130,8 +179,22 @@ def api_fabric_discover(
@router.post("/fabric/cleanup-duplicates")
def api_fabric_cleanup_duplicates(db: Session = Depends(get_db)) -> dict[str, Any]:
"""Merge duplicate fabric nodes (same managed/ume/name/ip) and retarget edges."""
from .topology_inventory_lifecycle import reconcile_dangling_fabric_links
link_stats = reconcile_dangling_fabric_links(db)
result = merge_duplicate_fabric_nodes(db)
return {"ok": True, **result}
db.commit()
return {"ok": True, **result, "reconcile_links": link_stats}
@router.post("/fabric/reconcile-links")
def api_fabric_reconcile_links(db: Session = Depends(get_db)) -> dict[str, Any]:
"""Detach fabric refs whose managed/UME inventory rows no longer exist."""
from .topology_inventory_lifecycle import reconcile_dangling_fabric_links
stats = reconcile_dangling_fabric_links(db)
db.commit()
return {"ok": True, **stats}
@router.get("/fabric/discover/{job_id}")
@ -139,7 +202,41 @@ def api_fabric_discover_job(job_id: str, db: Session = Depends(get_db)) -> dict[
return get_discover_job(db, job_id).model_dump()
# --- Views ------------------------------------------------------------------
# --- Tree / folders ---------------------------------------------------------
@router.get("/tree")
def api_topology_tree(db: Session = Depends(get_db)) -> dict[str, Any]:
return get_topology_tree(db).model_dump()
@router.post("/folders")
def api_create_folder(body: TopologyFolderCreate, db: Session = Depends(get_db)) -> dict[str, Any]:
return create_folder(db, body).model_dump()
@router.patch("/folders/{folder_id}")
def api_patch_folder(
folder_id: str, body: TopologyFolderUpdate, db: Session = Depends(get_db)
) -> dict[str, Any]:
return update_folder(db, folder_id, body).model_dump()
@router.delete("/folders/{folder_id}")
def api_delete_folder(
folder_id: str,
force: bool = Query(default=False),
db: Session = Depends(get_db),
) -> dict[str, Any]:
return delete_folder(db, folder_id, force=force)
@router.post("/tree/bootstrap")
def api_bootstrap_tree(db: Session = Depends(get_db)) -> dict[str, Any]:
return {"ok": True, **bootstrap_topology_tree(db)}
# --- Views (leaf canvases) --------------------------------------------------
@router.get("/views")
@ -165,8 +262,12 @@ def api_patch_view(
@router.delete("/views/{view_id}")
def api_delete_view(view_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
return delete_view(db, view_id)
def api_delete_view(
view_id: str,
force: bool = False,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return delete_view(db, view_id, force=force)
@router.patch("/views/{view_id}/positions")
@ -188,6 +289,15 @@ def api_project_neighbors(view_id: str, db: Session = Depends(get_db)) -> dict[s
return project_fabric_neighbors_to_view(db, view_id).model_dump()
@router.post("/views/{view_id}/populate")
def api_populate_view(
view_id: str,
body: ViewPopulateRequest | None = None,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return populate_view(db, view_id, body or ViewPopulateRequest()).model_dump()
@router.post("/views/{view_id}/nodes/remove")
def api_remove_nodes(
view_id: str,
@ -203,3 +313,107 @@ def api_edge_style(
view_id: str, body: ViewEdgeStylePatch, db: Session = Depends(get_db)
) -> dict[str, Any]:
return patch_view_edge_style(db, view_id, body).model_dump()
# --- Classify rules / slices / search ---------------------------------------
@router.get("/classify/rules")
def api_list_classify_rules(
scope: str = "",
db: Session = Depends(get_db),
) -> dict[str, Any]:
items = list_rules(db, scope=scope)
return {"items": [x.model_dump() for x in items]}
@router.post("/classify/rules")
def api_create_classify_rule(
body: ClassifyRuleCreate,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return create_rule(db, body).model_dump()
@router.patch("/classify/rules/{rule_id}")
def api_patch_classify_rule(
rule_id: str,
body: ClassifyRuleUpdate,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return update_rule(db, rule_id, body).model_dump()
@router.delete("/classify/rules/{rule_id}")
def api_delete_classify_rule(rule_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
return delete_rule(db, rule_id)
@router.post("/classify/preview")
def api_classify_preview(db: Session = Depends(get_db)) -> dict[str, Any]:
return preview_classify(db).model_dump()
@router.post("/classify/apply")
def api_classify_apply(
skip_manual: bool = Query(default=True),
fill_empty_only: bool = Query(default=False),
db: Session = Depends(get_db),
) -> dict[str, Any]:
return apply_classify(
db, skip_manual=skip_manual, fill_empty_only=fill_empty_only
).model_dump()
@router.get("/classify/unmatched")
def api_classify_unmatched(
kind: str = Query(default="any", description="any | role | region"),
page: int = Query(default=1, ge=1),
page_size: int = Query(default=50, ge=1, le=500),
db: Session = Depends(get_db),
) -> dict[str, Any]:
return list_unmatched(db, kind=kind, page=page, page_size=page_size)
@router.patch("/fabric/nodes/{fabric_node_id}/tags")
def api_patch_fabric_node_tags(
fabric_node_id: str,
body: FabricNodeTagPatch,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return patch_fabric_node_tags(db, fabric_node_id, body).model_dump()
@router.post("/fabric/nodes/match")
def api_fabric_nodes_match(
body: FabricNodesMatchRequest,
db: Session = Depends(get_db),
) -> dict[str, Any]:
"""Regex match over inventory (ephemeral; not stored as rules)."""
return match_fabric_nodes(db, body).model_dump()
@router.post("/fabric/nodes/tags/bulk")
def api_fabric_nodes_bulk_tag(
body: FabricNodesBulkTagRequest,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return bulk_tag_fabric_nodes(db, body).model_dump()
@router.post("/slices/generate")
def api_generate_slices(
body: SliceGenerateRequest,
db: Session = Depends(get_db),
) -> dict[str, Any]:
return generate_slices(db, body).model_dump()
@router.get("/fabric/nodes/search")
def api_fabric_nodes_search(
q: str = "",
page: int = Query(default=1, ge=1),
page_size: int = Query(default=50, ge=1, le=200),
db: Session = Depends(get_db),
) -> dict[str, Any]:
return search_fabric_nodes_with_views(db, keyword=q, page=page, page_size=page_size)

223
netx_api/topology_schemas.py
View file

@ -21,8 +21,17 @@ class FabricNodeOut(BaseModel):
ip: str = ""
vendor: str = ""
device_type: str = ""
role: str = ""
region_folder_id: str | None = None
role_source: str = ""
region_source: str = ""
attrs: dict[str, Any] = Field(default_factory=dict)
last_seen_at: datetime | None = None
# Inventory link diagnostics (list/enrich); empty when not enriched.
link_status: str = "" # managed | ume | both | orphaned
managed_alive: bool = False
ume_alive: bool = False
managed_source: str = "" # manual | ume_sync | lldp | webcrt | …
class FabricEdgeOut(BaseModel):
@ -125,14 +134,45 @@ class FabricDiscoverJobOut(BaseModel):
# ---------------------------------------------------------------------------
# Views
# Folders (tree grouping) + Views (leaf canvases)
# ---------------------------------------------------------------------------
class TopologyFolderCreate(BaseModel):
name: str = Field(min_length=1, max_length=256)
kind: str = Field(default="region", description="region only from API")
parent_id: str | None = None
sort_order: int = 0
class TopologyFolderUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=256)
parent_id: str | None = None
sort_order: int | None = None
class TopologyFolderOut(BaseModel):
id: str
parent_id: str = ""
kind: str
name: str
sort_order: int = 0
is_system: bool = False
created_at: datetime | None = None
updated_at: datetime | None = None
class TopologyViewCreate(BaseModel):
name: str = Field(min_length=1, max_length=256)
remark: str = Field(default="", max_length=1024)
filter: dict[str, Any] = Field(default_factory=dict)
folder_id: str = Field(..., min_length=1, description="Site/region folder id (required)")
kind: str = Field(default="custom", description="physical | custom")
role: str = Field(
default="core",
description="Optional filter preset label (legacy); not a tree level",
)
sort_order: int = 0
class TopologyViewUpdate(BaseModel):
@ -140,12 +180,20 @@ class TopologyViewUpdate(BaseModel):
remark: str | None = Field(default=None, max_length=1024)
filter: dict[str, Any] | None = None
viewport: dict[str, Any] | None = None
folder_id: str | None = None
kind: str | None = None
role: str | None = None
sort_order: int | None = None
class TopologyViewOut(BaseModel):
id: str
name: str
remark: str = ""
folder_id: str = ""
kind: str = "custom"
role: str = "core"
sort_order: int = 0
filter: dict[str, Any] = Field(default_factory=dict)
viewport: dict[str, Any] = Field(default_factory=dict)
node_count: int = 0
@ -153,6 +201,39 @@ class TopologyViewOut(BaseModel):
updated_at: datetime | None = None
class TopologyTreeViewOut(BaseModel):
id: str
name: str
kind: str = "custom"
role: str = "core"
sort_order: int = 0
node_count: int = 0
updated_at: datetime | None = None
class TopologyTreeFolderOut(BaseModel):
id: str
parent_id: str = ""
kind: str
name: str
sort_order: int = 0
is_system: bool = False
views: list[TopologyTreeViewOut] = Field(default_factory=list)
children: list["TopologyTreeFolderOut"] = Field(default_factory=list)
class TopologyTreeOut(BaseModel):
root: TopologyTreeFolderOut | None = None
class ViewPopulateRequest(BaseModel):
"""Fill a leaf view from membership rules (optional dry-run)."""
dry_run: bool = False
membership: dict[str, Any] | None = None
freeze_after: bool = True
class ViewNodeIn(BaseModel):
fabric_node_id: str = Field(min_length=1, max_length=64)
x: float = 0.0
@ -197,6 +278,19 @@ class TopologyViewGraphOut(BaseModel):
edges: list[ViewEdgeOut]
truncated: bool = False
truncate_reason: str = ""
outside_peers: list[dict[str, str]] = Field(default_factory=list)
class ViewPopulateOut(BaseModel):
view_id: str
dry_run: bool = False
candidate_count: int = 0
would_add: int = 0
added: int = 0
max_nodes: int = 0
truncated: bool = False
outside_peers: list[dict[str, str]] = Field(default_factory=list)
graph: TopologyViewGraphOut | None = None
class ViewPositionsPatch(BaseModel):
@ -228,3 +322,130 @@ class FabricManualEdgeIn(BaseModel):
b_node_id: str = Field(min_length=1, max_length=64)
a_port: str = ""
b_port: str = ""
# ---------------------------------------------------------------------------
# Classify rules + slices + search
# ---------------------------------------------------------------------------
class ClassifyRuleOut(BaseModel):
id: str
scope: str = "role"
name: str = ""
pattern: str = ""
match_field: str = "name"
priority: int = 100
enabled: bool = True
payload: dict[str, Any] = Field(default_factory=dict)
remark: str = ""
created_at: datetime | None = None
updated_at: datetime | None = None
class ClassifyRuleCreate(BaseModel):
scope: str = Field(default="role", description="role | region")
name: str = ""
pattern: str
match_field: str = "name"
priority: int = 100
enabled: bool = True
payload: dict[str, Any] = Field(default_factory=dict)
remark: str = ""
class ClassifyRuleUpdate(BaseModel):
name: str | None = None
pattern: str | None = None
match_field: str | None = None
priority: int | None = None
enabled: bool | None = None
payload: dict[str, Any] | None = None
remark: str | None = None
class ClassifyPreviewOut(BaseModel):
total_nodes: int = 0
role_matched: int = 0
role_unmatched: int = 0
role_conflicts: int = 0
region_matched: int = 0
region_unmatched: int = 0
region_conflicts: int = 0
role_samples: list[dict[str, Any]] = Field(default_factory=list)
region_samples: list[dict[str, Any]] = Field(default_factory=list)
unmatched_samples: list[dict[str, Any]] = Field(default_factory=list)
class ClassifyApplyOut(BaseModel):
role_updated: int = 0
region_updated: int = 0
skipped_manual: int = 0
total_nodes: int = 0
class FabricNodeTagPatch(BaseModel):
role: str | None = None
region_folder_id: str | None = None
class FabricNodesMatchRequest(BaseModel):
"""Ephemeral regex match over fabric inventory (not persisted as rules)."""
pattern: str
match_field: str = "name"
sample_limit: int = Field(default=50, ge=1, le=200)
class FabricNodesMatchOut(BaseModel):
pattern: str
match_field: str = "name"
total_matched: int = 0
samples: list[dict[str, Any]] = Field(default_factory=list)
fabric_node_ids: list[str] = Field(default_factory=list)
class FabricNodesBulkTagRequest(BaseModel):
"""Assign role/region to explicit ids or to an ephemeral regex match."""
fabric_node_ids: list[str] = Field(default_factory=list)
pattern: str = ""
match_field: str = "name"
role: str | None = None
region_folder_id: str | None = None
dry_run: bool = False
class FabricNodesBulkTagOut(BaseModel):
dry_run: bool = False
matched: int = 0
updated: int = 0
role: str | None = None
region_folder_id: str | None = None
samples: list[dict[str, Any]] = Field(default_factory=list)
class SliceGenerateRequest(BaseModel):
folder_id: str
template: str = Field(description="core_only | core_agg | agg_access")
dry_run: bool = True
max_nodes: int = Field(default=300, ge=1, le=2000)
seed_physical_cores: bool = False
class SliceMapPlan(BaseModel):
name: str
role: str = "core"
seed_fabric_node_ids: list[str] = Field(default_factory=list)
member_fabric_node_ids: list[str] = Field(default_factory=list)
node_count: int = 0
class SliceGenerateOut(BaseModel):
folder_id: str
template: str
dry_run: bool = True
maps: list[SliceMapPlan] = Field(default_factory=list)
map_count: int = 0
overlap_node_count: int = 0
created_view_ids: list[str] = Field(default_factory=list)

776
netx_api/topology_service.py
View file

@ -10,7 +10,7 @@ from typing import Any
from uuid import uuid4
from fastapi import HTTPException
from sqlalchemy import func, or_
from sqlalchemy import and_, func, or_
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
@ -26,11 +26,21 @@ from .models import (
TopoFabricEdge,
TopoFabricNode,
TopoFabricStats,
TopoFolder,
TopoView,
TopoViewEdgeStyle,
TopoViewNode,
UmeInventoryNE,
)
from .topology_membership import (
VIEW_KIND_CUSTOM,
VIEW_KIND_PHYSICAL,
has_hard_scope,
merge_filter_with_membership,
normalize_view_kind,
normalize_view_role,
parse_membership,
)
from .ne_exec import execute_managed_ne_commands
from .topology_lldp import (
NeighborHit,
@ -48,6 +58,12 @@ from .topology_schemas import (
FabricNeighborhoodOut,
FabricNodeOut,
FabricSummaryOut,
TopologyFolderCreate,
TopologyFolderOut,
TopologyFolderUpdate,
TopologyTreeFolderOut,
TopologyTreeOut,
TopologyTreeViewOut,
TopologyViewCreate,
TopologyViewGraphOut,
TopologyViewOut,
@ -57,9 +73,16 @@ from .topology_schemas import (
ViewNodeIn,
ViewNodeOut,
ViewNodesAdd,
ViewPopulateOut,
ViewPopulateRequest,
ViewPositionsPatch,
)
ROOT_FOLDER_NAME = "Network"
PHYSICAL_VIEW_NAME = "Physical topology"
# Legacy system region name (no longer auto-created; stripped on bootstrap when empty).
_LEGACY_UNASSIGNED_NAME = "Unassigned"
PAGE_DEFAULT = 100
PAGE_MAX = 2000
VIEW_GRAPH_NODE_HARD_CAP = 2000
@ -154,6 +177,10 @@ def _empty_to_none(s: str | None) -> str | None:
def _node_out(n: TopoFabricNode) -> FabricNodeOut:
return FabricNodeOut(
role=str(getattr(n, "role", "") or ""),
region_folder_id=str(getattr(n, "region_folder_id", None) or "") or None,
role_source=str(getattr(n, "role_source", "") or ""),
region_source=str(getattr(n, "region_source", "") or ""),
id=n.id,
managed_ne_id=n.managed_ne_id or "",
ume_ne_id=n.ume_ne_id or "",
@ -346,9 +373,15 @@ def list_fabric_nodes(
db: Session,
*,
keyword: str = "",
role: str = "",
region_folder_id: str = "",
unmatched: str = "",
link_status: str = "",
page: int = 1,
page_size: int = PAGE_DEFAULT,
) -> dict[str, Any]:
from .topology_inventory_lifecycle import enrich_fabric_node_dicts
page = max(1, int(page or 1))
page_size = max(1, min(PAGE_MAX, int(page_size or PAGE_DEFAULT)))
q = db.query(TopoFabricNode)
@ -363,18 +396,69 @@ def list_fabric_nodes(
TopoFabricNode.ume_ne_id.ilike(like),
)
)
role_v = str(role or "").strip().lower()
if role_v:
q = q.filter(TopoFabricNode.role == role_v)
region_v = str(region_folder_id or "").strip()
if region_v:
q = q.filter(TopoFabricNode.region_folder_id == region_v)
um = str(unmatched or "").strip().lower()
if um == "role":
q = q.filter(or_(TopoFabricNode.role == "", TopoFabricNode.role == "unknown"))
elif um == "region":
q = q.filter(
or_(TopoFabricNode.region_folder_id.is_(None), TopoFabricNode.region_folder_id == "")
)
elif um == "any":
q = q.filter(
or_(
TopoFabricNode.role == "",
TopoFabricNode.role == "unknown",
TopoFabricNode.region_folder_id.is_(None),
TopoFabricNode.region_folder_id == "",
)
)
ls = str(link_status or "").strip().lower()
if ls == "orphaned":
q = q.filter(
or_(TopoFabricNode.managed_ne_id.is_(None), TopoFabricNode.managed_ne_id == ""),
or_(TopoFabricNode.ume_ne_id.is_(None), TopoFabricNode.ume_ne_id == ""),
)
elif ls == "linked":
q = q.filter(
or_(
and_(TopoFabricNode.managed_ne_id.isnot(None), TopoFabricNode.managed_ne_id != ""),
and_(TopoFabricNode.ume_ne_id.isnot(None), TopoFabricNode.ume_ne_id != ""),
)
)
elif ls == "managed":
q = q.filter(
and_(TopoFabricNode.managed_ne_id.isnot(None), TopoFabricNode.managed_ne_id != ""),
or_(TopoFabricNode.ume_ne_id.is_(None), TopoFabricNode.ume_ne_id == ""),
)
elif ls == "ume":
q = q.filter(
and_(TopoFabricNode.ume_ne_id.isnot(None), TopoFabricNode.ume_ne_id != ""),
or_(TopoFabricNode.managed_ne_id.is_(None), TopoFabricNode.managed_ne_id == ""),
)
elif ls == "both":
q = q.filter(
and_(TopoFabricNode.managed_ne_id.isnot(None), TopoFabricNode.managed_ne_id != ""),
and_(TopoFabricNode.ume_ne_id.isnot(None), TopoFabricNode.ume_ne_id != ""),
)
total = int(q.count())
rows = (
q.order_by(TopoFabricNode.updated_at.desc())
q.order_by(TopoFabricNode.name.asc())
.offset((page - 1) * page_size)
.limit(page_size)
.all()
)
items = enrich_fabric_node_dicts(db, [_node_out(n).model_dump() for n in rows])
return {
"total": total,
"page": page,
"page_size": page_size,
"items": [_node_out(n).model_dump() for n in rows],
"items": items,
}
@ -632,15 +716,32 @@ def _apply_missing_and_purge(
# ---------------------------------------------------------------------------
# Views
# Folders (tree) + Views (leaf canvases)
# ---------------------------------------------------------------------------
def _folder_out(f: TopoFolder) -> TopologyFolderOut:
return TopologyFolderOut(
id=f.id,
parent_id=str(f.parent_id or ""),
kind=str(f.kind or "region"),
name=f.name or "",
sort_order=int(f.sort_order or 0),
is_system=bool(f.is_system),
created_at=f.created_at,
updated_at=f.updated_at,
)
def _view_out(v: TopoView, *, node_count: int = 0) -> TopologyViewOut:
return TopologyViewOut(
id=v.id,
name=v.name,
remark=v.remark or "",
folder_id=str(v.folder_id or ""),
kind=normalize_view_kind(getattr(v, "kind", None)),
role=normalize_view_role(v.role),
sort_order=int(v.sort_order or 0),
filter=dict(v.filter or {}),
viewport=dict(v.viewport or {}),
node_count=node_count,
@ -657,7 +758,281 @@ def _get_view_or_404(db: Session, view_id: str) -> TopoView:
return row
def _get_folder_or_404(db: Session, folder_id: str) -> TopoFolder:
fid = str(folder_id or "").strip()
row = db.get(TopoFolder, fid) if fid else None
if row is None:
raise HTTPException(status_code=404, detail="topology_folder_not_found")
return row
def ensure_region_physical_view(db: Session, folder_id: str, *, commit: bool = True) -> TopoView:
"""Ensure a site has exactly one default physical topology map."""
fid = str(folder_id or "").strip()
folder = _get_folder_or_404(db, fid)
if str(folder.kind or "") == "root":
raise HTTPException(status_code=400, detail="view_must_hang_under_region")
existing = (
db.query(TopoView)
.filter(TopoView.folder_id == folder.id, TopoView.kind == VIEW_KIND_PHYSICAL)
.order_by(TopoView.sort_order.asc(), TopoView.created_at.asc())
.first()
)
if existing is not None:
return existing
now = _utcnow()
role = "core"
row = TopoView(
id=uuid4().hex,
folder_id=folder.id,
parent_view_id=None,
kind=VIEW_KIND_PHYSICAL,
role=role,
name=PHYSICAL_VIEW_NAME,
remark="",
sort_order=0,
filter=merge_filter_with_membership({}, role=role, kind=VIEW_KIND_PHYSICAL),
viewport={},
created_at=now,
updated_at=now,
)
db.add(row)
if commit:
db.commit()
db.refresh(row)
else:
db.flush()
return row
def bootstrap_topology_tree(db: Session) -> dict[str, str]:
"""Ensure hidden system root; flatten legacy nesting; ensure physical map per site."""
now = _utcnow()
root = (
db.query(TopoFolder)
.filter(TopoFolder.kind == "root")
.order_by(TopoFolder.created_at.asc())
.first()
)
if root is None:
root = TopoFolder(
id=uuid4().hex,
parent_id=None,
kind="root",
name=ROOT_FOLDER_NAME,
sort_order=0,
is_system=True,
created_at=now,
updated_at=now,
)
db.add(root)
db.flush()
# Drop legacy auto-created Unassigned region when empty; otherwise demote to normal region.
legacy = (
db.query(TopoFolder)
.filter(
TopoFolder.kind == "region",
TopoFolder.name == _LEGACY_UNASSIGNED_NAME,
)
.all()
)
for folder in legacy:
view_cnt = db.query(TopoView).filter(TopoView.folder_id == folder.id).count()
if view_cnt == 0:
db.delete(folder)
elif bool(folder.is_system):
folder.is_system = False
folder.updated_at = now
# Flatten nesting + normalize kind for all views.
for v in db.query(TopoView).all():
changed = False
if v.parent_view_id:
v.parent_view_id = None
changed = True
kind = normalize_view_kind(getattr(v, "kind", None))
if str(getattr(v, "kind", "") or "") != kind:
v.kind = kind
changed = True
if not str(v.role or "").strip():
v.role = "core"
changed = True
filt = dict(v.filter or {})
if "membership" not in filt:
v.filter = merge_filter_with_membership(
filt, role=normalize_view_role(v.role), kind=kind
)
changed = True
if changed:
v.updated_at = now
# Ensure every region has a physical map.
regions = db.query(TopoFolder).filter(TopoFolder.kind == "region").all()
for region in regions:
ensure_region_physical_view(db, region.id, commit=False)
db.commit()
return {"root_id": root.id}
def create_folder(db: Session, body: TopologyFolderCreate) -> TopologyFolderOut:
bootstrap_topology_tree(db)
name = str(body.name or "").strip()
if not name:
raise HTTPException(status_code=400, detail="name_required")
kind = str(body.kind or "region").strip().lower()
if kind != "region":
raise HTTPException(status_code=400, detail="folder_kind_must_be_region")
root = db.query(TopoFolder).filter(TopoFolder.kind == "root").first()
if root is None:
raise HTTPException(status_code=500, detail="topology_root_missing")
parent_id = str(body.parent_id or "").strip() or root.id
parent = _get_folder_or_404(db, parent_id)
if str(parent.kind or "") != "root":
raise HTTPException(status_code=400, detail="region_must_hang_under_root")
now = _utcnow()
row = TopoFolder(
id=uuid4().hex,
parent_id=root.id,
kind="region",
name=name[:256],
sort_order=int(body.sort_order or 0),
is_system=False,
created_at=now,
updated_at=now,
)
db.add(row)
db.flush()
ensure_region_physical_view(db, row.id, commit=False)
db.commit()
db.refresh(row)
return _folder_out(row)
def update_folder(db: Session, folder_id: str, body: TopologyFolderUpdate) -> TopologyFolderOut:
row = _get_folder_or_404(db, folder_id)
if str(row.kind or "") == "root":
if body.parent_id is not None:
raise HTTPException(status_code=400, detail="cannot_reparent_root")
if body.name is not None:
name = str(body.name or "").strip()
if not name:
raise HTTPException(status_code=400, detail="name_required")
if bool(row.is_system) and str(row.kind or "") == "root":
row.name = name[:256]
elif bool(row.is_system):
raise HTTPException(status_code=400, detail="cannot_rename_system_folder")
else:
row.name = name[:256]
if body.sort_order is not None:
row.sort_order = int(body.sort_order)
if body.parent_id is not None and str(row.kind or "") == "region":
parent = _get_folder_or_404(db, body.parent_id)
if str(parent.kind or "") != "root":
raise HTTPException(status_code=400, detail="region_must_hang_under_root")
row.parent_id = parent.id
row.updated_at = _utcnow()
db.commit()
db.refresh(row)
return _folder_out(row)
def delete_folder(db: Session, folder_id: str, *, force: bool = False) -> dict[str, Any]:
"""Delete a region and cascade-delete its maps.
Every region has a default physical map, so folder delete must purge views
itself (cannot call ``delete_view``, which recreates physical).
``force`` is accepted for API compatibility; cascade always runs.
"""
row = _get_folder_or_404(db, folder_id)
if str(row.kind or "") == "root" or bool(row.is_system):
raise HTTPException(status_code=400, detail="cannot_delete_system_folder")
_ = force
views = db.query(TopoView).filter(TopoView.folder_id == row.id).all()
for v in views:
db.query(TopoViewEdgeStyle).filter(TopoViewEdgeStyle.view_id == v.id).delete(
synchronize_session=False
)
db.query(TopoViewNode).filter(TopoViewNode.view_id == v.id).delete(
synchronize_session=False
)
db.delete(v)
db.flush()
db.delete(row)
db.commit()
return {"ok": True, "folder_id": folder_id, "deleted": True}
def get_topology_tree(db: Session) -> TopologyTreeOut:
bootstrap_topology_tree(db)
folders = db.query(TopoFolder).order_by(TopoFolder.sort_order.asc(), TopoFolder.name.asc()).all()
views = db.query(TopoView).order_by(TopoView.sort_order.asc(), TopoView.name.asc()).all()
nc_map: dict[str, int] = {}
for vid, cnt in (
db.query(TopoViewNode.view_id, func.count(TopoViewNode.id))
.group_by(TopoViewNode.view_id)
.all()
):
nc_map[str(vid)] = int(cnt or 0)
by_parent: dict[str, list[TopoFolder]] = {}
root: TopoFolder | None = None
for f in folders:
if str(f.kind or "") == "root":
root = f
continue
pid = str(f.parent_id or "")
by_parent.setdefault(pid, []).append(f)
views_by_folder: dict[str, list[TopoView]] = {}
for v in views:
views_by_folder.setdefault(str(v.folder_id or ""), []).append(v)
def _flat_views(folder_views: list[TopoView]) -> list[TopologyTreeViewOut]:
# physical first, then custom; stable by sort_order/name.
ordered = sorted(
folder_views,
key=lambda x: (
0 if normalize_view_kind(getattr(x, "kind", None)) == VIEW_KIND_PHYSICAL else 1,
int(x.sort_order or 0),
x.name or "",
x.id,
),
)
return [
TopologyTreeViewOut(
id=v.id,
name=v.name or "",
kind=normalize_view_kind(getattr(v, "kind", None)),
role=normalize_view_role(v.role),
sort_order=int(v.sort_order or 0),
node_count=nc_map.get(v.id, 0),
updated_at=v.updated_at,
)
for v in ordered
]
def _build(folder: TopoFolder) -> TopologyTreeFolderOut:
kids = [_build(c) for c in by_parent.get(folder.id, [])]
return TopologyTreeFolderOut(
id=folder.id,
parent_id=str(folder.parent_id or ""),
kind=str(folder.kind or "region"),
name=folder.name or "",
sort_order=int(folder.sort_order or 0),
is_system=bool(folder.is_system),
views=_flat_views(views_by_folder.get(folder.id, [])),
children=kids,
)
if root is None:
return TopologyTreeOut(root=None)
return TopologyTreeOut(root=_build(root))
def list_views(db: Session) -> dict[str, Any]:
bootstrap_topology_tree(db)
rows = db.query(TopoView).order_by(TopoView.updated_at.desc()).all()
items = []
for v in rows:
@ -670,12 +1045,34 @@ def create_view(db: Session, body: TopologyViewCreate) -> TopologyViewOut:
name = str(body.name or "").strip()
if not name:
raise HTTPException(status_code=400, detail="name_required")
role = normalize_view_role(body.role)
kind = normalize_view_kind(body.kind)
folder_id = str(body.folder_id or "").strip()
if not folder_id:
raise HTTPException(status_code=400, detail="folder_id_required")
folder = _get_folder_or_404(db, folder_id)
if str(folder.kind or "") == "root":
raise HTTPException(status_code=400, detail="view_must_hang_under_region")
if kind == VIEW_KIND_PHYSICAL:
existing = (
db.query(TopoView)
.filter(TopoView.folder_id == folder.id, TopoView.kind == VIEW_KIND_PHYSICAL)
.first()
)
if existing is not None:
raise HTTPException(status_code=400, detail="region_already_has_physical_view")
filt = merge_filter_with_membership(dict(body.filter or {}), role=role, kind=kind)
now = _utcnow()
row = TopoView(
id=uuid4().hex,
folder_id=folder_id,
parent_view_id=None,
kind=kind,
role=role,
name=name[:256],
remark=str(body.remark or "")[:1024],
filter=dict(body.filter or {}),
sort_order=int(body.sort_order or 0),
filter=filt,
viewport={},
created_at=now,
updated_at=now,
@ -695,8 +1092,38 @@ def update_view(db: Session, view_id: str, body: TopologyViewUpdate) -> Topology
row.name = name[:256]
if body.remark is not None:
row.remark = str(body.remark or "")[:1024]
if body.role is not None:
row.role = normalize_view_role(body.role)
if body.sort_order is not None:
row.sort_order = int(body.sort_order)
if body.folder_id is not None:
fid = str(body.folder_id or "").strip()
folder = _get_folder_or_404(db, fid)
if str(folder.kind or "") == "root":
raise HTTPException(status_code=400, detail="view_must_hang_under_region")
row.folder_id = folder.id
if body.kind is not None:
new_kind = normalize_view_kind(body.kind)
if new_kind == VIEW_KIND_PHYSICAL and normalize_view_kind(row.kind) != VIEW_KIND_PHYSICAL:
clash = (
db.query(TopoView)
.filter(
TopoView.folder_id == row.folder_id,
TopoView.kind == VIEW_KIND_PHYSICAL,
TopoView.id != row.id,
)
.first()
)
if clash is not None:
raise HTTPException(status_code=400, detail="region_already_has_physical_view")
row.kind = new_kind
row.parent_view_id = None
if body.filter is not None:
row.filter = dict(body.filter or {})
row.filter = merge_filter_with_membership(
dict(body.filter or {}),
role=normalize_view_role(row.role),
kind=normalize_view_kind(row.kind),
)
if body.viewport is not None:
row.viewport = dict(body.viewport or {})
row.updated_at = _utcnow()
@ -706,13 +1133,20 @@ def update_view(db: Session, view_id: str, body: TopologyViewUpdate) -> Topology
return _view_out(row, node_count=nc)
def delete_view(db: Session, view_id: str) -> dict[str, Any]:
def delete_view(db: Session, view_id: str, *, force: bool = False) -> dict[str, Any]:
row = _get_view_or_404(db, view_id)
folder_id = str(row.folder_id or "")
is_physical = normalize_view_kind(row.kind) == VIEW_KIND_PHYSICAL
if is_physical and not force:
raise HTTPException(status_code=400, detail="cannot_delete_physical_view")
db.query(TopoViewEdgeStyle).filter(TopoViewEdgeStyle.view_id == row.id).delete(
synchronize_session=False
)
db.query(TopoViewNode).filter(TopoViewNode.view_id == row.id).delete(synchronize_session=False)
db.delete(row)
db.flush()
if folder_id and is_physical:
ensure_region_physical_view(db, folder_id, commit=False)
db.commit()
return {"ok": True, "view_id": view_id, "deleted": True}
@ -817,15 +1251,132 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut:
discovered_at=e.discovered_at,
)
)
outside = _outside_peers_for_view(db, view, member_ids=set(fids), layer=layer)
return TopologyViewGraphOut(
view=_view_out(view, node_count=len(nodes_out)),
nodes=nodes_out,
edges=edges_out,
truncated=truncated,
truncate_reason=reason,
outside_peers=outside,
)
def _membership_for_view(view: TopoView) -> dict[str, Any]:
return parse_membership(dict(view.filter or {}), role=normalize_view_role(view.role))
def _fabric_in_hard_scope(db: Session, fn: TopoFabricNode, mem: dict[str, Any]) -> bool:
"""If hard scope filters are set, node must match ALL set dimensions (AND)."""
if not has_hard_scope(mem):
return True
mid = str(fn.managed_ne_id or "").strip()
allowed_mids = set(mem.get("managed_ne_ids") or [])
if allowed_mids and mid not in allowed_mids:
return False
vendors = {str(x).lower() for x in (mem.get("vendors") or [])}
if vendors and str(fn.vendor or "").strip().lower() not in vendors:
return False
dtypes = {str(x).lower() for x in (mem.get("device_types") or [])}
if dtypes and str(fn.device_type or "").strip().lower() not in dtypes:
return False
keyword = str(mem.get("keyword") or "").strip().lower()
if keyword:
blob = f"{fn.name or ''} {fn.ip or ''}".lower()
if keyword not in blob:
return False
tags_any = {str(x).lower() for x in (mem.get("tags_any") or [])}
if tags_any:
ne = db.get(ManagedNE, mid) if mid else None
tag_blob = str(getattr(ne, "tags", "") or "").lower() if ne else ""
if not any(t in tag_blob for t in tags_any):
return False
return True
def _outside_peers_for_view(
db: Session,
view: TopoView,
*,
member_ids: set[str],
layer: str,
limit: int = 50,
) -> list[dict[str, str]]:
if not member_ids:
return []
edges = (
db.query(TopoFabricEdge)
.filter(
TopoFabricEdge.layer == layer,
or_(
TopoFabricEdge.a_node_id.in_(list(member_ids)),
TopoFabricEdge.b_node_id.in_(list(member_ids)),
),
)
.limit(5000)
.all()
)
out: list[dict[str, str]] = []
seen: set[str] = set()
for e in edges:
for peer_id, local_id in ((e.b_node_id, e.a_node_id), (e.a_node_id, e.b_node_id)):
if local_id not in member_ids or peer_id in member_ids:
continue
if peer_id in seen:
continue
seen.add(peer_id)
fn = db.get(TopoFabricNode, peer_id)
out.append(
{
"fabric_node_id": peer_id,
"name": (fn.name if fn else "") or "",
"ip": (fn.ip if fn else "") or "",
"via_node_id": local_id,
}
)
if len(out) >= limit:
return out
return out
def _place_fabric_ids_on_view(
db: Session,
view: TopoView,
fabric_ids: list[str],
*,
existing: set[str],
) -> int:
now = _utcnow()
added = 0
vnodes = db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all()
max_x = max((float(vn.x or 0) for vn in vnodes), default=40.0)
base_x = max_x + 200.0
cols = max(1, int(len(fabric_ids) ** 0.5) or 1)
for i, fid in enumerate(fabric_ids):
if fid in existing or db.get(TopoFabricNode, fid) is None:
continue
x = base_x + (i % cols) * 180.0
y = 40.0 + (i // cols) * 120.0
db.add(
TopoViewNode(
id=uuid4().hex,
view_id=view.id,
fabric_node_id=fid,
x=x,
y=y,
label="",
locked=False,
created_at=now,
updated_at=now,
)
)
existing.add(fid)
added += 1
if added:
view.updated_at = now
return added
def patch_view_positions(
db: Session, view_id: str, body: ViewPositionsPatch
) -> TopologyViewGraphOut:
@ -878,11 +1429,15 @@ def patch_view_positions(
def add_nodes_to_view(db: Session, view_id: str, body: ViewNodesAdd) -> TopologyViewGraphOut:
view = _get_view_or_404(db, view_id)
mem = _membership_for_view(view)
max_nodes = int(mem.get("max_nodes") or 300)
now = _utcnow()
existing = {
vn.fabric_node_id
for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all()
}
if len(existing) >= max_nodes:
raise HTTPException(status_code=400, detail="membership_max_nodes")
added_ids: list[str] = []
for mid in body.managed_ne_ids or []:
mid_s = str(mid or "").strip()
@ -919,6 +1474,11 @@ def add_nodes_to_view(db: Session, view_id: str, body: ViewNodesAdd) -> Topology
continue
added_ids.append(fid_s)
existing.add(fid_s)
# `existing` already includes ids in added_ids; cap new placements.
original_count = len(existing) - len(added_ids)
room = max(0, max_nodes - original_count)
if len(added_ids) > room:
added_ids = added_ids[:room]
cols = max(1, int(len(added_ids) ** 0.5) or 1)
for i, fid in enumerate(added_ids):
x = (i % cols) * 180.0 + 40.0
@ -941,19 +1501,51 @@ def add_nodes_to_view(db: Session, view_id: str, body: ViewNodesAdd) -> Topology
return get_view_graph(db, view.id)
def _neighbor_ids(
db: Session, *, seed_ids: set[str], layer: str, hops: int
) -> set[str]:
frontier = set(seed_ids)
found: set[str] = set()
for _ in range(max(0, hops)):
if not frontier:
break
rows = (
db.query(TopoFabricEdge)
.filter(
TopoFabricEdge.layer == layer,
or_(
TopoFabricEdge.a_node_id.in_(list(frontier)),
TopoFabricEdge.b_node_id.in_(list(frontier)),
),
)
.all()
)
nxt: set[str] = set()
for e in rows:
for a, b in ((e.a_node_id, e.b_node_id), (e.b_node_id, e.a_node_id)):
if a in frontier and b not in seed_ids and b not in found:
nxt.add(b)
found |= nxt
frontier = nxt
return found
def project_fabric_neighbors_to_view(db: Session, view_id: str) -> TopologyViewGraphOut:
"""Add fabric neighbors of current view nodes onto the view so edges can render."""
# Collapse duplicate fabric nodes first (fixes R1/r1 + twin R2 after raced discovers).
"""Add in-scope fabric neighbors onto the leaf view (bounded by membership)."""
merge_duplicate_fabric_nodes(db)
view = _get_view_or_404(db, view_id)
vnodes = db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all()
mem = _membership_for_view(view)
if bool(mem.get("frozen")):
return get_view_graph(db, view.id)
# Drop view placements that still point at LLDP orphans (no inventory link).
orphan_vns = []
for vn in vnodes:
fn = db.get(TopoFabricNode, vn.fabric_node_id)
if fn is None or not _is_inventory_node(fn):
orphan_vns.append(vn)
max_nodes = int(mem.get("max_nodes") or 300)
hops = int(mem.get("expand_hops") or 1)
filt = dict(view.filter or {})
layer = str(filt.get("layer") or "physical").strip() or "physical"
vnodes = db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all()
# Drop placements pointing at missing fabric rows only (keep LLDP placeholders).
orphan_vns = [vn for vn in vnodes if db.get(TopoFabricNode, vn.fabric_node_id) is None]
if orphan_vns:
for vn in orphan_vns:
db.delete(vn)
@ -964,60 +1556,130 @@ def project_fabric_neighbors_to_view(db: Session, view_id: str) -> TopologyViewG
existing = {vn.fabric_node_id for vn in vnodes}
if not existing:
return get_view_graph(db, view.id)
if len(existing) >= max_nodes:
g = get_view_graph(db, view.id)
g.truncated = True
g.truncate_reason = g.truncate_reason or "membership_cap"
return g
filt = dict(view.filter or {})
layer = str(filt.get("layer") or "physical").strip() or "physical"
peer_ids: set[str] = set()
for fid in existing:
rows = (
db.query(TopoFabricEdge)
.filter(
TopoFabricEdge.layer == layer,
or_(TopoFabricEdge.a_node_id == fid, TopoFabricEdge.b_node_id == fid),
peer_ids = _neighbor_ids(db, seed_ids=existing, layer=layer, hops=hops)
to_add: list[str] = []
for peer in sorted(peer_ids):
if peer in existing:
continue
fn = db.get(TopoFabricNode, peer)
if fn is None or not _is_inventory_node(fn):
continue
if _fabric_match_score(db, fn) < 2:
continue
if not _fabric_in_hard_scope(db, fn, mem):
continue
to_add.append(peer)
if len(existing) + len(to_add) >= max_nodes:
break
truncated = len(peer_ids) > len(to_add)
if to_add:
_place_fabric_ids_on_view(db, view, to_add, existing=existing)
db.commit()
g = get_view_graph(db, view.id)
if truncated:
g.truncated = True
g.truncate_reason = g.truncate_reason or "membership_cap"
return g
def populate_view(db: Session, view_id: str, body: ViewPopulateRequest) -> ViewPopulateOut:
"""Resolve membership candidates and optionally place them on the leaf view."""
view = _get_view_or_404(db, view_id)
role = normalize_view_role(view.role)
if body.membership is not None:
filt = merge_filter_with_membership(
dict(view.filter or {}), role=role, membership=parse_membership(
{"membership": body.membership}, role=role
)
.all()
)
for edge in rows:
peer = edge.b_node_id if edge.a_node_id == fid else edge.a_node_id
if not peer or peer in existing:
if not body.dry_run:
view.filter = filt
mem = parse_membership(dict(view.filter or {}), role=role)
max_nodes = int(mem.get("max_nodes") or 300)
hops = int(mem.get("expand_hops") or 1)
layer = str((view.filter or {}).get("layer") or "physical").strip() or "physical"
existing = {
vn.fabric_node_id
for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all()
}
seeds = set(mem.get("seed_fabric_node_ids") or []) | set(existing)
# Seed from managed_ne_ids
for mid in mem.get("managed_ne_ids") or []:
ne = db.get(ManagedNE, mid)
if ne is None:
continue
fn = ensure_fabric_node_for_managed(db, ne)
seeds.add(fn.id)
# Hard-scope scan when filters present
candidates: set[str] = set(seeds)
if has_hard_scope(mem):
for fn in db.query(TopoFabricNode).all():
if not _is_inventory_node(fn):
continue
if _fabric_in_hard_scope(db, fn, mem):
candidates.add(fn.id)
if hops > 0 and seeds:
for peer in _neighbor_ids(db, seed_ids=seeds, layer=layer, hops=hops):
fn = db.get(TopoFabricNode, peer)
# Project real inventory + LLDP placeholders; skip WebCRT twins / orphans.
if fn is None or not _is_inventory_node(fn):
continue
if has_hard_scope(mem) and not _fabric_in_hard_scope(db, fn, mem):
continue
if _fabric_match_score(db, fn) < 2:
continue
peer_ids.add(peer)
candidates.add(peer)
if not peer_ids:
return get_view_graph(db, view.id)
ordered = sorted(candidates)
truncated = len(ordered) > max_nodes
ordered = ordered[:max_nodes]
would_add = [fid for fid in ordered if fid not in existing]
now = _utcnow()
added = sorted(peer_ids)
cols = max(1, int(len(added) ** 0.5) or 1)
max_x = max((float(vn.x or 0) for vn in vnodes), default=40.0)
base_x = max_x + 200.0
for i, fid in enumerate(added):
if db.get(TopoFabricNode, fid) is None:
continue
x = base_x + (i % cols) * 180.0
y = 40.0 + (i // cols) * 120.0
db.add(
TopoViewNode(
id=uuid4().hex,
view_id=view.id,
fabric_node_id=fid,
x=x,
y=y,
label="",
locked=False,
created_at=now,
updated_at=now,
)
outside = _outside_peers_for_view(db, view, member_ids=set(ordered), layer=layer)
if body.dry_run:
return ViewPopulateOut(
view_id=view.id,
dry_run=True,
candidate_count=len(candidates),
would_add=len(would_add),
added=0,
max_nodes=max_nodes,
truncated=truncated,
outside_peers=outside,
graph=None,
)
view.updated_at = now
added = _place_fabric_ids_on_view(db, view, would_add, existing=existing)
if body.freeze_after:
mem["frozen"] = True
view.filter = merge_filter_with_membership(dict(view.filter or {}), role=role, membership=mem)
view.updated_at = _utcnow()
db.commit()
return get_view_graph(db, view.id)
g = get_view_graph(db, view.id)
if truncated:
g.truncated = True
g.truncate_reason = g.truncate_reason or "membership_cap"
return ViewPopulateOut(
view_id=view.id,
dry_run=False,
candidate_count=len(candidates),
would_add=len(would_add),
added=added,
max_nodes=max_nodes,
truncated=truncated,
outside_peers=g.outside_peers,
graph=g,
)
def remove_view_nodes(db: Session, view_id: str, fabric_node_ids: list[str]) -> TopologyViewGraphOut:

21
netx_api/ume_sync_service.py
View file

@ -564,14 +564,29 @@ def sync_inventory_full(db: Session, client: UMEClient, *, trigger_mode: str = "
db.flush()
deleted_ne = 0
if _snapshot_reconcile_ok(inv_meta):
from .topology_inventory_lifecycle import detach_fabric_from_ume
if seen_ne_ids:
stale_ids = [
str(x[0])
for x in db.query(UmeInventoryNE.ne_id)
.filter(~UmeInventoryNE.ne_id.in_(list(seen_ne_ids)))
.all()
if str(x[0] or "").strip()
]
else:
stale_ids = [
str(x[0])
for x in db.query(UmeInventoryNE.ne_id).all()
if str(x[0] or "").strip()
]
if stale_ids:
detach_fabric_from_ume(db, stale_ids)
deleted_ne = int(
db.query(UmeInventoryNE)
.filter(~UmeInventoryNE.ne_id.in_(list(seen_ne_ids)))
.filter(UmeInventoryNE.ne_id.in_(stale_ids))
.delete(synchronize_session=False)
)
else:
deleted_ne = int(db.query(UmeInventoryNE).delete(synchronize_session=False))
job.details_json = json.dumps(
{