Improve topology UX, traffic discover logs, and CLI wrap parsing.

Paginate LLDP job items, surface port-traffic raw logs with a 300s default, harden topology discover/edit/zoom/fullscreen, and join mid-token CLI wraps before TextFSM.

Co-authored-by: Cursor <cursoragent@cursor.com>

netx_api/cli_templates/README.md

@@ -18,10 +18,16 @@ cli_templates/
 
 ## Lookup order
 
-1. **This tree** (`index` → `vendor/*.textfsm`)
-2. Community `ntc-templates` package
+1. **Wrap-line join** (`netx_api.cli_wrap`) — flatten last-column wraps for registered
+   platform+command pairs (ZTE interface brief / LLDP brief today). Other vendors can
+   register rules the same way; do **not** apply blindly (Huawei LLDP detail uses real
+   indented fields).
+2. **This tree** (`index` → `vendor/*.textfsm`)
+3. Community `ntc-templates` package
 
-No regex CLI parsers. If both miss, the call returns empty — fix or add a template.
+No regex CLI parsers for field extraction. If both template trees miss, the call returns
+empty — fix or add a template. Wrap joining is the supported escape hatch when TextFSM
+alone cannot merge mid-token column wraps.
 
 ## Vendor coverage (blind fill from community + NetX)
 
@@ -45,7 +51,10 @@ No regex CLI parsers. If both miss, the call returns empty — fix or add a temp
 3. Register in root `index` with a **relative path**:
    `zte/zte_zxros_show_xxx.textfsm, .*, zte_zxros, sh[[ow]] ...`
 4. Align Value names with community templates when possible
-5. Add unit tests under `tests/`
+5. If the CLI wraps the last column onto the next line, add a `CliWrapRule` in
+   `netx_api/cli_wrap.py` (platform + command scoped) — do not try to solve mid-token
+   wraps in TextFSM alone
+6. Add unit tests under `tests/`
 
 When community already works, no file is required.
 When community is wrong/incomplete for our lab, add an override under the vendor folder
@@ -54,4 +63,4 @@ with the same Platform+Command so NetX wins.
 ## Concurrency
 
 Template files are read-only and safe across processes. Parsing goes through
-`netx_api.ntc_parse` (fresh `CliTable` + process lock; never share instances across threads).
+`netx_api.ntc_parse` (wrap-join → fresh `CliTable` + process lock; never share instances across threads).

netx_api/cli_templates/zte/zte_zxros_show_interface_brief.textfsm

@@ -8,13 +8,13 @@ Value PROT (up|down)
 Value DESCRIPTION (.*)
 
 Start
+  ^Interface\s+\S+\s+Mode\s+.*Admin -> Ports
   ^Interface\s+Attribute\s+Mode -> Ports
   ^\s*$$
   ^.* -> Start
 
 Ports
-  ^${INTERFACE}\s+${ATTRIBUTE}\s+${MODE}\s+${BW}\s+${ADMIN}\s+${PHY}\s+${PROT}(?:\s+${DESCRIPTION})?\s*$$ -> Record
-  ^${INTERFACE}\s+${ATTRIBUTE}\s+${MODE}\s+${ADMIN}\s+${PHY}\s+${PROT}(?:\s+${DESCRIPTION})?\s*$$ -> Record
+  ^${INTERFACE}\s+${ATTRIBUTE}\s+${MODE}(?:\s+${BW})?\s+${ADMIN}\s+${PHY}\s+${PROT}(?:\s+${DESCRIPTION})?\s*$$ -> Record
   ^\s*$$
   ^\S+\s*#\s*$$ -> End
   ^.*#\s*$$ -> End

netx_api/cli_wrap.py

@@ -0,0 +1,124 @@
+"""Pre-TextFSM CLI wrap-line joiner (vendor-agnostic).
+
+Many vendor CLIs wrap the last column (description / system name) onto the next
+line with leading spaces. Pure TextFSM cannot reliably merge those mid-token wraps
+without creating duplicate or incomplete records — especially when the wrap
+breaks inside a hostname (``AL5458-ACC-612`` + ``0HS``).
+
+This module flattens matching wrap lines **before** TextFSM runs. Rules are
+registered per platform + command so indented multi-field blocks (e.g. Huawei
+LLDP detail) are left alone.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class CliWrapRule:
+    """Join indented continuation lines into the previous table row."""
+
+    id: str
+    platforms: frozenset[str]
+    # Match ntc / NetX command string (already the resolved show/display command).
+    command: re.Pattern[str]
+    # A data row that may be continued (typically starts with interface name).
+    record_line: re.Pattern[str]
+    # Continuation fragment: leading whitespace + non-empty payload.
+    cont_line: re.Pattern[str]
+    # ZTE column wraps usually split mid-token → join with "" ; use " " if needed.
+    join_sep: str = ""
+    # Lines that must never be treated as records or continuations.
+    skip_line: re.Pattern[str] | None = None
+
+
+# ZTE ZXROS table rows: interface-like token at column 0.
+_ZTE_RECORD = re.compile(r"^(?P<head>[A-Za-z][\w./:-]*)\s+\S+")
+_ZTE_CONT = re.compile(r"^\s{8,}(?P<tail>\S.*?)\s*$")
+_ZTE_SKIP = re.compile(
+    r"^(?:"
+    r"-{3,}"
+    r"|Local\s+Interface\b"
+    r"|Interface\s+\S+\s+Mode\b"
+    r"|Interface\s+Attribute\b"
+    r"|Scope\s+codes\s*:"
+    r"|Total\s+neighbors\b"
+    r"|NB\s+=\s+"
+    r"|NC\s+=\s+"
+    r"|NTPMR\s+=\s+"
+    r"|PHY:\s*"
+    r"|.*#\s*$"
+    r")",
+    re.I,
+)
+
+CLI_WRAP_RULES: tuple[CliWrapRule, ...] = (
+    CliWrapRule(
+        id="zte_zxros_show_interface_brief",
+        platforms=frozenset({"zte_zxros"}),
+        command=re.compile(r"^show\s+interface\s+brief\b", re.I),
+        record_line=_ZTE_RECORD,
+        cont_line=_ZTE_CONT,
+        join_sep="",
+        skip_line=_ZTE_SKIP,
+    ),
+    CliWrapRule(
+        id="zte_zxros_show_lldp_neighbor_brief",
+        platforms=frozenset({"zte_zxros"}),
+        command=re.compile(r"^show\s+lldp\s+neighbor(?:s)?\s+brief\b", re.I),
+        record_line=_ZTE_RECORD,
+        cont_line=_ZTE_CONT,
+        join_sep="",
+        skip_line=_ZTE_SKIP,
+    ),
+)
+
+
+def _rules_for(*, platform: str, command: str) -> list[CliWrapRule]:
+    plat = str(platform or "").strip().lower()
+    cmd = str(command or "").strip()
+    if not plat or not cmd:
+        return []
+    return [
+        rule
+        for rule in CLI_WRAP_RULES
+        if plat in rule.platforms and rule.command.search(cmd)
+    ]
+
+
+def _is_skip(line: str, rule: CliWrapRule) -> bool:
+    if not rule.skip_line:
+        return False
+    if rule.skip_line.match(line):
+        return True
+    stripped = line.strip()
+    return bool(stripped and rule.skip_line.match(stripped))
+
+
+def _apply_rule(text: str, rule: CliWrapRule) -> str:
+    lines = str(text or "").splitlines()
+    if not lines:
+        return str(text or "")
+    out: list[str] = []
+    for ln in lines:
+        if _is_skip(ln, rule):
+            out.append(ln)
+            continue
+        cont = rule.cont_line.match(ln)
+        if cont and out:
+            prev = out[-1]
+            if rule.record_line.match(prev) and not _is_skip(prev, rule):
+                out[-1] = f"{prev.rstrip()}{rule.join_sep}{cont.group('tail')}"
+                continue
+        out.append(ln)
+    return "\n".join(out)
+
+
+def apply_cli_wrap(text: str, *, platform: str = "", command: str = "") -> str:
+    """Return text with registered wrap continuations flattened for TextFSM."""
+    raw = str(text or "")
+    for rule in _rules_for(platform=platform, command=command):
+        raw = _apply_rule(raw, rule)
+    return raw

netx_api/lldp_collect_router.py

@@ -53,5 +53,10 @@ def api_list_jobs(
 
 
 @router.get("/jobs/{job_id}")
-def api_get_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
-    return get_job_detail(db, job_id)
+def api_get_job(
+    job_id: str,
+    page: int = Query(default=1, ge=1),
+    page_size: int = Query(default=20, ge=1, le=100),
+    db: Session = Depends(get_db),
+) -> dict[str, Any]:
+    return get_job_detail(db, job_id, page=page, page_size=page_size)

netx_api/lldp_collect_service.py

@@ -287,5 +287,7 @@ def list_jobs(db: Session, *, page: int = 1, page_size: int = 20) -> dict:
     }
 
 
-def get_job_detail(db: Session, job_id: str) -> dict:
-    return get_discover_job(db, job_id).model_dump()
+def get_job_detail(
+    db: Session, job_id: str, *, page: int | None = None, page_size: int | None = None
+) -> dict:
+    return get_discover_job(db, job_id, page=page, page_size=page_size).model_dump()

netx_api/models/port_traffic.py

@@ -33,7 +33,7 @@ class PortTrafficDevice(Base):
     vendor: Mapped[str] = mapped_column(String(64), default="")
     note: Mapped[str] = mapped_column(String(256), default="")  # optional remark
     status: Mapped[str] = mapped_column(String(32), default="draft", index=True)  # draft|running|paused|stopped
-    interval_sec: Mapped[int] = mapped_column(Integer, default=60)
+    interval_sec: Mapped[int] = mapped_column(Integer, default=300)
     retention_days: Mapped[int] = mapped_column(Integer, default=7)
     concurrency: Mapped[int] = mapped_column(Integer, default=1)
     collect_running: Mapped[bool] = mapped_column(Boolean, default=False)

netx_api/ntc_parse.py

@@ -13,6 +13,8 @@ from functools import lru_cache
 from pathlib import Path
 from typing import Any, Callable, TypeVar
 
+from .cli_wrap import apply_cli_wrap
+
 _log = logging.getLogger("netx.ntc_parse")
 
 _PARSE_LOCK = threading.Lock()
@@ -107,10 +109,10 @@ def parse_cli(
     command: str,
     text: str,
 ) -> list[dict[str, Any]]:
-    """Parse CLI text: custom templates first, then community ntc-templates."""
+    """Parse CLI text: wrap-join → custom templates → community ntc-templates."""
     plat = str(platform or "").strip()
     cmd = str(command or "").strip()
-    raw = str(text or "")
+    raw = apply_cli_wrap(str(text or ""), platform=plat, command=cmd)
     if not plat or not cmd or not raw.strip():
         return []
 

netx_api/port_traffic_devices.py

@@ -481,30 +481,58 @@ def discover_ports(db: Session, body: DiscoverPortsRequest) -> DiscoverPortsResp
             detail=f"vendor_not_supported_for_port_traffic: {vendor or 'unknown'}",
         )
 
-    per_cmd = int(settings.ne_collect_read_timeout_sec or 120)
-    conn = open_netmiko_connection(creds, session_timeout=per_cmd + 60)
-    try:
-        raw = send_show_command(conn, cmds.brief, read_timeout=per_cmd)
-    finally:
-        close_netmiko_connection(conn)
-
-    ports = [
-        DiscoverPortItem(**brief_port_to_dict(p))
-        for p in parse_interface_brief(
-            raw,
-            cmds.vendor_key,
-            command=cmds.brief,
-            device_type=device_type,
-        )
-    ]
-    return DiscoverPortsResponse(
+    command = str(cmds.brief or "")
+    base = DiscoverPortsResponse(
         source=body.source,
         id=body.id,
         ne_name=ne_name,
         ne_ip=ne_ip,
         vendor=vendor,
         vendor_key=cmds.vendor_key,
-        ports=ports,
+        command=command,
     )
 
+    per_cmd = int(settings.ne_collect_read_timeout_sec or 120)
+    raw = ""
+    try:
+        conn = open_netmiko_connection(creds, session_timeout=per_cmd + 60)
+        try:
+            raw = send_show_command(conn, command, read_timeout=per_cmd)
+        finally:
+            close_netmiko_connection(conn)
+    except Exception as exc:
+        preview = str(raw or "")
+        if len(preview) > 12_000:
+            preview = f"{preview[:12_000]}\n...[truncated preview 12000/{len(raw)} chars]"
+        base.ok = False
+        base.error = f"cli_failed: {exc}"
+        base.raw_preview = preview
+        return base
+
+    preview = str(raw or "")
+    if len(preview) > 12_000:
+        preview = f"{preview[:12_000]}\n...[truncated preview 12000/{len(raw)} chars]"
+    base.raw_preview = preview
+
+    try:
+        ports = [
+            DiscoverPortItem(**brief_port_to_dict(p))
+            for p in parse_interface_brief(
+                raw,
+                cmds.vendor_key,
+                command=command,
+                device_type=device_type,
+            )
+        ]
+    except Exception as exc:
+        base.ok = False
+        base.error = f"parse_failed: {exc}"
+        return base
+
+    base.ports = ports
+    if not ports:
+        base.ok = False
+        base.error = "no_ports_parsed"
+    return base
+
 

netx_api/port_traffic_schemas.py

@@ -34,7 +34,7 @@ class PortTrafficDeviceCreate(BaseModel):
     ne_ip: str = ""
     vendor: str = ""
     note: str = Field(default="", max_length=256)
-    interval_sec: int = Field(default=60, ge=15, le=3600)
+    interval_sec: int = Field(default=300, ge=15, le=3600)
     retention_days: int = Field(default=7, ge=1, le=90)
     concurrency: int = Field(default=1, ge=1, le=5)
     interfaces: list[PortTrafficIfaceIn] = Field(default_factory=list)
@@ -147,6 +147,10 @@ class DiscoverPortsResponse(BaseModel):
     vendor: str = ""
     vendor_key: str = ""
     ports: list[DiscoverPortItem] = Field(default_factory=list)
+    ok: bool = True
+    command: str = ""
+    raw_preview: str = ""
+    error: str = ""
 
 
 class PortTrafficSamplePoint(BaseModel):

netx_api/topology_discover_common.py

@@ -24,15 +24,33 @@ def _raw_preview(raw: str, *, limit: int = _RAW_PREVIEW_MAX) -> str:
     return f"{text[:limit]}\n...[truncated preview {limit}/{len(text)} chars]"
 
 
-def _job_out(db: Session, job: TopoDiscoverJob, *, include_items: bool = True) -> FabricDiscoverJobOut:
+def _job_out(
+    db: Session,
+    job: TopoDiscoverJob,
+    *,
+    include_items: bool = True,
+    page: int | None = None,
+    page_size: int | None = None,
+) -> FabricDiscoverJobOut:
     items_out: list[FabricDiscoverJobItemOut] = []
+    items_total = 0
+    items_page = 1
+    items_page_size = 0
     if include_items:
-        items = (
+        q = (
             db.query(TopoDiscoverJobItem)
             .filter(TopoDiscoverJobItem.job_id == job.id)
             .order_by(TopoDiscoverJobItem.created_at.asc())
-            .all()
         )
+        items_total = int(q.count())
+        if page is not None and page_size is not None:
+            items_page = max(1, int(page or 1))
+            items_page_size = max(1, min(100, int(page_size or 20)))
+            items = q.offset((items_page - 1) * items_page_size).limit(items_page_size).all()
+        else:
+            items = q.all()
+            items_page = 1
+            items_page_size = items_total
         for it in items:
             unmatched = [
                 FabricDiscoverUnmatched.model_validate(x) for x in (it.unmatched_json or [])[:40]
@@ -74,14 +92,23 @@ def _job_out(db: Session, job: TopoDiscoverJob, *, include_items: bool = True) -
         started_at=job.started_at,
         ended_at=job.ended_at,
         items=items_out,
+        items_total=items_total,
+        items_page=items_page,
+        items_page_size=items_page_size,
     )
 
 
-def get_discover_job(db: Session, job_id: str) -> FabricDiscoverJobOut:
+def get_discover_job(
+    db: Session,
+    job_id: str,
+    *,
+    page: int | None = None,
+    page_size: int | None = None,
+) -> FabricDiscoverJobOut:
     job = db.get(TopoDiscoverJob, str(job_id or "").strip())
     if job is None:
         raise HTTPException(status_code=404, detail="discover_job_not_found")
-    return _job_out(db, job)
+    return _job_out(db, job, page=page, page_size=page_size)
 
 
 def _ume_target_dict(db: Session, uid: str, default_profile: Any) -> dict[str, str] | None:

netx_api/topology_schemas.py

@@ -132,6 +132,9 @@ class FabricDiscoverJobOut(BaseModel):
     started_at: datetime | None = None
     ended_at: datetime | None = None
     items: list[FabricDiscoverJobItemOut] = Field(default_factory=list)
+    items_total: int = 0
+    items_page: int = 1
+    items_page_size: int = 0
 
 
 # ---------------------------------------------------------------------------