feat(ops): add managed NE management and batch CLI collection

Introduce workbench operations for multi-vendor NE CRUD/connect-test and Netmiko batch collection with job lifecycle controls, log downloads, and paginated run filters.

Co-authored-by: Cursor <cursoragent@cursor.com>

netx_api/collection_job_state.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy.orm import Session
+
+from .config import settings
+from .models import NeCollectionJob, NeCollectionRun
+
+_TERMINAL = frozenset({"success", "fail", "cancelled"})
+
+
+def _sync_job_counts(job: NeCollectionJob, runs: list[NeCollectionRun]) -> None:
+    job.success_count = sum(1 for r in runs if str(r.status) == "success")
+    job.fail_count = sum(1 for r in runs if str(r.status) in ("fail", "cancelled"))
+
+
+def finalize_collection_job(db: Session, job_id: str) -> None:
+    runs = db.query(NeCollectionRun).filter(NeCollectionRun.job_id == job_id).all()
+    if not runs:
+        return
+    if any(str(r.status or "") not in _TERMINAL for r in runs):
+        return
+    job = db.get(NeCollectionJob, job_id)
+    if not job:
+        return
+    _sync_job_counts(job, runs)
+    job_status = str(job.status or "")
+    finish_at = datetime.now()
+    if job_status == "paused":
+        if not job.ended_at:
+            job.ended_at = finish_at
+    else:
+        success = int(job.success_count or 0)
+        job.status = "done" if success > 0 else "failed"
+        if not job.ended_at:
+            job.ended_at = finish_at
+    job.last_run_at = job.ended_at or finish_at
+    db.commit()
+
+
+def reconcile_stale_collection_job(db: Session, job_id: str) -> bool:
+    """Mark long-running pending/running rows as failed and finalize job if possible."""
+    job = db.get(NeCollectionJob, job_id)
+    if not job:
+        return False
+    if str(job.status or "") in ("done", "failed", "paused"):
+        return False
+    run_stale_sec = max(60, int(settings.ne_collect_stale_run_sec or 900))
+    pending_stale_sec = max(30, int(settings.ne_collect_pending_stale_sec or 180))
+    now = datetime.now()
+    runs = db.query(NeCollectionRun).filter(NeCollectionRun.job_id == job_id).all()
+    changed = False
+    for run in runs:
+        st = str(run.status or "")
+        if st in _TERMINAL:
+            continue
+        if st == "pending":
+            anchor = job.started_at or job.created_at
+            limit = pending_stale_sec
+            reason = "collection_pending_stale"
+        else:
+            anchor = run.started_at or job.started_at or job.created_at
+            limit = run_stale_sec
+            reason = "collection_timeout_stale"
+        if not anchor:
+            continue
+        age = (now - anchor).total_seconds()
+        if age < limit:
+            continue
+        run.status = "fail"
+        run.message = f"{reason} ({int(age)}s)"
+        run.ended_at = now
+        changed = True
+    if changed:
+        db.commit()
+        finalize_collection_job(db, job_id)
+    return changed