Add HTTP concurrency smoke probe for running netx API.

Measure public and authed read latency under worker load and classify Windows ephemeral-port exhaustion separately from server failures.

Co-authored-by: Cursor <cursoragent@cursor.com>

scripts/load_smoke.py

@@ -0,0 +1,298 @@
+"""HTTP concurrency smoke / load probe for a running netx API.
+
+Does not SSH to devices. Measures API responsiveness, DB readiness under
+concurrent reads, and /metrics pool+CLI budget snapshots.
+
+Usage (API already running):
+  .\\.venv\\Scripts\\python.exe scripts\\load_smoke.py
+  .\\.venv\\Scripts\\python.exe scripts\\load_smoke.py --base http://127.0.0.1:8890 --workers 40 --seconds 20
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import sys
+import threading
+import time
+import urllib.error
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass
+class Sample:
+    name: str
+    ok: int = 0
+    fail: int = 0
+    statuses: dict[int, int] = field(default_factory=dict)
+    lat_ms: list[float] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+
+    def add(self, status: int, ms: float, err: str = "") -> None:
+        self.lat_ms.append(ms)
+        self.statuses[status] = self.statuses.get(status, 0) + 1
+        # Windows ephemeral-port exhaustion under aggressive client concurrency.
+        if status == 0 and ("10048" in err or "Address already in use" in err or "只允许使用一次" in err):
+            self.client_port_exhaust = getattr(self, "client_port_exhaust", 0) + 1
+            if err and len(self.errors) < 8:
+                self.errors.append(err[:160])
+            return
+        if 200 <= status < 400:
+            self.ok += 1
+        else:
+            self.fail += 1
+            if err and len(self.errors) < 8:
+                self.errors.append(err[:160])
+
+    def summary(self) -> dict[str, Any]:
+        lat = sorted(self.lat_ms)
+        def pct(p: float) -> float | None:
+            if not lat:
+                return None
+            idx = min(len(lat) - 1, max(0, int(round((p / 100.0) * (len(lat) - 1)))))
+            return round(lat[idx], 1)
+
+        return {
+            "name": self.name,
+            "ok": self.ok,
+            "fail": self.fail,
+            "client_port_exhaust": int(getattr(self, "client_port_exhaust", 0)),
+            "total": self.ok + self.fail + int(getattr(self, "client_port_exhaust", 0)),
+            "rps": None,
+            "p50_ms": pct(50),
+            "p95_ms": pct(95),
+            "p99_ms": pct(99),
+            "avg_ms": round(statistics.fmean(lat), 1) if lat else None,
+            "statuses": dict(sorted(self.statuses.items())),
+            "errors": list(self.errors),
+        }
+
+
+def http_json(url: str, *, method: str = "GET", data: dict | None = None, token: str = "", timeout: float = 15.0) -> tuple[int, Any, float]:
+    body = None
+    headers = {"Accept": "application/json"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    if data is not None:
+        body = json.dumps(data).encode("utf-8")
+        headers["Content-Type"] = "application/json"
+    req = urllib.request.Request(url, data=body, headers=headers, method=method)
+    t0 = time.perf_counter()
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            raw = resp.read()
+            ms = (time.perf_counter() - t0) * 1000
+            try:
+                payload = json.loads(raw.decode("utf-8") or "null")
+            except Exception:
+                payload = raw.decode("utf-8", errors="replace")[:200]
+            return int(resp.status), payload, ms
+    except urllib.error.HTTPError as exc:
+        ms = (time.perf_counter() - t0) * 1000
+        raw = exc.read().decode("utf-8", errors="replace")[:200]
+        return int(exc.code), raw, ms
+    except Exception as exc:  # noqa: BLE001
+        ms = (time.perf_counter() - t0) * 1000
+        return 0, str(exc)[:200], ms
+
+
+def snapshot_metrics(base: str) -> dict[str, Any]:
+    code, payload, ms = http_json(f"{base}/metrics/json")
+    return {"http_status": code, "latency_ms": round(ms, 1), "body": payload if code == 200 else {"error": payload}}
+
+
+def login(base: str, username: str, password: str) -> str:
+    code, payload, _ = http_json(
+        f"{base}/v1/auth/login",
+        method="POST",
+        data={"username": username, "password": password},
+    )
+    if code != 200 or not isinstance(payload, dict):
+        raise RuntimeError(f"login_failed status={code} body={payload}")
+    token = str(payload.get("access_token") or payload.get("token") or "")
+    if not token:
+        raise RuntimeError(f"login_no_token body={payload}")
+    return token
+
+
+def run_wave(
+    *,
+    name: str,
+    urls: list[tuple[str, str]],
+    workers: int,
+    seconds: float,
+    token: str = "",
+) -> Sample:
+    sample = Sample(name=name)
+    stop_at = time.perf_counter() + max(1.0, seconds)
+    lock = threading.Lock()
+    idx = 0
+
+    def one() -> None:
+        nonlocal idx
+        while time.perf_counter() < stop_at:
+            with lock:
+                i = idx
+                idx += 1
+            path, method = urls[i % len(urls)]
+            code, body, ms = http_json(path, method=method, token=token, timeout=20.0)
+            err = "" if 200 <= code < 400 else str(body)
+            sample.add(code, ms, err)
+
+    with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
+        futs = [pool.submit(one) for _ in range(max(1, workers))]
+        for fut in as_completed(futs):
+            fut.result()
+    summary = sample.summary()
+    elapsed = max(0.001, seconds)
+    summary["rps"] = round((sample.ok + sample.fail) / elapsed, 1)
+    sample.rps = summary["rps"]  # type: ignore[attr-defined]
+    return sample
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="netx API concurrency smoke probe")
+    ap.add_argument("--base", default="http://127.0.0.1:8890")
+    ap.add_argument("--workers", type=int, default=40)
+    ap.add_argument("--seconds", type=float, default=15.0)
+    ap.add_argument("--user", default="admin")
+    ap.add_argument("--password", default="admin123")
+    ap.add_argument("--token", default="", help="Bearer token; default tries data/auth/mcp_token then login")
+    ap.add_argument("--out", default="scripts/.run/load_smoke_report.json")
+    args = ap.parse_args()
+    base = str(args.base).rstrip("/")
+
+    code, health, _ = http_json(f"{base}/health")
+    if code != 200:
+        print(f"[ERR] API not healthy at {base}/health -> {code} {health}", file=sys.stderr)
+        return 2
+
+    before = snapshot_metrics(base)
+    ready_code, ready_body, ready_ms = http_json(f"{base}/health/ready")
+
+    public = run_wave(
+        name="public_health_metrics",
+        urls=[
+            (f"{base}/health", "GET"),
+            (f"{base}/health/live", "GET"),
+            (f"{base}/health/ready", "GET"),
+            (f"{base}/metrics", "GET"),
+            (f"{base}/metrics/json", "GET"),
+        ],
+        workers=args.workers,
+        seconds=args.seconds,
+    )
+
+    auth_sample: dict[str, Any] | None = None
+    auth_err = ""
+    token = str(args.token or "").strip()
+    if not token:
+        mcp_path = Path("data/auth/mcp_token")
+        if mcp_path.is_file():
+            token = mcp_path.read_text(encoding="utf-8").strip()
+    if not token:
+        try:
+            token = login(base, args.user, args.password)
+        except Exception as exc:  # noqa: BLE001
+            auth_err = str(exc)[:240]
+            token = ""
+    try:
+        if not token:
+            raise RuntimeError(auth_err or "no_token")
+        auth = run_wave(
+            name="authed_read_apis",
+            urls=[
+                (f"{base}/v1/integrations/status", "GET"),
+                (f"{base}/v1/managed-ne?limit=50", "GET"),
+                (f"{base}/v1/topology/views", "GET"),
+                (f"{base}/v1/topology/fabric/summary", "GET"),
+                (f"{base}/v1/ume/inventory/ne?limit=50", "GET"),
+                (f"{base}/v1/port-traffic/devices", "GET"),
+                (f"{base}/v1/alarms?limit=50", "GET"),
+            ],
+            workers=max(8, args.workers // 2),
+            seconds=max(8.0, args.seconds * 0.7),
+            token=token,
+        )
+        auth_sample = auth.summary()
+        auth_sample["rps"] = round((auth.ok + auth.fail) / max(8.0, args.seconds * 0.7), 1)
+    except Exception as exc:  # noqa: BLE001
+        auth_err = str(exc)[:240]
+
+    after = snapshot_metrics(base)
+    report = {
+        "base": base,
+        "started_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
+        "config": {
+            "workers": args.workers,
+            "seconds": args.seconds,
+            "user": args.user,
+        },
+        "precheck": {
+            "health": health,
+            "ready_status": ready_code,
+            "ready_latency_ms": round(ready_ms, 1),
+            "ready": ready_body if ready_code == 200 else {"error": ready_body},
+        },
+        "metrics_before": before,
+        "public_wave": public.summary() | {"rps": round((public.ok + public.fail) / max(1.0, args.seconds), 1)},
+        "auth_wave": auth_sample,
+        "auth_error": auth_err,
+        "metrics_after": after,
+    }
+
+    # Simple verdicts
+    verdicts: list[str] = []
+    pub = report["public_wave"]
+    if pub["fail"] == 0 and (pub.get("p95_ms") or 0) < 500:
+        verdicts.append("public endpoints: PASS (no server failures, p95 < 500ms)")
+    elif pub["fail"] == 0:
+        verdicts.append(f"public endpoints: WARN (ok but p95={pub.get('p95_ms')}ms)")
+    else:
+        verdicts.append(f"public endpoints: FAIL ({pub['fail']} server errors)")
+    if int(pub.get("client_port_exhaust") or 0) > 0:
+        verdicts.append(
+            f"public client ports: EXHAUST ({pub['client_port_exhaust']} WinError 10048) — lower --workers on Windows"
+        )
+
+    if auth_sample:
+        if auth_sample["fail"] == 0 and (auth_sample.get("p95_ms") or 0) < 1000:
+            verdicts.append("authed reads: PASS")
+        elif auth_sample["fail"] == 0:
+            verdicts.append(f"authed reads: WARN (p95={auth_sample.get('p95_ms')}ms)")
+        else:
+            verdicts.append(f"authed reads: FAIL ({auth_sample['fail']} server errors)")
+        if int(auth_sample.get("client_port_exhaust") or 0) > 0:
+            verdicts.append(
+                f"authed client ports: EXHAUST ({auth_sample['client_port_exhaust']} WinError 10048)"
+            )
+    else:
+        verdicts.append(f"authed reads: SKIP ({auth_err or 'no token'})")
+
+    pool_after = ((after.get("body") or {}).get("db_pool") or {}) if isinstance(after.get("body"), dict) else {}
+    checked = pool_after.get("checked_out")
+    size = pool_after.get("size") or pool_after.get("pool_size_cfg")
+    if checked is not None and size is not None and int(size) > 0:
+        util = int(checked) / max(1, int(size))
+        if util < 0.7:
+            verdicts.append(f"db pool after load: OK (checked_out={checked}/{size})")
+        else:
+            verdicts.append(f"db pool after load: HOT (checked_out={checked}/{size})")
+
+    report["verdicts"] = verdicts
+
+    out = Path(args.out)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    print(json.dumps({"verdicts": verdicts, "public": report["public_wave"], "auth": auth_sample, "report": str(out)}, ensure_ascii=False, indent=2))
+    bad = [v for v in verdicts if ": FAIL" in v or ": HOT" in v]
+    return 1 if bad else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())