feat(webcrt): add CRT-style browser terminal for managed and UME NEs

Ship an ops WebCRT module with xterm.js UI, WebSocket session bridge reusing hop/bastion login, searchable paged targets, and session audit limits.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-07-29 17:23:45 +08:00
parent 88479245bb
commit 873f2bfe34
21 changed files with 1868 additions and 45 deletions

View file

@ -23,7 +23,6 @@ from .ne_service import (
_normalize_hop_vendor,
_normalize_protocol,
_require_crypto,
list_managed_ne,
)
@ -263,32 +262,55 @@ def list_cli_targets(
if src not in ("managed", "ume", "all"):
raise HTTPException(status_code=400, detail="invalid_source")
ready = cli_profile_ready(db)
items: list[dict[str, Any]] = []
total = 0
page = max(1, int(page or 1))
page_size = max(1, min(500, int(page_size or 50)))
offset = (page - 1) * page_size
kw = str(keyword or "").strip()
if src in ("managed", "all"):
managed = list_managed_ne(db, keyword=keyword, page=page, page_size=page_size)
total += int(managed["total"])
for row in managed["items"]:
items.append(
CliTargetOut(
source="managed",
id=str(row.id),
ume_ne_id=None,
name=str(row.name or row.ip_address),
ip_address=str(row.ip_address),
vendor=str(row.vendor),
device_type=str(row.device_type),
connect_status=str(row.connect_status),
cli_profile_ready=True,
).model_dump()
def _managed_item(row: Any) -> dict[str, Any]:
return CliTargetOut(
source="managed",
id=str(row.id),
ume_ne_id=None,
name=str(row.name or row.ip_address),
ip_address=str(row.ip_address),
vendor=str(row.vendor),
device_type=str(row.device_type),
connect_status=str(row.connect_status),
cli_profile_ready=True,
).model_dump()
def _ume_item(inv: UmeInventoryNE, ov: UmeCliOverride | None) -> dict[str, Any]:
return CliTargetOut(
source="ume",
id=str(inv.ne_id),
ume_ne_id=str(inv.ne_id),
name=str(inv.user_label or inv.ne_name or inv.host_name or inv.ip_address or inv.ne_id),
ip_address=str(inv.ip_address or ""),
ne_type=str(inv.ne_type or ""),
vendor=str(inv.vendor or ""),
connect_status=str(ov.connect_status if ov else "unknown"),
cli_profile_ready=ready,
).model_dump()
def _managed_query():
stmt = db.query(ManagedNE)
if kw:
like = f"%{kw}%"
stmt = stmt.filter(
ManagedNE.name.ilike(like)
| ManagedNE.ip_address.ilike(like)
| ManagedNE.username.ilike(like)
| ManagedNE.tags.ilike(like)
| ManagedNE.vendor.ilike(like)
| ManagedNE.device_type.ilike(like)
)
return stmt.order_by(ManagedNE.updated_at.desc())
if src in ("ume", "all"):
def _ume_query():
stmt = db.query(UmeInventoryNE, UmeCliOverride).outerjoin(
UmeCliOverride, UmeInventoryNE.ne_id == UmeCliOverride.ume_ne_id
)
kw = str(keyword or "").strip()
if kw:
like = f"%{kw}%"
stmt = stmt.filter(
@ -298,28 +320,41 @@ def list_cli_targets(
| UmeInventoryNE.ip_address.ilike(like)
| UmeInventoryNE.host_name.ilike(like)
)
ume_total = stmt.count()
total += ume_total if src == "ume" else ume_total
rows = (
stmt.order_by(UmeInventoryNE.ne_id.asc())
.offset((page - 1) * page_size)
.limit(page_size)
.all()
)
for inv, ov in rows:
items.append(
CliTargetOut(
source="ume",
id=str(inv.ne_id),
ume_ne_id=str(inv.ne_id),
name=str(inv.user_label or inv.ne_name or inv.host_name or inv.ip_address or inv.ne_id),
ip_address=str(inv.ip_address or ""),
ne_type=str(inv.ne_type or ""),
vendor=str(inv.vendor or ""),
connect_status=str(ov.connect_status if ov else "unknown"),
cli_profile_ready=ready,
).model_dump()
)
return stmt.order_by(UmeInventoryNE.ne_id.asc())
if src == "managed":
mq = _managed_query()
total = int(mq.count())
rows = mq.offset(offset).limit(page_size).all()
items = [_managed_item(x) for x in rows]
return {"items": items, "total": total, "page": page, "page_size": page_size}
if src == "ume":
uq = _ume_query()
total = int(uq.count())
rows = uq.offset(offset).limit(page_size).all()
items = [_ume_item(inv, ov) for inv, ov in rows]
return {"items": items, "total": total, "page": page, "page_size": page_size}
# source=all: managed first, then UME, with correct cross-list pagination
mq = _managed_query()
uq = _ume_query()
m_total = int(mq.count())
u_total = int(uq.count())
total = m_total + u_total
items: list[dict[str, Any]] = []
if offset < m_total:
take = min(page_size, m_total - offset)
for row in mq.offset(offset).limit(take).all():
items.append(_managed_item(row))
need = page_size - len(items)
if need > 0 and u_total > 0:
for inv, ov in uq.offset(0).limit(need).all():
items.append(_ume_item(inv, ov))
else:
u_off = offset - m_total
for inv, ov in uq.offset(u_off).limit(page_size).all():
items.append(_ume_item(inv, ov))
return {"items": items, "total": total, "page": page, "page_size": page_size}

View file

@ -74,6 +74,12 @@ class Settings(BaseSettings):
ne_collection_data_dir: str = "data/ne_collections"
# Managed NE exec: max CLI commands per request (lab can raise; hard-capped in ne_exec).
ne_exec_max_commands: int = 5
# WebCRT interactive terminal sessions
webcrt_max_sessions: int = 20
webcrt_idle_timeout_sec: int = 1800
webcrt_connect_timeout_sec: int = 90
webcrt_attach_timeout_sec: int = 60
webcrt_data_dir: str = "data/webcrt"
settings = Settings()

View file

@ -24,6 +24,7 @@ from .db import Base, SessionLocal, engine, get_db
from .collection_router import router as collection_router
from .cli_router import router as cli_router
from .managed_ne_router import router as managed_ne_router
from .webcrt_router import router as webcrt_router
from .importer import aggregate_alarms, import_alarm_excel, query_alarms
from .models import (
AiAnalyzeHistory,
@ -123,6 +124,7 @@ app = FastAPI(title="netx ops tool", version="0.1.0")
app.include_router(managed_ne_router)
app.include_router(cli_router)
app.include_router(collection_router)
app.include_router(webcrt_router)
parser_cfg = load_parser_config()
_UME_CLIENT_SINGLETON = UMEClient(
token_loader=lambda: load_shared_token(),

170
netx_api/webcrt_router.py Normal file
View file

@ -0,0 +1,170 @@
"""WebCRT HTTP + WebSocket routes."""
from __future__ import annotations
import asyncio
import json
import logging
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from .db import get_db
from .webcrt_service import (
close_session,
create_session,
list_sessions,
mark_attached,
)
_log = logging.getLogger("netx.webcrt.router")
router = APIRouter(prefix="/v1/webcrt", tags=["webcrt"])
class WebcrtSessionCreate(BaseModel):
ne_id: str | None = Field(default=None)
ume_ne_id: str | None = Field(default=None)
cols: int = Field(default=80, ge=20, le=500)
rows: int = Field(default=24, ge=5, le=200)
def _client_label(request: Request | None = None, websocket: WebSocket | None = None) -> str:
host = ""
if request is not None:
host = request.client.host if request.client else ""
elif websocket is not None:
host = websocket.client.host if websocket.client else ""
return str(host or "")
@router.get("/sessions")
def api_list_sessions() -> dict[str, Any]:
return list_sessions()
@router.post("/sessions")
def api_create_session(
body: WebcrtSessionCreate,
request: Request,
db: Session = Depends(get_db),
) -> dict[str, Any]:
mid = str(body.ne_id or "").strip()
uid = str(body.ume_ne_id or "").strip()
if bool(mid) == bool(uid):
raise HTTPException(status_code=400, detail="exactly_one_of_ne_id_or_ume_ne_id_required")
return create_session(
db,
ne_id=mid or None,
ume_ne_id=uid or None,
cols=body.cols,
rows=body.rows,
client=_client_label(request=request),
)
@router.delete("/sessions/{session_id}")
def api_close_session(session_id: str, request: Request) -> dict[str, Any]:
return close_session(session_id, reason="client_delete", client=_client_label(request=request))
@router.websocket("/sessions/{session_id}/ws")
async def websocket_session(websocket: WebSocket, session_id: str) -> None:
await websocket.accept()
try:
sess = mark_attached(session_id)
except HTTPException as exc:
await websocket.send_json({"type": "status", "state": "error", "message": str(exc.detail)})
await websocket.close(code=4404 if exc.status_code == 404 else 4409)
return
await websocket.send_json(
{
"type": "status",
"state": "connected",
"session_id": sess.session_id,
"ne_id": sess.ne_id,
"ne_name": sess.ne_name,
"ne_ip": sess.ne_ip,
"protocol": sess.protocol,
"cols": sess.cols,
"rows": sess.rows,
}
)
stop = asyncio.Event()
async def pump_stdout() -> None:
loop = asyncio.get_running_loop()
while not stop.is_set():
chunk = await loop.run_in_executor(None, sess.out_queue.get)
if chunk is None:
stop.set()
try:
await websocket.send_json(
{
"type": "status",
"state": "closed",
"message": sess.close_reason or "device_closed",
}
)
except Exception:
pass
break
try:
text = chunk.decode("utf-8", errors="replace")
await websocket.send_json({"type": "stdout", "data": text})
except Exception:
stop.set()
break
reader_task = asyncio.create_task(pump_stdout())
try:
while not stop.is_set():
raw = await websocket.receive_text()
try:
msg = json.loads(raw)
except json.JSONDecodeError:
# Treat plain text as stdin.
msg = {"type": "stdin", "data": raw}
mtype = str(msg.get("type") or "").strip().lower()
if mtype == "stdin":
data = msg.get("data")
if data is None:
continue
try:
await asyncio.get_running_loop().run_in_executor(
None, sess.write_stdin, str(data)
)
except Exception as exc:
await websocket.send_json(
{"type": "status", "state": "error", "message": f"write_failed:{exc}"}
)
break
elif mtype == "resize":
cols = int(msg.get("cols") or sess.cols)
rows = int(msg.get("rows") or sess.rows)
await asyncio.get_running_loop().run_in_executor(None, sess.resize, cols, rows)
elif mtype == "ping":
sess.touch()
await websocket.send_json({"type": "pong"})
elif mtype == "close":
break
except WebSocketDisconnect:
_log.info("webcrt ws disconnected session=%s", session_id)
except Exception:
_log.exception("webcrt ws error session=%s", session_id)
finally:
stop.set()
reader_task.cancel()
try:
await reader_task
except Exception:
pass
close_session(
session_id,
reason="ws_disconnect",
client=_client_label(websocket=websocket),
)

378
netx_api/webcrt_service.py Normal file
View file

@ -0,0 +1,378 @@
"""Interactive WebCRT sessions: bridge browser WebSocket <-> Netmiko device channel."""
from __future__ import annotations
import json
import logging
import queue
import threading
import time
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from fastapi import HTTPException
from netmiko import ConnectHandler
from sqlalchemy.orm import Session
from .config import settings
from .ne_crypto import CredentialCryptoError
from .ne_session_factory import close_netmiko_connection, open_netmiko_connection
_log = logging.getLogger("netx.webcrt")
_sessions_lock = threading.Lock()
_sessions: dict[str, "WebcrtSession"] = {}
_reaper_started = False
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
def _utc_iso() -> str:
return _utc_now().isoformat()
def webcrt_data_root() -> Path:
root = Path(str(settings.webcrt_data_dir or "data/webcrt"))
root.mkdir(parents=True, exist_ok=True)
return root.resolve()
def _audit(event: str, **fields: Any) -> None:
record = {"ts": _utc_iso(), "event": event, **fields}
try:
path = webcrt_data_root() / "audit.jsonl"
with path.open("a", encoding="utf-8") as fh:
fh.write(json.dumps(record, ensure_ascii=False) + "\n")
except Exception:
_log.exception("webcrt audit write failed")
_log.info("webcrt.%s %s", event, {k: v for k, v in fields.items() if k != "detail"})
@dataclass
class WebcrtSession:
session_id: str
ne_id: str
ne_name: str
ne_ip: str
protocol: str
cols: int
rows: int
conn: ConnectHandler | None = None
created_at: float = field(default_factory=time.time)
last_activity: float = field(default_factory=time.time)
attached: bool = False
closed: bool = False
close_reason: str = ""
out_queue: queue.Queue[bytes | None] = field(default_factory=queue.Queue)
_reader: threading.Thread | None = field(default=None, repr=False)
_write_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
def touch(self) -> None:
self.last_activity = time.time()
def write_stdin(self, data: str) -> None:
if self.closed or self.conn is None:
raise RuntimeError("session_closed")
text = str(data or "")
if not text:
return
with self._write_lock:
self.conn.write_channel(text)
self.touch()
def resize(self, cols: int, rows: int) -> None:
if self.closed or self.conn is None:
return
c = max(20, min(500, int(cols or 80)))
r = max(5, min(200, int(rows or 24)))
self.cols = c
self.rows = r
channel = getattr(self.conn, "remote_conn", None)
if channel is not None and hasattr(channel, "resize_pty"):
try:
channel.resize_pty(width=c, height=r)
except Exception:
_log.debug("resize_pty failed session=%s", self.session_id, exc_info=True)
self.touch()
def start_reader(self) -> None:
if self._reader and self._reader.is_alive():
return
self._reader = threading.Thread(
target=self._reader_loop,
name=f"webcrt-reader-{self.session_id[:8]}",
daemon=True,
)
self._reader.start()
def _reader_loop(self) -> None:
conn = self.conn
if conn is None:
self.out_queue.put(None)
return
channel = getattr(conn, "remote_conn", None)
try:
while not self.closed:
chunk = b""
try:
if channel is not None and hasattr(channel, "recv_ready") and hasattr(channel, "recv"):
if channel.recv_ready():
chunk = channel.recv(4096)
if not chunk:
break
elif hasattr(channel, "exit_status_ready") and channel.exit_status_ready():
break
else:
time.sleep(0.04)
continue
else:
text = conn.read_channel()
if text:
chunk = text.encode("utf-8", errors="replace")
else:
time.sleep(0.04)
continue
except Exception as exc:
if self.closed:
break
_log.debug("webcrt reader error session=%s: %s", self.session_id, exc)
time.sleep(0.1)
continue
if chunk:
self.touch()
self.out_queue.put(chunk)
finally:
self.out_queue.put(None)
def close(self, reason: str = "closed") -> None:
if self.closed:
return
self.closed = True
self.close_reason = reason or "closed"
try:
close_netmiko_connection(self.conn)
except Exception:
pass
self.conn = None
try:
self.out_queue.put_nowait(None)
except Exception:
pass
def _ensure_reaper() -> None:
global _reaper_started
with _sessions_lock:
if _reaper_started:
return
_reaper_started = True
t = threading.Thread(target=_reaper_loop, name="webcrt-reaper", daemon=True)
t.start()
def _reaper_loop() -> None:
while True:
try:
_reap_sessions()
except Exception:
_log.exception("webcrt reaper failed")
time.sleep(5)
def _reap_sessions() -> None:
idle = max(60, int(settings.webcrt_idle_timeout_sec or 1800))
attach = max(10, int(settings.webcrt_attach_timeout_sec or 60))
now = time.time()
to_close: list[tuple[WebcrtSession, str]] = []
with _sessions_lock:
for sess in list(_sessions.values()):
if sess.closed:
_sessions.pop(sess.session_id, None)
continue
if not sess.attached and (now - sess.created_at) > attach:
to_close.append((sess, "attach_timeout"))
elif (now - sess.last_activity) > idle:
to_close.append((sess, "idle_timeout"))
for sess, reason in to_close:
close_session(sess.session_id, reason=reason)
def active_session_count() -> int:
with _sessions_lock:
return sum(1 for s in _sessions.values() if not s.closed)
def get_session(session_id: str) -> WebcrtSession | None:
with _sessions_lock:
sess = _sessions.get(session_id)
if sess is None or sess.closed:
return None
return sess
def create_session(
db: Session,
*,
ne_id: str | None = None,
ume_ne_id: str | None = None,
cols: int = 80,
rows: int = 24,
client: str = "",
) -> dict[str, Any]:
from .cli_resolve import resolve_cli_target
_ensure_reaper()
max_sessions = max(1, int(settings.webcrt_max_sessions or 20))
if active_session_count() >= max_sessions:
raise HTTPException(status_code=429, detail="webcrt_session_limit")
mid = str(ne_id or "").strip()
uid = str(ume_ne_id or "").strip()
try:
creds, device = resolve_cli_target(db, managed_ne_id=mid or None, ume_ne_id=uid or None)
except HTTPException:
raise
except CredentialCryptoError as exc:
raise HTTPException(status_code=400, detail=str(exc) or "credential_crypto_error") from exc
except Exception as exc:
raise HTTPException(status_code=400, detail=f"credential_error:{exc}") from exc
if not str(creds.get("username") or "").strip() or not str(creds.get("password") or ""):
raise HTTPException(status_code=400, detail="credentials_incomplete")
session_id = str(uuid.uuid4())
c = max(20, min(500, int(cols or 80)))
r = max(5, min(200, int(rows or 24)))
connect_timeout = max(30, int(settings.webcrt_connect_timeout_sec or 90))
target_id = str(device.get("id") or mid or uid)
target_ip = str(device.get("ip_address") or "")
target_name = str(device.get("name") or target_ip)
protocol = str(device.get("protocol") or creds.get("protocol") or "ssh")
try:
conn = open_netmiko_connection(creds, session_timeout=connect_timeout)
except Exception as exc:
_audit(
"session_open_failed",
session_id=session_id,
ne_id=target_id,
ne_ip=target_ip,
source=str(device.get("source") or ""),
client=client or "",
error=str(exc)[:500],
)
raise HTTPException(status_code=502, detail=f"connect_failed:{exc}") from exc
channel = getattr(conn, "remote_conn", None)
if channel is not None and hasattr(channel, "resize_pty"):
try:
channel.resize_pty(width=c, height=r)
except Exception:
pass
try:
leftover = conn.read_channel()
except Exception:
leftover = ""
sess = WebcrtSession(
session_id=session_id,
ne_id=target_id,
ne_name=target_name,
ne_ip=target_ip,
protocol=protocol,
cols=c,
rows=r,
conn=conn,
)
if leftover:
sess.out_queue.put(str(leftover).encode("utf-8", errors="replace"))
sess.start_reader()
with _sessions_lock:
_sessions[session_id] = sess
_audit(
"session_created",
session_id=session_id,
ne_id=sess.ne_id,
ne_name=sess.ne_name,
ne_ip=sess.ne_ip,
protocol=sess.protocol,
source=str(device.get("source") or ""),
hop_enabled=bool(creds.get("hop_enabled")),
hop_vendor=str(creds.get("hop_vendor") or "") if creds.get("hop_enabled") else "",
client=client or "",
active=active_session_count(),
)
return {
"session_id": session_id,
"ne_id": sess.ne_id,
"ne_name": sess.ne_name,
"ne_ip": sess.ne_ip,
"source": str(device.get("source") or ""),
"protocol": sess.protocol,
"cols": sess.cols,
"rows": sess.rows,
"ws_path": f"/v1/webcrt/sessions/{session_id}/ws",
}
def mark_attached(session_id: str) -> WebcrtSession:
sess = get_session(session_id)
if sess is None:
raise HTTPException(status_code=404, detail="webcrt_session_not_found")
if sess.attached:
raise HTTPException(status_code=409, detail="webcrt_session_already_attached")
sess.attached = True
sess.touch()
_audit("session_attached", session_id=session_id, ne_id=sess.ne_id, ne_ip=sess.ne_ip)
return sess
def close_session(session_id: str, *, reason: str = "closed", client: str = "") -> dict[str, Any]:
with _sessions_lock:
sess = _sessions.pop(session_id, None)
if sess is None:
return {"ok": True, "session_id": session_id, "closed": False}
if not sess.closed:
sess.close(reason)
_audit(
"session_closed",
session_id=session_id,
ne_id=sess.ne_id,
ne_ip=sess.ne_ip,
reason=reason,
client=client or "",
active=active_session_count(),
)
return {"ok": True, "session_id": session_id, "closed": True, "reason": reason}
def list_sessions() -> dict[str, Any]:
with _sessions_lock:
items = [
{
"session_id": s.session_id,
"ne_id": s.ne_id,
"ne_name": s.ne_name,
"ne_ip": s.ne_ip,
"protocol": s.protocol,
"attached": s.attached,
"created_at": datetime.fromtimestamp(s.created_at, tz=timezone.utc).isoformat(),
"last_activity": datetime.fromtimestamp(s.last_activity, tz=timezone.utc).isoformat(),
}
for s in _sessions.values()
if not s.closed
]
return {
"total": len(items),
"max_sessions": max(1, int(settings.webcrt_max_sessions or 20)),
"idle_timeout_sec": max(60, int(settings.webcrt_idle_timeout_sec or 1800)),
"items": items,
}