Harden WebCRT reconnect, resource use, and SFTP UX.

Reuse detached sessions on reconnect, mount only the active tab, surface backpressure drops, gate SFTP for unsupported targets, and polish attach replay, find/recording performance, and terminal i18n.

Co-authored-by: Cursor <cursoragent@cursor.com>

netx_api/cli_schemas.py

@@ -121,5 +121,6 @@ class CliTargetOut(BaseModel):
     protocol: str = ""
     username: str = ""
     has_password: bool = False
+    hop_enabled: bool = False
     connect_status: str = "unknown"
     cli_profile_ready: bool = False

netx_api/cli_service.py

@@ -269,6 +269,10 @@ def list_cli_targets(
     offset = (page - 1) * page_size
     kw = str(keyword or "").strip()
 
+    default_profile = get_default_profile(db)
+    default_hop = bool(getattr(default_profile, "hop_enabled", False)) if default_profile else False
+    default_proto = str(getattr(default_profile, "protocol", "") or "ssh") if default_profile else "ssh"
+
     def _managed_item(row: Any, *, list_source: str = "managed") -> dict[str, Any]:
         return CliTargetOut(
             source=list_source,
@@ -281,11 +285,20 @@ def list_cli_targets(
             protocol=str(getattr(row, "protocol", "") or ""),
             username=str(getattr(row, "username", "") or ""),
             has_password=bool(str(getattr(row, "password_enc", "") or "").strip()),
+            hop_enabled=bool(getattr(row, "hop_enabled", False)),
             connect_status=str(row.connect_status),
             cli_profile_ready=True,
         ).model_dump()
 
     def _ume_item(inv: UmeInventoryNE, ov: UmeCliOverride | None) -> dict[str, Any]:
+        # UME hop/protocol come from the selected/default CLI profile (SFTP gating).
+        hop = default_hop
+        proto = default_proto
+        if ov and ov.profile_id:
+            pref = db.get(CliConnectProfile, str(ov.profile_id))
+            if pref is not None:
+                hop = bool(pref.hop_enabled)
+                proto = str(pref.protocol or proto or "ssh")
         return CliTargetOut(
             source="ume",
             id=str(inv.ne_id),
@@ -294,6 +307,8 @@ def list_cli_targets(
             ip_address=str(inv.ip_address or ""),
             ne_type=str(inv.ne_type or ""),
             vendor=str(inv.vendor or ""),
+            protocol=proto,
+            hop_enabled=hop,
             connect_status=str(ov.connect_status if ov else "unknown"),
             cli_profile_ready=ready,
         ).model_dump()

netx_api/config.py

@@ -87,6 +87,8 @@ class Settings(BaseSettings):
     webcrt_idle_timeout_sec: int = 1800
     webcrt_connect_timeout_sec: int = 90
     webcrt_attach_timeout_sec: int = 60
+    # Keep device PTY after WS drop so the UI can re-attach (reconnect / remount).
+    webcrt_detach_grace_sec: int = 120
     webcrt_data_dir: str = "data/webcrt"
     # SSH transport keepalive interval (seconds); 0 disables (default off).
     webcrt_keepalive_sec: int = 0

netx_api/webcrt_router.py

@@ -22,8 +22,10 @@ from .webcrt_service import (
     get_session,
     list_sessions,
     mark_attached,
+    read_session_log_tail,
     wait_session_ready,
     _decode_bytes,
+    _encode_text,
     _normalize_encoding,
 )
 
@@ -378,20 +380,32 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
         }
     )
 
-    # Replay full login transcript (kept for StrictMode remount / brief reconnect).
-    bootstrap = bytes(sess.bootstrap_output or b"")
-    if bootstrap:
-        try:
+    # First attach: login bootstrap. Later attaches (tab focus / StrictMode): session log tail.
+    first_attach = not bool(sess.bootstrap_replayed)
+    replay_bytes = b""
+    replay_text = ""
+    if first_attach:
+        raw_boot = bytes(sess.bootstrap_output or b"")
+        if raw_boot:
             if _normalize_encoding(sess.encoding) != "utf-8":
-                bootstrap = _decode_bytes(bootstrap, sess.encoding).encode("utf-8", errors="replace")
-            await websocket.send_bytes(bootstrap)
+                replay_text = _decode_bytes(raw_boot, sess.encoding)
+                replay_bytes = replay_text.encode("utf-8", errors="replace")
+            else:
+                replay_bytes = raw_boot
+                replay_text = _decode_bytes(raw_boot, "utf-8")
+        sess.bootstrap_replayed = True
+    else:
+        replay_text = read_session_log_tail(session_id, max_bytes=49152)
+        if replay_text:
+            replay_bytes = _encode_text(replay_text, "utf-8")
+    if replay_bytes:
+        try:
+            await websocket.send_bytes(replay_bytes)
         except Exception:
             try:
-                await websocket.send_json(
-                    {"type": "stdout", "data": _decode_bytes(bytes(sess.bootstrap_output or b""), sess.encoding)}
-                )
+                await websocket.send_json({"type": "stdout", "data": replay_text or ""})
             except Exception:
-                _log.debug("webcrt bootstrap send failed session=%s", session_id, exc_info=True)
+                _log.debug("webcrt bootstrap/replay send failed session=%s", session_id, exc_info=True)
 
     stop = asyncio.Event()
     stdin_buf: list[str] = []
@@ -438,9 +452,29 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
             except Exception:
                 return False
 
+        async def _notify_queue_drops() -> None:
+            try:
+                delta = int(sess.out_queue.take_drop_delta() or 0)
+            except Exception:
+                delta = 0
+            if delta <= 0:
+                return
+            try:
+                await websocket.send_json(
+                    {
+                        "type": "status",
+                        "state": "warning",
+                        "message": f"queue_dropped:{delta}",
+                        "dropped": delta,
+                    }
+                )
+            except Exception:
+                pass
+
         while not stop.is_set():
+            # Longer block is cheap now (Condition wait); cuts executor churn when idle.
             chunk = await loop.run_in_executor(
-                None, lambda: sess.take_stdout(attach_gen, timeout=0.05)
+                None, lambda: sess.take_stdout(attach_gen, timeout=0.2)
             )
             if chunk == "stale":
                 break
@@ -449,9 +483,11 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
                     if not await _flush_pending():
                         stop.set()
                         break
+                await _notify_queue_drops()
                 continue
             if chunk is None:
                 await _flush_pending()
+                await _notify_queue_drops()
                 stop.set()
                 try:
                     await websocket.send_json(
@@ -469,6 +505,7 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
                 if not await _flush_pending():
                     stop.set()
                     break
+                await _notify_queue_drops()
 
     reader_task = asyncio.create_task(pump_stdout())
     if sess.needs_live_prompt:
@@ -552,11 +589,12 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
             await reader_task
         except Exception:
             pass
-        # Keep device session briefly so React remount / blip can re-attach.
+        # Keep device session so UI reconnect / remount can re-attach.
         if get_session(session_id) is not None:
+            grace = float(getattr(settings, "webcrt_detach_grace_sec", 120) or 120)
             detach_session(
                 session_id,
-                grace_sec=8.0,
+                grace_sec=max(8.0, grace),
                 client=_client_label(websocket=websocket),
                 attach_gen=attach_gen,
             )

netx_api/webcrt_service.py

@@ -34,8 +34,6 @@ _log = logging.getLogger("netx.webcrt")
 _sessions_lock = threading.Lock()
 _sessions: dict[str, "WebcrtSession"] = {}
 _reaper_started = False
-# Sentinel for take_stdout timeout (distinct from device EOF None).
-_STDOUT_MISSING = object()
 
 # Network device CLI key rewrites (SecureCRT-like).
 # - Backspace: DEL(0x7f) -> BS(0x08)
@@ -347,11 +345,12 @@ class _BoundedByteQueue:
     def __init__(self, maxsize: int = 2000) -> None:
         self._q: queue.Queue[bytes | None] = queue.Queue()
         self._max = max(8, int(maxsize or 2000))
-        self._lock = threading.Lock()
+        self._cond = threading.Condition()
         self.dropped = 0
+        self._reported = 0
 
     def put(self, item: bytes | None) -> None:
-        with self._lock:
+        with self._cond:
             while self._q.qsize() >= self._max:
                 try:
                     self._q.get_nowait()
@@ -359,15 +358,38 @@ class _BoundedByteQueue:
                 except queue.Empty:
                     break
             self._q.put(item)
+            self._cond.notify()
 
     def put_nowait(self, item: bytes | None) -> None:
         self.put(item)
 
     def get_nowait(self) -> bytes | None:
-        return self._q.get_nowait()
+        with self._cond:
+            return self._q.get_nowait()
+
+    def get(self, timeout: float = 0.25) -> bytes | None:
+        """Block until a chunk is available or timeout (raises queue.Empty)."""
+        deadline = time.time() + max(0.0, float(timeout))
+        with self._cond:
+            while self._q.empty():
+                remaining = deadline - time.time()
+                if remaining <= 0:
+                    raise queue.Empty
+                self._cond.wait(timeout=remaining)
+            return self._q.get_nowait()
 
     def qsize(self) -> int:
-        return self._q.qsize()
+        with self._cond:
+            return self._q.qsize()
+
+    def take_drop_delta(self) -> int:
+        """Return newly dropped chunk count since last call (for client notice)."""
+        with self._cond:
+            delta = int(self.dropped) - int(self._reported)
+            if delta <= 0:
+                return 0
+            self._reported = int(self.dropped)
+            return delta
 
 
 def _utc_now() -> datetime:
@@ -390,6 +412,33 @@ def _session_log_path(session_id: str) -> Path:
     return folder / f"{session_id}.log"
 
 
+def read_session_log_tail(session_id: str, *, max_bytes: int = 49152) -> str:
+    """Best-effort UTF-8 tail of the on-disk session transcript (for WS re-attach)."""
+    path = _session_log_path(session_id)
+    try:
+        if not path.is_file():
+            return ""
+        size = path.stat().st_size
+        take = max(1024, min(int(max_bytes or 49152), 256 * 1024))
+        with path.open("rb") as fh:
+            if size > take:
+                fh.seek(size - take)
+                raw = fh.read()
+                # Drop partial first line after seek.
+                nl = raw.find(b"\n")
+                if 0 <= nl < len(raw) - 1:
+                    raw = raw[nl + 1 :]
+            else:
+                raw = fh.read()
+        text = raw.decode("utf-8", errors="replace")
+        # Strip header comment lines from the visible replay.
+        lines = [ln for ln in text.splitlines(keepends=True) if not ln.startswith("# session=")]
+        return "".join(lines)
+    except Exception:
+        _log.debug("webcrt session log tail failed session=%s", session_id, exc_info=True)
+        return ""
+
+
 def _audit(event: str, **fields: Any) -> None:
     record = {"ts": _utc_iso(), "event": event, **fields}
     try:
@@ -428,6 +477,8 @@ class WebcrtSession:
     connect_started_at: float = field(default_factory=time.time)
     connect_finished_at: float | None = None
     bootstrap_output: bytes = b""
+    # First WS attach gets login bootstrap; later attaches prefer session-log tail.
+    bootstrap_replayed: bool = False
     needs_live_prompt: bool = True
     # React StrictMode remounts open a second WS before the first fully tears down.
     # Only the newest attach_gen may consume out_queue / mark detach.
@@ -499,19 +550,20 @@ class WebcrtSession:
             with self._stdout_lock:
                 if attach_gen != self.attach_gen:
                     return "stale"
-                try:
-                    chunk = self.out_queue.get_nowait()
-                except queue.Empty:
-                    chunk = _STDOUT_MISSING
-                if chunk is not _STDOUT_MISSING:
-                    if attach_gen != self.attach_gen:
-                        # Put back including EOF sentinel so the new owner still sees close.
-                        self.out_queue.put(chunk)
-                        return "stale"
-                    return chunk  # bytes | None
-            if time.time() >= deadline:
+            remaining = deadline - time.time()
+            if remaining <= 0:
                 return "empty"
-            time.sleep(0.005)
+            # Slice waits so we can notice attach_gen bumps without busy-spinning.
+            try:
+                chunk = self.out_queue.get(timeout=min(0.05, remaining))
+            except queue.Empty:
+                continue
+            with self._stdout_lock:
+                if attach_gen != self.attach_gen:
+                    # Put back including EOF sentinel so the new owner still sees close.
+                    self.out_queue.put(chunk)
+                    return "stale"
+                return chunk  # bytes | None
 
     def write_stdin(self, data: str) -> None:
         if self.closed or self.conn is None:
@@ -803,10 +855,14 @@ def _reap_sessions() -> None:
             if sess.detach_deadline is not None:
                 if now >= sess.detach_deadline:
                     to_close.append((sess, "detach_timeout"))
-            elif (now - sess.created_at) > attach:
-                to_close.append((sess, "attach_timeout"))
-            elif (now - sess.last_activity) > idle:
-                to_close.append((sess, "idle_timeout"))
+            else:
+                # Start attach clock after connect finishes (not HTTP create time),
+                # so slow auth + UI mount does not race attach_timeout.
+                anchor = float(sess.connect_finished_at or sess.created_at or now)
+                if (now - anchor) > attach:
+                    to_close.append((sess, "attach_timeout"))
+                elif (now - sess.last_activity) > idle:
+                    to_close.append((sess, "idle_timeout"))
     for sess in to_nudge:
         try:
             # Touch without changing visible prompt when payload is empty/null-ish.