mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 03:10:46 +08:00
Harden WebCRT interactive sessions for more CRT-like device login.
Improve async connect, binary stdout coalescing, paste throttling with progress, SSH keepalive, login prompt settle, and optional SFTP while keeping core terminal UX focused. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
cb865254e4
commit
accd28c9a1
13 changed files with 1884 additions and 224 deletions
|
|
@ -121,7 +121,7 @@ def map_network_cli_enter(data: str, conn: ConnectHandler | None) -> str:
|
|||
return text.replace("\r\n", ret).replace("\r", ret)
|
||||
|
||||
|
||||
def _drain_channel(conn: ConnectHandler, *, rounds: int = 10, wait: float = 0.12) -> str:
|
||||
def _drain_channel(conn: ConnectHandler, *, rounds: int = 6, wait: float = 0.06) -> str:
|
||||
"""Read whatever is already sitting on the channel after login."""
|
||||
chunks: list[str] = []
|
||||
empty_streak = 0
|
||||
|
|
@ -173,7 +173,11 @@ def normalize_cli_transcript(text: str) -> str:
|
|||
lines = s.split("\n")
|
||||
while lines and not str(lines[-1]).strip():
|
||||
lines.pop()
|
||||
while len(lines) >= 2 and lines[-1] == lines[-2] and _looks_like_cli_prompt(lines[-1]):
|
||||
# Drop blank lines immediately before a final prompt (banner\n\nR2# -> banner\nR2#).
|
||||
while len(lines) >= 2 and not str(lines[-2]).strip() and _looks_like_cli_prompt(lines[-1]):
|
||||
lines.pop(-2)
|
||||
# Collapse trailing duplicate prompt lines (slow VMs often echo R2# several times).
|
||||
while len(lines) >= 2 and str(lines[-1]).strip() == str(lines[-2]).strip() and _looks_like_cli_prompt(lines[-1]):
|
||||
lines.pop()
|
||||
return "\n".join(lines)
|
||||
|
||||
|
|
@ -187,25 +191,131 @@ def prepare_bootstrap_output(text: str) -> str:
|
|||
return normalize_cli_transcript(text)
|
||||
|
||||
|
||||
def _prime_interactive_channel(conn: ConnectHandler) -> None:
|
||||
"""Send one RETURN after Netmiko login so the interactive channel is fully ready."""
|
||||
def _drain_raw_channel(conn: ConnectHandler, *, duration: float = 0.5) -> None:
|
||||
"""Discard leftover bytes on the live channel (SSH/Telnet) after login priming."""
|
||||
channel = getattr(conn, "remote_conn", None)
|
||||
if channel is None:
|
||||
try:
|
||||
_drain_channel(conn, rounds=max(2, int(duration / 0.05)), wait=0.05)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
end = time.time() + max(0.1, float(duration))
|
||||
while time.time() < end:
|
||||
got = False
|
||||
try:
|
||||
if hasattr(channel, "recv_ready") and hasattr(channel, "recv") and channel.recv_ready():
|
||||
channel.recv(65535)
|
||||
got = True
|
||||
elif hasattr(channel, "read_very_eager"):
|
||||
data = channel.read_very_eager()
|
||||
if data:
|
||||
got = True
|
||||
else:
|
||||
part = conn.read_channel()
|
||||
if part:
|
||||
got = True
|
||||
except Exception:
|
||||
break
|
||||
if not got:
|
||||
time.sleep(0.04)
|
||||
|
||||
|
||||
def _prime_interactive_channel(conn: ConnectHandler, *, already_prompted: bool = False) -> None:
|
||||
"""Send one RETURN after Netmiko login so the interactive channel is fully ready.
|
||||
|
||||
Skip the sync Enter when the login transcript already ends with a CLI prompt —
|
||||
otherwise slow Cisco VMs accumulate duplicate ``R2#`` lines in the bootstrap.
|
||||
"""
|
||||
try:
|
||||
_drain_channel(conn, rounds=4, wait=0.05)
|
||||
_drain_raw_channel(conn, duration=0.15)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn.write_channel(channel_return(conn))
|
||||
except Exception:
|
||||
if not already_prompted:
|
||||
try:
|
||||
conn.write_channel("\n")
|
||||
conn.write_channel(channel_return(conn))
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
conn.write_channel("\n")
|
||||
except Exception:
|
||||
return
|
||||
try:
|
||||
_drain_channel(conn, rounds=6, wait=0.08)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
_drain_channel(conn, rounds=8, wait=0.1)
|
||||
_drain_raw_channel(conn, duration=0.35)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _is_prompt_only_echo(text: str, prompt_hint: str = "") -> bool:
|
||||
"""True when chunk is only whitespace / CR / a repeated prompt (safe to drop after bootstrap)."""
|
||||
s = str(text or "").replace("\r\n", "\n").replace("\r", "\n").strip()
|
||||
if not s:
|
||||
return True
|
||||
hint = str(prompt_hint or "").strip()
|
||||
if hint and s == hint:
|
||||
return True
|
||||
# Single-line prompt echo only.
|
||||
if "\n" not in s and _looks_like_cli_prompt(s):
|
||||
return True
|
||||
if hint and all(line.strip() in ("", hint) for line in s.split("\n")):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _normalize_encoding(name: str) -> str:
|
||||
enc = str(name or "utf-8").strip().lower().replace("_", "-")
|
||||
if enc in ("gbk", "gb2312", "gb18030", "cp936"):
|
||||
return "gbk"
|
||||
return "utf-8"
|
||||
|
||||
|
||||
def _decode_bytes(data: bytes, encoding: str) -> str:
|
||||
enc = _normalize_encoding(encoding)
|
||||
try:
|
||||
return data.decode(enc, errors="replace")
|
||||
except Exception:
|
||||
return data.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _encode_text(text: str, encoding: str) -> bytes:
|
||||
enc = _normalize_encoding(encoding)
|
||||
try:
|
||||
return text.encode(enc, errors="replace")
|
||||
except Exception:
|
||||
return text.encode("utf-8", errors="replace")
|
||||
|
||||
|
||||
class _BoundedByteQueue:
|
||||
"""Thread-safe queue that drops oldest chunks when full (backpressure)."""
|
||||
|
||||
def __init__(self, maxsize: int = 2000) -> None:
|
||||
self._q: queue.Queue[bytes | None] = queue.Queue()
|
||||
self._max = max(8, int(maxsize or 2000))
|
||||
self._lock = threading.Lock()
|
||||
self.dropped = 0
|
||||
|
||||
def put(self, item: bytes | None) -> None:
|
||||
with self._lock:
|
||||
while self._q.qsize() >= self._max:
|
||||
try:
|
||||
self._q.get_nowait()
|
||||
self.dropped += 1
|
||||
except queue.Empty:
|
||||
break
|
||||
self._q.put(item)
|
||||
|
||||
def put_nowait(self, item: bytes | None) -> None:
|
||||
self.put(item)
|
||||
|
||||
def get_nowait(self) -> bytes | None:
|
||||
return self._q.get_nowait()
|
||||
|
||||
def qsize(self) -> int:
|
||||
return self._q.qsize()
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
|
@ -221,6 +331,12 @@ def webcrt_data_root() -> Path:
|
|||
return root.resolve()
|
||||
|
||||
|
||||
def _session_log_path(session_id: str) -> Path:
|
||||
folder = webcrt_data_root() / "sessions"
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
return folder / f"{session_id}.log"
|
||||
|
||||
|
||||
def _audit(event: str, **fields: Any) -> None:
|
||||
record = {"ts": _utc_iso(), "event": event, **fields}
|
||||
try:
|
||||
|
|
@ -244,6 +360,7 @@ class WebcrtSession:
|
|||
device_type: str = ""
|
||||
vendor: str = ""
|
||||
cli_keymap: bool = True
|
||||
encoding: str = "utf-8"
|
||||
conn: ConnectHandler | None = None
|
||||
created_at: float = field(default_factory=time.time)
|
||||
last_activity: float = field(default_factory=time.time)
|
||||
|
|
@ -251,24 +368,69 @@ class WebcrtSession:
|
|||
detach_deadline: float | None = None
|
||||
closed: bool = False
|
||||
close_reason: str = ""
|
||||
# connecting | ready | error | closed
|
||||
state: str = "ready"
|
||||
connect_error: str = ""
|
||||
connect_started_at: float = field(default_factory=time.time)
|
||||
connect_finished_at: float | None = None
|
||||
bootstrap_output: bytes = b""
|
||||
needs_live_prompt: bool = True
|
||||
# React StrictMode remounts open a second WS before the first fully tears down.
|
||||
# Only the newest attach_gen may consume out_queue / mark detach.
|
||||
attach_gen: int = 0
|
||||
out_queue: queue.Queue[bytes | None] = field(default_factory=queue.Queue)
|
||||
out_queue: _BoundedByteQueue = field(
|
||||
default_factory=lambda: _BoundedByteQueue(int(getattr(settings, "webcrt_out_queue_max", 2000) or 2000))
|
||||
)
|
||||
# Vendor CLI hop (Huawei/ZTE/Cisco): close when nested target session returns to hop.
|
||||
cli_hop_guard: bool = False
|
||||
cli_hop_prompt: str = ""
|
||||
post_login_commands: list[str] = field(default_factory=list)
|
||||
bytes_in: int = 0
|
||||
bytes_out: int = 0
|
||||
_reader: threading.Thread | None = field(default=None, repr=False)
|
||||
_write_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
||||
_stdout_lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
||||
_hop_scan_buf: str = field(default="", repr=False)
|
||||
_cli_hop_seen_other_prompt: bool = field(default=False, repr=False)
|
||||
_log_fh: Any = field(default=None, repr=False)
|
||||
_ready_event: threading.Event = field(default_factory=threading.Event, repr=False)
|
||||
|
||||
def touch(self) -> None:
|
||||
self.last_activity = time.time()
|
||||
|
||||
def open_session_log(self) -> None:
|
||||
if not bool(getattr(settings, "webcrt_session_log_enabled", True)):
|
||||
return
|
||||
if self._log_fh is not None:
|
||||
return
|
||||
try:
|
||||
self._log_fh = _session_log_path(self.session_id).open("a", encoding="utf-8", errors="replace")
|
||||
self._log_fh.write(f"# session={self.session_id} ne={self.ne_id} ip={self.ne_ip} ts={_utc_iso()}\n")
|
||||
self._log_fh.flush()
|
||||
except Exception:
|
||||
_log.debug("webcrt session log open failed", exc_info=True)
|
||||
self._log_fh = None
|
||||
|
||||
def append_session_log(self, text: str) -> None:
|
||||
if not text or self._log_fh is None:
|
||||
return
|
||||
try:
|
||||
self._log_fh.write(text)
|
||||
self._log_fh.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def close_session_log(self) -> None:
|
||||
fh = self._log_fh
|
||||
self._log_fh = None
|
||||
if fh is None:
|
||||
return
|
||||
try:
|
||||
fh.write(f"\n# closed reason={self.close_reason} ts={_utc_iso()}\n")
|
||||
fh.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def take_stdout(self, attach_gen: int, *, timeout: float = 0.25) -> bytes | None | str:
|
||||
"""Exclusive stdout take for one WS attach generation.
|
||||
|
||||
|
|
@ -295,7 +457,7 @@ class WebcrtSession:
|
|||
return chunk # bytes | None
|
||||
if time.time() >= deadline:
|
||||
return "empty"
|
||||
time.sleep(0.02)
|
||||
time.sleep(0.005)
|
||||
|
||||
def write_stdin(self, data: str) -> None:
|
||||
if self.closed or self.conn is None:
|
||||
|
|
@ -315,12 +477,10 @@ class WebcrtSession:
|
|||
return
|
||||
with self._write_lock:
|
||||
# Prefer raw channel I/O for interactive typing (char echo / backspace).
|
||||
# Netmiko write_channel is fine for automation but can feel "half-duplex"
|
||||
# on some Telnet/VRP sessions when used keystroke-by-keystroke.
|
||||
channel = getattr(self.conn, "remote_conn", None)
|
||||
try:
|
||||
if channel is not None and hasattr(channel, "send") and callable(channel.send):
|
||||
payload = text.encode(getattr(self.conn, "encoding", None) or "utf-8", errors="replace")
|
||||
payload = _encode_text(text, self.encoding)
|
||||
# Paramiko may write partially when the window is full.
|
||||
view = memoryview(payload)
|
||||
while len(view):
|
||||
|
|
@ -329,14 +489,47 @@ class WebcrtSession:
|
|||
time.sleep(0.01)
|
||||
continue
|
||||
view = view[n:]
|
||||
self.bytes_in += len(payload)
|
||||
elif channel is not None and hasattr(channel, "write") and callable(channel.write):
|
||||
encoding = getattr(self.conn, "encoding", None) or "utf-8"
|
||||
channel.write(text.encode(encoding, errors="replace") if isinstance(text, str) else text)
|
||||
payload = _encode_text(text, self.encoding)
|
||||
channel.write(payload)
|
||||
self.bytes_in += len(payload)
|
||||
else:
|
||||
self.conn.write_channel(text)
|
||||
self.bytes_in += len(text)
|
||||
except Exception:
|
||||
self.conn.write_channel(text)
|
||||
self.bytes_in += len(text)
|
||||
self.touch()
|
||||
|
||||
def send_break(self) -> None:
|
||||
"""Send SSH break / Telnet IAC BREAK to interrupt paging or hung commands."""
|
||||
if self.closed or self.conn is None:
|
||||
raise RuntimeError("session_closed")
|
||||
channel = getattr(self.conn, "remote_conn", None)
|
||||
with self._write_lock:
|
||||
sent = False
|
||||
if channel is not None and hasattr(channel, "send_break") and callable(channel.send_break):
|
||||
try:
|
||||
channel.send_break(0)
|
||||
sent = True
|
||||
except Exception:
|
||||
_log.debug("send_break failed session=%s", self.session_id, exc_info=True)
|
||||
if not sent and channel is not None and hasattr(channel, "send") and callable(channel.send):
|
||||
# Telnet IAC BREAK = 255 243
|
||||
try:
|
||||
channel.send(b"\xff\xf3")
|
||||
sent = True
|
||||
except Exception:
|
||||
pass
|
||||
if not sent:
|
||||
# Fallback: Ctrl-C often interrupts device CLI more-pages.
|
||||
try:
|
||||
self.conn.write_channel("\x03")
|
||||
except Exception:
|
||||
raise RuntimeError("break_failed")
|
||||
self.touch()
|
||||
|
||||
def resize(self, cols: int, rows: int) -> None:
|
||||
if self.closed or self.conn is None:
|
||||
return
|
||||
|
|
@ -369,53 +562,76 @@ class WebcrtSession:
|
|||
return
|
||||
channel = getattr(conn, "remote_conn", None)
|
||||
hop_return = False
|
||||
poll = max(0.002, float(getattr(settings, "webcrt_reader_poll_sec", 0.01) or 0.01))
|
||||
try:
|
||||
while not self.closed:
|
||||
chunk = b""
|
||||
try:
|
||||
if channel is not None and hasattr(channel, "recv_ready") and hasattr(channel, "recv"):
|
||||
# Paramiko SSH: raw bytes keep ANSI / backspace echo intact.
|
||||
if channel.recv_ready():
|
||||
chunk = channel.recv(4096)
|
||||
# Paramiko SSH: prefer short blocking recv over fixed spin-sleep.
|
||||
ready = False
|
||||
try:
|
||||
ready = bool(channel.recv_ready())
|
||||
except Exception:
|
||||
ready = False
|
||||
if ready:
|
||||
chunk = channel.recv(16384)
|
||||
if not chunk:
|
||||
break
|
||||
elif hasattr(channel, "exit_status_ready") and channel.exit_status_ready():
|
||||
break
|
||||
else:
|
||||
time.sleep(0.04)
|
||||
continue
|
||||
# Brief block: settimeout + recv wakes sooner than sleep(0.04).
|
||||
prev_timeout = None
|
||||
try:
|
||||
prev_timeout = channel.gettimeout()
|
||||
except Exception:
|
||||
prev_timeout = None
|
||||
try:
|
||||
channel.settimeout(poll)
|
||||
chunk = channel.recv(16384)
|
||||
except Exception:
|
||||
chunk = b""
|
||||
finally:
|
||||
try:
|
||||
channel.settimeout(prev_timeout)
|
||||
except Exception:
|
||||
pass
|
||||
if not chunk:
|
||||
continue
|
||||
elif channel is not None and hasattr(channel, "read_very_eager"):
|
||||
# Telnet: do NOT use conn.read_channel() — Netmiko strips ANSI
|
||||
# escape codes, which removes Huawei backspace echo (\x1b[1D \x1b[1D).
|
||||
# Telnet: do NOT use conn.read_channel() — Netmiko strips ANSI.
|
||||
data = channel.read_very_eager()
|
||||
if data:
|
||||
chunk = (
|
||||
data
|
||||
if isinstance(data, (bytes, bytearray))
|
||||
else str(data).encode(
|
||||
getattr(conn, "encoding", None) or "utf-8",
|
||||
errors="replace",
|
||||
)
|
||||
else _encode_text(str(data), self.encoding)
|
||||
)
|
||||
else:
|
||||
time.sleep(0.04)
|
||||
time.sleep(poll)
|
||||
continue
|
||||
else:
|
||||
text = conn.read_channel()
|
||||
if text:
|
||||
chunk = text.encode("utf-8", errors="replace")
|
||||
chunk = _encode_text(str(text), self.encoding)
|
||||
else:
|
||||
time.sleep(0.04)
|
||||
time.sleep(poll)
|
||||
continue
|
||||
except Exception as exc:
|
||||
if self.closed:
|
||||
break
|
||||
_log.debug("webcrt reader error session=%s: %s", self.session_id, exc)
|
||||
time.sleep(0.1)
|
||||
time.sleep(0.05)
|
||||
continue
|
||||
if chunk:
|
||||
self.touch()
|
||||
self.bytes_out += len(chunk)
|
||||
self.out_queue.put(chunk)
|
||||
try:
|
||||
self.append_session_log(_decode_bytes(chunk, self.encoding))
|
||||
except Exception:
|
||||
pass
|
||||
if self.cli_hop_guard and self._note_cli_hop_output(chunk):
|
||||
hop_return = True
|
||||
notice = (
|
||||
|
|
@ -426,7 +642,6 @@ class WebcrtSession:
|
|||
break
|
||||
finally:
|
||||
if hop_return and not self.closed:
|
||||
# Prefer registry close for audit + remove; fall back to local close.
|
||||
try:
|
||||
close_session(self.session_id, reason="cli_hop_return")
|
||||
except Exception:
|
||||
|
|
@ -436,12 +651,10 @@ class WebcrtSession:
|
|||
def _note_cli_hop_output(self, chunk: bytes) -> bool:
|
||||
"""Accumulate stdout and return True when nested CLI hop has returned to proxy."""
|
||||
try:
|
||||
text = chunk.decode("utf-8", errors="replace")
|
||||
text = _decode_bytes(chunk, self.encoding)
|
||||
except Exception:
|
||||
text = str(chunk)
|
||||
self._hop_scan_buf = (self._hop_scan_buf + text)[-12000:]
|
||||
# Track a prompt that differs from the hop so same-sysname labs still need
|
||||
# an explicit nested-close message before we tear down.
|
||||
marker = str(self.cli_hop_prompt or "").strip()
|
||||
last = extract_cli_prompt_marker(self._hop_scan_buf)
|
||||
if last and (not marker or last != marker):
|
||||
|
|
@ -452,11 +665,25 @@ class WebcrtSession:
|
|||
seen_other_prompt=self._cli_hop_seen_other_prompt,
|
||||
)
|
||||
|
||||
def run_post_login_commands(self) -> None:
|
||||
cmds = [str(c).rstrip("\r\n") for c in (self.post_login_commands or []) if str(c).strip()]
|
||||
if not cmds or self.closed or self.conn is None:
|
||||
return
|
||||
for cmd in cmds[:20]:
|
||||
try:
|
||||
self.write_stdin(cmd + "\r")
|
||||
time.sleep(0.15)
|
||||
except Exception:
|
||||
_log.debug("post_login command failed session=%s", self.session_id, exc_info=True)
|
||||
break
|
||||
|
||||
def close(self, reason: str = "closed") -> None:
|
||||
if self.closed:
|
||||
return
|
||||
self.closed = True
|
||||
self.state = "closed"
|
||||
self.close_reason = reason or "closed"
|
||||
self._ready_event.set()
|
||||
try:
|
||||
close_netmiko_connection(self.conn)
|
||||
except Exception:
|
||||
|
|
@ -466,6 +693,7 @@ class WebcrtSession:
|
|||
self.out_queue.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
self.close_session_log()
|
||||
|
||||
|
||||
def _ensure_reaper() -> None:
|
||||
|
|
@ -490,16 +718,32 @@ def _reaper_loop() -> None:
|
|||
def _reap_sessions() -> None:
|
||||
idle = max(60, int(settings.webcrt_idle_timeout_sec or 1800))
|
||||
attach = max(10, int(settings.webcrt_attach_timeout_sec or 60))
|
||||
anti_idle = max(0, int(getattr(settings, "webcrt_anti_idle_sec", 0) or 0))
|
||||
anti_payload = str(getattr(settings, "webcrt_anti_idle_payload", " ") or " ")
|
||||
now = time.time()
|
||||
to_close: list[tuple[WebcrtSession, str]] = []
|
||||
to_nudge: list[WebcrtSession] = []
|
||||
with _sessions_lock:
|
||||
for sess in list(_sessions.values()):
|
||||
if sess.closed:
|
||||
_sessions.pop(sess.session_id, None)
|
||||
continue
|
||||
if sess.state == "connecting":
|
||||
# Connecting sessions use connect timeout, not attach timeout alone.
|
||||
connect_budget = max(30, int(settings.webcrt_connect_timeout_sec or 90)) + 30
|
||||
if (now - sess.connect_started_at) > connect_budget:
|
||||
to_close.append((sess, "connect_timeout"))
|
||||
continue
|
||||
if sess.attached:
|
||||
if (now - sess.last_activity) > idle:
|
||||
to_close.append((sess, "idle_timeout"))
|
||||
elif (
|
||||
anti_idle > 0
|
||||
and sess.state == "ready"
|
||||
and sess.conn is not None
|
||||
and (now - sess.last_activity) >= anti_idle
|
||||
):
|
||||
to_nudge.append(sess)
|
||||
continue
|
||||
# Not attached: either never attached, or briefly detached for reconnect.
|
||||
if sess.detach_deadline is not None:
|
||||
|
|
@ -509,6 +753,18 @@ def _reap_sessions() -> None:
|
|||
to_close.append((sess, "attach_timeout"))
|
||||
elif (now - sess.last_activity) > idle:
|
||||
to_close.append((sess, "idle_timeout"))
|
||||
for sess in to_nudge:
|
||||
try:
|
||||
# Touch without changing visible prompt when payload is empty/null-ish.
|
||||
payload = anti_payload
|
||||
if payload == "\\0":
|
||||
payload = "\x00"
|
||||
if payload:
|
||||
sess.write_stdin(payload)
|
||||
else:
|
||||
sess.touch()
|
||||
except Exception:
|
||||
_log.debug("webcrt anti-idle failed session=%s", sess.session_id, exc_info=True)
|
||||
for sess, reason in to_close:
|
||||
close_session(sess.session_id, reason=reason)
|
||||
|
||||
|
|
@ -526,6 +782,21 @@ def get_session(session_id: str) -> WebcrtSession | None:
|
|||
return sess
|
||||
|
||||
|
||||
def wait_session_ready(session_id: str, *, timeout: float = 120.0) -> WebcrtSession:
|
||||
"""Block until async connect finishes (ready or error). Used by tests and WS."""
|
||||
deadline = time.time() + max(1.0, float(timeout))
|
||||
while time.time() < deadline:
|
||||
sess = get_session(session_id)
|
||||
if sess is None:
|
||||
raise HTTPException(status_code=404, detail="webcrt_session_not_found")
|
||||
if sess.state == "ready":
|
||||
return sess
|
||||
if sess.state == "error":
|
||||
raise HTTPException(status_code=502, detail=sess.connect_error or "connect_failed")
|
||||
sess._ready_event.wait(timeout=0.25)
|
||||
raise HTTPException(status_code=504, detail="connect_timeout")
|
||||
|
||||
|
||||
def _webcrt_creds_ready(creds: dict[str, Any]) -> bool:
|
||||
"""True when WebCRT can open a session with the resolved credentials.
|
||||
|
||||
|
|
@ -547,6 +818,133 @@ def _webcrt_creds_ready(creds: dict[str, Any]) -> bool:
|
|||
return bool(str(creds.get("password") or ""))
|
||||
|
||||
|
||||
def _finish_connect(
|
||||
sess: WebcrtSession,
|
||||
*,
|
||||
creds: dict[str, Any],
|
||||
device: dict[str, Any],
|
||||
connect_timeout: int,
|
||||
client: str,
|
||||
) -> None:
|
||||
log_buf = io.BytesIO()
|
||||
try:
|
||||
conn = open_netmiko_connection(
|
||||
creds,
|
||||
session_timeout=connect_timeout,
|
||||
session_log=log_buf,
|
||||
cols=sess.cols,
|
||||
rows=sess.rows,
|
||||
interactive=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
partial = _session_log_text(log_buf).strip()
|
||||
from .ne_cli_errors import format_cli_failure
|
||||
|
||||
classified = format_cli_failure(exc, partial)
|
||||
detail = f"connect_failed:{classified}"
|
||||
if partial:
|
||||
detail = f"{detail}\n--- device transcript ---\n{partial[-4000:]}"
|
||||
sess.state = "error"
|
||||
sess.connect_error = detail
|
||||
sess.connect_finished_at = time.time()
|
||||
sess._ready_event.set()
|
||||
_audit(
|
||||
"session_open_failed",
|
||||
session_id=sess.session_id,
|
||||
ne_id=sess.ne_id,
|
||||
ne_ip=sess.ne_ip,
|
||||
source=str(device.get("source") or ""),
|
||||
client=client or "",
|
||||
error=str(exc)[:500],
|
||||
transcript_len=len(partial),
|
||||
)
|
||||
return
|
||||
|
||||
channel = getattr(conn, "remote_conn", None)
|
||||
if channel is not None and hasattr(channel, "resize_pty"):
|
||||
try:
|
||||
channel.resize_pty(width=sess.cols, height=sess.rows)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
pre_log = _session_log_text(log_buf)
|
||||
already_prompted = _looks_like_cli_prompt(pre_log)
|
||||
_prime_interactive_channel(conn, already_prompted=already_prompted)
|
||||
bootstrap = prepare_bootstrap_output(_session_log_text(log_buf))
|
||||
if not bootstrap.strip():
|
||||
try:
|
||||
more = _drain_channel(conn, rounds=6, wait=0.08)
|
||||
except Exception:
|
||||
more = ""
|
||||
if more:
|
||||
bootstrap = prepare_bootstrap_output(more)
|
||||
# Slow Cisco VMs leave extra prompt bytes; discard before the live reader starts.
|
||||
try:
|
||||
_drain_raw_channel(conn, duration=0.55)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
hop_guard = get_cli_hop_guard(conn)
|
||||
sess.conn = conn
|
||||
sess.cli_hop_guard = bool(hop_guard)
|
||||
sess.cli_hop_prompt = str((hop_guard or {}).get("hop_prompt") or "")
|
||||
sess.bootstrap_output = _encode_text(str(bootstrap or ""), sess.encoding)
|
||||
# Bootstrap already ends at a prompt → never nudge another Enter on WS attach.
|
||||
sess.needs_live_prompt = not _looks_like_cli_prompt(bootstrap)
|
||||
sess.open_session_log()
|
||||
if bootstrap:
|
||||
sess.append_session_log(bootstrap if bootstrap.endswith("\n") else bootstrap + "\n")
|
||||
sess.start_reader()
|
||||
# Drop late prompt echoes that race into the queue right after reader start.
|
||||
prompt_hint = ""
|
||||
if bootstrap:
|
||||
prompt_hint = str(bootstrap).replace("\r\n", "\n").replace("\r", "\n").strip().split("\n")[-1].strip()
|
||||
settle_deadline = time.time() + 0.45
|
||||
while time.time() < settle_deadline:
|
||||
try:
|
||||
chunk = sess.out_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
time.sleep(0.02)
|
||||
continue
|
||||
if chunk is None:
|
||||
sess.out_queue.put(None)
|
||||
break
|
||||
try:
|
||||
text = _decode_bytes(chunk, sess.encoding)
|
||||
except Exception:
|
||||
text = ""
|
||||
if _is_prompt_only_echo(text, prompt_hint):
|
||||
continue
|
||||
# Non-prompt data: put back and stop settling.
|
||||
sess.out_queue.put(chunk)
|
||||
break
|
||||
try:
|
||||
sess.run_post_login_commands()
|
||||
except Exception:
|
||||
_log.debug("post_login failed session=%s", sess.session_id, exc_info=True)
|
||||
sess.state = "ready"
|
||||
sess.connect_finished_at = time.time()
|
||||
sess._ready_event.set()
|
||||
elapsed_ms = int((sess.connect_finished_at - sess.connect_started_at) * 1000)
|
||||
_audit(
|
||||
"session_created",
|
||||
session_id=sess.session_id,
|
||||
ne_id=sess.ne_id,
|
||||
ne_name=sess.ne_name,
|
||||
ne_ip=sess.ne_ip,
|
||||
protocol=sess.protocol,
|
||||
encoding=sess.encoding,
|
||||
source=str(device.get("source") or ""),
|
||||
hop_enabled=bool(creds.get("hop_enabled")),
|
||||
hop_vendor=str(creds.get("hop_vendor") or "") if creds.get("hop_enabled") else "",
|
||||
cli_hop_guard=bool(hop_guard),
|
||||
cli_hop_prompt=str((hop_guard or {}).get("hop_prompt") or ""),
|
||||
client=client or "",
|
||||
connect_ms=elapsed_ms,
|
||||
active=active_session_count(),
|
||||
)
|
||||
|
||||
|
||||
def create_session(
|
||||
db: Session,
|
||||
*,
|
||||
|
|
@ -555,6 +953,9 @@ def create_session(
|
|||
cols: int = 80,
|
||||
rows: int = 24,
|
||||
client: str = "",
|
||||
encoding: str = "utf-8",
|
||||
post_login_commands: list[str] | None = None,
|
||||
async_connect: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
from .cli_resolve import resolve_cli_target
|
||||
|
||||
|
|
@ -588,63 +989,8 @@ def create_session(
|
|||
device_type = str(device.get("device_type") or creds.get("device_type") or "")
|
||||
vendor = str(device.get("vendor") or creds.get("vendor") or "")
|
||||
cli_keymap = uses_network_cli_keymap(device_type, vendor)
|
||||
enc = _normalize_encoding(encoding)
|
||||
|
||||
# Capture real Telnet/SSH login I/O (banner, Username/Password, prompts).
|
||||
log_buf = io.BytesIO()
|
||||
try:
|
||||
conn = open_netmiko_connection(
|
||||
creds,
|
||||
session_timeout=connect_timeout,
|
||||
session_log=log_buf,
|
||||
cols=c,
|
||||
rows=r,
|
||||
interactive=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
partial = _session_log_text(log_buf).strip()
|
||||
_audit(
|
||||
"session_open_failed",
|
||||
session_id=session_id,
|
||||
ne_id=target_id,
|
||||
ne_ip=target_ip,
|
||||
source=str(device.get("source") or ""),
|
||||
client=client or "",
|
||||
error=str(exc)[:500],
|
||||
transcript_len=len(partial),
|
||||
)
|
||||
from .ne_cli_errors import format_cli_failure
|
||||
|
||||
classified = format_cli_failure(exc, partial)
|
||||
detail = f"connect_failed:{classified}"
|
||||
if partial:
|
||||
# Keep detail bounded; UI surfaces this on open failure.
|
||||
detail = f"{detail}\n--- device transcript ---\n{partial[-4000:]}"
|
||||
raise HTTPException(status_code=502, detail=detail) from exc
|
||||
|
||||
channel = getattr(conn, "remote_conn", None)
|
||||
if channel is not None and hasattr(channel, "resize_pty"):
|
||||
try:
|
||||
channel.resize_pty(width=c, height=r)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Prefer session_log (full login transcript including final prompt).
|
||||
_prime_interactive_channel(conn)
|
||||
bootstrap = prepare_bootstrap_output(_session_log_text(log_buf))
|
||||
if not bootstrap.strip():
|
||||
try:
|
||||
more = _drain_channel(conn, rounds=6, wait=0.1)
|
||||
except Exception:
|
||||
more = ""
|
||||
if more:
|
||||
bootstrap = prepare_bootstrap_output(more)
|
||||
# Discard any unread bytes so the live reader starts clean.
|
||||
try:
|
||||
_drain_channel(conn, rounds=3, wait=0.05)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
hop_guard = get_cli_hop_guard(conn)
|
||||
sess = WebcrtSession(
|
||||
session_id=session_id,
|
||||
ne_id=target_id,
|
||||
|
|
@ -656,34 +1002,51 @@ def create_session(
|
|||
device_type=device_type,
|
||||
vendor=vendor,
|
||||
cli_keymap=cli_keymap,
|
||||
conn=conn,
|
||||
bootstrap_output=str(bootstrap or "").encode("utf-8", errors="replace"),
|
||||
# Only nudge a live prompt when transcript has no recognizable prompt yet.
|
||||
needs_live_prompt=not _looks_like_cli_prompt(bootstrap),
|
||||
cli_hop_guard=bool(hop_guard),
|
||||
cli_hop_prompt=str((hop_guard or {}).get("hop_prompt") or ""),
|
||||
encoding=enc,
|
||||
state="connecting",
|
||||
post_login_commands=list(post_login_commands or [])[:20],
|
||||
)
|
||||
# Keep bootstrap for WS attach replay; do not rely solely on out_queue (StrictMode remount).
|
||||
sess.start_reader()
|
||||
|
||||
with _sessions_lock:
|
||||
_sessions[session_id] = sess
|
||||
|
||||
_audit(
|
||||
"session_created",
|
||||
"session_connecting",
|
||||
session_id=session_id,
|
||||
ne_id=sess.ne_id,
|
||||
ne_name=sess.ne_name,
|
||||
ne_ip=sess.ne_ip,
|
||||
protocol=sess.protocol,
|
||||
source=str(device.get("source") or ""),
|
||||
hop_enabled=bool(creds.get("hop_enabled")),
|
||||
hop_vendor=str(creds.get("hop_vendor") or "") if creds.get("hop_enabled") else "",
|
||||
cli_hop_guard=bool(hop_guard),
|
||||
cli_hop_prompt=str((hop_guard or {}).get("hop_prompt") or ""),
|
||||
encoding=enc,
|
||||
client=client or "",
|
||||
active=active_session_count(),
|
||||
async_connect=bool(async_connect),
|
||||
)
|
||||
|
||||
if async_connect:
|
||||
t = threading.Thread(
|
||||
target=_finish_connect,
|
||||
kwargs={
|
||||
"sess": sess,
|
||||
"creds": creds,
|
||||
"device": device,
|
||||
"connect_timeout": connect_timeout,
|
||||
"client": client or "",
|
||||
},
|
||||
name=f"webcrt-connect-{session_id[:8]}",
|
||||
daemon=True,
|
||||
)
|
||||
t.start()
|
||||
else:
|
||||
_finish_connect(
|
||||
sess,
|
||||
creds=creds,
|
||||
device=device,
|
||||
connect_timeout=connect_timeout,
|
||||
client=client or "",
|
||||
)
|
||||
if sess.state == "error":
|
||||
with _sessions_lock:
|
||||
_sessions.pop(session_id, None)
|
||||
raise HTTPException(status_code=502, detail=sess.connect_error or "connect_failed")
|
||||
|
||||
return {
|
||||
"session_id": session_id,
|
||||
"ne_id": sess.ne_id,
|
||||
|
|
@ -693,8 +1056,10 @@ def create_session(
|
|||
"protocol": sess.protocol,
|
||||
"cols": sess.cols,
|
||||
"rows": sess.rows,
|
||||
"encoding": enc,
|
||||
"state": sess.state,
|
||||
"ws_path": f"/v1/webcrt/sessions/{session_id}/ws",
|
||||
"cli_hop": bool(hop_guard),
|
||||
"cli_hop": bool(sess.cli_hop_guard),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -715,6 +1080,7 @@ def mark_attached(session_id: str) -> tuple[WebcrtSession, int]:
|
|||
ne_id=sess.ne_id,
|
||||
ne_ip=sess.ne_ip,
|
||||
attach_gen=attach_gen,
|
||||
state=sess.state,
|
||||
)
|
||||
return sess, attach_gen
|
||||
|
||||
|
|
@ -769,6 +1135,9 @@ def close_session(session_id: str, *, reason: str = "closed", client: str = "")
|
|||
ne_ip=sess.ne_ip,
|
||||
reason=reason,
|
||||
client=client or "",
|
||||
bytes_in=sess.bytes_in,
|
||||
bytes_out=sess.bytes_out,
|
||||
queue_dropped=getattr(sess.out_queue, "dropped", 0),
|
||||
active=active_session_count(),
|
||||
)
|
||||
return {"ok": True, "session_id": session_id, "closed": True, "reason": reason}
|
||||
|
|
@ -783,9 +1152,20 @@ def list_sessions() -> dict[str, Any]:
|
|||
"ne_name": s.ne_name,
|
||||
"ne_ip": s.ne_ip,
|
||||
"protocol": s.protocol,
|
||||
"encoding": s.encoding,
|
||||
"state": s.state,
|
||||
"attached": s.attached,
|
||||
"created_at": datetime.fromtimestamp(s.created_at, tz=timezone.utc).isoformat(),
|
||||
"last_activity": datetime.fromtimestamp(s.last_activity, tz=timezone.utc).isoformat(),
|
||||
"bytes_in": s.bytes_in,
|
||||
"bytes_out": s.bytes_out,
|
||||
"queue_depth": s.out_queue.qsize(),
|
||||
"queue_dropped": getattr(s.out_queue, "dropped", 0),
|
||||
"connect_ms": (
|
||||
int((s.connect_finished_at - s.connect_started_at) * 1000)
|
||||
if s.connect_finished_at
|
||||
else None
|
||||
),
|
||||
}
|
||||
for s in _sessions.values()
|
||||
if not s.closed
|
||||
|
|
@ -794,5 +1174,7 @@ def list_sessions() -> dict[str, Any]:
|
|||
"total": len(items),
|
||||
"max_sessions": max(1, int(settings.webcrt_max_sessions or 20)),
|
||||
"idle_timeout_sec": max(60, int(settings.webcrt_idle_timeout_sec or 1800)),
|
||||
"keepalive_sec": int(getattr(settings, "webcrt_keepalive_sec", 30) or 0),
|
||||
"anti_idle_sec": int(getattr(settings, "webcrt_anti_idle_sec", 0) or 0),
|
||||
"items": items,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue