Speed up WebCRT and record audited commands from the device echo.

Performance: event-driven stdout pump (no polling thread per socket, no 200ms
coalescing stall), keystrokes written immediately, blocking SSH reads, bounded
deque queue with batch drain, incremental UTF-8 decode, buffered session log,
non-blocking session teardown, shorter login/settle waits, xterm WebGL renderer
and idle-time direct writes. Zero-latency fake device: echo p50 11ms -> 2.3ms,
command RTT 219ms -> 20ms, bulk output 2.1 -> 8.3 MB/s.

Audit: track the device's own input row server-side (webcrt_cmd_audit) and
resolve each Enter from the echoed line feed, so Tab, backspace, cursor-key
edits, history recall, paste, Ctrl-C and pagers record what actually ran.
Password lines are redacted; pending Enters are flushed on close.

Hop: stop closing the hop session when the target prints the same prompt as the
hop (system-view); recognise the Huawei "closed by the remote host" message.
Generic NEs that show a network prompt now get the network key map.

Adds bench_webcrt.py, fake_ssh_device.py and e2e_webcrt_input_scenarios.py.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
oliver 2026-10-12 07:34:30 +08:00
parent 0d894efe5e
commit 45de8144a3
14 changed files with 2060 additions and 241 deletions

239
scripts/bench_webcrt.py Normal file
View file

@ -0,0 +1,239 @@
#!/usr/bin/env python3
"""WebCRT latency / throughput / audit benchmark against a live netx + lab device.
python scripts/bench_webcrt.py --ne-id <id> [--prompt-re "<r1>|\\[~?r1\\]"] [--audit]
Measures (client-side, real WebSocket):
* connect_ms create session -> first prompt visible
* echo latency single keystroke -> echoed byte (p50/p95/max)
* burst echo latency 20 keys sent 3ms apart -> last echo byte
* command RTT Enter -> next prompt for a short command
* bulk throughput long output command, bytes / seconds
With --audit it also types a set of tricky command lines (backspace, tab, history
recall, bare Enter) and prints what landed in audit.jsonl for the session.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import re
import statistics
import sys
import time
import httpx
import websockets
BASE = os.environ.get("NETX_BASE", "http://127.0.0.1:8890").rstrip("/")
USER = os.environ.get("NETX_USER", "admin")
PWD = os.environ.get("NETX_PASSWORD", "Admin@1234")
def pct(vals: list[float], p: float) -> float:
if not vals:
return float("nan")
s = sorted(vals)
return s[min(len(s) - 1, int(len(s) * p))]
async def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--ne-id", default="")
ap.add_argument("--quick", default="", help="ip:port:user:pass -> quick-connect instead of --ne-id")
ap.add_argument("--prompt-re", default=r"(?:<[^>\r\n]+>|\[[^\]\r\n]+\]|[\w.\-]+(?:\([^)]+\))*[#>])\s*$")
ap.add_argument("--bulk-cmd", default="display current-configuration")
ap.add_argument("--probe-cmd", default="display clock")
ap.add_argument("--audit", action="store_true")
ap.add_argument("--echo-only", action="store_true", help="only connect + echo latency")
ap.add_argument("--audit-cmds", default="")
ap.add_argument("--audit-file", default=os.path.join("data", "webcrt", "audit.jsonl"))
args = ap.parse_args()
prompt_re = re.compile(args.prompt_re)
c = httpx.Client(base_url=BASE, timeout=60)
tok = c.post("/v1/auth/login", json={"username": USER, "password": PWD}).json()["access_token"]
h = {"Authorization": f"Bearer {tok}"}
t_create = time.perf_counter()
if args.quick:
ip, port, user, pw = args.quick.split(":", 3)
r = c.post("/v1/webcrt/sessions/quick-connect", headers=h, json={
"name": "bench", "ip_address": ip, "port": int(port), "protocol": "ssh", "username": user,
"password": pw, "save_password": False, "cols": 160, "rows": 50})
else:
r = c.post("/v1/webcrt/sessions", headers=h, json={"ne_id": args.ne_id, "cols": 160, "rows": 50})
r.raise_for_status()
sid = r.json()["session_id"]
ticket = c.post("/v1/webcrt/ws-ticket", headers=h).json()["ticket"]
url = f"{BASE.replace('http', 'ws', 1)}/v1/webcrt/sessions/{sid}/ws?ws_ticket={ticket}"
print(f"session={sid}")
async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws:
buf = ""
ev = asyncio.Event()
last_rx = 0.0
rx_bytes = 0
async def reader() -> None:
nonlocal buf, last_rx, rx_bytes
dec = __import__("codecs").getincrementaldecoder("utf-8")(errors="replace")
async for raw in ws:
if isinstance(raw, bytes):
buf += dec.decode(raw)
rx_bytes += len(raw)
last_rx = time.perf_counter()
ev.set()
else:
m = json.loads(raw)
if m.get("type") == "status" and m.get("state") in ("error", "closed", "warning"):
print("STATUS", m)
if m.get("type") == "stdout":
buf += m.get("data", "")
last_rx = time.perf_counter()
ev.set()
rt = asyncio.create_task(reader())
def at_prompt() -> bool:
tail = re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", buf).replace("\r", "\n").rstrip(" ").split("\n")[-1]
return bool(prompt_re.search(tail)) and len(tail) < 80
async def wait_prompt(timeout: float = 60.0, quiet: float = 0.05) -> bool:
end = time.perf_counter() + timeout
while time.perf_counter() < end:
if at_prompt() and time.perf_counter() - last_rx >= quiet:
return True
await asyncio.sleep(0.005)
return False
async def send(data: str, audit_line: str | None = None) -> None:
p = {"type": "stdin", "data": data}
if audit_line:
p["audit_line"] = audit_line
await ws.send(json.dumps(p))
ok = await wait_prompt(90, 0.3)
connect_ms = (time.perf_counter() - t_create) * 1000
if not ok:
await send("\r")
ok = await wait_prompt(15, 0.3)
print(f"connect_ms={connect_ms:.0f} prompt={'yes' if ok else 'NO'} tail={buf[-80:]!r}")
if not ok:
return 2
# --- single-key echo latency ---
lat: list[float] = []
for ch in "abcdefghijklmnopqrst":
n0 = rx_bytes
t0 = time.perf_counter()
await send(ch)
while rx_bytes == n0 and time.perf_counter() - t0 < 2:
await asyncio.sleep(0.0005)
lat.append((time.perf_counter() - t0) * 1000)
await asyncio.sleep(0.05)
await send("\x08" * 20)
await asyncio.sleep(0.4)
print(f"echo_ms p50={statistics.median(lat):.1f} p95={pct(lat, .95):.1f} max={max(lat):.1f}")
if args.echo_only:
await ws.send(json.dumps({"type": "close"}))
await asyncio.sleep(0.3)
rt.cancel()
return 0
# --- burst typing: last echo arrival ---
await wait_prompt(5, 0.1)
n0 = len(buf)
t0 = time.perf_counter()
for ch in "abcdefghijklmnopqrst":
await send(ch)
await asyncio.sleep(0.003)
t_sent = time.perf_counter()
while time.perf_counter() - t_sent < 3 and not buf.endswith("abcdefghijklmnopqrst"):
await asyncio.sleep(0.002)
print(f"burst_echo_ms={(time.perf_counter() - t_sent) * 1000:.1f} (after last key sent)")
await send("\x08" * 20)
await asyncio.sleep(0.4)
# --- command RTT ---
rtts: list[float] = []
for _ in range(5):
await wait_prompt(10, 0.05)
n0 = len(buf)
t0 = time.perf_counter()
await send(args.probe_cmd + "\r")
await asyncio.sleep(0.02)
await wait_prompt(15, 0.0)
# prompt re-appeared after output
while len(buf) == n0 and time.perf_counter() - t0 < 10:
await asyncio.sleep(0.005)
rtts.append((time.perf_counter() - t0) * 1000)
print(f"cmd_rtt_ms p50={statistics.median(rtts):.0f} max={max(rtts):.0f} ({args.probe_cmd})")
# --- bulk ---
await wait_prompt(10, 0.2)
rx0 = rx_bytes
t0 = time.perf_counter()
await send(args.bulk_cmd + "\r")
await asyncio.sleep(0.2)
# page through "More" prompts if any
deadline = time.perf_counter() + 90
while time.perf_counter() < deadline:
tail = buf[-200:]
if "More" in tail[-40:]:
await send(" ")
await asyncio.sleep(0.01)
continue
if at_prompt() and time.perf_counter() - last_rx > 0.4 and rx_bytes - rx0 > 100:
break
await asyncio.sleep(0.01)
dt = time.perf_counter() - t0 - 0.4
print(f"bulk bytes={rx_bytes - rx0} secs={dt:.2f} KB/s={(rx_bytes - rx0) / 1024 / max(dt, 1e-3):.0f}")
if args.audit:
await wait_prompt(10, 0.2)
cmds = [__import__("codecs").decode(x, "unicode_escape") for x in args.audit_cmds.split("|") if x] or [
"display clock\r",
"disX\x08play ver\t\r",
"dis ip int br\t\r",
"\r",
"display clock\r",
"\x1b[A\r",
"\x1b[A\x1b[A\x1b[B\r",
"system-view\r",
"sysname r1\r",
"quit\r",
]
for cmd in cmds:
await send(cmd)
await asyncio.sleep(0.2)
await wait_prompt(10, 0.25)
if "More" in buf[-40:]:
await send(" ")
await asyncio.sleep(0.5)
await ws.send(json.dumps({"type": "close"}))
await asyncio.sleep(0.3)
rt.cancel()
time.sleep(0.8)
if args.audit:
print("--- audit.jsonl commands for session ---")
try:
with open(args.audit_file, encoding="utf-8") as fh:
for line in fh:
try:
o = json.loads(line)
except Exception:
continue
if o.get("event") == "command" and o.get("session_id") == sid:
print(repr(o.get("command")), o.get("source"))
except OSError as exc:
print("cannot read audit:", exc)
return 0
if __name__ == "__main__":
raise SystemExit(asyncio.run(main()))

View file

@ -0,0 +1,189 @@
#!/usr/bin/env python3
"""Keystroke-level WebCRT audit scenarios against a live device.
Every key is sent as its own stdin frame (~40ms apart, like a person typing) and the
command that landed in audit.jsonl for the session is compared with what the device
actually executed.
python scripts/e2e_webcrt_input_scenarios.py --ne-id <id> --vendor huawei
python scripts/e2e_webcrt_input_scenarios.py --ne-id <id> --vendor cisco
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import re
import sys
import time
import httpx
import websockets
BASE = os.environ.get("NETX_BASE", "http://127.0.0.1:8890").rstrip("/")
USER = os.environ.get("NETX_USER", "admin")
PWD = os.environ.get("NETX_PASSWORD", "Admin@1234")
LEFT, RIGHT, UP, DOWN = "\x1b[D", "\x1b[C", "\x1b[A", "\x1b[B"
HOME, END, DEL, BS, TAB, CR, CTRL_C = "\x1b[H", "\x1b[F", "\x1b[3~", "\x08", "\t", "\r", "\x03"
def keys(*parts: str | list[str]) -> list[str]:
"""Flatten: a plain str is typed char by char, escape sequences stay whole."""
out: list[str] = []
for p in parts:
if isinstance(p, list):
out.extend(p)
elif p.startswith("\x1b") or p in (BS, TAB, CR, CTRL_C):
out.append(p)
else:
out.extend(list(p))
return out
def scenarios(vendor: str) -> list[tuple[str, list[str], "str | list[str]"]]:
"""(name, keystrokes, expected command text after the prompt)."""
if vendor == "cisco":
full, partial, tabbed, other = "show clock", "show cl", "sh clo", "show version"
wrong = "show clxock"
else:
full, partial, tabbed, other = "display clock", "display cl", "display clo", "display version"
wrong = "display clxock"
word = full.split()[-1] # clock
return [
("plain", keys(full, CR), full),
("backspace fix", keys(full[:-3] + "x", BS, full[-3:], CR), full),
("tab completion", keys(tabbed, TAB, CR), "sh clock" if vendor == "cisco" else full),
("left-arrow insert mid-line", keys(full.replace("clock", "cock"), [LEFT] * 3, "l", CR), full),
("left-arrow + backspace delete", keys(wrong, [LEFT] * 3, BS, CR), full),
("left-arrow + Delete key", keys(wrong, [LEFT] * 4, DEL, CR), full),
("left-arrow replace char", keys(full.replace("clock", "clxck"), [LEFT] * 2, BS, "o", CR), full),
("home + insert first char", keys(full[1:], HOME, full[0], END, CR), full),
("right-arrow after left", keys(full, LEFT, LEFT, RIGHT, RIGHT, CR), full),
("up-arrow recall", keys(UP, CR), full),
("up-arrow recall then edit tail", keys(UP, [BS] * len(word), other.split()[-1], CR), other),
("up up down recall", keys(UP, UP, DOWN, CR), other),
("recall + left-arrow delete+insert", keys(UP, [LEFT] * 2, BS, "c", CR), other[:-3] + "con"),
("ctrl-c (device decides if line clears)", keys("garbage", CTRL_C, full, CR), ("garbage" + full) if vendor == "huawei" else full),
("empty enter x2 (not audited)", keys(CR, CR), ""),
("run then recall and run again", keys(full, CR, UP, CR), [full, full]),
]
async def run(args: argparse.Namespace) -> int:
c = httpx.Client(base_url=BASE, timeout=60)
tok = c.post("/v1/auth/login", json={"username": USER, "password": PWD}).json()["access_token"]
h = {"Authorization": f"Bearer {tok}"}
sid = c.post("/v1/webcrt/sessions", headers=h, json={"ne_id": args.ne_id, "cols": 160, "rows": 50}).json()["session_id"]
ticket = c.post("/v1/webcrt/ws-ticket", headers=h).json()["ticket"]
url = f"{BASE.replace('http', 'ws', 1)}/v1/webcrt/sessions/{sid}/ws?ws_ticket={ticket}"
prompt_re = re.compile(r"(?:<[^>\r\n]+>|\[[^\]\r\n]+\]|[\w.\-]+(?:\([^)]+\))*[#>])\s*$")
results: list[tuple[str, str | None, list[str]]] = []
async with websockets.connect(url, max_size=None) as ws:
buf = ""
last_rx = 0.0
async def reader() -> None:
nonlocal buf, last_rx
try:
async for raw in ws:
if isinstance(raw, bytes):
buf += raw.decode("utf-8", "replace")
last_rx = time.perf_counter()
except Exception:
pass
rt = asyncio.create_task(reader())
def at_prompt() -> bool:
plain = re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", buf).replace("\r", "\n")
tail = plain.rstrip(" ").split("\n")[-1]
return bool(prompt_re.search(tail)) and len(tail) < 80
async def settle(timeout: float = 20.0, quiet: float = 0.6) -> None:
end = time.perf_counter() + timeout
while time.perf_counter() < end:
if at_prompt() and time.perf_counter() - last_rx >= quiet:
return
await asyncio.sleep(0.02)
async def send(data: str) -> None:
await ws.send(json.dumps({"type": "stdin", "data": data}))
await settle(120, 1.0)
if not at_prompt():
await send("\r")
await settle(15, 1.0)
sel = [x.strip() for x in args.only.split(",") if x.strip()]
for name, ks, expected in scenarios(args.vendor):
if sel and not name.startswith(tuple(sel)):
continue
before = len(read_cmds(sid, args.audit_file))
for k in ks:
await send(k)
await asyncio.sleep(args.key_gap)
# Wait for the device (slow VMs echo seconds late), not a fixed delay: first for
# output after the last key, then until the expected audit rows exist.
sent_at = time.perf_counter()
while last_rx < sent_at and time.perf_counter() - sent_at < 15:
await asyncio.sleep(0.02)
await settle()
want = len(expected) if isinstance(expected, list) else (0 if expected == "" else 1)
waited = time.perf_counter()
while time.perf_counter() - waited < (2.0 if want == 0 else 15.0):
if len(read_cmds(sid, args.audit_file)) - before >= want:
break
await asyncio.sleep(0.1)
await asyncio.sleep(1.5) # Tab lines settle ~1.2s; also catches stray extra rows
got = read_cmds(sid, args.audit_file)[before:]
results.append((name, expected, got))
await ws.send(json.dumps({"type": "close"}))
await asyncio.sleep(0.5)
rt.cancel()
bad = 0
print(f"{'scenario':38} {'result':6} recorded")
for name, expected, got in results:
tails = [re.sub(r"^(?:<[^>]*>|\[[^\]]*\]|[\w.\-]+(?:\([^)]*\))*[#>])\s*", "", g) for g in got]
if isinstance(expected, list):
ok = tails == expected
elif expected == "":
ok = not got
else:
ok = tails == [expected]
bad += 0 if ok else 1
print(f"{name:38} {'OK' if ok else 'FAIL':6} {got!r}" + ("" if ok else f" expected {expected!r}"))
print(f"\n{len(results) - bad}/{len(results)} scenarios correct")
return 1 if bad else 0
def read_cmds(sid: str, path: str) -> list[str]:
out: list[str] = []
try:
with open(path, encoding="utf-8") as fh:
for line in fh:
if '"event": "command"' not in line:
continue
o = json.loads(line)
if o.get("session_id") == sid:
out.append(str(o.get("command") or ""))
except OSError:
pass
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--ne-id", required=True)
ap.add_argument("--vendor", choices=["huawei", "cisco"], required=True)
ap.add_argument("--only", default="", help="comma-separated scenario name prefixes")
ap.add_argument("--key-gap", type=float, default=0.04, help="seconds between keystrokes")
ap.add_argument("--audit-file", default=os.path.join("data", "webcrt", "audit.jsonl"))
return asyncio.run(run(ap.parse_args()))
if __name__ == "__main__":
sys.exit(main())

167
scripts/fake_ssh_device.py Normal file
View file

@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""Local fake network-device SSH server for WebCRT pipeline benchmarks.
Zero device latency, so a benchmark against it measures netx overhead only.
python scripts/fake_ssh_device.py [--port 2222] [--user lab] [--password lab]
Behaviour: Cisco-like ``test#`` prompt, server-side echo with Tab completion
(``sh<Tab>`` -> ``show ``), history on Up arrow, ``show clock`` (short output),
``bulk <MB>`` (large output) and ``quit``.
"""
from __future__ import annotations
import argparse
import socket
import threading
import time
import paramiko
HOST_KEY = paramiko.RSAKey.generate(2048)
PROMPT = b"test#"
class _Server(paramiko.ServerInterface):
def __init__(self, user: str, password: str) -> None:
self.user, self.password = user, password
self.event = threading.Event()
def check_auth_password(self, username, password): # noqa: ANN001
ok = username == self.user and password == self.password
return paramiko.AUTH_SUCCESSFUL if ok else paramiko.AUTH_FAILED
def get_allowed_auths(self, username): # noqa: ANN001
return "password"
def check_channel_request(self, kind, chanid): # noqa: ANN001
return paramiko.OPEN_SUCCEEDED if kind == "session" else paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED
def check_channel_pty_request(self, *a): # noqa: ANN002
return True
def check_channel_shell_request(self, channel): # noqa: ANN001
self.event.set()
return True
def check_channel_window_change_request(self, *a): # noqa: ANN002
return True
def _shell(chan: paramiko.Channel) -> None:
chan.send(b"\r\nFake device banner\r\n" + PROMPT)
line = b""
history: list[bytes] = []
hidx = 0
try:
while True:
data = chan.recv(1024)
if not data:
return
i = 0
while i < len(data):
b = data[i : i + 1]
if data[i : i + 3] == b"\x1b[A": # Up
if history:
hidx = max(0, hidx - 1) if hidx else len(history) - 1
chan.send(b"\x08" * len(line) + b" " * len(line) + b"\x08" * len(line))
line = history[hidx]
chan.send(line)
i += 3
continue
if b == b"\t":
if line.startswith(b"sh") and b" " not in line:
chan.send(b"ow "[len(line) - 2 :] if len(line) <= 4 else b"")
line = b"show "
i += 1
continue
if b in (b"\r", b"\n"):
chan.send(b"\r\n")
cmd = line.strip()
line = b""
if cmd:
history.append(cmd)
hidx = len(history)
if cmd == b"quit":
chan.close()
return
if cmd == b"show clock":
chan.send(b"2026-10-12 12:00:00\r\nMonday\r\n")
elif cmd.startswith(b"bulk"):
mb = float(cmd.split()[1]) if len(cmd.split()) > 1 else 1.0
row = b"Ethernet1/0/%d unassigned up down --\r\n"
sent = 0
n = 0
buf = []
while sent < mb * 1024 * 1024:
r = row % (n % 999)
buf.append(r)
sent += len(r)
n += 1
if len(buf) >= 400:
chan.sendall(b"".join(buf))
buf = []
if buf:
chan.sendall(b"".join(buf))
elif cmd:
chan.send(b"%% Unknown command\r\n")
chan.send(PROMPT)
i += 1
continue
if b in (b"\x08", b"\x7f"):
if line:
line = line[:-1]
chan.send(b"\x08 \x08")
i += 1
continue
line += b
chan.send(b)
i += 1
except Exception: # noqa: BLE001
pass
finally:
try:
chan.close()
except Exception: # noqa: BLE001
pass
def _serve(sock: socket.socket, user: str, password: str) -> None:
t = paramiko.Transport(sock)
t.add_server_key(HOST_KEY)
srv = _Server(user, password)
try:
t.start_server(server=srv)
chan = t.accept(30)
if chan is None:
return
srv.event.wait(10)
_shell(chan)
except Exception: # noqa: BLE001
pass
finally:
try:
t.close()
except Exception: # noqa: BLE001
pass
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--port", type=int, default=2222)
ap.add_argument("--user", default="lab")
ap.add_argument("--password", default="lab")
a = ap.parse_args()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(("127.0.0.1", a.port))
s.listen(20)
print(f"fake device on 127.0.0.1:{a.port} user={a.user}", flush=True)
while True:
c, _ = s.accept()
threading.Thread(target=_serve, args=(c, a.user, a.password), daemon=True).start()
if __name__ == "__main__":
main()