feat(ume): expose WSS runtime logs on subscription status UI

Ring-buffer ws_logs API, alarm raise/clear labels with alarmkey, and scrollable log panel on UME page.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-05-22 21:15:15 +08:00
parent 635cd52d4f
commit 5bd83c58dc
6 changed files with 203 additions and 31 deletions

View file

@ -38,6 +38,7 @@ from .ume_alarm_ws import (
cancel_alarm_subscription_manual,
establish_alarm_subscription_manual,
get_subscription_status,
get_ws_logs,
load_persisted_subscription,
shutdown_ws_consumer,
start_ume_alarm_ws_consumer,
@ -927,15 +928,17 @@ def ume_token_disconnect() -> dict[str, Any]:
@app.get("/v1/ume/alarm-subscription/status")
def ume_alarm_subscription_status() -> dict[str, Any]:
def ume_alarm_subscription_status(limit: int = 80) -> dict[str, Any]:
st = get_subscription_status()
ws_task = _UME_RUNTIME_TASKS.get("alarms_current_ws_consumer") or {}
log_limit = max(10, min(int(limit or 80), 100))
return {
"ok": True,
**st,
"ws_consumer_status": str(ws_task.get("status") or ""),
"ws_consumer_last_error": str(ws_task.get("last_error") or ""),
"ws_consumer_last_run_at": ws_task.get("last_run_at"),
"ws_logs": get_ws_logs(limit=log_limit),
}

View file

@ -1,5 +1,7 @@
from __future__ import annotations
from collections import deque
from datetime import datetime, timezone
import json
import logging
import re
@ -19,13 +21,67 @@ from .ume_alarm_subscription_store import (
save_subscription,
)
from .ume_client import UMEClient
from .ume_sync_service import apply_alarm_to_current, extract_alarm_from_notification, _utc_now_naive
from .ume_sync_service import (
_alarm_key,
apply_alarm_to_current,
extract_alarm_from_notification,
normalize_yang_alarm,
_utc_now_naive,
)
_WS_ALARM_ACTION_LABEL: dict[str, str] = {
"inserted": "上报(新增)",
"updated": "上报(更新)",
"deleted": "清除",
"skipped": "忽略",
}
_ws_log = logging.getLogger("netx.ume.alarm_ws")
_WS_LOG_LOCK = threading.Lock()
_WS_LOG_ENTRIES: deque[dict[str, Any]] = deque(maxlen=200)
_WS_LOG_MAX_RETURN = 100
_ORPHAN_SUB_ID_RE = re.compile(r"id:([0-9a-fA-F-]{8}-[0-9a-fA-F-]{4}-[0-9a-fA-F-]{4}-[0-9a-fA-F-]{4}-[0-9a-fA-F-]{12})")
def _utc_now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def append_ws_log(
message: str,
*,
level: str = "info",
subscription_id: str = "",
) -> None:
"""Ring buffer of recent WSS events for UI (newest last)."""
entry = {
"ts": _utc_now_iso(),
"level": str(level or "info").strip().lower() or "info",
"message": str(message or "").strip()[:500],
"subscription_id": str(subscription_id or "").strip(),
}
with _WS_LOG_LOCK:
_WS_LOG_ENTRIES.append(entry)
log_fn = _ws_log.info
if entry["level"] == "warning":
log_fn = _ws_log.warning
elif entry["level"] == "error":
log_fn = _ws_log.error
log_fn("%s%s", entry["message"], f" sub={entry['subscription_id']}" if entry["subscription_id"] else "")
def get_ws_logs(*, limit: int | None = None) -> list[dict[str, Any]]:
cap = int(limit if limit is not None else _WS_LOG_MAX_RETURN)
cap = max(1, min(cap, _WS_LOG_MAX_RETURN))
with _WS_LOG_LOCK:
items = list(_WS_LOG_ENTRIES)
if len(items) <= cap:
return items
return items[-cap:]
def parse_subscription_id_from_already_exists_error(message: str) -> str:
"""Extract subscription id from UME 400 'topic subscription already exist, id:...'."""
m = _ORPHAN_SUB_ID_RE.search(str(message or ""))
@ -107,7 +163,7 @@ def load_persisted_subscription() -> bool:
return False
sub_id, uri, topic = loaded
_set_active_subscription(sub_id, uri, topic=topic)
_ws_log.info("loaded persisted alarm subscription id=%s", sub_id)
append_ws_log(f"loaded persisted subscription id={sub_id}", subscription_id=sub_id)
return True
finally:
db.close()
@ -137,14 +193,14 @@ def establish_alarm_subscription_manual(client: UMEClient, db: Session) -> dict[
sub_id, uri, topic = existing
_set_active_subscription(sub_id, uri, topic=topic)
request_ws_reconnect()
_ws_log.info("establish skipped: subscription already exists id=%s", sub_id)
append_ws_log(f"establish skipped: already in DB id={sub_id}", subscription_id=sub_id)
st = get_subscription_status()
return {**st, "already_exists": True}
mem_id, mem_uri = get_active_subscription()
if mem_id and mem_uri:
request_ws_reconnect()
_ws_log.info("establish skipped: in-memory subscription id=%s", mem_id)
append_ws_log(f"establish skipped: in-memory id={mem_id}", subscription_id=mem_id)
st = get_subscription_status()
return {**st, "already_exists": True}
@ -161,7 +217,11 @@ def establish_alarm_subscription_manual(client: UMEClient, db: Session) -> dict[
orphan_id = parse_subscription_id_from_already_exists_error(str(exc))
if not orphan_id:
raise
_ws_log.warning("establish: UME reports existing subscription id=%s, deleting then retry", orphan_id)
append_ws_log(
f"establish: orphan on UME id={orphan_id}, deleting then retry",
level="warning",
subscription_id=orphan_id,
)
try:
_delete_subscription_on_ume(client, orphan_id)
except RuntimeError as del_exc:
@ -177,7 +237,7 @@ def establish_alarm_subscription_manual(client: UMEClient, db: Session) -> dict[
_active_client = client
_set_active_subscription(sub_id, uri, topic=topic)
request_ws_reconnect()
_ws_log.info("manual establish alarm subscription id=%s", sub_id)
append_ws_log(f"manual establish subscription id={sub_id}", subscription_id=sub_id)
return {**get_subscription_status(), "already_exists": False}
@ -195,7 +255,7 @@ def cancel_alarm_subscription_manual(client: UMEClient, db: Session) -> dict[str
db.commit()
_clear_active_subscription()
request_ws_reconnect()
_ws_log.info("manual cancel alarm subscription id=%s", sub_id)
append_ws_log(f"manual cancel subscription id={sub_id or '(none)'}", subscription_id=sub_id)
return get_subscription_status()
@ -224,35 +284,48 @@ def _run_ws_session(
def _on_message(_ws: Any, message: str) -> None:
payload = _parse_ws_message(message)
if payload is None:
append_ws_log("message ignored: invalid JSON", level="warning", subscription_id=subscription_id)
return
db = SessionLocal()
try:
action, changed = process_alarm_notification(db, payload)
alarm = extract_alarm_from_notification(payload)
if alarm is None:
append_ws_log("收包忽略: 非告警通知", level="warning", subscription_id=subscription_id)
return
norm = normalize_yang_alarm(alarm) or {}
alarm_key = _alarm_key(norm) or "?"
action, changed = apply_alarm_to_current(db, alarm, touch_ts=_utc_now_naive())
if changed:
db.commit()
else:
db.rollback()
label = _WS_ALARM_ACTION_LABEL.get(action, action)
status_msg = f"{label} key={alarm_key}" + ("" if changed else " (无变更)")
append_ws_log(status_msg, subscription_id=subscription_id)
if on_status is not None:
on_status(f"last={action}")
except Exception as exc:
db.rollback()
_ws_log.exception("ws alarm apply failed: %s", exc)
append_ws_log(f"alarm apply failed: {str(exc)[:200]}", level="error", subscription_id=subscription_id)
if on_status is not None:
on_status(f"apply_error:{str(exc)[:120]}")
finally:
db.close()
def _on_error(_ws: Any, error: Any) -> None:
_ws_log.warning("ws error subscription=%s: %s", subscription_id, error)
append_ws_log(f"ws error: {str(error)[:200]}", level="error", subscription_id=subscription_id)
if on_status is not None:
on_status(f"ws_error:{str(error)[:120]}")
def _on_close(_ws: Any, close_status_code: Any, close_msg: Any) -> None:
_ws_log.info("ws closed subscription=%s code=%s msg=%s", subscription_id, close_status_code, close_msg)
append_ws_log(
f"ws closed code={close_status_code} msg={str(close_msg or '')[:120]}",
subscription_id=subscription_id,
)
closed.set()
def _on_open(_ws: Any) -> None:
_ws_log.info("ws connected subscription=%s", subscription_id)
append_ws_log(f"ws connected uri={wss_uri[:120]}", subscription_id=subscription_id)
if on_status is not None:
on_status("connected")
@ -317,28 +390,31 @@ def run_alarm_ws_consumer_loop(
backoff_s = 2.0
max_backoff_s = 120.0
def _status(msg: str, *, level: str = "info", sub_id: str = "") -> None:
append_ws_log(msg, level=level, subscription_id=sub_id)
if on_status is not None:
on_status(msg)
while stop_event is None or not stop_event.is_set():
if is_paused is not None and is_paused():
if on_status is not None:
on_status("paused")
_status("paused", level="warning")
time.sleep(1.0)
continue
subscription_id, wss_uri = get_active_subscription()
if not subscription_id or not wss_uri:
if on_status is not None:
on_status("no_subscription")
_status("no_subscription")
_ws_wake_event.wait(timeout=2.0)
_ws_wake_event.clear()
continue
try:
if not _wait_for_shared_token(client, timeout_s=120.0):
if on_status is not None:
on_status("waiting_token")
_status("waiting_token", level="warning", sub_id=subscription_id)
time.sleep(2.0)
continue
append_ws_log(f"connecting {wss_uri[:160]}", subscription_id=subscription_id)
_run_ws_session(
client,
wss_uri=wss_uri,
@ -349,28 +425,21 @@ def run_alarm_ws_consumer_loop(
backoff_s = 2.0
except RuntimeError as exc:
if "ume_ws_no_valid_token" in str(exc):
if on_status is not None:
on_status("waiting_token")
_status("waiting_token", level="warning", sub_id=subscription_id)
time.sleep(2.0)
continue
_ws_log.exception("alarm ws session failed: %s", exc)
if on_status is not None:
on_status(f"error:{str(exc)[:120]}")
_status(f"session error: {str(exc)[:200]}", level="error", sub_id=subscription_id)
except Exception as exc:
_ws_log.exception("alarm ws session failed: %s", exc)
if on_status is not None:
on_status(f"error:{str(exc)[:120]}")
_status(f"session error: {str(exc)[:200]}", level="error", sub_id=subscription_id)
sub_id_after, uri_after = get_active_subscription()
if stop_event is not None and stop_event.is_set():
break
if not sub_id_after or not uri_after:
if on_status is not None:
on_status("no_subscription")
_status("no_subscription")
continue
if on_status is not None:
on_status(f"reconnect_ws_in_{int(backoff_s)}s")
_status(f"reconnect in {int(backoff_s)}s", sub_id=sub_id_after)
slept = 0.0
while slept < backoff_s:
if stop_event is not None and stop_event.is_set():
@ -402,9 +471,11 @@ def start_ume_alarm_ws_consumer(
if not bool(getattr(settings, "ume_alarm_ws_enabled", True)):
return
if not str(client.base_url or "").strip():
append_ws_log("consumer disabled: no UME base URL", level="warning")
if on_status is not None:
on_status("disabled:no_base_url")
return
append_ws_log("ws consumer thread started")
global _active_client
with _shutdown_lock:
_active_client = client

View file

@ -16,9 +16,11 @@ from netx_api import ume_alarm_ws
from netx_api.models import UmeAlarmSubscription
from netx_api.ume_alarm_subscription_store import clear_subscription, load_subscription, save_subscription
from netx_api.ume_alarm_ws import (
append_ws_log,
cancel_alarm_subscription_manual,
establish_alarm_subscription_manual,
get_subscription_status,
get_ws_logs,
load_persisted_subscription,
parse_subscription_id_from_already_exists_error,
process_alarm_notification,
@ -736,6 +738,22 @@ class UmeAlarmNotificationTests(unittest.TestCase):
self.assertTrue(changed)
self.assertIsNone(self.db.get(UmeAlarmCurrent, "AK-CLEARED-1"))
def test_ws_log_ring_buffer(self):
from netx_api import ume_alarm_ws as ws_mod
with ws_mod._WS_LOG_LOCK:
ws_mod._WS_LOG_ENTRIES.clear()
try:
for i in range(5):
append_ws_log(f"line-{i}")
logs = get_ws_logs(limit=3)
self.assertEqual(len(logs), 3)
self.assertEqual(logs[-1]["message"], "line-4")
self.assertIn("ts", logs[0])
finally:
with ws_mod._WS_LOG_LOCK:
ws_mod._WS_LOG_ENTRIES.clear()
def test_parse_subscription_id_from_already_exists_error(self):
msg = (
'ume_request_failed:400:{"error":{"errorInfo":"topic subscription already exist, '

View file

@ -189,6 +189,46 @@ pre {
padding: 10px;
}
.ws-log-panel {
margin-top: 12px;
max-height: 240px;
overflow-y: auto;
background: #091425;
border: 1px solid #223651;
border-radius: 8px;
padding: 8px 10px;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 12px;
line-height: 1.45;
}
.ws-log-line {
padding: 3px 0;
border-bottom: 1px solid #152238;
word-break: break-word;
}
.ws-log-line:last-child {
border-bottom: none;
}
.ws-log-line--error {
color: #ff8a9a;
}
.ws-log-line--warning {
color: #f0c060;
}
.ws-log-line--info {
color: #b8c9e0;
}
.ws-log-ts {
color: #6b8299;
margin-right: 8px;
}
.pill {
margin-top: 8px;
padding: 4px 8px;

View file

@ -197,6 +197,7 @@ export function UmePage({ toastOk, toastError }: UmePageProps) {
syncStatusQuery.data?.alarm_subscription ??
({ active: false } as const);
const wsConsumer = runtimeTasks.find((t) => t.task === "alarms_current_ws_consumer");
const wsLogs = [...(subscriptionStatusQuery.data?.ws_logs ?? alarmSub.ws_logs ?? [])].reverse();
const subscriptionActive = Boolean(alarmSub.active);
const subPending =
subscriptionEstablishMutation.isPending || subscriptionCancelMutation.isPending;
@ -364,6 +365,37 @@ export function UmePage({ toastOk, toastError }: UmePageProps) {
{subscriptionOpError ? (
<div className="pill pill--high">订阅操作失败: {subscriptionOpError}</div>
) : null}
<div style={{ marginTop: 12 }}>
<div className="actions-row actions-row--inline" style={{ marginTop: 0 }}>
<strong style={{ fontSize: 13 }}>WSS 运行日志</strong>
<span className="muted" style={{ fontSize: 12 }}>
{wsConsumer?.last_run_at
? `最近活动 ${formatSystemTime(String(wsConsumer.last_run_at))}`
: "自动刷新"}
</span>
</div>
<div className="ws-log-panel" role="log" aria-live="polite">
{wsLogs.length === 0 ? (
<div className="ws-log-line ws-log-line--info">暂无日志(建立订阅并连接 WSS 后会出现连接、收包、重连等记录)</div>
) : (
wsLogs.map((line, idx) => {
const lvl = String(line.level || "info").toLowerCase();
const cls =
lvl === "error" ? "ws-log-line--error" : lvl === "warning" ? "ws-log-line--warning" : "ws-log-line--info";
return (
<div key={`${line.ts}-${idx}`} className={`ws-log-line ${cls}`}>
<span className="ws-log-ts">{formatSystemTime(line.ts)}</span>
<span>[{lvl}] </span>
<span>{line.message}</span>
{line.subscription_id ? (
<span className="muted"> · {String(line.subscription_id).slice(0, 8)}…</span>
) : null}
</div>
);
})
)}
</div>
</div>
</article>
<article className="card card--full">
<h3>UME 同步</h3>

View file

@ -89,6 +89,13 @@ export type UmeSyncJobItem = {
ended_at?: string | null;
};
export type UmeWsLogEntry = {
ts: string;
level: string;
message: string;
subscription_id?: string;
};
export type UmeAlarmSubscriptionStatus = {
ok?: boolean;
created?: boolean;
@ -100,6 +107,7 @@ export type UmeAlarmSubscriptionStatus = {
ws_consumer_status?: string;
ws_consumer_last_error?: string;
ws_consumer_last_run_at?: string | null;
ws_logs?: UmeWsLogEntry[];
};
export type UmeSyncStatusResponse = {