feat(ume): OClaw forwarder runtime task and i18n status codes

Register oclaw_alarm_forwarder in background tasks with pause/resume, emit rt:/ws:/fwd: codes for last_error, and translate them in the UI for English and Chinese.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-06-24 10:29:10 +08:00
parent 5a00736f23
commit ba1a40f725
10 changed files with 380 additions and 37 deletions

View file

@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
from .key_alert_matcher import match_key_alert_rule from .key_alert_matcher import match_key_alert_rule
from .models import UmeInventoryNE, UmeKeyAlertForwardLog from .models import UmeInventoryNE, UmeKeyAlertForwardLog
from .oclaw_alarm_forwarder import enqueue_alarm_forward, is_forwarder_enabled from .oclaw_alarm_forwarder import enqueue_alarm_forward, is_forwarder_operational
from .ume_sync_service import ( from .ume_sync_service import (
_derive_ne_id_from_alarm, _derive_ne_id_from_alarm,
_pick, _pick,
@ -69,7 +69,7 @@ def maybe_forward_key_alert(
alarm_key: str, alarm_key: str,
action: str, action: str,
) -> bool: ) -> bool:
if not is_forwarder_enabled(): if not is_forwarder_operational():
return False return False
rule = match_key_alert_rule(db, norm=norm, action=action) rule = match_key_alert_rule(db, norm=norm, action=action)
if rule is None: if rule is None:

View file

@ -75,7 +75,29 @@ from .key_alert_matcher import (
rule_storage_key, rule_storage_key,
serialize_rule_ne_types, serialize_rule_ne_types,
) )
from .oclaw_alarm_forwarder import forwarder_status, shutdown_oclaw_alarm_forwarder, start_oclaw_alarm_forwarder from .runtime_task_messages import (
RT_ALARMS_SYNC_IN_PROGRESS_SKIP,
RT_OCLAW_FWD_DISABLED,
RT_PULLING_ALARMS_CURRENT,
RT_PULLING_INVENTORY,
RT_RESUMED,
RT_RESUMED_OCLAW_WSS_RECONNECT,
RT_RESUMED_SYNC_SOON,
RT_RESUMED_WSS_RECONNECT,
RT_STARTUP_ALARM_SYNC_BEFORE_WS,
RT_STARTUP_GATE_WAITING,
RT_KEEPALIVE_FAILED,
RT_UME_WS_DISABLED_NO_BASE_URL,
RT_WSS_ACTIVE_SKIP_REST,
)
from .oclaw_alarm_forwarder import (
forwarder_status,
is_forwarder_enabled,
request_forwarder_reconnect,
configure_oclaw_alarm_forwarder,
shutdown_oclaw_alarm_forwarder,
start_oclaw_alarm_forwarder,
)
from .ume_token_store import ( from .ume_token_store import (
clear_shared_token, clear_shared_token,
load_shared_token, load_shared_token,
@ -117,6 +139,7 @@ _UME_RUNTIME_TASKS: dict[str, dict[str, Any]] = {
"token_keepalive": {"task": "token_keepalive", "status": "init", "last_run_at": None, "last_error": ""}, "token_keepalive": {"task": "token_keepalive", "status": "init", "last_run_at": None, "last_error": ""},
"alarms_current_auto_sync": {"task": "alarms_current_auto_sync", "status": "init", "last_run_at": None, "last_error": ""}, "alarms_current_auto_sync": {"task": "alarms_current_auto_sync", "status": "init", "last_run_at": None, "last_error": ""},
"alarms_current_ws_consumer": {"task": "alarms_current_ws_consumer", "status": "init", "last_run_at": None, "last_error": ""}, "alarms_current_ws_consumer": {"task": "alarms_current_ws_consumer", "status": "init", "last_run_at": None, "last_error": ""},
"oclaw_alarm_forwarder": {"task": "oclaw_alarm_forwarder", "status": "init", "last_run_at": None, "last_error": ""},
"inventory_auto_sync": {"task": "inventory_auto_sync", "status": "init", "last_run_at": None, "last_error": ""}, "inventory_auto_sync": {"task": "inventory_auto_sync", "status": "init", "last_run_at": None, "last_error": ""},
} }
_UME_WS_STOP_EVENT: threading.Event | None = None _UME_WS_STOP_EVENT: threading.Event | None = None
@ -226,6 +249,10 @@ def _runtime_task_interval_fields(task_id: str) -> tuple[int | None, str]:
if not bool(getattr(settings, "ume_alarm_ws_enabled", True)): if not bool(getattr(settings, "ume_alarm_ws_enabled", True)):
return None, "disabled" return None, "disabled"
return None, "realtime" return None, "realtime"
if task_id == "oclaw_alarm_forwarder":
if not is_forwarder_enabled():
return None, "disabled"
return None, "realtime"
if task_id == "inventory_auto_sync": if task_id == "inventory_auto_sync":
if not bool(getattr(settings, "ume_sync_inventory_auto_enabled", True)): if not bool(getattr(settings, "ume_sync_inventory_auto_enabled", True)):
return None, "disabled" return None, "disabled"
@ -343,7 +370,7 @@ def _run_startup_alarm_sync_before_ws() -> None:
"alarms_current_auto_sync", "alarms_current_auto_sync",
status="running", status="running",
last_run_at=datetime.now(timezone.utc), last_run_at=datetime.now(timezone.utc),
last_error="启动:正在同步当前告警(完成后连接 WSS)…", last_error=RT_STARTUP_ALARM_SYNC_BEFORE_WS,
) )
db = SessionLocal() db = SessionLocal()
try: try:
@ -930,7 +957,7 @@ def on_startup() -> None:
client.renew_token() client.renew_token()
_set_runtime_task("token_keepalive", status="running", last_run_at=datetime.now(timezone.utc), last_error="") _set_runtime_task("token_keepalive", status="running", last_run_at=datetime.now(timezone.utc), last_error="")
except Exception: except Exception:
_set_runtime_task("token_keepalive", status="error", last_run_at=datetime.now(timezone.utc), last_error="keepalive_failed") _set_runtime_task("token_keepalive", status="error", last_run_at=datetime.now(timezone.utc), last_error=RT_KEEPALIVE_FAILED)
time.sleep(interval_keepalive_s) time.sleep(interval_keepalive_s)
t = threading.Thread(target=_keepalive_loop, name="ume-token-keepalive", daemon=True) t = threading.Thread(target=_keepalive_loop, name="ume-token-keepalive", daemon=True)
@ -983,7 +1010,7 @@ def on_startup() -> None:
_refresh_runtime_task_idle( _refresh_runtime_task_idle(
"alarms_current_auto_sync", "alarms_current_auto_sync",
"alarms_current", "alarms_current",
last_error="启动 REST 全量同步未完成,定时 REST 与 WSS 均待命", last_error=RT_STARTUP_GATE_WAITING,
) )
time.sleep(10) time.sleep(10)
continue continue
@ -994,7 +1021,7 @@ def on_startup() -> None:
_refresh_runtime_task_idle( _refresh_runtime_task_idle(
"alarms_current_auto_sync", "alarms_current_auto_sync",
"alarms_current", "alarms_current",
last_error="WSS 实时接收中,已跳过 REST 同步", last_error=RT_WSS_ACTIVE_SKIP_REST,
) )
time.sleep(max(30, min(alarms_interval_s, 300))) time.sleep(max(30, min(alarms_interval_s, 300)))
continue continue
@ -1012,7 +1039,7 @@ def on_startup() -> None:
"alarms_current_auto_sync", "alarms_current_auto_sync",
status="running", status="running",
last_run_at=datetime.now(timezone.utc), last_run_at=datetime.now(timezone.utc),
last_error="正在拉取 UME 当前告警…", last_error=RT_PULLING_ALARMS_CURRENT,
) )
db = SessionLocal() db = SessionLocal()
try: try:
@ -1032,7 +1059,7 @@ def on_startup() -> None:
_refresh_runtime_task_idle( _refresh_runtime_task_idle(
"alarms_current_auto_sync", "alarms_current_auto_sync",
"alarms_current", "alarms_current",
last_error="另一条当前告警 REST 同步进行中,已跳过", last_error=RT_ALARMS_SYNC_IN_PROGRESS_SKIP,
) )
time.sleep(30) time.sleep(30)
else: else:
@ -1091,7 +1118,7 @@ def on_startup() -> None:
"inventory_auto_sync", "inventory_auto_sync",
status="running", status="running",
last_run_at=datetime.now(timezone.utc), last_run_at=datetime.now(timezone.utc),
last_error="正在拉取 UME 网元清单…", last_error=RT_PULLING_INVENTORY,
) )
db = SessionLocal() db = SessionLocal()
try: try:
@ -1151,7 +1178,7 @@ def on_startup() -> None:
) )
_schedule_log.info("started thread %s alive=%s", t_ws.name, t_ws.is_alive()) _schedule_log.info("started thread %s alive=%s", t_ws.name, t_ws.is_alive())
else: else:
_set_runtime_task("alarms_current_ws_consumer", status="paused", last_error="未启用或未配置 UME_BASE_URL") _set_runtime_task("alarms_current_ws_consumer", status="paused", last_error=RT_UME_WS_DISABLED_NO_BASE_URL)
except Exception as exc: except Exception as exc:
_schedule_log.exception("startup: alarms_current_ws_consumer thread init failed: %s", exc) _schedule_log.exception("startup: alarms_current_ws_consumer thread init failed: %s", exc)
_set_runtime_task( _set_runtime_task(
@ -1161,11 +1188,47 @@ def on_startup() -> None:
last_error=f"startup_thread_init_failed: {str(exc)[:180]}", last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
) )
try: try:
def _fwd_on_status(msg: str) -> None:
paused = _runtime_is_paused("oclaw_alarm_forwarder")
fwd = forwarder_status()
if paused:
status = "paused"
elif not bool(fwd.get("enabled")):
status = "paused"
elif bool(fwd.get("connected")):
status = "running"
else:
status = "running"
_set_runtime_task(
"oclaw_alarm_forwarder",
status=status,
last_run_at=datetime.now(timezone.utc),
last_error=str(msg or "")[:240],
)
configure_oclaw_alarm_forwarder(
is_paused=lambda: _runtime_is_paused("oclaw_alarm_forwarder"),
on_status=_fwd_on_status,
)
if is_forwarder_enabled():
_set_runtime_task("oclaw_alarm_forwarder", status="running", last_error="")
else:
_set_runtime_task(
"oclaw_alarm_forwarder",
status="paused",
last_error=RT_OCLAW_FWD_DISABLED,
)
t_fwd = start_oclaw_alarm_forwarder() t_fwd = start_oclaw_alarm_forwarder()
if t_fwd is not None: if t_fwd is not None:
_schedule_log.info("started thread %s alive=%s", t_fwd.name, t_fwd.is_alive()) _schedule_log.info("started thread %s alive=%s", t_fwd.name, t_fwd.is_alive())
except Exception as exc: except Exception as exc:
_schedule_log.exception("startup: oclaw_alarm_forwarder thread init failed: %s", exc) _schedule_log.exception("startup: oclaw_alarm_forwarder thread init failed: %s", exc)
_set_runtime_task(
"oclaw_alarm_forwarder",
status="error",
last_run_at=datetime.now(timezone.utc),
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
)
@app.on_event("shutdown") @app.on_event("shutdown")
@ -1695,6 +1758,8 @@ def ume_runtime_task_pause(task: str) -> dict[str, Any]:
_clear_force_resume_hints(tid) _clear_force_resume_hints(tid)
if tid == "alarms_current_ws_consumer": if tid == "alarms_current_ws_consumer":
request_ws_reconnect() request_ws_reconnect()
if tid == "oclaw_alarm_forwarder":
request_forwarder_reconnect()
_set_runtime_task(tid, status="paused", last_error="") _set_runtime_task(tid, status="paused", last_error="")
return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()} return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()}
@ -1707,12 +1772,15 @@ def ume_runtime_task_resume(task: str) -> dict[str, Any]:
_runtime_resume_task(tid) _runtime_resume_task(tid)
if tid in ("alarms_current_auto_sync", "inventory_auto_sync"): if tid in ("alarms_current_auto_sync", "inventory_auto_sync"):
_request_force_sync_after_resume(tid) _request_force_sync_after_resume(tid)
resume_hint = "已恢复:将跳过本轮周期等待并尽快同步" resume_hint = RT_RESUMED_SYNC_SOON
elif tid == "alarms_current_ws_consumer": elif tid == "alarms_current_ws_consumer":
request_ws_reconnect() request_ws_reconnect()
resume_hint = "已恢复:将尽快重连 WSS" resume_hint = RT_RESUMED_WSS_RECONNECT
elif tid == "oclaw_alarm_forwarder":
request_forwarder_reconnect()
resume_hint = RT_RESUMED_OCLAW_WSS_RECONNECT
else: else:
resume_hint = "已恢复" resume_hint = RT_RESUMED
_set_runtime_task(tid, status="running", last_error=resume_hint) _set_runtime_task(tid, status="running", last_error=resume_hint)
return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()} return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()}
@ -2287,6 +2355,16 @@ def integrations_status(db: Session = Depends(get_db)) -> dict:
"error": "NETX_OCLAW_ALARM_WS_ENABLED=false or missing token/url", "error": "NETX_OCLAW_ALARM_WS_ENABLED=false or missing token/url",
"forwarder": fwd, "forwarder": fwd,
} }
elif bool(fwd.get("paused")):
oclaw_status = {
"status": "unknown",
"mode": "ws",
"enabled": True,
"connected": False,
"error_kind": "paused",
"error": "oclaw_alarm_forwarder runtime task paused",
"forwarder": fwd,
}
elif bool(fwd.get("connected")): elif bool(fwd.get("connected")):
oclaw_status = { oclaw_status = {
"status": "up", "status": "up",

View file

@ -5,6 +5,7 @@ import logging
import queue import queue
import threading import threading
import time import time
from collections.abc import Callable
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Any from typing import Any
@ -20,6 +21,8 @@ _THREAD: threading.Thread | None = None
_CONN_LOCK = threading.Lock() _CONN_LOCK = threading.Lock()
_WS: Any | None = None _WS: Any | None = None
_CONNECTED = threading.Event() _CONNECTED = threading.Event()
_IS_PAUSED: Callable[[], bool] | None = None
_ON_STATUS: Callable[[str], None] | None = None
_STATS_LOCK = threading.Lock() _STATS_LOCK = threading.Lock()
_STATS: dict[str, int] = { _STATS: dict[str, int] = {
"published_ok": 0, "published_ok": 0,
@ -46,8 +49,50 @@ def is_forwarder_enabled() -> bool:
return bool(_bridge_url()) and bool(_bridge_token()) return bool(_bridge_url()) and bool(_bridge_token())
def configure_oclaw_alarm_forwarder(
*,
is_paused: Callable[[], bool] | None = None,
on_status: Callable[[str], None] | None = None,
) -> None:
global _IS_PAUSED, _ON_STATUS
_IS_PAUSED = is_paused
_ON_STATUS = on_status
def _forwarder_paused() -> bool:
if _IS_PAUSED is None:
return False
try:
return bool(_IS_PAUSED())
except Exception:
return False
def is_forwarder_operational() -> bool:
return is_forwarder_enabled() and not _forwarder_paused()
def _notify_status(msg: str) -> None:
if _ON_STATUS is None:
return
try:
_ON_STATUS(str(msg or "")[:240])
except Exception:
pass
def request_forwarder_reconnect() -> None:
with _CONN_LOCK:
ws = _WS
if ws is not None:
try:
ws.close()
except Exception:
pass
def enqueue_alarm_forward(payload: dict[str, Any]) -> bool: def enqueue_alarm_forward(payload: dict[str, Any]) -> bool:
if not is_forwarder_enabled(): if not is_forwarder_operational():
return False return False
try: try:
_OUTBOUND_Q.put_nowait(dict(payload)) _OUTBOUND_Q.put_nowait(dict(payload))
@ -112,12 +157,18 @@ def _run_loop() -> None:
global _WS global _WS
backoff_s = 2.0 backoff_s = 2.0
while not _STOP_EVENT.is_set(): while not _STOP_EVENT.is_set():
if not is_forwarder_enabled(): if not is_forwarder_enabled() or _forwarder_paused():
_CONNECTED.clear()
if _forwarder_paused():
_notify_status("fwd:paused")
else:
_notify_status("fwd:disabled")
time.sleep(2.0) time.sleep(2.0)
continue continue
url = _bridge_url() url = _bridge_url()
ws = None ws = None
try: try:
_notify_status("fwd:connecting")
ws = websocket.create_connection(url, timeout=20) ws = websocket.create_connection(url, timeout=20)
ws.settimeout(30) ws.settimeout(30)
if not _send_auth(ws): if not _send_auth(ws):
@ -127,7 +178,14 @@ def _run_loop() -> None:
_CONNECTED.set() _CONNECTED.set()
backoff_s = 2.0 backoff_s = 2.0
_log.info("oclaw netx-bridge connected url=%s", url[:120]) _log.info("oclaw netx-bridge connected url=%s", url[:120])
_notify_status("fwd:connected")
while not _STOP_EVENT.is_set(): while not _STOP_EVENT.is_set():
if _forwarder_paused():
_notify_status("fwd:paused")
raise RuntimeError("forwarder paused")
if not is_forwarder_enabled():
_notify_status("fwd:disabled")
raise RuntimeError("forwarder disabled")
try: try:
payload = _OUTBOUND_Q.get(timeout=1.0) payload = _OUTBOUND_Q.get(timeout=1.0)
except queue.Empty: except queue.Empty:
@ -174,7 +232,9 @@ def _run_loop() -> None:
_OUTBOUND_Q.task_done() _OUTBOUND_Q.task_done()
except Exception as exc: except Exception as exc:
_CONNECTED.clear() _CONNECTED.clear()
_log.warning("oclaw netx-bridge disconnected: %s", str(exc)[:200]) err = str(exc)[:200]
_log.warning("oclaw netx-bridge disconnected: %s", err)
_notify_status(err)
time.sleep(backoff_s) time.sleep(backoff_s)
backoff_s = min(backoff_s * 1.5, 60.0) backoff_s = min(backoff_s * 1.5, 60.0)
finally: finally:
@ -192,9 +252,6 @@ def start_oclaw_alarm_forwarder() -> threading.Thread | None:
global _THREAD global _THREAD
if _THREAD is not None and _THREAD.is_alive(): if _THREAD is not None and _THREAD.is_alive():
return _THREAD return _THREAD
if not is_forwarder_enabled():
_log.info("oclaw alarm forwarder disabled")
return None
_STOP_EVENT.clear() _STOP_EVENT.clear()
_THREAD = threading.Thread(target=_run_loop, name="oclaw-alarm-forwarder", daemon=True) _THREAD = threading.Thread(target=_run_loop, name="oclaw-alarm-forwarder", daemon=True)
_THREAD.start() _THREAD.start()
@ -208,8 +265,12 @@ def shutdown_oclaw_alarm_forwarder() -> None:
def forwarder_status() -> dict[str, Any]: def forwarder_status() -> dict[str, Any]:
with _STATS_LOCK: with _STATS_LOCK:
stats = dict(_STATS) stats = dict(_STATS)
paused = _forwarder_paused()
enabled = is_forwarder_enabled()
return { return {
"enabled": is_forwarder_enabled(), "enabled": enabled,
"operational": bool(enabled and not paused),
"paused": paused,
"connected": _CONNECTED.is_set(), "connected": _CONNECTED.is_set(),
"queue_size": int(_OUTBOUND_Q.qsize()), "queue_size": int(_OUTBOUND_Q.qsize()),
"url": _bridge_url(), "url": _bridge_url(),

View file

@ -0,0 +1,23 @@
"""Stable runtime-task status codes for frontend i18n (rt:* / ws:* / fwd:*)."""
RT_STARTUP_ALARM_SYNC_BEFORE_WS = "rt:startup_alarm_sync_before_ws"
RT_STARTUP_GATE_WAITING = "rt:startup_gate_waiting"
RT_WSS_ACTIVE_SKIP_REST = "rt:wss_active_skip_rest"
RT_PULLING_ALARMS_CURRENT = "rt:pulling_alarms_current"
RT_ALARMS_SYNC_IN_PROGRESS_SKIP = "rt:alarms_sync_in_progress_skip"
RT_PULLING_INVENTORY = "rt:pulling_inventory"
RT_UME_WS_DISABLED_NO_BASE_URL = "rt:ume_ws_disabled_no_base_url"
RT_OCLAW_FWD_DISABLED = "rt:oclaw_fwd_disabled"
RT_RESUMED_SYNC_SOON = "rt:resumed_sync_soon"
RT_RESUMED_WSS_RECONNECT = "rt:resumed_wss_reconnect"
RT_RESUMED_OCLAW_WSS_RECONNECT = "rt:resumed_oclaw_wss_reconnect"
RT_RESUMED = "rt:resumed"
RT_KEEPALIVE_FAILED = "rt:keepalive_failed"
def ws_state_code(state: str) -> str:
return f"ws:{str(state or '').strip()}"
def fwd_state_code(state: str) -> str:
return f"fwd:{str(state or '').strip()}"

View file

@ -249,7 +249,7 @@ def get_ws_connection_status() -> dict[str, Any]:
detail = str(_ws_connection_detail or "") detail = str(_ws_connection_detail or "")
return { return {
"state": state, "state": state,
"label": _WS_CONNECTION_LABELS.get(state, state), "label": f"ws:{state}",
"detail": detail, "detail": detail,
} }
@ -284,7 +284,7 @@ def _notify_ws_connection(
dedup=False, dedup=False,
) )
if on_status is not None: if on_status is not None:
on_status(label) on_status(f"ws:{state}")
def _parse_ws_message(raw: str) -> dict[str, Any] | None: def _parse_ws_message(raw: str) -> dict[str, Any] | None:
@ -804,7 +804,7 @@ def start_ume_alarm_ws_consumer(
if not str(client.base_url or "").strip(): if not str(client.base_url or "").strip():
append_ws_log("consumer disabled: no UME base URL", level="warning") append_ws_log("consumer disabled: no UME base URL", level="warning")
if on_status is not None: if on_status is not None:
on_status("disabled:no_base_url") on_status("ws:disabled_no_base_url")
return return
append_ws_log("ws consumer thread started") append_ws_log("ws consumer thread started")
global _active_client global _active_client

View file

@ -342,6 +342,7 @@ const en = {
wsConnected: "connected", wsConnected: "connected",
wsDisconnected: "disconnected", wsDisconnected: "disconnected",
wsDisabled: "disabled", wsDisabled: "disabled",
wsPaused: "paused",
publishedOk: "published ok", publishedOk: "published ok",
publishedFail: "published fail", publishedFail: "published fail",
queue: "queue", queue: "queue",
@ -409,6 +410,48 @@ const en = {
intervalTitle: "Configured sleep between loop iterations (clamped at process start)", intervalTitle: "Configured sleep between loop iterations (clamped at process start)",
lastRunTitle: "Last finished sync for scheduled tasks; refreshed at each loop tick", lastRunTitle: "Last finished sync for scheduled tasks; refreshed at each loop tick",
deletedTitle: "Reconcile deletes: inventory = NE rows; alarms_current = current alarm rows", deletedTitle: "Reconcile deletes: inventory = NE rows; alarms_current = current alarm rows",
lastErrorCol: "last_error",
runtimeError: {
startup_alarm_sync_before_ws: "Starting up: syncing current alarms (WSS connects after completion)…",
startup_gate_waiting: "Startup REST snapshot pending; scheduled REST and WSS on standby",
wss_active_skip_rest: "WSS receiving live alarms; REST sync skipped",
pulling_alarms_current: "Pulling UME current alarms…",
alarms_sync_in_progress_skip: "Another current-alarm REST sync in progress; skipped",
pulling_inventory: "Pulling UME inventory…",
ume_ws_disabled_no_base_url: "Disabled or UME_BASE_URL not configured",
oclaw_fwd_disabled: "Disabled or NETX_OCLAW_ALARM_WS / token / url not configured",
resumed_sync_soon: "Resumed: skipping debounce wait, sync soon",
resumed_wss_reconnect: "Resumed: reconnecting WSS soon",
resumed_oclaw_wss_reconnect: "Resumed: reconnecting OClaw WSS soon",
resumed: "Resumed",
keepalive_failed: "Token keepalive failed",
},
wsState: {
init: "Initializing",
connected: "Connected",
connecting: "Connecting",
disconnected: "Disconnected",
no_subscription: "No subscription",
waiting_token: "Waiting for token",
paused: "Paused",
error: "Connection error",
reconnecting: "Reconnecting",
subscription_lost: "UME subscription lost",
disabled_no_base_url: "Disabled or UME_BASE_URL not configured",
},
fwdState: {
paused: "Paused",
disabled: "Disabled",
connecting: "Connecting",
connected: "Connected",
},
runtimeTask: {
token_keepalive: "UME token keepalive",
alarms_current_auto_sync: "Current alarms auto sync",
alarms_current_ws_consumer: "UME alarm WSS",
oclaw_alarm_forwarder: "OClaw key-alert WSS",
inventory_auto_sync: "Inventory auto sync",
},
}, },
syncStatus: { syncStatus: {
title: "Sync history", title: "Sync history",

View file

@ -340,6 +340,7 @@ const zh = {
wsConnected: "已连接", wsConnected: "已连接",
wsDisconnected: "未连接", wsDisconnected: "未连接",
wsDisabled: "未启用", wsDisabled: "未启用",
wsPaused: "已暂停",
publishedOk: "发布成功", publishedOk: "发布成功",
publishedFail: "发布失败", publishedFail: "发布失败",
queue: "待发队列", queue: "待发队列",
@ -407,6 +408,48 @@ const zh = {
intervalTitle: "Configured sleep between loop iterations (clamped at process start)", intervalTitle: "Configured sleep between loop iterations (clamped at process start)",
lastRunTitle: "Last finished sync for scheduled tasks; refreshed at each loop tick", lastRunTitle: "Last finished sync for scheduled tasks; refreshed at each loop tick",
deletedTitle: "全量对账删除条数:inventory 为网元行,alarms_current 为当前告警行,其余域多为 0", deletedTitle: "全量对账删除条数:inventory 为网元行,alarms_current 为当前告警行,其余域多为 0",
lastErrorCol: "last_error",
runtimeError: {
startup_alarm_sync_before_ws: "启动中:正在同步当前告警(完成后连接 WSS)…",
startup_gate_waiting: "启动 REST 全量同步未完成,定时 REST 与 WSS 均待命",
wss_active_skip_rest: "WSS 实时接收中,已跳过 REST 同步",
pulling_alarms_current: "正在拉取 UME 当前告警…",
alarms_sync_in_progress_skip: "另一条当前告警 REST 同步进行中,已跳过",
pulling_inventory: "正在拉取 UME 网元清单…",
ume_ws_disabled_no_base_url: "未启用或未配置 UME_BASE_URL",
oclaw_fwd_disabled: "未启用或未配置 NETX_OCLAW_ALARM_WS / token / url",
resumed_sync_soon: "已恢复:将跳过本轮周期等待并尽快同步",
resumed_wss_reconnect: "已恢复:将尽快重连 WSS",
resumed_oclaw_wss_reconnect: "已恢复:将尽快重连 OClaw WSS",
resumed: "已恢复",
keepalive_failed: "Token 保活失败",
},
wsState: {
init: "初始化",
connected: "已连接",
connecting: "连接中",
disconnected: "已断开",
no_subscription: "无订阅",
waiting_token: "等待 token",
paused: "已暂停",
error: "连接异常",
reconnecting: "重连等待",
subscription_lost: "UME 订阅已丢失",
disabled_no_base_url: "未启用或未配置 UME_BASE_URL",
},
fwdState: {
paused: "已暂停",
disabled: "未启用",
connecting: "连接中",
connected: "已连接",
},
runtimeTask: {
token_keepalive: "UME Token 保活",
alarms_current_auto_sync: "当前告警定时同步",
alarms_current_ws_consumer: "UME 告警 WSS",
oclaw_alarm_forwarder: "OClaw 关键告警 WSS",
inventory_auto_sync: "Inventory 定时同步",
},
}, },
syncStatus: { syncStatus: {
title: "同步状态", title: "同步状态",

View file

@ -27,6 +27,7 @@ import { useI18n } from "../i18n";
import { useToast } from "../hooks/useToast"; import { useToast } from "../hooks/useToast";
import type { UmeAlarmSubscriptionStatus, UmeKeyAlertRuleItem } from "../types"; import type { UmeAlarmSubscriptionStatus, UmeKeyAlertRuleItem } from "../types";
import { pageCount, runtimeIntervalLabel } from "../utils/display"; import { pageCount, runtimeIntervalLabel } from "../utils/display";
import { runtimeLastError, wsConnectionLabel } from "../utils/runtimeMessages";
import { formatSystemTime } from "../utils/time"; import { formatSystemTime } from "../utils/time";
export function UmePage() { export function UmePage() {
@ -278,8 +279,12 @@ export function UmePage() {
const wsState = String(wsConn?.state || ""); const wsState = String(wsConn?.state || "");
const wsPaused = Boolean(wsConsumer?.paused); const wsPaused = Boolean(wsConsumer?.paused);
const wsLabel = const wsLabel =
wsConn?.label || (wsPaused
(wsPaused ? t("ume.subscription.wssPaused") : wsConsumer?.last_error || wsConsumer?.status || t("common.empty")); ? t("ume.subscription.wssPaused")
: wsConnectionLabel(wsState, wsConn?.label, t)) ||
runtimeLastError(wsConsumer?.last_error, t) ||
wsConsumer?.status ||
t("common.empty");
const subscriptionActive = Boolean(alarmSub.active); const subscriptionActive = Boolean(alarmSub.active);
const serverSubLost = Boolean( const serverSubLost = Boolean(
subscriptionStatusQuery.data?.server_subscription_lost ?? alarmSub.server_subscription_lost, subscriptionStatusQuery.data?.server_subscription_lost ?? alarmSub.server_subscription_lost,
@ -303,7 +308,8 @@ export function UmePage() {
? "warn" ? "warn"
: wsState === "no_subscription" || wsState === "disconnected" || wsState === "error" || wsState === "init" : wsState === "no_subscription" || wsState === "disconnected" || wsState === "error" || wsState === "init"
? "down" ? "down"
: String(wsConsumer?.last_error || "").includes("connected") : String(wsConsumer?.last_error || "").includes("connected") ||
String(wsConsumer?.last_error || "").includes("ws:connected")
? "up" ? "up"
: "unknown"; : "unknown";
const wsLogs = [...(subscriptionStatusQuery.data?.ws_logs ?? alarmSub.ws_logs ?? [])].reverse(); const wsLogs = [...(subscriptionStatusQuery.data?.ws_logs ?? alarmSub.ws_logs ?? [])].reverse();
@ -420,9 +426,17 @@ export function UmePage() {
const oclawWsPill = const oclawWsPill =
!keyAlertForwarder?.enabled !keyAlertForwarder?.enabled
? "unknown" ? "unknown"
: keyAlertForwarder.connected : keyAlertForwarder.paused
? "up" ? "warn"
: "down"; : keyAlertForwarder.connected
? "up"
: "down";
const runtimeTaskLabel = (task: string) => {
const key = `ume.tasks.runtimeTask.${task}`;
const label = t(key);
return label === key ? task : label;
};
const perPage = (n: number) => t("common.perPage", { n: String(n) }); const perPage = (n: number) => t("common.perPage", { n: String(n) });
@ -524,7 +538,7 @@ export function UmePage() {
{subscriptionActive || wsConsumer ? ( {subscriptionActive || wsConsumer ? (
<span <span
className={`conn-pill conn-pill--${wsPillLevel}`} className={`conn-pill conn-pill--${wsPillLevel}`}
title={wsConn?.detail || wsConsumer?.last_error || alarmSub.wss_uri || ""} title={wsConn?.detail || runtimeLastError(wsConsumer?.last_error, t) || alarmSub.wss_uri || ""}
> >
WSS: {wsLabel} WSS: {wsLabel}
</span> </span>
@ -665,9 +679,11 @@ export function UmePage() {
{t("ume.keyAlert.ws")}:{" "} {t("ume.keyAlert.ws")}:{" "}
{!keyAlertForwarder?.enabled {!keyAlertForwarder?.enabled
? t("ume.keyAlert.wsDisabled") ? t("ume.keyAlert.wsDisabled")
: keyAlertForwarder.connected : keyAlertForwarder.paused
? t("ume.keyAlert.wsConnected") ? t("ume.keyAlert.wsPaused")
: t("ume.keyAlert.wsDisconnected")} : keyAlertForwarder.connected
? t("ume.keyAlert.wsConnected")
: t("ume.keyAlert.wsDisconnected")}
</span> </span>
{keyAlertForwarder?.enabled ? ( {keyAlertForwarder?.enabled ? (
<> <>
@ -1078,20 +1094,20 @@ export function UmePage() {
<th title={t("ume.tasks.intervalTitle")}>interval</th> <th title={t("ume.tasks.intervalTitle")}>interval</th>
<th>status</th> <th>status</th>
<th title={t("ume.tasks.lastRunTitle")}>last_run_at</th> <th title={t("ume.tasks.lastRunTitle")}>last_run_at</th>
<th>last_error</th> <th>{t("ume.tasks.lastErrorCol")}</th>
<th>actions</th> <th>actions</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{runtimeTasks.map((x) => ( {runtimeTasks.map((x) => (
<tr key={`runtime-${x.task}`}> <tr key={`runtime-${x.task}`}>
<td>{x.task}</td> <td title={x.task}>{runtimeTaskLabel(x.task)}</td>
<td title={typeof x.interval_s === "number" ? `${x.interval_s}s` : undefined}> <td title={typeof x.interval_s === "number" ? `${x.interval_s}s` : undefined}>
{runtimeIntervalLabel(x.interval_label)} {runtimeIntervalLabel(x.interval_label)}
</td> </td>
<td>{x.status}</td> <td>{x.status}</td>
<td>{x.last_run_at ? formatSystemTime(x.last_run_at) : t("common.empty")}</td> <td>{x.last_run_at ? formatSystemTime(x.last_run_at) : t("common.empty")}</td>
<td>{x.last_error || t("common.empty")}</td> <td>{runtimeLastError(x.last_error, t) || t("common.empty")}</td>
<td> <td>
{Boolean(x.paused) ? ( {Boolean(x.paused) ? (
<button <button

View file

@ -39,6 +39,8 @@ export type UmeInventoryNeTypeItem = {
export type UmeKeyAlertForwarderStatus = { export type UmeKeyAlertForwarderStatus = {
enabled: boolean; enabled: boolean;
operational?: boolean;
paused?: boolean;
connected: boolean; connected: boolean;
queue_size: number; queue_size: number;
url: string; url: string;

View file

@ -0,0 +1,77 @@
/** Map backend runtime status codes (rt:* / ws:* / fwd:*) and legacy Chinese to i18n keys. */
type TFunc = (key: string) => string;
const LEGACY_ZH_RUNTIME_ERROR: Record<string, string> = {
"启动:正在同步当前告警(完成后连接 WSS)…": "ume.tasks.runtimeError.startup_alarm_sync_before_ws",
"启动 REST 全量同步未完成,定时 REST 与 WSS 均待命": "ume.tasks.runtimeError.startup_gate_waiting",
"WSS 实时接收中,已跳过 REST 同步": "ume.tasks.runtimeError.wss_active_skip_rest",
"正在拉取 UME 当前告警…": "ume.tasks.runtimeError.pulling_alarms_current",
"另一条当前告警 REST 同步进行中,已跳过": "ume.tasks.runtimeError.alarms_sync_in_progress_skip",
"正在拉取 UME 网元清单…": "ume.tasks.runtimeError.pulling_inventory",
"未启用或未配置 UME_BASE_URL": "ume.tasks.runtimeError.ume_ws_disabled_no_base_url",
"未启用或未配置 NETX_OCLAW_ALARM_WS / token / url": "ume.tasks.runtimeError.oclaw_fwd_disabled",
"已恢复:将跳过本轮周期等待并尽快同步": "ume.tasks.runtimeError.resumed_sync_soon",
"已恢复:将尽快重连 WSS": "ume.tasks.runtimeError.resumed_wss_reconnect",
"已恢复:将尽快重连 OClaw WSS": "ume.tasks.runtimeError.resumed_oclaw_wss_reconnect",
已恢复: "ume.tasks.runtimeError.resumed",
初始化: "ume.tasks.wsState.init",
已连接: "ume.tasks.wsState.connected",
连接中: "ume.tasks.wsState.connecting",
已断开: "ume.tasks.wsState.disconnected",
无订阅: "ume.tasks.wsState.no_subscription",
"等待 token": "ume.tasks.wsState.waiting_token",
已暂停: "ume.tasks.wsState.paused",
连接异常: "ume.tasks.wsState.error",
重连等待: "ume.tasks.wsState.reconnecting",
UME订阅已丢失: "ume.tasks.wsState.subscription_lost",
};
function translatePrefixed(code: string, prefix: string, i18nSection: string, t: TFunc): string | null {
if (!code.startsWith(prefix)) return null;
const slug = code.slice(prefix.length);
if (!slug) return null;
const key = `${i18nSection}.${slug}`;
const tr = t(key);
return tr !== key ? tr : null;
}
export function runtimeLastError(raw: string | null | undefined, t: TFunc): string {
const s = String(raw ?? "").trim();
if (!s) return "";
const rt = translatePrefixed(s, "rt:", "ume.tasks.runtimeError", t);
if (rt) return rt;
const ws = translatePrefixed(s, "ws:", "ume.tasks.wsState", t);
if (ws) return ws;
const fwd = translatePrefixed(s, "fwd:", "ume.tasks.fwdState", t);
if (fwd) return fwd;
if (s === "keepalive_failed") {
const tr = t("ume.tasks.runtimeError.keepalive_failed");
if (tr !== "ume.tasks.runtimeError.keepalive_failed") return tr;
}
const legacyKey = LEGACY_ZH_RUNTIME_ERROR[s];
if (legacyKey) {
const tr = t(legacyKey);
if (tr !== legacyKey) return tr;
}
return s;
}
export function wsConnectionLabel(
state: string | null | undefined,
label: string | null | undefined,
t: TFunc,
): string {
const fromLabel = runtimeLastError(label, t);
if (fromLabel && fromLabel !== String(label ?? "").trim()) return fromLabel;
const st = String(state ?? "").trim();
if (st) {
const tr = runtimeLastError(`ws:${st}`, t);
if (tr) return tr;
}
return fromLabel || st || "";
}