mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 03:10:46 +08:00
Remove legacy OClaw alarm WSS and fix key-alert rules table layout.
Key alerts now deliver only via the DSH hub; Modal Body no longer flex-shrinks the rules rows to empty. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
f937efa0f9
commit
0a4561f385
25 changed files with 36 additions and 729 deletions
|
|
@ -13,9 +13,6 @@ NETX_PARSER_CONFIG=netx_api/config/parsers/zte_alarm_monitor_v1.yaml
|
||||||
NETX_OCLAW_ANALYZE_URL=http://127.0.0.1:8787/admin/api/ops-ai/analyze-sync
|
NETX_OCLAW_ANALYZE_URL=http://127.0.0.1:8787/admin/api/ops-ai/analyze-sync
|
||||||
NETX_OCLAW_ANALYZE_TOKEN=
|
NETX_OCLAW_ANALYZE_TOKEN=
|
||||||
NETX_OCLAW_HEALTH_URL=http://127.0.0.1:8787/admin/api/ops-ai/health
|
NETX_OCLAW_HEALTH_URL=http://127.0.0.1:8787/admin/api/ops-ai/health
|
||||||
NETX_OCLAW_ALARM_WS_ENABLED=false
|
|
||||||
NETX_OCLAW_ALARM_WS_URL=ws://127.0.0.1:8787/ws/netx-bridge
|
|
||||||
# 告警 WSS 鉴权与 NETX_OCLAW_ANALYZE_TOKEN 共用(OClaw 侧 OCLAW_OPS_AI_SHARED_TOKEN)
|
|
||||||
NETX_UME_BASE_URL=https://10.227.157.143:18014
|
NETX_UME_BASE_URL=https://10.227.157.143:18014
|
||||||
NETX_UME_USERNAME=
|
NETX_UME_USERNAME=
|
||||||
NETX_UME_PASSWORD=
|
NETX_UME_PASSWORD=
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@
|
||||||
- `netx` liveness: `/health/live`
|
- `netx` liveness: `/health/live`
|
||||||
- `netx` readiness: `/health/ready`
|
- `netx` readiness: `/health/ready`
|
||||||
- `netx` integrations: `/v1/integrations/status`
|
- `netx` integrations: `/v1/integrations/status`
|
||||||
- Alert when `oclaw_bridge.status != up` for more than 2 polling cycles.
|
- Alert when `dsh_alarm_hub` has no subscribers while key-alert rules are enabled (optional).
|
||||||
- Alert when `db.status != up` or `latency_ms` exceeds threshold.
|
- Alert when `db.status != up` or `latency_ms` exceeds threshold.
|
||||||
|
|
||||||
## Operations
|
## Operations
|
||||||
|
|
|
||||||
|
|
@ -67,13 +67,6 @@ def shutdown_runtime(*, reason: str = "lifespan") -> None:
|
||||||
except Exception: # noqa: BLE001
|
except Exception: # noqa: BLE001
|
||||||
_log.exception("shutdown_ws_consumer failed")
|
_log.exception("shutdown_ws_consumer failed")
|
||||||
|
|
||||||
try:
|
|
||||||
from .oclaw_alarm_forwarder import shutdown_oclaw_alarm_forwarder
|
|
||||||
|
|
||||||
shutdown_oclaw_alarm_forwarder()
|
|
||||||
except Exception: # noqa: BLE001
|
|
||||||
_log.exception("shutdown_oclaw_alarm_forwarder failed")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from .webcrt_session_registry import close_all_sessions
|
from .webcrt_session_registry import close_all_sessions
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,6 @@ class Settings(BaseSettings):
|
||||||
oclaw_connect_timeout_sec: float = 15.0
|
oclaw_connect_timeout_sec: float = 15.0
|
||||||
oclaw_analyze_read_timeout_sec: float = 180.0
|
oclaw_analyze_read_timeout_sec: float = 180.0
|
||||||
oclaw_health_timeout_sec: float = 8.0
|
oclaw_health_timeout_sec: float = 8.0
|
||||||
oclaw_alarm_ws_enabled: bool = False
|
|
||||||
oclaw_alarm_ws_url: str = "ws://127.0.0.1:8787/ws/netx-bridge"
|
|
||||||
# UME RESTCONF integration
|
# UME RESTCONF integration
|
||||||
ume_base_url: str = ""
|
ume_base_url: str = ""
|
||||||
ume_username: str = ""
|
ume_username: str = ""
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,6 @@ from sqlalchemy.orm import Session
|
||||||
from .config import settings
|
from .config import settings
|
||||||
from .db import get_db
|
from .db import get_db
|
||||||
from .dsh_alarm_hub import dsh_alarm_ws_loop, hub_status
|
from .dsh_alarm_hub import dsh_alarm_ws_loop, hub_status
|
||||||
from .oclaw_alarm_forwarder import forwarder_status
|
|
||||||
|
|
||||||
router = APIRouter(tags=["health"])
|
router = APIRouter(tags=["health"])
|
||||||
|
|
||||||
|
|
@ -79,7 +78,7 @@ def health_ready(db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||||
|
|
||||||
@router.get("/v1/integrations/status")
|
@router.get("/v1/integrations/status")
|
||||||
def integrations_status(db: Session = Depends(get_db)) -> dict:
|
def integrations_status(db: Session = Depends(get_db)) -> dict:
|
||||||
"""netx API + DB + oclaw bridge status."""
|
"""netx API + DB + DSH alarm hub status."""
|
||||||
netx_api = {"status": "up"}
|
netx_api = {"status": "up"}
|
||||||
|
|
||||||
db_status: dict = {"status": "unknown"}
|
db_status: dict = {"status": "unknown"}
|
||||||
|
|
@ -90,56 +89,8 @@ def integrations_status(db: Session = Depends(get_db)) -> dict:
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
db_status = {"status": "down", "error": str(exc)[:240]}
|
db_status = {"status": "down", "error": str(exc)[:240]}
|
||||||
|
|
||||||
oclaw_status: dict = {"status": "unknown"}
|
|
||||||
fwd = forwarder_status()
|
|
||||||
if not bool(fwd.get("enabled")):
|
|
||||||
oclaw_status = {
|
|
||||||
"status": "unknown",
|
|
||||||
"mode": "ws",
|
|
||||||
"enabled": False,
|
|
||||||
"connected": False,
|
|
||||||
"error_kind": "disabled",
|
|
||||||
"error": "NETX_OCLAW_ALARM_WS_ENABLED=false or missing token/url",
|
|
||||||
"forwarder": fwd,
|
|
||||||
}
|
|
||||||
elif bool(fwd.get("paused")):
|
|
||||||
oclaw_status = {
|
|
||||||
"status": "unknown",
|
|
||||||
"mode": "ws",
|
|
||||||
"enabled": True,
|
|
||||||
"connected": False,
|
|
||||||
"error_kind": "paused",
|
|
||||||
"error": "oclaw_alarm_forwarder runtime task paused",
|
|
||||||
"forwarder": fwd,
|
|
||||||
}
|
|
||||||
elif bool(fwd.get("connected")):
|
|
||||||
oclaw_status = {
|
|
||||||
"status": "up",
|
|
||||||
"mode": "ws",
|
|
||||||
"enabled": True,
|
|
||||||
"connected": True,
|
|
||||||
"queue_size": int(fwd.get("queue_size") or 0),
|
|
||||||
"published_ok": int(fwd.get("published_ok") or 0),
|
|
||||||
"published_fail": int(fwd.get("published_fail") or 0),
|
|
||||||
"url": str(fwd.get("url") or ""),
|
|
||||||
"forwarder": fwd,
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
oclaw_status = {
|
|
||||||
"status": "down",
|
|
||||||
"mode": "ws",
|
|
||||||
"enabled": True,
|
|
||||||
"connected": False,
|
|
||||||
"error_kind": "ws_disconnected",
|
|
||||||
"error": "oclaw netx-bridge WebSocket not connected",
|
|
||||||
"queue_size": int(fwd.get("queue_size") or 0),
|
|
||||||
"url": str(fwd.get("url") or ""),
|
|
||||||
"forwarder": fwd,
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"netx_api": netx_api,
|
"netx_api": netx_api,
|
||||||
"db": db_status,
|
"db": db_status,
|
||||||
"oclaw_bridge": oclaw_status,
|
|
||||||
"dsh_alarm_hub": hub_status(),
|
"dsh_alarm_hub": hub_status(),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -7,10 +7,9 @@ from typing import Any
|
||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from .dsh_alarm_hub import publish_alarm
|
||||||
from .key_alert_matcher import match_key_alert_rule
|
from .key_alert_matcher import match_key_alert_rule
|
||||||
from .models import UmeInventoryNE, UmeKeyAlertForwardLog
|
from .models import UmeInventoryNE, UmeKeyAlertForwardLog
|
||||||
from .oclaw_alarm_forwarder import enqueue_alarm_forward, is_forwarder_operational
|
|
||||||
from .dsh_alarm_hub import publish_alarm
|
|
||||||
from .ume_sync_service import (
|
from .ume_sync_service import (
|
||||||
_derive_ne_id_from_alarm,
|
_derive_ne_id_from_alarm,
|
||||||
_pick,
|
_pick,
|
||||||
|
|
@ -82,6 +81,7 @@ def maybe_forward_key_alert(
|
||||||
)
|
)
|
||||||
.first()
|
.first()
|
||||||
)
|
)
|
||||||
|
# Column name is historical (oclaw_ok); now means DSH hub delivery succeeded.
|
||||||
if existing is not None and int(existing.oclaw_ok or 0) == 1:
|
if existing is not None and int(existing.oclaw_ok or 0) == 1:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -96,22 +96,10 @@ def maybe_forward_key_alert(
|
||||||
payload["rule_key"] = str(rule.notification_id or "")
|
payload["rule_key"] = str(rule.notification_id or "")
|
||||||
|
|
||||||
hub_sent = publish_alarm(payload)
|
hub_sent = publish_alarm(payload)
|
||||||
oclaw_queued = False
|
if hub_sent <= 0:
|
||||||
if is_forwarder_operational():
|
|
||||||
oclaw_queued = bool(enqueue_alarm_forward(payload))
|
|
||||||
if hub_sent <= 0 and not oclaw_queued:
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Hub-only delivery already reached DSH clients — mark ok for dedup.
|
status_text = f"dsh_hub:{hub_sent}"
|
||||||
# Oclaw path stays pending until the bridge records a result.
|
|
||||||
delivered_ok = hub_sent > 0 and not oclaw_queued
|
|
||||||
status = []
|
|
||||||
if hub_sent > 0:
|
|
||||||
status.append(f"dsh_hub:{hub_sent}")
|
|
||||||
if oclaw_queued:
|
|
||||||
status.append("oclaw_queued")
|
|
||||||
status_text = ",".join(status) if status else "queued"
|
|
||||||
|
|
||||||
row = existing
|
row = existing
|
||||||
if row is None:
|
if row is None:
|
||||||
row = UmeKeyAlertForwardLog(
|
row = UmeKeyAlertForwardLog(
|
||||||
|
|
@ -120,62 +108,21 @@ def maybe_forward_key_alert(
|
||||||
rule_key=str(rule.notification_id or ""),
|
rule_key=str(rule.notification_id or ""),
|
||||||
notification_id=notification_id_from_norm(norm),
|
notification_id=notification_id_from_norm(norm),
|
||||||
forwarded_at=_utc_now_naive(),
|
forwarded_at=_utc_now_naive(),
|
||||||
oclaw_ok=1 if delivered_ok else 0,
|
oclaw_ok=1,
|
||||||
error="" if delivered_ok else status_text,
|
error="",
|
||||||
)
|
)
|
||||||
db.add(row)
|
db.add(row)
|
||||||
else:
|
else:
|
||||||
row.notification_id = notification_id_from_norm(norm)
|
row.notification_id = notification_id_from_norm(norm)
|
||||||
row.rule_key = str(rule.notification_id or "")
|
row.rule_key = str(rule.notification_id or "")
|
||||||
row.forwarded_at = _utc_now_naive()
|
row.forwarded_at = _utc_now_naive()
|
||||||
row.oclaw_ok = 1 if delivered_ok else 0
|
row.oclaw_ok = 1
|
||||||
row.error = "" if delivered_ok else status_text
|
row.error = ""
|
||||||
try:
|
try:
|
||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
|
_log.debug("key_alert forward log race alarm_key=%s action=%s", alarm_key, act)
|
||||||
|
else:
|
||||||
|
_log.debug("key_alert forwarded via DSH hub=%s status=%s", hub_sent, status_text)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def record_forward_result(
|
|
||||||
*,
|
|
||||||
alarm_key: str,
|
|
||||||
action: str,
|
|
||||||
ok: bool,
|
|
||||||
error: str = "",
|
|
||||||
rule_key: str = "",
|
|
||||||
) -> None:
|
|
||||||
from .db import SessionLocal
|
|
||||||
|
|
||||||
key = str(alarm_key or "").strip()
|
|
||||||
act = str(action or "").strip().lower()
|
|
||||||
if not key or not act:
|
|
||||||
return
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
row = (
|
|
||||||
db.query(UmeKeyAlertForwardLog)
|
|
||||||
.filter(UmeKeyAlertForwardLog.alarm_key == key, UmeKeyAlertForwardLog.action == act)
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
if row is None:
|
|
||||||
row = UmeKeyAlertForwardLog(
|
|
||||||
alarm_key=key,
|
|
||||||
action=act,
|
|
||||||
rule_key=str(rule_key or "").strip(),
|
|
||||||
forwarded_at=_utc_now_naive(),
|
|
||||||
oclaw_ok=1 if ok else 0,
|
|
||||||
error="" if ok else str(error or "forward_failed")[:240],
|
|
||||||
)
|
|
||||||
db.add(row)
|
|
||||||
else:
|
|
||||||
if rule_key and not str(row.rule_key or "").strip():
|
|
||||||
row.rule_key = str(rule_key or "").strip()
|
|
||||||
row.forwarded_at = _utc_now_naive()
|
|
||||||
row.oclaw_ok = 1 if ok else 0
|
|
||||||
row.error = "" if ok else str(error or "forward_failed")[:240]
|
|
||||||
db.commit()
|
|
||||||
except Exception:
|
|
||||||
db.rollback()
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ from .config import settings
|
||||||
from .db import db_pool_status
|
from .db import db_pool_status
|
||||||
from .db_storage_metrics import collect_db_storage_metrics
|
from .db_storage_metrics import collect_db_storage_metrics
|
||||||
from .host_metrics import collect_host_metrics
|
from .host_metrics import collect_host_metrics
|
||||||
from .oclaw_alarm_forwarder import forwarder_status
|
|
||||||
|
|
||||||
router = APIRouter(tags=["metrics"])
|
router = APIRouter(tags=["metrics"])
|
||||||
|
|
||||||
|
|
@ -30,7 +29,6 @@ def collect_runtime_metrics() -> dict[str, Any]:
|
||||||
"db_storage": collect_db_storage_metrics(),
|
"db_storage": collect_db_storage_metrics(),
|
||||||
"cli_budget": cli_budget_status(),
|
"cli_budget": cli_budget_status(),
|
||||||
"audit_queue": audit_queue_status(),
|
"audit_queue": audit_queue_status(),
|
||||||
"oclaw_forwarder": forwarder_status(),
|
|
||||||
"schedulers_inline": bool(getattr(settings, "run_inline_schedulers", True)),
|
"schedulers_inline": bool(getattr(settings, "run_inline_schedulers", True)),
|
||||||
"host": collect_host_metrics(),
|
"host": collect_host_metrics(),
|
||||||
}
|
}
|
||||||
|
|
@ -92,17 +90,6 @@ def _prom_lines(metrics: dict[str, Any]) -> str:
|
||||||
for key in ("depth", "dropped", "maxsize"):
|
for key in ("depth", "dropped", "maxsize"):
|
||||||
if key in audit:
|
if key in audit:
|
||||||
lines.append(f"netx_audit_queue_{key} {audit[key]}")
|
lines.append(f"netx_audit_queue_{key} {audit[key]}")
|
||||||
fwd = metrics.get("oclaw_forwarder") or {}
|
|
||||||
for key, prom in (
|
|
||||||
("queue_size", "netx_oclaw_forwarder_queue_size"),
|
|
||||||
("published_ok", "netx_oclaw_forwarder_published_ok"),
|
|
||||||
("published_fail", "netx_oclaw_forwarder_published_fail"),
|
|
||||||
("dropped", "netx_oclaw_forwarder_dropped"),
|
|
||||||
("requeued", "netx_oclaw_forwarder_requeued"),
|
|
||||||
("retry_exhausted", "netx_oclaw_forwarder_retry_exhausted"),
|
|
||||||
):
|
|
||||||
if key in fwd:
|
|
||||||
lines.append(f"{prom} {int(fwd.get(key) or 0)}")
|
|
||||||
sched = metrics.get("device_schedulers") or {}
|
sched = metrics.get("device_schedulers") or {}
|
||||||
if "stale" in sched:
|
if "stale" in sched:
|
||||||
lines.append(f'netx_device_schedulers_stale {1 if sched.get("stale") else 0}')
|
lines.append(f'netx_device_schedulers_stale {1 if sched.get("stale") else 0}')
|
||||||
|
|
|
||||||
|
|
@ -1,350 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import queue
|
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
from collections.abc import Callable
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
import websocket
|
|
||||||
|
|
||||||
from .config import settings
|
|
||||||
from .runtime_task_messages import fwd_state_code
|
|
||||||
|
|
||||||
_log = logging.getLogger("netx.oclaw.alarm_forwarder")
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_bridge_error(exc: BaseException | str) -> str:
|
|
||||||
"""Map low-level WS errors to stable fwd:* codes for UI i18n."""
|
|
||||||
raw = str(exc or "").strip()
|
|
||||||
low = raw.lower()
|
|
||||||
name = type(exc).__name__ if isinstance(exc, BaseException) else ""
|
|
||||||
if (
|
|
||||||
"10061" in raw
|
|
||||||
or "actively refused" in low
|
|
||||||
or "积极拒绝" in raw
|
|
||||||
or "connection refused" in low
|
|
||||||
or name == "ConnectionRefusedError"
|
|
||||||
):
|
|
||||||
return fwd_state_code("connect_refused")
|
|
||||||
if "timed out" in low or "timeout" in low or name in {"TimeoutError", "socket.timeout"}:
|
|
||||||
return fwd_state_code("connect_timeout")
|
|
||||||
if "auth failed" in low or "auth-fail" in low or "invalid_token" in low:
|
|
||||||
return fwd_state_code("auth_failed")
|
|
||||||
if raw:
|
|
||||||
return raw[:200]
|
|
||||||
return fwd_state_code("disconnected")
|
|
||||||
|
|
||||||
|
|
||||||
_OUTBOUND_Q: "queue.Queue[dict[str, Any]] | None" = None
|
|
||||||
_Q_LOCK = threading.Lock()
|
|
||||||
_STOP_EVENT = threading.Event()
|
|
||||||
_THREAD: threading.Thread | None = None
|
|
||||||
_CONN_LOCK = threading.Lock()
|
|
||||||
_WS: Any | None = None
|
|
||||||
_CONNECTED = threading.Event()
|
|
||||||
_IS_PAUSED: Callable[[], bool] | None = None
|
|
||||||
_ON_STATUS: Callable[[str], None] | None = None
|
|
||||||
_STATS_LOCK = threading.Lock()
|
|
||||||
_STATS: dict[str, int] = {
|
|
||||||
"published_ok": 0,
|
|
||||||
"published_fail": 0,
|
|
||||||
"queued": 0,
|
|
||||||
"dropped": 0,
|
|
||||||
"requeued": 0,
|
|
||||||
"retry_exhausted": 0,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _outbound_q() -> "queue.Queue[dict[str, Any]]":
|
|
||||||
global _OUTBOUND_Q
|
|
||||||
with _Q_LOCK:
|
|
||||||
if _OUTBOUND_Q is None:
|
|
||||||
maxsize = max(100, int(getattr(settings, "oclaw_forward_queue_max", 2000) or 2000))
|
|
||||||
_OUTBOUND_Q = queue.Queue(maxsize=maxsize)
|
|
||||||
return _OUTBOUND_Q
|
|
||||||
|
|
||||||
|
|
||||||
def _utc_now_iso() -> str:
|
|
||||||
return datetime.now(timezone.utc).isoformat()
|
|
||||||
|
|
||||||
|
|
||||||
def _bridge_token() -> str:
|
|
||||||
return str(getattr(settings, "oclaw_analyze_token", "") or "").strip()
|
|
||||||
|
|
||||||
|
|
||||||
def _bridge_url() -> str:
|
|
||||||
return str(getattr(settings, "oclaw_alarm_ws_url", "") or "").strip()
|
|
||||||
|
|
||||||
|
|
||||||
def is_forwarder_enabled() -> bool:
|
|
||||||
if not bool(getattr(settings, "oclaw_alarm_ws_enabled", False)):
|
|
||||||
return False
|
|
||||||
return bool(_bridge_url()) and bool(_bridge_token())
|
|
||||||
|
|
||||||
|
|
||||||
def configure_oclaw_alarm_forwarder(
|
|
||||||
*,
|
|
||||||
is_paused: Callable[[], bool] | None = None,
|
|
||||||
on_status: Callable[[str], None] | None = None,
|
|
||||||
) -> None:
|
|
||||||
global _IS_PAUSED, _ON_STATUS
|
|
||||||
_IS_PAUSED = is_paused
|
|
||||||
_ON_STATUS = on_status
|
|
||||||
|
|
||||||
|
|
||||||
def _forwarder_paused() -> bool:
|
|
||||||
if _IS_PAUSED is None:
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
return bool(_IS_PAUSED())
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def is_forwarder_operational() -> bool:
|
|
||||||
return is_forwarder_enabled() and not _forwarder_paused()
|
|
||||||
|
|
||||||
|
|
||||||
def _notify_status(msg: str) -> None:
|
|
||||||
if _ON_STATUS is None:
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
_ON_STATUS(str(msg or "")[:240])
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def request_forwarder_reconnect() -> None:
|
|
||||||
with _CONN_LOCK:
|
|
||||||
ws = _WS
|
|
||||||
if ws is not None:
|
|
||||||
try:
|
|
||||||
ws.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def enqueue_alarm_forward(payload: dict[str, Any]) -> bool:
|
|
||||||
if not is_forwarder_operational():
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
_outbound_q().put_nowait(dict(payload))
|
|
||||||
with _STATS_LOCK:
|
|
||||||
_STATS["queued"] = int(_STATS.get("queued", 0)) + 1
|
|
||||||
return True
|
|
||||||
except queue.Full:
|
|
||||||
with _STATS_LOCK:
|
|
||||||
_STATS["dropped"] = int(_STATS.get("dropped", 0)) + 1
|
|
||||||
_log.warning("oclaw alarm forward queue full; dropping alarm_key=%s", payload.get("alarm_key"))
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _requeue_or_drop(payload: dict[str, Any], *, reason: str) -> None:
|
|
||||||
max_retries = max(0, int(getattr(settings, "oclaw_forward_max_retries", 3) or 3))
|
|
||||||
item = dict(payload)
|
|
||||||
attempts = int(item.get("_fwd_attempts") or 0) + 1
|
|
||||||
item["_fwd_attempts"] = attempts
|
|
||||||
if attempts > max_retries:
|
|
||||||
with _STATS_LOCK:
|
|
||||||
_STATS["retry_exhausted"] = int(_STATS.get("retry_exhausted", 0)) + 1
|
|
||||||
_STATS["dropped"] = int(_STATS.get("dropped", 0)) + 1
|
|
||||||
_log.warning(
|
|
||||||
"oclaw forward drop after retries alarm_key=%s attempts=%s reason=%s",
|
|
||||||
item.get("alarm_key"),
|
|
||||||
attempts,
|
|
||||||
reason[:80],
|
|
||||||
)
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
_outbound_q().put_nowait(item)
|
|
||||||
with _STATS_LOCK:
|
|
||||||
_STATS["requeued"] = int(_STATS.get("requeued", 0)) + 1
|
|
||||||
except queue.Full:
|
|
||||||
with _STATS_LOCK:
|
|
||||||
_STATS["dropped"] = int(_STATS.get("dropped", 0)) + 1
|
|
||||||
_log.warning(
|
|
||||||
"oclaw forward requeue full; dropping alarm_key=%s",
|
|
||||||
item.get("alarm_key"),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _send_auth(ws: Any) -> bool:
|
|
||||||
token = _bridge_token()
|
|
||||||
ws.send(json.dumps({"type": "auth", "token": token}, ensure_ascii=False))
|
|
||||||
deadline = time.time() + 10.0
|
|
||||||
while time.time() < deadline:
|
|
||||||
raw = ws.recv()
|
|
||||||
if not raw:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
msg = json.loads(raw)
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
if not isinstance(msg, dict):
|
|
||||||
continue
|
|
||||||
if str(msg.get("type") or "").strip() == "auth-ok":
|
|
||||||
return True
|
|
||||||
if str(msg.get("type") or "").strip() == "auth-fail":
|
|
||||||
return False
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _dispatch_one(ws: Any, payload: dict[str, Any]) -> dict[str, Any]:
|
|
||||||
envelope = {
|
|
||||||
"type": "event",
|
|
||||||
"event": "netx.alarm",
|
|
||||||
"payload": payload,
|
|
||||||
}
|
|
||||||
ws.send(json.dumps(envelope, ensure_ascii=False, default=str))
|
|
||||||
deadline = time.time() + 15.0
|
|
||||||
alarm_key = str(payload.get("alarm_key") or "")
|
|
||||||
while time.time() < deadline:
|
|
||||||
raw = ws.recv()
|
|
||||||
if not raw:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
msg = json.loads(raw)
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
if not isinstance(msg, dict):
|
|
||||||
continue
|
|
||||||
if str(msg.get("type") or "").strip() == "pong":
|
|
||||||
continue
|
|
||||||
if str(msg.get("type") or "").strip() == "ack":
|
|
||||||
if alarm_key and str(msg.get("alarm_key") or "").strip() not in {"", alarm_key}:
|
|
||||||
continue
|
|
||||||
return msg
|
|
||||||
return {"type": "ack", "alarm_key": alarm_key, "ok": False, "error": "ack_timeout"}
|
|
||||||
|
|
||||||
|
|
||||||
def _run_loop() -> None:
|
|
||||||
global _WS
|
|
||||||
backoff_s = 2.0
|
|
||||||
while not _STOP_EVENT.is_set():
|
|
||||||
if not is_forwarder_enabled() or _forwarder_paused():
|
|
||||||
_CONNECTED.clear()
|
|
||||||
if _forwarder_paused():
|
|
||||||
_notify_status("fwd:paused")
|
|
||||||
else:
|
|
||||||
_notify_status("fwd:disabled")
|
|
||||||
time.sleep(2.0)
|
|
||||||
continue
|
|
||||||
url = _bridge_url()
|
|
||||||
ws = None
|
|
||||||
try:
|
|
||||||
_notify_status("fwd:connecting")
|
|
||||||
ws = websocket.create_connection(url, timeout=20)
|
|
||||||
ws.settimeout(30)
|
|
||||||
if not _send_auth(ws):
|
|
||||||
raise RuntimeError("oclaw netx-bridge auth failed")
|
|
||||||
with _CONN_LOCK:
|
|
||||||
_WS = ws
|
|
||||||
_CONNECTED.set()
|
|
||||||
backoff_s = 2.0
|
|
||||||
_log.info("oclaw netx-bridge connected url=%s", url[:120])
|
|
||||||
_notify_status("fwd:connected")
|
|
||||||
while not _STOP_EVENT.is_set():
|
|
||||||
if _forwarder_paused():
|
|
||||||
_notify_status("fwd:paused")
|
|
||||||
raise RuntimeError("forwarder paused")
|
|
||||||
if not is_forwarder_enabled():
|
|
||||||
_notify_status("fwd:disabled")
|
|
||||||
raise RuntimeError("forwarder disabled")
|
|
||||||
try:
|
|
||||||
payload = _outbound_q().get(timeout=1.0)
|
|
||||||
except queue.Empty:
|
|
||||||
try:
|
|
||||||
ws.send(json.dumps({"type": "ping", "ts": _utc_now_iso()}, ensure_ascii=False))
|
|
||||||
except Exception:
|
|
||||||
raise
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
ack = _dispatch_one(ws, payload)
|
|
||||||
ok = bool(ack.get("ok"))
|
|
||||||
err = str(ack.get("error") or "")[:240]
|
|
||||||
with _STATS_LOCK:
|
|
||||||
if ok:
|
|
||||||
_STATS["published_ok"] = int(_STATS.get("published_ok", 0)) + 1
|
|
||||||
else:
|
|
||||||
_STATS["published_fail"] = int(_STATS.get("published_fail", 0)) + 1
|
|
||||||
try:
|
|
||||||
from .key_alert_forward import record_forward_result
|
|
||||||
|
|
||||||
record_forward_result(
|
|
||||||
alarm_key=str(payload.get("alarm_key") or ""),
|
|
||||||
action=str(payload.get("action") or ""),
|
|
||||||
ok=ok,
|
|
||||||
error=err,
|
|
||||||
rule_key=str(payload.get("rule_key") or ""),
|
|
||||||
)
|
|
||||||
except Exception as rec_exc:
|
|
||||||
_log.warning("forward result record failed: %s", str(rec_exc)[:120])
|
|
||||||
if not ok:
|
|
||||||
_log.warning(
|
|
||||||
"oclaw ack failed alarm_key=%s error=%s",
|
|
||||||
payload.get("alarm_key"),
|
|
||||||
str(ack.get("error") or "")[:120],
|
|
||||||
)
|
|
||||||
# Soft ack failure: do not infinite-requeue; count as fail only.
|
|
||||||
except Exception as exc:
|
|
||||||
_log.warning("oclaw forward failed alarm_key=%s err=%s", payload.get("alarm_key"), str(exc)[:120])
|
|
||||||
_requeue_or_drop(payload, reason=str(exc)[:120])
|
|
||||||
raise
|
|
||||||
finally:
|
|
||||||
_outbound_q().task_done()
|
|
||||||
except Exception as exc:
|
|
||||||
_CONNECTED.clear()
|
|
||||||
err = _normalize_bridge_error(exc)
|
|
||||||
_log.warning("oclaw netx-bridge disconnected: %s", str(exc)[:200])
|
|
||||||
_notify_status(err)
|
|
||||||
time.sleep(backoff_s)
|
|
||||||
backoff_s = min(backoff_s * 1.5, 60.0)
|
|
||||||
finally:
|
|
||||||
with _CONN_LOCK:
|
|
||||||
_WS = None
|
|
||||||
_CONNECTED.clear()
|
|
||||||
if ws is not None:
|
|
||||||
try:
|
|
||||||
ws.close()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def start_oclaw_alarm_forwarder() -> threading.Thread | None:
|
|
||||||
global _THREAD
|
|
||||||
if _THREAD is not None and _THREAD.is_alive():
|
|
||||||
return _THREAD
|
|
||||||
_STOP_EVENT.clear()
|
|
||||||
_THREAD = threading.Thread(target=_run_loop, name="oclaw-alarm-forwarder", daemon=True)
|
|
||||||
_THREAD.start()
|
|
||||||
return _THREAD
|
|
||||||
|
|
||||||
|
|
||||||
def shutdown_oclaw_alarm_forwarder() -> None:
|
|
||||||
_STOP_EVENT.set()
|
|
||||||
|
|
||||||
|
|
||||||
def forwarder_status() -> dict[str, Any]:
|
|
||||||
with _STATS_LOCK:
|
|
||||||
stats = dict(_STATS)
|
|
||||||
paused = _forwarder_paused()
|
|
||||||
enabled = is_forwarder_enabled()
|
|
||||||
return {
|
|
||||||
"enabled": enabled,
|
|
||||||
"operational": bool(enabled and not paused),
|
|
||||||
"paused": paused,
|
|
||||||
"connected": _CONNECTED.is_set(),
|
|
||||||
"queue_size": int(_outbound_q().qsize()),
|
|
||||||
"queue_max": max(100, int(getattr(settings, "oclaw_forward_queue_max", 2000) or 2000)),
|
|
||||||
"url": _bridge_url(),
|
|
||||||
"published_ok": int(stats.get("published_ok", 0)),
|
|
||||||
"published_fail": int(stats.get("published_fail", 0)),
|
|
||||||
"queued_total": int(stats.get("queued", 0)),
|
|
||||||
"dropped": int(stats.get("dropped", 0)),
|
|
||||||
"requeued": int(stats.get("requeued", 0)),
|
|
||||||
"retry_exhausted": int(stats.get("retry_exhausted", 0)),
|
|
||||||
}
|
|
||||||
|
|
@ -336,13 +336,6 @@ def _ume_runtime_items() -> list[dict[str, Any]]:
|
||||||
from .ume_support import _list_runtime_tasks
|
from .ume_support import _list_runtime_tasks
|
||||||
except Exception:
|
except Exception:
|
||||||
return []
|
return []
|
||||||
fwd: dict[str, Any] | None = None
|
|
||||||
try:
|
|
||||||
from .oclaw_alarm_forwarder import forwarder_status
|
|
||||||
|
|
||||||
fwd = forwarder_status()
|
|
||||||
except Exception:
|
|
||||||
fwd = None
|
|
||||||
ws_conn: dict[str, Any] | None = None
|
ws_conn: dict[str, Any] | None = None
|
||||||
try:
|
try:
|
||||||
from .ume_alarm_ws import get_ws_connection_status
|
from .ume_alarm_ws import get_ws_connection_status
|
||||||
|
|
@ -356,24 +349,7 @@ def _ume_runtime_items() -> list[dict[str, Any]]:
|
||||||
status = str(row.get("status") or "unknown")
|
status = str(row.get("status") or "unknown")
|
||||||
progress = str(row.get("interval_label") or "")
|
progress = str(row.get("interval_label") or "")
|
||||||
detail = str(row.get("last_error") or "")[:240]
|
detail = str(row.get("last_error") or "")[:240]
|
||||||
if task == "oclaw_alarm_forwarder" and isinstance(fwd, dict):
|
if task == "alarms_current_ws_consumer" and isinstance(ws_conn, dict):
|
||||||
bits: list[str] = []
|
|
||||||
if not bool(fwd.get("enabled")):
|
|
||||||
bits.append("disabled")
|
|
||||||
elif bool(fwd.get("paused")):
|
|
||||||
bits.append("paused")
|
|
||||||
elif bool(fwd.get("connected")):
|
|
||||||
bits.append("connected")
|
|
||||||
else:
|
|
||||||
bits.append("disconnected")
|
|
||||||
q = int(fwd.get("queue_size") or 0)
|
|
||||||
if q > 0:
|
|
||||||
bits.append(f"q={q}")
|
|
||||||
pub_ok = int(fwd.get("published_ok") or 0)
|
|
||||||
if pub_ok > 0:
|
|
||||||
bits.append(f"pub={pub_ok}")
|
|
||||||
progress = " · ".join([p for p in (progress, *bits) if p])
|
|
||||||
elif task == "alarms_current_ws_consumer" and isinstance(ws_conn, dict):
|
|
||||||
state = str(ws_conn.get("state") or ws_conn.get("status") or "").strip()
|
state = str(ws_conn.get("state") or ws_conn.get("status") or "").strip()
|
||||||
if state:
|
if state:
|
||||||
progress = " · ".join([p for p in (progress, state) if p])
|
progress = " · ".join([p for p in (progress, state) if p])
|
||||||
|
|
|
||||||
|
|
@ -8,17 +8,11 @@ RT_ALARMS_SYNC_IN_PROGRESS_SKIP = "rt:alarms_sync_in_progress_skip"
|
||||||
RT_PULLING_INVENTORY = "rt:pulling_inventory"
|
RT_PULLING_INVENTORY = "rt:pulling_inventory"
|
||||||
RT_PULLING_TOPOLOGY = "rt:pulling_topology"
|
RT_PULLING_TOPOLOGY = "rt:pulling_topology"
|
||||||
RT_UME_WS_DISABLED_NO_BASE_URL = "rt:ume_ws_disabled_no_base_url"
|
RT_UME_WS_DISABLED_NO_BASE_URL = "rt:ume_ws_disabled_no_base_url"
|
||||||
RT_OCLAW_FWD_DISABLED = "rt:oclaw_fwd_disabled"
|
|
||||||
RT_RESUMED_SYNC_SOON = "rt:resumed_sync_soon"
|
RT_RESUMED_SYNC_SOON = "rt:resumed_sync_soon"
|
||||||
RT_RESUMED_WSS_RECONNECT = "rt:resumed_wss_reconnect"
|
RT_RESUMED_WSS_RECONNECT = "rt:resumed_wss_reconnect"
|
||||||
RT_RESUMED_OCLAW_WSS_RECONNECT = "rt:resumed_oclaw_wss_reconnect"
|
|
||||||
RT_RESUMED = "rt:resumed"
|
RT_RESUMED = "rt:resumed"
|
||||||
RT_KEEPALIVE_FAILED = "rt:keepalive_failed"
|
RT_KEEPALIVE_FAILED = "rt:keepalive_failed"
|
||||||
|
|
||||||
|
|
||||||
def ws_state_code(state: str) -> str:
|
def ws_state_code(state: str) -> str:
|
||||||
return f"ws:{str(state or '').strip()}"
|
return f"ws:{str(state or '').strip()}"
|
||||||
|
|
||||||
|
|
||||||
def fwd_state_code(state: str) -> str:
|
|
||||||
return f"fwd:{str(state or '').strip()}"
|
|
||||||
|
|
@ -34,10 +34,6 @@ from .models import (
|
||||||
UmeKeyAlertRule,
|
UmeKeyAlertRule,
|
||||||
UmeSyncJob,
|
UmeSyncJob,
|
||||||
)
|
)
|
||||||
from .oclaw_alarm_forwarder import (
|
|
||||||
forwarder_status,
|
|
||||||
request_forwarder_reconnect,
|
|
||||||
)
|
|
||||||
from .ume_alarm_ws import (
|
from .ume_alarm_ws import (
|
||||||
cancel_alarm_subscription_manual,
|
cancel_alarm_subscription_manual,
|
||||||
clear_local_alarm_subscription_manual,
|
clear_local_alarm_subscription_manual,
|
||||||
|
|
|
||||||
|
|
@ -34,10 +34,6 @@ from .models import (
|
||||||
UmeKeyAlertRule,
|
UmeKeyAlertRule,
|
||||||
UmeSyncJob,
|
UmeSyncJob,
|
||||||
)
|
)
|
||||||
from .oclaw_alarm_forwarder import (
|
|
||||||
forwarder_status,
|
|
||||||
request_forwarder_reconnect,
|
|
||||||
)
|
|
||||||
from .ume_alarm_ws import (
|
from .ume_alarm_ws import (
|
||||||
cancel_alarm_subscription_manual,
|
cancel_alarm_subscription_manual,
|
||||||
clear_local_alarm_subscription_manual,
|
clear_local_alarm_subscription_manual,
|
||||||
|
|
|
||||||
|
|
@ -35,10 +35,6 @@ from .models import (
|
||||||
UmeSyncJob,
|
UmeSyncJob,
|
||||||
)
|
)
|
||||||
from .dsh_alarm_hub import hub_status as dsh_alarm_hub_status
|
from .dsh_alarm_hub import hub_status as dsh_alarm_hub_status
|
||||||
from .oclaw_alarm_forwarder import (
|
|
||||||
forwarder_status,
|
|
||||||
request_forwarder_reconnect,
|
|
||||||
)
|
|
||||||
from .ume_alarm_ws import (
|
from .ume_alarm_ws import (
|
||||||
cancel_alarm_subscription_manual,
|
cancel_alarm_subscription_manual,
|
||||||
clear_local_alarm_subscription_manual,
|
clear_local_alarm_subscription_manual,
|
||||||
|
|
@ -146,14 +142,12 @@ def ume_list_key_alert_rules(
|
||||||
}
|
}
|
||||||
for row in rows
|
for row in rows
|
||||||
]
|
]
|
||||||
fwd = forwarder_status()
|
|
||||||
hub = dsh_alarm_hub_status()
|
hub = dsh_alarm_hub_status()
|
||||||
return {
|
return {
|
||||||
"items": items,
|
"items": items,
|
||||||
"total": total,
|
"total": total,
|
||||||
"page": page,
|
"page": page,
|
||||||
"page_size": page_size,
|
"page_size": page_size,
|
||||||
"forwarder": fwd,
|
|
||||||
"dsh_alarm_hub": hub,
|
"dsh_alarm_hub": hub,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -183,8 +177,6 @@ def ume_key_alert_monitor(
|
||||||
"page_size": int(base.get("page_size") or page_size),
|
"page_size": int(base.get("page_size") or page_size),
|
||||||
"config": get_key_alert_monitor_config(db),
|
"config": get_key_alert_monitor_config(db),
|
||||||
"dsh_alarm_hub": base.get("dsh_alarm_hub") or dsh_alarm_hub_status(),
|
"dsh_alarm_hub": base.get("dsh_alarm_hub") or dsh_alarm_hub_status(),
|
||||||
# Legacy OClaw outbound bridge (optional; demoted in UI).
|
|
||||||
"forwarder": base.get("forwarder") or forwarder_status(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -345,6 +337,6 @@ def ume_list_notification_ids(
|
||||||
for nid, cause in rows
|
for nid, cause in rows
|
||||||
if str(nid or "").strip()
|
if str(nid or "").strip()
|
||||||
]
|
]
|
||||||
return {"items": items, "total": len(items), "forwarder": forwarder_status()}
|
return {"items": items, "total": len(items)}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ Device collectors (config_sync / LLDP / ne_collect / port_traffic) run via ``sta
|
||||||
(API inline by default; set ``NETX_RUN_INLINE_SCHEDULERS=false`` and run
|
(API inline by default; set ``NETX_RUN_INLINE_SCHEDULERS=false`` and run
|
||||||
``python -m netx_api.worker`` for a split process).
|
``python -m netx_api.worker`` for a split process).
|
||||||
API process also owns UME keepalive, alarm WSS, current-alarm/inventory sync loops,
|
API process also owns UME keepalive, alarm WSS, current-alarm/inventory sync loops,
|
||||||
and oclaw forwarder via ``start_api_sideband_threads``.
|
and DSH alarm hub via ``start_api_sideband_threads``.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
@ -29,7 +29,6 @@ from .ume_sync_topology import fail_stale_topology_running_jobs
|
||||||
from .runtime_task_messages import (
|
from .runtime_task_messages import (
|
||||||
RT_ALARMS_SYNC_IN_PROGRESS_SKIP,
|
RT_ALARMS_SYNC_IN_PROGRESS_SKIP,
|
||||||
RT_KEEPALIVE_FAILED,
|
RT_KEEPALIVE_FAILED,
|
||||||
RT_OCLAW_FWD_DISABLED,
|
|
||||||
RT_PULLING_ALARMS_CURRENT,
|
RT_PULLING_ALARMS_CURRENT,
|
||||||
RT_PULLING_INVENTORY,
|
RT_PULLING_INVENTORY,
|
||||||
RT_PULLING_TOPOLOGY,
|
RT_PULLING_TOPOLOGY,
|
||||||
|
|
@ -37,12 +36,6 @@ from .runtime_task_messages import (
|
||||||
RT_UME_WS_DISABLED_NO_BASE_URL,
|
RT_UME_WS_DISABLED_NO_BASE_URL,
|
||||||
RT_WSS_ACTIVE_SKIP_REST,
|
RT_WSS_ACTIVE_SKIP_REST,
|
||||||
)
|
)
|
||||||
from .oclaw_alarm_forwarder import (
|
|
||||||
configure_oclaw_alarm_forwarder,
|
|
||||||
forwarder_status,
|
|
||||||
is_forwarder_enabled,
|
|
||||||
start_oclaw_alarm_forwarder,
|
|
||||||
)
|
|
||||||
|
|
||||||
_log = logging.getLogger("netx.ume.runtime")
|
_log = logging.getLogger("netx.ume.runtime")
|
||||||
_schedule_log = logging.getLogger("netx.ume.schedule")
|
_schedule_log = logging.getLogger("netx.ume.schedule")
|
||||||
|
|
@ -420,48 +413,6 @@ def start_api_sideband_threads() -> None:
|
||||||
last_run_at=datetime.now(timezone.utc),
|
last_run_at=datetime.now(timezone.utc),
|
||||||
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
|
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
|
||||||
)
|
)
|
||||||
try:
|
|
||||||
def _fwd_on_status(msg: str) -> None:
|
|
||||||
paused = ume_support._runtime_is_paused("oclaw_alarm_forwarder")
|
|
||||||
fwd = forwarder_status()
|
|
||||||
if paused:
|
|
||||||
status = "paused"
|
|
||||||
elif not bool(fwd.get("enabled")):
|
|
||||||
status = "paused"
|
|
||||||
elif bool(fwd.get("connected")):
|
|
||||||
status = "running"
|
|
||||||
else:
|
|
||||||
status = "running"
|
|
||||||
ume_support._set_runtime_task(
|
|
||||||
"oclaw_alarm_forwarder",
|
|
||||||
status=status,
|
|
||||||
last_run_at=datetime.now(timezone.utc),
|
|
||||||
last_error=str(msg or "")[:240],
|
|
||||||
)
|
|
||||||
|
|
||||||
configure_oclaw_alarm_forwarder(
|
|
||||||
is_paused=lambda: ume_support._runtime_is_paused("oclaw_alarm_forwarder"),
|
|
||||||
on_status=_fwd_on_status,
|
|
||||||
)
|
|
||||||
if is_forwarder_enabled():
|
|
||||||
ume_support._set_runtime_task("oclaw_alarm_forwarder", status="running", last_error="")
|
|
||||||
else:
|
|
||||||
ume_support._set_runtime_task(
|
|
||||||
"oclaw_alarm_forwarder",
|
|
||||||
status="paused",
|
|
||||||
last_error=RT_OCLAW_FWD_DISABLED,
|
|
||||||
)
|
|
||||||
t_fwd = start_oclaw_alarm_forwarder()
|
|
||||||
if t_fwd is not None:
|
|
||||||
_schedule_log.info("started thread %s alive=%s", t_fwd.name, t_fwd.is_alive())
|
|
||||||
except Exception as exc:
|
|
||||||
_schedule_log.exception("startup: oclaw_alarm_forwarder thread init failed: %s", exc)
|
|
||||||
ume_support._set_runtime_task(
|
|
||||||
"oclaw_alarm_forwarder",
|
|
||||||
status="error",
|
|
||||||
last_run_at=datetime.now(timezone.utc),
|
|
||||||
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Dock already has topology but Fabric/World empty (or last apply partial): heal without waiting 24h.
|
# Dock already has topology but Fabric/World empty (or last apply partial): heal without waiting 24h.
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -18,11 +18,9 @@ from .timeutil import utcnow_naive
|
||||||
from .runtime_task_messages import (
|
from .runtime_task_messages import (
|
||||||
RT_ALARMS_SYNC_IN_PROGRESS_SKIP,
|
RT_ALARMS_SYNC_IN_PROGRESS_SKIP,
|
||||||
RT_KEEPALIVE_FAILED,
|
RT_KEEPALIVE_FAILED,
|
||||||
RT_OCLAW_FWD_DISABLED,
|
|
||||||
RT_PULLING_ALARMS_CURRENT,
|
RT_PULLING_ALARMS_CURRENT,
|
||||||
RT_PULLING_INVENTORY,
|
RT_PULLING_INVENTORY,
|
||||||
RT_RESUMED,
|
RT_RESUMED,
|
||||||
RT_RESUMED_OCLAW_WSS_RECONNECT,
|
|
||||||
RT_RESUMED_SYNC_SOON,
|
RT_RESUMED_SYNC_SOON,
|
||||||
RT_RESUMED_WSS_RECONNECT,
|
RT_RESUMED_WSS_RECONNECT,
|
||||||
RT_STARTUP_ALARM_SYNC_BEFORE_WS,
|
RT_STARTUP_ALARM_SYNC_BEFORE_WS,
|
||||||
|
|
@ -30,7 +28,6 @@ from .runtime_task_messages import (
|
||||||
RT_UME_WS_DISABLED_NO_BASE_URL,
|
RT_UME_WS_DISABLED_NO_BASE_URL,
|
||||||
RT_WSS_ACTIVE_SKIP_REST,
|
RT_WSS_ACTIVE_SKIP_REST,
|
||||||
)
|
)
|
||||||
from .oclaw_alarm_forwarder import is_forwarder_enabled
|
|
||||||
from .ume_alarm_ws import (
|
from .ume_alarm_ws import (
|
||||||
begin_startup_alarm_sync_gate,
|
begin_startup_alarm_sync_gate,
|
||||||
complete_startup_alarm_sync_gate,
|
complete_startup_alarm_sync_gate,
|
||||||
|
|
@ -64,7 +61,6 @@ _UME_RUNTIME_TASKS: dict[str, dict[str, Any]] = {
|
||||||
"token_keepalive": {"task": "token_keepalive", "status": "init", "last_run_at": None, "last_error": ""},
|
"token_keepalive": {"task": "token_keepalive", "status": "init", "last_run_at": None, "last_error": ""},
|
||||||
"alarms_current_auto_sync": {"task": "alarms_current_auto_sync", "status": "init", "last_run_at": None, "last_error": ""},
|
"alarms_current_auto_sync": {"task": "alarms_current_auto_sync", "status": "init", "last_run_at": None, "last_error": ""},
|
||||||
"alarms_current_ws_consumer": {"task": "alarms_current_ws_consumer", "status": "init", "last_run_at": None, "last_error": ""},
|
"alarms_current_ws_consumer": {"task": "alarms_current_ws_consumer", "status": "init", "last_run_at": None, "last_error": ""},
|
||||||
"oclaw_alarm_forwarder": {"task": "oclaw_alarm_forwarder", "status": "init", "last_run_at": None, "last_error": ""},
|
|
||||||
"inventory_auto_sync": {"task": "inventory_auto_sync", "status": "init", "last_run_at": None, "last_error": ""},
|
"inventory_auto_sync": {"task": "inventory_auto_sync", "status": "init", "last_run_at": None, "last_error": ""},
|
||||||
"topology_auto_sync": {"task": "topology_auto_sync", "status": "init", "last_run_at": None, "last_error": ""},
|
"topology_auto_sync": {"task": "topology_auto_sync", "status": "init", "last_run_at": None, "last_error": ""},
|
||||||
}
|
}
|
||||||
|
|
@ -175,10 +171,6 @@ def _runtime_task_interval_fields(task_id: str) -> tuple[int | None, str]:
|
||||||
if not bool(getattr(settings, "ume_alarm_ws_enabled", True)):
|
if not bool(getattr(settings, "ume_alarm_ws_enabled", True)):
|
||||||
return None, "disabled"
|
return None, "disabled"
|
||||||
return None, "realtime"
|
return None, "realtime"
|
||||||
if task_id == "oclaw_alarm_forwarder":
|
|
||||||
if not is_forwarder_enabled():
|
|
||||||
return None, "disabled"
|
|
||||||
return None, "realtime"
|
|
||||||
if task_id == "inventory_auto_sync":
|
if task_id == "inventory_auto_sync":
|
||||||
if not bool(getattr(settings, "ume_sync_inventory_auto_enabled", True)):
|
if not bool(getattr(settings, "ume_sync_inventory_auto_enabled", True)):
|
||||||
return None, "disabled"
|
return None, "disabled"
|
||||||
|
|
|
||||||
|
|
@ -11,10 +11,8 @@ from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from .db import get_db
|
from .db import get_db
|
||||||
from .models import UmeSyncJob
|
from .models import UmeSyncJob
|
||||||
from .oclaw_alarm_forwarder import request_forwarder_reconnect
|
|
||||||
from .runtime_task_messages import (
|
from .runtime_task_messages import (
|
||||||
RT_RESUMED,
|
RT_RESUMED,
|
||||||
RT_RESUMED_OCLAW_WSS_RECONNECT,
|
|
||||||
RT_RESUMED_SYNC_SOON,
|
RT_RESUMED_SYNC_SOON,
|
||||||
RT_RESUMED_WSS_RECONNECT,
|
RT_RESUMED_WSS_RECONNECT,
|
||||||
)
|
)
|
||||||
|
|
@ -236,8 +234,6 @@ def ume_runtime_task_pause(task: str) -> dict[str, Any]:
|
||||||
_clear_force_resume_hints(tid)
|
_clear_force_resume_hints(tid)
|
||||||
if tid == "alarms_current_ws_consumer":
|
if tid == "alarms_current_ws_consumer":
|
||||||
request_ws_reconnect()
|
request_ws_reconnect()
|
||||||
if tid == "oclaw_alarm_forwarder":
|
|
||||||
request_forwarder_reconnect()
|
|
||||||
_set_runtime_task(tid, status="paused", last_error="")
|
_set_runtime_task(tid, status="paused", last_error="")
|
||||||
return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()}
|
return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()}
|
||||||
|
|
||||||
|
|
@ -254,9 +250,6 @@ def ume_runtime_task_resume(task: str) -> dict[str, Any]:
|
||||||
elif tid == "alarms_current_ws_consumer":
|
elif tid == "alarms_current_ws_consumer":
|
||||||
request_ws_reconnect()
|
request_ws_reconnect()
|
||||||
resume_hint = RT_RESUMED_WSS_RECONNECT
|
resume_hint = RT_RESUMED_WSS_RECONNECT
|
||||||
elif tid == "oclaw_alarm_forwarder":
|
|
||||||
request_forwarder_reconnect()
|
|
||||||
resume_hint = RT_RESUMED_OCLAW_WSS_RECONNECT
|
|
||||||
else:
|
else:
|
||||||
resume_hint = RT_RESUMED
|
resume_hint = RT_RESUMED
|
||||||
_set_runtime_task(tid, status="running", last_error=resume_hint)
|
_set_runtime_task(tid, status="running", last_error=resume_hint)
|
||||||
|
|
|
||||||
|
|
@ -34,10 +34,6 @@ from .models import (
|
||||||
UmeKeyAlertRule,
|
UmeKeyAlertRule,
|
||||||
UmeSyncJob,
|
UmeSyncJob,
|
||||||
)
|
)
|
||||||
from .oclaw_alarm_forwarder import (
|
|
||||||
forwarder_status,
|
|
||||||
request_forwarder_reconnect,
|
|
||||||
)
|
|
||||||
from .ume_alarm_ws import (
|
from .ume_alarm_ws import (
|
||||||
cancel_alarm_subscription_manual,
|
cancel_alarm_subscription_manual,
|
||||||
clear_local_alarm_subscription_manual,
|
clear_local_alarm_subscription_manual,
|
||||||
|
|
|
||||||
|
|
@ -71,7 +71,6 @@ class StabilityHardeningTests(unittest.TestCase):
|
||||||
self.assertIn("netx_uptime_seconds", body)
|
self.assertIn("netx_uptime_seconds", body)
|
||||||
self.assertIn("netx_thread_count", body)
|
self.assertIn("netx_thread_count", body)
|
||||||
self.assertIn("netx_cli_budget_limit", body)
|
self.assertIn("netx_cli_budget_limit", body)
|
||||||
self.assertIn("netx_oclaw_forwarder_dropped", body)
|
|
||||||
self.assertIn("netx_device_schedulers_stale", body)
|
self.assertIn("netx_device_schedulers_stale", body)
|
||||||
|
|
||||||
def test_log_runtime_budget_does_not_raise(self) -> None:
|
def test_log_runtime_budget_does_not_raise(self) -> None:
|
||||||
|
|
|
||||||
|
|
@ -80,8 +80,8 @@ src/
|
||||||
- API 仅写在 `services/api.ts`
|
- API 仅写在 `services/api.ts`
|
||||||
- Query key 集中在 `constants/queryKeys.ts`
|
- Query key 集中在 `constants/queryKeys.ts`
|
||||||
- 失效缓存用 prefix key(如 `queryKeys.umeSyncStatusAll`)
|
- 失效缓存用 prefix key(如 `queryKeys.umeSyncStatusAll`)
|
||||||
- 顶栏连接状态:`App` 轮询 `GET /v1/integrations/status`(5s),展示 **netx api** 与 **oclaw bridge**(含延迟 / 错误类型)
|
- 顶栏连接状态:`App` 轮询 `GET /v1/integrations/status`(5s),展示 **netx api** 与 **DSH alarm hub**
|
||||||
- 关键告警推送主路径:NetX **DSH alarm hub**(`/v1/integrations/dsh-alarm/ws`)为服务器,Netx Ops 外拨订阅;UME 页展示多订阅连接列表。OClaw 出站 WSS 为遗留旁路(`NETX_OCLAW_ALARM_WS_ENABLED`)
|
- 关键告警推送:NetX **DSH alarm hub**(`/v1/integrations/dsh-alarm/ws`)为服务器,Netx Ops 外拨订阅;UME 页展示多订阅连接列表
|
||||||
|
|
||||||
## 网元管理(独立于 UME)
|
## 网元管理(独立于 UME)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1087,15 +1087,6 @@ const en = {
|
||||||
hubColLastSeen: "last seen",
|
hubColLastSeen: "last seen",
|
||||||
hubEmptyConnections: "No Netx Ops clients are connected to the alarm hub",
|
hubEmptyConnections: "No Netx Ops clients are connected to the alarm hub",
|
||||||
hubPath: "path",
|
hubPath: "path",
|
||||||
legacyOclaw: "Legacy OClaw outbound",
|
|
||||||
ws: "OClaw WSS",
|
|
||||||
wsConnected: "connected",
|
|
||||||
wsDisconnected: "disconnected",
|
|
||||||
wsDisabled: "disabled",
|
|
||||||
wsPaused: "paused",
|
|
||||||
publishedOk: "published ok",
|
|
||||||
publishedFail: "published fail",
|
|
||||||
queue: "queue",
|
|
||||||
matchType: "Match by",
|
matchType: "Match by",
|
||||||
matchNotificationId: "notificationId",
|
matchNotificationId: "notificationId",
|
||||||
matchKeyword: "description keyword",
|
matchKeyword: "description keyword",
|
||||||
|
|
@ -1171,10 +1162,8 @@ const en = {
|
||||||
pulling_topology: "Pulling UME topology (nodes/links)…",
|
pulling_topology: "Pulling UME topology (nodes/links)…",
|
||||||
topology_sync_busy: "Topology sync already running; skipped",
|
topology_sync_busy: "Topology sync already running; skipped",
|
||||||
ume_ws_disabled_no_base_url: "Disabled or UME_BASE_URL not configured",
|
ume_ws_disabled_no_base_url: "Disabled or UME_BASE_URL not configured",
|
||||||
oclaw_fwd_disabled: "Disabled or NETX_OCLAW_ALARM_WS / token / url not configured",
|
|
||||||
resumed_sync_soon: "Resumed: skipping debounce wait, sync soon",
|
resumed_sync_soon: "Resumed: skipping debounce wait, sync soon",
|
||||||
resumed_wss_reconnect: "Resumed: reconnecting WSS soon",
|
resumed_wss_reconnect: "Resumed: reconnecting WSS soon",
|
||||||
resumed_oclaw_wss_reconnect: "Resumed: reconnecting OClaw WSS soon",
|
|
||||||
resumed: "Resumed",
|
resumed: "Resumed",
|
||||||
keepalive_failed: "Token keepalive failed",
|
keepalive_failed: "Token keepalive failed",
|
||||||
},
|
},
|
||||||
|
|
@ -1191,21 +1180,10 @@ const en = {
|
||||||
subscription_lost: "UME subscription lost",
|
subscription_lost: "UME subscription lost",
|
||||||
disabled_no_base_url: "Disabled or UME_BASE_URL not configured",
|
disabled_no_base_url: "Disabled or UME_BASE_URL not configured",
|
||||||
},
|
},
|
||||||
fwdState: {
|
|
||||||
paused: "Paused",
|
|
||||||
disabled: "Disabled",
|
|
||||||
connecting: "Connecting",
|
|
||||||
connected: "Connected",
|
|
||||||
connect_refused: "OClaw gateway down or port 8787 refused",
|
|
||||||
connect_timeout: "Timed out connecting to OClaw gateway",
|
|
||||||
auth_failed: "OClaw netx-bridge auth failed (check shared token)",
|
|
||||||
disconnected: "OClaw WSS disconnected",
|
|
||||||
},
|
|
||||||
runtimeTask: {
|
runtimeTask: {
|
||||||
token_keepalive: "UME token keepalive",
|
token_keepalive: "UME token keepalive",
|
||||||
alarms_current_auto_sync: "Current alarms auto sync",
|
alarms_current_auto_sync: "Current alarms auto sync",
|
||||||
alarms_current_ws_consumer: "UME alarm WSS",
|
alarms_current_ws_consumer: "UME alarm WSS",
|
||||||
oclaw_alarm_forwarder: "OClaw key-alert WSS",
|
|
||||||
inventory_auto_sync: "Inventory auto sync",
|
inventory_auto_sync: "Inventory auto sync",
|
||||||
topology_auto_sync: "Topology auto sync",
|
topology_auto_sync: "Topology auto sync",
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1079,15 +1079,6 @@ const zh = {
|
||||||
hubColLastSeen: "最近心跳",
|
hubColLastSeen: "最近心跳",
|
||||||
hubEmptyConnections: "当前没有 Netx Ops 客户端连入告警 hub",
|
hubEmptyConnections: "当前没有 Netx Ops 客户端连入告警 hub",
|
||||||
hubPath: "路径",
|
hubPath: "路径",
|
||||||
legacyOclaw: "遗留 OClaw 出站",
|
|
||||||
ws: "OClaw WSS",
|
|
||||||
wsConnected: "已连接",
|
|
||||||
wsDisconnected: "未连接",
|
|
||||||
wsDisabled: "未启用",
|
|
||||||
wsPaused: "已暂停",
|
|
||||||
publishedOk: "发布成功",
|
|
||||||
publishedFail: "发布失败",
|
|
||||||
queue: "待发队列",
|
|
||||||
matchType: "匹配方式",
|
matchType: "匹配方式",
|
||||||
matchNotificationId: "告警码 (notificationId)",
|
matchNotificationId: "告警码 (notificationId)",
|
||||||
matchKeyword: "告警描述关键字",
|
matchKeyword: "告警描述关键字",
|
||||||
|
|
@ -1163,10 +1154,8 @@ const zh = {
|
||||||
pulling_topology: "正在拉取 UME 拓扑(节点/链路)…",
|
pulling_topology: "正在拉取 UME 拓扑(节点/链路)…",
|
||||||
topology_sync_busy: "拓扑同步进行中,已跳过",
|
topology_sync_busy: "拓扑同步进行中,已跳过",
|
||||||
ume_ws_disabled_no_base_url: "未启用或未配置 UME_BASE_URL",
|
ume_ws_disabled_no_base_url: "未启用或未配置 UME_BASE_URL",
|
||||||
oclaw_fwd_disabled: "未启用或未配置 NETX_OCLAW_ALARM_WS / token / url",
|
|
||||||
resumed_sync_soon: "已恢复:将跳过本轮周期等待并尽快同步",
|
resumed_sync_soon: "已恢复:将跳过本轮周期等待并尽快同步",
|
||||||
resumed_wss_reconnect: "已恢复:将尽快重连 WSS",
|
resumed_wss_reconnect: "已恢复:将尽快重连 WSS",
|
||||||
resumed_oclaw_wss_reconnect: "已恢复:将尽快重连 OClaw WSS",
|
|
||||||
resumed: "已恢复",
|
resumed: "已恢复",
|
||||||
keepalive_failed: "Token 保活失败",
|
keepalive_failed: "Token 保活失败",
|
||||||
},
|
},
|
||||||
|
|
@ -1183,21 +1172,10 @@ const zh = {
|
||||||
subscription_lost: "UME 订阅已丢失",
|
subscription_lost: "UME 订阅已丢失",
|
||||||
disabled_no_base_url: "未启用或未配置 UME_BASE_URL",
|
disabled_no_base_url: "未启用或未配置 UME_BASE_URL",
|
||||||
},
|
},
|
||||||
fwdState: {
|
|
||||||
paused: "已暂停",
|
|
||||||
disabled: "未启用",
|
|
||||||
connecting: "连接中",
|
|
||||||
connected: "已连接",
|
|
||||||
connect_refused: "OClaw 网关未启动或 8787 拒绝连接",
|
|
||||||
connect_timeout: "连接 OClaw 网关超时",
|
|
||||||
auth_failed: "OClaw netx-bridge 鉴权失败(检查共享 token)",
|
|
||||||
disconnected: "OClaw WSS 已断开",
|
|
||||||
},
|
|
||||||
runtimeTask: {
|
runtimeTask: {
|
||||||
token_keepalive: "UME Token 保活",
|
token_keepalive: "UME Token 保活",
|
||||||
alarms_current_auto_sync: "当前告警定时同步",
|
alarms_current_auto_sync: "当前告警定时同步",
|
||||||
alarms_current_ws_consumer: "UME 告警 WSS",
|
alarms_current_ws_consumer: "UME 告警 WSS",
|
||||||
oclaw_alarm_forwarder: "OClaw 关键告警 WSS",
|
|
||||||
inventory_auto_sync: "Inventory 定时同步",
|
inventory_auto_sync: "Inventory 定时同步",
|
||||||
topology_auto_sync: "拓扑定时同步",
|
topology_auto_sync: "拓扑定时同步",
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -11312,10 +11312,25 @@ option {
|
||||||
max-height: min(78vh, 780px);
|
max-height: min(78vh, 780px);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Modal.Body often uses flex column; default flex-shrink:1 crushes the rules
|
||||||
|
* table to thead-only while the pager still shows total > 0.
|
||||||
|
*/
|
||||||
|
.app-heroui-modal__dialog [data-slot="modal-body"] > *,
|
||||||
|
.app-heroui-modal__dialog .modal__body > * {
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.app-heroui-modal--xl .app-heroui-modal__dialog [data-slot="modal-body"],
|
||||||
|
.app-heroui-modal--xl .app-heroui-modal__dialog .modal__body {
|
||||||
|
max-height: min(86vh, 900px);
|
||||||
|
}
|
||||||
|
|
||||||
.app-heroui-modal .pt-list-table-wrap {
|
.app-heroui-modal .pt-list-table-wrap {
|
||||||
overflow-x: auto;
|
overflow-x: auto;
|
||||||
max-width: 100%;
|
max-width: 100%;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
|
min-height: 10rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.app-heroui-modal .pt-list-table {
|
.app-heroui-modal .pt-list-table {
|
||||||
|
|
|
||||||
|
|
@ -437,21 +437,11 @@ export function UmePage() {
|
||||||
const keyAlertHub = keyAlertMonitorQuery.data?.dsh_alarm_hub;
|
const keyAlertHub = keyAlertMonitorQuery.data?.dsh_alarm_hub;
|
||||||
const keyAlertHubConnections = keyAlertHub?.connections || [];
|
const keyAlertHubConnections = keyAlertHub?.connections || [];
|
||||||
const keyAlertHubSubscribers = Number(keyAlertHub?.subscribers || keyAlertHubConnections.length || 0);
|
const keyAlertHubSubscribers = Number(keyAlertHub?.subscribers || keyAlertHubConnections.length || 0);
|
||||||
const keyAlertForwarder = keyAlertMonitorQuery.data?.forwarder;
|
|
||||||
const keyAlertRules = keyAlertMonitorQuery.data?.rules || [];
|
const keyAlertRules = keyAlertMonitorQuery.data?.rules || [];
|
||||||
const keyAlertTotal = Number(keyAlertMonitorQuery.data?.total || keyAlertRules.length);
|
const keyAlertTotal = Number(keyAlertMonitorQuery.data?.total || keyAlertRules.length);
|
||||||
const keyAlertPages = pageCount(keyAlertTotal, keyAlertPageSize);
|
const keyAlertPages = pageCount(keyAlertTotal, keyAlertPageSize);
|
||||||
const keyAlertForwardOnClear = Boolean(keyAlertMonitorQuery.data?.config?.forward_on_clear);
|
const keyAlertForwardOnClear = Boolean(keyAlertMonitorQuery.data?.config?.forward_on_clear);
|
||||||
const hubWsPill = keyAlertHubSubscribers > 0 ? "up" : "down";
|
const hubWsPill = keyAlertHubSubscribers > 0 ? "up" : "down";
|
||||||
const oclawWsPill =
|
|
||||||
!keyAlertForwarder?.enabled
|
|
||||||
? "unknown"
|
|
||||||
: keyAlertForwarder.paused
|
|
||||||
? "warn"
|
|
||||||
: keyAlertForwarder.connected
|
|
||||||
? "up"
|
|
||||||
: "down";
|
|
||||||
const showLegacyOclaw = Boolean(keyAlertForwarder?.enabled);
|
|
||||||
|
|
||||||
const runtimeTaskLabel = (task: string) => {
|
const runtimeTaskLabel = (task: string) => {
|
||||||
const key = `ume.tasks.runtimeTask.${task}`;
|
const key = `ume.tasks.runtimeTask.${task}`;
|
||||||
|
|
@ -878,16 +868,6 @@ export function UmePage() {
|
||||||
<span className="conn-pill">
|
<span className="conn-pill">
|
||||||
{t("ume.keyAlert.hubDeliverOk")}: {Number(keyAlertHub?.deliver_ok || 0)}
|
{t("ume.keyAlert.hubDeliverOk")}: {Number(keyAlertHub?.deliver_ok || 0)}
|
||||||
</span>
|
</span>
|
||||||
{showLegacyOclaw ? (
|
|
||||||
<span className={`conn-pill conn-pill--${oclawWsPill}`} title={t("ume.keyAlert.legacyOclaw")}>
|
|
||||||
{t("ume.keyAlert.ws")}:{" "}
|
|
||||||
{keyAlertForwarder?.paused
|
|
||||||
? t("ume.keyAlert.wsPaused")
|
|
||||||
: keyAlertForwarder?.connected
|
|
||||||
? t("ume.keyAlert.wsConnected")
|
|
||||||
: t("ume.keyAlert.wsDisconnected")}
|
|
||||||
</span>
|
|
||||||
) : null}
|
|
||||||
</div>
|
</div>
|
||||||
</article>
|
</article>
|
||||||
|
|
||||||
|
|
@ -984,15 +964,6 @@ export function UmePage() {
|
||||||
{` · ${t("ume.keyAlert.hubPublished")} ${Number(keyAlertHub?.published || 0)}`}
|
{` · ${t("ume.keyAlert.hubPublished")} ${Number(keyAlertHub?.published || 0)}`}
|
||||||
{` · ${t("ume.keyAlert.hubDeliverOk")} ${Number(keyAlertHub?.deliver_ok || 0)}`}
|
{` · ${t("ume.keyAlert.hubDeliverOk")} ${Number(keyAlertHub?.deliver_ok || 0)}`}
|
||||||
{keyAlertHub?.path ? ` · ${t("ume.keyAlert.hubPath")} ${keyAlertHub.path}` : ""}
|
{keyAlertHub?.path ? ` · ${t("ume.keyAlert.hubPath")} ${keyAlertHub.path}` : ""}
|
||||||
{showLegacyOclaw
|
|
||||||
? ` · ${t("ume.keyAlert.legacyOclaw")}: ${
|
|
||||||
keyAlertForwarder?.paused
|
|
||||||
? t("ume.keyAlert.wsPaused")
|
|
||||||
: keyAlertForwarder?.connected
|
|
||||||
? t("ume.keyAlert.wsConnected")
|
|
||||||
: t("ume.keyAlert.wsDisconnected")
|
|
||||||
}`
|
|
||||||
: ""}
|
|
||||||
</p>
|
</p>
|
||||||
<div className="ume-modal-form">
|
<div className="ume-modal-form">
|
||||||
<div className="muted ume-modal-form__label">{t("ume.keyAlert.hubConnections")}</div>
|
<div className="muted ume-modal-form__label">{t("ume.keyAlert.hubConnections")}</div>
|
||||||
|
|
@ -1036,18 +1007,6 @@ export function UmePage() {
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{showLegacyOclaw ? (
|
|
||||||
<p className="muted ume-modal-form__hint">
|
|
||||||
{t("ume.keyAlert.legacyOclaw")}:{" "}
|
|
||||||
{keyAlertForwarder?.paused
|
|
||||||
? t("ume.keyAlert.wsPaused")
|
|
||||||
: keyAlertForwarder?.connected
|
|
||||||
? t("ume.keyAlert.wsConnected")
|
|
||||||
: t("ume.keyAlert.wsDisconnected")}
|
|
||||||
{` · ${t("ume.keyAlert.publishedOk")} ${Number(keyAlertForwarder?.published_ok || 0)}`}
|
|
||||||
{` · ${t("ume.keyAlert.queue")} ${Number(keyAlertForwarder?.queue_size || 0)}`}
|
|
||||||
</p>
|
|
||||||
) : null}
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="ume-modal-form">
|
<div className="ume-modal-form">
|
||||||
|
|
|
||||||
|
|
@ -21,19 +21,6 @@ export type IntegrationStatus = {
|
||||||
netx_api: { status: "up" | "down" | "unknown"; [k: string]: unknown };
|
netx_api: { status: "up" | "down" | "unknown"; [k: string]: unknown };
|
||||||
db: { status: "up" | "down" | "unknown"; latency_ms?: number; error?: string; [k: string]: unknown };
|
db: { status: "up" | "down" | "unknown"; latency_ms?: number; error?: string; [k: string]: unknown };
|
||||||
dsh_alarm_hub?: DshAlarmHubStatus;
|
dsh_alarm_hub?: DshAlarmHubStatus;
|
||||||
oclaw_bridge?: {
|
|
||||||
status: "up" | "down" | "unknown";
|
|
||||||
mode?: string;
|
|
||||||
enabled?: boolean;
|
|
||||||
connected?: boolean;
|
|
||||||
queue_size?: number;
|
|
||||||
published_ok?: number;
|
|
||||||
published_fail?: number;
|
|
||||||
latency_ms?: number;
|
|
||||||
error_kind?: string;
|
|
||||||
error?: string;
|
|
||||||
[k: string]: unknown;
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export type UmeKeyAlertRuleItem = {
|
export type UmeKeyAlertRuleItem = {
|
||||||
|
|
@ -57,18 +44,6 @@ export type UmeInventoryNeTypeItem = {
|
||||||
ne_count: number;
|
ne_count: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type UmeKeyAlertForwarderStatus = {
|
|
||||||
enabled: boolean;
|
|
||||||
operational?: boolean;
|
|
||||||
paused?: boolean;
|
|
||||||
connected: boolean;
|
|
||||||
queue_size: number;
|
|
||||||
url: string;
|
|
||||||
published_ok?: number;
|
|
||||||
published_fail?: number;
|
|
||||||
queued_total?: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type UmeKeyAlertMonitorResponse = {
|
export type UmeKeyAlertMonitorResponse = {
|
||||||
ok: boolean;
|
ok: boolean;
|
||||||
rules: UmeKeyAlertRuleItem[];
|
rules: UmeKeyAlertRuleItem[];
|
||||||
|
|
@ -78,10 +53,8 @@ export type UmeKeyAlertMonitorResponse = {
|
||||||
config?: {
|
config?: {
|
||||||
forward_on_clear: boolean;
|
forward_on_clear: boolean;
|
||||||
};
|
};
|
||||||
/** Primary: NetX hub ← netxops clients (multi-subscriber). */
|
/** NetX hub ← netxops clients (multi-subscriber). */
|
||||||
dsh_alarm_hub?: DshAlarmHubStatus;
|
dsh_alarm_hub?: DshAlarmHubStatus;
|
||||||
/** Legacy: NetX → OClaw outbound bridge (single link). */
|
|
||||||
forwarder: UmeKeyAlertForwarderStatus;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export type UmeSyncJobItem = {
|
export type UmeSyncJobItem = {
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
/** Map backend runtime status codes (rt:* / ws:* / fwd:*) and legacy Chinese to i18n keys. */
|
/** Map backend runtime status codes (rt:* / ws:*) and legacy Chinese to i18n keys. */
|
||||||
|
|
||||||
type TFunc = (key: string) => string;
|
type TFunc = (key: string) => string;
|
||||||
|
|
||||||
|
|
@ -11,10 +11,8 @@ const LEGACY_ZH_RUNTIME_ERROR: Record<string, string> = {
|
||||||
"正在拉取 UME 网元清单…": "ume.tasks.runtimeError.pulling_inventory",
|
"正在拉取 UME 网元清单…": "ume.tasks.runtimeError.pulling_inventory",
|
||||||
"正在拉取 UME 拓扑(节点/链路)…": "ume.tasks.runtimeError.pulling_topology",
|
"正在拉取 UME 拓扑(节点/链路)…": "ume.tasks.runtimeError.pulling_topology",
|
||||||
"未启用或未配置 UME_BASE_URL": "ume.tasks.runtimeError.ume_ws_disabled_no_base_url",
|
"未启用或未配置 UME_BASE_URL": "ume.tasks.runtimeError.ume_ws_disabled_no_base_url",
|
||||||
"未启用或未配置 NETX_OCLAW_ALARM_WS / token / url": "ume.tasks.runtimeError.oclaw_fwd_disabled",
|
|
||||||
"已恢复:将跳过本轮周期等待并尽快同步": "ume.tasks.runtimeError.resumed_sync_soon",
|
"已恢复:将跳过本轮周期等待并尽快同步": "ume.tasks.runtimeError.resumed_sync_soon",
|
||||||
"已恢复:将尽快重连 WSS": "ume.tasks.runtimeError.resumed_wss_reconnect",
|
"已恢复:将尽快重连 WSS": "ume.tasks.runtimeError.resumed_wss_reconnect",
|
||||||
"已恢复:将尽快重连 OClaw WSS": "ume.tasks.runtimeError.resumed_oclaw_wss_reconnect",
|
|
||||||
已恢复: "ume.tasks.runtimeError.resumed",
|
已恢复: "ume.tasks.runtimeError.resumed",
|
||||||
初始化: "ume.tasks.wsState.init",
|
初始化: "ume.tasks.wsState.init",
|
||||||
已连接: "ume.tasks.wsState.connected",
|
已连接: "ume.tasks.wsState.connected",
|
||||||
|
|
@ -45,8 +43,6 @@ export function runtimeLastError(raw: string | null | undefined, t: TFunc): stri
|
||||||
if (rt) return rt;
|
if (rt) return rt;
|
||||||
const ws = translatePrefixed(s, "ws:", "ume.tasks.wsState", t);
|
const ws = translatePrefixed(s, "ws:", "ume.tasks.wsState", t);
|
||||||
if (ws) return ws;
|
if (ws) return ws;
|
||||||
const fwd = translatePrefixed(s, "fwd:", "ume.tasks.fwdState", t);
|
|
||||||
if (fwd) return fwd;
|
|
||||||
|
|
||||||
if (s === "keepalive_failed") {
|
if (s === "keepalive_failed") {
|
||||||
const tr = t("ume.tasks.runtimeError.keepalive_failed");
|
const tr = t("ume.tasks.runtimeError.keepalive_failed");
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue