mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 03:10:46 +08:00
Extract shared brownfield patches, add the legacy revision, and default to upgrade-head plus skip of duplicate inline DDL (with legacy fallback if Alembic fails). Co-authored-by: Cursor <cursoragent@cursor.com>
668 lines
27 KiB
Python
668 lines
27 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timezone
|
|
from io import StringIO
|
|
import time
|
|
import re
|
|
import threading
|
|
|
|
_schedule_log = logging.getLogger("netx.ume.schedule")
|
|
_BOOT_MONO = time.monotonic()
|
|
from fastapi import Depends, FastAPI, File, HTTPException, Query, UploadFile
|
|
from fastapi.responses import Response
|
|
from sqlalchemy import text as sql_text
|
|
from sqlalchemy.orm import Session
|
|
from typing import Any
|
|
import uvicorn
|
|
|
|
from .ap_client import analyze_with_oclaw, health_with_oclaw
|
|
from .auth_middleware import AuthAuditMiddleware
|
|
from .auth_router import router as auth_router
|
|
from .auth_service import bootstrap_admin_if_needed
|
|
from .config import settings
|
|
from .db import Base, SessionLocal, engine, get_db
|
|
from .collection_router import router as collection_router
|
|
from .cli_router import router as cli_router
|
|
from .config_sync_router import router as config_sync_router
|
|
from .port_traffic_router import router as port_traffic_router
|
|
from .managed_ne_router import router as managed_ne_router
|
|
from .webcrt_router import router as webcrt_router
|
|
from .topology_router import router as topology_router
|
|
from .lldp_collect_router import router as lldp_collect_router
|
|
from .ops_router import router as ops_router
|
|
from .sql_router import router as sql_router
|
|
from .sql_router import sql_query, sql_ume_query # noqa: F401 — tests import from main
|
|
from .security_bootstrap import assert_secure_defaults_or_exit
|
|
from .integrations_router import router as integrations_router
|
|
from .ume_router import router as ume_router
|
|
from .alarms_router import router as alarms_router
|
|
from .ume_router import ( # noqa: F401 — tests import from main
|
|
_extract_ume_raw_group_field,
|
|
_serialize_ume_alarm_raw_row,
|
|
ume_alarms_fields,
|
|
)
|
|
from .ume_support import ( # noqa: F401 — tests import from main
|
|
_classify_protocol_bucket,
|
|
_protocol_bucket_label,
|
|
)
|
|
import netx_api.ume_support as ume_support
|
|
from .ume_runtime import start_device_schedulers
|
|
from .importer import aggregate_alarms, import_alarm_excel, query_alarms
|
|
from .models import (
|
|
AiAnalyzeHistory,
|
|
AlarmBatch,
|
|
AlarmNorm,
|
|
ApiToken,
|
|
AppUser,
|
|
AuditLog,
|
|
ImportErrorRow,
|
|
ManagedNE,
|
|
NeCollectionJob,
|
|
NeCollectionRun,
|
|
UmeAlarmCurrent,
|
|
UmeAlarmHistory,
|
|
UmeInventoryNE,
|
|
UmeKeyAlertRule,
|
|
UmeKeyAlertForwardLog,
|
|
UmeSyncJob,
|
|
)
|
|
from .models import ImportJob
|
|
from .parser_config import load_parser_config
|
|
from .ume_client import UMEClient
|
|
from .ume_alarm_ws import (
|
|
begin_startup_alarm_sync_gate,
|
|
cancel_alarm_subscription_manual,
|
|
clear_local_alarm_subscription_manual,
|
|
complete_startup_alarm_sync_gate,
|
|
establish_alarm_subscription_manual,
|
|
get_alarms_coordination_status,
|
|
get_subscription_status,
|
|
get_ws_connection_status,
|
|
get_ws_logs,
|
|
is_startup_alarm_sync_pending,
|
|
is_wss_active_for_current_alarms,
|
|
load_persisted_subscription,
|
|
request_ws_reconnect,
|
|
shutdown_ws_consumer,
|
|
start_ume_alarm_ws_consumer,
|
|
)
|
|
from .ume_sync_service import sync_alarms_current, sync_alarms_history_full, sync_inventory_full
|
|
from .runtime_task_messages import (
|
|
RT_ALARMS_SYNC_IN_PROGRESS_SKIP,
|
|
RT_OCLAW_FWD_DISABLED,
|
|
RT_PULLING_ALARMS_CURRENT,
|
|
RT_PULLING_INVENTORY,
|
|
RT_RESUMED,
|
|
RT_RESUMED_OCLAW_WSS_RECONNECT,
|
|
RT_RESUMED_SYNC_SOON,
|
|
RT_RESUMED_WSS_RECONNECT,
|
|
RT_STARTUP_ALARM_SYNC_BEFORE_WS,
|
|
RT_STARTUP_GATE_WAITING,
|
|
RT_KEEPALIVE_FAILED,
|
|
RT_UME_WS_DISABLED_NO_BASE_URL,
|
|
RT_WSS_ACTIVE_SKIP_REST,
|
|
)
|
|
from .oclaw_alarm_forwarder import (
|
|
forwarder_status,
|
|
is_forwarder_enabled,
|
|
request_forwarder_reconnect,
|
|
configure_oclaw_alarm_forwarder,
|
|
shutdown_oclaw_alarm_forwarder,
|
|
start_oclaw_alarm_forwarder,
|
|
)
|
|
from .ume_token_store import (
|
|
clear_shared_token,
|
|
load_shared_token,
|
|
release_refresh_lock,
|
|
save_shared_token,
|
|
try_acquire_refresh_lock,
|
|
wait_for_token_update,
|
|
)
|
|
from .schemas import (
|
|
AlarmAggregateBucket,
|
|
AlarmAggregateResponse,
|
|
AiAnalyzeHistoryItem,
|
|
AiAnalyzeHistoryResponse,
|
|
AlarmItem,
|
|
AlarmQueryResponse,
|
|
BatchSummary,
|
|
ImportJobItem,
|
|
ImportJobListResponse,
|
|
)
|
|
|
|
app = FastAPI(
|
|
title="netx ops tool",
|
|
version="0.1.0",
|
|
docs_url="/docs" if bool(settings.docs_enabled) else None,
|
|
redoc_url="/redoc" if bool(settings.docs_enabled) else None,
|
|
openapi_url="/openapi.json" if bool(settings.docs_enabled) else None,
|
|
)
|
|
app.add_middleware(AuthAuditMiddleware)
|
|
app.include_router(auth_router)
|
|
app.include_router(managed_ne_router)
|
|
app.include_router(cli_router)
|
|
app.include_router(collection_router)
|
|
app.include_router(config_sync_router)
|
|
app.include_router(port_traffic_router)
|
|
app.include_router(webcrt_router)
|
|
app.include_router(topology_router)
|
|
app.include_router(lldp_collect_router)
|
|
app.include_router(ops_router)
|
|
app.include_router(sql_router)
|
|
app.include_router(integrations_router)
|
|
app.include_router(ume_router)
|
|
app.include_router(alarms_router)
|
|
parser_cfg = load_parser_config()
|
|
def _configure_ume_diag_logging() -> None:
|
|
"""Emit netx.ume.* INFO to stderr so background scripts/.run/*.log and consoles show scheduler lines."""
|
|
fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
for name in ("netx.ume.schedule", "netx.ume.sync"):
|
|
lg = logging.getLogger(name)
|
|
if lg.handlers:
|
|
continue
|
|
h = logging.StreamHandler()
|
|
h.setFormatter(fmt)
|
|
lg.addHandler(h)
|
|
lg.setLevel(logging.INFO)
|
|
lg.propagate = False
|
|
|
|
|
|
@app.on_event("startup")
|
|
def on_startup() -> None:
|
|
assert_secure_defaults_or_exit()
|
|
_configure_ume_diag_logging()
|
|
Base.metadata.create_all(bind=engine)
|
|
from .schema_patches import (
|
|
apply_all_legacy_startup_ddl,
|
|
apply_auth_schema_patches,
|
|
run_alembic_upgrade_to_head,
|
|
)
|
|
|
|
alembic_ok = True
|
|
if bool(getattr(settings, "alembic_upgrade_on_start", True)):
|
|
try:
|
|
run_alembic_upgrade_to_head()
|
|
except Exception:
|
|
alembic_ok = False
|
|
_schedule_log.exception("startup: alembic upgrade head failed")
|
|
|
|
skip_ddl = bool(getattr(settings, "skip_legacy_startup_ddl", True))
|
|
# Auth columns must exist before bootstrap even when legacy DDL is skipped.
|
|
try:
|
|
with engine.begin() as conn:
|
|
apply_auth_schema_patches(conn)
|
|
except Exception:
|
|
_schedule_log.exception("startup: auth schema patches failed")
|
|
if skip_ddl and alembic_ok:
|
|
_schedule_log.info("startup: schema via Alembic (legacy inline DDL skipped)")
|
|
else:
|
|
if skip_ddl and not alembic_ok:
|
|
_schedule_log.warning(
|
|
"startup: Alembic failed — falling back to legacy schema patches"
|
|
)
|
|
try:
|
|
apply_all_legacy_startup_ddl(engine)
|
|
except Exception:
|
|
_schedule_log.exception("startup: legacy schema patches failed")
|
|
ume_support._reset_runtime_pause_flags()
|
|
ume_support._fail_stale_running_sync_jobs_on_startup()
|
|
try:
|
|
from .topology_service import bootstrap_topology_tree, reclaim_stale_discover_jobs
|
|
|
|
db_topo = SessionLocal()
|
|
try:
|
|
bootstrap_topology_tree(db_topo)
|
|
closed = reclaim_stale_discover_jobs(db_topo, force_all_open=True)
|
|
if closed:
|
|
_schedule_log.warning(
|
|
"startup: closed %s orphaned topology discover jobs", closed
|
|
)
|
|
finally:
|
|
db_topo.close()
|
|
except Exception:
|
|
_schedule_log.exception("startup: topology discover job cleanup failed")
|
|
if ume_support._needs_startup_alarm_sync_before_ws():
|
|
begin_startup_alarm_sync_gate()
|
|
_schedule_log.info(
|
|
"startup: WSS blocked until initial REST current-alarm sync completes (delay=%ss)",
|
|
ume_support._startup_alarm_pull_delay_s(),
|
|
)
|
|
else:
|
|
complete_startup_alarm_sync_gate()
|
|
db = SessionLocal()
|
|
try:
|
|
try:
|
|
bootstrap_admin_if_needed(db)
|
|
except Exception:
|
|
_schedule_log.exception("startup: auth bootstrap admin failed")
|
|
from .collection_recovery import recover_collection_jobs_on_startup
|
|
|
|
resumed = recover_collection_jobs_on_startup(db)
|
|
if resumed:
|
|
_schedule_log.info("startup: resumed %s pending ne collection runs", resumed)
|
|
from .config_sync_recovery import recover_config_sync_on_startup
|
|
from .config_sync_service import ensure_policy
|
|
from .port_traffic_recovery import recover_port_traffic_on_startup
|
|
|
|
ensure_policy(db)
|
|
cfg_resumed = recover_config_sync_on_startup(db)
|
|
if cfg_resumed:
|
|
_schedule_log.info("startup: resumed %s config_sync task(s) from interrupted cycle", cfg_resumed)
|
|
try:
|
|
from .lldp_collect_service import ensure_policy as ensure_lldp_collect_policy
|
|
|
|
ensure_lldp_collect_policy(db)
|
|
except Exception:
|
|
_schedule_log.exception("startup: lldp_collect policy ensure failed")
|
|
pt_cleared = recover_port_traffic_on_startup(db)
|
|
if pt_cleared:
|
|
_schedule_log.info("startup: cleared %s port_traffic stuck collect_running flag(s)", pt_cleared)
|
|
try:
|
|
from .port_traffic_migrate import backfill_port_traffic_series
|
|
|
|
backfill_port_traffic_series(db)
|
|
except Exception:
|
|
_schedule_log.exception("startup: port_traffic series backfill failed")
|
|
except Exception:
|
|
_schedule_log.exception("startup: ne collection / config_sync recovery failed")
|
|
finally:
|
|
db.close()
|
|
if bool(getattr(settings, "run_inline_schedulers", True)):
|
|
try:
|
|
start_device_schedulers()
|
|
except Exception:
|
|
_schedule_log.exception("startup: device schedulers init failed")
|
|
else:
|
|
_schedule_log.info(
|
|
"startup: inline schedulers disabled — run `python -m netx_api.worker` for "
|
|
"config_sync / lldp_collect / port_traffic"
|
|
)
|
|
try:
|
|
if bool(getattr(settings, "ume_keepalive_enabled", True)):
|
|
interval_keepalive_s = int(getattr(settings, "ume_keepalive_interval_s", 600) or 600)
|
|
interval_keepalive_s = max(30, min(interval_keepalive_s, 3600))
|
|
renew_before_s = int(getattr(settings, "ume_keepalive_renew_before_s", 900) or 900)
|
|
renew_before_s = max(30, min(renew_before_s, 86400))
|
|
|
|
def _keepalive_loop() -> None:
|
|
# Best-effort keepalive: if token exists, periodically handshake to extend TTL.
|
|
while True:
|
|
try:
|
|
if ume_support._runtime_is_paused("token_keepalive"):
|
|
time.sleep(1)
|
|
continue
|
|
client = ume_support._ume_client()
|
|
st = client.token_status()
|
|
expires_in = int(st.get("expires_in_s") or 0)
|
|
# Renew when missing/invalid TTL (0) or nearing expiry — previously 0 skipped renew forever.
|
|
if bool(st.get("has_token")) and (expires_in <= 0 or expires_in < renew_before_s):
|
|
client.renew_token()
|
|
ume_support._set_runtime_task("token_keepalive", status="running", last_run_at=datetime.now(timezone.utc), last_error="")
|
|
except Exception:
|
|
ume_support._set_runtime_task("token_keepalive", status="error", last_run_at=datetime.now(timezone.utc), last_error=RT_KEEPALIVE_FAILED)
|
|
time.sleep(interval_keepalive_s)
|
|
|
|
t = threading.Thread(target=_keepalive_loop, name="ume-token-keepalive", daemon=True)
|
|
t.start()
|
|
except Exception as exc:
|
|
_schedule_log.exception("startup: token_keepalive thread init failed: %s", exc)
|
|
ume_support._set_runtime_task(
|
|
"token_keepalive",
|
|
status="error",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
|
|
)
|
|
try:
|
|
|
|
def _startup_alarm_sync_worker() -> None:
|
|
try:
|
|
ume_support._run_startup_alarm_sync_before_ws()
|
|
except Exception as exc:
|
|
_schedule_log.exception("startup: alarm sync before WSS failed: %s", exc)
|
|
complete_startup_alarm_sync_gate()
|
|
|
|
# Do not block HTTP /health on slow UME REST pull; WSS waits on startup_alarm_sync_gate.
|
|
t_startup_sync = threading.Thread(
|
|
target=_startup_alarm_sync_worker,
|
|
name="ume-startup-alarm-sync",
|
|
daemon=True,
|
|
)
|
|
t_startup_sync.start()
|
|
except Exception as exc:
|
|
_schedule_log.exception("startup: alarm sync thread init failed: %s", exc)
|
|
complete_startup_alarm_sync_gate()
|
|
try:
|
|
if bool(getattr(settings, "ume_sync_alarms_current_enabled", True)):
|
|
alarms_interval_s = int(getattr(settings, "ume_sync_alarms_current_interval_s", 18000) or 18000)
|
|
alarms_interval_s = max(30, min(alarms_interval_s, 86400))
|
|
|
|
def _alarms_current_sync_loop() -> None:
|
|
ume_support._refresh_runtime_task_idle("alarms_current_auto_sync", "alarms_current")
|
|
ume_support._wait_until_startup_alarm_pull_allowed("alarms_current_auto_sync")
|
|
while True:
|
|
try:
|
|
_schedule_log.info(
|
|
"alarms_current_auto_sync: loop tick paused=%s",
|
|
ume_support._runtime_is_paused("alarms_current_auto_sync"),
|
|
)
|
|
if ume_support._runtime_is_paused("alarms_current_auto_sync"):
|
|
time.sleep(1)
|
|
continue
|
|
if is_startup_alarm_sync_pending():
|
|
ume_support._refresh_runtime_task_idle(
|
|
"alarms_current_auto_sync",
|
|
"alarms_current",
|
|
last_error=RT_STARTUP_GATE_WAITING,
|
|
)
|
|
time.sleep(10)
|
|
continue
|
|
if (
|
|
bool(getattr(settings, "ume_sync_alarms_current_skip_when_ws", True))
|
|
and is_wss_active_for_current_alarms()
|
|
):
|
|
ume_support._refresh_runtime_task_idle(
|
|
"alarms_current_auto_sync",
|
|
"alarms_current",
|
|
last_error=RT_WSS_ACTIVE_SKIP_REST,
|
|
)
|
|
time.sleep(max(30, min(alarms_interval_s, 300)))
|
|
continue
|
|
ume_support._maybe_wait_for_sync_interval(
|
|
task_id="alarms_current_auto_sync",
|
|
domain="alarms_current",
|
|
interval_s=alarms_interval_s,
|
|
label="alarms_current_auto_sync",
|
|
)
|
|
_schedule_log.info(
|
|
"alarms_current_auto_sync: iteration start (interval=%ss)",
|
|
alarms_interval_s,
|
|
)
|
|
ume_support._set_runtime_task(
|
|
"alarms_current_auto_sync",
|
|
status="running",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=RT_PULLING_ALARMS_CURRENT,
|
|
)
|
|
db = SessionLocal()
|
|
try:
|
|
client = ume_support._ume_client()
|
|
sync_alarms_current(db, client, trigger_mode="schedule")
|
|
_schedule_log.info("alarms_current_auto_sync: sync finished ok")
|
|
ume_support._set_runtime_task(
|
|
"alarms_current_auto_sync",
|
|
status="running",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error="",
|
|
)
|
|
finally:
|
|
db.close()
|
|
except RuntimeError as exc:
|
|
if str(exc) == "alarms_current_sync_busy":
|
|
ume_support._refresh_runtime_task_idle(
|
|
"alarms_current_auto_sync",
|
|
"alarms_current",
|
|
last_error=RT_ALARMS_SYNC_IN_PROGRESS_SKIP,
|
|
)
|
|
time.sleep(30)
|
|
else:
|
|
raise
|
|
except Exception as exc:
|
|
_schedule_log.exception("alarms_current_auto_sync: sync failed: %s", exc)
|
|
ume_support._set_runtime_task(
|
|
"alarms_current_auto_sync",
|
|
status="error",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=str(exc)[:240],
|
|
)
|
|
|
|
t2 = threading.Thread(target=_alarms_current_sync_loop, name="ume-alarms-current-sync", daemon=True)
|
|
t2.start()
|
|
_schedule_log.info("started thread %s alive=%s", t2.name, t2.is_alive())
|
|
if not t2.is_alive():
|
|
_schedule_log.error("ume-alarms-current-sync thread exited immediately (check uncaught errors above)")
|
|
except Exception as exc:
|
|
_schedule_log.exception("startup: alarms_current_auto_sync thread init failed: %s", exc)
|
|
ume_support._set_runtime_task(
|
|
"alarms_current_auto_sync",
|
|
status="error",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
|
|
)
|
|
try:
|
|
if bool(getattr(settings, "ume_sync_inventory_auto_enabled", True)):
|
|
hours = int(getattr(settings, "ume_sync_inventory_every_hours", 48) or 48)
|
|
hours = max(1, min(hours, 168))
|
|
inventory_interval_s = int(hours * 3600)
|
|
ume_support._refresh_runtime_task_idle("inventory_auto_sync", "inventory")
|
|
|
|
def _inventory_auto_sync_loop() -> None:
|
|
ume_support._refresh_runtime_task_idle("inventory_auto_sync", "inventory")
|
|
while True:
|
|
try:
|
|
_schedule_log.info(
|
|
"inventory_auto_sync: loop tick paused=%s",
|
|
ume_support._runtime_is_paused("inventory_auto_sync"),
|
|
)
|
|
if ume_support._runtime_is_paused("inventory_auto_sync"):
|
|
time.sleep(1)
|
|
continue
|
|
ume_support._maybe_wait_for_sync_interval(
|
|
task_id="inventory_auto_sync",
|
|
domain="inventory",
|
|
interval_s=inventory_interval_s,
|
|
label="inventory_auto_sync",
|
|
)
|
|
_schedule_log.info(
|
|
"inventory_auto_sync: iteration start (interval=%ss)",
|
|
inventory_interval_s,
|
|
)
|
|
ume_support._set_runtime_task(
|
|
"inventory_auto_sync",
|
|
status="running",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=RT_PULLING_INVENTORY,
|
|
)
|
|
db = SessionLocal()
|
|
try:
|
|
client = ume_support._ume_client()
|
|
sync_inventory_full(db, client, trigger_mode="schedule")
|
|
_schedule_log.info("inventory_auto_sync: sync finished ok")
|
|
ume_support._set_runtime_task(
|
|
"inventory_auto_sync",
|
|
status="running",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error="",
|
|
)
|
|
finally:
|
|
db.close()
|
|
except Exception as exc:
|
|
_schedule_log.exception("inventory_auto_sync: sync failed: %s", exc)
|
|
ume_support._set_runtime_task(
|
|
"inventory_auto_sync",
|
|
status="error",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=str(exc)[:240],
|
|
)
|
|
|
|
t3 = threading.Thread(target=_inventory_auto_sync_loop, name="ume-inventory-auto-sync", daemon=True)
|
|
t3.start()
|
|
_schedule_log.info("started thread %s alive=%s", t3.name, t3.is_alive())
|
|
if not t3.is_alive():
|
|
_schedule_log.error("ume-inventory-auto-sync thread exited immediately (check uncaught errors above)")
|
|
except Exception as exc:
|
|
_schedule_log.exception("startup: inventory_auto_sync thread init failed: %s", exc)
|
|
ume_support._set_runtime_task(
|
|
"inventory_auto_sync",
|
|
status="error",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
|
|
)
|
|
try:
|
|
if bool(getattr(settings, "ume_alarm_ws_enabled", True)) and str(getattr(settings, "ume_base_url", "") or "").strip():
|
|
if load_persisted_subscription():
|
|
_schedule_log.info("startup: loaded persisted UME alarm subscription")
|
|
ume_support._UME_WS_STOP_EVENT = threading.Event()
|
|
|
|
def _ws_on_status(msg: str) -> None:
|
|
ume_support._set_runtime_task(
|
|
"alarms_current_ws_consumer",
|
|
status="running",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=str(msg or "")[:240],
|
|
)
|
|
|
|
t_ws = start_ume_alarm_ws_consumer(
|
|
ume_support._ume_client(),
|
|
on_status=_ws_on_status,
|
|
stop_event=ume_support._UME_WS_STOP_EVENT,
|
|
is_paused=lambda: ume_support._runtime_is_paused("alarms_current_ws_consumer"),
|
|
)
|
|
_schedule_log.info("started thread %s alive=%s", t_ws.name, t_ws.is_alive())
|
|
else:
|
|
ume_support._set_runtime_task("alarms_current_ws_consumer", status="paused", last_error=RT_UME_WS_DISABLED_NO_BASE_URL)
|
|
except Exception as exc:
|
|
_schedule_log.exception("startup: alarms_current_ws_consumer thread init failed: %s", exc)
|
|
ume_support._set_runtime_task(
|
|
"alarms_current_ws_consumer",
|
|
status="error",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
|
|
)
|
|
try:
|
|
def _fwd_on_status(msg: str) -> None:
|
|
paused = ume_support._runtime_is_paused("oclaw_alarm_forwarder")
|
|
fwd = forwarder_status()
|
|
if paused:
|
|
status = "paused"
|
|
elif not bool(fwd.get("enabled")):
|
|
status = "paused"
|
|
elif bool(fwd.get("connected")):
|
|
status = "running"
|
|
else:
|
|
status = "running"
|
|
ume_support._set_runtime_task(
|
|
"oclaw_alarm_forwarder",
|
|
status=status,
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=str(msg or "")[:240],
|
|
)
|
|
|
|
configure_oclaw_alarm_forwarder(
|
|
is_paused=lambda: ume_support._runtime_is_paused("oclaw_alarm_forwarder"),
|
|
on_status=_fwd_on_status,
|
|
)
|
|
if is_forwarder_enabled():
|
|
ume_support._set_runtime_task("oclaw_alarm_forwarder", status="running", last_error="")
|
|
else:
|
|
ume_support._set_runtime_task(
|
|
"oclaw_alarm_forwarder",
|
|
status="paused",
|
|
last_error=RT_OCLAW_FWD_DISABLED,
|
|
)
|
|
t_fwd = start_oclaw_alarm_forwarder()
|
|
if t_fwd is not None:
|
|
_schedule_log.info("started thread %s alive=%s", t_fwd.name, t_fwd.is_alive())
|
|
except Exception as exc:
|
|
_schedule_log.exception("startup: oclaw_alarm_forwarder thread init failed: %s", exc)
|
|
ume_support._set_runtime_task(
|
|
"oclaw_alarm_forwarder",
|
|
status="error",
|
|
last_run_at=datetime.now(timezone.utc),
|
|
last_error=f"startup_thread_init_failed: {str(exc)[:180]}",
|
|
)
|
|
|
|
|
|
@app.on_event("shutdown")
|
|
def on_shutdown() -> None:
|
|
shutdown_oclaw_alarm_forwarder()
|
|
if ume_support._UME_WS_STOP_EVENT is not None:
|
|
ume_support._UME_WS_STOP_EVENT.set()
|
|
shutdown_ws_consumer()
|
|
|
|
|
|
@app.get("/health", status_code=200)
|
|
def health() -> dict[str, str]:
|
|
return {"status": "ok"}
|
|
|
|
|
|
|
|
@app.get("/v1/integrations/status")
|
|
def integrations_status(db: Session = Depends(get_db)) -> dict:
|
|
# netx api is up if this handler executes; still verify DB + oclaw bridge separately.
|
|
netx_api = {"status": "up"}
|
|
|
|
db_status: dict = {"status": "unknown"}
|
|
try:
|
|
t0 = time.monotonic()
|
|
db.execute(sql_text("select 1"))
|
|
db_status = {"status": "up", "latency_ms": int((time.monotonic() - t0) * 1000)}
|
|
except Exception as exc:
|
|
db_status = {"status": "down", "error": str(exc)[:240]}
|
|
|
|
oclaw_status: dict = {"status": "unknown"}
|
|
fwd = forwarder_status()
|
|
if not bool(fwd.get("enabled")):
|
|
oclaw_status = {
|
|
"status": "unknown",
|
|
"mode": "ws",
|
|
"enabled": False,
|
|
"connected": False,
|
|
"error_kind": "disabled",
|
|
"error": "NETX_OCLAW_ALARM_WS_ENABLED=false or missing token/url",
|
|
"forwarder": fwd,
|
|
}
|
|
elif bool(fwd.get("paused")):
|
|
oclaw_status = {
|
|
"status": "unknown",
|
|
"mode": "ws",
|
|
"enabled": True,
|
|
"connected": False,
|
|
"error_kind": "paused",
|
|
"error": "oclaw_alarm_forwarder runtime task paused",
|
|
"forwarder": fwd,
|
|
}
|
|
elif bool(fwd.get("connected")):
|
|
oclaw_status = {
|
|
"status": "up",
|
|
"mode": "ws",
|
|
"enabled": True,
|
|
"connected": True,
|
|
"queue_size": int(fwd.get("queue_size") or 0),
|
|
"published_ok": int(fwd.get("published_ok") or 0),
|
|
"published_fail": int(fwd.get("published_fail") or 0),
|
|
"url": str(fwd.get("url") or ""),
|
|
"forwarder": fwd,
|
|
}
|
|
else:
|
|
oclaw_status = {
|
|
"status": "down",
|
|
"mode": "ws",
|
|
"enabled": True,
|
|
"connected": False,
|
|
"error_kind": "ws_disconnected",
|
|
"error": "oclaw netx-bridge WebSocket not connected",
|
|
"queue_size": int(fwd.get("queue_size") or 0),
|
|
"url": str(fwd.get("url") or ""),
|
|
"forwarder": fwd,
|
|
}
|
|
|
|
return {"netx_api": netx_api, "db": db_status, "oclaw_bridge": oclaw_status}
|
|
|
|
|
|
@app.get("/")
|
|
def root() -> dict:
|
|
return {
|
|
"ok": True,
|
|
"mode": "api_only",
|
|
"message": "netx UI is served by Vite frontend only",
|
|
"frontend_url": settings.frontend_url,
|
|
"api_health": "/health",
|
|
"api_status": "/v1/integrations/status",
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
uvicorn.run("netx_api.main:app", host=settings.host, port=settings.port, reload=False)
|