fix(startup): defer alarm pull 60s; health probe accepts only HTTP 200

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-06-04 21:32:07 +08:00
parent f562f50953
commit 65e0713e94
3 changed files with 30 additions and 3 deletions

View file

@ -44,6 +44,8 @@ class Settings(BaseSettings):
ume_sync_alarms_current_interval_s: int = 18000 ume_sync_alarms_current_interval_s: int = 18000
ume_sync_alarms_current_skip_when_ws: bool = True ume_sync_alarms_current_skip_when_ws: bool = True
ume_startup_sync_alarms_before_ws: bool = True ume_startup_sync_alarms_before_ws: bool = True
# Defer first REST alarm pull after process start (WSS may connect earlier).
ume_startup_alarm_sync_delay_s: int = 60
ume_alarm_cleared_tombstone_s: int = 300 ume_alarm_cleared_tombstone_s: int = 300
ume_alarm_ws_enabled: bool = True ume_alarm_ws_enabled: bool = True
ume_notification_establish_path: str = "/restconf/operations/zte-notifications:establish-subscription" ume_notification_establish_path: str = "/restconf/operations/zte-notifications:establish-subscription"

View file

@ -10,6 +10,7 @@ import re
import threading import threading
_schedule_log = logging.getLogger("netx.ume.schedule") _schedule_log = logging.getLogger("netx.ume.schedule")
_BOOT_MONO = time.monotonic()
from fastapi import Depends, FastAPI, File, HTTPException, Query, UploadFile from fastapi import Depends, FastAPI, File, HTTPException, Query, UploadFile
from fastapi.responses import Response from fastapi.responses import Response
from sqlalchemy import text as sql_text from sqlalchemy import text as sql_text
@ -285,6 +286,21 @@ def _fail_stale_running_sync_jobs_on_startup() -> None:
db.close() db.close()
def _startup_alarm_pull_delay_s() -> int:
return max(0, min(3600, int(getattr(settings, "ume_startup_alarm_sync_delay_s", 60) or 60)))
def _wait_until_startup_alarm_pull_allowed(label: str) -> None:
delay_s = _startup_alarm_pull_delay_s()
if delay_s <= 0:
return
remaining = float(delay_s) - (time.monotonic() - _BOOT_MONO)
if remaining <= 0:
return
_schedule_log.info("%s: defer alarm pull %.0fs after process start", label, remaining)
time.sleep(remaining)
def _run_startup_alarm_sync_before_ws() -> None: def _run_startup_alarm_sync_before_ws() -> None:
"""REST-sync current alarms once on boot before WSS connects (avoids stale/reconcile races).""" """REST-sync current alarms once on boot before WSS connects (avoids stale/reconcile races)."""
ume_url = str(getattr(settings, "ume_base_url", "") or "").strip() ume_url = str(getattr(settings, "ume_base_url", "") or "").strip()
@ -296,6 +312,7 @@ def _run_startup_alarm_sync_before_ws() -> None:
complete_startup_alarm_sync_gate() complete_startup_alarm_sync_gate()
return return
_wait_until_startup_alarm_pull_allowed("startup_alarm_sync")
begin_startup_alarm_sync_gate() begin_startup_alarm_sync_gate()
try: try:
_schedule_log.info("startup: syncing current alarms before WSS") _schedule_log.info("startup: syncing current alarms before WSS")
@ -840,6 +857,7 @@ def on_startup() -> None:
def _alarms_current_sync_loop() -> None: def _alarms_current_sync_loop() -> None:
_refresh_runtime_task_idle("alarms_current_auto_sync", "alarms_current") _refresh_runtime_task_idle("alarms_current_auto_sync", "alarms_current")
_wait_until_startup_alarm_pull_allowed("alarms_current_auto_sync")
while True: while True:
try: try:
_schedule_log.info( _schedule_log.info(
@ -1022,7 +1040,7 @@ def on_shutdown() -> None:
shutdown_ws_consumer() shutdown_ws_consumer()
@app.get("/health") @app.get("/health", status_code=200)
def health() -> dict[str, str]: def health() -> dict[str, str]:
return {"status": "ok"} return {"status": "ok"}

View file

@ -95,10 +95,17 @@ function Show-LogTail {
function Test-NetxApiListening { function Test-NetxApiListening {
param([string]$HostName, [int]$LocalPort, [int]$WaitSec = 45) param([string]$HostName, [int]$LocalPort, [int]$WaitSec = 45)
$deadline = (Get-Date).AddSeconds($WaitSec) $deadline = (Get-Date).AddSeconds($WaitSec)
$healthUrl = "http://${HostName}:${LocalPort}/health"
while ((Get-Date) -lt $deadline) { while ((Get-Date) -lt $deadline) {
try { try {
$r = Invoke-WebRequest -Uri "http://${HostName}:${LocalPort}/health" -UseBasicParsing -TimeoutSec 2 # Only HTTP 200 with {"status":"ok"} counts; no redirects / other codes.
if ($r.StatusCode -eq 200) { return $true } $r = Invoke-WebRequest -Uri $healthUrl -UseBasicParsing -TimeoutSec 2 -MaximumRedirection 0
if ($r.StatusCode -ne 200) {
Start-Sleep -Milliseconds 500
continue
}
$body = $r.Content | ConvertFrom-Json -ErrorAction Stop
if ($body.status -eq "ok") { return $true }
} catch {} } catch {}
Start-Sleep -Milliseconds 500 Start-Sleep -Milliseconds 500
} }