fix(startup): defer alarm pull 60s; health probe accepts only HTTP 200

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-06-04 21:32:07 +08:00
parent f562f50953
commit 65e0713e94
3 changed files with 30 additions and 3 deletions

View file

@ -44,6 +44,8 @@ class Settings(BaseSettings):
ume_sync_alarms_current_interval_s: int = 18000
ume_sync_alarms_current_skip_when_ws: bool = True
ume_startup_sync_alarms_before_ws: bool = True
# Defer first REST alarm pull after process start (WSS may connect earlier).
ume_startup_alarm_sync_delay_s: int = 60
ume_alarm_cleared_tombstone_s: int = 300
ume_alarm_ws_enabled: bool = True
ume_notification_establish_path: str = "/restconf/operations/zte-notifications:establish-subscription"

View file

@ -10,6 +10,7 @@ import re
import threading
_schedule_log = logging.getLogger("netx.ume.schedule")
_BOOT_MONO = time.monotonic()
from fastapi import Depends, FastAPI, File, HTTPException, Query, UploadFile
from fastapi.responses import Response
from sqlalchemy import text as sql_text
@ -285,6 +286,21 @@ def _fail_stale_running_sync_jobs_on_startup() -> None:
db.close()
def _startup_alarm_pull_delay_s() -> int:
return max(0, min(3600, int(getattr(settings, "ume_startup_alarm_sync_delay_s", 60) or 60)))
def _wait_until_startup_alarm_pull_allowed(label: str) -> None:
delay_s = _startup_alarm_pull_delay_s()
if delay_s <= 0:
return
remaining = float(delay_s) - (time.monotonic() - _BOOT_MONO)
if remaining <= 0:
return
_schedule_log.info("%s: defer alarm pull %.0fs after process start", label, remaining)
time.sleep(remaining)
def _run_startup_alarm_sync_before_ws() -> None:
"""REST-sync current alarms once on boot before WSS connects (avoids stale/reconcile races)."""
ume_url = str(getattr(settings, "ume_base_url", "") or "").strip()
@ -296,6 +312,7 @@ def _run_startup_alarm_sync_before_ws() -> None:
complete_startup_alarm_sync_gate()
return
_wait_until_startup_alarm_pull_allowed("startup_alarm_sync")
begin_startup_alarm_sync_gate()
try:
_schedule_log.info("startup: syncing current alarms before WSS")
@ -840,6 +857,7 @@ def on_startup() -> None:
def _alarms_current_sync_loop() -> None:
_refresh_runtime_task_idle("alarms_current_auto_sync", "alarms_current")
_wait_until_startup_alarm_pull_allowed("alarms_current_auto_sync")
while True:
try:
_schedule_log.info(
@ -1022,7 +1040,7 @@ def on_shutdown() -> None:
shutdown_ws_consumer()
@app.get("/health")
@app.get("/health", status_code=200)
def health() -> dict[str, str]:
return {"status": "ok"}

View file

@ -95,10 +95,17 @@ function Show-LogTail {
function Test-NetxApiListening {
param([string]$HostName, [int]$LocalPort, [int]$WaitSec = 45)
$deadline = (Get-Date).AddSeconds($WaitSec)
$healthUrl = "http://${HostName}:${LocalPort}/health"
while ((Get-Date) -lt $deadline) {
try {
$r = Invoke-WebRequest -Uri "http://${HostName}:${LocalPort}/health" -UseBasicParsing -TimeoutSec 2
if ($r.StatusCode -eq 200) { return $true }
# Only HTTP 200 with {"status":"ok"} counts; no redirects / other codes.
$r = Invoke-WebRequest -Uri $healthUrl -UseBasicParsing -TimeoutSec 2 -MaximumRedirection 0
if ($r.StatusCode -ne 200) {
Start-Sleep -Milliseconds 500
continue
}
$body = $r.Content | ConvertFrom-Json -ErrorAction Stop
if ($body.status -eq "ok") { return $true }
} catch {}
Start-Sleep -Milliseconds 500
}