mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 02:00:46 +08:00
fix(startup): defer alarm pull 60s; health probe accepts only HTTP 200
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
f562f50953
commit
65e0713e94
3 changed files with 30 additions and 3 deletions
|
|
@ -44,6 +44,8 @@ class Settings(BaseSettings):
|
|||
ume_sync_alarms_current_interval_s: int = 18000
|
||||
ume_sync_alarms_current_skip_when_ws: bool = True
|
||||
ume_startup_sync_alarms_before_ws: bool = True
|
||||
# Defer first REST alarm pull after process start (WSS may connect earlier).
|
||||
ume_startup_alarm_sync_delay_s: int = 60
|
||||
ume_alarm_cleared_tombstone_s: int = 300
|
||||
ume_alarm_ws_enabled: bool = True
|
||||
ume_notification_establish_path: str = "/restconf/operations/zte-notifications:establish-subscription"
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import re
|
|||
import threading
|
||||
|
||||
_schedule_log = logging.getLogger("netx.ume.schedule")
|
||||
_BOOT_MONO = time.monotonic()
|
||||
from fastapi import Depends, FastAPI, File, HTTPException, Query, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy import text as sql_text
|
||||
|
|
@ -285,6 +286,21 @@ def _fail_stale_running_sync_jobs_on_startup() -> None:
|
|||
db.close()
|
||||
|
||||
|
||||
def _startup_alarm_pull_delay_s() -> int:
|
||||
return max(0, min(3600, int(getattr(settings, "ume_startup_alarm_sync_delay_s", 60) or 60)))
|
||||
|
||||
|
||||
def _wait_until_startup_alarm_pull_allowed(label: str) -> None:
|
||||
delay_s = _startup_alarm_pull_delay_s()
|
||||
if delay_s <= 0:
|
||||
return
|
||||
remaining = float(delay_s) - (time.monotonic() - _BOOT_MONO)
|
||||
if remaining <= 0:
|
||||
return
|
||||
_schedule_log.info("%s: defer alarm pull %.0fs after process start", label, remaining)
|
||||
time.sleep(remaining)
|
||||
|
||||
|
||||
def _run_startup_alarm_sync_before_ws() -> None:
|
||||
"""REST-sync current alarms once on boot before WSS connects (avoids stale/reconcile races)."""
|
||||
ume_url = str(getattr(settings, "ume_base_url", "") or "").strip()
|
||||
|
|
@ -296,6 +312,7 @@ def _run_startup_alarm_sync_before_ws() -> None:
|
|||
complete_startup_alarm_sync_gate()
|
||||
return
|
||||
|
||||
_wait_until_startup_alarm_pull_allowed("startup_alarm_sync")
|
||||
begin_startup_alarm_sync_gate()
|
||||
try:
|
||||
_schedule_log.info("startup: syncing current alarms before WSS")
|
||||
|
|
@ -840,6 +857,7 @@ def on_startup() -> None:
|
|||
|
||||
def _alarms_current_sync_loop() -> None:
|
||||
_refresh_runtime_task_idle("alarms_current_auto_sync", "alarms_current")
|
||||
_wait_until_startup_alarm_pull_allowed("alarms_current_auto_sync")
|
||||
while True:
|
||||
try:
|
||||
_schedule_log.info(
|
||||
|
|
@ -1022,7 +1040,7 @@ def on_shutdown() -> None:
|
|||
shutdown_ws_consumer()
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
@app.get("/health", status_code=200)
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
|
|
|||
|
|
@ -95,10 +95,17 @@ function Show-LogTail {
|
|||
function Test-NetxApiListening {
|
||||
param([string]$HostName, [int]$LocalPort, [int]$WaitSec = 45)
|
||||
$deadline = (Get-Date).AddSeconds($WaitSec)
|
||||
$healthUrl = "http://${HostName}:${LocalPort}/health"
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
try {
|
||||
$r = Invoke-WebRequest -Uri "http://${HostName}:${LocalPort}/health" -UseBasicParsing -TimeoutSec 2
|
||||
if ($r.StatusCode -eq 200) { return $true }
|
||||
# Only HTTP 200 with {"status":"ok"} counts; no redirects / other codes.
|
||||
$r = Invoke-WebRequest -Uri $healthUrl -UseBasicParsing -TimeoutSec 2 -MaximumRedirection 0
|
||||
if ($r.StatusCode -ne 200) {
|
||||
Start-Sleep -Milliseconds 500
|
||||
continue
|
||||
}
|
||||
$body = $r.Content | ConvertFrom-Json -ErrorAction Stop
|
||||
if ($body.status -eq "ok") { return $true }
|
||||
} catch {}
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue