mirror of
https://github.com/hansjone/netx.git
synced 2026-10-19 13:10:46 +08:00
fix(startup): defer alarm pull 60s; health probe accepts only HTTP 200
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
f562f50953
commit
65e0713e94
3 changed files with 30 additions and 3 deletions
|
|
@ -44,6 +44,8 @@ class Settings(BaseSettings):
|
||||||
ume_sync_alarms_current_interval_s: int = 18000
|
ume_sync_alarms_current_interval_s: int = 18000
|
||||||
ume_sync_alarms_current_skip_when_ws: bool = True
|
ume_sync_alarms_current_skip_when_ws: bool = True
|
||||||
ume_startup_sync_alarms_before_ws: bool = True
|
ume_startup_sync_alarms_before_ws: bool = True
|
||||||
|
# Defer first REST alarm pull after process start (WSS may connect earlier).
|
||||||
|
ume_startup_alarm_sync_delay_s: int = 60
|
||||||
ume_alarm_cleared_tombstone_s: int = 300
|
ume_alarm_cleared_tombstone_s: int = 300
|
||||||
ume_alarm_ws_enabled: bool = True
|
ume_alarm_ws_enabled: bool = True
|
||||||
ume_notification_establish_path: str = "/restconf/operations/zte-notifications:establish-subscription"
|
ume_notification_establish_path: str = "/restconf/operations/zte-notifications:establish-subscription"
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,7 @@ import re
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
_schedule_log = logging.getLogger("netx.ume.schedule")
|
_schedule_log = logging.getLogger("netx.ume.schedule")
|
||||||
|
_BOOT_MONO = time.monotonic()
|
||||||
from fastapi import Depends, FastAPI, File, HTTPException, Query, UploadFile
|
from fastapi import Depends, FastAPI, File, HTTPException, Query, UploadFile
|
||||||
from fastapi.responses import Response
|
from fastapi.responses import Response
|
||||||
from sqlalchemy import text as sql_text
|
from sqlalchemy import text as sql_text
|
||||||
|
|
@ -285,6 +286,21 @@ def _fail_stale_running_sync_jobs_on_startup() -> None:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _startup_alarm_pull_delay_s() -> int:
|
||||||
|
return max(0, min(3600, int(getattr(settings, "ume_startup_alarm_sync_delay_s", 60) or 60)))
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_until_startup_alarm_pull_allowed(label: str) -> None:
|
||||||
|
delay_s = _startup_alarm_pull_delay_s()
|
||||||
|
if delay_s <= 0:
|
||||||
|
return
|
||||||
|
remaining = float(delay_s) - (time.monotonic() - _BOOT_MONO)
|
||||||
|
if remaining <= 0:
|
||||||
|
return
|
||||||
|
_schedule_log.info("%s: defer alarm pull %.0fs after process start", label, remaining)
|
||||||
|
time.sleep(remaining)
|
||||||
|
|
||||||
|
|
||||||
def _run_startup_alarm_sync_before_ws() -> None:
|
def _run_startup_alarm_sync_before_ws() -> None:
|
||||||
"""REST-sync current alarms once on boot before WSS connects (avoids stale/reconcile races)."""
|
"""REST-sync current alarms once on boot before WSS connects (avoids stale/reconcile races)."""
|
||||||
ume_url = str(getattr(settings, "ume_base_url", "") or "").strip()
|
ume_url = str(getattr(settings, "ume_base_url", "") or "").strip()
|
||||||
|
|
@ -296,6 +312,7 @@ def _run_startup_alarm_sync_before_ws() -> None:
|
||||||
complete_startup_alarm_sync_gate()
|
complete_startup_alarm_sync_gate()
|
||||||
return
|
return
|
||||||
|
|
||||||
|
_wait_until_startup_alarm_pull_allowed("startup_alarm_sync")
|
||||||
begin_startup_alarm_sync_gate()
|
begin_startup_alarm_sync_gate()
|
||||||
try:
|
try:
|
||||||
_schedule_log.info("startup: syncing current alarms before WSS")
|
_schedule_log.info("startup: syncing current alarms before WSS")
|
||||||
|
|
@ -840,6 +857,7 @@ def on_startup() -> None:
|
||||||
|
|
||||||
def _alarms_current_sync_loop() -> None:
|
def _alarms_current_sync_loop() -> None:
|
||||||
_refresh_runtime_task_idle("alarms_current_auto_sync", "alarms_current")
|
_refresh_runtime_task_idle("alarms_current_auto_sync", "alarms_current")
|
||||||
|
_wait_until_startup_alarm_pull_allowed("alarms_current_auto_sync")
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
_schedule_log.info(
|
_schedule_log.info(
|
||||||
|
|
@ -1022,7 +1040,7 @@ def on_shutdown() -> None:
|
||||||
shutdown_ws_consumer()
|
shutdown_ws_consumer()
|
||||||
|
|
||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health", status_code=200)
|
||||||
def health() -> dict[str, str]:
|
def health() -> dict[str, str]:
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -95,10 +95,17 @@ function Show-LogTail {
|
||||||
function Test-NetxApiListening {
|
function Test-NetxApiListening {
|
||||||
param([string]$HostName, [int]$LocalPort, [int]$WaitSec = 45)
|
param([string]$HostName, [int]$LocalPort, [int]$WaitSec = 45)
|
||||||
$deadline = (Get-Date).AddSeconds($WaitSec)
|
$deadline = (Get-Date).AddSeconds($WaitSec)
|
||||||
|
$healthUrl = "http://${HostName}:${LocalPort}/health"
|
||||||
while ((Get-Date) -lt $deadline) {
|
while ((Get-Date) -lt $deadline) {
|
||||||
try {
|
try {
|
||||||
$r = Invoke-WebRequest -Uri "http://${HostName}:${LocalPort}/health" -UseBasicParsing -TimeoutSec 2
|
# Only HTTP 200 with {"status":"ok"} counts; no redirects / other codes.
|
||||||
if ($r.StatusCode -eq 200) { return $true }
|
$r = Invoke-WebRequest -Uri $healthUrl -UseBasicParsing -TimeoutSec 2 -MaximumRedirection 0
|
||||||
|
if ($r.StatusCode -ne 200) {
|
||||||
|
Start-Sleep -Milliseconds 500
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
$body = $r.Content | ConvertFrom-Json -ErrorAction Stop
|
||||||
|
if ($body.status -eq "ok") { return $true }
|
||||||
} catch {}
|
} catch {}
|
||||||
Start-Sleep -Milliseconds 500
|
Start-Sleep -Milliseconds 500
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue