mirror of
https://github.com/hansjone/netx.git
synced 2026-10-13 02:03:23 +08:00
UME scheduler: resume wakes debounce sleep + skip wait; pause clears hints; loop tick logs
- Event.wait debounce sleep so 开始 interrupts long interval wait\n- resume sets skip+event; wake path discards skip to avoid double-sync\n- pause clears skip/wake; boot resets debounce via _reset_runtime_pause_flags\n- loop tick + thread is_alive logging for netx.ume.schedule Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
b17aef0eae
commit
936cee1aa2
1 changed files with 83 additions and 5 deletions
|
|
@ -78,6 +78,49 @@ _UME_RUNTIME_TASKS: dict[str, dict[str, Any]] = {
|
||||||
_UME_RUNTIME_PAUSED: dict[str, bool] = {}
|
_UME_RUNTIME_PAUSED: dict[str, bool] = {}
|
||||||
UME_KNOWN_RUNTIME_TASKS: tuple[str, ...] = tuple(_UME_RUNTIME_TASKS.keys())
|
UME_KNOWN_RUNTIME_TASKS: tuple[str, ...] = tuple(_UME_RUNTIME_TASKS.keys())
|
||||||
_UME_RUNTIME_LOCK = threading.Lock()
|
_UME_RUNTIME_LOCK = threading.Lock()
|
||||||
|
# Debounce skip / wake for scheduled sync threads (resume should not wait full interval).
|
||||||
|
_UME_DEBOUNCE_MUTEX = threading.Lock()
|
||||||
|
_UME_SYNC_SKIP_DEBOUNCE: set[str] = set()
|
||||||
|
_UME_DEBOUNCE_WAKE: dict[str, threading.Event] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def _debounce_wake_event(task_id: str) -> threading.Event:
|
||||||
|
with _UME_DEBOUNCE_MUTEX:
|
||||||
|
ev = _UME_DEBOUNCE_WAKE.get(task_id)
|
||||||
|
if ev is None:
|
||||||
|
ev = threading.Event()
|
||||||
|
_UME_DEBOUNCE_WAKE[task_id] = ev
|
||||||
|
return ev
|
||||||
|
|
||||||
|
|
||||||
|
def _request_force_sync_after_resume(task_id: str) -> None:
|
||||||
|
"""Skip next debounce wait and interrupt an in-progress debounce sleep (UI 开始)."""
|
||||||
|
with _UME_DEBOUNCE_MUTEX:
|
||||||
|
_UME_SYNC_SKIP_DEBOUNCE.add(task_id)
|
||||||
|
try:
|
||||||
|
_debounce_wake_event(task_id).set()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _clear_force_resume_hints(task_id: str) -> None:
|
||||||
|
"""Pause: drop pending skip/wake so state is predictable."""
|
||||||
|
with _UME_DEBOUNCE_MUTEX:
|
||||||
|
_UME_SYNC_SKIP_DEBOUNCE.discard(task_id)
|
||||||
|
try:
|
||||||
|
_debounce_wake_event(task_id).clear()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_debounce_wakeup() -> None:
|
||||||
|
with _UME_DEBOUNCE_MUTEX:
|
||||||
|
_UME_SYNC_SKIP_DEBOUNCE.clear()
|
||||||
|
for ev in _UME_DEBOUNCE_WAKE.values():
|
||||||
|
try:
|
||||||
|
ev.clear()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _set_runtime_task(task: str, *, status: str, last_run_at: datetime | None = None, last_error: str = "") -> None:
|
def _set_runtime_task(task: str, *, status: str, last_run_at: datetime | None = None, last_error: str = "") -> None:
|
||||||
|
|
@ -184,6 +227,7 @@ def _reset_runtime_pause_flags() -> None:
|
||||||
with _UME_RUNTIME_LOCK:
|
with _UME_RUNTIME_LOCK:
|
||||||
for tid in UME_KNOWN_RUNTIME_TASKS:
|
for tid in UME_KNOWN_RUNTIME_TASKS:
|
||||||
_UME_RUNTIME_PAUSED[tid] = False
|
_UME_RUNTIME_PAUSED[tid] = False
|
||||||
|
_reset_debounce_wakeup()
|
||||||
|
|
||||||
|
|
||||||
def _fail_stale_running_sync_jobs_on_startup() -> None:
|
def _fail_stale_running_sync_jobs_on_startup() -> None:
|
||||||
|
|
@ -213,13 +257,26 @@ def _fail_stale_running_sync_jobs_on_startup() -> None:
|
||||||
|
|
||||||
|
|
||||||
def _sleep_or_until_paused(task_id: str, total_s: float) -> None:
|
def _sleep_or_until_paused(task_id: str, total_s: float) -> None:
|
||||||
"""Sleep up to total_s wall seconds, but wake every ~2s to honor pause."""
|
"""Sleep up to total_s wall seconds; honor pause; wake early on resume (debounce interrupt)."""
|
||||||
deadline = time.time() + max(0.0, float(total_s))
|
deadline = time.time() + max(0.0, float(total_s))
|
||||||
|
ev = _debounce_wake_event(task_id)
|
||||||
|
ev.clear()
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
if _runtime_is_paused(task_id):
|
if _runtime_is_paused(task_id):
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
continue
|
continue
|
||||||
time.sleep(min(2.0, max(0.0, deadline - time.time())))
|
remaining = deadline - time.time()
|
||||||
|
if remaining <= 0:
|
||||||
|
break
|
||||||
|
timeout = min(2.0, remaining)
|
||||||
|
if ev.wait(timeout=timeout):
|
||||||
|
ev.clear()
|
||||||
|
_schedule_log.info("%s: debounce wait interrupted (resume)", task_id)
|
||||||
|
with _UME_DEBOUNCE_MUTEX:
|
||||||
|
_UME_SYNC_SKIP_DEBOUNCE.discard(task_id)
|
||||||
|
return
|
||||||
|
if ev.is_set():
|
||||||
|
ev.clear()
|
||||||
|
|
||||||
|
|
||||||
def _seconds_since_last_finished_job(db: Session, domain: str) -> float | None:
|
def _seconds_since_last_finished_job(db: Session, domain: str) -> float | None:
|
||||||
|
|
@ -250,6 +307,11 @@ def _maybe_wait_for_sync_interval(
|
||||||
label: str,
|
label: str,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Sleep until interval elapsed since last finished job (ended_at), if any."""
|
"""Sleep until interval elapsed since last finished job (ended_at), if any."""
|
||||||
|
with _UME_DEBOUNCE_MUTEX:
|
||||||
|
if task_id in _UME_SYNC_SKIP_DEBOUNCE:
|
||||||
|
_UME_SYNC_SKIP_DEBOUNCE.discard(task_id)
|
||||||
|
_schedule_log.info("%s: debounce skipped (resume/kick)", label)
|
||||||
|
return
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
elapsed = _seconds_since_last_finished_job(db, domain)
|
elapsed = _seconds_since_last_finished_job(db, domain)
|
||||||
|
|
@ -541,6 +603,10 @@ def on_startup() -> None:
|
||||||
def _alarms_current_sync_loop() -> None:
|
def _alarms_current_sync_loop() -> None:
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
|
_schedule_log.info(
|
||||||
|
"alarms_current_auto_sync: loop tick paused=%s",
|
||||||
|
_runtime_is_paused("alarms_current_auto_sync"),
|
||||||
|
)
|
||||||
if _runtime_is_paused("alarms_current_auto_sync"):
|
if _runtime_is_paused("alarms_current_auto_sync"):
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
continue
|
continue
|
||||||
|
|
@ -584,7 +650,9 @@ def on_startup() -> None:
|
||||||
|
|
||||||
t2 = threading.Thread(target=_alarms_current_sync_loop, name="ume-alarms-current-sync", daemon=True)
|
t2 = threading.Thread(target=_alarms_current_sync_loop, name="ume-alarms-current-sync", daemon=True)
|
||||||
t2.start()
|
t2.start()
|
||||||
_schedule_log.info("started thread %s", t2.name)
|
_schedule_log.info("started thread %s alive=%s", t2.name, t2.is_alive())
|
||||||
|
if not t2.is_alive():
|
||||||
|
_schedule_log.error("ume-alarms-current-sync thread exited immediately (check uncaught errors above)")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
try:
|
try:
|
||||||
|
|
@ -596,6 +664,10 @@ def on_startup() -> None:
|
||||||
def _inventory_auto_sync_loop() -> None:
|
def _inventory_auto_sync_loop() -> None:
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
|
_schedule_log.info(
|
||||||
|
"inventory_auto_sync: loop tick paused=%s",
|
||||||
|
_runtime_is_paused("inventory_auto_sync"),
|
||||||
|
)
|
||||||
if _runtime_is_paused("inventory_auto_sync"):
|
if _runtime_is_paused("inventory_auto_sync"):
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
continue
|
continue
|
||||||
|
|
@ -639,7 +711,9 @@ def on_startup() -> None:
|
||||||
|
|
||||||
t3 = threading.Thread(target=_inventory_auto_sync_loop, name="ume-inventory-auto-sync", daemon=True)
|
t3 = threading.Thread(target=_inventory_auto_sync_loop, name="ume-inventory-auto-sync", daemon=True)
|
||||||
t3.start()
|
t3.start()
|
||||||
_schedule_log.info("started thread %s", t3.name)
|
_schedule_log.info("started thread %s alive=%s", t3.name, t3.is_alive())
|
||||||
|
if not t3.is_alive():
|
||||||
|
_schedule_log.error("ume-inventory-auto-sync thread exited immediately (check uncaught errors above)")
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
@ -812,6 +886,8 @@ def ume_runtime_task_pause(task: str) -> dict[str, Any]:
|
||||||
if tid not in UME_KNOWN_RUNTIME_TASKS:
|
if tid not in UME_KNOWN_RUNTIME_TASKS:
|
||||||
raise HTTPException(status_code=404, detail="unknown_runtime_task")
|
raise HTTPException(status_code=404, detail="unknown_runtime_task")
|
||||||
_runtime_pause_task(tid)
|
_runtime_pause_task(tid)
|
||||||
|
if tid in ("alarms_current_auto_sync", "inventory_auto_sync"):
|
||||||
|
_clear_force_resume_hints(tid)
|
||||||
_set_runtime_task(tid, status="paused", last_error="")
|
_set_runtime_task(tid, status="paused", last_error="")
|
||||||
return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()}
|
return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()}
|
||||||
|
|
||||||
|
|
@ -822,7 +898,9 @@ def ume_runtime_task_resume(task: str) -> dict[str, Any]:
|
||||||
if tid not in UME_KNOWN_RUNTIME_TASKS:
|
if tid not in UME_KNOWN_RUNTIME_TASKS:
|
||||||
raise HTTPException(status_code=404, detail="unknown_runtime_task")
|
raise HTTPException(status_code=404, detail="unknown_runtime_task")
|
||||||
_runtime_resume_task(tid)
|
_runtime_resume_task(tid)
|
||||||
_set_runtime_task(tid, status="running", last_error="")
|
if tid in ("alarms_current_auto_sync", "inventory_auto_sync"):
|
||||||
|
_request_force_sync_after_resume(tid)
|
||||||
|
_set_runtime_task(tid, status="running", last_error="已恢复:将跳过本轮周期等待并尽快同步")
|
||||||
return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()}
|
return {"ok": True, "task": tid, "runtime_tasks": _list_runtime_tasks()}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue