mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 00:50:46 +08:00
oliver
3413d5bf32
Fix pending mislabeled as fail due to timezone and queue stale checks, show full error details, sync live progress counts, and add retry-failed endpoint for failed NEs only. Co-authored-by: Cursor <cursoragent@cursor.com>
92 lines
3.2 KiB
Python
92 lines
3.2 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from .config import settings
|
|
from .models import NeCollectionJob, NeCollectionRun
|
|
|
|
_TERMINAL = frozenset({"success", "fail", "cancelled"})
|
|
|
|
|
|
def _sync_job_counts(job: NeCollectionJob, runs: list[NeCollectionRun]) -> None:
|
|
job.success_count = sum(1 for r in runs if str(r.status) == "success")
|
|
job.fail_count = sum(1 for r in runs if str(r.status) == "fail")
|
|
# cancelled rows are tracked separately via ne_count - success - fail - active
|
|
|
|
|
|
def sync_job_progress(db: Session, job_id: str) -> None:
|
|
"""Refresh success/fail counters while a job is still running."""
|
|
job = db.get(NeCollectionJob, job_id)
|
|
if not job or str(job.status or "") != "running":
|
|
return
|
|
runs = db.query(NeCollectionRun).filter(NeCollectionRun.job_id == job_id).all()
|
|
_sync_job_counts(job, runs)
|
|
db.commit()
|
|
|
|
|
|
def finalize_collection_job(db: Session, job_id: str) -> None:
|
|
runs = db.query(NeCollectionRun).filter(NeCollectionRun.job_id == job_id).all()
|
|
if not runs:
|
|
return
|
|
if any(str(r.status or "") not in _TERMINAL for r in runs):
|
|
return
|
|
job = db.get(NeCollectionJob, job_id)
|
|
if not job:
|
|
return
|
|
_sync_job_counts(job, runs)
|
|
job_status = str(job.status or "")
|
|
finish_at = datetime.now()
|
|
if job_status == "paused":
|
|
if not job.ended_at:
|
|
job.ended_at = finish_at
|
|
else:
|
|
success = int(job.success_count or 0)
|
|
job.status = "done" if success > 0 else "failed"
|
|
if not job.ended_at:
|
|
job.ended_at = finish_at
|
|
job.last_run_at = job.ended_at or finish_at
|
|
db.commit()
|
|
|
|
|
|
def reconcile_stale_collection_job(db: Session, job_id: str) -> bool:
|
|
"""Mark long-running pending/running rows as failed and finalize job if possible."""
|
|
job = db.get(NeCollectionJob, job_id)
|
|
if not job:
|
|
return False
|
|
if str(job.status or "") in ("done", "failed", "paused"):
|
|
return False
|
|
run_stale_sec = max(60, int(settings.ne_collect_stale_run_sec or 900))
|
|
pending_stale_sec = max(30, int(settings.ne_collect_pending_stale_sec or 180))
|
|
now = datetime.now()
|
|
runs = db.query(NeCollectionRun).filter(NeCollectionRun.job_id == job_id).all()
|
|
changed = False
|
|
for run in runs:
|
|
st = str(run.status or "")
|
|
if st in _TERMINAL:
|
|
continue
|
|
if st == "pending":
|
|
# Pending while job is running means queued in the worker pool, not stuck.
|
|
if str(job.status or "") == "running":
|
|
continue
|
|
anchor = job.started_at or job.created_at
|
|
limit = pending_stale_sec
|
|
reason = "collection_pending_stale"
|
|
else:
|
|
anchor = run.started_at or job.started_at or job.created_at
|
|
limit = run_stale_sec
|
|
reason = "collection_timeout_stale"
|
|
if not anchor:
|
|
continue
|
|
age = (now - anchor).total_seconds()
|
|
if age < limit:
|
|
continue
|
|
run.status = "fail"
|
|
run.message = f"{reason} ({int(age)}s)"
|
|
run.ended_at = now
|
|
changed = True
|
|
if changed:
|
|
db.commit()
|
|
finalize_collection_job(db, job_id)
|
|
return changed
|