mirror of
https://github.com/hansjone/netx.git
synced 2026-10-11 16:40:46 +08:00
104 lines
4.5 KiB
Python
104 lines
4.5 KiB
Python
"""Coalesced collect-completion events for live cutover evaluation.
|
|
|
|
One worker per process bounds concurrency. Project row locks in run_evaluate
|
|
serialize persistence across processes. No device collection runs here.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import threading
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from ..models import BizMigrationBatch, BizMigrationProject, BizMigrationRun, BizMonitorTemplate, BizStateBatch
|
|
from . import service
|
|
|
|
_log = logging.getLogger(__name__)
|
|
_lock = threading.Lock()
|
|
_pending: set[str] = set()
|
|
_worker_running = False
|
|
|
|
|
|
def try_auto_monitor_for_task(db: Session, task_id: str) -> int:
|
|
count = 0
|
|
# Bindings are JSON for multiple sampling intervals; match in Python after
|
|
# limiting to projects with an active operation batch.
|
|
projects = db.query(BizMigrationProject).filter(
|
|
BizMigrationProject.status != "done",
|
|
BizMigrationProject.id.in_(db.query(BizMigrationBatch.project_id).filter(
|
|
BizMigrationBatch.status == "active")),
|
|
).all()
|
|
for candidate in projects:
|
|
tids = service._all_hf_task_ids(candidate, "old") + service._all_hf_task_ids(candidate, "new")
|
|
if task_id not in tids or not candidate.old_baseline_batch_id or not candidate.new_baseline_batch_id:
|
|
continue
|
|
# Lock before deduplication so two worker processes cannot save the same
|
|
# snapshot. run_evaluate holds the same lock through the transaction.
|
|
proj = db.query(BizMigrationProject).filter(
|
|
BizMigrationProject.id == candidate.id).with_for_update().populate_existing().first()
|
|
batch = db.query(BizMigrationBatch).filter(
|
|
BizMigrationBatch.project_id == proj.id, BizMigrationBatch.status == "active"
|
|
).order_by(BizMigrationBatch.started_at.desc()).first()
|
|
if not batch or proj.status == "done":
|
|
db.rollback()
|
|
continue
|
|
mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None
|
|
if not mt:
|
|
db.rollback()
|
|
_log.error("cutover monitor template missing project=%s", proj.id)
|
|
continue
|
|
sheets, overrides, defaults, norms = service.resolve_evaluate_sheets(db, mt)
|
|
samples = []
|
|
for tid in tids:
|
|
sample = db.query(BizStateBatch).filter(
|
|
BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued"))
|
|
).order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first()
|
|
samples.append([tid, sample.id if sample else "", sample.status if sample else ""])
|
|
payload = [samples, proj.old_baseline_batch_id, proj.new_baseline_batch_id,
|
|
batch.expect_set_json, service._port_map_dict(db, proj.mapping_id),
|
|
sheets, overrides, defaults, norms, service.resolve_collect_metric_ids(db, proj)]
|
|
fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
|
last = db.query(BizMigrationRun).filter(BizMigrationRun.batch_id == batch.id).order_by(
|
|
BizMigrationRun.created_at.desc()).first()
|
|
if last and (last.summary_json or {}).get("auto_fingerprint") == fingerprint:
|
|
db.rollback()
|
|
continue
|
|
try:
|
|
result = service.run_evaluate(db, batch_id=batch.id, purpose="auto", _commit=False)
|
|
run = db.get(BizMigrationRun, result["id"])
|
|
run.summary_json = {**run.summary_json, "auto_fingerprint": fingerprint}
|
|
db.commit()
|
|
count += 1
|
|
except Exception:
|
|
db.rollback()
|
|
_log.exception("cutover monitor evaluate failed project=%s", proj.id)
|
|
return count
|
|
|
|
|
|
def schedule_auto_monitor_for_task(task_id: str) -> None:
|
|
global _worker_running
|
|
with _lock:
|
|
_pending.add(str(task_id))
|
|
if _worker_running:
|
|
return
|
|
_worker_running = True
|
|
|
|
def work() -> None:
|
|
global _worker_running
|
|
from ..db import SessionLocal
|
|
|
|
while True:
|
|
with _lock:
|
|
if not _pending:
|
|
_worker_running = False
|
|
return
|
|
tid = _pending.pop()
|
|
try:
|
|
with SessionLocal() as db:
|
|
try_auto_monitor_for_task(db, tid)
|
|
except Exception:
|
|
_log.exception("cutover monitor worker failed task=%s", tid)
|
|
|
|
threading.Thread(target=work, name="biz-cutover-monitor", daemon=True).start()
|