mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 04:20:45 +08:00
Mirror LLDP job retention so finished sync cycles are pruned by policy while keeping per-NE config version history separate. Co-authored-by: Cursor <cursoragent@cursor.com>
737 lines
25 KiB
Python
737 lines
25 KiB
Python
"""Config sync policy, cycles, dashboard, and snapshot queries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import logging
|
|
import re
|
|
import zipfile
|
|
from datetime import datetime, timedelta
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from fastapi import HTTPException
|
|
from sqlalchemy import func, or_
|
|
from sqlalchemy.orm import Session
|
|
|
|
from .config_sync_codec import decompress_text
|
|
from .config_sync_schemas import (
|
|
ConfigSyncCycleCreate,
|
|
ConfigSyncCycleOut,
|
|
ConfigSyncDashboardOut,
|
|
ConfigSyncPolicyOut,
|
|
ConfigSyncPolicyUpdate,
|
|
ConfigSyncTargetRef,
|
|
ConfigSyncTaskOut,
|
|
NeConfigHistoryOut,
|
|
NeConfigSnapshotDetailOut,
|
|
NeConfigSnapshotMetaOut,
|
|
)
|
|
from .models import (
|
|
ConfigSyncCycle,
|
|
ConfigSyncPolicy,
|
|
ConfigSyncTask,
|
|
ManagedNE,
|
|
NeConfigHistory,
|
|
NeConfigSnapshot,
|
|
UmeInventoryNE,
|
|
)
|
|
from .cli_resolve import cli_profile_ready
|
|
|
|
_log = logging.getLogger("netx.config_sync")
|
|
|
|
POLICY_ID = 1
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.utcnow()
|
|
|
|
|
|
DEFAULT_CYCLE_KEEP = 30
|
|
|
|
|
|
def ensure_policy(db: Session) -> ConfigSyncPolicy:
|
|
row = db.get(ConfigSyncPolicy, POLICY_ID)
|
|
if row is None:
|
|
row = ConfigSyncPolicy(id=POLICY_ID, enabled=False)
|
|
db.add(row)
|
|
db.commit()
|
|
db.refresh(row)
|
|
return row
|
|
|
|
|
|
def prune_config_sync_cycles(db: Session, *, keep: int = DEFAULT_CYCLE_KEEP) -> int:
|
|
"""Delete finished cycles beyond ``keep`` (newest kept). Active cycles always retained."""
|
|
keep = max(0, min(200, int(keep)))
|
|
finished = (
|
|
db.query(ConfigSyncCycle)
|
|
.filter(ConfigSyncCycle.status.in_(("success", "fail", "cancelled")))
|
|
.order_by(ConfigSyncCycle.created_at.desc())
|
|
.all()
|
|
)
|
|
to_drop = finished if keep == 0 else finished[keep:]
|
|
if not to_drop:
|
|
return 0
|
|
dropped = 0
|
|
for cycle in to_drop:
|
|
cid = str(cycle.id)
|
|
db.query(ConfigSyncTask).filter(ConfigSyncTask.cycle_id == cid).delete(
|
|
synchronize_session=False
|
|
)
|
|
db.delete(cycle)
|
|
dropped += 1
|
|
if dropped:
|
|
db.commit()
|
|
return dropped
|
|
|
|
|
|
def _cycle_keep_value(row: ConfigSyncPolicy) -> int:
|
|
return max(0, min(200, int(getattr(row, "cycle_keep", None) or DEFAULT_CYCLE_KEEP)))
|
|
|
|
|
|
def _targets_from_json(raw: Any) -> list[ConfigSyncTargetRef]:
|
|
items: list[ConfigSyncTargetRef] = []
|
|
if not isinstance(raw, list):
|
|
return items
|
|
for x in raw:
|
|
if not isinstance(x, dict):
|
|
continue
|
|
src = str(x.get("source") or "").strip().lower()
|
|
tid = str(x.get("id") or "").strip()
|
|
if src not in ("managed", "ume") or not tid:
|
|
continue
|
|
items.append(ConfigSyncTargetRef(source=src, id=tid)) # type: ignore[arg-type]
|
|
return items
|
|
|
|
|
|
def policy_to_out(row: ConfigSyncPolicy) -> ConfigSyncPolicyOut:
|
|
return ConfigSyncPolicyOut(
|
|
enabled=bool(row.enabled),
|
|
interval_days=max(1, int(row.interval_days or 3)),
|
|
concurrency=max(1, min(30, int(row.concurrency or 5))),
|
|
scope_mode=str(row.scope_mode or "all"),
|
|
selected_targets=_targets_from_json(row.selected_targets),
|
|
history_keep=max(0, min(30, int(row.history_keep if row.history_keep is not None else 3))),
|
|
cycle_keep=_cycle_keep_value(row),
|
|
updated_at=row.updated_at,
|
|
)
|
|
|
|
|
|
def get_policy(db: Session) -> ConfigSyncPolicyOut:
|
|
return policy_to_out(ensure_policy(db))
|
|
|
|
|
|
def update_policy(db: Session, body: ConfigSyncPolicyUpdate) -> ConfigSyncPolicyOut:
|
|
row = ensure_policy(db)
|
|
data = body.model_dump(exclude_unset=True)
|
|
if "enabled" in data and data["enabled"] is not None:
|
|
row.enabled = bool(data["enabled"])
|
|
if "interval_days" in data and data["interval_days"] is not None:
|
|
row.interval_days = int(data["interval_days"])
|
|
if "concurrency" in data and data["concurrency"] is not None:
|
|
row.concurrency = max(1, min(30, int(data["concurrency"])))
|
|
if "scope_mode" in data and data["scope_mode"] is not None:
|
|
row.scope_mode = str(data["scope_mode"])
|
|
if "selected_targets" in data and data["selected_targets"] is not None:
|
|
refs = data["selected_targets"]
|
|
row.selected_targets = [
|
|
{"source": r.source if hasattr(r, "source") else r["source"], "id": r.id if hasattr(r, "id") else r["id"]}
|
|
for r in refs
|
|
]
|
|
if "history_keep" in data and data["history_keep"] is not None:
|
|
row.history_keep = max(0, min(30, int(data["history_keep"])))
|
|
if "cycle_keep" in data and data["cycle_keep"] is not None:
|
|
row.cycle_keep = max(0, min(200, int(data["cycle_keep"])))
|
|
row.updated_at = _utcnow()
|
|
db.commit()
|
|
db.refresh(row)
|
|
prune_config_sync_cycles(db, keep=_cycle_keep_value(row))
|
|
return policy_to_out(row)
|
|
|
|
|
|
def cycle_to_out(row: ConfigSyncCycle) -> ConfigSyncCycleOut:
|
|
return ConfigSyncCycleOut(
|
|
id=str(row.id),
|
|
trigger_mode=str(row.trigger_mode or ""),
|
|
status=str(row.status or ""),
|
|
concurrency=int(row.concurrency or 0),
|
|
planned_count=int(row.planned_count or 0),
|
|
success_count=int(row.success_count or 0),
|
|
fail_count=int(row.fail_count or 0),
|
|
skip_count=int(row.skip_count or 0),
|
|
error_message=str(row.error_message or ""),
|
|
started_at=row.started_at,
|
|
ended_at=row.ended_at,
|
|
created_at=row.created_at,
|
|
)
|
|
|
|
|
|
def task_to_out(row: ConfigSyncTask) -> ConfigSyncTaskOut:
|
|
return ConfigSyncTaskOut(
|
|
id=str(row.id),
|
|
cycle_id=str(row.cycle_id),
|
|
source=str(row.source),
|
|
target_id=str(row.target_id),
|
|
ne_name=str(row.ne_name or ""),
|
|
ne_ip=str(row.ne_ip or ""),
|
|
vendor=str(row.vendor or ""),
|
|
status=str(row.status or ""),
|
|
message=str(row.message or ""),
|
|
started_at=row.started_at,
|
|
ended_at=row.ended_at,
|
|
)
|
|
|
|
|
|
def expand_targets(db: Session, policy: ConfigSyncPolicy) -> list[dict[str, str]]:
|
|
"""Return list of {source, id, ne_name, ne_ip, vendor, device_type}."""
|
|
mode = str(policy.scope_mode or "all").strip().lower()
|
|
out: list[dict[str, str]] = []
|
|
seen: set[tuple[str, str]] = set()
|
|
|
|
def _add(source: str, tid: str, name: str, ip: str, vendor: str, device_type: str) -> None:
|
|
key = (source, tid)
|
|
if key in seen:
|
|
return
|
|
seen.add(key)
|
|
out.append(
|
|
{
|
|
"source": source,
|
|
"id": tid,
|
|
"ne_name": name,
|
|
"ne_ip": ip,
|
|
"vendor": vendor,
|
|
"device_type": device_type,
|
|
}
|
|
)
|
|
|
|
if mode == "selected":
|
|
for ref in _targets_from_json(policy.selected_targets):
|
|
if ref.source == "managed":
|
|
ne = db.get(ManagedNE, ref.id)
|
|
if ne:
|
|
_add("managed", str(ne.id), str(ne.name or ""), str(ne.ip_address or ""), str(ne.vendor or ""), str(ne.device_type or ""))
|
|
else:
|
|
inv = db.get(UmeInventoryNE, ref.id)
|
|
if inv:
|
|
_add(
|
|
"ume",
|
|
str(inv.ne_id),
|
|
str(inv.host_name or inv.ne_name or inv.user_label or inv.ne_id or ""),
|
|
str(inv.ip_address or ""),
|
|
str(inv.vendor or ""),
|
|
str(inv.ne_type or ""),
|
|
)
|
|
return out
|
|
|
|
for ne in db.query(ManagedNE).order_by(ManagedNE.updated_at.desc()).all():
|
|
_add("managed", str(ne.id), str(ne.name or ""), str(ne.ip_address or ""), str(ne.vendor or ""), str(ne.device_type or ""))
|
|
|
|
if cli_profile_ready(db):
|
|
for inv in db.query(UmeInventoryNE).order_by(UmeInventoryNE.ne_id.asc()).all():
|
|
if not str(inv.ip_address or "").strip():
|
|
continue
|
|
_add(
|
|
"ume",
|
|
str(inv.ne_id),
|
|
str(inv.host_name or inv.ne_name or inv.user_label or inv.ne_id or ""),
|
|
str(inv.ip_address or ""),
|
|
str(inv.vendor or ""),
|
|
str(inv.ne_type or ""),
|
|
)
|
|
return out
|
|
|
|
|
|
def has_active_cycle(db: Session) -> ConfigSyncCycle | None:
|
|
"""Any non-terminal cycle occupies the single-flight slot (incl. paused)."""
|
|
return (
|
|
db.query(ConfigSyncCycle)
|
|
.filter(ConfigSyncCycle.status.in_(("running", "pending", "paused")))
|
|
.order_by(ConfigSyncCycle.created_at.desc())
|
|
.first()
|
|
)
|
|
|
|
|
|
def has_running_cycle(db: Session) -> ConfigSyncCycle | None:
|
|
"""Backward-compatible alias: treat paused as active so a new cycle cannot start."""
|
|
return has_active_cycle(db)
|
|
|
|
|
|
def last_finished_cycle(db: Session) -> ConfigSyncCycle | None:
|
|
return (
|
|
db.query(ConfigSyncCycle)
|
|
.filter(ConfigSyncCycle.status.in_(("success", "fail", "cancelled")))
|
|
.order_by(ConfigSyncCycle.ended_at.desc().nullslast(), ConfigSyncCycle.created_at.desc())
|
|
.first()
|
|
)
|
|
|
|
|
|
def next_due_at(db: Session, policy: ConfigSyncPolicy | None = None) -> datetime | None:
|
|
pol = policy or ensure_policy(db)
|
|
if not pol.enabled:
|
|
return None
|
|
from .config_sync_scheduler import startup_grace_until
|
|
|
|
last = (
|
|
db.query(ConfigSyncCycle)
|
|
.filter(ConfigSyncCycle.status == "success", ConfigSyncCycle.ended_at.isnot(None))
|
|
.order_by(ConfigSyncCycle.ended_at.desc())
|
|
.first()
|
|
)
|
|
days = max(1, int(pol.interval_days or 3))
|
|
if last and last.ended_at:
|
|
due = last.ended_at + timedelta(days=days)
|
|
else:
|
|
# Never synced successfully: do not fire immediately on enable / first boot.
|
|
due = _utcnow() + timedelta(days=days)
|
|
grace_until = startup_grace_until()
|
|
if grace_until is not None and due < grace_until:
|
|
return grace_until
|
|
return due
|
|
|
|
|
|
def create_cycle(db: Session, body: ConfigSyncCycleCreate) -> ConfigSyncCycleOut:
|
|
if has_running_cycle(db):
|
|
raise HTTPException(status_code=409, detail="config_sync_cycle_already_running")
|
|
policy = ensure_policy(db)
|
|
mode = str(body.mode or "full").strip().lower()
|
|
trigger = "retry_failed" if mode == "retry_failed" else "manual"
|
|
concurrency = max(1, min(30, int(policy.concurrency or 5)))
|
|
|
|
targets: list[dict[str, str]] = []
|
|
if mode == "retry_failed":
|
|
src_cycle_id = str(body.cycle_id or "").strip()
|
|
src = None
|
|
if src_cycle_id:
|
|
src = db.get(ConfigSyncCycle, src_cycle_id)
|
|
if src is None:
|
|
src = (
|
|
db.query(ConfigSyncCycle)
|
|
.filter(ConfigSyncCycle.fail_count > 0)
|
|
.order_by(ConfigSyncCycle.created_at.desc())
|
|
.first()
|
|
)
|
|
if src is None:
|
|
raise HTTPException(status_code=404, detail="no_failed_cycle")
|
|
fails = (
|
|
db.query(ConfigSyncTask)
|
|
.filter(ConfigSyncTask.cycle_id == src.id, ConfigSyncTask.status == "fail")
|
|
.all()
|
|
)
|
|
for t in fails:
|
|
targets.append(
|
|
{
|
|
"source": str(t.source),
|
|
"id": str(t.target_id),
|
|
"ne_name": str(t.ne_name or ""),
|
|
"ne_ip": str(t.ne_ip or ""),
|
|
"vendor": str(t.vendor or ""),
|
|
"device_type": "",
|
|
}
|
|
)
|
|
if not targets:
|
|
raise HTTPException(status_code=400, detail="no_failed_tasks")
|
|
else:
|
|
targets = expand_targets(db, policy)
|
|
if not targets:
|
|
raise HTTPException(status_code=400, detail="no_targets")
|
|
|
|
cycle = ConfigSyncCycle(
|
|
id=uuid4().hex,
|
|
trigger_mode=trigger,
|
|
status="running",
|
|
concurrency=concurrency,
|
|
planned_count=len(targets),
|
|
started_at=_utcnow(),
|
|
created_at=_utcnow(),
|
|
)
|
|
db.add(cycle)
|
|
db.flush()
|
|
for t in targets:
|
|
db.add(
|
|
ConfigSyncTask(
|
|
id=uuid4().hex,
|
|
cycle_id=cycle.id,
|
|
source=t["source"],
|
|
target_id=t["id"],
|
|
ne_name=t.get("ne_name") or "",
|
|
ne_ip=t.get("ne_ip") or "",
|
|
vendor=t.get("vendor") or "",
|
|
status="pending",
|
|
)
|
|
)
|
|
db.commit()
|
|
db.refresh(cycle)
|
|
return cycle_to_out(cycle)
|
|
|
|
|
|
def list_cycles(db: Session, *, page: int, page_size: int) -> dict[str, Any]:
|
|
q = db.query(ConfigSyncCycle).order_by(ConfigSyncCycle.created_at.desc())
|
|
total = int(q.count())
|
|
rows = q.offset((page - 1) * page_size).limit(page_size).all()
|
|
return {"total": total, "page": page, "page_size": page_size, "items": [cycle_to_out(r) for r in rows]}
|
|
|
|
|
|
def get_cycle(db: Session, cycle_id: str) -> ConfigSyncCycleOut:
|
|
row = db.get(ConfigSyncCycle, cycle_id)
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="cycle_not_found")
|
|
return cycle_to_out(row)
|
|
|
|
|
|
def list_cycle_tasks(
|
|
db: Session,
|
|
cycle_id: str,
|
|
*,
|
|
page: int,
|
|
page_size: int,
|
|
status: str = "",
|
|
keyword: str = "",
|
|
) -> dict[str, Any]:
|
|
if not db.get(ConfigSyncCycle, cycle_id):
|
|
raise HTTPException(status_code=404, detail="cycle_not_found")
|
|
q = db.query(ConfigSyncTask).filter(ConfigSyncTask.cycle_id == cycle_id)
|
|
st = str(status or "").strip()
|
|
if st:
|
|
q = q.filter(ConfigSyncTask.status == st)
|
|
kw = str(keyword or "").strip()
|
|
if kw:
|
|
like = f"%{kw}%"
|
|
q = q.filter(
|
|
or_(
|
|
ConfigSyncTask.ne_name.ilike(like),
|
|
ConfigSyncTask.ne_ip.ilike(like),
|
|
ConfigSyncTask.target_id.ilike(like),
|
|
ConfigSyncTask.message.ilike(like),
|
|
)
|
|
)
|
|
total = int(q.count())
|
|
rows = q.order_by(ConfigSyncTask.ne_name.asc()).offset((page - 1) * page_size).limit(page_size).all()
|
|
return {"total": total, "page": page, "page_size": page_size, "items": [task_to_out(r) for r in rows]}
|
|
|
|
|
|
def pause_cycle(db: Session, cycle_id: str) -> ConfigSyncCycleOut:
|
|
row = db.get(ConfigSyncCycle, cycle_id)
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="cycle_not_found")
|
|
if str(row.status) not in ("running", "pending"):
|
|
raise HTTPException(status_code=400, detail="cycle_not_running")
|
|
row.status = "paused"
|
|
db.commit()
|
|
db.refresh(row)
|
|
return cycle_to_out(row)
|
|
|
|
|
|
def resume_cycle(db: Session, cycle_id: str) -> ConfigSyncCycleOut:
|
|
row = db.get(ConfigSyncCycle, cycle_id)
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="cycle_not_found")
|
|
if str(row.status) != "paused":
|
|
raise HTTPException(status_code=400, detail="cycle_not_paused")
|
|
pending = (
|
|
db.query(ConfigSyncTask)
|
|
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status == "pending")
|
|
.count()
|
|
)
|
|
if pending <= 0:
|
|
raise HTTPException(status_code=400, detail="no_pending_tasks")
|
|
other = has_running_cycle(db)
|
|
if other and str(other.id) != cycle_id:
|
|
raise HTTPException(status_code=409, detail="config_sync_cycle_already_running")
|
|
row.status = "running"
|
|
db.commit()
|
|
db.refresh(row)
|
|
return cycle_to_out(row)
|
|
|
|
|
|
def stop_cycle(db: Session, cycle_id: str) -> ConfigSyncCycleOut:
|
|
"""Cancel remaining work and close the cycle (running/paused/pending)."""
|
|
row = db.get(ConfigSyncCycle, cycle_id)
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="cycle_not_found")
|
|
if str(row.status) not in ("running", "paused", "pending"):
|
|
raise HTTPException(status_code=400, detail="cycle_not_active")
|
|
now = _utcnow()
|
|
pending = (
|
|
db.query(ConfigSyncTask)
|
|
.filter(
|
|
ConfigSyncTask.cycle_id == cycle_id,
|
|
ConfigSyncTask.status.in_(("pending", "running")),
|
|
)
|
|
.all()
|
|
)
|
|
for task in pending:
|
|
# In-flight workers may still finish and overwrite; pending must not start.
|
|
if str(task.status) == "pending":
|
|
task.status = "cancelled"
|
|
task.message = "stopped_by_user"
|
|
task.ended_at = now
|
|
else:
|
|
task.message = (str(task.message or "") + " · stop_requested")[:1020]
|
|
row.status = "cancelled"
|
|
row.error_message = "stopped_by_user"
|
|
row.ended_at = now
|
|
db.commit()
|
|
sync_cycle_progress(db, cycle_id)
|
|
db.refresh(row)
|
|
try:
|
|
from .config_sync_runner import _release_pool
|
|
|
|
_release_pool(cycle_id)
|
|
except Exception:
|
|
pass
|
|
out = cycle_to_out(row)
|
|
try:
|
|
prune_config_sync_cycles(db, keep=_cycle_keep_value(ensure_policy(db)))
|
|
except Exception:
|
|
_log.exception("prune_config_sync_cycles after stop failed")
|
|
return out
|
|
|
|
|
|
def dashboard(db: Session) -> ConfigSyncDashboardOut:
|
|
policy = ensure_policy(db)
|
|
snap_count = int(db.query(func.count()).select_from(NeConfigSnapshot).scalar() or 0)
|
|
running = (
|
|
db.query(ConfigSyncCycle)
|
|
.filter(ConfigSyncCycle.status.in_(("running", "paused", "pending")))
|
|
.order_by(ConfigSyncCycle.created_at.desc())
|
|
.first()
|
|
)
|
|
last = last_finished_cycle(db)
|
|
fail_by_vendor: dict[str, int] = {}
|
|
if last:
|
|
rows = (
|
|
db.query(ConfigSyncTask.vendor, func.count())
|
|
.filter(ConfigSyncTask.cycle_id == last.id, ConfigSyncTask.status == "fail")
|
|
.group_by(ConfigSyncTask.vendor)
|
|
.all()
|
|
)
|
|
for vendor, cnt in rows:
|
|
fail_by_vendor[str(vendor or "unknown") or "unknown"] = int(cnt)
|
|
return ConfigSyncDashboardOut(
|
|
policy=policy_to_out(policy),
|
|
snapshot_count=snap_count,
|
|
last_cycle=cycle_to_out(last) if last else None,
|
|
running_cycle=cycle_to_out(running) if running else None,
|
|
next_due_at=next_due_at(db, policy),
|
|
fail_by_vendor=fail_by_vendor,
|
|
)
|
|
|
|
|
|
def _snap_meta(row: NeConfigSnapshot) -> NeConfigSnapshotMetaOut:
|
|
cmds = row.commands_json if isinstance(row.commands_json, list) else []
|
|
return NeConfigSnapshotMetaOut(
|
|
source=str(row.source),
|
|
target_id=str(row.target_id),
|
|
vendor=str(row.vendor or ""),
|
|
device_type=str(row.device_type or ""),
|
|
ne_name=str(row.ne_name or ""),
|
|
ne_ip=str(row.ne_ip or ""),
|
|
config_sha256=str(row.config_sha256 or ""),
|
|
config_alt_sha256=str(row.config_alt_sha256 or ""),
|
|
plain_size=int(row.plain_size or 0),
|
|
plain_alt_size=int(row.plain_alt_size or 0),
|
|
zlib_size=int(row.zlib_size or 0),
|
|
zlib_alt_size=int(row.zlib_alt_size or 0),
|
|
has_alt=bool(row.config_alt_zlib),
|
|
commands=[str(c) for c in cmds],
|
|
collected_at=row.collected_at,
|
|
last_cycle_id=str(row.last_cycle_id or ""),
|
|
)
|
|
|
|
|
|
def list_snapshots(
|
|
db: Session,
|
|
*,
|
|
page: int,
|
|
page_size: int,
|
|
keyword: str = "",
|
|
source: str = "",
|
|
vendor: str = "",
|
|
) -> dict[str, Any]:
|
|
q = db.query(NeConfigSnapshot)
|
|
src = str(source or "").strip().lower()
|
|
if src in ("managed", "ume"):
|
|
q = q.filter(NeConfigSnapshot.source == src)
|
|
vend = str(vendor or "").strip()
|
|
if vend:
|
|
q = q.filter(NeConfigSnapshot.vendor.ilike(f"%{vend}%"))
|
|
kw = str(keyword or "").strip()
|
|
if kw:
|
|
like = f"%{kw}%"
|
|
q = q.filter(
|
|
or_(
|
|
NeConfigSnapshot.ne_name.ilike(like),
|
|
NeConfigSnapshot.ne_ip.ilike(like),
|
|
NeConfigSnapshot.target_id.ilike(like),
|
|
)
|
|
)
|
|
total = int(q.count())
|
|
rows = q.order_by(NeConfigSnapshot.collected_at.desc()).offset((page - 1) * page_size).limit(page_size).all()
|
|
return {"total": total, "page": page, "page_size": page_size, "items": [_snap_meta(r) for r in rows]}
|
|
|
|
|
|
def get_snapshot_detail(
|
|
db: Session,
|
|
source: str,
|
|
target_id: str,
|
|
*,
|
|
field: str = "both",
|
|
) -> NeConfigSnapshotDetailOut:
|
|
src = str(source or "").strip().lower()
|
|
tid = str(target_id or "").strip()
|
|
row = db.get(NeConfigSnapshot, {"source": src, "target_id": tid})
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="snapshot_not_found")
|
|
meta = _snap_meta(row)
|
|
primary = ""
|
|
alt = ""
|
|
f = str(field or "both").strip().lower()
|
|
if f in ("primary", "both", ""):
|
|
primary = decompress_text(row.config_zlib)
|
|
if f in ("alt", "both") and row.config_alt_zlib:
|
|
alt = decompress_text(row.config_alt_zlib)
|
|
return NeConfigSnapshotDetailOut(**meta.model_dump(), config_text=primary, config_alt_text=alt)
|
|
|
|
|
|
def _safe_export_part(text: str) -> str:
|
|
s = re.sub(r'[<>:"/\\|?*\s]+', "_", str(text or "").strip())
|
|
return (s[:80] or "ne").strip("._") or "ne"
|
|
|
|
|
|
def build_snapshot_export(
|
|
db: Session,
|
|
source: str,
|
|
target_id: str,
|
|
*,
|
|
field: str = "primary",
|
|
) -> tuple[str, bytes, str]:
|
|
"""Return (filename, payload, media_type) for download."""
|
|
detail = get_snapshot_detail(db, source, target_id, field="both")
|
|
name = _safe_export_part(detail.ne_name or detail.target_id)
|
|
ip = _safe_export_part(detail.ne_ip or "ip")
|
|
base = f"{name}-{ip}-{detail.source}"
|
|
f = str(field or "primary").strip().lower()
|
|
|
|
if f == "alt":
|
|
if not detail.has_alt or not detail.config_alt_text:
|
|
raise HTTPException(status_code=404, detail="alt_config_not_found")
|
|
filename = f"{base}-hierarchical.txt"
|
|
return filename, detail.config_alt_text.encode("utf-8"), "text/plain; charset=utf-8"
|
|
|
|
if f == "both" and detail.has_alt and detail.config_alt_text:
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
|
zf.writestr(f"{base}-set.txt", detail.config_text or "")
|
|
zf.writestr(f"{base}-hierarchical.txt", detail.config_alt_text or "")
|
|
return f"{base}-configs.zip", buf.getvalue(), "application/zip"
|
|
|
|
filename = f"{base}-config.txt"
|
|
return filename, (detail.config_text or "").encode("utf-8"), "text/plain; charset=utf-8"
|
|
|
|
|
|
def list_snapshot_history(
|
|
db: Session,
|
|
source: str,
|
|
target_id: str,
|
|
*,
|
|
page: int,
|
|
page_size: int,
|
|
) -> dict[str, Any]:
|
|
src = str(source or "").strip().lower()
|
|
tid = str(target_id or "").strip()
|
|
q = (
|
|
db.query(NeConfigHistory)
|
|
.filter(NeConfigHistory.source == src, NeConfigHistory.target_id == tid)
|
|
.order_by(NeConfigHistory.collected_at.desc())
|
|
)
|
|
total = int(q.count())
|
|
rows = q.offset((page - 1) * page_size).limit(page_size).all()
|
|
items: list[NeConfigHistoryOut] = []
|
|
for row in rows:
|
|
cmds = row.commands_json if isinstance(row.commands_json, list) else []
|
|
items.append(
|
|
NeConfigHistoryOut(
|
|
id=str(row.id),
|
|
source=str(row.source),
|
|
target_id=str(row.target_id),
|
|
vendor=str(row.vendor or ""),
|
|
device_type=str(row.device_type or ""),
|
|
ne_name=str(row.ne_name or ""),
|
|
ne_ip=str(row.ne_ip or ""),
|
|
config_sha256=str(row.config_sha256 or ""),
|
|
config_alt_sha256=str(row.config_alt_sha256 or ""),
|
|
plain_size=int(row.plain_size or 0),
|
|
plain_alt_size=int(row.plain_alt_size or 0),
|
|
zlib_size=int(row.zlib_size or 0),
|
|
zlib_alt_size=int(row.zlib_alt_size or 0),
|
|
has_alt=bool(row.config_alt_zlib),
|
|
commands=[str(c) for c in cmds],
|
|
collected_at=row.collected_at,
|
|
last_cycle_id=str(row.cycle_id or ""),
|
|
cycle_id=str(row.cycle_id or ""),
|
|
)
|
|
)
|
|
return {"total": total, "page": page, "page_size": page_size, "items": items}
|
|
|
|
|
|
def sync_cycle_progress(db: Session, cycle_id: str) -> None:
|
|
cycle = db.get(ConfigSyncCycle, cycle_id)
|
|
if not cycle:
|
|
return
|
|
success = (
|
|
db.query(func.count())
|
|
.select_from(ConfigSyncTask)
|
|
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status == "success")
|
|
.scalar()
|
|
or 0
|
|
)
|
|
fail = (
|
|
db.query(func.count())
|
|
.select_from(ConfigSyncTask)
|
|
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status == "fail")
|
|
.scalar()
|
|
or 0
|
|
)
|
|
skip = (
|
|
db.query(func.count())
|
|
.select_from(ConfigSyncTask)
|
|
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status.in_(("skipped", "cancelled")))
|
|
.scalar()
|
|
or 0
|
|
)
|
|
cycle.success_count = int(success)
|
|
cycle.fail_count = int(fail)
|
|
cycle.skip_count = int(skip)
|
|
db.commit()
|
|
|
|
|
|
def finalize_cycle(db: Session, cycle_id: str) -> None:
|
|
cycle = db.get(ConfigSyncCycle, cycle_id)
|
|
if not cycle:
|
|
return
|
|
if str(cycle.status) in ("paused", "cancelled"):
|
|
return
|
|
pending = (
|
|
db.query(func.count())
|
|
.select_from(ConfigSyncTask)
|
|
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status.in_(("pending", "running")))
|
|
.scalar()
|
|
or 0
|
|
)
|
|
if int(pending) > 0:
|
|
return
|
|
sync_cycle_progress(db, cycle_id)
|
|
db.refresh(cycle)
|
|
if str(cycle.status) in ("paused", "cancelled"):
|
|
return
|
|
# Cycle outcome is about finishing the run, not per-NE results.
|
|
# Individual task failures stay in fail_count for retry/dashboard.
|
|
cycle.status = "success"
|
|
if cycle.error_message == "completed_with_failures":
|
|
cycle.error_message = ""
|
|
cycle.ended_at = _utcnow()
|
|
db.commit()
|
|
try:
|
|
prune_config_sync_cycles(db, keep=_cycle_keep_value(ensure_policy(db)))
|
|
except Exception:
|
|
_log.exception("prune_config_sync_cycles after finish failed")
|