mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 04:20:45 +08:00
Harden retention under one-click start: UME raw caps and collection TTL.
Keep inline schedulers as the start_netx.ps1 default while bounding UME raw_json and pruning old NE collection dirs on startup. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
e0503df8a5
commit
96be90bd1a
11 changed files with 167 additions and 14 deletions
|
|
@ -1,9 +1,14 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .config import settings
|
||||
|
||||
_log = logging.getLogger("netx.ne.collection.paths")
|
||||
|
||||
|
||||
def collection_data_root() -> Path:
|
||||
root = Path(str(settings.ne_collection_data_dir or "data/ne_collections"))
|
||||
|
|
@ -24,3 +29,42 @@ def clear_run_output_files(job_id: str, run_id: str) -> None:
|
|||
for path in out_dir.iterdir():
|
||||
if path.is_file():
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def prune_old_collection_dirs(*, keep_days: int | None = None) -> int:
|
||||
"""Remove top-level job directories older than ``keep_days``.
|
||||
|
||||
Returns number of job directories removed. Safe no-op when keep_days <= 0.
|
||||
"""
|
||||
days = int(
|
||||
keep_days
|
||||
if keep_days is not None
|
||||
else (getattr(settings, "ne_collection_keep_days", 14) or 0)
|
||||
)
|
||||
if days <= 0:
|
||||
return 0
|
||||
root = collection_data_root()
|
||||
cutoff = time.time() - (days * 86400)
|
||||
removed = 0
|
||||
try:
|
||||
children = list(root.iterdir())
|
||||
except Exception: # noqa: BLE001
|
||||
_log.exception("list collection root failed path=%s", root)
|
||||
return 0
|
||||
for path in children:
|
||||
if not path.is_dir():
|
||||
continue
|
||||
try:
|
||||
mtime = path.stat().st_mtime
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
if mtime >= cutoff:
|
||||
continue
|
||||
try:
|
||||
shutil.rmtree(path, ignore_errors=False)
|
||||
removed += 1
|
||||
except Exception: # noqa: BLE001
|
||||
_log.warning("prune collection dir failed path=%s", path, exc_info=True)
|
||||
if removed:
|
||||
_log.info("pruned %s ne_collection job dir(s) older than %s day(s)", removed, days)
|
||||
return removed
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue