"""Background jobs for managed-NE exec (NetX-native; Linux + RouterOS + batch). Agents poll GET /v1/managed-ne/exec-jobs/{job_id} (MCP: getNeExecJob). Sync exec stays on POST /exec and /exec-batch; long or multi-NE work should use jobs. """ from __future__ import annotations import json import os import re import threading import time import uuid from pathlib import Path from typing import Any, Callable _STATUS_RUNNING = "running" _STATUS_SUCCEEDED = "succeeded" _STATUS_FAILED = "failed" _STATUS_TIMEOUT = "timeout" _TERMINAL = {_STATUS_SUCCEEDED, _STATUS_FAILED, _STATUS_TIMEOUT} _LOCK = threading.Lock() _INFLIGHT = 0 _DEFAULT_TIMEOUT_S = 900 _DEFAULT_MAX_CONCURRENT = 3 _DEFAULT_ASYNC_MIN_NES = 4 _JOB_ID_RE = re.compile(r"[0-9a-f]{32}") _MAX_JOB_TARGETS = 20 # Same combined limit as ne_exec._normalize_batch_targets. def _env_int(name: str, default: int, *, min_v: int, max_v: int) -> int: raw = str(os.getenv(name) or "").strip() if not raw: try: from .config import settings attr = name.lower() # Map common env → settings when present. mapping = { "NETX_NE_EXEC_JOB_MAX_CONCURRENT": "ne_exec_job_max_concurrent", "NETX_NE_EXEC_ASYNC_MIN_NES": "ne_exec_async_min_nes", "NETX_NE_EXEC_JOB_TIMEOUT_S": "ne_exec_job_timeout_s", } key = mapping.get(name) if key and hasattr(settings, key): return max(min_v, min(int(getattr(settings, key)), max_v)) except Exception: pass return default try: n = int(raw) except Exception: return default return max(min_v, min(int(n), max_v)) def _jobs_dir() -> Path: override = str(os.getenv("NETX_NE_EXEC_JOB_DIR") or "").strip() if override: p = Path(override).expanduser().resolve() else: try: from .config import settings raw = str(getattr(settings, "ne_exec_job_dir", "") or "").strip() if raw: p = Path(raw).expanduser().resolve() else: p = Path("data/ne_exec_jobs").resolve() except Exception: p = Path("data/ne_exec_jobs").resolve() p.mkdir(parents=True, exist_ok=True) return p def _job_path(job_id: str) -> Path: if not isinstance(job_id, str) or not _JOB_ID_RE.fullmatch(job_id): raise ValueError("invalid_job_id") directory = _jobs_dir().resolve() path = (directory / f"{job_id}.json").resolve() # Also reject a valid-looking filename symlinked outside the job directory. if path.parent != directory: raise ValueError("invalid_job_id") return path def _atomic_write(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(path.suffix + ".tmp") tmp.write_text(json.dumps(payload, ensure_ascii=False, default=str), encoding="utf-8") tmp.replace(path) def _read_job(job_id: str) -> dict[str, Any] | None: path = _job_path(job_id) if not path.is_file(): return None try: data = json.loads(path.read_text(encoding="utf-8")) except Exception: return None return data if isinstance(data, dict) and data.get("job_id") == job_id else None def _write_job(job: dict[str, Any]) -> None: jid = job.get("job_id") _atomic_write(_job_path(jid), job) def async_min_ne_count() -> int: """Auto-async when batch NE count >= this (0 disables auto; async=true still works).""" return _env_int("NETX_NE_EXEC_ASYNC_MIN_NES", _DEFAULT_ASYNC_MIN_NES, min_v=0, max_v=50) def max_concurrent_jobs() -> int: return _env_int("NETX_NE_EXEC_JOB_MAX_CONCURRENT", _DEFAULT_MAX_CONCURRENT, min_v=1, max_v=16) def default_job_timeout_s() -> int: return _env_int("NETX_NE_EXEC_JOB_TIMEOUT_S", _DEFAULT_TIMEOUT_S, min_v=60, max_v=1800) def count_exec_ne_targets(args: dict[str, Any] | None) -> int: a = args if isinstance(args, dict) else {} n = 0 for key in ("ne_ids", "ume_ne_ids"): val = a.get(key) if isinstance(val, list): n += len([x for x in val if str(x or "").strip()]) targets = a.get("targets") if isinstance(targets, list): n += len([t for t in targets if isinstance(t, dict)]) if n == 0 and ( str(a.get("ne_id") or "").strip() or str(a.get("ume_ne_id") or "").strip() ): return 1 return int(n) def _truthy_async_flag(raw: Any) -> bool | None: if raw is None: return None if isinstance(raw, bool): return raw text = str(raw).strip().lower() if not text: return None if text in {"1", "true", "yes", "on"}: return True if text in {"0", "false", "no", "off"}: return False return None def should_run_exec_async(args: dict[str, Any] | None) -> bool: a = dict(args or {}) flag = _truthy_async_flag(a.get("async")) if flag is False: return False if flag is True: return True min_n = async_min_ne_count() if min_n <= 0: return False return count_exec_ne_targets(a) >= int(min_n) def start_ne_exec_job( *, kind: str, arguments: dict[str, Any], owner_user_id: str, runner: Callable[[], dict[str, Any]], timeout_s: int | None = None, ) -> dict[str, Any]: """Start background runner; returns immediate ack with job_id.""" global _INFLIGHT owner = str(owner_user_id or "").strip() if not owner: return {"ok": False, "error_code": "job_owner_required", "error": "job_owner_required"} ne_count = count_exec_ne_targets(arguments) if ne_count > _MAX_JOB_TARGETS: return {"ok": False, "error_code": "too_many_targets", "error": "too_many_targets", "max_targets": _MAX_JOB_TARGETS} max_c = max_concurrent_jobs() with _LOCK: if _INFLIGHT >= max_c: return { "ok": False, "error_code": "ne_exec_job_busy", "error": "ne_exec_job_busy", "hint": ( f"Too many concurrent NE exec jobs (max {max_c}). " "Poll getNeExecJob for running jobs or shrink the batch." ), } _INFLIGHT += 1 job_id = uuid.uuid4().hex timeout = int(timeout_s) if timeout_s is not None else default_job_timeout_s() timeout = max(60, min(timeout, 1800)) now = int(time.time() * 1000) job = { "job_id": job_id, "owner_user_id": owner, "status": _STATUS_RUNNING, "kind": str(kind or "exec"), "arguments": dict(arguments or {}), "ne_count": ne_count, "created_at_ms": now, "updated_at_ms": now, "timeout_s": timeout, "result": None, "error": "", } _write_job(job) def _worker() -> None: global _INFLIGHT started = time.time() try: result = runner() if not isinstance(result, dict): result = { "ok": False, "error": "invalid_runner_result", "payload_type": type(result).__name__, } status = _STATUS_SUCCEEDED if result.get("ok") is not False else _STATUS_FAILED if time.time() - started > timeout: status = _STATUS_TIMEOUT cur = _read_job(job_id) or job cur.update( { "status": status, "updated_at_ms": int(time.time() * 1000), "result": result, "error": str(result.get("error") or "") if status != _STATUS_SUCCEEDED else "", "duration_ms": int((time.time() - started) * 1000), } ) _write_job(cur) except Exception as exc: cur = _read_job(job_id) or job cur.update( { "status": _STATUS_FAILED, "updated_at_ms": int(time.time() * 1000), "error": f"{type(exc).__name__}: {exc}", "result": {"ok": False, "error": f"{type(exc).__name__}: {exc}"}, "duration_ms": int((time.time() - started) * 1000), } ) _write_job(cur) finally: with _LOCK: _INFLIGHT = max(0, _INFLIGHT - 1) threading.Thread(target=_worker, name=f"ne-exec-{job_id[:8]}", daemon=True).start() return { "ok": True, "async": True, "status": _STATUS_RUNNING, "job_id": job_id, "ne_count": int(job["ne_count"]), "poll_tool": "getNeExecJob", "poll_path": f"/v1/managed-ne/exec-jobs/{job_id}", "hint": ( f"exec started in background (job_id={job_id}, ne_count={job['ne_count']}). " "Tell the user the job_id and end the turn; later call getNeExecJob(job_id=...) " "or ask the user to continue. Do not sleep/busy-wait in this turn." ), "example_poll": {"job_id": job_id}, } def get_ne_exec_job(job_id: str, *, user_id: str, is_admin: bool = False) -> dict[str, Any]: """Read only the caller's jobs; explicit admins may inspect any persisted job. Ownerless legacy records are never granted to ordinary callers. """ jid = job_id if not jid: return {"ok": False, "error_code": "job_id_required", "error": "job_id_required"} try: job = _read_job(jid) except ValueError: return {"ok": False, "error_code": "invalid_job_id", "error": "invalid_job_id"} requester = str(user_id or "").strip() if not requester: return {"ok": False, "error_code": "job_forbidden", "error": "job_forbidden"} if not job: return {"ok": False, "error_code": "job_not_found", "error": "job_not_found", "job_id": jid} owner = str(job.get("owner_user_id") or "").strip() if not is_admin and (not owner or owner != requester): return {"ok": False, "error_code": "job_forbidden", "error": "job_forbidden"} status = str(job.get("status") or "") out: dict[str, Any] = { "ok": True, "job_id": jid, "status": status, "kind": job.get("kind"), "ne_count": job.get("ne_count"), "created_at_ms": job.get("created_at_ms"), "updated_at_ms": job.get("updated_at_ms"), "duration_ms": job.get("duration_ms"), "terminal": status in _TERMINAL, } if status in _TERMINAL: out["result"] = job.get("result") if job.get("error"): out["error"] = job.get("error") else: out["hint"] = "Still running; poll getNeExecJob again later or ask the user to continue." return out __all__ = [ "async_min_ne_count", "count_exec_ne_targets", "get_ne_exec_job", "should_run_exec_async", "start_ne_exec_job", ]