Add NE capability descriptors and native async exec jobs for agents.

Expose recommended_mode/hints on getManagedNe, support background exec-jobs with getNeExecJob in MCP, and allow linux_shell on MikroTik as well as Linux.

Co-authored-by: Cursor <cursoragent@cursor.com>

netx_api/auth_scopes.py

@@ -118,6 +118,8 @@ def required_scope_for_request(method: str, path: str) -> str | None:
 
     if p.startswith("/v1/managed-ne"):
         path_tail = p.rstrip("/")
+        if "/exec-jobs" in p:
+            return SCOPE_NE_EXEC
         if m == "POST" and (path_tail.endswith("/exec") or path_tail.endswith("/exec-batch")):
             return SCOPE_NE_EXEC
         if m in ("POST", "PUT", "PATCH", "DELETE"):

netx_api/config.py

@@ -146,6 +146,12 @@ class Settings(BaseSettings):
     ne_exec_max_commands: int = 5
     # Opt-in: allow per-NE exec_policy (linux_shell/unrestricted). Default off — UI hidden.
     ne_exec_policy_enabled: bool = False
+    # Background exec jobs (MCP getNeExecJob / POST /v1/managed-ne/exec-jobs).
+    ne_exec_job_dir: str = "data/ne_exec_jobs"
+    ne_exec_job_max_concurrent: int = 3
+    ne_exec_job_timeout_s: int = 900
+    # Auto-async when batch NE count >= this (0 = only when async=true).
+    ne_exec_async_min_nes: int = 4
     # WebCRT interactive terminal sessions (multi-operator concurrent terminals).
     webcrt_max_sessions: int = 40
     # Per-user cap (0 = unlimited beyond global max).

netx_api/managed_ne_router.py

@@ -12,13 +12,16 @@ from .db import get_db
 from .device_types import SUPPORTED_VENDORS
 from .ne_connect import schedule_connect_tests
 from .ne_crypto import credentials_configured
+from .ne_capability import build_ne_capability
 from .ne_exec import execute_managed_ne_commands, execute_managed_ne_commands_batch
+from .ne_exec_jobs import get_ne_exec_job, start_ne_exec_job
 from .ne_schemas import (
     BatchAccountApplyRequest,
     BatchHopApplyRequest,
     ConnectTestRequest,
     ManagedNeCreate,
     ManagedNeExecBatchRequest,
+    ManagedNeExecJobCreate,
     ManagedNeExecRequest,
     ManagedNeUpdate,
 )
@@ -149,6 +152,39 @@ def api_delete_ume_synced_managed_ne(db: Session = Depends(get_db)):
     return delete_ume_synced_managed_ne(db).model_dump()
 
 
+def _audit_exec(
+    db: Session,
+    *,
+    uid: str,
+    uname: str,
+    path: str,
+    out: dict,
+    body_ne_id: str = "",
+    body_ume_ne_id: str = "",
+    commands: list | None = None,
+) -> None:
+    device = out.get("device") if isinstance(out.get("device"), dict) else {}
+    write_audit(
+        db,
+        action="ne.exec",
+        actor_user_id=uid,
+        actor_username=uname,
+        method="POST",
+        path=path,
+        status_code=200 if out.get("ok") else 502,
+        detail={
+            "ne_id": body_ne_id,
+            "ume_ne_id": body_ume_ne_id,
+            "ne_name": str(device.get("name") or device.get("ne_name") or ""),
+            "ne_ip": str(device.get("ip_address") or device.get("ip") or device.get("mgmt_ip") or ""),
+            "commands": list(out.get("commands") or commands or [])[:20],
+            "ok": bool(out.get("ok")),
+            "error": str(out.get("error") or "")[:500],
+            "output_len": len(str(out.get("output") or "")),
+        },
+    )
+
+
 @router.post("/exec")
 def api_exec_managed_ne(
     body: ManagedNeExecRequest,
@@ -164,25 +200,15 @@ def api_exec_managed_ne(
         ume_ne_id=body.ume_ne_id,
         read_timeout_sec=body.read_timeout_sec,
     )
-    device = out.get("device") if isinstance(out.get("device"), dict) else {}
-    write_audit(
+    _audit_exec(
         db,
-        action="ne.exec",
-        actor_user_id=uid,
-        actor_username=uname,
-        method="POST",
+        uid=uid,
+        uname=uname,
         path="/v1/managed-ne/exec",
-        status_code=200 if out.get("ok") else 502,
-        detail={
-            "ne_id": body.ne_id or "",
-            "ume_ne_id": body.ume_ne_id or "",
-            "ne_name": str(device.get("name") or device.get("ne_name") or ""),
-            "ne_ip": str(device.get("ip_address") or device.get("ip") or device.get("mgmt_ip") or ""),
-            "commands": list(out.get("commands") or body.commands or [])[:20],
-            "ok": bool(out.get("ok")),
-            "error": str(out.get("error") or "")[:500],
-            "output_len": len(str(out.get("output") or "")),
-        },
+        out=out,
+        body_ne_id=body.ne_id or "",
+        body_ume_ne_id=body.ume_ne_id or "",
+        commands=list(body.commands or []),
     )
     return out
 
@@ -235,6 +261,96 @@ def api_exec_managed_ne_batch(
     return out
 
 
+@router.post("/exec-jobs")
+def api_create_exec_job(
+    body: ManagedNeExecJobCreate,
+    ctx: Annotated[AuthContext, Depends(require_user)],
+    db: Session = Depends(get_db),
+):
+    """Start a background exec (single or batch). Poll GET /exec-jobs/{job_id}."""
+    from .db import SessionLocal
+
+    uid, uname = _actor(ctx)
+    payload = body.model_dump(exclude_none=True)
+    is_batch = bool(
+        payload.get("targets")
+        or payload.get("ne_ids")
+        or payload.get("ume_ne_ids")
+    )
+    kind = "exec-batch" if is_batch else "exec"
+
+    def _runner() -> dict:
+        with SessionLocal() as job_db:
+            if is_batch:
+                targets = payload.get("targets")
+                out = execute_managed_ne_commands_batch(
+                    targets=targets,
+                    ne_ids=payload.get("ne_ids"),
+                    ume_ne_ids=payload.get("ume_ne_ids"),
+                    commands=payload.get("commands"),
+                    read_timeout_sec=payload.get("read_timeout_sec"),
+                    concurrency=payload.get("concurrency"),
+                )
+                write_audit(
+                    job_db,
+                    action="ne.exec_batch",
+                    actor_user_id=uid,
+                    actor_username=uname,
+                    method="POST",
+                    path="/v1/managed-ne/exec-jobs",
+                    status_code=200,
+                    detail={
+                        "async": True,
+                        "ne_ids": list(payload.get("ne_ids") or [])[:100],
+                        "ume_ne_ids": list(payload.get("ume_ne_ids") or [])[:100],
+                        "ok": bool(out.get("ok", True)),
+                    },
+                )
+                return out
+            cmds = list(payload.get("commands") or [])
+            out = execute_managed_ne_commands(
+                job_db,
+                cmds,
+                ne_id=payload.get("ne_id"),
+                ume_ne_id=payload.get("ume_ne_id"),
+                read_timeout_sec=payload.get("read_timeout_sec"),
+            )
+            _audit_exec(
+                job_db,
+                uid=uid,
+                uname=uname,
+                path="/v1/managed-ne/exec-jobs",
+                out=out,
+                body_ne_id=str(payload.get("ne_id") or ""),
+                body_ume_ne_id=str(payload.get("ume_ne_id") or ""),
+                commands=cmds,
+            )
+            return out
+
+    ack = start_ne_exec_job(kind=kind, arguments=payload, runner=_runner)
+    write_audit(
+        db,
+        action="ne.exec_job_start",
+        actor_user_id=uid,
+        actor_username=uname,
+        method="POST",
+        path="/v1/managed-ne/exec-jobs",
+        status_code=200 if ack.get("ok") else 429,
+        detail={"job_id": ack.get("job_id") or "", "kind": kind, "ok": bool(ack.get("ok"))},
+    )
+    return ack
+
+
+@router.get("/exec-jobs/{job_id}")
+def api_get_exec_job(
+    job_id: str,
+    ctx: Annotated[AuthContext, Depends(require_user)],
+):
+    """Poll a background exec job started via POST /exec-jobs."""
+    _ = ctx
+    return get_ne_exec_job(job_id)
+
+
 @router.post("/connect-test")
 def api_connect_test(body: ConnectTestRequest, db: Session = Depends(get_db)):
     ids = [str(x).strip() for x in body.ids if str(x).strip()]
@@ -251,7 +367,14 @@ def api_connect_test(body: ConnectTestRequest, db: Session = Depends(get_db)):
 
 @router.get("/{ne_id}")
 def api_get_managed_ne(ne_id: str, db: Session = Depends(get_db)):
-    return get_managed_ne(db, ne_id).model_dump()
+    out = get_managed_ne(db, ne_id).model_dump()
+    out["capability"] = build_ne_capability(
+        device_type=out.get("device_type"),
+        exec_policy=out.get("exec_policy"),
+        vendor=out.get("vendor"),
+        hop_enabled=bool(out.get("hop_enabled")),
+    )
+    return out
 
 
 @router.patch("/{ne_id}")

netx_api/ne_capability.py

@@ -0,0 +1,110 @@
+"""Agent-facing capability descriptor for managed NE exec.
+
+Derived from device_type + exec_policy + runtime settings — not stored in DB.
+Callers (MCP / UI) should read this before choosing short CLI vs on-device script vs async job.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .ne_exec import _EXEC_BATCH_MAX_TARGETS, _EXEC_READ_TIMEOUT_DEFAULT, _EXEC_READ_TIMEOUT_MAX
+from .ne_exec_guard import (
+    EXEC_POLICY_LINUX_SHELL,
+    EXEC_POLICY_READONLY,
+    EXEC_POLICY_UNRESTRICTED,
+    _LINUX_SHELL_MAX_LEN,
+    _LINUX_SHELL_MAX_LINES,
+    _READONLY_MAX_LEN,
+    allows_open_exec_policy,
+    effective_exec_policy,
+    is_linux_device_type,
+    is_mikrotik_device_type,
+    normalize_exec_policy,
+)
+
+
+def _device_family(device_type: str | None) -> str:
+    if is_linux_device_type(device_type):
+        return "linux"
+    if is_mikrotik_device_type(device_type):
+        return "mikrotik"
+    return "network_cli"
+
+
+def _max_commands() -> int:
+    from .config import settings
+    from .ne_exec import _EXEC_MAX_COMMANDS_CAP
+
+    raw = int(getattr(settings, "ne_exec_max_commands", 5) or 5)
+    return max(1, min(_EXEC_MAX_COMMANDS_CAP, raw))
+
+
+def build_ne_capability(
+    *,
+    device_type: str | None,
+    exec_policy: str | None,
+    vendor: str | None = None,
+    hop_enabled: bool = False,
+) -> dict[str, Any]:
+    """Return a stable capability object for agents and ops UIs."""
+    stored = normalize_exec_policy(exec_policy)
+    effective = effective_exec_policy(exec_policy, device_type=device_type)
+    family = _device_family(device_type)
+    shell_ok = effective in (EXEC_POLICY_LINUX_SHELL, EXEC_POLICY_UNRESTRICTED)
+    open_eligible = allows_open_exec_policy(device_type)
+    max_cmds = _max_commands()
+
+    if effective == EXEC_POLICY_READONLY:
+        recommended = "show_only"
+        hints = [
+            "readonly: only show/display/ping/traceroute (optional pipe filters).",
+            "For shell/scripts set exec_policy=linux_shell on a Linux or MikroTik NE "
+            "(requires NETX_NE_EXEC_POLICY_ENABLED).",
+        ]
+    elif family == "linux":
+        recommended = "script_on_device"
+        hints = [
+            "Prefer short on-device scripts for multi-step work; avoid huge inline heredocs in MCP args.",
+            "Long / multi-NE work: execManagedNe async=true then poll getNeExecJob.",
+            f"Max {max_cmds} commands per call; combine steps in one script when needed.",
+        ]
+    elif family == "mikrotik":
+        recommended = "short_cli"
+        hints = [
+            "Prefer short RouterOS lines (comment=/find=); avoid one ultra-long line that may truncate.",
+            "Multiline scripts allowed under linux_shell; keep each logical change small and idempotent.",
+            "Long / multi-NE work: execManagedNe async=true then poll getNeExecJob.",
+        ]
+    else:
+        recommended = "show_only"
+        hints = ["network_cli: use vendor show/display; write changes are blocked under readonly."]
+
+    if hop_enabled:
+        hints.append("hop_enabled: expect higher latency; raise read_timeout_sec toward 90–120 if needed.")
+
+    return {
+        "vendor": str(vendor or "").strip() or None,
+        "device_type": str(device_type or "").strip() or None,
+        "device_family": family,
+        "exec_policy_stored": stored,
+        "exec_policy_effective": effective,
+        "allows_shell_scripts": bool(shell_ok),
+        "allows_multiline": bool(shell_ok),
+        "open_policy_eligible": bool(open_eligible),
+        "max_commands_per_call": max_cmds,
+        "max_command_len": _LINUX_SHELL_MAX_LEN if shell_ok else _READONLY_MAX_LEN,
+        "max_command_lines": _LINUX_SHELL_MAX_LINES if shell_ok else 1,
+        "read_timeout_sec": {
+            "default": _EXEC_READ_TIMEOUT_DEFAULT,
+            "max": _EXEC_READ_TIMEOUT_MAX,
+        },
+        "supports_batch_exec": True,
+        "batch_max_targets": _EXEC_BATCH_MAX_TARGETS,
+        "supports_async_job": True,
+        "recommended_mode": recommended,
+        "hints": hints,
+    }
+
+
+__all__ = ["build_ne_capability"]

netx_api/ne_exec_guard.py

@@ -3,7 +3,9 @@
 Policies (per managed NE ``exec_policy``):
 
 - ``readonly`` (default): network CLI only — show/display/ping/traceroute.
-- ``linux_shell``: single-line shell; no network prefix/pipe rules; no write-deny list.
+- ``linux_shell``: open shell/script on capable NEs (Linux, MikroTik RouterOS) —
+  pipes/&&/;/quotes/heredoc / RouterOS multiline scripts allowed;
+  no network prefix/pipe rules; no write-deny list.
 - ``unrestricted``: same as linux_shell (lab open); kept distinct for audit/UI.
 """
 
@@ -20,6 +22,11 @@ EXEC_POLICIES = frozenset(
     {EXEC_POLICY_READONLY, EXEC_POLICY_LINUX_SHELL, EXEC_POLICY_UNRESTRICTED}
 )
 
+# readonly: short show/ping lines. linux_shell: scripts / heredoc file writes for agents.
+_READONLY_MAX_LEN = 500
+_LINUX_SHELL_MAX_LEN = 65_536
+_LINUX_SHELL_MAX_LINES = 2_000
+
 # Block obvious config-change / destructive patterns (case-insensitive).
 _BLOCKED_RE = re.compile(
     r"(?i)("
@@ -63,6 +70,16 @@ def is_linux_device_type(device_type: str | None) -> bool:
     return low in ("linux", "linux_ssh", "linux_telnet") or low.startswith("linux_")
 
 
+def is_mikrotik_device_type(device_type: str | None) -> bool:
+    low = str(device_type or "").strip().lower()
+    return low in ("mikrotik_routeros", "mikrotik_switchos") or low.startswith("mikrotik_")
+
+
+def allows_open_exec_policy(device_type: str | None) -> bool:
+    """Device types that may use linux_shell / unrestricted (multiline scripts)."""
+    return is_linux_device_type(device_type) or is_mikrotik_device_type(device_type)
+
+
 def exec_policy_feature_enabled() -> bool:
     """Global kill-switch: off → always readonly (UI hidden, API rejects open policies)."""
     from .config import settings
@@ -71,11 +88,11 @@ def exec_policy_feature_enabled() -> bool:
 
 
 def effective_exec_policy(raw: str | None, *, device_type: str | None = None) -> str:
-    """Policy used at exec time (forces readonly when feature off or non-linux)."""
+    """Policy used at exec time (forces readonly when feature off or ineligible type)."""
     if not exec_policy_feature_enabled():
         return EXEC_POLICY_READONLY
     pol = normalize_exec_policy(raw)
-    if pol != EXEC_POLICY_READONLY and not is_linux_device_type(device_type):
+    if pol != EXEC_POLICY_READONLY and not allows_open_exec_policy(device_type):
         return EXEC_POLICY_READONLY
     return pol
 
@@ -85,14 +102,14 @@ def require_exec_policy_writable(
     *,
     device_type: str | None = None,
 ) -> str:
-    """Normalize for create/update; reject open policies when feature off or non-linux."""
+    """Normalize for create/update; reject open policies when feature off or ineligible type."""
     pol = normalize_exec_policy(raw)
     if pol == EXEC_POLICY_READONLY:
         return pol
     if not exec_policy_feature_enabled():
         raise HTTPException(status_code=400, detail="exec_policy_feature_disabled")
-    if not is_linux_device_type(device_type):
-        raise HTTPException(status_code=400, detail="exec_policy_requires_linux_device_type")
+    if not allows_open_exec_policy(device_type):
+        raise HTTPException(status_code=400, detail="exec_policy_requires_shell_device_type")
     return pol
 
 
@@ -109,11 +126,14 @@ def _validate_pipe_segments(cmd: str) -> None:
             raise HTTPException(status_code=400, detail="command_pipe_not_allowed")
 
 
-def _validate_single_line(cmd: str) -> None:
-    if any(ch in cmd for ch in ("\n", "\r")):
-        raise HTTPException(status_code=400, detail="command_chars_not_allowed")
+def _validate_linux_shell_command(cmd: str) -> None:
+    """Shell policy: allow multiline (heredoc / scripts); still reject exotic separators."""
     if any(sep in cmd for sep in _FORBIDDEN_LINE_SEPARATORS):
         raise HTTPException(status_code=400, detail="command_chars_not_allowed")
+    # Count lines after normalizing CRLF; blank trailing newline from strip() is already gone.
+    line_count = cmd.count("\n") + 1
+    if line_count > _LINUX_SHELL_MAX_LINES:
+        raise HTTPException(status_code=400, detail="command_too_many_lines")
 
 
 def _validate_readonly_command(cmd: str) -> None:
@@ -133,12 +153,17 @@ def validate_ne_exec_command(command: str, *, policy: str = EXEC_POLICY_READONLY
     cmd = str(command or "").strip()
     if not cmd:
         raise HTTPException(status_code=400, detail="empty_command")
-    if len(cmd) > 500:
-        raise HTTPException(status_code=400, detail="command_too_long")
     pol = normalize_exec_policy(policy)
+    max_len = (
+        _LINUX_SHELL_MAX_LEN
+        if pol in (EXEC_POLICY_LINUX_SHELL, EXEC_POLICY_UNRESTRICTED)
+        else _READONLY_MAX_LEN
+    )
+    if len(cmd) > max_len:
+        raise HTTPException(status_code=400, detail="command_too_long")
     if pol in (EXEC_POLICY_LINUX_SHELL, EXEC_POLICY_UNRESTRICTED):
-        # One command string per slot; shell metacharacters (|;&&`$) allowed.
-        _validate_single_line(cmd)
+        # Shell metacharacters (|;&&`$'"<<) and newlines (heredoc) allowed.
+        _validate_linux_shell_command(cmd)
         return
     _validate_readonly_command(cmd)
 

netx_api/ne_exec_jobs.py

@@ -0,0 +1,297 @@
+"""Background jobs for managed-NE exec (NetX-native; Linux + RouterOS + batch).
+
+Agents poll GET /v1/managed-ne/exec-jobs/{job_id} (MCP: getNeExecJob).
+Sync exec stays on POST /exec and /exec-batch; long or multi-NE work should use jobs.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+import uuid
+from pathlib import Path
+from typing import Any, Callable
+
+_STATUS_RUNNING = "running"
+_STATUS_SUCCEEDED = "succeeded"
+_STATUS_FAILED = "failed"
+_STATUS_TIMEOUT = "timeout"
+_TERMINAL = {_STATUS_SUCCEEDED, _STATUS_FAILED, _STATUS_TIMEOUT}
+_LOCK = threading.Lock()
+_INFLIGHT = 0
+_DEFAULT_TIMEOUT_S = 900
+_DEFAULT_MAX_CONCURRENT = 3
+_DEFAULT_ASYNC_MIN_NES = 4
+
+
+def _env_int(name: str, default: int, *, min_v: int, max_v: int) -> int:
+    raw = str(os.getenv(name) or "").strip()
+    if not raw:
+        try:
+            from .config import settings
+
+            attr = name.lower()
+            # Map common env → settings when present.
+            mapping = {
+                "NETX_NE_EXEC_JOB_MAX_CONCURRENT": "ne_exec_job_max_concurrent",
+                "NETX_NE_EXEC_ASYNC_MIN_NES": "ne_exec_async_min_nes",
+                "NETX_NE_EXEC_JOB_TIMEOUT_S": "ne_exec_job_timeout_s",
+            }
+            key = mapping.get(name)
+            if key and hasattr(settings, key):
+                return max(min_v, min(int(getattr(settings, key)), max_v))
+        except Exception:
+            pass
+        return default
+    try:
+        n = int(raw)
+    except Exception:
+        return default
+    return max(min_v, min(int(n), max_v))
+
+
+def _jobs_dir() -> Path:
+    override = str(os.getenv("NETX_NE_EXEC_JOB_DIR") or "").strip()
+    if override:
+        p = Path(override).expanduser().resolve()
+    else:
+        try:
+            from .config import settings
+
+            raw = str(getattr(settings, "ne_exec_job_dir", "") or "").strip()
+            if raw:
+                p = Path(raw).expanduser().resolve()
+            else:
+                p = Path("data/ne_exec_jobs").resolve()
+        except Exception:
+            p = Path("data/ne_exec_jobs").resolve()
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def _job_path(job_id: str) -> Path:
+    return _jobs_dir() / f"{job_id}.json"
+
+
+def _atomic_write(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(payload, ensure_ascii=False, default=str), encoding="utf-8")
+    tmp.replace(path)
+
+
+def _read_job(job_id: str) -> dict[str, Any] | None:
+    path = _job_path(job_id)
+    if not path.is_file():
+        return None
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def _write_job(job: dict[str, Any]) -> None:
+    jid = str(job.get("job_id") or "").strip()
+    if not jid:
+        return
+    _atomic_write(_job_path(jid), job)
+
+
+def async_min_ne_count() -> int:
+    """Auto-async when batch NE count >= this (0 disables auto; async=true still works)."""
+    return _env_int("NETX_NE_EXEC_ASYNC_MIN_NES", _DEFAULT_ASYNC_MIN_NES, min_v=0, max_v=50)
+
+
+def max_concurrent_jobs() -> int:
+    return _env_int("NETX_NE_EXEC_JOB_MAX_CONCURRENT", _DEFAULT_MAX_CONCURRENT, min_v=1, max_v=16)
+
+
+def default_job_timeout_s() -> int:
+    return _env_int("NETX_NE_EXEC_JOB_TIMEOUT_S", _DEFAULT_TIMEOUT_S, min_v=60, max_v=1800)
+
+
+def count_exec_ne_targets(args: dict[str, Any] | None) -> int:
+    a = args if isinstance(args, dict) else {}
+    n = 0
+    for key in ("ne_ids", "ume_ne_ids", "nms_ne_ids"):
+        val = a.get(key)
+        if isinstance(val, list):
+            n = max(n, len([x for x in val if str(x or "").strip()]))
+    targets = a.get("targets")
+    if isinstance(targets, list):
+        n = max(n, len([t for t in targets if isinstance(t, dict)]))
+    if n == 0 and (
+        str(a.get("ne_id") or "").strip()
+        or str(a.get("ume_ne_id") or "").strip()
+        or str(a.get("nms_ne_id") or "").strip()
+    ):
+        return 1
+    return int(n)
+
+
+def _truthy_async_flag(raw: Any) -> bool | None:
+    if raw is None:
+        return None
+    if isinstance(raw, bool):
+        return raw
+    text = str(raw).strip().lower()
+    if not text:
+        return None
+    if text in {"1", "true", "yes", "on"}:
+        return True
+    if text in {"0", "false", "no", "off"}:
+        return False
+    return None
+
+
+def should_run_exec_async(args: dict[str, Any] | None) -> bool:
+    a = dict(args or {})
+    flag = _truthy_async_flag(a.get("async"))
+    if flag is False:
+        return False
+    if flag is True:
+        return True
+    min_n = async_min_ne_count()
+    if min_n <= 0:
+        return False
+    return count_exec_ne_targets(a) >= int(min_n)
+
+
+def start_ne_exec_job(
+    *,
+    kind: str,
+    arguments: dict[str, Any],
+    runner: Callable[[], dict[str, Any]],
+    timeout_s: int | None = None,
+) -> dict[str, Any]:
+    """Start background runner; returns immediate ack with job_id."""
+    global _INFLIGHT
+    max_c = max_concurrent_jobs()
+    with _LOCK:
+        if _INFLIGHT >= max_c:
+            return {
+                "ok": False,
+                "error_code": "ne_exec_job_busy",
+                "error": "ne_exec_job_busy",
+                "hint": (
+                    f"Too many concurrent NE exec jobs (max {max_c}). "
+                    "Poll getNeExecJob for running jobs or shrink the batch."
+                ),
+            }
+        _INFLIGHT += 1
+
+    job_id = uuid.uuid4().hex
+    timeout = int(timeout_s) if timeout_s is not None else default_job_timeout_s()
+    timeout = max(60, min(timeout, 1800))
+    now = int(time.time() * 1000)
+    job = {
+        "job_id": job_id,
+        "status": _STATUS_RUNNING,
+        "kind": str(kind or "exec"),
+        "arguments": dict(arguments or {}),
+        "ne_count": count_exec_ne_targets(arguments),
+        "created_at_ms": now,
+        "updated_at_ms": now,
+        "timeout_s": timeout,
+        "result": None,
+        "error": "",
+    }
+    _write_job(job)
+
+    def _worker() -> None:
+        global _INFLIGHT
+        started = time.time()
+        try:
+            result = runner()
+            if not isinstance(result, dict):
+                result = {
+                    "ok": False,
+                    "error": "invalid_runner_result",
+                    "payload_type": type(result).__name__,
+                }
+            status = _STATUS_SUCCEEDED if result.get("ok") is not False else _STATUS_FAILED
+            if time.time() - started > timeout:
+                status = _STATUS_TIMEOUT
+            cur = _read_job(job_id) or job
+            cur.update(
+                {
+                    "status": status,
+                    "updated_at_ms": int(time.time() * 1000),
+                    "result": result,
+                    "error": str(result.get("error") or "") if status != _STATUS_SUCCEEDED else "",
+                    "duration_ms": int((time.time() - started) * 1000),
+                }
+            )
+            _write_job(cur)
+        except Exception as exc:
+            cur = _read_job(job_id) or job
+            cur.update(
+                {
+                    "status": _STATUS_FAILED,
+                    "updated_at_ms": int(time.time() * 1000),
+                    "error": f"{type(exc).__name__}: {exc}",
+                    "result": {"ok": False, "error": f"{type(exc).__name__}: {exc}"},
+                    "duration_ms": int((time.time() - started) * 1000),
+                }
+            )
+            _write_job(cur)
+        finally:
+            with _LOCK:
+                _INFLIGHT = max(0, _INFLIGHT - 1)
+
+    threading.Thread(target=_worker, name=f"ne-exec-{job_id[:8]}", daemon=True).start()
+    return {
+        "ok": True,
+        "async": True,
+        "status": _STATUS_RUNNING,
+        "job_id": job_id,
+        "ne_count": int(job["ne_count"]),
+        "poll_tool": "getNeExecJob",
+        "poll_path": f"/v1/managed-ne/exec-jobs/{job_id}",
+        "hint": (
+            f"exec started in background (job_id={job_id}, ne_count={job['ne_count']}). "
+            "Tell the user the job_id and end the turn; later call getNeExecJob(job_id=...) "
+            "or ask the user to continue. Do not sleep/busy-wait in this turn."
+        ),
+        "example_poll": {"job_id": job_id},
+    }
+
+
+def get_ne_exec_job(job_id: str) -> dict[str, Any]:
+    jid = str(job_id or "").strip()
+    if not jid:
+        return {"ok": False, "error_code": "job_id_required", "error": "job_id_required"}
+    job = _read_job(jid)
+    if not job:
+        return {"ok": False, "error_code": "job_not_found", "error": "job_not_found", "job_id": jid}
+    status = str(job.get("status") or "")
+    out: dict[str, Any] = {
+        "ok": True,
+        "job_id": jid,
+        "status": status,
+        "kind": job.get("kind"),
+        "ne_count": job.get("ne_count"),
+        "created_at_ms": job.get("created_at_ms"),
+        "updated_at_ms": job.get("updated_at_ms"),
+        "duration_ms": job.get("duration_ms"),
+        "terminal": status in _TERMINAL,
+    }
+    if status in _TERMINAL:
+        out["result"] = job.get("result")
+        if job.get("error"):
+            out["error"] = job.get("error")
+    else:
+        out["hint"] = "Still running; poll getNeExecJob again later or ask the user to continue."
+    return out
+
+
+__all__ = [
+    "async_min_ne_count",
+    "count_exec_ne_targets",
+    "get_ne_exec_job",
+    "should_run_exec_async",
+    "start_ne_exec_job",
+]

netx_api/ne_netmiko.py

@@ -149,12 +149,15 @@ def _send_command_expect_prompt(conn: Any, cmd: str, *, read_timeout: int) -> st
 
 
 def send_show_command(conn: Any, command: str, *, read_timeout: int = 120) -> str:
-    """Send a show/display command via ``send_command`` (wait for device prompt).
+    """Send one CLI command via ``send_command`` (wait for device prompt).
 
-    Do not use ``send_command_timing`` as the primary path for Cisco config
-    collection: long idle during ``Building configuration...`` is treated as
-    end-of-output and truncates the config. Timing is only a fallback when
-    expect-prompt returns empty after a channel drain (IOSv leftover-prompt bug).
+    Never re-sends the same command on empty/short output: this path is shared by
+    show collection and open-policy config/scripts (MikroTik ``add``, etc.). A
+    leftover-prompt empty read must be fixed by draining *before* send, not by
+    retrying — retries duplicate side-effecting commands.
+
+    Prefer ``send_command`` over ``send_command_timing`` for long config dumps:
+    idle during ``Building configuration...`` truncates timing reads.
 
     ``cmd_verify=False``: Netmiko's default echo check often raises
     ``Pattern not detected: 'show\\ lldp\\ ...'`` on IOSv / hop / slow echo paths.
@@ -163,19 +166,7 @@ def send_show_command(conn: Any, command: str, *, read_timeout: int = 120) -> st
     if not cmd:
         return ""
 
+    # IOSv / hop: find_prompt / paging-off often leave an extra prompt in the
+    # channel; drain so send_command does not match that leftover immediately.
     drain_read_channel(conn)
-    out = _send_command_expect_prompt(conn, cmd, read_timeout=read_timeout)
-    if out.strip():
-        return out
-
-    # Leftover prompt matched before command echo — drain again and retry once.
-    drain_read_channel(conn)
-    out = _send_command_expect_prompt(conn, cmd, read_timeout=read_timeout)
-    if out.strip():
-        return out
-
-    drain_read_channel(conn)
-    try:
-        return str(conn.send_command_timing(cmd, read_timeout=read_timeout) or "")
-    except Exception:
-        return ""
+    return _send_command_expect_prompt(conn, cmd, read_timeout=read_timeout)

netx_api/ne_schemas.py

@@ -176,6 +176,19 @@ class ManagedNeExecBatchRequest(BaseModel):
     concurrency: int | None = Field(default=4, ge=1, le=8)
 
 
+class ManagedNeExecJobCreate(BaseModel):
+    """Start a background single or batch exec (poll GET /v1/managed-ne/exec-jobs/{job_id})."""
+
+    ne_id: str | None = None
+    ume_ne_id: str | None = None
+    targets: list[ManagedNeExecBatchTarget] | None = Field(default=None, max_length=20)
+    ne_ids: list[str] | None = Field(default=None, max_length=20)
+    ume_ne_ids: list[str] | None = Field(default=None, max_length=20)
+    commands: list[str] | None = Field(default=None, max_length=50)
+    read_timeout_sec: int | None = Field(default=None, ge=10, le=120)
+    concurrency: int | None = Field(default=4, ge=1, le=8)
+
+
 class HopProxyConfig(BaseModel):
     """Shared jump-host (proxy) settings applied to one or many NEs."""
 

netx_api/ne_service_crud.py

@@ -35,7 +35,7 @@ from .ne_service_common import (
 )
 from .ne_exec_guard import (
     EXEC_POLICY_READONLY,
-    is_linux_device_type,
+    allows_open_exec_policy,
     require_exec_policy_writable,
 )
 
@@ -173,8 +173,8 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed
             str(data["exec_policy"]),
             device_type=row.device_type,
         )
-    elif "device_type" in data and not is_linux_device_type(row.device_type):
-        # Leaving linux clears any previously open policy.
+    elif "device_type" in data and not allows_open_exec_policy(row.device_type):
+        # Leaving linux/mikrotik clears any previously open policy.
         row.exec_policy = EXEC_POLICY_READONLY
     if "password" in data and data["password"]:
         _require_crypto()