mirror of
https://github.com/hansjone/netx.git
synced 2026-10-15 14:33:13 +08:00
Run heavy CLI dumps on a dedicated long-timeout SSH lane.
Split interface detail/route-class collects from the light lane so they no longer block other checks, and scope Collect Now busy state per task. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
99cec1c324
commit
f279431c70
5 changed files with 675 additions and 301 deletions
|
|
@ -3,6 +3,8 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
@ -39,6 +41,42 @@ from .profiles import get_profile
|
||||||
|
|
||||||
_log = logging.getLogger("netx.biz_state.runner")
|
_log = logging.getLogger("netx.biz_state.runner")
|
||||||
|
|
||||||
|
# concrete, params, profile_id, item_id, mode
|
||||||
|
WorkItem = tuple[str, dict[str, str], str, str, str]
|
||||||
|
|
||||||
|
_heavy_pool_lock = threading.Lock()
|
||||||
|
_heavy_pool: ThreadPoolExecutor | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _heavy_cli_pool() -> ThreadPoolExecutor:
|
||||||
|
global _heavy_pool
|
||||||
|
with _heavy_pool_lock:
|
||||||
|
if _heavy_pool is None:
|
||||||
|
n = max(1, int(getattr(settings, "biz_state_heavy_workers", 4) or 4))
|
||||||
|
_heavy_pool = ThreadPoolExecutor(max_workers=n, thread_name_prefix="biz-heavy")
|
||||||
|
return _heavy_pool
|
||||||
|
|
||||||
|
|
||||||
|
def work_item_lane(profile_id: str, mode: str) -> str:
|
||||||
|
"""Return collect lane for a work item (custom_raw always light)."""
|
||||||
|
if str(mode or "").strip() == "custom":
|
||||||
|
return "light"
|
||||||
|
profile = get_profile(profile_id) or _resolve_collect_profile(profile_id)
|
||||||
|
lane = str(getattr(profile, "collect_lane", "light") or "light").strip().lower() if profile else "light"
|
||||||
|
return "heavy" if lane == "heavy" else "light"
|
||||||
|
|
||||||
|
|
||||||
|
def partition_work(work: list[WorkItem]) -> tuple[list[WorkItem], list[WorkItem]]:
|
||||||
|
"""Split work into (light, heavy) lanes."""
|
||||||
|
light: list[WorkItem] = []
|
||||||
|
heavy: list[WorkItem] = []
|
||||||
|
for item in work:
|
||||||
|
if work_item_lane(item[2], item[4]) == "heavy":
|
||||||
|
heavy.append(item)
|
||||||
|
else:
|
||||||
|
light.append(item)
|
||||||
|
return light, heavy
|
||||||
|
|
||||||
|
|
||||||
def _utcnow() -> datetime:
|
def _utcnow() -> datetime:
|
||||||
return datetime.utcnow()
|
return datetime.utcnow()
|
||||||
|
|
@ -311,113 +349,23 @@ def dispatch_collect(task_id: str, *, manual: bool = False) -> None:
|
||||||
_finish_task(task_id, error=error)
|
_finish_task(task_id, error=error)
|
||||||
|
|
||||||
|
|
||||||
def _run_collect_session(
|
def _run_collect_lane(
|
||||||
*,
|
*,
|
||||||
task_id: str,
|
work: list[WorkItem],
|
||||||
batch_id: str,
|
batch_id: str,
|
||||||
source: str,
|
creds: dict[str, Any],
|
||||||
ne_id: str,
|
vendor_eff: str,
|
||||||
vendor: str,
|
device_type_eff: str,
|
||||||
device_type: str,
|
vendor_key: str,
|
||||||
) -> None:
|
per_cmd: int,
|
||||||
per_cmd = int(settings.ne_collect_read_timeout_sec or 120)
|
cap: int,
|
||||||
cap = int(settings.ne_collect_run_timeout_cap_sec or 600)
|
label: str,
|
||||||
|
) -> tuple[int, int, bool, bool]:
|
||||||
db = SessionLocal()
|
"""Run one SSH lane (own connection + CollectSession + timeout budget)."""
|
||||||
try:
|
|
||||||
task = db.get(BizStateTask, task_id)
|
|
||||||
batch = db.get(BizStateBatch, batch_id)
|
|
||||||
if not task or not batch:
|
|
||||||
return
|
|
||||||
|
|
||||||
try:
|
|
||||||
if source == "managed":
|
|
||||||
creds, info = resolve_cli_target(db, managed_ne_id=ne_id)
|
|
||||||
elif source == "ume":
|
|
||||||
creds, info = resolve_cli_target(db, ume_ne_id=ne_id)
|
|
||||||
else:
|
|
||||||
raise RuntimeError("invalid_source")
|
|
||||||
except HTTPException as exc:
|
|
||||||
raise RuntimeError(str(exc.detail or "resolve_failed")) from exc
|
|
||||||
|
|
||||||
skip = cli_creds_skip_reason(creds, interactive=False)
|
|
||||||
if skip:
|
|
||||||
raise RuntimeError(skip)
|
|
||||||
|
|
||||||
vendor_eff = str(info.get("vendor") or vendor or "")
|
|
||||||
device_type_eff = str(info.get("device_type") or device_type or "")
|
|
||||||
if vendor_eff and vendor_eff != task.vendor:
|
|
||||||
task.vendor = vendor_eff
|
|
||||||
if device_type_eff and device_type_eff != task.device_type:
|
|
||||||
task.device_type = device_type_eff
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
vendor_key = resolve_vendor_key(vendor_eff, device_type_eff)
|
|
||||||
|
|
||||||
items = (
|
|
||||||
db.query(BizStateTaskItem)
|
|
||||||
.filter(
|
|
||||||
BizStateTaskItem.task_id == task_id,
|
|
||||||
BizStateTaskItem.enabled.is_(True),
|
|
||||||
)
|
|
||||||
.order_by(BizStateTaskItem.sort_order.asc())
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
|
|
||||||
# Build work list before opening session
|
|
||||||
work: list[tuple[str, dict[str, str], str, str, str]] = []
|
|
||||||
# concrete, params, profile_id, item_id, mode
|
|
||||||
# Dedupe same CLI → same metric (e.g. legacy if_intf + config_interface).
|
|
||||||
seen_work: set[tuple[str, str]] = set()
|
|
||||||
for item in items:
|
|
||||||
if item.kind == "custom_raw":
|
|
||||||
cmd = normalize_command(item.command_override)
|
|
||||||
if cmd:
|
|
||||||
key = (cmd, "__custom__")
|
|
||||||
if key in seen_work:
|
|
||||||
continue
|
|
||||||
seen_work.add(key)
|
|
||||||
work.append((cmd, {}, "", item.id, "custom"))
|
|
||||||
continue
|
|
||||||
profile = _resolve_collect_profile(item.source_profile_id)
|
|
||||||
if profile is None:
|
|
||||||
_append_event(
|
|
||||||
db,
|
|
||||||
task_id=task_id,
|
|
||||||
message=f"skip profile {item.source_profile_id} (missing or disabled)",
|
|
||||||
level="info",
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
binds = _bindings_for_item(db, item.id)
|
|
||||||
try:
|
|
||||||
pairs = expand_from_bindings(
|
|
||||||
profile=profile,
|
|
||||||
bindings=binds,
|
|
||||||
command_override=item.command_override,
|
|
||||||
)
|
|
||||||
except ValueError as exc:
|
|
||||||
_append_event(db, task_id=task_id, message=str(exc), level="error")
|
|
||||||
continue
|
|
||||||
for concrete, params in pairs:
|
|
||||||
cmd = normalize_command(concrete)
|
|
||||||
# Prefer match_command metric so remapped if_intf shares key with config_interface
|
|
||||||
hit = match_command(vendor_key=vendor_key, command=cmd)
|
|
||||||
mid = str((hit.profile.metric_id if hit else profile.metric_id) or "").strip()
|
|
||||||
pid = str((hit.profile.profile_id if hit else profile.profile_id) or "").strip()
|
|
||||||
key = (cmd, mid or pid)
|
|
||||||
if key in seen_work:
|
|
||||||
continue
|
|
||||||
seen_work.add(key)
|
|
||||||
work.append((cmd, params, pid or profile.profile_id, item.id, "normal"))
|
|
||||||
|
|
||||||
if not work:
|
if not work:
|
||||||
batch.status = "failed"
|
return 0, 0, False, False
|
||||||
batch.message = "no commands to run"
|
|
||||||
batch.ended_at = _utcnow()
|
|
||||||
db.commit()
|
|
||||||
raise RuntimeError("no commands to run")
|
|
||||||
|
|
||||||
budget = min(cap, per_cmd * max(1, len(work)) + 90)
|
budget = min(int(cap), int(per_cmd) * max(1, len(work)) + 90)
|
||||||
holder: dict[str, Any] = {}
|
holder: dict[str, Any] = {}
|
||||||
|
|
||||||
def _session() -> tuple[int, int, bool, bool]:
|
def _session() -> tuple[int, int, bool, bool]:
|
||||||
|
|
@ -457,7 +405,7 @@ def _run_collect_session(
|
||||||
return 0, 0, True, False
|
return 0, 0, True, False
|
||||||
for concrete, params, profile_id, item_id, mode in work:
|
for concrete, params, profile_id, item_id, mode in work:
|
||||||
if holder.get("timed_out"):
|
if holder.get("timed_out"):
|
||||||
raise TimeoutError("biz_state_aborted")
|
raise TimeoutError(f"{label}_aborted")
|
||||||
cmd_count += 1
|
cmd_count += 1
|
||||||
cmd_row = BizStateBatchCommand(
|
cmd_row = BizStateBatchCommand(
|
||||||
id=uuid4().hex,
|
id=uuid4().hex,
|
||||||
|
|
@ -517,7 +465,6 @@ def _run_collect_session(
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if cache_hit_primary:
|
if cache_hit_primary:
|
||||||
# Keep existing cache entry; primary will re-parse from shared raw.
|
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
session.remember(
|
session.remember(
|
||||||
|
|
@ -587,7 +534,6 @@ def _run_collect_session(
|
||||||
aux_row.message = f"aux_for={cmd_row.id}"[:1020]
|
aux_row.message = f"aux_for={cmd_row.id}"[:1020]
|
||||||
aux_row.raw_text = entry.raw
|
aux_row.raw_text = entry.raw
|
||||||
aux_row.row_count = len(entry.records or [])
|
aux_row.row_count = len(entry.records or [])
|
||||||
# refresh cache row id to this aux row on first fetch
|
|
||||||
entry.cmd_row_id = aux_row.id
|
entry.cmd_row_id = aux_row.id
|
||||||
sdb.add(aux_row)
|
sdb.add(aux_row)
|
||||||
sdb.commit()
|
sdb.commit()
|
||||||
|
|
@ -672,16 +618,234 @@ def _run_collect_session(
|
||||||
close_netmiko_connection(conn)
|
close_netmiko_connection(conn)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
total_rows, cmd_count, any_fail, any_ok = run_cli_with_timeout(
|
return run_cli_with_timeout(
|
||||||
_session,
|
_session,
|
||||||
timeout_sec=budget,
|
timeout_sec=budget,
|
||||||
conn_holder=holder,
|
conn_holder=holder,
|
||||||
label="biz_state",
|
label=label,
|
||||||
acquire_budget=True,
|
acquire_budget=True,
|
||||||
)
|
)
|
||||||
except TimeoutError as exc:
|
except TimeoutError as exc:
|
||||||
raise RuntimeError(str(exc)[:1020]) from exc
|
raise RuntimeError(str(exc)[:1020]) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _absorb_lane_result(
|
||||||
|
result: tuple[int, int, bool, bool] | BaseException,
|
||||||
|
*,
|
||||||
|
total_rows: int,
|
||||||
|
cmd_count: int,
|
||||||
|
any_fail: bool,
|
||||||
|
any_ok: bool,
|
||||||
|
lane_errors: list[str],
|
||||||
|
) -> tuple[int, int, bool, bool]:
|
||||||
|
if isinstance(result, BaseException):
|
||||||
|
lane_errors.append(_format_error(result))
|
||||||
|
return total_rows, cmd_count, True, any_ok
|
||||||
|
rows, cmds, fail, ok = result
|
||||||
|
return (
|
||||||
|
total_rows + int(rows or 0),
|
||||||
|
cmd_count + int(cmds or 0),
|
||||||
|
any_fail or bool(fail),
|
||||||
|
any_ok or bool(ok),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_collect_session(
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
batch_id: str,
|
||||||
|
source: str,
|
||||||
|
ne_id: str,
|
||||||
|
vendor: str,
|
||||||
|
device_type: str,
|
||||||
|
) -> None:
|
||||||
|
light_per = int(settings.ne_collect_read_timeout_sec or 120)
|
||||||
|
light_cap = int(settings.ne_collect_run_timeout_cap_sec or 600)
|
||||||
|
heavy_per = int(getattr(settings, "biz_state_heavy_read_timeout_sec", 300) or 300)
|
||||||
|
heavy_cap = int(getattr(settings, "biz_state_heavy_run_timeout_cap_sec", 900) or 900)
|
||||||
|
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
task = db.get(BizStateTask, task_id)
|
||||||
|
batch = db.get(BizStateBatch, batch_id)
|
||||||
|
if not task or not batch:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
if source == "managed":
|
||||||
|
creds, info = resolve_cli_target(db, managed_ne_id=ne_id)
|
||||||
|
elif source == "ume":
|
||||||
|
creds, info = resolve_cli_target(db, ume_ne_id=ne_id)
|
||||||
|
else:
|
||||||
|
raise RuntimeError("invalid_source")
|
||||||
|
except HTTPException as exc:
|
||||||
|
raise RuntimeError(str(exc.detail or "resolve_failed")) from exc
|
||||||
|
|
||||||
|
skip = cli_creds_skip_reason(creds, interactive=False)
|
||||||
|
if skip:
|
||||||
|
raise RuntimeError(skip)
|
||||||
|
|
||||||
|
vendor_eff = str(info.get("vendor") or vendor or "")
|
||||||
|
device_type_eff = str(info.get("device_type") or device_type or "")
|
||||||
|
if vendor_eff and vendor_eff != task.vendor:
|
||||||
|
task.vendor = vendor_eff
|
||||||
|
if device_type_eff and device_type_eff != task.device_type:
|
||||||
|
task.device_type = device_type_eff
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
vendor_key = resolve_vendor_key(vendor_eff, device_type_eff)
|
||||||
|
|
||||||
|
items = (
|
||||||
|
db.query(BizStateTaskItem)
|
||||||
|
.filter(
|
||||||
|
BizStateTaskItem.task_id == task_id,
|
||||||
|
BizStateTaskItem.enabled.is_(True),
|
||||||
|
)
|
||||||
|
.order_by(BizStateTaskItem.sort_order.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Build work list before opening session
|
||||||
|
work: list[WorkItem] = []
|
||||||
|
# Dedupe same CLI → same metric (e.g. legacy if_intf + config_interface).
|
||||||
|
seen_work: set[tuple[str, str]] = set()
|
||||||
|
for item in items:
|
||||||
|
if item.kind == "custom_raw":
|
||||||
|
cmd = normalize_command(item.command_override)
|
||||||
|
if cmd:
|
||||||
|
key = (cmd, "__custom__")
|
||||||
|
if key in seen_work:
|
||||||
|
continue
|
||||||
|
seen_work.add(key)
|
||||||
|
work.append((cmd, {}, "", item.id, "custom"))
|
||||||
|
continue
|
||||||
|
profile = _resolve_collect_profile(item.source_profile_id)
|
||||||
|
if profile is None:
|
||||||
|
_append_event(
|
||||||
|
db,
|
||||||
|
task_id=task_id,
|
||||||
|
message=f"skip profile {item.source_profile_id} (missing or disabled)",
|
||||||
|
level="info",
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
binds = _bindings_for_item(db, item.id)
|
||||||
|
try:
|
||||||
|
pairs = expand_from_bindings(
|
||||||
|
profile=profile,
|
||||||
|
bindings=binds,
|
||||||
|
command_override=item.command_override,
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
_append_event(db, task_id=task_id, message=str(exc), level="error")
|
||||||
|
continue
|
||||||
|
for concrete, params in pairs:
|
||||||
|
cmd = normalize_command(concrete)
|
||||||
|
# Prefer match_command metric so remapped if_intf shares key with config_interface
|
||||||
|
hit = match_command(vendor_key=vendor_key, command=cmd)
|
||||||
|
mid = str((hit.profile.metric_id if hit else profile.metric_id) or "").strip()
|
||||||
|
pid = str((hit.profile.profile_id if hit else profile.profile_id) or "").strip()
|
||||||
|
key = (cmd, mid or pid)
|
||||||
|
if key in seen_work:
|
||||||
|
continue
|
||||||
|
seen_work.add(key)
|
||||||
|
work.append((cmd, params, pid or profile.profile_id, item.id, "normal"))
|
||||||
|
|
||||||
|
if not work:
|
||||||
|
batch.status = "failed"
|
||||||
|
batch.message = "no commands to run"
|
||||||
|
batch.ended_at = _utcnow()
|
||||||
|
db.commit()
|
||||||
|
raise RuntimeError("no commands to run")
|
||||||
|
|
||||||
|
light_work, heavy_work = partition_work(work)
|
||||||
|
lane_kwargs = dict(
|
||||||
|
batch_id=batch_id,
|
||||||
|
creds=creds,
|
||||||
|
vendor_eff=vendor_eff,
|
||||||
|
device_type_eff=device_type_eff,
|
||||||
|
vendor_key=vendor_key,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _run_light() -> tuple[int, int, bool, bool]:
|
||||||
|
return _run_collect_lane(
|
||||||
|
work=light_work,
|
||||||
|
per_cmd=light_per,
|
||||||
|
cap=light_cap,
|
||||||
|
label="biz_state_light",
|
||||||
|
**lane_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _run_heavy() -> tuple[int, int, bool, bool]:
|
||||||
|
return _run_collect_lane(
|
||||||
|
work=heavy_work,
|
||||||
|
per_cmd=heavy_per,
|
||||||
|
cap=heavy_cap,
|
||||||
|
label="biz_state_heavy",
|
||||||
|
**lane_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
total_rows = 0
|
||||||
|
cmd_count = 0
|
||||||
|
any_fail = False
|
||||||
|
any_ok = False
|
||||||
|
lane_errors: list[str] = []
|
||||||
|
|
||||||
|
if light_work and heavy_work:
|
||||||
|
heavy_fut = _heavy_cli_pool().submit(_run_heavy)
|
||||||
|
light_res: tuple[int, int, bool, bool] | BaseException
|
||||||
|
try:
|
||||||
|
light_res = _run_light()
|
||||||
|
except Exception as exc:
|
||||||
|
light_res = exc
|
||||||
|
heavy_res: tuple[int, int, bool, bool] | BaseException
|
||||||
|
try:
|
||||||
|
heavy_res = heavy_fut.result()
|
||||||
|
except Exception as exc:
|
||||||
|
heavy_res = exc
|
||||||
|
total_rows, cmd_count, any_fail, any_ok = _absorb_lane_result(
|
||||||
|
light_res,
|
||||||
|
total_rows=total_rows,
|
||||||
|
cmd_count=cmd_count,
|
||||||
|
any_fail=any_fail,
|
||||||
|
any_ok=any_ok,
|
||||||
|
lane_errors=lane_errors,
|
||||||
|
)
|
||||||
|
total_rows, cmd_count, any_fail, any_ok = _absorb_lane_result(
|
||||||
|
heavy_res,
|
||||||
|
total_rows=total_rows,
|
||||||
|
cmd_count=cmd_count,
|
||||||
|
any_fail=any_fail,
|
||||||
|
any_ok=any_ok,
|
||||||
|
lane_errors=lane_errors,
|
||||||
|
)
|
||||||
|
elif heavy_work:
|
||||||
|
try:
|
||||||
|
total_rows, cmd_count, any_fail, any_ok = _run_heavy()
|
||||||
|
except Exception as exc:
|
||||||
|
total_rows, cmd_count, any_fail, any_ok = _absorb_lane_result(
|
||||||
|
exc,
|
||||||
|
total_rows=0,
|
||||||
|
cmd_count=0,
|
||||||
|
any_fail=False,
|
||||||
|
any_ok=False,
|
||||||
|
lane_errors=lane_errors,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
total_rows, cmd_count, any_fail, any_ok = _run_light()
|
||||||
|
except Exception as exc:
|
||||||
|
total_rows, cmd_count, any_fail, any_ok = _absorb_lane_result(
|
||||||
|
exc,
|
||||||
|
total_rows=0,
|
||||||
|
cmd_count=0,
|
||||||
|
any_fail=False,
|
||||||
|
any_ok=False,
|
||||||
|
lane_errors=lane_errors,
|
||||||
|
)
|
||||||
|
|
||||||
|
if lane_errors and not any_ok and cmd_count == 0:
|
||||||
|
raise RuntimeError("; ".join(lane_errors)[:1020])
|
||||||
|
|
||||||
batch = db.get(BizStateBatch, batch_id)
|
batch = db.get(BizStateBatch, batch_id)
|
||||||
if batch:
|
if batch:
|
||||||
batch.command_count = cmd_count
|
batch.command_count = cmd_count
|
||||||
|
|
@ -689,11 +853,16 @@ def _run_collect_session(
|
||||||
batch.ended_at = _utcnow()
|
batch.ended_at = _utcnow()
|
||||||
if any_fail and any_ok:
|
if any_fail and any_ok:
|
||||||
batch.status = "partial"
|
batch.status = "partial"
|
||||||
|
if lane_errors:
|
||||||
|
batch.message = "; ".join(lane_errors)[:1020]
|
||||||
elif any_fail and not any_ok:
|
elif any_fail and not any_ok:
|
||||||
batch.status = "failed"
|
batch.status = "failed"
|
||||||
batch.message = "all commands failed"
|
batch.message = (
|
||||||
|
"; ".join(lane_errors)[:1020] if lane_errors else "all commands failed"
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
batch.status = "success"
|
batch.status = "success"
|
||||||
|
batch.message = ""
|
||||||
db.commit()
|
db.commit()
|
||||||
if batch.status in ("success", "partial"):
|
if batch.status in ("success", "partial"):
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -64,6 +64,8 @@ class ParseProfile:
|
||||||
kind: str = "collect" # collect | discover
|
kind: str = "collect" # collect | discover
|
||||||
aux_commands: list[AuxCommand] = field(default_factory=list)
|
aux_commands: list[AuxCommand] = field(default_factory=list)
|
||||||
enrich_joins: list[EnrichJoin] = field(default_factory=list)
|
enrich_joins: list[EnrichJoin] = field(default_factory=list)
|
||||||
|
# light: default shared SSH lane; heavy: dedicated long-timeout connection
|
||||||
|
collect_lane: str = "light"
|
||||||
|
|
||||||
|
|
||||||
_LLDP_FIELDS: list[FieldDef] = [
|
_LLDP_FIELDS: list[FieldDef] = [
|
||||||
|
|
@ -1270,11 +1272,32 @@ def _zte_status_profiles() -> list[ParseProfile]:
|
||||||
|
|
||||||
_PROFILES: list[ParseProfile] | None = None
|
_PROFILES: list[ParseProfile] | None = None
|
||||||
|
|
||||||
|
# Huge CLI dumps: dedicated heavy lane (longer read_timeout / own SSH).
|
||||||
|
_HEAVY_LANE_METRICS = frozenset(
|
||||||
|
{
|
||||||
|
"interface_detail",
|
||||||
|
"ip_route",
|
||||||
|
"ipv6_route",
|
||||||
|
"bgp_route",
|
||||||
|
"l2vpn_mac",
|
||||||
|
"evpn_mac",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_collect_lanes(profiles: list[ParseProfile]) -> list[ParseProfile]:
|
||||||
|
for p in profiles:
|
||||||
|
if str(p.metric_id or "").strip() in _HEAVY_LANE_METRICS:
|
||||||
|
p.collect_lane = "heavy"
|
||||||
|
return profiles
|
||||||
|
|
||||||
|
|
||||||
def all_profiles() -> list[ParseProfile]:
|
def all_profiles() -> list[ParseProfile]:
|
||||||
global _PROFILES
|
global _PROFILES
|
||||||
if _PROFILES is None:
|
if _PROFILES is None:
|
||||||
_PROFILES = _lldp_profiles() + _vrf_profiles() + _zte_status_profiles()
|
_PROFILES = _apply_collect_lanes(
|
||||||
|
_lldp_profiles() + _vrf_profiles() + _zte_status_profiles()
|
||||||
|
)
|
||||||
return list(_PROFILES)
|
return list(_PROFILES)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -1357,6 +1380,7 @@ def profile_to_public_dict(p: ParseProfile, *, overrides: dict[str, Any] | None
|
||||||
"tags": list(p.tags),
|
"tags": list(p.tags),
|
||||||
"sort_order": p.sort_order,
|
"sort_order": p.sort_order,
|
||||||
"enabled": bool(ov.get("enabled")) if "enabled" in ov else p.enabled,
|
"enabled": bool(ov.get("enabled")) if "enabled" in ov else p.enabled,
|
||||||
|
"collect_lane": str(p.collect_lane or "light"),
|
||||||
"kind": p.kind,
|
"kind": p.kind,
|
||||||
"match": p.match,
|
"match": p.match,
|
||||||
"textfsm_command": p.textfsm_command or p.command_template,
|
"textfsm_command": p.textfsm_command or p.command_template,
|
||||||
|
|
|
||||||
|
|
@ -107,6 +107,10 @@ class Settings(BaseSettings):
|
||||||
biz_state_scheduler_enabled: bool = True
|
biz_state_scheduler_enabled: bool = True
|
||||||
biz_state_scheduler_tick_sec: int = 15
|
biz_state_scheduler_tick_sec: int = 15
|
||||||
biz_state_dispatch_workers: int = 2
|
biz_state_dispatch_workers: int = 2
|
||||||
|
# Heavy CLI lane (interface detail / routes / MAC): own SSH + longer timeouts.
|
||||||
|
biz_state_heavy_read_timeout_sec: int = 300
|
||||||
|
biz_state_heavy_run_timeout_cap_sec: int = 900
|
||||||
|
biz_state_heavy_workers: int = 4
|
||||||
# Managed NE exec: max CLI commands per request (lab can raise; hard-capped in ne_exec).
|
# Managed NE exec: max CLI commands per request (lab can raise; hard-capped in ne_exec).
|
||||||
ne_exec_max_commands: int = 5
|
ne_exec_max_commands: int = 5
|
||||||
# WebCRT interactive terminal sessions (multi-operator concurrent terminals).
|
# WebCRT interactive terminal sessions (multi-operator concurrent terminals).
|
||||||
|
|
|
||||||
162
tests/test_collect_lanes.py
Normal file
162
tests/test_collect_lanes.py
Normal file
|
|
@ -0,0 +1,162 @@
|
||||||
|
"""Light/heavy collect lane partition and dual-lane scheduling."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from netx_api.biz_state.collect_runner import (
|
||||||
|
partition_work,
|
||||||
|
work_item_lane,
|
||||||
|
)
|
||||||
|
from netx_api.biz_state.profiles import get_profile, reload_profiles
|
||||||
|
|
||||||
|
|
||||||
|
class CollectLaneProfileTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
reload_profiles()
|
||||||
|
|
||||||
|
def test_interface_detail_is_heavy(self) -> None:
|
||||||
|
p = get_profile("zte.interface_detail")
|
||||||
|
self.assertIsNotNone(p)
|
||||||
|
assert p is not None
|
||||||
|
self.assertEqual(p.collect_lane, "heavy")
|
||||||
|
|
||||||
|
def test_route_profiles_are_heavy(self) -> None:
|
||||||
|
for pid in (
|
||||||
|
"zte.ip_route_vrf",
|
||||||
|
"zte.ip_route",
|
||||||
|
"zte.ipv6_route_vrf",
|
||||||
|
"zte.ipv6_route",
|
||||||
|
):
|
||||||
|
p = get_profile(pid)
|
||||||
|
self.assertIsNotNone(p, pid)
|
||||||
|
assert p is not None
|
||||||
|
self.assertEqual(p.collect_lane, "heavy", pid)
|
||||||
|
|
||||||
|
def test_arp_is_light(self) -> None:
|
||||||
|
p = get_profile("zte.arp")
|
||||||
|
self.assertIsNotNone(p)
|
||||||
|
assert p is not None
|
||||||
|
self.assertEqual(p.collect_lane, "light")
|
||||||
|
|
||||||
|
|
||||||
|
class PartitionWorkTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls) -> None:
|
||||||
|
reload_profiles()
|
||||||
|
|
||||||
|
def test_custom_raw_always_light(self) -> None:
|
||||||
|
self.assertEqual(work_item_lane("", "custom"), "light")
|
||||||
|
self.assertEqual(work_item_lane("zte.interface_detail", "custom"), "light")
|
||||||
|
|
||||||
|
def test_partition_mixed(self) -> None:
|
||||||
|
work = [
|
||||||
|
("show arp", {}, "zte.arp", "i1", "normal"),
|
||||||
|
(
|
||||||
|
"show interface | include ifindex",
|
||||||
|
{},
|
||||||
|
"zte.interface_detail",
|
||||||
|
"i2",
|
||||||
|
"normal",
|
||||||
|
),
|
||||||
|
("show ip forwarding route vrf a", {}, "zte.ip_route_vrf", "i3", "normal"),
|
||||||
|
("show version", {}, "", "i4", "custom"),
|
||||||
|
]
|
||||||
|
light, heavy = partition_work(work)
|
||||||
|
self.assertEqual(len(light), 2)
|
||||||
|
self.assertEqual({x[2] for x in light}, {"zte.arp", ""})
|
||||||
|
self.assertEqual(len(heavy), 2)
|
||||||
|
self.assertEqual(
|
||||||
|
{x[2] for x in heavy},
|
||||||
|
{"zte.interface_detail", "zte.ip_route_vrf"},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_partition_light_only(self) -> None:
|
||||||
|
work = [("show arp", {}, "zte.arp", "i1", "normal")]
|
||||||
|
light, heavy = partition_work(work)
|
||||||
|
self.assertEqual(len(light), 1)
|
||||||
|
self.assertEqual(heavy, [])
|
||||||
|
|
||||||
|
def test_partition_heavy_only(self) -> None:
|
||||||
|
work = [
|
||||||
|
(
|
||||||
|
"show interface | include ifindex",
|
||||||
|
{},
|
||||||
|
"zte.interface_detail",
|
||||||
|
"i1",
|
||||||
|
"normal",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
light, heavy = partition_work(work)
|
||||||
|
self.assertEqual(light, [])
|
||||||
|
self.assertEqual(len(heavy), 1)
|
||||||
|
|
||||||
|
|
||||||
|
class DualLaneParallelTests(unittest.TestCase):
|
||||||
|
def test_light_not_blocked_by_heavy_sleep(self) -> None:
|
||||||
|
"""When both lanes run, light completes without waiting for heavy."""
|
||||||
|
from netx_api.biz_state import collect_runner as cr
|
||||||
|
|
||||||
|
order: list[str] = []
|
||||||
|
|
||||||
|
def fake_lane(*, work, label, **_kwargs):
|
||||||
|
if "heavy" in label:
|
||||||
|
time.sleep(0.35)
|
||||||
|
order.append("heavy_done")
|
||||||
|
return (0, len(work), False, True)
|
||||||
|
order.append("light_done")
|
||||||
|
return (0, len(work), False, True)
|
||||||
|
|
||||||
|
light_work = [("show arp", {}, "zte.arp", "i1", "normal")]
|
||||||
|
heavy_work = [
|
||||||
|
(
|
||||||
|
"show interface | include ifindex",
|
||||||
|
{},
|
||||||
|
"zte.interface_detail",
|
||||||
|
"i2",
|
||||||
|
"normal",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
with patch.object(cr, "_run_collect_lane", side_effect=fake_lane):
|
||||||
|
t0 = time.perf_counter()
|
||||||
|
heavy_fut = cr._heavy_cli_pool().submit(
|
||||||
|
lambda: cr._run_collect_lane(
|
||||||
|
work=heavy_work,
|
||||||
|
batch_id="b",
|
||||||
|
creds={},
|
||||||
|
vendor_eff="zte",
|
||||||
|
device_type_eff="",
|
||||||
|
vendor_key="zte",
|
||||||
|
per_cmd=1,
|
||||||
|
cap=10,
|
||||||
|
label="biz_state_heavy",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
light_res = cr._run_collect_lane(
|
||||||
|
work=light_work,
|
||||||
|
batch_id="b",
|
||||||
|
creds={},
|
||||||
|
vendor_eff="zte",
|
||||||
|
device_type_eff="",
|
||||||
|
vendor_key="zte",
|
||||||
|
per_cmd=1,
|
||||||
|
cap=10,
|
||||||
|
label="biz_state_light",
|
||||||
|
)
|
||||||
|
light_elapsed = time.perf_counter() - t0
|
||||||
|
heavy_res = heavy_fut.result()
|
||||||
|
total_elapsed = time.perf_counter() - t0
|
||||||
|
|
||||||
|
self.assertEqual(light_res[1], 1)
|
||||||
|
self.assertEqual(heavy_res[1], 1)
|
||||||
|
self.assertEqual(order[0], "light_done")
|
||||||
|
self.assertLess(light_elapsed, 0.25)
|
||||||
|
self.assertGreaterEqual(total_elapsed, 0.3)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
|
|
@ -197,6 +197,7 @@ export function BizStatePage() {
|
||||||
|
|
||||||
const [tasks, setTasks] = useState<TaskRow[]>([]);
|
const [tasks, setTasks] = useState<TaskRow[]>([]);
|
||||||
const [busy, setBusy] = useState(false);
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [collectingIds, setCollectingIds] = useState<Record<string, true>>({});
|
||||||
const [listKeyword, setListKeyword] = useState("");
|
const [listKeyword, setListKeyword] = useState("");
|
||||||
const debouncedListKw = useDebouncedValue(listKeyword, 250);
|
const debouncedListKw = useDebouncedValue(listKeyword, 250);
|
||||||
const [purposeFilter, setPurposeFilter] = useState<"all" | "portrait" | "cutover_hf">("all");
|
const [purposeFilter, setPurposeFilter] = useState<"all" | "portrait" | "cutover_hf">("all");
|
||||||
|
|
@ -253,7 +254,9 @@ export function BizStatePage() {
|
||||||
const purpose =
|
const purpose =
|
||||||
purposeFilter === "all" ? "" : purposeFilter === "portrait" ? "portrait" : "cutover_hf";
|
purposeFilter === "all" ? "" : purposeFilter === "portrait" ? "portrait" : "cutover_hf";
|
||||||
const res = await bizStateListTasks(purpose);
|
const res = await bizStateListTasks(purpose);
|
||||||
setTasks((res.items || []) as TaskRow[]);
|
const items = (res.items || []) as TaskRow[];
|
||||||
|
setTasks(items);
|
||||||
|
return items;
|
||||||
}, [purposeFilter]);
|
}, [purposeFilter]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|
@ -619,27 +622,58 @@ export function BizStatePage() {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const collectNow = async () => {
|
const setTaskCollecting = (id: string, on: boolean) => {
|
||||||
if (!taskId) return;
|
setCollectingIds((prev) => {
|
||||||
setBusy(true);
|
if (on) {
|
||||||
|
if (prev[id]) return prev;
|
||||||
|
return { ...prev, [id]: true };
|
||||||
|
}
|
||||||
|
if (!prev[id]) return prev;
|
||||||
|
const next = { ...prev };
|
||||||
|
delete next[id];
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const collectNowForTask = async (id: string, fromModal = false) => {
|
||||||
|
setTaskCollecting(id, true);
|
||||||
try {
|
try {
|
||||||
await bizStateCollectNow(taskId);
|
await bizStateCollectNow(id);
|
||||||
|
showOk(t("bizState.collecting"));
|
||||||
for (let i = 0; i < 20; i++) {
|
for (let i = 0; i < 20; i++) {
|
||||||
await new Promise((r) => setTimeout(r, 1500));
|
await new Promise((r) => setTimeout(r, 1500));
|
||||||
const task = await bizStateGetTask(taskId);
|
if (fromModal && taskId === id) {
|
||||||
|
try {
|
||||||
|
const task = await bizStateGetTask(id);
|
||||||
setDetail(task);
|
setDetail(task);
|
||||||
if (!task.collect_running) break;
|
|
||||||
}
|
|
||||||
await loadTask(taskId);
|
|
||||||
await refreshTasks();
|
await refreshTasks();
|
||||||
|
if (!task.collect_running) break;
|
||||||
|
} catch {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const items = await refreshTasks();
|
||||||
|
const latest = items.find((x) => x.id === id);
|
||||||
|
if (!latest?.collect_running) break;
|
||||||
|
}
|
||||||
|
await refreshTasks();
|
||||||
|
if (fromModal && taskId === id) {
|
||||||
|
await loadTask(id);
|
||||||
setTaskTab("batches");
|
setTaskTab("batches");
|
||||||
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
showError(formatErr(e));
|
showError(formatErr(e));
|
||||||
} finally {
|
} finally {
|
||||||
setBusy(false);
|
setTaskCollecting(id, false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const collectNow = async () => {
|
||||||
|
if (!taskId) return;
|
||||||
|
await collectNowForTask(taskId, true);
|
||||||
|
};
|
||||||
|
|
||||||
const removeTask = async (id: string) => {
|
const removeTask = async (id: string) => {
|
||||||
if (!window.confirm(t("bizState.confirmDelete"))) return;
|
if (!window.confirm(t("bizState.confirmDelete"))) return;
|
||||||
setBusy(true);
|
setBusy(true);
|
||||||
|
|
@ -939,27 +973,8 @@ export function BizStatePage() {
|
||||||
<Button
|
<Button
|
||||||
size="sm"
|
size="sm"
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
isDisabled={busy || row.collect_running}
|
isDisabled={Boolean(row.collect_running || collectingIds[row.id])}
|
||||||
onPress={async () => {
|
onPress={() => void collectNowForTask(row.id, false)}
|
||||||
setBusy(true);
|
|
||||||
try {
|
|
||||||
await bizStateCollectNow(row.id);
|
|
||||||
showOk(t("bizState.collecting"));
|
|
||||||
for (let i = 0; i < 20; i++) {
|
|
||||||
await new Promise((r) => setTimeout(r, 1500));
|
|
||||||
await refreshTasks();
|
|
||||||
const cur = (await bizStateListTasks(purposeFilter === "all" ? "" : purposeFilter)).items?.find(
|
|
||||||
(x: any) => x.id === row.id,
|
|
||||||
);
|
|
||||||
if (!cur?.collect_running) break;
|
|
||||||
}
|
|
||||||
await refreshTasks();
|
|
||||||
} catch (e) {
|
|
||||||
showError(formatErr(e));
|
|
||||||
} finally {
|
|
||||||
setBusy(false);
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
>
|
>
|
||||||
{t("bizState.collectNow")}
|
{t("bizState.collectNow")}
|
||||||
</Button>
|
</Button>
|
||||||
|
|
@ -1439,7 +1454,7 @@ export function BizStatePage() {
|
||||||
<Button
|
<Button
|
||||||
size="sm"
|
size="sm"
|
||||||
variant="primary"
|
variant="primary"
|
||||||
isDisabled={busy || Boolean(detail?.collect_running)}
|
isDisabled={Boolean(detail?.collect_running || (taskId && collectingIds[taskId]))}
|
||||||
onPress={() => void collectNow()}
|
onPress={() => void collectNow()}
|
||||||
>
|
>
|
||||||
{t("bizState.collectNow")}
|
{t("bizState.collectNow")}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue