mirror of
https://github.com/hansjone/netx.git
synced 2026-10-19 06:53:19 +08:00
Run heavy CLI dumps on a dedicated long-timeout SSH lane.
Split interface detail/route-class collects from the light lane so they no longer block other checks, and scope Collect Now busy state per task. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
99cec1c324
commit
f279431c70
5 changed files with 675 additions and 301 deletions
|
|
@ -3,6 +3,8 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
|
@ -39,6 +41,42 @@ from .profiles import get_profile
|
|||
|
||||
_log = logging.getLogger("netx.biz_state.runner")
|
||||
|
||||
# concrete, params, profile_id, item_id, mode
|
||||
WorkItem = tuple[str, dict[str, str], str, str, str]
|
||||
|
||||
_heavy_pool_lock = threading.Lock()
|
||||
_heavy_pool: ThreadPoolExecutor | None = None
|
||||
|
||||
|
||||
def _heavy_cli_pool() -> ThreadPoolExecutor:
|
||||
global _heavy_pool
|
||||
with _heavy_pool_lock:
|
||||
if _heavy_pool is None:
|
||||
n = max(1, int(getattr(settings, "biz_state_heavy_workers", 4) or 4))
|
||||
_heavy_pool = ThreadPoolExecutor(max_workers=n, thread_name_prefix="biz-heavy")
|
||||
return _heavy_pool
|
||||
|
||||
|
||||
def work_item_lane(profile_id: str, mode: str) -> str:
|
||||
"""Return collect lane for a work item (custom_raw always light)."""
|
||||
if str(mode or "").strip() == "custom":
|
||||
return "light"
|
||||
profile = get_profile(profile_id) or _resolve_collect_profile(profile_id)
|
||||
lane = str(getattr(profile, "collect_lane", "light") or "light").strip().lower() if profile else "light"
|
||||
return "heavy" if lane == "heavy" else "light"
|
||||
|
||||
|
||||
def partition_work(work: list[WorkItem]) -> tuple[list[WorkItem], list[WorkItem]]:
|
||||
"""Split work into (light, heavy) lanes."""
|
||||
light: list[WorkItem] = []
|
||||
heavy: list[WorkItem] = []
|
||||
for item in work:
|
||||
if work_item_lane(item[2], item[4]) == "heavy":
|
||||
heavy.append(item)
|
||||
else:
|
||||
light.append(item)
|
||||
return light, heavy
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.utcnow()
|
||||
|
|
@ -311,113 +349,23 @@ def dispatch_collect(task_id: str, *, manual: bool = False) -> None:
|
|||
_finish_task(task_id, error=error)
|
||||
|
||||
|
||||
def _run_collect_session(
|
||||
def _run_collect_lane(
|
||||
*,
|
||||
task_id: str,
|
||||
work: list[WorkItem],
|
||||
batch_id: str,
|
||||
source: str,
|
||||
ne_id: str,
|
||||
vendor: str,
|
||||
device_type: str,
|
||||
) -> None:
|
||||
per_cmd = int(settings.ne_collect_read_timeout_sec or 120)
|
||||
cap = int(settings.ne_collect_run_timeout_cap_sec or 600)
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
task = db.get(BizStateTask, task_id)
|
||||
batch = db.get(BizStateBatch, batch_id)
|
||||
if not task or not batch:
|
||||
return
|
||||
|
||||
try:
|
||||
if source == "managed":
|
||||
creds, info = resolve_cli_target(db, managed_ne_id=ne_id)
|
||||
elif source == "ume":
|
||||
creds, info = resolve_cli_target(db, ume_ne_id=ne_id)
|
||||
else:
|
||||
raise RuntimeError("invalid_source")
|
||||
except HTTPException as exc:
|
||||
raise RuntimeError(str(exc.detail or "resolve_failed")) from exc
|
||||
|
||||
skip = cli_creds_skip_reason(creds, interactive=False)
|
||||
if skip:
|
||||
raise RuntimeError(skip)
|
||||
|
||||
vendor_eff = str(info.get("vendor") or vendor or "")
|
||||
device_type_eff = str(info.get("device_type") or device_type or "")
|
||||
if vendor_eff and vendor_eff != task.vendor:
|
||||
task.vendor = vendor_eff
|
||||
if device_type_eff and device_type_eff != task.device_type:
|
||||
task.device_type = device_type_eff
|
||||
db.commit()
|
||||
|
||||
vendor_key = resolve_vendor_key(vendor_eff, device_type_eff)
|
||||
|
||||
items = (
|
||||
db.query(BizStateTaskItem)
|
||||
.filter(
|
||||
BizStateTaskItem.task_id == task_id,
|
||||
BizStateTaskItem.enabled.is_(True),
|
||||
)
|
||||
.order_by(BizStateTaskItem.sort_order.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
# Build work list before opening session
|
||||
work: list[tuple[str, dict[str, str], str, str, str]] = []
|
||||
# concrete, params, profile_id, item_id, mode
|
||||
# Dedupe same CLI → same metric (e.g. legacy if_intf + config_interface).
|
||||
seen_work: set[tuple[str, str]] = set()
|
||||
for item in items:
|
||||
if item.kind == "custom_raw":
|
||||
cmd = normalize_command(item.command_override)
|
||||
if cmd:
|
||||
key = (cmd, "__custom__")
|
||||
if key in seen_work:
|
||||
continue
|
||||
seen_work.add(key)
|
||||
work.append((cmd, {}, "", item.id, "custom"))
|
||||
continue
|
||||
profile = _resolve_collect_profile(item.source_profile_id)
|
||||
if profile is None:
|
||||
_append_event(
|
||||
db,
|
||||
task_id=task_id,
|
||||
message=f"skip profile {item.source_profile_id} (missing or disabled)",
|
||||
level="info",
|
||||
)
|
||||
continue
|
||||
binds = _bindings_for_item(db, item.id)
|
||||
try:
|
||||
pairs = expand_from_bindings(
|
||||
profile=profile,
|
||||
bindings=binds,
|
||||
command_override=item.command_override,
|
||||
)
|
||||
except ValueError as exc:
|
||||
_append_event(db, task_id=task_id, message=str(exc), level="error")
|
||||
continue
|
||||
for concrete, params in pairs:
|
||||
cmd = normalize_command(concrete)
|
||||
# Prefer match_command metric so remapped if_intf shares key with config_interface
|
||||
hit = match_command(vendor_key=vendor_key, command=cmd)
|
||||
mid = str((hit.profile.metric_id if hit else profile.metric_id) or "").strip()
|
||||
pid = str((hit.profile.profile_id if hit else profile.profile_id) or "").strip()
|
||||
key = (cmd, mid or pid)
|
||||
if key in seen_work:
|
||||
continue
|
||||
seen_work.add(key)
|
||||
work.append((cmd, params, pid or profile.profile_id, item.id, "normal"))
|
||||
|
||||
creds: dict[str, Any],
|
||||
vendor_eff: str,
|
||||
device_type_eff: str,
|
||||
vendor_key: str,
|
||||
per_cmd: int,
|
||||
cap: int,
|
||||
label: str,
|
||||
) -> tuple[int, int, bool, bool]:
|
||||
"""Run one SSH lane (own connection + CollectSession + timeout budget)."""
|
||||
if not work:
|
||||
batch.status = "failed"
|
||||
batch.message = "no commands to run"
|
||||
batch.ended_at = _utcnow()
|
||||
db.commit()
|
||||
raise RuntimeError("no commands to run")
|
||||
return 0, 0, False, False
|
||||
|
||||
budget = min(cap, per_cmd * max(1, len(work)) + 90)
|
||||
budget = min(int(cap), int(per_cmd) * max(1, len(work)) + 90)
|
||||
holder: dict[str, Any] = {}
|
||||
|
||||
def _session() -> tuple[int, int, bool, bool]:
|
||||
|
|
@ -457,7 +405,7 @@ def _run_collect_session(
|
|||
return 0, 0, True, False
|
||||
for concrete, params, profile_id, item_id, mode in work:
|
||||
if holder.get("timed_out"):
|
||||
raise TimeoutError("biz_state_aborted")
|
||||
raise TimeoutError(f"{label}_aborted")
|
||||
cmd_count += 1
|
||||
cmd_row = BizStateBatchCommand(
|
||||
id=uuid4().hex,
|
||||
|
|
@ -517,7 +465,6 @@ def _run_collect_session(
|
|||
continue
|
||||
|
||||
if cache_hit_primary:
|
||||
# Keep existing cache entry; primary will re-parse from shared raw.
|
||||
pass
|
||||
else:
|
||||
session.remember(
|
||||
|
|
@ -587,7 +534,6 @@ def _run_collect_session(
|
|||
aux_row.message = f"aux_for={cmd_row.id}"[:1020]
|
||||
aux_row.raw_text = entry.raw
|
||||
aux_row.row_count = len(entry.records or [])
|
||||
# refresh cache row id to this aux row on first fetch
|
||||
entry.cmd_row_id = aux_row.id
|
||||
sdb.add(aux_row)
|
||||
sdb.commit()
|
||||
|
|
@ -672,16 +618,234 @@ def _run_collect_session(
|
|||
close_netmiko_connection(conn)
|
||||
|
||||
try:
|
||||
total_rows, cmd_count, any_fail, any_ok = run_cli_with_timeout(
|
||||
return run_cli_with_timeout(
|
||||
_session,
|
||||
timeout_sec=budget,
|
||||
conn_holder=holder,
|
||||
label="biz_state",
|
||||
label=label,
|
||||
acquire_budget=True,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise RuntimeError(str(exc)[:1020]) from exc
|
||||
|
||||
|
||||
def _absorb_lane_result(
|
||||
result: tuple[int, int, bool, bool] | BaseException,
|
||||
*,
|
||||
total_rows: int,
|
||||
cmd_count: int,
|
||||
any_fail: bool,
|
||||
any_ok: bool,
|
||||
lane_errors: list[str],
|
||||
) -> tuple[int, int, bool, bool]:
|
||||
if isinstance(result, BaseException):
|
||||
lane_errors.append(_format_error(result))
|
||||
return total_rows, cmd_count, True, any_ok
|
||||
rows, cmds, fail, ok = result
|
||||
return (
|
||||
total_rows + int(rows or 0),
|
||||
cmd_count + int(cmds or 0),
|
||||
any_fail or bool(fail),
|
||||
any_ok or bool(ok),
|
||||
)
|
||||
|
||||
|
||||
def _run_collect_session(
|
||||
*,
|
||||
task_id: str,
|
||||
batch_id: str,
|
||||
source: str,
|
||||
ne_id: str,
|
||||
vendor: str,
|
||||
device_type: str,
|
||||
) -> None:
|
||||
light_per = int(settings.ne_collect_read_timeout_sec or 120)
|
||||
light_cap = int(settings.ne_collect_run_timeout_cap_sec or 600)
|
||||
heavy_per = int(getattr(settings, "biz_state_heavy_read_timeout_sec", 300) or 300)
|
||||
heavy_cap = int(getattr(settings, "biz_state_heavy_run_timeout_cap_sec", 900) or 900)
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
task = db.get(BizStateTask, task_id)
|
||||
batch = db.get(BizStateBatch, batch_id)
|
||||
if not task or not batch:
|
||||
return
|
||||
|
||||
try:
|
||||
if source == "managed":
|
||||
creds, info = resolve_cli_target(db, managed_ne_id=ne_id)
|
||||
elif source == "ume":
|
||||
creds, info = resolve_cli_target(db, ume_ne_id=ne_id)
|
||||
else:
|
||||
raise RuntimeError("invalid_source")
|
||||
except HTTPException as exc:
|
||||
raise RuntimeError(str(exc.detail or "resolve_failed")) from exc
|
||||
|
||||
skip = cli_creds_skip_reason(creds, interactive=False)
|
||||
if skip:
|
||||
raise RuntimeError(skip)
|
||||
|
||||
vendor_eff = str(info.get("vendor") or vendor or "")
|
||||
device_type_eff = str(info.get("device_type") or device_type or "")
|
||||
if vendor_eff and vendor_eff != task.vendor:
|
||||
task.vendor = vendor_eff
|
||||
if device_type_eff and device_type_eff != task.device_type:
|
||||
task.device_type = device_type_eff
|
||||
db.commit()
|
||||
|
||||
vendor_key = resolve_vendor_key(vendor_eff, device_type_eff)
|
||||
|
||||
items = (
|
||||
db.query(BizStateTaskItem)
|
||||
.filter(
|
||||
BizStateTaskItem.task_id == task_id,
|
||||
BizStateTaskItem.enabled.is_(True),
|
||||
)
|
||||
.order_by(BizStateTaskItem.sort_order.asc())
|
||||
.all()
|
||||
)
|
||||
|
||||
# Build work list before opening session
|
||||
work: list[WorkItem] = []
|
||||
# Dedupe same CLI → same metric (e.g. legacy if_intf + config_interface).
|
||||
seen_work: set[tuple[str, str]] = set()
|
||||
for item in items:
|
||||
if item.kind == "custom_raw":
|
||||
cmd = normalize_command(item.command_override)
|
||||
if cmd:
|
||||
key = (cmd, "__custom__")
|
||||
if key in seen_work:
|
||||
continue
|
||||
seen_work.add(key)
|
||||
work.append((cmd, {}, "", item.id, "custom"))
|
||||
continue
|
||||
profile = _resolve_collect_profile(item.source_profile_id)
|
||||
if profile is None:
|
||||
_append_event(
|
||||
db,
|
||||
task_id=task_id,
|
||||
message=f"skip profile {item.source_profile_id} (missing or disabled)",
|
||||
level="info",
|
||||
)
|
||||
continue
|
||||
binds = _bindings_for_item(db, item.id)
|
||||
try:
|
||||
pairs = expand_from_bindings(
|
||||
profile=profile,
|
||||
bindings=binds,
|
||||
command_override=item.command_override,
|
||||
)
|
||||
except ValueError as exc:
|
||||
_append_event(db, task_id=task_id, message=str(exc), level="error")
|
||||
continue
|
||||
for concrete, params in pairs:
|
||||
cmd = normalize_command(concrete)
|
||||
# Prefer match_command metric so remapped if_intf shares key with config_interface
|
||||
hit = match_command(vendor_key=vendor_key, command=cmd)
|
||||
mid = str((hit.profile.metric_id if hit else profile.metric_id) or "").strip()
|
||||
pid = str((hit.profile.profile_id if hit else profile.profile_id) or "").strip()
|
||||
key = (cmd, mid or pid)
|
||||
if key in seen_work:
|
||||
continue
|
||||
seen_work.add(key)
|
||||
work.append((cmd, params, pid or profile.profile_id, item.id, "normal"))
|
||||
|
||||
if not work:
|
||||
batch.status = "failed"
|
||||
batch.message = "no commands to run"
|
||||
batch.ended_at = _utcnow()
|
||||
db.commit()
|
||||
raise RuntimeError("no commands to run")
|
||||
|
||||
light_work, heavy_work = partition_work(work)
|
||||
lane_kwargs = dict(
|
||||
batch_id=batch_id,
|
||||
creds=creds,
|
||||
vendor_eff=vendor_eff,
|
||||
device_type_eff=device_type_eff,
|
||||
vendor_key=vendor_key,
|
||||
)
|
||||
|
||||
def _run_light() -> tuple[int, int, bool, bool]:
|
||||
return _run_collect_lane(
|
||||
work=light_work,
|
||||
per_cmd=light_per,
|
||||
cap=light_cap,
|
||||
label="biz_state_light",
|
||||
**lane_kwargs,
|
||||
)
|
||||
|
||||
def _run_heavy() -> tuple[int, int, bool, bool]:
|
||||
return _run_collect_lane(
|
||||
work=heavy_work,
|
||||
per_cmd=heavy_per,
|
||||
cap=heavy_cap,
|
||||
label="biz_state_heavy",
|
||||
**lane_kwargs,
|
||||
)
|
||||
|
||||
total_rows = 0
|
||||
cmd_count = 0
|
||||
any_fail = False
|
||||
any_ok = False
|
||||
lane_errors: list[str] = []
|
||||
|
||||
if light_work and heavy_work:
|
||||
heavy_fut = _heavy_cli_pool().submit(_run_heavy)
|
||||
light_res: tuple[int, int, bool, bool] | BaseException
|
||||
try:
|
||||
light_res = _run_light()
|
||||
except Exception as exc:
|
||||
light_res = exc
|
||||
heavy_res: tuple[int, int, bool, bool] | BaseException
|
||||
try:
|
||||
heavy_res = heavy_fut.result()
|
||||
except Exception as exc:
|
||||
heavy_res = exc
|
||||
total_rows, cmd_count, any_fail, any_ok = _absorb_lane_result(
|
||||
light_res,
|
||||
total_rows=total_rows,
|
||||
cmd_count=cmd_count,
|
||||
any_fail=any_fail,
|
||||
any_ok=any_ok,
|
||||
lane_errors=lane_errors,
|
||||
)
|
||||
total_rows, cmd_count, any_fail, any_ok = _absorb_lane_result(
|
||||
heavy_res,
|
||||
total_rows=total_rows,
|
||||
cmd_count=cmd_count,
|
||||
any_fail=any_fail,
|
||||
any_ok=any_ok,
|
||||
lane_errors=lane_errors,
|
||||
)
|
||||
elif heavy_work:
|
||||
try:
|
||||
total_rows, cmd_count, any_fail, any_ok = _run_heavy()
|
||||
except Exception as exc:
|
||||
total_rows, cmd_count, any_fail, any_ok = _absorb_lane_result(
|
||||
exc,
|
||||
total_rows=0,
|
||||
cmd_count=0,
|
||||
any_fail=False,
|
||||
any_ok=False,
|
||||
lane_errors=lane_errors,
|
||||
)
|
||||
else:
|
||||
try:
|
||||
total_rows, cmd_count, any_fail, any_ok = _run_light()
|
||||
except Exception as exc:
|
||||
total_rows, cmd_count, any_fail, any_ok = _absorb_lane_result(
|
||||
exc,
|
||||
total_rows=0,
|
||||
cmd_count=0,
|
||||
any_fail=False,
|
||||
any_ok=False,
|
||||
lane_errors=lane_errors,
|
||||
)
|
||||
|
||||
if lane_errors and not any_ok and cmd_count == 0:
|
||||
raise RuntimeError("; ".join(lane_errors)[:1020])
|
||||
|
||||
batch = db.get(BizStateBatch, batch_id)
|
||||
if batch:
|
||||
batch.command_count = cmd_count
|
||||
|
|
@ -689,11 +853,16 @@ def _run_collect_session(
|
|||
batch.ended_at = _utcnow()
|
||||
if any_fail and any_ok:
|
||||
batch.status = "partial"
|
||||
if lane_errors:
|
||||
batch.message = "; ".join(lane_errors)[:1020]
|
||||
elif any_fail and not any_ok:
|
||||
batch.status = "failed"
|
||||
batch.message = "all commands failed"
|
||||
batch.message = (
|
||||
"; ".join(lane_errors)[:1020] if lane_errors else "all commands failed"
|
||||
)
|
||||
else:
|
||||
batch.status = "success"
|
||||
batch.message = ""
|
||||
db.commit()
|
||||
if batch.status in ("success", "partial"):
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -64,6 +64,8 @@ class ParseProfile:
|
|||
kind: str = "collect" # collect | discover
|
||||
aux_commands: list[AuxCommand] = field(default_factory=list)
|
||||
enrich_joins: list[EnrichJoin] = field(default_factory=list)
|
||||
# light: default shared SSH lane; heavy: dedicated long-timeout connection
|
||||
collect_lane: str = "light"
|
||||
|
||||
|
||||
_LLDP_FIELDS: list[FieldDef] = [
|
||||
|
|
@ -1270,11 +1272,32 @@ def _zte_status_profiles() -> list[ParseProfile]:
|
|||
|
||||
_PROFILES: list[ParseProfile] | None = None
|
||||
|
||||
# Huge CLI dumps: dedicated heavy lane (longer read_timeout / own SSH).
|
||||
_HEAVY_LANE_METRICS = frozenset(
|
||||
{
|
||||
"interface_detail",
|
||||
"ip_route",
|
||||
"ipv6_route",
|
||||
"bgp_route",
|
||||
"l2vpn_mac",
|
||||
"evpn_mac",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _apply_collect_lanes(profiles: list[ParseProfile]) -> list[ParseProfile]:
|
||||
for p in profiles:
|
||||
if str(p.metric_id or "").strip() in _HEAVY_LANE_METRICS:
|
||||
p.collect_lane = "heavy"
|
||||
return profiles
|
||||
|
||||
|
||||
def all_profiles() -> list[ParseProfile]:
|
||||
global _PROFILES
|
||||
if _PROFILES is None:
|
||||
_PROFILES = _lldp_profiles() + _vrf_profiles() + _zte_status_profiles()
|
||||
_PROFILES = _apply_collect_lanes(
|
||||
_lldp_profiles() + _vrf_profiles() + _zte_status_profiles()
|
||||
)
|
||||
return list(_PROFILES)
|
||||
|
||||
|
||||
|
|
@ -1357,6 +1380,7 @@ def profile_to_public_dict(p: ParseProfile, *, overrides: dict[str, Any] | None
|
|||
"tags": list(p.tags),
|
||||
"sort_order": p.sort_order,
|
||||
"enabled": bool(ov.get("enabled")) if "enabled" in ov else p.enabled,
|
||||
"collect_lane": str(p.collect_lane or "light"),
|
||||
"kind": p.kind,
|
||||
"match": p.match,
|
||||
"textfsm_command": p.textfsm_command or p.command_template,
|
||||
|
|
|
|||
|
|
@ -107,6 +107,10 @@ class Settings(BaseSettings):
|
|||
biz_state_scheduler_enabled: bool = True
|
||||
biz_state_scheduler_tick_sec: int = 15
|
||||
biz_state_dispatch_workers: int = 2
|
||||
# Heavy CLI lane (interface detail / routes / MAC): own SSH + longer timeouts.
|
||||
biz_state_heavy_read_timeout_sec: int = 300
|
||||
biz_state_heavy_run_timeout_cap_sec: int = 900
|
||||
biz_state_heavy_workers: int = 4
|
||||
# Managed NE exec: max CLI commands per request (lab can raise; hard-capped in ne_exec).
|
||||
ne_exec_max_commands: int = 5
|
||||
# WebCRT interactive terminal sessions (multi-operator concurrent terminals).
|
||||
|
|
|
|||
162
tests/test_collect_lanes.py
Normal file
162
tests/test_collect_lanes.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
"""Light/heavy collect lane partition and dual-lane scheduling."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from netx_api.biz_state.collect_runner import (
|
||||
partition_work,
|
||||
work_item_lane,
|
||||
)
|
||||
from netx_api.biz_state.profiles import get_profile, reload_profiles
|
||||
|
||||
|
||||
class CollectLaneProfileTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
reload_profiles()
|
||||
|
||||
def test_interface_detail_is_heavy(self) -> None:
|
||||
p = get_profile("zte.interface_detail")
|
||||
self.assertIsNotNone(p)
|
||||
assert p is not None
|
||||
self.assertEqual(p.collect_lane, "heavy")
|
||||
|
||||
def test_route_profiles_are_heavy(self) -> None:
|
||||
for pid in (
|
||||
"zte.ip_route_vrf",
|
||||
"zte.ip_route",
|
||||
"zte.ipv6_route_vrf",
|
||||
"zte.ipv6_route",
|
||||
):
|
||||
p = get_profile(pid)
|
||||
self.assertIsNotNone(p, pid)
|
||||
assert p is not None
|
||||
self.assertEqual(p.collect_lane, "heavy", pid)
|
||||
|
||||
def test_arp_is_light(self) -> None:
|
||||
p = get_profile("zte.arp")
|
||||
self.assertIsNotNone(p)
|
||||
assert p is not None
|
||||
self.assertEqual(p.collect_lane, "light")
|
||||
|
||||
|
||||
class PartitionWorkTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
reload_profiles()
|
||||
|
||||
def test_custom_raw_always_light(self) -> None:
|
||||
self.assertEqual(work_item_lane("", "custom"), "light")
|
||||
self.assertEqual(work_item_lane("zte.interface_detail", "custom"), "light")
|
||||
|
||||
def test_partition_mixed(self) -> None:
|
||||
work = [
|
||||
("show arp", {}, "zte.arp", "i1", "normal"),
|
||||
(
|
||||
"show interface | include ifindex",
|
||||
{},
|
||||
"zte.interface_detail",
|
||||
"i2",
|
||||
"normal",
|
||||
),
|
||||
("show ip forwarding route vrf a", {}, "zte.ip_route_vrf", "i3", "normal"),
|
||||
("show version", {}, "", "i4", "custom"),
|
||||
]
|
||||
light, heavy = partition_work(work)
|
||||
self.assertEqual(len(light), 2)
|
||||
self.assertEqual({x[2] for x in light}, {"zte.arp", ""})
|
||||
self.assertEqual(len(heavy), 2)
|
||||
self.assertEqual(
|
||||
{x[2] for x in heavy},
|
||||
{"zte.interface_detail", "zte.ip_route_vrf"},
|
||||
)
|
||||
|
||||
def test_partition_light_only(self) -> None:
|
||||
work = [("show arp", {}, "zte.arp", "i1", "normal")]
|
||||
light, heavy = partition_work(work)
|
||||
self.assertEqual(len(light), 1)
|
||||
self.assertEqual(heavy, [])
|
||||
|
||||
def test_partition_heavy_only(self) -> None:
|
||||
work = [
|
||||
(
|
||||
"show interface | include ifindex",
|
||||
{},
|
||||
"zte.interface_detail",
|
||||
"i1",
|
||||
"normal",
|
||||
)
|
||||
]
|
||||
light, heavy = partition_work(work)
|
||||
self.assertEqual(light, [])
|
||||
self.assertEqual(len(heavy), 1)
|
||||
|
||||
|
||||
class DualLaneParallelTests(unittest.TestCase):
|
||||
def test_light_not_blocked_by_heavy_sleep(self) -> None:
|
||||
"""When both lanes run, light completes without waiting for heavy."""
|
||||
from netx_api.biz_state import collect_runner as cr
|
||||
|
||||
order: list[str] = []
|
||||
|
||||
def fake_lane(*, work, label, **_kwargs):
|
||||
if "heavy" in label:
|
||||
time.sleep(0.35)
|
||||
order.append("heavy_done")
|
||||
return (0, len(work), False, True)
|
||||
order.append("light_done")
|
||||
return (0, len(work), False, True)
|
||||
|
||||
light_work = [("show arp", {}, "zte.arp", "i1", "normal")]
|
||||
heavy_work = [
|
||||
(
|
||||
"show interface | include ifindex",
|
||||
{},
|
||||
"zte.interface_detail",
|
||||
"i2",
|
||||
"normal",
|
||||
)
|
||||
]
|
||||
|
||||
with patch.object(cr, "_run_collect_lane", side_effect=fake_lane):
|
||||
t0 = time.perf_counter()
|
||||
heavy_fut = cr._heavy_cli_pool().submit(
|
||||
lambda: cr._run_collect_lane(
|
||||
work=heavy_work,
|
||||
batch_id="b",
|
||||
creds={},
|
||||
vendor_eff="zte",
|
||||
device_type_eff="",
|
||||
vendor_key="zte",
|
||||
per_cmd=1,
|
||||
cap=10,
|
||||
label="biz_state_heavy",
|
||||
)
|
||||
)
|
||||
light_res = cr._run_collect_lane(
|
||||
work=light_work,
|
||||
batch_id="b",
|
||||
creds={},
|
||||
vendor_eff="zte",
|
||||
device_type_eff="",
|
||||
vendor_key="zte",
|
||||
per_cmd=1,
|
||||
cap=10,
|
||||
label="biz_state_light",
|
||||
)
|
||||
light_elapsed = time.perf_counter() - t0
|
||||
heavy_res = heavy_fut.result()
|
||||
total_elapsed = time.perf_counter() - t0
|
||||
|
||||
self.assertEqual(light_res[1], 1)
|
||||
self.assertEqual(heavy_res[1], 1)
|
||||
self.assertEqual(order[0], "light_done")
|
||||
self.assertLess(light_elapsed, 0.25)
|
||||
self.assertGreaterEqual(total_elapsed, 0.3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -197,6 +197,7 @@ export function BizStatePage() {
|
|||
|
||||
const [tasks, setTasks] = useState<TaskRow[]>([]);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [collectingIds, setCollectingIds] = useState<Record<string, true>>({});
|
||||
const [listKeyword, setListKeyword] = useState("");
|
||||
const debouncedListKw = useDebouncedValue(listKeyword, 250);
|
||||
const [purposeFilter, setPurposeFilter] = useState<"all" | "portrait" | "cutover_hf">("all");
|
||||
|
|
@ -253,7 +254,9 @@ export function BizStatePage() {
|
|||
const purpose =
|
||||
purposeFilter === "all" ? "" : purposeFilter === "portrait" ? "portrait" : "cutover_hf";
|
||||
const res = await bizStateListTasks(purpose);
|
||||
setTasks((res.items || []) as TaskRow[]);
|
||||
const items = (res.items || []) as TaskRow[];
|
||||
setTasks(items);
|
||||
return items;
|
||||
}, [purposeFilter]);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -619,27 +622,58 @@ export function BizStatePage() {
|
|||
}
|
||||
};
|
||||
|
||||
const collectNow = async () => {
|
||||
if (!taskId) return;
|
||||
setBusy(true);
|
||||
const setTaskCollecting = (id: string, on: boolean) => {
|
||||
setCollectingIds((prev) => {
|
||||
if (on) {
|
||||
if (prev[id]) return prev;
|
||||
return { ...prev, [id]: true };
|
||||
}
|
||||
if (!prev[id]) return prev;
|
||||
const next = { ...prev };
|
||||
delete next[id];
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const collectNowForTask = async (id: string, fromModal = false) => {
|
||||
setTaskCollecting(id, true);
|
||||
try {
|
||||
await bizStateCollectNow(taskId);
|
||||
await bizStateCollectNow(id);
|
||||
showOk(t("bizState.collecting"));
|
||||
for (let i = 0; i < 20; i++) {
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
const task = await bizStateGetTask(taskId);
|
||||
if (fromModal && taskId === id) {
|
||||
try {
|
||||
const task = await bizStateGetTask(id);
|
||||
setDetail(task);
|
||||
if (!task.collect_running) break;
|
||||
}
|
||||
await loadTask(taskId);
|
||||
await refreshTasks();
|
||||
if (!task.collect_running) break;
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const items = await refreshTasks();
|
||||
const latest = items.find((x) => x.id === id);
|
||||
if (!latest?.collect_running) break;
|
||||
}
|
||||
await refreshTasks();
|
||||
if (fromModal && taskId === id) {
|
||||
await loadTask(id);
|
||||
setTaskTab("batches");
|
||||
}
|
||||
} catch (e) {
|
||||
showError(formatErr(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
setTaskCollecting(id, false);
|
||||
}
|
||||
};
|
||||
|
||||
const collectNow = async () => {
|
||||
if (!taskId) return;
|
||||
await collectNowForTask(taskId, true);
|
||||
};
|
||||
|
||||
const removeTask = async (id: string) => {
|
||||
if (!window.confirm(t("bizState.confirmDelete"))) return;
|
||||
setBusy(true);
|
||||
|
|
@ -939,27 +973,8 @@ export function BizStatePage() {
|
|||
<Button
|
||||
size="sm"
|
||||
variant="secondary"
|
||||
isDisabled={busy || row.collect_running}
|
||||
onPress={async () => {
|
||||
setBusy(true);
|
||||
try {
|
||||
await bizStateCollectNow(row.id);
|
||||
showOk(t("bizState.collecting"));
|
||||
for (let i = 0; i < 20; i++) {
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
await refreshTasks();
|
||||
const cur = (await bizStateListTasks(purposeFilter === "all" ? "" : purposeFilter)).items?.find(
|
||||
(x: any) => x.id === row.id,
|
||||
);
|
||||
if (!cur?.collect_running) break;
|
||||
}
|
||||
await refreshTasks();
|
||||
} catch (e) {
|
||||
showError(formatErr(e));
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}}
|
||||
isDisabled={Boolean(row.collect_running || collectingIds[row.id])}
|
||||
onPress={() => void collectNowForTask(row.id, false)}
|
||||
>
|
||||
{t("bizState.collectNow")}
|
||||
</Button>
|
||||
|
|
@ -1439,7 +1454,7 @@ export function BizStatePage() {
|
|||
<Button
|
||||
size="sm"
|
||||
variant="primary"
|
||||
isDisabled={busy || Boolean(detail?.collect_running)}
|
||||
isDisabled={Boolean(detail?.collect_running || (taskId && collectingIds[taskId]))}
|
||||
onPress={() => void collectNow()}
|
||||
>
|
||||
{t("bizState.collectNow")}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue