Cache biz-state discover CLI results and improve bind table UX.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-22 09:48:43 +08:00
parent b9be545a75
commit 908771baea
8 changed files with 316 additions and 63 deletions

View file

@ -1,7 +1,13 @@
"""One-shot discover for placeholder candidates (VRF list, etc.)."""
"""One-shot discover for placeholder candidates (VRF list, etc.).
Same NE + discover profile reuses CLI/parse results within a TTL so binding
multiple AF monitor items (vpnv4/vpnv6/VRF) does not re-login the device.
"""
from __future__ import annotations
import threading
import time
from typing import Any
from fastapi import HTTPException
@ -21,6 +27,81 @@ from .command_match import (
from .parsers import get_parser, run_parser
from .profiles import get_profile
# Process-local cache: (source, ne_id, discover_profile_id) → parsed rows.
# Binding UI opens many collect profiles that share config_vrf / config_bgp_peer.
_CACHE_LOCK = threading.Lock()
_DISCOVER_CACHE: dict[str, dict[str, Any]] = {}
_DISCOVER_CACHE_TTL_SEC = 1800 # 30 min — covers a typical bind session
def _cache_key(source: str, ne_id: str, discover_profile_id: str) -> str:
return f"{source}|{ne_id}|{discover_profile_id}"
def clear_discover_cache(
*,
source: str = "",
ne_id: str = "",
discover_profile_id: str = "",
) -> int:
"""Drop cache entries; empty args clear all. Returns removed count."""
src = str(source or "").strip().lower()
nid = str(ne_id or "").strip()
pid = str(discover_profile_id or "").strip()
with _CACHE_LOCK:
if not src and not nid and not pid:
n = len(_DISCOVER_CACHE)
_DISCOVER_CACHE.clear()
return n
drop = [
k
for k, v in _DISCOVER_CACHE.items()
if (not src or v.get("source") == src)
and (not nid or v.get("ne_id") == nid)
and (not pid or v.get("discover_profile_id") == pid)
]
for k in drop:
_DISCOVER_CACHE.pop(k, None)
return len(drop)
def _cache_get(key: str) -> dict[str, Any] | None:
now = time.time()
with _CACHE_LOCK:
hit = _DISCOVER_CACHE.get(key)
if not hit:
return None
if float(hit.get("expires_at") or 0) <= now:
_DISCOVER_CACHE.pop(key, None)
return None
return dict(hit)
def _cache_put(
key: str,
*,
source: str,
ne_id: str,
discover_profile_id: str,
command: str,
vendor: str,
device_type: str,
records: list[dict[str, Any]],
raw_preview: str,
) -> None:
with _CACHE_LOCK:
_DISCOVER_CACHE[key] = {
"source": source,
"ne_id": ne_id,
"discover_profile_id": discover_profile_id,
"command": command,
"vendor": vendor,
"device_type": device_type,
"records": list(records),
"raw_preview": raw_preview,
"expires_at": time.time() + _DISCOVER_CACHE_TTL_SEC,
}
def resolve_discover_profile(
*,
@ -56,6 +137,7 @@ def discover_params(
discover_profile_id: str = "",
collect_profile_id: str = "",
placeholder: str = "",
force_refresh: bool = False,
) -> dict[str, Any]:
src = str(source or "managed").strip().lower() or "managed"
nid = str(ne_id or "").strip()
@ -94,69 +176,97 @@ def discover_params(
value_field = str(active_ph.discover_value_field or active_ph.name or value_field)
label_field = str(active_ph.discover_label_field or label_field)
try:
if src == "managed":
creds, info = resolve_cli_target(db, managed_ne_id=nid)
elif src == "ume":
creds, info = resolve_cli_target(db, ume_ne_id=nid)
else:
raise HTTPException(status_code=400, detail="invalid_source")
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=400, detail=f"resolve_failed: {exc}") from exc
skip = cli_creds_skip_reason(creds, interactive=False)
if skip:
raise HTTPException(status_code=400, detail=skip)
vendor = str(info.get("vendor") or creds.get("vendor") or "")
device_type = str(info.get("device_type") or creds.get("device_type") or "")
cache_key = _cache_key(src, nid, disc.profile_id)
cache_hit = False
vendor = ""
device_type = ""
command = str(disc.command_template or "").strip()
per_cmd = int(settings.ne_collect_read_timeout_sec or 120)
raw = ""
try:
conn = open_netmiko_connection(creds, session_timeout=per_cmd + 60)
try:
try:
disable_target_paging(conn, vendor=vendor, device_type=device_type)
except Exception:
pass
raw = send_show_command(conn, command, read_timeout=per_cmd)
finally:
close_netmiko_connection(conn)
except Exception as exc:
return {
"ok": False,
"error": f"cli_failed: {exc}",
"discover_profile_id": disc.profile_id,
"command": command,
"candidates": [],
"raw_preview": str(raw or "")[:4000],
}
records: list[dict[str, Any]] = []
if get_parser(disc.parser_id):
cached = None if force_refresh else _cache_get(cache_key)
if cached:
cache_hit = True
records = list(cached.get("records") or [])
command = str(cached.get("command") or command)
vendor = str(cached.get("vendor") or "")
device_type = str(cached.get("device_type") or "")
raw = str(cached.get("raw_preview") or "")
else:
try:
records, _fsm_tables, _rule_keys = run_parser(
disc.parser_id,
raw_text=raw,
vendor=vendor,
device_type=device_type,
command=disc.textfsm_command or command,
params={},
textfsm_command=disc.textfsm_command or "",
)
if src == "managed":
creds, info = resolve_cli_target(db, managed_ne_id=nid)
elif src == "ume":
creds, info = resolve_cli_target(db, ume_ne_id=nid)
else:
raise HTTPException(status_code=400, detail="invalid_source")
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=400, detail=f"resolve_failed: {exc}") from exc
skip = cli_creds_skip_reason(creds, interactive=False)
if skip:
raise HTTPException(status_code=400, detail=skip)
vendor = str(info.get("vendor") or creds.get("vendor") or "")
device_type = str(info.get("device_type") or creds.get("device_type") or "")
per_cmd = int(settings.ne_collect_read_timeout_sec or 120)
try:
conn = open_netmiko_connection(creds, session_timeout=per_cmd + 60)
try:
try:
disable_target_paging(conn, vendor=vendor, device_type=device_type)
except Exception:
pass
raw = send_show_command(conn, command, read_timeout=per_cmd)
finally:
close_netmiko_connection(conn)
except Exception as exc:
return {
"ok": False,
"error": f"parse_failed: {exc}",
"error": f"cli_failed: {exc}",
"discover_profile_id": disc.profile_id,
"command": command,
"cache_hit": False,
"candidates": [],
"raw_preview": str(raw or "")[:4000],
}
if get_parser(disc.parser_id):
try:
records, _fsm_tables, _rule_keys = run_parser(
disc.parser_id,
raw_text=raw,
vendor=vendor,
device_type=device_type,
command=disc.textfsm_command or command,
params={},
textfsm_command=disc.textfsm_command or "",
)
except Exception as exc:
return {
"ok": False,
"error": f"parse_failed: {exc}",
"discover_profile_id": disc.profile_id,
"command": command,
"cache_hit": False,
"candidates": [],
"raw_preview": str(raw or "")[:4000],
}
_cache_put(
cache_key,
source=src,
ne_id=nid,
discover_profile_id=disc.profile_id,
command=command,
vendor=vendor,
device_type=device_type,
records=records,
raw_preview=str(raw or "")[:4000],
)
candidates = []
seen: set[str] = set()
# Shared discover profile → one candidate per (placeholder fields) tuple.
@ -164,7 +274,6 @@ def discover_params(
filter_ph = pair_phs[0]
for rec in records:
if not _record_passes_discover_filter(rec, filter_ph):
# Also require every paired field non-empty.
continue
bind: dict[str, str] = {}
ok = True
@ -240,6 +349,7 @@ def discover_params(
"vendor_key": resolve_vendor_key(vendor, device_type),
"value_field": value_field,
"pair_mode": bool(pair_phs and len(pair_phs) >= 2),
"cache_hit": cache_hit,
"candidates": candidates,
"raw_preview": str(raw or "")[:4000],
}

View file

@ -85,6 +85,7 @@ class DiscoverIn(BaseModel):
discover_profile_id: str = ""
collect_profile_id: str = ""
placeholder: str = ""
force_refresh: bool = False
class BindingsIn(BaseModel):
@ -139,6 +140,7 @@ def api_discover(body: DiscoverIn, db: Session = Depends(get_db)) -> dict[str, A
discover_profile_id=body.discover_profile_id,
collect_profile_id=body.collect_profile_id,
placeholder=body.placeholder,
force_refresh=bool(body.force_refresh),
)

View file

@ -0,0 +1,91 @@
"""Discover CLI/parse cache reuse across collect profiles."""
from __future__ import annotations
import unittest
from unittest.mock import MagicMock, patch
from netx_api.biz_state import discover as disc_mod
from netx_api.biz_state.discover import clear_discover_cache, discover_params
from netx_api.biz_state.profiles import get_profile
class DiscoverCacheTests(unittest.TestCase):
def setUp(self) -> None:
clear_discover_cache()
def tearDown(self) -> None:
clear_discover_cache()
def test_shared_bgp_peer_discover_reuses_cli(self) -> None:
v4 = get_profile("zte.bgp_vpnv4_neighbor_in")
v6 = get_profile("zte.bgp_vpnv6_neighbor_in")
assert v4 is not None and v6 is not None
self.assertEqual(v4.placeholders[0].discover_profile_id, "zte.config_bgp_peer")
self.assertEqual(v6.placeholders[0].discover_profile_id, "zte.config_bgp_peer")
records = [
{"afi": "vpnv4", "vrf": "", "neighbor": "10.0.0.1", "remote_as": "65001"},
{"afi": "vpnv6", "vrf": "", "neighbor": "FC00::1", "remote_as": "65002"},
]
db = MagicMock()
with (
patch.object(disc_mod, "resolve_cli_target", return_value=({"vendor": "zte"}, {"vendor": "zte", "device_type": "zte_zxros"})),
patch.object(disc_mod, "cli_creds_skip_reason", return_value=None),
patch.object(disc_mod, "open_netmiko_connection", return_value=MagicMock()) as open_conn,
patch.object(disc_mod, "disable_target_paging"),
patch.object(disc_mod, "send_show_command", return_value="raw-bgp") as send_cmd,
patch.object(disc_mod, "close_netmiko_connection"),
patch.object(disc_mod, "get_parser", return_value=True),
patch.object(
disc_mod,
"run_parser",
return_value=(records, {}, ()),
) as run_parser,
):
first = discover_params(
db,
source="managed",
ne_id="ne1",
collect_profile_id="zte.bgp_vpnv4_neighbor_in",
placeholder="neighbor",
)
self.assertTrue(first["ok"])
self.assertFalse(first.get("cache_hit"))
self.assertEqual([c["value"] for c in first["candidates"]], ["10.0.0.1"])
self.assertEqual(open_conn.call_count, 1)
self.assertEqual(send_cmd.call_count, 1)
self.assertEqual(run_parser.call_count, 1)
second = discover_params(
db,
source="managed",
ne_id="ne1",
collect_profile_id="zte.bgp_vpnv6_neighbor_in",
placeholder="neighbor",
)
self.assertTrue(second["ok"])
self.assertTrue(second.get("cache_hit"))
self.assertEqual([c["value"] for c in second["candidates"]], ["FC00::1"])
# Same discover profile → no second SSH/parse
self.assertEqual(open_conn.call_count, 1)
self.assertEqual(send_cmd.call_count, 1)
self.assertEqual(run_parser.call_count, 1)
forced = discover_params(
db,
source="managed",
ne_id="ne1",
collect_profile_id="zte.bgp_vpnv4_neighbor_in",
placeholder="neighbor",
force_refresh=True,
)
self.assertTrue(forced["ok"])
self.assertFalse(forced.get("cache_hit"))
self.assertEqual(open_conn.call_count, 2)
self.assertEqual(send_cmd.call_count, 2)
if __name__ == "__main__":
unittest.main()

View file

@ -239,11 +239,14 @@ const en = {
bindTitle: "Select bindings",
unbound: "Not bound",
bindHintRequired: "Bind params before collect",
paramsDblClickHint: "Double-click to view all bindings",
paramsDblClickHint: "Click to view all bindings",
paramsClickHint: "Click to view all bindings",
paramsDetailTitle: "Bound parameters",
paramsDetailCount: "{{n}} binding(s)",
discoverLoading: "Discovering…",
discoverEmpty: "No candidates discovered",
discoverCacheHit: "reused (no re-login)",
discoverRefresh: "Refresh from device",
selectAllVrfs: "Select all",
deselectAllVrfs: "Deselect all",
batches: "Batches",

View file

@ -239,11 +239,14 @@ const zh = {
bindTitle: "选择参数关联",
unbound: "未关联",
bindHintRequired: "需关联参数后才可采集",
paramsDblClickHint: "双击查看全部关联明细",
paramsDblClickHint: "单击查看全部关联明细",
paramsClickHint: "单击查看全部关联明细",
paramsDetailTitle: "参数关联明细",
paramsDetailCount: "共 {{n}} 条",
discoverLoading: "正在发现…",
discoverEmpty: "未发现可用选项",
discoverCacheHit: "已复用(未重新登录)",
discoverRefresh: "重新从设备拉取",
selectAllVrfs: "全选",
deselectAllVrfs: "取消全选",
batches: "采集批次",

View file

@ -11292,28 +11292,44 @@ html.login-page--paused .login-page__flare {
}
.bs-profiles-table {
min-width: 1100px;
min-width: 1280px;
table-layout: fixed;
border-collapse: separate;
border-spacing: 0;
}
.bs-profiles-table thead th {
position: sticky;
top: 0;
z-index: 3;
background: rgba(15, 23, 42, 0.97);
box-shadow: 0 1px 0 rgba(148, 163, 184, 0.25);
}
.bs-profiles-table th:nth-child(1),
.bs-profiles-table td:nth-child(1) {
width: 56px;
width: 52px;
}
.bs-profiles-table th:nth-child(2),
.bs-profiles-table td:nth-child(2) {
width: 28%;
width: 24%;
}
.bs-profiles-table th:nth-child(3),
.bs-profiles-table td:nth-child(3) {
width: 180px;
width: 34%;
min-width: 320px;
}
.bs-profiles-table th:nth-child(4),
.bs-profiles-table td:nth-child(4) {
width: 110px;
width: 120px;
}
.bs-profiles-table th:nth-child(5),
.bs-profiles-table td:nth-child(5) {
width: auto;
}
.bs-profiles-table .bs-cmd-cell {

View file

@ -246,6 +246,7 @@ export function BizStatePage() {
const [discoverCmd, setDiscoverCmd] = useState("");
const [discoverLoading, setDiscoverLoading] = useState(false);
const [discoverError, setDiscoverError] = useState("");
const [discoverCacheHit, setDiscoverCacheHit] = useState(false);
// Bound-params detail modal (double-click params cell)
const [paramsDetail, setParamsDetail] = useState<{
@ -775,10 +776,11 @@ export function BizStatePage() {
setSelectedVrfs([]);
setDiscoverCmd("");
setDiscoverError("");
setDiscoverCacheHit(false);
setDiscoverLoading(false);
};
const startDiscover = async (item: any) => {
const startDiscover = async (item: any, forceRefresh = false) => {
if (!taskId) return;
const prof = profiles.find((p) => p.profile_id === item.source_profile_id);
const phs = prof?.placeholders || [];
@ -792,6 +794,7 @@ export function BizStatePage() {
setSelectedVrfs([]);
setDiscoverCmd("");
setDiscoverError("");
setDiscoverCacheHit(false);
setDiscoverLoading(true);
setBusy(true);
try {
@ -801,6 +804,7 @@ export function BizStatePage() {
task_id: taskId,
collect_profile_id: item.source_profile_id,
placeholder: sharedDisc ? "" : ph.name,
force_refresh: forceRefresh,
});
if (!res.ok) {
const err = res.error || t("bizState.discoverFailed");
@ -809,6 +813,7 @@ export function BizStatePage() {
return;
}
setDiscoverCmd(res.command || "");
setDiscoverCacheHit(Boolean(res.cache_hit));
const cand = (res.candidates || []) as Candidate[];
setCandidates(cand);
const existing = (item.bindings || []) as { placeholder?: string; value?: string }[];
@ -1459,10 +1464,10 @@ export function BizStatePage() {
}${bindLines.length ? " bs-params-scroll--clickable" : ""}`}
title={
bindLines.length
? `${bindHint}\n${t("bizState.paramsDblClickHint")}`
? `${bindHint}\n${t("bizState.paramsClickHint")}`
: bindHint
}
onDoubleClick={() => {
onClick={() => {
if (!bindLines.length) return;
setParamsDetail({
title: prof.title,
@ -1689,6 +1694,7 @@ export function BizStatePage() {
<p className="muted">
<code>{discoverCmd}</code>
{candidates.length ? ` · ${selectedVrfs.length}/${candidates.length}` : null}
{discoverCacheHit ? ` · ${t("bizState.discoverCacheHit")}` : null}
</p>
) : null}
{discoverError ? <p className="bs-params-warn">{discoverError}</p> : null}
@ -1711,6 +1717,19 @@ export function BizStatePage() {
? t("bizState.deselectAllVrfs")
: t("bizState.selectAllVrfs")}
</Button>
{bindItemId ? (
<Button
size="sm"
variant="ghost"
isDisabled={busy || discoverLoading}
onPress={() => {
const item = (detail?.items || []).find((it: any) => it.id === bindItemId);
if (item) void startDiscover(item, true);
}}
>
{t("bizState.discoverRefresh")}
</Button>
) : null}
</div>
<div className="bs-bind-list bs-bind-list--modal">
{candidates.map((c) => (

View file

@ -1777,7 +1777,16 @@ export const bizStateDiscover = (body: Record<string, unknown>) =>
ok: boolean;
error?: string;
command?: string;
candidates?: Array<{ value: string; label: string; rd?: string; protocols?: string }>;
cache_hit?: boolean;
pair_mode?: boolean;
discover_profile_id?: string;
candidates?: Array<{
value: string;
label: string;
rd?: string;
protocols?: string;
bindings?: Record<string, string>;
}>;
raw_preview?: string;
}>("/v1/biz-state/discover", body);