hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-09 02:00:46 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 2

Add ZTE BGP VRF summary aux FIB collect with optional all-VRF expand.

Browse source 
Wire config_vrf plus ipv4/ipv6 forwarding as aux for VPN VRF summaries, and allow unbound items to expand all AF-filtered VRFs at collect time.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-21 17:23:34 +08:00
parent 78f7032d7b
commit 3576e4a69c
9 changed files with 390 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

97
netx_api/biz_state/collect_runner.py
View file

@ -35,7 +35,13 @@ from .collect_session import (
resolve_aux_command,
run_primary_with_bundle,
)
from .command_match import expand_from_bindings, match_command, normalize_command
from .command_match import (
EXPAND_ALL_COMMAND,
expand_bindings_from_discover_records,
expand_from_bindings,
match_command,
normalize_command,
)
from .parsers import get_parser
from .profiles import get_profile
@ -404,7 +410,69 @@ def _run_collect_lane(
batch_row = sdb.get(BizStateBatch, batch_id)
if not batch_row:
return 0, 0, True, False
# Resolve expand_all → concrete per-VRF commands via discover profile.
flat_work: list[WorkItem] = []
aux_persisted: set[tuple[str, str]] = set()
for concrete, params, profile_id, item_id, mode in work:
if mode != "expand_all":
flat_work.append((concrete, params, profile_id, item_id, mode))
continue
profile = get_profile(profile_id)
if profile is None or not profile.placeholders:
any_fail = True
_append_event(
sdb,
task_id=str(batch_row.task_id or ""),
message=f"expand_all missing profile {profile_id}",
level="error",
)
continue
ph = profile.placeholders[0]
disc = get_profile(str(ph.discover_profile_id or "").strip())
if disc is None:
any_fail = True
_append_event(
sdb,
task_id=str(batch_row.task_id or ""),
message=f"expand_all discover profile missing for {profile_id}",
level="error",
)
continue
disc_cmd = normalize_command(disc.command_template)
entry, _ = session.fetch_and_parse(
disc_cmd,
parser_id=disc.parser_id,
textfsm_command=disc.textfsm_command or disc_cmd,
params={},
)
if not entry.ok:
any_fail = True
_append_event(
sdb,
task_id=str(batch_row.task_id or ""),
message=f"expand_all discover failed: {entry.error}",
level="error",
)
continue
try:
pairs = expand_bindings_from_discover_records(
profile=profile,
records=entry.records,
)
except ValueError as exc:
any_fail = True
_append_event(
sdb,
task_id=str(batch_row.task_id or ""),
message=str(exc),
level="error",
)
continue
for cmd, p in pairs:
flat_work.append((cmd, p, profile_id, item_id, "normal"))
for concrete, params, profile_id, item_id, mode in flat_work:
if holder.get("timed_out"):
raise TimeoutError(f"{label}_aborted")
cmd_count += 1
@ -530,6 +598,9 @@ def _run_collect_lane(
cmd_row_id=aux_row.id,
)
aux_results[ra.key] = entry
aux_mid = str(getattr(ra.profile, "metric_id", "") or "").strip()
if aux_mid:
aux_row.metric_id = aux_mid
if cache_hit:
aux_row.parse_status = "aux_cached"
aux_row.message = (
@ -549,6 +620,26 @@ def _run_collect_lane(
aux_row.raw_text = entry.raw
aux_row.row_count = len(entry.records or [])
entry.cmd_row_id = aux_row.id
# Persist aux metrics once per CLI (config_vrf / FIB shared across VRFs).
if (
entry.ok
and entry.records
and aux_mid in _GENERIC_METRICS
):
persist_key = (normalize_command(ra.command), aux_mid)
if persist_key not in aux_persisted:
n_aux = _persist_metric_rows(
sdb,
batch=batch_row,
cmd_row=aux_row,
metric_id=aux_mid,
records=entry.records,
)
aux_row.row_count = n_aux
total_rows += n_aux
aux_persisted.add(persist_key)
if n_aux:
_bump_batch_progress(batch_id, add_rows=n_aux)
sdb.add(aux_row)
sdb.commit()
@ -786,6 +877,10 @@ def _run_collect_session(
_append_event(db, task_id=task_id, message=str(exc), level="error")
continue
for concrete, params in pairs:
if concrete == EXPAND_ALL_COMMAND:
# Defer VRF list expansion until CollectSession is open.
work.append(("", dict(params or {}), profile.profile_id, item.id, "expand_all"))
continue
cmd = normalize_command(concrete)
# Prefer match_command metric so remapped if_intf shares key with config_interface
hit = match_command(vendor_key=vendor_key, command=cmd)

78
netx_api/biz_state/command_match.py
View file

@ -6,7 +6,10 @@ import re
from dataclasses import dataclass
from typing import Any
from .profiles import ParseProfile, all_profiles, get_profile
from .profiles import ParseProfile, PlaceholderDef, all_profiles, get_profile
# Sentinel concrete command: collect expands all discover values at runtime.
EXPAND_ALL_COMMAND = "__expand_all__"
@dataclass(frozen=True)
@ -85,13 +88,52 @@ def normalize_binding_dicts(
return converted
def _optional_discover_placeholders(profile: ParseProfile) -> list[PlaceholderDef]:
return [
ph
for ph in (profile.placeholders or [])
if (not ph.required)
and ph.bind_mode == "discover_select"
and str(ph.discover_profile_id or "").strip()
]
def filter_discover_records(
records: list[dict[str, Any]] | None,
ph: PlaceholderDef,
) -> list[str]:
"""Apply placeholder discover filter; return unique values for ``ph``."""
value_field = str(ph.discover_value_field or ph.name or "").strip() or "vrf_name"
filt_field = str(ph.discover_filter_field or "").strip()
filt_contains = str(ph.discover_filter_contains or "").strip().lower()
out: list[str] = []
seen: set[str] = set()
for rec in records or []:
if not isinstance(rec, dict):
continue
if filt_field and filt_contains:
hay = str(rec.get(filt_field) or "").strip().lower()
if filt_contains not in hay:
continue
val = str(rec.get(value_field) or "").strip()
if not val or val in seen:
continue
seen.add(val)
out.append(val)
return out
def expand_from_bindings(
*,
profile: ParseProfile,
bindings: list[dict[str, str]] | None = None,
command_override: str = "",
) -> list[tuple[str, dict[str, str]]]:
"""Return list of (concrete_command, params). Reject leftover placeholders."""
"""Return list of (concrete_command, params). Reject leftover placeholders.
When all placeholders are optional discover_select and bindings are empty,
returns a single ``(EXPAND_ALL_COMMAND, {})`` sentinel for collect-time expansion.
"""
override = normalize_command(command_override)
if override:
if "<" in override and ">" in override:
@ -107,6 +149,9 @@ def expand_from_bindings(
binds = normalize_binding_dicts(bindings, placeholders=profile.placeholders)
if not binds:
optional = _optional_discover_placeholders(profile)
if optional and len(optional) == len(profile.placeholders):
return [(EXPAND_ALL_COMMAND, {"__expand_all__": "1"})]
raise ValueError(f"profile {profile.profile_id} requires parameter bindings")
out: list[tuple[str, dict[str, str]]] = []
@ -124,6 +169,25 @@ def expand_from_bindings(
return out
def expand_bindings_from_discover_records(
*,
profile: ParseProfile,
records: list[dict[str, Any]] | None,
) -> list[tuple[str, dict[str, str]]]:
"""Build concrete commands from discover/parser records (e.g. config_vrf)."""
optional = _optional_discover_placeholders(profile)
if not optional:
raise ValueError(f"profile {profile.profile_id} has no optional discover placeholders")
if len(optional) != 1 or len(profile.placeholders) != 1:
raise ValueError(f"expand-all only supports a single optional placeholder: {profile.profile_id}")
ph = optional[0]
values = filter_discover_records(records, ph)
if not values:
raise ValueError(f"no discover values for {profile.profile_id} ({ph.discover_profile_id})")
bindings = [{ph.name: v} for v in values]
return expand_from_bindings(profile=profile, bindings=bindings)
def preview_task_item(
*,
vendor_key: str,
@ -173,6 +237,16 @@ def preview_task_item(
"message": str(exc),
}
if pairs and pairs[0][0] == EXPAND_ALL_COMMAND:
return {
"ok": True,
"kind": kind,
"profile_id": profile.profile_id,
"commands": [],
"parse": "expand_all",
"message": "no bindings: collect will expand all discover VRFs",
}
previews = []
for concrete, params in pairs:
hit = match_command(vendor_key=vendor_key, command=concrete)

17
netx_api/biz_state/discover.py
View file

@ -148,10 +148,25 @@ def discover_params(
candidates = []
seen: set[str] = set()
filt_field = ""
filt_contains = ""
if collect_profile_id:
collect = get_profile(collect_profile_id)
if collect:
for ph in collect.placeholders:
if placeholder and ph.name != placeholder:
continue
filt_field = str(ph.discover_filter_field or "").strip()
filt_contains = str(ph.discover_filter_contains or "").strip().lower()
break
for rec in records:
val = str(rec.get(value_field) or "").strip()
if not val or val in seen:
continue
if filt_field and filt_contains:
hay = str(rec.get(filt_field) or "").strip().lower()
if filt_contains not in hay:
continue
seen.add(val)
label = str(rec.get(label_field) or val).strip() or val
candidates.append(
@ -159,7 +174,7 @@ def discover_params(
"value": val,
"label": label,
"rd": str(rec.get("rd") or ""),
"protocols": str(rec.get("protocols") or ""),
"protocols": str(rec.get("protocols") or rec.get("address_families") or ""),
"extra": rec,
}
)

120
netx_api/biz_state/profiles.py
View file

@ -35,6 +35,9 @@ class PlaceholderDef:
discover_profile_id: str = ""
discover_value_field: str = ""
discover_label_field: str = ""
# When required=False and no bindings: collect expands all discover values.
discover_filter_field: str = ""
discover_filter_contains: str = ""
@dataclass(frozen=True)
@ -272,18 +275,57 @@ _BGP_PEER_FIELDS: list[FieldDef] = [
FieldDef("msg_rcvd", length=32, role="meta", display_name="MsgRcvd"),
FieldDef("msg_send", length=32, role="meta", display_name="MsgSend"),
FieldDef("up_down", length=32, role="meta", display_name="Up/Down"),
# From config_vrf enrich (VRF summary aux).
FieldDef("rd", length=64, role="meta", display_name="RD"),
FieldDef("address_families", length=64, role="meta", display_name="AF"),
]
# Neighbor / dual-placeholder profiles still require an explicit VRF bind.
_BGP_VRF_PLACEHOLDER = PlaceholderDef(
name="vrf",
schema_field="vrf",
required=True,
bind_mode="discover_select",
discover_profile_id="zte.vrf_list",
discover_profile_id="zte.config_vrf",
discover_value_field="vrf_name",
discover_label_field="vrf_name",
)
# Single-VRF collect: bind selected VRFs, or leave empty → all (from config_vrf).
_VRF_PLACEHOLDER_ALL = PlaceholderDef(
name="vrf",
schema_field="vrf",
required=False,
bind_mode="discover_select",
discover_profile_id="zte.config_vrf",
discover_value_field="vrf_name",
discover_label_field="vrf_name",
)
_VRF_PLACEHOLDER_IPV4 = PlaceholderDef(
name="vrf",
schema_field="vrf",
required=False,
bind_mode="discover_select",
discover_profile_id="zte.config_vrf",
discover_value_field="vrf_name",
discover_label_field="vrf_name",
discover_filter_field="address_families",
discover_filter_contains="ipv4",
)
_VRF_PLACEHOLDER_IPV6 = PlaceholderDef(
name="vrf",
schema_field="vrf",
required=False,
bind_mode="discover_select",
discover_profile_id="zte.config_vrf",
discover_value_field="vrf_name",
discover_label_field="vrf_name",
discover_filter_field="address_families",
discover_filter_contains="ipv6",
)
_BGP_NEIGHBOR_PLACEHOLDER = PlaceholderDef(
name="neighbor",
schema_field="neighbor",
@ -351,6 +393,8 @@ _IP_ROUTE_FIELDS: list[FieldDef] = [
FieldDef("pri", length=16, role="meta", display_name="Pri"),
FieldDef("metric", length=32, role="meta", display_name="Metric"),
FieldDef("flags", length=16, role="meta", display_name="Flags"),
FieldDef("rd", length=64, role="meta", display_name="RD"),
FieldDef("address_families", length=64, role="meta", display_name="AF"),
]
_IPV6_ROUTE_FIELDS: list[FieldDef] = [
@ -362,6 +406,8 @@ _IPV6_ROUTE_FIELDS: list[FieldDef] = [
FieldDef("pri", length=16, role="meta", display_name="Pri"),
FieldDef("metric", length=32, role="meta", display_name="Metric"),
FieldDef("flags", length=16, role="meta", display_name="Flags"),
FieldDef("rd", length=64, role="meta", display_name="RD"),
FieldDef("address_families", length=64, role="meta", display_name="AF"),
]
_L2VPN_PW_FIELDS: list[FieldDef] = [
@ -707,13 +753,28 @@ def _zte_status_profiles() -> list[ParseProfile]:
command_template="show bgp vpnv4 unicast vrf <vrf> summary | one-line",
match=r"(?i)^\s*show\s+bgp\s+vpnv4\s+unicast\s+vrf\s+(?P<vrf>\S+)\s+summary(?:\s*\|\s*one-line)?\s*$",
textfsm_command="show bgp vpnv4 unicast summary",
description="Per-VRF BGP VPNv4 peer summary.",
placeholders=[_BGP_VRF_PLACEHOLDER],
description=(
"Per-VRF BGP VPNv4 peer summary. Bind VRFs or leave empty for all "
"(from config VRF intent); aux: IPv4 FIB + config_vrf."
),
placeholders=[_VRF_PLACEHOLDER_IPV4],
fields=list(_BGP_PEER_FIELDS),
tags=["bgp", "vpnv4", "vrf", "status"],
sort_order=370,
enabled=True,
kind="collect",
aux_commands=[
AuxCommand(key="config_vrf", profile_id="zte.config_vrf"),
AuxCommand(key="ip_route", profile_id="zte.ip_route_vrf"),
],
enrich_joins=[
EnrichJoin(
from_aux="config_vrf",
left_on="vrf",
right_on="vrf_name",
take=("rd", "address_families"),
),
],
),
ParseProfile(
profile_id="zte.bgp_vpnv6_vrf_summary",
@ -724,13 +785,28 @@ def _zte_status_profiles() -> list[ParseProfile]:
command_template="show bgp vpnv6 unicast vrf <vrf> summary | one-line",
match=r"(?i)^\s*show\s+bgp\s+vpnv6\s+unicast\s+vrf\s+(?P<vrf>\S+)\s+summary(?:\s*\|\s*one-line)?\s*$",
textfsm_command="show bgp vpnv6 unicast summary",
description="Per-VRF BGP VPNv6 peer summary.",
placeholders=[_BGP_VRF_PLACEHOLDER],
description=(
"Per-VRF BGP VPNv6 peer summary. Bind VRFs or leave empty for all "
"(from config VRF intent); aux: IPv6 FIB + config_vrf."
),
placeholders=[_VRF_PLACEHOLDER_IPV6],
fields=list(_BGP_PEER_FIELDS),
tags=["bgp", "vpnv6", "vrf", "status"],
sort_order=375,
enabled=True,
kind="collect",
aux_commands=[
AuxCommand(key="config_vrf", profile_id="zte.config_vrf"),
AuxCommand(key="ipv6_route", profile_id="zte.ipv6_route_vrf"),
],
enrich_joins=[
EnrichJoin(
from_aux="config_vrf",
left_on="vrf",
right_on="vrf_name",
take=("rd", "address_families"),
),
],
),
ParseProfile(
profile_id="zte.bgp_evpn_summary",
@ -1052,13 +1128,25 @@ def _zte_status_profiles() -> list[ParseProfile]:
command_template="show ip forwarding route vrf <vrf> | one-line",
match=r"(?i)^\s*show\s+ip\s+forwarding\s+route\s+vrf\s+(?P<vrf>\S+)(?:\s*\|\s*one-line)?\s*$",
textfsm_command="show ip forwarding route",
description="IPv4 FIB for one VRF.",
placeholders=[_BGP_VRF_PLACEHOLDER],
description=(
"IPv4 FIB per VRF. Bind VRFs or leave empty for all ipv4 VRFs "
"(config VRF intent aux)."
),
placeholders=[_VRF_PLACEHOLDER_IPV4],
fields=list(_IP_ROUTE_FIELDS),
tags=["route", "ipv4", "vrf"],
sort_order=450,
enabled=True,
kind="collect",
aux_commands=[AuxCommand(key="config_vrf", profile_id="zte.config_vrf")],
enrich_joins=[
EnrichJoin(
from_aux="config_vrf",
left_on="vrf",
right_on="vrf_name",
take=("rd", "address_families"),
),
],
),
ParseProfile(
profile_id="zte.ip_route",
@ -1085,13 +1173,25 @@ def _zte_status_profiles() -> list[ParseProfile]:
command_template="show ipv6 forwarding route vrf <vrf> | one-line",
match=r"(?i)^\s*show\s+ipv6\s+forwarding\s+route\s+vrf\s+(?P<vrf>\S+)(?:\s*\|\s*one-line)?\s*$",
textfsm_command="show ipv6 forwarding route",
description="IPv6 FIB for one VRF.",
placeholders=[_BGP_VRF_PLACEHOLDER],
description=(
"IPv6 FIB per VRF. Bind VRFs or leave empty for all ipv6 VRFs "
"(config VRF intent aux)."
),
placeholders=[_VRF_PLACEHOLDER_IPV6],
fields=list(_IPV6_ROUTE_FIELDS),
tags=["route", "ipv6", "vrf"],
sort_order=460,
enabled=True,
kind="collect",
aux_commands=[AuxCommand(key="config_vrf", profile_id="zte.config_vrf")],
enrich_joins=[
EnrichJoin(
from_aux="config_vrf",
left_on="vrf",
right_on="vrf_name",
take=("rd", "address_families"),
),
],
),
ParseProfile(
profile_id="zte.ipv6_route",
@ -1416,6 +1516,8 @@ def profile_to_public_dict(p: ParseProfile, *, overrides: dict[str, Any] | None
"discover_profile_id": ph.discover_profile_id,
"discover_value_field": ph.discover_value_field,
"discover_label_field": ph.discover_label_field,
"discover_filter_field": ph.discover_filter_field,
"discover_filter_contains": ph.discover_filter_contains,
}
for ph in p.placeholders
],

3
netx_api/biz_state/service.py
View file

@ -290,6 +290,9 @@ def _assert_bindings_ready(db: Session, task_id: str) -> None:
profile = get_profile(it.source_profile_id)
if not profile or not profile.placeholders:
continue
# Optional discover placeholders: empty bindings → expand all at collect.
if all(not ph.required for ph in profile.placeholders):
continue
binds = (
db.query(BizStateTaskItemBinding)
.filter(BizStateTaskItemBinding.item_id == it.id)