hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-09 04:20:45 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 2

Derive discover-backed aux automatically and fix BGP enrich join keys.

Browse source 
Overwrite explicit aux from discover_select at load time; join VRF CE on vrf,neighbor and global neighbors on afi,neighbor to avoid cross-AF collisions.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-22 14:17:12 +08:00
parent c14d4e81ff
commit f274d8ea72
5 changed files with 420 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

38
netx_api/biz_state/enrich.py
View file

@ -13,6 +13,15 @@ class EnrichJoin:
Example (ARP ← if_intf)::
EnrichJoin(from_aux="if_intf", on="interface", take=("vrf",))
Composite keys (VRF CE peer ← config_bgp_peer)::
EnrichJoin(
from_aux="config_bgp_peer",
left_on="vrf,neighbor",
right_on="vrf,neighbor",
take=("remote_as", "activate"),
)
"""
from_aux: str
@ -23,12 +32,29 @@ class EnrichJoin:
fill_missing: bool = True # on miss, set take fields to ""
def _sides(join: EnrichJoin) -> tuple[str, str]:
def _field_list(spec: str) -> list[str]:
return [p.strip() for p in str(spec or "").split(",") if p.strip()]
def _sides(join: EnrichJoin) -> tuple[list[str], list[str]]:
left = str(join.left_on or join.on or "").strip()
right = str(join.right_on or join.on or "").strip()
if not left or not right:
lf = _field_list(left)
rf = _field_list(right)
if not lf or not rf:
raise ValueError(f"EnrichJoin {join.from_aux!r} needs on= or left_on/right_on")
return left, right
if len(lf) != len(rf):
raise ValueError(
f"EnrichJoin {join.from_aux!r} left/right field count mismatch: {lf} vs {rf}"
)
return lf, rf
def _row_key(row: Mapping[str, Any], fields: list[str]) -> str:
parts = [str(row.get(f) or "").strip() for f in fields]
if not any(parts):
return ""
return "\0".join(parts)
def apply_enrich_joins(
@ -41,7 +67,7 @@ def apply_enrich_joins(
return records
aux_map = aux_records or {}
for join in joins:
left, right = _sides(join)
left_fields, right_fields = _sides(join)
take = [str(t).strip() for t in (join.take or ()) if str(t).strip()]
if not take:
continue
@ -49,13 +75,13 @@ def apply_enrich_joins(
for row in aux_map.get(str(join.from_aux or "").strip()) or []:
if not isinstance(row, dict):
continue
key = str(row.get(right) or "").strip()
key = _row_key(row, right_fields)
if key and key not in index:
index[key] = row
for rec in records:
if not isinstance(rec, dict):
continue
hit = index.get(str(rec.get(left) or "").strip())
hit = index.get(_row_key(rec, left_fields))
for field in take:
if hit is not None:
rec[field] = str(hit.get(field) or "").strip()

192
netx_api/biz_state/profiles.py
View file

@ -44,7 +44,13 @@ class PlaceholderDef:
@dataclass(frozen=True)
class AuxCommand:
"""Secondary collect: only key + profile_id (rest from that profile)."""
"""Secondary collect: only key + profile_id (rest from that profile).
When the primary profile uses ``discover_select`` placeholders, aux is
**framework-derived** from those ``discover_profile_id`` values at load
time (explicit ``aux_commands`` for that case are overwritten). Profiles
without discover (e.g. ARP → config_interface) keep explicit aux.
"""
key: str
profile_id: str
@ -820,9 +826,6 @@ def _zte_status_profiles() -> list[ParseProfile]:
sort_order=370,
enabled=True,
kind="collect",
aux_commands=[
AuxCommand(key="config_vrf", profile_id="zte.config_vrf"),
],
enrich_joins=[
EnrichJoin(
from_aux="config_vrf",
@ -851,9 +854,6 @@ def _zte_status_profiles() -> list[ParseProfile]:
sort_order=375,
enabled=True,
kind="collect",
aux_commands=[
AuxCommand(key="config_vrf", profile_id="zte.config_vrf"),
],
enrich_joins=[
EnrichJoin(
from_aux="config_vrf",
@ -970,19 +970,22 @@ def _zte_status_profiles() -> list[ParseProfile]:
command_template="show bgp vpnv4 unicast neighbor in <neighbor> | one-line",
match=r"(?i)^\s*show\s+bgp\s+vpnv4\s+unicast\s+neighbor\s+(?P<direction>in)\s+(?P<neighbor>\S+)(?:\s*\|\s*one-line)?\s*$",
textfsm_command="show bgp vpnv4 unicast neighbor in",
description="Routes learned from VPNv4 neighbor; summary aux for peer state.",
description=(
"Routes learned from VPNv4 neighbor; discover neighbor from BGP peer intent; "
"aux: show running-config bgp."
),
placeholders=[_BGP_NEIGHBOR_VPNV4],
fields=list(_BGP_ROUTE_FIELDS),
tags=["bgp", "vpnv4", "route"],
sort_order=420,
enabled=True,
kind="collect",
aux_commands=[AuxCommand(key="bgp_summary", profile_id="zte.bgp_vpnv4_summary")],
enrich_joins=[
EnrichJoin(
from_aux="bgp_summary",
on="neighbor",
take=("as_num", "state", "pfx_rcd"),
from_aux="config_bgp_peer",
left_on="afi,neighbor",
right_on="afi,neighbor",
take=("remote_as", "activate", "route_map_in", "route_map_out"),
),
],
),
@ -995,7 +998,10 @@ def _zte_status_profiles() -> list[ParseProfile]:
command_template="show bgp vpnv4 unicast neighbor out <neighbor> | one-line",
match=r"(?i)^\s*show\s+bgp\s+vpnv4\s+unicast\s+neighbor\s+(?P<direction>out)\s+(?P<neighbor>\S+)(?:\s*\|\s*one-line)?\s*$",
textfsm_command="show bgp vpnv4 unicast neighbor out",
description="Routes advertised to VPNv4 neighbor (large; bind neighbor).",
description=(
"Routes advertised to VPNv4 neighbor; discover from BGP peer intent; "
"aux: show running-config bgp."
),
placeholders=[_BGP_NEIGHBOR_VPNV4],
fields=list(_BGP_ROUTE_FIELDS),
tags=["bgp", "vpnv4", "route"],
@ -1003,12 +1009,12 @@ def _zte_status_profiles() -> list[ParseProfile]:
enabled=True,
kind="collect",
collect_lane="heavy",
aux_commands=[AuxCommand(key="bgp_summary", profile_id="zte.bgp_vpnv4_summary")],
enrich_joins=[
EnrichJoin(
from_aux="bgp_summary",
on="neighbor",
take=("as_num", "state", "pfx_rcd"),
from_aux="config_bgp_peer",
left_on="afi,neighbor",
right_on="afi,neighbor",
take=("remote_as", "activate", "route_map_in", "route_map_out"),
),
],
),
@ -1021,19 +1027,22 @@ def _zte_status_profiles() -> list[ParseProfile]:
command_template="show bgp vpnv6 unicast neighbor in <neighbor> | one-line",
match=r"(?i)^\s*show\s+bgp\s+vpnv6\s+unicast\s+neighbor\s+(?P<direction>in)\s+(?P<neighbor>\S+)(?:\s*\|\s*one-line)?\s*$",
textfsm_command="show bgp vpnv6 unicast neighbor in",
description="Routes learned from VPNv6 neighbor; summary aux for peer state.",
description=(
"Routes learned from VPNv6 neighbor; discover from BGP peer intent; "
"aux: show running-config bgp."
),
placeholders=[_BGP_NEIGHBOR_VPNV6],
fields=list(_BGP_ROUTE_FIELDS),
tags=["bgp", "vpnv6", "route"],
sort_order=426,
enabled=True,
kind="collect",
aux_commands=[AuxCommand(key="bgp_summary", profile_id="zte.bgp_vpnv6_summary")],
enrich_joins=[
EnrichJoin(
from_aux="bgp_summary",
on="neighbor",
take=("as_num", "state", "pfx_rcd"),
from_aux="config_bgp_peer",
left_on="afi,neighbor",
right_on="afi,neighbor",
take=("remote_as", "activate", "route_map_in", "route_map_out"),
),
],
),
@ -1046,7 +1055,10 @@ def _zte_status_profiles() -> list[ParseProfile]:
command_template="show bgp vpnv6 unicast neighbor out <neighbor> | one-line",
match=r"(?i)^\s*show\s+bgp\s+vpnv6\s+unicast\s+neighbor\s+(?P<direction>out)\s+(?P<neighbor>\S+)(?:\s*\|\s*one-line)?\s*$",
textfsm_command="show bgp vpnv6 unicast neighbor out",
description="Routes advertised to VPNv6 neighbor (large; bind neighbor).",
description=(
"Routes advertised to VPNv6 neighbor; discover from BGP peer intent; "
"aux: show running-config bgp."
),
placeholders=[_BGP_NEIGHBOR_VPNV6],
fields=list(_BGP_ROUTE_FIELDS),
tags=["bgp", "vpnv6", "route"],
@ -1054,12 +1066,12 @@ def _zte_status_profiles() -> list[ParseProfile]:
enabled=True,
kind="collect",
collect_lane="heavy",
aux_commands=[AuxCommand(key="bgp_summary", profile_id="zte.bgp_vpnv6_summary")],
enrich_joins=[
EnrichJoin(
from_aux="bgp_summary",
on="neighbor",
take=("as_num", "state", "pfx_rcd"),
from_aux="config_bgp_peer",
left_on="afi,neighbor",
right_on="afi,neighbor",
take=("remote_as", "activate", "route_map_in", "route_map_out"),
),
],
),
@ -1085,12 +1097,11 @@ def _zte_status_profiles() -> list[ParseProfile]:
sort_order=430,
enabled=True,
kind="collect",
aux_commands=[AuxCommand(key="config_bgp_peer", profile_id="zte.config_bgp_peer")],
enrich_joins=[
EnrichJoin(
from_aux="config_bgp_peer",
left_on="neighbor",
right_on="neighbor",
left_on="vrf,neighbor",
right_on="vrf,neighbor",
take=("remote_as", "activate", "route_map_in", "route_map_out"),
),
],
@ -1118,12 +1129,11 @@ def _zte_status_profiles() -> list[ParseProfile]:
enabled=True,
kind="collect",
collect_lane="heavy",
aux_commands=[AuxCommand(key="config_bgp_peer", profile_id="zte.config_bgp_peer")],
enrich_joins=[
EnrichJoin(
from_aux="config_bgp_peer",
left_on="neighbor",
right_on="neighbor",
left_on="vrf,neighbor",
right_on="vrf,neighbor",
take=("remote_as", "activate", "route_map_in", "route_map_out"),
),
],
@ -1150,12 +1160,11 @@ def _zte_status_profiles() -> list[ParseProfile]:
sort_order=440,
enabled=True,
kind="collect",
aux_commands=[AuxCommand(key="config_bgp_peer", profile_id="zte.config_bgp_peer")],
enrich_joins=[
EnrichJoin(
from_aux="config_bgp_peer",
left_on="neighbor",
right_on="neighbor",
left_on="vrf,neighbor",
right_on="vrf,neighbor",
take=("remote_as", "activate", "route_map_in", "route_map_out"),
),
],
@ -1183,12 +1192,11 @@ def _zte_status_profiles() -> list[ParseProfile]:
enabled=True,
kind="collect",
collect_lane="heavy",
aux_commands=[AuxCommand(key="config_bgp_peer", profile_id="zte.config_bgp_peer")],
enrich_joins=[
EnrichJoin(
from_aux="config_bgp_peer",
left_on="neighbor",
right_on="neighbor",
left_on="vrf,neighbor",
right_on="vrf,neighbor",
take=("remote_as", "activate", "route_map_in", "route_map_out"),
),
],
@ -1210,7 +1218,6 @@ def _zte_status_profiles() -> list[ParseProfile]:
sort_order=450,
enabled=True,
kind="collect",
aux_commands=[AuxCommand(key="config_vrf", profile_id="zte.config_vrf")],
enrich_joins=[
EnrichJoin(
from_aux="config_vrf",
@ -1252,7 +1259,6 @@ def _zte_status_profiles() -> list[ParseProfile]:
sort_order=460,
enabled=True,
kind="collect",
aux_commands=[AuxCommand(key="config_vrf", profile_id="zte.config_vrf")],
enrich_joins=[
EnrichJoin(
from_aux="config_vrf",
@ -1515,10 +1521,112 @@ def _apply_collect_lanes(profiles: list[ParseProfile]) -> list[ParseProfile]:
return profiles
_AUX_PLACEHOLDER_RE = re.compile(r"<([^>]+)>")
def _discover_profile_ids(p: ParseProfile) -> list[str]:
"""Ordered unique discover_profile_id from discover_select placeholders."""
out: list[str] = []
seen: set[str] = set()
for ph in p.placeholders or []:
if str(ph.bind_mode or "").strip() != "discover_select":
continue
dpi = str(ph.discover_profile_id or "").strip()
if not dpi or dpi in seen:
continue
seen.add(dpi)
out.append(dpi)
return out
def _aux_key_for_profile_id(profile_id: str) -> str:
pid = str(profile_id or "").strip()
return pid.rsplit(".", 1)[-1] if pid else "aux"
def _normalize_discover_aux(p: ParseProfile) -> None:
"""If profile discovers params, aux_commands MUST be those discover profiles.
Prevents drift (e.g. discover=config_bgp_peer but aux=bgp summary).
Explicit aux is still used when there is no discover_select (ARP pattern).
"""
disc = _discover_profile_ids(p)
if not disc:
return
p.aux_commands = [
AuxCommand(key=_aux_key_for_profile_id(dpi), profile_id=dpi) for dpi in disc
]
def _validate_profiles(profiles: list[ParseProfile]) -> None:
by_id = {p.profile_id: p for p in profiles}
errors: list[str] = []
for p in profiles:
if not p.enabled:
continue
primary_ph = {ph.name for ph in (p.placeholders or [])}
aux_keys = {a.key for a in (p.aux_commands or [])}
disc = _discover_profile_ids(p)
if disc:
aux_pids = [a.profile_id for a in (p.aux_commands or [])]
if aux_pids != disc:
errors.append(
f"{p.profile_id}: aux_commands {aux_pids} must equal "
f"discover_profile_ids {disc} (framework normalize failed)"
)
for a in p.aux_commands or []:
ap = by_id.get(a.profile_id)
if ap is None:
errors.append(
f"{p.profile_id}: aux {a.key!r} profile not found: {a.profile_id}"
)
continue
tmpl = str(ap.command_template or "")
extra = set(_AUX_PLACEHOLDER_RE.findall(tmpl)) - primary_ph
if extra:
errors.append(
f"{p.profile_id}: aux {a.key!r} ({a.profile_id}) has "
f"placeholders {sorted(extra)} not in primary {sorted(primary_ph)}"
)
for j in p.enrich_joins or []:
fk = str(j.from_aux or "").strip()
if fk and fk not in aux_keys:
errors.append(
f"{p.profile_id}: enrich_joins from_aux={fk!r} not in "
f"aux keys {sorted(aux_keys)}"
)
for ph in p.placeholders or []:
if str(ph.bind_mode or "").strip() != "discover_select":
continue
dpi = str(ph.discover_profile_id or "").strip()
if not dpi:
errors.append(
f"{p.profile_id}: placeholder {ph.name!r} discover_select "
f"missing discover_profile_id"
)
elif dpi not in by_id:
errors.append(
f"{p.profile_id}: placeholder {ph.name!r} discover_profile_id "
f"not found: {dpi}"
)
if errors:
raise RuntimeError(
"biz_state profile validation failed:\n- " + "\n- ".join(errors)
)
def _finalize_profiles(profiles: list[ParseProfile]) -> list[ParseProfile]:
"""Normalize discover→aux, validate, apply collect lanes."""
for p in profiles:
_normalize_discover_aux(p)
_validate_profiles(profiles)
return _apply_collect_lanes(profiles)
def all_profiles() -> list[ParseProfile]:
global _PROFILES
if _PROFILES is None:
_PROFILES = _apply_collect_lanes(
_PROFILES = _finalize_profiles(
_lldp_profiles() + _vrf_profiles() + _zte_status_profiles()
)
return list(_PROFILES)