Fix ZTE BGP RR *i/RD parsing and add batch collect detail UX.

Decouple heavy CLI parse via ParsePool, keep ECMP/multi-RD keys, and expose per-command CLI lines vs DB rows with raw.txt export.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-23 10:52:43 +08:00
parent 4c5738e48e
commit 05357e971e
31 changed files with 1952 additions and 269 deletions

View file

@ -65,11 +65,115 @@ class BatchWorkbookApiTests(unittest.TestCase):
self.assertNotIn("metrics", out)
self.assertNotIn("lldp_neighbors", out)
self.assertEqual(out["commands"][0]["has_raw"], True)
self.assertEqual(out["commands"][0]["raw_line_count"], 1)
self.assertEqual(out["sheets"][0]["metric_id"], "arp")
self.assertEqual(out["sheets"][0]["row_count"], 2)
self.assertEqual(out["sheets"][0]["commands"][0]["raw_command"], "show arp | one-line")
self.assertEqual(out["sheets"][0]["commands"][0]["raw_line_count"], 1)
self.assertTrue(out["sheets"][0].get("title"))
def test_get_batch_raw_line_count_splitlines(self) -> None:
batch = BizStateBatch(
id="b1",
task_id="t1",
status="partial",
command_count=1,
row_count=0,
message="stopped",
)
cmd = BizStateBatchCommand(
id="c1",
batch_id="b1",
profile_id="zte.arp",
parser_id="arp",
metric_id="arp",
raw_command="show arp",
parse_status="ok",
row_count=3,
raw_text="a\nb\nc\n",
message="",
)
db = MagicMock()
db.get.side_effect = lambda model, pk: batch if pk == "b1" else None
cmd_q = MagicMock()
cmd_q.filter.return_value.order_by.return_value.all.return_value = [cmd]
metric_count_q = MagicMock()
metric_count_q.filter.return_value.group_by.return_value.all.return_value = [("arp", 3)]
lldp_count_q = MagicMock()
lldp_count_q.filter.return_value.scalar.return_value = 0
def query(*_args, **_kwargs):
n = query.n
query.n += 1
if n == 0:
return cmd_q
if n == 1:
return metric_count_q
return lldp_count_q
query.n = 0
db.query.side_effect = query
with patch(
"netx_api.biz_state.service.batch_protect_info",
return_value={"protected": False, "reasons": []},
):
out = get_batch(db, "b1")
self.assertEqual(out["message"], "stopped")
self.assertEqual(out["commands"][0]["raw_line_count"], 3)
self.assertEqual(out["commands"][0]["row_count"], 3)
def test_get_batch_command_and_raw_download(self) -> None:
from netx_api.biz_state.service import get_batch_command
from netx_api.biz_state_router import api_download_batch_command_raw
batch = BizStateBatch(id="b1", task_id="t1", status="ok")
cmd = BizStateBatchCommand(
id="c1",
batch_id="b1",
profile_id="zte.arp",
parser_id="arp",
metric_id="arp",
raw_command="show arp | one-line",
parse_status="ok",
row_count=2,
raw_text="line1\nline2",
message="hint",
)
db = MagicMock()
def _get(model, pk):
if model is BizStateBatch and pk == "b1":
return batch
if model is BizStateBatchCommand and pk == "c1":
return cmd
return None
db.get.side_effect = _get
detail = get_batch_command(db, "b1", "c1")
self.assertEqual(detail["raw_line_count"], 2)
self.assertEqual(detail["row_count"], 2)
self.assertEqual(detail["message"], "hint")
self.assertIn("line1", detail["raw_text"])
resp = api_download_batch_command_raw("b1", "c1", db)
self.assertEqual(resp.media_type, "text/plain; charset=utf-8")
cd = (resp.headers.get("content-disposition") or "").lower()
self.assertIn("attachment", cd)
self.assertIn(".txt", cd)
# StreamingResponse may expose async iterator; content already covered by get_batch_command.
body_iter = getattr(resp, "body_iterator", None)
if body_iter is not None and hasattr(body_iter, "__iter__") and not hasattr(body_iter, "__aiter__"):
body = b"".join(body_iter)
self.assertEqual(body.decode("utf-8"), "line1\nline2")
else:
# Fallback: reconstruct what the route encodes
from netx_api.biz_state.service import get_batch_command as _gbc
raw = str(_gbc(db, "b1", "c1").get("raw_text") or "")
self.assertEqual(raw, "line1\nline2")
def test_bgp_peer_sheet_uses_status_summary_title(self) -> None:
"""Shared metric_id bgp_peer must not inherit first AF profile title."""
batch = BizStateBatch(

View file

@ -5,6 +5,7 @@ from __future__ import annotations
import unittest
from netx_api.biz_migration.evaluate import (
KEY_SEP,
classify_status,
dual_verdict,
evaluate_metric_dual,
@ -90,7 +91,7 @@ class ParseExpectSetTests(unittest.TestCase):
]
}
)
self.assertEqual(got["bgp_peer"], {"AS1|1.1.1.1"})
self.assertEqual(got["bgp_peer"], {KEY_SEP.join(["AS1", "1.1.1.1"])})
self.assertEqual(got["arp"], {"10.0.0.1"})
def test_multi_select_each_key_is_separate(self):
@ -111,7 +112,7 @@ class ParseExpectSetTests(unittest.TestCase):
got = parse_expect_set(
{"items": [{"metric_id": "bgp_peer", "keys": ["1.1.1.1", "2.2.2.2"]}]}
)
self.assertEqual(got["bgp_peer"], {"1.1.1.1|2.2.2.2"})
self.assertEqual(got["bgp_peer"], {KEY_SEP.join(["1.1.1.1", "2.2.2.2"])})
def test_nested_keys_list(self):
got = parse_expect_set(
@ -122,8 +123,20 @@ class ParseExpectSetTests(unittest.TestCase):
]
}
)
self.assertEqual(got["isis_adjacency"], {"p1|gei-1|sys1", "p1|gei-2|sys2"})
self.assertEqual(got["arp"], {"10.0.0.1|vrf1"})
self.assertEqual(
got["isis_adjacency"],
{
KEY_SEP.join(["p1", "gei-1", "sys1"]),
KEY_SEP.join(["p1", "gei-2", "sys2"]),
},
)
self.assertEqual(got["arp"], {KEY_SEP.join(["10.0.0.1", "vrf1"])})
def test_legacy_pipe_key_normalized(self):
got = parse_expect_set(
{"items": [{"metric_id": "bgp_peer", "key": "AS1|1.1.1.1"}]}
)
self.assertEqual(got["bgp_peer"], {KEY_SEP.join(["AS1", "1.1.1.1"])})
def test_ports_only_apply_to_interface_brief(self):
expect = parse_expect_set({"ports": ["gei-1"]})
@ -855,12 +868,20 @@ class EvaluateMetricDualTests(unittest.TestCase):
def test_composite_key_port_remap(self):
mapped = _remap_key_str(
KEY_SEP.join(["gei-old", "vrf1"]),
key_fields=["interface", "vrf"],
iface_fields=["interface"],
port_map={"gei-old": "gei-new"},
)
self.assertEqual(mapped, KEY_SEP.join(["gei-new", "vrf1"]))
# Legacy pipe-separated input still remaps
mapped2 = _remap_key_str(
"gei-old|vrf1",
key_fields=["interface", "vrf"],
iface_fields=["interface"],
port_map={"gei-old": "gei-new"},
)
self.assertEqual(mapped, "gei-new|vrf1")
self.assertEqual(mapped2, KEY_SEP.join(["gei-new", "vrf1"]))
def test_anomaly_not_gated_by_field_tokens_on_removed(self):
self.assertEqual(

View file

@ -57,9 +57,8 @@ class BizStateCollectFinalizeTests(unittest.TestCase):
def test_finalize_partial_after_heavy_timeout(self) -> None:
with patch.object(runner, "SessionLocal", self.Session):
with patch(
"netx_api.biz_state.compare_service.try_auto_compare_for_task",
return_value=None,
):
"netx_api.biz_state.compare_service.schedule_auto_compare_for_task",
) as sched:
status = runner._finalize_batch_status(
batch_id="b-finalize",
task_id="t-finalize",
@ -70,6 +69,7 @@ class BizStateCollectFinalizeTests(unittest.TestCase):
lane_errors=["RuntimeError: biz_state_heavy_timeout (2400s)"],
)
self.assertEqual(status, "partial")
sched.assert_not_called()
self.db.expire_all()
batch = self.db.get(BizStateBatch, "b-finalize")
assert batch is not None
@ -127,8 +127,7 @@ class BizStateCollectFinalizeTests(unittest.TestCase):
with patch.object(runner, "SessionLocal", session_factory):
with patch(
"netx_api.biz_state.compare_service.try_auto_compare_for_task",
return_value=None,
"netx_api.biz_state.compare_service.schedule_auto_compare_for_task",
):
status = runner._finalize_batch_status(
batch_id="b-finalize",
@ -147,6 +146,23 @@ class BizStateCollectFinalizeTests(unittest.TestCase):
self.assertEqual(batch.status, "partial")
self.assertIn("biz_state_heavy_timeout", batch.message or "")
def test_finalize_success_schedules_auto_compare(self) -> None:
with patch.object(runner, "SessionLocal", self.Session):
with patch(
"netx_api.biz_state.compare_service.schedule_auto_compare_for_task",
) as sched:
status = runner._finalize_batch_status(
batch_id="b-finalize",
task_id="t-finalize",
cmd_count=55,
total_rows=6606,
any_fail=False,
any_ok=True,
lane_errors=[],
)
self.assertEqual(status, "success")
sched.assert_called_once_with("t-finalize", "b-finalize")
def test_fail_batch_retries_on_stale_connection(self) -> None:
calls = {"n": 0}
real_session = self.Session

View file

@ -391,6 +391,33 @@ class CompareSheetDefaultsTests(unittest.TestCase):
)
self.assertEqual(out["summary"]["duplicate_keys_before"], 1)
self.assertEqual(out["summary"]["duplicate_keys_after"], 1)
self.assertEqual(out["summary"]["duplicate"], 2)
self.assertIn("a|X|1", out["summary"]["duplicate_key_list"])
kinds = [d["kind"] for d in out["diffs"]]
self.assertEqual(kinds.count("duplicate"), 2)
# First before wins → matches first after → unchanged (same remote_ip)
self.assertEqual(out["summary"]["unchanged"], 1)
self.assertEqual(out["summary"]["changed"], 0)
def test_ignore_port_changes_false_keeps_iface(self) -> None:
before = [
{"local_if": "old-1", "remote_sys": "Peer", "remote_if": "p1", "remote_ip": "1.1.1.1"},
]
after = [
{"local_if": "new-1", "remote_sys": "Peer", "remote_if": "p1", "remote_ip": "1.1.1.1"},
]
out = compare_rows(
before_rows=before,
after_rows=after,
key_fields=["local_if", "remote_sys", "remote_if"],
iface_fields=["local_if"],
compare_fields=["remote_ip"],
port_map={},
ignore_port_changes=False,
)
self.assertEqual(out["summary"]["removed"], 1)
self.assertEqual(out["summary"]["added"], 1)
self.assertFalse(out["mapping_stats"].get("ignore_port_changes"))
def test_normalize_allows_duplicate_metric_with_distinct_sheet_id(self) -> None:
from netx_api.biz_state.compare_service import _normalize_sheet, sheet_key

View file

@ -0,0 +1,100 @@
"""Collect/parse decoupling: fetch_raw + parse pool job."""
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
from netx_api.biz_state.collect_session import CollectSession
from netx_api.biz_state.parse_pool import AuxRawCapture, PrimaryParseJob, parse_async_enabled
class FetchRawTests(unittest.TestCase):
def test_fetch_raw_skips_parser(self) -> None:
conn = object()
sends: list[str] = []
def _send(_c, command, read_timeout=0):
sends.append(command)
return f"RAW:{command}"
session = CollectSession(conn, send_fn=_send, read_timeout=10)
with patch("netx_api.biz_state.collect_session.run_parser") as rp:
entry, hit = session.fetch_raw("show ip route")
rp.assert_not_called()
self.assertFalse(hit)
self.assertTrue(entry.ok)
self.assertEqual(entry.raw, "RAW:show ip route")
self.assertEqual(entry.records, [])
# Second call hits cache without re-CLI.
entry2, hit2 = session.fetch_raw("show ip route")
self.assertTrue(hit2)
self.assertEqual(len(sends), 1)
self.assertEqual(entry2.raw, entry.raw)
class ParseJobTests(unittest.TestCase):
def test_run_primary_parse_job_persists(self) -> None:
from netx_api.biz_state import collect_runner as cr
with tempfile.TemporaryDirectory() as tmp:
with patch("netx_api.biz_state.spool.settings") as st:
st.biz_state_spool_dir = tmp
st.biz_state_persist_every_cmds = 8
st.biz_state_raw_max_bytes = 0
submitted: list = []
mock_pool = MagicMock()
mock_pool.submit.side_effect = lambda bid, items: submitted.append(
(bid, list(items))
)
with patch.object(cr, "run_primary_with_bundle") as rpb:
rpb.return_value = (
[{"prefix": "1.1.1.1/32"}],
{"rule_a": [{"PREFIX": "1.1.1.1/32"}]},
["rule_a"],
)
with patch(
"netx_api.biz_state.persist_pool.get_persist_pool",
return_value=mock_pool,
):
job = PrimaryParseJob(
batch_id="batch1",
cmd_id="cmd1",
task_item_id="item1",
profile_id="zte.ip_route",
parser_id="ip_route",
metric_id="ip_route",
concrete="show ip forwarding route",
merged_params={},
raw_text="DESTINATION\n1.1.1.1/32",
raw_rel_path="",
textfsm_command="show ip forwarding route",
vendor="ZTE",
device_type="zte_zxros",
enrich_joins=[],
aux_captures=[],
persisted=set(),
)
ok, fail = cr._run_primary_parse_job(job)
self.assertTrue(ok)
self.assertFalse(fail)
self.assertEqual(len(submitted), 1)
_bid, items = submitted[0]
self.assertEqual(_bid, "batch1")
self.assertEqual(len(items), 1)
self.assertEqual(items[0].parse_status, "ok")
self.assertEqual(items[0].row_count, 1)
rec_path = Path(tmp) / "batch1" / "cmd1.records.jsonl"
self.assertTrue(rec_path.is_file())
def test_parse_async_default_on(self) -> None:
self.assertTrue(parse_async_enabled())
if __name__ == "__main__":
unittest.main()

View file

@ -49,6 +49,19 @@ class PortMapSubifTests(unittest.TestCase):
self.assertEqual(resolve_mapped_iface("gei-0/0/0/1.100", pmap), "xgei-0/1/0/1.100")
self.assertEqual(resolve_mapped_iface("gei-0/0/0/2.100", pmap), "gei-0/0/0/2.100")
def test_qinq_multi_level_parent(self) -> None:
"""QinQ a.b.c: map parent a.b or a, keep remaining suffix."""
pmap = {"gei-0/1/0/1": "xgei-0/2/0/1"}
self.assertEqual(
resolve_mapped_iface("gei-0/1/0/1.100.200", pmap),
"xgei-0/2/0/1.100.200",
)
pmap2 = {"gei-0/1/0/1.100": "xgei-0/2/0/1.100"}
self.assertEqual(
resolve_mapped_iface("gei-0/1/0/1.100.200", pmap2),
"xgei-0/2/0/1.100.200",
)
def test_apply_port_map_row(self) -> None:
row = apply_port_map(
{"interface": "gei-0/0/0/1.55", "admin": "up"},

View file

@ -1134,6 +1134,69 @@ Routes Learned From This Neighbor:
self.assertEqual(v4_wrap[0]["next_hop"], "10.0.0.1")
self.assertEqual(v4_wrap[0]["afi"], "ipv4")
def test_bgp_route_rr_status_i_and_rd(self) -> None:
"""RR neighbor-in uses '* i prefix' and Route Distinguisher fill-down."""
from pathlib import Path
from netx_api.biz_state.profiles import metric_field_map
fixture = Path(__file__).resolve().parents[2] / "test" / "show-zte" / "show-bgp-vpnv4-neighbor-router-in"
if not fixture.is_file():
# Workspace layout: chatgpt/test/show-zte vs netx/tests
fixture = Path(__file__).resolve().parents[3] / "test" / "show-zte" / "show-bgp-vpnv4-neighbor-router-in"
raw = fixture.read_text(encoding="utf-8", errors="replace")
routes = normalize_bgp_route(
raw_text=raw,
command="show bgp vpnv4 unicast neighbor in 114.0.24.93 | one-line",
vendor="ZTE",
device_type="zte_zxros",
params={"neighbor": "114.0.24.93", "direction": "in", "afi": "vpnv4"},
)
self.assertEqual(len(routes), 1905, "must match Total number of routes")
self.assertTrue(all("/" in str(r.get("network") or "") for r in routes))
self.assertTrue(all(str(r.get("network") or "").lower() != "i" for r in routes))
self.assertTrue(all(str(r.get("status_codes") or "") == "*i" for r in routes))
self.assertTrue(all(str(r.get("rd") or "").strip() for r in routes))
# Known multi-RD prefix must survive under both RDs (not collapsed by uniqueness)
multi = [r for r in routes if r.get("network") == "100.127.58.68/30"]
self.assertEqual(len(multi), 2)
self.assertEqual(
{r["rd"] for r in multi},
{"114.0.141.200:65013", "114.14.249.211:65013"},
)
by_key = {(r["rd"], r["network"], r["next_hop"]) for r in routes}
self.assertEqual(len(by_key), len(routes))
# Profile uniqueness keys must include RD + next_hop for ECMP / multi-RD
keys = {f.name for f in metric_field_map().get("bgp_route", []) if f.is_key}
self.assertTrue({"rd", "network", "next_hop", "neighbor", "direction"} <= keys)
def test_bgp_route_ecmp_keeps_distinct_next_hops(self) -> None:
"""Same RD+prefix with two next-hops must both persist (load-share)."""
# Match real ZTE layout: RD only captured after entering Routes state
# (via "Routes Learned…" / Network header), same as production fixture.
raw = """
Routes Learned From This Neighbor:
Status codes: * valid, i - internal
Network Next Hop Metric LocPrf RtPrf Path
Route Distinguisher:10.0.0.1:100
* i 192.0.2.0/24 10.1.1.1 0 100 0 65001 i
* i 192.0.2.0/24 10.1.1.2 0 100 0 65001 i
Total number of routes: 2
"""
routes = normalize_bgp_route(
raw_text=raw,
command="show bgp vpnv4 unicast neighbor in 10.0.0.1 | one-line",
vendor="ZTE",
device_type="zte_zxros",
params={"neighbor": "10.0.0.1", "direction": "in", "afi": "vpnv4"},
)
self.assertEqual(len(routes), 2)
self.assertEqual({r["next_hop"] for r in routes}, {"10.1.1.1", "10.1.1.2"})
self.assertEqual({r["rd"] for r in routes}, {"10.0.0.1:100"})
self.assertEqual({r["network"] for r in routes}, {"192.0.2.0/24"})
self.assertTrue(all(r.get("status_codes") == "*i" for r in routes))
by_key = {(r["rd"], r["network"], r["next_hop"]) for r in routes}
self.assertEqual(len(by_key), 2)
if __name__ == "__main__":
unittest.main()