From 285c19e5a3621b05f46b7d56d97273ac60c73358 Mon Sep 17 00:00:00 2001 From: oliver Date: Sat, 19 Sep 2026 21:11:31 +0800 Subject: [PATCH] Decouple cutover HF from portrait and tighten ops UI. Portrait stays on biz-monitor; cutover create/ensure owns purpose=cutover_hf slots with project collect override, multi-interval bindings, window pause, and industrial detail UX. Also expand ZTE collect/config parsers and TextFSM for route/BGP/L2VPN/VRRP coverage. Co-authored-by: Cursor --- netx_api/biz_migration/evaluate.py | 64 +- netx_api/biz_migration/monitor_templates.py | 110 +- netx_api/biz_migration/service.py | 1016 +++++++++-- netx_api/biz_migration_router.py | 69 +- netx_api/biz_state/collect_runner.py | 20 +- netx_api/biz_state/collect_session.py | 22 +- netx_api/biz_state/compare_engine.py | 98 +- netx_api/biz_state/compare_service.py | 185 +- netx_api/biz_state/iface_normalize.py | 137 ++ netx_api/biz_state/parsers/cisco/__init__.py | 4 +- netx_api/biz_state/parsers/common/vrf_list.py | 30 +- netx_api/biz_state/parsers/huawei/__init__.py | 9 +- netx_api/biz_state/parsers/zte/__init__.py | 51 + netx_api/biz_state/parsers/zte/bgp_peer.py | 239 ++- netx_api/biz_state/parsers/zte/bgp_route.py | 199 +++ .../biz_state/parsers/zte/config_bgp_peer.py | 216 +++ .../biz_state/parsers/zte/config_common.py | 46 + .../biz_state/parsers/zte/config_interface.py | 148 ++ netx_api/biz_state/parsers/zte/config_isis.py | 210 +++ .../biz_state/parsers/zte/config_l2vpn_pw.py | 140 ++ netx_api/biz_state/parsers/zte/config_ospf.py | 267 +++ .../parsers/zte/config_static_route.py | 283 +++ netx_api/biz_state/parsers/zte/config_vrf.py | 107 ++ netx_api/biz_state/parsers/zte/evpn_mac.py | 119 ++ .../biz_state/parsers/zte/interface_detail.py | 195 ++ netx_api/biz_state/parsers/zte/ip_route.py | 107 ++ netx_api/biz_state/parsers/zte/ipv6_route.py | 128 ++ netx_api/biz_state/parsers/zte/l2vpn_mac.py | 95 + netx_api/biz_state/parsers/zte/l2vpn_pw.py | 106 ++ .../biz_state/parsers/zte/optical_brief.py | 93 + .../biz_state/parsers/zte/ospf_neighbor.py | 108 ++ netx_api/biz_state/parsers/zte/vrrp.py | 122 ++ netx_api/biz_state/profiles.py | 895 +++++++++- netx_api/biz_state/schema_ensure.py | 14 + netx_api/biz_state/service.py | 31 +- netx_api/biz_state_router.py | 9 +- netx_api/biz_state_scheduler.py | 50 + netx_api/cli_templates/index | 10 + .../zte/zte_zxros_show_bgp_evpn_mac.textfsm | 23 + ...zte_zxros_show_bgp_neighbor_routes.textfsm | 29 + .../zte/zte_zxros_show_bgp_summary.textfsm | 22 + .../zte/zte_zxros_show_interface.textfsm | 17 +- ...zte_zxros_show_ip_forwarding_route.textfsm | 23 + .../zte/zte_zxros_show_ip_vrf.textfsm | 6 +- ...e_zxros_show_ipv6_forwarding_route.textfsm | 23 + .../zte_zxros_show_l2vpn_forwardinfo.textfsm | 23 + .../zte/zte_zxros_show_mac_l2vpn.textfsm | 19 + .../zte_zxros_show_opticalinfo_brief.textfsm | 18 + .../zte/zte_zxros_show_ospf_neighbor.textfsm | 23 + .../zte/zte_zxros_show_vrrp_brief.textfsm | 21 + netx_api/models/biz_migration.py | 22 +- netx_api/models/biz_state.py | 4 + tests/test_biz_migration_hf_decouple.py | 499 ++++++ tests/test_biz_state_compare.py | 58 +- tests/test_iface_normalize.py | 80 + tests/test_zte_config_intent.py | 417 +++++ tests/test_zte_config_real_dump.py | 248 +++ tests/test_zte_extended_parsers.py | 326 ++++ web/src/i18n/en.ts | 181 +- web/src/i18n/zh.ts | 176 +- web/src/index.css | 329 +++- web/src/pages/network/BizComparePage.tsx | 65 +- web/src/pages/network/BizMigrationPage.tsx | 1581 +++++++++++++---- .../pages/network/BizMonitorTemplatesPage.tsx | 149 +- web/src/pages/network/BizStatePage.tsx | 37 +- web/src/services/api.ts | 34 +- web/src/utils/time.ts | 28 + 67 files changed, 9510 insertions(+), 723 deletions(-) create mode 100644 netx_api/biz_state/iface_normalize.py create mode 100644 netx_api/biz_state/parsers/zte/bgp_route.py create mode 100644 netx_api/biz_state/parsers/zte/config_bgp_peer.py create mode 100644 netx_api/biz_state/parsers/zte/config_common.py create mode 100644 netx_api/biz_state/parsers/zte/config_interface.py create mode 100644 netx_api/biz_state/parsers/zte/config_isis.py create mode 100644 netx_api/biz_state/parsers/zte/config_l2vpn_pw.py create mode 100644 netx_api/biz_state/parsers/zte/config_ospf.py create mode 100644 netx_api/biz_state/parsers/zte/config_static_route.py create mode 100644 netx_api/biz_state/parsers/zte/config_vrf.py create mode 100644 netx_api/biz_state/parsers/zte/evpn_mac.py create mode 100644 netx_api/biz_state/parsers/zte/interface_detail.py create mode 100644 netx_api/biz_state/parsers/zte/ip_route.py create mode 100644 netx_api/biz_state/parsers/zte/ipv6_route.py create mode 100644 netx_api/biz_state/parsers/zte/l2vpn_mac.py create mode 100644 netx_api/biz_state/parsers/zte/l2vpn_pw.py create mode 100644 netx_api/biz_state/parsers/zte/optical_brief.py create mode 100644 netx_api/biz_state/parsers/zte/ospf_neighbor.py create mode 100644 netx_api/biz_state/parsers/zte/vrrp.py create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_bgp_evpn_mac.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_bgp_neighbor_routes.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_bgp_summary.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_ip_forwarding_route.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_ipv6_forwarding_route.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_l2vpn_forwardinfo.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_mac_l2vpn.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_opticalinfo_brief.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_ospf_neighbor.textfsm create mode 100644 netx_api/cli_templates/zte/zte_zxros_show_vrrp_brief.textfsm create mode 100644 tests/test_biz_migration_hf_decouple.py create mode 100644 tests/test_iface_normalize.py create mode 100644 tests/test_zte_config_intent.py create mode 100644 tests/test_zte_config_real_dump.py create mode 100644 tests/test_zte_extended_parsers.py diff --git a/netx_api/biz_migration/evaluate.py b/netx_api/biz_migration/evaluate.py index 9c21859..6423324 100644 --- a/netx_api/biz_migration/evaluate.py +++ b/netx_api/biz_migration/evaluate.py @@ -796,11 +796,21 @@ def _key_in_port_map( iface_fields: list[str], port_map: dict[str, str], ) -> bool: - """True when every interface segment of the key is an old-side map entry.""" + """True when every interface segment of the key is covered by the map. + + Covered = exact map key, or parent (before last ``.``) is a map key. + """ vals = _iface_values(key_str, key_fields=key_fields, iface_fields=iface_fields) if not vals: return False - return all(v in port_map for v in vals) + for v in vals: + if v in port_map: + continue + parent = v.rsplit(".", 1)[0] if "." in v else "" + if parent and parent in port_map: + continue + return False + return True def _port_identity(key_fields: list[str], iface_fields: list[str]) -> bool: @@ -832,24 +842,28 @@ def _remap_key_str( port_map: dict[str, str], ) -> str: """Map old-side key_str → new-side using iface segments (not whole-string only).""" + from ..biz_state.iface_normalize import resolve_mapped_iface + if not key_str: return key_str if not port_map: return key_str - if key_str in port_map and (not key_fields or len(key_fields) == 1): - return port_map[key_str] + if (not key_fields or len(key_fields) == 1) and ( + key_str in port_map or ("." in key_str and key_str.rsplit(".", 1)[0] in port_map) + ): + return resolve_mapped_iface(key_str, port_map) parts = str(key_str).split("|") if key_fields and len(parts) == len(key_fields): iface_set = {str(f) for f in iface_fields} out: list[str] = [] for i, f in enumerate(key_fields): v = parts[i] - if f in iface_set and v in port_map: - out.append(port_map[v]) + if f in iface_set: + out.append(resolve_mapped_iface(v, port_map)) else: out.append(v) return "|".join(out) - return port_map.get(key_str, key_str) + return resolve_mapped_iface(key_str, port_map) def _reverse_remap_key_str( @@ -860,24 +874,28 @@ def _reverse_remap_key_str( rev_map: dict[str, str], ) -> str: """Map new-side key_str → old-side canon identity.""" + from ..biz_state.iface_normalize import resolve_mapped_iface + if not key_str: return key_str if not rev_map: return key_str - if key_str in rev_map and (not key_fields or len(key_fields) == 1): - return rev_map[key_str] + if (not key_fields or len(key_fields) == 1) and ( + key_str in rev_map or ("." in key_str and key_str.rsplit(".", 1)[0] in rev_map) + ): + return resolve_mapped_iface(key_str, rev_map) parts = str(key_str).split("|") if key_fields and len(parts) == len(key_fields): iface_set = {str(f) for f in iface_fields} out: list[str] = [] for i, f in enumerate(key_fields): v = parts[i] - if f in iface_set and v in rev_map: - out.append(rev_map[v]) + if f in iface_set: + out.append(resolve_mapped_iface(v, rev_map)) else: out.append(v) return "|".join(out) - return rev_map.get(key_str, key_str) + return resolve_mapped_iface(key_str, rev_map) def _current_row(diff: dict[str, Any] | None) -> dict[str, Any]: @@ -938,8 +956,14 @@ def evaluate_metric_dual( sheet_override: dict[str, Any] | None = None, out_of_expect: str = "strict", sheet_id: str = "", + iface_normalize_rules: list[dict[str, str]] | None = None, ) -> dict[str, Any]: """Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge.""" + from ..biz_state.iface_normalize import ( + apply_iface_normalize_rows, + normalize_iface_rules, + ) + sid = str(sheet_id or "").strip() or str(metric_id or "").strip() expect_keys = expect_keys_for_metric( expect, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid @@ -962,6 +986,21 @@ def evaluate_metric_dual( success_patterns = list(ov.get("success") or []) if isinstance(ov.get("success"), list) else [] anomaly_patterns = list(ov.get("anomaly") or []) if isinstance(ov.get("anomaly"), list) else [] + norm_rules = normalize_iface_rules(iface_normalize_rules) + old_baseline_rows = apply_iface_normalize_rows( + old_baseline_rows, iface_fields=iface_fields, rules=norm_rules + ) + old_current_rows = apply_iface_normalize_rows( + old_current_rows, iface_fields=iface_fields, rules=norm_rules + ) + new_current_rows = apply_iface_normalize_rows( + new_current_rows, iface_fields=iface_fields, rules=norm_rules + ) + if new_baseline_rows is not None: + new_baseline_rows = apply_iface_normalize_rows( + new_baseline_rows, iface_fields=iface_fields, rules=norm_rules + ) + old_cmp = compare_rows( before_rows=old_baseline_rows, after_rows=old_current_rows, @@ -970,6 +1009,7 @@ def evaluate_metric_dual( compare_fields=compare_fields, port_map=None, field_rules=field_rules, + iface_normalize_rules=None, # already applied ) old_idx = build_diff_index_from_compare(old_cmp) diff --git a/netx_api/biz_migration/monitor_templates.py b/netx_api/biz_migration/monitor_templates.py index 740c511..ac0bce2 100644 --- a/netx_api/biz_migration/monitor_templates.py +++ b/netx_api/biz_migration/monitor_templates.py @@ -11,6 +11,7 @@ from sqlalchemy.orm import Session from ..models import BizCompareTemplate, BizMonitorTemplate from ..timeutil import utcnow_naive from ..biz_state import compare_service as cmp_svc +from ..biz_state.iface_normalize import default_zte_iface_normalize_rules from .evaluate import PORT_METRIC_ID, PORT_STATUS_FIELDS @@ -18,13 +19,27 @@ def _utcnow(): return utcnow_naive() -def _out(row: BizMonitorTemplate, compare_name: str = "") -> dict[str, Any]: +def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None = None) -> dict[str, Any]: + collect = [str(x).strip() for x in (row.collect_metric_ids_json or []) if str(x).strip()] + effective = list(collect) + if not effective and db is not None: + ct = db.get(BizCompareTemplate, row.compare_template_id) if row.compare_template_id else None + if ct: + seen: list[str] = [] + for s in cmp_svc.template_metrics(ct): + mid = str(s.get("metric_id") or "").strip() + if mid and mid not in seen: + seen.append(mid) + effective = seen + if not effective: + effective = [PORT_METRIC_ID] return { "id": row.id, "name": row.name, "compare_template_id": row.compare_template_id or "", "compare_template_name": compare_name, - "collect_metric_ids": list(row.collect_metric_ids_json or []), + "collect_metric_ids": collect, + "collect_metric_ids_effective": effective, "defaults": dict(row.defaults_json or {}), "sheet_overrides": list(row.sheet_overrides_json or []), "note": row.note or "", @@ -52,6 +67,11 @@ def ensure_port_compare_template(db: Session) -> BizCompareTemplate: "field_rules": [], } if row: + if not cmp_svc.template_iface_normalize(row): + cmp_svc._set_template_iface_normalize(row, default_zte_iface_normalize_rules()) # noqa: SLF001 + row.updated_at = _utcnow() + db.commit() + db.refresh(row) return row row = BizCompareTemplate( id=uuid4().hex, @@ -61,6 +81,7 @@ def ensure_port_compare_template(db: Session) -> BizCompareTemplate: updated_at=_utcnow(), ) cmp_svc._apply_sheets_to_row(row, [sheet]) # noqa: SLF001 — shared normalizer + cmp_svc._set_template_iface_normalize(row, default_zte_iface_normalize_rules()) # noqa: SLF001 db.add(row) db.commit() db.refresh(row) @@ -262,68 +283,61 @@ def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[st ) +DEFAULT_MONITOR_TEMPLATE_NAME = "默认割接监控" + + def ensure_default_monitor_templates(db: Session) -> None: - """Seed once when monitor-template table is empty.""" + """Seed once when monitor-template table is empty — one template on the full status compare.""" if db.query(BizMonitorTemplate.id).limit(1).first(): return cmp_svc.ensure_default_templates(db) - port_tpl = ensure_port_compare_template(db) zte = ( db.query(BizCompareTemplate) .filter(BizCompareTemplate.name == "ZTE status default") .one_or_none() ) - seeds = [ + if not zte: + return + zte_sheets = cmp_svc.template_metrics(zte) + zte_overrides = [ + preset_override_for_metric( + str(s.get("metric_id") or ""), + sheet_id=str(s.get("sheet_id") or s.get("metric_id") or ""), + ) + for s in zte_sheets + if s.get("metric_id") + ] + db.add( BizMonitorTemplate( id=uuid4().hex, - name="端口割接监控", - compare_template_id=port_tpl.id, - collect_metric_ids_json=[PORT_METRIC_ID], + name=DEFAULT_MONITOR_TEMPLATE_NAME, + compare_template_id=zte.id, + collect_metric_ids_json=[], defaults_json={"dual_mode": "migrate_pair", "out_of_expect": "strict"}, - sheet_overrides_json=[preset_override_for_metric(PORT_METRIC_ID)], - note="Default: port status dual-verdict with up/down semantics", + sheet_overrides_json=zte_overrides, + note="Default: full status dual-verdict (port/ARP/BGP/…)", created_at=_utcnow(), updated_at=_utcnow(), - ), - ] - if zte: - zte_sheets = cmp_svc.template_metrics(zte) - zte_overrides = [ - preset_override_for_metric( - str(s.get("metric_id") or ""), - sheet_id=str(s.get("sheet_id") or s.get("metric_id") or ""), - ) - for s in zte_sheets - if s.get("metric_id") - ] - seeds.append( - BizMonitorTemplate( - id=uuid4().hex, - name="ZTE 状态割接监控", - compare_template_id=zte.id, - collect_metric_ids_json=[], - defaults_json={"dual_mode": "migrate_pair", "out_of_expect": "strict"}, - sheet_overrides_json=zte_overrides, - note="ZTE multi-sheet: port/ARP/BGP/… dual presets", - created_at=_utcnow(), - updated_at=_utcnow(), - ) ) - for s in seeds: - db.add(s) + ) db.commit() def default_port_monitor_template_id(db: Session) -> str: - """Ensure seeds exist and return id of 「端口割接监控」 (or first template).""" + """Ensure seeds exist and return the default monitor template id.""" ensure_default_monitor_templates(db) - row = ( - db.query(BizMonitorTemplate) - .filter(BizMonitorTemplate.name == "端口割接监控") - .one_or_none() - ) - if row: - return row.id + for name in ( + DEFAULT_MONITOR_TEMPLATE_NAME, + "ZTE 状态割接监控", + "端口割接监控", + ): + row = ( + db.query(BizMonitorTemplate) + .filter(BizMonitorTemplate.name == name) + .one_or_none() + ) + if row: + return row.id first = db.query(BizMonitorTemplate).order_by(BizMonitorTemplate.created_at.asc()).first() return first.id if first else "" @@ -339,7 +353,7 @@ def list_monitor_templates(db: Session) -> list[dict[str, Any]]: ensure_default_monitor_templates(db) names = _compare_name_map(db) rows = db.query(BizMonitorTemplate).order_by(BizMonitorTemplate.name.asc()).all() - return [_out(r, names.get(r.compare_template_id or "", "")) for r in rows] + return [_out(r, names.get(r.compare_template_id or "", ""), db=db) for r in rows] def get_monitor_template(db: Session, template_id: str) -> dict[str, Any]: @@ -347,7 +361,7 @@ def get_monitor_template(db: Session, template_id: str) -> dict[str, Any]: if not row: raise HTTPException(status_code=404, detail="monitor_template_not_found") names = _compare_name_map(db) - return _out(row, names.get(row.compare_template_id or "", "")) + return _out(row, names.get(row.compare_template_id or "", ""), db=db) def create_monitor_template(db: Session, body: dict[str, Any]) -> dict[str, Any]: @@ -381,7 +395,7 @@ def create_monitor_template(db: Session, body: dict[str, Any]) -> dict[str, Any] db.commit() db.refresh(row) names = _compare_name_map(db) - return _out(row, names.get(row.compare_template_id or "", "")) + return _out(row, names.get(row.compare_template_id or "", ""), db=db) def update_monitor_template(db: Session, template_id: str, body: dict[str, Any]) -> dict[str, Any]: @@ -417,7 +431,7 @@ def update_monitor_template(db: Session, template_id: str, body: dict[str, Any]) db.commit() db.refresh(row) names = _compare_name_map(db) - return _out(row, names.get(row.compare_template_id or "", "")) + return _out(row, names.get(row.compare_template_id or "", ""), db=db) def delete_monitor_template(db: Session, template_id: str) -> None: diff --git a/netx_api/biz_migration/service.py b/netx_api/biz_migration/service.py index 952c852..2acce52 100644 --- a/netx_api/biz_migration/service.py +++ b/netx_api/biz_migration/service.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import datetime, timezone from typing import Any from uuid import uuid4 @@ -32,11 +33,296 @@ from .evaluate import ( port_sheet_def, ) +PURPOSE_CUTOVER_HF = "cutover_hf" + + +def _parse_dt(raw: Any) -> datetime | None: + """Parse ISO datetime to naive UTC for DB storage. + + Accepts ``Z`` / offset-aware ISO (preferred from clients) or naive ISO + (treated as already-UTC). Comparisons use :func:`utcnow_naive`. + """ + if raw is None or raw == "": + return None + if isinstance(raw, datetime): + if raw.tzinfo is not None: + return raw.astimezone(timezone.utc).replace(tzinfo=None) + return raw + s = str(raw).strip() + if not s: + return None + try: + if s.endswith("Z"): + s = s[:-1] + "+00:00" + dt = datetime.fromisoformat(s) + if dt.tzinfo is not None: + return dt.astimezone(timezone.utc).replace(tzinfo=None) + return dt + except ValueError: + return None + + +def _dt_iso(dt: datetime | None) -> str | None: + """Serialize naive-UTC datetime with explicit ``Z`` for global clients.""" + if not dt: + return None + return dt.replace(tzinfo=None).isoformat() + "Z" + + +def _task_looks_like_hf(task: BizStateTask | None) -> bool: + """Legacy recovery heuristic. Prefer purpose; note match only when purpose empty.""" + if not task: + return False + purpose = str(getattr(task, "purpose", None) or "").strip() + if purpose == PURPOSE_CUTOVER_HF: + return True + if purpose: + return False + note = str(task.note or "") + return note.startswith("割接高频") + + +def _find_portrait_sibling(db: Session, hf_task: BizStateTask) -> BizStateTask | None: + """Prefer a non-HF sibling task on the same NE (legacy recovery).""" + rows = ( + db.query(BizStateTask) + .filter( + BizStateTask.source == hf_task.source, + BizStateTask.ne_id == hf_task.ne_id, + BizStateTask.id != hf_task.id, + ) + .order_by(BizStateTask.created_at.asc()) + .all() + ) + for r in rows: + if not _task_looks_like_hf(r): + return r + return None + + +def _migrate_legacy_hf_slot(db: Session, proj: BizMigrationProject, side: str) -> None: + """If portrait slot holds an HF task and hf slot empty, move HF id; restore portrait sibling if found.""" + portrait_attr = f"{side}_task_id" + hf_attr = f"{side}_hf_task_id" + portrait_id = str(getattr(proj, portrait_attr, None) or "").strip() + hf_id = str(getattr(proj, hf_attr, None) or "").strip() + if hf_id or not portrait_id: + return + task = db.get(BizStateTask, portrait_id) + if not task or not _task_looks_like_hf(task): + return + setattr(proj, hf_attr, portrait_id) + sibling = _find_portrait_sibling(db, task) + if sibling: + setattr(proj, portrait_attr, sibling.id) + # else keep portrait_id pointing at HF until user rebinds — hf slot is correct + + +def migrate_project_hf_slots(db: Session, proj: BizMigrationProject, *, commit: bool = False) -> BizMigrationProject: + """Idempotent legacy repair: split HF out of portrait slots when needed.""" + before = (proj.old_task_id, proj.new_task_id, getattr(proj, "old_hf_task_id", None), getattr(proj, "new_hf_task_id", None)) + _migrate_legacy_hf_slot(db, proj, "old") + _migrate_legacy_hf_slot(db, proj, "new") + after = (proj.old_task_id, proj.new_task_id, getattr(proj, "old_hf_task_id", None), getattr(proj, "new_hf_task_id", None)) + if before != after: + proj.updated_at = utcnow_naive() + if commit: + db.commit() + db.refresh(proj) + return proj + + +def portrait_task_id(proj: BizMigrationProject, side: str) -> str: + return str(getattr(proj, f"{side}_task_id", None) or "").strip() + + +def hf_task_id(proj: BizMigrationProject, side: str) -> str: + return str(getattr(proj, f"{side}_hf_task_id", None) or "").strip() + + +def current_task_id(proj: BizMigrationProject, side: str) -> str: + """Evaluate current batches: prefer HF, fall back to portrait.""" + return hf_task_id(proj, side) or portrait_task_id(proj, side) + + +def _metric_interval_map(proj: BizMigrationProject) -> dict[str, int]: + raw = getattr(proj, "metric_interval_sec_json", None) + out: dict[str, int] = {} + if isinstance(raw, dict): + for k, v in raw.items(): + mid = str(k).strip() + if not mid: + continue + try: + out[mid] = max(60, int(v)) + except (TypeError, ValueError): + continue + return out + + +def _normalize_metric_intervals(raw: Any) -> dict[str, int]: + if not isinstance(raw, dict): + return {} + out: dict[str, int] = {} + for k, v in raw.items(): + mid = str(k).strip() + if not mid: + continue + try: + out[mid] = max(60, int(v)) + except (TypeError, ValueError): + continue + return out + + +def _group_metrics_by_interval( + metric_ids: list[str], + *, + default_interval: int, + by_metric: dict[str, int], +) -> list[tuple[int, list[str]]]: + """Group metrics by effective interval (ascending).""" + groups: dict[int, list[str]] = {} + default_iv = max(60, int(default_interval or 60)) + for mid in metric_ids: + m = str(mid).strip() + if not m: + continue + iv = max(60, int(by_metric.get(m) or default_iv)) + groups.setdefault(iv, []).append(m) + return sorted(((iv, mids) for iv, mids in groups.items()), key=lambda x: x[0]) + + +def _hf_bindings(proj: BizMigrationProject, side: str) -> list[dict[str, Any]]: + attr = f"{side}_hf_bindings_json" + raw = getattr(proj, attr, None) + if isinstance(raw, list) and raw: + out: list[dict[str, Any]] = [] + for row in raw: + if not isinstance(row, dict): + continue + tid = str(row.get("task_id") or "").strip() + if not tid: + continue + mids = [str(x).strip() for x in (row.get("metric_ids") or []) if str(x).strip()] + try: + iv = max(60, int(row.get("interval_sec") or 60)) + except (TypeError, ValueError): + iv = 60 + out.append({"task_id": tid, "metric_ids": mids, "interval_sec": iv}) + if out: + return out + # Legacy single slot + tid = hf_task_id(proj, side) + if tid: + return [ + { + "task_id": tid, + "metric_ids": _project_collect_override(proj), + "interval_sec": int(getattr(proj, "hf_interval_sec", None) or 60), + } + ] + return [] + + +def _all_hf_task_ids(proj: BizMigrationProject, side: str) -> list[str]: + seen: list[str] = [] + for b in _hf_bindings(proj, side): + tid = str(b.get("task_id") or "").strip() + if tid and tid not in seen: + seen.append(tid) + return seen + + +def _hf_task_id_for_metric(proj: BizMigrationProject, side: str, metric_id: str) -> str: + mid = str(metric_id or "").strip() + for b in _hf_bindings(proj, side): + mids = b.get("metric_ids") or [] + if mid and mid in mids: + return str(b.get("task_id") or "").strip() + if not mids: + # empty metric_ids = covers all for that binding + return str(b.get("task_id") or "").strip() + return hf_task_id(proj, side) or portrait_task_id(proj, side) + + +def _current_batch_for_metric( + db: Session, + proj: BizMigrationProject, + side: str, + metric_id: str, + *, + pinned_batch_id: str = "", +) -> BizStateBatch | None: + tid = _hf_task_id_for_metric(proj, side, metric_id) + pin = pinned_batch_id.strip() + if pin: + pinned = db.get(BizStateBatch, pin) + # Only honor pin when it belongs to this metric's HF task (multi-interval safe) + if pinned and (not tid or str(pinned.task_id or "") == tid): + return pinned + return _latest_success_batch(db, tid) if tid else None + + +def find_portrait_task_for_ne(db: Session, *, source: str, ne_id: str) -> BizStateTask | None: + """Best portrait (non-HF) biz_state task for an NE — for baseline binding.""" + src = str(source or "managed").strip().lower() or "managed" + nid = str(ne_id or "").strip() + if not nid: + return None + rows = ( + db.query(BizStateTask) + .filter(BizStateTask.source == src, BizStateTask.ne_id == nid) + .order_by(BizStateTask.created_at.asc()) + .all() + ) + for r in rows: + if not _task_looks_like_hf(r): + return r + return None + + +def list_ne_portrait_options(db: Session, *, source: str, ne_id: str, limit: int = 30) -> dict[str, Any]: + """Portrait task + recent batches for baseline picker when NE is selected.""" + task = find_portrait_task_for_ne(db, source=source, ne_id=ne_id) + if not task: + return { + "source": source, + "ne_id": ne_id, + "task": None, + "batches": [], + "hint": "no_portrait_task", + } + batches = ( + db.query(BizStateBatch) + .filter( + BizStateBatch.task_id == task.id, + BizStateBatch.status.in_(("success", "partial")), + ) + .order_by(BizStateBatch.started_at.desc()) + .limit(max(1, min(100, int(limit or 30)))) + .all() + ) + return { + "source": source, + "ne_id": ne_id, + "task": _task_brief(db, task.id), + "batches": [_batch_brief(db, b.id) for b in batches], + "hint": "", + } + + +def _project_collect_override(proj: BizMigrationProject) -> list[str]: + raw = getattr(proj, "collect_metric_ids_json", None) + if isinstance(raw, list): + return [str(x).strip() for x in raw if str(x).strip()] + return [] + def _task_brief(db: Session, task_id: str) -> dict[str, Any]: t = db.get(BizStateTask, task_id) if task_id else None if not t: - return {"id": task_id or "", "ne_name": "", "ne_ip": "", "vendor": ""} + return {"id": task_id or "", "ne_name": "", "ne_ip": "", "vendor": "", "purpose": ""} return { "id": t.id, "ne_name": t.ne_name, @@ -44,6 +330,7 @@ def _task_brief(db: Session, task_id: str) -> dict[str, Any]: "vendor": t.vendor, "status": t.status, "note": t.note, + "purpose": str(getattr(t, "purpose", None) or ""), "interval_sec": t.interval_sec, "collect_running": bool(t.collect_running), } @@ -64,21 +351,29 @@ def _batch_brief(db: Session, batch_id: str) -> dict[str, Any]: def _monitor_template_brief(db: Session, template_id: str) -> dict[str, Any]: tid = str(template_id or "").strip() if not tid: - return {"id": "", "name": "", "compare_template_id": "", "compare_template_name": ""} + return { + "id": "", + "name": "", + "compare_template_id": "", + "compare_template_name": "", + "collect_metric_ids": [], + "collect_metric_ids_effective": [], + } row = db.get(BizMonitorTemplate, tid) if not row: - return {"id": tid, "name": "", "compare_template_id": "", "compare_template_name": ""} + return { + "id": tid, + "name": "", + "compare_template_id": "", + "compare_template_name": "", + "collect_metric_ids": [], + "collect_metric_ids_effective": [], + } cmp_name = "" if row.compare_template_id: ct = db.get(BizCompareTemplate, row.compare_template_id) cmp_name = (ct.name if ct else "") or "" - return { - "id": row.id, - "name": row.name or "", - "compare_template_id": row.compare_template_id or "", - "compare_template_name": cmp_name, - "collect_metric_ids": list(row.collect_metric_ids_json or []), - } + return mon_tpl._out(row, cmp_name, db=db) # noqa: SLF001 def resolve_project_monitor_template(db: Session, proj: BizMigrationProject) -> BizMonitorTemplate: @@ -103,8 +398,10 @@ def resolve_project_monitor_template(db: Session, proj: BizMigrationProject) -> def resolve_evaluate_sheets( db: Session, mt: BizMonitorTemplate -) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]: - """Return (sheets, sheet_overrides, defaults) from monitor → compare template.""" +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any], list[dict[str, str]]]: + """Return (sheets, sheet_overrides, defaults, iface_normalize_rules).""" + from ..biz_state.compare_service import template_iface_normalize + overrides = list(mt.sheet_overrides_json or []) if isinstance(mt.sheet_overrides_json, list) else [] defaults = dict(mt.defaults_json or {}) if isinstance(mt.defaults_json, dict) else {} cid = str(mt.compare_template_id or "").strip() @@ -113,17 +410,21 @@ def resolve_evaluate_sheets( if ct: sheets = template_metrics(ct) if sheets: - return sheets, overrides, defaults + return sheets, overrides, defaults, template_iface_normalize(ct) # Fallback: built-in port sheet (legacy) - return [port_sheet_def()], overrides, defaults + return [port_sheet_def()], overrides, defaults, [] def resolve_collect_metric_ids(db: Session, proj: BizMigrationProject) -> list[str]: + """Project-level override wins; else monitor template; else sheet metric ids.""" + override = _project_collect_override(proj) + if override: + return override mt = resolve_project_monitor_template(db, proj) collect = [str(x).strip() for x in (mt.collect_metric_ids_json or []) if str(x).strip()] if collect: return collect - sheets, _, _ = resolve_evaluate_sheets(db, mt) + sheets, _, _, _ = resolve_evaluate_sheets(db, mt) seen: list[str] = [] for s in sheets: mid = str(s.get("metric_id") or "").strip() @@ -133,20 +434,37 @@ def resolve_collect_metric_ids(db: Session, proj: BizMigrationProject) -> list[s def project_to_dict(db: Session, p: BizMigrationProject) -> dict[str, Any]: + """Serialize project. Does not migrate slots (GET stays read-only).""" + collect_override = _project_collect_override(p) + mt_brief = _monitor_template_brief(db, getattr(p, "monitor_template_id", None) or "") + # Same resolution as ensure/evaluate (empty → compare sheets) + effective_collect = resolve_collect_metric_ids(db, p) return { "id": p.id, "name": p.name, "old_task_id": p.old_task_id, "new_task_id": p.new_task_id, + "old_hf_task_id": getattr(p, "old_hf_task_id", None) or "", + "new_hf_task_id": getattr(p, "new_hf_task_id", None) or "", + "old_hf_bindings": _hf_bindings(p, "old"), + "new_hf_bindings": _hf_bindings(p, "new"), "old_baseline_batch_id": p.old_baseline_batch_id, "new_baseline_batch_id": p.new_baseline_batch_id, "mapping_id": p.mapping_id, "monitor_template_id": getattr(p, "monitor_template_id", None) or "", - "monitor_template": _monitor_template_brief(db, getattr(p, "monitor_template_id", None) or ""), + "monitor_template": mt_brief, + "collect_metric_ids": collect_override, + "collect_metric_ids_effective": effective_collect, + "metric_interval_sec": _metric_interval_map(p), + "hf_interval_sec": int(getattr(p, "hf_interval_sec", None) or 60), + "hf_start_at": _dt_iso(getattr(p, "hf_start_at", None)), + "hf_end_at": _dt_iso(getattr(p, "hf_end_at", None)), "status": p.status, "note": p.note, "old_task": _task_brief(db, p.old_task_id), "new_task": _task_brief(db, p.new_task_id), + "old_hf_task": _task_brief(db, getattr(p, "old_hf_task_id", None) or ""), + "new_hf_task": _task_brief(db, getattr(p, "new_hf_task_id", None) or ""), "old_baseline": _batch_brief(db, p.old_baseline_batch_id), "new_baseline": _batch_brief(db, p.new_baseline_batch_id), "created_at": p.created_at.isoformat() if p.created_at else None, @@ -178,39 +496,129 @@ def list_projects(db: Session) -> list[dict[str, Any]]: def create_project(db: Session, body: dict[str, Any]) -> dict[str, Any]: + """One-shot cutover project: NE + monitor template + metrics + schedule + baseline + mapping. + + Auto-binds portrait tasks when found for the NEs (baselines come from them). + Always creates HF biz_state tasks from the monitor template collect set. + """ name = str(body.get("name") or "").strip() if not name: raise HTTPException(status_code=400, detail="name_required") - old_task_id = str(body.get("old_task_id") or "").strip() - new_task_id = str(body.get("new_task_id") or "").strip() - if not old_task_id or not new_task_id: - raise HTTPException(status_code=400, detail="old_new_task_required") - if not db.get(BizStateTask, old_task_id) or not db.get(BizStateTask, new_task_id): - raise HTTPException(status_code=404, detail="task_not_found") + + old_ne = body.get("old_ne") if isinstance(body.get("old_ne"), dict) else None + new_ne = body.get("new_ne") if isinstance(body.get("new_ne"), dict) else None + old_task_id = str(body.get("old_task_id") or body.get("old_portrait_task_id") or "").strip() + new_task_id = str(body.get("new_task_id") or body.get("new_portrait_task_id") or "").strip() + + # Auto-bind portrait by NE when not explicitly provided + if not old_task_id and old_ne and old_ne.get("ne_id"): + pt = find_portrait_task_for_ne( + db, + source=str(old_ne.get("source") or "managed"), + ne_id=str(old_ne.get("ne_id") or ""), + ) + if pt: + old_task_id = pt.id + if not new_task_id and new_ne and new_ne.get("ne_id"): + pt = find_portrait_task_for_ne( + db, + source=str(new_ne.get("source") or "managed"), + ne_id=str(new_ne.get("ne_id") or ""), + ) + if pt: + new_task_id = pt.id + + if old_task_id and not db.get(BizStateTask, old_task_id): + raise HTTPException(status_code=404, detail="old_task_not_found") + if new_task_id and not db.get(BizStateTask, new_task_id): + raise HTTPException(status_code=404, detail="new_task_not_found") + + has_old = bool(old_task_id or (old_ne and old_ne.get("ne_id"))) + has_new = bool(new_task_id or (new_ne and new_ne.get("ne_id"))) + if not has_old or not has_new: + raise HTTPException(status_code=400, detail="old_new_ne_required") + mapping_id = str(body.get("mapping_id") or "").strip() if mapping_id and not db.get(BizPortMapping, mapping_id): raise HTTPException(status_code=404, detail="mapping_not_found") + + # Inline mapping rows → create BizPortMapping + mapping_rows = body.get("mapping_rows") + mapping_name = str(body.get("mapping_name") or "").strip() + if not mapping_id and isinstance(mapping_rows, list) and mapping_rows: + from ..biz_state import compare_service as cmp_svc + + created_map = cmp_svc.create_mapping( + db, + { + "name": mapping_name or f"{name}-ports", + "rows": mapping_rows, + }, + ) + mapping_id = str(created_map.get("id") or "") + monitor_template_id = str(body.get("monitor_template_id") or "").strip() if monitor_template_id: if not db.get(BizMonitorTemplate, monitor_template_id): raise HTTPException(status_code=404, detail="monitor_template_not_found") else: monitor_template_id = mon_tpl.default_port_monitor_template_id(db) + if not monitor_template_id: + raise HTTPException(status_code=400, detail="monitor_template_required") + + collect_ids = body.get("collect_metric_ids") + if not isinstance(collect_ids, list): + collect_ids = [] + collect_ids = [str(x).strip() for x in collect_ids if str(x).strip()] + + metric_intervals = _normalize_metric_intervals(body.get("metric_interval_sec")) + hf_interval = max(60, int(body.get("hf_interval_sec") or body.get("interval_sec") or 60)) + hf_start = _parse_dt(body.get("hf_start_at")) + hf_end = _parse_dt(body.get("hf_end_at")) + + old_baseline = str(body.get("old_baseline_batch_id") or "").strip() + new_baseline = str(body.get("new_baseline_batch_id") or "").strip() + if old_baseline and not db.get(BizStateBatch, old_baseline): + raise HTTPException(status_code=404, detail="old_baseline_not_found") + if new_baseline and not db.get(BizStateBatch, new_baseline): + raise HTTPException(status_code=404, detail="new_baseline_not_found") + p = BizMigrationProject( id=uuid4().hex, name=name, old_task_id=old_task_id, new_task_id=new_task_id, - old_baseline_batch_id=str(body.get("old_baseline_batch_id") or "").strip(), - new_baseline_batch_id=str(body.get("new_baseline_batch_id") or "").strip(), + old_hf_task_id="", + new_hf_task_id="", + old_baseline_batch_id=old_baseline, + new_baseline_batch_id=new_baseline, mapping_id=mapping_id, monitor_template_id=monitor_template_id, - status=str(body.get("status") or "draft").strip() or "draft", + collect_metric_ids_json=collect_ids, + metric_interval_sec_json=metric_intervals, + old_hf_bindings_json=[], + new_hf_bindings_json=[], + hf_interval_sec=hf_interval, + hf_start_at=hf_start, + hf_end_at=hf_end, + status=str(body.get("status") or "active").strip() or "active", note=str(body.get("note") or "")[:500], ) db.add(p) db.commit() db.refresh(p) + + # Always create HF from NE / portrait — this is the cutover collect path + ensure_highfreq( + db, + p.id, + interval_sec=hf_interval, + retention_days=int(body.get("retention_days") or 7), + collect_now=bool(body.get("collect_now", False)), + old_ne=old_ne, + new_ne=new_ne, + ) + p = db.get(BizMigrationProject, p.id) or p return project_to_dict(db, p) @@ -251,16 +659,64 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st if mid and not db.get(BizMonitorTemplate, mid): raise HTTPException(status_code=404, detail="monitor_template_not_found") p.monitor_template_id = mid + if "old_task_id" in body and body["old_task_id"] is not None: + tid = str(body["old_task_id"] or "").strip() + if tid and not db.get(BizStateTask, tid): + raise HTTPException(status_code=404, detail="task_not_found") + p.old_task_id = tid + if "new_task_id" in body and body["new_task_id"] is not None: + tid = str(body["new_task_id"] or "").strip() + if tid and not db.get(BizStateTask, tid): + raise HTTPException(status_code=404, detail="task_not_found") + p.new_task_id = tid + if "old_hf_task_id" in body and body["old_hf_task_id"] is not None: + tid = str(body["old_hf_task_id"] or "").strip() + if tid and not db.get(BizStateTask, tid): + raise HTTPException(status_code=404, detail="task_not_found") + p.old_hf_task_id = tid + if "new_hf_task_id" in body and body["new_hf_task_id"] is not None: + tid = str(body["new_hf_task_id"] or "").strip() + if tid and not db.get(BizStateTask, tid): + raise HTTPException(status_code=404, detail="task_not_found") + p.new_hf_task_id = tid + if "collect_metric_ids" in body and body["collect_metric_ids"] is not None: + raw = body["collect_metric_ids"] + if not isinstance(raw, list): + raise HTTPException(status_code=400, detail="collect_metric_ids_must_be_list") + p.collect_metric_ids_json = [str(x).strip() for x in raw if str(x).strip()] + if "metric_interval_sec" in body and body["metric_interval_sec"] is not None: + p.metric_interval_sec_json = _normalize_metric_intervals(body["metric_interval_sec"]) + if "hf_interval_sec" in body and body["hf_interval_sec"] is not None: + p.hf_interval_sec = max(60, int(body["hf_interval_sec"] or 60)) + if "hf_start_at" in body: + p.hf_start_at = _parse_dt(body.get("hf_start_at")) + if "hf_end_at" in body: + p.hf_end_at = _parse_dt(body.get("hf_end_at")) p.updated_at = utcnow_naive() db.commit() db.refresh(p) + # Status / schedule changes should stop or resume HF without waiting for scheduler tick + if any(k in body for k in ("status", "hf_start_at", "hf_end_at")): + _apply_hf_window_to_tasks(db, p) + db.refresh(p) return project_to_dict(db, p) def delete_project(db: Session, project_id: str) -> dict[str, Any]: + from ..biz_state import service as biz_svc + p = db.get(BizMigrationProject, project_id) if not p: raise HTTPException(status_code=404, detail="project_not_found") + # Stop cutover HF collects before dropping the project + for tid in _all_hf_task_ids(p, "old") + _all_hf_task_ids(p, "new"): + t = db.get(BizStateTask, tid) + if t and t.status == "running": + try: + biz_svc.update_task(db, tid, {"status": "paused"}) + except Exception: # noqa: BLE001 + pass + db.query(BizMigrationRedTicket).filter(BizMigrationRedTicket.project_id == project_id).delete() batches = db.query(BizMigrationBatch).filter(BizMigrationBatch.project_id == project_id).all() for b in batches: runs = db.query(BizMigrationRun).filter(BizMigrationRun.batch_id == b.id).all() @@ -350,25 +806,35 @@ def run_evaluate( raise HTTPException(status_code=400, detail="new_baseline_required") old_batch = db.get(BizStateBatch, old_batch_id.strip()) if old_batch_id.strip() else None - if not old_batch: - old_batch = _latest_success_batch(db, proj.old_task_id) new_batch = db.get(BizStateBatch, new_batch_id.strip()) if new_batch_id.strip() else None + # Primary current batch ids (for run record / pin UX); per-metric may differ when multi-HF + if not old_batch: + for tid in _all_hf_task_ids(proj, "old") or [current_task_id(proj, "old")]: + old_batch = _latest_success_batch(db, tid) + if old_batch: + break if not new_batch: - new_batch = _latest_success_batch(db, proj.new_task_id) + for tid in _all_hf_task_ids(proj, "new") or [current_task_id(proj, "new")]: + new_batch = _latest_success_batch(db, tid) + if new_batch: + break if not old_batch: raise HTTPException(status_code=400, detail="old_current_batch_required") if not new_batch: raise HTTPException(status_code=400, detail="new_current_batch_required") old_cur = old_batch.id new_cur = new_batch.id + pin_old = old_batch_id.strip() + pin_new = new_batch_id.strip() port_map = _port_map_dict(db, proj.mapping_id) expect = parse_expect_set(mb.expect_set_json if isinstance(mb.expect_set_json, dict) else {}) # Final acceptance: window closed → unfinished expect = red window_active = (mb.status == "active") and (not acceptance) mt = resolve_project_monitor_template(db, proj) - sheets, sheet_overrides, defaults = resolve_evaluate_sheets(db, mt) + sheets, sheet_overrides, defaults, iface_norm = resolve_evaluate_sheets(db, mt) out_of_expect = str((defaults or {}).get("out_of_expect") or "strict").strip().lower() + collect_ids = set(resolve_collect_metric_ids(db, proj)) sheet_cards: list[dict[str, Any]] = [] all_rows: list[dict[str, Any]] = [] @@ -389,12 +855,57 @@ def run_evaluate( if sheet_ov.get("skip_dual"): continue + # Metric not in HF collect set → skip dual eval (avoid false red) + if collect_ids and mid not in collect_ids: + sheet_cards.append( + { + "metric_id": mid, + "sheet_id": sid, + "title": sheet_title(sheet), + "progress_ok": 0, + "progress_total": 0, + "anomaly": 0, + "anomaly_in_expect": 0, + "old_summary": {}, + "new_summary": {}, + "new_baseline_mode": "skipped", + "new_baseline_missing": False, + "collect_skipped": True, + } + ) + continue + + old_cur_batch = _current_batch_for_metric(db, proj, "old", mid, pinned_batch_id=pin_old) + new_cur_batch = _current_batch_for_metric(db, proj, "new", mid, pinned_batch_id=pin_new) + # Do NOT fall back to another HF task's batch — empty/wrong metric → false red + if not old_cur_batch or not new_cur_batch: + sheet_cards.append( + { + "metric_id": mid, + "sheet_id": sid, + "title": sheet_title(sheet), + "progress_ok": 0, + "progress_total": 0, + "anomaly": 0, + "anomaly_in_expect": 0, + "old_summary": {}, + "new_summary": {}, + "new_baseline_mode": "missing_current", + "new_baseline_missing": False, + "current_missing": True, + "collect_skipped": False, + } + ) + continue + old_cur_mid = old_cur_batch.id + new_cur_mid = new_cur_batch.id + old_base = apply_row_filters( _load_metric_rows(db, batch_id=proj.old_baseline_batch_id, metric_id=mid), row_filters, ) old_now = apply_row_filters( - _load_metric_rows(db, batch_id=old_cur, metric_id=mid), + _load_metric_rows(db, batch_id=old_cur_mid, metric_id=mid), row_filters, ) new_base_rows = apply_row_filters( @@ -402,7 +913,7 @@ def run_evaluate( row_filters, ) new_now = apply_row_filters( - _load_metric_rows(db, batch_id=new_cur, metric_id=mid), + _load_metric_rows(db, batch_id=new_cur_mid, metric_id=mid), row_filters, ) @@ -423,6 +934,7 @@ def run_evaluate( field_rules=field_rules, sheet_override=sheet_ov, out_of_expect=out_of_expect, + iface_normalize_rules=iface_norm, ) sheet_cards.append( { @@ -437,6 +949,7 @@ def run_evaluate( "new_summary": one["new_summary"], "new_baseline_mode": one.get("new_baseline_mode") or "provided", "new_baseline_missing": bool(one.get("new_baseline_missing")), + "collect_skipped": False, } ) for r in one["rows"]: @@ -448,6 +961,9 @@ def run_evaluate( if v: verdict_counts[v] = verdict_counts.get(v, 0) + 1 + active_cards = [ + c for c in sheet_cards if not c.get("collect_skipped") and not c.get("current_missing") + ] run = BizMigrationRun( id=uuid4().hex, project_id=proj.id, @@ -463,19 +979,31 @@ def run_evaluate( "acceptance": acceptance, "sheet_cards": sheet_cards, "progress": { - "ok": sum(c["progress_ok"] for c in sheet_cards), - "total": sum(c["progress_total"] for c in sheet_cards), + "ok": sum(c["progress_ok"] for c in active_cards), + "total": sum(c["progress_total"] for c in active_cards), }, - "anomaly": sum(c["anomaly"] for c in sheet_cards), - "new_baseline_missing": any(bool(c.get("new_baseline_missing")) for c in sheet_cards), + "anomaly": sum(c["anomaly"] for c in active_cards), + "new_baseline_missing": any(bool(c.get("new_baseline_missing")) for c in active_cards), "missing_metrics": [ str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "") - for c in sheet_cards + for c in active_cards if c.get("new_baseline_missing") ], + "current_missing_metrics": [ + str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "") + for c in sheet_cards + if c.get("current_missing") + ], + "collect_skipped_metrics": [ + str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "") + for c in sheet_cards + if c.get("collect_skipped") + ], "verdict_counts": verdict_counts, "window_active": window_active, "expect_ports": sorted(expect.get("_ports") or ()), + "old_current_task_id": current_task_id(proj, "old"), + "new_current_task_id": current_task_id(proj, "new"), }, message="", ) @@ -616,7 +1144,8 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] if not p.old_baseline_batch_id: return {"batch_id": "", "sheets": [], "mapped": {}} mt = resolve_project_monitor_template(db, p) - sheets, sheet_overrides, _ = resolve_evaluate_sheets(db, mt) + sheets, sheet_overrides, _defaults, _norm = resolve_evaluate_sheets(db, mt) + collect_ids = set(resolve_collect_metric_ids(db, p)) port_map = _port_map_dict(db, p.mapping_id) out_sheets: list[dict[str, Any]] = [] for sheet in sheets: @@ -625,6 +1154,9 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] key_fields = [str(k) for k in (sheet.get("key_fields") or []) if str(k).strip()] if not mid or not key_fields: continue + # Only offer expect keys for metrics that are actually HF-collected + if collect_ids and mid not in collect_ids: + continue sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid) if sheet_ov.get("skip_dual"): continue @@ -678,6 +1210,12 @@ def _catalog_item_for_metric(*, vendor: str, device_type: str, metric_id: str) - vkey = resolve_vendor_key(vendor, device_type) for p in profiles_for_vendor(vkey): if p.metric_id == mid and p.kind == "collect": + if getattr(p, "placeholders", None): + ph = ",".join(str(x.name) for x in (p.placeholders or []) if getattr(x, "name", None)) + raise HTTPException( + status_code=400, + detail=f"metric_needs_bindings:{mid}:{ph or 'required'}", + ) return { "source_profile_id": p.profile_id, "kind": "catalog", @@ -715,7 +1253,10 @@ def _enabled_metric_ids(db: Session, task_id: str) -> set[str]: def _is_highfreq_task(db: Session, task: BizStateTask, want_metrics: set[str]) -> bool: - """True when task metrics match want set and interval is short (cutover HF).""" + """True when task is cutover HF with matching metrics (or legacy short-interval match).""" + if str(getattr(task, "purpose", None) or "").strip() == PURPOSE_CUTOVER_HF: + metrics = _enabled_metric_ids(db, task.id) + return metrics == set(want_metrics) metrics = _enabled_metric_ids(db, task.id) return metrics == set(want_metrics) and int(task.interval_sec or 0) <= 300 @@ -725,33 +1266,126 @@ def _is_port_highfreq_task(db: Session, task: BizStateTask) -> bool: return _is_highfreq_task(db, task, {PORT_METRIC_ID}) +def _ne_spec_to_template_fields(db: Session, ne: dict[str, Any]) -> dict[str, Any]: + """Resolve NE dict into fields suitable for create_task.""" + from ..biz_state import service as biz_svc + + source = str(ne.get("source") or "managed").strip().lower() or "managed" + ne_id = str(ne.get("ne_id") or "").strip() + if not ne_id: + raise HTTPException(status_code=400, detail="ne_id_required") + meta = biz_svc._ne_meta(db, source=source, ne_id=ne_id) # noqa: SLF001 + return { + "source": source, + "ne_id": ne_id, + "ne_name": str(ne.get("ne_name") or meta.get("ne_name") or ""), + "ne_ip": str(ne.get("ne_ip") or meta.get("ne_ip") or ""), + "vendor": str(ne.get("vendor") or meta.get("vendor") or ""), + "device_type": str(ne.get("device_type") or meta.get("device_type") or ""), + } + + +def _template_from_task_or_ne( + db: Session, + *, + portrait: BizStateTask | None, + existing_hf: BizStateTask | None, + ne: dict[str, Any] | None, +) -> dict[str, Any]: + """Build NE identity for HF create: prefer portrait, else existing HF, else NE spec.""" + if portrait and not _task_looks_like_hf(portrait): + return { + "source": portrait.source, + "ne_id": portrait.ne_id, + "ne_name": portrait.ne_name, + "ne_ip": portrait.ne_ip, + "vendor": portrait.vendor, + "device_type": portrait.device_type, + } + if existing_hf: + return { + "source": existing_hf.source, + "ne_id": existing_hf.ne_id, + "ne_name": existing_hf.ne_name, + "ne_ip": existing_hf.ne_ip, + "vendor": existing_hf.vendor, + "device_type": existing_hf.device_type, + } + if portrait: + return { + "source": portrait.source, + "ne_id": portrait.ne_id, + "ne_name": portrait.ne_name, + "ne_ip": portrait.ne_ip, + "vendor": portrait.vendor, + "device_type": portrait.device_type, + } + if ne: + return _ne_spec_to_template_fields(db, ne) + raise HTTPException(status_code=400, detail="old_new_ne_or_task_required") + + def _ensure_side_highfreq( db: Session, *, - template: BizStateTask, + existing_hf: BizStateTask | None, + template_fields: dict[str, Any], project_name: str, interval_sec: int, retention_days: int, metric_ids: list[str], + status: str = "running", ) -> tuple[BizStateTask, bool]: - """Return (task, created). Reuse if already HF for metric set; else create sibling.""" + """Return (hf_task, created). Reuse existing HF if metrics match; never mutate portrait.""" from ..biz_state import service as biz_svc want = {str(m).strip() for m in metric_ids if str(m).strip()} if not want: want = {PORT_METRIC_ID} - if _is_highfreq_task(db, template, want): - if template.status != "running": - biz_svc.update_task(db, template.id, {"status": "running"}) - refreshed = db.get(BizStateTask, template.id) - return refreshed or template, False - return template, False + if existing_hf and _is_highfreq_task(db, existing_hf, want): + patch: dict[str, Any] = { + "interval_sec": interval_sec, + "retention_days": retention_days, + "purpose": PURPOSE_CUTOVER_HF, + } + if status: + patch["status"] = status + biz_svc.update_task(db, existing_hf.id, patch) + refreshed = db.get(BizStateTask, existing_hf.id) + return refreshed or existing_hf, False + + # Existing HF (purpose already cutover_hf) with wrong metrics → update items in place. + # Never rewrite a task that only "looks like" HF by note — create a sibling instead. + if existing_hf and str(getattr(existing_hf, "purpose", None) or "").strip() == PURPOSE_CUTOVER_HF: + items = [ + _catalog_item_for_metric( + vendor=str(template_fields.get("vendor") or existing_hf.vendor), + device_type=str(template_fields.get("device_type") or existing_hf.device_type), + metric_id=mid, + ) + for mid in sorted(want) + ] + note = f"割接高频/{'+'.join(sorted(want)[:3])}/{project_name}"[:256] + biz_svc.update_task( + db, + existing_hf.id, + { + "note": note, + "purpose": PURPOSE_CUTOVER_HF, + "status": status or "running", + "interval_sec": interval_sec, + "retention_days": retention_days, + "items": items, + }, + ) + refreshed = db.get(BizStateTask, existing_hf.id) + return refreshed or existing_hf, False items = [ _catalog_item_for_metric( - vendor=template.vendor, - device_type=template.device_type, + vendor=str(template_fields.get("vendor") or ""), + device_type=str(template_fields.get("device_type") or ""), metric_id=mid, ) for mid in sorted(want) @@ -760,14 +1394,15 @@ def _ensure_side_highfreq( created = biz_svc.create_task( db, { - "source": template.source, - "ne_id": template.ne_id, - "ne_name": template.ne_name, - "ne_ip": template.ne_ip, - "vendor": template.vendor, - "device_type": template.device_type, + "source": template_fields["source"], + "ne_id": template_fields["ne_id"], + "ne_name": template_fields.get("ne_name") or "", + "ne_ip": template_fields.get("ne_ip") or "", + "vendor": template_fields.get("vendor") or "", + "device_type": template_fields.get("device_type") or "", "note": note, - "status": "running", + "purpose": PURPOSE_CUTOVER_HF, + "status": status or "running", "interval_sec": interval_sec, "retention_days": retention_days, "items": items, @@ -779,6 +1414,52 @@ def _ensure_side_highfreq( return task, True +def _hf_window_status(proj: BizMigrationProject) -> str: + """Return desired HF task status based on project window: running|paused.""" + status = str(getattr(proj, "status", None) or "").strip().lower() + if status in ("done", "completed", "archived", "cancelled"): + return "paused" + now = utcnow_naive() + start = getattr(proj, "hf_start_at", None) + end = getattr(proj, "hf_end_at", None) + if end and now > end: + return "paused" + if start and now < start: + return "paused" + return "running" + + +def _apply_hf_window_to_tasks(db: Session, proj: BizMigrationProject) -> None: + """Immediately pause/resume bound cutover HF tasks to match project window/status.""" + import logging + + from ..biz_state import service as biz_svc + + log = logging.getLogger("netx.biz_migration.hf_window") + want = _hf_window_status(proj) + for tid in _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new"): + task = db.get(BizStateTask, tid) + if not task: + continue + purpose = str(getattr(task, "purpose", None) or "").strip() + if purpose and purpose != PURPOSE_CUTOVER_HF: + continue + patch: dict[str, Any] = {} + # Legacy empty-purpose HF: stamp purpose so resume skips bindings assert + if purpose != PURPOSE_CUTOVER_HF: + patch["purpose"] = PURPOSE_CUTOVER_HF + if want == "paused" and task.status == "running": + patch["status"] = "paused" + elif want == "running" and task.status == "paused": + patch["status"] = "running" + if not patch: + continue + try: + biz_svc.update_task(db, tid, patch) + except Exception: # noqa: BLE001 + log.exception("hf window sync failed project=%s task=%s want=%s", proj.id, tid, want) + + def ensure_port_highfreq( db: Session, project_id: str, @@ -801,97 +1482,210 @@ def ensure_highfreq( db: Session, project_id: str, *, - interval_sec: int = 60, + interval_sec: int | None = None, retention_days: int = 7, collect_now: bool = True, + old_ne: dict[str, Any] | None = None, + new_ne: dict[str, Any] | None = None, ) -> dict[str, Any]: - """Create/bind HF collect tasks from project's monitor template collect_metric_ids.""" + """Create/bind HF tasks (one per interval group) — never overwrites portrait ids.""" from ..biz_state.collect_runner import dispatch_collect proj = db.get(BizMigrationProject, project_id) if not proj: raise HTTPException(status_code=404, detail="project_not_found") - old_tpl = db.get(BizStateTask, proj.old_task_id) if proj.old_task_id else None - new_tpl = db.get(BizStateTask, proj.new_task_id) if proj.new_task_id else None - if not old_tpl or not new_tpl: - raise HTTPException(status_code=400, detail="old_new_task_required") + migrate_project_hf_slots(db, proj, commit=False) - metric_ids = resolve_collect_metric_ids(db, proj) - iv = max(60, int(interval_sec or 60)) + if interval_sec is not None: + proj.hf_interval_sec = max(60, int(interval_sec)) + iv_default = max(60, int(getattr(proj, "hf_interval_sec", None) or interval_sec or 60)) ret = max(1, int(retention_days or 7)) - old_task, old_created = _ensure_side_highfreq( + metric_ids = resolve_collect_metric_ids(db, proj) + if not metric_ids: + metric_ids = [PORT_METRIC_ID] + by_metric = _metric_interval_map(proj) + groups = _group_metrics_by_interval(metric_ids, default_interval=iv_default, by_metric=by_metric) + want_status = _hf_window_status(proj) + + old_portrait = db.get(BizStateTask, proj.old_task_id) if proj.old_task_id else None + new_portrait = db.get(BizStateTask, proj.new_task_id) if proj.new_task_id else None + + # Reuse existing bindings by interval when possible + def _existing_by_interval(side: str) -> dict[int, BizStateTask]: + out: dict[int, BizStateTask] = {} + for b in _hf_bindings(proj, side): + tid = str(b.get("task_id") or "").strip() + t = db.get(BizStateTask, tid) if tid else None + if not t: + continue + try: + iv = max(60, int(b.get("interval_sec") or t.interval_sec or 60)) + except (TypeError, ValueError): + iv = 60 + out[iv] = t + # legacy primary slot + primary = db.get(BizStateTask, hf_task_id(proj, side)) if hf_task_id(proj, side) else None + if primary and iv_default not in out: + out[iv_default] = primary + return out + + old_existing = _existing_by_interval("old") + new_existing = _existing_by_interval("new") + old_prev_ids = set(_all_hf_task_ids(proj, "old")) + new_prev_ids = set(_all_hf_task_ids(proj, "new")) + + old_fields = _template_from_task_or_ne( db, - template=old_tpl, - project_name=proj.name, - interval_sec=iv, - retention_days=ret, - metric_ids=metric_ids, + portrait=old_portrait, + existing_hf=next(iter(old_existing.values()), None), + ne=old_ne, ) - new_tpl = db.get(BizStateTask, proj.new_task_id) - if not new_tpl: - raise HTTPException(status_code=400, detail="new_task_required") - new_task, new_created = _ensure_side_highfreq( + new_fields = _template_from_task_or_ne( db, - template=new_tpl, - project_name=proj.name, - interval_sec=iv, - retention_days=ret, - metric_ids=metric_ids, + portrait=new_portrait, + existing_hf=next(iter(new_existing.values()), None), + ne=new_ne, ) + old_bindings: list[dict[str, Any]] = [] + new_bindings: list[dict[str, Any]] = [] + old_any_created = False + new_any_created = False + + for g_iv, g_metrics in groups: + old_task, old_created = _ensure_side_highfreq( + db, + existing_hf=old_existing.get(g_iv), + template_fields=old_fields, + project_name=proj.name, + interval_sec=g_iv, + retention_days=ret, + metric_ids=g_metrics, + status=want_status, + ) + new_task, new_created = _ensure_side_highfreq( + db, + existing_hf=new_existing.get(g_iv), + template_fields=new_fields, + project_name=proj.name, + interval_sec=g_iv, + retention_days=ret, + metric_ids=g_metrics, + status=want_status, + ) + old_any_created = old_any_created or old_created + new_any_created = new_any_created or new_created + old_bindings.append( + {"task_id": old_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv} + ) + new_bindings.append( + {"task_id": new_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv} + ) + proj = db.get(BizMigrationProject, project_id) if not proj: raise HTTPException(status_code=404, detail="project_not_found") - proj.old_task_id = old_task.id - proj.new_task_id = new_task.id + proj.old_hf_bindings_json = old_bindings + proj.new_hf_bindings_json = new_bindings + proj.old_hf_task_id = old_bindings[0]["task_id"] if old_bindings else "" + proj.new_hf_task_id = new_bindings[0]["task_id"] if new_bindings else "" + proj.hf_interval_sec = iv_default proj.updated_at = utcnow_naive() db.commit() - collect: dict[str, Any] = {"old": None, "new": None} - if collect_now: - for side, tid in (("old", old_task.id), ("new", new_task.id)): + # Pause HF tasks dropped from bindings (interval regroup / metric shrink) + from ..biz_state import service as biz_svc + + keep_ids = {b["task_id"] for b in old_bindings} | {b["task_id"] for b in new_bindings} + for tid in (old_prev_ids | new_prev_ids) - keep_ids: + t = db.get(BizStateTask, tid) + if not t: + continue + if str(getattr(t, "purpose", None) or "").strip() not in ("", PURPOSE_CUTOVER_HF): + continue + if t.status == "running": try: - dispatch_collect(tid) - collect[side] = {"ok": True, "task_id": tid} - except Exception as exc: # noqa: BLE001 - collect[side] = {"ok": False, "task_id": tid, "error": str(exc)[:200]} + biz_svc.update_task(db, tid, {"status": "paused"}) + except Exception: # noqa: BLE001 + pass + + collect: dict[str, Any] = {"old": [], "new": []} + if collect_now and want_status == "running": + for side, binds in (("old", old_bindings), ("new", new_bindings)): + for b in binds: + tid = b["task_id"] + try: + dispatch_collect(tid) + collect[side].append({"ok": True, "task_id": tid}) + except Exception as exc: # noqa: BLE001 + collect[side].append({"ok": False, "task_id": tid, "error": str(exc)[:200]}) + elif collect_now and want_status != "running": + collect = { + "old": [{"ok": False, "error": "hf_window_inactive"}], + "new": [{"ok": False, "error": "hf_window_inactive"}], + } return { "project": project_to_dict(db, proj), - "old_task": _task_brief(db, old_task.id), - "new_task": _task_brief(db, new_task.id), - "old_created": old_created, - "new_created": new_created, - "interval_sec": iv, + "old_task": _task_brief(db, proj.old_hf_task_id), + "new_task": _task_brief(db, proj.new_hf_task_id), + "old_hf_task": _task_brief(db, proj.old_hf_task_id), + "new_hf_task": _task_brief(db, proj.new_hf_task_id), + "old_hf_bindings": old_bindings, + "new_hf_bindings": new_bindings, + "old_created": old_any_created, + "new_created": new_any_created, + "interval_sec": iv_default, + "hf_status": want_status, "collect_metric_ids": metric_ids, "collect": collect, } def collect_project_now(db: Session, project_id: str) -> dict[str, Any]: - """Trigger immediate collect on project's old/new biz_state tasks.""" + """Trigger immediate collect on all HF binding tasks (never portrait).""" from ..biz_state.collect_runner import dispatch_collect proj = db.get(BizMigrationProject, project_id) if not proj: raise HTTPException(status_code=404, detail="project_not_found") - out: dict[str, Any] = {"old": None, "new": None} - for side, tid in (("old", proj.old_task_id), ("new", proj.new_task_id)): - if not tid: - out[side] = {"ok": False, "error": "task_missing"} + migrate_project_hf_slots(db, proj, commit=True) + want_status = _hf_window_status(proj) + all_ids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new") + if want_status != "running": + from ..biz_state import service as biz_svc + + for tid in all_ids: + t = db.get(BizStateTask, tid) + if t and t.status == "running": + biz_svc.update_task(db, tid, {"status": "paused"}) + return { + "old": [{"ok": False, "error": "hf_window_inactive"}], + "new": [{"ok": False, "error": "hf_window_inactive"}], + "hf_status": want_status, + } + + out: dict[str, Any] = {"old": [], "new": [], "hf_status": want_status} + for side in ("old", "new"): + tids = _all_hf_task_ids(proj, side) + if not tids: + out[side] = [{"ok": False, "error": "hf_task_missing"}] continue - task = db.get(BizStateTask, tid) - if not task: - out[side] = {"ok": False, "error": "task_not_found"} - continue - if bool(task.collect_running): - out[side] = {"ok": False, "error": "collect_already_running", "task_id": tid} - continue - try: - dispatch_collect(tid) - out[side] = {"ok": True, "task_id": tid} - except Exception as exc: # noqa: BLE001 - out[side] = {"ok": False, "task_id": tid, "error": str(exc)[:200]} + for tid in tids: + task = db.get(BizStateTask, tid) + if not task: + out[side].append({"ok": False, "error": "task_not_found", "task_id": tid}) + continue + if bool(task.collect_running): + out[side].append( + {"ok": False, "error": "collect_already_running", "task_id": tid} + ) + continue + try: + dispatch_collect(tid) + out[side].append({"ok": True, "task_id": tid}) + except Exception as exc: # noqa: BLE001 + out[side].append({"ok": False, "task_id": tid, "error": str(exc)[:200]}) return out @@ -1037,11 +1831,14 @@ def finish_batch( ok = int(progress.get("ok") or 0) total = int(progress.get("total") or 0) missing_metrics = [str(x) for x in (summary.get("missing_metrics") or []) if str(x)] + current_missing = [str(x) for x in (summary.get("current_missing_metrics") or []) if str(x)] fail_reasons: list[str] = [] if total <= 0: fail_reasons.append("empty_expect") if missing_metrics: fail_reasons.append("new_baseline_missing") + if current_missing: + fail_reasons.append("current_missing") if anomaly: fail_reasons.append("anomaly") if total > 0 and ok < total: @@ -1052,6 +1849,7 @@ def finish_batch( "passed": passed, "fail_reasons": fail_reasons, "missing_metrics": missing_metrics, + "current_missing_metrics": current_missing, "progress_ok": ok, "progress_total": total, "anomaly": anomaly, diff --git a/netx_api/biz_migration_router.py b/netx_api/biz_migration_router.py index 633cdd9..0a0c1df 100644 --- a/netx_api/biz_migration_router.py +++ b/netx_api/biz_migration_router.py @@ -17,22 +17,45 @@ router = APIRouter(prefix="/v1/biz-migration", tags=["biz-migration"]) class ProjectIn(BaseModel): name: str - old_task_id: str - new_task_id: str + old_task_id: str = "" + new_task_id: str = "" + old_hf_task_id: str = "" + new_hf_task_id: str = "" old_baseline_batch_id: str = "" new_baseline_batch_id: str = "" mapping_id: str = "" + mapping_name: str = "" + mapping_rows: list[dict[str, Any]] = Field(default_factory=list) monitor_template_id: str = "" - status: str = "draft" + collect_metric_ids: list[str] = Field(default_factory=list) + metric_interval_sec: dict[str, int] = Field(default_factory=dict) + hf_interval_sec: int = 60 + hf_start_at: str | None = None + hf_end_at: str | None = None + create_hf: bool = True + collect_now: bool = False + retention_days: int = 7 + old_ne: dict[str, Any] | None = None + new_ne: dict[str, Any] | None = None + status: str = "active" note: str = "" class ProjectPatchIn(BaseModel): name: str | None = None + old_task_id: str | None = None + new_task_id: str | None = None + old_hf_task_id: str | None = None + new_hf_task_id: str | None = None old_baseline_batch_id: str | None = None new_baseline_batch_id: str | None = None mapping_id: str | None = None monitor_template_id: str | None = None + collect_metric_ids: list[str] | None = None + metric_interval_sec: dict[str, int] | None = None + hf_interval_sec: int | None = None + hf_start_at: str | None = None + hf_end_at: str | None = None status: str | None = None note: str | None = None @@ -79,6 +102,19 @@ def api_list_projects(db: Session = Depends(get_db)): return {"items": svc.list_projects(db)} +@router.get("/ne-portrait") +def api_ne_portrait( + source: str = "managed", + ne_id: str = "", + limit: int = 30, + db: Session = Depends(get_db), +): + """Portrait biz_state task + recent batches for a NE (baseline picker).""" + if not str(ne_id or "").strip(): + return {"source": source, "ne_id": "", "task": None, "batches": [], "hint": "ne_id_required"} + return svc.list_ne_portrait_options(db, source=source, ne_id=ne_id, limit=limit) + + @router.post("/projects") def api_create_project(body: ProjectIn, db: Session = Depends(get_db)): return svc.create_project(db, body.model_dump()) @@ -112,9 +148,14 @@ def api_baseline_expect(project_id: str, db: Session = Depends(get_db)): class EnsureHighfreqIn(BaseModel): - interval_sec: int = 60 + interval_sec: int | None = None retention_days: int = 7 collect_now: bool = True + old_ne: dict[str, Any] | None = None + new_ne: dict[str, Any] | None = None + collect_metric_ids: list[str] | None = None + hf_start_at: str | None = None + hf_end_at: str | None = None @router.post("/projects/{project_id}/ensure-port-highfreq") @@ -124,14 +165,28 @@ def api_ensure_port_highfreq( body: EnsureHighfreqIn | None = None, db: Session = Depends(get_db), ): - """Create/bind HF biz_state tasks from monitor template collect_metric_ids.""" - payload = body.model_dump() if body else {} + """Create/bind HF biz_state tasks into hf slots (portrait slots untouched).""" + payload = body.model_dump(exclude_unset=True) if body else {} + # Persist schedule / collect override before ensure + patch: dict[str, Any] = {} + if "collect_metric_ids" in payload and payload["collect_metric_ids"] is not None: + patch["collect_metric_ids"] = payload["collect_metric_ids"] + if "hf_start_at" in payload: + patch["hf_start_at"] = payload.get("hf_start_at") + if "hf_end_at" in payload: + patch["hf_end_at"] = payload.get("hf_end_at") + if "interval_sec" in payload and payload["interval_sec"] is not None: + patch["hf_interval_sec"] = payload["interval_sec"] + if patch: + svc.patch_project(db, project_id, patch) return svc.ensure_highfreq( db, project_id, - interval_sec=int(payload.get("interval_sec") or 60), + interval_sec=payload.get("interval_sec"), retention_days=int(payload.get("retention_days") or 7), collect_now=bool(payload.get("collect_now", True)), + old_ne=payload.get("old_ne"), + new_ne=payload.get("new_ne"), ) diff --git a/netx_api/biz_state/collect_runner.py b/netx_api/biz_state/collect_runner.py index eba8379..e4bd1e7 100644 --- a/netx_api/biz_state/collect_runner.py +++ b/netx_api/biz_state/collect_runner.py @@ -158,10 +158,27 @@ def _persist_vrf_route_summary( _GENERIC_METRICS = { "isis_adjacency", "interface_brief", + "interface_detail", "arp", "if_intf", "nd6_cache", "bgp_peer", + "ospf_neighbor", + "vrrp", + "optical_brief", + "bgp_route", + "ip_route", + "ipv6_route", + "l2vpn_pw", + "l2vpn_mac", + "evpn_mac", + "config_vrf", + "config_interface", + "config_bgp_peer", + "config_l2vpn_pw", + "config_static_route", + "config_ospf", + "config_isis", } _METRIC_CHUNK = 2000 @@ -500,7 +517,7 @@ def _run_collect_session( aux_results: dict[str, Any] = {} for aux in list(hit.profile.aux_commands or []): try: - ra = resolve_aux_command(aux) + ra = resolve_aux_command(aux, params=merged) except ValueError as exc: aux_row = BizStateBatchCommand( id=uuid4().hex, @@ -534,6 +551,7 @@ def _run_collect_session( ra.command, parser_id=ra.parser_id, textfsm_command=ra.textfsm_command, + params=merged, cmd_row_id=aux_row.id, ) aux_results[ra.key] = entry diff --git a/netx_api/biz_state/collect_session.py b/netx_api/biz_state/collect_session.py index b52dcca..4fe88ed 100644 --- a/netx_api/biz_state/collect_session.py +++ b/netx_api/biz_state/collect_session.py @@ -4,6 +4,7 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Any, Callable +import re from .command_match import normalize_command from .enrich import EnrichJoin, apply_enrich_joins @@ -34,8 +35,11 @@ class ResolvedAux: profile: ParseProfile -def resolve_aux_command(aux: AuxCommand) -> ResolvedAux: - """Resolve aux from ``profile_id`` (single source of truth).""" +def resolve_aux_command( + aux: AuxCommand, + params: dict[str, str] | None = None, +) -> ResolvedAux: + """Resolve aux from ``profile_id``; render ```` from primary params.""" key = str(aux.key or "").strip() pid = str(aux.profile_id or "").strip() if not key or not pid: @@ -43,10 +47,22 @@ def resolve_aux_command(aux: AuxCommand) -> ResolvedAux: prof = get_profile(pid) if not prof: raise ValueError(f"aux profile not found: {pid}") - cmd = normalize_command(prof.command_template) + tmpl = str(prof.command_template or "") + rendered = tmpl + for name, val in dict(params or {}).items(): + n = str(name or "").strip() + if not n: + continue + rendered = rendered.replace(f"<{n}>", str(val or "").strip()) + cmd = normalize_command(rendered) if not cmd: raise ValueError(f"aux profile {pid} has empty command_template") + if re.search(r"<[^>]+>", cmd): + raise ValueError(f"aux profile {pid} unresolved placeholders: {cmd}") textfsm = str(prof.textfsm_command or cmd).strip() + # Prefer concrete rendered command for FSM when aux had placeholders + if params and any(f"<{k}>" in tmpl for k in (params or {})): + textfsm = cmd parser_id = str(prof.parser_id or "").strip() meta = get_parser_meta(parser_id) if parser_id else None rule_keys = tuple((meta or {}).get("rule_keys") or ()) diff --git a/netx_api/biz_state/compare_engine.py b/netx_api/biz_state/compare_engine.py index b494d5a..3a8a15d 100644 --- a/netx_api/biz_state/compare_engine.py +++ b/netx_api/biz_state/compare_engine.py @@ -5,6 +5,11 @@ from __future__ import annotations from typing import Any, Mapping, Sequence from .compare_rules import field_rule_map, values_equal, explain_diff +from .iface_normalize import ( + apply_iface_normalize_rows, + normalize_iface_rules, + resolve_mapped_iface, +) def apply_port_map( @@ -13,11 +18,12 @@ def apply_port_map( iface_fields: list[str], port_map: dict[str, str], ) -> dict[str, Any]: + """Rewrite iface columns on a row (exact map or parent.subif inheritance).""" out = dict(row) for f in iface_fields: val = str(out.get(f) or "").strip() - if val and val in port_map: - out[f] = port_map[val] + if val: + out[f] = resolve_mapped_iface(val, port_map) return out @@ -32,7 +38,11 @@ def mapping_stats( iface_fields: list[str], port_map: dict[str, str], ) -> dict[str, Any]: - """hit/miss/unused for port mapping validation.""" + """hit/miss/unused for port mapping validation. + + ``hit_before``: map key appears as a normalized before iface, or as the + parent of a before subinterface (so main-port-only maps still validate). + """ before_ifaces: set[str] = set() after_ifaces: set[str] = set() for f in iface_fields: @@ -45,13 +55,25 @@ def mapping_stats( if v: after_ifaces.add(v) + before_bases: set[str] = set() + for v in before_ifaces: + before_bases.add(v) + if "." in v: + before_bases.add(v.rsplit(".", 1)[0]) + + after_bases: set[str] = set() + for v in after_ifaces: + after_bases.add(v) + if "." in v: + after_bases.add(v.rsplit(".", 1)[0]) + mapped_before = set(port_map.keys()) mapped_after = set(port_map.values()) - hit_before = sorted(mapped_before & before_ifaces) - miss_before = sorted(mapped_before - before_ifaces) - unused = sorted(mapped_before - before_ifaces) # same as miss for before presence - hit_after = sorted(mapped_after & after_ifaces) - miss_after = sorted(mapped_after - after_ifaces) + hit_before = sorted(mapped_before & before_bases) + miss_before = sorted(mapped_before - before_bases) + unused = list(miss_before) + hit_after = sorted(mapped_after & after_bases) + miss_after = sorted(mapped_after - after_bases) return { "before_iface_count": len(before_ifaces), "after_iface_count": len(after_ifaces), @@ -74,13 +96,18 @@ def compare_rows( compare_fields: list[str], port_map: dict[str, str] | None = None, field_rules: Sequence[Mapping[str, Any]] | None = None, + iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, ) -> dict[str, Any]: """Return summary + diffs list. Diff kinds: added | removed | changed | unchanged - Empty ``port_map`` means ignore port/iface changes: matching keys exclude - ``iface_fields`` (same-device batches compare without a rename map). + Pipeline: iface normalize (both sides) → port map (before) → match. + + Empty ``port_map``: try to ignore port renames by dropping ``iface_fields`` + from the match key (LLDP-style). If that would collapse distinct rows + (OSPF/VRRP/config where the same id appears on many interfaces), keep + iface columns so identity compare stays correct. ``field_rules`` drives normalize / numeric tolerance / per-field compare mode (template-driven; no metric-specific branches here). @@ -89,24 +116,49 @@ def compare_rows( raise ValueError("key_fields required") pmap = dict(port_map or {}) rules = field_rule_map(field_rules) - iface_set = {str(f) for f in (iface_fields or []) if str(f).strip()} - # No map → ignore port renames: drop iface columns from the match key. + norm_rules = normalize_iface_rules(iface_normalize_rules) + iface_list = [str(f) for f in (iface_fields or []) if str(f).strip()] + iface_set = set(iface_list) + + before_norm = apply_iface_normalize_rows( + before_rows, iface_fields=iface_list, rules=norm_rules + ) + after_norm = apply_iface_normalize_rows( + after_rows, iface_fields=iface_list, rules=norm_rules + ) + + ignore_port_changes = False + # No map → optionally ignore port renames by dropping iface from match key. if not pmap and iface_set: - match_keys = [k for k in key_fields if k not in iface_set] - if not match_keys: + candidate = [k for k in key_fields if k not in iface_set] + if not candidate: match_keys = list(key_fields) + else: + before_c = [row_key(r, candidate) for r in before_norm] + after_c = [row_key(r, candidate) for r in after_norm] + if len(before_c) == len(set(before_c)) and len(after_c) == len(set(after_c)): + match_keys = candidate + ignore_port_changes = True + else: + # Remaining keys are not unique — iface is required for identity. + match_keys = list(key_fields) else: match_keys = list(key_fields) before_mapped: list[dict[str, Any]] = [ - apply_port_map(r, iface_fields=iface_fields, port_map=pmap) for r in before_rows + apply_port_map(r, iface_fields=iface_list, port_map=pmap) for r in before_norm ] after_index: dict[tuple[str, ...], dict[str, Any]] = {} - for r in after_rows: - after_index[row_key(r, match_keys)] = r + after_dup = 0 + for r in after_norm: + k = row_key(r, match_keys) + if k in after_index: + after_dup += 1 + after_index[k] = r before_keys: set[tuple[str, ...]] = set() + before_dup = 0 diffs: list[dict[str, Any]] = [] added = removed = changed = unchanged = 0 @@ -115,6 +167,8 @@ def compare_rows( for orig, mapped in zip(before_rows, before_mapped): k = row_key(mapped, match_keys) + if k in before_keys: + before_dup += 1 before_keys.add(k) after = after_index.get(k) if after is None: @@ -184,13 +238,13 @@ def compare_rows( ) stats = mapping_stats( - before_rows=before_rows, - after_rows=after_rows, - iface_fields=iface_fields, + before_rows=before_norm, + after_rows=after_norm, + iface_fields=iface_list, port_map=pmap, ) if not pmap: - stats = {**stats, "ok": True, "ignore_port_changes": True} + stats = {**stats, "ok": True, "ignore_port_changes": ignore_port_changes} return { "summary": { "before_count": len(before_rows), @@ -200,6 +254,8 @@ def compare_rows( "changed": changed, "unchanged": unchanged, "match_key_fields": match_keys, + "duplicate_keys_before": before_dup, + "duplicate_keys_after": after_dup, }, "diffs": diffs, "mapping_stats": stats, diff --git a/netx_api/biz_state/compare_service.py b/netx_api/biz_state/compare_service.py index 816cefc..0f70591 100644 --- a/netx_api/biz_state/compare_service.py +++ b/netx_api/biz_state/compare_service.py @@ -32,6 +32,11 @@ from .compare_rules import ( effective_compare_fields, effective_display_fields, ) +from .iface_normalize import ( + apply_iface_normalize_rows, + default_zte_iface_normalize_rules, + normalize_iface_rules, +) from .profiles import metric_field_map @@ -397,6 +402,15 @@ def _default_sheet_for_metric(metric_id: str, *, compare_roles: tuple[str, ...] # Context columns: show but not necessarily compare ctx = [n for n in ("vrf", "entry_type", "age") if n not in keys and n not in compare] extra["display_fields"] = list(keys) + list(compare) + ctx + elif metric_id in ("interface_detail", "optical_brief", "bgp_peer"): + # Counters (rates / optical power / pfx_rcd) stay visible but out of default compare. + counters = [f.name for f in fields if (not f.is_key) and f.role == "counter"] + meta = [ + f.name + for f in fields + if (not f.is_key) and f.role == "meta" and f.name not in compare + ] + extra["display_fields"] = list(keys) + list(compare) + counters + meta return _sheet_def( metric_id=metric_id, key_fields=keys, @@ -449,6 +463,20 @@ def _bgp_afi_sheets() -> list[dict[str, Any]]: ("bgp_peer.ipv6", "BGP IPv6", "ipv6"), ("bgp_peer.vpnv4", "BGP VPNv4", "vpnv4"), ("bgp_peer.vpnv6", "BGP VPNv6", "vpnv6"), + ("bgp_peer.evpn", "BGP EVPN", "evpn"), + ("bgp_peer.vpls", "BGP VPLS", "vpls"), + ), + op="eq", + ) + + +def _vrrp_af_sheets() -> list[dict[str, Any]]: + return _sheets_split_by_field( + "vrrp", + "af", + ( + ("vrrp.ipv4", "VRRP IPv4", "ipv4"), + ("vrrp.ipv6", "VRRP IPv6", "ipv6"), ), op="eq", ) @@ -470,6 +498,7 @@ def _builtin_source_splits() -> dict[str, list[dict[str, Any]]]: return { "bgp_peer": _bgp_afi_sheets(), "isis_adjacency": _isis_af_sheets(), + "vrrp": _vrrp_af_sheets(), } @@ -477,9 +506,44 @@ def _default_zte_status_sheets() -> list[dict[str, Any]]: return [ *_isis_af_sheets(), _default_sheet_for_metric("interface_brief", compare_roles=("state",)), + _default_sheet_for_metric("interface_detail", compare_roles=("state",)), _default_sheet_for_metric("arp", compare_roles=("state",)), _default_sheet_for_metric("nd6_cache", compare_roles=("state",)), + _default_sheet_for_metric("ospf_neighbor", compare_roles=("state",)), + *_vrrp_af_sheets(), + _default_sheet_for_metric("optical_brief", compare_roles=("state",)), *_bgp_afi_sheets(), + _default_sheet_for_metric("l2vpn_pw", compare_roles=("state",)), + _default_lldp_sheet(), + ] + + +def _default_zte_config_sheets() -> list[dict[str, Any]]: + """Config-intent metrics for cutover / intent-vs-intent compare.""" + return [ + _default_sheet_for_metric("config_vrf", compare_roles=("state",)), + _default_sheet_for_metric("config_interface", compare_roles=("state",)), + _default_sheet_for_metric("config_bgp_peer", compare_roles=("state",)), + _default_sheet_for_metric("config_l2vpn_pw", compare_roles=("state",)), + *_sheets_split_by_field( + "config_static_route", + "af", + ( + ("config_static_route.ipv4", "Static IPv4", "ipv4"), + ("config_static_route.ipv6", "Static IPv6", "ipv6"), + ), + compare_roles=("state",), + ), + *_sheets_split_by_field( + "config_ospf", + "af", + ( + ("config_ospf.ipv4", "OSPF IPv4", "ipv4"), + ("config_ospf.ipv6", "OSPF IPv6", "ipv6"), + ), + compare_roles=("state",), + ), + _default_sheet_for_metric("config_isis", compare_roles=("state",)), ] @@ -622,6 +686,17 @@ def _parse_metrics_body(body: dict[str, Any]) -> list[dict[str, Any]]: ] +def template_iface_normalize(t: BizCompareTemplate | None) -> list[dict[str, str]]: + """Resolved iface type-alias rules for a compare template.""" + if t is None: + return [] + return normalize_iface_rules(getattr(t, "iface_normalize_json", None) or []) + + +def _set_template_iface_normalize(t: BizCompareTemplate, raw: Any) -> None: + t.iface_normalize_json = normalize_iface_rules(raw) + + def _template_out(t: BizCompareTemplate) -> dict[str, Any]: sheets = template_metrics(t) first = sheets[0] if sheets else None @@ -636,6 +711,7 @@ def _template_out(t: BizCompareTemplate) -> dict[str, Any]: "iface_fields": list((first or {}).get("iface_fields") or t.iface_fields or []), "compare_fields": list((first or {}).get("compare_fields") or t.compare_fields or []), "ignore_fields": [], + "iface_normalize_rules": template_iface_normalize(t), "note": t.note, "updated_at": t.updated_at.isoformat() + "Z" if t.updated_at else None, } @@ -730,21 +806,37 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: name = "ZTE status default" row = db.query(BizCompareTemplate).filter(BizCompareTemplate.name == name).one_or_none() sheets = _default_zte_status_sheets() + # Volatile counters must not stay in compare_fields on upgraded installs. + _STRIP_COMPARE: dict[str, frozenset[str]] = { + "interface_detail": frozenset({"input_bps", "output_bps", "in_util", "out_util"}), + "optical_brief": frozenset({"rx_power", "tx_power"}), + "bgp_peer": frozenset({"pfx_rcd"}), + } if row: existing = template_metrics(row) want = {s["metric_id"] for s in sheets} have = {s["metric_id"] for s in existing} changed = bool(want - have) - # Migrate ARP sheet: inject template row_filters if missing (replaces code filter) upgraded: list[dict[str, Any]] = [] by_want = {s["metric_id"]: s for s in sheets} for s in existing: cur = dict(s) - if cur.get("metric_id") == "arp" and not cur.get("row_filters"): + mid = str(cur.get("metric_id") or "") + if mid == "arp" and not cur.get("row_filters"): cur["row_filters"] = list(by_want.get("arp", {}).get("row_filters") or arp_dynamic_row_filters()) if not cur.get("field_rules") and by_want.get("arp", {}).get("field_rules"): cur["field_rules"] = list(by_want["arp"]["field_rules"]) changed = True + strip = _STRIP_COMPARE.get(mid) + if strip: + old_cmp = list(cur.get("compare_fields") or []) + new_cmp = [f for f in old_cmp if f not in strip] + if new_cmp != old_cmp: + cur["compare_fields"] = new_cmp + want_disp = list((by_want.get(mid) or {}).get("display_fields") or []) + if want_disp: + cur["display_fields"] = want_disp + changed = True upgraded.append(_normalize_sheet(cur) or cur) if want - have: for s in sheets: @@ -753,7 +845,12 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: changed = True if changed: _apply_sheets_to_row(row, upgraded if upgraded else sheets) - row.note = "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP)" + row.note = "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)" + row.updated_at = _utcnow() + db.commit() + db.refresh(row) + if not template_iface_normalize(row): + _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) row.updated_at = _utcnow() db.commit() db.refresh(row) @@ -761,11 +858,51 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: row = BizCompareTemplate( id=uuid4().hex, name=name, - note="Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP)", + note="Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)", created_at=_utcnow(), updated_at=_utcnow(), ) _apply_sheets_to_row(row, sheets) + _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) + db.add(row) + db.commit() + db.refresh(row) + return row + + +def ensure_default_zte_config_template(db: Session) -> BizCompareTemplate: + name = "ZTE config intent default" + row = db.query(BizCompareTemplate).filter(BizCompareTemplate.name == name).one_or_none() + sheets = _default_zte_config_sheets() + if row: + existing = template_metrics(row) + want = {s["metric_id"] for s in sheets} + have = {s["metric_id"] for s in existing} + if want - have: + upgraded = list(existing) + for s in sheets: + if s["metric_id"] not in have: + upgraded.append(s) + _apply_sheets_to_row(row, upgraded) + row.note = "Built-in ZTE config intent (VRF/IF/BGP/L2VPN PW)" + row.updated_at = _utcnow() + db.commit() + db.refresh(row) + if not template_iface_normalize(row): + _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) + row.updated_at = _utcnow() + db.commit() + db.refresh(row) + return row + row = BizCompareTemplate( + id=uuid4().hex, + name=name, + note="Built-in ZTE config intent (VRF/IF/BGP/L2VPN PW)", + created_at=_utcnow(), + updated_at=_utcnow(), + ) + _apply_sheets_to_row(row, sheets) + _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) db.add(row) db.commit() db.refresh(row) @@ -773,16 +910,11 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: def ensure_default_templates(db: Session) -> None: - """Seed built-ins only on first install (empty table). + """Seed the single built-in compare template (most complete status sheet set). - Operators own templates after that — do not recreate deleted built-ins - or overwrite customized sheets on every list call. + Operators own additional templates after that — do not recreate deleted + siblings (port-only / LLDP / VRF / config) on every list call. """ - if db.query(BizCompareTemplate.id).limit(1).first(): - return - ensure_default_cutover_template(db) - ensure_default_lldp_template(db) - ensure_default_vrf_template(db) ensure_default_zte_status_template(db) @@ -871,6 +1003,10 @@ def create_template(db: Session, body: dict[str, Any]) -> dict[str, Any]: updated_at=_utcnow(), ) _apply_sheets_to_row(t, sheets) + if "iface_normalize_rules" in body or "iface_normalize_json" in body: + _set_template_iface_normalize( + t, body.get("iface_normalize_rules", body.get("iface_normalize_json")) + ) db.add(t) db.commit() return _template_out(t) @@ -884,6 +1020,11 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict t.name = str(body.get("name") or "")[:256] if "note" in body: t.note = str(body.get("note") or "")[:512] + if "iface_normalize_rules" in body or "iface_normalize_json" in body: + _set_template_iface_normalize( + t, body.get("iface_normalize_rules", body.get("iface_normalize_json")) + ) + t.updated_at = _utcnow() if any( k in body for k in ( @@ -1105,12 +1246,22 @@ def validate_mapping( pmap = _port_map_dict(db, mapping_id) # Validate against first sheet that has iface fields (or first sheet) target = next((s for s in sheets if s.get("iface_fields")), sheets[0]) - before = _load_metric_rows(db, batch_id=before_batch_id, metric_id=target["metric_id"]) - after = _load_metric_rows(db, batch_id=after_batch_id, metric_id=target["metric_id"]) + iface_fields = list(target.get("iface_fields") or []) + norm = template_iface_normalize(tpl) + before = apply_iface_normalize_rows( + _load_metric_rows(db, batch_id=before_batch_id, metric_id=target["metric_id"]), + iface_fields=iface_fields, + rules=norm, + ) + after = apply_iface_normalize_rows( + _load_metric_rows(db, batch_id=after_batch_id, metric_id=target["metric_id"]), + iface_fields=iface_fields, + rules=norm, + ) return mapping_stats( before_rows=before, after_rows=after, - iface_fields=list(target.get("iface_fields") or []), + iface_fields=iface_fields, port_map=pmap, ) @@ -1232,6 +1383,7 @@ def _run_sheet( before_batch_id: str, after_batch_id: str, port_map: dict[str, str], + iface_normalize_rules: list[dict[str, str]] | None = None, ) -> dict[str, Any]: key_fields = list(sheet.get("key_fields") or []) iface_fields = list(sheet.get("iface_fields") or []) @@ -1261,6 +1413,7 @@ def _run_sheet( compare_fields=compare_fields, port_map=port_map, field_rules=field_rules, + iface_normalize_rules=iface_normalize_rules, ) summary = dict(result["summary"]) summary["before_raw_count"] = len(before_raw) @@ -1302,6 +1455,7 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> raise HTTPException(status_code=400, detail="template_has_no_metrics") pmap = _port_map_dict(db, j.mapping_id) + norm_rules = template_iface_normalize(tpl) sheet_results: list[dict[str, Any]] = [] agg = { "before_count": 0, @@ -1319,6 +1473,7 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> before_batch_id=before_batch_id, after_batch_id=after_batch_id, port_map=pmap, + iface_normalize_rules=norm_rules, ) sheet_results.append(one) s = one["summary"] diff --git a/netx_api/biz_state/iface_normalize.py b/netx_api/biz_state/iface_normalize.py new file mode 100644 index 0000000..199a9a1 --- /dev/null +++ b/netx_api/biz_state/iface_normalize.py @@ -0,0 +1,137 @@ +"""Interface name normalize + port-map resolve (exact + parent.subif). + +Normalize rules (template-owned): rewrite type aliases like ``GE`` → ``gei``. +Port map (job-owned): rewrite before → after, with subinterface suffix inheritance. +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + + +def normalize_iface_rules(raw: Any) -> list[dict[str, str]]: + """Accept list of {from,to} or ``from,to`` lines; drop empties; longest ``from`` first.""" + out: list[dict[str, str]] = [] + if not raw: + return out + if isinstance(raw, str): + items: list[Any] = [] + for line in raw.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if "," in line: + a, b = line.split(",", 1) + items.append({"from": a.strip(), "to": b.strip()}) + elif "\t" in line: + a, b = line.split("\t", 1) + items.append({"from": a.strip(), "to": b.strip()}) + else: + parts = line.split() + if len(parts) >= 2: + items.append({"from": parts[0], "to": parts[1]}) + raw = items + if not isinstance(raw, (list, tuple)): + return out + for item in raw: + if not isinstance(item, Mapping): + continue + fr = str(item.get("from") or item.get("src") or "").strip() + to = str(item.get("to") or item.get("dst") or "").strip() + if fr and to: + out.append({"from": fr, "to": to}) + # Longest prefix first so XXVGE wins over XGE / GE. + out.sort(key=lambda r: len(r["from"]), reverse=True) + return out + + +def _prefix_ok(name: str, prefix: str) -> bool: + """True when ``name`` starts with ``prefix`` (ci) and boundary is safe.""" + if not name or not prefix: + return False + n = name + p = prefix + if len(n) < len(p): + return False + if n[: len(p)].lower() != p.lower(): + return False + if len(n) == len(p): + return True + nxt = n[len(p)] + return nxt.isdigit() or nxt in "-/" + + +def normalize_iface_name(name: str, rules: Sequence[Mapping[str, str]] | None) -> str: + """Apply first matching type-prefix rule; leave subif / rest intact.""" + text = str(name or "").strip() + if not text or not rules: + return text + for rule in rules: + fr = str(rule.get("from") or "").strip() + to = str(rule.get("to") or "").strip() + if not fr or not to: + continue + if _prefix_ok(text, fr): + return to + text[len(fr) :] + return text + + +def apply_iface_normalize( + row: dict[str, Any], + *, + iface_fields: Sequence[str], + rules: Sequence[Mapping[str, str]] | None, +) -> dict[str, Any]: + if not rules or not iface_fields: + return dict(row) + out = dict(row) + for f in iface_fields: + key = str(f or "").strip() + if not key: + continue + val = str(out.get(key) or "").strip() + if val: + out[key] = normalize_iface_name(val, rules) + return out + + +def apply_iface_normalize_rows( + rows: Sequence[Mapping[str, Any]], + *, + iface_fields: Sequence[str], + rules: Sequence[Mapping[str, str]] | None, +) -> list[dict[str, Any]]: + return [ + apply_iface_normalize(dict(r), iface_fields=iface_fields, rules=rules) + for r in rows + ] + + +def resolve_mapped_iface(name: str, port_map: Mapping[str, str] | None) -> str: + """Exact map hit, else parent + ``.suffix`` if parent mapped, else unchanged.""" + text = str(name or "").strip() + if not text or not port_map: + return text + if text in port_map: + return str(port_map[text]) + if "." not in text: + return text + parent, suffix = text.rsplit(".", 1) + if not parent or not suffix: + return text + if parent in port_map: + return f"{port_map[parent]}.{suffix}" + return text + + +def default_zte_iface_normalize_rules() -> list[dict[str, str]]: + """Common ZTE brief aliases → canonical CLI names.""" + return normalize_iface_rules( + [ + {"from": "XXVGE", "to": "xxvgei"}, + {"from": "XGE", "to": "xgei"}, + {"from": "CGE", "to": "cgei"}, + {"from": "GE", "to": "gei"}, + {"from": "SG", "to": "smartgroup"}, + ] + ) diff --git a/netx_api/biz_state/parsers/cisco/__init__.py b/netx_api/biz_state/parsers/cisco/__init__.py index 69f0964..61aff10 100644 --- a/netx_api/biz_state/parsers/cisco/__init__.py +++ b/netx_api/biz_state/parsers/cisco/__init__.py @@ -2,7 +2,9 @@ LLDP already works via ``common.lldp_neighbors`` (profile ``parser_id=lldp_neighbors``). -Stub modules (not registered until implemented):: +Shared ``metric_id`` schemas live in ``profiles.py`` (ZTE is the reference +implementation). Stub modules keep normalize signatures aligned — do **not** +register in ``PARSERS`` until implemented:: interface_brief.py arp.py isis_adjacency.py nd6_cache.py bgp_peer.py diff --git a/netx_api/biz_state/parsers/common/vrf_list.py b/netx_api/biz_state/parsers/common/vrf_list.py index c6449ff..8ae8535 100644 --- a/netx_api/biz_state/parsers/common/vrf_list.py +++ b/netx_api/biz_state/parsers/common/vrf_list.py @@ -11,6 +11,14 @@ from .pipeline import prefer_fsm RULE_KEYS = ("zte_zxros_show_ip_vrf",) +_VRF_LINE_RE = re.compile( + r"^\s*(?P[A-Za-z0-9_./:-]+)\s+" + r"(?P\d+:\d+||\S+:\S+)\s*" + r"(?:(?Pipv4(?:,ipv6)?|ipv6(?:,ipv4)?|ipv4|ipv6)\s*)?" + r"(?P\d+)?\s*$", + re.I, +) + def _map_fsm_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] @@ -22,11 +30,16 @@ def _map_fsm_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: if name in seen: continue seen.add(name) + protocols = row_get(r, "PROTOCOLS", "ADDRESS_FAMILY", "protocols") + # Legacy templates may put protocols in INTERFACES + if not protocols: + protocols = row_get(r, "INTERFACES", "interfaces") out.append( { "vrf_name": name[:128], - "rd": row_get(r, "DEFAULT_RD", "RD", "rd")[:64], - "protocols": row_get(r, "PROTOCOLS", "ADDRESS_FAMILY", "INTERFACES", "protocols")[:64], + "rd": row_get(r, "RD", "DEFAULT_RD", "rd")[:64], + "protocols": protocols[:64], + "vrf_id": row_get(r, "VRF_ID", "vrf_id")[:32], } ) return out @@ -36,16 +49,23 @@ def _hand_parse(*, raw_text: str, **_kw: Any) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] seen: set[str] = set() for line in str(raw_text or "").splitlines(): - m = re.match(r"^\s*([A-Za-z0-9_./:-]+)\s+(\d+:\d+||\S+:\S+)\s*", line) + m = _VRF_LINE_RE.match(line) if not m: continue - name = m.group(1) + name = m.group("name") if name.lower() in ("name", "vrf", "vpn-instance", "total"): continue if name in seen: continue seen.add(name) - out.append({"vrf_name": name[:128], "rd": m.group(2)[:64], "protocols": ""}) + out.append( + { + "vrf_name": name[:128], + "rd": (m.group("rd") or "")[:64], + "protocols": (m.group("protocols") or "")[:64], + "vrf_id": (m.group("vrf_id") or "")[:32], + } + ) return out diff --git a/netx_api/biz_state/parsers/huawei/__init__.py b/netx_api/biz_state/parsers/huawei/__init__.py index f9028c7..fdebf94 100644 --- a/netx_api/biz_state/parsers/huawei/__init__.py +++ b/netx_api/biz_state/parsers/huawei/__init__.py @@ -2,11 +2,16 @@ LLDP already works via ``common.lldp_neighbors`` (``display lldp neighbor``). -Stub modules (not registered until implemented):: +Shared ``metric_id`` / FieldDef schemas (cross-vendor compare) are defined in +``profiles.py`` for ZTE first. Stub modules below keep the same normalize +signatures for future Huawei implementations — do **not** register in +``PARSERS`` until TextFSM + prefer_fsm paths exist (avoids NotImplementedError +at collect time):: interface_brief.py arp.py isis_adjacency.py nd6_cache.py bgp_peer.py -When ready, import and add to ``PARSERS``. +When ready, import and add to ``PARSERS``. Prefer same field names as ZTE +(``bgp_peer``, ``ospf_neighbor``, ``vrrp``, ``ip_route``, …). """ from __future__ import annotations diff --git a/netx_api/biz_state/parsers/zte/__init__.py b/netx_api/biz_state/parsers/zte/__init__.py index 5e31dd4..3ef653c 100644 --- a/netx_api/biz_state/parsers/zte/__init__.py +++ b/netx_api/biz_state/parsers/zte/__init__.py @@ -10,20 +10,54 @@ from typing import Any, Callable from .arp import is_valid_arp_age, normalize_arp from .bgp_peer import normalize_bgp_peer +from .bgp_route import normalize_bgp_route +from .config_bgp_peer import normalize_config_bgp_peer +from .config_interface import normalize_config_interface +from .config_isis import normalize_config_isis +from .config_l2vpn_pw import normalize_config_l2vpn_pw +from .config_ospf import normalize_config_ospf +from .config_static_route import normalize_config_static_route +from .config_vrf import normalize_config_vrf +from .evpn_mac import normalize_evpn_mac from .if_intf import normalize_if_intf, parse_if_intf_vrf_map from .interface_brief import normalize_interface_brief +from .interface_detail import normalize_interface_detail +from .ip_route import normalize_ip_route +from .ipv6_route import normalize_ipv6_route from .isis_adjacency import normalize_isis_adjacency +from .l2vpn_mac import normalize_l2vpn_mac +from .l2vpn_pw import normalize_l2vpn_pw from .nd6_cache import normalize_nd6_cache +from .optical_brief import normalize_optical_brief +from .ospf_neighbor import normalize_ospf_neighbor +from .vrrp import normalize_vrrp NormalizeFn = Callable[..., list[dict[str, Any]]] PARSERS: dict[str, NormalizeFn] = { "isis_adjacency": normalize_isis_adjacency, "interface_brief": normalize_interface_brief, + "interface_detail": normalize_interface_detail, "arp": normalize_arp, "if_intf": normalize_if_intf, "nd6_cache": normalize_nd6_cache, "bgp_peer": normalize_bgp_peer, + "ospf_neighbor": normalize_ospf_neighbor, + "vrrp": normalize_vrrp, + "optical_brief": normalize_optical_brief, + "bgp_route": normalize_bgp_route, + "ip_route": normalize_ip_route, + "ipv6_route": normalize_ipv6_route, + "l2vpn_pw": normalize_l2vpn_pw, + "l2vpn_mac": normalize_l2vpn_mac, + "evpn_mac": normalize_evpn_mac, + "config_vrf": normalize_config_vrf, + "config_interface": normalize_config_interface, + "config_bgp_peer": normalize_config_bgp_peer, + "config_l2vpn_pw": normalize_config_l2vpn_pw, + "config_static_route": normalize_config_static_route, + "config_ospf": normalize_config_ospf, + "config_isis": normalize_config_isis, } __all__ = [ @@ -31,9 +65,26 @@ __all__ = [ "is_valid_arp_age", "normalize_arp", "normalize_bgp_peer", + "normalize_bgp_route", + "normalize_config_bgp_peer", + "normalize_config_interface", + "normalize_config_isis", + "normalize_config_l2vpn_pw", + "normalize_config_ospf", + "normalize_config_static_route", + "normalize_config_vrf", + "normalize_evpn_mac", "normalize_if_intf", "normalize_interface_brief", + "normalize_interface_detail", + "normalize_ip_route", + "normalize_ipv6_route", "normalize_isis_adjacency", + "normalize_l2vpn_mac", + "normalize_l2vpn_pw", "normalize_nd6_cache", + "normalize_optical_brief", + "normalize_ospf_neighbor", + "normalize_vrrp", "parse_if_intf_vrf_map", ] diff --git a/netx_api/biz_state/parsers/zte/bgp_peer.py b/netx_api/biz_state/parsers/zte/bgp_peer.py index 2374e12..2a7d6b9 100644 --- a/netx_api/biz_state/parsers/zte/bgp_peer.py +++ b/netx_api/biz_state/parsers/zte/bgp_peer.py @@ -1,23 +1,39 @@ -"""ZTE: show bgp {vpnv4|ipv4|vpnv6} unicast summary.""" +"""ZTE: show bgp {afi} [unicast] [vrf ] summary (and l2vpn summaries).""" from __future__ import annotations import re -from typing import Any +from typing import Any, Mapping -_BGP_PEER_RE = re.compile( - r"^(?P\d{1,3}(?:\.\d{1,3}){3})\s+(?P\d+)\s+(?P\d+)\s+" +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm + +RULE_KEYS = ("zte_zxros_show_bgp_summary",) + +_PEER_LINE_RE = re.compile( + r"^(?P\d{1,3}(?:\.\d{1,3}){3}|[0-9A-Fa-f:]+)\s+" + r"(?P\d+)\s+(?P\S+)\s+" r"(?P\d+)\s+(?P\d+)\s+(?P\S+)\s+(?P\S+)\s*$", re.I, ) - -RULE_KEYS: tuple[str, ...] = () +_NEI_ONLY_RE = re.compile(r"^(?P[0-9A-Fa-f:]+)\s*$", re.I) +_CONT_RE = re.compile( + r"^\s+(?P\d+)\s+(?P\S+)\s+" + r"(?P\d+)\s+(?P\d+)\s+(?P\S+)\s+(?P\S+)\s*$", + re.I, +) +_VRF_IN_CMD_RE = re.compile(r"(?i)\bvrf\s+(\S+)") def _detect_bgp_afi(command: str, params: dict[str, str] | None) -> str: if params and params.get("afi"): return str(params.get("afi") or "").strip().lower() low = str(command or "").lower() + if "l2vpn" in low and "evpn" in low: + return "evpn" + if "l2vpn" in low and "vpls" in low: + return "vpls" if "vpnv6" in low: return "vpnv6" if "vpnv4" in low: @@ -29,55 +45,186 @@ def _detect_bgp_afi(command: str, params: dict[str, str] | None) -> str: return "unknown" +def _detect_vrf(command: str, params: dict[str, str] | None) -> str: + if params and params.get("vrf"): + return str(params.get("vrf") or "").strip() + m = _VRF_IN_CMD_RE.search(str(command or "")) + return m.group(1).strip() if m else "" + + +def _state_and_pfx(state_raw: str) -> tuple[str, str]: + raw = str(state_raw or "").strip() + if raw.isdigit(): + return "Established", raw + # e.g. (NoNeg) + return raw, "" + + +def _peer_row( + *, + afi: str, + vrf: str, + nei: str, + ver: str, + asn: str, + rx: str, + tx: str, + up: str, + state_raw: str, +) -> dict[str, Any]: + state, pfx = _state_and_pfx(state_raw) + return { + "afi": afi[:32], + "vrf": vrf[:128], + "neighbor": nei[:128], + "ver": ver[:8], + "as_num": asn[:16], + "msg_rcvd": rx[:32], + "msg_send": tx[:32], + "up_down": up[:32], + "state": state[:64], + "pfx_rcd": pfx[:32], + } + + +def _map_fsm_rows( + rows: list[dict[str, Any]], + *, + afi: str, + vrf: str, +) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + pending = "" + for r in rows: + nei = row_get(r, "NEIGHBOR", "neighbor") + ver = row_get(r, "VER", "ver") + if nei and not ver: + pending = nei + continue + if not nei and pending and ver: + nei = pending + pending = "" + if not nei or not ver: + continue + if nei in seen: + continue + seen.add(nei) + state_raw = row_get(r, "STATE_PFX", "STATE", "state", "pfx_rcd") + out.append( + _peer_row( + afi=afi, + vrf=vrf, + nei=nei, + ver=ver, + asn=row_get(r, "ASN", "AS", "as_num"), + rx=row_get(r, "MSG_RCVD", "msg_rcvd"), + tx=row_get(r, "MSG_SEND", "msg_send"), + up=row_get(r, "UP_DOWN", "up_down"), + state_raw=state_raw, + ) + ) + return out + + +def _hand_parse( + *, + raw_text: str, + afi: str = "unknown", + vrf: str = "", + **_kw: Any, +) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + pending = "" + for raw in str(raw_text or "").splitlines(): + line = raw.rstrip() + if not line.strip(): + continue + low = line.strip().lower() + if low.startswith("bgp router") or low.startswith("local as") or low.startswith("all "): + continue + if low.startswith("neighbor") and "msg" in low: + continue + m = _PEER_LINE_RE.match(line.strip()) + if m: + nei = m.group("nei") + if nei in seen: + continue + seen.add(nei) + pending = "" + out.append( + _peer_row( + afi=afi, + vrf=vrf, + nei=nei, + ver=m.group("ver"), + asn=m.group("asn"), + rx=m.group("rx"), + tx=m.group("tx"), + up=m.group("up"), + state_raw=m.group("state"), + ) + ) + continue + m_only = _NEI_ONLY_RE.match(line.strip()) + if m_only: + pending = m_only.group("nei") + continue + m_cont = _CONT_RE.match(line) + if m_cont and pending: + nei = pending + pending = "" + if nei in seen: + continue + seen.add(nei) + out.append( + _peer_row( + afi=afi, + vrf=vrf, + nei=nei, + ver=m_cont.group("ver"), + asn=m_cont.group("asn"), + rx=m_cont.group("rx"), + tx=m_cont.group("tx"), + up=m_cont.group("up"), + state_raw=m_cont.group("state"), + ) + ) + return out + + def normalize_bgp_peer( *, raw_text: str, - fsm_tables=None, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, vendor: str = "", device_type: str = "", command: str = "", params: dict[str, str] | None = None, ) -> list[dict[str, Any]]: - _ = (vendor, device_type, fsm_tables) afi = _detect_bgp_afi(command, params) - out: list[dict[str, Any]] = [] - seen: set[str] = set() - for raw in str(raw_text or "").splitlines(): - line = raw.strip() - if not line: - continue - low = line.lower() - if low.startswith("bgp router") or low.startswith("local as") or low.startswith("all "): - continue - if low.startswith("neighbor") and "msg" in low: - continue - m = _BGP_PEER_RE.match(line) - if not m: - continue - nei = m.group("nei") - if nei in seen: - continue - seen.add(nei) - state_raw = m.group("state") - if state_raw.isdigit(): - state = "Established" - pfx = state_raw - else: - state = state_raw - pfx = "" - out.append( - { - "afi": afi[:32], - "neighbor": nei[:64], - "ver": m.group("ver")[:8], - "as_num": m.group("asn")[:16], - "msg_rcvd": m.group("rx")[:32], - "msg_send": m.group("tx")[:32], - "up_down": m.group("up")[:32], - "state": state[:64], - "pfx_rcd": pfx[:32], - } + vrf = _detect_vrf(command, params) + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), ) - return out + cmd = str(command or "show bgp summary").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + + def _map(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return _map_fsm_rows(rows, afi=afi, vrf=vrf) + + def _hand(*, raw_text: str, **kw: Any) -> list[dict[str, Any]]: + return _hand_parse(raw_text=raw_text, afi=afi, vrf=vrf, **kw) + + return prefer_fsm(tables, RULE_KEYS, _map, _hand, raw_text=raw_text) + normalize_bgp_peer.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/bgp_route.py b/netx_api/biz_state/parsers/zte/bgp_route.py new file mode 100644 index 0000000..17483db --- /dev/null +++ b/netx_api/biz_state/parsers/zte/bgp_route.py @@ -0,0 +1,199 @@ +"""ZTE: show bgp … neighbor {in|out} [vrf] routes.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm +from .bgp_peer import _detect_bgp_afi, _detect_vrf + +RULE_KEYS = ("zte_zxros_show_bgp_neighbor_routes",) + +_ROUTE_RE = re.compile( + r"^\s*(?P[*<>isd]*)\s*" + r"(?P\d{1,3}(?:\.\d{1,3}){3}/\d+|[0-9A-Fa-f:]+(?:/\d+)?)\s+" + r"(?P\S+)\s+" + r"(?P.*)$" +) +_DIR_RE = re.compile(r"(?i)\bneighbor\s+(in|out)\s+") +_NEI_RE = re.compile(r"(?i)\bneighbor\s+(?:in|out)\s+(\S+)") + + +def _detect_direction(command: str, params: dict[str, str] | None) -> str: + if params and params.get("direction"): + return str(params.get("direction") or "").strip().lower() + m = _DIR_RE.search(str(command or "")) + return m.group(1).lower() if m else "" + + +def _detect_neighbor(command: str, params: dict[str, str] | None) -> str: + if params and params.get("neighbor"): + return str(params.get("neighbor") or "").strip() + m = _NEI_RE.search(str(command or "")) + return m.group(1).strip() if m else "" + + +def _split_rest(rest: str) -> tuple[str, str, str, str]: + """Parse trailing Metric LocPrf Tag/RtPrf Path columns (some may be blank).""" + parts = str(rest or "").split() + if not parts: + return "", "", "", "" + # Last token(s) are AS path + origin; path ends with i|e|? + path = " ".join(parts) + metric = loc = tag = "" + # Heuristic: numeric-only leading fields are metric/loc/tag when present + nums: list[str] = [] + path_parts: list[str] = [] + for p in parts: + if not path_parts and re.fullmatch(r"\d+", p): + nums.append(p) + else: + path_parts.append(p) + if len(nums) >= 1: + metric = nums[0] + if len(nums) >= 2: + loc = nums[1] + if len(nums) >= 3: + tag = nums[2] + path = " ".join(path_parts) + return metric, loc, tag, path + + +def _map_fsm_rows( + rows: list[dict[str, Any]], + *, + afi: str, + vrf: str, + neighbor: str, + direction: str, +) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for r in rows: + net = row_get(r, "NETWORK", "network") + if not net or net.lower() == "network" or net in seen: + continue + seen.add(net) + path = row_get(r, "PATH", "path") + out.append( + { + "afi": afi[:32], + "vrf": vrf[:128], + "neighbor": neighbor[:128], + "direction": direction[:8], + "network": net[:128], + "next_hop": row_get(r, "NEXT_HOP", "next_hop")[:128], + "metric": row_get(r, "METRIC", "metric")[:32], + "loc_prf": row_get(r, "LOC_PRF", "loc_prf")[:32], + "tag": row_get(r, "TAG", "RT_PRF", "tag")[:32], + "path": path[:256], + "status_codes": "", + "as_num": "", + "state": "", + "pfx_rcd": "", + } + ) + return out + + +def _hand_parse( + *, + raw_text: str, + afi: str = "unknown", + vrf: str = "", + neighbor: str = "", + direction: str = "", + **_kw: Any, +) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for raw in str(raw_text or "").splitlines(): + line = raw.rstrip() + if not line.strip(): + continue + low = line.strip().lower() + if low.startswith("network") or "next hop" in low: + continue + if low.startswith("status") or low.startswith("origin") or low.startswith("routes "): + continue + if low.startswith("local ") or low.startswith("remote ") or low.startswith("total "): + continue + if low.startswith("route distinguisher"): + continue + m = _ROUTE_RE.match(line) + if not m: + continue + net = m.group("net") + if net in seen: + continue + seen.add(net) + metric, loc, tag, path = _split_rest(m.group("rest")) + out.append( + { + "afi": afi[:32], + "vrf": vrf[:128], + "neighbor": neighbor[:128], + "direction": direction[:8], + "network": net[:128], + "next_hop": m.group("nh")[:128], + "metric": metric[:32], + "loc_prf": loc[:32], + "tag": tag[:32], + "path": path[:256], + "status_codes": (m.group("flags") or "").strip()[:16], + "as_num": "", + "state": "", + "pfx_rcd": "", + } + ) + return out + + +def normalize_bgp_route( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + afi = _detect_bgp_afi(command, params) + vrf = _detect_vrf(command, params) + neighbor = _detect_neighbor(command, params) + direction = _detect_direction(command, params) + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show bgp neighbor routes").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + + def _map(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return _map_fsm_rows( + rows, afi=afi, vrf=vrf, neighbor=neighbor, direction=direction + ) + + def _hand(*, raw_text: str, **kw: Any) -> list[dict[str, Any]]: + return _hand_parse( + raw_text=raw_text, + afi=afi, + vrf=vrf, + neighbor=neighbor, + direction=direction, + **kw, + ) + + return prefer_fsm(tables, RULE_KEYS, _map, _hand, raw_text=raw_text) + + +normalize_bgp_route.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/config_bgp_peer.py b/netx_api/biz_state/parsers/zte/config_bgp_peer.py new file mode 100644 index 0000000..1d15bee --- /dev/null +++ b/netx_api/biz_state/parsers/zte/config_bgp_peer.py @@ -0,0 +1,216 @@ +"""ZTE config intent: ``show running-config bgp`` / MIM ``!``. + +Emits one row per ``(afi, vrf, neighbor|peer_group)`` activation under +address-family. IP literals go in ``neighbor``; non-IP names (peer-groups) +go in ``peer_group``. Never captures password / secret lines. + +Route-maps under an address-family are scoped to that ``(afi, vrf)``; +global (top-level) route-maps apply to all AF rows for that neighbor and +are overlaid by AF-specific maps when both exist. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from .config_common import extract_mim_section, is_secret_line + +RULE_KEYS: tuple[str, ...] = () + +_NEI_REMOTE_RE = re.compile(r"^\s*neighbor\s+(\S+)\s+remote-as\s+(\S+)\s*$", re.I) +_NEI_UPD_RE = re.compile(r"^\s*neighbor\s+(\S+)\s+update-source\s+(\S+)\s*$", re.I) +_NEI_PG_RE = re.compile(r"^\s*neighbor\s+(\S+)\s+peer-group(?:\s+(\S+))?\s*$", re.I) +_NEI_RM_RE = re.compile( + r"^\s*neighbor\s+(\S+)\s+route-map\s+(\S+)\s+(in|out)\s*$", re.I +) +_NEI_ACT_RE = re.compile( + r"^\s*neighbor\s+(\S+)\s+activate(?:\s+(disable))?\s*$", re.I +) +_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$") + +_AfKey = tuple[str, str, str] # (afi, vrf, token) + + +def _is_ip_neighbor(token: str) -> bool: + """True for IPv4/IPv6 literals; false for peer-group names (even with ':').""" + tok = str(token or "").strip() + if not tok: + return False + if _IPV4_RE.match(tok): + return True + # IPv6 must look like an address, not ``asn:name`` peer-group labels. + if ":" not in tok: + return False + try: + import ipaddress + + ipaddress.ip_address(tok.split("%", 1)[0]) + return True + except ValueError: + return False + + +def _split_neighbor_cols(token: str, info: Mapping[str, str]) -> tuple[str, str]: + """Return ``(neighbor, peer_group)`` columns for a neighbor token.""" + tok = str(token or "").strip() + if _is_ip_neighbor(tok): + return tok, str(info.get("peer_group") or "") + # Non-IP token is a peer-group name + return "", tok + + +def _merge_info( + token: str, + *, + afi: str, + vrf: str, + meta: Mapping[str, dict[str, str]], + af_rm: Mapping[_AfKey, dict[str, str]], +) -> dict[str, str]: + """Global neighbor meta + AF-scoped route-maps (AF wins on conflict).""" + info = dict(meta.get(token) or {}) + scoped = af_rm.get((afi, vrf, token)) or {} + for key in ("route_map_in", "route_map_out"): + if scoped.get(key): + info[key] = scoped[key] + return info + + +def _row( + *, + afi: str, + vrf: str, + token: str, + act: str, + info: Mapping[str, str], +) -> dict[str, Any]: + neighbor, peer_group = _split_neighbor_cols(token, info) + return { + "afi": afi[:32], + "vrf": vrf[:128], + "neighbor": neighbor[:128], + "peer_group": peer_group[:64], + "remote_as": str(info.get("remote_as") or "")[:16], + "activate": act[:16], + "update_source": str(info.get("update_source") or "")[:64], + "route_map_in": str(info.get("route_map_in") or "")[:128], + "route_map_out": str(info.get("route_map_out") or "")[:128], + } + + +def normalize_config_bgp_peer( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = (fsm_tables, vendor, device_type, command, params) + body = extract_mim_section(raw_text, "bgp") + meta: dict[str, dict[str, str]] = {} + af_rm: dict[_AfKey, dict[str, str]] = {} + afi = "" + vrf = "" + activations: list[tuple[str, str, str, str]] = [] + # (afi, vrf, token, activate) + + for raw in body.splitlines(): + line = raw.rstrip() + if is_secret_line(line): + continue + m = _NEI_REMOTE_RE.match(line) + if m: + nei = m.group(1).strip() + meta.setdefault(nei, {}) + meta[nei]["remote_as"] = m.group(2).strip() + continue + m = _NEI_UPD_RE.match(line) + if m: + nei = m.group(1).strip() + meta.setdefault(nei, {}) + meta[nei]["update_source"] = m.group(2).strip() + continue + m = _NEI_PG_RE.match(line) + if m: + nei = m.group(1).strip() + meta.setdefault(nei, {}) + group = (m.group(2) or "").strip() + if group: + # neighbor peer-group + meta[nei]["peer_group"] = group + else: + # neighbor peer-group → peer-group definition + meta[nei]["is_group"] = "1" + continue + m = _NEI_RM_RE.match(line) + if m and not afi: + nei = m.group(1).strip() + meta.setdefault(nei, {}) + direction = m.group(3).lower() + key = "route_map_in" if direction == "in" else "route_map_out" + meta[nei][key] = m.group(2).strip() + continue + m = re.match(r"^\s*address-family\s+ipv4\s+vrf\s+(\S+)\s*$", line, re.I) + if m: + afi, vrf = "ipv4", m.group(1).strip() + continue + m = re.match(r"^\s*address-family\s+ipv6\s+vrf\s+(\S+)\s*$", line, re.I) + if m: + afi, vrf = "ipv6", m.group(1).strip() + continue + m = re.match(r"^\s*address-family\s+l2vpn\s+(\S+)\s*$", line, re.I) + if m: + afi, vrf = f"l2vpn-{m.group(1).lower()}", "" + continue + m = re.match(r"^\s*address-family\s+(\S+)\s*$", line, re.I) + if m: + afi, vrf = m.group(1).lower(), "" + continue + if re.match(r"^\s*\$\s*$", line) and afi: + if line.startswith(" ") and not line.startswith(" "): + afi, vrf = "", "" + continue + m = _NEI_ACT_RE.match(line) + if m and afi: + nei = m.group(1).strip() + act = "disable" if m.group(2) else "enable" + activations.append((afi, vrf, nei, act)) + continue + m = _NEI_RM_RE.match(line) + if m and afi: + nei = m.group(1).strip() + direction = m.group(3).lower() + rk = "route_map_in" if direction == "in" else "route_map_out" + slot = af_rm.setdefault((afi, vrf, nei), {}) + slot[rk] = m.group(2).strip() + + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str, str]] = set() + for afi_s, vrf_s, token, act in activations: + info = _merge_info(token, afi=afi_s, vrf=vrf_s, meta=meta, af_rm=af_rm) + row = _row(afi=afi_s, vrf=vrf_s, token=token, act=act, info=info) + key = (row["afi"], row["vrf"], row["neighbor"], row["peer_group"]) + if key in seen: + continue + seen.add(key) + out.append(row) + + # Global peers with remote-as but no AF activate + for token, info in meta.items(): + if not info.get("remote_as"): + continue + if any(n == token for _, _, n, _ in activations): + continue + row = _row(afi="global", vrf="", token=token, act="", info=info) + key = (row["afi"], row["vrf"], row["neighbor"], row["peer_group"]) + if key in seen: + continue + seen.add(key) + out.append(row) + return out + + +normalize_config_bgp_peer.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/config_common.py b/netx_api/biz_state/parsers/zte/config_common.py new file mode 100644 index 0000000..9c2816a --- /dev/null +++ b/netx_api/biz_state/parsers/zte/config_common.py @@ -0,0 +1,46 @@ +"""Helpers for ZTE MIM running-config section parsing.""" + +from __future__ import annotations + +import re + +_MIM_OPEN = re.compile(r"^!<([^/>][^>]*)>\s*$") +_MIM_CLOSE = re.compile(r"^!]+)>\s*$") + + +def extract_mim_section(raw_text: str, section: str) -> str: + """Return body of ``!
`` … ``!
`` if present; else original text.""" + name = str(section or "").strip().lower() + if not name: + return str(raw_text or "") + text = str(raw_text or "") + lines = text.splitlines() + start = -1 + for i, line in enumerate(lines): + m = _MIM_OPEN.match(line.strip()) + if m and m.group(1).strip().lower() == name: + start = i + 1 + break + if start < 0: + return text + out: list[str] = [] + for line in lines[start:]: + m = _MIM_CLOSE.match(line.strip()) + if m and m.group(1).strip().lower() == name: + break + out.append(line) + return "\n".join(out) + + +def is_secret_line(line: str) -> bool: + low = str(line or "").lower() + return any( + k in low + for k in ( + "password", + "secret", + "encrypted", + "authentication-key", + "pre-shared-key", + ) + ) diff --git a/netx_api/biz_state/parsers/zte/config_interface.py b/netx_api/biz_state/parsers/zte/config_interface.py new file mode 100644 index 0000000..6f2b394 --- /dev/null +++ b/netx_api/biz_state/parsers/zte/config_interface.py @@ -0,0 +1,148 @@ +"""ZTE config intent: ``show running-config if-intf`` / MIM ``!``. + +Duplicate ``interface`` blocks (common for tunnels) are merged: later values +override empty ones; ``admin`` always takes the last explicit setting. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from .config_common import extract_mim_section, is_secret_line + +RULE_KEYS: tuple[str, ...] = () + +_IFACE_RE = re.compile(r"^\s*interface\s+(\S+)\s*$", re.I) +_VRF_RE = re.compile(r"^\s*ip\s+vrf\s+forwarding\s+(\S+)\s*$", re.I) +_IP_RE = re.compile(r"^\s*ip\s+address\s+(\S+)(?:\s+(\S+))?\s*$", re.I) +_IP6_RE = re.compile(r"^\s*ipv6\s+address\s+(\S+)\s*$", re.I) +_DESC_RE = re.compile(r"^\s*description\s+(.*?)\s*$", re.I) +_MTU_RE = re.compile(r"^\s*mtu\s+(\d+)\s*$", re.I) + + +def _row_from_cur(cur: dict[str, Any]) -> dict[str, Any]: + ips = cur.get("_ips") or [] + ip6s = cur.get("_ip6s") or [] + return { + "interface": str(cur.get("interface") or "")[:128], + "vrf": str(cur.get("vrf") or "")[:128], + "description": str(cur.get("description") or "")[:256], + "admin": str(cur.get("admin") or "up")[:16], + "ip_address": ",".join(ips)[:256], + "ipv6_address": ",".join(ip6s)[:256], + "mtu": str(cur.get("mtu") or "")[:16], + "_admin_set": bool(cur.get("_admin_set")), + } + + +def _flush( + cur: dict[str, Any] | None, + out: list[dict[str, Any]], + by_iface: dict[str, int], +) -> None: + if not cur: + return + iface = str(cur.get("interface") or "") + if not iface: + return + row = _row_from_cur(cur) + admin_set = bool(row.pop("_admin_set", False)) + if iface in by_iface: + prev = out[by_iface[iface]] + # Only override admin when this block explicitly set shutdown / no shutdown. + if admin_set: + prev["admin"] = row["admin"] + for key in ("vrf", "description", "mtu"): + if row[key]: + prev[key] = row[key] + if row["ip_address"]: + existing = [x for x in (prev["ip_address"] or "").split(",") if x] + for ip in row["ip_address"].split(","): + if ip and ip not in existing: + existing.append(ip) + prev["ip_address"] = ",".join(existing)[:256] + if row["ipv6_address"]: + existing = [x for x in (prev["ipv6_address"] or "").split(",") if x] + for ip in row["ipv6_address"].split(","): + if ip and ip not in existing: + existing.append(ip) + prev["ipv6_address"] = ",".join(existing)[:256] + return + by_iface[iface] = len(out) + out.append(row) + + +def normalize_config_interface( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = (fsm_tables, vendor, device_type, command, params) + body = extract_mim_section(raw_text, "if-intf") + out: list[dict[str, Any]] = [] + by_iface: dict[str, int] = {} + cur: dict[str, Any] | None = None + for raw in body.splitlines(): + line = raw.rstrip() + if is_secret_line(line): + continue + m = _IFACE_RE.match(line) + if m: + _flush(cur, out, by_iface) + cur = { + "interface": m.group(1).strip(), + "vrf": "", + "description": "", + "admin": "up", + "_admin_set": False, + "mtu": "", + "_ips": [], + "_ip6s": [], + } + continue + if not cur: + continue + if re.match(r"^\$\s*$", line): + _flush(cur, out, by_iface) + cur = None + continue + low = line.strip().lower() + if low == "shutdown": + cur["admin"] = "down" + cur["_admin_set"] = True + continue + if low == "no shutdown": + cur["admin"] = "up" + cur["_admin_set"] = True + continue + m = _VRF_RE.match(line) + if m: + cur["vrf"] = m.group(1).strip() + continue + m = _DESC_RE.match(line) + if m: + cur["description"] = (m.group(1) or "").strip().strip('"')[:256] + continue + m = _IP_RE.match(line) + if m: + addr = m.group(1).strip() + mask = (m.group(2) or "").strip() + cur["_ips"].append(f"{addr}/{mask}" if mask else addr) + continue + m = _IP6_RE.match(line) + if m: + cur["_ip6s"].append(m.group(1).strip()) + continue + m = _MTU_RE.match(line) + if m: + cur["mtu"] = m.group(1) + _flush(cur, out, by_iface) + return out + + +normalize_config_interface.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/config_isis.py b/netx_api/biz_state/parsers/zte/config_isis.py new file mode 100644 index 0000000..dd3cf78 --- /dev/null +++ b/netx_api/biz_state/parsers/zte/config_isis.py @@ -0,0 +1,210 @@ +"""ZTE config intent: IS-IS from ``!`` / ``show running-config isis``. + +One row per ``(process, vrf, interface)``. Process-level attrs (area, system-id, +router-id, is-type) are copied onto each interface row. +Never captures hello-authentication / encrypted secrets. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from .config_common import extract_mim_section, is_secret_line + +RULE_KEYS: tuple[str, ...] = () + +_PROC_RE = re.compile( + r"(?i)^\s*router\s+isis\s+(\S+)(?:\s+vrf\s+(\S+))?\s*$" +) +_AREA_RE = re.compile(r"(?i)^\s*area\s+(\S+)\s*$") +_SYS_RE = re.compile(r"(?i)^\s*system-id\s+(\S+)\s*$") +_RID_RE = re.compile(r"(?i)^\s*router-id\s+(\S+)\s*$") +_ISTYPE_RE = re.compile(r"(?i)^\s*is-type\s+(\S+)\s*$") +_IFACE_RE = re.compile(r"(?i)^\s*interface\s+(\S+)\s*$") +_CIRCUIT_RE = re.compile(r"(?i)^\s*circuit-type\s+(\S+)\s*$") +_NET_RE = re.compile(r"(?i)^\s*network\s+(\S+)\s*$") +_METRIC_RE = re.compile(r"(?i)^\s*metric\s+(\d+)\s*$") +_V6_METRIC_RE = re.compile(r"(?i)^\s*ipv6\s+metric\s+(\d+)\s*$") +_DOLLAR_RE = re.compile(r"^(\s*)\$\s*$") + + +def _indent(spaces: str) -> int: + return len(spaces.expandtabs(2)) + + +def _flush( + proc: dict[str, Any] | None, + iface: dict[str, Any] | None, + out: list[dict[str, Any]], + seen: set[tuple[str, ...]], +) -> None: + if not proc or not iface: + return + key = (proc["process_id"], proc["vrf"], iface["interface"]) + if key in seen: + return + seen.add(key) + out.append( + { + "process_id": str(proc["process_id"])[:32], + "vrf": str(proc.get("vrf") or "")[:128], + "area": str(proc.get("area") or "")[:64], + "system_id": str(proc.get("system_id") or "")[:64], + "router_id": str(proc.get("router_id") or "")[:64], + "is_type": str(proc.get("is_type") or "")[:32], + "interface": str(iface.get("interface") or "")[:128], + "circuit_type": str(iface.get("circuit_type") or "")[:32], + "network_type": str(iface.get("network_type") or "")[:32], + "ip_enable": str(iface.get("ip_enable") or "")[:8], + "ipv6_enable": str(iface.get("ipv6_enable") or "")[:8], + "metric": str(iface.get("metric") or "")[:16], + "ipv6_metric": str(iface.get("ipv6_metric") or "")[:16], + "passive": str(iface.get("passive") or "")[:8], + "bfd": str(iface.get("bfd") or "")[:16], + "ipv6_bfd": str(iface.get("ipv6_bfd") or "")[:16], + } + ) + + +def normalize_config_isis( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = (fsm_tables, vendor, device_type, command, params) + body = extract_mim_section(raw_text, "isis") + out: list[dict[str, Any]] = [] + seen: set[tuple[str, ...]] = set() + proc: dict[str, Any] | None = None + iface: dict[str, Any] | None = None + in_af = False + af_indent = 0 + + for raw in body.splitlines(): + line = raw.rstrip() + if is_secret_line(line): + continue + m = _PROC_RE.match(line) + if m: + _flush(proc, iface, out, seen) + iface = None + in_af = False + af_indent = 0 + proc = { + "process_id": m.group(1).strip(), + "vrf": (m.group(2) or "").strip(), + "area": "", + "system_id": "", + "router_id": "", + "is_type": "", + } + continue + if not proc: + continue + + # Skip address-family subtree (nested $ must not close the process). + if in_af: + m = _DOLLAR_RE.match(line) + if m and _indent(m.group(1)) <= af_indent: + in_af = False + af_indent = 0 + continue + + if re.match(r"(?i)^\s*address-family\s+", line): + in_af = True + af_indent = len(line) - len(line.lstrip(" \t")) + continue + + m = _DOLLAR_RE.match(line) + if m: + ind = _indent(m.group(1)) + if iface is not None and ind >= 2: + _flush(proc, iface, out, seen) + iface = None + continue + _flush(proc, iface, out, seen) + iface = None + proc = None + continue + + if iface is not None: + m = _CIRCUIT_RE.match(line) + if m: + iface["circuit_type"] = m.group(1).lower() + continue + m = _NET_RE.match(line) + if m: + iface["network_type"] = m.group(1).lower() + continue + m = _V6_METRIC_RE.match(line) + if m: + iface["ipv6_metric"] = m.group(1) + continue + m = _METRIC_RE.match(line) + if m: + iface["metric"] = m.group(1) + continue + low = line.strip().lower() + if low == "ip router isis" or low.startswith("ip router isis "): + iface["ip_enable"] = "yes" + continue + if low == "ipv6 router isis" or low.startswith("ipv6 router isis "): + iface["ipv6_enable"] = "yes" + continue + if low == "passive-mode" or low == "passive": + iface["passive"] = "yes" + continue + if low == "bfd-enable" or low.startswith("bfd-enable"): + iface["bfd"] = "enable" + continue + if low == "ipv6 bfd-enable" or low.startswith("ipv6 bfd-enable"): + iface["ipv6_bfd"] = "enable" + continue + continue + + m = _IFACE_RE.match(line) + if m: + _flush(proc, iface, out, seen) + iface = { + "interface": m.group(1).strip(), + "circuit_type": "", + "network_type": "", + "ip_enable": "", + "ipv6_enable": "", + "metric": "", + "ipv6_metric": "", + "passive": "", + "bfd": "", + "ipv6_bfd": "", + } + continue + m = _AREA_RE.match(line) + if m: + proc["area"] = m.group(1).strip() + continue + m = _SYS_RE.match(line) + if m: + proc["system_id"] = m.group(1).strip() + continue + m = _RID_RE.match(line) + if m: + # prefer IPv4-looking router-id at process level + rid = m.group(1).strip() + if ":" not in rid or not proc.get("router_id"): + proc["router_id"] = rid + continue + m = _ISTYPE_RE.match(line) + if m: + proc["is_type"] = m.group(1).lower() + continue + + _flush(proc, iface, out, seen) + return out + + +normalize_config_isis.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/config_l2vpn_pw.py b/netx_api/biz_state/parsers/zte/config_l2vpn_pw.py new file mode 100644 index 0000000..a6fd518 --- /dev/null +++ b/netx_api/biz_state/parsers/zte/config_l2vpn_pw.py @@ -0,0 +1,140 @@ +"""ZTE config intent: ``show running-config l2vpn`` / MIM ``!``. + +One row per ``pseudo-wire`` / ``backup-pw`` under ``vpws`` / ``vpls``. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from .config_common import extract_mim_section, is_secret_line + +RULE_KEYS: tuple[str, ...] = () + +_VPN_RE = re.compile(r'^\s*(vpws|vpls)\s+("?)([^"\s]+)\2\s*$', re.I) +_DESC_RE = re.compile(r"^\s*description\s+(.*?)\s*$", re.I) +_AP_RE = re.compile(r"^\s*access-point\s+(\S+)\s*$", re.I) +_PW_RE = re.compile(r"^\s*pseudo-wire\s+(\S+)", re.I) +_BACKUP_PW_RE = re.compile(r"^\s*backup-pw\s+(\S+)", re.I) +_NEI_RE = re.compile( + r"^\s*neighbou?r\s+(\S+)\s+vcid\s+(\S+)\s*$", + re.I, +) +_ENC_RE = re.compile(r"^\s*encapsulation\s+(\S+)\s*$", re.I) + + +def _flush_pw( + cur_vpn: dict[str, Any] | None, + cur_pw: dict[str, Any] | None, + out: list[dict[str, Any]], + seen: set[tuple[str, str]], +) -> None: + if not cur_vpn or not cur_pw: + return + pw = str(cur_pw.get("pw_name") or "") + vpn = str(cur_vpn.get("vpn_name") or "") + key = (vpn, pw) + if not pw or not vpn or key in seen: + return + seen.add(key) + out.append( + { + "vpn_type": str(cur_vpn.get("vpn_type") or "")[:16], + "vpn_name": vpn[:128], + "pw_name": pw[:128], + "peer": str(cur_pw.get("peer") or "")[:64], + "vcid": str(cur_pw.get("vcid") or "")[:64], + "encapsulation": str(cur_pw.get("encapsulation") or "")[:32], + "access_point": str(cur_vpn.get("access_point") or "")[:128], + "description": str(cur_vpn.get("description") or "")[:256], + } + ) + + +def _start_pw( + cur_vpn: dict[str, Any] | None, + cur_pw: dict[str, Any] | None, + out: list[dict[str, Any]], + seen: set[tuple[str, str]], + pw_name: str, +) -> dict[str, Any]: + _flush_pw(cur_vpn, cur_pw, out, seen) + return {"pw_name": pw_name.strip(), "peer": "", "vcid": "", "encapsulation": ""} + + +def normalize_config_l2vpn_pw( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = (fsm_tables, vendor, device_type, command, params) + body = extract_mim_section(raw_text, "l2vpn") + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + cur_vpn: dict[str, Any] | None = None + cur_pw: dict[str, Any] | None = None + for raw in body.splitlines(): + line = raw.rstrip() + if is_secret_line(line): + continue + m = _VPN_RE.match(line) + if m: + _flush_pw(cur_vpn, cur_pw, out, seen) + cur_pw = None + cur_vpn = { + "vpn_type": m.group(1).lower(), + "vpn_name": m.group(3).strip(), + "description": "", + "access_point": "", + } + continue + if not cur_vpn: + continue + if re.match(r"^\$\s*$", line): + _flush_pw(cur_vpn, cur_pw, out, seen) + cur_pw = None + cur_vpn = None + continue + m = _DESC_RE.match(line) + if m and cur_pw is None: + cur_vpn["description"] = (m.group(1) or "").strip().strip('"')[:256] + continue + m = _AP_RE.match(line) + if m: + cur_vpn["access_point"] = m.group(1).strip() + continue + m = _PW_RE.match(line) + if m: + cur_pw = _start_pw(cur_vpn, cur_pw, out, seen, m.group(1)) + continue + m = _BACKUP_PW_RE.match(line) + if m: + cur_pw = _start_pw(cur_vpn, cur_pw, out, seen, m.group(1)) + continue + # indented $ closes nested block; flush PW once peer is known so + # backup-pw / redundancy neighbours do not overwrite the primary. + if re.match(r"^\s+\$\s*$", line): + if cur_pw and cur_pw.get("peer"): + _flush_pw(cur_vpn, cur_pw, out, seen) + cur_pw = None + continue + if not cur_pw: + continue + m = _NEI_RE.match(line) + if m: + cur_pw["peer"] = m.group(1).strip() + cur_pw["vcid"] = m.group(2).strip() + continue + m = _ENC_RE.match(line) + if m: + cur_pw["encapsulation"] = m.group(1).strip() + _flush_pw(cur_vpn, cur_pw, out, seen) + return out + + +normalize_config_l2vpn_pw.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/config_ospf.py b/netx_api/biz_state/parsers/zte/config_ospf.py new file mode 100644 index 0000000..c3a2877 --- /dev/null +++ b/netx_api/biz_state/parsers/zte/config_ospf.py @@ -0,0 +1,267 @@ +"""ZTE config intent: OSPFv2/v3 from ``!`` / ``!``. + +One row per ``(af, process, vrf, area, interface)``. Areas with no interfaces +still emit a row with empty ``interface``. Redistribute is process-scoped and +applied when the process block closes (it often appears after areas). +Never captures digest keys / encrypted auth material. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from .config_common import extract_mim_section, is_secret_line + +RULE_KEYS: tuple[str, ...] = () + +_V2_RE = re.compile(r"(?i)^\s*router\s+ospf\s+(\S+)(?:\s+vrf\s+(\S+))?\s*$") +_V3_RE = re.compile(r"(?i)^\s*ipv6\s+router\s+ospf\s+(\S+)(?:\s+vrf\s+(\S+))?\s*$") +_RID_RE = re.compile(r"(?i)^\s*router-id\s+(\S+)\s*$") +_AREA_RE = re.compile(r"(?i)^\s*area\s+(\S+)\s*$") +_IFACE_RE = re.compile(r"(?i)^\s*interface\s+(\S+)\s*$") +_COST_RE = re.compile(r"(?i)^\s*cost\s+(\d+)\s*$") +_HELLO_RE = re.compile(r"(?i)^\s*hello-interval\s+(\d+)\s*$") +_DEAD_RE = re.compile(r"(?i)^\s*dead-interval\s+(\d+)\s*$") +_NET_RE = re.compile(r"(?i)^\s*network\s+(\S+)\s*$") +_REDIST_RE = re.compile(r"(?i)^\s*redistribute\s+(\S+)") +_DOLLAR_RE = re.compile(r"^(\s*)\$\s*$") + + +def _indent(spaces: str) -> int: + return len(spaces.expandtabs(2)) + + +def _queue_iface( + proc: dict[str, Any], + area: dict[str, Any], + iface: dict[str, Any], +) -> None: + proc.setdefault("_pending", []).append( + { + "area": str(area.get("area") or ""), + "area_type": str(area.get("area_type") or "normal"), + "interface": str(iface.get("interface") or ""), + "network_type": str(iface.get("network_type") or ""), + "cost": str(iface.get("cost") or ""), + "hello_interval": str(iface.get("hello_interval") or ""), + "dead_interval": str(iface.get("dead_interval") or ""), + "bfd": str(iface.get("bfd") or ""), + } + ) + + +def _emit_process( + proc: dict[str, Any] | None, + out: list[dict[str, Any]], + seen: set[tuple[str, ...]], +) -> None: + if not proc: + return + redist = ",".join(sorted(proc.get("_redistribute") or []))[:256] + for item in proc.get("_pending") or []: + key = ( + proc["af"], + proc["process_id"], + proc["vrf"], + item["area"], + item["interface"], + ) + if key in seen: + continue + seen.add(key) + out.append( + { + "af": proc["af"][:8], + "process_id": str(proc["process_id"])[:32], + "vrf": str(proc.get("vrf") or "")[:128], + "router_id": str(proc.get("router_id") or "")[:64], + "area": item["area"][:64], + "area_type": item["area_type"][:32], + "interface": item["interface"][:128], + "network_type": item["network_type"][:32], + "cost": item["cost"][:16], + "hello_interval": item["hello_interval"][:16], + "dead_interval": item["dead_interval"][:16], + "bfd": item["bfd"][:16], + "redistribute": redist, + } + ) + + +def _close_area(proc: dict[str, Any] | None, area: dict[str, Any] | None) -> None: + if not proc or not area: + return + if int(area.get("_iface_count") or 0) == 0: + _queue_iface( + proc, + area, + { + "interface": "", + "network_type": "", + "cost": "", + "hello_interval": "", + "dead_interval": "", + "bfd": "", + }, + ) + + +def _parse_ospf_body(body: str, *, af: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, ...]] = set() + proc: dict[str, Any] | None = None + area: dict[str, Any] | None = None + iface: dict[str, Any] | None = None + head_re = _V3_RE if af == "ipv6" else _V2_RE + + for raw in body.splitlines(): + line = raw.rstrip() + if is_secret_line(line): + continue + m = head_re.match(line) + if m: + if iface and proc and area: + _queue_iface(proc, area, iface) + _close_area(proc, area) + _emit_process(proc, out, seen) + iface = None + area = None + proc = { + "af": af, + "process_id": m.group(1).strip(), + "vrf": (m.group(2) or "").strip(), + "router_id": "", + "_redistribute": set(), + "_pending": [], + } + continue + if not proc: + continue + + m = _DOLLAR_RE.match(line) + if m: + ind = _indent(m.group(1)) + if iface is not None and ind >= 4: + _queue_iface(proc, area, iface) # type: ignore[arg-type] + iface = None + elif area is not None and ind >= 2: + if iface and area: + _queue_iface(proc, area, iface) + iface = None + _close_area(proc, area) + area = None + else: + if iface and area: + _queue_iface(proc, area, iface) + iface = None + _close_area(proc, area) + area = None + _emit_process(proc, out, seen) + proc = None + continue + + if iface is not None: + m = _COST_RE.match(line) + if m: + iface["cost"] = m.group(1) + continue + m = _HELLO_RE.match(line) + if m: + iface["hello_interval"] = m.group(1) + continue + m = _DEAD_RE.match(line) + if m: + iface["dead_interval"] = m.group(1) + continue + m = _NET_RE.match(line) + if m: + iface["network_type"] = m.group(1).lower() + continue + low = line.strip().lower() + if low == "bfd" or low.startswith("bfd interval") or low.startswith("bfd enable"): + iface["bfd"] = "enable" + continue + if low == "bfd disable": + iface["bfd"] = "disable" + continue + continue + + m = _AREA_RE.match(line) + if m: + if iface and area: + _queue_iface(proc, area, iface) + iface = None + _close_area(proc, area) + area = { + "area": m.group(1).strip(), + "area_type": "normal", + "_iface_count": 0, + } + continue + + if area is not None: + m = _IFACE_RE.match(line) + if m: + if iface: + _queue_iface(proc, area, iface) + iface = { + "interface": m.group(1).strip(), + "network_type": "", + "cost": "", + "hello_interval": "", + "dead_interval": "", + "bfd": "", + } + area["_iface_count"] = int(area.get("_iface_count") or 0) + 1 + continue + low = line.strip().lower() + if low.startswith("stub"): + area["area_type"] = "stub" + continue + if low.startswith("nssa"): + area["area_type"] = "nssa" + continue + + # process-level (also after areas closed) + if iface is None: + m = _RID_RE.match(line) + if m: + proc["router_id"] = m.group(1).strip() + continue + m = _REDIST_RE.match(line) + if m: + proc["_redistribute"].add(m.group(1).lower()) + continue + + if iface and proc and area: + _queue_iface(proc, area, iface) + _close_area(proc, area) + _emit_process(proc, out, seen) + return out + + +def normalize_config_ospf( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = (fsm_tables, vendor, device_type, params) + cmd = str(command or "").lower() + text = str(raw_text or "") + if "ospfv3" in cmd: + return _parse_ospf_body(extract_mim_section(text, "ospfv3"), af="ipv6") + if "ospfv2" in cmd or re.search(r"(?i)running-config\s+ospf(?:v2)?\b", cmd): + return _parse_ospf_body(extract_mim_section(text, "ospfv2"), af="ipv4") + if re.search(r"(?i)^!\s*$", text, re.M) and not re.search( + r"(?i)^!\s*$", text, re.M + ): + return _parse_ospf_body(extract_mim_section(text, "ospfv3"), af="ipv6") + return _parse_ospf_body(extract_mim_section(text, "ospfv2"), af="ipv4") + + +normalize_config_ospf.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/config_static_route.py b/netx_api/biz_state/parsers/zte/config_static_route.py new file mode 100644 index 0000000..47b13fe --- /dev/null +++ b/netx_api/biz_state/parsers/zte/config_static_route.py @@ -0,0 +1,283 @@ +"""ZTE config intent: static routes from ``!`` / ``!``. + +Commands: + ``show running-config static`` + ``show running-config ipv6-static-route`` +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from .config_common import extract_mim_section, is_secret_line + +RULE_KEYS: tuple[str, ...] = () + +_V4_HEAD = re.compile( + r"(?i)^\s*ip\s+route(?:\s+vrf\s+(?P\S+))?\s+" + r"(?P\d{1,3}(?:\.\d{1,3}){3})\s+" + r"(?P\d{1,3}(?:\.\d{1,3}){3})\s+" + r"(?P.*)$" +) +_V6_HEAD = re.compile( + r"(?i)^\s*ipv6\s+route(?:\s+vrf\s+(?P\S+))?\s+" + r"(?P\S+)\s+" + r"(?P.*)$" +) + +_IPV4_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$") +_IFACE_RE = re.compile( + r"(?i)^(null\d*|gei-\S+|xgei-\S+|cgei-\S+|bvi\S*|vlan\S*|smartgroup\S*|" + r"loopback\S*|mgmt\S*|pos\S*|eth\S*|tunnel\S*|irb\S*|ve\S*)$" +) +_KW = frozenset( + { + "name", + "track", + "tag", + "metric", + "distance", + "preference", + "nexthop-vrf", + "bfd", + "enable", + "disable", + "permanent", + } +) + + +def _is_ipv4(tok: str) -> bool: + return bool(_IPV4_RE.match(tok or "")) + + +def _is_iface(tok: str) -> bool: + return bool(_IFACE_RE.match(tok or "")) + + +def _take_kw(tokens: list[str], i: int, row: dict[str, str]) -> int: + """Consume optional keyword/value pairs starting at ``i``; return new index.""" + n = len(tokens) + while i < n: + low = tokens[i].lower() + if low in ("bfd",): + # bfd [enable|disable] + row["bfd"] = "enable" + i += 1 + if i < n and tokens[i].lower() in ("enable", "disable"): + row["bfd"] = tokens[i].lower() + i += 1 + continue + if low in ("enable", "disable", "permanent"): + if low == "permanent": + row["permanent"] = "yes" + i += 1 + continue + if low in ("name", "track", "tag", "metric", "distance", "preference", "nexthop-vrf"): + if i + 1 >= n: + break + val = tokens[i + 1] + if low == "nexthop-vrf": + row["nexthop_vrf"] = val + elif low == "name": + row["route_name"] = val + else: + row[low] = val + i += 2 + continue + break + return i + + +def _parse_v4_rest(rest: str) -> dict[str, str]: + tokens = (rest or "").split() + row: dict[str, str] = { + "next_hop": "", + "interface": "", + "nexthop_vrf": "", + "metric": "", + "track": "", + "tag": "", + "route_name": "", + "bfd": "", + "distance": "", + } + if not tokens: + return row + i = 0 + # nexthop-vrf without explicit IP + if tokens[0].lower() == "nexthop-vrf" and len(tokens) >= 2: + row["nexthop_vrf"] = tokens[1] + _take_kw(tokens, 2, row) + return row + if _is_iface(tokens[0]): + row["interface"] = tokens[0] + i = 1 + if i < len(tokens) and _is_ipv4(tokens[i]) and tokens[i].lower() not in _KW: + row["next_hop"] = tokens[i] + i += 1 + elif _is_ipv4(tokens[0]): + row["next_hop"] = tokens[0] + i = 1 + _take_kw(tokens, i, row) + return row + + +def _parse_v6_rest(rest: str) -> dict[str, str]: + tokens = (rest or "").split() + row: dict[str, str] = { + "next_hop": "", + "interface": "", + "nexthop_vrf": "", + "metric": "", + "track": "", + "tag": "", + "route_name": "", + "bfd": "", + "distance": "", + } + if not tokens: + return row + i = 0 + if tokens[0].lower() == "nexthop-vrf" and len(tokens) >= 2: + row["nexthop_vrf"] = tokens[1] + _take_kw(tokens, 2, row) + return row + if _is_iface(tokens[0]): + row["interface"] = tokens[0] + i = 1 + if i < len(tokens) and ":" in tokens[i] and tokens[i].lower() not in _KW: + row["next_hop"] = tokens[i] + i += 1 + elif ":" in tokens[0]: + row["next_hop"] = tokens[0] + i = 1 + _take_kw(tokens, i, row) + return row + + +def _row( + *, + af: str, + vrf: str, + prefix: str, + mask: str, + extra: dict[str, str], +) -> dict[str, Any]: + return { + "af": af[:8], + "vrf": (vrf or "")[:128], + "prefix": prefix[:128], + "mask": (mask or "")[:64], + "next_hop": (extra.get("next_hop") or "")[:64], + "interface": (extra.get("interface") or "")[:128], + "nexthop_vrf": (extra.get("nexthop_vrf") or "")[:128], + "metric": (extra.get("metric") or "")[:16], + "bfd": (extra.get("bfd") or "")[:16], + "track": (extra.get("track") or "")[:64], + "route_name": (extra.get("route_name") or "")[:128], + "tag": (extra.get("tag") or "")[:32], + "distance": (extra.get("distance") or extra.get("preference") or "")[:16], + } + + +def _parse_v4(body: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, ...]] = set() + for raw in body.splitlines(): + line = raw.rstrip() + if is_secret_line(line): + continue + m = _V4_HEAD.match(line) + if not m: + continue + extra = _parse_v4_rest(m.group("rest") or "") + row = _row( + af="ipv4", + vrf=(m.group("vrf") or "").strip(), + prefix=m.group("prefix").strip(), + mask=m.group("mask").strip(), + extra=extra, + ) + key = ( + row["vrf"], + row["prefix"], + row["mask"], + row["next_hop"], + row["interface"], + row["nexthop_vrf"], + ) + if key in seen: + continue + seen.add(key) + out.append(row) + return out + + +def _parse_v6(body: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, ...]] = set() + for raw in body.splitlines(): + line = raw.rstrip() + if is_secret_line(line): + continue + m = _V6_HEAD.match(line) + if not m: + continue + extra = _parse_v6_rest(m.group("rest") or "") + row = _row( + af="ipv6", + vrf=(m.group("vrf") or "").strip(), + prefix=m.group("prefix").strip(), + mask="", + extra=extra, + ) + key = ( + row["vrf"], + row["prefix"], + row["next_hop"], + row["interface"], + row["nexthop_vrf"], + ) + if key in seen: + continue + seen.add(key) + out.append(row) + return out + + +def _detect_af(command: str, raw_text: str) -> str: + cmd = str(command or "").lower() + if "ipv6-static" in cmd or "ipv6 static" in cmd: + return "ipv6" + if re.search(r"(?i)^\s*show\s+running-config\s+static\b", cmd): + return "ipv4" + # Full dump / ambiguous: prefer section presence + text = str(raw_text or "") + if re.search(r"(?i)^!\s*$", text, re.M) and not re.search( + r"(?i)^!\s*$", text, re.M + ): + return "ipv6" + return "ipv4" + + +def normalize_config_static_route( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = (fsm_tables, vendor, device_type, params) + af = _detect_af(command, raw_text) + if af == "ipv6": + body = extract_mim_section(raw_text, "ipv6-static-route") + return _parse_v6(body) + body = extract_mim_section(raw_text, "static") + return _parse_v4(body) + + +normalize_config_static_route.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/config_vrf.py b/netx_api/biz_state/parsers/zte/config_vrf.py new file mode 100644 index 0000000..261ddf3 --- /dev/null +++ b/netx_api/biz_state/parsers/zte/config_vrf.py @@ -0,0 +1,107 @@ +"""ZTE config intent: ``show running-config vrf`` / MIM ``!``.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from .config_common import extract_mim_section, is_secret_line + +RULE_KEYS: tuple[str, ...] = () + +_VRF_RE = re.compile(r"^\s*ip\s+vrf\s+(\S+)\s*$", re.I) +_RD_RE = re.compile(r"^\s*rd\s+(\S+)\s*$", re.I) +_DESC_RE = re.compile(r'^\s*description\s+("?)(.*?)\1\s*$', re.I) +_AF_RE = re.compile(r"^\s*address-family\s+(ipv4|ipv6)\s*$", re.I) +_RT_RE = re.compile(r"^\s*route-target\s+(export|import)\s+(\S+)\s*$", re.I) + + +def _flush(cur: dict[str, Any] | None, out: list[dict[str, Any]], seen: set[str]) -> None: + if not cur: + return + name = str(cur.get("vrf_name") or "") + if not name or name in seen: + return + seen.add(name) + afs = sorted(cur.get("_afs") or []) + exports = sorted(cur.get("_rt_export") or []) + imports = sorted(cur.get("_rt_import") or []) + out.append( + { + "vrf_name": name[:128], + "rd": str(cur.get("rd") or "")[:64], + "description": str(cur.get("description") or "")[:256], + "address_families": ",".join(afs)[:64], + "rt_export": ",".join(exports)[:512], + "rt_import": ",".join(imports)[:512], + } + ) + + +def normalize_config_vrf( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = (fsm_tables, vendor, device_type, command, params) + body = extract_mim_section(raw_text, "vrf") + out: list[dict[str, Any]] = [] + seen: set[str] = set() + cur: dict[str, Any] | None = None + for raw in body.splitlines(): + line = raw.rstrip() + if is_secret_line(line): + continue + m = _VRF_RE.match(line) + if m: + _flush(cur, out, seen) + cur = { + "vrf_name": m.group(1).strip(), + "rd": "", + "description": "", + "_afs": set(), + "_rt_export": set(), + "_rt_import": set(), + } + continue + if not cur: + continue + if line.strip() == "$" and not line.startswith(" "): + # top-level close for vrf block (indent 0 $) + if not line.startswith(" ") and not line.startswith("\t"): + _flush(cur, out, seen) + cur = None + continue + # ZTE uses bare `$` at column 0 to end blocks; nested `$` are indented + if re.match(r"^\$\s*$", line): + _flush(cur, out, seen) + cur = None + continue + m = _RD_RE.match(line) + if m: + cur["rd"] = m.group(1).strip() + continue + m = _DESC_RE.match(line) + if m: + cur["description"] = (m.group(2) or "").strip()[:256] + continue + m = _AF_RE.match(line) + if m: + cur["_afs"].add(m.group(1).lower()) + continue + m = _RT_RE.match(line) + if m: + direction, value = m.group(1).lower(), m.group(2).strip() + if direction == "export": + cur["_rt_export"].add(value) + else: + cur["_rt_import"].add(value) + _flush(cur, out, seen) + return out + + +normalize_config_vrf.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/evpn_mac.py b/netx_api/biz_state/parsers/zte/evpn_mac.py new file mode 100644 index 0000000..08cecaa --- /dev/null +++ b/netx_api/biz_state/parsers/zte/evpn_mac.py @@ -0,0 +1,119 @@ +"""ZTE: show bgp evpn mac.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm + +RULE_KEYS = ("zte_zxros_show_bgp_evpn_mac",) + +_ROUTE_RE = re.compile( + r"^\s*(?P[*>isd]*)\s*" + r"(?P\S+)\s+(?P\S+)\s+(?P.*)$" +) + + +def _map_fsm_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for r in rows: + net = row_get(r, "NETWORK", "network") + if not net or net.lower() == "network" or net in seen: + continue + seen.add(net) + out.append( + { + "network": net[:256], + "next_hop": row_get(r, "NEXT_HOP", "next_hop")[:128], + "metric": row_get(r, "METRIC", "metric")[:32], + "loc_prf": row_get(r, "LOC_PRF", "loc_prf")[:32], + "rt_prf": row_get(r, "RT_PRF", "rt_prf")[:32], + "path": row_get(r, "PATH", "path")[:256], + "status_codes": "", + } + ) + return out + + +def _hand_parse(*, raw_text: str, **_kw: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for raw in str(raw_text or "").splitlines(): + line = raw.rstrip() + if not line.strip(): + continue + low = line.strip().lower() + if low.startswith("network") or low.startswith("status") or low.startswith("origin"): + continue + if low.startswith("route distinguisher"): + continue + m = _ROUTE_RE.match(line) + if not m: + continue + net = m.group("net") + if "/" not in net and ":" not in net and not re.search(r"[0-9a-fA-F]{4}\.", net): + # skip non-NLRI-looking tokens + if not re.match(r"^\[", net): + continue + if net in seen: + continue + seen.add(net) + rest = m.group("rest").split() + metric = loc = rt = "" + path_parts: list[str] = [] + for p in rest: + if not path_parts and re.fullmatch(r"\d+", p): + if not metric: + metric = p + elif not loc: + loc = p + elif not rt: + rt = p + else: + path_parts.append(p) + else: + path_parts.append(p) + out.append( + { + "network": net[:256], + "next_hop": m.group("nh")[:128], + "metric": metric[:32], + "loc_prf": loc[:32], + "rt_prf": rt[:32], + "path": " ".join(path_parts)[:256], + "status_codes": (m.group("flags") or "").strip()[:16], + } + ) + return out + + +def normalize_evpn_mac( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = params + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show bgp evpn mac").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + return prefer_fsm(tables, RULE_KEYS, _map_fsm_rows, _hand_parse, raw_text=raw_text) + + +normalize_evpn_mac.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/interface_detail.py b/netx_api/biz_state/parsers/zte/interface_detail.py new file mode 100644 index 0000000..4b990ee --- /dev/null +++ b/netx_api/biz_state/parsers/zte/interface_detail.py @@ -0,0 +1,195 @@ +"""ZTE: filtered ``show interface | include …`` (rate / util / BW).""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm + +RULE_KEYS = ("zte_zxros_show_interface",) + +_IF_HDR_RE = re.compile( + r"^(?P\S+)\s+is\s+(?Padministratively\s+down|up|down)" + r"(?:,\s*ifindex:\s*(?P\d+))?\s*$", + re.I, +) +_DESC_RE = re.compile(r"^\s*Description:\s*(?P.*?)\s*$", re.I) +_BW_RE = re.compile(r"^\s*BW\s+(?P.+?)\s*$", re.I) +_MTU_RE = re.compile(r"^\s*MTU\s+(?P\d+)\s+bytes", re.I) +_MEDIA_RE = re.compile(r"^\s*The\s+port\s+is\s+(?P\S+)", re.I) +_NEG_RE = re.compile(r"^\s*Negotiation\s+(?P\S+)", re.I) +_PERIOD_RE = re.compile(r"^\s*Rate\s+period\s*:\s*(?P

\d+)", re.I) +_IN_RE = re.compile(r"^\s*Input\s*:\s*(?P[\d.]+)\s*bit/s", re.I) +_OUT_RE = re.compile(r"^\s*Output\s*:\s*(?P[\d.]+)\s*bit/s", re.I) +_UTIL_RE = re.compile( + r"^\s*Intf\s+utilization:\s*input\s*(?P[\d.]+)%\s+output\s*(?P[\d.]+)%", + re.I, +) + + +def _empty_row(iface: str) -> dict[str, Any]: + return { + "interface": iface[:128], + "admin": "", + "ifindex": "", + "description": "", + "port_media": "", + "negotiation": "", + "bw": "", + "mtu": "", + "rate_period": "", + "input_bps": "", + "output_bps": "", + "in_util": "", + "out_util": "", + } + + +def _normalize_admin(state: str) -> str: + s = str(state or "").strip().lower() + # Guard against FSM greedily eating ", ifindex:" + if "," in s: + s = s.split(",", 1)[0].strip() + if "administratively" in s: + return "admin-down" + if s == "up": + return "up" + if s == "down": + return "down" + return s[:32] + + +def _map_fsm_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for r in rows: + iface = row_get(r, "IFNAME", "interface") + in_util = row_get(r, "IN_UTIL", "in_util") + # Incomplete FSM rows (no util yet) are filldown noise — skip + if not iface or not in_util or iface in seen: + continue + seen.add(iface) + admin_raw = row_get(r, "ADMIN_STATE", "ADMIN_OPER", "admin") + out.append( + { + "interface": iface[:128], + "admin": _normalize_admin(admin_raw)[:32], + "ifindex": row_get(r, "IFINDEX", "ifindex")[:32], + "description": row_get(r, "DESCRIPTION", "description")[:256], + "port_media": row_get(r, "PORT_MEDIA", "port_media")[:32], + "negotiation": row_get(r, "NEGOTIATION", "negotiation")[:32], + "bw": row_get(r, "BW_RAW", "bw")[:64], + "mtu": row_get(r, "MTU", "mtu")[:16], + "rate_period": row_get(r, "RATE_PERIOD", "rate_period")[:16], + "input_bps": row_get(r, "INPUT_BPS", "input_bps")[:32], + "output_bps": row_get(r, "OUTPUT_BPS", "output_bps")[:32], + "in_util": in_util[:16], + "out_util": row_get(r, "OUT_UTIL", "out_util")[:16], + } + ) + return out + + +def _hand_parse(*, raw_text: str, **_kw: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + cur: dict[str, Any] | None = None + in_rates = False + saw_input = False + + def _flush() -> None: + nonlocal cur + if cur and cur.get("interface"): + out.append(cur) + cur = None + + for raw in str(raw_text or "").splitlines(): + line = raw.rstrip() + if not line.strip(): + continue + m = _IF_HDR_RE.match(line.strip()) + if m: + _flush() + cur = _empty_row(m.group("iface")) + cur["admin"] = _normalize_admin(m.group("state")) + cur["ifindex"] = (m.group("ifindex") or "")[:32] + in_rates = False + saw_input = False + continue + if not cur: + continue + m = _DESC_RE.match(line) + if m: + cur["description"] = (m.group("desc") or "").strip()[:256] + continue + m = _MEDIA_RE.match(line) + if m: + cur["port_media"] = m.group("media")[:32] + continue + m = _NEG_RE.match(line) + if m: + cur["negotiation"] = m.group("neg")[:32] + continue + m = _BW_RE.match(line) + if m: + cur["bw"] = m.group("bw").strip()[:64] + continue + m = _MTU_RE.match(line) + if m: + cur["mtu"] = m.group("mtu")[:16] + continue + m = _PERIOD_RE.match(line) + if m: + cur["rate_period"] = m.group("p")[:16] + in_rates = True + saw_input = False + continue + if in_rates and "Peak rate" in line: + continue + m = _IN_RE.match(line) + if m and in_rates and not saw_input: + cur["input_bps"] = m.group("bps")[:32] + saw_input = True + continue + m = _OUT_RE.match(line) + if m and in_rates and saw_input and not cur.get("output_bps"): + cur["output_bps"] = m.group("bps")[:32] + continue + m = _UTIL_RE.match(line) + if m: + cur["in_util"] = m.group("inu")[:16] + cur["out_util"] = m.group("outu")[:16] + in_rates = False + continue + _flush() + return out + + +def normalize_interface_detail( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = params + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show interface").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + return prefer_fsm(tables, RULE_KEYS, _map_fsm_rows, _hand_parse, raw_text=raw_text) + + +normalize_interface_detail.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/ip_route.py b/netx_api/biz_state/parsers/zte/ip_route.py new file mode 100644 index 0000000..e50dc27 --- /dev/null +++ b/netx_api/biz_state/parsers/zte/ip_route.py @@ -0,0 +1,107 @@ +"""ZTE: show ip forwarding route [vrf ].""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm +from .bgp_peer import _detect_vrf + +RULE_KEYS = ("zte_zxros_show_ip_forwarding_route",) + +_ROUTE_RE = re.compile( + r"^\s*(?P[*>R]*)\s*" + r"(?P\d{1,3}(?:\.\d{1,3}){3}/\d+)\s+" + r"(?P\S+)\s+(?P\S+)\s+(?P\S+)\s+" + r"(?P\S+)\s+(?P\S+)\s*$" +) + + +def _map_fsm_rows(rows: list[dict[str, Any]], *, vrf: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for r in rows: + dest = row_get(r, "DEST", "dest") + if not dest or dest.lower() == "dest" or dest in seen: + continue + seen.add(dest) + out.append( + { + "vrf": vrf[:128], + "dest": dest[:64], + "gateway": row_get(r, "GATEWAY", "gateway")[:64], + "interface": row_get(r, "INTERFACE", "interface")[:128], + "owner": row_get(r, "OWNER", "owner")[:64], + "pri": row_get(r, "PRI", "pri")[:16], + "metric": row_get(r, "METRIC", "metric")[:32], + "flags": row_get(r, "FLAGS", "flags")[:16], + } + ) + return out + + +def _hand_parse(*, raw_text: str, vrf: str = "", **_kw: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for raw in str(raw_text or "").splitlines(): + line = raw.rstrip() + if not line.strip() or line.strip().lower().startswith("dest"): + continue + m = _ROUTE_RE.match(line) + if not m: + continue + dest = m.group("dest") + if dest in seen: + continue + seen.add(dest) + out.append( + { + "vrf": vrf[:128], + "dest": dest[:64], + "gateway": m.group("gw")[:64], + "interface": m.group("iface")[:128], + "owner": m.group("owner")[:64], + "pri": m.group("pri")[:16], + "metric": m.group("metric")[:32], + "flags": (m.group("flags") or "").strip()[:16], + } + ) + return out + + +def normalize_ip_route( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + vrf = _detect_vrf(command, params) + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show ip forwarding route").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + + def _map(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return _map_fsm_rows(rows, vrf=vrf) + + def _hand(*, raw_text: str, **kw: Any) -> list[dict[str, Any]]: + return _hand_parse(raw_text=raw_text, vrf=vrf, **kw) + + return prefer_fsm(tables, RULE_KEYS, _map, _hand, raw_text=raw_text) + + +normalize_ip_route.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/ipv6_route.py b/netx_api/biz_state/parsers/zte/ipv6_route.py new file mode 100644 index 0000000..605f993 --- /dev/null +++ b/netx_api/biz_state/parsers/zte/ipv6_route.py @@ -0,0 +1,128 @@ +"""ZTE: show ipv6 forwarding route [vrf ].""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm +from .bgp_peer import _detect_vrf + +RULE_KEYS = ("zte_zxros_show_ipv6_forwarding_route",) + +_DEST_RE = re.compile( + r"^\s*(?P[0-9A-Fa-f:]+/\d+)\s+(?P\S+)\s+(?P\S+)\s+" + r"(?P\S+)(?:\s+(?P\S+))?\s*$" +) +_NH_RE = re.compile(r"^\s+(?P[0-9A-Fa-f:]+|\S+),(?P\S+)\s*$") + + +def _map_fsm_rows(rows: list[dict[str, Any]], *, vrf: str) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + cur: dict[str, Any] | None = None + for r in rows: + dest = row_get(r, "DEST", "dest") + gw = row_get(r, "GATEWAY", "gateway") + iface = row_get(r, "INTERFACE", "interface") + if dest: + cur = { + "vrf": vrf[:128], + "dest": dest[:128], + "gateway": "", + "interface": "", + "owner": row_get(r, "OWNER", "owner")[:64], + "pri": row_get(r, "PRI", "pri")[:16], + "metric": row_get(r, "METRIC", "metric")[:32], + "flags": row_get(r, "FLAGS", "flags")[:16], + } + if gw or iface: + cur["gateway"] = gw[:128] + cur["interface"] = iface[:128] + key = (dest, gw) + if key not in seen: + seen.add(key) + out.append(cur) + cur = None + continue + if cur and (gw or iface): + cur["gateway"] = gw[:128] + cur["interface"] = iface[:128] + key = (cur["dest"], gw) + if key not in seen: + seen.add(key) + out.append(dict(cur)) + cur = None + return out + + +def _hand_parse(*, raw_text: str, vrf: str = "", **_kw: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + pending: dict[str, Any] | None = None + for raw in str(raw_text or "").splitlines(): + line = raw.rstrip() + if not line.strip(): + continue + m = _DEST_RE.match(line) + if m: + pending = { + "vrf": vrf[:128], + "dest": m.group("dest")[:128], + "gateway": "", + "interface": "", + "owner": m.group("owner")[:64], + "pri": m.group("pri")[:16], + "metric": m.group("metric")[:32], + "flags": (m.group("flags") or "")[:16], + } + continue + m2 = _NH_RE.match(line) + if m2 and pending: + gw = m2.group("gw") + key = (pending["dest"], gw) + if key not in seen: + seen.add(key) + row = dict(pending) + row["gateway"] = gw[:128] + row["interface"] = m2.group("iface")[:128] + out.append(row) + pending = None + return out + + +def normalize_ipv6_route( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + vrf = _detect_vrf(command, params) + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show ipv6 forwarding route").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + + def _map(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return _map_fsm_rows(rows, vrf=vrf) + + def _hand(*, raw_text: str, **kw: Any) -> list[dict[str, Any]]: + return _hand_parse(raw_text=raw_text, vrf=vrf, **kw) + + return prefer_fsm(tables, RULE_KEYS, _map, _hand, raw_text=raw_text) + + +normalize_ipv6_route.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/l2vpn_mac.py b/netx_api/biz_state/parsers/zte/l2vpn_mac.py new file mode 100644 index 0000000..4820155 --- /dev/null +++ b/netx_api/biz_state/parsers/zte/l2vpn_mac.py @@ -0,0 +1,95 @@ +"""ZTE: show mac l2vpn.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm + +RULE_KEYS = ("zte_zxros_show_mac_l2vpn",) + +_MAC_RE = re.compile( + r"^(?P[0-9a-fA-F]{4}\.[0-9a-fA-F]{4}\.[0-9a-fA-F]{4})\s+" + r"(?P\S+)\s+(?P\S+)\s+(?P.+?)\s+(?P\S+)\s*$" +) + + +def _map_fsm_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + for r in rows: + mac = row_get(r, "MAC", "mac") + vpn = row_get(r, "VPN", "vpn") + if not mac or mac.lower() == "mac": + continue + key = (mac, vpn) + if key in seen: + continue + seen.add(key) + out.append( + { + "mac": mac[:64], + "vpn": vpn[:128], + "vlan": row_get(r, "VLAN", "vlan")[:32], + "outgoing": row_get(r, "OUTGOING", "outgoing")[:256], + "attribute": row_get(r, "ATTRIBUTE", "attribute")[:64], + } + ) + return out + + +def _hand_parse(*, raw_text: str, **_kw: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + for raw in str(raw_text or "").splitlines(): + line = raw.strip() + if not line or line.lower().startswith("mac "): + continue + m = _MAC_RE.match(line) + if not m: + continue + key = (m.group("mac"), m.group("vpn")) + if key in seen: + continue + seen.add(key) + out.append( + { + "mac": m.group("mac")[:64], + "vpn": m.group("vpn")[:128], + "vlan": m.group("vlan")[:32], + "outgoing": m.group("out").strip()[:256], + "attribute": m.group("attr")[:64], + } + ) + return out + + +def normalize_l2vpn_mac( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = params + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show mac l2vpn").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + return prefer_fsm(tables, RULE_KEYS, _map_fsm_rows, _hand_parse, raw_text=raw_text) + + +normalize_l2vpn_mac.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/l2vpn_pw.py b/netx_api/biz_state/parsers/zte/l2vpn_pw.py new file mode 100644 index 0000000..649429b --- /dev/null +++ b/netx_api/biz_state/parsers/zte/l2vpn_pw.py @@ -0,0 +1,106 @@ +"""ZTE: show l2vpn forwardinfo (PW state).""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm + +RULE_KEYS = ("zte_zxros_show_l2vpn_forwardinfo",) + +_PW_RE = re.compile( + r"^(?P\S+)\s+(?P\S+)\s+(?P\S+)\s+(?P\S+)\s+" + r"(?:(?P[A-Za-z$]+)\s+)?" + r"(?PUP|DOWN)\s+(?P\S+)\s+(?P\S+)\s+(?P\S+)\s*$", + re.I, +) + + +def _map_fsm_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for r in rows: + name = row_get(r, "PW_NAME", "pw_name") + if not name or name.lower() == "pwname" or name in seen: + continue + seen.add(name) + state = row_get(r, "STATE", "state") + # FSM may glue mode into pw_type + pwt = row_get(r, "PW_TYPE", "pw_type") + out.append( + { + "pw_name": name[:128], + "peer": row_get(r, "PEER", "peer")[:64], + "fec": row_get(r, "FEC", "fec")[:32], + "pw_type": pwt[:64], + "state": state[:32], + "local_label": row_get(r, "LLABEL", "local_label")[:32], + "remote_label": row_get(r, "RLABEL", "remote_label")[:32], + "vpn_owner": row_get(r, "VPN_OWNER", "vpn_owner")[:256], + } + ) + return out + + +def _hand_parse(*, raw_text: str, **_kw: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for raw in str(raw_text or "").splitlines(): + line = raw.strip() + if not line or line.lower().startswith("pwname"): + continue + m = _PW_RE.match(line) + if not m: + continue + name = m.group("name") + if name in seen: + continue + seen.add(name) + mode = (m.group("mode") or "").strip() + pwt = m.group("pwt") + if mode: + pwt = f"{pwt} {mode}" + out.append( + { + "pw_name": name[:128], + "peer": m.group("peer")[:64], + "fec": m.group("fec")[:32], + "pw_type": pwt[:64], + "state": m.group("state")[:32], + "local_label": m.group("ll")[:32], + "remote_label": m.group("rl")[:32], + "vpn_owner": m.group("owner")[:256], + } + ) + return out + + +def normalize_l2vpn_pw( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = params + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show l2vpn forwardinfo").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + return prefer_fsm(tables, RULE_KEYS, _map_fsm_rows, _hand_parse, raw_text=raw_text) + + +normalize_l2vpn_pw.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/optical_brief.py b/netx_api/biz_state/parsers/zte/optical_brief.py new file mode 100644 index 0000000..3f78a6e --- /dev/null +++ b/netx_api/biz_state/parsers/zte/optical_brief.py @@ -0,0 +1,93 @@ +"""ZTE: show opticalinfo brief.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm + +RULE_KEYS = ("zte_zxros_show_opticalinfo_brief",) + +_OPT_RE = re.compile( + r"^(?P\S+)\s+(?P\S+)\s+(?P\S+)\s+" + r"(?P\S+)\s+(?P\S+)\s+(?P\S+)", +) + + +def _map_fsm_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for r in rows: + iface = row_get(r, "INTERFACE", "interface") + if not iface or iface.lower() == "interface" or iface in seen: + continue + seen.add(iface) + out.append( + { + "interface": iface[:128], + "optic_type": row_get(r, "OPTIC_TYPE", "optic_type")[:64], + "wavelength": row_get(r, "WAVELENGTH", "wavelength")[:32], + "rx_power": row_get(r, "RX_POWER", "rx_power")[:64], + "tx_power": row_get(r, "TX_POWER", "tx_power")[:64], + "status": row_get(r, "STATUS", "status")[:32], + } + ) + return out + + +def _hand_parse(*, raw_text: str, **_kw: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for raw in str(raw_text or "").splitlines(): + line = raw.strip() + if not line or line.lower().startswith("interface"): + continue + m = _OPT_RE.match(line) + if not m: + continue + iface = m.group("iface") + if iface in seen: + continue + seen.add(iface) + out.append( + { + "interface": iface[:128], + "optic_type": m.group("otype")[:64], + "wavelength": m.group("wave")[:32], + "rx_power": m.group("rx")[:64], + "tx_power": m.group("tx")[:64], + "status": m.group("status")[:32], + } + ) + return out + + +def normalize_optical_brief( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = params + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show opticalinfo brief").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + return prefer_fsm(tables, RULE_KEYS, _map_fsm_rows, _hand_parse, raw_text=raw_text) + + +normalize_optical_brief.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/ospf_neighbor.py b/netx_api/biz_state/parsers/zte/ospf_neighbor.py new file mode 100644 index 0000000..857e50d --- /dev/null +++ b/netx_api/biz_state/parsers/zte/ospf_neighbor.py @@ -0,0 +1,108 @@ +"""ZTE: show ip ospf neighbor.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm + +RULE_KEYS = ("zte_zxros_show_ospf_neighbor",) + +_PROC_RE = re.compile(r"Process\s+ID\s+(\d+)", re.I) +_NEI_RE = re.compile( + r"^(?P\d{1,3}(?:\.\d{1,3}){3})\s+(?P\S+)\s+(?P\S+)\s+" + r"(?P\S+)\s+(?P\S+)\s+(?P\S+)\s*$" +) + + +def _map_fsm_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str]] = set() + for r in rows: + nid = row_get(r, "NEIGHBOR_ID", "neighbor_id") + iface = row_get(r, "INTERFACE", "interface") + proc = row_get(r, "PROCESS_ID", "process_id") or "0" + if not nid or not iface: + continue + key = (proc, nid, iface) + if key in seen: + continue + seen.add(key) + out.append( + { + "process_id": proc[:32], + "neighbor_id": nid[:64], + "pri": row_get(r, "PRI", "pri")[:16], + "state": row_get(r, "STATE", "state")[:64], + "dead_time": row_get(r, "DEAD_TIME", "dead_time")[:32], + "address": row_get(r, "ADDRESS", "address")[:64], + "interface": iface[:128], + } + ) + return out + + +def _hand_parse(*, raw_text: str, **_kw: Any) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[tuple[str, str, str]] = set() + proc = "0" + for raw in str(raw_text or "").splitlines(): + line = raw.strip() + if not line: + continue + pm = _PROC_RE.search(line) + if pm: + proc = pm.group(1) + continue + if line.lower().startswith("neighbor id"): + continue + m = _NEI_RE.match(line) + if not m: + continue + key = (proc, m.group("nid"), m.group("iface")) + if key in seen: + continue + seen.add(key) + out.append( + { + "process_id": proc[:32], + "neighbor_id": m.group("nid")[:64], + "pri": m.group("pri")[:16], + "state": m.group("state")[:64], + "dead_time": m.group("dead")[:32], + "address": m.group("addr")[:64], + "interface": m.group("iface")[:128], + } + ) + return out + + +def normalize_ospf_neighbor( + *, + raw_text: str, + fsm_tables: Mapping[str, list[dict[str, Any]]] | None = None, + vendor: str = "", + device_type: str = "", + command: str = "", + params: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + _ = params + tables = dict(fsm_tables or {}) + if not any(tables.get(k) for k in RULE_KEYS): + platform = resolve_cli_platform( + vendor=vendor, + device_type=device_type, + vendor_key=resolve_vendor_key(vendor, device_type), + ) + cmd = str(command or "show ip ospf neighbor").strip() + if platform and cmd: + tables = apply_rules( + platform=platform, text=raw_text, rule_keys=RULE_KEYS, command=cmd + ) + return prefer_fsm(tables, RULE_KEYS, _map_fsm_rows, _hand_parse, raw_text=raw_text) + + +normalize_ospf_neighbor.RULE_KEYS = RULE_KEYS diff --git a/netx_api/biz_state/parsers/zte/vrrp.py b/netx_api/biz_state/parsers/zte/vrrp.py new file mode 100644 index 0000000..305ca8c --- /dev/null +++ b/netx_api/biz_state/parsers/zte/vrrp.py @@ -0,0 +1,122 @@ +"""ZTE: show vrrp {ipv4|ipv6} brief.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from ....lldp_shared import resolve_vendor_key +from ....ntc_parse import apply_rules, resolve_cli_platform, row_get +from ..common.pipeline import prefer_fsm + +RULE_KEYS = ("zte_zxros_show_vrrp_brief",) + +_VRRP_RE = re.compile( + r"^(?P\S+)\s+(?P\d+)\s+(?P\d+)\s+(?P

+
{t("bizCompare.ifaceNormalize")}
+

{t("bizCompare.ifaceNormalizeHint")}

+