Fix ZTE BGP RR *i/RD parsing and add batch collect detail UX.

Decouple heavy CLI parse via ParsePool, keep ECMP/multi-RD keys, and expose per-command CLI lines vs DB rows with raw.txt export.

Co-authored-by: Cursor <cursoragent@cursor.com>

netx_api/biz_state/collect_runner.py

@@ -30,8 +30,11 @@ from ..models import (
 )
 from ..ne_netmiko import disable_target_paging, send_show_command
 from ..ne_session_factory import close_netmiko_connection, open_netmiko_connection
+from ..timeutil import utcnow_naive
 from .collect_session import (
+    CachedCommand,
     CollectSession,
+    ResolvedAux,
     build_parse_bundle,
     resolve_aux_command,
     run_primary_with_bundle,
@@ -43,7 +46,7 @@ from .command_match import (
     match_command,
     normalize_command,
 )
-from .parsers import get_parser
+from .parsers import get_parser, run_parser
 from .profiles import get_profile
 from .collect_stop import (
     STOP_USER_MESSAGE,
@@ -93,7 +96,7 @@ def partition_work(work: list[WorkItem]) -> tuple[list[WorkItem], list[WorkItem]
 
 
 def _utcnow() -> datetime:
-    return datetime.utcnow()
+    return utcnow_naive()
 
 
 def _format_error(exc: BaseException) -> str:
@@ -215,6 +218,201 @@ _GENERIC_METRICS = {
 _METRIC_CHUNK = 2000
 
 
+def _run_primary_parse_job(job: Any) -> tuple[bool, bool]:
+    """Parse primary+aux raws, write spool, submit persist. Returns (any_ok, any_fail)."""
+    from .parse_pool import AuxRawCapture, PrimaryParseJob
+    from .persist_pool import get_persist_pool
+    from .spool import SpooledCommand, write_meta, write_records
+
+    if not isinstance(job, PrimaryParseJob):
+        return False, True
+
+    batch_id = job.batch_id
+    cmd_id = job.cmd_id
+    pending: list[SpooledCommand] = []
+    any_ok = False
+    any_fail = False
+
+    def _flush_item(item: SpooledCommand, *, records: list[dict[str, Any]] | None = None) -> None:
+        if records is not None and item.persist_kind:
+            item.records_rel_path = write_records(batch_id, item.id, records)
+            item.row_count = len(records)
+        try:
+            write_meta(batch_id, item.id, item.to_meta())
+        except Exception:
+            _log.exception("biz_state write meta failed cmd=%s", item.id)
+        pending.append(item)
+
+    aux_results: dict[str, CachedCommand] = {}
+    resolved_aux: list[ResolvedAux] = []
+
+    for cap in list(job.aux_captures or []):
+        if not isinstance(cap, AuxRawCapture):
+            continue
+        from .profiles import get_profile as _gp
+
+        prof = _gp(cap.profile_id)
+        entry = CachedCommand(
+            raw=cap.raw,
+            records=list(cap.records or []),
+            fsm_tables=dict(cap.fsm_tables or {}),
+            ok=bool(cap.ok),
+            error=str(cap.error or ""),
+            cmd_row_id=cap.aux_id,
+        )
+        if entry.ok and not entry.records and cap.parser_id and get_parser(cap.parser_id):
+            try:
+                records, fsm_tables, _keys = run_parser(
+                    cap.parser_id,
+                    raw_text=cap.raw,
+                    vendor=job.vendor,
+                    device_type=job.device_type,
+                    command=cap.textfsm_command or cap.command,
+                    textfsm_command=cap.textfsm_command or "",
+                    params=dict(job.merged_params or {}),
+                )
+                entry.records = list(records or [])
+                entry.fsm_tables = dict(fsm_tables or {})
+            except Exception as exc:
+                entry.ok = False
+                entry.error = f"parse: {type(exc).__name__}: {exc}"
+
+        aux_results[cap.key] = entry
+        if prof is not None:
+            resolved_aux.append(
+                ResolvedAux(
+                    key=cap.key,
+                    profile_id=cap.profile_id,
+                    command=cap.command,
+                    textfsm_command=cap.textfsm_command or cap.command,
+                    parser_id=cap.parser_id,
+                    rule_keys=tuple(cap.rule_keys or ()),
+                    profile=prof,
+                )
+            )
+
+        aux_sp = SpooledCommand(
+            id=cap.aux_id,
+            batch_id=batch_id,
+            task_item_id=job.task_item_id,
+            profile_id=cap.profile_id,
+            parser_id=cap.parser_id,
+            metric_id=cap.metric_id,
+            raw_command=cap.command[:512],
+            params_json={},
+            raw_rel_path=cap.raw_rel_path,
+        )
+        if cap.cache_hit and entry.ok and entry.records:
+            aux_sp.parse_status = "aux_cached"
+            aux_sp.message = (
+                f"aux_for={cmd_id};cache_hit;src={entry.cmd_row_id}"
+            )[:1020]
+            aux_sp.row_count = len(entry.records or [])
+        elif not entry.ok:
+            aux_sp.parse_status = "aux_failed"
+            aux_sp.message = f"aux_for={cmd_id};{entry.error}"[:1020]
+            any_fail = True
+        else:
+            aux_sp.parse_status = "aux"
+            aux_sp.message = f"aux_for={cmd_id}"[:1020]
+            aux_sp.row_count = len(entry.records or [])
+
+        persist_recs: list[dict[str, Any]] | None = None
+        if entry.ok and entry.records and cap.metric_id in _GENERIC_METRICS:
+            persist_key = (normalize_command(cap.command), cap.metric_id)
+            do_persist = False
+            persisted = job.persisted
+            if persisted is not None:
+                if job.cache_lock is not None:
+                    with job.cache_lock:
+                        if persist_key not in persisted:
+                            persisted.add(persist_key)
+                            do_persist = True
+                elif persist_key not in persisted:
+                    persisted.add(persist_key)
+                    do_persist = True
+            if do_persist:
+                aux_sp.persist_kind = "metric"
+                persist_recs = list(entry.records)
+        _flush_item(aux_sp, records=persist_recs)
+
+    bundle = build_parse_bundle(
+        primary_raw=job.raw_text,
+        primary_parser_id=job.parser_id,
+        aux_results=aux_results,
+        resolved_aux=resolved_aux,
+    )
+    # Include aux raws that lacked a profile (still needed for multi-raw parsers).
+    for cap in list(job.aux_captures or []):
+        if not isinstance(cap, AuxRawCapture):
+            continue
+        if cap.key in bundle.raws:
+            continue
+        entry = aux_results.get(cap.key) or CachedCommand(ok=False)
+        bundle.raws[cap.key] = entry.raw
+        bundle.command_rules[cap.key] = list(cap.rule_keys or [])
+        if entry.records:
+            bundle.aux_records[cap.key] = list(entry.records)
+        bundle.fsm_extra.update(entry.fsm_tables or {})
+    primary = SpooledCommand(
+        id=cmd_id,
+        batch_id=batch_id,
+        task_item_id=job.task_item_id,
+        profile_id=job.profile_id,
+        parser_id=job.parser_id,
+        metric_id=job.metric_id,
+        raw_command=job.concrete[:512],
+        params_json=dict(job.merged_params or {}),
+        raw_rel_path=job.raw_rel_path,
+    )
+    try:
+        records, fsm_tables, rule_keys = run_primary_with_bundle(
+            job.parser_id,
+            bundle=bundle,
+            vendor=job.vendor,
+            device_type=job.device_type,
+            command=job.textfsm_command or job.concrete,
+            textfsm_command=job.textfsm_command or "",
+            params=dict(job.merged_params or {}),
+            enrich_joins=list(job.enrich_joins or []),
+        )
+        hints = []
+        if rule_keys:
+            nonempty = [k for k in rule_keys if fsm_tables.get(k)]
+            hints.append(f"fsm_keys={','.join(rule_keys)};hit={','.join(nonempty)}")
+        if bundle.aux_records:
+            hints.append(
+                "aux="
+                + ",".join(f"{k}:{len(v)}" for k, v in bundle.aux_records.items())
+            )
+        if job.enrich_joins:
+            hints.append(
+                "enrich=" + ",".join(getattr(j, "from_aux", "") for j in job.enrich_joins)
+            )
+        if hints:
+            primary.message = ";".join(hints)[:1020]
+        primary.parse_status = "ok"
+        any_ok = True
+        persist_recs = None
+        if job.metric_id == "lldp_neighbor":
+            primary.persist_kind = "lldp"
+            persist_recs = list(records or [])
+        elif job.metric_id in _GENERIC_METRICS:
+            primary.persist_kind = "metric"
+            persist_recs = list(records or [])
+        _flush_item(primary, records=persist_recs)
+        _ = fsm_tables  # kept for hints above
+    except Exception as exc:
+        any_fail = True
+        primary.parse_status = "failed"
+        primary.message = f"parse: {_format_error(exc)}"
+        _flush_item(primary)
+
+    if pending:
+        get_persist_pool().submit(batch_id, pending)
+    return any_ok, any_fail
+
+
 def _emit_task_event(*, task_id: str, message: str, level: str = "error") -> None:
     """Short-lived session for lane events (no long-held DB during CLI)."""
     if not task_id or not str(message or "").strip():
@@ -547,9 +745,26 @@ def _run_collect_lane(
         any_ok = False
         pending: list[SpooledCommand] = []
         task_id = ""
+        from .parse_pool import (
+            AuxRawCapture,
+            PrimaryParseJob,
+            get_parse_pool,
+            parse_async_enabled,
+        )
         from .persist_pool import get_persist_pool
 
         persist = get_persist_pool()
+        parse_pool = get_parse_pool() if parse_async_enabled() else None
+        parse_stats_lock = threading.Lock()
+        parse_stats = {"ok": False, "fail": False, "pending": 0}
+
+        def _on_parse_done(ok: bool, fail: bool) -> None:
+            with parse_stats_lock:
+                if ok:
+                    parse_stats["ok"] = True
+                if fail:
+                    parse_stats["fail"] = True
+                parse_stats["pending"] = max(0, int(parse_stats["pending"]) - 1)
 
         def _submit_pending() -> None:
             nonlocal pending
@@ -763,8 +978,8 @@ def _run_collect_lane(
                         cmd_row_id=cmd_id,
                     )
 
-                resolved_aux = []
-                aux_results: dict[str, Any] = {}
+                # Collect aux raws on the SSH thread (no TextFSM); parse overlaps next CLI.
+                aux_captures: list[AuxRawCapture] = []
                 for aux in list(hit.profile.aux_commands or []):
                     try:
                         ra = resolve_aux_command(aux, params=merged)
@@ -781,145 +996,105 @@ def _run_collect_lane(
                             )
                         )
                         continue
-                    resolved_aux.append(ra)
                     aux_id = uuid4().hex
-                    entry, cache_hit = session.fetch_and_parse(
-                        ra.command,
-                        parser_id=ra.parser_id,
-                        textfsm_command=ra.textfsm_command,
-                        params=merged,
-                        cmd_row_id=aux_id,
-                    )
-                    aux_results[ra.key] = entry
+                    entry, cache_hit = session.fetch_raw(ra.command, cmd_row_id=aux_id)
                     aux_mid = str(getattr(ra.profile, "metric_id", "") or "").strip()
-                    aux_sp = SpooledCommand(
-                        id=aux_id,
-                        batch_id=batch_id,
-                        task_item_id=item_id,
-                        profile_id=ra.profile_id,
-                        parser_id=ra.parser_id,
-                        metric_id=aux_mid,
-                        raw_command=ra.command[:512],
-                        params_json={},
+                    aux_rel = ""
+                    if entry.raw:
+                        try:
+                            aux_rel = write_raw_text(batch_id, aux_id, entry.raw or "")
+                        except Exception:
+                            _log.exception(
+                                "biz_state spool aux raw failed cmd=%s", aux_id
+                            )
+                    if not entry.ok:
+                        any_fail = True
+                        _queue(
+                            SpooledCommand(
+                                id=aux_id,
+                                batch_id=batch_id,
+                                task_item_id=item_id,
+                                profile_id=ra.profile_id,
+                                parser_id=ra.parser_id,
+                                metric_id=aux_mid,
+                                raw_command=ra.command[:512],
+                                parse_status="aux_failed",
+                                message=f"aux_for={cmd_id};{entry.error}"[:1020],
+                                raw_rel_path=aux_rel,
+                            )
+                        )
+                        continue
+                    # If cache already has parsed records, pass them through.
+                    aux_captures.append(
+                        AuxRawCapture(
+                            key=ra.key,
+                            aux_id=aux_id,
+                            profile_id=ra.profile_id,
+                            parser_id=ra.parser_id,
+                            metric_id=aux_mid,
+                            command=ra.command,
+                            textfsm_command=ra.textfsm_command,
+                            rule_keys=tuple(ra.rule_keys or ()),
+                            raw=entry.raw,
+                            raw_rel_path=aux_rel,
+                            cache_hit=bool(cache_hit and entry.records),
+                            records=list(entry.records or []),
+                            fsm_tables=dict(entry.fsm_tables or {}),
+                            ok=True,
+                        )
                     )
-                    if cache_hit:
-                        aux_sp.parse_status = "aux_cached"
-                        aux_sp.message = (
-                            f"aux_for={cmd_id};cache_hit;src={entry.cmd_row_id}"
-                        )[:1020]
-                        aux_sp.row_count = len(entry.records or [])
-                    elif not entry.ok:
-                        aux_sp.parse_status = "aux_failed"
-                        aux_sp.message = f"aux_for={cmd_id};{entry.error}"[:1020]
-                        try:
-                            aux_sp.raw_rel_path = write_raw_text(
-                                batch_id, aux_id, entry.raw or ""
-                            )
-                        except Exception:
-                            pass
-                    else:
-                        aux_sp.parse_status = "aux"
-                        aux_sp.message = f"aux_for={cmd_id}"[:1020]
-                        try:
-                            aux_sp.raw_rel_path = write_raw_text(
-                                batch_id, aux_id, entry.raw or ""
-                            )
-                        except Exception:
-                            pass
-                        aux_sp.row_count = len(entry.records or [])
-                        entry.cmd_row_id = aux_id
 
-                    persist_recs: list[dict[str, Any]] | None = None
-                    if entry.ok and entry.records and aux_mid in _GENERIC_METRICS:
-                        persist_key = (normalize_command(ra.command), aux_mid)
-                        do_persist = False
-                        if cache_lock is not None:
-                            with cache_lock:
-                                if persist_key not in persisted:
-                                    persisted.add(persist_key)
-                                    do_persist = True
-                        elif persist_key not in persisted:
-                            persisted.add(persist_key)
-                            do_persist = True
-                        if do_persist:
-                            aux_sp.persist_kind = "metric"
-                            persist_recs = list(entry.records)
-                    _queue(aux_sp, records=persist_recs)
-
-                bundle = build_parse_bundle(
-                    primary_raw=raw_text,
-                    primary_parser_id=hit.profile.parser_id,
-                    aux_results=aux_results,
-                    resolved_aux=resolved_aux,
-                )
-                primary = SpooledCommand(
-                    id=cmd_id,
+                job = PrimaryParseJob(
                     batch_id=batch_id,
+                    cmd_id=cmd_id,
                     task_item_id=item_id,
                     profile_id=hit.profile.profile_id,
                     parser_id=hit.profile.parser_id,
                     metric_id=hit.profile.metric_id,
-                    raw_command=concrete[:512],
-                    params_json=merged,
+                    concrete=concrete,
+                    merged_params=dict(merged or {}),
+                    raw_text=raw_text,
                     raw_rel_path=raw_rel,
+                    textfsm_command=hit.profile.textfsm_command or concrete,
+                    vendor=vendor_eff,
+                    device_type=device_type_eff,
+                    enrich_joins=list(hit.profile.enrich_joins or []),
+                    aux_captures=aux_captures,
+                    persisted=persisted,
+                    cache_lock=cache_lock,
+                    on_done=_on_parse_done if parse_pool is not None else None,
                 )
-                try:
-                    records, fsm_tables, rule_keys = run_primary_with_bundle(
-                        hit.profile.parser_id,
-                        bundle=bundle,
-                        vendor=vendor_eff,
-                        device_type=device_type_eff,
-                        command=hit.profile.textfsm_command or concrete,
-                        textfsm_command=hit.profile.textfsm_command or "",
-                        params=merged,
-                        enrich_joins=list(hit.profile.enrich_joins or []),
-                    )
-                    session.remember(
-                        concrete,
-                        raw=raw_text,
-                        fsm_tables=fsm_tables,
-                        records=records,
-                        ok=True,
-                        cmd_row_id=cmd_id,
-                    )
-                    hints = []
-                    if rule_keys:
-                        nonempty = [k for k in rule_keys if fsm_tables.get(k)]
-                        hints.append(
-                            f"fsm_keys={','.join(rule_keys)};hit={','.join(nonempty)}"
-                        )
-                    if bundle.aux_records:
-                        hints.append(
-                            "aux="
-                            + ",".join(
-                                f"{k}:{len(v)}" for k, v in bundle.aux_records.items()
-                            )
-                        )
-                    if hit.profile.enrich_joins:
-                        hints.append(
-                            "enrich="
-                            + ",".join(j.from_aux for j in hit.profile.enrich_joins)
-                        )
-                    if hints:
-                        primary.message = ";".join(hints)[:1020]
-                    primary.parse_status = "ok"
-                    any_ok = True
-                    persist_recs = None
-                    if hit.profile.metric_id == "lldp_neighbor":
-                        primary.persist_kind = "lldp"
-                        persist_recs = list(records or [])
-                    elif hit.profile.metric_id in _GENERIC_METRICS:
-                        primary.persist_kind = "metric"
-                        persist_recs = list(records or [])
-                    _queue(primary, records=persist_recs)
-                except Exception as exc:
-                    any_fail = True
-                    primary.parse_status = "failed"
-                    primary.message = f"parse: {_format_error(exc)}"
-                    _queue(primary)
+                if parse_pool is not None:
+                    with parse_stats_lock:
+                        parse_stats["pending"] += 1
+                    parse_pool.submit(job)
+                    # Count primary (+ aux will be counted in parse worker via persist).
+                    # cmd_count: bump for primary + each aux capture so progress is visible.
+                    cmd_count += 1 + len(aux_captures)
+                else:
+                    ok, fail = _run_primary_parse_job(job)
+                    if ok:
+                        any_ok = True
+                    if fail:
+                        any_fail = True
+                    # Sync path: parse job already submitted persist; count cmds.
+                    cmd_count += 1 + len(aux_captures)
 
-            # Final flush for this lane → persist pool; wait so rows land before return.
+            # Drain CLI spool leftovers, then wait parse + persist pools.
             _submit_pending()
+            if parse_pool is not None:
+                if not parse_pool.wait_idle(timeout=max(30.0, float(budget))):
+                    _log.warning(
+                        "biz_state parse barrier timed out batch=%s lane=%s",
+                        batch_id,
+                        label,
+                    )
+                    any_fail = True
+                with parse_stats_lock:
+                    if parse_stats["ok"]:
+                        any_ok = True
+                    if parse_stats["fail"]:
+                        any_fail = True
             if not persist.wait_idle(timeout=max(30.0, float(budget))):
                 _log.warning(
                     "biz_state persist barrier timed out batch=%s lane=%s",
@@ -933,6 +1108,8 @@ def _run_collect_lane(
             # Best-effort: enqueue leftover spool before connection teardown.
             try:
                 _submit_pending()
+                if parse_pool is not None:
+                    parse_pool.wait_idle(timeout=60.0)
                 persist.wait_idle(timeout=60.0)
             except Exception:
                 _log.exception(
@@ -1115,13 +1292,14 @@ def _finalize_batch_status(
             batch.message = ""
         status = str(batch.status or "")
         db.commit()
-        if status in ("success", "partial") and not stopped:
+        # Only full success triggers auto compare; never block the collect thread.
+        if status == "success" and not stopped:
             try:
-                from .compare_service import try_auto_compare_for_task
+                from .compare_service import schedule_auto_compare_for_task
 
-                try_auto_compare_for_task(db, task_id, batch_id)
+                schedule_auto_compare_for_task(task_id, batch_id)
             except Exception:
-                _log.exception("biz_state auto compare hook failed task=%s", task_id)
+                _log.exception("biz_state auto compare schedule failed task=%s", task_id)
         return status
 
     try:
@@ -1405,10 +1583,20 @@ def _run_collect_session(
             raise RuntimeError("; ".join(lane_errors)[:1020])
         any_fail = True
 
-    # Ensure persist pool drained before terminal status write.
+    # Ensure parse + persist pools drained before terminal status write.
     try:
+        from .parse_pool import get_parse_pool, parse_async_enabled
         from .persist_pool import get_persist_pool
 
+        if parse_async_enabled():
+            if not get_parse_pool().wait_idle(timeout=120.0):
+                _log.warning(
+                    "biz_state parse barrier before finalize timed out batch=%s",
+                    batch_id,
+                )
+                any_fail = True
+                if "parse_barrier_timeout" not in lane_errors:
+                    lane_errors.append("RuntimeError: parse_barrier_timeout")
         if not get_persist_pool().wait_idle(timeout=120.0):
             _log.warning(
                 "biz_state persist barrier before finalize timed out batch=%s",
@@ -1418,7 +1606,7 @@ def _run_collect_session(
             if "persist_barrier_timeout" not in lane_errors:
                 lane_errors.append("RuntimeError: persist_barrier_timeout")
     except Exception:
-        _log.exception("biz_state persist barrier before finalize failed batch=%s", batch_id)
+        _log.exception("biz_state parse/persist barrier before finalize failed batch=%s", batch_id)
 
     stopped = is_stop_requested(batch_id) or any("_stopped" in e for e in lane_errors)
     _finalize_batch_status(

netx_api/biz_state/collect_session.py

@@ -174,6 +174,40 @@ class CollectSession:
                 return hit
         return None
 
+    def fetch_raw(
+        self,
+        command: str,
+        *,
+        cmd_row_id: str = "",
+    ) -> tuple[CachedCommand, bool]:
+        """CLI-only fetch (no TextFSM). Return ``(entry, cache_hit)``.
+
+        Used so the SSH thread can overlap subsequent commands while a parse
+        pool parses prior raws.
+        """
+        with self._command_lock(command):
+            cached = self.get_cached(command)
+            if cached is not None and str(cached.raw or "").strip():
+                return cached, True
+            try:
+                raw = self._send_show(command)
+            except Exception as exc:
+                entry = self.remember(
+                    command,
+                    raw="",
+                    ok=False,
+                    error=f"{type(exc).__name__}: {exc}",
+                    cmd_row_id=cmd_row_id,
+                )
+                return entry, False
+            entry = self.remember(
+                command,
+                raw=raw,
+                ok=True,
+                cmd_row_id=cmd_row_id,
+            )
+            return entry, False
+
     def fetch_and_parse(
         self,
         command: str,
@@ -187,11 +221,46 @@ class CollectSession:
 
         Same concrete CLI is serialized across shared-cache lanes so aux of
         one monitor item can be reused by the next without re-collecting.
+        Prefer ``fetch_raw`` on the collect hot path when parse can be async.
         """
         with self._command_lock(command):
             cached = self.get_cached(command)
             if cached is not None:
-                return cached, True
+                # Full parse hit (records or explicit prior parse failure with raw).
+                if cached.records or (cached.ok and not parser_id):
+                    return cached, True
+                if cached.ok and str(cached.raw or "").strip() and parser_id and get_parser(parser_id):
+                    # Raw-only cache (async collect): parse now without re-CLI.
+                    try:
+                        records, fsm_tables, _keys = run_parser(
+                            parser_id,
+                            raw_text=cached.raw,
+                            vendor=self.vendor,
+                            device_type=self.device_type,
+                            command=textfsm_command or command,
+                            textfsm_command=textfsm_command or "",
+                            params=params or {},
+                        )
+                    except Exception as exc:
+                        entry = self.remember(
+                            command,
+                            raw=cached.raw,
+                            ok=False,
+                            error=f"parse: {type(exc).__name__}: {exc}",
+                            cmd_row_id=cmd_row_id or cached.cmd_row_id,
+                        )
+                        return entry, True
+                    entry = self.remember(
+                        command,
+                        raw=cached.raw,
+                        fsm_tables=fsm_tables,
+                        records=records,
+                        ok=True,
+                        cmd_row_id=cmd_row_id or cached.cmd_row_id,
+                    )
+                    return entry, True
+                if cached.ok:
+                    return cached, True
             try:
                 raw = self._send_show(command)
             except Exception as exc:

netx_api/biz_state/compare_engine.py

@@ -42,6 +42,9 @@ def mapping_stats(
 
     ``hit_before``: map key appears as a normalized before iface, or as the
     parent of a before subinterface (so main-port-only maps still validate).
+
+    Map keys are matched against **normalized** iface values (same pipeline as
+    compare). Enter mapping keys in normalized form.
     """
     before_ifaces: set[str] = set()
     after_ifaces: set[str] = set()
@@ -58,14 +61,17 @@ def mapping_stats(
     before_bases: set[str] = set()
     for v in before_ifaces:
         before_bases.add(v)
-        if "." in v:
-            before_bases.add(v.rsplit(".", 1)[0])
+        # Multi-level parents: a.b.c → a.b, a
+        parts = v.split(".")
+        for i in range(len(parts) - 1, 0, -1):
+            before_bases.add(".".join(parts[:i]))
 
     after_bases: set[str] = set()
     for v in after_ifaces:
         after_bases.add(v)
-        if "." in v:
-            after_bases.add(v.rsplit(".", 1)[0])
+        parts = v.split(".")
+        for i in range(len(parts) - 1, 0, -1):
+            after_bases.add(".".join(parts[:i]))
 
     mapped_before = set(port_map.keys())
     mapped_after = set(port_map.values())
@@ -84,6 +90,7 @@ def mapping_stats(
         "miss_after": miss_after,
         "unused_before_keys": unused,
         "ok": not miss_before and not miss_after,
+        "hint": "map keys must match normalized iface names (post iface_normalize)",
     }
 
 
@@ -97,17 +104,22 @@ def compare_rows(
     port_map: dict[str, str] | None = None,
     field_rules: Sequence[Mapping[str, Any]] | None = None,
     iface_normalize_rules: Sequence[Mapping[str, str]] | None = None,
+    ignore_port_changes: bool | None = None,
 ) -> dict[str, Any]:
     """Return summary + diffs list.
 
-    Diff kinds: added | removed | changed | unchanged
+    Diff kinds: added | removed | changed | unchanged | duplicate
 
     Pipeline: iface normalize (both sides) → port map (before) → match.
 
-    Empty ``port_map``: try to ignore port renames by dropping ``iface_fields``
-    from the match key (LLDP-style). If that would collapse distinct rows
-    (OSPF/VRRP/config where the same id appears on many interfaces), keep
-    iface columns so identity compare stays correct.
+    ``ignore_port_changes``:
+      - ``None`` (default): auto — drop iface from match key only when remaining
+        keys stay unique on both sides (legacy LLDP-style heuristic).
+      - ``True``: force drop iface from match key when a non-empty candidate exists.
+      - ``False``: never drop iface from match key.
+
+    Duplicate match keys are not silently discarded: extras become ``duplicate``
+    diffs and ``summary.duplicate_key_list`` lists the colliding keys.
 
     ``field_rules`` drives normalize / numeric tolerance / per-field compare mode
     (template-driven; no metric-specific branches here).
@@ -127,20 +139,25 @@ def compare_rows(
         after_rows, iface_fields=iface_list, rules=norm_rules
     )
 
-    ignore_port_changes = False
+    ignore_ports = False
     # No map → optionally ignore port renames by dropping iface from match key.
     if not pmap and iface_set:
         candidate = [k for k in key_fields if k not in iface_set]
         if not candidate:
             match_keys = list(key_fields)
+        elif ignore_port_changes is True:
+            match_keys = candidate
+            ignore_ports = True
+        elif ignore_port_changes is False:
+            match_keys = list(key_fields)
         else:
+            # Auto heuristic (legacy default)
             before_c = [row_key(r, candidate) for r in before_norm]
             after_c = [row_key(r, candidate) for r in after_norm]
             if len(before_c) == len(set(before_c)) and len(after_c) == len(set(after_c)):
                 match_keys = candidate
-                ignore_port_changes = True
+                ignore_ports = True
             else:
-                # Remaining keys are not unique — iface is required for identity.
                 match_keys = list(key_fields)
     else:
         match_keys = list(key_fields)
@@ -151,16 +168,22 @@ def compare_rows(
 
     after_index: dict[tuple[str, ...], dict[str, Any]] = {}
     after_dup = 0
+    after_dup_keys: list[tuple[str, ...]] = []
+    after_dup_rows: list[tuple[tuple[str, ...], dict[str, Any]]] = []
     for r in after_norm:
         k = row_key(r, match_keys)
         if k in after_index:
             after_dup += 1
+            after_dup_keys.append(k)
+            after_dup_rows.append((k, r))
+            continue  # first wins — do not overwrite
         after_index[k] = r
 
     before_keys: set[tuple[str, ...]] = set()
     before_dup = 0
+    before_dup_keys: list[tuple[str, ...]] = []
     diffs: list[dict[str, Any]] = []
-    added = removed = changed = unchanged = 0
+    added = removed = changed = unchanged = duplicate = 0
 
     def _key_obj(row: dict[str, Any]) -> dict[str, Any]:
         return {f: row.get(f, "") for f in key_fields}
@@ -169,6 +192,20 @@ def compare_rows(
         k = row_key(mapped, match_keys)
         if k in before_keys:
             before_dup += 1
+            before_dup_keys.append(k)
+            duplicate += 1
+            diffs.append(
+                {
+                    "kind": "duplicate",
+                    "side": "before",
+                    "key": _key_obj(mapped),
+                    "before": orig,
+                    "after": after_index.get(k),
+                    "mapped_before": mapped,
+                    "changes": {},
+                }
+            )
+            continue  # only first before row participates in match
         before_keys.add(k)
         after = after_index.get(k)
         if after is None:
@@ -237,6 +274,30 @@ def compare_rows(
             }
         )
 
+    for k, after in after_dup_rows:
+        duplicate += 1
+        diffs.append(
+            {
+                "kind": "duplicate",
+                "side": "after",
+                "key": _key_obj(after),
+                "before": None,
+                "after": after,
+                "mapped_before": None,
+                "changes": {},
+            }
+        )
+
+    def _fmt_keys(keys: list[tuple[str, ...]]) -> list[str]:
+        seen: set[str] = set()
+        out: list[str] = []
+        for k in keys:
+            s = "|".join(k)
+            if s not in seen:
+                seen.add(s)
+                out.append(s)
+        return out
+
     stats = mapping_stats(
         before_rows=before_norm,
         after_rows=after_norm,
@@ -244,7 +305,7 @@ def compare_rows(
         port_map=pmap,
     )
     if not pmap:
-        stats = {**stats, "ok": True, "ignore_port_changes": ignore_port_changes}
+        stats = {**stats, "ok": True, "ignore_port_changes": ignore_ports}
     return {
         "summary": {
             "before_count": len(before_rows),
@@ -253,9 +314,11 @@ def compare_rows(
             "removed": removed,
             "changed": changed,
             "unchanged": unchanged,
+            "duplicate": duplicate,
             "match_key_fields": match_keys,
             "duplicate_keys_before": before_dup,
             "duplicate_keys_after": after_dup,
+            "duplicate_key_list": _fmt_keys(before_dup_keys + after_dup_keys),
         },
         "diffs": diffs,
         "mapping_stats": stats,

netx_api/biz_state/compare_service.py

@@ -4,6 +4,8 @@ from __future__ import annotations
 
 import io
 import json
+import logging
+import threading
 import zipfile
 from datetime import datetime
 from typing import Any
@@ -20,6 +22,7 @@ from ..models import (
     BizPortMapping,
     BizPortMappingRow,
     BizStateBatch,
+    BizStateBatchCommand,
     BizStateLldpNeighbor,
     BizStateTask,
 )
@@ -39,11 +42,69 @@ from .iface_normalize import (
 )
 from .profiles import metric_field_map
 
+_log = logging.getLogger("netx.biz_state.compare")
+
+# Per-job mutex so auto + manual run_compare do not dual-write the same job.
+_auto_compare_locks: dict[str, threading.Lock] = {}
+_auto_compare_locks_guard = threading.Lock()
+
+
+def _job_compare_lock(job_id: str) -> threading.Lock:
+    jid = str(job_id or "").strip()
+    with _auto_compare_locks_guard:
+        lock = _auto_compare_locks.get(jid)
+        if lock is None:
+            lock = threading.Lock()
+            _auto_compare_locks[jid] = lock
+        return lock
+
 
 def _utcnow() -> datetime:
     return utcnow_naive()
 
 
+# parse_status values that mean the command produced usable (possibly empty) rows
+_METRIC_OK_STATUSES = frozenset(
+    {"ok", "unmatched", "skipped_custom", "ok_aux", "unmatched_aux"}
+)
+_METRIC_FAIL_STATUSES = frozenset({"failed", "failed_aux"})
+
+
+def batch_metric_collect_ok(db: Session, batch_id: str, metric_id: str) -> bool:
+    """True when this metric is safe to use from the batch.
+
+    - Batch ``success``: OK (all commands finished).
+    - Batch ``partial``/other: require at least one non-failed command for the metric.
+    - No command row for the metric on a non-success batch → incomplete.
+    """
+    bid = str(batch_id or "").strip()
+    mid = str(metric_id or "").strip()
+    if not bid or not mid:
+        return False
+    batch = db.get(BizStateBatch, bid)
+    if not batch:
+        return False
+    status = str(batch.status or "")
+    if status == "success":
+        return True
+    cmds = (
+        db.query(BizStateBatchCommand)
+        .filter(
+            BizStateBatchCommand.batch_id == bid,
+            BizStateBatchCommand.metric_id == mid,
+        )
+        .all()
+    )
+    if not cmds:
+        return False
+    statuses = [str(c.parse_status or "").strip().lower() for c in cmds]
+    if any(s in _METRIC_OK_STATUSES or s.startswith("ok") for s in statuses):
+        return True
+    if all(s in _METRIC_FAIL_STATUSES or s.startswith("failed") for s in statuses):
+        return False
+    # Unknown status with rows still present — allow; empty unknown on partial — deny
+    return any(int(c.row_count or 0) > 0 for c in cmds)
+
 def _compare_side(
     db: Session,
     batch_id: str,
@@ -314,6 +375,7 @@ def _sheet_def(
     display_fields: list[str] | None = None,
     row_filters: list[dict[str, Any]] | None = None,
     field_rules: list[dict[str, Any]] | None = None,
+    ignore_port_changes: bool | None = None,
 ) -> dict[str, Any]:
     mid = str(metric_id or "").strip()
     sid = str(sheet_id or "").strip() or mid
@@ -352,6 +414,8 @@ def _sheet_def(
         "row_filters": _normalize_row_filters(row_filters),
         "field_rules": rules,
     }
+    if ignore_port_changes is not None:
+        sheet["ignore_port_changes"] = bool(ignore_port_changes)
     return sheet
 
 
@@ -553,6 +617,9 @@ def _normalize_sheet(raw: Any) -> dict[str, Any] | None:
         disp_arg = _str_list(raw.get("display_fields"))
     else:
         disp_arg = None
+    ignore_ports: bool | None = None
+    if "ignore_port_changes" in raw and raw.get("ignore_port_changes") is not None:
+        ignore_ports = bool(raw.get("ignore_port_changes"))
     return _sheet_def(
         metric_id=mid,
         sheet_id=str(raw.get("sheet_id") or "").strip() or mid,
@@ -563,6 +630,7 @@ def _normalize_sheet(raw: Any) -> dict[str, Any] | None:
         display_fields=disp_arg,
         row_filters=_normalize_row_filters(raw.get("row_filters")),
         field_rules=rules,
+        ignore_port_changes=ignore_ports,
     )
 
 
@@ -1368,7 +1436,7 @@ def _resolve_after_batch(db: Session, job: BizCompareJob) -> str:
         db.query(BizStateBatch)
         .filter(
             BizStateBatch.task_id == task_id,
-            BizStateBatch.status.in_(("success", "partial")),
+            BizStateBatch.status == "success",
         )
         .order_by(BizStateBatch.started_at.desc())
         .first()
@@ -1401,6 +1469,9 @@ def _run_sheet(
     )
     row_filters = list(sheet.get("row_filters") or [])
     mode = "presence" if not compare_fields else "fields"
+    ignore_ports = sheet.get("ignore_port_changes")
+    if ignore_ports is not None:
+        ignore_ports = bool(ignore_ports)
     before_raw = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"])
     after_raw = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"])
     before_rows = apply_row_filters(before_raw, row_filters)
@@ -1414,6 +1485,7 @@ def _run_sheet(
         port_map=port_map,
         field_rules=field_rules,
         iface_normalize_rules=iface_normalize_rules,
+        ignore_port_changes=ignore_ports,
     )
     summary = dict(result["summary"])
     summary["before_raw_count"] = len(before_raw)
@@ -1429,6 +1501,7 @@ def _run_sheet(
         "display_fields": display_fields,
         "row_filters": row_filters,
         "field_rules": field_rules,
+        "ignore_port_changes": ignore_ports,
         "mode": mode,
         "summary": summary,
         "diffs": result["diffs"],
@@ -1437,6 +1510,18 @@ def _run_sheet(
 
 
 def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]:
+    lock = _job_compare_lock(job_id)
+    if not lock.acquire(blocking=False):
+        raise HTTPException(status_code=409, detail="compare_already_running")
+    try:
+        return _run_compare_unlocked(db, job_id, force_after_batch_id=force_after_batch_id)
+    finally:
+        lock.release()
+
+
+def _run_compare_unlocked(
+    db: Session, job_id: str, *, force_after_batch_id: str = ""
+) -> dict[str, Any]:
     j = db.get(BizCompareJob, job_id)
     if not j:
         raise HTTPException(status_code=404, detail="job_not_found")
@@ -1467,6 +1552,7 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
         "removed": 0,
         "changed": 0,
         "unchanged": 0,
+        "duplicate": 0,
     }
     mapping_by_metric: dict[str, Any] = {}
     for sheet in sheets_cfg:
@@ -1513,6 +1599,7 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
                 "compare_fields": s["compare_fields"],
                 "display_fields": s.get("display_fields") or [],
                 "field_rules": s.get("field_rules") or [],
+                "ignore_port_changes": s.get("ignore_port_changes"),
                 "mode": s["mode"],
                 "summary": s["summary"],
             }
@@ -1957,7 +2044,14 @@ def list_runs(db: Session, job_id: str, *, limit: int = 20) -> list[dict[str, An
 
 
 def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int:
-    """When a new after batch lands, run auto jobs pinned to that after task."""
+    """When a new *success* after batch lands, run auto jobs pinned to that task.
+
+    Skips jobs already being compared (non-blocking lock). Call from a background
+    thread so the collect finalize path is not blocked.
+    """
+    batch = db.get(BizStateBatch, batch_id)
+    if not batch or str(batch.status or "") != "success":
+        return 0
     jobs = (
         db.query(BizCompareJob)
         .filter(BizCompareJob.mode == "auto", BizCompareJob.after_task_id == task_id)
@@ -1970,6 +2064,34 @@ def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int:
         try:
             run_compare(db, j.id, force_after_batch_id=batch_id)
             n += 1
+        except HTTPException as exc:
+            if int(getattr(exc, "status_code", 0) or 0) == 409:
+                _log.info("auto compare skipped (busy) job=%s", j.id)
+            continue
         except Exception:
+            _log.exception("auto compare failed job=%s task=%s", j.id, task_id)
             continue
     return n
+
+
+def schedule_auto_compare_for_task(task_id: str, batch_id: str) -> None:
+    """Fire-and-forget auto compare on a daemon thread (own DB session)."""
+
+    def _run() -> None:
+        from ..db import SessionLocal
+
+        db = SessionLocal()
+        try:
+            try_auto_compare_for_task(db, task_id, batch_id)
+        except Exception:
+            _log.exception(
+                "bg auto compare failed task=%s batch=%s", task_id, batch_id
+            )
+        finally:
+            db.close()
+
+    threading.Thread(
+        target=_run,
+        name=f"biz-auto-cmp-{str(batch_id)[:8]}",
+        daemon=True,
+    ).start()

netx_api/biz_state/iface_normalize.py

@@ -108,7 +108,11 @@ def apply_iface_normalize_rows(
 
 
 def resolve_mapped_iface(name: str, port_map: Mapping[str, str] | None) -> str:
-    """Exact map hit, else parent + ``.suffix`` if parent mapped, else unchanged."""
+    """Exact map hit, else longest mapped parent + remaining suffix (QinQ-safe).
+
+    ``gei-0/1/0/1.100.200`` tries exact, then ``gei-0/1/0/1.100``, then
+    ``gei-0/1/0/1`` — not only the last ``.`` segment.
+    """
     text = str(name or "").strip()
     if not text or not port_map:
         return text
@@ -116,11 +120,14 @@ def resolve_mapped_iface(name: str, port_map: Mapping[str, str] | None) -> str:
         return str(port_map[text])
     if "." not in text:
         return text
-    parent, suffix = text.rsplit(".", 1)
-    if not parent or not suffix:
-        return text
-    if parent in port_map:
-        return f"{port_map[parent]}.{suffix}"
+    parts = text.split(".")
+    for i in range(len(parts) - 1, 0, -1):
+        parent = ".".join(parts[:i])
+        suffix = ".".join(parts[i:])
+        if not parent or not suffix:
+            continue
+        if parent in port_map:
+            return f"{port_map[parent]}.{suffix}"
     return text
 
 

netx_api/biz_state/parse_pool.py

@@ -0,0 +1,181 @@
+"""Parse pool: TextFSM / normalize off the collect (SSH) threads."""
+
+from __future__ import annotations
+
+import logging
+import queue
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from ..config import settings
+
+_log = logging.getLogger("netx.biz_state.parse")
+
+_SENTINEL = object()
+
+
+@dataclass
+class AuxRawCapture:
+    """CLI-only aux capture passed to the parse worker."""
+
+    key: str
+    aux_id: str
+    profile_id: str
+    parser_id: str
+    metric_id: str
+    command: str
+    textfsm_command: str
+    rule_keys: tuple[str, ...] = ()
+    raw: str = ""
+    raw_rel_path: str = ""
+    cache_hit: bool = False
+    # Pre-parsed records when cache already had a full parse hit.
+    records: list[dict[str, Any]] = field(default_factory=list)
+    fsm_tables: dict[str, list[dict[str, Any]]] = field(default_factory=dict)
+    ok: bool = True
+    error: str = ""
+
+
+@dataclass
+class PrimaryParseJob:
+    """One primary command (with aux raws) ready for CPU parse + spool."""
+
+    batch_id: str
+    cmd_id: str
+    task_item_id: str
+    profile_id: str
+    parser_id: str
+    metric_id: str
+    concrete: str
+    merged_params: dict[str, Any]
+    raw_text: str
+    raw_rel_path: str
+    textfsm_command: str
+    vendor: str
+    device_type: str
+    enrich_joins: list[Any] = field(default_factory=list)
+    aux_captures: list[AuxRawCapture] = field(default_factory=list)
+    # Shared across lanes for aux metric de-dupe (may be None).
+    persisted: set[tuple[str, str]] | None = None
+    cache_lock: Any | None = None
+    on_done: Callable[[bool, bool], None] | None = None
+
+
+class ParsePool:
+    """Background workers that parse + enqueue persist for primary jobs."""
+
+    def __init__(self, *, workers: int | None = None) -> None:
+        n = max(
+            1,
+            int(
+                workers
+                if workers is not None
+                else (getattr(settings, "biz_state_parse_workers", 8) or 8)
+            ),
+        )
+        self._q: queue.Queue[Any] = queue.Queue()
+        self._inflight = 0
+        self._lock = threading.Lock()
+        self._cv = threading.Condition(self._lock)
+        self._workers: list[threading.Thread] = []
+        self._stopped = False
+        for i in range(n):
+            t = threading.Thread(
+                target=self._loop,
+                name=f"biz-parse-{i}",
+                daemon=True,
+            )
+            t.start()
+            self._workers.append(t)
+        _log.info("biz_state parse pool started workers=%s", n)
+
+    def submit(self, job: PrimaryParseJob) -> None:
+        if self._stopped or job is None:
+            return
+        with self._cv:
+            self._inflight += 1
+        self._q.put(job)
+
+    def wait_idle(self, *, timeout: float | None = None) -> bool:
+        end = None
+        if timeout is not None:
+            end = time.monotonic() + max(0.0, float(timeout))
+        with self._cv:
+            while self._inflight > 0 or not self._q.empty():
+                remaining = None
+                if end is not None:
+                    remaining = end - time.monotonic()
+                    if remaining <= 0:
+                        return False
+                self._cv.wait(timeout=remaining)
+            return True
+
+    def shutdown(self, *, wait: bool = True) -> None:
+        self._stopped = True
+        for _ in self._workers:
+            self._q.put(_SENTINEL)
+        if wait:
+            for t in self._workers:
+                t.join(timeout=5.0)
+
+    def _loop(self) -> None:
+        from .collect_runner import _run_primary_parse_job
+
+        while True:
+            job = self._q.get()
+            if job is _SENTINEL:
+                self._q.task_done()
+                break
+            ok = False
+            fail = False
+            try:
+                ok, fail = _run_primary_parse_job(job)
+            except Exception:
+                fail = True
+                _log.exception(
+                    "biz_state parse job failed batch=%s cmd=%s",
+                    getattr(job, "batch_id", ""),
+                    getattr(job, "cmd_id", ""),
+                )
+                try:
+                    if job.on_done:
+                        job.on_done(False, True)
+                except Exception:
+                    pass
+            else:
+                try:
+                    if job.on_done:
+                        job.on_done(ok, fail)
+                except Exception:
+                    _log.exception("biz_state parse on_done failed")
+            finally:
+                with self._cv:
+                    self._inflight = max(0, self._inflight - 1)
+                    self._cv.notify_all()
+                self._q.task_done()
+
+
+_pool_lock = threading.Lock()
+_pool: ParsePool | None = None
+
+
+def parse_async_enabled() -> bool:
+    return bool(getattr(settings, "biz_state_parse_async", True))
+
+
+def get_parse_pool() -> ParsePool:
+    global _pool
+    with _pool_lock:
+        if _pool is None:
+            _pool = ParsePool()
+        return _pool
+
+
+def shutdown_parse_pool(*, wait: bool = False) -> None:
+    global _pool
+    with _pool_lock:
+        if _pool is not None:
+            _pool.shutdown(wait=wait)
+            _pool = None

netx_api/biz_state/parsers/zte/bgp_route.py

@@ -13,21 +13,22 @@ from .bgp_peer import _detect_bgp_afi, _detect_local_as, _detect_vrf
 
 RULE_KEYS = ("zte_zxros_show_bgp_neighbor_routes",)
 
-# Single-line IPv4-style: * 10.1.0.0/24  10.0.0.1  … path
+# Status codes may be glued or spaced: "*", "* i", "*>i", "* s", "< *"
+_FLAGS = r"(?P<flags>(?:[*<>isd]+(?:\s+[*<>isd]+)*)?)"
+# Single-line IPv4-style: * i  10.1.0.0/24  10.0.0.1  … path
 _ROUTE_ONE_LINE_RE = re.compile(
-    r"^\s*(?P<flags>[*<>isd]*)\s*"
+    rf"^\s*{_FLAGS}\s+"
     r"(?P<net>\S+)\s+"
     r"(?P<nh>\S+)\s+"
     r"(?P<rest>.*)$"
 )
-# Network alone (often IPv6 wrap): * 2407::1/128   or bare prefix for "out"
-_NET_ONLY_RE = re.compile(
-    r"^\s*(?P<flags>[*<>isd]*)\s*(?P<net>\S+)\s*$"
-)
+# Network alone (often IPv6 wrap): * i  2407::1/128
+_NET_ONLY_RE = re.compile(rf"^\s*{_FLAGS}\s*(?P<net>\S+)\s*$")
 _DIR_RE = re.compile(r"(?i)\bneighbor\s+(in|out)\s+")
 # Neighbor may be IPv4 or IPv6 (consume until EOL / pipe)
 _NEI_RE = re.compile(r"(?i)\bneighbor\s+(?:in|out)\s+(\S+)")
 _TOTAL_RE = re.compile(r"(?i)total\s+number\s+of\s+routes\s*:\s*(\d+)")
+_RD_RE = re.compile(r"(?i)^Route\s+Distinguisher\s*:\s*(\S+)")
 _HEADER_NETS = frozenset(
     {
         "network",
@@ -80,6 +81,21 @@ def _looks_like_prefix(net: str) -> bool:
     return _looks_like_ip_or_prefix(net)
 
 
+def _declared_total(raw_text: str) -> int | None:
+    m = _TOTAL_RE.search(str(raw_text or ""))
+    if not m:
+        return None
+    try:
+        return int(m.group(1))
+    except ValueError:
+        return None
+
+
+def _route_dedupe_key(*, rd: str, net: str, nh: str) -> str:
+    """ECMP / multi-RD: same prefix with different RD or next-hop is distinct."""
+    return f"{rd}\0{net}\0{nh}"
+
+
 def _split_rest(rest: str, *, path_continuation: bool = False) -> tuple[str, str, str, str]:
     """Parse trailing Metric LocPrf Tag/RtPrf Path columns (some may be blank).
 
@@ -118,8 +134,8 @@ def _split_rest(rest: str, *, path_continuation: bool = False) -> tuple[str, str
 
 def _empty_if_total_zero(raw_text: str) -> bool:
     """True when device reports Total number of routes: 0."""
-    m = _TOTAL_RE.search(str(raw_text or ""))
-    return bool(m and int(m.group(1)) == 0)
+    total = _declared_total(raw_text)
+    return total == 0
 
 
 def _skip_noise_line(line: str) -> bool:
@@ -134,9 +150,7 @@ def _skip_noise_line(line: str) -> bool:
         return True
     if low.startswith("local ") or low.startswith("remote ") or low.startswith("total "):
         return True
-    if low.startswith("route distinguisher") or low.startswith("valid ") or low.startswith(
-        "invalid "
-    ):
+    if low.startswith("valid ") or low.startswith("invalid "):
         return True
     # Banner / clock lines (e.g. "09:50:02 Indonesia Sat Sep 19 2026")
     if re.match(r"^\d{1,2}:\d{2}:\d{2}\b", low):
@@ -157,19 +171,24 @@ def _emit_route(
     vrf: str,
     neighbor: str,
     direction: str,
+    rd: str,
     net: str,
     nh: str,
     rest: str,
     flags: str,
     path_continuation: bool = False,
 ) -> None:
-    if not _looks_like_prefix(net) or net in seen:
+    if not _looks_like_prefix(net):
         return
     if nh and not _looks_like_ip_or_prefix(nh):
         # Path/metric-only continuation without a real next-hop — keep empty nh
         if re.search(r"[A-Za-z]", nh):
             return
-    seen.add(net)
+        nh = ""
+    key = _route_dedupe_key(rd=rd, net=net, nh=nh or "")
+    if key in seen:
+        return
+    seen.add(key)
     metric, loc, tag, path = _split_rest(rest, path_continuation=path_continuation)
     out.append(
         {
@@ -178,13 +197,14 @@ def _emit_route(
             "vrf": vrf[:128],
             "neighbor": neighbor[:128],
             "direction": direction[:8],
+            "rd": (rd or "")[:64],
             "network": net[:128],
             "next_hop": (nh or "")[:128],
             "metric": metric[:32],
             "loc_prf": loc[:32],
             "tag": tag[:32],
             "path": path[:256],
-            "status_codes": (flags or "").strip()[:16],
+            "status_codes": re.sub(r"\s+", "", (flags or "").strip())[:16],
             "as_num": "",
             "state": "",
             "pfx_rcd": "",
@@ -205,7 +225,7 @@ def _map_fsm_rows(
     seen: set[str] = set()
     for r in rows:
         net = row_get(r, "NETWORK", "network")
-        if not _looks_like_prefix(net) or net in seen:
+        if not _looks_like_prefix(net):
             continue
         nh = row_get(r, "NEXT_HOP", "next_hop")
         if str(nh or "").strip().lower() in _HEADER_NETS:
@@ -213,8 +233,13 @@ def _map_fsm_rows(
         if nh and not _looks_like_ip_or_prefix(nh):
             # Reject FSM false hits like NETWORK=20 NEXT_HOP=65254
             continue
-        seen.add(net)
+        rd = row_get(r, "RD", "rd")
+        key = _route_dedupe_key(rd=rd, net=net, nh=nh or "")
+        if key in seen:
+            continue
+        seen.add(key)
         path = row_get(r, "PATH", "path")
+        flags = row_get(r, "STATUS", "status_codes", "FLAGS")
         out.append(
             {
                 "local_as": local_as[:16],
@@ -222,13 +247,14 @@ def _map_fsm_rows(
                 "vrf": vrf[:128],
                 "neighbor": neighbor[:128],
                 "direction": direction[:8],
+                "rd": (rd or "")[:64],
                 "network": net[:128],
                 "next_hop": nh[:128],
                 "metric": row_get(r, "METRIC", "metric")[:32],
                 "loc_prf": row_get(r, "LOC_PRF", "loc_prf")[:32],
                 "tag": row_get(r, "TAG", "RT_PRF", "tag")[:32],
                 "path": path[:256],
-                "status_codes": "",
+                "status_codes": re.sub(r"\s+", "", (flags or "").strip())[:16],
                 "as_num": "",
                 "state": "",
                 "pfx_rcd": "",
@@ -253,6 +279,7 @@ def _hand_parse(
     pending_net = ""
     pending_flags = ""
     pending_nh = ""
+    current_rd = ""
 
     def _flush_pending(*, rest: str = "", path_continuation: bool = False) -> None:
         nonlocal pending_net, pending_flags, pending_nh
@@ -266,6 +293,7 @@ def _hand_parse(
             vrf=vrf,
             neighbor=neighbor,
             direction=direction,
+            rd=current_rd,
             net=pending_net,
             nh=pending_nh,
             rest=rest,
@@ -278,6 +306,11 @@ def _hand_parse(
 
     for raw in str(raw_text or "").splitlines():
         line = raw.rstrip()
+        rd_m = _RD_RE.match(line.strip())
+        if rd_m:
+            _flush_pending()
+            current_rd = rd_m.group(1).strip()
+            continue
         if _skip_noise_line(line):
             continue
 
@@ -299,7 +332,7 @@ def _hand_parse(
                 _flush_pending(rest=tok, path_continuation=True)
                 continue
 
-        # Full one-liner (typical IPv4)
+        # Full one-liner (typical IPv4 / RR "* i  prefix …")
         m = _ROUTE_ONE_LINE_RE.match(line)
         if m and _looks_like_prefix(m.group("net")) and _looks_like_ip_or_prefix(m.group("nh")):
             _flush_pending()
@@ -311,6 +344,7 @@ def _hand_parse(
                 vrf=vrf,
                 neighbor=neighbor,
                 direction=direction,
+                rd=current_rd,
                 net=m.group("net"),
                 nh=m.group("nh"),
                 rest=m.group("rest"),
@@ -382,7 +416,14 @@ def normalize_bgp_route(
             **kw,
         )
 
-    return prefer_fsm(tables, RULE_KEYS, _map, _hand, raw_text=raw_text)
+    rows = prefer_fsm(tables, RULE_KEYS, _map, _hand, raw_text=raw_text)
+    # If device declared a large table but FSM/hand returned a tiny subset, prefer hand.
+    declared = _declared_total(raw_text)
+    if declared is not None and declared > 0 and len(rows) < max(1, declared // 2):
+        hand_rows = _hand(raw_text=raw_text)
+        if len(hand_rows) > len(rows):
+            return hand_rows
+    return rows
 
 
 normalize_bgp_route.RULE_KEYS = RULE_KEYS

netx_api/biz_state/profiles.py

@@ -535,8 +535,9 @@ _BGP_ROUTE_FIELDS: list[FieldDef] = [
     FieldDef("vrf", length=128, indexed=True, is_key=True, display_name="VRF", from_command_param=True),
     FieldDef("neighbor", length=128, indexed=True, is_key=True, display_name="Neighbor", from_command_param=True),
     FieldDef("direction", length=8, indexed=True, is_key=True, display_name="Dir", from_command_param=True),
+    FieldDef("rd", length=64, indexed=True, is_key=True, display_name="RD"),
     FieldDef("network", length=128, indexed=True, is_key=True, display_name="Network"),
-    FieldDef("next_hop", length=128, role="state", display_name="NextHop"),
+    FieldDef("next_hop", length=128, indexed=True, is_key=True, display_name="NextHop"),
     FieldDef("metric", length=32, role="meta", display_name="Metric"),
     FieldDef("loc_prf", length=32, role="meta", display_name="LocPrf"),
     FieldDef("tag", length=32, role="meta", display_name="Tag"),

netx_api/biz_state/schema_ensure.py

@@ -79,6 +79,8 @@ def apply_biz_state_schema(conn: Connection) -> None:
         "ALTER TABLE biz_migration_project ADD COLUMN IF NOT EXISTS new_hf_bindings_json JSON DEFAULT '[]'",
         "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS purpose VARCHAR(32) DEFAULT ''",
         "CREATE INDEX IF NOT EXISTS ix_biz_state_task_purpose ON biz_state_task (purpose)",
+        "ALTER TABLE biz_migration_red_ticket ADD COLUMN IF NOT EXISTS match_key_str VARCHAR(256) DEFAULT ''",
+        "CREATE INDEX IF NOT EXISTS ix_biz_migration_red_match ON biz_migration_red_ticket (project_id, metric_id, match_key_str)",
     ):
         try:
             _run_sql(conn, sql)

netx_api/biz_state/service.py

@@ -593,6 +593,14 @@ def _metric_sheet_title(metric_id: str, fallback: str = "") -> str:
     return fb or mid
 
 
+def _raw_line_count(raw: str | None) -> int:
+    """CLI text lines collected (splitlines-compatible, no list materialization)."""
+    s = raw or ""
+    if not s:
+        return 0
+    return s.count("\n") + (0 if s.endswith("\n") else 1)
+
+
 def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
     """Batch workbook summary: meta + commands + sheet catalog (no metric row payload)."""
     b = db.get(BizStateBatch, batch_id)
@@ -648,6 +656,7 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
     for c in cmds:
         status = str(c.parse_status or "").strip().lower()
         is_aux = status.startswith("aux")
+        raw = c.raw_text or ""
         info = {
             "id": c.id,
             "profile_id": c.profile_id,
@@ -657,8 +666,9 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
             "params": c.params_json or {},
             "parse_status": c.parse_status,
             "row_count": c.row_count,
+            "raw_line_count": _raw_line_count(raw),
             "message": c.message,
-            "has_raw": bool(str(c.raw_text or "").strip()),
+            "has_raw": bool(str(raw).strip()),
             "is_aux": is_aux,
         }
         cmd_n = normalize_command(str(c.raw_command or ""))
@@ -688,6 +698,7 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
                     "raw_command": c.raw_command,
                     "parse_status": c.parse_status,
                     "row_count": c.row_count,
+                    "raw_line_count": info["raw_line_count"],
                     "message": c.message,
                     "has_raw": info["has_raw"],
                     "profile_id": c.profile_id,
@@ -713,6 +724,7 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
                 "params": c.params_json or {},
                 "parse_status": c.parse_status,
                 "row_count": c.row_count,
+                "raw_line_count": _raw_line_count(c.raw_text),
                 "message": c.message,
                 "has_raw": bool(str(c.raw_text or "").strip()),
                 "is_aux": True,
@@ -906,6 +918,7 @@ def get_batch_command(db: Session, batch_id: str, command_id: str) -> dict[str,
     if not c or c.batch_id != batch_id:
         raise HTTPException(status_code=404, detail="command_not_found")
     task = db.get(BizStateTask, b.task_id) if b.task_id else None
+    raw = c.raw_text or ""
     return {
         "id": c.id,
         "batch_id": batch_id,
@@ -922,8 +935,9 @@ def get_batch_command(db: Session, batch_id: str, command_id: str) -> dict[str,
         "params": c.params_json or {},
         "parse_status": c.parse_status,
         "row_count": c.row_count,
+        "raw_line_count": _raw_line_count(raw),
         "message": c.message,
-        "raw_text": c.raw_text or "",
+        "raw_text": raw,
         "collected_at": c.created_at.isoformat() + "Z" if c.created_at else None,
         "batch_started_at": b.started_at.isoformat() + "Z" if b.started_at else None,
         "batch_ended_at": b.ended_at.isoformat() + "Z" if b.ended_at else None,