Fix compare load crash from ORM yield_per with unique().

Load metric rows in LIMIT/OFFSET chunks so large-table compares no longer hit the SQLAlchemy yield_per/unique conflict.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-10-08 11:17:29 +08:00
parent 227959b9a2
commit c5efbbd016

View file

@ -1348,15 +1348,22 @@ def _port_map_dict(db: Session, mapping_id: str) -> dict[str, str]:
def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dict[str, Any]]:
"""Load metric rows with streaming fetch to avoid holding the full ORM set."""
"""Load metric rows in LIMIT/OFFSET chunks (avoid ORM yield_per + unique() clash)."""
if metric_id == "lldp_neighbor":
q = (
out: list[dict[str, Any]] = []
offset = 0
while True:
chunk = (
db.query(BizStateLldpNeighbor)
.filter(BizStateLldpNeighbor.batch_id == batch_id)
.execution_options(stream_results=True, yield_per=_LOAD_YIELD_PER)
.order_by(BizStateLldpNeighbor.id.asc())
.offset(offset)
.limit(_LOAD_YIELD_PER)
.all()
)
out: list[dict[str, Any]] = []
for n in q:
if not chunk:
break
for n in chunk:
out.append(
{
"local_if": n.local_if,
@ -1377,21 +1384,30 @@ def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dic
}
)
db.expunge(n)
offset += len(chunk)
if len(chunk) < _LOAD_YIELD_PER:
break
return out
# Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …)
from ..models import BizStateMetricRow
q = (
out = []
offset = 0
while True:
chunk = (
db.query(BizStateMetricRow)
.filter(
BizStateMetricRow.batch_id == batch_id,
BizStateMetricRow.metric_id == metric_id,
)
.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc())
.execution_options(stream_results=True, yield_per=_LOAD_YIELD_PER)
.offset(offset)
.limit(_LOAD_YIELD_PER)
.all()
)
out = []
for r in q:
if not chunk:
break
for r in chunk:
# Raw rows only — filtering belongs to the compare sheet template
# (``row_filters``), not metric-specific branches here.
# ``_netx`` is collector provenance (stripped before field compare).
@ -1411,6 +1427,9 @@ def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dic
}
)
db.expunge(r)
offset += len(chunk)
if len(chunk) < _LOAD_YIELD_PER:
break
if out:
return out
# Known metric with zero rows is OK; unknown metric still errors