mirror of
https://github.com/hansjone/netx.git
synced 2026-10-09 00:50:46 +08:00
Fix compare load crash from ORM yield_per with unique().
Load metric rows in LIMIT/OFFSET chunks so large-table compares no longer hit the SQLAlchemy yield_per/unique conflict. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
227959b9a2
commit
c5efbbd016
1 changed files with 73 additions and 54 deletions
|
|
@ -1348,15 +1348,22 @@ def _port_map_dict(db: Session, mapping_id: str) -> dict[str, str]:
|
||||||
|
|
||||||
|
|
||||||
def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dict[str, Any]]:
|
def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dict[str, Any]]:
|
||||||
"""Load metric rows with streaming fetch to avoid holding the full ORM set."""
|
"""Load metric rows in LIMIT/OFFSET chunks (avoid ORM yield_per + unique() clash)."""
|
||||||
if metric_id == "lldp_neighbor":
|
if metric_id == "lldp_neighbor":
|
||||||
q = (
|
out: list[dict[str, Any]] = []
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
chunk = (
|
||||||
db.query(BizStateLldpNeighbor)
|
db.query(BizStateLldpNeighbor)
|
||||||
.filter(BizStateLldpNeighbor.batch_id == batch_id)
|
.filter(BizStateLldpNeighbor.batch_id == batch_id)
|
||||||
.execution_options(stream_results=True, yield_per=_LOAD_YIELD_PER)
|
.order_by(BizStateLldpNeighbor.id.asc())
|
||||||
|
.offset(offset)
|
||||||
|
.limit(_LOAD_YIELD_PER)
|
||||||
|
.all()
|
||||||
)
|
)
|
||||||
out: list[dict[str, Any]] = []
|
if not chunk:
|
||||||
for n in q:
|
break
|
||||||
|
for n in chunk:
|
||||||
out.append(
|
out.append(
|
||||||
{
|
{
|
||||||
"local_if": n.local_if,
|
"local_if": n.local_if,
|
||||||
|
|
@ -1377,21 +1384,30 @@ def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dic
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
db.expunge(n)
|
db.expunge(n)
|
||||||
|
offset += len(chunk)
|
||||||
|
if len(chunk) < _LOAD_YIELD_PER:
|
||||||
|
break
|
||||||
return out
|
return out
|
||||||
# Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …)
|
# Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …)
|
||||||
from ..models import BizStateMetricRow
|
from ..models import BizStateMetricRow
|
||||||
|
|
||||||
q = (
|
out = []
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
chunk = (
|
||||||
db.query(BizStateMetricRow)
|
db.query(BizStateMetricRow)
|
||||||
.filter(
|
.filter(
|
||||||
BizStateMetricRow.batch_id == batch_id,
|
BizStateMetricRow.batch_id == batch_id,
|
||||||
BizStateMetricRow.metric_id == metric_id,
|
BizStateMetricRow.metric_id == metric_id,
|
||||||
)
|
)
|
||||||
.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc())
|
.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc())
|
||||||
.execution_options(stream_results=True, yield_per=_LOAD_YIELD_PER)
|
.offset(offset)
|
||||||
|
.limit(_LOAD_YIELD_PER)
|
||||||
|
.all()
|
||||||
)
|
)
|
||||||
out = []
|
if not chunk:
|
||||||
for r in q:
|
break
|
||||||
|
for r in chunk:
|
||||||
# Raw rows only — filtering belongs to the compare sheet template
|
# Raw rows only — filtering belongs to the compare sheet template
|
||||||
# (``row_filters``), not metric-specific branches here.
|
# (``row_filters``), not metric-specific branches here.
|
||||||
# ``_netx`` is collector provenance (stripped before field compare).
|
# ``_netx`` is collector provenance (stripped before field compare).
|
||||||
|
|
@ -1411,6 +1427,9 @@ def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dic
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
db.expunge(r)
|
db.expunge(r)
|
||||||
|
offset += len(chunk)
|
||||||
|
if len(chunk) < _LOAD_YIELD_PER:
|
||||||
|
break
|
||||||
if out:
|
if out:
|
||||||
return out
|
return out
|
||||||
# Known metric with zero rows is OK; unknown metric still errors
|
# Known metric with zero rows is OK; unknown metric still errors
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue