Stream biz_state metric persist to cut peak memory on large BGP tables.

Write JSONL iteratively, flush via chunked PG execute_values, skip mega raw in DB, and yield bgp_route rows instead of building a full list.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-09-23 16:23:55 +08:00
parent 08231c779e
commit 9b6452971f
7 changed files with 423 additions and 136 deletions

View file

@ -34,6 +34,7 @@ Complex joins that cannot be expressed as equal-field copy still go in
from __future__ import annotations
import inspect
from collections.abc import Iterable
from typing import Any, Callable, Mapping, Sequence
from ...lldp_shared import resolve_vendor_key
@ -231,6 +232,11 @@ def run_parser(
"params": params or {},
}
records = fn(**filtered)
if not isinstance(records, list):
# Allow list or streaming Iterable (generator); reject bare str/bytes.
if records is None:
records = []
elif isinstance(records, (str, bytes)):
records = []
elif not isinstance(records, list) and not isinstance(records, Iterable):
records = []
return records, fsm_tables, used_keys