From eebfbf25fa001396815ddcffe87a183f5d575eb7 Mon Sep 17 00:00:00 2001 From: oliver Date: Wed, 13 May 2026 15:59:20 +0800 Subject: [PATCH] fix(netx): avoid alarm sync failures from long UME fields Store UME alarm payload fields as TEXT and run startup ALTER COLUMN upgrades so large alarm records no longer fail with StringDataRightTruncation. Keep full values during sync while hashing only pathological ultra-long alarm keys to preserve idempotent primary keys. Co-authored-by: Cursor --- netx_api/main.py | 17 +++++++++++++++++ netx_api/models.py | 32 ++++++++++++++++---------------- netx_api/ume_sync_service.py | 11 +++++++++-- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/netx_api/main.py b/netx_api/main.py index 6e8a236..ea2e95e 100644 --- a/netx_api/main.py +++ b/netx_api/main.py @@ -550,6 +550,23 @@ def on_startup() -> None: conn.exec_driver_sql("ALTER TABLE ume_inventory_ne ADD COLUMN IF NOT EXISTS net_mask VARCHAR(128) DEFAULT ''") conn.exec_driver_sql("ALTER TABLE ume_inventory_ne ADD COLUMN IF NOT EXISTS create_time VARCHAR(64) DEFAULT ''") conn.exec_driver_sql("ALTER TABLE ume_inventory_ne ADD COLUMN IF NOT EXISTS creator VARCHAR(128) DEFAULT ''") + # Allow long UME alarm fields; avoid StringDataRightTruncation on large payloads. + conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN alarm_key TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN object_name TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN event_type TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN native_probable_cause TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN perceived_severity TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN is_cleared TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN time_created TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN root_cause_alarm_indication TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN alarm_key TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN object_name TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN event_type TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN native_probable_cause TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN perceived_severity TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN is_cleared TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN time_created TYPE TEXT") + conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN root_cause_alarm_indication TYPE TEXT") # Simplify alarm tables: display fields come from runtime join with inventory table. conn.exec_driver_sql("ALTER TABLE ume_alarms_current DROP COLUMN IF EXISTS ne_name") conn.exec_driver_sql("ALTER TABLE ume_alarms_current DROP COLUMN IF EXISTS user_label") diff --git a/netx_api/models.py b/netx_api/models.py index 64f858b..70a0d69 100644 --- a/netx_api/models.py +++ b/netx_api/models.py @@ -166,15 +166,15 @@ class UmeInventoryNE(Base): class UmeAlarmCurrent(Base): __tablename__ = "ume_alarms_current" - alarm_key: Mapped[str] = mapped_column(String(256), primary_key=True) + alarm_key: Mapped[str] = mapped_column(Text, primary_key=True) ne_id: Mapped[str] = mapped_column(String(128), default="", index=True) - object_name: Mapped[str] = mapped_column(String(512), default="", index=True) - event_type: Mapped[str] = mapped_column(String(128), default="") - native_probable_cause: Mapped[str] = mapped_column(String(256), default="") - perceived_severity: Mapped[str] = mapped_column(String(64), default="", index=True) - is_cleared: Mapped[str] = mapped_column(String(16), default="", index=True) - time_created: Mapped[str] = mapped_column(String(64), default="", index=True) - root_cause_alarm_indication: Mapped[str] = mapped_column(String(32), default="") + object_name: Mapped[str] = mapped_column(Text, default="", index=True) + event_type: Mapped[str] = mapped_column(Text, default="") + native_probable_cause: Mapped[str] = mapped_column(Text, default="") + perceived_severity: Mapped[str] = mapped_column(Text, default="", index=True) + is_cleared: Mapped[str] = mapped_column(Text, default="", index=True) + time_created: Mapped[str] = mapped_column(Text, default="", index=True) + root_cause_alarm_indication: Mapped[str] = mapped_column(Text, default="") first_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) last_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, index=True) raw_json: Mapped[str] = mapped_column(Text, default="{}") @@ -183,15 +183,15 @@ class UmeAlarmCurrent(Base): class UmeAlarmHistory(Base): __tablename__ = "ume_alarms_history" - alarm_key: Mapped[str] = mapped_column(String(256), primary_key=True) + alarm_key: Mapped[str] = mapped_column(Text, primary_key=True) ne_id: Mapped[str] = mapped_column(String(128), default="", index=True) - object_name: Mapped[str] = mapped_column(String(512), default="", index=True) - event_type: Mapped[str] = mapped_column(String(128), default="") - native_probable_cause: Mapped[str] = mapped_column(String(256), default="") - perceived_severity: Mapped[str] = mapped_column(String(64), default="", index=True) - is_cleared: Mapped[str] = mapped_column(String(16), default="", index=True) - time_created: Mapped[str] = mapped_column(String(64), default="", index=True) - root_cause_alarm_indication: Mapped[str] = mapped_column(String(32), default="") + object_name: Mapped[str] = mapped_column(Text, default="", index=True) + event_type: Mapped[str] = mapped_column(Text, default="") + native_probable_cause: Mapped[str] = mapped_column(Text, default="") + perceived_severity: Mapped[str] = mapped_column(Text, default="", index=True) + is_cleared: Mapped[str] = mapped_column(Text, default="", index=True) + time_created: Mapped[str] = mapped_column(Text, default="", index=True) + root_cause_alarm_indication: Mapped[str] = mapped_column(Text, default="") first_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) last_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, index=True) raw_json: Mapped[str] = mapped_column(Text, default="{}") diff --git a/netx_api/ume_sync_service.py b/netx_api/ume_sync_service.py index db4d184..dbded27 100644 --- a/netx_api/ume_sync_service.py +++ b/netx_api/ume_sync_service.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import logging +import hashlib from datetime import datetime, timezone from typing import Any @@ -43,6 +44,9 @@ def _pick(d: dict[str, Any], *keys: str) -> Any: def _alarm_key(alarm: dict[str, Any]) -> str: key = _s(_pick(alarm, "alarmKey", "alarm-key","alarmkey","id")) if key: + # Keep full upstream key; use a stable digest only for pathological ultra-long keys. + if len(key) > 512: + return "sha256:" + hashlib.sha256(key.encode("utf-8", errors="ignore")).hexdigest() return key parts = [ _s(_pick(alarm, "objectName", "object-name")), @@ -51,7 +55,10 @@ def _alarm_key(alarm: dict[str, Any]) -> str: _s(_pick(alarm, "nativeProbableCause", "native-probable-cause")), ] merged = "|".join(x for x in parts if x) - return merged or f"fallback-{datetime.utcnow().timestamp()}" + raw = merged or f"fallback-{datetime.utcnow().timestamp()}" + if len(raw) > 512: + return "sha256:" + hashlib.sha256(raw.encode("utf-8", errors="ignore")).hexdigest() + return raw def _derive_ne_id_from_alarm(alarm: dict[str, Any]) -> str: @@ -338,7 +345,7 @@ def _sync_alarms_common( inserted += 1 else: updated += 1 - existing.ne_id = _derive_ne_id_from_alarm(alarm) + existing.ne_id = _s(_derive_ne_id_from_alarm(alarm)) existing.object_name = _s(_pick(alarm, "objectName", "object-name")) existing.event_type = _s(_pick(alarm, "eventType", "event-type")) existing.native_probable_cause = _s(_pick(alarm, "nativeProbableCause", "native-probable-cause"))