hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-09 03:10:46 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 2

fix(netx): avoid alarm sync failures from long UME fields

Browse source 
Store UME alarm payload fields as TEXT and run startup ALTER COLUMN upgrades so large alarm records no longer fail with StringDataRightTruncation. Keep full values during sync while hashing only pathological ultra-long alarm keys to preserve idempotent primary keys.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-05-13 15:59:20 +08:00
parent b253ee7bff
commit eebfbf25fa
3 changed files with 42 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

17
netx_api/main.py
View file

@ -550,6 +550,23 @@ def on_startup() -> None:
conn.exec_driver_sql("ALTER TABLE ume_inventory_ne ADD COLUMN IF NOT EXISTS net_mask VARCHAR(128) DEFAULT ''")
conn.exec_driver_sql("ALTER TABLE ume_inventory_ne ADD COLUMN IF NOT EXISTS create_time VARCHAR(64) DEFAULT ''")
conn.exec_driver_sql("ALTER TABLE ume_inventory_ne ADD COLUMN IF NOT EXISTS creator VARCHAR(128) DEFAULT ''")
# Allow long UME alarm fields; avoid StringDataRightTruncation on large payloads.
conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN alarm_key TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN object_name TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN event_type TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN native_probable_cause TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN perceived_severity TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN is_cleared TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN time_created TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_current ALTER COLUMN root_cause_alarm_indication TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN alarm_key TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN object_name TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN event_type TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN native_probable_cause TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN perceived_severity TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN is_cleared TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN time_created TYPE TEXT")
conn.exec_driver_sql("ALTER TABLE ume_alarms_history ALTER COLUMN root_cause_alarm_indication TYPE TEXT")
# Simplify alarm tables: display fields come from runtime join with inventory table.
conn.exec_driver_sql("ALTER TABLE ume_alarms_current DROP COLUMN IF EXISTS ne_name")
conn.exec_driver_sql("ALTER TABLE ume_alarms_current DROP COLUMN IF EXISTS user_label")

32
netx_api/models.py
View file

@ -166,15 +166,15 @@ class UmeInventoryNE(Base):
class UmeAlarmCurrent(Base):
__tablename__ = "ume_alarms_current"
alarm_key: Mapped[str] = mapped_column(String(256), primary_key=True)
alarm_key: Mapped[str] = mapped_column(Text, primary_key=True)
ne_id: Mapped[str] = mapped_column(String(128), default="", index=True)
object_name: Mapped[str] = mapped_column(String(512), default="", index=True)
event_type: Mapped[str] = mapped_column(String(128), default="")
native_probable_cause: Mapped[str] = mapped_column(String(256), default="")
perceived_severity: Mapped[str] = mapped_column(String(64), default="", index=True)
is_cleared: Mapped[str] = mapped_column(String(16), default="", index=True)
time_created: Mapped[str] = mapped_column(String(64), default="", index=True)
root_cause_alarm_indication: Mapped[str] = mapped_column(String(32), default="")
object_name: Mapped[str] = mapped_column(Text, default="", index=True)
event_type: Mapped[str] = mapped_column(Text, default="")
native_probable_cause: Mapped[str] = mapped_column(Text, default="")
perceived_severity: Mapped[str] = mapped_column(Text, default="", index=True)
is_cleared: Mapped[str] = mapped_column(Text, default="", index=True)
time_created: Mapped[str] = mapped_column(Text, default="", index=True)
root_cause_alarm_indication: Mapped[str] = mapped_column(Text, default="")
first_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
last_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, index=True)
raw_json: Mapped[str] = mapped_column(Text, default="{}")
@ -183,15 +183,15 @@ class UmeAlarmCurrent(Base):
class UmeAlarmHistory(Base):
__tablename__ = "ume_alarms_history"
alarm_key: Mapped[str] = mapped_column(String(256), primary_key=True)
alarm_key: Mapped[str] = mapped_column(Text, primary_key=True)
ne_id: Mapped[str] = mapped_column(String(128), default="", index=True)
object_name: Mapped[str] = mapped_column(String(512), default="", index=True)
event_type: Mapped[str] = mapped_column(String(128), default="")
native_probable_cause: Mapped[str] = mapped_column(String(256), default="")
perceived_severity: Mapped[str] = mapped_column(String(64), default="", index=True)
is_cleared: Mapped[str] = mapped_column(String(16), default="", index=True)
time_created: Mapped[str] = mapped_column(String(64), default="", index=True)
root_cause_alarm_indication: Mapped[str] = mapped_column(String(32), default="")
object_name: Mapped[str] = mapped_column(Text, default="", index=True)
event_type: Mapped[str] = mapped_column(Text, default="")
native_probable_cause: Mapped[str] = mapped_column(Text, default="")
perceived_severity: Mapped[str] = mapped_column(Text, default="", index=True)
is_cleared: Mapped[str] = mapped_column(Text, default="", index=True)
time_created: Mapped[str] = mapped_column(Text, default="", index=True)
root_cause_alarm_indication: Mapped[str] = mapped_column(Text, default="")
first_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
last_seen_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, index=True)
raw_json: Mapped[str] = mapped_column(Text, default="{}")

11
netx_api/ume_sync_service.py
View file

@ -2,6 +2,7 @@ from __future__ import annotations
import json
import logging
import hashlib
from datetime import datetime, timezone
from typing import Any
@ -43,6 +44,9 @@ def _pick(d: dict[str, Any], *keys: str) -> Any:
def _alarm_key(alarm: dict[str, Any]) -> str:
key = _s(_pick(alarm, "alarmKey", "alarm-key","alarmkey","id"))
if key:
# Keep full upstream key; use a stable digest only for pathological ultra-long keys.
if len(key) > 512:
return "sha256:" + hashlib.sha256(key.encode("utf-8", errors="ignore")).hexdigest()
return key
parts = [
_s(_pick(alarm, "objectName", "object-name")),
@ -51,7 +55,10 @@ def _alarm_key(alarm: dict[str, Any]) -> str:
_s(_pick(alarm, "nativeProbableCause", "native-probable-cause")),
]
merged = "|".join(x for x in parts if x)
return merged or f"fallback-{datetime.utcnow().timestamp()}"
raw = merged or f"fallback-{datetime.utcnow().timestamp()}"
if len(raw) > 512:
return "sha256:" + hashlib.sha256(raw.encode("utf-8", errors="ignore")).hexdigest()
return raw
def _derive_ne_id_from_alarm(alarm: dict[str, Any]) -> str:
@ -338,7 +345,7 @@ def _sync_alarms_common(
inserted += 1
else:
updated += 1
existing.ne_id = _derive_ne_id_from_alarm(alarm)
existing.ne_id = _s(_derive_ne_id_from_alarm(alarm))
existing.object_name = _s(_pick(alarm, "objectName", "object-name"))
existing.event_type = _s(_pick(alarm, "eventType", "event-type"))
existing.native_probable_cause = _s(_pick(alarm, "nativeProbableCause", "native-probable-cause"))