"""UME alarm normalize / upsert / notification apply.""" from __future__ import annotations import hashlib import json import re import threading import time from datetime import datetime from typing import Any from sqlalchemy import func from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from .config import settings from .models import UmeAlarmCurrent from .ume_raw import dumps_ume_raw from .ume_sync_common import _lookup_host_name, _pick, _s, _utc_now_naive def _alarm_key(alarm: dict[str, Any]) -> str: key = _s(_pick(alarm, "alarmKey", "alarm-key","alarmkey","id")) if key: # Keep full upstream key; use a stable digest only for pathological ultra-long keys. if len(key) > 512: return "sha256:" + hashlib.sha256(key.encode("utf-8", errors="ignore")).hexdigest() return key parts = [ _s(_pick(alarm, "objectName", "object-name")), _s(_pick(alarm, "eventType", "event-type")), _s(_pick(alarm, "timeCreated", "time-created")), _s(_pick(alarm, "nativeProbableCause", "native-probable-cause")), ] merged = "|".join(x for x in parts if x) raw = merged or f"fallback-{_utc_now_naive().timestamp()}" if len(raw) > 512: return "sha256:" + hashlib.sha256(raw.encode("utf-8", errors="ignore")).hexdigest() return raw def _derive_ne_id_from_alarm(alarm: dict[str, Any]) -> str: ne_id = _s(_pick(alarm, "ne-id", "neId", "ne_id")) if ne_id: return ne_id def _uuid_like(s: str) -> bool: return bool( re.fullmatch( r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}", str(s or "").strip(), ) ) # Prefer UUID net_id in objectName, e.g. "... ME{33a3e8f4-a76e-40fd-a0ba-045371a5f234} ..." object_name = _s(_pick(alarm, "objectName", "object-name")) if object_name: m_obj = re.search(r"ME\{([^}]+)\}", object_name, flags=re.IGNORECASE) if m_obj: candidate = _s(m_obj.group(1)) if _uuid_like(candidate): return candidate alarm_key = _s(_pick(alarm, "alarmKey", "alarm-key", "alarmkey")) if not alarm_key: return "" # If alarmkey contains ME{...}, only accept it when it looks like a UUID. m0 = re.search(r"ME\{([^}]+)\}", alarm_key, flags=re.IGNORECASE) if m0: candidate = _s(m0.group(1)) if _uuid_like(candidate): return candidate # Common UME formats observed: # 1) "#" # 2) ", , " # 3) " " if "#" in alarm_key: candidate = _s(alarm_key.split("#", 1)[0]) return candidate if _uuid_like(candidate) else "" if "," in alarm_key: candidate = _s(alarm_key.split(",", 1)[0]) return candidate if _uuid_like(candidate) else "" parts = [p for p in alarm_key.split() if p] if len(parts) >= 2 and ":" not in parts[0]: candidate = _s(parts[0]) return candidate if _uuid_like(candidate) else "" return "" def _normalize_yang_key(key: str) -> str: raw = str(key or "").strip() if not raw: return "" if ":" in raw: return raw.rsplit(":", 1)[-1] return raw def normalize_yang_alarm(raw: dict[str, Any]) -> dict[str, Any]: """Flatten YANG namespace-prefixed keys (e.g. zte-alarms:alarmkey) for _pick().""" out: dict[str, Any] = {} for k, v in raw.items(): nk = _normalize_yang_key(str(k)) if not nk: continue if nk in out and out[nk] not in (None, ""): continue out[nk] = v return out def _is_alarm_cleared(alarm: dict[str, Any]) -> bool: val = _pick(alarm, "isCleared", "is-cleared") if isinstance(val, bool): return val text = _s(val).lower() return text in {"true", "1", "yes"} _cleared_tombstone_lock = threading.Lock() _cleared_tombstones: dict[str, float] = {} def _mark_alarm_cleared_tombstone(alarm_key: str) -> None: key = str(alarm_key or "").strip() if not key: return ttl_s = max(60, int(getattr(settings, "ume_alarm_cleared_tombstone_s", 300) or 300)) expires = time.time() + ttl_s with _cleared_tombstone_lock: _cleared_tombstones[key] = expires now = time.time() # Always sweep expired keys; hard-cap with oldest-first eviction. stale = [k for k, exp in _cleared_tombstones.items() if exp <= now] for k in stale: _cleared_tombstones.pop(k, None) max_keys = 50000 if len(_cleared_tombstones) > max_keys: overflow = len(_cleared_tombstones) - max_keys oldest = sorted(_cleared_tombstones.items(), key=lambda kv: kv[1])[:overflow] for k, _ in oldest: _cleared_tombstones.pop(k, None) def _is_alarm_cleared_tombstone(alarm_key: str) -> bool: key = str(alarm_key or "").strip() if not key: return False now = time.time() with _cleared_tombstone_lock: exp = _cleared_tombstones.get(key) if exp is None: return False if exp <= now: _cleared_tombstones.pop(key, None) return False return True def notification_id_from_norm(norm: dict[str, Any]) -> str: return _s(_pick(norm, "notificationId", "notification-id")) def _alarm_row_from_norm(key: str, norm: dict[str, Any], *, touch_ts: datetime, first_seen_at: datetime) -> dict[str, Any]: return { "alarm_key": key, "ne_id": _s(_derive_ne_id_from_alarm(norm)), "host_name": "", "object_name": _s(_pick(norm, "objectName", "object-name")), "event_type": _s(_pick(norm, "eventType", "event-type")), "native_probable_cause": _s(_pick(norm, "nativeProbableCause", "native-probable-cause")), "perceived_severity": _s(_pick(norm, "perceivedSeverity", "perceived-severity")), "is_cleared": _s(_pick(norm, "isCleared", "is-cleared")), "time_created": _s(_pick(norm, "timeCreated", "time-created")), "root_cause_alarm_indication": _s( _pick(norm, "rootCauseAlarmIndication", "root-cause-alarm-indication") ), "notification_id": notification_id_from_norm(norm), "first_seen_at": first_seen_at, "last_seen_at": touch_ts, "raw_json": dumps_ume_raw(norm), } def _apply_row_to_model(db: Session, existing: UmeAlarmCurrent, norm: dict[str, Any], *, touch_ts: datetime) -> None: existing.ne_id = _s(_derive_ne_id_from_alarm(norm)) existing.object_name = _s(_pick(norm, "objectName", "object-name")) existing.event_type = _s(_pick(norm, "eventType", "event-type")) existing.native_probable_cause = _s(_pick(norm, "nativeProbableCause", "native-probable-cause")) existing.perceived_severity = _s(_pick(norm, "perceivedSeverity", "perceived-severity")) existing.is_cleared = _s(_pick(norm, "isCleared", "is-cleared")) existing.time_created = _s(_pick(norm, "timeCreated", "time-created")) existing.root_cause_alarm_indication = _s( _pick(norm, "rootCauseAlarmIndication", "root-cause-alarm-indication") ) existing.notification_id = notification_id_from_norm(norm) prev_seen = existing.last_seen_at if prev_seen is None or touch_ts >= prev_seen: existing.last_seen_at = touch_ts existing.raw_json = dumps_ume_raw(norm) def _upsert_alarm_current(db: Session, key: str, norm: dict[str, Any], *, touch_ts: datetime) -> tuple[str, bool]: bind = db.get_bind() dialect = str(getattr(getattr(bind, "dialect", None), "name", "") or "").lower() existing = db.get(UmeAlarmCurrent, key) if existing is not None: _apply_row_to_model(db, existing, norm, touch_ts=touch_ts) existing.host_name = _lookup_host_name(db, existing.ne_id) return "updated", True if dialect == "postgresql": row = _alarm_row_from_norm(key, norm, touch_ts=touch_ts, first_seen_at=touch_ts) row["host_name"] = _lookup_host_name(db, row["ne_id"]) ins = pg_insert(UmeAlarmCurrent).values(**row) excluded = ins.excluded stmt = ins.on_conflict_do_update( index_elements=[UmeAlarmCurrent.alarm_key], set_={ "ne_id": excluded.ne_id, "host_name": excluded.host_name, "object_name": excluded.object_name, "event_type": excluded.event_type, "native_probable_cause": excluded.native_probable_cause, "perceived_severity": excluded.perceived_severity, "is_cleared": excluded.is_cleared, "time_created": excluded.time_created, "root_cause_alarm_indication": excluded.root_cause_alarm_indication, "notification_id": excluded.notification_id, "last_seen_at": func.greatest(UmeAlarmCurrent.last_seen_at, excluded.last_seen_at), "raw_json": excluded.raw_json, }, ) db.execute(stmt) return "inserted", True try: with db.begin_nested(): model = UmeAlarmCurrent(alarm_key=key, first_seen_at=touch_ts) db.add(model) db.flush() _apply_row_to_model(db, model, norm, touch_ts=touch_ts) model.host_name = _lookup_host_name(db, model.ne_id) return "inserted", True except IntegrityError: existing = db.get(UmeAlarmCurrent, key) if existing is None: return "skipped", False _apply_row_to_model(db, existing, norm, touch_ts=touch_ts) existing.host_name = _lookup_host_name(db, existing.ne_id) return "updated", True def extract_alarm_from_notification(payload: dict[str, Any]) -> dict[str, Any] | None: """Parse alarm-notification from a WS/REST notification envelope.""" if not isinstance(payload, dict): return None def _find_alarm_notification(node: Any) -> dict[str, Any] | None: if isinstance(node, dict): for k, v in node.items(): key = str(k).lower() if key in {"alarm-notification", "alarm_notification"} and isinstance(v, dict): return normalize_yang_alarm(v) found = _find_alarm_notification(v) if found is not None: return found elif isinstance(node, list): for item in node: found = _find_alarm_notification(item) if found is not None: return found return None direct = _find_alarm_notification(payload) if direct is not None: return direct return normalize_yang_alarm(payload) if payload else None def apply_alarm_to_current( db: Session, alarm: dict[str, Any], *, touch_ts: datetime, source: str = "", ) -> tuple[str, bool]: """ Apply one alarm to ume_alarms_current. Returns (action, changed) where action is inserted|updated|deleted|skipped. """ norm = normalize_yang_alarm(alarm) if alarm else {} if not norm: return "skipped", False if _is_alarm_cleared(norm): key = _alarm_key(norm) if not key: return "skipped", False existing = db.get(UmeAlarmCurrent, key) if existing is None: _mark_alarm_cleared_tombstone(key) return "deleted", False db.delete(existing) _mark_alarm_cleared_tombstone(key) return "deleted", True key = _alarm_key(norm) if not key: return "skipped", False if str(source or "").strip().lower() == "rest" and _is_alarm_cleared_tombstone(key): return "skipped", False return _upsert_alarm_current(db, key, norm, touch_ts=touch_ts)