Make biz compare template-driven with field rules, board UX, and import/export.

Operators pick metrics then Key/Compare/Display rules; seed built-ins only once; fail-first verdict board for cutover review.

Co-authored-by: Cursor <cursoragent@cursor.com>

netx_api/biz_state/compare_engine.py

@@ -2,7 +2,9 @@
 
 from __future__ import annotations
 
-from typing import Any
+from typing import Any, Mapping, Sequence
+
+from .compare_rules import field_rule_map, values_equal, explain_diff
 
 
 def apply_port_map(
@@ -71,6 +73,7 @@ def compare_rows(
     iface_fields: list[str],
     compare_fields: list[str],
     port_map: dict[str, str] | None = None,
+    field_rules: Sequence[Mapping[str, Any]] | None = None,
 ) -> dict[str, Any]:
     """Return summary + diffs list.
 
@@ -78,10 +81,14 @@ def compare_rows(
 
     Empty ``port_map`` means ignore port/iface changes: matching keys exclude
     ``iface_fields`` (same-device batches compare without a rename map).
+
+    ``field_rules`` drives normalize / numeric tolerance / per-field compare mode
+    (template-driven; no metric-specific branches here).
     """
     if not key_fields:
         raise ValueError("key_fields required")
     pmap = dict(port_map or {})
+    rules = field_rule_map(field_rules)
     iface_set = {str(f) for f in (iface_fields or []) if str(f).strip()}
     # No map → ignore port renames: drop iface columns from the match key.
     if not pmap and iface_set:
@@ -129,9 +136,13 @@ def compare_rows(
         for f in compare_fields:
             bv = mapped.get(f, "")
             av = after.get(f, "")
-            # Compare using mapped before for iface fields already rewritten
-            if str(bv) != str(av):
-                field_changes[f] = {"before": bv, "after": av}
+            rule = rules.get(f)
+            if not values_equal(bv, av, rule=rule):
+                entry: dict[str, Any] = {"before": bv, "after": av}
+                reason = explain_diff(bv, av, rule=rule)
+                if reason:
+                    entry["reason"] = reason
+                field_changes[f] = entry
         if field_changes:
             changed += 1
             diffs.append(

netx_api/biz_state/compare_rules.py

@@ -0,0 +1,289 @@
+"""Template-driven compare rules: row filters + per-field compare/normalize.
+
+All metric-specific compare behavior belongs in the sheet template
+(``row_filters`` / ``field_rules``), not in hardcoded service branches.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Mapping, Sequence
+
+
+_AGE_TIMER_RE = re.compile(r"^\d{1,2}:\d{2}:\d{2}$")
+
+
+def _as_list(raw: Any) -> list[Any]:
+    if raw is None:
+        return []
+    if isinstance(raw, (list, tuple)):
+        return list(raw)
+    return [raw]
+
+
+def _field_val(row: Mapping[str, Any], field: str) -> str:
+    return str((row or {}).get(field) or "").strip()
+
+
+def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool:
+    """Evaluate one leaf predicate. Unknown ops → True (do not drop)."""
+    field = str(filt.get("field") or "").strip()
+    op = str(filt.get("op") or "eq").strip().lower()
+    if not field and op not in ("any", "all"):
+        return True
+    raw = _field_val(row, field)
+    expect = filt.get("value")
+
+    if op in ("eq", "=="):
+        return raw.lower() == str(expect or "").strip().lower()
+    if op in ("ne", "!="):
+        return raw.lower() != str(expect or "").strip().lower()
+    if op == "in":
+        opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()}
+        return raw.lower() in opts
+    if op in ("not_in", "nin"):
+        opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()}
+        return raw.lower() not in opts
+    if op == "empty":
+        return not raw
+    if op in ("not_empty", "nonempty"):
+        return bool(raw)
+    if op == "regex":
+        pat = str(expect or "")
+        if not pat:
+            return True
+        try:
+            return bool(re.search(pat, raw, re.I))
+        except re.error:
+            return True
+    if op == "age_timer":
+        # HH:MM:SS dynamic ARP age
+        return bool(_AGE_TIMER_RE.match(raw))
+    if op == "ci_eq":
+        return raw.lower() == str(expect or "").strip().lower()
+    return True
+
+
+def row_matches_filter(row: Mapping[str, Any], filt: Mapping[str, Any] | None) -> bool:
+    if not filt or not isinstance(filt, dict):
+        return True
+    if "any" in filt:
+        kids = filt.get("any") or []
+        if not isinstance(kids, list) or not kids:
+            return True
+        return any(row_matches_filter(row, k) for k in kids if isinstance(k, dict))
+    if "all" in filt:
+        kids = filt.get("all") or []
+        if not isinstance(kids, list) or not kids:
+            return True
+        return all(row_matches_filter(row, k) for k in kids if isinstance(k, dict))
+    return eval_leaf_filter(row, filt)
+
+
+def apply_row_filters(
+    rows: Sequence[Mapping[str, Any]],
+    filters: Sequence[Mapping[str, Any]] | None,
+) -> list[dict[str, Any]]:
+    """Keep rows matching all top-level filters (AND). Nested any/all supported."""
+    fl = [f for f in (filters or []) if isinstance(f, dict)]
+    out: list[dict[str, Any]] = []
+    for r in rows or []:
+        if not isinstance(r, dict):
+            continue
+        if all(row_matches_filter(r, f) for f in fl):
+            out.append(dict(r))
+    return out
+
+
+def normalize_value(value: Any, how: str) -> str:
+    text = str(value if value is not None else "").strip()
+    mode = str(how or "").strip().lower()
+    if not mode or mode in ("none", "strip"):
+        return text
+    if mode == "lower":
+        return text.lower()
+    if mode == "upper":
+        return text.upper()
+    if mode == "mac":
+        # 0011.2233.4455 / 00-11-22-33-44-55 / 00:11:… → lowercase hex only
+        hex_only = re.sub(r"[^0-9a-fA-F]", "", text).lower()
+        return hex_only
+    if mode == "empty_as_blank":
+        if text.lower() in ("n/a", "na", "-", "--", "none", "null"):
+            return ""
+        return text
+    return text
+
+
+def field_rule_map(rules: Sequence[Mapping[str, Any]] | None) -> dict[str, dict[str, Any]]:
+    out: dict[str, dict[str, Any]] = {}
+    for raw in rules or []:
+        if not isinstance(raw, dict):
+            continue
+        name = str(raw.get("field") or "").strip()
+        if not name:
+            continue
+        out[name] = dict(raw)
+    return out
+
+
+def effective_compare_fields(
+    compare_fields: Sequence[str],
+    field_rules: Sequence[Mapping[str, Any]] | None,
+) -> list[str]:
+    """Drop fields marked compare=ignore (or legacy ignore:true)."""
+    rules = field_rule_map(field_rules)
+    out: list[str] = []
+    for f in compare_fields or []:
+        name = str(f).strip()
+        if not name:
+            continue
+        rule = rules.get(name) or {}
+        mode = str(rule.get("compare") or "").strip().lower()
+        if mode in ("ignore", "skip", "off"):
+            continue
+        if rule.get("ignore") is True:
+            continue
+        out.append(name)
+    return out
+
+
+def effective_display_fields(
+    *,
+    key_fields: Sequence[str],
+    compare_fields: Sequence[str],
+    display_fields: Sequence[str] | None = None,
+) -> list[str]:
+    """Result-table columns: explicit display, else key+compare (legacy).
+
+    Keys always lead; remaining display/compare fields follow in given order
+    without duplicates.
+    """
+    keys = [str(x).strip() for x in (key_fields or []) if str(x).strip()]
+    key_set = set(keys)
+    compare = [str(x).strip() for x in (compare_fields or []) if str(x).strip()]
+    raw_disp = display_fields
+    if raw_disp is None:
+        # Legacy templates: show key + compare only
+        extra = [f for f in compare if f not in key_set]
+        return keys + extra
+    disp = [str(x).strip() for x in raw_disp if str(x).strip()]
+    # Force keys first (always visible)
+    out: list[str] = list(keys)
+    seen = set(keys)
+    for f in disp:
+        if f in seen:
+            continue
+        out.append(f)
+        seen.add(f)
+    # Ensure compare fields appear even if UI forgot to tick display
+    for f in compare:
+        if f in seen or f in key_set:
+            continue
+        out.append(f)
+        seen.add(f)
+    return out
+
+
+def _parse_float(text: str) -> float | None:
+    try:
+        return float(text) if text else 0.0
+    except ValueError:
+        return None
+
+
+def explain_diff(
+    before: Any,
+    after: Any,
+    *,
+    rule: Mapping[str, Any] | None = None,
+) -> str:
+    """Human-readable reason when values_equal is False (for UI / export)."""
+    rule = rule or {}
+    norm = str(rule.get("normalize") or "strip").strip().lower() or "strip"
+    bv = normalize_value(before, norm)
+    av = normalize_value(after, norm)
+    mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
+    if mode in ("percent", "pct", "rel"):
+        bn = _parse_float(bv)
+        an = _parse_float(av)
+        if bn is None or an is None:
+            return "neq"
+        if bn == 0.0:
+            return "pct_base_zero"
+        try:
+            t = float(rule.get("tolerance") or 0)
+        except (TypeError, ValueError):
+            t = 0.0
+        pct = abs(an - bn) / abs(bn) * 100.0
+        return f"pct {pct:.1f}% > {t:g}%"
+    if mode in ("numeric", "number", "int", "float"):
+        bn = _parse_float(bv)
+        an = _parse_float(av)
+        if bn is None or an is None:
+            return "neq"
+        try:
+            t = float(rule.get("tolerance") or 0)
+        except (TypeError, ValueError):
+            t = 0.0
+        delta = abs(an - bn)
+        return f"abs Δ{delta:g} > {t:g}"
+    return "neq"
+
+
+def values_equal(
+    before: Any,
+    after: Any,
+    *,
+    rule: Mapping[str, Any] | None = None,
+) -> bool:
+    rule = rule or {}
+    norm = str(rule.get("normalize") or "strip").strip().lower() or "strip"
+    bv = normalize_value(before, norm)
+    av = normalize_value(after, norm)
+    mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
+    if mode in ("ignore", "skip", "off"):
+        return True
+    if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"):
+        bn = _parse_float(bv)
+        an = _parse_float(av)
+        if bn is None or an is None:
+            return bv == av
+        tol = rule.get("tolerance", 0)
+        try:
+            t = float(tol or 0)
+        except (TypeError, ValueError):
+            t = 0.0
+        if mode in ("percent", "pct", "rel"):
+            # Relative % vs before: |a-b|/max(|b|,eps)*100 <= tol
+            # before==0: both zero → ok; else fail (undefined relative base)
+            if bn == 0.0:
+                return an == 0.0
+            pct = abs(an - bn) / abs(bn) * 100.0
+            return pct <= t
+        return abs(bn - an) <= t
+    return bv == av
+
+
+def arp_dynamic_row_filters() -> list[dict[str, Any]]:
+    """Canonical ARP compare filter (replaces hardcoded service filter)."""
+    return [
+        {
+            "any": [
+                {"field": "entry_type", "op": "eq", "value": "dynamic"},
+                {
+                    "all": [
+                        {"field": "entry_type", "op": "empty"},
+                        {"field": "age", "op": "age_timer"},
+                    ]
+                },
+            ]
+        }
+    ]
+
+
+# Presets for seeding defaults (not a UI "apply preset" button)
+ROW_FILTER_PRESETS: dict[str, list[dict[str, Any]]] = {
+    "arp_dynamic": arp_dynamic_row_filters(),
+    "bgp_established": [{"field": "state", "op": "eq", "value": "Established"}],
+}

netx_api/biz_state/compare_service.py

@@ -24,6 +24,13 @@ from ..models import (
 )
 from ..timeutil import utcnow_naive
 from .compare_engine import compare_rows, mapping_stats
+from .compare_rules import (
+    ROW_FILTER_PRESETS,
+    apply_row_filters,
+    arp_dynamic_row_filters,
+    effective_compare_fields,
+    effective_display_fields,
+)
 from .profiles import metric_field_map
 
 
@@ -158,12 +165,58 @@ def _str_list(raw: Any) -> list[str]:
     return [str(x).strip() for x in (raw or []) if str(x).strip()]
 
 
+def _normalize_row_filters(raw: Any) -> list[dict[str, Any]]:
+    if not isinstance(raw, list):
+        return []
+    out: list[dict[str, Any]] = []
+    for item in raw:
+        if isinstance(item, dict) and item:
+            out.append(dict(item))
+    return out
+
+
+def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]:
+    if not isinstance(raw, list):
+        return []
+    out: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for item in raw:
+        if not isinstance(item, dict):
+            continue
+        name = str(item.get("field") or "").strip()
+        if not name or name in seen:
+            continue
+        seen.add(name)
+        rule: dict[str, Any] = {"field": name}
+        compare = str(item.get("compare") or "").strip().lower()
+        if compare:
+            rule["compare"] = compare
+        if item.get("ignore") is True:
+            rule["ignore"] = True
+            rule.setdefault("compare", "ignore")
+        norm = str(item.get("normalize") or "").strip().lower()
+        if norm and norm not in ("none", "strip"):
+            rule["normalize"] = norm
+        if item.get("tolerance") is not None and str(item.get("tolerance")).strip() != "":
+            try:
+                rule["tolerance"] = float(item.get("tolerance"))
+            except (TypeError, ValueError):
+                pass
+        # Drop empty rules (only field name)
+        if len(rule) > 1:
+            out.append(rule)
+    return out
+
+
 def _sheet_def(
     *,
     metric_id: str,
     key_fields: list[str],
     iface_fields: list[str] | None = None,
     compare_fields: list[str] | None = None,
+    display_fields: list[str] | None = None,
+    row_filters: list[dict[str, Any]] | None = None,
+    field_rules: list[dict[str, Any]] | None = None,
 ) -> dict[str, Any]:
     mid = str(metric_id or "").strip()
     keys = _str_list(key_fields)
@@ -173,12 +226,32 @@ def _sheet_def(
     # Keys are identity only; strip them from compare so UI/engine stay clear
     key_set = set(keys)
     compare = [f for f in compare if f not in key_set]
-    return {
+    rules = _normalize_field_rules(field_rules)
+    # Drop ignored fields from compare list (single source of truth for UI)
+    compare = effective_compare_fields(compare, rules)
+    # None = legacy (derive key+compare); explicit list (even empty extras) preserved
+    if display_fields is None:
+        display = effective_display_fields(
+            key_fields=keys,
+            compare_fields=compare,
+            display_fields=None,
+        )
+    else:
+        display = effective_display_fields(
+            key_fields=keys,
+            compare_fields=compare,
+            display_fields=_str_list(display_fields),
+        )
+    sheet: dict[str, Any] = {
         "metric_id": mid,
         "key_fields": keys,
         "iface_fields": ifaces,
         "compare_fields": compare,
+        "display_fields": display,
+        "row_filters": _normalize_row_filters(row_filters),
+        "field_rules": rules,
     }
+    return sheet
 
 
 def _default_lldp_sheet() -> dict[str, Any]:
@@ -217,11 +290,25 @@ def _default_sheet_for_metric(metric_id: str, *, compare_roles: tuple[str, ...]
     keys = [f.name for f in fields if f.is_key]
     ifaces = [f.name for f in fields if f.is_interface]
     compare = [f.name for f in fields if (not f.is_key) and f.role in compare_roles]
+    extra: dict[str, Any] = {}
+    if metric_id == "arp":
+        # Template-owned ARP filter (was hardcoded in _load_metric_rows)
+        extra["row_filters"] = arp_dynamic_row_filters()
+        # MAC often differs in format across vendors / reloads
+        extra["field_rules"] = [{"field": "mac", "normalize": "mac"}] if "mac" in {
+            f.name for f in fields
+        } else []
+        # Context columns: show but not necessarily compare
+        ctx = [n for n in ("vrf", "entry_type", "age") if n not in keys and n not in compare]
+        extra["display_fields"] = list(keys) + list(compare) + ctx
     return _sheet_def(
         metric_id=metric_id,
         key_fields=keys,
         iface_fields=ifaces,
         compare_fields=compare,
+        display_fields=extra.get("display_fields"),
+        row_filters=extra.get("row_filters"),
+        field_rules=extra.get("field_rules"),
     )
 
 
@@ -242,11 +329,27 @@ def _normalize_sheet(raw: Any) -> dict[str, Any] | None:
     keys = _str_list(raw.get("key_fields"))
     if not mid or not keys:
         return None
+    # Legacy ignore_fields → field_rules compare=ignore
+    rules = list(_normalize_field_rules(raw.get("field_rules")))
+    ignore = set(_str_list(raw.get("ignore_fields")))
+    by_field = {str(r.get("field")): r for r in rules}
+    for name in ignore:
+        if name not in by_field:
+            rules.append({"field": name, "compare": "ignore", "ignore": True})
+    # display_fields: missing key → legacy derive; present → explicit
+    disp_arg: list[str] | None
+    if "display_fields" in raw:
+        disp_arg = _str_list(raw.get("display_fields"))
+    else:
+        disp_arg = None
     return _sheet_def(
         metric_id=mid,
         key_fields=keys,
         iface_fields=_str_list(raw.get("iface_fields")),
         compare_fields=_str_list(raw.get("compare_fields")),
+        display_fields=disp_arg,
+        row_filters=_normalize_row_filters(raw.get("row_filters")),
+        field_rules=rules,
     )
 
 
@@ -257,12 +360,14 @@ def _legacy_sheets(t: BizCompareTemplate) -> list[dict[str, Any]]:
         return []
     ignore = set(_str_list(t.ignore_fields))
     compare = [f for f in _str_list(t.compare_fields) if f not in ignore]
+    rules = [{"field": f, "compare": "ignore", "ignore": True} for f in sorted(ignore)]
     return [
         _sheet_def(
             metric_id=mid,
             key_fields=keys,
             iface_fields=_str_list(t.iface_fields),
             compare_fields=compare,
+            field_rules=rules,
         )
     ]
 
@@ -329,12 +434,25 @@ def _parse_metrics_body(body: dict[str, Any]) -> list[dict[str, Any]]:
         raise HTTPException(status_code=400, detail="key_fields_required")
     ignore = set(_str_list(body.get("ignore_fields")))
     compare = [f for f in _str_list(body.get("compare_fields")) if f not in ignore]
+    rules = _normalize_field_rules(body.get("field_rules"))
+    by_field = {str(r.get("field")): r for r in rules}
+    for name in ignore:
+        if name not in by_field:
+            rules.append({"field": name, "compare": "ignore", "ignore": True})
+    disp_arg: list[str] | None
+    if "display_fields" in body:
+        disp_arg = _str_list(body.get("display_fields"))
+    else:
+        disp_arg = None
     return [
         _sheet_def(
             metric_id=mid,
             key_fields=keys,
             iface_fields=_str_list(body.get("iface_fields")),
             compare_fields=compare,
+            display_fields=disp_arg,
+            row_filters=_normalize_row_filters(body.get("row_filters")),
+            field_rules=rules,
         )
     ]
 
@@ -451,8 +569,25 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate:
         existing = template_metrics(row)
         want = {s["metric_id"] for s in sheets}
         have = {s["metric_id"] for s in existing}
+        changed = bool(want - have)
+        # Migrate ARP sheet: inject template row_filters if missing (replaces code filter)
+        upgraded: list[dict[str, Any]] = []
+        by_want = {s["metric_id"]: s for s in sheets}
+        for s in existing:
+            cur = dict(s)
+            if cur.get("metric_id") == "arp" and not cur.get("row_filters"):
+                cur["row_filters"] = list(by_want.get("arp", {}).get("row_filters") or arp_dynamic_row_filters())
+                if not cur.get("field_rules") and by_want.get("arp", {}).get("field_rules"):
+                    cur["field_rules"] = list(by_want["arp"]["field_rules"])
+                changed = True
+            upgraded.append(_normalize_sheet(cur) or cur)
         if want - have:
-            _apply_sheets_to_row(row, sheets)
+            for s in sheets:
+                if s["metric_id"] not in have:
+                    upgraded.append(s)
+            changed = True
+        if changed:
+            _apply_sheets_to_row(row, upgraded if upgraded else sheets)
             row.note = "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP)"
             row.updated_at = _utcnow()
             db.commit()
@@ -473,6 +608,13 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate:
 
 
 def ensure_default_templates(db: Session) -> None:
+    """Seed built-ins only on first install (empty table).
+
+    Operators own templates after that — do not recreate deleted built-ins
+    or overwrite customized sheets on every list call.
+    """
+    if db.query(BizCompareTemplate.id).limit(1).first():
+        return
     ensure_default_cutover_template(db)
     ensure_default_lldp_template(db)
     ensure_default_vrf_template(db)
@@ -510,6 +652,14 @@ def list_metric_schemas() -> list[dict[str, Any]]:
     return out
 
 
+def list_row_filter_presets() -> list[dict[str, Any]]:
+    """Named row_filter bundles for the template UI (ARP dynamic, BGP Established, …)."""
+    return [
+        {"id": pid, "label": pid, "row_filters": filters}
+        for pid, filters in ROW_FILTER_PRESETS.items()
+    ]
+
+
 def create_template(db: Session, body: dict[str, Any]) -> dict[str, Any]:
     sheets = _parse_metrics_body(body)
     t = BizCompareTemplate(
@@ -533,7 +683,20 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict
         t.name = str(body.get("name") or "")[:256]
     if "note" in body:
         t.note = str(body.get("note") or "")[:512]
-    if any(k in body for k in ("metrics", "metric_id", "key_fields", "iface_fields", "compare_fields", "ignore_fields")):
+    if any(
+        k in body
+        for k in (
+            "metrics",
+            "metric_id",
+            "key_fields",
+            "iface_fields",
+            "compare_fields",
+            "ignore_fields",
+            "display_fields",
+            "row_filters",
+            "field_rules",
+        )
+    ):
         # Prefer explicit metrics; otherwise merge into current sheets from legacy keys
         if "metrics" in body and body.get("metrics") is not None:
             sheets = _parse_metrics_body(body)
@@ -557,8 +720,18 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict
                     first["iface_fields"] = _str_list(body.get("iface_fields"))
                 if "compare_fields" in body or "ignore_fields" in body:
                     ignore = set(_str_list(body.get("ignore_fields"))) if "ignore_fields" in body else set()
-                    compare = _str_list(body.get("compare_fields")) if "compare_fields" in body else list(first.get("compare_fields") or [])
+                    compare = (
+                        _str_list(body.get("compare_fields"))
+                        if "compare_fields" in body
+                        else list(first.get("compare_fields") or [])
+                    )
                     first["compare_fields"] = [f for f in compare if f not in ignore]
+                if "display_fields" in body:
+                    first["display_fields"] = _str_list(body.get("display_fields"))
+                if "row_filters" in body:
+                    first["row_filters"] = _normalize_row_filters(body.get("row_filters"))
+                if "field_rules" in body:
+                    first["field_rules"] = _normalize_field_rules(body.get("field_rules"))
                 sheets[0] = _normalize_sheet(first) or first
         _apply_sheets_to_row(t, sheets)
     t.updated_at = _utcnow()
@@ -704,21 +877,9 @@ def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dic
         .all()
     )
     if rows:
-        out = [dict(r.data_json or {}) for r in rows]
-        if metric_id == "arp":
-            from .parsers.zte.arp import is_valid_arp_age
-
-            # Compare only dynamic ARP (Age is HH:MM:SS); drop static H / incomplete flags
-            out = [
-                r
-                for r in out
-                if str(r.get("entry_type") or "").lower() == "dynamic"
-                or (
-                    not r.get("entry_type")
-                    and is_valid_arp_age(str(r.get("age") or ""))
-                )
-            ]
-        return out
+        # Raw rows only — filtering belongs to the compare sheet template
+        # (``row_filters``), not metric-specific branches here.
+        return [dict(r.data_json or {}) for r in rows]
     # Known metric with zero rows is OK; unknown metric still errors
     if metric_id in metric_field_map():
         return []
@@ -873,10 +1034,24 @@ def _run_sheet(
 ) -> dict[str, Any]:
     key_fields = list(sheet.get("key_fields") or [])
     iface_fields = list(sheet.get("iface_fields") or [])
-    compare_fields = list(sheet.get("compare_fields") or [])
+    field_rules = list(sheet.get("field_rules") or [])
+    compare_fields = effective_compare_fields(
+        list(sheet.get("compare_fields") or []),
+        field_rules,
+    )
+    display_fields = effective_display_fields(
+        key_fields=key_fields,
+        compare_fields=compare_fields,
+        display_fields=list(sheet.get("display_fields"))
+        if "display_fields" in sheet
+        else None,
+    )
+    row_filters = list(sheet.get("row_filters") or [])
     mode = "presence" if not compare_fields else "fields"
-    before_rows = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"])
-    after_rows = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"])
+    before_raw = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"])
+    after_raw = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"])
+    before_rows = apply_row_filters(before_raw, row_filters)
+    after_rows = apply_row_filters(after_raw, row_filters)
     result = compare_rows(
         before_rows=before_rows,
         after_rows=after_rows,
@@ -884,14 +1059,22 @@ def _run_sheet(
         iface_fields=iface_fields,
         compare_fields=compare_fields,
         port_map=port_map,
+        field_rules=field_rules,
     )
+    summary = dict(result["summary"])
+    summary["before_raw_count"] = len(before_raw)
+    summary["after_raw_count"] = len(after_raw)
+    summary["row_filters"] = len(row_filters)
     return {
         "metric_id": sheet["metric_id"],
         "key_fields": key_fields,
         "iface_fields": iface_fields,
         "compare_fields": compare_fields,
+        "display_fields": display_fields,
+        "row_filters": row_filters,
+        "field_rules": field_rules,
         "mode": mode,
-        "summary": result["summary"],
+        "summary": summary,
         "diffs": result["diffs"],
         "mapping_stats": result["mapping_stats"],
     }
@@ -965,6 +1148,8 @@ def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") ->
                 "key_fields": s["key_fields"],
                 "iface_fields": s["iface_fields"],
                 "compare_fields": s["compare_fields"],
+                "display_fields": s.get("display_fields") or [],
+                "field_rules": s.get("field_rules") or [],
                 "mode": s["mode"],
                 "summary": s["summary"],
             }
@@ -1012,11 +1197,23 @@ def _csv_cell(v: Any) -> str:
 
 def _sheet_csv(sheet: dict[str, Any]) -> str:
     keys = list(sheet.get("key_fields") or [])
+    key_set = set(keys)
     compare = list(sheet.get("compare_fields") or [])
+    compare_set = set(compare)
+    display = effective_display_fields(
+        key_fields=keys,
+        compare_fields=compare,
+        display_fields=list(sheet.get("display_fields") or []) or None,
+    )
+    # Non-key display columns: compare fields get pre/post; display-only get single value col
+    extra = [f for f in display if f not in key_set]
     headers = ["kind", *keys]
-    for f in compare:
-        headers.append(f"{f}__pre")
-        headers.append(f"{f}__post")
+    for f in extra:
+        if f in compare_set:
+            headers.append(f"{f}__pre")
+            headers.append(f"{f}__post")
+        else:
+            headers.append(f)
     lines = [",".join(_csv_cell(h) for h in headers)]
     for d in list(sheet.get("diffs") or []):
         kind = str(d.get("kind") or "")
@@ -1026,16 +1223,23 @@ def _sheet_csv(sheet: dict[str, Any]) -> str:
         row = [kind]
         for k in keys:
             row.append(key.get(k, pre.get(k, post.get(k, ""))))
-        for f in compare:
-            if kind == "added":
-                row.append("")
-                row.append(post.get(f, ""))
-            elif kind == "removed":
-                row.append(pre.get(f, ""))
-                row.append("")
+        for f in extra:
+            if f in compare_set:
+                if kind == "added":
+                    row.append("")
+                    row.append(post.get(f, ""))
+                elif kind == "removed":
+                    row.append(pre.get(f, ""))
+                    row.append("")
+                else:
+                    row.append(pre.get(f, ""))
+                    row.append(post.get(f, ""))
             else:
-                row.append(pre.get(f, ""))
-                row.append(post.get(f, ""))
+                # Display-only: prefer after, then before
+                if kind == "removed":
+                    row.append(pre.get(f, ""))
+                else:
+                    row.append(post.get(f, pre.get(f, "")))
         lines.append(",".join(_csv_cell(x) for x in row))
     return "\ufeff" + "\n".join(lines) + "\n"
 
@@ -1125,6 +1329,8 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]:
             "key_fields": list(sh.get("key_fields") or []),
             "iface_fields": list(sh.get("iface_fields") or []),
             "compare_fields": list(sh.get("compare_fields") or []),
+            "display_fields": list(sh.get("display_fields") or []),
+            "field_rules": list(sh.get("field_rules") or []),
             "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"),
             "summary": dict(sh.get("summary") or {}),
         }

netx_api/biz_state_router.py

@@ -212,11 +212,34 @@ def api_export_batch(batch_id: str, db: Session = Depends(get_db)) -> StreamingR
 from .biz_state import compare_service as cmp_svc  # noqa: E402
 
 
+class FieldRuleIn(BaseModel):
+    field: str
+    compare: str = ""
+    normalize: str = ""
+    ignore: bool = False
+    tolerance: float | None = None
+
+
+class RowFilterIn(BaseModel):
+    """Leaf or nested (any/all) row filter — kept open for nested dicts."""
+
+    model_config = {"extra": "allow"}
+
+    field: str | None = None
+    op: str | None = None
+    value: Any = None
+    any: list[dict[str, Any]] | None = None
+    all: list[dict[str, Any]] | None = None
+
+
 class TemplateMetricIn(BaseModel):
     metric_id: str
     key_fields: list[str] = Field(default_factory=list)
     iface_fields: list[str] = Field(default_factory=list)
     compare_fields: list[str] = Field(default_factory=list)
+    display_fields: list[str] = Field(default_factory=list)
+    row_filters: list[dict[str, Any]] = Field(default_factory=list)
+    field_rules: list[FieldRuleIn] = Field(default_factory=list)
 
 
 class TemplateIn(BaseModel):
@@ -230,6 +253,9 @@ class TemplateIn(BaseModel):
     iface_fields: list[str] = Field(default_factory=list)
     compare_fields: list[str] = Field(default_factory=list)
     ignore_fields: list[str] = Field(default_factory=list)
+    display_fields: list[str] = Field(default_factory=list)
+    row_filters: list[dict[str, Any]] = Field(default_factory=list)
+    field_rules: list[FieldRuleIn] = Field(default_factory=list)
 
 
 class TemplatePatchIn(BaseModel):
@@ -241,6 +267,9 @@ class TemplatePatchIn(BaseModel):
     iface_fields: list[str] | None = None
     compare_fields: list[str] | None = None
     ignore_fields: list[str] | None = None
+    display_fields: list[str] | None = None
+    row_filters: list[dict[str, Any]] | None = None
+    field_rules: list[FieldRuleIn] | None = None
 
 
 class MappingRowIn(BaseModel):
@@ -275,7 +304,10 @@ class CompareJobIn(BaseModel):
 
 @router.get("/compare/metrics")
 def api_list_compare_metrics() -> dict[str, Any]:
-    return {"items": cmp_svc.list_metric_schemas()}
+    return {
+        "items": cmp_svc.list_metric_schemas(),
+        "row_filter_presets": cmp_svc.list_row_filter_presets(),
+    }
 
 
 @router.get("/compare/templates")