diff --git a/.env.example b/.env.example index 7ede38d..23d7cfd 100644 --- a/.env.example +++ b/.env.example @@ -120,8 +120,16 @@ NETX_UME_NOTIFICATION_TOPIC=ALARM # bundled | external; unset = external (compatible with existing deploys) # NETX_BUNDLED_PG_PORT=15432 # NETX_BUNDLED_PG_DATA_DIR= +# Update: GitHub primary + Forgejo mirror (git.avelo.top); pick newest reachable +# NETX_UPDATE_SOURCES=github,forgejo +# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest +# NETX_UPDATE_GITHUB_REPO=hansjone/netx # NETX_UPDATE_URL= +# NETX_UPDATE_FALLBACK_URL= +# NETX_UPDATE_TOKEN= # NETX_UPDATE_CHANNEL=stable +# NETX_UPDATE_AUTO=false +# When true: tray/scheduled task may download+apply zip updates silently. # Heavier fleets: raise CLI/DB together; also ensure Postgres max_connections and bastion session limits. # One-click start (recommended): .\scripts\start_netx.ps1 -Background -WithWeb # -> API + netx_api.worker + N× netx_api.biz_state_worker + optional Vite diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index 12e2fa9..3c8444b 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -22,10 +22,11 @@ jobs: with: python-version: "3.12" - - name: Build release zip + - name: Build release stage shell: pwsh run: | - powershell -ExecutionPolicy Bypass -File ./packaging/build_release.ps1 -CreateVenv + # Use pwsh -File (UTF-8). Nested Windows PowerShell 5.1 mis-parses UTF-8 scripts without BOM. + & ./packaging/build_release.ps1 -CreateVenv - name: Install Inno Setup shell: pwsh @@ -43,6 +44,5 @@ jobs: uses: softprops/action-gh-release@v2 with: files: | - packaging/release/NetX-*-win64.zip packaging/release/NetX-Setup-*.exe generate_release_notes: true diff --git a/.gitignore b/.gitignore index 92197b4..6720b41 100644 --- a/.gitignore +++ b/.gitignore @@ -11,13 +11,18 @@ scripts/.run/ scripts/_* scripts/archive/ packages/netx-topology-mcp/tests/_* +_tmp_* +.idea/ ume/ data/ne_collections/ data/webcrt/ data/auth/ data/runtime/ +data/ne_exec_jobs/ # Windows packaging artifacts (binaries / release trees) packaging/cache/ packaging/release/ packaging/postgres/pgsql/ +packaging/winsw/ *.exe +!packaging/installer/*.iss diff --git a/README.md b/README.md index 2ce7550..8a4d5e9 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,9 @@ API base: `http://127.0.0.1:8890/` After `npm run build` in `web/`, the API can also serve the UI at `http://127.0.0.1:8890/` (same origin). See `NETX_UI_DIST_DIR` in `.env.example`. -### Windows installer / portable package (optional) +### Windows installer (optional) -Fool-proof Windows delivery (bundled or external Postgres, program/data split, manual update) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`. +Fool-proof Windows delivery via `NetX-Setup-*.exe` (bundled or external Postgres, program/data split, offline upgrade over existing installs) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`. ### 6) MCP（Cursor / oclaw / Claude） diff --git a/netx_api/app_startup.py b/netx_api/app_startup.py index ea16cd6..8acc5f5 100644 --- a/netx_api/app_startup.py +++ b/netx_api/app_startup.py @@ -15,6 +15,7 @@ from .schema_patches import ( apply_topology_schema_safety_net, run_alembic_upgrade_to_head, ) +from .ne_crypto import ensure_credential_secret_key from .security_bootstrap import assert_secure_defaults_or_exit from .ume_runtime import start_api_sideband_threads, start_device_schedulers import netx_api.ume_support as ume_support @@ -42,6 +43,8 @@ def _configure_ume_diag_logging() -> None: def run_api_startup() -> None: """Full API boot sequence previously inlined in ``main.on_startup``.""" assert_secure_defaults_or_exit() + # Persist Fernet key for managed-NE passwords (same idea as JWT secret file). + ensure_credential_secret_key() _configure_ume_diag_logging() _log.info( "startup: ne_exec_policy_enabled=%s", @@ -156,6 +159,17 @@ def run_api_startup() -> None: ) except Exception: _log.exception("startup: biz_state collect recovery failed") + try: + from .biz_state.compare_service import recover_interrupted_compares_on_startup + + cmp_rec = recover_interrupted_compares_on_startup(db) + if cmp_rec.get("runs"): + _log.info( + "startup: cancelled %s interrupted biz compare run(s)", + cmp_rec.get("runs"), + ) + except Exception: + _log.exception("startup: biz compare recovery failed") try: from .port_traffic_migrate import backfill_port_traffic_series diff --git a/netx_api/biz_state/compare_engine.py b/netx_api/biz_state/compare_engine.py index b11cae8..3871cc4 100644 --- a/netx_api/biz_state/compare_engine.py +++ b/netx_api/biz_state/compare_engine.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections import defaultdict, deque from typing import Any, Mapping, Sequence from .compare_rules import field_rule_map, values_equal, explain_diff @@ -11,6 +12,53 @@ from .iface_normalize import ( resolve_mapped_iface, ) +# Prefer these fields when stratifying success samples (BGP multipath / multi-cmd). +_STRATUM_FIELDS = ("neighbor", "direction", "afi", "vrf") + + +def stratum_key(row: Mapping[str, Any] | None) -> str: + """Bucket key for stratified unchanged sampling.""" + if not isinstance(row, Mapping): + return "_" + parts: list[str] = [] + for f in _STRATUM_FIELDS: + v = str(row.get(f) or "").strip() + if v: + parts.append(f"{f}={v}") + return "|".join(parts) if parts else "_" + + +def stratify_take(items: list[Any], limit: int, *, key_fn) -> list[Any]: + """Round-robin across strata so one neighbor/direction cannot consume the whole sample.""" + lim = max(0, int(limit)) + if lim <= 0 or not items: + return [] + if len(items) <= lim: + return list(items) + buckets: dict[str, deque[Any]] = defaultdict(deque) + order: list[str] = [] + for it in items: + sk = str(key_fn(it) or "_") + if sk not in buckets: + order.append(sk) + buckets[sk].append(it) + out: list[Any] = [] + while len(out) < lim and buckets: + drained: list[str] = [] + for sk in order: + q = buckets.get(sk) + if not q: + drained.append(sk) + continue + out.append(q.popleft()) + if len(out) >= lim: + break + for sk in drained: + buckets.pop(sk, None) + if sk in order: + order = [x for x in order if x != sk] + return out + def apply_port_map( row: dict[str, Any], @@ -105,12 +153,28 @@ def compare_rows( field_rules: Sequence[Mapping[str, Any]] | None = None, iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, ignore_port_changes: bool | None = None, + include_unchanged: bool = False, + unchanged_limit: int | None = None, + compact_unchanged: bool = False, ) -> dict[str, Any]: """Return summary + diffs list. - Diff kinds: added | removed | changed | unchanged | duplicate + Diff kinds: added | removed | changed | unchanged - Pipeline: iface normalize (both sides) → port map (before) → match. + Pipeline: iface normalize (both sides) → port map (before) → ordered + same-key pairing (load / list order within each match key). + + Same match key with N before and M after rows: zip by index + ``0..min(N,M)-1`` for field compare; extras become ``removed`` (before) + or ``added`` (after). Example: 5 vs 2 → 2 compared + 3 removed. + + ``include_unchanged``: when False, matching rows still increment + ``summary.unchanged`` but are omitted from ``diffs``. + + ``unchanged_limit``: max unchanged diffs to emit (None = no cap). Use with + large sheets so the UI can browse a sample without writing millions of rows. + + ``compact_unchanged``: emit key only (empty before/after) to cut storage. ``ignore_port_changes``: - ``None`` (default): auto — drop iface from match key only when remaining @@ -118,8 +182,8 @@ def compare_rows( - ``True``: force drop iface from match key when a non-empty candidate exists. - ``False``: never drop iface from match key. - Duplicate match keys are not silently discarded: extras become ``duplicate`` - diffs and ``summary.duplicate_key_list`` lists the colliding keys. + ``summary.duplicate`` is always 0. ``duplicate_keys_*`` count match keys + that appear more than once on a side (diagnostic only). ``field_rules`` drives normalize / numeric tolerance / per-field compare mode (template-driven; no metric-specific branches here). @@ -166,127 +230,166 @@ def compare_rows( apply_port_map(r, iface_fields=iface_list, port_map=pmap) for r in before_norm ] - after_index: dict[tuple[str, ...], dict[str, Any]] = {} - after_dup = 0 - after_dup_keys: list[tuple[str, ...]] = [] - after_dup_rows: list[tuple[tuple[str, ...], dict[str, Any]]] = [] + before_groups: dict[tuple[str, ...], list[tuple[dict[str, Any], dict[str, Any]]]] = ( + defaultdict(list) + ) + after_groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) + for orig, mapped in zip(before_rows, before_mapped): + before_groups[row_key(mapped, match_keys)].append((orig, mapped)) for r in after_norm: - k = row_key(r, match_keys) - if k in after_index: - after_dup += 1 - after_dup_keys.append(k) - after_dup_rows.append((k, r)) - continue # first wins — do not overwrite - after_index[k] = r + after_groups[row_key(r, match_keys)].append(r) - before_keys: set[tuple[str, ...]] = set() - before_dup = 0 - before_dup_keys: list[tuple[str, ...]] = [] diffs: list[dict[str, Any]] = [] - added = removed = changed = unchanged = duplicate = 0 + added = removed = changed = unchanged = 0 + unchanged_listed = 0 + limit_n = None if unchanged_limit is None else max(0, int(unchanged_limit)) + multi_before_keys: list[tuple[str, ...]] = [] + multi_after_keys: list[tuple[str, ...]] = [] + unchanged_candidates: list[tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = [] def _key_obj(row: dict[str, Any]) -> dict[str, Any]: return {f: row.get(f, "") for f in key_fields} - for orig, mapped in zip(before_rows, before_mapped): - k = row_key(mapped, match_keys) - if k in before_keys: - before_dup += 1 - before_dup_keys.append(k) - duplicate += 1 + def _row_id(row: dict[str, Any] | None) -> str: + if not isinstance(row, dict): + return "" + netx = row.get("_netx") + if isinstance(netx, dict): + return str(netx.get("row_id") or "") + return "" + + def _append_unchanged_diff( + orig: dict[str, Any], mapped: dict[str, Any], after_row: dict[str, Any] + ) -> None: + nonlocal unchanged_listed + unchanged_listed += 1 + before_rid = _row_id(orig) + after_rid = _row_id(after_row) + if compact_unchanged: diffs.append( { - "kind": "duplicate", - "side": "before", + "kind": "unchanged", "key": _key_obj(mapped), - "before": orig, - "after": after_index.get(k), - "mapped_before": mapped, + "before": {}, + "after": {}, + "mapped_before": {}, "changes": {}, - } - ) - continue # only first before row participates in match - before_keys.add(k) - after = after_index.get(k) - if after is None: - removed += 1 - diffs.append( - { - "kind": "removed", - "key": _key_obj(mapped), - "before": orig, - "after": None, - "mapped_before": mapped, - "changes": {}, - } - ) - continue - # Empty compare_fields = presence-only: keyed rows that exist on both - # sides are unchanged (no value checks). - field_changes: dict[str, dict[str, Any]] = {} - for f in compare_fields: - bv = mapped.get(f, "") - av = after.get(f, "") - rule = rules.get(f) - if not values_equal(bv, av, rule=rule): - entry: dict[str, Any] = {"before": bv, "after": av} - reason = explain_diff(bv, av, rule=rule) - if reason: - entry["reason"] = reason - field_changes[f] = entry - if field_changes: - changed += 1 - diffs.append( - { - "kind": "changed", - "key": _key_obj(mapped), - "before": orig, - "after": after, - "mapped_before": mapped, - "changes": field_changes, + "compact": True, + "before_row_id": before_rid, + "after_row_id": after_rid, } ) else: - unchanged += 1 diffs.append( { "kind": "unchanged", "key": _key_obj(mapped), "before": orig, - "after": after, + "after": after_row, "mapped_before": mapped, "changes": {}, + "before_row_id": before_rid, + "after_row_id": after_rid, } ) - for k, after in after_index.items(): - if k in before_keys: - continue - added += 1 - diffs.append( - { - "kind": "added", - "key": _key_obj(after), - "before": None, - "after": after, - "mapped_before": None, - "changes": {}, - } - ) + # Stable key order: before encounter order, then after-only keys + seen_keys: set[tuple[str, ...]] = set() + ordered_keys: list[tuple[str, ...]] = [] + for orig, mapped in zip(before_rows, before_mapped): + k = row_key(mapped, match_keys) + if k not in seen_keys: + seen_keys.add(k) + ordered_keys.append(k) + for r in after_norm: + k = row_key(r, match_keys) + if k not in seen_keys: + seen_keys.add(k) + ordered_keys.append(k) + for k in ordered_keys: + b_list = before_groups.get(k) or [] + a_list = after_groups.get(k) or [] + if len(b_list) > 1: + multi_before_keys.append(k) + if len(a_list) > 1: + multi_after_keys.append(k) + n = max(len(b_list), len(a_list)) + for i in range(n): + if i >= len(b_list): + after = a_list[i] + added += 1 + diffs.append( + { + "kind": "added", + "key": _key_obj(after), + "before": None, + "after": after, + "mapped_before": None, + "changes": {}, + "before_row_id": "", + "after_row_id": _row_id(after), + } + ) + continue + if i >= len(a_list): + orig, mapped = b_list[i] + removed += 1 + diffs.append( + { + "kind": "removed", + "key": _key_obj(mapped), + "before": orig, + "after": None, + "mapped_before": mapped, + "changes": {}, + "before_row_id": _row_id(orig), + "after_row_id": "", + } + ) + continue + orig, mapped = b_list[i] + after = a_list[i] + field_changes: dict[str, dict[str, Any]] = {} + for f in compare_fields: + bv = mapped.get(f, "") + av = after.get(f, "") + rule = rules.get(f) + if not values_equal(bv, av, rule=rule): + entry: dict[str, Any] = {"before": bv, "after": av} + reason = explain_diff(bv, av, rule=rule) + if reason: + entry["reason"] = reason + field_changes[f] = entry + if field_changes: + changed += 1 + diffs.append( + { + "kind": "changed", + "key": _key_obj(mapped), + "before": orig, + "after": after, + "mapped_before": mapped, + "changes": field_changes, + "before_row_id": _row_id(orig), + "after_row_id": _row_id(after), + } + ) + else: + unchanged += 1 + if include_unchanged: + unchanged_candidates.append((orig, mapped, after)) - for k, after in after_dup_rows: - duplicate += 1 - diffs.append( - { - "kind": "duplicate", - "side": "after", - "key": _key_obj(after), - "before": None, - "after": after, - "mapped_before": None, - "changes": {}, - } - ) + if include_unchanged and unchanged_candidates: + if limit_n is None: + picked = unchanged_candidates + else: + picked = stratify_take( + unchanged_candidates, + limit_n, + key_fn=lambda t: stratum_key(t[1]), + ) + for orig, mapped, after_row in picked: + _append_unchanged_diff(orig, mapped, after_row) def _fmt_keys(keys: list[tuple[str, ...]]) -> list[str]: seen: set[str] = set() @@ -296,7 +399,7 @@ def compare_rows( if s not in seen: seen.add(s) out.append(s) - return out + return out[:64] stats = mapping_stats( before_rows=before_norm, @@ -314,11 +417,21 @@ def compare_rows( "removed": removed, "changed": changed, "unchanged": unchanged, - "duplicate": duplicate, + "duplicate": 0, "match_key_fields": match_keys, - "duplicate_keys_before": before_dup, - "duplicate_keys_after": after_dup, - "duplicate_key_list": _fmt_keys(before_dup_keys + after_dup_keys), + "duplicate_keys_before": len(multi_before_keys), + "duplicate_keys_after": len(multi_after_keys), + "duplicate_key_list": _fmt_keys(multi_before_keys + multi_after_keys), + "unchanged_listed": unchanged_listed, + "unchanged_truncated": bool( + include_unchanged and limit_n is not None and unchanged > unchanged_listed + ), + "unchanged_compact": bool(compact_unchanged and unchanged_listed > 0), + "unchanged_sample_mode": ( + "stratified" + if include_unchanged and limit_n is not None and unchanged > unchanged_listed + else ("full" if include_unchanged else "none") + ), }, "diffs": diffs, "mapping_stats": stats, diff --git a/netx_api/biz_state/compare_service.py b/netx_api/biz_state/compare_service.py index 195e752..85472da 100644 --- a/netx_api/biz_state/compare_service.py +++ b/netx_api/biz_state/compare_service.py @@ -6,12 +6,14 @@ import io import json import logging import threading +import time import zipfile from datetime import datetime -from typing import Any +from typing import Any, Callable from uuid import uuid4 from fastapi import HTTPException +from sqlalchemy import and_, or_ from sqlalchemy.orm import Session from ..models import ( @@ -34,6 +36,7 @@ from .compare_rules import ( arp_dynamic_row_filters, effective_compare_fields, effective_display_fields, + row_matches_filter, ) from .iface_normalize import ( apply_iface_normalize_rows, @@ -181,7 +184,68 @@ def _compare_side( } _DIFF_CHUNK = 2000 +_LOAD_YIELD_PER = 5000 _SEARCH_TEXT_MAX = 4000 +# Heartbeat while bulk-inserting large fail/ok diff sets (vpnv4-scale). +_PERSIST_PROGRESS_EVERY = 10_000 +# Above this, store fail diffs as key + row_id + changes (hydrate sides on read). +_FAIL_COMPACT_MIN = 50_000 +# Live search (kw): load at most this many matching rows per side, return ≤ this many pairs. +_LIVE_SEARCH_LOAD_CAP = 2_000 +_LIVE_SEARCH_RESULT_CAP = 200 +# Success-row persist policy (see resolve_unchanged_policy) +_STORE_UNCHANGED_MODES = frozenset({"auto", "always", "never", "sample", "keys"}) +_UNCHANGED_FULL_MAX = 20_000 +_UNCHANGED_SAMPLE_MAX = 5_000 + + +def normalize_store_unchanged(raw: Any) -> str: + m = str(raw or "auto").strip().lower() + return m if m in _STORE_UNCHANGED_MODES else "auto" + + +def resolve_unchanged_policy( + mode: str, *, before_n: int, after_n: int +) -> dict[str, Any]: + """Decide whether / how many success rows to emit for one sheet. + + - always: full before/after for every match (slow on million-row sheets) + - never: count only + - sample: key + row_id sample (cap) — cutover spot-check default for large sheets + - keys: all success as key + row_id (full browse; write still heavy) + - auto: full when sheet is small; otherwise sample compact + """ + m = normalize_store_unchanged(mode) + n = max(int(before_n or 0), int(after_n or 0)) + if m == "never": + return {"mode": m, "include": False, "limit": None, "compact": False} + if m == "always": + return {"mode": m, "include": True, "limit": None, "compact": False} + if m == "keys": + return {"mode": m, "include": True, "limit": None, "compact": True} + if m == "sample": + return { + "mode": m, + "include": True, + "limit": _UNCHANGED_SAMPLE_MAX, + "compact": True, + } + # auto — cutover-oriented: large sheets sample, not full key dump + if n <= _UNCHANGED_FULL_MAX: + return {"mode": m, "include": True, "limit": None, "compact": False} + return { + "mode": m, + "include": True, + "limit": _UNCHANGED_SAMPLE_MAX, + "compact": True, + } + + +def _strip_netx(row: Any) -> dict[str, Any]: + """Drop collector provenance before persisting compare payloads.""" + if not isinstance(row, dict): + return {} + return {k: v for k, v in row.items() if k != "_netx"} def _diff_search_text(d: dict[str, Any]) -> str: @@ -215,30 +279,150 @@ def _persist_sheet_diffs( run_id: str, metric_id: str, diffs: list[dict[str, Any]], -) -> None: - """Bulk-insert diff rows; avoids embedding million-row arrays in summary_json.""" + seq_start: int = 0, + on_progress: Callable[[int, int], None] | None = None, +) -> int: + """Bulk-insert diff rows already selected by the engine policy. + + Compact success rows carry key + before/after_row_id; JSON sides stay empty + and are hydrated from metric tables on read. + + ``on_progress(written, total)`` fires periodically so UI elapsed time moves + during multi-minute inserts (e.g. large vpnv4 fail sets). + """ buf: list[dict[str, Any]] = [] - for i, d in enumerate(diffs): + seq = int(seq_start or 0) + written = 0 + total = len(diffs) + last_prog = 0 + last_prog_t = time.monotonic() + + def _maybe_prog(force: bool = False) -> None: + nonlocal last_prog, last_prog_t + if not on_progress: + return + now = time.monotonic() + if ( + not force + and written - last_prog < _PERSIST_PROGRESS_EVERY + and now - last_prog_t < 2.0 + ): + return + last_prog = written + last_prog_t = now + try: + on_progress(written, total) + except Exception: + _log.exception("persist progress callback failed run=%s metric=%s", run_id, metric_id) + + for d in diffs: + kind = str(d.get("kind") or "") + before = _strip_netx(d.get("before")) + after = _strip_netx(d.get("after")) + mapped = _strip_netx(d.get("mapped_before")) + payload = { + "kind": kind, + "key": dict(d.get("key") or {}), + "before": before, + "after": after, + "mapped_before": mapped, + "changes": dict(d.get("changes") or {}), + } + # Compact rows: search_text = kind + key (+ changes) only — no fat sides + search_src = ( + { + "kind": kind, + "key": payload["key"], + **({"changes": payload["changes"]} if payload["changes"] else {}), + } + if bool(d.get("compact")) + else payload + ) buf.append( { "id": uuid4().hex, "run_id": run_id, "metric_id": metric_id, - "seq": i, - "kind": str(d.get("kind") or ""), - "key_json": dict(d.get("key") or {}), - "before_json": dict(d.get("before") or {}), - "after_json": dict(d.get("after") or {}), - "mapped_before_json": dict(d.get("mapped_before") or {}), - "changes_json": dict(d.get("changes") or {}), - "search_text": _diff_search_text(d), + "seq": seq, + "kind": kind, + "key_json": payload["key"], + "before_json": before, + "after_json": after, + "mapped_before_json": mapped, + "changes_json": payload["changes"], + "before_row_id": str(d.get("before_row_id") or "")[:64], + "after_row_id": str(d.get("after_row_id") or "")[:64], + "search_text": _diff_search_text(search_src), } ) + seq += 1 + written += 1 if len(buf) >= _DIFF_CHUNK: db.bulk_insert_mappings(BizCompareDiff, buf) buf.clear() + # Commit chunks so progress/UI can see mid-write fail rows and + # elapsed_ms advances (otherwise persisting_* looks frozen). + db.commit() + _maybe_prog() if buf: db.bulk_insert_mappings(BizCompareDiff, buf) + db.commit() + _maybe_prog(force=True) + return written + + +def _metric_rows_by_ids(db: Session, ids: list[str]) -> dict[str, dict[str, Any]]: + """Load BizStateMetricRow / LLDP rows by primary key for hydrate.""" + clean = [str(i).strip() for i in ids if str(i or "").strip()] + if not clean: + return {} + from ..models import BizStateMetricRow + + out: dict[str, dict[str, Any]] = {} + # Chunk IN lists for large pages + for i in range(0, len(clean), 500): + chunk = clean[i : i + 500] + for r in db.query(BizStateMetricRow).filter(BizStateMetricRow.id.in_(chunk)).all(): + out[str(r.id)] = dict(r.data_json or {}) + missing = [x for x in chunk if x not in out] + if missing: + for n in ( + db.query(BizStateLldpNeighbor) + .filter(BizStateLldpNeighbor.id.in_(missing)) + .all() + ): + out[str(n.id)] = { + "local_if": n.local_if, + "remote_sys": n.remote_sys, + "remote_if": n.remote_if, + "remote_ip": n.remote_ip, + "protocol": n.protocol, + } + return out + + +def _hydrate_diff_rows(db: Session, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Fill empty before/after from metric tables when row_ids are present.""" + need: list[str] = [] + for it in items: + if not it.get("before") and it.get("before_row_id"): + need.append(str(it["before_row_id"])) + if not it.get("after") and it.get("after_row_id"): + need.append(str(it["after_row_id"])) + if not need: + return items + by_id = _metric_rows_by_ids(db, need) + for it in items: + brid = str(it.get("before_row_id") or "") + arid = str(it.get("after_row_id") or "") + if not it.get("before") and brid and brid in by_id: + it["before"] = by_id[brid] + if not it.get("after") and arid and arid in by_id: + it["after"] = by_id[arid] + # Success compact: no mapped_before stored — UI falls back to before + if not it.get("mapped_before") and it.get("before"): + it["mapped_before"] = dict(it["before"]) + return items def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]: @@ -249,6 +433,8 @@ def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]: "after": r.after_json or {}, "mapped_before": r.mapped_before_json or {}, "changes": r.changes_json or {}, + "before_row_id": getattr(r, "before_row_id", "") or "", + "after_row_id": getattr(r, "after_row_id", "") or "", } @@ -288,6 +474,10 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A sheets = list(summary.get("sheets") or []) if sheets: return sheets + # Running / empty: do NOT invent a fake first-metric sheet (was showing + # isis 100% with 0→0 while BGP was still loading). + if str(run.status or "") in ("running", "queued"): + return [] return [ { "metric_id": run.metric_id, @@ -297,6 +487,7 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A "iface_fields": list((tpl.iface_fields if tpl else None) or []), "compare_fields": list((tpl.compare_fields if tpl else None) or []), "mode": "fields", + "status": "done", "summary": { k: summary.get(k, 0) for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count") @@ -306,6 +497,81 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A ] +def _metric_row_estimate( + db: Session, batch_ids: list[str], metric_id: str +) -> int: + """Cheap size hint from batch command row_count (max across sides).""" + mid = str(metric_id or "").strip() + if not mid: + return 0 + best = 0 + for bid in batch_ids: + bid = str(bid or "").strip() + if not bid: + continue + rows = ( + db.query(BizStateBatchCommand) + .filter( + BizStateBatchCommand.batch_id == bid, + BizStateBatchCommand.metric_id == mid, + ) + .all() + ) + if not rows: + continue + n = sum(int(c.row_count or 0) for c in rows) + if n > best: + best = n + return best + + +def _order_sheets_small_first( + db: Session, + sheets: list[dict[str, Any]], + *, + before_batch_id: str, + after_batch_id: str, +) -> list[dict[str, Any]]: + """Run smaller metrics first so field engineers can review early results.""" + if len(sheets) <= 1: + return list(sheets) + batches = [before_batch_id, after_batch_id] + scored: list[tuple[int, int, dict[str, Any]]] = [] + for i, sheet in enumerate(sheets): + n = _metric_row_estimate(db, batches, str(sheet.get("metric_id") or "")) + scored.append((n, i, sheet)) + scored.sort(key=lambda x: (x[0], x[1])) + return [s for _, _, s in scored] + + +def _pending_sheet_meta(sheet: dict[str, Any]) -> dict[str, Any]: + """Placeholder meta so the UI lists all check items while a run is in flight.""" + key_fields = list(sheet.get("key_fields") or []) + compare_fields = list(sheet.get("compare_fields") or []) + return { + "sheet_id": sheet_key(sheet), + "title": sheet_title(sheet), + "metric_id": sheet.get("metric_id") or "", + "key_fields": key_fields, + "iface_fields": list(sheet.get("iface_fields") or []), + "compare_fields": compare_fields, + "display_fields": list(sheet.get("display_fields") or []), + "field_rules": list(sheet.get("field_rules") or []), + "row_filters": list(sheet.get("row_filters") or []), + "ignore_port_changes": sheet.get("ignore_port_changes"), + "mode": "presence" if not compare_fields else "fields", + "status": "pending", + "summary": { + "added": 0, + "removed": 0, + "changed": 0, + "unchanged": 0, + "before_count": 0, + "after_count": 0, + }, + } + + def _str_list(raw: Any) -> list[str]: return [str(x).strip() for x in (raw or []) if str(x).strip()] @@ -551,7 +817,35 @@ def _builtin_source_splits() -> dict[str, list[dict[str, Any]]]: } +def _packaged_zte_status_template_path(): + from pathlib import Path + + return Path(__file__).resolve().parent / "data" / "default_zte_status_template.json" + + +def _load_packaged_zte_status_template() -> dict[str, Any]: + """IOH CN migration sheet set shipped as the built-in status default.""" + path = _packaged_zte_status_template_path() + if not path.is_file(): + return {} + try: + return dict(json.loads(path.read_text(encoding="utf-8")) or {}) + except Exception: + _log.exception("failed to load packaged ZTE status template %s", path) + return {} + + def _default_zte_status_sheets() -> list[dict[str, Any]]: + """Built-in status sheets — prefer packaged IOH CN migration rules.""" + raw = _load_packaged_zte_status_template() + out: list[dict[str, Any]] = [] + for item in list(raw.get("metrics") or []): + sheet = _normalize_sheet(item) + if sheet: + out.append(sheet) + if out: + return out + # Fallback if package missing (tests / incomplete install) return [ *_isis_af_sheets(), _default_sheet_for_metric("interface_brief", compare_roles=("state",)), @@ -568,6 +862,23 @@ def _default_zte_status_sheets() -> list[dict[str, Any]]: ] +def _builtin_status_needs_packaged_upgrade(existing: list[dict[str, Any]]) -> bool: + """True when built-in template still lacks filtered BGP route sheets.""" + mids = {str(s.get("metric_id") or "") for s in existing} + if "bgp_route" not in mids and "l2vpn_mac" not in mids: + return True + has_filtered_route = any( + str(s.get("metric_id") or "") == "bgp_route" and list(s.get("row_filters") or []) + for s in existing + ) + if not has_filtered_route: + return True + packaged_keys = {sheet_key(s) for s in _default_zte_status_sheets()} + have_keys = {sheet_key(s) for s in existing} + # Missing several packaged sheet ids → sync to packaged default + return len(packaged_keys - have_keys) >= 3 + + def _default_zte_config_sheets() -> list[dict[str, Any]]: """Config-intent metrics for cutover / intent-vs-intent compare.""" return [ @@ -834,52 +1145,63 @@ def ensure_default_lldp_template(db: Session) -> BizCompareTemplate: def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: name = "ZTE status default" row = db.query(BizCompareTemplate).filter(BizCompareTemplate.name == name).one_or_none() + packaged = _load_packaged_zte_status_template() sheets = _default_zte_status_sheets() - # Volatile counters must not stay in compare_fields on upgraded installs. - _STRIP_COMPARE: dict[str, frozenset[str]] = { - "interface_detail": frozenset({"input_bps", "output_bps", "in_util", "out_util"}), - "optical_brief": frozenset({"rx_power", "tx_power"}), - "bgp_peer": frozenset({"pfx_rcd"}), - } + note = str( + packaged.get("note") + or "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP route AF sheets)" + )[:512] + iface_rules = list(packaged.get("iface_normalize_rules") or []) if row: existing = template_metrics(row) - want = {s["metric_id"] for s in sheets} - have = {s["metric_id"] for s in existing} - changed = bool(want - have) + if _builtin_status_needs_packaged_upgrade(existing) and sheets: + _apply_sheets_to_row(row, sheets) + row.note = note + row.updated_at = _utcnow() + if iface_rules: + _set_template_iface_normalize(row, iface_rules) + elif not template_iface_normalize(row): + # Packaged IOH rules use empty normalize; leave empty when explicit + _set_template_iface_normalize(row, []) + db.commit() + db.refresh(row) + return row + # Incremental patches for already-upgraded installs + changed = False upgraded: list[dict[str, Any]] = [] - by_want = {s["metric_id"]: s for s in sheets} + by_sid = {sheet_key(s): s for s in sheets} for s in existing: cur = dict(s) mid = str(cur.get("metric_id") or "") + sid = sheet_key(cur) if mid == "arp" and not cur.get("row_filters"): - cur["row_filters"] = list(by_want.get("arp", {}).get("row_filters") or arp_dynamic_row_filters()) - if not cur.get("field_rules") and by_want.get("arp", {}).get("field_rules"): - cur["field_rules"] = list(by_want["arp"]["field_rules"]) + src = by_sid.get(sid) or next( + (x for x in sheets if x.get("metric_id") == "arp"), None + ) + cur["row_filters"] = list( + (src or {}).get("row_filters") or arp_dynamic_row_filters() + ) + if not cur.get("field_rules") and src and src.get("field_rules"): + cur["field_rules"] = list(src["field_rules"]) changed = True - strip = _STRIP_COMPARE.get(mid) - if strip: - old_cmp = list(cur.get("compare_fields") or []) - new_cmp = [f for f in old_cmp if f not in strip] - if new_cmp != old_cmp: - cur["compare_fields"] = new_cmp - want_disp = list((by_want.get(mid) or {}).get("display_fields") or []) - if want_disp: - cur["display_fields"] = want_disp - changed = True upgraded.append(_normalize_sheet(cur) or cur) - if want - have: - for s in sheets: - if s["metric_id"] not in have: + have_mids = {str(s.get("metric_id") or "") for s in upgraded} + for s in sheets: + if str(s.get("metric_id") or "") not in have_mids: + # Only append wholly missing metrics (e.g. bgp_route family) + if str(s.get("metric_id") or "") == "bgp_route" and "bgp_route" not in have_mids: + upgraded.extend( + [x for x in sheets if x.get("metric_id") == "bgp_route"] + ) + have_mids.add("bgp_route") + changed = True + elif str(s.get("metric_id") or "") not in have_mids: upgraded.append(s) - changed = True + have_mids.add(str(s.get("metric_id") or "")) + changed = True if changed: _apply_sheets_to_row(row, upgraded if upgraded else sheets) - row.note = "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)" - row.updated_at = _utcnow() - db.commit() - db.refresh(row) - if not template_iface_normalize(row): - _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) + row.note = note row.updated_at = _utcnow() db.commit() db.refresh(row) @@ -887,12 +1209,12 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: row = BizCompareTemplate( id=uuid4().hex, name=name, - note="Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)", + note=note, created_at=_utcnow(), updated_at=_utcnow(), ) _apply_sheets_to_row(row, sheets) - _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) + _set_template_iface_normalize(row, iface_rules) db.add(row) db.commit() db.refresh(row) @@ -948,10 +1270,10 @@ def ensure_default_templates(db: Session) -> None: def upgrade_builtin_split_sheets(db: Session) -> None: - """Split unfiltered whole-table sheets on the built-in ZTE template only. + """Upgrade built-in ZTE status template to packaged AF / BGP route sheets. - A sheet is replaced when its id is still the source metric and it has no - row filters. Custom templates and already-split sheets are left alone. + Custom templates are left alone. Built-in is replaced wholesale when it + still lacks filtered ``bgp_route`` sheets (IOH CN migration default). """ row = ( db.query(BizCompareTemplate) @@ -961,6 +1283,18 @@ def upgrade_builtin_split_sheets(db: Session) -> None: if not row: return existing = template_metrics(row) + packaged_sheets = _default_zte_status_sheets() + if _builtin_status_needs_packaged_upgrade(existing) and packaged_sheets: + packaged = _load_packaged_zte_status_template() + _apply_sheets_to_row(row, packaged_sheets) + row.note = str( + packaged.get("note") + or "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP route AF sheets)" + )[:512] + _set_template_iface_normalize(row, list(packaged.get("iface_normalize_rules") or [])) + row.updated_at = _utcnow() + db.commit() + return splits = _builtin_source_splits() out: list[dict[str, Any]] = [] changed = False @@ -1206,61 +1540,191 @@ def _port_map_dict(db: Session, mapping_id: str) -> dict[str, str]: return {str(r.before_if): str(r.after_if) for r in rows if r.before_if and r.after_if} -def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dict[str, Any]]: - if metric_id == "lldp_neighbor": - rows = ( - db.query(BizStateLldpNeighbor) - .filter(BizStateLldpNeighbor.batch_id == batch_id) - .all() - ) - return [ - { - "local_if": n.local_if, - "remote_sys": n.remote_sys, - "remote_if": n.remote_if, - "remote_ip": n.remote_ip, - "protocol": n.protocol, - "_netx": { - "batch_id": batch_id, - "batch_command_id": n.batch_command_id or "", - "task_id": n.task_id or "", - "ne_id": n.ne_id or "", - "collected_at": n.collected_at.isoformat() + "Z" if n.collected_at else None, - "row_id": n.id, - }, - } - for n in rows - ] - # Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …) - from ..models import BizStateMetricRow +def _load_metric_rows( + db: Session, + *, + batch_id: str, + metric_id: str, + on_chunk: Callable[[int], None] | None = None, + row_filters: list[dict[str, Any]] | None = None, +) -> list[dict[str, Any]]: + """Load metric rows in keyset chunks (stable on million-row sheets). - rows = ( - db.query(BizStateMetricRow) - .filter( + When ``row_filters`` are SQL-pushdown-safe on PostgreSQL, they are applied in + the SELECT (critical for BGP afi/vrf sheet splits — avoids loading 1M+ then + discarding). Otherwise filters are applied in Python after each chunk. + """ + from .compare_sql import ( + _dialect_is_postgres, + _filters_sql_compatible, + compile_row_filters_sql, + ) + + filters = [f for f in (row_filters or []) if isinstance(f, dict)] + pushdown = bool( + filters and _dialect_is_postgres(db) and _filters_sql_compatible(filters) + ) + filter_sql, filter_params = ("TRUE", {}) + if pushdown: + filter_sql, filter_params = compile_row_filters_sql(filters) + + if metric_id == "lldp_neighbor": + out: list[dict[str, Any]] = [] + last_id = "" + while True: + q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id) + if last_id: + q = q.filter(BizStateLldpNeighbor.id > last_id) + chunk = q.order_by(BizStateLldpNeighbor.id.asc()).limit(_LOAD_YIELD_PER).all() + if not chunk: + break + for n in chunk: + row = { + "local_if": n.local_if, + "remote_sys": n.remote_sys, + "remote_if": n.remote_if, + "remote_ip": n.remote_ip, + "protocol": n.protocol, + "_netx": { + "batch_id": batch_id, + "batch_command_id": n.batch_command_id or "", + "task_id": n.task_id or "", + "ne_id": n.ne_id or "", + "collected_at": n.collected_at.isoformat() + "Z" + if n.collected_at + else None, + "row_id": n.id, + }, + } + if filters and not pushdown and not all( + row_matches_filter(row, f) for f in filters + ): + db.expunge(n) + continue + out.append(row) + db.expunge(n) + last_id = str(chunk[-1].id) + if on_chunk: + on_chunk(len(out)) + if len(chunk) < _LOAD_YIELD_PER: + break + if filters and not pushdown: + return apply_row_filters(out, filters) + return out + + # Generic tabular metrics — PG + pushdown uses SQL keyset with JSON filters + from ..models import BizStateMetricRow + from sqlalchemy import text as sql_text + + out: list[dict[str, Any]] = [] + last_seq = -1 + last_id = "" + while True: + if pushdown: + params = { + "bid": batch_id, + "mid": metric_id, + "last_seq": last_seq, + "last_id": last_id, + "lim": int(_LOAD_YIELD_PER), + **filter_params, + } + keyset = ( + "(seq > :last_seq OR (seq = :last_seq AND id > :last_id))" + if last_id + else "TRUE" + ) + rows = db.execute( + sql_text( + f""" + SELECT id, batch_command_id, task_id, ne_id, seq, data_json, collected_at + FROM biz_state_metric_row + WHERE batch_id = :bid + AND metric_id = :mid + AND ({filter_sql}) + AND ({keyset}) + ORDER BY seq ASC, id ASC + LIMIT :lim + """ + ), + params, + ).mappings().all() + if not rows: + break + for r in rows: + data = dict(r["data_json"] or {}) + collected = r["collected_at"] + out.append( + { + **data, + "_netx": { + "batch_id": batch_id, + "batch_command_id": str(r["batch_command_id"] or ""), + "task_id": str(r["task_id"] or ""), + "ne_id": str(r["ne_id"] or ""), + "collected_at": collected.isoformat() + "Z" + if collected is not None + else None, + "row_id": str(r["id"]), + }, + } + ) + last_seq = int(rows[-1]["seq"] or 0) + last_id = str(rows[-1]["id"]) + if on_chunk: + on_chunk(len(out)) + if len(rows) < _LOAD_YIELD_PER: + break + continue + + q = db.query(BizStateMetricRow).filter( BizStateMetricRow.batch_id == batch_id, BizStateMetricRow.metric_id == metric_id, ) - .order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) - .all() - ) - if rows: - # Raw rows only — filtering belongs to the compare sheet template - # (``row_filters``), not metric-specific branches here. - # ``_netx`` is collector provenance (stripped before field compare). - return [ - { + if last_id: + q = q.filter( + or_( + BizStateMetricRow.seq > last_seq, + and_( + BizStateMetricRow.seq == last_seq, + BizStateMetricRow.id > last_id, + ), + ) + ) + chunk = ( + q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) + .limit(_LOAD_YIELD_PER) + .all() + ) + if not chunk: + break + for r in chunk: + row = { **dict(r.data_json or {}), "_netx": { "batch_id": batch_id, "batch_command_id": r.batch_command_id or "", "task_id": r.task_id or "", "ne_id": r.ne_id or "", - "collected_at": r.collected_at.isoformat() + "Z" if r.collected_at else None, + "collected_at": r.collected_at.isoformat() + "Z" + if r.collected_at + else None, "row_id": r.id, }, } - for r in rows - ] + if filters and not all(row_matches_filter(row, f) for f in filters): + db.expunge(r) + continue + out.append(row) + db.expunge(r) + last_seq = int(chunk[-1].seq or 0) + last_id = str(chunk[-1].id) + if on_chunk: + on_chunk(len(out)) + if len(chunk) < _LOAD_YIELD_PER: + break + if out: + return out # Known metric with zero rows is OK; unknown metric still errors if metric_id in metric_field_map(): return [] @@ -1318,6 +1782,7 @@ def _job_out(j: BizCompareJob) -> dict[str, Any]: "mode": j.mode, "status": j.status, "enabled_sheet_ids": _str_list(getattr(j, "enabled_sheet_ids", None)), + "store_unchanged": normalize_store_unchanged(getattr(j, "store_unchanged", None)), "note": j.note, "updated_at": j.updated_at.isoformat() + "Z" if j.updated_at else None, } @@ -1360,6 +1825,7 @@ def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: mode=str(body.get("mode") or "manual")[:16], status="ready", enabled_sheet_ids=enabled, + store_unchanged=normalize_store_unchanged(body.get("store_unchanged")), note=str(body.get("note") or "")[:512], created_at=_utcnow(), updated_at=_utcnow(), @@ -1391,6 +1857,8 @@ def update_job(db: Session, job_id: str, body: dict[str, Any]) -> dict[str, Any] setattr(j, key, str(body.get(key) or "")) if "enabled_sheet_ids" in body: j.enabled_sheet_ids = _str_list(body.get("enabled_sheet_ids")) + if "store_unchanged" in body: + j.store_unchanged = normalize_store_unchanged(body.get("store_unchanged")) j.updated_at = _utcnow() db.commit() return _job_out(j) @@ -1444,6 +1912,40 @@ def _resolve_after_batch(db: Session, job: BizCompareJob) -> str: return str(latest.id) if latest else "" +def _sheet_result_envelope( + sheet: dict[str, Any], + *, + key_fields: list[str], + iface_fields: list[str], + compare_fields: list[str], + display_fields: list[str], + row_filters: list[Any], + field_rules: list[Any], + ignore_ports: bool | None, + mode: str, + summary: dict[str, Any], + diffs: list[Any], + mapping_stats: dict[str, Any], +) -> dict[str, Any]: + return { + "sheet_id": sheet_key(sheet), + "title": sheet_title(sheet), + "metric_id": sheet["metric_id"], + "key_fields": key_fields, + "iface_fields": iface_fields, + "compare_fields": compare_fields, + "display_fields": display_fields, + "row_filters": row_filters, + "field_rules": field_rules, + "ignore_port_changes": ignore_ports, + "mode": mode, + "status": "done", + "summary": summary, + "diffs": diffs, + "mapping_stats": mapping_stats, + } + + def _run_sheet( db: Session, *, @@ -1452,6 +1954,8 @@ def _run_sheet( after_batch_id: str, port_map: dict[str, str], iface_normalize_rules: list[dict[str, str]] | None = None, + store_unchanged: str = "auto", + on_load_progress: Callable[..., None] | None = None, ) -> dict[str, Any]: key_fields = list(sheet.get("key_fields") or []) iface_fields = list(sheet.get("iface_fields") or []) @@ -1472,10 +1976,117 @@ def _run_sheet( ignore_ports = sheet.get("ignore_port_changes") if ignore_ports is not None: ignore_ports = bool(ignore_ports) - before_raw = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"]) - after_raw = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"]) - before_rows = apply_row_filters(before_raw, row_filters) - after_rows = apply_row_filters(after_raw, row_filters) + mid = sheet["metric_id"] + + def _emit_load(side: str, n: int, **meta: Any) -> None: + if not on_load_progress: + return + try: + on_load_progress(side, n, **meta) + except TypeError: + on_load_progress(side, n) + + # PostgreSQL path: pushdown-safe sheets join in-DB (BGP-scale). + from .compare_sql import SqlCompareSkip, run_sql_sheet_compare, sql_compare_skip_reason + + skip_reason = sql_compare_skip_reason( + db, + sheet, + port_map=port_map, + iface_normalize_rules=iface_normalize_rules, + ) + if not skip_reason: + try: + result = run_sql_sheet_compare( + db, + sheet=sheet, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + store_unchanged=store_unchanged, + on_progress=_emit_load, + ) + summary = dict(result["summary"]) + # Engine already sets raw counts / policy; keep keys stable + if "unchanged_policy" not in summary: + summary["unchanged_policy"] = resolve_unchanged_policy( + store_unchanged, + before_n=int(summary.get("before_count") or 0), + after_n=int(summary.get("after_count") or 0), + ) + return _sheet_result_envelope( + sheet, + key_fields=key_fields, + iface_fields=iface_fields, + compare_fields=compare_fields, + display_fields=display_fields, + row_filters=row_filters, + field_rules=field_rules, + ignore_ports=ignore_ports, + mode=mode, + summary=summary, + diffs=list(result.get("diffs") or []), + mapping_stats=dict(result.get("mapping_stats") or {}), + ) + except SqlCompareSkip as skip: + skip_reason = skip.reason or "skip" + _log.info( + "python compare sheet=%s metric=%s skip_sql=%s", + sheet_key(sheet), + mid, + skip_reason, + ) + try: + db.rollback() + except Exception: + pass + except Exception: + _log.exception( + "sql compare fallback sheet=%s metric=%s — using Python engine", + sheet_key(sheet), + mid, + ) + skip_reason = "sql_error_fallback" + # Roll back aborted SQL transaction so Python path can use the session + try: + db.rollback() + except Exception: + pass + else: + _log.info( + "python compare sheet=%s metric=%s skip_sql=%s", + sheet_key(sheet), + mid, + skip_reason, + ) + + _emit_load("before", 0, engine="python", note=skip_reason or "python", phase="loading") + + def _before_chunk(n: int) -> None: + _emit_load("before", n, engine="python", note=skip_reason or "python", phase="loading") + + def _after_chunk(n: int) -> None: + _emit_load("after", n, engine="python", note=skip_reason or "python", phase="loading") + + before_raw = _load_metric_rows( + db, + batch_id=before_batch_id, + metric_id=mid, + on_chunk=_before_chunk, + row_filters=row_filters, + ) + after_raw = _load_metric_rows( + db, + batch_id=after_batch_id, + metric_id=mid, + on_chunk=_after_chunk, + row_filters=row_filters, + ) + # Filters already applied in load when pushdown-safe; keep apply for safety + before_rows = apply_row_filters(before_raw, row_filters) if row_filters else before_raw + after_rows = apply_row_filters(after_raw, row_filters) if row_filters else after_raw + policy = resolve_unchanged_policy( + store_unchanged, before_n=len(before_rows), after_n=len(after_rows) + ) result = compare_rows( before_rows=before_rows, after_rows=after_rows, @@ -1486,42 +2097,36 @@ def _run_sheet( field_rules=field_rules, iface_normalize_rules=iface_normalize_rules, ignore_port_changes=ignore_ports, + include_unchanged=bool(policy["include"]), + unchanged_limit=policy.get("limit"), + compact_unchanged=bool(policy.get("compact")), ) summary = dict(result["summary"]) summary["before_raw_count"] = len(before_raw) summary["after_raw_count"] = len(after_raw) summary["row_filters"] = len(row_filters) - return { - "sheet_id": sheet_key(sheet), - "title": sheet_title(sheet), - "metric_id": sheet["metric_id"], - "key_fields": key_fields, - "iface_fields": iface_fields, - "compare_fields": compare_fields, - "display_fields": display_fields, - "row_filters": row_filters, - "field_rules": field_rules, - "ignore_port_changes": ignore_ports, - "mode": mode, - "summary": summary, - "diffs": result["diffs"], - "mapping_stats": result["mapping_stats"], - } + summary["unchanged_policy"] = policy + summary.setdefault("engine", "python") + return _sheet_result_envelope( + sheet, + key_fields=key_fields, + iface_fields=iface_fields, + compare_fields=compare_fields, + display_fields=display_fields, + row_filters=row_filters, + field_rules=field_rules, + ignore_ports=ignore_ports, + mode=mode, + summary=summary, + diffs=list(result.get("diffs") or []), + mapping_stats=dict(result.get("mapping_stats") or {}), + ) -def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]: - lock = _job_compare_lock(job_id) - if not lock.acquire(blocking=False): - raise HTTPException(status_code=409, detail="compare_already_running") - try: - return _run_compare_unlocked(db, job_id, force_after_batch_id=force_after_batch_id) - finally: - lock.release() - - -def _run_compare_unlocked( +def _validate_compare_job( db: Session, job_id: str, *, force_after_batch_id: str = "" ) -> dict[str, Any]: + """Resolve job/template/batches/sheets; raises HTTPException on bad input.""" j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") @@ -1541,10 +2146,185 @@ def _run_compare_unlocked( sheets_cfg = _filter_enabled_sheets(sheets_cfg, getattr(j, "enabled_sheet_ids", None)) if not sheets_cfg: raise HTTPException(status_code=400, detail="no_enabled_sheets") + sheets_cfg = _order_sheets_small_first( + db, + sheets_cfg, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + ) + return { + "job": j, + "template": tpl, + "before_batch_id": before_batch_id, + "after_batch_id": after_batch_id, + "sheets_cfg": sheets_cfg, + } - pmap = _port_map_dict(db, j.mapping_id) + +def _create_running_run( + db: Session, + *, + job: BizCompareJob, + tpl: BizCompareTemplate, + before_batch_id: str, + after_batch_id: str, + sheets_cfg: list[dict[str, Any]], +) -> BizCompareRun: + first_metric = str(sheets_cfg[0].get("metric_id") or "") + pending_sheets = [_pending_sheet_meta(s) for s in sheets_cfg] + run = BizCompareRun( + id=uuid4().hex, + job_id=job.id, + template_id=tpl.id, + mapping_id=job.mapping_id, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + metric_id=first_metric, + status="running", + summary_json={ + "progress": { + "phase": "queued", + "sheet_index": 0, + "sheet_total": len(sheets_cfg), + "sheet_id": "", + "sheet_title": "", + "elapsed_ms": 0, + }, + "sheet_count": len(sheets_cfg), + "added": 0, + "removed": 0, + "changed": 0, + "unchanged": 0, + "duplicate": 0, + "before_count": 0, + "after_count": 0, + "sheets": pending_sheets, + }, + diffs_json=[], + mapping_stats_json={}, + message="queued", + created_at=_utcnow(), + ) + db.add(run) + db.commit() + db.refresh(run) + return run + + +def _set_run_progress( + db: Session, + run: BizCompareRun, + *, + phase: str, + sheet_index: int, + sheet_total: int, + sheet: dict[str, Any] | None, + started_mono: float, + extra: dict[str, Any] | None = None, + detach: bool = False, +) -> None: + """Update run progress. + + ``detach=True`` writes via a fresh session so SQL compare can keep an open + transaction (TEMP CTAS) without mid-flight commits on the worker ``db``. + """ + elapsed_ms = int((time.monotonic() - started_mono) * 1000) + progress = { + "phase": phase, + "sheet_index": sheet_index, + "sheet_total": sheet_total, + "sheet_id": sheet_key(sheet) if sheet else "", + "sheet_title": sheet_title(sheet) if sheet else "", + "elapsed_ms": elapsed_ms, + } + if extra: + progress.update(extra) + title = progress["sheet_title"] or progress["sheet_id"] or "" + message = ( + f"{phase} {sheet_index}/{sheet_total}" + + (f" · {title}" if title else "") + + f" · {elapsed_ms // 1000}s" + )[:1024] + + if detach: + from ..db import SessionLocal + + s = SessionLocal() + try: + r = s.get(BizCompareRun, str(run.id)) + if not r: + return + prev = dict(r.summary_json or {}) + prev["progress"] = progress + r.summary_json = prev + if str(r.status or "") != "cancelled": + r.status = "running" + r.message = message + s.commit() + # Mirror into worker instance for later in-memory reads (do not commit db) + prev_w = dict(run.summary_json or {}) + prev_w["progress"] = progress + run.summary_json = prev_w + if str(run.status or "") != "cancelled": + run.message = message + finally: + s.close() + return + + prev = dict(run.summary_json or {}) + prev["progress"] = progress + run.summary_json = prev + # Re-read status from DB — cancel may have been committed by another session + # (UI cancel / startup recovery) while this worker still holds a stale "running". + db.expire(run, ["status", "message"]) + if str(run.status or "") != "cancelled": + run.status = "running" + run.message = message + db.commit() + + +def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]: + """Run compare into an existing ``running`` BizCompareRun; persist sheet-by-sheet.""" + run = db.get(BizCompareRun, run_id) + if not run: + raise HTTPException(status_code=404, detail="run_not_found") + j = db.get(BizCompareJob, run.job_id) + if not j: + run.status = "failed" + run.message = "job_not_found" + db.commit() + raise HTTPException(status_code=404, detail="job_not_found") + tpl = db.get(BizCompareTemplate, run.template_id) + if not tpl: + run.status = "failed" + run.message = "template_not_found" + db.commit() + raise HTTPException(status_code=404, detail="template_not_found") + + before_batch_id = str(run.before_batch_id or "") + after_batch_id = str(run.after_batch_id or "") + sheets_cfg = _filter_enabled_sheets( + template_metrics(tpl), getattr(j, "enabled_sheet_ids", None) + ) + if not sheets_cfg: + run.status = "failed" + run.message = "no_enabled_sheets" + db.commit() + raise HTTPException(status_code=400, detail="no_enabled_sheets") + sheets_cfg = _order_sheets_small_first( + db, + sheets_cfg, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + ) + + started_mono = time.monotonic() + store_mode = normalize_store_unchanged(getattr(j, "store_unchanged", None)) + pmap = _port_map_dict(db, run.mapping_id) norm_rules = template_iface_normalize(tpl) - sheet_results: list[dict[str, Any]] = [] + unchanged_listed_total = 0 + unchanged_truncated_any = False + unchanged_compact_any = False agg = { "before_count": 0, "after_count": 0, @@ -1555,87 +2335,483 @@ def _run_compare_unlocked( "duplicate": 0, } mapping_by_metric: dict[str, Any] = {} - for sheet in sheets_cfg: - one = _run_sheet( - db, - sheet=sheet, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - port_map=pmap, - iface_normalize_rules=norm_rules, - ) - sheet_results.append(one) - s = one["summary"] - for k in agg: - agg[k] += int(s.get(k) or 0) - mapping_by_metric[sheet_key(one)] = one["mapping_stats"] - - first = sheet_results[0] field_counts: dict[str, int] = {} - for s in sheet_results: - for d in list(s.get("diffs") or []): - if str(d.get("kind") or "") != "changed": - continue - for fname in d.get("changes") or {}: - field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1 - top_fields = sorted( - [{"field": k, "count": v} for k, v in field_counts.items()], - key=lambda x: (-int(x["count"]), str(x["field"])), - )[:8] + total = len(sheets_cfg) - run_id = uuid4().hex - # Persist counts/meta only — diffs go to biz_compare_diff rows - summary_payload = { - **agg, - "sheet_count": len(sheet_results), - "top_changed_fields": top_fields, - "sheets": [ - { - "sheet_id": s.get("sheet_id") or s["metric_id"], - "title": s.get("title") or s.get("sheet_id") or s["metric_id"], - "metric_id": s["metric_id"], - "key_fields": s["key_fields"], - "iface_fields": s["iface_fields"], - "compare_fields": s["compare_fields"], - "display_fields": s.get("display_fields") or [], - "field_rules": s.get("field_rules") or [], - "ignore_port_changes": s.get("ignore_port_changes"), - "mode": s["mode"], - "summary": s["summary"], + # Prefer seeded pending sheets from create; realign to small-first order + prev_metas = list((run.summary_json or {}).get("sheets") or []) + by_key = {sheet_key(m): m for m in prev_metas} + sheet_metas = [ + by_key.get(sheet_key(s)) or _pending_sheet_meta(s) for s in sheets_cfg + ] + + def _publish_sheets() -> None: + prev = dict(run.summary_json or {}) + prev["sheets"] = list(sheet_metas) + prev.update({k: agg[k] for k in agg}) + run.summary_json = prev + db.expire(run, ["status"]) + # Never resurrect cancelled while publishing incremental sheet metas + if str(run.status or "") == "cancelled": + db.commit() + return + db.commit() + + try: + for idx, sheet in enumerate(sheets_cfg, start=1): + if _run_is_cancelled(db, run_id): + run = db.get(BizCompareRun, run_id) or run + return get_run(db, run.id) + sid = sheet_key(sheet) + # Mark current sheet running in the sidebar list + for meta in sheet_metas: + if sheet_key(meta) == sid: + meta["status"] = "running" + break + _publish_sheets() + + _load_pub = {"t": 0.0, "n": -1, "engine": ""} + + def _on_load( + side: str, + n: int, + *, + engine: str = "python", + note: str = "", + phase: str | None = None, + _idx: int = idx, + _sheet: dict = sheet, + ) -> None: + now = time.monotonic() + eng = str(engine or "python") + # SQL emits sparse updates; Python still throttle chunk spam + if eng != "sql": + if n - _load_pub["n"] < 25_000 and now - _load_pub["t"] < 2.0: + return + _load_pub["t"] = now + _load_pub["n"] = n + _load_pub["engine"] = eng + extra: dict[str, Any] = { + "load_side": side, + "rows_loaded": n, + "engine": eng, + } + if note: + extra["engine_note"] = str(note)[:128] + # SQL path: detach progress commits so TEMP CTAS stays in one txn + _set_run_progress( + db, + run, + phase=str(phase or ("loading" if eng != "sql" else "sql_count")), + sheet_index=_idx, + sheet_total=total, + sheet=_sheet, + started_mono=started_mono, + extra=extra, + detach=(eng == "sql"), + ) + + _set_run_progress( + db, + run, + phase="loading", + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={"engine": "", "engine_note": ""}, + ) + one = _run_sheet( + db, + sheet=sheet, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + port_map=pmap, + iface_normalize_rules=norm_rules, + store_unchanged=store_mode, + on_load_progress=_on_load, + ) + s = one["summary"] + listed = int(s.get("unchanged_listed") or 0) + unchanged_listed_total += listed + if s.get("unchanged_truncated"): + unchanged_truncated_any = True + if s.get("unchanged_compact"): + unchanged_compact_any = True + _set_run_progress( + db, + run, + phase="comparing", + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={ + "before_count": int(s.get("before_count") or 0), + "after_count": int(s.get("after_count") or 0), + "diff_rows": int(s.get("added") or 0) + + int(s.get("removed") or 0) + + int(s.get("changed") or 0) + + int(s.get("duplicate") or 0) + + listed, + }, + ) + diffs = list(one.get("diffs") or []) + for d in diffs: + if str(d.get("kind") or "") != "changed": + continue + for fname in d.get("changes") or {}: + field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1 + # Cutover-first: persist fails so the UI can open the fail tab ASAP, + # then write success slim keys (sample / keys mode). + fail_diffs = [d for d in diffs if str(d.get("kind") or "") != "unchanged"] + ok_diffs = [d for d in diffs if str(d.get("kind") or "") == "unchanged"] + mid = sheet_key(one) + # Million-row vpnv4 with many diffs: keep key/row_id/changes only. + if len(fail_diffs) >= _FAIL_COMPACT_MIN: + for d in fail_diffs: + d["before"] = {} + d["after"] = {} + d["mapped_before"] = {} + d["compact"] = True + + def _on_persist( + written: int, + total_n: int, + *, + phase: str, + kind_key: str, + ) -> None: + _set_run_progress( + db, + run, + phase=phase, + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={ + kind_key: total_n, + "persisted": written, + "persist_total": total_n, + }, + # Separate session so chunk commits in persist do not race + # with progress JSON writes on the worker session. + detach=True, + ) + + _set_run_progress( + db, + run, + phase="persisting_fail", + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={ + "fail_rows": len(fail_diffs), + "persisted": 0, + "persist_total": len(fail_diffs), + }, + ) + n_fail = _persist_sheet_diffs( + db, + run_id=run.id, + metric_id=mid, + diffs=fail_diffs, + seq_start=0, + on_progress=lambda w, n: _on_persist( + w, n, phase="persisting_fail", kind_key="fail_rows" + ), + ) + if ok_diffs: + _set_run_progress( + db, + run, + phase="persisting_ok", + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={ + "ok_rows": len(ok_diffs), + "persisted": 0, + "persist_total": len(ok_diffs), + }, + ) + _persist_sheet_diffs( + db, + run_id=run.id, + metric_id=mid, + diffs=ok_diffs, + seq_start=n_fail, + on_progress=lambda w, n: _on_persist( + w, n, phase="persisting_ok", kind_key="ok_rows" + ), + ) + for k in agg: + agg[k] += int(s.get(k) or 0) + mapping_by_metric[mid] = one["mapping_stats"] + done_meta = { + "sheet_id": one.get("sheet_id") or one["metric_id"], + "title": one.get("title") or one.get("sheet_id") or one["metric_id"], + "metric_id": one["metric_id"], + "key_fields": one["key_fields"], + "iface_fields": one["iface_fields"], + "compare_fields": one["compare_fields"], + "display_fields": one.get("display_fields") or [], + "field_rules": one.get("field_rules") or [], + "row_filters": one.get("row_filters") or [], + "ignore_port_changes": one.get("ignore_port_changes"), + "mode": one["mode"], + "status": "done", + "summary": one["summary"], } - for s in sheet_results - ], - } + replaced = False + for i, meta in enumerate(sheet_metas): + if sheet_key(meta) == mid: + sheet_metas[i] = done_meta + replaced = True + break + if not replaced: + sheet_metas.append(done_meta) + _publish_sheets() + # Drop heavy diffs before next sheet + one.clear() + diffs.clear() + fail_diffs.clear() + ok_diffs.clear() + db.commit() + if _run_is_cancelled(db, run_id): + run = db.get(BizCompareRun, run_id) or run + return get_run(db, run.id) - run = BizCompareRun( - id=run_id, - job_id=j.id, - template_id=tpl.id, - mapping_id=j.mapping_id, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - metric_id=first["metric_id"], - status="success", - summary_json=summary_payload, - diffs_json=[], - mapping_stats_json=mapping_by_metric, - message="", - created_at=_utcnow(), - ) - db.add(run) - db.flush() - for s in sheet_results: - _persist_sheet_diffs( - db, - run_id=run_id, - metric_id=sheet_key(s), - diffs=list(s.get("diffs") or []), + if _run_is_cancelled(db, run_id): + run = db.get(BizCompareRun, run_id) or run + return get_run(db, run.id) + + duration_ms = int((time.monotonic() - started_mono) * 1000) + top_fields = sorted( + [{"field": k, "count": v} for k, v in field_counts.items()], + key=lambda x: (-int(x["count"]), str(x["field"])), + )[:8] + summary_payload = { + **agg, + "sheet_count": len(sheet_metas), + "top_changed_fields": top_fields, + "duration_ms": duration_ms, + "store_unchanged": store_mode, + "unchanged_stored": unchanged_listed_total > 0, + "unchanged_listed": unchanged_listed_total, + "unchanged_truncated": unchanged_truncated_any, + "unchanged_compact": unchanged_compact_any, + "progress": { + "phase": "done", + "sheet_index": total, + "sheet_total": total, + "sheet_id": "", + "sheet_title": "", + "elapsed_ms": duration_ms, + }, + "sheets": sheet_metas, + } + run = db.get(BizCompareRun, run_id) or run + if str(run.status or "") == "cancelled": + return get_run(db, run.id) + run.status = "success" + run.summary_json = summary_payload + run.mapping_stats_json = mapping_by_metric + run.metric_id = str(sheet_metas[0]["metric_id"]) if sheet_metas else run.metric_id + run.message = f"done · {duration_ms // 1000}s" + run.diffs_json = [] + j.updated_at = _utcnow() + if j.mode == "manual": + j.after_batch_id = after_batch_id + db.commit() + return get_run(db, run.id) + except HTTPException as exc: + run = db.get(BizCompareRun, run_id) or run + if str(run.status or "") != "cancelled": + run.status = "failed" + run.message = str(getattr(exc, "detail", "") or exc)[:1024] + prev = dict(run.summary_json or {}) + prog = dict(prev.get("progress") or {}) + prog["phase"] = "failed" + prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000) + prev["progress"] = prog + prev["duration_ms"] = prog["elapsed_ms"] + run.summary_json = prev + db.commit() + raise + except Exception as exc: + _log.exception("compare run failed run=%s job=%s", run_id, j.id) + run = db.get(BizCompareRun, run_id) or run + if str(run.status or "") != "cancelled": + run.status = "failed" + run.message = str(exc)[:1024] + prev = dict(run.summary_json or {}) + prog = dict(prev.get("progress") or {}) + prog["phase"] = "failed" + prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000) + prev["progress"] = prog + prev["duration_ms"] = prog["elapsed_ms"] + run.summary_json = prev + db.commit() + raise + + +_INTERRUPT_MARK = "interrupted_by_restart" + + +def _job_has_active_run(db: Session, job_id: str) -> BizCompareRun | None: + return ( + db.query(BizCompareRun) + .filter( + BizCompareRun.job_id == str(job_id or ""), + BizCompareRun.status.in_(("running", "queued")), ) - j.updated_at = _utcnow() - if j.mode == "manual": - j.after_batch_id = after_batch_id + .order_by(BizCompareRun.created_at.desc()) + .first() + ) + + +def _run_is_cancelled(db: Session, run_id: str) -> bool: + """Re-read status so user/startup cancel is visible to the worker thread.""" + db.expire_all() + r = db.get(BizCompareRun, run_id) + return bool(r and str(r.status or "") == "cancelled") + + +def recover_interrupted_compares_on_startup(db: Session) -> dict[str, Any]: + """Mark orphaned running/queued compare runs as cancelled after process restart. + + In-memory job locks die with the process; without this, the UI stays on + 「比对中」and blocks a new run. + """ + now = _utcnow() + rows = ( + db.query(BizCompareRun) + .filter(BizCompareRun.status.in_(("running", "queued"))) + .all() + ) + n = 0 + for r in rows: + r.status = "cancelled" + msg = str(r.message or "").strip() + if _INTERRUPT_MARK not in msg: + r.message = f"{msg} | {_INTERRUPT_MARK}".strip(" |")[:1024] + prev = dict(r.summary_json or {}) + prog = dict(prev.get("progress") or {}) + prog["phase"] = "cancelled" + prog["elapsed_ms"] = int(prog.get("elapsed_ms") or 0) + prev["progress"] = prog + # Mark in-flight sheet placeholders so UI does not show fake pass + sheets = list(prev.get("sheets") or []) + for sh in sheets: + st = str(sh.get("status") or "") + if st in ("pending", "running", "queued"): + sh["status"] = "cancelled" + prev["sheets"] = sheets + r.summary_json = prev + n += 1 + if n: + db.commit() + _log.info("startup: cancelled %s interrupted compare run(s)", n) + return {"runs": n, "at": now.isoformat() + "Z"} + + +def cancel_compare_run(db: Session, run_id: str) -> dict[str, Any]: + """Cancel a running/queued compare so a new run can start.""" + r = db.get(BizCompareRun, run_id) + if not r: + raise HTTPException(status_code=404, detail="run_not_found") + st = str(r.status or "") + if st not in ("running", "queued"): + return get_run(db, run_id) + r.status = "cancelled" + msg = str(r.message or "").strip() + r.message = f"{msg} | cancelled_by_user".strip(" |")[:1024] + prev = dict(r.summary_json or {}) + prog = dict(prev.get("progress") or {}) + prog["phase"] = "cancelled" + prev["progress"] = prog + for sh in list(prev.get("sheets") or []): + if str(sh.get("status") or "") in ("pending", "running", "queued"): + sh["status"] = "cancelled" + r.summary_json = prev db.commit() - return get_run(db, run.id) + return get_run(db, run_id) + + +def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]: + """Synchronous compare (auto-compare / tests). Blocks the caller until done.""" + if _job_has_active_run(db, job_id): + raise HTTPException(status_code=409, detail="compare_already_running") + lock = _job_compare_lock(job_id) + if not lock.acquire(blocking=False): + raise HTTPException(status_code=409, detail="compare_already_running") + try: + ctx = _validate_compare_job(db, job_id, force_after_batch_id=force_after_batch_id) + run = _create_running_run( + db, + job=ctx["job"], + tpl=ctx["template"], + before_batch_id=ctx["before_batch_id"], + after_batch_id=ctx["after_batch_id"], + sheets_cfg=ctx["sheets_cfg"], + ) + return _execute_compare_into_run(db, run.id) + finally: + lock.release() + + +def enqueue_compare( + db: Session, job_id: str, *, force_after_batch_id: str = "" +) -> dict[str, Any]: + """Create a ``running`` run and execute compare on a daemon thread. + + Returns immediately so the HTTP worker / UI stay responsive. Poll + ``GET /compare/runs/{id}`` for progress (``summary.progress``). + """ + if _job_has_active_run(db, job_id): + raise HTTPException(status_code=409, detail="compare_already_running") + lock = _job_compare_lock(job_id) + if not lock.acquire(blocking=False): + raise HTTPException(status_code=409, detail="compare_already_running") + run_id = "" + try: + ctx = _validate_compare_job(db, job_id, force_after_batch_id=force_after_batch_id) + run = _create_running_run( + db, + job=ctx["job"], + tpl=ctx["template"], + before_batch_id=ctx["before_batch_id"], + after_batch_id=ctx["after_batch_id"], + sheets_cfg=ctx["sheets_cfg"], + ) + run_id = run.id + except Exception: + lock.release() + raise + + def _bg() -> None: + from ..db import SessionLocal + + s = SessionLocal() + try: + _execute_compare_into_run(s, run_id) + except Exception: + _log.exception("bg compare failed job=%s run=%s", job_id, run_id) + finally: + s.close() + lock.release() + + threading.Thread( + target=_bg, + name=f"biz-cmp-{run_id[:8]}", + daemon=True, + ).start() + return get_run(db, run_id) def _csv_cell(v: Any) -> str: @@ -1723,12 +2899,15 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di st = sa + sr + sc + su sf = sr + sc sj = sf + su + status = str(sh.get("status") or "done") + pending = status in ("pending", "running", "queued") sheet_cards.append( { "sheet_id": sheet_key(sh), "title": sheet_title(sh), "metric_id": sh.get("metric_id") or "", "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"), + "status": status, "added": sa, "removed": sr, "changed": sc, @@ -1738,7 +2917,10 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di "fail_count": sf, "success_count": su, "diff_count": sf, - "pass_rate": round((su / sj) * 100, 1) if sj else (100.0 if st == 0 else 0.0), + # Pending sheets must not look like "100% pass" + "pass_rate": None + if pending + else (round((su / sj) * 100, 1) if sj else (100.0 if st == 0 else 0.0)), } ) @@ -1757,6 +2939,9 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di key=lambda x: (-int(x["count"]), str(x["field"])), )[:8] + any_pending = any( + str(sh.get("status") or "") in ("pending", "running", "queued") for sh in sheets + ) return { "added": added, "removed": removed, @@ -1770,11 +2955,18 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di "fail_count": fail_count, "success_count": success_count, "diff_count": diff_count, - "pass_rate": pass_rate, + "pass_rate": None if any_pending else pass_rate, "diff_rate": diff_rate, - "ok": fail_count == 0, + "ok": False if any_pending else fail_count == 0, "sheet_cards": sheet_cards, "top_changed_fields": top_fields, + "duration_ms": int(summary.get("duration_ms") or 0), + "store_unchanged": str(summary.get("store_unchanged") or ""), + "unchanged_stored": bool(summary.get("unchanged_stored", True)), + "unchanged_listed": int(summary.get("unchanged_listed") or 0), + "unchanged_truncated": bool(summary.get("unchanged_truncated")), + "unchanged_compact": bool(summary.get("unchanged_compact")), + "progress": dict(summary.get("progress") or {}), } @@ -1797,6 +2989,7 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: "display_fields": list(sh.get("display_fields") or []), "field_rules": list(sh.get("field_rules") or []), "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"), + "status": str(sh.get("status") or "done"), "summary": dict(sh.get("summary") or {}), } for sh in raw_sheets @@ -1850,6 +3043,358 @@ def _lookup_sheet(sheets: list[dict[str, Any]], key: str) -> dict[str, Any] | No return None +def _kind_allows(kind_n: str, diff_kind: str) -> bool: + dk = str(diff_kind or "") + if kind_n == "all": + return True + if kind_n == "diff": + return dk in ("removed", "changed") + return dk == kind_n + + +def _parse_qf(raw: Any) -> dict[str, str]: + """Normalize field query map: {field: value} with safe names only.""" + from .compare_sql import _FIELD_RE + + if raw is None or raw == "": + return {} + obj: Any = raw + if isinstance(raw, str): + s = raw.strip() + if not s: + return {} + try: + obj = json.loads(s) + except Exception: + return {} + if not isinstance(obj, dict): + return {} + out: dict[str, str] = {} + for k, v in obj.items(): + name = str(k or "").strip() + val = str(v or "").strip() + if not name or not val or not _FIELD_RE.match(name): + continue + out[name] = val + return out + + +def _split_kw_and_field_tokens(kw: str) -> tuple[str, dict[str, str]]: + """Parse ``direction:out network:1.1.1.1 foo`` → free kw + field map.""" + from .compare_sql import _FIELD_RE + + free: list[str] = [] + fields: dict[str, str] = {} + for tok in str(kw or "").split(): + if ":" in tok: + name, _, val = tok.partition(":") + name = name.strip() + val = val.strip() + if name and val and _FIELD_RE.match(name): + fields[name] = val + continue + if tok.strip(): + free.append(tok.strip()) + return " ".join(free), fields + + +def _kw_match_sql(key_fields: list[str], *, param: str = "kw") -> str: + """OR of ILIKE on key fields (and data_json::text fallback).""" + from .compare_sql import _FIELD_RE, _safe_field + + parts: list[str] = [] + for f in key_fields: + name = str(f or "").strip() + if not name or not _FIELD_RE.match(name): + continue + sf = _safe_field(name) + parts.append(f"lower(trim(both from coalesce(data_json->>'{sf}', ''))) LIKE :{param}") + # Broad fallback so free-text still hits non-key columns (path, next_hop, …) + parts.append(f"lower(data_json::text) LIKE :{param}") + return "(" + " OR ".join(parts) + ")" if parts else f"(lower(data_json::text) LIKE :{param})" + + +def _field_qf_sql(field_q: dict[str, str], *, prefix: str = "qf") -> tuple[str, dict[str, Any]]: + """AND of ILIKE contains on each field.""" + from .compare_sql import _safe_field + + if not field_q: + return "TRUE", {} + parts: list[str] = [] + params: dict[str, Any] = {} + for i, (name, val) in enumerate(field_q.items()): + sf = _safe_field(name) + key = f"{prefix}_{i}" + params[key] = f"%{val.lower()}%" + parts.append( + f"lower(trim(both from coalesce(data_json->>'{sf}', ''))) LIKE :{key}" + ) + return "(" + " AND ".join(parts) + ")", params + + +def _load_metric_rows_for_search( + db: Session, + *, + batch_id: str, + metric_id: str, + row_filters: list[dict[str, Any]] | None, + key_fields: list[str], + kw: str = "", + field_q: dict[str, str] | None = None, + cap: int = _LIVE_SEARCH_LOAD_CAP, +) -> tuple[list[dict[str, Any]], bool]: + """Load rows matching sheet filters + optional free kw / field_q. Returns (rows, truncated).""" + from .compare_sql import ( + _dialect_is_postgres, + _filters_sql_compatible, + compile_row_filters_sql, + ) + from ..models import BizStateMetricRow + from sqlalchemy import text as sql_text + + bid = str(batch_id or "").strip() + mid = str(metric_id or "").strip() + needle = str(kw or "").strip() + fq = {k: v for k, v in (field_q or {}).items() if str(v or "").strip()} + if not bid or not mid or (not needle and not fq): + return [], False + lim = max(1, min(int(cap), _LIVE_SEARCH_LOAD_CAP)) + filters = [f for f in (row_filters or []) if isinstance(f, dict)] + + if _dialect_is_postgres(db): + filter_sql, filter_params = ("TRUE", {}) + if filters and _filters_sql_compatible(filters): + filter_sql, filter_params = compile_row_filters_sql(filters) + qf_sql, qf_params = _field_qf_sql(fq) + search_parts = [qf_sql] + params: dict[str, Any] = { + "bid": bid, + "mid": mid, + "lim": lim + 1, + **filter_params, + **qf_params, + } + if needle: + params["kw"] = f"%{needle.lower()}%" + search_parts.append(_kw_match_sql(key_fields)) + search_sql = " AND ".join(f"({p})" for p in search_parts if p and p != "TRUE") + if not search_sql: + search_sql = "TRUE" + rows = db.execute( + sql_text( + f""" + SELECT id, batch_command_id, task_id, ne_id, seq, data_json, collected_at + FROM biz_state_metric_row + WHERE batch_id = :bid + AND metric_id = :mid + AND ({filter_sql}) + AND ({search_sql}) + ORDER BY seq ASC, id ASC + LIMIT :lim + """ + ), + params, + ).mappings().all() + truncated = len(rows) > lim + rows = rows[:lim] + out: list[dict[str, Any]] = [] + for r in rows: + data = dict(r["data_json"] or {}) + collected = r["collected_at"] + out.append( + { + **data, + "_netx": { + "batch_id": bid, + "batch_command_id": str(r["batch_command_id"] or ""), + "task_id": str(r["task_id"] or ""), + "ne_id": str(r["ne_id"] or ""), + "collected_at": collected.isoformat() + "Z" + if collected is not None + else None, + "row_id": str(r["id"]), + }, + } + ) + return out, truncated + + # Non-PG / fallback: scan with early stop (OK for tests / small sheets) + q = ( + db.query(BizStateMetricRow) + .filter( + BizStateMetricRow.batch_id == bid, + BizStateMetricRow.metric_id == mid, + ) + .order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) + ) + out = [] + truncated = False + needle_l = needle.lower() + key_set = [str(k).strip() for k in key_fields if str(k).strip()] + for r in q.yield_per(500): + data = dict(r.data_json or {}) + row = { + **data, + "_netx": { + "batch_id": bid, + "batch_command_id": r.batch_command_id or "", + "task_id": r.task_id or "", + "ne_id": r.ne_id or "", + "collected_at": r.collected_at.isoformat() + "Z" + if r.collected_at + else None, + "row_id": r.id, + }, + } + if filters and not all(row_matches_filter(row, f) for f in filters): + db.expunge(r) + continue + if fq: + ok_f = True + for fname, fval in fq.items(): + if fval.lower() not in str(row.get(fname) or "").lower(): + ok_f = False + break + if not ok_f: + db.expunge(r) + continue + if needle_l: + hit = False + for kf in key_set: + if needle_l in str(row.get(kf) or "").lower(): + hit = True + break + if not hit: + blob = json.dumps(data, ensure_ascii=False, default=str).lower() + hit = needle_l in blob + if not hit: + db.expunge(r) + continue + out.append(row) + db.expunge(r) + if len(out) >= lim: + truncated = True + break + return out, truncated + + +def _live_search_sheet_diffs( + db: Session, + run: BizCompareRun, + sheet: dict[str, Any], + *, + kind: str, + kw: str = "", + field_q: dict[str, str] | None = None, + page: int, + page_size: int, +) -> dict[str, Any]: + """Search before/after metric tables, zip-compare, filter by kind tab.""" + mid_src = str(sheet.get("metric_id") or "").strip() + sid = sheet_key(sheet) + key_fields = list(sheet.get("key_fields") or []) + iface_fields = list(sheet.get("iface_fields") or []) + field_rules = list(sheet.get("field_rules") or []) + compare_fields = effective_compare_fields( + list(sheet.get("compare_fields") or []), + field_rules, + ) + row_filters = list(sheet.get("row_filters") or []) + # Older runs may lack row_filters on sheet meta — fall back to template + if not row_filters and run.template_id: + tpl = db.get(BizCompareTemplate, run.template_id) + if tpl: + for s in template_metrics(tpl): + if sheet_key(s) == sid: + row_filters = list(s.get("row_filters") or []) + if not key_fields: + key_fields = list(s.get("key_fields") or []) + if not field_rules: + field_rules = list(s.get("field_rules") or []) + compare_fields = effective_compare_fields( + list(s.get("compare_fields") or compare_fields), + field_rules, + ) + break + + free_kw, tok_fields = _split_kw_and_field_tokens(kw) + merged_q = {**tok_fields, **(field_q or {})} + # Drop empty + merged_q = {k: v for k, v in merged_q.items() if str(v or "").strip()} + + if not key_fields or not mid_src: + return { + "total": 0, + "page": page, + "page_size": page_size, + "metric_id": sid, + "items": [], + "source": "live", + "truncated": False, + } + + ignore_ports = sheet.get("ignore_port_changes") + if ignore_ports is not None: + ignore_ports = bool(ignore_ports) + pmap = _port_map_dict(db, str(run.mapping_id or "")) + tpl = db.get(BizCompareTemplate, run.template_id) if run.template_id else None + norm_rules = template_iface_normalize(tpl) + + before_rows, trunc_b = _load_metric_rows_for_search( + db, + batch_id=str(run.before_batch_id or ""), + metric_id=mid_src, + row_filters=row_filters, + key_fields=key_fields, + kw=free_kw, + field_q=merged_q, + ) + after_rows, trunc_a = _load_metric_rows_for_search( + db, + batch_id=str(run.after_batch_id or ""), + metric_id=mid_src, + row_filters=row_filters, + key_fields=key_fields, + kw=free_kw, + field_q=merged_q, + ) + result = compare_rows( + before_rows=before_rows, + after_rows=after_rows, + key_fields=key_fields, + iface_fields=iface_fields, + compare_fields=compare_fields, + port_map=pmap, + field_rules=field_rules, + iface_normalize_rules=norm_rules, + ignore_port_changes=ignore_ports, + include_unchanged=True, + unchanged_limit=None, + compact_unchanged=False, + ) + kind_n = (kind or "diff").strip().lower() + filtered = [ + d for d in list(result.get("diffs") or []) if _kind_allows(kind_n, str(d.get("kind") or "")) + ] + # Cap pairs returned to keep UI snappy + truncated = bool(trunc_b or trunc_a or len(filtered) > _LIVE_SEARCH_RESULT_CAP) + filtered = filtered[:_LIVE_SEARCH_RESULT_CAP] + total = len(filtered) + start = (page - 1) * page_size + page_items = filtered[start : start + page_size] + return { + "total": total, + "page": page, + "page_size": page_size, + "metric_id": sid, + "items": page_items, + "source": "live", + "truncated": truncated, + "live_before_matched": len(before_rows), + "live_after_matched": len(after_rows), + } + + def list_run_diffs( db: Session, run_id: str, @@ -1857,6 +3402,7 @@ def list_run_diffs( metric_id: str = "", kind: str = "diff", kw: str = "", + qf: Any = None, page: int = 1, page_size: int = 100, ) -> dict[str, Any]: @@ -1867,6 +3413,12 @@ def list_run_diffs( size_n = max(1, min(500, int(page_size or 100))) kind_n = (kind or "diff").strip().lower() kw_n = (kw or "").strip() + field_q = _parse_qf(qf) + # Also accept field:value tokens inside kw + free_from_kw, tok_fields = _split_kw_and_field_tokens(kw_n) + if tok_fields: + field_q = {**tok_fields, **field_q} + kw_n = free_from_kw summary = dict(r.summary_json or {}) tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None @@ -1875,6 +3427,19 @@ def list_run_diffs( sheet = _lookup_sheet(sheets, asked) if asked else (sheets[0] if sheets else None) mid = sheet_key(sheet) if sheet else (asked or str(r.metric_id or "")) + # Unified search: kw and/or field filters → live source lookup; kind tab only filters. + if (kw_n or field_q) and sheet: + return _live_search_sheet_diffs( + db, + r, + sheet, + kind=kind_n, + kw=kw_n, + field_q=field_q, + page=page_n, + page_size=size_n, + ) + if _run_has_diff_rows(db, run_id): q = db.query(BizCompareDiff).filter( BizCompareDiff.run_id == run_id, @@ -1882,10 +3447,12 @@ def list_run_diffs( ) if kind_n == "diff": q = q.filter(BizCompareDiff.kind.in_(("removed", "changed"))) + elif kind_n == "removed": + q = q.filter(BizCompareDiff.kind == "removed") + elif kind_n == "changed": + q = q.filter(BizCompareDiff.kind == "changed") elif kind_n != "all": q = q.filter(BizCompareDiff.kind == kind_n) - if kw_n: - q = q.filter(BizCompareDiff.search_text.ilike(f"%{kw_n}%")) total = q.count() rows = ( q.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc()) @@ -1893,12 +3460,15 @@ def list_run_diffs( .limit(size_n) .all() ) + items = _hydrate_diff_rows(db, [_diff_row_out(x) for x in rows]) return { "total": total, "page": page_n, "page_size": size_n, "metric_id": mid, - "items": [_diff_row_out(x) for x in rows], + "items": items, + "source": "stored", + "truncated": False, } # Legacy: diffs embedded in summary_json / diffs_json @@ -1908,7 +3478,7 @@ def list_run_diffs( inline = list((sheet or {}).get("diffs") or []) if not inline and mid == r.metric_id: inline = list(r.diffs_json or []) - filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n) + filtered = _filter_inline_diffs(inline, kind=kind_n, kw="") total = len(filtered) start = (page_n - 1) * size_n page_items = filtered[start : start + size_n] @@ -1918,11 +3488,16 @@ def list_run_diffs( "page_size": size_n, "metric_id": mid, "items": page_items, + "source": "stored", + "truncated": False, } def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str, Any]]: - """Load all diffs for one sheet (export). Prefer row table; fall back to inline.""" + """Load all diffs for one sheet (export). Prefer row table; fall back to inline. + + Compact success rows are hydrated in chunks from metric tables. + """ if _run_has_diff_rows(db, run_id): out: list[dict[str, Any]] = [] offset = 0 @@ -1937,7 +3512,8 @@ def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str ) if not rows: break - out.extend(_diff_row_out(x) for x in rows) + chunk = _hydrate_diff_rows(db, [_diff_row_out(x) for x in rows]) + out.extend(chunk) offset += len(rows) if len(rows) < _DIFF_CHUNK: break @@ -2025,22 +3601,34 @@ def list_runs(db: Session, job_id: str, *, limit: int = 20) -> list[dict[str, An job = db.get(BizCompareJob, job_id) before_tid = str(job.before_task_id or "") if job else "" after_tid = str(job.after_task_id or "") if job else "" - return [ - { - "id": r.id, - "before_batch_id": r.before_batch_id, - "after_batch_id": r.after_batch_id, - "before": _compare_side(db, r.before_batch_id, fallback_task_id=before_tid), - "after": _compare_side(db, r.after_batch_id, fallback_task_id=after_tid), - "status": r.status, - "summary": { - k: (r.summary_json or {}).get(k, 0) - for k in ("added", "removed", "changed", "unchanged", "sheet_count") - }, - "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, - } - for r in rows - ] + out: list[dict[str, Any]] = [] + for r in rows: + summary = dict(r.summary_json or {}) + out.append( + { + "id": r.id, + "before_batch_id": r.before_batch_id, + "after_batch_id": r.after_batch_id, + "before": _compare_side(db, r.before_batch_id, fallback_task_id=before_tid), + "after": _compare_side(db, r.after_batch_id, fallback_task_id=after_tid), + "status": r.status, + "message": r.message or "", + "summary": { + k: summary.get(k, 0) + for k in ( + "added", + "removed", + "changed", + "unchanged", + "sheet_count", + "duration_ms", + ) + }, + "progress": dict(summary.get("progress") or {}), + "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, + } + ) + return out def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int: diff --git a/netx_api/biz_state/compare_sql.py b/netx_api/biz_state/compare_sql.py new file mode 100644 index 0000000..b7ba9d3 --- /dev/null +++ b/netx_api/biz_state/compare_sql.py @@ -0,0 +1,753 @@ +"""PostgreSQL-backed sheet compare (FULL OUTER JOIN) for pushdown-safe sheets. + +Falls back to the Python engine when port-map / complex rules cannot be expressed +in SQL. See ``can_sql_compare``. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Callable, Mapping, Sequence +from uuid import uuid4 + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from .compare_rules import effective_compare_fields, field_rule_map + +_log = logging.getLogger("netx.biz_state.compare_sql") + +# Below this, Python hash-join is faster and avoids TEMP CTAS / progress races. +_SQL_MIN_ROWS = 20_000 +# Cap a single SQL statement so a stuck planner/lock surfaces as fallback. +_SQL_STATEMENT_TIMEOUT_MS = 180_000 + +_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class SqlCompareSkip(Exception): + """Soft skip — caller should use the Python engine (not an error).""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = str(reason or "skip") +_SQL_FILTER_OPS = frozenset( + {"eq", "==", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "ci_eq"} +) +_SQL_NORMALIZE = frozenset({"", "none", "strip", "lower", "upper", "empty_as_blank"}) +_SQL_COMPARE_MODES = frozenset( + { + "", + "eq", + "ignore", + "skip", + "off", + "numeric", + "number", + "int", + "float", + "percent", + "pct", + "rel", + } +) +_EMPTY_AS_BLANK = ("n/a", "na", "-", "--", "none", "null") +_NUM_RE_SQL = r"^-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9]+)?$" + + +def _dialect_is_postgres(db: Session) -> bool: + bind = db.get_bind() + if bind is None: + return False + return str(getattr(bind.dialect, "name", "") or "").lower() in ("postgresql", "postgres") + + +def _safe_field(name: str) -> str: + f = str(name or "").strip() + if not f or not _FIELD_RE.match(f): + raise ValueError(f"unsafe_json_field:{name!r}") + return f + + +def _json_text_expr(alias: str, field: str) -> str: + """SQL expression: trimmed text from JSONB column ``alias.data`` / ``data_json``.""" + f = _safe_field(field) + # alias is caller-controlled (_b / data_json) — not user input + return f"trim(both from coalesce({alias}->>'{f}', ''))" + + +def _norm_expr(alias: str, field: str, normalize: str) -> str: + base = _json_text_expr(alias, field) + mode = str(normalize or "strip").strip().lower() or "strip" + if mode in ("", "none", "strip"): + return base + if mode == "lower": + return f"lower({base})" + if mode == "upper": + return f"upper({base})" + if mode == "empty_as_blank": + opts = ", ".join(f"'{x}'" for x in _EMPTY_AS_BLANK) + return ( + f"CASE WHEN lower({base}) IN ({opts}) THEN '' ELSE {base} END" + ) + raise ValueError(f"unsupported_normalize:{mode}") + + +def _filters_sql_compatible(filters: Sequence[Mapping[str, Any]] | None) -> bool: + fl = [f for f in (filters or []) if isinstance(f, dict)] + return all(_filter_node_compatible(f) for f in fl) + + +def _filter_node_compatible(filt: Mapping[str, Any]) -> bool: + if "any" in filt: + kids = filt.get("any") or [] + return isinstance(kids, list) and all( + isinstance(k, dict) and _filter_node_compatible(k) for k in kids + ) + if "all" in filt: + kids = filt.get("all") or [] + return isinstance(kids, list) and all( + isinstance(k, dict) and _filter_node_compatible(k) for k in kids + ) + field = str(filt.get("field") or "").strip() + op = str(filt.get("op") or "eq").strip().lower() + if op not in _SQL_FILTER_OPS: + return False + if op in ("empty", "not_empty", "nonempty"): + return bool(field) and bool(_FIELD_RE.match(field)) + if not field or not _FIELD_RE.match(field): + return False + if op in ("in", "not_in", "nin"): + vals = filt.get("value") + if vals is None: + return True + if not isinstance(vals, (list, tuple)): + vals = [vals] + return all(isinstance(x, (str, int, float, bool)) or x is None for x in vals) + return True + + +def _field_rules_sql_compatible(rules: Sequence[Mapping[str, Any]] | None) -> bool: + for raw in rules or []: + if not isinstance(raw, dict): + return False + name = str(raw.get("field") or "").strip() + if name and not _FIELD_RE.match(name): + return False + mode = str(raw.get("compare") or "eq").strip().lower() or "eq" + if mode not in _SQL_COMPARE_MODES: + return False + if raw.get("ignore") is True: + continue + if mode in ("ignore", "skip", "off"): + continue + norm = str(raw.get("normalize") or "strip").strip().lower() or "strip" + if norm not in _SQL_NORMALIZE: + return False + return True + + +def sql_compare_skip_reason( + db: Session, + sheet: Mapping[str, Any], + *, + port_map: Mapping[str, str] | None = None, + iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, +) -> str: + """Empty string when SQL is allowed; otherwise a short reason for progress/logs. + + Simple ``row_filters`` (eq/in/contains/…) used for BGP sheet splits are fine — + they do **not** force Python by themselves. + """ + if not _dialect_is_postgres(db): + return "not_postgres" + if port_map: + return "port_map" + key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] + if not key_fields: + return "no_key_fields" + if any(not _FIELD_RE.match(k) for k in key_fields): + return "unsafe_key_field" + iface_fields = [str(f).strip() for f in (sheet.get("iface_fields") or []) if str(f).strip()] + iface_set = set(iface_fields) + ignore_ports = sheet.get("ignore_port_changes") + if ignore_ports is True: + return "ignore_port_changes" + if ignore_ports is None and iface_set and any(k in iface_set for k in key_fields): + return "auto_ignore_ports" + field_rules = list(sheet.get("field_rules") or []) + if not _field_rules_sql_compatible(field_rules): + return "field_rules" + compare_fields = effective_compare_fields( + list(sheet.get("compare_fields") or []), + field_rules, + ) + if any(not _FIELD_RE.match(str(f).strip()) for f in compare_fields if str(f).strip()): + return "unsafe_compare_field" + used = set(key_fields) | {str(f).strip() for f in compare_fields if str(f).strip()} + if iface_normalize_rules and (used & iface_set): + return "iface_normalize" + if not _filters_sql_compatible(list(sheet.get("row_filters") or [])): + return "row_filters" + if not str(sheet.get("metric_id") or "").strip(): + return "no_metric_id" + return "" + + +def can_sql_compare( + db: Session, + sheet: Mapping[str, Any], + *, + port_map: Mapping[str, str] | None = None, + iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, +) -> bool: + """True when this sheet can run entirely as a PostgreSQL JOIN.""" + return not bool( + sql_compare_skip_reason( + db, + sheet, + port_map=port_map, + iface_normalize_rules=iface_normalize_rules, + ) + ) + + +def compile_row_filters_sql( + filters: Sequence[Mapping[str, Any]] | None, + *, + json_col: str = "data_json", + param_prefix: str = "f", +) -> tuple[str, dict[str, Any]]: + """Compile template row_filters to SQL AND-clause + bind params. + + Returns ``(sql_fragment, params)``. Empty filters → ``(\"TRUE\", {})``. + ``json_col`` is the JSONB column/expression name (trusted). + """ + fl = [f for f in (filters or []) if isinstance(f, dict)] + if not fl: + return "TRUE", {} + params: dict[str, Any] = {} + counter = {"n": 0} + + def _next(name: str) -> str: + counter["n"] += 1 + return f"{param_prefix}_{counter['n']}_{name}" + + def _node(filt: Mapping[str, Any]) -> str: + if "any" in filt: + kids = [k for k in (filt.get("any") or []) if isinstance(k, dict)] + if not kids: + return "TRUE" + return "(" + " OR ".join(_node(k) for k in kids) + ")" + if "all" in filt: + kids = [k for k in (filt.get("all") or []) if isinstance(k, dict)] + if not kids: + return "TRUE" + return "(" + " AND ".join(_node(k) for k in kids) + ")" + field = _safe_field(str(filt.get("field") or "")) + op = str(filt.get("op") or "eq").strip().lower() + expr = _json_text_expr(json_col, field) + # _json_text_expr uses alias->> ; for bare column use data_json directly + if json_col == "data_json": + expr = f"trim(both from coalesce(data_json->>'{field}', ''))" + if op in ("empty",): + return f"({expr} = '')" + if op in ("not_empty", "nonempty"): + return f"({expr} <> '')" + expect = filt.get("value") + if op in ("eq", "==", "ci_eq"): + key = _next("v") + params[key] = str(expect or "").strip() + if op == "ci_eq" or op in ("eq", "=="): + # Python eq is case-insensitive via .lower() + params[key] = str(expect or "").strip().lower() + return f"(lower({expr}) = :{key})" + if op in ("ne", "!="): + key = _next("v") + params[key] = str(expect or "").strip().lower() + return f"(lower({expr}) <> :{key})" + if op == "contains": + key = _next("v") + needle = str(expect or "").strip().lower() + params[key] = f"%{needle}%" + return f"(lower({expr}) LIKE :{key})" + if op in ("in", "not_in", "nin"): + vals = expect + if vals is None: + vals = [] + if not isinstance(vals, (list, tuple)): + vals = [vals] + clean = [str(x).strip().lower() for x in vals if str(x).strip()] + if not clean: + return "FALSE" if op == "in" else "TRUE" + keys = [] + for i, v in enumerate(clean): + k = _next(f"in{i}") + params[k] = v + keys.append(f":{k}") + inside = f"lower({expr}) IN ({', '.join(keys)})" + return f"({inside})" if op == "in" else f"(NOT {inside})" + raise ValueError(f"unsupported_filter_op:{op}") + + parts = [_node(f) for f in fl] + return "(" + " AND ".join(parts) + ")", params + + +def _rk_sql(key_fields: list[str], *, json_col: str = "data_json") -> str: + parts = [] + for k in key_fields: + f = _safe_field(k) + if json_col == "data_json": + parts.append(f"trim(both from coalesce(data_json->>'{f}', ''))") + else: + parts.append(f"trim(both from coalesce({json_col}->>'{f}', ''))") + if len(parts) == 1: + return parts[0] + return "concat_ws('|', " + ", ".join(parts) + ")" + + +def _field_differs_sql(bv: str, av: str, rule: Mapping[str, Any]) -> str: + """SQL boolean: True when before/after values differ under the field rule.""" + mode = str(rule.get("compare") or "eq").strip().lower() or "eq" + if mode in ("ignore", "skip", "off") or rule.get("ignore") is True: + return "FALSE" + try: + tol = float(rule.get("tolerance") or 0) + except (TypeError, ValueError): + tol = 0.0 + if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"): + # Match Python values_equal: parseable → numeric compare; else string eq + num_b = f"(({bv}) ~ '{_NUM_RE_SQL}')" + num_a = f"(({av}) ~ '{_NUM_RE_SQL}')" + bn = f"({bv})::double precision" + an = f"({av})::double precision" + if mode in ("percent", "pct", "rel"): + # differ when relative % > tol (before==0 → after must be 0) + num_diff = ( + f"(CASE WHEN {bn} = 0 THEN {an} IS DISTINCT FROM 0 " + f"ELSE (abs({an} - {bn}) / abs({bn}) * 100.0) > {tol} END)" + ) + else: + num_diff = f"(abs({an} - {bn}) > {tol})" + return ( + f"(CASE WHEN {num_b} AND {num_a} THEN {num_diff} " + f"ELSE ({bv} IS DISTINCT FROM {av}) END)" + ) + return f"({bv} IS DISTINCT FROM {av})" + + +def _changed_predicate( + compare_fields: list[str], + rules: dict[str, dict[str, Any]], + *, + before_alias: str = "b", + after_alias: str = "a", +) -> str: + """SQL boolean: True when any compare field differs (IS DISTINCT FROM).""" + if not compare_fields: + return "FALSE" + clauses: list[str] = [] + for f in compare_fields: + name = str(f).strip() + if not name: + continue + rule = rules.get(name) or {} + mode = str(rule.get("compare") or "eq").strip().lower() or "eq" + if mode in ("ignore", "skip", "off") or rule.get("ignore") is True: + continue + norm = str(rule.get("normalize") or "strip").strip().lower() or "strip" + bv = _norm_expr(f"{before_alias}.data", name, norm) + av = _norm_expr(f"{after_alias}.data", name, norm) + clauses.append(_field_differs_sql(bv, av, rule)) + if not clauses: + return "FALSE" + return "(" + " OR ".join(clauses) + ")" + + +def _key_obj_from_data(data: dict[str, Any] | None, key_fields: list[str]) -> dict[str, Any]: + row = data if isinstance(data, dict) else {} + return {f: row.get(f, "") for f in key_fields} + + +def _changes_from_rows( + before: dict[str, Any] | None, + after: dict[str, Any] | None, + compare_fields: list[str], + rules: dict[str, dict[str, Any]], +) -> dict[str, dict[str, Any]]: + from .compare_rules import explain_diff, values_equal + + out: dict[str, dict[str, Any]] = {} + b = before or {} + a = after or {} + for f in compare_fields: + rule = rules.get(f) + bv = b.get(f, "") + av = a.get(f, "") + if values_equal(bv, av, rule=rule): + continue + entry: dict[str, Any] = {"before": bv, "after": av} + reason = explain_diff(bv, av, rule=rule) + if reason: + entry["reason"] = reason + out[f] = entry + return out + + +def run_sql_sheet_compare( + db: Session, + *, + sheet: Mapping[str, Any], + before_batch_id: str, + after_batch_id: str, + store_unchanged: str = "auto", + on_progress: Callable[..., None] | None = None, +) -> dict[str, Any]: + """Compare one sheet via PostgreSQL TEMP tables + FULL OUTER JOIN. + + Same-key multipath: rows get ``dup_rn`` by ``seq,id`` and join on + ``(rk, dup_rn)`` (ordered zip). Extras are added/removed, not duplicate. + + Returns the same ``{summary, diffs, mapping_stats}`` shape as ``compare_rows``. + ``on_progress(side, n, *, engine=\"sql\", note=..., phase=...)``. + """ + if not _dialect_is_postgres(db): + raise RuntimeError("sql_compare_requires_postgres") + + def _prog(side: str, n: int, *, phase: str = "sql_count", note: str = "") -> None: + if not on_progress: + return + try: + on_progress(side, n, engine="sql", note=note, phase=phase) + except TypeError: + on_progress(side, n) + + key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] + field_rules = list(sheet.get("field_rules") or []) + rules = field_rule_map(field_rules) + compare_fields = effective_compare_fields( + list(sheet.get("compare_fields") or []), + field_rules, + ) + row_filters = list(sheet.get("row_filters") or []) + mid = str(sheet.get("metric_id") or "").strip() + bid_b = str(before_batch_id or "").strip() + bid_a = str(after_batch_id or "").strip() + if not mid or not bid_b or not bid_a or not key_fields: + raise ValueError("sql_compare_missing_args") + + filter_sql, filter_params = compile_row_filters_sql(row_filters) + rk = _rk_sql(key_fields) + tag = uuid4().hex[:8] + tb = f"_netx_cmp_b_{tag}" + ta = f"_netx_cmp_a_{tag}" + + # Keep worker transaction open across CTAS — progress must NOT commit this session. + db.execute(text(f"SET LOCAL statement_timeout = '{int(_SQL_STATEMENT_TIMEOUT_MS)}'")) + + _prog("before", 0, phase="sql_count", note="count") + + # Raw counts (no row_filters) + raw_b = int( + db.execute( + text( + "SELECT count(*) FROM biz_state_metric_row " + "WHERE batch_id = :bid AND metric_id = :mid" + ), + {"bid": bid_b, "mid": mid}, + ).scalar() + or 0 + ) + _prog("before", raw_b, phase="sql_count") + raw_a = int( + db.execute( + text( + "SELECT count(*) FROM biz_state_metric_row " + "WHERE batch_id = :bid AND metric_id = :mid" + ), + {"bid": bid_a, "mid": mid}, + ).scalar() + or 0 + ) + _prog("after", raw_a, phase="sql_count") + + if max(raw_b, raw_a) < int(_SQL_MIN_ROWS): + raise SqlCompareSkip("small_sheet") + + base_params = {"bid": bid_b, "mid": mid, **filter_params} + # Build TEMP sides (one transaction — no mid-flight commits on ``db``) + _prog("before", raw_b, phase="sql_project", note="temp_before") + for tname, batch_id, side, note in ( + (tb, bid_b, "before", "temp_before"), + (ta, bid_a, "after", "temp_after"), + ): + db.execute(text(f"DROP TABLE IF EXISTS {tname}")) + params = {**base_params, "bid": batch_id} + if side == "after": + _prog(side, raw_a, phase="sql_project", note=note) + db.execute( + text( + f""" + CREATE TEMP TABLE {tname} ON COMMIT PRESERVE ROWS AS + SELECT + id, + ({rk}) AS rk, + data_json AS data, + row_number() OVER ( + PARTITION BY ({rk}) + ORDER BY seq ASC, id ASC + ) AS dup_rn + FROM biz_state_metric_row + WHERE batch_id = :bid + AND metric_id = :mid + AND ({filter_sql}) + """ + ), + params, + ) + db.execute(text(f"CREATE INDEX IF NOT EXISTS {tname}_rk ON {tname} (rk, dup_rn)")) + + before_n = int( + db.execute(text(f"SELECT count(*) FROM {tb}")).scalar() or 0 + ) + after_n = int( + db.execute(text(f"SELECT count(*) FROM {ta}")).scalar() or 0 + ) + _prog("after", after_n, phase="sql_join", note="join") + + changed_pred = _changed_predicate(compare_fields, rules) + kind_expr = f""" + CASE + WHEN b.id IS NULL THEN 'added' + WHEN a.id IS NULL THEN 'removed' + WHEN {changed_pred} THEN 'changed' + ELSE 'unchanged' + END + """ + + # Ordered same-key pairing: join on (rk, dup_rn) so 5 vs 2 → 2 compared + 3 removed + agg_rows = db.execute( + text( + f""" + SELECT {kind_expr} AS kind, count(*)::bigint AS n + FROM {tb} b + FULL OUTER JOIN {ta} a + ON b.rk = a.rk AND b.dup_rn = a.dup_rn + GROUP BY 1 + """ + ) + ).mappings().all() + counts = {str(r["kind"]): int(r["n"] or 0) for r in agg_rows} + added = counts.get("added", 0) + removed = counts.get("removed", 0) + changed = counts.get("changed", 0) + unchanged = counts.get("unchanged", 0) + + # Diagnostic: count of match keys with >1 row (not fail rows) + multi_b = int( + db.execute( + text(f"SELECT count(*) FROM (SELECT rk FROM {tb} GROUP BY rk HAVING count(*) > 1) t") + ).scalar() + or 0 + ) + multi_a = int( + db.execute( + text(f"SELECT count(*) FROM (SELECT rk FROM {ta} GROUP BY rk HAVING count(*) > 1) t") + ).scalar() + or 0 + ) + + # Lazy import — avoid circular import with compare_service + from .compare_service import resolve_unchanged_policy + + policy = resolve_unchanged_policy( + store_unchanged, before_n=before_n, after_n=after_n + ) + diffs: list[dict[str, Any]] = [] + + # Fail rows (stream into Python — should be << million) + fail_sql = text( + f""" + SELECT + {kind_expr} AS kind, + b.id AS before_row_id, + a.id AS after_row_id, + b.data AS before_data, + a.data AS after_data, + COALESCE(b.rk, a.rk) AS rk + FROM {tb} b + FULL OUTER JOIN {ta} a + ON b.rk = a.rk AND b.dup_rn = a.dup_rn + WHERE {kind_expr} IN ('added', 'removed', 'changed') + """ + ) + fail_n = 0 + _prog("after", after_n, phase="sql_fail_fetch", note="fetch_fails") + for row in db.execute(fail_sql).mappings(): + kind = str(row["kind"] or "") + before = dict(row["before_data"] or {}) if row["before_data"] is not None else None + after = dict(row["after_data"] or {}) if row["after_data"] is not None else None + key_src = after if kind == "added" else (before or after or {}) + item: dict[str, Any] = { + "kind": kind, + "key": _key_obj_from_data(key_src, key_fields), + "before": before, + "after": after, + "mapped_before": before, + "changes": {}, + "before_row_id": str(row["before_row_id"] or ""), + "after_row_id": str(row["after_row_id"] or ""), + } + if kind == "changed": + item["changes"] = _changes_from_rows(before, after, compare_fields, rules) + diffs.append(item) + fail_n += 1 + if fail_n == 1 or fail_n % 25_000 == 0: + _prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails") + if fail_n: + _prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails") + + unchanged_listed = 0 + include_u = bool(policy.get("include")) + compact = bool(policy.get("compact")) + limit_n = policy.get("limit") + if include_u and unchanged > 0: + params_u: dict[str, Any] = {} + # Stratified sample: round-robin by neighbor|direction|afi|vrf so one + # BGP command cannot consume the whole success sample. + stratum_expr = """ + concat_ws('|', + coalesce(nullif(trim(both from coalesce(b.data->>'neighbor','')), ''), '_'), + coalesce(nullif(trim(both from coalesce(b.data->>'direction','')), ''), '_'), + coalesce(nullif(trim(both from coalesce(b.data->>'afi','')), ''), '_'), + coalesce(nullif(trim(both from coalesce(b.data->>'vrf','')), ''), '_') + ) + """ + if limit_n is not None: + params_u["lim"] = max(0, int(limit_n)) + u_sql = text( + f""" + WITH u AS ( + SELECT + b.id AS before_row_id, + a.id AS after_row_id, + b.data AS before_data, + a.data AS after_data, + {stratum_expr} AS stratum + FROM {tb} b + INNER JOIN {ta} a + ON b.rk = a.rk AND b.dup_rn = a.dup_rn + WHERE NOT ({changed_pred}) + ), + ranked AS ( + SELECT *, + row_number() OVER ( + PARTITION BY stratum ORDER BY before_row_id ASC + ) AS rn_in + FROM u + ), + picked AS ( + SELECT *, + row_number() OVER ( + ORDER BY rn_in ASC, stratum ASC, before_row_id ASC + ) AS pick_ord + FROM ranked + ) + SELECT before_row_id, after_row_id, before_data, after_data + FROM picked + WHERE pick_ord <= :lim + """ + ) + else: + u_sql = text( + f""" + SELECT + b.id AS before_row_id, + a.id AS after_row_id, + b.data AS before_data, + a.data AS after_data + FROM {tb} b + INNER JOIN {ta} a + ON b.rk = a.rk AND b.dup_rn = a.dup_rn + WHERE NOT ({changed_pred}) + """ + ) + for row in db.execute(u_sql, params_u).mappings(): + before = dict(row["before_data"] or {}) + after = dict(row["after_data"] or {}) + unchanged_listed += 1 + if compact: + diffs.append( + { + "kind": "unchanged", + "key": _key_obj_from_data(before, key_fields), + "before": {}, + "after": {}, + "mapped_before": {}, + "changes": {}, + "compact": True, + "before_row_id": str(row["before_row_id"] or ""), + "after_row_id": str(row["after_row_id"] or ""), + } + ) + else: + diffs.append( + { + "kind": "unchanged", + "key": _key_obj_from_data(before, key_fields), + "before": before, + "after": after, + "mapped_before": before, + "changes": {}, + "before_row_id": str(row["before_row_id"] or ""), + "after_row_id": str(row["after_row_id"] or ""), + } + ) + + # Cleanup (also ON COMMIT DROP) + db.execute(text(f"DROP TABLE IF EXISTS {tb}")) + db.execute(text(f"DROP TABLE IF EXISTS {ta}")) + + summary = { + "before_count": before_n, + "after_count": after_n, + "added": added, + "removed": removed, + "changed": changed, + "unchanged": unchanged, + "duplicate": 0, + "match_key_fields": list(key_fields), + "duplicate_keys_before": multi_b, + "duplicate_keys_after": multi_a, + "duplicate_key_list": [], + "unchanged_listed": unchanged_listed, + "unchanged_truncated": bool( + include_u and limit_n is not None and unchanged > unchanged_listed + ), + "unchanged_compact": bool(compact and unchanged_listed > 0), + "unchanged_sample_mode": ( + "stratified" + if include_u and limit_n is not None and unchanged > unchanged_listed + else ("full" if include_u else "none") + ), + "engine": "sql", + "before_raw_count": raw_b, + "after_raw_count": raw_a, + "row_filters": len(row_filters), + "unchanged_policy": policy, + } + mapping_stats = { + "before_iface_count": 0, + "after_iface_count": 0, + "map_pairs": 0, + "hit_before": [], + "miss_before": [], + "hit_after": [], + "miss_after": [], + "unused_before_keys": [], + "ok": True, + "ignore_port_changes": False, + "engine": "sql", + } + return {"summary": summary, "diffs": diffs, "mapping_stats": mapping_stats} diff --git a/netx_api/biz_state/data/default_zte_status_template.json b/netx_api/biz_state/data/default_zte_status_template.json new file mode 100644 index 0000000..d643cbc --- /dev/null +++ b/netx_api/biz_state/data/default_zte_status_template.json @@ -0,0 +1,1051 @@ +{ + "format": "netx.biz_compare_template", + "version": 1, + "name": "ZTE status default", + "note": "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP, address-family sheets)", + "iface_normalize_rules": [], + "metrics": [ + { + "sheet_id": "isis_adjacency.ipv4", + "title": "ISIS IPv4", + "metric_id": "isis_adjacency", + "key_fields": [ + "process_id", + "interface", + "system_id" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state", + "lev", + "af" + ], + "display_fields": [ + "process_id", + "interface", + "system_id", + "state", + "lev", + "af" + ], + "row_filters": [ + { + "op": "contains", + "field": "af", + "value": "IPv4" + } + ], + "field_rules": [] + }, + { + "sheet_id": "isis_adjacency.ipv6", + "title": "ISIS IPv6", + "metric_id": "isis_adjacency", + "key_fields": [ + "process_id", + "interface", + "system_id" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state", + "lev", + "af" + ], + "display_fields": [ + "process_id", + "interface", + "system_id", + "state", + "lev", + "af" + ], + "row_filters": [ + { + "op": "contains", + "field": "af", + "value": "IPv6" + } + ], + "field_rules": [] + }, + { + "sheet_id": "interface_brief", + "title": "interface_brief", + "metric_id": "interface_brief", + "key_fields": [ + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "prot" + ], + "display_fields": [ + "interface", + "prot", + "admin", + "phy" + ], + "row_filters": [ + { + "op": "regex", + "field": "interface", + "value": "^[^.]+$" + }, + { + "op": "contains", + "field": "interface", + "value": "gei" + } + ], + "field_rules": [] + }, + { + "sheet_id": "arp", + "title": "arp", + "metric_id": "arp", + "key_fields": [ + "ip", + "interface", + "vrf" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "mac" + ], + "display_fields": [ + "ip", + "interface", + "vrf", + "mac", + "age", + "entry_type" + ], + "row_filters": [ + { + "op": "age_timer", + "field": "age", + "value": "" + } + ], + "field_rules": [ + { + "field": "mac", + "normalize": "mac" + } + ] + }, + { + "sheet_id": "nd6_cache", + "title": "nd6_cache", + "metric_id": "nd6_cache", + "key_fields": [ + "address", + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "link_address", + "status" + ], + "display_fields": [ + "address", + "interface", + "link_address", + "status", + "type", + "age" + ], + "row_filters": [ + { + "op": "ne", + "field": "status", + "value": " Incomplete" + } + ], + "field_rules": [] + }, + { + "sheet_id": "bgp_peer.ipv4", + "title": "BGP IPv4", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "ipv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.ipv6", + "title": "BGP IPv6", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "ipv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.vpnv4", + "title": "BGP VPNv4", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpnv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.vpnv6", + "title": "BGP VPNv6", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpnv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.evpn", + "title": "BGP EVPN", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "evpn" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.vpls", + "title": "BGP VPLS", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpls" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "interface_detail", + "title": "interface_detail", + "metric_id": "interface_detail", + "key_fields": [ + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "admin", + "input_bps", + "output_bps", + "port_status", + "in_util", + "out_util", + "ip_mtu", + "mtu", + "mpls_mtu", + "ipv6_mtu", + "port_media", + "negotiation" + ], + "display_fields": [ + "interface", + "admin", + "input_bps", + "output_bps", + "port_status", + "in_util", + "out_util", + "ip_mtu", + "mtu", + "mpls_mtu", + "ipv6_mtu", + "port_media", + "negotiation", + "bw", + "description", + "rate_period" + ], + "row_filters": [], + "field_rules": [ + { + "field": "input_bps", + "compare": "percent", + "tolerance": 5 + }, + { + "field": "output_bps", + "compare": "numeric", + "tolerance": 5 + }, + { + "field": "in_util", + "compare": "numeric", + "tolerance": 5 + }, + { + "field": "out_util", + "compare": "numeric", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "ospf_neighbor", + "title": "ospf_neighbor", + "metric_id": "ospf_neighbor", + "key_fields": [ + "neighbor_id", + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state" + ], + "display_fields": [ + "neighbor_id", + "interface", + "state", + "process_id", + "address" + ], + "row_filters": [], + "field_rules": [] + }, + { + "sheet_id": "vrrp.ipv4", + "title": "VRRP IPv4", + "metric_id": "vrrp", + "key_fields": [ + "af", + "interface", + "vr_id" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state", + "priority", + "master_addr", + "vrouter_addr" + ], + "display_fields": [ + "af", + "interface", + "vr_id", + "state", + "priority", + "master_addr", + "vrouter_addr" + ], + "row_filters": [ + { + "op": "eq", + "field": "af", + "value": "ipv4" + } + ], + "field_rules": [] + }, + { + "sheet_id": "vrrp.ipv6", + "title": "VRRP IPv6", + "metric_id": "vrrp", + "key_fields": [ + "af", + "interface", + "vr_id" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state", + "priority", + "master_addr", + "vrouter_addr" + ], + "display_fields": [ + "af", + "interface", + "vr_id", + "state", + "priority", + "master_addr", + "vrouter_addr" + ], + "row_filters": [ + { + "op": "eq", + "field": "af", + "value": "ipv6" + } + ], + "field_rules": [] + }, + { + "sheet_id": "optical_brief", + "title": "optical_brief", + "metric_id": "optical_brief", + "key_fields": [ + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "status", + "rx_power" + ], + "display_fields": [ + "interface", + "status", + "rx_power", + "tx_power", + "optic_type", + "wavelength", + "rx_threshold", + "tx_threshold" + ], + "row_filters": [], + "field_rules": [ + { + "field": "rx_power", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "lldp_neighbor", + "title": "lldp_neighbor", + "metric_id": "lldp_neighbor", + "key_fields": [ + "local_if", + "remote_sys", + "remote_if" + ], + "iface_fields": [ + "local_if" + ], + "compare_fields": [], + "display_fields": [ + "local_if", + "remote_sys", + "remote_if", + "remote_ip", + "protocol" + ], + "row_filters": [], + "field_rules": [] + }, + { + "sheet_id": "l2vpn_pw_detail", + "title": "l2vpn_pw_detail", + "metric_id": "l2vpn_pw_detail", + "key_fields": [ + "peer", + "vcid", + "service_instance" + ], + "iface_fields": [], + "compare_fields": [ + "vc_status", + "remote_status", + "vc_type", + "control_word" + ], + "display_fields": [ + "peer", + "vcid", + "service_instance", + "vc_status", + "remote_status", + "vc_type", + "control_word", + "pw_name", + "activation_status", + "service_instance_type", + "conn_mode", + "signaling", + "tunnel_dest" + ], + "row_filters": [], + "field_rules": [] + }, + { + "sheet_id": "l2vpn_mac", + "title": "l2vpn_mac", + "metric_id": "l2vpn_mac", + "key_fields": [ + "mac", + "vpn", + "neighbor", + "ac_port", + "exter_vlan", + "vpn_sid", + "neighbor_sid" + ], + "iface_fields": [], + "compare_fields": [], + "display_fields": [ + "mac", + "vpn", + "neighbor", + "ac_port", + "exter_vlan", + "vpn_sid", + "neighbor_sid", + "vlan", + "pw", + "attribute" + ], + "row_filters": [], + "field_rules": [] + }, + { + "sheet_id": "bgp_route", + "title": "bgp_route_ipv4", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "empty", + "field": "vrf", + "value": "" + }, + { + "op": "eq", + "field": "afi", + "value": "ipv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.vpnv4", + "title": "bgp_route_vpnv4", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpnv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.ipv6", + "title": "bgp_route_ipv6", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "empty", + "field": "vrf", + "value": "" + }, + { + "op": "eq", + "field": "afi", + "value": "ipv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.vpnv6", + "title": "bgp_route_vpnv6", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpnv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.vpnv6_vrf", + "title": "bgp_route_vpnv6_vrf", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "not_empty", + "field": "vrf", + "value": "" + }, + { + "op": "eq", + "field": "afi", + "value": "ipv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.vpnv4_vrf", + "title": "bgp_route_vpnv4_vrf", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "not_empty", + "field": "vrf", + "value": "" + }, + { + "op": "eq", + "field": "afi", + "value": "ipv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + } + ] +} diff --git a/netx_api/biz_state/schema_ensure.py b/netx_api/biz_state/schema_ensure.py index 6b1013f..2179310 100644 --- a/netx_api/biz_state/schema_ensure.py +++ b/netx_api/biz_state/schema_ensure.py @@ -23,6 +23,7 @@ def apply_biz_state_schema(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_command_batch_id ON biz_state_batch_command (batch_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_lldp_neighbor_batch_id ON biz_state_lldp_neighbor (batch_id)", "ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS enabled_sheet_ids JSON DEFAULT '[]'", + "ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS store_unchanged VARCHAR(16) DEFAULT 'auto'", "CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)", @@ -31,6 +32,8 @@ def apply_biz_state_schema(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_id ON biz_compare_diff (run_id)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_kind ON biz_compare_diff (run_id, metric_id, kind)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_seq ON biz_compare_diff (run_id, metric_id, seq)", + "ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS before_row_id VARCHAR(64) DEFAULT ''", + "ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS after_row_id VARCHAR(64) DEFAULT ''", "CREATE INDEX IF NOT EXISTS ix_biz_migration_project_status ON biz_migration_project (status)", "CREATE INDEX IF NOT EXISTS ix_biz_migration_batch_project_id ON biz_migration_batch (project_id)", "CREATE INDEX IF NOT EXISTS ix_biz_migration_run_batch_id ON biz_migration_run (batch_id)", diff --git a/netx_api/biz_state_router.py b/netx_api/biz_state_router.py index 4a4f3ed..20a0319 100644 --- a/netx_api/biz_state_router.py +++ b/netx_api/biz_state_router.py @@ -633,6 +633,8 @@ class CompareJobIn(BaseModel): mode: str = "manual" # Empty = all template sheets; non-empty = only these sheet_id values enabled_sheet_ids: list[str] = Field(default_factory=list) + # auto|always|never|sample|keys — how to persist matching (success) rows + store_unchanged: str = "auto" note: str = "" @@ -723,8 +725,15 @@ def api_delete_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any] @router.post("/compare/jobs/{job_id}/run") -def api_run_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: - return cmp_svc.run_compare(db, job_id) +def api_run_job( + job_id: str, + sync: bool = Query(False, description="If true, block until compare finishes (tests/debug)"), + db: Session = Depends(get_db), +) -> dict[str, Any]: + """Start a compare run. Default is async (returns status=running immediately).""" + if sync: + return cmp_svc.run_compare(db, job_id) + return cmp_svc.enqueue_compare(db, job_id) @router.get("/compare/jobs/{job_id}/runs") @@ -737,6 +746,12 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return cmp_svc.get_run(db, run_id) +@router.post("/compare/runs/{run_id}/cancel") +def api_cancel_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: + """Cancel a stuck/running compare so a new run can start.""" + return cmp_svc.cancel_compare_run(db, run_id) + + @router.delete("/compare/runs/{run_id}") def api_delete_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return cmp_svc.delete_run(db, run_id) @@ -748,6 +763,7 @@ def api_list_run_diffs( metric_id: str = "", kind: str = "diff", kw: str = "", + qf: str = "", page: int = 1, page_size: int = 100, db: Session = Depends(get_db), @@ -758,6 +774,7 @@ def api_list_run_diffs( metric_id=metric_id, kind=kind, kw=kw, + qf=qf, page=page, page_size=page_size, ) diff --git a/netx_api/config.py b/netx_api/config.py index 1126fc5..e12b5fd 100644 --- a/netx_api/config.py +++ b/netx_api/config.py @@ -80,8 +80,9 @@ class Settings(BaseSettings): ume_topo_nodes_path: str = "/restconf/data/zte-resources-module:TopoNodes" ume_topological_links_path: str = "/restconf/data/zte-resources-module:TopologicalLinks" ume_sync_alarms_history_every_hours: int = 24 - # Managed NE credentials (Fernet key; generate with cryptography.fernet.Fernet.generate_key()) + # Managed NE credentials (Fernet). Empty = auto-generate & persist to credential_secret_file. credential_secret_key: str = "" + credential_secret_file: str = "data/auth/credential_secret" # Shared-server worker caps (sized for multi-operator use; raise if bastion/DB allow). ne_connect_max_workers: int = 8 ne_connect_timeout_sec: int = 30 diff --git a/netx_api/models/biz_state.py b/netx_api/models/biz_state.py index bd8c311..a434dc7 100644 --- a/netx_api/models/biz_state.py +++ b/netx_api/models/biz_state.py @@ -303,6 +303,8 @@ class BizCompareJob(Base): status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|ready|auto # Empty = all template sheets; non-empty = only these sheet_id values enabled_sheet_ids: Mapped[list] = mapped_column(_JsonType, default=list) + # auto|always|never|sample — how to persist matching (success) rows + store_unchanged: Mapped[str] = mapped_column(String(16), default="auto") note: Mapped[str] = mapped_column(String(512), default="") created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) @@ -347,4 +349,7 @@ class BizCompareDiff(Base): after_json: Mapped[dict] = mapped_column(_JsonType, default=dict) mapped_before_json: Mapped[dict] = mapped_column(_JsonType, default=dict) changes_json: Mapped[dict] = mapped_column(_JsonType, default=dict) + # Pointers into BizStateMetricRow / LLDP tables for compact success hydrate + before_row_id: Mapped[str] = mapped_column(String(64), default="") + after_row_id: Mapped[str] = mapped_column(String(64), default="") search_text: Mapped[str] = mapped_column(Text, default="") diff --git a/netx_api/ne_crypto.py b/netx_api/ne_crypto.py index 4ba7617..7624c2a 100644 --- a/netx_api/ne_crypto.py +++ b/netx_api/ne_crypto.py @@ -1,18 +1,73 @@ from __future__ import annotations +import logging +from pathlib import Path + from cryptography.fernet import Fernet, InvalidToken from .config import settings +_log = logging.getLogger("netx.crypto") +_cached_credential_key: str | None = None + class CredentialCryptoError(RuntimeError): pass +def credential_secret_file_path() -> Path: + raw = str(getattr(settings, "credential_secret_file", None) or "data/auth/credential_secret").strip() + path = Path(raw) + if not path.is_absolute(): + path = Path.cwd() / path + return path + + +def ensure_credential_secret_key() -> str: + """Resolve Fernet key: env ``NETX_CREDENTIAL_SECRET_KEY`` > file > generate once. + + Packaged installs should also write the key into ``.env`` via setup_first_run; + this startup/path fallback covers upgrades and missed first-run keys. + """ + global _cached_credential_key + configured = str(settings.credential_secret_key or "").strip() + if configured: + return configured + if _cached_credential_key: + return _cached_credential_key + + path = credential_secret_file_path() + try: + if path.is_file(): + existing = path.read_text(encoding="utf-8").strip() + if existing: + _cached_credential_key = existing + settings.credential_secret_key = existing + return existing + except Exception: + _log.exception("read credential secret file failed path=%s", path) + + generated = Fernet.generate_key().decode("ascii") + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(generated + "\n", encoding="utf-8") + try: + path.chmod(0o600) + except Exception: + pass + _log.info("wrote per-install credential Fernet key to %s", path) + except Exception: + _log.exception( + "write credential secret file failed path=%s; using in-memory key only", + path, + ) + _cached_credential_key = generated + settings.credential_secret_key = generated + return generated + + def _fernet() -> Fernet: - key = str(settings.credential_secret_key or "").strip() - if not key: - raise CredentialCryptoError("credential_secret_key_not_configured") + key = ensure_credential_secret_key() try: return Fernet(key.encode("ascii")) except Exception as exc: @@ -37,4 +92,7 @@ def decrypt_secret(value: str) -> str: def credentials_configured() -> bool: - return bool(str(settings.credential_secret_key or "").strip()) + try: + return bool(ensure_credential_secret_key()) + except Exception: + return False diff --git a/packaging/README.md b/packaging/README.md index 4f22d6f..ec2353b 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -7,14 +7,22 @@ This directory builds a Windows deliverable with: - Optional **bundled** portable PostgreSQL **or** **external** existing Postgres - API-hosted UI (`web/dist`) — no separate Vite process for end users - Program / data split so upgrades do not wipe the database -- Manual update script + auto-update **manifest contract** (fetch not implemented yet) +- Manual update script + check/apply updates (GitHub Releases or custom manifest) +- Optional system tray + start-at-logon ## What users get | Artifact | How | |----------|-----| -| `NetX-x.y.z-win64.zip` | `build_release.ps1` | -| `NetX-Setup-x.y.z.exe` | Compile `installer/netx.iss` with [Inno Setup](https://jrsoftware.org/isinfo.php) after staging | +| `NetX-Setup-x.y.z.exe` | Stage with `build_release.ps1`, then compile `installer/netx.iss` with [Inno Setup](https://jrsoftware.org/isinfo.php) | + +Zip packages are **not** published. Releases ship **Setup.exe only**. + +**Installer:** English + 简体中文 (language dialog). Icons use `packaging/assets/netx.ico`. + +**Offline:** Setup ships portable PostgreSQL, **`python/runtime` + `.venv`** (portable; not tied to the build PC’s user profile), and WinSW. **First install** shows the Database page (built-in or external PostgreSQL; external credentials validated with `psql SELECT 1`). **Re-running Setup when `%ProgramData%\NetX\.env` already has `NETX_DB_MODE`** skips the DB wizard and updates program files in place — **no GitHub/EDB download on the target PC**. Service install uses bundled WinSW (pass `-AllowDownload` only on a build/dev machine if the binary is missing). Auto-update still needs network later, and is unchecked by default. + +**OS:** Windows 10/11 or Windows Server **2016+** recommended. Packaging scripts are UTF-8 **with BOM** so Chinese UI works on Windows PowerShell 5.x. Bundled Python in current releases is **3.13+**, which does **not** support Windows Server 2012 R2 — use Server 2016+ or a newer desktop OS. ## Build (developer machine) @@ -25,14 +33,13 @@ cd netx # Optional: download portable Postgres into packaging\postgres\pgsql powershell -ExecutionPolicy Bypass -File .\packaging\download_postgres.ps1 -# Build web + stage + zip (-CreateVenv ships a ready .venv; large) +# Build web + stage (-CreateVenv ships a ready .venv; large). No zip by default. powershell -ExecutionPolicy Bypass -File .\packaging\build_release.ps1 -CreateVenv ``` Output: -- `packaging/release/netx-win64/` — stage tree -- `packaging/release/NetX--win64.zip` +- `packaging/release/netx-win64/` — stage tree (input to Inno Setup) Inno Setup: @@ -40,29 +47,27 @@ Inno Setup: ISCC.exe packaging\installer\netx.iss ``` -Override version in the `.iss` or edit `#define MyAppVersion`. +Override version in the `.iss` or edit `#define MyAppVersion`. +Optional local zip only: `build_release.ps1 -CreateZip` (not for release upload). ## End-user install ### Setup.exe 1. Run `NetX-Setup-x.y.z.exe` (admin). -2. Files → `%ProgramFiles%\NetX\` (program root). -3. Data → `%ProgramData%\NetX\` (`.env`, `pgdata`, spool, secrets). -4. Optional post-install task runs `setup_first_run.ps1` (choose bundled vs external DB). -5. Start menu: **Start NetX** / **Stop NetX** / **Open NetX UI**. +2. **First install:** Database page — choose built-in (offline) or external PostgreSQL (host/port/user/password/db; connection must succeed to continue). +3. **Already installed:** if `%ProgramData%\NetX\.env` already has `NETX_DB_MODE`, Setup shows an **Install mode** page: + - **Update** (default) — skip Database page; stop NetX; replace program files; **keep** ProgramData / DB settings. + - **Reinstall** — same Database wizard as first install (`ApplyDatabaseConfig`); does **not** delete ProgramData (use Uninstall → delete data for a wipe). +4. Files → `%ProgramFiles%\NetX\` (program root). +5. Data → `%ProgramData%\NetX\` (`.env`, `pgdata`, spool, secrets). +6. Start menu: **Start NetX** / **Stop NetX** / **Open NetX UI** / **Reconfigure database**. -### Zip (portable) - -1. Unpack anywhere. -2. Marker `.portable` → data root is sibling `NetXData\`. -3. Target needs **Python 3.11+** on PATH unless the zip was built with `-CreateVenv`. -4. Run: - -```powershell -.\packaging\setup_first_run.ps1 -.\packaging\start_netx_app.ps1 -``` +Silent first install (bundled default): `/SILENT /DbMode=bundled` +Silent external: `/SILENT /DbMode=external /DbHost=... /DbPort=5432 /DbUser=... /DbPassword=... /DbName=...` +Silent update over existing data: `/VERYSILENT /NORESTART /InstallMode=update` (also `/SkipDbPage=1`) +Silent reinstall (DB wizard via params): `/VERYSILENT /InstallMode=reinstall /DbMode=bundled` (or external `/DbHost`…) — also `/ForceDbPage=1` +Optional credential key (reuse encrypted NE passwords from another install): `/CredentialSecretKey=...` — omit to auto-generate. ## Database modes @@ -76,15 +81,122 @@ Existing Windows deploys that only set `NETX_DATABASE_URL` keep working: never s ## Manual update +Preferred: run a newer `NetX-Setup-*.exe` and choose **Update** (or silent `/InstallMode=update`) so ProgramData is kept. Setup (elevated) always relinks `.venv` → `python/runtime` after file copy so tray start works without write access under Program Files. + +Legacy/dev zip (only if you built with `-CreateZip`): + ```powershell -.\packaging\update_netx.ps1 -PackagePath .\NetX-0.3.0-win64.zip +.\packaging\update_netx.ps1 -PackagePath .\NetX-0.4.0-win64.zip ``` Stops services, replaces program folders (`netx_api`, `web`, `packaging`, `postgres`, …), **keeps** the data root, restarts. Schema migrations still run via Alembic on API start. -## Auto-update (reserved) +## Check / apply updates -See `manifest.example.json`. Future: set `NETX_UPDATE_URL` + `NETX_UPDATE_CHANNEL`; a checker will download the zip and call `update_netx.ps1`. Not implemented in this phase. +**Defaults (no `.env` needed):** probe **GitHub** (`hansjone/netx`) and Forgejo mirror (`https://git.avelo.top/hansjone/netx`), then use the **newest reachable** version. Same version → prefer GitHub. + +```env +# Optional overrides in ProgramData\NetX\.env +# NETX_UPDATE_SOURCES=github,forgejo +# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest +# NETX_UPDATE_GITHUB_REPO=hansjone/netx +# NETX_UPDATE_URL=https://cdn.example.com/netx/manifest.json +# NETX_UPDATE_TOKEN=****** +``` + +| Variable | Role | +|----------|------| +| `NETX_UPDATE_FORGEJO_URL` | Forgejo/Gitea `releases/latest` API (default: git.avelo.top mirror) | +| `NETX_UPDATE_GITHUB_REPO` | GitHub `owner/repo` (default `hansjone/netx`) | +| `NETX_UPDATE_SOURCES` | Probe order / tie-break order, e.g. `github,forgejo` | +| `NETX_UPDATE_URL` | Optional custom JSON manifest | + +```powershell +.\packaging\check_update.ps1 +.\packaging\check_update.ps1 -Apply +``` + +Both sides should publish the same **Setup.exe**. `check_update.ps1` prefers `NetX-Setup-*.exe` (legacy `win64.zip` still works if present). **Code mirror alone is not enough** — Forgejo pull-mirror syncs git/tags only; Release assets must be uploaded separately (or clients can only fall back to GitHub). + +### Maintainer release checklist (Windows) + +Do this on the **dev PC on the home LAN** after bumping version: + +1. **Build** — `.\packaging\build_release.ps1 -CreateVenv` + Inno Setup → `NetX-Setup-*.exe` +2. **GitHub** — `.\packaging\publish_release.ps1 -Version x.y.z` (uploads Setup.exe only) +3. **Forgejo** — upload the same Setup.exe (default: public domain `https://git.avelo.top`): + +```powershell +# One-time: User env var (never commit the token) +# Forgejo → Settings → Applications → Generate New Token (repo write) +[Environment]::SetEnvironmentVariable("NETX_FORGEJO_TOKEN", "your_token", "User") +# Restart Cursor / open a new terminal so the agent/scripts see it + +# Default: https://git.avelo.top +.\packaging\publish_forgejo_release.ps1 -Version 0.4.11 + +# Optional when LAN IP is reachable: +# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 -ForgejoBase "http://10.0.0.131:3000" +``` + +| Role | URL | +|------|-----| +| Publish default | `https://git.avelo.top` (`-ForgejoBase` default) | +| Optional LAN | `http://10.0.0.131:3000` | +| Update probe (default) | GitHub + `https://git.avelo.top/.../releases/latest` | + +Verify: + +- https://git.avelo.top/hansjone/netx/releases +- Probe: `.\packaging\check_update.ps1` → both `github` and `forgejo` reachable with the new version + +Token: `NETX_FORGEJO_TOKEN` (or `FORGEJO_TOKEN`). + +## Tray & autostart + +```powershell +# System tray: Start / Stop / Open UI / Check updates +.\packaging\netx_tray.ps1 -StartOnLaunch + +# Start tray at Windows logon (current user) +.\packaging\install_autostart.ps1 +# Remove: .\packaging\install_autostart.ps1 -Remove +``` + +## Windows Service (admin) + +Uses [WinSW](https://github.com/winsw/winsw) (downloaded on first install into `packaging/cache`). +`service_run.ps1` probes `/health` and restarts children after consecutive failures. + +```powershell +# Elevated PowerShell +.\packaging\install_service.ps1 -Start +# Uninstall: .\packaging\install_service.ps1 -Uninstall + +# Fallback without WinSW binary management: SYSTEM scheduled task at startup +.\packaging\install_service.ps1 -Mode task -Start +``` + +## Silent auto-update + +```powershell +# Writes NETX_UPDATE_AUTO=true and registers a daily task (default 03:30) +.\packaging\install_update_task.ps1 + +# Manual silent path (only applies when NETX_UPDATE_AUTO=true) +.\packaging\check_update.ps1 -Apply -Quiet -AutoOnly +``` + +Tray also auto-applies on launch when `NETX_UPDATE_AUTO=true`. + +## Code signing (optional, publisher machine) + +```powershell +.\packaging\sign_release.ps1 -Files .\packaging\release\NetX-Setup-0.4.0.exe -Thumbprint +# or: -PfxPath .\certs\code.pfx -PfxPassword *** +``` + +Needs `signtool.exe` (Windows SDK) and a real code-signing certificate. Without a trusted cert, SmartScreen may still warn. ## Layout reminder diff --git a/packaging/_common.ps1 b/packaging/_common.ps1 index 4a1ef66..0a22f55 100644 --- a/packaging/_common.ps1 +++ b/packaging/_common.ps1 @@ -1,4 +1,4 @@ -# Shared path helpers for Windows packaging scripts. +# Shared path helpers for Windows packaging scripts. # Dot-source: . "$PSScriptRoot\_common.ps1" $ErrorActionPreference = "Stop" @@ -114,7 +114,33 @@ function Write-DotEnvValue { if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } - Set-Content -Path $Path -Value ($lines -join "`r`n") -Encoding utf8 + $text = ($lines -join "`r`n") + try { + Set-Content -LiteralPath $Path -Value $text -Encoding utf8 + } catch { + # Admin-created .env is often Users:RX only — repair ACL then retry once. + $root = $dir + if ((Split-Path -Leaf $dir) -eq "NetX" -or $dir -match '\\NetX$') { + $root = $dir + } else { + $parent = Split-Path -Parent $dir + if ($parent -and ((Split-Path -Leaf $parent) -eq "NetX")) { $root = $parent } + } + $null = Ensure-NetxDataAcl -DataRoot $root -Quiet + try { + # Reset .env ACL explicitly if still blocked. + $icacls = Join-Path $env:SystemRoot "System32\icacls.exe" + if (Test-Path -LiteralPath $icacls) { + & $icacls $Path /grant "*S-1-5-32-545:M" /C /Q 2>$null | Out-Null + } + Set-Content -LiteralPath $Path -Value $text -Encoding utf8 + } catch { + throw (New-Object System.UnauthorizedAccessException( + ("access_denied: cannot write {0}. Run Start Menu → NetX → Reconfigure database as Administrator, or: icacls `"{1}`" /grant Users:(OI)(CI)M /T" -f $Path, $root), + $_.Exception + )) + } + } } function Get-DbMode { @@ -135,3 +161,382 @@ function Import-NetxEnvFile { } return $map } + +function ConvertTo-NetxFsPath([string]$Path) { + # Forward slashes work in .env and most Windows APIs via Python pathlib. + return ($Path -replace '\\', '/') +} + +function Get-NetxDataEnvMap { + param([string]$DataRoot) + $d = $DataRoot + return @{ + "NETX_AUTH_MCP_TOKEN_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\mcp_token")) + "NETX_AUTH_SECRET_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\jwt_secret")) + "NETX_CREDENTIAL_SECRET_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\credential_secret")) + "NETX_SCHEDULER_HEARTBEAT_PATH" = (ConvertTo-NetxFsPath (Join-Path $d "data\runtime\scheduler_heartbeat.json")) + "NETX_RUN_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\runtime")) + "NETX_BIZ_STATE_SPOOL_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\biz_state_spool")) + "NETX_NE_COLLECTION_DATA_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\ne_collections")) + "NETX_NE_EXEC_JOB_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\ne_exec_jobs")) + "NETX_WEBCRT_DATA_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\webcrt")) + } +} + +function New-NetxFernetKey { + # cryptography.fernet.Fernet.generate_key() == urlsafe_b64encode(os.urandom(32)) + $bytes = New-Object byte[] 32 + $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() + try { + $rng.GetBytes($bytes) + } finally { + $rng.Dispose() + } + return [Convert]::ToBase64String($bytes).Replace('+', '-').Replace('/', '_') +} + +function Ensure-NetxDataAcl { + param( + [Parameter(Mandatory = $true)][string]$DataRoot, + [switch]$Quiet = $false + ) + # Installer creates %ProgramData%\NetX as Administrators. Tray / normal users must + # be able to update .env and runtime files when reconfiguring DB. + if (-not (Test-Path -LiteralPath $DataRoot)) { return $false } + try { + $acl = Get-Acl -LiteralPath $DataRoot + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( + "BUILTIN\Users", + "Modify", + "ContainerInherit,ObjectInherit", + "None", + "Allow" + ) + $acl.SetAccessRule($rule) + Set-Acl -LiteralPath $DataRoot -AclObject $acl + + # Also fix already-created files that only inherited RX for Users. + $icacls = Join-Path $env:SystemRoot "System32\icacls.exe" + if (Test-Path -LiteralPath $icacls) { + & $icacls $DataRoot /grant "*S-1-5-32-545:(OI)(CI)M" /T /C /Q 2>$null | Out-Null + } + if (-not $Quiet) { + Write-Host "==> Data ACL: BUILTIN\Users Modify on $DataRoot" -ForegroundColor DarkGray + } + return $true + } catch { + if (-not $Quiet) { + Write-Host "[WARN] Ensure-NetxDataAcl: $($_.Exception.Message)" -ForegroundColor Yellow + } + return $false + } +} + +function Ensure-NetxDataDirectories { + param([string]$DataRoot) + if (-not (Test-Path -LiteralPath $DataRoot)) { + New-Item -ItemType Directory -Path $DataRoot -Force | Out-Null + } + $subs = @( + "data", "data\auth", "data\runtime", "data\biz_state_spool", + "data\ne_collections", "data\ne_exec_jobs", "data\webcrt", + "backups", "pgdata" + ) + foreach ($sub in $subs) { + $p = Join-Path $DataRoot $sub + if (-not (Test-Path $p)) { + New-Item -ItemType Directory -Path $p -Force | Out-Null + } + } + # Best-effort; succeeds when running elevated (installer / first-run as admin). + $null = Ensure-NetxDataAcl -DataRoot $DataRoot -Quiet +} + +function Test-NetxTcpPortFree { + param([string]$HostName = "127.0.0.1", [int]$Port) + try { + $client = New-Object System.Net.Sockets.TcpClient + $iar = $client.BeginConnect($HostName, $Port, $null, $null) + $ok = $iar.AsyncWaitHandle.WaitOne(400) + if ($ok -and $client.Connected) { + $client.Close() + return $false + } + $client.Close() + return $true + } catch { + return $true + } +} + +function Test-NetxApiHealthy { + param([string]$HostName = "127.0.0.1", [int]$Port = 8890, [int]$TimeoutSec = 2) + try { + $url = "http://${HostName}:${Port}/health" + $r = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec $TimeoutSec -MaximumRedirection 0 + if ($r.StatusCode -ne 200) { return $false } + $body = $r.Content | ConvertFrom-Json -ErrorAction Stop + return ($body.status -eq "ok") + } catch { + return $false + } +} + +function Wait-NetxApiHealthy { + param( + [string]$HostName = "127.0.0.1", + [int]$Port = 8890, + [int]$TimeoutSec = 180, + [int]$PollMs = 500 + ) + $deadline = (Get-Date).AddSeconds($TimeoutSec) + while ((Get-Date) -lt $deadline) { + if (Test-NetxApiHealthy -HostName $HostName -Port $Port -TimeoutSec 2) { + return $true + } + Start-Sleep -Milliseconds $PollMs + } + return $false +} + +function ConvertTo-NetxProcessArgument { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) + # Start-Process joins string[] args with spaces and does NOT quote values that + # contain spaces (e.g. C:\Program Files\NetX). Always emit one argv token. + if ($Value -match '[\s"]') { + return '"' + ($Value -replace '"', '\"') + '"' + } + return $Value +} + +function ConvertTo-NetxProcessArgumentString { + param([Parameter(Mandatory = $true)][string[]]$Arguments) + return (($Arguments | ForEach-Object { ConvertTo-NetxProcessArgument -Value "$_" }) -join " ") +} + +function Start-NetxPowerShell { + param( + [Parameter(Mandatory = $true)][string]$File, + [string[]]$Arguments = @(), + [string]$WorkingDirectory = "", + [ValidateSet("Hidden", "Normal", "Minimized")] + [string]$WindowStyle = "Hidden", + [switch]$Wait = $false, + [switch]$PassThru = $false + ) + $parts = @( + "-NoProfile", + "-WindowStyle", $WindowStyle, + "-ExecutionPolicy", "Bypass", + "-File", $File + ) + $Arguments + $sp = @{ + FilePath = "powershell.exe" + ArgumentList = (ConvertTo-NetxProcessArgumentString -Arguments $parts) + WindowStyle = $WindowStyle + } + if ($WorkingDirectory) { $sp.WorkingDirectory = $WorkingDirectory } + if ($Wait) { $sp.Wait = $true } + if ($PassThru -or $Wait) { $sp.PassThru = $true } + return Start-Process @sp +} + +function Get-NetxSystemPython { + # Prefer a real interpreter; skip the WindowsApps Store stub. + $candidates = @() + foreach ($cmd in @("python", "python3")) { + $c = Get-Command $cmd -ErrorAction SilentlyContinue + if ($c -and $c.Source -and ($c.Source -notmatch '\\WindowsApps\\')) { + $candidates += $c.Source + } + } + $pyLauncher = Get-Command "py" -ErrorAction SilentlyContinue + if ($pyLauncher -and $pyLauncher.Source -and ($pyLauncher.Source -notmatch '\\WindowsApps\\')) { + try { + $resolved = (& $pyLauncher.Source -3 -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() + if ($resolved -and (Test-Path -LiteralPath $resolved)) { + $candidates += $resolved + } + } catch {} + } + foreach ($p in @( + "$env:LOCALAPPDATA\Python\pythoncore-3.14-64\python.exe", + "$env:LOCALAPPDATA\Programs\Python\Python312\python.exe", + "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", + "$env:ProgramFiles\Python312\python.exe", + "$env:ProgramFiles\Python311\python.exe" + )) { + if ($p -and (Test-Path $p)) { $candidates += $p } + } + foreach ($p in ($candidates | Select-Object -Unique)) { + try { + $v = & $p -c "import sys; print('%d.%d'%sys.version_info[:2])" 2>$null + if ($v -match '^(3\.(1[1-9]|[2-9]\d))$') { return $p } + } catch {} + } + return $null +} + +function Get-NetxBundledPythonRoot { + param([Parameter(Mandatory = $true)][string]$ProgramRoot) + return Join-Path $ProgramRoot "python\runtime" +} + +function Repair-NetxShippedVenv { + param([Parameter(Mandatory = $true)][string]$ProgramRoot) + $cfgPath = Join-Path $ProgramRoot ".venv\pyvenv.cfg" + $rtRoot = Get-NetxBundledPythonRoot -ProgramRoot $ProgramRoot + $rtPy = Join-Path $rtRoot "python.exe" + if (-not (Test-Path -LiteralPath $rtPy)) { return $false } + if (-not (Test-Path -LiteralPath $cfgPath)) { return $false } + + $rtRootAbs = (Resolve-Path -LiteralPath $rtRoot).Path + $rtPyAbs = (Resolve-Path -LiteralPath $rtPy).Path + $ver = "3.14.0" + try { + $verOut = & $rtPyAbs -c "import sys; print('%d.%d.%d' % sys.version_info[:3])" 2>$null + if ($verOut) { $ver = ($verOut | Out-String).Trim() } + } catch {} + + $lines = @( + "home = $rtRootAbs", + "include-system-site-packages = false", + "version = $ver", + "executable = $rtPyAbs" + ) + $desired = ($lines -join "`r`n") + # Skip write when already correct — Users cannot modify Program Files after Setup. + try { + $cur = ([IO.File]::ReadAllText($cfgPath) -replace "`r`n", "`n" -replace "`r", "`n").Trim() + $want = ($desired -replace "`r`n", "`n" -replace "`r", "`n").Trim() + if ($cur -eq $want) { + return $true + } + } catch {} + + try { + Set-Content -LiteralPath $cfgPath -Value $desired -Encoding ascii -ErrorAction Stop + return $true + } catch { + # Non-elevated tray/start: leave cfg as-is; caller may still run if paths happen to work. + return $false + } +} + +function Test-NetxVenvRunnable { + param([Parameter(Mandatory = $true)][string]$VenvPython) + if (-not (Test-Path -LiteralPath $VenvPython)) { return $false } + + $outFile = Join-Path $env:TEMP ("netx-venv-out-" + [Guid]::NewGuid().ToString("n") + ".txt") + $errFile = Join-Path $env:TEMP ("netx-venv-err-" + [Guid]::NewGuid().ToString("n") + ".txt") + try { + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $VenvPython + $psi.Arguments = '-c "import encodings, sys; sys.exit(0 if sys.prefix else 1)"' + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + $p = [Diagnostics.Process]::Start($psi) + $stderr = $p.StandardError.ReadToEnd() + $null = $p.StandardOutput.ReadToEnd() + $p.WaitForExit() + if ($stderr -match 'did not find executable|No Python at|Fatal Python error') { + return $false + } + if ($p.ExitCode -ne 0) { return $false } + return $true + } finally { + Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue + } +} + +function Ensure-NetxVenv { + param( + [Parameter(Mandatory = $true)][string]$ProgramRoot, + [switch]$ForcePip = $false + ) + $venvPy = Join-Path $ProgramRoot ".venv\Scripts\python.exe" + $req = Join-Path $ProgramRoot "requirements.txt" + $rtPy = Join-Path (Get-NetxBundledPythonRoot -ProgramRoot $ProgramRoot) "python.exe" + + if (Test-Path -LiteralPath $rtPy) { + try { + $null = Repair-NetxShippedVenv -ProgramRoot $ProgramRoot + } catch { + # Access denied under Program Files when not elevated — ignore if venv already runs. + } + } + + if ((Test-Path -LiteralPath $venvPy) -and -not $ForcePip) { + if (Test-NetxVenvRunnable -VenvPython $venvPy) { + return $venvPy + } + Write-Host "[WARN] Shipped .venv exists but Python cannot start (broken pyvenv.cfg or missing runtime)." -ForegroundColor Yellow + if (Test-Path -LiteralPath $rtPy) { + $repaired = $false + try { + $repaired = [bool](Repair-NetxShippedVenv -ProgramRoot $ProgramRoot) + } catch { + $repaired = $false + } + if ($repaired -and (Test-NetxVenvRunnable -VenvPython $venvPy)) { + Write-Host "==> Repaired .venv to use bundled python/runtime" -ForegroundColor Green + return $venvPy + } + if (-not $repaired) { + throw "venv_needs_admin_repair: pyvenv.cfg still points at the build PC. Re-run Setup (Update) as Administrator, or Start Menu → Reconfigure database once elevated." + } + } + if (-not $ForcePip) { + throw "venv_not_runnable: reinstall NetX Setup (ships python/runtime + .venv) or run repair as admin" + } + } + if (-not (Test-Path $venvPy)) { + $py = Get-NetxSystemPython + if (-not $py) { + throw "python_not_found: install Python 3.11+ (or ship Setup built with -CreateVenv)" + } + Write-Host "==> Creating .venv with $py" + $venvDir = Join-Path $ProgramRoot ".venv" + try { + & $py -m venv $venvDir + } catch { + throw "venv_create_failed: cannot write $venvDir (need admin / ship -CreateVenv). $($_.Exception.Message)" + } + if (-not (Test-Path $venvPy)) { + throw "venv_create_failed: missing $venvPy (Program Files may be read-only without elevation)" + } + $ForcePip = $true + } + if ($ForcePip) { + if (-not (Test-Path $req)) { throw "requirements_missing: $req" } + Write-Host "==> pip install -r requirements.txt" + & $venvPy -m pip install --upgrade pip + & $venvPy -m pip install -r $req + if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } + } + return $venvPy +} + +function Stop-NetxTrayProcesses { + param([string]$ProgramRoot = "") + $hits = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { + $_.CommandLine -and $_.CommandLine -match 'netx_tray\.ps1' + }) + if ($ProgramRoot) { + $esc = [regex]::Escape($ProgramRoot) + $hits = @($hits | Where-Object { $_.CommandLine -match $esc -or $_.CommandLine -match 'netx_tray\.ps1' }) + } + foreach ($p in $hits) { + try { + Stop-Process -Id $p.ProcessId -Force -ErrorAction Stop + Write-Host "Stopped NetX tray PID=$($p.ProcessId)" + } catch {} + } +} + +function ConvertTo-NetxSqlLiteral([string]$Value) { + # Single-quote escaping for PostgreSQL string literals. + return ($Value -replace "'", "''") +} diff --git a/packaging/assets/README.md b/packaging/assets/README.md new file mode 100644 index 0000000..2310893 --- /dev/null +++ b/packaging/assets/README.md @@ -0,0 +1,6 @@ +# NetX packaging assets + +- `netx.ico` — installer / shortcuts / tray (regenerate with `python gen_icon.py`) +- `netx-256.png` / `netx-64.png` — previews + +Mark matches `web/public/favicon.svg` (network “N” nodes on navy). diff --git a/packaging/assets/gen_icon.py b/packaging/assets/gen_icon.py new file mode 100644 index 0000000..f6e556c --- /dev/null +++ b/packaging/assets/gen_icon.py @@ -0,0 +1,56 @@ +"""Generate NetX Windows .ico from brand mark (matches web/public/favicon.svg).""" +from __future__ import annotations + +import os +from PIL import Image, ImageDraw + +OUT_DIR = os.path.dirname(os.path.abspath(__file__)) +BG = (15, 39, 68, 255) +LINE = (126, 182, 232, 255) +NODE = (232, 242, 252, 255) +ACCENT = (61, 130, 247, 255) + + +def make(size: int) -> Image.Image: + img = Image.new("RGBA", (size, size), (0, 0, 0, 0)) + d = ImageDraw.Draw(img) + radius = max(2, int(size * 0.22)) + d.rounded_rectangle([0, 0, size - 1, size - 1], radius=radius, fill=BG) + s = size / 32.0 + + def xy(x: float, y: float) -> tuple[float, float]: + return (x * s, y * s) + + stroke = max(2, int(round(2.2 * s))) + for a, b in ( + (xy(9.5, 8.5), xy(9.5, 23.5)), + (xy(9.5, 8.5), xy(22.5, 23.5)), + (xy(22.5, 8.5), xy(22.5, 23.5)), + ): + d.line([a, b], fill=LINE, width=stroke) + + def circle(cx: float, cy: float, rad: float, fill: tuple[int, int, int, int]) -> None: + x, y = xy(cx, cy) + rr = rad * s + d.ellipse([x - rr, y - rr, x + rr, y + rr], fill=fill) + + circle(9.5, 8.5, 2.35, NODE) + circle(9.5, 23.5, 2.35, NODE) + circle(22.5, 8.5, 2.35, NODE) + circle(22.5, 23.5, 2.35, NODE) + circle(16, 16, 1.7, ACCENT) + return img + + +def main() -> None: + sizes = [16, 24, 32, 48, 64, 128, 256] + images = [make(s) for s in sizes] + ico_path = os.path.join(OUT_DIR, "netx.ico") + images[-1].save(ico_path, format="ICO", sizes=[(s, s) for s in sizes]) + images[-1].save(os.path.join(OUT_DIR, "netx-256.png")) + make(64).save(os.path.join(OUT_DIR, "netx-64.png")) + print(f"wrote {ico_path} ({os.path.getsize(ico_path)} bytes)") + + +if __name__ == "__main__": + main() diff --git a/packaging/assets/netx-256.png b/packaging/assets/netx-256.png new file mode 100644 index 0000000..167327b Binary files /dev/null and b/packaging/assets/netx-256.png differ diff --git a/packaging/assets/netx-64.png b/packaging/assets/netx-64.png new file mode 100644 index 0000000..afeb76f Binary files /dev/null and b/packaging/assets/netx-64.png differ diff --git a/packaging/assets/netx.ico b/packaging/assets/netx.ico new file mode 100644 index 0000000..179e51b Binary files /dev/null and b/packaging/assets/netx.ico differ diff --git a/packaging/build_release.ps1 b/packaging/build_release.ps1 index a347f03..3f35475 100644 --- a/packaging/build_release.ps1 +++ b/packaging/build_release.ps1 @@ -1,10 +1,14 @@ -param( +param( [string]$Version = "", [string]$OutDir = "", [switch]$SkipWebBuild = $false, [switch]$SkipPostgresDownload = $false, + # Zip is no longer published; stage + Setup.exe only. Opt-in for local/dev testing. + [switch]$CreateZip = $false, + # Deprecated no-op (zip is off by default). Kept so older scripts that pass -SkipZip still run. [switch]$SkipZip = $false, - [switch]$CreateVenv = $false + # Default on: offline Setup must ship .venv so target PCs never pip-install. + [switch]$CreateVenv = $true ) $ErrorActionPreference = "Stop" @@ -51,6 +55,20 @@ if (-not $SkipPostgresDownload) { } } +# Bundle WinSW so offline installs can register a Windows Service without GitHub. +$winswCache = Join-Path $PSScriptRoot "cache\WinSW-x64.exe" +$winswShip = Join-Path $PSScriptRoot "winsw\WinSW-x64.exe" +if (-not (Test-Path $winswShip)) { + if (-not (Test-Path $winswCache)) { + $winswUrl = "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW-x64.exe" + Write-Host "==> Downloading WinSW for offline bundle: $winswUrl" + New-Item -ItemType Directory -Path (Split-Path $winswCache -Parent) -Force | Out-Null + Invoke-WebRequest -Uri $winswUrl -OutFile $winswCache -UseBasicParsing + } + New-Item -ItemType Directory -Path (Split-Path $winswShip -Parent) -Force | Out-Null + Copy-Item -Path $winswCache -Destination $winswShip -Force +} + # Flat layout = same as repo so scripts\start_netx.ps1 works unchanged. foreach ($d in @("netx_api", "alembic", "scripts")) { Copy-Item -Path (Join-Path $repo $d) -Destination (Join-Path $stage $d) -Recurse -Force @@ -66,26 +84,70 @@ $webOut = Join-Path $stage "web\dist" New-Item -ItemType Directory -Path (Split-Path $webOut -Parent) -Force | Out-Null Copy-Item -Path $dist -Destination $webOut -Recurse -Force +function Write-Utf8BomFile { + param([string]$Path) + # Windows PowerShell 4/5 (Server 2012+) mis-parses UTF-8 without BOM when scripts contain CJK. + $utf8Bom = New-Object System.Text.UTF8Encoding $true + $bytes = [IO.File]::ReadAllBytes($Path) + $hasBom = ($bytes.Length -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) + $text = if ($hasBom) { + [Text.Encoding]::UTF8.GetString($bytes, 3, $bytes.Length - 3) + } else { + [Text.Encoding]::UTF8.GetString($bytes) + } + $text = $text -replace "`r`n", "`n" -replace "`n", "`r`n" + if (-not $text.EndsWith("`r`n")) { $text += "`r`n" } + [IO.File]::WriteAllText($Path, $text, $utf8Bom) +} + $packOut = Join-Path $stage "packaging" New-Item -ItemType Directory -Path $packOut -Force | Out-Null Copy-Item -Path (Join-Path $PSScriptRoot "_common.ps1") -Destination $packOut -Force foreach ($name in @( "download_postgres.ps1", "setup_first_run.ps1", "start_netx_app.ps1", - "stop_netx_app.ps1", "update_netx.ps1", "build_release.ps1", - "README.md", "manifest.example.json" + "stop_netx_app.ps1", "update_netx.ps1", "check_update.ps1", + "launch_shortcut.ps1", "netx_tray.ps1", "install_autostart.ps1", "install_service.ps1", + "service_run.ps1", "install_update_task.ps1", "uninstall_prepare.ps1", "uninstall_delete_data.ps1", "sign_release.ps1", + "build_release.ps1", "publish_release.ps1", "publish_forgejo_release.ps1", "README.md", "manifest.example.json" )) { $src = Join-Path $PSScriptRoot $name if (Test-Path $src) { Copy-Item $src (Join-Path $packOut $name) -Force } } +Get-ChildItem -Path $packOut -Filter *.ps1 -File | ForEach-Object { Write-Utf8BomFile -Path $_.FullName } +Get-ChildItem -Path (Join-Path $stage "scripts") -Filter *.ps1 -File -ErrorAction SilentlyContinue | + ForEach-Object { Write-Utf8BomFile -Path $_.FullName } Copy-Item -Path (Join-Path $PSScriptRoot "config") -Destination (Join-Path $packOut "config") -Recurse -Force Copy-Item -Path (Join-Path $PSScriptRoot "installer") -Destination (Join-Path $packOut "installer") -Recurse -Force +$assetsSrc = Join-Path $PSScriptRoot "assets" +if (Test-Path $assetsSrc) { + Copy-Item -Path $assetsSrc -Destination (Join-Path $packOut "assets") -Recurse -Force +} +$winswOut = Join-Path $packOut "winsw" +New-Item -ItemType Directory -Path $winswOut -Force | Out-Null +if (Test-Path $winswShip) { + Copy-Item -Path $winswShip -Destination (Join-Path $winswOut "WinSW-x64.exe") -Force + Write-Host "==> Bundled WinSW for offline service install" +} else { + Write-Host "[WARN] WinSW missing - offline service install will fail" -ForegroundColor Yellow +} New-Item -ItemType Directory -Path (Join-Path $packOut "postgres") -Force | Out-Null Copy-Item -Path (Join-Path $PSScriptRoot "postgres\README.md") -Destination (Join-Path $packOut "postgres\README.md") -Force if (Test-Path (Join-Path $pgsql "bin\pg_ctl.exe")) { - Write-Host "==> Copying bundled PostgreSQL" - New-Item -ItemType Directory -Path (Join-Path $stage "postgres") -Force | Out-Null - Copy-Item -Path $pgsql -Destination (Join-Path $stage "postgres\pgsql") -Recurse -Force + Write-Host '==> Copying bundled PostgreSQL (bin/lib/share; skip doc/)' + $pgStage = Join-Path $stage "postgres\pgsql" + New-Item -ItemType Directory -Path $pgStage -Force | Out-Null + foreach ($sub in @("bin", "lib", "share")) { + $srcSub = Join-Path $pgsql $sub + if (Test-Path $srcSub) { + Copy-Item -Path $srcSub -Destination (Join-Path $pgStage $sub) -Recurse -Force + } + } + Get-ChildItem -Path $pgsql -File -ErrorAction SilentlyContinue | ForEach-Object { + Copy-Item -Path $_.FullName -Destination (Join-Path $pgStage $_.Name) -Force + } +} else { + throw "bundled_postgres_missing: Setup.exe must ship postgres for offline install. Run download_postgres.ps1 (build machine only)." } $builtAt = (Get-Date).ToUniversalTime().ToString("o") @@ -99,26 +161,93 @@ $builtAt = (Get-Date).ToUniversalTime().ToString("o") Set-Content -Path (Join-Path $stage ".portable") -Value "1" -Encoding ascii +# Root .cmd launchers (Explorer / Start Menu friendly; ASCII-only). +$cmdSrc = Join-Path $PSScriptRoot "cmd" +foreach ($cmdName in @("NetX-FirstRun.cmd", "NetX-Start.cmd", "NetX-Tray.cmd", "NetX-Stop.cmd")) { + $c = Join-Path $cmdSrc $cmdName + if (Test-Path $c) { + Copy-Item -Path $c -Destination (Join-Path $stage $cmdName) -Force + } +} + if ($CreateVenv) { - Write-Host "==> Creating .venv in stage (requires Python 3.11+ on PATH)" - $py = (Get-Command python -ErrorAction SilentlyContinue) - if (-not $py) { throw "python_not_found_for_venv" } - & $py.Source -m venv (Join-Path $stage ".venv") - & (Join-Path $stage ".venv\Scripts\python.exe") -m pip install --upgrade pip - & (Join-Path $stage ".venv\Scripts\python.exe") -m pip install -r (Join-Path $stage "requirements.txt") + Write-Host "==> Creating portable python/runtime + .venv (must not reference build-machine paths)" + $pyPath = Get-NetxSystemPython + if (-not $pyPath) { throw "python_not_found_for_venv: need Python 3.11+ (not WindowsApps stub)" } + Write-Host " Build Python: $pyPath" + + $prefix = (& $pyPath -c "import sys; print(sys.base_prefix)" 2>$null | Out-String).Trim() + if (-not $prefix -or -not (Test-Path -LiteralPath $prefix)) { + throw "python_base_prefix_not_found: $prefix" + } + $rtDir = Join-Path $stage "python\runtime" + if (Test-Path $rtDir) { Remove-Item -Recurse -Force $rtDir } + New-Item -ItemType Directory -Path $rtDir -Force | Out-Null + + Write-Host "==> Copying Python stdlib/runtime to $rtDir (exclude base site-packages)" + $spEx = Join-Path $prefix "Lib\site-packages" + $robocopy = Join-Path $env:SystemRoot "System32\robocopy.exe" + if (-not (Test-Path -LiteralPath $robocopy)) { throw "robocopy_not_found" } + & $robocopy $prefix $rtDir /E /XD $spEx __pycache__ /XF *.pyc /NFL /NDL /NJH /NJS /nc /ns /np | Out-Null + if ($LASTEXITCODE -ge 8) { throw "robocopy_python_runtime_failed: exit $LASTEXITCODE" } + + $rtPy = Join-Path $rtDir "python.exe" + if (-not (Test-Path -LiteralPath $rtPy)) { throw "python_runtime_missing: $rtPy" } + + $venvDir = Join-Path $stage ".venv" + if (Test-Path $venvDir) { Remove-Item -Recurse -Force $venvDir } + Write-Host '==> Creating .venv from shipped runtime (--copies)' + & $rtPy -m venv $venvDir --copies + if ($LASTEXITCODE -ne 0) { throw "venv_create_failed" } + + $venvPy = Join-Path $venvDir "Scripts\python.exe" + $null = Repair-NetxShippedVenv -ProgramRoot $stage + if (-not (Test-NetxVenvRunnable -VenvPython $venvPy)) { + throw "venv_not_runnable_after_build: check python/runtime copy" + } + + & $venvPy -m pip install --upgrade pip + & $venvPy -m pip install -r (Join-Path $stage "requirements.txt") + if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } + + $null = Repair-NetxShippedVenv -ProgramRoot $stage + if (-not (Test-NetxVenvRunnable -VenvPython $venvPy)) { + throw "venv_not_runnable_after_pip" + } + + Write-Host "==> Slimming .venv (drop tests / __pycache__ / *.pyc)" + $site = Join-Path $stage ".venv\Lib\site-packages" + if (Test-Path $site) { + Get-ChildItem -Path $site -Recurse -Directory -Force -ErrorAction SilentlyContinue | + Where-Object { + $_.Name -in @('__pycache__', 'tests', 'test', 'testing', 'Tests') -or + $_.FullName -match '\\pandas\\tests(\\|$)' -or + $_.FullName -match '\\numpy\\(_*tests|tests)(\\|$)' -or + $_.FullName -match '\\scipy\\(_*tests|tests)(\\|$)' + } | + Sort-Object { $_.FullName.Length } -Descending | + ForEach-Object { + Remove-Item -LiteralPath $_.FullName -Recurse -Force -ErrorAction SilentlyContinue + } + Get-ChildItem -Path $site -Recurse -Include *.pyc,*.pyo -Force -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue + } } Write-Host "==> Stage ready: $stage" -ForegroundColor Green -if (-not $SkipZip) { +if ($SkipZip -and $CreateZip) { + Write-Host "[WARN] -SkipZip ignored because -CreateZip was set" -ForegroundColor Yellow +} +if ($CreateZip) { $zip = Join-Path (Split-Path -Parent $stage) "NetX-$Version-win64.zip" if (Test-Path $zip) { Remove-Item -Force $zip } Compress-Archive -Path $stage -DestinationPath $zip -Force - Write-Host "==> Zip: $zip" -ForegroundColor Green + Write-Host "==> Zip (optional/dev): $zip" -ForegroundColor Yellow } Write-Host "" -Write-Host "Ship options:" -Write-Host " Zip: user unpacks, runs packaging\setup_first_run.ps1 then start_netx_app.ps1" +Write-Host "Ship:" Write-Host " Exe: compile packaging\installer\netx.iss with Inno Setup (needs stage above)" +Write-Host " (Zip packages are not published; use -CreateZip only for local testing.)" Write-Host "Python: install 3.11+ on target, or rebuild with -CreateVenv and ship .venv" diff --git a/packaging/check_update.ps1 b/packaging/check_update.ps1 new file mode 100644 index 0000000..0d45bf9 --- /dev/null +++ b/packaging/check_update.ps1 @@ -0,0 +1,436 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [string]$UpdateUrl = "", + [string]$FallbackUrl = "", + [string]$ForgejoUrl = "", + [string]$GithubRepo = "", + [string]$Channel = "", + [string]$Sources = "", + [switch]$Apply = $false, + [switch]$Quiet = $false, + # Only apply when NETX_UPDATE_AUTO=true (scheduled silent updates). + [switch]$AutoOnly = $false +) + +# Check for a newer NetX Windows package. +# +# Default (no config needed): +# GitHub primary + Forgejo mirror fallback (git.avelo.top). +# Probe every reachable source and pick the highest version; +# on equal versions prefer GitHub (listed first). +# +# Optional overrides: +# NETX_UPDATE_SOURCES=github,forgejo +# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest +# NETX_UPDATE_GITHUB_REPO=hansjone/netx +# NETX_UPDATE_URL=... manifest JSON +# NETX_UPDATE_TOKEN=... private API token + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$envPath = Join-Path $data ".env" +$map = @{} +if (Test-Path $envPath) { $map = Read-DotEnv -Path $envPath } + +function Get-Cfg([string]$Key, [string]$ParamVal = "") { + if ($ParamVal) { return $ParamVal } + if ($map.ContainsKey($Key) -and $map[$Key]) { return [string]$map[$Key] } + $envVal = [Environment]::GetEnvironmentVariable($Key) + if ($envVal) { return $envVal } + return "" +} + +$UpdateUrl = Get-Cfg "NETX_UPDATE_URL" $UpdateUrl +$FallbackUrl = Get-Cfg "NETX_UPDATE_FALLBACK_URL" $FallbackUrl +$ForgejoUrl = Get-Cfg "NETX_UPDATE_FORGEJO_URL" $ForgejoUrl +if (-not $ForgejoUrl) { + $ForgejoUrl = "https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest" +} +$GithubRepo = Get-Cfg "NETX_UPDATE_GITHUB_REPO" $GithubRepo +if (-not $GithubRepo) { $GithubRepo = "hansjone/netx" } +$Channel = Get-Cfg "NETX_UPDATE_CHANNEL" $Channel +if (-not $Channel) { $Channel = "stable" } +$Sources = Get-Cfg "NETX_UPDATE_SOURCES" $Sources +$UpdateToken = Get-Cfg "NETX_UPDATE_TOKEN" "" + +$current = Get-NetxVersion -ProgramRoot $prog + +function Compare-SemVer { + param([string]$A, [string]$B) + $pa = @($A.TrimStart('v', 'V').Split('.') | ForEach-Object { try { [int]$_ } catch { 0 } }) + $pb = @($B.TrimStart('v', 'V').Split('.') | ForEach-Object { try { [int]$_ } catch { 0 } }) + while ($pa.Count -lt 3) { $pa += 0 } + while ($pb.Count -lt 3) { $pb += 0 } + for ($i = 0; $i -lt 3; $i++) { + if ($pa[$i] -lt $pb[$i]) { return -1 } + if ($pa[$i] -gt $pb[$i]) { return 1 } + } + return 0 +} + +function Get-AuthHeaders { + param([string]$Style = "bearer") + $h = @{ "User-Agent" = "NetX-UpdateCheck" } + if (-not $UpdateToken) { return $h } + if ($Style -eq "token") { + $h["Authorization"] = "token $UpdateToken" + } else { + $h["Authorization"] = "Bearer $UpdateToken" + } + return $h +} + +function Resolve-AssetUrl { + param($Asset, $Rel, [string]$SourceName, [string]$ApiLatestUrl = "") + $url = [string]$Asset.browser_download_url + if ($url) { return $url } + # Forgejo/Gitea sometimes omit browser_download_url; synthesize release download URL. + if ($SourceName -eq "forgejo" -and $ApiLatestUrl -and $Rel.tag_name -and $Asset.name) { + if ($ApiLatestUrl -match '^(https?://[^/]+)/api/v1/repos/([^/]+)/([^/]+)/releases/latest') { + $base = $Matches[1] + $owner = $Matches[2] + $repo = $Matches[3] + $tag = [string]$Rel.tag_name + $name = [uri]::EscapeDataString([string]$Asset.name).Replace('%2F', '/') + # EscapeDataString encodes too much; use raw name (assets shouldn't need query encoding). + $name = [string]$Asset.name + return "$base/$owner/$repo/releases/download/$tag/$name" + } + } + return "" +} + +function Convert-ReleaseToManifest { + param($Rel, [string]$SourceName, [string]$ApiLatestUrl = "") + $tag = [string]$Rel.tag_name + $ver = $tag.TrimStart('v', 'V') + $assets = @($Rel.assets) + # Prefer Setup.exe (current ship format). Legacy win64.zip still accepted. + $setupAsset = $assets | Where-Object { $_.name -match 'Setup-.*\.exe$' } | Select-Object -First 1 + $zipAsset = $assets | Where-Object { $_.name -match 'win64\.zip$' } | Select-Object -First 1 + if (-not $setupAsset -and -not $zipAsset) { + throw "${SourceName}_release_missing_setup_or_zip" + } + + $setupUrl = "" + if ($setupAsset) { + $setupUrl = Resolve-AssetUrl -Asset $setupAsset -Rel $Rel -SourceName $SourceName -ApiLatestUrl $ApiLatestUrl + if (-not $setupUrl) { throw "${SourceName}_setup_url_missing" } + } + $zipUrl = "" + if ($zipAsset) { + $zipUrl = Resolve-AssetUrl -Asset $zipAsset -Rel $Rel -SourceName $SourceName -ApiLatestUrl $ApiLatestUrl + } + + if ($setupUrl) { + $primaryUrl = $setupUrl + $packageKind = "setup" + $primarySize = [int64]$(if ($setupAsset.size) { $setupAsset.size } else { 0 }) + } else { + if (-not $zipUrl) { throw "${SourceName}_zip_url_missing" } + $primaryUrl = $zipUrl + $packageKind = "zip" + $primarySize = [int64]$(if ($zipAsset.size) { $zipAsset.size } else { 0 }) + } + + return [pscustomobject]@{ + channel = "stable" + latest = $ver + min_compatible = $ver + notes_url = [string]$Rel.html_url + source = $SourceName + windows = [pscustomobject]@{ + url = $primaryUrl + sha256 = "" + size = $primarySize + setup_url = $setupUrl + package = $packageKind + } + } +} + +function Get-ManifestFromUrl { + param([string]$Url) + $headers = Get-AuthHeaders -Style "bearer" + $resp = Invoke-WebRequest -Uri $Url -Headers $headers -UseBasicParsing -TimeoutSec 30 + $m = $resp.Content | ConvertFrom-Json + if (-not $m.source) { + $m | Add-Member -NotePropertyName source -NotePropertyValue "manifest" -Force + } + return $m +} + +function Get-FromGitHubReleases { + $api = "https://api.github.com/repos/$GithubRepo/releases/latest" + $headers = Get-AuthHeaders -Style "bearer" + $headers["Accept"] = "application/vnd.github+json" + $rel = Invoke-RestMethod -Uri $api -Headers $headers -TimeoutSec 30 + return Convert-ReleaseToManifest -Rel $rel -SourceName "github" +} + +function Get-FromForgejoReleases { + param([string]$ApiUrl) + if (-not $ApiUrl) { throw "forgejo_url_empty" } + $headers = Get-AuthHeaders -Style "token" + $headers["Accept"] = "application/json" + $rel = Invoke-RestMethod -Uri $ApiUrl -Headers $headers -TimeoutSec 30 + return Convert-ReleaseToManifest -Rel $rel -SourceName "forgejo" -ApiLatestUrl $ApiUrl +} + +function Get-UpdateSourceList { + if ($Sources) { + return @($Sources.Split(',') | ForEach-Object { $_.Trim().ToLowerInvariant() } | Where-Object { $_ }) + } + # Default: GitHub primary, Forgejo mirror backup. Optional manifests prepended if configured. + $list = [System.Collections.Generic.List[string]]::new() + if ($UpdateUrl) { [void]$list.Add("manifest") } + [void]$list.Add("github") + [void]$list.Add("forgejo") + if ($FallbackUrl) { [void]$list.Add("manifest_fallback") } + return @($list) +} + +function Get-ManifestFromSource([string]$src) { + switch ($src) { + "manifest" { + if (-not $UpdateUrl) { throw "NETX_UPDATE_URL empty" } + Write-Host "==> Probing manifest: $UpdateUrl" + return Get-ManifestFromUrl -Url $UpdateUrl + } + "manifest_fallback" { + if (-not $FallbackUrl) { throw "NETX_UPDATE_FALLBACK_URL empty" } + Write-Host "==> Probing fallback manifest: $FallbackUrl" + $m = Get-ManifestFromUrl -Url $FallbackUrl + if ($m -and -not $m.source) { + $m | Add-Member -NotePropertyName source -NotePropertyValue "manifest_fallback" -Force + } + return $m + } + "forgejo" { + Write-Host "==> Probing Forgejo: $ForgejoUrl" + return Get-FromForgejoReleases -ApiUrl $ForgejoUrl + } + "github" { + Write-Host "==> Probing GitHub: $GithubRepo" + return Get-FromGitHubReleases + } + default { throw "unknown_source: $src" } + } +} + +Write-Host "==> Current version: $current" +$sourceList = Get-UpdateSourceList +Write-Host "==> Probe sources (pick newest reachable; tie → earlier in list): $($sourceList -join ', ')" + +$candidates = @() +$errors = @() +foreach ($src in $sourceList) { + try { + $m = Get-ManifestFromSource -src $src + if ($m.channel -and $Channel -and ($m.channel -ne $Channel)) { + Write-Host "[WARN] $src channel=$($m.channel) requested=$Channel" + } + if (-not $m.windows -or -not $m.windows.url) { + throw "missing_windows_download_url" + } + Write-Host " OK $($m.source) latest=$($m.latest)" -ForegroundColor DarkGreen + $candidates += $m + } catch { + $msg = $_.Exception.Message + $errors += "${src}: $msg" + Write-Host "[WARN] source '$src' unreachable/failed: $msg" -ForegroundColor Yellow + } +} + +if ($candidates.Count -eq 0) { + $joined = ($errors -join "; ") + if ($Quiet) { exit 2 } + throw "update_check_failed: all sources failed ($joined)" +} + +# Prefer highest semver; on tie keep earlier source in $sourceList (GitHub before Forgejo by default). +$manifest = $candidates[0] +foreach ($c in $candidates) { + $cmpCand = Compare-SemVer -A ([string]$manifest.latest) -B ([string]$c.latest) + if ($cmpCand -lt 0) { + $manifest = $c + } +} + +Write-Host "==> Selected source=$($manifest.source) latest=$($manifest.latest) (from $($candidates.Count) reachable)" -ForegroundColor Green + +# Prefer Setup.exe when a manifest still lists a legacy zip as windows.url but also has setup_url. +if ($manifest.windows -and $manifest.windows.setup_url) { + $setupCandidate = [string]$manifest.windows.setup_url + $urlNow = [string]$manifest.windows.url + $pkgNow = if ($manifest.windows.package) { [string]$manifest.windows.package } else { "" } + if ($setupCandidate -and ($pkgNow -eq "zip" -or $urlNow -match '\.zip(\?|$)' -or -not $urlNow)) { + $manifest.windows | Add-Member -NotePropertyName url -NotePropertyValue $setupCandidate -Force + $manifest.windows | Add-Member -NotePropertyName package -NotePropertyValue "setup" -Force + } +} + +$latest = [string]$manifest.latest +$cmp = Compare-SemVer -A $current -B $latest +$packageKind = "setup" +if ($manifest.windows.package) { + $packageKind = [string]$manifest.windows.package +} elseif ([string]$manifest.windows.url -match '\.zip(\?|$)') { + $packageKind = "zip" +} + +$result = [pscustomobject]@{ + current = $current + latest = $latest + update_available = ($cmp -lt 0) + source = $(if ($manifest.source) { [string]$manifest.source } else { "unknown" }) + download_url = [string]$manifest.windows.url + setup_url = $(if ($manifest.windows.setup_url) { [string]$manifest.windows.setup_url } else { "" }) + package = $packageKind + notes_url = $(if ($manifest.notes_url) { [string]$manifest.notes_url } else { "" }) + sha256 = $(if ($manifest.windows.sha256) { [string]$manifest.windows.sha256 } else { "" }) + reachable = @($candidates | ForEach-Object { "$($_.source)=$($_.latest)" }) +} + +if (-not $result.update_available) { + Write-Host "==> Up to date (latest=$latest, source=$($result.source))" -ForegroundColor Green + if (-not $Quiet) { + $result | ConvertTo-Json -Compress | Write-Output + } + exit 0 +} + +Write-Host "==> Update available: $current -> $latest (source=$($result.source))" -ForegroundColor Cyan +if ($result.notes_url) { Write-Host " Notes: $($result.notes_url)" } + +$autoEnabled = $false +$autoVal = Get-Cfg "NETX_UPDATE_AUTO" "" +if ($autoVal -match '^(1|true|yes|on)$') { $autoEnabled = $true } + +if (-not $Apply) { + Write-Host " Download ($($result.package)): $($result.download_url)" + Write-Host " To apply: .\packaging\check_update.ps1 -Apply" + if (-not $Quiet) { + $result | ConvertTo-Json -Compress | Write-Output + } + exit 10 +} + +if ($AutoOnly -and -not $autoEnabled) { + Write-Host "==> Update available but NETX_UPDATE_AUTO is not enabled; skip apply" + if (-not $Quiet) { + $result | ConvertTo-Json -Compress | Write-Output + } + exit 10 +} + +$lockFile = Join-Path $data "data\runtime\update.lock" +$lockDir = Split-Path -Parent $lockFile +if (-not (Test-Path $lockDir)) { + New-Item -ItemType Directory -Path $lockDir -Force | Out-Null +} +if (Test-Path $lockFile) { + $ageHrs = ((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalHours + if ($ageHrs -lt 2) { + Write-Host "==> Another update appears in progress ($lockFile); abort" + exit 3 + } + Remove-Item -Force $lockFile -ErrorAction SilentlyContinue +} +Set-Content -Path $lockFile -Value (Get-Date).ToString("o") -Encoding ascii + +try { + $dlDir = Join-Path $data "backups\downloads" + if (-not (Test-Path $dlDir)) { + New-Item -ItemType Directory -Path $dlDir -Force | Out-Null + } + + $isSetup = ($result.package -eq "setup") -or ($result.download_url -match '\.exe(\?|$)') + if ($isSetup) { + $pkgName = "NetX-Setup-$latest.exe" + } else { + $pkgName = "NetX-$latest-win64.zip" + } + $pkgPath = Join-Path $dlDir $pkgName + + $needDownload = $true + if ((Test-Path -LiteralPath $pkgPath) -and ((Get-Item -LiteralPath $pkgPath).Length -gt 1MB)) { + Write-Host "==> Using existing download: $pkgPath ($([math]::Round((Get-Item $pkgPath).Length/1MB,1)) MB)" + $needDownload = $false + } + if ($needDownload) { + Write-Host "==> Downloading $pkgName from $($result.source) ..." + # Only attach token for non-GitHub hosts (Forgejo/private). Public GitHub assets need no auth; + # a Forgejo token would break anonymous GitHub downloads. + $dlHeaders = @{ "User-Agent" = "NetX-UpdateCheck" } + $dlUri = [uri]$result.download_url + $isGithub = ($dlUri.Host -match '(^|\.)github\.com$' -or $dlUri.Host -match '(^|\.)githubusercontent\.com$') + if ($UpdateToken -and -not $isGithub) { + $dlHeaders["Authorization"] = "token $UpdateToken" + } + Invoke-WebRequest -Uri $result.download_url -OutFile $pkgPath -Headers $dlHeaders -UseBasicParsing + } + if ($result.sha256 -and $result.sha256 -notmatch 'REPLACE' -and $result.sha256.Trim()) { + $hash = (Get-FileHash -Path $pkgPath -Algorithm SHA256).Hash.ToLowerInvariant() + $expect = $result.sha256.ToLowerInvariant() + if ($hash -ne $expect) { + throw "sha256_mismatch: got $hash expected $expect" + } + Write-Host "==> SHA256 OK" + } + + # Program Files installs need admin; elevate once (avoid loop via NETX_UPDATE_ELEVATED). + $progWritable = $false + try { + $probe = Join-Path $prog (".netx_w_" + [guid]::NewGuid().ToString("n")) + [IO.File]::WriteAllText($probe, "x") + Remove-Item -LiteralPath $probe -Force -ErrorAction SilentlyContinue + $progWritable = $true + } catch { + $progWritable = $false + } + if (-not $progWritable -and -not $env:NETX_UPDATE_ELEVATED) { + Write-Host "==> Program directory is not writable; relaunching update as Administrator (UAC)..." -ForegroundColor Yellow + $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply" + $ep = Start-Process -FilePath "powershell.exe" -ArgumentList $arg -WorkingDirectory $prog ` + -Verb RunAs -Wait -PassThru + if ($null -eq $ep.ExitCode) { exit 1 } + exit $ep.ExitCode + } + + $env:NETX_UPDATE_ELEVATED = "1" + if ($isSetup) { + # Silent Setup update mode: skip DB wizard, keep ProgramData + # (see installer/netx.iss /InstallMode=update). + Write-Host "==> Applying update via silent Setup.exe" + $setupArgs = "/VERYSILENT /NORESTART /SUPPRESSMSGBOXES /DIR=`"$prog`" /InstallMode=update" + $sp = Start-Process -FilePath $pkgPath -ArgumentList $setupArgs -Wait -PassThru + if ($null -eq $sp.ExitCode -or $sp.ExitCode -ne 0) { + $code = if ($null -eq $sp.ExitCode) { "null" } else { $sp.ExitCode } + throw "setup_update_failed: exit $code" + } + # Setup post-install also repairs; belt-and-suspenders when running elevated apply. + $repairPs1 = Join-Path $prog "packaging\repair_venv.ps1" + if (Test-Path -LiteralPath $repairPs1) { + Write-Host "==> Relinking .venv to bundled python/runtime" + & powershell -NoProfile -ExecutionPolicy Bypass -File $repairPs1 ` + -ProgramRoot $prog -DataRoot $data + } + Write-Host "==> Restarting NetX after Setup" + & (Join-Path $PSScriptRoot "start_netx_app.ps1") ` + -ProgramRoot $prog -DataRoot $data -SkipBrowser + if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } else { + Write-Host "==> Applying legacy zip update via update_netx.ps1" + & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "update_netx.ps1") ` + -PackagePath $pkgPath -ProgramRoot $prog -DataRoot $data + } + Write-Host "==> Update applied to $latest" -ForegroundColor Green +} finally { + Remove-Item -Force $lockFile -ErrorAction SilentlyContinue +} +exit 0 diff --git a/packaging/cmd/NetX-FirstRun.cmd b/packaging/cmd/NetX-FirstRun.cmd new file mode 100644 index 0000000..63fa9e5 --- /dev/null +++ b/packaging/cmd/NetX-FirstRun.cmd @@ -0,0 +1,28 @@ +@echo off +setlocal +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" +cd /d "%ROOT%" +title NetX — Reconfigure database +echo. +echo NetX database reconfigure / 重新配置数据库 +echo Prefer the installer wizard for first-time setup. +echo 首次安装请用安装向导选择内置或外置数据库。 +echo This window is for repair / reconfigure only. +echo 本窗口仅用于修复或重新配置。 +echo. +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\setup_first_run.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" +set "EC=%ERRORLEVEL%" +if not "%EC%"=="0" ( + echo. + echo Setup failed / 配置失败 exit=%EC% + pause + exit /b %EC% +) +echo. +echo Starting NetX tray... / 正在启动托盘... +start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\netx_tray.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -StartOnLaunch +echo. +echo Done. You can close this window. / 完成，可关闭本窗口。 +pause +exit /b 0 diff --git a/packaging/cmd/NetX-Start.cmd b/packaging/cmd/NetX-Start.cmd new file mode 100644 index 0000000..b489f7d --- /dev/null +++ b/packaging/cmd/NetX-Start.cmd @@ -0,0 +1,7 @@ +@echo off +setlocal +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" +cd /d "%ROOT%" +start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action start -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" +exit /b 0 diff --git a/packaging/cmd/NetX-Stop.cmd b/packaging/cmd/NetX-Stop.cmd new file mode 100644 index 0000000..7722e38 --- /dev/null +++ b/packaging/cmd/NetX-Stop.cmd @@ -0,0 +1,7 @@ +@echo off +setlocal +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" +cd /d "%ROOT%" +start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action stop -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" +exit /b 0 diff --git a/packaging/cmd/NetX-Tray.cmd b/packaging/cmd/NetX-Tray.cmd new file mode 100644 index 0000000..7cdd5af --- /dev/null +++ b/packaging/cmd/NetX-Tray.cmd @@ -0,0 +1,7 @@ +@echo off +setlocal +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" +cd /d "%ROOT%" +start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action tray -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -StartOnLaunch +exit /b 0 diff --git a/packaging/config/database.example.env b/packaging/config/database.example.env index a9d072a..f5e5555 100644 --- a/packaging/config/database.example.env +++ b/packaging/config/database.example.env @@ -5,5 +5,11 @@ # NETX_HOST=127.0.0.1 # NETX_PORT=8890 # NETX_UI_DIST_DIR=web/dist +# Defaults already: github + forgejo mirror git.avelo.top (newest reachable wins) +# NETX_UPDATE_SOURCES=github,forgejo +# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest +# NETX_UPDATE_GITHUB_REPO=hansjone/netx # NETX_UPDATE_URL= +# NETX_UPDATE_TOKEN= # NETX_UPDATE_CHANNEL=stable +# NETX_UPDATE_AUTO=false diff --git a/packaging/download_postgres.ps1 b/packaging/download_postgres.ps1 index 38c7b41..6398fd1 100644 --- a/packaging/download_postgres.ps1 +++ b/packaging/download_postgres.ps1 @@ -1,4 +1,4 @@ -param( +param( [string]$Version = "16.4-1", [string]$OutDir = "" ) diff --git a/packaging/install_autostart.ps1 b/packaging/install_autostart.ps1 new file mode 100644 index 0000000..50510df --- /dev/null +++ b/packaging/install_autostart.ps1 @@ -0,0 +1,25 @@ +param( + [string]$ProgramRoot = "", + [switch]$Remove = $false +) + +# Register / unregister NetX tray at current-user logon. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$tray = Join-Path $PSScriptRoot "netx_tray.ps1" +$runKey = "HKCU:\Software\Microsoft\Windows\CurrentVersion\Run" +$name = "NetX" + +if ($Remove) { + Remove-ItemProperty -Path $runKey -Name $name -ErrorAction SilentlyContinue + Write-Host "==> Removed NetX from startup" + exit 0 +} + +$cmd = "powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$tray`" -ProgramRoot `"$prog`" -StartOnLaunch" +New-ItemProperty -Path $runKey -Name $name -Value $cmd -PropertyType String -Force | Out-Null +Write-Host "==> NetX tray will start at logon" +Write-Host " $cmd" diff --git a/packaging/install_service.ps1 b/packaging/install_service.ps1 new file mode 100644 index 0000000..f9e87f6 --- /dev/null +++ b/packaging/install_service.ps1 @@ -0,0 +1,141 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [ValidateSet("winsw", "task")] + [string]$Mode = "winsw", + [switch]$Uninstall = $false, + [switch]$Start = $false, + # Offline-safe by default: use WinSW shipped in the package. Opt-in to download from GitHub. + [switch]$AllowDownload = $false +) + +# Install NetX as a Windows Service (WinSW) or as a SYSTEM startup Scheduled Task. +# Requires elevation for winsw / task modes that run as SYSTEM. +# WinSW binary is bundled at packaging\winsw\WinSW-x64.exe by build_release.ps1 — no network needed. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$svcName = "NetX" +$winswDir = Join-Path $prog "packaging\winsw" +$winswExe = Join-Path $winswDir "NetX.exe" +$winswXml = Join-Path $winswDir "NetX.xml" +$cacheDir = Join-Path $PSScriptRoot "cache" + +function Test-IsAdmin { + $id = [Security.Principal.WindowsIdentity]::GetCurrent() + $p = New-Object Security.Principal.WindowsPrincipal($id) + return $p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) +} + +function ConvertTo-WinSWPath([string]$Path) { + return ($Path -replace '\\', '/') +} + +function Ensure-WinSW { + if (-not (Test-Path $winswDir)) { + New-Item -ItemType Directory -Path $winswDir -Force | Out-Null + } + if (-not (Test-Path $cacheDir)) { + New-Item -ItemType Directory -Path $cacheDir -Force | Out-Null + } + $bundled = Join-Path $PSScriptRoot "winsw\WinSW-x64.exe" + $dl = Join-Path $cacheDir "WinSW-x64.exe" + $src = $null + if (Test-Path $bundled) { + $src = $bundled + Write-Host "==> Using bundled WinSW: $bundled" + } elseif (Test-Path $dl) { + $src = $dl + Write-Host "==> Using cached WinSW: $dl" + } elseif ($AllowDownload) { + $url = "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW-x64.exe" + Write-Host "==> Downloading WinSW: $url" + Invoke-WebRequest -Uri $url -OutFile $dl -UseBasicParsing + $src = $dl + } else { + throw "winsw_missing_offline: expected $bundled (rebuild with build_release.ps1). Or pass -AllowDownload when online." + } + Copy-Item -Path $src -Destination $winswExe -Force + + $runPs1 = ConvertTo-WinSWPath (Join-Path $PSScriptRoot "service_run.ps1") + $stopPs1 = ConvertTo-WinSWPath (Join-Path $PSScriptRoot "stop_netx_app.ps1") + $progFs = ConvertTo-WinSWPath $prog + $dataFs = ConvertTo-WinSWPath $data + $logPath = ConvertTo-WinSWPath (Join-Path $data "data\runtime") + if (-not (Test-Path (Join-Path $data "data\runtime"))) { + New-Item -ItemType Directory -Path (Join-Path $data "data\runtime") -Force | Out-Null + } + + $xml = @" + + $svcName + NetX Ops + NetX API, workers, and optional bundled PostgreSQL + powershell.exe + -NoProfile -ExecutionPolicy Bypass -File "$runPs1" -ProgramRoot "$progFs" -DataRoot "$dataFs" + $logPath + + 10240 + 4 + + + + 1 hour + 60sec + powershell.exe + -NoProfile -ExecutionPolicy Bypass -File "$stopPs1" -ProgramRoot "$progFs" -DataRoot "$dataFs" + $progFs + +"@ + # WinSW expects UTF-8 without BOM issues; ASCII-compatible paths preferred. + [System.IO.File]::WriteAllText($winswXml, $xml) +} + +if ($Uninstall) { + if (-not (Test-IsAdmin)) { throw "admin_required_for_uninstall" } + if (Test-Path $winswExe) { + Write-Host "==> Stopping/uninstalling WinSW service" + & $winswExe stop 2>$null + & $winswExe uninstall 2>$null + } + Unregister-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -Confirm:$false -ErrorAction SilentlyContinue + Write-Host "==> Service/task removed" + exit 0 +} + +if (-not (Test-IsAdmin)) { + throw "admin_required: run elevated PowerShell to install the NetX service" +} + +if ($Mode -eq "winsw") { + Ensure-WinSW + Write-Host "==> Installing Windows Service via WinSW" + & $winswExe stop 2>$null + & $winswExe uninstall 2>$null + & $winswExe install + if ($LASTEXITCODE -ne 0) { throw "winsw_install_failed" } + if ($Start) { + & $winswExe start + Write-Host "==> Service started" -ForegroundColor Green + } else { + Write-Host "==> Installed. Start with: Start-Service NetX or `"$winswExe`" start" + } +} else { + Write-Host "==> Registering Scheduled Task NetX\NetXService (At startup, SYSTEM)" + $runPs1 = Join-Path $PSScriptRoot "service_run.ps1" + $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$runPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`"" + $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog + $trigger = New-ScheduledTaskTrigger -AtStartup + $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest + $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 1) + Register-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Force | Out-Null + if ($Start) { + Start-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" + Write-Host "==> Task started" -ForegroundColor Green + } else { + Write-Host "==> Task registered. Start with: Start-ScheduledTask -TaskPath '\NetX\' -TaskName NetXService" + } +} diff --git a/packaging/install_update_task.ps1 b/packaging/install_update_task.ps1 new file mode 100644 index 0000000..07b3ab9 --- /dev/null +++ b/packaging/install_update_task.ps1 @@ -0,0 +1,57 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [ValidateSet("Daily", "AtLogOn", "Hourly")] + [string]$Trigger = "Daily", + [string]$At = "03:30", + [switch]$Remove = $false, + [switch]$EnableAutoEnv = $true +) + +# Schedule silent update checks. With NETX_UPDATE_AUTO=true, applies updates when available. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$taskName = "NetXUpdate" +$taskPath = "\NetX\" +$checkPs1 = Join-Path $PSScriptRoot "check_update.ps1" + +if ($Remove) { + Unregister-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Confirm:$false -ErrorAction SilentlyContinue + Write-Host "==> Update task removed" + exit 0 +} + +if ($EnableAutoEnv) { + $envFile = Join-Path $data ".env" + if (-not (Test-Path $data)) { + New-Item -ItemType Directory -Path $data -Force | Out-Null + } + Write-DotEnvValue -Path $envFile -Values @{ "NETX_UPDATE_AUTO" = "true" } + Write-Host "==> Set NETX_UPDATE_AUTO=true in $envFile" +} + +$arg = "-NoProfile -ExecutionPolicy Bypass -File `"$checkPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply -Quiet -AutoOnly" +$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog + +switch ($Trigger) { + "Hourly" { + $trig = New-ScheduledTaskTrigger -Once -At (Get-Date).Date.AddHours((Get-Date).Hour + 1) ` + -RepetitionInterval (New-TimeSpan -Hours 1) -RepetitionDuration ([TimeSpan]::MaxValue) + } + "AtLogOn" { + $trig = New-ScheduledTaskTrigger -AtLogOn + } + default { + $trig = New-ScheduledTaskTrigger -Daily -At $At + } +} + +$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -StartWhenAvailable +$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive -RunLevel Limited +Register-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Action $action -Trigger $trig -Settings $settings -Principal $principal -Force | Out-Null +Write-Host "==> Scheduled update task: $taskPath$taskName ($Trigger)" -ForegroundColor Green +Write-Host " Manual run: .\packaging\check_update.ps1 -Apply -Quiet -AutoOnly" diff --git a/packaging/installer/Languages/ChineseSimplified.isl b/packaging/installer/Languages/ChineseSimplified.isl new file mode 100644 index 0000000..30d9973 --- /dev/null +++ b/packaging/installer/Languages/ChineseSimplified.isl @@ -0,0 +1,417 @@ +; *** Inno Setup version 6.5.0+ Chinese Simplified messages *** +; +; To download user-contributed translations of this file, go to: +; https://jrsoftware.org/files/istrans/ +; +; Note: When translating this text, do not add periods (.) to the end of +; messages that didn't have them already, because on those messages Inno +; Setup adds the periods automatically (appending a period would result in +; two periods being displayed). +; +; Maintainer: Zhenghan Yang (Kira) +; Email: 847320916@QQ.com +; Github: https://github.com/kira-96/Inno-Setup-Chinese-Simplified-Translation +; Encoding: UTF-8 +; Translation based on network resource +; + +[LangOptions] +; The following three entries are very important. Be sure to read and +; understand the '[LangOptions] section' topic in the help file. +LanguageName=简体中文 +; About LanguageID, to reference link: +; https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c +LanguageID=$0804 +; LanguageCodePage should always be set if possible, even if this file is Unicode +; For English it's set to zero anyway because English only uses ASCII characters +LanguageCodePage=936 +; If the language you are translating to requires special font faces or +; sizes, uncomment any of the following entries and change them accordingly. +;DialogFontName= +;DialogFontSize=9 +;DialogFontBaseScaleWidth=7 +;DialogFontBaseScaleHeight=15 +;WelcomeFontName=Segoe UI +;WelcomeFontSize=14 + +[Messages] + +; *** Application titles +SetupAppTitle=安装 +SetupWindowTitle=安装 - %1 +UninstallAppTitle=卸载 +UninstallAppFullTitle=%1 卸载 + +; *** Misc. common +InformationTitle=信息 +ConfirmTitle=确认 +ErrorTitle=错误 + +; *** SetupLdr messages +SetupLdrStartupMessage=现在将安装 %1。您想要继续吗？ +LdrCannotCreateTemp=无法创建临时文件。安装程序已中止 +LdrCannotExecTemp=无法执行临时目录中的文件。安装程序已中止 +HelpTextNote= + +; *** Startup error messages +LastErrorMessage=%1。%n%n错误 %2: %3 +SetupFileMissing=安装目录中缺少文件 %1。请修正这个问题或者获取程序的新副本。 +SetupFileCorrupt=安装文件已损坏。请获取程序的新副本。 +SetupFileCorruptOrWrongVer=安装文件已损坏，或是与这个安装程序的版本不兼容。请修正这个问题或获取新的程序副本。 +InvalidParameter=无效的命令行参数：%n%n%1 +SetupAlreadyRunning=安装程序已在运行。 +WindowsVersionNotSupported=此程序不支持当前计算机运行的 Windows 版本。 +WindowsServicePackRequired=此程序需要 %1 服务包 %2 或更高版本。 +NotOnThisPlatform=此程序不能在 %1 上运行。 +OnlyOnThisPlatform=此程序只能在 %1 上运行。 +OnlyOnTheseArchitectures=此程序只能安装到为下列处理器架构设计的 Windows 版本中：%n%n%1 +WinVersionTooLowError=此程序需要 %1 版本 %2 或更高。 +WinVersionTooHighError=此程序不能安装于 %1 版本 %2 或更高。 +AdminPrivilegesRequired=在安装此程序时您必须以管理员身份登录。 +PowerUserPrivilegesRequired=在安装此程序时您必须以管理员身份或高级用户组身份登录。 +SetupAppRunningError=安装程序检测到 %1 当前正在运行。%n%n请先关闭正在运行的程序，然后点击“确定”继续，或点击“取消”退出。 +UninstallAppRunningError=卸载程序检测到 %1 当前正在运行。%n%n请先关闭正在运行的程序，然后点击“确定”继续，或点击“取消”退出。 + +; *** Startup questions +PrivilegesRequiredOverrideTitle=选择安装程序安装模式 +PrivilegesRequiredOverrideInstruction=选择安装模式 +PrivilegesRequiredOverrideText1=%1 可以为所有用户安装（需要管理员权限），或仅为您安装。 +PrivilegesRequiredOverrideText2=%1 可以仅为您安装，或为所有用户安装（需要管理员权限）。 +PrivilegesRequiredOverrideAllUsers=为所有用户安装(&A) +PrivilegesRequiredOverrideAllUsersRecommended=为所有用户安装(&A)（推荐） +PrivilegesRequiredOverrideCurrentUser=仅为我安装(&M) +PrivilegesRequiredOverrideCurrentUserRecommended=仅为我安装(&M)（推荐） + +; *** Misc. errors +ErrorCreatingDir=安装程序无法创建目录“%1” +ErrorTooManyFilesInDir=无法在目录“%1”中创建文件，因为里面包含太多文件。 + +; *** Setup common messages +ExitSetupTitle=退出安装程序 +ExitSetupMessage=安装程序尚未完成。如果现在退出，将不会安装该程序。%n%n您之后可以再次运行安装程序完成安装。%n%n现在退出安装程序吗？ +AboutSetupMenuItem=关于安装程序(&A)... +AboutSetupTitle=关于安装程序 +AboutSetupMessage=%1 版本 %2%n%3%n%n%1 主页：%n%4 +AboutSetupNote= +TranslatorNote=简体中文翻译由 Kira（847320916@qq.com）维护。项目地址：https://github.com/kira-96/Inno-Setup-Chinese-Simplified-Translation + +; *** Buttons +ButtonBack=< 上一步(&B) +ButtonNext=下一步(&N) > +ButtonInstall=安装(&I) +ButtonOK=确定 +ButtonCancel=取消 +ButtonYes=是(&Y) +ButtonYesToAll=全是(&A) +ButtonNo=否(&N) +ButtonNoToAll=全否(&O) +ButtonFinish=完成(&F) +ButtonBrowse=浏览(&B)... +ButtonWizardBrowse=浏览(&R)... +ButtonNewFolder=新建文件夹(&M) + +; *** "Select Language" dialog messages +SelectLanguageTitle=选择安装语言 +SelectLanguageLabel=选择安装时使用的语言。 + +; *** Common wizard text +ClickNext=点击“下一步”继续，或点击“取消”退出安装程序。 +BeveledLabel= +BrowseDialogTitle=浏览文件夹 +BrowseDialogLabel=在下面的列表中选择一个文件夹，然后点击“确定”。 +NewFolderName=新建文件夹 + +; *** "Welcome" wizard page +WelcomeLabel1=欢迎使用 [name] 安装向导 +WelcomeLabel2=即将在您的计算机上安装 [name/ver]。%n%n建议您在继续安装前关闭所有其他应用程序。 + +; *** "Password" wizard page +WizardPassword=密码 +PasswordLabel1=此安装程序需要密码验证。 +PasswordLabel3=请输入密码，然后点击“下一步”继续。密码区分大小写。 +PasswordEditLabel=密码(&P)： +IncorrectPassword=您输入的密码不正确，请重新输入。 + +; *** "License Agreement" wizard page +WizardLicense=许可协议 +LicenseLabel=请在继续安装前阅读以下重要信息。 +LicenseLabel3=请阅读下列许可协议。在继续安装前您必须同意这些协议条款。 +LicenseAccepted=我同意此协议(&A) +LicenseNotAccepted=我不同意此协议(&D) + +; *** "Information" wizard pages +WizardInfoBefore=信息 +InfoBeforeLabel=请在继续安装前阅读以下重要信息。 +InfoBeforeClickLabel=准备好继续安装后，点击“下一步”。 +WizardInfoAfter=信息 +InfoAfterLabel=请在继续安装前阅读以下重要信息。 +InfoAfterClickLabel=准备好继续安装后，点击“下一步”。 + +; *** "User Information" wizard page +WizardUserInfo=用户信息 +UserInfoDesc=请输入您的信息。 +UserInfoName=用户名(&U)： +UserInfoOrg=组织(&O)： +UserInfoSerial=序列号(&S)： +UserInfoNameRequired=请输入用户名。 + +; *** "Select Destination Location" wizard page +WizardSelectDir=选择目标位置 +SelectDirDesc=您想将 [name] 安装在哪里？ +SelectDirLabel3=安装程序将安装 [name] 到下面的文件夹中。 +SelectDirBrowseLabel=点击“下一步”继续。如果您想选择其他文件夹，点击“浏览”。 +DiskSpaceGBLabel=至少需要有 [gb] GB 的可用磁盘空间。 +DiskSpaceMBLabel=至少需要有 [mb] MB 的可用磁盘空间。 +CannotInstallToNetworkDrive=安装程序无法安装到一个网络驱动器。 +CannotInstallToUNCPath=安装程序无法安装到一个 UNC 路径。 +InvalidPath=您必须输入一个带驱动器盘符的完整路径，例如：%n%nC:\App%n%n或UNC路径：%n%n\\server\share +InvalidDrive=您选定的驱动器或 UNC 共享不存在或不能访问。请选择其他位置。 +DiskSpaceWarningTitle=磁盘空间不足 +DiskSpaceWarning=安装程序至少需要 %1 KB 的可用空间才能安装，但选定驱动器只有 %2 KB 的可用空间。%n%n您确定要继续吗？ +DirNameTooLong=文件夹名称或路径太长。 +InvalidDirName=文件夹名称无效。 +BadDirName32=文件夹名称不能包含下列任何字符：%n%n%1 +DirExistsTitle=文件夹已存在 +DirExists=文件夹：%n%n%1%n%n已经存在。您确定安装到这个文件夹中吗？ +DirDoesntExistTitle=文件夹不存在 +DirDoesntExist=文件夹：%n%n%1%n%n不存在。您想要创建此文件夹吗？ + +; *** "Select Components" wizard page +WizardSelectComponents=选择组件 +SelectComponentsDesc=您想安装哪些程序组件？ +SelectComponentsLabel2=选中您想安装的组件；取消您不想安装的组件。然后点击“下一步”继续。 +FullInstallation=完全安装 +; if possible don't translate 'Compact' as 'Minimal' (I mean 'Minimal' in your language) +CompactInstallation=简洁安装 +CustomInstallation=自定义安装 +NoUninstallWarningTitle=组件已存在 +NoUninstallWarning=安装程序检测到下列组件已安装在您的计算机中：%n%n%1%n%n取消选中这些组件不会卸载它们。%n%n您确定要继续吗？ +ComponentSize1=%1 KB +ComponentSize2=%1 MB +ComponentsDiskSpaceGBLabel=当前选择的组件需要至少 [gb] GB 的磁盘空间。 +ComponentsDiskSpaceMBLabel=当前选择的组件需要至少 [mb] MB 的磁盘空间。 + +; *** "Select Additional Tasks" wizard page +WizardSelectTasks=选择附加任务 +SelectTasksDesc=您想要安装程序执行哪些附加任务？ +SelectTasksLabel2=选择您想要安装程序在安装 [name] 时执行的附加任务，然后点击“下一步”。 + +; *** "Select Start Menu Folder" wizard page +WizardSelectProgramGroup=选择开始菜单文件夹 +SelectStartMenuFolderDesc=安装程序应该在哪里放置程序的快捷方式？ +SelectStartMenuFolderLabel3=安装程序将在下列“开始”菜单文件夹中创建程序的快捷方式。 +SelectStartMenuFolderBrowseLabel=点击“下一步”继续。如果您想选择其他文件夹，点击“浏览”。 +MustEnterGroupName=您必须输入一个文件夹名称。 +GroupNameTooLong=文件夹名称或路径太长。 +InvalidGroupName=文件夹名称无效。 +BadGroupName=文件夹名称不能包含下列任何字符：%n%n%1 +NoProgramGroupCheck2=不创建开始菜单文件夹(&D) + +; *** "Ready to Install" wizard page +WizardReady=准备安装 +ReadyLabel1=安装程序准备就绪，现在可以开始安装 [name] 到您的计算机。 +ReadyLabel2a=点击“安装”继续此安装程序。如果您想重新查看或修改任何设置，点击“上一步”。 +ReadyLabel2b=点击“安装”继续此安装程序。 +ReadyMemoUserInfo=用户信息： +ReadyMemoDir=目标位置： +ReadyMemoType=安装类型： +ReadyMemoComponents=已选择组件： +ReadyMemoGroup=开始菜单文件夹： +ReadyMemoTasks=附加任务： + +; *** TDownloadWizardPage wizard page and DownloadTemporaryFile +DownloadingLabel2=正在下载文件... +ButtonStopDownload=停止下载(&S) +StopDownload=您确定要停止下载吗？ +ErrorDownloadAborted=下载已中止。 +ErrorDownloadFailed=下载失败：%1 %2。 +ErrorDownloadSizeFailed=获取大小失败：%1 %2。 +ErrorProgress=无效的进度：%1 / %2。 +ErrorFileSize=文件大小错误：预期 %1，实际 %2。 + +; *** TExtractionWizardPage wizard page and ExtractArchive +ExtractingLabel=正在提取文件... +ButtonStopExtraction=停止提取(&S) +StopExtraction=您确定要停止提取吗？ +ErrorExtractionAborted=提取已中止。 +ErrorExtractionFailed=提取失败：%1 + +; *** Archive extraction failure details +ArchiveIncorrectPassword=密码不正确。 +ArchiveIsCorrupted=压缩包已损坏。 +ArchiveUnsupportedFormat=不支持的压缩包格式。 + +; *** "Preparing to Install" wizard page +WizardPreparing=正在准备安装 +PreparingDesc=安装程序正在准备安装 [name] 到您的计算机。 +PreviousInstallNotCompleted=先前的程序安装或卸载未完成，需要您重启计算机以完成该安装。%n%n在重启计算机后，再次运行安装程序以完成 [name] 的安装。 +CannotContinue=安装程序不能继续。请点击“取消”退出。 +ApplicationsFound=以下应用程序正在使用将由安装程序更新的文件。建议您允许安装程序自动关闭这些应用程序。 +ApplicationsFound2=以下应用程序正在使用将由安装程序更新的文件。建议您允许安装程序自动关闭这些应用程序。安装完成后，安装程序将尝试重新启动这些应用程序。 +CloseApplications=自动关闭应用程序(&A) +DontCloseApplications=不要关闭应用程序(&D) +ErrorCloseApplications=安装程序无法自动关闭所有应用程序。建议您在继续之前，关闭所有在使用需要由安装程序更新的文件的应用程序。 +PrepareToInstallNeedsRestart=安装程序必须重启您的计算机。计算机重启后，请再次运行安装程序以完成 [name] 的安装。%n%n要立即重启吗？ + +; *** "Installing" wizard page +WizardInstalling=正在安装 +InstallingLabel=安装程序正在安装 [name] 到您的计算机，请稍候。 + +; *** "Setup Completed" wizard page +FinishedHeadingLabel=完成 [name] 安装向导 +FinishedLabelNoIcons=安装程序已在您的计算机中安装了 [name]。 +FinishedLabel=安装程序已在您的计算机中安装了 [name]。您可以通过已安装的快捷方式运行此应用程序。 +ClickFinish=点击“完成”退出安装程序。 +FinishedRestartLabel=为完成 [name] 的安装，安装程序必须重新启动您的计算机。要立即重启吗？ +FinishedRestartMessage=为完成 [name] 的安装，安装程序必须重新启动您的计算机。%n%n要立即重启吗？ +ShowReadmeCheck=是，我想查阅自述文件 +YesRadio=是，立即重启计算机(&Y) +NoRadio=否，稍后重启计算机(&N) +; used for example as 'Run MyProg.exe' +RunEntryExec=运行 %1 +; used for example as 'View Readme.txt' +RunEntryShellExec=查阅 %1 + +; *** "Setup Needs the Next Disk" stuff +ChangeDiskTitle=安装程序需要下一张磁盘 +SelectDiskLabel2=请插入磁盘 %1 并点击“确定”。%n%n如果这个磁盘中的文件可以在下列文件夹之外的文件夹中找到，请输入正确的路径或点击“浏览”。 +PathLabel=路径(&P)： +FileNotInDir2=“%2”中找不到文件“%1”。请插入正确的磁盘或选择其他文件夹。 +SelectDirectoryLabel=请指定下一张磁盘的位置。 + +; *** Installation phase messages +SetupAborted=安装程序未完成安装。%n%n请修正这个问题并重新运行安装程序。 +AbortRetryIgnoreSelectAction=选择操作 +AbortRetryIgnoreRetry=重试(&T) +AbortRetryIgnoreIgnore=忽略错误并继续(&I) +AbortRetryIgnoreCancel=取消安装 +RetryCancelSelectAction=选择操作 +RetryCancelRetry=重试(&T) +RetryCancelCancel=取消 + +; *** Installation status messages +StatusClosingApplications=正在关闭应用程序... +StatusCreateDirs=正在创建目录... +StatusExtractFiles=正在提取文件... +StatusDownloadFiles=正在下载文件... +StatusCreateIcons=正在创建快捷方式... +StatusCreateIniEntries=正在创建 INI 条目... +StatusCreateRegistryEntries=正在创建注册表条目... +StatusRegisterFiles=正在注册文件... +StatusSavingUninstall=正在保存卸载信息... +StatusRunProgram=正在完成安装... +StatusRestartingApplications=正在重启应用程序... +StatusRollback=正在撤销更改... + +; *** Misc. errors +ErrorInternal2=内部错误：%1。 +ErrorFunctionFailedNoCode=%1 失败。 +ErrorFunctionFailed=%1 失败；错误代码 %2。 +ErrorFunctionFailedWithMessage=%1 失败；错误代码 %2。%n%3 +ErrorExecutingProgram=无法执行文件：%n%1 + +; *** Registry errors +ErrorRegOpenKey=打开注册表项时出错：%n%1\%2 +ErrorRegCreateKey=创建注册表项时出错：%n%1\%2 +ErrorRegWriteKey=写入注册表项时出错：%n%1\%2 + +; *** INI errors +ErrorIniEntry=在文件“%1”中创建 INI 条目时出错。 + +; *** File copying errors +FileAbortRetryIgnoreSkipNotRecommended=跳过此文件(&S)（不推荐） +FileAbortRetryIgnoreIgnoreNotRecommended=忽略错误并继续(&I)（不推荐） +SourceIsCorrupted=源文件已损坏。 +SourceDoesntExist=源文件“%1”不存在。 +SourceVerificationFailed=源文件验证失败：%1 +VerificationSignatureDoesntExist=签名文件“%1”不存在。 +VerificationSignatureInvalid=签名文件“%1”无效。 +VerificationKeyNotFound=签名文件“%1”使用了未知的密钥。 +VerificationFileNameIncorrect=文件名不正确。 +VerificationFileTagIncorrect=文件标签不正确。 +VerificationFileSizeIncorrect=文件大小不正确。 +VerificationFileHashIncorrect=文件哈希值不正确。 +ExistingFileReadOnly2=无法替换已存在的文件，它是只读的。 +ExistingFileReadOnlyRetry=移除只读属性并重试(&R) +ExistingFileReadOnlyKeepExisting=保留已存在的文件(&K) +ErrorReadingExistingDest=尝试读取已存在的文件时出错： +FileExistsSelectAction=选择操作 +FileExists2=文件已经存在。 +FileExistsOverwriteExisting=覆盖已存在的文件(&O) +FileExistsKeepExisting=保留已存在的文件(&K) +FileExistsOverwriteOrKeepAll=为接下来的冲突文件执行此操作(&D) +ExistingFileNewerSelectAction=选择操作 +ExistingFileNewer2=已存在的文件比安装程序将要安装的文件还要新。 +ExistingFileNewerOverwriteExisting=覆盖已存在的文件(&O) +ExistingFileNewerKeepExisting=保留已存在的文件(&K)（推荐） +ExistingFileNewerOverwriteOrKeepAll=为接下来的冲突文件执行此操作(&D) +ErrorChangingAttr=尝试更改下列已存在的文件属性时出错： +ErrorCreatingTemp=尝试在目标目录创建文件时出错： +ErrorReadingSource=尝试读取下列源文件时出错： +ErrorCopying=尝试复制下列文件时出错： +ErrorDownloading=尝试下载文件时出错： +ErrorExtracting=尝试提取压缩包时出错： +ErrorReplacingExistingFile=尝试替换已存在的文件时出错： +ErrorRestartReplace=重启并替换失败： +ErrorRenamingTemp=尝试重命名下列目标目录中的一个文件时出错： +ErrorRegisterServer=无法注册 DLL/OCX：%1 +ErrorRegSvr32Failed=RegSvr32 失败；退出代码 %1。 +ErrorRegisterTypeLib=无法注册类型库：%1 + +; *** Uninstall display name markings +; used for example as 'My Program (32-bit)' +UninstallDisplayNameMark=%1 (%2) +; used for example as 'My Program (32-bit, All users)' +UninstallDisplayNameMarks=%1 (%2, %3) +UninstallDisplayNameMark32Bit=32 位 +UninstallDisplayNameMark64Bit=64 位 +UninstallDisplayNameMarkAllUsers=所有用户 +UninstallDisplayNameMarkCurrentUser=当前用户 + +; *** Post-installation errors +ErrorOpeningReadme=尝试打开自述文件时出错。 +ErrorRestartingComputer=安装程序无法重启计算机，请手动重启。 + +; *** Uninstaller messages +UninstallNotFound=文件“%1”不存在。无法卸载。 +UninstallOpenError=文件“%1”不能被打开。无法卸载 +UninstallUnsupportedVer=此版本的卸载程序无法识别卸载日志文件“%1”的格式。无法卸载。 +UninstallUnknownEntry=卸载日志中遇到一个未知条目（%1）。 +ConfirmUninstall=您确认要完全移除 %1 及其所有组件吗？ +UninstallOnlyOnWin64=仅允许在 64 位 Windows 中卸载此程序。 +OnlyAdminCanUninstall=仅使用管理员权限的用户能完成此卸载。 +UninstallStatusLabel=正在从您的计算机中移除 %1，请稍候。 +UninstalledAll=已顺利从您的计算机中移除 %1。 +UninstalledMost=%1 卸载完成。%n%n有部分内容未能被删除，但您可以手动删除它们。 +UninstalledAndNeedsRestart=为完成 %1 的卸载，需要重启您的计算机。%n%n要立即重启吗？ +UninstallDataCorrupted=文件“%1”已损坏。无法卸载。 + +; *** Uninstallation phase messages +ConfirmDeleteSharedFileTitle=删除共享文件？ +ConfirmDeleteSharedFile2=系统表示下列共享文件已不再有任何程序使用。您希望卸载程序删除此共享文件吗？%n%n如果仍有程序正在使用此文件，删除后这些程序可能无法正常运行。如果您不能确定，请选择“否”，保留此文件在系统中不会造成任何损害。 +SharedFileNameLabel=文件名： +SharedFileLocationLabel=位置： +WizardUninstalling=卸载状态 +StatusUninstalling=正在卸载 %1... + +; *** Shutdown block reasons +ShutdownBlockReasonInstallingApp=正在安装 %1。 +ShutdownBlockReasonUninstallingApp=正在卸载 %1。 + +; The custom messages below aren't used by Setup itself, but if you make +; use of them in your scripts, you'll want to translate them. + +[CustomMessages] + +NameAndVersion=%1 版本 %2 +AdditionalIcons=附加快捷方式： +CreateDesktopIcon=创建桌面快捷方式(&D) +CreateQuickLaunchIcon=创建快速启动栏快捷方式(&Q) +ProgramOnTheWeb=%1 网站 +UninstallProgram=卸载 %1 +LaunchProgram=运行 %1 +AssocFileExtension=将 %2 文件扩展名与 %1 建立关联(&A) +AssocingFileExtension=正在将 %2 文件扩展名与 %1 建立关联... +AutoStartProgramGroupDescription=启动： +AutoStartProgram=自动启动 %1 +AddonHostProgramNotFound=您选择的文件夹中无法找到 %1。%n%n您确定要继续吗？ diff --git a/packaging/installer/netx.iss b/packaging/installer/netx.iss index 55090ea..42b836e 100644 --- a/packaging/installer/netx.iss +++ b/packaging/installer/netx.iss @@ -1,10 +1,11 @@ -; NetX Windows installer (Inno Setup 6+) +; NetX Windows installer (Inno Setup 6+) ; Compile after: packaging\build_release.ps1 ; ISCC.exe packaging\installer\netx.iss +; Encoding: UTF-8 with BOM (required for Chinese CustomMessages) #define MyAppName "NetX" #ifndef MyAppVersion - #define MyAppVersion "0.3.0" + #define MyAppVersion "0.4.12" #endif #define MyAppPublisher "NetX" #define MyAppURL "https://github.com/hansjone/netx" @@ -12,6 +13,7 @@ ; Stage directory produced by build_release.ps1 (relative to this .iss) #define StageDir "..\release\netx-win64" +#define IconFile "..\assets\netx.ico" [Setup] AppId={{A7E3C2D1-9F40-4B8E-9C1A-NETXWIN64001} @@ -24,46 +26,1048 @@ DefaultGroupName=NetX DisableProgramGroupPage=yes LicenseFile={#StageDir}\LICENSE OutputDir=..\release -OutputBaseFilename=NetX-Setup-{#MyAppVersion} +#ifndef OutputBaseFilename + #define OutputBaseFilename "NetX-Setup-" + MyAppVersion +#endif +OutputBaseFilename={#OutputBaseFilename} +SetupIconFile={#IconFile} +UninstallDisplayIcon={app}\packaging\assets\netx.ico Compression=lzma2 SolidCompression=yes WizardStyle=modern ArchitecturesAllowed=x64compatible ArchitecturesInstallIn64BitMode=x64compatible PrivilegesRequired=admin -; Data lives under {commonappdata}\NetX — not overwritten by upgrades -CloseApplications=yes +; Data lives under {commonappdata}\NetX - not overwritten by upgrades. +; Do NOT use CloseApplications=yes — it can freeze/beep on the Tasks/Ready +; pages while scanning locked NetX tray/service processes with no visible dialog. +CloseApplications=no +ShowLanguageDialog=yes [Languages] Name: "english"; MessagesFile: "compiler:Default.isl" +Name: "chinesesimplified"; MessagesFile: "Languages\ChineseSimplified.isl" + +[CustomMessages] +english.CreateDesktopIcon=Create a desktop shortcut +english.SetupOptions=After install: +english.StartTrayNow=Start NetX tray now +english.EnableAutostart=Start NetX tray at Windows logon +english.EnableAutoUpdate=Enable daily silent auto-update (needs network later) +english.InstallService=Install as Windows Service (uses bundled WinSW, offline) +english.UninstallDeleteData=Also delete NetX data (%ProgramData%\NetX)?%n%nYes = remove database, settings, and secrets%nNo = keep data for a later reinstall +english.DbPageCaption=Database +english.DbPageDescription=Choose built-in or external PostgreSQL. Optional credential key: paste from an existing install, or leave empty to auto-generate. +english.DbBundled=Built-in PostgreSQL (portable, offline) +english.DbExternal=External PostgreSQL (existing server) +english.DbHost=Host +english.DbPort=Port +english.DbUser=User +english.DbPassword=Password +english.DbName=Database +english.CredKey=Credential key +english.CredKeyHint=NETX_CREDENTIAL_SECRET_KEY (optional — empty = auto-generate) +english.DbFieldsRequired=Please fill in host, port, user, password, and database name. +english.DbTesting=Testing PostgreSQL connection, please wait… +english.DbTestFailed=Cannot connect to PostgreSQL with these settings.%n%n%1%n%nFix the settings and try again. +english.DbTestExtractFailed=Could not extract PostgreSQL client tools for connection test. +english.DbSilentExternalMissing=Silent install with external DB requires /DbHost /DbUser /DbPassword /DbName (optional /DbPort). +english.DbApplyFailed=Database configuration failed after file install (exit %1).%n%nLog: %2%n%nFix the problem, then use Start Menu → Reconfigure database — or reinstall. +english.DbApplyExecFailed=Could not start database configuration script. +english.DbBundledMissing=Built-in PostgreSQL files are missing from the install folder:%n%1%n%nThe Setup package is incomplete. Re-download NetX-Setup and install again. +english.DbEnvMissing=Database configuration did not write %ProgramData%\NetX\.env. +english.ReconfigureDb=Reconfigure database +english.UpgradeKeepDb=Existing NetX data found - database settings will be kept (no wizard). +english.ModePageCaption=Install mode +english.ModePageDescription=An existing NetX installation was found. Choose update or reinstall. +english.ModeUpdate=Update existing installation (keep database settings) +english.ModeReinstall=Reinstall and reconfigure database (same as first install) +english.ModeUpdateHint=Replaces program files only. ProgramData\.env and database data are kept. +english.ModeReinstallHint=Shows the Database page again. Does not delete ProgramData unless you uninstall and choose to. +chinesesimplified.CreateDesktopIcon=创建桌面快捷方式 +chinesesimplified.SetupOptions=安装完成后: +chinesesimplified.StartTrayNow=立即启动 NetX 托盘 +chinesesimplified.EnableAutostart=开机时自动启动 NetX 托盘 +chinesesimplified.EnableAutoUpdate=启用每日静默自动更新（以后需要联网） +chinesesimplified.InstallService=安装为 Windows 服务（使用已捆绑的 WinSW，可离线） +chinesesimplified.UninstallDeleteData=是否同时删除 NetX 数据（%ProgramData%\NetX）？%n%n是 = 删除数据库、配置和密钥%n否 = 保留数据以便以后重装 +chinesesimplified.DbPageCaption=数据库 +chinesesimplified.DbPageDescription=选择内置或外置 PostgreSQL。可选粘贴已有 NETX_CREDENTIAL_SECRET_KEY；留空则自动生成。 +chinesesimplified.DbBundled=内置 PostgreSQL（便携，可离线） +chinesesimplified.DbExternal=外置 PostgreSQL（已有服务器） +chinesesimplified.DbHost=主机 +chinesesimplified.DbPort=端口 +chinesesimplified.DbUser=用户 +chinesesimplified.DbPassword=密码 +chinesesimplified.DbName=数据库名 +chinesesimplified.CredKey=凭据密钥 +chinesesimplified.CredKeyHint=NETX_CREDENTIAL_SECRET_KEY（可选，留空=自动生成） +chinesesimplified.DbFieldsRequired=请填写主机、端口、用户、密码和数据库名。 +chinesesimplified.DbTesting=正在测试 PostgreSQL 连接，请稍候… +chinesesimplified.DbTestFailed=无法用当前设置连接 PostgreSQL。%n%n%1%n%n请改正后重试。 +chinesesimplified.DbTestExtractFailed=无法解压 PostgreSQL 客户端以测试连接。 +chinesesimplified.DbSilentExternalMissing=静默安装外置库需要参数 /DbHost /DbUser /DbPassword /DbName（可选 /DbPort）。 +chinesesimplified.DbApplyFailed=文件安装后数据库自动配置失败（退出码 %1）。%n%n日志: %2%n%n请处理后使用开始菜单 → 重新配置数据库，或重新安装。 +chinesesimplified.DbApplyExecFailed=无法启动数据库配置脚本。 +chinesesimplified.DbBundledMissing=安装目录中缺少内置 PostgreSQL 文件：%n%1%n%n安装包不完整，请重新下载 NetX-Setup 后再装。 +chinesesimplified.DbEnvMissing=数据库配置未写入 %ProgramData%\NetX\.env。 +chinesesimplified.ReconfigureDb=重新配置数据库 +chinesesimplified.UpgradeKeepDb=检测到已有 NetX 数据 - 将保留数据库配置（跳过向导）。 +chinesesimplified.ModePageCaption=安装模式 +chinesesimplified.ModePageDescription=检测到已有 NetX 安装。请选择更新或重装。 +chinesesimplified.ModeUpdate=更新现有安装（保留数据库配置） +chinesesimplified.ModeReinstall=重装并重新配置数据库（与首次安装相同） +chinesesimplified.ModeUpdateHint=仅替换程序文件，保留 ProgramData\.env 与数据库数据。 +chinesesimplified.ModeReinstallHint=再次显示数据库页。不会删除 ProgramData（除非卸载时选择删除）。 [Tasks] -Name: "desktopicon"; Description: "Create a desktop shortcut"; GroupDescription: "Additional icons:"; Flags: unchecked -Name: "firstrun"; Description: "Run first-time setup (database mode) after install"; GroupDescription: "Setup:"; Flags: checkedonce +Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: checkedonce [Files] -; Entire release stage → {app}. Exclude .portable so installed builds use ProgramData. +; Entire release stage -> {app}. Exclude .portable so installed builds use ProgramData. Source: "{#StageDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs; Excludes: ".portable" +; Connection-test tools (wizard Next / silent PrepareToInstall only) +Source: "test_pg_conn.ps1"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\psql.exe"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libpq.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libssl-3-x64.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libcrypto-3-x64.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libintl-9.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libiconv-2.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libwinpthread-1.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\zlib1.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\liblz4.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libzstd.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libxml2.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libxslt.dll"; Flags: dontcopy [Dirs] -Name: "{commonappdata}\NetX" -Name: "{commonappdata}\NetX\data" -Name: "{commonappdata}\NetX\data\auth" -Name: "{commonappdata}\NetX\data\runtime" -Name: "{commonappdata}\NetX\pgdata" -Name: "{commonappdata}\NetX\backups" +; users-modify so tray / non-admin reconfigure can update .env and runtime files. +Name: "{commonappdata}\NetX"; Permissions: users-modify +Name: "{commonappdata}\NetX\data"; Permissions: users-modify +Name: "{commonappdata}\NetX\data\auth"; Permissions: users-modify +Name: "{commonappdata}\NetX\data\runtime"; Permissions: users-modify +Name: "{commonappdata}\NetX\pgdata"; Permissions: users-modify +Name: "{commonappdata}\NetX\backups"; Permissions: users-modify + +[InstallDelete] +; Remove legacy desktop shortcuts from older Setup builds (keep a single NetX icon). +Type: files; Name: "{commondesktop}\NetX Start.lnk" +Type: files; Name: "{commondesktop}\NetX First-time setup.lnk" +Type: files; Name: "{commondesktop}\首次配置（内置或外置数据库）.lnk" [Icons] -Name: "{group}\Start NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" -Name: "{group}\Stop NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\stop_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" -Name: "{group}\Open NetX UI"; Filename: "http://127.0.0.1:8890/" -Name: "{group}\First-time setup"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\setup_first_run.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" -Name: "{autodesktop}\Start NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Tasks: desktopicon +; Prefer .cmd targets so Server 2012 / classic Start Menu shows normal program entries. +Name: "{group}\NetX Tray"; Filename: "{app}\NetX-Tray.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\Start NetX"; Filename: "{app}\NetX-Start.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\Stop NetX"; Filename: "{app}\NetX-Stop.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\Check for updates"; Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\packaging\launch_shortcut.ps1"" -Action update -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\Open NetX UI"; Filename: "http://127.0.0.1:8890/"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\{cm:ReconfigureDb}"; Filename: "{app}\NetX-FirstRun.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\Install Windows Service (admin)"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\install_service.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"" -Start"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\Enable silent auto-update (daily)"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\install_update_task.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\Enable start at logon"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\install_autostart.ps1"" -ProgramRoot ""{app}"""; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +; One desktop shortcut only (tray = start + tray UI). Start Menu keeps Start/Stop entries. +Name: "{commondesktop}\NetX"; Filename: "{app}\NetX-Tray.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico"; Tasks: desktopicon [Run] -Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\setup_first_run.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Flags: postinstall skipifsilent; Tasks: firstrun -Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Description: "Start NetX now"; Flags: postinstall nowait skipifsilent unchecked +Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\netx_tray.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"" -StartOnLaunch"; WorkingDir: "{app}"; Description: "{cm:StartTrayNow}"; Flags: postinstall runhidden nowait skipifsilent; Check: NetxShouldStartTray +Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\install_autostart.ps1"" -ProgramRoot ""{app}"""; WorkingDir: "{app}"; Description: "{cm:EnableAutostart}"; Flags: postinstall runhidden skipifsilent unchecked +Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\install_update_task.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Description: "{cm:EnableAutoUpdate}"; Flags: postinstall runhidden skipifsilent unchecked +Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\install_service.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"" -Start"; WorkingDir: "{app}"; Description: "{cm:InstallService}"; Flags: postinstall runhidden skipifsilent unchecked + +; Stop runtime BEFORE files are deleted. Do not also Exec prepare from +; InitializeUninstall (that previously raced/killed unins000 or hung the UI). +[UninstallRun] +Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\uninstall_prepare.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; RunOnceId: "StopNetX"; Flags: runhidden waituntilterminated [UninstallDelete] -; Do NOT delete {commonappdata}\NetX — preserves DB and secrets across reinstall +; Program files always removed. Data dir only when user chose Yes in InitializeUninstall. Type: filesandordirs; Name: "{app}" + +[Code] +var + GDeleteNetxData: Boolean; + GTempDeleteDataScript: String; + DbPage: TWizardPage; + RbBundled: TNewRadioButton; + RbExternal: TNewRadioButton; + LblHost: TNewStaticText; + LblPort: TNewStaticText; + LblUser: TNewStaticText; + LblPassword: TNewStaticText; + LblDbName: TNewStaticText; + EdHost: TNewEdit; + EdPort: TNewEdit; + EdUser: TNewEdit; + EdPassword: TNewEdit; + EdDbName: TNewEdit; + LblCredKey: TNewStaticText; + LblCredHint: TNewStaticText; + EdCredKey: TNewEdit; + GDbMode: String; + GDbUrl: String; + GDbHost: String; + GDbPort: String; + GDbUser: String; + GDbPassword: String; + GDbName: String; + GCredKey: String; + GSkipDbPage: Boolean; + GIsUpgrade: Boolean; + GInstallMode: String; + ModePage: TWizardPage; + RbModeUpdate: TNewRadioButton; + RbModeReinstall: TNewRadioButton; + LblModeHint: TNewStaticText; + GDbConfigured: Boolean; + GPgToolsReady: Boolean; + GDbConnOk: Boolean; + GNextBtnCaption: String; + +function EnvHasDbMode(): Boolean; +var + Lines: TArrayOfString; + i: Integer; + Line: String; +begin + Result := False; + if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then + Exit; + for i := 0 to GetArrayLength(Lines) - 1 do + begin + Line := Trim(Lines[i]); + if Pos('NETX_DB_MODE=', Line) = 1 then + begin + if Length(Line) > Length('NETX_DB_MODE=') then + begin + Result := True; + Exit; + end; + end; + end; +end; + +function EnvModeIsExternal(): Boolean; +var + Lines: TArrayOfString; + i: Integer; + Line: String; +begin + Result := False; + if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then + Exit; + for i := 0 to GetArrayLength(Lines) - 1 do + begin + Line := Trim(Lines[i]); + if CompareText(Line, 'NETX_DB_MODE=external') = 0 then + begin + Result := True; + Exit; + end; + end; +end; + +function ParamIsTruthy(const ParamName: String): Boolean; +var + V: String; +begin + V := LowerCase(Trim(ExpandConstant('{param:' + ParamName + '|}'))); + Result := (V = '1') or (V = 'true') or (V = 'yes') or (V = 'on'); +end; + +function DetectExistingConfiguredInstall(): Boolean; +begin + { Configured install = ProgramData\.env already has NETX_DB_MODE. } + Result := EnvHasDbMode(); +end; + +procedure SyncSkipDbFromInstallMode(); +begin + GSkipDbPage := (GInstallMode = 'update'); +end; + +procedure ApplyInstallModeFromChoice(); +begin + if (RbModeReinstall <> nil) and RbModeReinstall.Checked then + GInstallMode := 'reinstall' + else if GIsUpgrade then + GInstallMode := 'update' + else + GInstallMode := 'fresh'; + SyncSkipDbFromInstallMode(); +end; + +procedure ModeChoiceClick(Sender: TObject); +begin + ApplyInstallModeFromChoice(); + if LblModeHint = nil then + Exit; + if GInstallMode = 'reinstall' then + LblModeHint.Caption := ExpandConstant('{cm:ModeReinstallHint}') + else + LblModeHint.Caption := ExpandConstant('{cm:ModeUpdateHint}'); +end; + +function ResolveInstallModeFromParams(): Boolean; +{ Returns True if a CLI param forced the mode. } +var + Mode: String; +begin + Result := False; + Mode := LowerCase(Trim(ExpandConstant('{param:InstallMode|}'))); + if (Mode = 'update') or (Mode = 'upgrade') then + begin + GInstallMode := 'update'; + Result := True; + end + else if (Mode = 'reinstall') or (Mode = 'fresh') then + begin + GInstallMode := 'reinstall'; + Result := True; + end + else if ParamIsTruthy('SkipDbPage') then + begin + GInstallMode := 'update'; + Result := True; + end + else if ParamIsTruthy('ForceDbPage') then + begin + GInstallMode := 'reinstall'; + Result := True; + end; + if Result then + SyncSkipDbFromInstallMode(); +end; + +procedure StopNetxBeforeFileReplace(); +var + ResultCode: Integer; + StopScript: String; + Params: String; +begin + StopScript := ExpandConstant('{app}\packaging\stop_netx_app.ps1'); + if not FileExists(StopScript) then + Exit; + Params := + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + StopScript + '"' + + ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + + ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"' + + ' -KillTray'; + Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode); +end; + +function NetxShouldStartTray(): Boolean; +begin + Result := GDbConfigured or + FileExists(ExpandConstant('{commonappdata}\NetX\.env')); +end; + +procedure UpdateDbFieldState(); +var + Ext: Boolean; +begin + Ext := RbExternal.Checked; + LblHost.Enabled := Ext; + LblPort.Enabled := Ext; + LblUser.Enabled := Ext; + LblPassword.Enabled := Ext; + LblDbName.Enabled := Ext; + EdHost.Enabled := Ext; + EdPort.Enabled := Ext; + EdUser.Enabled := Ext; + EdPassword.Enabled := Ext; + EdDbName.Enabled := Ext; +end; + +procedure DbModeClick(Sender: TObject); +begin + UpdateDbFieldState(); +end; + +function ExtractPgClientTools(): Boolean; +begin + if GPgToolsReady and FileExists(ExpandConstant('{tmp}\psql.exe')) and + FileExists(ExpandConstant('{tmp}\test_pg_conn.ps1')) then + begin + Result := True; + Exit; + end; + Result := False; + try + ExtractTemporaryFile('test_pg_conn.ps1'); + ExtractTemporaryFile('psql.exe'); + ExtractTemporaryFile('libpq.dll'); + ExtractTemporaryFile('libssl-3-x64.dll'); + ExtractTemporaryFile('libcrypto-3-x64.dll'); + ExtractTemporaryFile('libintl-9.dll'); + ExtractTemporaryFile('libiconv-2.dll'); + ExtractTemporaryFile('libwinpthread-1.dll'); + ExtractTemporaryFile('zlib1.dll'); + ExtractTemporaryFile('liblz4.dll'); + ExtractTemporaryFile('libzstd.dll'); + ExtractTemporaryFile('libxml2.dll'); + ExtractTemporaryFile('libxslt.dll'); + Result := FileExists(ExpandConstant('{tmp}\psql.exe')) and + FileExists(ExpandConstant('{tmp}\test_pg_conn.ps1')); + GPgToolsReady := Result; + except + Result := False; + GPgToolsReady := False; + end; +end; + +procedure SetWizardBusy(const Busy: Boolean; const StatusText: String); +begin + if Busy then + begin + if GNextBtnCaption = '' then + GNextBtnCaption := WizardForm.NextButton.Caption; + WizardForm.NextButton.Caption := StatusText; + WizardForm.NextButton.Enabled := False; + WizardForm.BackButton.Enabled := False; + WizardForm.CancelButton.Enabled := False; + end + else + begin + if GNextBtnCaption <> '' then + WizardForm.NextButton.Caption := GNextBtnCaption; + WizardForm.NextButton.Enabled := True; + WizardForm.BackButton.Enabled := True; + WizardForm.CancelButton.Enabled := True; + end; + WizardForm.Refresh; +end; + +function TestExternalDb(const Host, Port, User, Password, DbName: String; var ErrMsg: String): Boolean; +var + ResultCode: Integer; + UrlFile: String; + ErrFile: String; + PwFile: String; + Params: String; + Lines: TArrayOfString; + i: Integer; +begin + Result := False; + ErrMsg := ''; + if not ExtractPgClientTools() then + begin + ErrMsg := ExpandConstant('{cm:DbTestExtractFailed}'); + Exit; + end; + + UrlFile := ExpandConstant('{tmp}\netx_db_url.txt'); + ErrFile := ExpandConstant('{tmp}\netx_db_test_err.txt'); + PwFile := ExpandConstant('{tmp}\netx_db_pw.txt'); + DeleteFile(UrlFile); + DeleteFile(ErrFile); + SaveStringToFile(PwFile, Password, False); + + Params := + '-NoProfile -ExecutionPolicy Bypass -File "' + ExpandConstant('{tmp}\test_pg_conn.ps1') + '"' + + ' -PgHost "' + Host + '"' + + ' -Port ' + Port + + ' -User "' + User + '"' + + ' -PasswordFile "' + PwFile + '"' + + ' -Database "' + DbName + '"' + + ' -PsqlPath "' + ExpandConstant('{tmp}\psql.exe') + '"' + + ' -OutUrlFile "' + UrlFile + '"' + + ' -OutErrFile "' + ErrFile + '"'; + + if not Exec('powershell.exe', Params, ExpandConstant('{tmp}'), SW_HIDE, ewWaitUntilTerminated, ResultCode) then + begin + ErrMsg := 'powershell_exec_failed'; + DeleteFile(PwFile); + Exit; + end; + DeleteFile(PwFile); + if ResultCode <> 0 then + begin + ErrMsg := 'psql/test exit ' + IntToStr(ResultCode); + if LoadStringsFromFile(ErrFile, Lines) then + begin + ErrMsg := ''; + for i := 0 to GetArrayLength(Lines) - 1 do + begin + if ErrMsg <> '' then + ErrMsg := ErrMsg + #13#10; + ErrMsg := ErrMsg + Lines[i]; + end; + if ErrMsg = '' then + ErrMsg := 'psql/test exit ' + IntToStr(ResultCode); + end; + Exit; + end; + if not LoadStringsFromFile(UrlFile, Lines) or (GetArrayLength(Lines) < 1) or (Trim(Lines[0]) = '') then + begin + ErrMsg := 'url_file_missing'; + Exit; + end; + GDbUrl := Trim(Lines[0]); + Result := True; +end; + +function SaveExternalFieldsFromControls(var ErrMsg: String): Boolean; +begin + Result := False; + GDbHost := Trim(EdHost.Text); + GDbPort := Trim(EdPort.Text); + GDbUser := Trim(EdUser.Text); + GDbPassword := EdPassword.Text; + GDbName := Trim(EdDbName.Text); + GDbConnOk := False; + GDbUrl := ''; + if (GDbHost = '') or (GDbPort = '') or (GDbUser = '') or + (GDbPassword = '') or (GDbName = '') then + begin + ErrMsg := ExpandConstant('{cm:DbFieldsRequired}'); + Exit; + end; + GDbMode := 'external'; + Result := True; +end; + +function ValidateExternalDbSaved(var ErrMsg: String): Boolean; +begin + Result := False; + if GDbConnOk and (GDbUrl <> '') then + begin + Result := True; + Exit; + end; + if not TestExternalDb(GDbHost, GDbPort, GDbUser, GDbPassword, GDbName, ErrMsg) then + begin + if ErrMsg = '' then + ErrMsg := 'connection_failed'; + ErrMsg := FmtMessage(ExpandConstant('{cm:DbTestFailed}'), [ErrMsg]); + GDbConnOk := False; + Exit; + end; + GDbConnOk := True; + Result := True; +end; + +function ApplySilentDbParams(var ErrMsg: String): Boolean; +var + Mode, Host, Port, User, Password, DbName: String; +begin + Result := False; + GCredKey := Trim(ExpandConstant('{param:CredentialSecretKey|}')); + Mode := LowerCase(ExpandConstant('{param:DbMode|bundled}')); + if Mode = '' then + Mode := 'bundled'; + if Mode = 'bundled' then + begin + GDbMode := 'bundled'; + GDbUrl := ''; + Result := True; + Exit; + end; + if Mode <> 'external' then + begin + ErrMsg := 'invalid DbMode (use bundled or external)'; + Exit; + end; + Host := ExpandConstant('{param:DbHost|}'); + Port := ExpandConstant('{param:DbPort|5432}'); + User := ExpandConstant('{param:DbUser|}'); + Password := ExpandConstant('{param:DbPassword|}'); + DbName := ExpandConstant('{param:DbName|}'); + if (Host = '') or (User = '') or (Password = '') or (DbName = '') then + begin + ErrMsg := ExpandConstant('{cm:DbSilentExternalMissing}'); + Exit; + end; + if not TestExternalDb(Host, Port, User, Password, DbName, ErrMsg) then + begin + if ErrMsg = '' then + ErrMsg := 'connection_failed'; + ErrMsg := FmtMessage(ExpandConstant('{cm:DbTestFailed}'), [ErrMsg]); + Exit; + end; + GDbMode := 'external'; + Result := True; +end; + +procedure CaptureCredKeyFromControls(); +begin + if EdCredKey <> nil then + GCredKey := Trim(EdCredKey.Text) + else + GCredKey := ''; +end; + +procedure InitializeWizard(); +var + TopPos: Integer; +begin + GDbMode := 'bundled'; + GDbUrl := ''; + GDbHost := ''; + GDbPort := '5432'; + GDbUser := ''; + GDbPassword := ''; + GDbName := ''; + GCredKey := ''; + GDbConfigured := False; + GPgToolsReady := False; + GDbConnOk := False; + { Existing install: interactive Mode page (Update vs Reinstall). + Silent: /InstallMode=update|reinstall (or /SkipDbPage=1 /ForceDbPage=1). } + GIsUpgrade := DetectExistingConfiguredInstall(); + ModePage := nil; + RbModeUpdate := nil; + RbModeReinstall := nil; + LblModeHint := nil; + if not ResolveInstallModeFromParams() then + begin + if GIsUpgrade then + GInstallMode := 'update' + else + GInstallMode := 'fresh'; + SyncSkipDbFromInstallMode(); + end; + + if GIsUpgrade and (not WizardSilent) then + begin + ModePage := CreateCustomPage(wpSelectDir, + ExpandConstant('{cm:ModePageCaption}'), + ExpandConstant('{cm:ModePageDescription}')); + + RbModeUpdate := TNewRadioButton.Create(ModePage); + RbModeUpdate.Parent := ModePage.Surface; + RbModeUpdate.Caption := ExpandConstant('{cm:ModeUpdate}'); + RbModeUpdate.Checked := (GInstallMode <> 'reinstall'); + RbModeUpdate.Top := ScaleY(8); + RbModeUpdate.Left := ScaleX(0); + RbModeUpdate.Width := ModePage.SurfaceWidth; + RbModeUpdate.OnClick := @ModeChoiceClick; + + RbModeReinstall := TNewRadioButton.Create(ModePage); + RbModeReinstall.Parent := ModePage.Surface; + RbModeReinstall.Caption := ExpandConstant('{cm:ModeReinstall}'); + RbModeReinstall.Checked := (GInstallMode = 'reinstall'); + RbModeReinstall.Top := RbModeUpdate.Top + ScaleY(28); + RbModeReinstall.Left := ScaleX(0); + RbModeReinstall.Width := ModePage.SurfaceWidth; + RbModeReinstall.OnClick := @ModeChoiceClick; + + LblModeHint := TNewStaticText.Create(ModePage); + LblModeHint.Parent := ModePage.Surface; + LblModeHint.Top := RbModeReinstall.Top + ScaleY(36); + LblModeHint.Left := ScaleX(0); + LblModeHint.Width := ModePage.SurfaceWidth; + LblModeHint.Height := ScaleY(60); + LblModeHint.AutoSize := False; + LblModeHint.WordWrap := True; + ModeChoiceClick(nil); + + DbPage := CreateCustomPage(ModePage.ID, + ExpandConstant('{cm:DbPageCaption}'), + ExpandConstant('{cm:DbPageDescription}')); + end + else + DbPage := CreateCustomPage(wpSelectDir, + ExpandConstant('{cm:DbPageCaption}'), + ExpandConstant('{cm:DbPageDescription}')); + + RbBundled := TNewRadioButton.Create(DbPage); + RbBundled.Parent := DbPage.Surface; + RbBundled.Caption := ExpandConstant('{cm:DbBundled}'); + RbBundled.Checked := True; + RbBundled.Top := ScaleY(8); + RbBundled.Left := ScaleX(0); + RbBundled.Width := DbPage.SurfaceWidth; + RbBundled.OnClick := @DbModeClick; + + RbExternal := TNewRadioButton.Create(DbPage); + RbExternal.Parent := DbPage.Surface; + RbExternal.Caption := ExpandConstant('{cm:DbExternal}'); + RbExternal.Top := RbBundled.Top + ScaleY(28); + RbExternal.Left := ScaleX(0); + RbExternal.Width := DbPage.SurfaceWidth; + RbExternal.OnClick := @DbModeClick; + + TopPos := RbExternal.Top + ScaleY(36); + + LblHost := TNewStaticText.Create(DbPage); + LblHost.Parent := DbPage.Surface; + LblHost.Caption := ExpandConstant('{cm:DbHost}'); + LblHost.Top := TopPos; + LblHost.Left := ScaleX(16); + + EdHost := TNewEdit.Create(DbPage); + EdHost.Parent := DbPage.Surface; + EdHost.Top := TopPos; + EdHost.Left := ScaleX(120); + EdHost.Width := ScaleX(220); + EdHost.Text := '127.0.0.1'; + + TopPos := TopPos + ScaleY(28); + LblPort := TNewStaticText.Create(DbPage); + LblPort.Parent := DbPage.Surface; + LblPort.Caption := ExpandConstant('{cm:DbPort}'); + LblPort.Top := TopPos; + LblPort.Left := ScaleX(16); + + EdPort := TNewEdit.Create(DbPage); + EdPort.Parent := DbPage.Surface; + EdPort.Top := TopPos; + EdPort.Left := ScaleX(120); + EdPort.Width := ScaleX(80); + EdPort.Text := '5432'; + + TopPos := TopPos + ScaleY(28); + LblUser := TNewStaticText.Create(DbPage); + LblUser.Parent := DbPage.Surface; + LblUser.Caption := ExpandConstant('{cm:DbUser}'); + LblUser.Top := TopPos; + LblUser.Left := ScaleX(16); + + EdUser := TNewEdit.Create(DbPage); + EdUser.Parent := DbPage.Surface; + EdUser.Top := TopPos; + EdUser.Left := ScaleX(120); + EdUser.Width := ScaleX(220); + EdUser.Text := 'netx'; + + TopPos := TopPos + ScaleY(28); + LblPassword := TNewStaticText.Create(DbPage); + LblPassword.Parent := DbPage.Surface; + LblPassword.Caption := ExpandConstant('{cm:DbPassword}'); + LblPassword.Top := TopPos; + LblPassword.Left := ScaleX(16); + + EdPassword := TNewEdit.Create(DbPage); + EdPassword.Parent := DbPage.Surface; + EdPassword.Top := TopPos; + EdPassword.Left := ScaleX(120); + EdPassword.Width := ScaleX(220); + EdPassword.PasswordChar := '*'; + + TopPos := TopPos + ScaleY(28); + LblDbName := TNewStaticText.Create(DbPage); + LblDbName.Parent := DbPage.Surface; + LblDbName.Caption := ExpandConstant('{cm:DbName}'); + LblDbName.Top := TopPos; + LblDbName.Left := ScaleX(16); + + EdDbName := TNewEdit.Create(DbPage); + EdDbName.Parent := DbPage.Surface; + EdDbName.Top := TopPos; + EdDbName.Left := ScaleX(120); + EdDbName.Width := ScaleX(220); + EdDbName.Text := 'netx'; + + TopPos := TopPos + ScaleY(36); + LblCredKey := TNewStaticText.Create(DbPage); + LblCredKey.Parent := DbPage.Surface; + LblCredKey.Caption := ExpandConstant('{cm:CredKey}'); + LblCredKey.Top := TopPos; + LblCredKey.Left := ScaleX(0); + + EdCredKey := TNewEdit.Create(DbPage); + EdCredKey.Parent := DbPage.Surface; + EdCredKey.Top := TopPos; + EdCredKey.Left := ScaleX(120); + EdCredKey.Width := DbPage.SurfaceWidth - ScaleX(120); + EdCredKey.Text := ''; + + TopPos := TopPos + ScaleY(24); + LblCredHint := TNewStaticText.Create(DbPage); + LblCredHint.Parent := DbPage.Surface; + LblCredHint.Caption := ExpandConstant('{cm:CredKeyHint}'); + LblCredHint.Top := TopPos; + LblCredHint.Left := ScaleX(120); + LblCredHint.Width := DbPage.SurfaceWidth - ScaleX(120); + LblCredHint.AutoSize := False; + + UpdateDbFieldState(); + + { Prefill from existing ProgramData\.env when present. } + if EnvModeIsExternal() then + begin + RbExternal.Checked := True; + UpdateDbFieldState(); + end + else if EnvHasDbMode() then + begin + RbBundled.Checked := True; + UpdateDbFieldState(); + end; + + if WizardSilent then + begin + { Params applied in PrepareToInstall } + end; +end; + +function ShouldSkipPage(PageID: Integer): Boolean; +begin + Result := False; + if (ModePage <> nil) and (PageID = ModePage.ID) then + begin + { Only interactive existing installs show the mode page. } + Result := (not GIsUpgrade) or WizardSilent; + Exit; + end; + if (DbPage <> nil) and (PageID = DbPage.ID) then + Result := GSkipDbPage; +end; + +function NextButtonClick(CurPageID: Integer): Boolean; +var + ErrMsg: String; +begin + Result := True; + if (ModePage <> nil) and (CurPageID = ModePage.ID) then + begin + ApplyInstallModeFromChoice(); + Exit; + end; + { Only the DB page runs validation. Other pages must always proceed + (Tasks/Ready used to appear "dead" when CloseApplications hung). } + if (DbPage = nil) or (CurPageID <> DbPage.ID) or GSkipDbPage then + Exit; + + CaptureCredKeyFromControls(); + if RbBundled.Checked then + begin + GDbMode := 'bundled'; + GDbUrl := ''; + GDbConnOk := True; + Result := True; + Exit; + end; + + if not SaveExternalFieldsFromControls(ErrMsg) then + begin + MsgBox(ErrMsg, mbError, MB_OK); + Result := False; + Exit; + end; + + { Connection test here so bad settings never reach Tasks/Ready. + Show busy state — ExtractTemporaryFile + psql can take a few seconds. } + SetWizardBusy(True, ExpandConstant('{cm:DbTesting}')); + try + Result := ValidateExternalDbSaved(ErrMsg); + finally + SetWizardBusy(False, ''); + end; + if not Result then + MsgBox(ErrMsg, mbError, MB_OK); +end; + +function PrepareToInstall(var NeedsRestart: Boolean): String; +var + ErrMsg: String; +begin + Result := ''; + NeedsRestart := False; + + if (ModePage <> nil) and (not WizardSilent) then + ApplyInstallModeFromChoice(); + + { Stop tray / API / bundled PG so Program Files can be replaced safely. } + if GSkipDbPage or GIsUpgrade or DirExists(ExpandConstant('{app}\packaging')) then + StopNetxBeforeFileReplace(); + + if GSkipDbPage then + Exit; + + if WizardSilent then + begin + if not ApplySilentDbParams(ErrMsg) then + Result := ErrMsg; + Exit; + end; + + if GDbMode = '' then + GDbMode := 'bundled'; + + { Re-check only if user never got a successful test (should be rare). } + if (GDbMode = 'external') and (not GDbConnOk) then + begin + if not ValidateExternalDbSaved(ErrMsg) then + Result := ErrMsg; + end; +end; + +function ApplyDatabaseConfig(): Boolean; +var + ResultCode: Integer; + Params: String; + UrlFile: String; + KeyFile: String; + LogFile: String; + InitDb: String; + EnvFile: String; + Wrapper: String; + WrapBody: String; +begin + Result := False; + GDbConfigured := False; + if GDbMode = '' then + GDbMode := 'bundled'; + + InitDb := ExpandConstant('{app}\postgres\pgsql\bin\initdb.exe'); + if (GDbMode = 'bundled') and (not FileExists(InitDb)) then + begin + MsgBox(FmtMessage(ExpandConstant('{cm:DbBundledMissing}'), [InitDb]), mbError, MB_OK); + Exit; + end; + + ForceDirectories(ExpandConstant('{commonappdata}\NetX\data\runtime')); + LogFile := ExpandConstant('{commonappdata}\NetX\data\runtime\setup_first_run.log'); + Wrapper := ExpandConstant('{tmp}\netx_apply_db.ps1'); + + WrapBody := + '$ErrorActionPreference = ''Stop''' + #13#10 + + '$log = ''' + LogFile + '''' + #13#10 + + 'Start-Transcript -Path $log -Force | Out-Null' + #13#10 + + 'try {' + #13#10 + + ' & ''' + ExpandConstant('{app}\packaging\setup_first_run.ps1') + '''' + + ' -ProgramRoot ''' + ExpandConstant('{app}') + '''' + + ' -DataRoot ''' + ExpandConstant('{commonappdata}\NetX') + '''' + + ' -NonInteractive -DbMode ' + GDbMode; + + if GDbMode = 'external' then + begin + UrlFile := ExpandConstant('{tmp}\netx_db_url.txt'); + if GDbUrl <> '' then + SaveStringToFile(UrlFile, GDbUrl, False); + WrapBody := WrapBody + + ' -ExternalDatabaseUrlFile ''' + UrlFile + ''' -SkipExternalProbe'; + end; + + if GCredKey <> '' then + begin + KeyFile := ExpandConstant('{tmp}\netx_cred_key.txt'); + SaveStringToFile(KeyFile, GCredKey, False); + WrapBody := WrapBody + ' -CredentialSecretKeyFile ''' + KeyFile + ''''; + end; + + WrapBody := WrapBody + #13#10 + + ' if ($null -ne $LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE }' + #13#10 + + '} catch {' + #13#10 + + ' Write-Error $_' + #13#10 + + ' exit 1' + #13#10 + + '} finally {' + #13#10 + + ' Stop-Transcript | Out-Null' + #13#10 + + '}' + #13#10; + + SaveStringToFile(Wrapper, WrapBody, False); + Params := '-NoProfile -ExecutionPolicy Bypass -File "' + Wrapper + '"'; + + if not Exec( + 'powershell.exe', Params, ExpandConstant('{app}'), + SW_HIDE, ewWaitUntilTerminated, ResultCode) then + begin + MsgBox(ExpandConstant('{cm:DbApplyExecFailed}'), mbError, MB_OK); + Exit; + end; + if ResultCode <> 0 then + begin + MsgBox(FmtMessage(ExpandConstant('{cm:DbApplyFailed}'), [IntToStr(ResultCode), LogFile]), mbError, MB_OK); + Exit; + end; + + EnvFile := ExpandConstant('{commonappdata}\NetX\.env'); + if not FileExists(EnvFile) then + begin + MsgBox(ExpandConstant('{cm:DbEnvMissing}'), mbError, MB_OK); + Exit; + end; + + GDbConfigured := True; + Result := True; +end; + +function InitializeUninstall(): Boolean; +var + DeleteSrc: String; +begin + Result := True; + GDeleteNetxData := False; + GTempDeleteDataScript := ExpandConstant('{tmp}\netx_uninstall_delete_data.ps1'); + + // Copy data-wipe helper to TEMP before program files are removed. + DeleteSrc := ExpandConstant('{app}\packaging\uninstall_delete_data.ps1'); + if FileExists(DeleteSrc) then + CopyFile(DeleteSrc, GTempDeleteDataScript, False); + + if UninstallSilent then + begin + // Settings often uses QuietUninstallString; keep data by default. + GDeleteNetxData := False; + end + else + begin + GDeleteNetxData := + MsgBox(ExpandConstant('{cm:UninstallDeleteData}'), mbConfirmation, MB_YESNO) = IDYES; + end; +end; + +procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep); +var + ResultCode: Integer; + DataRoot: String; +begin + if (CurUninstallStep = usPostUninstall) and GDeleteNetxData then + begin + DataRoot := ExpandConstant('{commonappdata}\NetX'); + if FileExists(GTempDeleteDataScript) then + begin + Exec( + 'powershell.exe', + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + + GTempDeleteDataScript + '" -DataRoot "' + DataRoot + '"', + ExpandConstant('{tmp}'), + SW_HIDE, + ewWaitUntilTerminated, + ResultCode + ); + end; + if DirExists(DataRoot) then + DelTree(DataRoot, True, True, True); + end; +end; + +procedure RepairNetxVenvAfterInstall(); +var + ResultCode: Integer; + RepairScript: String; + Params: String; +begin + RepairScript := ExpandConstant('{app}\packaging\repair_venv.ps1'); + if not FileExists(RepairScript) then + Exit; + Params := + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + RepairScript + '"' + + ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + + ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; + Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode); +end; + +procedure CurStepChanged(CurStep: TSetupStep); +var + UninstallKey: String; +begin + if CurStep = ssPostInstall then + begin + { Always relink .venv -> local python/runtime (Setup is elevated). } + RepairNetxVenvAfterInstall(); + + { Upgrade keeps existing .env / pgdata - do not re-run DB wizard. } + if not GSkipDbPage then + ApplyDatabaseConfig() + else + GDbConfigured := FileExists(ExpandConstant('{commonappdata}\NetX\.env')); + + UninstallKey := + 'Software\Microsoft\Windows\CurrentVersion\Uninstall\{A7E3C2D1-9F40-4B8E-9C1A-NETXWIN64001}_is1'; + RegDeleteValue(HKLM, UninstallKey, 'QuietUninstallString'); + end; +end; diff --git a/packaging/installer/test_pg_conn.ps1 b/packaging/installer/test_pg_conn.ps1 new file mode 100644 index 0000000..efc8323 --- /dev/null +++ b/packaging/installer/test_pg_conn.ps1 @@ -0,0 +1,104 @@ +param( + [Parameter(Mandatory = $true)][string]$PgHost, + [int]$Port = 5432, + [Parameter(Mandatory = $true)][string]$User, + [string]$Password = "", + [string]$PasswordFile = "", + [Parameter(Mandatory = $true)][string]$Database, + [Parameter(Mandatory = $true)][string]$PsqlPath, + [string]$OutUrlFile = "", + [string]$OutErrFile = "" +) + +$ErrorActionPreference = "Stop" + +function Write-ErrFile([string]$Message) { + if ($OutErrFile) { + Set-Content -LiteralPath $OutErrFile -Value $Message -Encoding utf8 + } + [Console]::Error.WriteLine($Message) +} + +if (-not (Test-Path -LiteralPath $PsqlPath)) { + Write-ErrFile "psql_not_found: $PsqlPath" + exit 2 +} +if ($PasswordFile) { + if (-not (Test-Path -LiteralPath $PasswordFile)) { + Write-ErrFile "password_file_missing" + exit 3 + } + $Password = [IO.File]::ReadAllText($PasswordFile).TrimEnd("`r", "`n") +} +if ([string]::IsNullOrWhiteSpace($PgHost)) { + Write-ErrFile "host_required" + exit 3 +} +if ([string]::IsNullOrWhiteSpace($User) -or [string]::IsNullOrWhiteSpace($Database)) { + Write-ErrFile "user_and_database_required" + exit 3 +} +if ([string]::IsNullOrWhiteSpace($Password)) { + Write-ErrFile "password_required" + exit 3 +} +if ($Port -lt 1 -or $Port -gt 65535) { + Write-ErrFile "invalid_port: $Port" + exit 3 +} + +$encUser = [uri]::EscapeDataString($User) +$encPw = [uri]::EscapeDataString($Password) +$encDb = [uri]::EscapeDataString($Database) +$url = "postgresql+psycopg://${encUser}:${encPw}@${PgHost}:${Port}/${encDb}" + +$psqlDir = Split-Path -Parent $PsqlPath +$prevPath = $env:PATH +$prevPw = $env:PGPASSWORD +try { + $env:PATH = "$psqlDir;$env:PATH" + $env:PGPASSWORD = $Password + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $PsqlPath + $psi.Arguments = "-h `"$PgHost`" -p $Port -U `"$User`" -d `"$Database`" -v ON_ERROR_STOP=1 -t -A -c `"SELECT 1`"" + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + $psi.WorkingDirectory = $psqlDir + $p = [Diagnostics.Process]::Start($psi) + $stdout = $p.StandardOutput.ReadToEnd() + $stderr = $p.StandardError.ReadToEnd() + $p.WaitForExit() + if ($p.ExitCode -ne 0) { + $msg = (($stderr + "`n" + $stdout).Trim()) + if (-not $msg) { $msg = "psql_exit_$($p.ExitCode)" } + Write-ErrFile "connection_failed: $msg" + exit 1 + } + if (($stdout.Trim()) -notmatch '1') { + Write-ErrFile "unexpected_result: $($stdout.Trim())" + exit 1 + } +} catch { + Write-ErrFile ("connection_failed: " + $_.Exception.Message) + exit 1 +} finally { + $env:PATH = $prevPath + if ($null -eq $prevPw) { + Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue + } else { + $env:PGPASSWORD = $prevPw + } +} + +if ($OutUrlFile) { + $dir = Split-Path -Parent $OutUrlFile + if ($dir -and -not (Test-Path -LiteralPath $dir)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + Set-Content -LiteralPath $OutUrlFile -Value $url -Encoding ascii -NoNewline +} + +Write-Output "ok" +exit 0 diff --git a/packaging/launch_shortcut.ps1 b/packaging/launch_shortcut.ps1 new file mode 100644 index 0000000..f4ae945 --- /dev/null +++ b/packaging/launch_shortcut.ps1 @@ -0,0 +1,127 @@ +param( + [ValidateSet("tray", "start", "stop", "setup", "update")] + [string]$Action = "tray", + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [switch]$StartOnLaunch = $true +) + +# Visible entrypoint for Start Menu / desktop shortcuts. +# Nested PowerShell runs are hidden; only failures show a message box. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +Add-Type -AssemblyName System.Windows.Forms + +$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') +function T([string]$En, [string]$Zh) { + if ($zh) { return $Zh } else { return $En } +} + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$logDir = Join-Path $data "data\runtime" +if (-not (Test-Path $logDir)) { + New-Item -ItemType Directory -Path $logDir -Force | Out-Null +} +$log = Join-Path $logDir "launch.log" + +function Write-LaunchLog([string]$Msg) { + $line = "[{0}] {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Msg + Add-Content -Path $log -Value $line -Encoding utf8 +} + +function Show-NetxError([string]$Msg) { + Write-LaunchLog "ERROR $Msg" + [void][System.Windows.Forms.MessageBox]::Show( + $Msg, + "NetX", + [System.Windows.Forms.MessageBoxButtons]::OK, + [System.Windows.Forms.MessageBoxIcon]::Error + ) +} + +function Invoke-NetxHiddenPs1 { + param( + [string]$File, + [string[]]$ExtraArgs = @(), + [switch]$NoWait = $false + ) + return Start-NetxPowerShell -File $File ` + -Arguments (@("-ProgramRoot", $prog, "-DataRoot", $data) + $ExtraArgs) ` + -WorkingDirectory $prog -WindowStyle Hidden -PassThru -Wait:(-not $NoWait) +} + +function Ensure-NetxEnv { + $envPath = Join-Path $data ".env" + if (Test-Path $envPath) { return } + $cmd = Join-Path $prog "NetX-FirstRun.cmd" + throw (T ` + "NetX is not configured yet (missing $envPath).`nRe-run Setup and choose built-in or external DB,`nor Start Menu → Reconfigure database:`n$cmd" ` + "尚未完成数据库配置（缺少 $envPath）。`n请重新运行安装向导选择内置/外置库，`n或开始菜单 → 重新配置数据库：`n$cmd") +} + +try { + Write-LaunchLog "action=$Action prog=$prog data=$data" + switch ($Action) { + "setup" { + # Visible console so user can pick bundled vs external DB. + $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot "setup_first_run.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` + -WorkingDirectory $prog -WindowStyle Normal -Wait -PassThru + if ($p.ExitCode -ne 0) { throw "setup_exit_$($p.ExitCode)" } + if (Test-Path (Join-Path $data ".env")) { + Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-StartOnLaunch") ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } + } + "stop" { + $p = Invoke-NetxHiddenPs1 -File (Join-Path $PSScriptRoot "stop_netx_app.ps1") + if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0) { throw "stop_exit_$($p.ExitCode)" } + } + "update" { + Ensure-NetxEnv + # Update UI may need a visible window for prompts. + $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot "check_update.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` + -WorkingDirectory $prog -WindowStyle Normal -Wait -PassThru + if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0 -and $p.ExitCode -ne 10) { + throw "update_exit_$($p.ExitCode)" + } + } + "start" { + Ensure-NetxEnv + $hostBind = "127.0.0.1" + $port = 8890 + $envPath = Join-Path $data ".env" + if (Test-Path $envPath) { + $map = Read-DotEnv -Path $envPath + if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } + if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } + } + $p = Invoke-NetxHiddenPs1 -File (Join-Path $PSScriptRoot "start_netx_app.ps1") -ExtraArgs @("-SkipBrowser") + if ($p.ExitCode -ne 0) { throw "start_exit_$($p.ExitCode)" } + if (Wait-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 180) { + try { Start-Process "http://${hostBind}:${port}/" } catch {} + } else { + throw (T ` + "NetX started but /health not ready within 180s.`nSee: $logDir\netx.err.log" ` + "NetX 已启动但 /health 在 180 秒内未就绪。`n请查看: $logDir\netx.err.log") + } + } + "tray" { + Ensure-NetxEnv + $extra = @("-ProgramRoot", $prog, "-DataRoot", $data) + if ($StartOnLaunch) { $extra += "-StartOnLaunch" } + # Detach tray; do not leave a success MessageBox (keeps a console open). + Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` + -Arguments $extra -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } + } + Write-LaunchLog "action=$Action ok" +} catch { + Show-NetxError ((T "NetX failed to start:`n$($_.Exception.Message)" "NetX 启动失败：`n$($_.Exception.Message)") + "`n`n$log") + exit 1 +} diff --git a/packaging/manifest.example.json b/packaging/manifest.example.json index 9f51c14..410b5ee 100644 --- a/packaging/manifest.example.json +++ b/packaging/manifest.example.json @@ -1,11 +1,13 @@ { "channel": "stable", - "latest": "0.3.0", + "latest": "0.4.12", "min_compatible": "0.3.0", - "notes_url": "", + "notes_url": "https://github.com/hansjone/netx/releases/tag/v0.4.12", "windows": { - "url": "https://example.com/netx/NetX-0.2.0-win64.zip", - "sha256": "REPLACE_WITH_SHA256", + "url": "https://github.com/hansjone/netx/releases/download/v0.4.12/NetX-Setup-0.4.12.exe", + "setup_url": "https://github.com/hansjone/netx/releases/download/v0.4.12/NetX-Setup-0.4.12.exe", + "package": "setup", + "sha256": "", "size": 0 } } diff --git a/packaging/netx_tray.ps1 b/packaging/netx_tray.ps1 new file mode 100644 index 0000000..889e65d --- /dev/null +++ b/packaging/netx_tray.ps1 @@ -0,0 +1,747 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [switch]$StartOnLaunch = $true, + [switch]$CheckUpdateOnLaunch = $false, + [switch]$AutoUpdate = $false +) + +# System-tray controller for NetX (Windows packaged installs). + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +Add-Type -AssemblyName System.Windows.Forms +Add-Type -AssemblyName System.Drawing + +# Custom tray menu: hover/selected highlight + flat light chrome (default OS menu looks dead). +if (-not ("NetxMenuRenderer" -as [type])) { + Add-Type -ReferencedAssemblies System.Windows.Forms, System.Drawing -TypeDefinition @" +using System.Drawing; +using System.Drawing.Drawing2D; +using System.Windows.Forms; + +public sealed class NetxMenuColorTable : ProfessionalColorTable { + static readonly Color Hover = Color.FromArgb(232, 240, 252); + static readonly Color Press = Color.FromArgb(214, 228, 248); + static readonly Color Border = Color.FromArgb(170, 198, 240); + static readonly Color Edge = Color.FromArgb(210, 214, 220); + static readonly Color Sep = Color.FromArgb(228, 230, 235); + static readonly Color Bg = Color.FromArgb(252, 252, 253); + + public override Color MenuItemSelected { get { return Hover; } } + public override Color MenuItemSelectedGradientBegin { get { return Hover; } } + public override Color MenuItemSelectedGradientEnd { get { return Hover; } } + public override Color MenuItemBorder { get { return Border; } } + public override Color MenuItemPressedGradientBegin { get { return Press; } } + public override Color MenuItemPressedGradientMiddle { get { return Press; } } + public override Color MenuItemPressedGradientEnd { get { return Press; } } + public override Color MenuBorder { get { return Edge; } } + public override Color ToolStripDropDownBackground { get { return Bg; } } + public override Color ImageMarginGradientBegin { get { return Bg; } } + public override Color ImageMarginGradientMiddle { get { return Bg; } } + public override Color ImageMarginGradientEnd { get { return Bg; } } + public override Color SeparatorDark { get { return Sep; } } + public override Color SeparatorLight { get { return Sep; } } +} + +public sealed class NetxMenuRenderer : ToolStripProfessionalRenderer { + public NetxMenuRenderer() : base(new NetxMenuColorTable()) { + RoundedEdges = false; + } + + protected override void OnRenderMenuItemBackground(ToolStripItemRenderEventArgs e) { + ToolStripMenuItem item = e.Item as ToolStripMenuItem; + if (item == null || !item.Selected || !item.Enabled) { + base.OnRenderMenuItemBackground(e); + return; + } + Rectangle r = new Rectangle(2, 0, e.Item.Width - 4, e.Item.Height); + using (SolidBrush brush = new SolidBrush(Color.FromArgb(232, 240, 252))) + using (Pen pen = new Pen(Color.FromArgb(170, 198, 240))) { + e.Graphics.SmoothingMode = SmoothingMode.AntiAlias; + e.Graphics.FillRectangle(brush, r); + e.Graphics.DrawRectangle(pen, r.X, r.Y, r.Width - 1, r.Height - 1); + } + } + + protected override void OnRenderToolStripBorder(ToolStripRenderEventArgs e) { + Rectangle r = new Rectangle(0, 0, e.AffectedBounds.Width - 1, e.AffectedBounds.Height - 1); + using (Pen pen = new Pen(Color.FromArgb(210, 214, 220))) { + e.Graphics.DrawRectangle(pen, r); + } + } + + protected override void OnRenderSeparator(ToolStripSeparatorRenderEventArgs e) { + int y = e.Item.Height / 2; + using (Pen pen = new Pen(Color.FromArgb(228, 230, 235))) { + e.Graphics.DrawLine(pen, 10, y, e.Item.Width - 10, y); + } + } +} +"@ +} + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$hostBind = "127.0.0.1" +$port = 8890 +$envPath = Join-Path $data ".env" +$map = @{} +if (Test-Path $envPath) { + $map = Read-DotEnv -Path $envPath + if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } + if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } +} +$autoVal = "" +if ($map["NETX_UPDATE_AUTO"]) { $autoVal = $map["NETX_UPDATE_AUTO"] } +elseif ($env:NETX_UPDATE_AUTO) { $autoVal = $env:NETX_UPDATE_AUTO } +if ($autoVal -match '^(1|true|yes|on)$') { + $AutoUpdate = $true + $CheckUpdateOnLaunch = $true +} +$uiUrl = "http://${hostBind}:${port}/" +$script:ver = Get-NetxVersion -ProgramRoot $prog +$ver = $script:ver +$dbMode = Get-DbMode -EnvMap $map +$dbModeLabel = if ($dbMode -eq "bundled") { + if (([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)')) { "内置库" } else { "built-in DB" } +} else { + if (([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)')) { "外置库" } else { "external DB" } +} +$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') +function T([string]$En, [string]$Zh) { + if ($zh) { return $Zh } else { return $En } +} + +if (-not (Test-Path $envPath)) { + [void][System.Windows.Forms.MessageBox]::Show( + (T ` + "NetX is not configured yet (missing $envPath).`nRe-run the installer and choose built-in or external DB,`nor use Start Menu → NetX → Reconfigure database." ` + "尚未完成数据库配置（缺少 $envPath）。`n请重新运行安装向导并选择内置或外置数据库，`n或使用「开始菜单 → NetX → 重新配置数据库」。"), + "NetX", + [System.Windows.Forms.MessageBoxButtons]::OK, + [System.Windows.Forms.MessageBoxIcon]::Warning + ) + exit 1 +} + +# Only one tray icon per machine session (desktop + postinstall + autostart can race). +$script:netxTrayMutex = $null +try { + $createdNew = $false + $script:netxTrayMutex = New-Object System.Threading.Mutex($true, "Local\NetX.WinTray.SingleInstance", [ref]$createdNew) + if (-not $createdNew) { + if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { + try { Start-Process $uiUrl } catch {} + } elseif ($StartOnLaunch) { + Start-NetxPowerShell -File (Join-Path $PSScriptRoot "start_netx_app.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-SkipBrowser") ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } + exit 0 + } +} catch { + # If mutex fails, continue with a tray (better than no UI control). +} + +$startPs1 = Join-Path $PSScriptRoot "start_netx_app.ps1" +$stopPs1 = Join-Path $PSScriptRoot "stop_netx_app.ps1" +$checkPs1 = Join-Path $PSScriptRoot "check_update.ps1" +$setupPs1 = Join-Path $PSScriptRoot "setup_first_run.ps1" + +function Invoke-NetxScript { + param( + [string]$File, + [string[]]$ExtraArgs = @(), + [switch]$Wait = $false + ) + return Start-NetxPowerShell -File $File -Arguments (@("-ProgramRoot", $prog, "-DataRoot", $data) + $ExtraArgs) ` + -WorkingDirectory $prog -WindowStyle Hidden -PassThru -Wait:$Wait +} + +function Update-NetxTrayTip { + # Refresh version after in-place updates (tray process may outlive version.json). + try { + $nowVer = Get-NetxVersion -ProgramRoot $prog + if ($nowVer -and $nowVer -ne $script:ver) { + $script:ver = $nowVer + if ($null -ne $script:miHeader) { $script:miHeader.Text = "NetX $script:ver" } + } + } catch {} + $running = Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 1 + $state = if ($running) { + (T "Running" "运行中") + } else { + (T "Stopped" "已停止") + } + $v = $script:ver + $tip = "NetX $v · $state · $dbModeLabel`n$uiUrl" + if ($tip.Length -gt 63) { + # NotifyIcon.Text max ~63 chars on older Windows. + $tip = "NetX $v · $state · $dbModeLabel" + } + $notify.Text = $tip +} + +function Close-NetxTrayMenu { + # ContextMenuStrip stays modal if Click handlers block; close before dialogs/waits. + try { $menu.Close() } catch {} + try { [System.Windows.Forms.Application]::DoEvents() } catch {} +} + +function Start-NetxTrayDeferred { + param([scriptblock]$Work) + Close-NetxTrayMenu + $timer = New-Object System.Windows.Forms.Timer + $timer.Interval = 50 + $script:netxTrayDeferredWork = $Work + $timer.add_Tick({ + $t = $script:netxTrayDeferredTimer + try { if ($t) { $t.Stop(); $t.Dispose() } } catch {} + $script:netxTrayDeferredTimer = $null + $w = $script:netxTrayDeferredWork + $script:netxTrayDeferredWork = $null + if ($w) { & $w } + }) + $script:netxTrayDeferredTimer = $timer + $timer.Start() +} + +function Start-NetxTrayWatchProcess { + # Never Start-Process -Wait on the WinForms UI thread — it freezes the tray menu. + param( + [Parameter(Mandatory = $true)]$Process, + [scriptblock]$OnExit + ) + if ($null -eq $Process) { + if ($OnExit) { & $OnExit $null } + return + } + $script:netxWatchProc = $Process + $script:netxWatchOnExit = $OnExit + if ($script:netxWatchTimer) { + try { $script:netxWatchTimer.Stop(); $script:netxWatchTimer.Dispose() } catch {} + } + $timer = New-Object System.Windows.Forms.Timer + $timer.Interval = 400 + $timer.add_Tick({ + $p = $script:netxWatchProc + if ($null -eq $p) { + try { $script:netxWatchTimer.Stop(); $script:netxWatchTimer.Dispose() } catch {} + $script:netxWatchTimer = $null + return + } + $done = $false + try { $done = [bool]$p.HasExited } catch { $done = $true } + if (-not $done) { return } + try { $script:netxWatchTimer.Stop(); $script:netxWatchTimer.Dispose() } catch {} + $script:netxWatchTimer = $null + $cb = $script:netxWatchOnExit + $script:netxWatchOnExit = $null + $script:netxWatchProc = $null + if ($cb) { & $cb $p } + }) + $script:netxWatchTimer = $timer + $timer.Start() +} + +function Restart-NetxTraySelf { + # Fresh process picks up new version.json / scripts after in-place update. + try { + $notify.ShowBalloonTip( + 2500, "NetX", + (T "Restarting tray…" "正在重启托盘…"), + [System.Windows.Forms.ToolTipIcon]::Info + ) + } catch {} + $trayFile = Join-Path $PSScriptRoot "netx_tray.ps1" + $arg = "-NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -File `"$trayFile`" -ProgramRoot `"$prog`" -DataRoot `"$data`"" + try { + Start-Process -FilePath "powershell.exe" -ArgumentList $arg -WindowStyle Hidden | Out-Null + } catch {} + try { $notify.Visible = $false } catch {} + try { $form.Close() } catch {} + [System.Windows.Forms.Application]::Exit() +} + +function Test-NetxSetupCancelled { + param([AllowNull()][Nullable[int]]$ExitCode) + if ($null -eq $ExitCode) { return $false } + # STATUS_CONTROL_C_EXIT (0xC000013A): console closed / Ctrl+C — not a setup failure. + return ([int]$ExitCode -eq -1073741510) +} + +function Complete-NetxTrayReconfig { + # Reload .env after reconfigure (host/port/mode may change). + if (Test-Path $envPath) { + $map = Read-DotEnv -Path $envPath + if ($map["NETX_HOST"]) { $script:hostBind = $map["NETX_HOST"]; $hostBind = $script:hostBind } + if ($map["NETX_PORT"]) { + try { + $script:port = [int]$map["NETX_PORT"] + $port = $script:port + } catch {} + } + $script:uiUrl = "http://${hostBind}:${port}/" + $uiUrl = $script:uiUrl + $dbMode = Get-DbMode -EnvMap $map + $script:dbModeLabel = if ($dbMode -eq "bundled") { + (T "built-in DB" "内置库") + } else { + (T "external DB" "外置库") + } + $dbModeLabel = $script:dbModeLabel + if ($miSub) { $miSub.Text = "$dbModeLabel · ${hostBind}:$port" } + } + + $restart = [System.Windows.Forms.MessageBox]::Show( + (T "Database configuration saved.`nStart NetX now?" "数据库配置已保存。`n是否立即启动 NetX？"), + "NetX", + [System.Windows.Forms.MessageBoxButtons]::YesNo, + [System.Windows.Forms.MessageBoxIcon]::Information + ) + if ($restart -eq [System.Windows.Forms.DialogResult]::Yes) { + Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null + Open-NetxUiWhenReady + } else { + Update-NetxTrayTip + } +} + +function Open-NetxUiWhenReady { + param([int]$TimeoutSec = 180) + $notify.ShowBalloonTip( + 5000, + "NetX", + (T "Starting… first run may take a minute." "正在启动…首次迁移可能需要一分钟。"), + [System.Windows.Forms.ToolTipIcon]::Info + ) + # Poll on a timer — do not block the tray UI thread with Wait-NetxApiHealthy. + if ($script:netxUiReadyTimer) { + try { $script:netxUiReadyTimer.Stop(); $script:netxUiReadyTimer.Dispose() } catch {} + } + $script:netxUiReadyTicks = 0 + $script:netxUiReadyMax = [Math]::Max(1, [int]($TimeoutSec * 2)) + $timer = New-Object System.Windows.Forms.Timer + $timer.Interval = 500 + $timer.add_Tick({ + $script:netxUiReadyTicks++ + if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { + try { $script:netxUiReadyTimer.Stop(); $script:netxUiReadyTimer.Dispose() } catch {} + $script:netxUiReadyTimer = $null + try { Start-Process $uiUrl } catch {} + Update-NetxTrayTip + return + } + if ($script:netxUiReadyTicks -ge $script:netxUiReadyMax) { + try { $script:netxUiReadyTimer.Stop(); $script:netxUiReadyTimer.Dispose() } catch {} + $script:netxUiReadyTimer = $null + $notify.ShowBalloonTip( + 8000, + "NetX", + (T "UI not ready yet. Use tray → Open UI when ready, or check data\runtime\netx.err.log." "界面尚未就绪。就绪后请用托盘「打开界面」，或查看 data\runtime\netx.err.log。"), + [System.Windows.Forms.ToolTipIcon]::Warning + ) + Update-NetxTrayTip + } + }) + $script:netxUiReadyTimer = $timer + $timer.Start() +} + +$form = New-Object System.Windows.Forms.Form +$form.Text = "NetX" +$form.ShowInTaskbar = $false +$form.WindowState = "Minimized" +$form.Visible = $false +$form.Size = New-Object System.Drawing.Size(0, 0) + +$notify = New-Object System.Windows.Forms.NotifyIcon +$notify.Visible = $true +$iconPath = Join-Path $PSScriptRoot "assets\netx.ico" +try { + if (Test-Path $iconPath) { + $notify.Icon = New-Object System.Drawing.Icon $iconPath + } else { + $notify.Icon = [System.Drawing.SystemIcons]::Application + } +} catch { + try { $notify.Icon = [System.Drawing.SystemIcons]::Application } catch {} +} +Update-NetxTrayTip + +function New-NetxMenuItem { + param( + [string]$Text, + [switch]$Header, + [switch]$Primary, + [switch]$Muted + ) + $item = New-Object System.Windows.Forms.ToolStripMenuItem + $item.Text = $Text + $item.AutoSize = $true + $item.Padding = New-Object System.Windows.Forms.Padding(10, 5, 28, 5) + $item.Margin = New-Object System.Windows.Forms.Padding(0) + if ($Header) { + $item.Enabled = $false + $item.Font = New-Object System.Drawing.Font("Segoe UI", 9.0, [System.Drawing.FontStyle]::Bold) + $item.ForeColor = [System.Drawing.Color]::FromArgb(55, 65, 80) + $item.Padding = New-Object System.Windows.Forms.Padding(10, 6, 28, 2) + } elseif ($Muted) { + $item.Enabled = $false + $item.Font = New-Object System.Drawing.Font("Segoe UI", 8.25) + $item.ForeColor = [System.Drawing.Color]::FromArgb(120, 128, 140) + $item.Padding = New-Object System.Windows.Forms.Padding(10, 1, 28, 6) + } elseif ($Primary) { + $item.Font = New-Object System.Drawing.Font("Segoe UI", 9.0, [System.Drawing.FontStyle]::Bold) + $item.ForeColor = [System.Drawing.Color]::FromArgb(20, 40, 70) + } else { + $item.Font = New-Object System.Drawing.Font("Segoe UI", 9.0) + $item.ForeColor = [System.Drawing.Color]::FromArgb(35, 40, 48) + } + return $item +} + +function New-NetxMenuSeparator { + $sep = New-Object System.Windows.Forms.ToolStripSeparator + $sep.Margin = New-Object System.Windows.Forms.Padding(0, 3, 0, 3) + $sep.Padding = New-Object System.Windows.Forms.Padding(0) + return $sep +} + +$menu = New-Object System.Windows.Forms.ContextMenuStrip +$menu.ShowImageMargin = $false +$menu.ShowCheckMargin = $false +$menu.Font = New-Object System.Drawing.Font("Segoe UI", 9.0) +$menu.BackColor = [System.Drawing.Color]::FromArgb(252, 252, 253) +$menu.ForeColor = [System.Drawing.Color]::FromArgb(35, 40, 48) +$menu.Padding = New-Object System.Windows.Forms.Padding(4, 4, 4, 4) +$menu.Renderer = New-Object NetxMenuRenderer + +# --- header (info only) --- +$script:miHeader = New-NetxMenuItem -Text "NetX $script:ver" -Header +$miHeader = $script:miHeader +[void]$menu.Items.Add($miHeader) + +$miSub = New-NetxMenuItem -Text "$dbModeLabel · ${hostBind}:$port" -Muted +[void]$menu.Items.Add($miSub) + +[void]$menu.Items.Add((New-NetxMenuSeparator)) + +# --- primary --- +$miOpen = New-NetxMenuItem -Text (T "Open UI" "打开界面") -Primary +$miOpen.add_Click({ + Start-NetxTrayDeferred { + if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { + Start-Process $uiUrl + } else { + $notify.ShowBalloonTip(4000, "NetX", (T "NetX is not running. Starting…" "NetX 未运行，正在启动…"), [System.Windows.Forms.ToolTipIcon]::Info) + Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null + Open-NetxUiWhenReady + } + } +}) +[void]$menu.Items.Add($miOpen) + +[void]$menu.Items.Add((New-NetxMenuSeparator)) + +# --- service control --- +$miStart = New-NetxMenuItem -Text (T "Start" "启动") +$miStart.add_Click({ + Start-NetxTrayDeferred { + Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null + Open-NetxUiWhenReady + } +}) +[void]$menu.Items.Add($miStart) + +$miStop = New-NetxMenuItem -Text (T "Stop" "停止") +$miStop.add_Click({ + Start-NetxTrayDeferred { + $notify.ShowBalloonTip(3000, "NetX", (T "Stopping…" "正在停止…"), [System.Windows.Forms.ToolTipIcon]::Info) + Invoke-NetxScript -File $stopPs1 | Out-Null + Update-NetxTrayTip + } +}) +[void]$menu.Items.Add($miStop) + +[void]$menu.Items.Add((New-NetxMenuSeparator)) + +# --- setup / update --- +$miReconfig = New-NetxMenuItem -Text (T "Reconfigure database…" "重新配置数据库…") +$miReconfig.add_Click({ + Start-NetxTrayDeferred { + $ans = [System.Windows.Forms.MessageBox]::Show( + (T ` + "NetX will stop while you reconfigure the database.`n`nChoose built-in or external PostgreSQL in the console window.`nContinue?" ` + "重新配置数据库时会先停止 NetX。`n`n请在随后的控制台窗口中选择内置或外置 PostgreSQL。`n是否继续？"), + (T "Reconfigure database" "重新配置数据库"), + [System.Windows.Forms.MessageBoxButtons]::YesNo, + [System.Windows.Forms.MessageBoxIcon]::Question + ) + if ($ans -ne [System.Windows.Forms.DialogResult]::Yes) { return } + + $notify.ShowBalloonTip(3000, "NetX", (T "Stopping…" "正在停止…"), [System.Windows.Forms.ToolTipIcon]::Info) + Invoke-NetxScript -File $stopPs1 | Out-Null + + # ProgramData\.env is often admin-owned after Setup; repair ACL before reconfigure. + $null = Ensure-NetxDataAcl -DataRoot $data -Quiet + + try { + $p = Start-NetxPowerShell -File $setupPs1 ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` + -WorkingDirectory $prog -WindowStyle Normal -PassThru + Start-NetxTrayWatchProcess -Process $p -OnExit { + param($sp) + if ($null -eq $sp) { Update-NetxTrayTip; return } + if (Test-NetxSetupCancelled -ExitCode $sp.ExitCode) { + $notify.ShowBalloonTip( + 4000, "NetX", + (T "Database setup cancelled." "已取消数据库配置。"), + [System.Windows.Forms.ToolTipIcon]::Info + ) + Update-NetxTrayTip + return + } + if ($null -ne $sp.ExitCode -and $sp.ExitCode -ne 0) { + $elev = [System.Windows.Forms.MessageBox]::Show( + (T ` + "Database setup failed (exit $($sp.ExitCode)).`n`nIf this was a permission error on %ProgramData%\NetX\.env, click Yes to retry as Administrator." ` + "数据库配置失败（退出码 $($sp.ExitCode)）。`n`n若是 %ProgramData%\NetX\.env 权限问题，点「是」将以管理员身份重试。"), + "NetX", + [System.Windows.Forms.MessageBoxButtons]::YesNo, + [System.Windows.Forms.MessageBoxIcon]::Warning + ) + if ($elev -ne [System.Windows.Forms.DialogResult]::Yes) { + Update-NetxTrayTip + return + } + $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$setupPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`"" + try { + $ep = Start-Process -FilePath "powershell.exe" -ArgumentList $arg ` + -WorkingDirectory $prog -Verb RunAs -PassThru + } catch { + Update-NetxTrayTip + return + } + Start-NetxTrayWatchProcess -Process $ep -OnExit { + param($ep2) + if ($null -eq $ep2 -or (Test-NetxSetupCancelled -ExitCode $ep2.ExitCode)) { + Update-NetxTrayTip + return + } + if ($null -ne $ep2.ExitCode -and $ep2.ExitCode -ne 0) { + [System.Windows.Forms.MessageBox]::Show( + (T "Elevated database setup still failed (exit $($ep2.ExitCode))." "管理员配置仍失败（退出码 $($ep2.ExitCode)）。"), + "NetX", + [System.Windows.Forms.MessageBoxButtons]::OK, + [System.Windows.Forms.MessageBoxIcon]::Error + ) + Update-NetxTrayTip + return + } + Complete-NetxTrayReconfig + } + return + } + Complete-NetxTrayReconfig + } + } catch { + [System.Windows.Forms.MessageBox]::Show( + (T "Database setup failed: $($_.Exception.Message)" "数据库配置失败：$($_.Exception.Message)"), + "NetX", + [System.Windows.Forms.MessageBoxButtons]::OK, + [System.Windows.Forms.MessageBoxIcon]::Error + ) + Update-NetxTrayTip + } + } +}) +[void]$menu.Items.Add($miReconfig) + +$miUpdate = New-NetxMenuItem -Text (T "Check for updates…" "检查更新…") +$miUpdate.add_Click({ + Start-NetxTrayDeferred { + try { + $notify.ShowBalloonTip( + 3000, "NetX", + (T "Checking for updates…" "正在检查更新…"), + [System.Windows.Forms.ToolTipIcon]::Info + ) + # No -Wait on UI thread (freezes ContextMenuStrip / tray). + $p = Start-NetxPowerShell -File $checkPs1 -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` + -WorkingDirectory $prog -WindowStyle Normal -PassThru + Start-NetxTrayWatchProcess -Process $p -OnExit { + param($cp) + try { + if ($null -eq $cp) { return } + if (Test-NetxSetupCancelled -ExitCode $cp.ExitCode) { return } + if ($cp.ExitCode -eq 10) { + $ans = [System.Windows.Forms.MessageBox]::Show( + (T "A newer NetX is available. Download and apply now?" "发现新版本，是否立即下载并更新？"), + (T "NetX update" "NetX 更新"), + [System.Windows.Forms.MessageBoxButtons]::YesNo, + [System.Windows.Forms.MessageBoxIcon]::Question + ) + if ($ans -ne [System.Windows.Forms.DialogResult]::Yes) { return } + $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$checkPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply" + $notify.ShowBalloonTip( + 5000, "NetX", + (T "Downloading / applying update… keep the console open." "正在下载/应用更新…请勿关闭控制台窗口。"), + [System.Windows.Forms.ToolTipIcon]::Info + ) + try { + $ap = Start-Process -FilePath "powershell.exe" -ArgumentList $arg ` + -WorkingDirectory $prog -WindowStyle Normal -Verb RunAs -PassThru + } catch { + $notify.ShowBalloonTip(4000, "NetX", (T "Update cancelled (UAC)." "已取消更新（UAC）。"), [System.Windows.Forms.ToolTipIcon]::Info) + return + } + Start-NetxTrayWatchProcess -Process $ap -OnExit { + param($ap2) + if ($null -eq $ap2) { return } + if (Test-NetxSetupCancelled -ExitCode $ap2.ExitCode) { + $notify.ShowBalloonTip(4000, "NetX", (T "Update cancelled." "已取消更新。"), [System.Windows.Forms.ToolTipIcon]::Info) + return + } + if ($null -ne $ap2.ExitCode -and $ap2.ExitCode -eq 0) { + $newVer = Get-NetxVersion -ProgramRoot $prog + [System.Windows.Forms.MessageBox]::Show( + (T "Updated to $newVer.`nTray will restart." "已更新到 $newVer。`n即将重启托盘。"), + (T "NetX update" "NetX 更新"), + [System.Windows.Forms.MessageBoxButtons]::OK, + [System.Windows.Forms.MessageBoxIcon]::Information + ) + Restart-NetxTraySelf + return + } + [System.Windows.Forms.MessageBox]::Show( + (T "Update failed (exit $($ap2.ExitCode)). Re-run Check for updates, or install NetX-Setup manually." "更新失败（退出码 $($ap2.ExitCode)）。请重试「检查更新」，或手动运行 Setup 安装包。"), + (T "NetX update" "NetX 更新"), + [System.Windows.Forms.MessageBoxButtons]::OK, + [System.Windows.Forms.MessageBoxIcon]::Warning + ) + Update-NetxTrayTip + } + } elseif ($cp.ExitCode -eq 0) { + [System.Windows.Forms.MessageBox]::Show( + (T "NetX is up to date ($script:ver)." "已是最新版本 ($script:ver)。"), + (T "NetX update" "NetX 更新") + ) + } + } catch { + [System.Windows.Forms.MessageBox]::Show( + (T "Update check failed: $($_.Exception.Message)" "检查更新失败：$($_.Exception.Message)"), + "NetX" + ) + } + } + } catch { + [System.Windows.Forms.MessageBox]::Show( + (T "Update check failed: $($_.Exception.Message)" "检查更新失败：$($_.Exception.Message)"), + "NetX" + ) + } + } +}) +[void]$menu.Items.Add($miUpdate) + +[void]$menu.Items.Add((New-NetxMenuSeparator)) + +# --- exit --- +$miExit = New-NetxMenuItem -Text (T "Exit tray" "退出托盘") +$miExit.ToolTipText = (T "Leave NetX services running" "NetX 服务继续运行") +$miExit.add_Click({ + Close-NetxTrayMenu + $notify.Visible = $false + $form.Close() + [System.Windows.Forms.Application]::Exit() +}) +[void]$menu.Items.Add($miExit) + +$miStopExit = New-NetxMenuItem -Text (T "Stop and exit" "停止并退出") +$miStopExit.add_Click({ + Start-NetxTrayDeferred { + $notify.ShowBalloonTip(3000, "NetX", (T "Stopping…" "正在停止…"), [System.Windows.Forms.ToolTipIcon]::Info) + Invoke-NetxScript -File $stopPs1 -Wait | Out-Null + $notify.Visible = $false + $form.Close() + [System.Windows.Forms.Application]::Exit() + } +}) +[void]$menu.Items.Add($miStopExit) + +$notify.ContextMenuStrip = $menu +$notify.add_DoubleClick({ + if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { + Start-Process $uiUrl + } else { + Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null + Open-NetxUiWhenReady + } +}) + +# Refresh tooltip periodically (running / stopped). +$script:tipTimer = New-Object System.Windows.Forms.Timer +$script:tipTimer.Interval = 5000 +$script:tipTimer.add_Tick({ Update-NetxTrayTip }) +$script:tipTimer.Start() + +$form.add_Shown({ + if ($AutoUpdate) { + $notify.ShowBalloonTip(4000, "NetX", (T "Checking for updates…" "正在检查更新…"), [System.Windows.Forms.ToolTipIcon]::Info) + Start-NetxPowerShell -File $checkPs1 ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-Apply", "-Quiet", "-AutoOnly") ` + -WorkingDirectory $prog -WindowStyle Hidden -Wait | Out-Null + } elseif ($CheckUpdateOnLaunch) { + Start-NetxPowerShell -File $checkPs1 ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-Quiet") ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } + if ($StartOnLaunch) { + Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null + $script:netxUiWaitTicks = 0 + $script:netxUiWaitMax = 360 # 360 * 500ms = 180s + $script:netxUiTimer = New-Object System.Windows.Forms.Timer + $script:netxUiTimer.Interval = 500 + $script:netxUiTimer.add_Tick({ + $script:netxUiWaitTicks++ + if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { + $script:netxUiTimer.Stop() + $script:netxUiTimer.Dispose() + try { Start-Process $uiUrl } catch {} + Update-NetxTrayTip + return + } + if ($script:netxUiWaitTicks -eq 1) { + $notify.ShowBalloonTip( + 5000, + "NetX", + (T "Starting… first run may take a minute." "正在启动…首次迁移可能需要一分钟。"), + [System.Windows.Forms.ToolTipIcon]::Info + ) + } + if ($script:netxUiWaitTicks -ge $script:netxUiWaitMax) { + $script:netxUiTimer.Stop() + $script:netxUiTimer.Dispose() + $notify.ShowBalloonTip( + 8000, + "NetX", + (T "UI not ready yet. Use tray → Open UI later, or check data\runtime\netx.err.log." "界面尚未就绪。请稍后用托盘「打开界面」，或查看 data\runtime\netx.err.log。"), + [System.Windows.Forms.ToolTipIcon]::Warning + ) + Update-NetxTrayTip + } + }) + $script:netxUiTimer.Start() + } +}) + +$form.add_FormClosing({ + try { $script:tipTimer.Stop(); $script:tipTimer.Dispose() } catch {} + $notify.Visible = $false + $notify.Dispose() +}) + +[System.Windows.Forms.Application]::Run($form) diff --git a/packaging/publish_forgejo_release.ps1 b/packaging/publish_forgejo_release.ps1 new file mode 100644 index 0000000..0d633af --- /dev/null +++ b/packaging/publish_forgejo_release.ps1 @@ -0,0 +1,129 @@ +param( + [string]$Version = "", + [string]$ForgejoBase = "https://git.avelo.top", + [string]$Owner = "hansjone", + [string]$Repo = "netx", + [string]$Token = "", + [string]$ReleaseDir = "" +) + +# Publish Windows Setup.exe to Forgejo/Gitea (mirror sync does NOT copy GitHub Release files). +# Requires a Forgejo token with repo write (Settings → Applications → Generate New Token). +# Default: https://git.avelo.top (public domain). LAN IP only when explicitly reachable. +# +# Example: +# $env:NETX_FORGEJO_TOKEN = "..." # or User env NETX_FORGEJO_TOKEN +# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 +# +# Optional LAN (when 10.0.0.131 is reachable): +# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 -ForgejoBase "http://10.0.0.131:3000" + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$repoRoot = Get-NetxRepoRoot +if (-not $Version) { + $Version = Get-NetxVersion -ProgramRoot $repoRoot +} +$tag = "v$Version" +$relDir = if ($ReleaseDir) { $ReleaseDir } else { Join-Path $PSScriptRoot "release" } +$setup = Join-Path $relDir "NetX-Setup-$Version.exe" + +if (-not $Token) { + $Token = $env:NETX_FORGEJO_TOKEN + if (-not $Token) { $Token = $env:FORGEJO_TOKEN } +} +if (-not $Token) { + throw "forgejo_token_required: set NETX_FORGEJO_TOKEN or pass -Token" +} +if (-not (Test-Path $setup)) { + throw "missing_setup: $setup (run build_release.ps1 + ISCC first)" +} + +$apiBase = "$ForgejoBase/api/v1/repos/$Owner/$Repo" +$headers = @{ + "Authorization" = "token $Token" + "Accept" = "application/json" +} + +function Invoke-ForgejoJson { + param( + [string]$Method, + [string]$Url, + [object]$Body = $null + ) + $params = @{ + Method = $Method + Uri = $Url + Headers = $headers + ContentType = "application/json" + } + if ($null -ne $Body) { + $params["Body"] = ($Body | ConvertTo-Json -Depth 5) + } + return Invoke-RestMethod @params +} + +Write-Host "==> Forgejo publish $tag -> $ForgejoBase/$Owner/$Repo" + +# Ensure git tag exists on Forgejo (mirror usually already has it). +try { + $null = Invoke-ForgejoJson -Method GET -Url "$apiBase/git/refs/tags/$tag" + Write-Host " tag $tag present on Forgejo" +} catch { + Write-Host "[WARN] tag $tag not found on Forgejo yet — run mirror-sync or push tag first" -ForegroundColor Yellow +} + +$existing = $null +try { + $existing = Invoke-ForgejoJson -Method GET -Url "$apiBase/releases/tags/$tag" +} catch { + $existing = $null +} + +$notes = @" +NetX $Version Windows installer (published from GitHub release build). + +- NetX-Setup-$Version.exe +"@ + +if ($existing -and $existing.id) { + Write-Host "==> Release $tag already exists (id=$($existing.id)); deleting old release to re-upload assets" + Invoke-ForgejoJson -Method DELETE -Url "$apiBase/releases/$($existing.id)" | Out-Null +} + +Write-Host "==> Creating release $tag" +$rel = Invoke-ForgejoJson -Method POST -Url "$apiBase/releases" -Body @{ + tag_name = $tag + target = "main" + name = "NetX $Version" + body = $notes + draft = $false + prerelease = $false +} +$relId = $rel.id +if (-not $relId) { throw "forgejo_create_release_failed" } + +function Upload-ForgejoAsset { + param([string]$Path) + if (-not (Test-Path $Path)) { return } + $name = Split-Path -Leaf $Path + Write-Host "==> Uploading $name ..." + $url = "$apiBase/releases/$relId/assets" + $curl = Get-Command curl.exe -ErrorAction SilentlyContinue + if (-not $curl) { throw "curl.exe required for asset upload" } + # --ssl-no-revoke: schannel CRYPT_E_REVOCATION_OFFLINE often fails via proxy/offline CA. + & $curl.Source -f -sS --ssl-no-revoke -X POST ` + -H "Authorization: token $Token" ` + -F "attachment=@$Path" ` + $url + if ($LASTEXITCODE -ne 0) { throw "upload_failed: $name" } + Write-Host " OK $name" -ForegroundColor Green +} + +Upload-ForgejoAsset -Path $setup + +$releaseUrl = "$ForgejoBase/$Owner/$Repo/releases/tag/$tag" +Write-Host "" +Write-Host "==> Forgejo release ready: $releaseUrl" -ForegroundColor Green +Write-Host " Update API: $ForgejoBase/api/v1/repos/$Owner/$Repo/releases/latest" diff --git a/packaging/publish_release.ps1 b/packaging/publish_release.ps1 index 193c36c..122df57 100644 --- a/packaging/publish_release.ps1 +++ b/packaging/publish_release.ps1 @@ -1,9 +1,10 @@ -param( +param( [string]$Version = "", [switch]$SkipBuild = $false, [switch]$SkipInstaller = $false, [switch]$SkipGitHub = $false, - [switch]$Draft = $false + [switch]$Draft = $false, + [switch]$Sign = $false ) $ErrorActionPreference = "Stop" @@ -15,7 +16,6 @@ if (-not $Version) { } $tag = "v$Version" $releaseDir = Join-Path $PSScriptRoot "release" -$zip = Join-Path $releaseDir "NetX-$Version-win64.zip" $setup = Join-Path $releaseDir "NetX-Setup-$Version.exe" Write-Host "==> NetX publish $tag" @@ -25,29 +25,33 @@ if (-not $SkipBuild) { -Version $Version -CreateVenv } -if (-not (Test-Path $zip)) { - throw "missing_zip: $zip" -} - if (-not $SkipInstaller) { $isccCmd = Get-Command ISCC.exe -ErrorAction SilentlyContinue $isccCandidates = @( $(if ($isccCmd) { $isccCmd.Source }), + "$env:LOCALAPPDATA\Programs\Inno Setup 6\ISCC.exe", "${env:ProgramFiles(x86)}\Inno Setup 6\ISCC.exe", "$env:ProgramFiles\Inno Setup 6\ISCC.exe" ) | Where-Object { $_ -and (Test-Path $_) } $iscc = $isccCandidates | Select-Object -First 1 if (-not $iscc) { - Write-Host "[WARN] Inno Setup (ISCC) not found. Install: winget install JRSoftware.InnoSetup" -ForegroundColor Yellow - Write-Host " Skipping Setup.exe; zip-only release." - } else { - Write-Host "==> Compiling installer: $iscc" - & $iscc "/DMyAppVersion=$Version" (Join-Path $PSScriptRoot "installer\netx.iss") - if (-not (Test-Path $setup)) { - throw "installer_build_failed: expected $setup" - } - Write-Host "==> Setup.exe: $setup" -ForegroundColor Green + throw "innosetup_not_found: install with winget install JRSoftware.InnoSetup" } + Write-Host "==> Compiling installer: $iscc" + & $iscc "/DMyAppVersion=$Version" (Join-Path $PSScriptRoot "installer\netx.iss") + if (-not (Test-Path $setup)) { + throw "installer_build_failed: expected $setup" + } + Write-Host "==> Setup.exe: $setup" -ForegroundColor Green +} + +if (-not (Test-Path $setup)) { + throw "missing_setup: $setup" +} + +if ($Sign) { + Write-Host "==> Signing release artifacts" + & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "sign_release.ps1") -Files @($setup) } if ($SkipGitHub) { @@ -60,30 +64,32 @@ if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { } Set-Location $repo +# gh writes "release not found" to stderr; keep Stop for the rest of the script. +$prevEap = $ErrorActionPreference +$ErrorActionPreference = "Continue" $existing = gh release view $tag 2>$null -if ($LASTEXITCODE -eq 0) { +$viewCode = $LASTEXITCODE +$ErrorActionPreference = $prevEap +if ($viewCode -eq 0) { Write-Host "==> Release $tag exists; uploading assets" - gh release upload $tag $zip --clobber - if (Test-Path $setup) { - gh release upload $tag $setup --clobber - } + gh release upload $tag $setup --clobber } else { $notes = @" -## NetX $Version — first Windows installable release +## NetX $Version ### Downloads -- **NetX-Setup-$Version.exe** — recommended installer (Program Files + ProgramData) -- **NetX-$Version-win64.zip** — portable directory package +- **NetX-Setup-$Version.exe** — Windows installer (Program Files + ProgramData) ### Requirements -- Windows 10/11 x64 -- No separate Python or PostgreSQL install required (bundled in package) +- Windows 10/11 x64 (or Windows Server 2016+) +- No separate Python or PostgreSQL install required (bundled) -### Quick start (installer) +### Quick start 1. Run ``NetX-Setup-$Version.exe`` as administrator. -2. Complete first-time setup (choose **bundled** or **external** PostgreSQL). -3. Start menu → **Start NetX** → browser opens ``http://127.0.0.1:8890/`` -4. Default login: ``admin`` / ``admin123`` (change after first login) +2. **First install:** choose built-in or external PostgreSQL. +3. **Already installed:** Setup detects existing data and updates in place (keeps DB settings). +4. Start menu → **Start NetX** → ``http://127.0.0.1:8890/`` +5. Default login: ``admin`` / ``admin123`` (change after first login) ### Linux Unchanged — use your own PostgreSQL and ``scripts/start_netx.sh``. @@ -92,8 +98,7 @@ See [packaging/README.md](https://github.com/hansjone/netx/blob/main/packaging/R "@ $args = @("release", "create", $tag, "--title", "NetX $Version", "--notes", $notes) if ($Draft) { $args += "--draft" } - $args += $zip - if (Test-Path $setup) { $args += $setup } + $args += $setup & gh @args } diff --git a/packaging/repair_venv.ps1 b/packaging/repair_venv.ps1 new file mode 100644 index 0000000..4172d3a --- /dev/null +++ b/packaging/repair_venv.ps1 @@ -0,0 +1,25 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "" +) + +# Relink shipped .venv to local python/runtime (run elevated after Setup/update). +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot + +Write-Host "==> Repairing .venv under $prog" +if (Repair-NetxShippedVenv -ProgramRoot $prog) { + $venvPy = Join-Path $prog ".venv\Scripts\python.exe" + if (Test-NetxVenvRunnable -VenvPython $venvPy) { + Write-Host "==> .venv OK -> bundled python/runtime" -ForegroundColor Green + Get-Content (Join-Path $prog ".venv\pyvenv.cfg") + exit 0 + } + Write-Host "[ERR] pyvenv.cfg written but venv still not runnable" -ForegroundColor Red + exit 2 +} +Write-Host "[ERR] repair failed (missing runtime/.venv or access denied)" -ForegroundColor Red +exit 1 diff --git a/packaging/service_run.ps1 b/packaging/service_run.ps1 new file mode 100644 index 0000000..e514352 --- /dev/null +++ b/packaging/service_run.ps1 @@ -0,0 +1,100 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [int]$HealthFailLimit = 6, + [int]$HealthIntervalSec = 10 +) + +# Long-running supervisor for Windows Service / Task Scheduler. +# Starts NetX, probes /health; repeated failures trigger restart (or exit for WinSW). + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$logDir = Join-Path $data "data\runtime" +if (-not (Test-Path $logDir)) { + New-Item -ItemType Directory -Path $logDir -Force | Out-Null +} +$logFile = Join-Path $logDir "service_run.log" +$stopFlag = Join-Path $logDir "service.stop" + +function Write-SvcLog([string]$Msg) { + $line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Msg + Add-Content -Path $logFile -Value $line -Encoding utf8 + Write-Host $line +} + +function Get-BindInfo { + $envPath = Join-Path $data ".env" + $hostBind = "127.0.0.1" + $port = 8890 + if (Test-Path $envPath) { + $map = Read-DotEnv -Path $envPath + if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } + if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } + } + return @{ Host = $hostBind; Port = $port } +} + +Remove-Item -Force $stopFlag -ErrorAction SilentlyContinue +Write-SvcLog "service_run starting program=$prog data=$data" + +$startPs1 = Join-Path $PSScriptRoot "start_netx_app.ps1" +$stopPs1 = Join-Path $PSScriptRoot "stop_netx_app.ps1" +$bind = Get-BindInfo +$failStreak = 0 +$restartCount = 0 + +function Start-NetxChildren { + & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startPs1 ` + -ProgramRoot $prog -DataRoot $data -SkipBrowser -Force + if ($LASTEXITCODE -ne 0) { + throw "start_netx_app_failed exit=$LASTEXITCODE" + } +} + +function Stop-NetxChildren { + & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $stopPs1 ` + -ProgramRoot $prog -DataRoot $data +} + +try { + Start-NetxChildren + Write-SvcLog "NetX started; health watch host=$($bind.Host) port=$($bind.Port)" + + while ($true) { + if (Test-Path $stopFlag) { + Write-SvcLog "stop flag detected" + break + } + + if (Test-NetxApiHealthy -HostName $bind.Host -Port $bind.Port -TimeoutSec 3) { + $failStreak = 0 + } else { + $failStreak++ + Write-SvcLog "health fail streak=$failStreak/$HealthFailLimit" + if ($failStreak -ge $HealthFailLimit) { + $restartCount++ + Write-SvcLog "restarting NetX (attempt=$restartCount)" + try { Stop-NetxChildren } catch { Write-SvcLog "stop warning: $($_.Exception.Message)" } + Start-Sleep -Seconds 3 + Start-NetxChildren + $failStreak = 0 + # Give API time to come up before counting failures again. + Start-Sleep -Seconds ([Math]::Max(15, $HealthIntervalSec)) + continue + } + } + Start-Sleep -Seconds $HealthIntervalSec + } +} catch { + Write-SvcLog "ERROR: $($_.Exception.Message)" + throw +} finally { + Write-SvcLog "stopping NetX" + try { Stop-NetxChildren } catch { Write-SvcLog "stop warning: $($_.Exception.Message)" } + Remove-Item -Force $stopFlag -ErrorAction SilentlyContinue + Write-SvcLog "service_run exited" +} diff --git a/packaging/setup_first_run.ps1 b/packaging/setup_first_run.ps1 index fa8e0f5..8c58bf5 100644 --- a/packaging/setup_first_run.ps1 +++ b/packaging/setup_first_run.ps1 @@ -1,12 +1,16 @@ -param( +param( [ValidateSet("bundled", "external", "")] [string]$DbMode = "", [string]$ProgramRoot = "", [string]$DataRoot = "", [string]$ExternalDatabaseUrl = "", + [string]$ExternalDatabaseUrlFile = "", + [string]$CredentialSecretKey = "", + [string]$CredentialSecretKeyFile = "", [string]$BundledPassword = "", [int]$BundledPort = 15432, - [switch]$NonInteractive = $false + [switch]$NonInteractive = $false, + [switch]$SkipExternalProbe = $false ) $ErrorActionPreference = "Stop" @@ -15,20 +19,19 @@ $ErrorActionPreference = "Stop" $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot $envPath = Join-Path $data ".env" +$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') -Write-Host "==> Program root: $prog" -Write-Host "==> Data root: $data" -Write-Host "==> Env file: $envPath" +function T([string]$En, [string]$Zh) { + if ($zh) { return $Zh } else { return $En } +} -if (-not (Test-Path $data)) { - New-Item -ItemType Directory -Path $data -Force | Out-Null -} -foreach ($sub in @("data", "data\auth", "data\runtime", "backups", "pgdata")) { - $p = Join-Path $data $sub - if (-not (Test-Path $p)) { - New-Item -ItemType Directory -Path $p -Force | Out-Null - } -} +try { + +Write-Host ("==> " + (T "Program root:" "程序目录:") + " $prog") +Write-Host ("==> " + (T "Data root:" "数据目录:") + " $data") +Write-Host ("==> " + (T "Env file:" "环境文件:") + " $envPath") + +Ensure-NetxDataDirectories -DataRoot $data $existing = Read-DotEnv -Path $envPath if (-not $DbMode) { @@ -42,10 +45,11 @@ if (-not $DbMode) { } } else { Write-Host "" - Write-Host "Choose database mode:" - Write-Host " 1) bundled — use NetX portable PostgreSQL (default for new installs)" - Write-Host " 2) external — connect to an existing PostgreSQL (Linux / already deployed)" - $choice = Read-Host "Enter 1 or 2" + Write-Host (T "Choose database mode:" "选择数据库模式:") -ForegroundColor Cyan + Write-Host (T " 1) bundled — NetX portable PostgreSQL (offline / default)" " 1) bundled — NetX 内置便携 PostgreSQL（可离线，默认）") + Write-Host (T " 2) external — existing PostgreSQL (you provide connection URL)" " 2) external — 已有外置 PostgreSQL（需填写连接串）") + Write-Host "" + $choice = Read-Host (T "Enter 1 or 2" "请输入 1 或 2") if ($choice -eq "2") { $DbMode = "external" } else { $DbMode = "bundled" } } } @@ -58,28 +62,44 @@ $values = @{} $values["NETX_DB_MODE"] = $DbMode $values["NETX_HOST"] = "127.0.0.1" $values["NETX_PORT"] = "8890" -$values["NETX_UI_DIST_DIR"] = "web/dist" +# Absolute UI path when present; else relative for source trees. +$distAbs = Join-Path $prog "web\dist" +if (Test-Path (Join-Path $distAbs "index.html")) { + $values["NETX_UI_DIST_DIR"] = (ConvertTo-NetxFsPath $distAbs) +} else { + $values["NETX_UI_DIST_DIR"] = "web/dist" +} -# Point runtime data dirs into the data root (absolute). -$values["NETX_AUTH_MCP_TOKEN_FILE"] = ((Join-Path $data "data\auth\mcp_token") -replace '\\', '/') -$values["NETX_SCHEDULER_HEARTBEAT_PATH"] = ((Join-Path $data "data\runtime\scheduler_heartbeat.json") -replace '\\', '/') +# All runtime data under data root (never under Program Files). +$dataEnv = Get-NetxDataEnvMap -DataRoot $data +foreach ($k in $dataEnv.Keys) { $values[$k] = $dataEnv[$k] } + +# Preload credential key from file/CLI only; interactive prompt comes AFTER DB settings +# so users are not left thinking the key alone finished setup. +if ($CredentialSecretKeyFile) { + if (-not (Test-Path -LiteralPath $CredentialSecretKeyFile)) { + throw "credential_secret_key_file_missing: $CredentialSecretKeyFile" + } + $CredentialSecretKey = [IO.File]::ReadAllText($CredentialSecretKeyFile).Trim() +} if ($DbMode -eq "bundled") { $pgsql = Join-Path $prog "postgres\pgsql" if (-not (Test-Path (Join-Path $pgsql "bin\initdb.exe"))) { - # In-repo layout $alt = Join-Path $PSScriptRoot "postgres\pgsql" if (Test-Path (Join-Path $alt "bin\initdb.exe")) { $pgsql = $alt } else { - throw "bundled_postgres_missing: run packaging\download_postgres.ps1 first (expected $pgsql)" + throw (T ` + "bundled_postgres_missing: incomplete package (expected $pgsql). Offline Setup must include postgres; rebuild with build_release.ps1 on a machine that already ran download_postgres.ps1." ` + "缺少内置 PostgreSQL（期望路径 $pgsql）。离线安装包必须自带数据库；请在已运行过 download_postgres.ps1 的机器上重新 build_release。") } } if (-not $BundledPassword) { if ($NonInteractive) { $BundledPassword = -join ((48..57) + (65..90) + (97..122) | Get-Random -Count 24 | ForEach-Object { [char]$_ }) } else { - $sec = Read-Host -Prompt "Password for bundled role 'netx' (empty = auto-generate)" -AsSecureString + $sec = Read-Host -Prompt (T "Password for bundled role 'netx' (empty = auto-generate)" "内置数据库用户 netx 的密码（回车=自动生成）") -AsSecureString $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($sec) try { $BundledPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto($bstr) @@ -88,24 +108,22 @@ if ($DbMode -eq "bundled") { } if (-not $BundledPassword) { $BundledPassword = -join ((48..57) + (65..90) + (97..122) | Get-Random -Count 24 | ForEach-Object { [char]$_ }) - Write-Host "Generated password (saved in .env only)." + Write-Host (T "Generated password (saved in .env only)." "已自动生成密码（仅保存在 .env）。") } } } $pgData = Join-Path $data "pgdata" $values["NETX_BUNDLED_PG_PORT"] = "$BundledPort" - $values["NETX_BUNDLED_PG_DATA_DIR"] = ($pgData -replace '\\', '/') + $values["NETX_BUNDLED_PG_DATA_DIR"] = (ConvertTo-NetxFsPath $pgData) $pwFile = Join-Path $data "pg_netx.pw" Set-Content -Path $pwFile -Value $BundledPassword -Encoding ascii -NoNewline $encPw = [uri]::EscapeDataString($BundledPassword) $values["NETX_DATABASE_URL"] = "postgresql+psycopg://netx:${encPw}@127.0.0.1:${BundledPort}/netx" - # Initialize cluster if needed $initdb = Join-Path $pgsql "bin\initdb.exe" $pgCtl = Join-Path $pgsql "bin\pg_ctl.exe" $psql = Join-Path $pgsql "bin\psql.exe" - $createdb = Join-Path $pgsql "bin\createdb.exe" - $createuser = Join-Path $pgsql "bin\createuser.exe" + $sqlPw = ConvertTo-NetxSqlLiteral $BundledPassword if (-not (Test-Path (Join-Path $pgData "PG_VERSION"))) { Write-Host "==> initdb $pgData" @@ -115,7 +133,6 @@ if ($DbMode -eq "bundled") { if ($LASTEXITCODE -ne 0) { throw "initdb_failed" } } - # Ensure listen / port in postgresql.conf $conf = Join-Path $pgData "postgresql.conf" $hba = Join-Path $pgData "pg_hba.conf" if (Test-Path $conf) { @@ -126,6 +143,8 @@ if ($DbMode -eq "bundled") { $confText = $confText -replace '(?m)^\s*port\s*=\s*\d+', "port = $BundledPort" if ($confText -notmatch "(?m)^\s*listen_addresses\s*=") { $confText += "`nlisten_addresses = '127.0.0.1'`n" + } else { + $confText = $confText -replace "(?m)^\s*listen_addresses\s*=\s*'[^']*'", "listen_addresses = '127.0.0.1'" } Set-Content -Path $conf -Value $confText -Encoding utf8 } @@ -137,49 +156,216 @@ if ($DbMode -eq "bundled") { } } - Write-Host "==> Starting bundled PostgreSQL for bootstrap" - & $pgCtl -D $pgData -l (Join-Path $data "pgdata\pg.log") start - Start-Sleep -Seconds 2 + $status = & $pgCtl -D $pgData status 2>&1 | Out-String + if ($status -notmatch "server is running") { + if (-not (Test-NetxTcpPortFree -HostName "127.0.0.1" -Port $BundledPort)) { + throw "port_in_use: 127.0.0.1:$BundledPort already occupied (bundled Postgres)" + } + Write-Host "==> Starting bundled PostgreSQL for bootstrap" + & $pgCtl -D $pgData -l (Join-Path $pgData "pg.log") start + if ($LASTEXITCODE -ne 0) { throw "pg_start_failed" } + Start-Sleep -Seconds 2 + } $env:PGPASSWORD = $BundledPassword try { - $role = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='netx'" - if ($role -notmatch "1") { - & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` - -c "CREATE ROLE netx LOGIN PASSWORD '$BundledPassword';" + function Invoke-NetxPsql { + param([string]$Sql, [string]$Database = "postgres") + $out = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d $Database -v ON_ERROR_STOP=1 -tAc $Sql 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "psql_failed ($LASTEXITCODE): $Sql`n$out" + } + return (($out | Out-String).Trim()) + } + + $role = Invoke-NetxPsql -Sql "SELECT 1 FROM pg_roles WHERE rolname='netx'" + if ($role -eq "1") { + Invoke-NetxPsql -Sql "ALTER ROLE netx WITH LOGIN PASSWORD '$sqlPw'" | Out-Null } else { - & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` - -c "ALTER ROLE netx WITH LOGIN PASSWORD '$BundledPassword';" + Invoke-NetxPsql -Sql "CREATE ROLE netx LOGIN PASSWORD '$sqlPw'" | Out-Null } - $db = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='netx'" - if ($db -notmatch "1") { - & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` - -c "CREATE DATABASE netx OWNER netx;" + $db = Invoke-NetxPsql -Sql "SELECT 1 FROM pg_database WHERE datname='netx'" + if ($db -ne "1") { + Invoke-NetxPsql -Sql "CREATE DATABASE netx OWNER netx" | Out-Null + } + Invoke-NetxPsql -Sql "GRANT ALL PRIVILEGES ON DATABASE netx TO netx" | Out-Null + # Verify login as app role (catches auth/hba mismatches early). + $prev = $env:PGPASSWORD + $env:PGPASSWORD = $BundledPassword + try { + $ping = & $psql -h 127.0.0.1 -p $BundledPort -U netx -d netx -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>&1 + if ($LASTEXITCODE -ne 0 -or (($ping | Out-String).Trim()) -ne "1") { + throw "netx_role_login_failed: $ping" + } + } finally { + $env:PGPASSWORD = $prev } - & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` - -c "GRANT ALL PRIVILEGES ON DATABASE netx TO netx;" } finally { Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue } Write-Host "==> Bundled Postgres ready on 127.0.0.1:$BundledPort" -ForegroundColor Green } else { + if ($ExternalDatabaseUrlFile) { + if (-not (Test-Path -LiteralPath $ExternalDatabaseUrlFile)) { + throw "external_url_file_missing: $ExternalDatabaseUrlFile" + } + $ExternalDatabaseUrl = [IO.File]::ReadAllText($ExternalDatabaseUrlFile).Trim() + } if (-not $ExternalDatabaseUrl) { - if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { - $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] - } elseif ($NonInteractive) { - throw "external_url_required" + if ($NonInteractive) { + if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { + $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] + } else { + throw "external_url_required" + } } else { - $ExternalDatabaseUrl = Read-Host "NETX_DATABASE_URL (postgresql+psycopg://user:pass@host:5432/netx)" + # Interactive: always ask. Empty = keep existing URL (never silent). + Write-Host "" + Write-Host (T ` + "Enter PostgreSQL URL (database/role must already exist):" ` + "请输入 PostgreSQL 连接串（库和用户需事先建好）:") -ForegroundColor Cyan + Write-Host " postgresql+psycopg://USER:PASSWORD@HOST:5432/DBNAME" + Write-Host (T ` + "Example: postgresql+psycopg://netx:secret@10.0.0.8:5432/netx" ` + "示例: postgresql+psycopg://netx:secret@10.0.0.8:5432/netx") + if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { + Write-Host (T ` + "Current: $($existing['NETX_DATABASE_URL'])" ` + "当前: $($existing['NETX_DATABASE_URL'])") -ForegroundColor DarkGray + Write-Host (T ` + "Press Enter to keep the current URL, or paste a new one." ` + "直接回车 = 保留当前连接串；或粘贴新的连接串。") + } + $entered = (Read-Host "NETX_DATABASE_URL").Trim() + if ($entered) { + $ExternalDatabaseUrl = $entered + } elseif ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { + $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] + Write-Host (T "==> Keeping existing NETX_DATABASE_URL" "==> 保留已有 NETX_DATABASE_URL") -ForegroundColor Green + } } } if (-not $ExternalDatabaseUrl) { throw "external_url_required" } + $ExternalDatabaseUrl = $ExternalDatabaseUrl.Trim() $values["NETX_DATABASE_URL"] = $ExternalDatabaseUrl - Write-Host "==> External Postgres configured (ensure role/db exist; see scripts\init_pg.ps1)" -ForegroundColor Green + Write-Host "==> External Postgres configured" -ForegroundColor Green + + if (-not $SkipExternalProbe) { + # Probe with bundled psql when available (wizard already tested; this covers CLI reconfigure). + $psqlProbe = Join-Path $prog "postgres\pgsql\bin\psql.exe" + $testHelper = Join-Path $PSScriptRoot "installer\test_pg_conn.ps1" + if (-not (Test-Path $testHelper)) { + $testHelper = Join-Path $PSScriptRoot "test_pg_conn.ps1" + } + if ((Test-Path $psqlProbe) -and ($ExternalDatabaseUrl -match '^(?:postgresql(?:\+psycopg)?|postgres)://([^:]+):([^@]*)@([^:/]+):?(\d+)?/([^?\s]+)')) { + $u = [uri]::UnescapeDataString($Matches[1]) + $pw = [uri]::UnescapeDataString($Matches[2]) + $h = $Matches[3] + $po = if ($Matches[4]) { [int]$Matches[4] } else { 5432 } + $dbn = [uri]::UnescapeDataString($Matches[5]) + Write-Host "==> Probing external PostgreSQL ${h}:${po}/${dbn} ..." + if (Test-Path $testHelper) { + $probeErr = Join-Path $env:TEMP ("netx-pg-probe-" + [Guid]::NewGuid().ToString("n") + ".txt") + & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $testHelper ` + -PgHost $h -Port $po -User $u -Password $pw -Database $dbn ` + -PsqlPath $psqlProbe -OutErrFile $probeErr + if ($LASTEXITCODE -ne 0) { + $detail = if (Test-Path $probeErr) { (Get-Content -LiteralPath $probeErr -Raw) } else { "exit $LASTEXITCODE" } + Remove-Item -LiteralPath $probeErr -Force -ErrorAction SilentlyContinue + throw (T "external_db_probe_failed: $detail" "外置数据库连接失败: $detail") + } + Remove-Item -LiteralPath $probeErr -Force -ErrorAction SilentlyContinue + Write-Host "==> External PostgreSQL OK" -ForegroundColor Green + } else { + $env:PGPASSWORD = $pw + try { + $ping = & $psqlProbe -h $h -p $po -U $u -d $dbn -t -A -c "SELECT 1" 2>&1 + if ($LASTEXITCODE -ne 0 -or (($ping | Out-String).Trim()) -notmatch '1') { + throw (T "external_db_probe_failed: $ping" "外置数据库连接失败: $ping") + } + } finally { + Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue + } + } + } else { + Write-Host (T ` + "[WARN] Skipped connection probe (no bundled psql or URL parse failed)." ` + "[WARN] 已跳过连接探测（缺少捆绑 psql 或连接串无法解析）。") -ForegroundColor Yellow + } + } +} + +# Fernet key AFTER database prompts (interactive UX). +# Priority: CLI/file > interactive prompt > existing .env > generate. +if (-not $CredentialSecretKey -and -not $NonInteractive) { + Write-Host "" + Write-Host (T ` + "Optional: paste NETX_CREDENTIAL_SECRET_KEY from an existing install" ` + "可选: 粘贴已有安装的 NETX_CREDENTIAL_SECRET_KEY（便于沿用已加密凭据）") -ForegroundColor Cyan + Write-Host (T ` + "Leave empty to keep existing .env key, or auto-generate if none." ` + "留空则保留已有 .env 中的密钥；若没有则自动生成。") + $CredentialSecretKey = (Read-Host "NETX_CREDENTIAL_SECRET_KEY").Trim() +} +if ($CredentialSecretKey) { + $values["NETX_CREDENTIAL_SECRET_KEY"] = $CredentialSecretKey.Trim() + Write-Host "==> Using provided NETX_CREDENTIAL_SECRET_KEY" -ForegroundColor Green +} elseif ($existing.ContainsKey("NETX_CREDENTIAL_SECRET_KEY") -and $existing["NETX_CREDENTIAL_SECRET_KEY"]) { + $values["NETX_CREDENTIAL_SECRET_KEY"] = $existing["NETX_CREDENTIAL_SECRET_KEY"] + Write-Host "==> Keeping existing NETX_CREDENTIAL_SECRET_KEY from .env" -ForegroundColor Green +} else { + $values["NETX_CREDENTIAL_SECRET_KEY"] = New-NetxFernetKey + Write-Host "==> Generated NETX_CREDENTIAL_SECRET_KEY (saved in .env)" -ForegroundColor Green } Write-DotEnvValue -Path $envPath -Values $values Write-Host "" Write-Host "Wrote $envPath" -ForegroundColor Green -Write-Host "Next: .\packaging\start_netx_app.ps1" + +# Official Setup ships python/runtime + .venv (build_release -CreateVenv). +$venvPy = Join-Path $prog ".venv\Scripts\python.exe" +try { + $null = Ensure-NetxVenv -ProgramRoot $prog + Write-Host "==> Bundled Python venv OK: $venvPy" -ForegroundColor Green +} catch { + if (Test-Path $venvPy) { + throw + } + Write-Host "==> Bundled .venv missing — creating one (Setup should normally ship it)." -ForegroundColor Yellow + Write-Host "[WARN] $($_.Exception.Message)" -ForegroundColor Yellow + Write-Host " Install a Setup built with: packaging\build_release.ps1 -CreateVenv" -ForegroundColor Yellow +} + +Write-Host "" +Write-Host (T "Setup complete." "首次配置完成。") -ForegroundColor Green +if (-not $NonInteractive) { + Write-Host (T ` + "Next: Start Menu → NetX Tray (or press Y below)." ` + "下一步: 开始菜单 → NetX 托盘 （或在下方按 Y 立即启动）。") + $ans = Read-Host (T "Start NetX tray now? [Y/n]" "现在启动 NetX 托盘？[Y/n]") + if ($ans -notmatch '^[Nn]') { + try { + Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-StartOnLaunch") ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + Write-Host (T "Tray started." "托盘已启动。") -ForegroundColor Green + } catch { + Write-Host "[WARN] $($_.Exception.Message)" -ForegroundColor Yellow + } + } +} else { + Write-Host "Next: .\packaging\start_netx_app.ps1 or NetX-Tray.cmd" +} + +} catch { + Write-Host "" + Write-Host ("[ERR] " + $_.Exception.Message) -ForegroundColor Red + if (-not $NonInteractive) { + try { + [void](Read-Host (T "Press Enter to close" "按回车关闭窗口")) + } catch {} + } + exit 1 +} diff --git a/packaging/sign_release.ps1 b/packaging/sign_release.ps1 new file mode 100644 index 0000000..a859447 --- /dev/null +++ b/packaging/sign_release.ps1 @@ -0,0 +1,78 @@ +param( + [Parameter(Mandatory = $true)] + [string[]]$Files, + [string]$Thumbprint = "", + [string]$PfxPath = "", + [string]$PfxPassword = "", + [string]$TimestampUrl = "http://timestamp.digicert.com", + [string]$Description = "NetX" +) + +# Sign release artifacts with signtool (Authenticode). +# Requires Windows SDK / signtool.exe and a code-signing certificate. +# +# Examples: +# .\sign_release.ps1 -Files .\packaging\release\NetX-Setup-0.4.0.exe -Thumbprint ABCDEF... +# .\sign_release.ps1 -Files .\packaging\release\*.exe -PfxPath .\certs\netx.pfx -PfxPassword *** + +$ErrorActionPreference = "Stop" + +function Find-SignTool { + $cmd = Get-Command signtool.exe -ErrorAction SilentlyContinue + if ($cmd) { return $cmd.Source } + $roots = @( + "${env:ProgramFiles(x86)}\Windows Kits\10\bin", + "${env:ProgramFiles}\Windows Kits\10\bin" + ) + foreach ($root in $roots) { + if (-not (Test-Path $root)) { continue } + $hit = Get-ChildItem -Path $root -Recurse -Filter signtool.exe -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -match '\\x64\\signtool\.exe$' } | + Select-Object -First 1 + if ($hit) { return $hit.FullName } + } + return $null +} + +$signtool = Find-SignTool +if (-not $signtool) { + throw "signtool_not_found: install Windows SDK or add signtool.exe to PATH" +} + +if (-not $Thumbprint -and -not $PfxPath) { + if ($env:NETX_SIGN_THUMBPRINT) { $Thumbprint = $env:NETX_SIGN_THUMBPRINT } + if ($env:NETX_SIGN_PFX) { $PfxPath = $env:NETX_SIGN_PFX } + if ($env:NETX_SIGN_PFX_PASSWORD) { $PfxPassword = $env:NETX_SIGN_PFX_PASSWORD } +} +if (-not $Thumbprint -and -not $PfxPath) { + throw "provide -Thumbprint or -PfxPath (or NETX_SIGN_THUMBPRINT / NETX_SIGN_PFX)" +} + +$resolved = @() +foreach ($pattern in $Files) { + $resolved += @(Resolve-Path -Path $pattern -ErrorAction Stop) +} +if ($resolved.Count -eq 0) { throw "no_files_to_sign" } + +foreach ($f in $resolved) { + $path = $f.Path + Write-Host "==> Signing $path" + $args = @( + "sign", "/fd", "SHA256", "/td", "SHA256", "/tr", $TimestampUrl, + "/d", $Description + ) + if ($PfxPath) { + $args += @("/f", $PfxPath) + if ($PfxPassword) { $args += @("/p", $PfxPassword) } + } else { + $args += @("/sha1", $Thumbprint) + } + $args += $path + & $signtool @args + if ($LASTEXITCODE -ne 0) { throw "sign_failed: $path" } + & $signtool verify /pa $path + if ($LASTEXITCODE -ne 0) { throw "verify_failed: $path" } + Write-Host " OK" -ForegroundColor Green +} + +Write-Host "==> Done. Without a trusted CA certificate, Windows SmartScreen may still warn." diff --git a/packaging/start_netx_app.ps1 b/packaging/start_netx_app.ps1 index 3fe2313..a795e44 100644 --- a/packaging/start_netx_app.ps1 +++ b/packaging/start_netx_app.ps1 @@ -1,8 +1,9 @@ -param( +param( [string]$ProgramRoot = "", [string]$DataRoot = "", [switch]$SkipBrowser = $false, - [switch]$InlineSchedulers = $false + [switch]$InlineSchedulers = $false, + [switch]$Force = $false ) $ErrorActionPreference = "Stop" @@ -21,25 +22,39 @@ if (-not (Test-Path (Join-Path $prog "netx_api"))) { throw "netx_api not found under program root: $prog" } +Ensure-NetxDataDirectories -DataRoot $data + $map = Import-NetxEnvFile -Path $envPath +# Force data-root paths even if an older .env missed them. +$dataEnv = Get-NetxDataEnvMap -DataRoot $data +foreach ($k in $dataEnv.Keys) { + Set-Item -Path "Env:$k" -Value $dataEnv[$k] +} + Set-Location $prog $env:PYTHONPATH = $prog -# Keep runtime artifacts in the data root (not under Program Files). -$env:NETX_AUTH_MCP_TOKEN_FILE = Join-Path $data "data\auth\mcp_token" -$env:NETX_SCHEDULER_HEARTBEAT_PATH = Join-Path $data "data\runtime\scheduler_heartbeat.json" -$env:NETX_AUTH_SECRET_FILE = Join-Path $data "data\auth\jwt_secret" - $dist = Join-Path $prog "web\dist" if (Test-Path (Join-Path $dist "index.html")) { $env:NETX_UI_DIST_DIR = $dist } $mode = Get-DbMode -EnvMap $map +$hostBind = if ($env:NETX_HOST) { $env:NETX_HOST } else { "127.0.0.1" } +$port = if ($env:NETX_PORT) { [int]$env:NETX_PORT } else { 8890 } + Write-Host "==> Program: $prog" Write-Host "==> Data: $data" Write-Host "==> DB mode: $mode" +if (-not $Force -and (Test-NetxApiHealthy -HostName $hostBind -Port $port)) { + Write-Host "==> NetX already healthy at http://${hostBind}:${port}/ — skip start (use -Force to restart)" -ForegroundColor Green + if (-not $SkipBrowser) { + try { Start-Process "http://${hostBind}:${port}/" } catch {} + } + exit 0 +} + function Get-PgsqlBin { foreach ($b in @( (Join-Path $prog "postgres\pgsql\bin"), @@ -54,13 +69,20 @@ if ($mode -eq "bundled") { $pgBin = Get-PgsqlBin if (-not $pgBin) { throw "bundled_postgres_missing" } $pgData = if ($map["NETX_BUNDLED_PG_DATA_DIR"]) { - $map["NETX_BUNDLED_PG_DATA_DIR"] + $map["NETX_BUNDLED_PG_DATA_DIR"] -replace '/', '\' } else { Join-Path $data "pgdata" } + $pgPort = 15432 + if ($map["NETX_BUNDLED_PG_PORT"]) { + try { $pgPort = [int]$map["NETX_BUNDLED_PG_PORT"] } catch {} + } $pgCtl = Join-Path $pgBin "pg_ctl.exe" $status = & $pgCtl -D $pgData status 2>&1 | Out-String if ($status -notmatch "server is running") { + if (-not (Test-NetxTcpPortFree -HostName "127.0.0.1" -Port $pgPort)) { + throw "port_in_use: 127.0.0.1:$pgPort (bundled Postgres)" + } Write-Host "==> Starting bundled PostgreSQL" if (-not (Test-Path $pgData)) { New-Item -ItemType Directory -Path $pgData -Force | Out-Null @@ -74,38 +96,48 @@ if ($mode -eq "bundled") { } } -# Ensure venv exists for start_netx.ps1 -$venvPy = Join-Path $prog ".venv\Scripts\python.exe" -if (-not (Test-Path $venvPy)) { - Write-Host "==> Creating .venv (one-time)" - $pyCmd = Get-Command python -ErrorAction SilentlyContinue - if (-not $pyCmd) { throw "python_not_found: install Python 3.11+ and re-run" } - & $pyCmd.Source -m venv (Join-Path $prog ".venv") - & $venvPy -m pip install --upgrade pip - & $venvPy -m pip install -r (Join-Path $prog "requirements.txt") - if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } +try { + $null = Ensure-NetxVenv -ProgramRoot $prog +} catch { + Write-Host "[ERR] $($_.Exception.Message)" -ForegroundColor Red + Write-Host " Tip: run Start Menu → NetX → First-time setup once as Administrator," -ForegroundColor Yellow + Write-Host " or install a Setup built with packaging\\build_release.ps1 -CreateVenv." -ForegroundColor Yellow + exit 1 } -$hostBind = if ($env:NETX_HOST) { $env:NETX_HOST } else { "127.0.0.1" } -$port = if ($env:NETX_PORT) { [int]$env:NETX_PORT } else { 8890 } +if (-not (Test-NetxTcpPortFree -HostName $hostBind -Port $port)) { + if (Test-NetxApiHealthy -HostName $hostBind -Port $port) { + Write-Host "==> Port $port already serves NetX — skip start" -ForegroundColor Green + exit 0 + } + throw "port_in_use: ${hostBind}:${port} occupied by non-NetX process" +} $startScript = Join-Path $prog "scripts\start_netx.ps1" if (-not (Test-Path $startScript)) { throw "start_netx.ps1 not found at $startScript" } -$psArgs = @( - "-ExecutionPolicy", "Bypass", "-File", $startScript, - "-SkipInstall", "-Background", - "-BindHost", $hostBind, "-Port", "$port" -) -if ($InlineSchedulers) { $psArgs += "-InlineSchedulers" } - Write-Host "==> Starting NetX (API + workers; UI via API on :$port)" -& powershell @psArgs -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# Run in-process with -Background so this console exits after /health (nested +# Start-Process -Wait on a Hidden child often never returns under UAC update). +$startArgs = @{ + SkipInstall = $true + Background = $true + BindHost = $hostBind + Port = $port +} +if ($InlineSchedulers) { $startArgs["InlineSchedulers"] = $true } +& $startScript @startArgs +if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $url = "http://${hostBind}:${port}/" +if (-not (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2)) { + if (-not (Wait-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 30)) { + Write-Host "[ERR] NetX started but /health not ready: $url" -ForegroundColor Red + exit 1 + } +} Write-Host "==> Open: $url" -ForegroundColor Green if (-not $SkipBrowser) { try { Start-Process $url } catch {} diff --git a/packaging/stop_netx_app.ps1 b/packaging/stop_netx_app.ps1 index 174c631..5da0525 100644 --- a/packaging/stop_netx_app.ps1 +++ b/packaging/stop_netx_app.ps1 @@ -1,7 +1,8 @@ param( [string]$ProgramRoot = "", [string]$DataRoot = "", - [switch]$KeepPostgres = $false + [switch]$KeepPostgres = $false, + [switch]$KillTray = $false ) $ErrorActionPreference = "Continue" @@ -21,11 +22,17 @@ if (-not (Test-Path $stopScript)) { } if (Test-Path $stopScript) { Write-Host "==> Stopping NetX processes" - & powershell -ExecutionPolicy Bypass -File $stopScript -Force + Start-NetxPowerShell -File $stopScript -Arguments @("-Force") ` + -WorkingDirectory $prog -WindowStyle Hidden -Wait | Out-Null } else { Write-Host "[WARN] stop_netx.ps1 not found" } +if ($KillTray) { + Write-Host "==> Stopping NetX tray icons" + Stop-NetxTrayProcesses -ProgramRoot $prog +} + $mode = Get-DbMode -EnvMap $map if ($mode -eq "bundled" -and -not $KeepPostgres) { $pgBinCandidates = @( diff --git a/packaging/uninstall_delete_data.ps1 b/packaging/uninstall_delete_data.ps1 new file mode 100644 index 0000000..707d0cd --- /dev/null +++ b/packaging/uninstall_delete_data.ps1 @@ -0,0 +1,53 @@ +param( + [string]$DataRoot = "" +) + +# Post-uninstall optional data wipe with retries (handles briefly locked runtime logs). + +$ErrorActionPreference = "Continue" +if (-not $DataRoot) { + $DataRoot = Join-Path ([Environment]::GetFolderPath("CommonApplicationData")) "NetX" +} + +if (-not (Test-Path -LiteralPath $DataRoot)) { + Write-Host "==> Data root already gone: $DataRoot" + exit 0 +} + +Write-Host "==> Deleting data root: $DataRoot" + +# Kill anything still holding files under data root. +Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { + $_.CommandLine -and ($_.CommandLine.IndexOf($DataRoot, [StringComparison]::OrdinalIgnoreCase) -ge 0) +} | ForEach-Object { + try { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } catch {} + try { & taskkill.exe /F /PID $_.ProcessId 2>$null | Out-Null } catch {} +} + +$ok = $false +for ($i = 1; $i -le 5; $i++) { + try { + cmd /c "rmdir /s /q `"$DataRoot`"" + } catch {} + if (-not (Test-Path -LiteralPath $DataRoot)) { + $ok = $true + break + } + Start-Sleep -Seconds 1 +} + +if (-not $ok -and (Test-Path -LiteralPath $DataRoot)) { + try { + takeown /F $DataRoot /R /D Y | Out-Null + icacls $DataRoot /grant Administrators:F /T | Out-Null + cmd /c "rmdir /s /q `"$DataRoot`"" + } catch {} +} + +if (Test-Path -LiteralPath $DataRoot) { + Write-Host "[WARN] Could not fully delete $DataRoot" + exit 1 +} + +Write-Host "==> Data root deleted" +exit 0 diff --git a/packaging/uninstall_prepare.ps1 b/packaging/uninstall_prepare.ps1 new file mode 100644 index 0000000..05e581b --- /dev/null +++ b/packaging/uninstall_prepare.ps1 @@ -0,0 +1,71 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "" +) + +# Fast, non-blocking uninstall prep for Inno [UninstallRun]. +# Must return within a few seconds and never kill unins000.exe / _unins.tmp. + +$ErrorActionPreference = "Continue" + +$prog = if ($ProgramRoot) { $ProgramRoot } else { "C:\Program Files\NetX" } +$data = if ($DataRoot) { $DataRoot } else { Join-Path $env:ProgramData "NetX" } +$prog = ($prog -replace '/', '\').TrimEnd('\') +$data = ($data -replace '/', '\').TrimEnd('\') + +function Test-Protected([string]$Name, [string]$Cmd) { + if ($Name -match '^(unins\d*|_unins\.tmp|setup)\.exe$') { return $true } + if ($Cmd -match 'unins\d*\.exe|_unins\.tmp|uninstall_prepare\.ps1|uninstall_delete_data\.ps1') { return $true } + return $false +} + +function Stop-Pid([int]$Id, [string]$Label) { + if ($Id -le 4 -or $Id -eq $PID) { return } + Write-Host "stop $Id $Label" + try { Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue } catch {} +} + +Write-Host "==> uninstall_prepare fast-stop prog=$prog" + +$patterns = @( + 'netx_tray\.ps1', + 'netx_api\.(main|worker|biz_state_worker)', + 'start_netx_app\.ps1', + 'stop_netx_app\.ps1', + 'launch_shortcut\.ps1', + [regex]::Escape((Join-Path $prog '.venv\Scripts\python.exe')), + [regex]::Escape((Join-Path $prog 'postgres\pgsql\bin')) +) + +Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | ForEach-Object { + $cl = [string]$_.CommandLine + $name = [string]$_.Name + if (-not $cl) { return } + if (Test-Protected -Name $name -Cmd $cl) { return } + foreach ($pat in $patterns) { + if ($cl -match $pat) { + Stop-Pid -Id ([int]$_.ProcessId) -Label $name + break + } + } +} + +# Best-effort postgres stop (no hang: immediate + ignore) +$pgCtl = Join-Path $prog "postgres\pgsql\bin\pg_ctl.exe" +$pgData = Join-Path $data "pgdata" +if ((Test-Path $pgCtl) -and (Test-Path $pgData)) { + try { + $p = Start-Process -FilePath $pgCtl -ArgumentList @('-D', $pgData, 'stop', '-m', 'immediate') ` + -WindowStyle Hidden -PassThru + if (-not $p.WaitForExit(5000)) { + try { Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue } catch {} + } + } catch {} +} + +try { + Remove-ItemProperty -Path "HKCU:\Software\Microsoft\Windows\CurrentVersion\Run" -Name "NetX" -ErrorAction SilentlyContinue +} catch {} + +Write-Host "==> uninstall_prepare done" +exit 0 diff --git a/packaging/update_netx.ps1 b/packaging/update_netx.ps1 index 1ddd0ec..db003c6 100644 --- a/packaging/update_netx.ps1 +++ b/packaging/update_netx.ps1 @@ -1,4 +1,4 @@ -param( +param( [Parameter(Mandatory = $true)] [string]$PackagePath, [string]$ProgramRoot = "", @@ -22,10 +22,14 @@ New-Item -ItemType Directory -Path $work -Force | Out-Null try { Write-Host "==> Extracting update package" + # Preferred end-user path is NetX-Setup-*.exe (check_update / offline installer). + # This script remains for legacy/dev .zip packages (build_release.ps1 -CreateZip). if ($PackagePath.ToLowerInvariant().EndsWith(".zip")) { Expand-Archive -Path $PackagePath -DestinationPath $work -Force + } elseif ($PackagePath.ToLowerInvariant().EndsWith(".exe")) { + throw "unsupported_package: run NetX-Setup-*.exe (upgrade keeps DB), or use check_update.ps1 -Apply" } else { - throw "unsupported_package: use a .zip built by build_release.ps1" + throw "unsupported_package: use a .zip from build_release.ps1 -CreateZip, or NetX-Setup-*.exe" } $src = $work @@ -54,7 +58,35 @@ try { if (Test-Path $envFile) { Copy-Item $envFile (Join-Path $bak ".env") } - Write-Host "==> Backup marker: $bak (data/pgdata left in place; optional pg_dump not run)" + # Best-effort logical backup when psql is available (bundled or PATH). + $map = Read-DotEnv -Path $envFile + $pgDump = $null + foreach ($c in @( + (Join-Path $prog "postgres\pgsql\bin\pg_dump.exe"), + (Join-Path $PSScriptRoot "postgres\pgsql\bin\pg_dump.exe") + )) { + if (Test-Path $c) { $pgDump = $c; break } + } + if (-not $pgDump) { + $cmd = Get-Command pg_dump -ErrorAction SilentlyContinue + if ($cmd) { $pgDump = $cmd.Source } + } + if ($pgDump -and $map["NETX_DATABASE_URL"] -match 'postgresql\+?[^:]*://([^:]+):([^@]+)@([^:/]+):?(\d+)?/([^?\s]+)') { + $u = $Matches[1]; $p = [uri]::UnescapeDataString($Matches[2]); $h = $Matches[3] + $pt = if ($Matches[4]) { $Matches[4] } else { "5432" } + $dbn = $Matches[5] + $dumpOut = Join-Path $bak "netx.dump" + Write-Host "==> pg_dump -> $dumpOut" + $env:PGPASSWORD = $p + try { + & $pgDump -h $h -p $pt -U $u -d $dbn -Fc -f $dumpOut + } catch { + Write-Host "[WARN] pg_dump failed: $($_.Exception.Message)" -ForegroundColor Yellow + } finally { + Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue + } + } + Write-Host "==> Backup: $bak" } Write-Host "==> Stopping services" @@ -62,7 +94,8 @@ try { -ProgramRoot $prog -DataRoot $data # Replace program layers only — never wipe data root. - $replaceDirs = @("netx_api", "alembic", "web", "packaging", "scripts", "postgres", "packages") + # Include python/ (portable runtime) — required with shipped .venv since 0.4.6. + $replaceDirs = @("netx_api", "alembic", "web", "packaging", "scripts", "postgres", "packages", "python") foreach ($name in $replaceDirs) { $from = Join-Path $src $name $to = Join-Path $prog $name @@ -79,7 +112,7 @@ try { Copy-Item -Path $from -Destination (Join-Path $prog $f) -Force } } - # Optional runtime / .venv shipped in package + # Legacy top-level runtime/ (older packages) + shipped .venv if (Test-Path (Join-Path $src "runtime")) { $rt = Join-Path $prog "runtime" if (Test-Path $rt) { Remove-Item -Recurse -Force $rt } @@ -92,11 +125,21 @@ try { Copy-Item -Path (Join-Path $src ".venv") -Destination $venvTo -Recurse -Force } + # Builder-path pyvenv.cfg → local python/runtime (same as first install). + if (Get-Command Repair-NetxShippedVenv -ErrorAction SilentlyContinue) { + if (Repair-NetxShippedVenv -ProgramRoot $prog) { + Write-Host "==> Relinked .venv to bundled python/runtime" -ForegroundColor Green + } + } + Write-Host "==> Update files applied" -ForegroundColor Green if (-not $NoStart) { - & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "start_netx_app.ps1") ` + # Same process (no nested powershell) so the update console can exit cleanly. + & (Join-Path $PSScriptRoot "start_netx_app.ps1") ` -ProgramRoot $prog -DataRoot $data -SkipBrowser + if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } + Write-Host "==> Update complete. You can close this window." -ForegroundColor Green } finally { if (Test-Path $work) { Remove-Item -Recurse -Force $work -ErrorAction SilentlyContinue diff --git a/pyproject.toml b/pyproject.toml index a2836f0..9263218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netx-ops" -version = "0.3.0" +version = "0.4.12" description = "netx operations tool: alarm-centric workflows with REST API and stdio MCP" readme = "README.md" requires-python = ">=3.11" @@ -43,3 +43,4 @@ netx-topology-mcp = "netx_topology_mcp.server:main" [tool.setuptools.packages.find] where = ["."] include = ["netx_api*"] + diff --git a/scripts/init_pg.ps1 b/scripts/init_pg.ps1 index d9c02de..cfcb909 100644 --- a/scripts/init_pg.ps1 +++ b/scripts/init_pg.ps1 @@ -1,4 +1,4 @@ -param( +param( [string]$PgHost = "127.0.0.1", [int]$PgPort = 5432, [string]$SuperUser = "postgres", diff --git a/scripts/start_netx.ps1 b/scripts/start_netx.ps1 index ed2caec..d1ba8ea 100644 --- a/scripts/start_netx.ps1 +++ b/scripts/start_netx.ps1 @@ -23,9 +23,24 @@ if ($Backgtound -and -not $Background) { $projectRoot = Split-Path -Parent $PSScriptRoot Set-Location $projectRoot -$runDir = Join-Path $PSScriptRoot ".run" +$packCommon = Join-Path $projectRoot "packaging\_common.ps1" +if (Test-Path -LiteralPath $packCommon) { + . $packCommon + if (Get-Command Repair-NetxShippedVenv -ErrorAction SilentlyContinue) { + $null = Repair-NetxShippedVenv -ProgramRoot $projectRoot + } +} + +# Prefer writable data-root runtime dir (Program Files is not writable after Setup install). +if ($env:NETX_RUN_DIR) { + $runDir = $env:NETX_RUN_DIR +} elseif ($env:NETX_SCHEDULER_HEARTBEAT_PATH) { + $runDir = Split-Path -Parent ($env:NETX_SCHEDULER_HEARTBEAT_PATH -replace '/', '\') +} else { + $runDir = Join-Path $PSScriptRoot ".run" +} if (-not (Test-Path $runDir)) { - New-Item -ItemType Directory -Path $runDir | Out-Null + New-Item -ItemType Directory -Path $runDir -Force | Out-Null } $pidFile = Join-Path $runDir "netx.pid" @@ -39,13 +54,17 @@ $webPidFile = Join-Path $runDir "web.pid" $webLogFile = Join-Path $runDir "web.out.log" $webErrFile = Join-Path $runDir "web.err.log" -$venvPython = Join-Path $projectRoot ".venv\\Scripts\\python.exe" -if (-not (Test-Path $venvPython)) { +$venvPython = Join-Path $projectRoot ".venv\Scripts\python.exe" +if (Get-Command Test-NetxVenvRunnable -ErrorAction SilentlyContinue) { + if (-not (Test-NetxVenvRunnable -VenvPython $venvPython)) { + throw "venv_not_runnable: reinstall NetX or run setup as administrator" + } +} elseif (-not (Test-Path -LiteralPath $venvPython)) { Write-Host "==> .venv not found, creating virtual environment" python -m venv (Join-Path $projectRoot ".venv") -} -if (-not (Test-Path $venvPython)) { - throw "failed_to_create_venv" + if (-not (Test-Path -LiteralPath $venvPython)) { + throw "failed_to_create_venv" + } } $pythonExe = $venvPython @@ -253,7 +272,11 @@ if ($Background) { Show-LogTail -Path $logFile exit 1 } - $healthWaitSec = 45 + # Fresh installs run Alembic upgrades on first boot; 45s is often too short. + $healthWaitSec = 180 + if ($env:NETX_START_HEALTH_WAIT_SEC) { + try { $healthWaitSec = [int]$env:NETX_START_HEALTH_WAIT_SEC } catch {} + } if (-not (Test-NetxApiListening -HostName $BindHost -LocalPort $Port -WaitSec $healthWaitSec)) { Write-Host "[ERR] Port $Port not listening /health not OK within ${healthWaitSec}s (process may be stuck on DB or schema migration)." -ForegroundColor Red Show-LogTail -Path $errFile diff --git a/scripts/stop_netx.ps1 b/scripts/stop_netx.ps1 index 2d91466..1106bb3 100644 --- a/scripts/stop_netx.ps1 +++ b/scripts/stop_netx.ps1 @@ -1,4 +1,4 @@ -param( +param( [int]$Port = 8890, [int]$WebPort = 5173, # Windows often ignores graceful Stop-Process on python; default hard-kill. @@ -9,7 +9,13 @@ param( $ErrorActionPreference = "Continue" $useForce = if ($NoForce) { $false } else { [bool]$Force } -$runDir = Join-Path $PSScriptRoot ".run" +if ($env:NETX_RUN_DIR) { + $runDir = $env:NETX_RUN_DIR +} elseif ($env:NETX_SCHEDULER_HEARTBEAT_PATH) { + $runDir = Split-Path -Parent ($env:NETX_SCHEDULER_HEARTBEAT_PATH -replace '/', '\') +} else { + $runDir = Join-Path $PSScriptRoot ".run" +} $pidFile = Join-Path $runDir "netx.pid" $workerPidFile = Join-Path $runDir "worker.pid" $webPidFile = Join-Path $runDir "web.pid" diff --git a/tests/test_biz_state_compare.py b/tests/test_biz_state_compare.py index 8d610ad..dcb07e1 100644 --- a/tests/test_biz_state_compare.py +++ b/tests/test_biz_state_compare.py @@ -5,7 +5,8 @@ from __future__ import annotations import copy import unittest -from netx_api.biz_state.compare_engine import compare_rows, mapping_stats +from netx_api.biz_state.compare_engine import compare_rows, mapping_stats, stratify_take +from netx_api.biz_state.compare_service import _kind_allows from netx_api.biz_state.compare_rules import ( apply_row_filters, arp_dynamic_row_filters, @@ -96,7 +97,18 @@ class CompareEngineTests(unittest.TestCase): self.assertEqual(s["removed"], 1) self.assertEqual(s["added"], 1) kinds = {d["kind"] for d in out["diffs"]} - self.assertIn("unchanged", kinds) + # Default: unchanged counted but not listed (million-row safe) + self.assertNotIn("unchanged", kinds) + out_full = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["local_if", "remote_sys", "remote_if"], + iface_fields=["local_if"], + compare_fields=[], + port_map={}, + include_unchanged=True, + ) + self.assertIn("unchanged", {d["kind"] for d in out_full["diffs"]}) def test_empty_port_map_ignores_iface_in_key(self) -> None: """No port map → ignore local_if when matching (same neighbor, renamed port).""" @@ -149,10 +161,107 @@ class CompareEngineTests(unittest.TestCase): iface_fields=["local_if"], compare_fields=["remote_ip"], port_map={}, + include_unchanged=True, ) self.assertEqual(out["summary"]["unchanged"], 1) self.assertEqual(len(out["diffs"]), 1) self.assertEqual(out["diffs"][0]["kind"], "unchanged") + slim = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["local_if", "remote_sys", "remote_if"], + iface_fields=["local_if"], + compare_fields=["remote_ip"], + port_map={}, + ) + self.assertEqual(slim["summary"]["unchanged"], 1) + self.assertEqual(slim["diffs"], []) + + def test_unchanged_sample_limit_and_compact(self) -> None: + before = [ + {"k": str(i), "v": "1", "_netx": {"row_id": f"b{i}"}} for i in range(5) + ] + after = [ + {"k": str(i), "v": "1", "_netx": {"row_id": f"a{i}"}} for i in range(5) + ] + out = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["k"], + iface_fields=[], + compare_fields=["v"], + port_map={}, + include_unchanged=True, + unchanged_limit=2, + compact_unchanged=True, + ) + self.assertEqual(out["summary"]["unchanged"], 5) + self.assertEqual(out["summary"]["unchanged_listed"], 2) + self.assertTrue(out["summary"]["unchanged_truncated"]) + self.assertTrue(out["summary"]["unchanged_compact"]) + self.assertEqual(out["summary"]["unchanged_sample_mode"], "stratified") + self.assertEqual(len(out["diffs"]), 2) + self.assertEqual(out["diffs"][0]["before"], {}) + self.assertEqual(out["diffs"][0]["after"], {}) + self.assertEqual(out["diffs"][0]["before_row_id"], "b0") + self.assertEqual(out["diffs"][0]["after_row_id"], "a0") + + def test_stratify_take_round_robin(self) -> None: + items = [("a", i) for i in range(5)] + [("b", i) for i in range(5)] + got = stratify_take(items, 4, key_fn=lambda x: x[0]) + self.assertEqual([x[0] for x in got], ["a", "b", "a", "b"]) + + def test_kind_allows_tabs(self) -> None: + self.assertTrue(_kind_allows("all", "unchanged")) + self.assertTrue(_kind_allows("diff", "removed")) + self.assertTrue(_kind_allows("diff", "changed")) + self.assertFalse(_kind_allows("diff", "added")) + self.assertFalse(_kind_allows("diff", "unchanged")) + self.assertTrue(_kind_allows("unchanged", "unchanged")) + self.assertFalse(_kind_allows("unchanged", "removed")) + self.assertTrue(_kind_allows("removed", "removed")) + self.assertTrue(_kind_allows("changed", "changed")) + + def test_unchanged_sample_stratified_across_neighbors(self) -> None: + """Sample must cover multiple neighbors, not only the first command's rows.""" + before = [] + after = [] + for n_i, neigh in enumerate(("1.1.1.1", "2.2.2.2", "3.3.3.3")): + for j in range(10): + net = f"10.{n_i}.{j}.0/24" + before.append( + { + "neighbor": neigh, + "direction": "in", + "network": net, + "v": "1", + "_netx": {"row_id": f"b-{neigh}-{j}"}, + } + ) + after.append( + { + "neighbor": neigh, + "direction": "in", + "network": net, + "v": "1", + "_netx": {"row_id": f"a-{neigh}-{j}"}, + } + ) + out = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["neighbor", "direction", "network"], + iface_fields=[], + compare_fields=["v"], + port_map={}, + include_unchanged=True, + unchanged_limit=6, + compact_unchanged=True, + ) + self.assertEqual(out["summary"]["unchanged"], 30) + self.assertEqual(out["summary"]["unchanged_listed"], 6) + keys = [d["key"]["neighbor"] for d in out["diffs"]] + self.assertEqual(set(keys), {"1.1.1.1", "2.2.2.2", "3.3.3.3"}) def test_mac_normalize_via_field_rules(self) -> None: before = [{"ip": "1.1.1.1", "mac": "00:11:22:33:44:55", "iface": "gei-0/1"}] @@ -309,6 +418,57 @@ class CompareRulesTests(unittest.TestCase): class CompareDiffPagingTests(unittest.TestCase): + def test_resolve_unchanged_policy(self) -> None: + from netx_api.biz_state.compare_service import resolve_unchanged_policy + + small = resolve_unchanged_policy("auto", before_n=100, after_n=100) + self.assertTrue(small["include"]) + self.assertIsNone(small["limit"]) + self.assertFalse(small["compact"]) + large = resolve_unchanged_policy("auto", before_n=1_500_000, after_n=1_500_000) + self.assertTrue(large["include"]) + self.assertEqual(large["limit"], 5000) + self.assertTrue(large["compact"]) + self.assertFalse(resolve_unchanged_policy("never", before_n=10, after_n=10)["include"]) + keys = resolve_unchanged_policy("keys", before_n=1_500_000, after_n=1_500_000) + self.assertTrue(keys["include"]) + self.assertIsNone(keys["limit"]) + self.assertTrue(keys["compact"]) + + def test_hydrate_diff_rows_fills_sides(self) -> None: + from netx_api.biz_state.compare_service import _hydrate_diff_rows + + class _FakeDb: + pass + + items = [ + { + "kind": "unchanged", + "key": {"k": "1"}, + "before": {}, + "after": {}, + "mapped_before": {}, + "changes": {}, + "before_row_id": "b1", + "after_row_id": "a1", + } + ] + # Patch loader + import netx_api.biz_state.compare_service as cs + + orig = cs._metric_rows_by_ids + try: + cs._metric_rows_by_ids = lambda _db, ids: { # type: ignore[assignment] + "b1": {"k": "1", "v": "pre"}, + "a1": {"k": "1", "v": "post"}, + } + out = _hydrate_diff_rows(_FakeDb(), items) + finally: + cs._metric_rows_by_ids = orig + self.assertEqual(out[0]["before"]["v"], "pre") + self.assertEqual(out[0]["after"]["v"], "post") + self.assertEqual(out[0]["mapped_before"]["v"], "pre") + def test_filter_inline_diffs_kind_and_kw(self) -> None: from netx_api.biz_state.compare_service import _filter_inline_diffs @@ -354,25 +514,43 @@ class CompareSheetDefaultsTests(unittest.TestCase): self.assertIn("isis_adjacency.ipv6", ids) # Same source metric may appear multiple times (afi splits) self.assertEqual(sum(1 for s in sheets if s["metric_id"] == "bgp_peer"), 6) + self.assertGreaterEqual(sum(1 for s in sheets if s["metric_id"] == "bgp_route"), 4) self.assertIn("bgp_peer.evpn", ids) self.assertIn("bgp_peer.vpls", ids) self.assertIn("vrrp.ipv4", ids) self.assertIn("lldp_neighbor", ids) - detail = next(s for s in sheets if sheet_key(s) == "interface_detail") - self.assertEqual(detail["compare_fields"], ["port_status"]) - self.assertIn("input_bps", detail["display_fields"]) - optical = next(s for s in sheets if sheet_key(s) == "optical_brief") - self.assertEqual(optical["compare_fields"], ["status"]) - self.assertIn("rx_power", optical["display_fields"]) - bgp4 = next(s for s in sheets if sheet_key(s) == "bgp_peer.ipv4") - self.assertEqual(bgp4["compare_fields"], ["as_num", "state"]) - self.assertIn("pfx_rcd", bgp4["display_fields"]) + # Packaged IOH default: filtered bgp_route ipv4 + percent pfx_rcd + route4 = next(s for s in sheets if sheet_key(s) == "bgp_route") + self.assertEqual(route4["metric_id"], "bgp_route") + self.assertTrue( + any( + f.get("field") == "afi" and f.get("value") == "ipv4" + for f in (route4.get("row_filters") or []) + ) + ) + self.assertTrue( + any( + r.get("field") == "pfx_rcd" and r.get("compare") == "percent" + for r in (route4.get("field_rules") or []) + ) + ) + route_vpn = next(s for s in sheets if sheet_key(s) == "bgp_route.vpnv4") + self.assertTrue( + any( + f.get("field") == "afi" and f.get("value") == "vpnv4" + for f in (route_vpn.get("row_filters") or []) + ) + ) vpnv4 = next(s for s in sheets if sheet_key(s) == "bgp_peer.vpnv4") - self.assertEqual(vpnv4["row_filters"], [{"field": "afi", "op": "eq", "value": "vpnv4"}]) + self.assertEqual( + vpnv4["row_filters"], + [{"field": "afi", "op": "eq", "value": "vpnv4"}], + ) isis4 = next(s for s in sheets if sheet_key(s) == "isis_adjacency.ipv4") self.assertEqual(isis4["row_filters"][0]["op"], "contains") - def test_duplicate_match_keys_are_reported(self) -> None: + def test_ordered_same_key_pairing_2v2(self) -> None: + """Same key, two rows each: zip by order (not first-wins + duplicate).""" before = [ {"local_if": "a", "remote_sys": "X", "remote_if": "1", "remote_ip": "1"}, {"local_if": "a", "remote_sys": "X", "remote_if": "1", "remote_ip": "9"}, @@ -389,15 +567,47 @@ class CompareSheetDefaultsTests(unittest.TestCase): compare_fields=["remote_ip"], port_map={"a": "a"}, ) + self.assertEqual(out["summary"]["before_count"], 2) + self.assertEqual(out["summary"]["after_count"], 2) + self.assertEqual(out["summary"]["duplicate"], 0) self.assertEqual(out["summary"]["duplicate_keys_before"], 1) self.assertEqual(out["summary"]["duplicate_keys_after"], 1) - self.assertEqual(out["summary"]["duplicate"], 2) self.assertIn("a|X|1", out["summary"]["duplicate_key_list"]) kinds = [d["kind"] for d in out["diffs"]] - self.assertEqual(kinds.count("duplicate"), 2) - # First before wins → matches first after → unchanged (same remote_ip) + self.assertNotIn("duplicate", kinds) + # Pair 0: 1==1 unchanged; pair 1: 9!=2 changed self.assertEqual(out["summary"]["unchanged"], 1) + self.assertEqual(out["summary"]["changed"], 1) + self.assertEqual(out["summary"]["added"], 0) + self.assertEqual(out["summary"]["removed"], 0) + + def test_ordered_multipath_5_vs_2(self) -> None: + """5 before / 2 after same key → 2 compared + 3 removed failures.""" + key = {"local_if": "a", "remote_sys": "X", "remote_if": "1"} + before = [{**key, "remote_ip": str(i)} for i in range(5)] + after = [{**key, "remote_ip": "0"}, {**key, "remote_ip": "1"}] + out = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["local_if", "remote_sys", "remote_if"], + iface_fields=["local_if"], + compare_fields=["remote_ip"], + port_map={"a": "a"}, + ) + self.assertEqual(out["summary"]["before_count"], 5) + self.assertEqual(out["summary"]["after_count"], 2) + self.assertEqual(out["summary"]["removed"], 3) + self.assertEqual(out["summary"]["added"], 0) + self.assertEqual( + out["summary"]["unchanged"] + out["summary"]["changed"], + 2, + ) + self.assertEqual(out["summary"]["unchanged"], 2) self.assertEqual(out["summary"]["changed"], 0) + self.assertEqual(out["summary"]["duplicate"], 0) + kinds = [d["kind"] for d in out["diffs"]] + self.assertEqual(kinds.count("removed"), 3) + self.assertNotIn("duplicate", kinds) def test_ignore_port_changes_false_keeps_iface(self) -> None: before = [ diff --git a/tests/test_biz_state_compare_recovery.py b/tests/test_biz_state_compare_recovery.py new file mode 100644 index 0000000..e24ea40 --- /dev/null +++ b/tests/test_biz_state_compare_recovery.py @@ -0,0 +1,98 @@ +"""Startup recovery for interrupted biz-state compare runs.""" + +from __future__ import annotations + +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_state.compare_service import ( + cancel_compare_run, + recover_interrupted_compares_on_startup, +) +from netx_api.db import Base +from netx_api.models import BizCompareJob, BizCompareRun + + +class BizStateCompareRecoveryTests(unittest.TestCase): + def setUp(self) -> None: + engine = create_engine("sqlite+pysqlite:///:memory:", future=True) + TestingSession = sessionmaker( + bind=engine, autoflush=False, autocommit=False, expire_on_commit=False + ) + Base.metadata.create_all(bind=engine) + self.db = TestingSession() + self.job = BizCompareJob( + id="j1", + name="cutover", + template_id="tpl1", + status="active", + ) + self.db.add(self.job) + self.running = BizCompareRun( + id="r_run", + job_id="j1", + status="running", + message="loading 1/2 · BGP", + summary_json={ + "progress": {"phase": "loading", "sheet_index": 1, "sheet_total": 2}, + "sheets": [ + {"sheet_id": "bgp", "status": "running"}, + {"sheet_id": "isis", "status": "pending"}, + ], + }, + ) + self.queued = BizCompareRun( + id="r_q", + job_id="j1", + status="queued", + message="queued", + summary_json={"sheets": [{"sheet_id": "bgp", "status": "pending"}]}, + ) + self.ok = BizCompareRun( + id="r_ok", + job_id="j1", + status="success", + message="done", + summary_json={"added": 0, "removed": 0, "changed": 0}, + ) + self.db.add_all([self.running, self.queued, self.ok]) + self.db.commit() + + def tearDown(self) -> None: + self.db.close() + + def test_startup_cancels_running_and_queued(self) -> None: + out = recover_interrupted_compares_on_startup(self.db) + self.assertEqual(out["runs"], 2) + + self.db.refresh(self.running) + self.db.refresh(self.queued) + self.db.refresh(self.ok) + self.assertEqual(self.running.status, "cancelled") + self.assertIn("interrupted_by_restart", self.running.message or "") + self.assertEqual( + (self.running.summary_json or {}).get("progress", {}).get("phase"), + "cancelled", + ) + sheets = list((self.running.summary_json or {}).get("sheets") or []) + self.assertEqual(sheets[0].get("status"), "cancelled") + self.assertEqual(sheets[1].get("status"), "cancelled") + self.assertEqual(self.queued.status, "cancelled") + self.assertEqual(self.ok.status, "success") + + def test_cancel_compare_run_user(self) -> None: + out = cancel_compare_run(self.db, "r_run") + self.assertEqual(out["status"], "cancelled") + self.db.refresh(self.running) + self.assertEqual(self.running.status, "cancelled") + self.assertIn("cancelled_by_user", self.running.message or "") + + def test_cancel_idempotent_on_success(self) -> None: + out = cancel_compare_run(self.db, "r_ok") + self.assertEqual(out["status"], "success") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_biz_state_compare_sql.py b/tests/test_biz_state_compare_sql.py new file mode 100644 index 0000000..44b2891 --- /dev/null +++ b/tests/test_biz_state_compare_sql.py @@ -0,0 +1,228 @@ +"""Unit tests for SQL compare eligibility and filter compilation.""" + +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock + +from netx_api.biz_state.compare_sql import ( + can_sql_compare, + compile_row_filters_sql, + sql_compare_skip_reason, + _field_rules_sql_compatible, + _filters_sql_compatible, + _safe_field, +) + + +class _FakeDialect: + def __init__(self, name: str) -> None: + self.name = name + + +class _FakeBind: + def __init__(self, name: str) -> None: + self.dialect = _FakeDialect(name) + + +def _db(dialect: str = "postgresql") -> MagicMock: + db = MagicMock() + db.get_bind.return_value = _FakeBind(dialect) + return db + + +class CompareSqlGateTests(unittest.TestCase): + def test_safe_field_rejects_injection(self) -> None: + with self.assertRaises(ValueError): + _safe_field("a'; drop table x;--") + with self.assertRaises(ValueError): + _safe_field("x.y") + self.assertEqual(_safe_field("network"), "network") + + def test_requires_postgres(self) -> None: + sheet = { + "metric_id": "bgp_route", + "key_fields": ["network", "next_hop"], + "compare_fields": ["path"], + "iface_fields": [], + "field_rules": [], + "row_filters": [], + } + self.assertFalse(can_sql_compare(_db("sqlite"), sheet, port_map={})) + self.assertTrue(can_sql_compare(_db("postgresql"), sheet, port_map={})) + + def test_rejects_port_map(self) -> None: + sheet = { + "metric_id": "lldp_neighbor", + "key_fields": ["local_if", "remote_sys"], + "compare_fields": [], + "iface_fields": ["local_if"], + "ignore_port_changes": False, + "field_rules": [], + "row_filters": [], + } + self.assertFalse( + can_sql_compare(_db(), sheet, port_map={"gei-0/1": "gei-0/2"}) + ) + + def test_rejects_auto_ignore_ports_with_iface_key(self) -> None: + sheet = { + "metric_id": "lldp_neighbor", + "key_fields": ["local_if", "remote_sys"], + "compare_fields": [], + "iface_fields": ["local_if"], + "ignore_port_changes": None, + "field_rules": [], + "row_filters": [], + } + self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) + + def test_rejects_mac_normalize(self) -> None: + sheet = { + "metric_id": "arp", + "key_fields": ["ip", "vrf"], + "compare_fields": ["mac"], + "iface_fields": [], + "field_rules": [{"field": "mac", "normalize": "mac"}], + "row_filters": [], + } + self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) + + def test_rejects_age_timer_filter(self) -> None: + sheet = { + "metric_id": "arp", + "key_fields": ["ip"], + "compare_fields": [], + "iface_fields": [], + "field_rules": [], + "row_filters": [{"field": "age", "op": "age_timer"}], + } + self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) + + def test_accepts_bgp_route_style(self) -> None: + sheet = { + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "vrf", + "neighbor", + "direction", + "rd", + "network", + "next_hop", + ], + "compare_fields": ["path", "as_num", "pfx_rcd"], + "iface_fields": [], + "field_rules": [ + {"field": "pfx_rcd", "compare": "percent", "tolerance": 5}, + ], + "row_filters": [ + {"field": "vrf", "op": "empty"}, + {"field": "afi", "op": "eq", "value": "ipv4"}, + ], + } + self.assertTrue(can_sql_compare(_db(), sheet, port_map={})) + # BGP afi/vrf sheet splits + percent rules must not force Python + self.assertEqual(sql_compare_skip_reason(_db(), sheet, port_map={}), "") + + def test_rejects_iface_normalize_when_key_uses_iface(self) -> None: + sheet = { + "metric_id": "interface_brief", + "key_fields": ["interface"], + "compare_fields": ["admin"], + "iface_fields": ["interface"], + "ignore_port_changes": False, + "field_rules": [], + "row_filters": [], + } + self.assertFalse( + can_sql_compare( + _db(), + sheet, + port_map={}, + iface_normalize_rules=[{"from": "GE", "to": "gei"}], + ) + ) + + def test_accepts_ignore_port_changes_false_without_normalize(self) -> None: + sheet = { + "metric_id": "interface_brief", + "key_fields": ["interface"], + "compare_fields": ["admin"], + "iface_fields": ["interface"], + "ignore_port_changes": False, + "field_rules": [], + "row_filters": [], + } + self.assertTrue(can_sql_compare(_db(), sheet, port_map={}, iface_normalize_rules=[])) + + +class CompareSqlFilterCompileTests(unittest.TestCase): + def test_empty_filters(self) -> None: + sql, params = compile_row_filters_sql([]) + self.assertEqual(sql, "TRUE") + self.assertEqual(params, {}) + + def test_eq_case_insensitive(self) -> None: + sql, params = compile_row_filters_sql( + [{"field": "afi", "op": "eq", "value": "IPv4"}] + ) + self.assertIn("lower(", sql) + self.assertIn("afi", sql) + self.assertEqual(list(params.values()), ["ipv4"]) + + def test_contains(self) -> None: + sql, params = compile_row_filters_sql( + [{"field": "af", "op": "contains", "value": "IPv4"}] + ) + self.assertIn("LIKE", sql) + self.assertEqual(list(params.values()), ["%ipv4%"]) + + def test_any_all_nesting(self) -> None: + sql, params = compile_row_filters_sql( + [ + { + "any": [ + {"field": "entry_type", "op": "eq", "value": "dynamic"}, + { + "all": [ + {"field": "entry_type", "op": "empty"}, + {"field": "ip", "op": "not_empty"}, + ] + }, + ] + } + ] + ) + self.assertIn(" OR ", sql) + self.assertIn(" AND ", sql) + self.assertTrue(_filters_sql_compatible([{"any": [{"field": "a", "op": "eq", "value": "1"}]}])) + + def test_in_list(self) -> None: + sql, params = compile_row_filters_sql( + [{"field": "afi", "op": "in", "value": ["ipv4", "ipv6"]}] + ) + self.assertIn(" IN (", sql) + self.assertEqual(sorted(params.values()), ["ipv4", "ipv6"]) + + def test_field_rules_matrix(self) -> None: + self.assertTrue(_field_rules_sql_compatible([{"field": "mac", "normalize": "lower"}])) + self.assertFalse(_field_rules_sql_compatible([{"field": "mac", "normalize": "mac"}])) + self.assertTrue( + _field_rules_sql_compatible( + [{"field": "rx", "compare": "numeric", "tolerance": 1}] + ) + ) + self.assertTrue( + _field_rules_sql_compatible( + [{"field": "pfx_rcd", "compare": "percent", "tolerance": 5}] + ) + ) + self.assertTrue( + _field_rules_sql_compatible([{"field": "x", "compare": "ignore"}]) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 8e14743..5246ade 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -494,6 +494,53 @@ const en = { needBeforeBatch: "Select before task and batch", needAfterBatch: "Select after batch", runNow: "Run now", + runStarted: "Compare started in background — you can close this page; check run history for progress", + runFailed: "Compare failed", + runStatusRunning: "Running", + runStatusFailed: "Failed", + runStatusDoneSec: "Done {{s}}s", + runProgress: "{{phase}} · sheet {{i}}/{{n}} · {{sheet}} · {{s}}s elapsed", + runSheetProgress: "sheet {{i}}/{{n}}", + runElapsed: "{{s}}s elapsed", + runRowsLoaded: "Loaded {{side}} {{n}} rows", + runRowsSqlCount: "DB count {{side}} {{n}} rows", + runPersisting: "Wrote {{done}}/{{total}} rows", + runEngineSql: "engine SQL", + runEnginePython: "engine Python", + runEngineNote: "reason {{note}}", + phaseQueued: "Queued", + phaseLoading: "Loading", + phaseSqlCount: "Counting", + phaseSqlProject: "Preparing tables", + phaseSqlJoin: "Joining", + phaseSqlFailFetch: "Fetching fails", + phaseComparing: "Comparing", + phasePersisting: "Writing", + phasePersistingFail: "Writing fails", + phasePersistingOk: "Writing passes", + phaseDone: "Done", + phaseFailed: "Failed", + phaseCancelled: "Cancelled", + sheetPending: "Pending", + colProgress: "Progress / result", + passRateScope: "all rows", + keyFiltersToggle: "Field filters", + hideDisplayCols: "Hide display cols", + showDisplayCols: "Show display cols", + hideDisplayColsHint: "Compact: hide display-only columns so keys and diffs stand out", + ranWithDuration: "Compare finished ({{s}}s)", + unchangedNotStored: "Success rows were counted but not stored. Set “Store success rows” to sample and re-run for spot-check.", + unchangedSampleHint: "{{total}} success rows total; browsing a stratified sample of {{listed}}. Search any route for live source lookup. Pass rate uses all {{total}}.", + liveSearchHint: "Field filters look up source batches live; tabs only filter kind. E.g. direction=out, network=1.1.1.1.", + liveSearchTruncatedHint: "Search results truncated — narrow the field filters.", + clearKeyFilters: "Clear fields", + storeUnchanged: "Store success rows", + storeUnchangedAuto: "Auto (full if small / sample 5k + hydrate if large)", + storeUnchangedSample: "Sample only (up to 5k keys + hydrate)", + storeUnchangedKeys: "All success keys (full browse; large write still slow)", + storeUnchangedAlways: "Always full JSON (slow on huge sheets)", + storeUnchangedNever: "Never (counts only)", + storeUnchangedHint: "Cutover focuses on fails and pass rate; success defaults to a stratified sample. Type a filter to look up any route from source tables. Pass rate uses the full success count.", saveJob: "Save config", jobSaved: "Job config saved", jobDeleted: "Compare job deleted", @@ -502,15 +549,25 @@ const en = { edit: "Edit", delete: "Delete", tabConfig: "Job config", + tabRuns: "Compare batches", tabResult: "Result", runs: "Run history", + runsHint: "Each “Run now” creates a batch. Stuck “Running” after restart is auto-cancelled; you can also cancel manually and re-run.", noRuns: "No runs yet — run a compare first", result: "Result", + viewResult: "View result", + cancelRun: "Cancel", + confirmCancelRun: "Cancel this compare batch? You can start a new run afterward.", + runCancelled: "Compare cancelled", + runStatusCancelled: "Cancelled", + compareAlreadyRunning: "A compare is already running — cancel it under Compare batches or wait", + runBatchSummary: "Fail {{fail}} · Pass {{ok}} · Added {{added}}", + colTime: "Time", pickBatchRun: "Select compare run", pickRun: "Select run…", runCount: "{{n}} runs", resultEmpty: "No matching diff rows", - resultFilterPh: "Filter key / values…", + resultFilterPh: "Free text, or direction:out network:1.1.1.1", kindAll: "All", kindDiff: "Fail", kindAdded: "Added", @@ -542,7 +599,7 @@ const en = { filterFailField: "Failed compare field", filterFailFieldAll: "Any failed field", resultEmpty: "No matching rows", - resultFilterPh: "Filter identity / values…", + resultFilterPh: "Free text, or direction:out network:1.1.1.1", diffCount: "{{n}} failed", passRate: "Pass rate", passOk: "All passed", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 89e55aa..168cdf4 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -493,6 +493,53 @@ const zh = { needBeforeBatch: "请选择操作前任务与批次", needAfterBatch: "请选择操作后批次", runNow: "立即比对", + runStarted: "比对已在后台启动，可关闭本页；进度见历史记录", + runFailed: "比对失败", + runStatusRunning: "比对中", + runStatusFailed: "失败", + runStatusDoneSec: "完成 {{s}}s", + runProgress: "{{phase}} · 表 {{i}}/{{n}} · {{sheet}} · 已用 {{s}}s", + runSheetProgress: "表 {{i}}/{{n}}", + runElapsed: "已用 {{s}}s", + runRowsLoaded: "已加载 {{side}} {{n}} 行", + runRowsSqlCount: "库内统计 {{side}} {{n}} 行", + runPersisting: "已写入 {{done}}/{{total}} 行", + runEngineSql: "引擎 SQL", + runEnginePython: "引擎 Python", + runEngineNote: "原因 {{note}}", + phaseQueued: "排队中", + phaseLoading: "加载数据", + phaseSqlCount: "库内统计", + phaseSqlProject: "准备比对表", + phaseSqlJoin: "库内比对", + phaseSqlFailFetch: "拉取失败行", + phaseComparing: "比对中", + phasePersisting: "写入结果", + phasePersistingFail: "写入失败行", + phasePersistingOk: "写入成功行", + phaseDone: "完成", + phaseFailed: "失败", + phaseCancelled: "已取消", + sheetPending: "等待中", + colProgress: "进度 / 结果", + passRateScope: "全表", + keyFiltersToggle: "字段筛选", + hideDisplayCols: "隐藏展示列", + showDisplayCols: "显示展示列", + hideDisplayColsHint: "紧凑模式：默认隐藏不参与比对的展示列，突出 Key 与差异", + ranWithDuration: "比对完成（耗时 {{s}} 秒）", + unchangedNotStored: "成功行仅统计数量未落库。可在任务配置将「成功行保存」改为抽样后重新比对（抽查用）。", + unchangedSampleHint: "成功共 {{total}} 条，明细抽样 {{listed}} 条（分层抽查）。要查任意路由请输入筛选条件——将按原表即时判定。通过率按全部 {{total}} 计。", + liveSearchHint: "按字段回查原表即时判定；页签只过滤种类。例：direction=out，network=1.1.1.1。", + liveSearchTruncatedHint: "搜索结果已截断，请再收窄字段条件。", + clearKeyFilters: "清空字段", + storeUnchanged: "成功行保存", + storeUnchangedAuto: "自动（小表全量 / 大表抽样 5000+原表补全）", + storeUnchangedSample: "仅抽样（最多 5000，身份键+原表补全）", + storeUnchangedKeys: "全量身份键（可翻完全部成功，大表写入仍较久）", + storeUnchangedAlways: "始终全量 JSON（大表很慢，慎用）", + storeUnchangedNever: "不保存（只计数量）", + storeUnchangedHint: "割接优先看失败与通过率；成功默认分层抽样。输入筛选可回查原表（不依赖抽样）。通过率按全部成功计数。", saveJob: "保存配置", jobSaved: "任务配置已保存", jobDeleted: "比对任务已删除", @@ -501,15 +548,25 @@ const zh = { edit: "编辑", delete: "删除", tabConfig: "任务配置", + tabRuns: "对比批次", tabResult: "比对结果", runs: "历史比对", + runsHint: "每次「立即比对」生成一条批次。重启后卡住的「比对中」会自动标为已取消；也可手动终止后重跑。", noRuns: "尚无比对记录，请先执行比对", result: "比对结果", + viewResult: "查看结果", + cancelRun: "终止", + confirmCancelRun: "确定终止该比对批次？终止后可重新发起比对。", + runCancelled: "比对已取消", + runStatusCancelled: "已取消", + compareAlreadyRunning: "已有比对在进行中，请先在「对比批次」中终止或等待完成", + runBatchSummary: "失败 {{fail}} · 成功 {{ok}} · 新增 {{added}}", + colTime: "时间", pickBatchRun: "选择比对记录", pickRun: "选择比对记录…", runCount: "{{n}} 次", resultEmpty: "无匹配失败/结果行", - resultFilterPh: "筛选身份 / 字段值…", + resultFilterPh: "自由词，或 direction:out network:1.1.1.1", kindAll: "全部", kindDiff: "失败", kindAdded: "新增", diff --git a/web/src/index.css b/web/src/index.css index eb97361..2046460 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -11698,6 +11698,164 @@ html.login-page--paused .login-page__flare { rgba(8, 13, 24, 0.4); } +.bs-cmp-progress { + display: flex; + flex-direction: column; + gap: 6px; + padding: 10px 12px; + border-radius: 8px; + border: 1px solid rgba(59, 130, 246, 0.35); + background: rgba(37, 99, 235, 0.12); + color: var(--bs-cmp-ink, #e2e8f0); + font-size: 0.875rem; +} + +.bs-cmp-progress.is-failed { + border-color: rgba(239, 68, 68, 0.4); + background: rgba(239, 68, 68, 0.12); +} + +.bs-cmp-progress.is-cancelled { + border-color: rgba(245, 158, 11, 0.4); + background: rgba(245, 158, 11, 0.1); +} + +.bs-cmp-progress__head { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px 12px; +} + +.bs-cmp-progress__actions { + margin-left: auto; + display: flex; + align-items: center; + gap: 6px; +} + +.bs-cmp-progress__engine, +.bs-cmp-progress__elapsed { + font-size: 0.8125rem; +} + +.bs-cmp-progress__track, +.bs-cmp-run-prog__track { + height: 6px; + border-radius: 999px; + background: rgba(148, 163, 184, 0.28); + overflow: hidden; +} + +.bs-cmp-progress__fill, +.bs-cmp-run-prog__fill { + height: 100%; + width: 0; + border-radius: inherit; + background: linear-gradient(90deg, #2563eb, #60a5fa); + transition: width 0.35s ease; +} + +.bs-cmp-progress__fill.is-indeterminate, +.bs-cmp-run-prog__fill.is-indeterminate { + width: 36%; + animation: bs-cmp-prog-slide 1.25s ease-in-out infinite; +} + +@keyframes bs-cmp-prog-slide { + 0% { + transform: translateX(-120%); + } + 100% { + transform: translateX(320%); + } +} + +.bs-cmp-progress__meta { + font-size: 0.8125rem; + color: var(--bs-cmp-muted, #94a3b8); + line-height: 1.4; + word-break: break-word; +} + +.bs-cmp-progress__msg { + flex: 1 1 100%; + font-size: 0.8125rem; +} + +/* Compare batch list (job detail → 对比批次) */ +.bs-cmp-runs { + display: flex; + flex-direction: column; + gap: 10px; + min-height: 260px; +} + +.bs-cmp-runs__toolbar { + display: flex; + flex-wrap: wrap; + align-items: flex-start; + justify-content: space-between; + gap: 10px 14px; +} + +.bs-cmp-runs__hint { + margin: 0; + flex: 1 1 220px; + line-height: 1.45; +} + +.bs-cmp-runs__table tbody tr.is-selected td { + background: rgba(37, 99, 235, 0.08); +} + +.bs-cmp-runs__elapsed { + margin-top: 4px; + font-size: 0.75rem; + color: var(--muted, #64748b); + font-variant-numeric: tabular-nums; +} + +.bs-cmp-runs__sides { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 2px 6px; + font-size: 0.8125rem; + line-height: 1.35; + max-width: 22rem; +} + +.bs-cmp-runs__arrow { + color: var(--muted, #94a3b8); + flex: 0 0 auto; +} + +.bs-cmp-runs__summary { + font-size: 0.8125rem; + line-height: 1.4; + color: var(--ink, #334155); + max-width: 18rem; +} + +.bs-cmp-run-prog { + display: flex; + flex-direction: column; + gap: 5px; + min-width: 160px; + max-width: 22rem; +} + +.bs-cmp-run-prog__meta { + font-size: 0.75rem; + line-height: 1.35; + color: var(--muted, #64748b); + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; +} + /* Native :fullscreen + CSS immersive fallback (class only when FS API blocked) */ .bs-cmp-board.is-fullscreen, .bs-cmp-board:fullscreen { @@ -12129,6 +12287,15 @@ body:has(.bs-cmp-board:fullscreen) { box-shadow: 0 0 0 2px rgba(245, 158, 11, 0.22); } +.bs-cmp-nav__item.is-pending .bs-cmp-nav__dot { + background: #64748b; + box-shadow: 0 0 0 2px rgba(100, 116, 139, 0.25); +} + +.bs-cmp-nav__item.is-pending .bs-cmp-nav__name { + opacity: 0.75; +} + .bs-cmp-nav__name { font-size: 12px; font-weight: 600; @@ -12472,6 +12639,36 @@ body:has(.bs-cmp-board:fullscreen) { white-space: nowrap; } +.bs-cmp-key-filters { + display: flex; + flex-wrap: wrap; + gap: 8px 10px; + align-items: flex-end; + width: 100%; +} + +.bs-cmp-key-filter { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 110px; + max-width: 160px; + flex: 0 1 140px; +} + +.bs-cmp-key-filter > span { + font-size: 11px; + line-height: 1.2; +} + +.bs-cmp-key-filter .input, +.bs-cmp-key-filter [data-slot="input"], +.bs-cmp-key-filter input { + min-width: 0; + width: 100%; + font-size: 12px; +} + .bs-cmp-kind-pills { display: flex; flex-wrap: wrap; @@ -12773,10 +12970,6 @@ body:has(.bs-cmp-board:fullscreen) { vertical-align: middle; } -.bs-cmp-val-cell--diff { - background: rgba(245, 158, 11, 0.08); -} - .bs-cmp-pair { display: flex; flex-direction: column; @@ -12784,6 +12977,10 @@ body:has(.bs-cmp-board:fullscreen) { min-width: 4.5rem; } +.bs-cmp-pair--same { + min-width: 3rem; +} + .bs-cmp-pair__row { display: grid; grid-template-columns: 1.6em minmax(0, 1fr); @@ -12808,6 +13005,20 @@ body:has(.bs-cmp-board:fullscreen) { color: #86efac; } +.bs-cmp-val-cell--pair.is-same { + background: transparent; +} + +.bs-cmp-val--same { + color: #cbd5e1; + opacity: 0.92; +} + +.bs-cmp-val-cell--diff { + background: rgba(245, 158, 11, 0.1); + box-shadow: inset 0 0 0 1px rgba(245, 158, 11, 0.22); +} + .bs-cmp-val { font-family: ui-monospace, SFMono-Regular, Consolas, monospace; color: #e2e8f0; @@ -12815,6 +13026,63 @@ body:has(.bs-cmp-board:fullscreen) { line-height: 1.35; } +.bs-cmp-progress.is-compact { + padding: 6px 10px; + gap: 4px; +} + +.bs-cmp-progress.is-compact .bs-cmp-progress__track { + height: 4px; +} + +.bs-cmp-progress.is-compact .bs-cmp-progress__meta { + font-size: 0.75rem; +} + +.bs-cmp-strip__pass-scope { + margin-left: 4px; + font-size: 9px; + font-weight: 600; + color: #64748b; + text-transform: none; +} + +.bs-cmp-strip__toggle { + flex: 0 0 auto; + height: 28px; + padding: 0 8px; + border-radius: 6px; + border: 1px solid rgba(148, 163, 184, 0.28); + background: rgba(15, 23, 42, 0.45); + color: #cbd5e1; + font-size: 11px; + cursor: pointer; + white-space: nowrap; +} + +.bs-cmp-strip__toggle:hover { + border-color: rgba(96, 165, 250, 0.45); + color: #f1f5f9; +} + +.bs-cmp-strip__toggle.is-active { + border-color: rgba(59, 130, 246, 0.5); + background: rgba(37, 99, 235, 0.18); + color: #93c5fd; +} + +.bs-cmp-strip__hint { + flex: 1 1 100%; + margin: 0; + font-size: 0.75rem; + line-height: 1.35; +} + +.bs-cmp-strip .bs-cmp-key-filters { + flex: 1 1 100%; + margin-top: 0; +} + .bs-cmp-val.is-empty { color: #64748b; font-style: italic; diff --git a/web/src/pages/network/BizComparePage.tsx b/web/src/pages/network/BizComparePage.tsx index 8ffddee..aa3a018 100644 --- a/web/src/pages/network/BizComparePage.tsx +++ b/web/src/pages/network/BizComparePage.tsx @@ -7,6 +7,7 @@ import { useDebouncedValue } from "../../hooks/useDebouncedValue"; import { useToast } from "../../hooks/useToast"; import { useI18n } from "../../i18n"; import { + bizCompareCancelRun, bizCompareCreateJob, bizCompareCreateMapping, bizCompareCreateTemplate, @@ -35,8 +36,9 @@ import { cutoverCachedGet, cutoverCachedGetSWR, invalidateCutoverCache } from ". import { jobChipColor, NmStatusChip } from "./nmChips"; type PageTab = "templates" | "jobs"; -type JobDetailTab = "config" | "result"; +type JobDetailTab = "config" | "runs" | "result"; type KindFilter = "diff" | "all" | "added" | "removed" | "changed" | "unchanged"; +type DiffListSource = "stored" | "live" | ""; type CreateJobStep = 0 | 1 | 2 | 3; const CREATE_JOB_STEPS = 4; @@ -100,6 +102,8 @@ type Template = { }; type Mapping = { id: string; name: string; rows: { before_if: string; after_if: string }[] }; +type StoreUnchanged = "auto" | "always" | "never" | "sample" | "keys"; + type Job = { id: string; name: string; @@ -112,6 +116,7 @@ type Job = { mode: string; status: string; enabled_sheet_ids?: string[]; + store_unchanged?: StoreUnchanged | string; note?: string; }; @@ -134,6 +139,8 @@ type RunSheet = { display_fields?: string[]; field_rules?: FieldRule[]; mode?: string; + /** pending|running|queued|done|cancelled — set while overall run is active */ + status?: string; summary?: Record; diffs?: DiffRow[]; }; @@ -189,6 +196,8 @@ function PairCell(props: { const post = afterText || "—"; const isAdded = kind === "added"; const isRemoved = kind === "removed"; + const same = + !mismatch && !isAdded && !isRemoved && String(beforeText || "") === String(afterText || ""); // Whole-row missing/extra: emphasize the present side; do not strike it out. const preClass = [ "bs-cmp-val", @@ -210,20 +219,31 @@ function PairCell(props: { -
-
- {beforeLabel} - {isAdded ? "—" : pre} + {same ? ( +
+ {pre || "—"}
-
- {afterLabel} - {isRemoved ? "—" : post} + ) : ( +
+
+ {beforeLabel} + {isAdded ? "—" : pre} +
+
+ {afterLabel} + {isRemoved ? "—" : post} +
-
+ )} {reason ?
{reason}
: null} ); @@ -329,6 +349,109 @@ function sheetLabel(s: { title?: string; sheet_id?: string; metric_id?: string } return String(s?.title || s?.sheet_id || s?.metric_id || "").trim() || "—"; } +type RunProgressInfo = { + phase?: string; + sheet_index?: number; + sheet_total?: number; + sheet_title?: string; + sheet_id?: string; + elapsed_ms?: number; + load_side?: string; + rows_loaded?: number; + engine?: string; + engine_note?: string; + persisted?: number; + persist_total?: number; +}; + +function localizeRunPhase(phase: string | undefined, t: (key: string) => string): string { + const p = String(phase || "").trim().toLowerCase(); + if (!p || p === "…") return "…"; + if (p.startsWith("persisting")) { + if (p.includes("fail")) return t("bizCompare.phasePersistingFail"); + if (p.includes("ok")) return t("bizCompare.phasePersistingOk"); + return t("bizCompare.phasePersisting"); + } + const map: Record = { + queued: "bizCompare.phaseQueued", + loading: "bizCompare.phaseLoading", + sql_count: "bizCompare.phaseSqlCount", + sql_project: "bizCompare.phaseSqlProject", + sql_join: "bizCompare.phaseSqlJoin", + sql_fail_fetch: "bizCompare.phaseSqlFailFetch", + comparing: "bizCompare.phaseComparing", + done: "bizCompare.phaseDone", + failed: "bizCompare.phaseFailed", + cancelled: "bizCompare.phaseCancelled", + }; + const key = map[p]; + return key ? t(key) : String(phase); +} + +function runProgressPercent(prog: RunProgressInfo): number { + const total = Number(prog.sheet_total || 0); + const idx = Number(prog.sheet_index || 0); + if (total <= 0) return 8; + const phase = String(prog.phase || "").toLowerCase(); + if (phase === "queued") return 4; + if (phase === "done") return 100; + let base = Math.max(0, Math.min(idx, total)) / total; + const persistTotal = Number(prog.persist_total || 0); + const persisted = Number(prog.persisted || 0); + if (persistTotal > 0 && phase.startsWith("persisting")) { + const within = Math.min(1, persisted / persistTotal) * (1 / total); + base = Math.max(0, (idx - 1) / total) + within; + } else if (phase === "loading" || phase === "sql_count" || phase === "comparing") { + base = Math.max(0, (idx - 1) / total) + 0.35 / total; + } + return Math.max(4, Math.min(99, Math.round(base * 100))); +} + +function runProgressMetaLine( + prog: RunProgressInfo, + t: (key: string, vars?: Record) => string, +): string { + const parts: string[] = []; + const phase = localizeRunPhase(prog.phase, t); + if (phase && phase !== "…") parts.push(phase); + const total = Number(prog.sheet_total || 0); + if (total > 0) { + parts.push( + t("bizCompare.runSheetProgress", { + i: String(prog.sheet_index || 0), + n: String(total), + }), + ); + } + const title = String(prog.sheet_title || prog.sheet_id || "").trim(); + if (title) parts.push(title); + const rows = Number(prog.rows_loaded || 0); + if (rows > 0) { + const eng = String(prog.engine || "").toLowerCase(); + parts.push( + eng === "sql" + ? t("bizCompare.runRowsSqlCount", { + side: String(prog.load_side || "—"), + n: String(rows), + }) + : t("bizCompare.runRowsLoaded", { + side: String(prog.load_side || "—"), + n: String(rows), + }), + ); + } + const persistTotal = Number(prog.persist_total || 0); + if (String(prog.phase || "").startsWith("persisting") && persistTotal > 0) { + parts.push( + t("bizCompare.runPersisting", { + done: String(prog.persisted || 0), + total: String(persistTotal), + }), + ); + } + return parts.join(" · "); +} + function templateSheets(tpl?: Template | null): MetricSheet[] { if (!tpl) return []; if (tpl.metrics?.length) return tpl.metrics; @@ -824,6 +947,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const [beforeBatchId, setBeforeBatchId] = useState(""); const [afterBatchId, setAfterBatchId] = useState(""); const [mode, setMode] = useState<"manual" | "auto">("manual"); + const [storeUnchanged, setStoreUnchanged] = useState("auto"); const [mapName, setMapName] = useState("端口映射"); const [mapText, setMapText] = useState(""); const [validateOut, setValidateOut] = useState(null); @@ -835,11 +959,35 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const [kindFilter, setKindFilter] = useState("diff"); const [resultKw, setResultKw] = useState(""); const debouncedResultKw = useDebouncedValue(resultKw, 300); + const [resultKeyFilters, setResultKeyFilters] = useState>({}); + const [keyFiltersOpen, setKeyFiltersOpen] = useState(false); + const [hideDisplayCols, setHideDisplayCols] = useState(true); + const debouncedKeyFiltersJson = useDebouncedValue(JSON.stringify(resultKeyFilters), 300); + const debouncedKeyFilters = useMemo(() => { + try { + const o = JSON.parse(debouncedKeyFiltersJson || "{}") as Record; + const out: Record = {}; + for (const [k, v] of Object.entries(o || {})) { + if (String(k || "").trim() && String(v || "").trim()) out[String(k)] = String(v).trim(); + } + return out; + } catch { + return {} as Record; + } + }, [debouncedKeyFiltersJson]); + const hasResultSearch = + Boolean(debouncedResultKw.trim()) || Object.keys(debouncedKeyFilters).length > 0; + const activeKeyFilterCount = Object.keys(resultKeyFilters).filter((k) => + String(resultKeyFilters[k] || "").trim(), + ).length; + const keyFiltersVisible = keyFiltersOpen || activeKeyFilterCount > 0; const [resultPage, setResultPage] = useState(1); const [resultPageSize, setResultPageSize] = useState(100); const [resultTotal, setResultTotal] = useState(0); const [pagedDiffs, setPagedDiffs] = useState([]); const [diffsLoading, setDiffsLoading] = useState(false); + const [diffsSource, setDiffsSource] = useState(""); + const [diffsTruncated, setDiffsTruncated] = useState(false); const boardRef = useRef(null); const tableScrollRef = useRef(null); const tableScrollPosRef = useRef({ top: 0, left: 0 }); @@ -1048,17 +1196,34 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage [runSheets, resultSheetId], ); - // Reset page when sheet / filter / page size changes + // Clear field filters when switching sheet + useEffect(() => { + setResultKeyFilters({}); + }, [resultSheetId]); + + // Reset page when sheet / filter / page size changes useEffect(() => { setResultPage(1); - }, [resultSheetId, kindFilter, debouncedResultKw, resultPageSize, runDetail?.id]); + }, [ + resultSheetId, + kindFilter, + debouncedResultKw, + debouncedKeyFiltersJson, + resultPageSize, + runDetail?.id, + ]); useEffect(() => { const runId = String(runDetail?.id || ""); const mid = resultSheetId || sheetIdentity(activeRunSheet) || ""; - if (!runId || !mid || jobDetailTab !== "result") { + const sheetSt = String(activeRunSheet?.status || ""); + // Block only while *this* sheet is still in flight — done sheets are readable mid-run + const sheetStillRunning = ["pending", "running", "queued"].includes(sheetSt); + if (!runId || !mid || jobDetailTab !== "result" || sheetStillRunning) { setPagedDiffs([]); setResultTotal(0); + setDiffsSource(""); + setDiffsTruncated(false); return; } let cancelled = false; @@ -1070,12 +1235,17 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage metricId: mid, kind: kindFilter, kw: debouncedResultKw.trim(), + qf: debouncedKeyFilters, page: resultPage, pageSize: resultPageSize, }); if (cancelled) return; setPagedDiffs((res.items || []) as DiffRow[]); setResultTotal(Number(res.total || 0)); + setDiffsSource( + String((res as any).source || "").toLowerCase() === "live" ? "live" : "stored", + ); + setDiffsTruncated(Boolean((res as any).truncated)); const pages = Math.max( 1, Math.ceil(Number(res.total || 0) / Number(res.page_size || resultPageSize)), @@ -1095,10 +1265,13 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; }, [ runDetail?.id, + runDetail?.status, resultSheetId, activeRunSheet?.metric_id, + activeRunSheet?.status, kindFilter, debouncedResultKw, + debouncedKeyFiltersJson, resultPage, resultPageSize, jobDetailTab, @@ -1186,7 +1359,8 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage Number(c.success_count ?? c.unchanged ?? 0); const sheetPassRateOf = (c: { - pass_rate?: number; + pass_rate?: number | null; + status?: string; fail_count?: number; diff_count?: number; removed?: number; @@ -1194,7 +1368,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage success_count?: number; unchanged?: number; added?: number; - }) => { + }): number | null => { + const st = String(c.status || ""); + if (st === "pending" || st === "running" || st === "queued") return null; + if (c.pass_rate === null) return null; // Always recompute from fail/success so added never skews pass rate const fail = sheetFailOf(c); const ok = sheetSuccessOf(c); @@ -1210,6 +1387,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage title?: string; metric_id: string; mode?: string; + status?: string; added?: number; removed?: number; changed?: number; @@ -1219,10 +1397,13 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage fail_count?: number; success_count?: number; diff_count?: number; - pass_rate?: number; + pass_rate?: number | null; }>; - // Failures first so ops can scan quickly when many sheets + // Failures first; keep pending/running at the end so completed fails stay visible return [...raw].sort((a, b) => { + const pa = ["pending", "running", "queued"].includes(String(a.status || "")) ? 1 : 0; + const pb = ["pending", "running", "queued"].includes(String(b.status || "")) ? 1 : 0; + if (pa !== pb) return pa - pb; const da = sheetFailOf(a); const db = sheetFailOf(b); if (da !== db) return db - da; @@ -1233,14 +1414,39 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const activeFail = sheetFailOf(activeSheetCard || {}); const activeSuccess = sheetSuccessOf(activeSheetCard || {}); const activePassRate = sheetPassRateOf(activeSheetCard || {}); + const activeSheetPending = ["pending", "running", "queued"].includes( + String(activeSheetCard?.status || ""), + ); const activeAdded = Number(activeSheetCard?.added || 0); + const activeRemoved = Number(activeSheetCard?.removed || 0); + const activeChanged = Number(activeSheetCard?.changed || 0); const showFailCol = - kindFilter === "diff" || kindFilter === "all" || kindFilter === "added"; + kindFilter === "diff" || + kindFilter === "all" || + kindFilter === "added" || + kindFilter === "removed" || + kindFilter === "changed"; + const isLiveSearch = diffsSource === "live" && hasResultSearch; + const resultSearchKeyFields = useMemo(() => { + const keys = (activeRunSheet?.key_fields || []).map((f) => String(f || "").trim()).filter(Boolean); + // Prefer BGP-ish fields first for compact UI + const prefer = ["direction", "neighbor", "network", "rd", "afi", "vrf", "local_as"]; + const ranked = [ + ...prefer.filter((p) => keys.includes(p)), + ...keys.filter((k) => !prefer.includes(k)), + ]; + return ranked.slice(0, 8); + }, [activeRunSheet?.key_fields]); const resultEmptyColSpan = 1 + (showFailCol ? 1 : 0) + resultColumns.keys.length + - Math.max(resultColumns.extras.length, 0); + Math.max( + resultColumns.extras.filter( + (f) => resultColumns.compareSet.has(f) || !hideDisplayCols, + ).length, + 0, + ); const rememberTableScroll = useCallback(() => { const wrap = tableScrollRef.current; @@ -1649,6 +1855,14 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage setBeforeBatchId(preset?.before_batch_id || ""); setAfterBatchId(preset?.after_batch_id || ""); setMode(preset?.mode === "auto" ? "auto" : "manual"); + { + const su = String(preset?.store_unchanged || "auto"); + setStoreUnchanged( + su === "always" || su === "never" || su === "sample" || su === "keys" + ? su + : "auto", + ); + } setValidateOut(null); if (preset?.mapping_id) loadMappingText(preset.mapping_id); else { @@ -1667,8 +1881,24 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage after_batch_id: mode === "manual" ? afterBatchId : "", mode, enabled_sheet_ids: enabledSheetIds, + store_unchanged: storeUnchanged, }); + const renderStoreUnchangedField = () => ( + setStoreUnchanged(e.target.value as StoreUnchanged)} + fullWidth + > + + + + + + + ); + const closeCreateJob = () => { setJobCreateOpen(false); setCreateStep(0); @@ -1755,12 +1985,14 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const job = jobs.find((x) => x.id === id); if (job) resetJobForm(job); try { - const r = await bizCompareListRuns(id); - setRuns(r.items || []); - if ((r.items || []).length) { - const latest = await bizCompareGetRun(String((r.items as any[])[0].id)); + const r = await bizCompareListRuns(id, 50); + const items = r.items || []; + setRuns(items); + if (items.length) { + const latest = await bizCompareGetRun(String((items as any[])[0].id)); setRunDetail(latest); - setJobDetailTab("result"); + // Land on batch list so stuck/cancelled runs are visible and actionable + setJobDetailTab("runs"); } } catch (e) { showError(formatErr(e)); @@ -1796,28 +2028,108 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage } }; + const runStatus = String(runDetail?.status || ""); + const runIsActive = runStatus === "running" || runStatus === "queued"; + const jobHasActiveRun = runs.some((r) => { + const st = String((r as any).status || ""); + return st === "running" || st === "queued"; + }); + const runNow = async () => { if (!jobId) return; if (!enabledJobSheetCount) { showError(t("bizCompare.needSheets")); return; } + if (jobHasActiveRun) { + showError(t("bizCompare.compareAlreadyRunning")); + setJobDetailTab("runs"); + return; + } setBusy(true); try { await bizCompareUpdateJob(jobId, jobConfigBody()); + // Async enqueue — returns immediately with status=running; poll below. const run = await bizCompareRunJob(jobId); setRunDetail(run); - setJobDetailTab("result"); - showOk(t("bizCompare.ran")); - const r = await bizCompareListRuns(jobId); + setJobDetailTab("runs"); + showOk(t("bizCompare.runStarted")); + const r = await bizCompareListRuns(jobId, 50); setRuns(r.items || []); await refresh({ force: true }); } catch (e) { showError(formatErr(e)); + setJobDetailTab("runs"); } finally { setBusy(false); } }; + const runProgress = (runDetail?.summary?.progress || {}) as RunProgressInfo; + const runEngine = String(runProgress.engine || "").toLowerCase(); + + // Poll active compare runs so the modal can be closed and reopened safely. + useEffect(() => { + if (!runIsActive || !runDetail?.id) return; + let cancelled = false; + let notified = false; + const tick = async () => { + try { + const d = await bizCompareGetRun(String(runDetail.id)); + if (cancelled) return; + setRunDetail(d); + if (jobId) { + const r = await bizCompareListRuns(jobId); + if (!cancelled) setRuns(r.items || []); + } + const st = String(d.status || ""); + if (!notified && st === "success") { + notified = true; + const ms = Number((d.summary as any)?.duration_ms || 0); + const sec = ms > 0 ? Math.round(ms / 1000) : 0; + showOk(sec > 0 ? t("bizCompare.ranWithDuration", { s: String(sec) }) : t("bizCompare.ran")); + } else if (!notified && st === "failed") { + notified = true; + showError(String(d.message || t("bizCompare.runFailed"))); + } else if (!notified && st === "cancelled") { + notified = true; + showError(String(d.message || t("bizCompare.runCancelled"))); + } + } catch (e) { + if (!cancelled) showError(formatErr(e)); + } + }; + void tick(); + const id = window.setInterval(() => void tick(), 2000); + return () => { + cancelled = true; + window.clearInterval(id); + }; + }, [runDetail?.id, runIsActive, jobId, showOk, showError, t]); + + // Keep the runs list fresh while any batch on this job is active + useEffect(() => { + if (!jobId || !jobHasActiveRun || jobDetailTab !== "runs") return; + let cancelled = false; + const tick = async () => { + try { + const r = await bizCompareListRuns(jobId, 50); + if (!cancelled) setRuns(r.items || []); + const curId = String(runDetail?.id || ""); + if (curId) { + const d = await bizCompareGetRun(curId); + if (!cancelled) setRunDetail(d); + } + } catch { + /* ignore list poll errors */ + } + }; + void tick(); + const id = window.setInterval(() => void tick(), 2500); + return () => { + cancelled = true; + window.clearInterval(id); + }; + }, [jobId, jobHasActiveRun, jobDetailTab, runDetail?.id]); const loadRun = async (runId: string) => { try { @@ -1831,6 +2143,25 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage } }; + const cancelRun = async (runId: string) => { + if (!runId) return; + if (!window.confirm(t("bizCompare.confirmCancelRun"))) return; + setBusy(true); + try { + const d = await bizCompareCancelRun(runId); + if (String(runDetail?.id || "") === runId) setRunDetail(d); + if (jobId) { + const r = await bizCompareListRuns(jobId, 50); + setRuns(r.items || []); + } + showOk(t("bizCompare.runCancelled")); + } catch (e) { + showError(formatErr(e)); + } finally { + setBusy(false); + } + }; + const removeRun = async (runId: string) => { if (!runId) return; if (!window.confirm(t("bizCompare.confirmDeleteRun"))) return; @@ -1840,19 +2171,16 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage await bizCompareDeleteRun(runId); let nextRuns: typeof runs = []; if (jobId) { - const r = await bizCompareListRuns(jobId); + const r = await bizCompareListRuns(jobId, 50); nextRuns = r.items || []; } else { nextRuns = (runs || []).filter((r) => String(r.id) !== runId); } setRuns(nextRuns); if (wasCurrent) { - if (nextRuns.length) { - await loadRun(String(nextRuns[0].id)); - } else { - setRunDetail(null); - setResultSheetId(""); - } + setRunDetail(null); + setResultSheetId(""); + setJobDetailTab("runs"); } showOk(t("bizCompare.runDeleted")); } catch (e) { @@ -2034,6 +2362,8 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage + {renderStoreUnchangedField()} +

{t("bizCompare.storeUnchangedHint")}

{renderSheetChips()} {renderBatchFields()} {renderMappingBlock()} @@ -2701,6 +3031,8 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage + {renderStoreUnchangedField()} +

{t("bizCompare.storeUnchangedHint")}

) : null} @@ -2760,6 +3092,18 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage > {t("bizCompare.tabConfig")} + +
+ {!runs.length ? ( +
{t("bizCompare.noRuns")}
+ ) : ( +
+ + + + + + + + + + + + {runs.map((r) => { + const st = String((r as any).status || ""); + const active = st === "running" || st === "queued"; + const selected = String(runDetail?.id || "") === String(r.id); + const before = enrichSide( + (r as any).before, + beforeTaskId, + tasks, + beforeBatches, + ); + const after = enrichSide( + (r as any).after, + afterTaskId || beforeTaskId, + tasks, + afterBatches.length ? afterBatches : beforeBatches, + ); + const when = formatSystemTime((r as any).created_at) || "—"; + const sum = ((r as any).summary || {}) as Record; + const fail = + Number(sum.removed || 0) + Number(sum.changed || 0); + const ok = Number(sum.unchanged || 0); + const durMs = Number(sum.duration_ms || 0); + const prog = ((r as any).progress || + (r as any).summary?.progress || + {}) as RunProgressInfo; + const pct = active ? runProgressPercent(prog) : 100; + const elapsedSec = Math.round(Number(prog.elapsed_ms || 0) / 1000); + const stLabel = + st === "running" || st === "queued" + ? t("bizCompare.runStatusRunning") + : st === "failed" + ? t("bizCompare.runStatusFailed") + : st === "cancelled" + ? t("bizCompare.runStatusCancelled") + : t("bizCompare.runStatusDoneSec", { + s: String(durMs > 0 ? Math.round(durMs / 1000) : 0), + }); + const chipColor = active + ? "accent" + : st === "cancelled" + ? "warning" + : jobChipColor(st); + const meta = active ? runProgressMetaLine(prog, t) : ""; + return ( + + + + + + + + ); + })} + +
{t("bizCompare.colStatus")}{t("bizCompare.colTime")}{t("bizCompare.sidesTitle")}{t("bizCompare.colProgress")}{t("bizCompare.colActions")}
+ {stLabel} + {active && elapsedSec > 0 ? ( +
+ {t("bizCompare.runElapsed", { s: String(elapsedSec) })} +
+ ) : null} + 		{when} +
+ {sideDeviceName(before)} + {sideCollectTime(before)} + + → + + {sideDeviceName(after)} + {sideCollectTime(after)} +
+ 		+ {active ? ( +
+
+
0 + ? { width: `${pct}%` } + : undefined + } + /> +
+
+ {meta || "…"} +
+
+ ) : ( +
+ {st === "failed" || st === "cancelled" + ? String((r as any).message || stLabel).slice(0, 96) + : t("bizCompare.runBatchSummary", { + fail: String(fail), + ok: String(ok), + added: String(Number(sum.added || 0)), + })} +
+ )} +
+
+ + {active ? ( + + ) : ( + + )} +
+
+
+ )} + ) : (
0 + ? t("bizCompare.runStatusDoneSec", { + s: String(Math.round(durMs / 1000)), + }) + : ""; return ( @@ -2876,7 +3404,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
+ {runDetail && runIsActive ? ( +
+
+ {t("bizCompare.runStatusRunning")} + {runEngine === "sql" || runEngine === "python" ? ( + + {runEngine === "sql" + ? t("bizCompare.runEngineSql") + : t("bizCompare.runEnginePython")} + {runProgress.engine_note + ? ` · ${t("bizCompare.runEngineNote", { + note: String(runProgress.engine_note), + })}` + : ""} + + ) : null} + + {t("bizCompare.runElapsed", { + s: String(Math.round(Number(runProgress.elapsed_ms || 0) / 1000)), + })} + +
+ +
+
+
+
0 + ? { width: `${runProgressPercent(runProgress)}%` } + : undefined + } + /> +
+
+ {runProgressMetaLine(runProgress, t)} +
+
+ ) : null} + {runDetail && runStatus === "failed" ? ( +
+
+ {t("bizCompare.runStatusFailed")} +
+
{String(runDetail.message || "")}
+
+ ) : null} + {runDetail && runStatus === "cancelled" ? ( +
+
+ {t("bizCompare.runStatusCancelled")} +
+
+ {String(runDetail.message || t("bizCompare.runCancelled"))} +
+
+ ) : null} {runDetail ? (
{(sheetCards.length ? sheetCards - : runSheets.map((s) => { - const removed = Number(s.summary?.removed || 0); - const changed = Number(s.summary?.changed || 0); - const unchanged = Number(s.summary?.unchanged || 0); - const fail = removed + changed; - const judged = fail + unchanged; - return { - sheet_id: sheetIdentity(s), - title: sheetLabel(s), - metric_id: s.metric_id, - mode: s.mode, - added: s.summary?.added, - removed, - changed, - unchanged, - before_count: s.summary?.before_count, - after_count: s.summary?.after_count, - fail_count: fail, - success_count: unchanged, - diff_count: fail, - pass_rate: judged - ? Math.round((unchanged / judged) * 1000) / 10 - : 100, - }; - }) + : [...runSheets] + .map((s) => { + const removed = Number(s.summary?.removed || 0); + const changed = Number(s.summary?.changed || 0); + const unchanged = Number(s.summary?.unchanged || 0); + const fail = removed + changed; + const judged = fail + unchanged; + const st = String((s as any).status || "done"); + const pending = ["pending", "running", "queued"].includes(st); + return { + sheet_id: sheetIdentity(s), + title: sheetLabel(s), + metric_id: s.metric_id, + mode: s.mode, + status: st, + added: s.summary?.added, + removed, + changed, + unchanged, + before_count: s.summary?.before_count, + after_count: s.summary?.after_count, + fail_count: fail, + success_count: unchanged, + diff_count: fail, + pass_rate: pending + ? null + : judged + ? Math.round((unchanged / judged) * 1000) / 10 + : 100, + }; + }) + .sort((a, b) => { + const pa = ["pending", "running", "queued"].includes( + String(a.status || ""), + ) + ? 1 + : 0; + const pb = ["pending", "running", "queued"].includes( + String(b.status || ""), + ) + ? 1 + : 0; + if (pa !== pb) return pa - pb; + const da = Number(a.fail_count || 0); + const db = Number(b.fail_count || 0); + if (da !== db) return db - da; + return String(a.metric_id).localeCompare(String(b.metric_id)); + }) ).map((c) => { const fail = sheetFailOf(c); const ok = sheetSuccessOf(c); @@ -2995,6 +3619,9 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const id = sheetIdentity(c); const label = sheetLabel(c); const active = resultSheetId === id; + const pending = ["pending", "running", "queued"].includes( + String((c as any).status || ""), + ); return ( @@ -3032,7 +3669,13 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage
0 ? " is-warn" : " is-ok"}`} + className={`bs-cmp-strip${ + activeSheetPending + ? " is-pending" + : activeFail > 0 + ? " is-warn" + : " is-ok" + }`} >
@@ -3045,23 +3688,41 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage {sheetLabel(activeSheetCard)} - {activeFail > 0 ? t("bizCompare.kindFail") : t("bizCompare.kindPass")} + {activeSheetPending + ? t("bizCompare.sheetPending") + : activeFail > 0 + ? t("bizCompare.kindFail") + : t("bizCompare.kindPass")} {activeSheetCard?.mode === "presence" ? ` · ${t("bizCompare.presenceShort")}` : ""} - {` · ${activeSheetCard?.before_count ?? 0}→${activeSheetCard?.after_count ?? 0}`} + {activeSheetPending + ? "" + : ` · ${activeSheetCard?.before_count ?? 0}→${activeSheetCard?.after_count ?? 0}`}
-
+
{t("bizCompare.passRate")} + + {t("bizCompare.passRateScope")} + + + + {activeSheetPending || activePassRate === null + ? "…" + : `${activePassRate}%`} - {activePassRate}%
{( [ ["diff", activeFail, "diff"], + ["removed", activeRemoved, "removed"], + ["changed", activeChanged, "changed"], ["added", activeAdded, "added"], ["unchanged", activeSuccess, "unchanged"], ["all", null, "all"], @@ -3081,7 +3742,11 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage ? t("bizCompare.kindAll") : id === "added" ? t("bizCompare.kindAddedShort") - : t("bizCompare.kindSuccess")} + : id === "removed" + ? t("bizCompare.kindRemovedShort") + : id === "changed" + ? t("bizCompare.kindChangedShort") + : t("bizCompare.kindSuccess")} {n !== null ? ( <> {" "} @@ -3100,7 +3765,110 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage {diffsLoading ? "…" : `${pagedDiffs.length}/${resultTotal}`} + {resultSearchKeyFields.length ? ( + + ) : null} +
+ {resultSearchKeyFields.length && keyFiltersVisible ? ( +
+ {resultSearchKeyFields.map((f) => ( + + ))} + {activeKeyFilterCount ? ( + + ) : null} +
+ ) : null} + {isLiveSearch ? ( +

+ {diffsTruncated + ? t("bizCompare.liveSearchTruncatedHint") + : t("bizCompare.liveSearchHint")} +

+ ) : null} + {!isLiveSearch && + kindFilter === "unchanged" && + (runDetail?.summary?.unchanged_truncated || + (Number( + (activeRunSheet?.summary as any)?.unchanged || + runDetail?.summary?.unchanged || + 0, + ) > + Number( + (activeRunSheet?.summary as any)?.unchanged_listed ?? + runDetail?.summary?.unchanged_listed ?? + 0, + ) && + Number( + (activeRunSheet?.summary as any)?.unchanged_listed ?? + runDetail?.summary?.unchanged_listed ?? + 0, + ) > 0)) ? ( +

+ {t("bizCompare.unchangedSampleHint", { + listed: String( + Number( + (activeRunSheet?.summary as any)?.unchanged_listed ?? + runDetail?.summary?.unchanged_listed ?? + resultTotal, + ), + ), + total: String( + Number( + (activeRunSheet?.summary as any)?.unchanged ?? + runDetail?.summary?.unchanged ?? + 0, + ), + ), + })} +

+ ) : null}
resultColumns.compareSet.has(f), ); - const displayCols = resultColumns.extras.filter( - (f) => !resultColumns.compareSet.has(f), - ); + const displayCols = hideDisplayCols + ? [] + : resultColumns.extras.filter( + (f) => !resultColumns.compareSet.has(f), + ); const verdictColSpan = showFailCol ? 2 : 1; const hasGroups = keyCols.length + compareCols.length + displayCols.length > 0; @@ -3251,43 +4021,57 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage {cellText(d.key?.[k] ?? pre[k] ?? post[k]) || "—"} ))} - {resultColumns.extras.map((f, fi) => { - const pv = cellText(pre[f]); - const av = cellText(post[f]); - const ch = d.changes?.[f]; - const isCmp = resultColumns.compareSet.has(f); - const prev = resultColumns.extras[fi - 1]; - const prevCmp = prev - ? resultColumns.compareSet.has(prev) - : null; - const zoneStart = fi === 0 || prevCmp !== isCmp; - const mismatch = - d.kind === "added" || d.kind === "removed" - ? Boolean(pv || av) - : isCmp - ? Boolean(ch) - : pv !== av; - return ( - - ); - })} + {resultColumns.extras + .filter( + (f) => + resultColumns.compareSet.has(f) || !hideDisplayCols, + ) + .map((f, fi, visibleExtras) => { + const pv = cellText(pre[f]); + const av = cellText(post[f]); + const ch = d.changes?.[f]; + const isCmp = resultColumns.compareSet.has(f); + const prev = visibleExtras[fi - 1]; + const prevCmp = prev + ? resultColumns.compareSet.has(prev) + : null; + const zoneStart = fi === 0 || prevCmp !== isCmp; + const mismatch = + d.kind === "added" || d.kind === "removed" + ? Boolean(pv || av) + : isCmp + ? Boolean(ch) + : pv !== av; + return ( + + ); + })} ); })} {runDetail && !diffsLoading && !pagedDiffs.length ? ( -
{t("bizCompare.resultEmpty")}
+
+ {kindFilter === "unchanged" + ? Number(runDetail?.summary?.unchanged || 0) > 0 && + !Number(runDetail?.summary?.unchanged_listed || 0) + ? t("bizCompare.unchangedNotStored") + : t("bizCompare.resultEmpty") + : activeSheetPending + ? t("bizCompare.runStatusRunning") + : t("bizCompare.resultEmpty")} +
) : null} @@ -3322,8 +4106,13 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage - {jobId ? ( ) : null} + ) : jobDetailTab === "runs" ? ( + ) : ( <> - )} diff --git a/web/src/services/api.ts b/web/src/services/api.ts index be84ad7..c4aec6b 100644 --- a/web/src/services/api.ts +++ b/web/src/services/api.ts @@ -2154,6 +2154,12 @@ export const bizCompareListRuns = (jobId: string, limit = 20) => export const bizCompareGetRun = (runId: string) => apiGet>(`/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`); +export const bizCompareCancelRun = (runId: string) => + apiPost>( + `/v1/biz-state/compare/runs/${encodeURIComponent(runId)}/cancel`, + {}, + ); + export const bizCompareDeleteRun = (runId: string) => apiDelete<{ ok: boolean; job_id?: string; run_id?: string }>( `/v1/biz-state/compare/runs/${encodeURIComponent(runId)}`, @@ -2164,6 +2170,8 @@ export const bizCompareListRunDiffs = (params: { metricId?: string; kind?: string; kw?: string; + /** Field filters e.g. { direction: "out", network: "1.1.1.1" } */ + qf?: Record; page?: number; pageSize?: number; }) => { @@ -2171,6 +2179,15 @@ export const bizCompareListRunDiffs = (params: { if (params.metricId) p.set("metric_id", params.metricId); if (params.kind) p.set("kind", params.kind); if (params.kw) p.set("kw", params.kw); + if (params.qf && Object.keys(params.qf).length) { + const cleaned: Record = {}; + for (const [k, v] of Object.entries(params.qf)) { + const key = String(k || "").trim(); + const val = String(v || "").trim(); + if (key && val) cleaned[key] = val; + } + if (Object.keys(cleaned).length) p.set("qf", JSON.stringify(cleaned)); + } p.set("page", String(Math.max(1, Number(params.page || 1)))); p.set("page_size", String(Math.max(1, Math.min(500, Number(params.pageSize || 100))))); return apiGet<{ @@ -2179,6 +2196,10 @@ export const bizCompareListRunDiffs = (params: { page_size: number; metric_id: string; items: Record[]; + source?: string; + truncated?: boolean; + live_before_matched?: number; + live_after_matched?: number; }>(`/v1/biz-state/compare/runs/${encodeURIComponent(params.runId)}/diffs?${p.toString()}`); };