diff --git a/.env.example b/.env.example index 23d7cfd..7ede38d 100644 --- a/.env.example +++ b/.env.example @@ -120,16 +120,8 @@ NETX_UME_NOTIFICATION_TOPIC=ALARM # bundled | external; unset = external (compatible with existing deploys) # NETX_BUNDLED_PG_PORT=15432 # NETX_BUNDLED_PG_DATA_DIR= -# Update: GitHub primary + Forgejo mirror (git.avelo.top); pick newest reachable -# NETX_UPDATE_SOURCES=github,forgejo -# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest -# NETX_UPDATE_GITHUB_REPO=hansjone/netx # NETX_UPDATE_URL= -# NETX_UPDATE_FALLBACK_URL= -# NETX_UPDATE_TOKEN= # NETX_UPDATE_CHANNEL=stable -# NETX_UPDATE_AUTO=false -# When true: tray/scheduled task may download+apply zip updates silently. # Heavier fleets: raise CLI/DB together; also ensure Postgres max_connections and bastion session limits. # One-click start (recommended): .\scripts\start_netx.ps1 -Background -WithWeb # -> API + netx_api.worker + N× netx_api.biz_state_worker + optional Vite diff --git a/.gitattributes b/.gitattributes index 67733fc..dfdb8b7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1 @@ *.sh text eol=lf -packaging/linux/** text eol=lf diff --git a/.github/workflows/release-linux.yml b/.github/workflows/release-linux.yml deleted file mode 100644 index 8eb23bc..0000000 --- a/.github/workflows/release-linux.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Release Linux - -on: - push: - tags: - - "v*" - workflow_dispatch: - -permissions: - contents: write - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: "20" - - - name: Check tag matches pyproject version - if: startsWith(github.ref, 'refs/tags/') - run: | - ver="$(sed -n 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' pyproject.toml | head -n1)" - [ "${GITHUB_REF_NAME}" = "v${ver}" ] || { echo "tag ${GITHUB_REF_NAME} != v${ver}"; exit 1; } - - - name: Cache PostgreSQL build + downloads - uses: actions/cache@v4 - with: - path: packaging/cache/linux - key: linux-x86_64-${{ hashFiles('packaging/linux/versions.env', 'packaging/linux/build_in_container.sh') }}-${{ hashFiles('requirements.txt') }} - restore-keys: | - linux-x86_64-${{ hashFiles('packaging/linux/versions.env', 'packaging/linux/build_in_container.sh') }}- - - - name: Build package - run: bash packaging/linux/build_linux_release.sh - - - uses: actions/upload-artifact@v4 - with: - name: netx-linux-x86_64 - path: | - packaging/release/netx-*-linux-x86_64.tar.gz - packaging/release/netx-*-linux-x86_64.tar.gz.sha256 - retention-days: 7 - - smoke: - needs: build - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image: - - centos:7 - - rockylinux:9 - - ubuntu:18.04 - - ubuntu:24.04 - - debian:12 - - opensuse/leap:15 - - openeuler/openeuler:22.03 - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: netx-linux-x86_64 - path: pkg - - name: Offline install smoke (${{ matrix.image }}) - run: | - tarball="$(ls pkg/netx-*-linux-x86_64.tar.gz)" - # Minimal images may not ship tar: unpack on the runner and mount both. - mkdir -p pkgx && tar xzf "$tarball" -C pkgx - docker run --rm --network none \ - -v "$PWD/pkg:/pkg:ro" -v "$PWD/pkgx:/pkgx:ro" \ - -v "$PWD/packaging/linux/smoke_test.sh:/smoke.sh:ro" \ - "${{ matrix.image }}" bash /smoke.sh "/pkg/$(basename "$tarball")" "/pkgx/$(ls pkgx)" - - release: - needs: smoke - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - steps: - - uses: actions/download-artifact@v4 - with: - name: netx-linux-x86_64 - path: pkg - # Release notes come from the Windows workflow; only attach files here. - - name: Upload to GitHub Release - uses: softprops/action-gh-release@v2 - with: - files: | - pkg/netx-*-linux-x86_64.tar.gz - pkg/netx-*-linux-x86_64.tar.gz.sha256 diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index 12b777e..12e2fa9 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -22,11 +22,10 @@ jobs: with: python-version: "3.12" - - name: Build release stage + - name: Build release zip shell: pwsh run: | - # Use pwsh -File (UTF-8). Nested Windows PowerShell 5.1 mis-parses UTF-8 scripts without BOM. - & ./packaging/build_release.ps1 -CreateVenv + powershell -ExecutionPolicy Bypass -File ./packaging/build_release.ps1 -CreateVenv - name: Install Inno Setup shell: pwsh @@ -38,18 +37,12 @@ jobs: shell: pwsh run: | $ver = (Get-Content pyproject.toml | Select-String '^\s*version\s*=').Line -replace '.*"(.+)"','$1' - if ($env:GITHUB_REF_NAME -ne "v$ver") { throw "Release tag $env:GITHUB_REF_NAME does not match pyproject version $ver" } ISCC.exe "/DMyAppVersion=$ver" packaging/installer/netx.iss - if ($LASTEXITCODE -ne 0) { throw "Inno Setup failed with exit code $LASTEXITCODE" } - $setup = "packaging/release/NetX-Setup-$ver.exe" - if (-not (Test-Path $setup) -or (Get-Item $setup).Length -lt 1MB) { throw "Missing or invalid installer: $setup" } - $hash = (Get-FileHash $setup -Algorithm SHA256).Hash.ToLowerInvariant() - Set-Content "$setup.sha256" -Value $hash -Encoding ascii -NoNewline - name: Upload to GitHub Release uses: softprops/action-gh-release@v2 with: files: | + packaging/release/NetX-*-win64.zip packaging/release/NetX-Setup-*.exe - packaging/release/NetX-Setup-*.exe.sha256 generate_release_notes: true diff --git a/.gitignore b/.gitignore index c72cfd0..92197b4 100644 --- a/.gitignore +++ b/.gitignore @@ -8,25 +8,16 @@ build/ dist/ scripts/.run/ # Local one-off probes / staging dumps (not for commit) -artifacts/ scripts/_* scripts/archive/ packages/netx-topology-mcp/tests/_* -_tmp_* -.idea/ -/ume/ +ume/ data/ne_collections/ data/webcrt/ data/auth/ data/runtime/ -data/ne_exec_jobs/ # Windows packaging artifacts (binaries / release trees) packaging/cache/ packaging/release/ packaging/postgres/pgsql/ -packaging/winsw/ *.exe -!packaging/installer/*.iss - -# Claude Code personal overrides -.claude/settings.local.json diff --git a/README.md b/README.md index 8a4d5e9..2ce7550 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,9 @@ API base: `http://127.0.0.1:8890/` After `npm run build` in `web/`, the API can also serve the UI at `http://127.0.0.1:8890/` (same origin). See `NETX_UI_DIST_DIR` in `.env.example`. -### Windows installer (optional) +### Windows installer / portable package (optional) -Fool-proof Windows delivery via `NetX-Setup-*.exe` (bundled or external Postgres, program/data split, offline upgrade over existing installs) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`. +Fool-proof Windows delivery (bundled or external Postgres, program/data split, manual update) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`. ### 6) MCP（Cursor / oclaw / Claude） diff --git a/alembic/versions/20261013_user_prefs.py b/alembic/versions/20261013_user_prefs.py deleted file mode 100644 index abd5d96..0000000 --- a/alembic/versions/20261013_user_prefs.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Add app_user.prefs JSON for per-user UI preferences. - -Revision ID: 20261013_user_prefs -Revises: 20260922_biz_state_queued -Create Date: 2026-10-13 -""" - -from __future__ import annotations - -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "20261013_user_prefs" -down_revision: Union[str, Sequence[str], None] = "20260922_biz_state_queued" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - bind = op.get_bind() - dialect = str(getattr(bind.dialect, "name", "") or "").lower() - if dialect.startswith("postgres"): - op.execute( - "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'::json NOT NULL" - ) - # Ensure NOT NULL + default for brownfield rows that already had a nullable column. - op.execute("UPDATE app_user SET prefs = '{}'::json WHERE prefs IS NULL") - op.execute("ALTER TABLE app_user ALTER COLUMN prefs SET DEFAULT '{}'::json") - op.execute("ALTER TABLE app_user ALTER COLUMN prefs SET NOT NULL") - else: - insp = sa.inspect(bind) - if "app_user" in insp.get_table_names(): - cols = {c["name"] for c in insp.get_columns("app_user")} - if "prefs" not in cols: - op.add_column( - "app_user", - sa.Column("prefs", sa.JSON(), nullable=False, server_default="{}"), - ) - - -def downgrade() -> None: - bind = op.get_bind() - dialect = str(getattr(bind.dialect, "name", "") or "").lower() - if dialect.startswith("postgres"): - op.execute("ALTER TABLE app_user DROP COLUMN IF EXISTS prefs") - else: - try: - op.drop_column("app_user", "prefs") - except Exception: - pass diff --git a/docs/biz-state-canonical-fields.md b/docs/biz-state-canonical-fields.md deleted file mode 100644 index 3670f0a..0000000 --- a/docs/biz-state-canonical-fields.md +++ /dev/null @@ -1,68 +0,0 @@ -# 割接对比：规范值契约 - -适用范围：业务对比（`BizCompare*`）和割接监控（`BizMigration*`）。两种模式用同一条身份流水线（`netx_api/biz_state/identity_pipeline.py`），跨厂商对比靠它把两侧设备的原始输出对齐到同一套“规范值”。 - -## 1. 流水线 - -同一行在某一侧（`before` / `after`）依次经过： - -1. **归一化**：接口字段去内部空格 → 厂商内规范名（`VENDOR_CANONICAL`，如华为 `GigabitEthernet`→`GE`）→ 模板别名规则（`iface_normalize_rules`，如 `GE`→`gei`）；键字段按 `field_rules[].normalize`（`lower`/`upper`/`mac`…）；再按 `field_rules[].value_map` 改写状态值。 -2. **行过滤**：`row_filters` 在归一化之后求值，所以按规范值书写；带 `side` 的过滤只作用于该侧。 -3. **端口映射**：只作用于 `before` 侧，键用 before 侧规则归一化、值用 after 侧规则归一化。 -4. **取键**：`scalar_text(value).strip()` 逐字段取值，得到 match key。 - -快照版本 `config_snapshot.version >= 2` 的 run 走完整流水线；`version == 1` 的历史 run 保持旧语义（过滤看原始行、键不归一化、同键按采集顺序配对），回填、检索、导出结果不变。 - -## 2. 规范值 - -比对层只认下面这些取值。设备原始值不同的，用 `value_map` 或 parser 输出规范值解决，**引擎里不写厂商特例**。 - -| metric | 字段 | 规范取值 | -|---|---|---| -| `interface_brief` | `admin` / `phy` / `prot` | `up` / `down`（华为 `*down` 记 `admin=down`） | -| `bgp_peer` | `state` | `established` / `idle` / `active` / `connect` / `opensent` / `openconfirm` | -| `bgp_peer` | `afi` | `ipv4` / `ipv6` / `vpnv4` / `vpnv6` / `evpn` / `vpls` | -| `arp` | `entry_type` | `dynamic` / `static` / `interface` | -| `isis_adjacency` / `ospf_neighbor` | `state` | `up` / `full` / `2way` / `down` / `init` / `idle` | -| `vrrp` | `af` | `ipv4` / `ipv6` | - -### 中兴（已实现）现状 - -中兴 parser 输出设备打印的原值，没有改写： - -- `interface_brief.admin/phy/prot`：`up` / `down`，已是规范值。 -- `arp.entry_type`：由 `age` 推断，`HH:MM:SS` 为 `dynamic`，否则 `static`。 -- `bgp_peer.state`：设备原值，如 `Established`（首字母大写）。与中兴自身对比时一致；与其他厂商对比时在模板里加值映射： - - ```json - {"field": "state", "value_map": {"established": "established", "estab": "established", "*down": "down"}} - ``` - - `value_map` 大小写不敏感匹配，输出为右侧的规范值；两侧都经过同一张表。 - -> 没有改动中兴 parser 的输出值：已采集批次里的值是原值，直接改成小写会让“旧批次 vs 新批次”出现假差异。需要统一时在模板里加 `value_map`。 - -### 其他厂商（待办） - -华为 VRP 的状态 parser（`interface_brief`、`arp`、`bgp_peer`、`isis_adjacency`、`ospf_neighbor`）**尚未实现**：仓库里没有真实输出样本，按规则不凭空编写 TextFSM。收到样本后按 `skills/biz-state/netx-biz-state/SKILL.md` 第 2 节逐项完成，输出与中兴相同的字段名，状态值输出上表的规范值，并给 profile 补 `slice`。 - -目前只有 LLDP 在跨厂商场景下有真实 parser。 - -## 3. 命令切片（`ParseProfile.slice`） - -一个采集命令产出 metric 的哪一部分，由 profile 的 `slice` 声明，例如 `zte.bgp_ipv4_summary` → `{"afi": "ipv4"}`、`zte.vrrp_ipv6` → `{"af": "ipv6"}`。比对表单（sheet）用顶层 `eq` 行过滤声明自己要的切片。 - -- 一侧没有任何命令覆盖该切片 → 该 sheet 状态 `not_collected`，不判业务缺失、不计入通过率。 -- 割接监控创建 HF 采集任务时，取两侧画像命令切片的并集；某侧找不到对应切片的 profile 时返回 `no_profile_for_slice:::`。 -- 没声明 `slice` 的命令视为覆盖整个 metric。 - -## 4. sheet 数据状态 - -| 状态 | 条件 | 通过率 | -|---|---|---| -| `ok` | 两侧采集正常 | 正常计算 | -| `no_data` | 两侧过滤后都是 0 行，命令都成功 | `null`，不计入总通过率 | -| `suspect_parse` | 某侧 `raw_line_count > 5` 但 `row_count == 0`，或全部命令失败 | 失败行照常显示，任务级 `ok = null` | -| `not_collected` | 某侧没有任何命令（或切片）产出该 metric | `null`，不计入总通过率 | - -没有任何命令回执的批次（导入或历史数据）无法判断，按正常处理。 diff --git a/docs/reviews/assets/biz-compare-config-mobile.png b/docs/reviews/assets/biz-compare-config-mobile.png deleted file mode 100644 index 29539dc..0000000 Binary files a/docs/reviews/assets/biz-compare-config-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-config.png b/docs/reviews/assets/biz-compare-config.png deleted file mode 100644 index 98fe764..0000000 Binary files a/docs/reviews/assets/biz-compare-config.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-desktop.png b/docs/reviews/assets/biz-compare-desktop.png deleted file mode 100644 index 20904ac..0000000 Binary files a/docs/reviews/assets/biz-compare-desktop.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-error.png b/docs/reviews/assets/biz-compare-error.png deleted file mode 100644 index 4396388..0000000 Binary files a/docs/reviews/assets/biz-compare-error.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-jobs.png b/docs/reviews/assets/biz-compare-jobs.png deleted file mode 100644 index ace1467..0000000 Binary files a/docs/reviews/assets/biz-compare-jobs.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-legacy.png b/docs/reviews/assets/biz-compare-legacy.png deleted file mode 100644 index a6c03ba..0000000 Binary files a/docs/reviews/assets/biz-compare-legacy.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-memory-benchmark.json b/docs/reviews/assets/biz-compare-memory-benchmark.json deleted file mode 100644 index 70290b7..0000000 --- a/docs/reviews/assets/biz-compare-memory-benchmark.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "date": "2026-10-10", - "baseline_ref": "afa8e06b2de64100f963ae30b3babfcb5d7009f4", - "python": "3.14.3", - "dataset": { - "rows_per_side": 1000000, - "fields": ["id", "neighbor", "state", "afi"], - "strata": 50, - "unchanged_limit": 5000, - "compact_unchanged": true, - "interface_transforms": false, - "all_rows_unchanged": true - }, - "measurement": "tracemalloc peak allocated during compare_rows; excludes input allocations, DB loading and process overhead", - "results": [ - { - "engine": "baseline", - "peak_engine_mib": 1144.43, - "seconds_with_tracemalloc": 92.4 - }, - { - "engine": "optimized", - "peak_engine_mib": 420.49, - "seconds_with_tracemalloc": 63.681 - } - ], - "results_and_sample_order_identical": true, - "test_result": "1 passed in 157.88s" -} diff --git a/docs/reviews/assets/biz-compare-narrow.png b/docs/reviews/assets/biz-compare-narrow.png deleted file mode 100644 index bfec41c..0000000 Binary files a/docs/reviews/assets/biz-compare-narrow.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-runs-mobile.png b/docs/reviews/assets/biz-compare-runs-mobile.png deleted file mode 100644 index 9e53762..0000000 Binary files a/docs/reviews/assets/biz-compare-runs-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-runs.png b/docs/reviews/assets/biz-compare-runs.png deleted file mode 100644 index 9042eca..0000000 Binary files a/docs/reviews/assets/biz-compare-runs.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-batches.png b/docs/reviews/assets/biz-state-batches.png deleted file mode 100644 index e6b7248..0000000 Binary files a/docs/reviews/assets/biz-state-batches.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-config-mobile.png b/docs/reviews/assets/biz-state-config-mobile.png deleted file mode 100644 index 19244fc..0000000 Binary files a/docs/reviews/assets/biz-state-config-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-config.png b/docs/reviews/assets/biz-state-config.png deleted file mode 100644 index a4eb2e0..0000000 Binary files a/docs/reviews/assets/biz-state-config.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-error.png b/docs/reviews/assets/biz-state-error.png deleted file mode 100644 index 62cd975..0000000 Binary files a/docs/reviews/assets/biz-state-error.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-tasks.png b/docs/reviews/assets/biz-state-tasks.png deleted file mode 100644 index be971dc..0000000 Binary files a/docs/reviews/assets/biz-state-tasks.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-workbook-mobile.png b/docs/reviews/assets/biz-state-workbook-mobile.png deleted file mode 100644 index 5b2a506..0000000 Binary files a/docs/reviews/assets/biz-state-workbook-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-workbook.png b/docs/reviews/assets/biz-state-workbook.png deleted file mode 100644 index cad4dee..0000000 Binary files a/docs/reviews/assets/biz-state-workbook.png and /dev/null differ diff --git a/docs/reviews/assets/compare-template-editor-mobile.png b/docs/reviews/assets/compare-template-editor-mobile.png deleted file mode 100644 index 3dc4212..0000000 Binary files a/docs/reviews/assets/compare-template-editor-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/compare-template-editor.png b/docs/reviews/assets/compare-template-editor.png deleted file mode 100644 index a6419a9..0000000 Binary files a/docs/reviews/assets/compare-template-editor.png and /dev/null differ diff --git a/docs/reviews/assets/cutover-monitor-board.png b/docs/reviews/assets/cutover-monitor-board.png deleted file mode 100644 index 9fe1dcf..0000000 Binary files a/docs/reviews/assets/cutover-monitor-board.png and /dev/null differ diff --git a/docs/reviews/assets/cutover-monitor-mobile.png b/docs/reviews/assets/cutover-monitor-mobile.png deleted file mode 100644 index e9e2c1b..0000000 Binary files a/docs/reviews/assets/cutover-monitor-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/cutover-monitor-wall.png b/docs/reviews/assets/cutover-monitor-wall.png deleted file mode 100644 index f35d5db..0000000 Binary files a/docs/reviews/assets/cutover-monitor-wall.png and /dev/null differ diff --git a/docs/reviews/biz-compare-review-2026-10-10.md b/docs/reviews/biz-compare-review-2026-10-10.md deleted file mode 100644 index 65e342e..0000000 --- a/docs/reviews/biz-compare-review-2026-10-10.md +++ /dev/null @@ -1,131 +0,0 @@ -# 业务对比检查与优化记录 - -检查日期：2026-10-10。范围：网络管理 → 割接对比 → 业务对比，包括前端任务/批次/结果界面、Python 与 PostgreSQL 对比路径、结果分页与搜索、短期列表缓存。 - -本次直接修改项目代码并添加回归测试，没有连接设备执行采集，也没有修改生产数据库。浏览器截图与交互测试使用固定 API 响应。 - -## 已修复的问题 - -### 1. 搜索会改变业务判定（P1） - -原实现分别按搜索条件过滤割接前、后的原表，再重新对比。例如 `state=up → down`，搜索 `up` 后只剩前侧，原本的“变化”就可能变成“删除”。重复身份键的业务还可能在筛选后错位配对。 - -现在：已存储的判定直接在结果表分页筛选；需要回查未保存全量明细的成功行时，先找到候选身份键，再补齐两侧的完整分组，按原有配对规则对比，最后筛选。达到读取上限的分组整体跳过，并返回截断标记，避免产生虚假的新增或删除。 - -### 2. PostgreSQL 搜索漏用部分模板过滤条件（P1） - -原搜索只下推 SQL 支持的过滤规则，对 `age_timer` 等不支持的规则直接忽略，可能将静态 ARP 等不属于模板范围的数据纳入结果。现在不支持下推的规则走流式读取并在 Python 中完整过滤。 - -### 3. 快速切换时旧请求覆盖当前任务（P1） - -前后采集任务的批次列表、打开任务、加载历史结果以及轮询缺少完整的过期保护。现在对批次请求使用清理标记，对任务与结果使用请求序号；关闭弹窗、切换路由或打开其他任务后，旧响应不能覆盖当前界面。 - -### 4. 结果加载错误容易被误读（P2） - -原查询失败时保留上一组明细，界面已经显示新筛选条件；切到未完成的表后，加载状态也可能无法结束。现在加载和失败都有独立状态，错误时清除旧明细并提供重试入口；切换表/页签后清理加载状态。 - -### 5. LLDP 搜索漏查（P2） - -LLDP 数据存于 `biz_state_lldp_neighbor`，原搜索只读 `biz_state_metric_row`。现在原表回查与精简结果的字段搜索都支持 LLDP 表。 - -### 6. 其他已修复边界 - -- 恰好命中读取上限时，原非 PostgreSQL 路径错误地报告“已截断”；现在多读一条判断。 -- 无效或有歧义的表标识可能落到其他表；现在返回 `sheet_not_found`。 -- `%`、`_` 等搜索字符按字面值匹配，避免不同查询路径含义不一致。 -- 关闭最后一项检查后，空数组被解释为“启用全部”；现在阻止关闭最后一项并提示至少保留一项。 -- 两个 HeroUI Input 使用了 `isDisabled`，实际需要 `disabled`；过滤值与容差输入的禁用状态已修正。 -- 列表缓存缺少并发合并，慢请求可以覆盖强制刷新结果或在失效后重新填充缓存；现在共享请求、保护失效状态并抑制过期的后台通知。 -- 清理中英文文案中的重复键，解决相关类型检查错误。 -- 操作后任务选择“同操作前”时原来没有加载批次列表；现在按实际使用的任务读取并共享缓存。切换采集任务时清除原批次选择，避免把上一个任务的批次带入配置。 -- 原通用下拉框嵌套了两个 label，标签不能正确关联控件；现在使用明确的 `htmlFor/id`，并关联提示文本。 - -## 性能改动与实际收益范围 - -- 合并两套轮询，下一轮在本轮请求完成后启动。模拟浏览器测试中，运行任务在 5.2 秒内历史列表和详情各请求 2–3 次（包含可能尚未完成的初始加载），没有重复轮询，也不会因慢请求持续叠加。 -- 已保存的失败、新增、变化等结果在数据库中筛选并分页，避免每次翻页重新加载和对比原始批次。回归测试明确验证这一查询路径不调用 `compare_rows`。 -- 同一前后采集任务的批次请求共用短期缓存，列表后台刷新合并同一请求；6 项缓存测试覆盖并发、强制刷新、失败重试与失效竞态。 -- 模板查找使用 Map，避免列表搜索与绘制逐项线性查找模板。 -- 修改筛选后先回到第一页，避免先请求旧页码再重复请求第一页。 - -这些改动减少了重复工作。以下后续基准使用合成数据，没有测量生产环境延迟。 - -## 后续优化（2026-10-10） - -### 历史执行配置与精简行还原 - -新运行在创建时将模板、启用的检查项、字段规则/过滤、接口归一化、端口映射、成功行保存策略和名称存入 `summary_json.config_snapshot`。后台执行、成功行回查、精简行还原以及导出共用该配置；排队之后修改配置，甚至删除原模板/映射，也不会改变这一运行使用的配置。无需数据库结构迁移。 - -修复了精简存储明细还原时将 `mapped_before` 直接设为原始 `before` 的问题。现在前侧依次归一化和映射，后侧归一化，保持与完整结果的显示及 CSV 一致。ZIP 新增 `config_snapshot.json`，便于核对当时配置；详情接口只返回快照模板与版本，不将完整端口映射加入轮询响应。 - -页面显示“执行配置已保存”；旧记录显示“旧记录：未保存执行配置”。旧运行无法可靠补造当时的配置，仍保留兼容查询，成功行回查及精简行还原会使用当前配置。快照也不保存原始采集行的副本，若原始批次数据被清理，依赖行 ID 的还原仍会缺少内容。 - -### Python 大表内存 - -成功样本按“层内序号、层首次出现顺序”排序，只保留样本上限内的候选和分层状态，避免先积累全部成功行。保留原分层轮询顺序；未截断的小表仍保持采集顺序。不需要接口变换时复用输入行；有变换时创建新行，不修改输入数据。稳定身份键顺序直接使用分组字典，省去再次计算全表键、额外键列表与集合。 - -每侧 1,000,000 行、50 个邻居分层、全部成功、保存 5,000 条精简样本的测试：优化前引擎新增分配峰值 **1144.43 MiB**，优化后 **420.49 MiB**，下降约 **63.3%**。完整结果与样本顺序的 SHA-256 一致。开启 `tracemalloc` 时耗时为 92.400 秒和 63.681 秒；该计时含内存追踪开销，不代表生产耗时。输入行在追踪开始前创建，峰值不包含输入行、数据库加载或 Python 进程的总内存。 - -基线为 `afa8e06b2de64100f963ae30b3babfcb5d7009f4` 中的对比引擎。原始指标见 [百万行测试结果](assets/biz-compare-memory-benchmark.json)。身份键分组仍需要随原表行数增长的内存；这次不是完全流式对比。 - -### 大表导出 - -导出改为按 `(seq, id)` 游标分批查询、分批还原精简行、逐行将 UTF-8 CSV 写入 ZIP，避免将全部明细与完整 CSV 同时保存在内存。相同 `seq` 的多条结果按 `id` 连续读取，不漏掉跨批次的同行序号记录。保留 BOM、字段顺序及逗号/引号/换行转义。最终压缩包仍由现有接口在内存中返回，压缩文件本身的大小仍计入内存。 - -## UI 改动 - -任务列表、创建向导、任务配置、对比批次和结果区统一使用深色平面、细边框、小圆角、低饱和蓝色和等宽数字。任务配置按作业参数、前后数据源、检查范围、端口映射四个区块组织；数据源左右配对，显示设备地址和采集数量。保存配置作为配置页主操作，删除操作降低视觉权重。 - -批次页增加最近 50 条记录的统计、刷新、加载、错误重试；刷新失败保留已载入记录。已完成只表示执行结束，业务失败/成功/新增数量单独显示。排队与运行使用不同文案，未知进度使用不确定进度条；时间、批次标识和前后采集信息便于核查。页签支持方向键及 Home/End，窄屏配置改为单列，批次表在独立容器中横向滚动。 - -结果区分为当前检查项/全表通过率、结果类型、搜索与辅助操作三层。增加明确的分页范围、清空筛选、加载状态、错误重试和搜索无结果提示。筛选按钮增加按下状态，字段筛选增加展开状态与键盘焦点提示。后续增加了执行配置保存状态及旧记录限制提示。 - -窄屏检查项导航改为紧凑的横向排列，避免一个检查项占据整行的大块区域。浏览器检查覆盖 1440px、768px、390px；筛选栏没有横向溢出。结果表保留必要的横向滚动。 - -- [任务列表](assets/biz-compare-jobs.png) -- [桌面任务配置](assets/biz-compare-config.png) -- [390px 任务配置](assets/biz-compare-config-mobile.png) -- [桌面对比批次](assets/biz-compare-runs.png) -- [390px 对比批次](assets/biz-compare-runs-mobile.png) -- [桌面结果区](assets/biz-compare-desktop.png) -- [768px 结果区](assets/biz-compare-narrow.png) -- [加载失败与重试](assets/biz-compare-error.png) -- [旧记录执行配置提示](assets/biz-compare-legacy.png) - -## 验证 - -- Python：87 项通过，1 项百万行基准默认跳过；单独启用百万行基准后通过。新增测试覆盖搜索、历史配置快照、有界抽样与游标导出。 -- 缓存：6 项 Node 测试通过。 -- 浏览器：27 个场景通过，包括任务列表文案、配置保存数据、同任务批次加载与切换清理、自动模式、检查项开关、创建向导、键盘页签、批次统计/状态/刷新/错误重试、配置保存状态、旧记录提示、结果布局、筛选、窄屏布局、旧任务响应保护、单套轮询、无运行时异常。 -- Vite 生产打包成功。 -- 本次业务对比页面的类型错误已经修复。全项目 `tsc -b` 仍被 `BizStatePage.tsx`、`BizMigrationPage.tsx` 中原有的类型/未使用变量错误阻塞。 -- 页面 ESLint 原有 34 个错误，主要是 `any` 与现有 React Hook 写法；批次列表增加类型后减少了 `any`，本次未增加错误。结果请求依赖警告已修正。通用下拉框和缓存模块没有 lint 错误。 -- PostgreSQL 路径验证了 SQL 编译与规则降级分支，没有在真实 PostgreSQL 数据库上执行查询或测量执行计划。 - -可重复执行的检查： - -```powershell -# 项目根目录 -.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare_export.py tests/test_biz_state_compare_snapshot.py tests/test_biz_state_compare_memory.py tests/test_biz_state_compare_search.py tests/test_biz_state_compare.py tests/test_biz_state_compare_sql.py tests/test_biz_state_compare_recovery.py -q - -# 可选：百万行基准，耗时约数分钟，内存占用较大 -$env:NETX_BENCH_ROWS = '1000000' -$env:NETX_BENCH_REF = 'afa8e06b2de64100f963ae30b3babfcb5d7009f4' -.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare_memory.py::test_memory_benchmark_against_git_baseline -q -s -Remove-Item Env:NETX_BENCH_ROWS, Env:NETX_BENCH_REF - -# web 目录 -node --experimental-strip-types --test tests/cutoverDataCache.test.mjs -node node_modules/vite/bin/vite.js build -node node_modules/typescript/bin/tsc -b -``` - -浏览器脚本为 `web/tests/bizCompare.browser.mjs`。先启动本地 Vite 服务，再设置 `NETX_PLAYWRIGHT_MODULE` 为可用的 Playwright 包路径；`NETX_TEST_URL` 默认是 `http://127.0.0.1:5179`，浏览器默认使用已安装的 Chrome。所有 `/v1` 请求由脚本提供固定响应，不会访问实际设备或数据库。 - -## 后续值得优化的部分 - -1. **Python 大表分组。** 有界抽样和冗余复制优化已完成，百万行基准下降约 63% 的引擎新增分配峰值。分组与输入仍随原表规模增长；进一步降低内存可研究排序流式配对或分区执行，并保持重复键配对语义。 -2. **PostgreSQL 搜索与深分页。** JSON 字段包含搜索和精确总数统计仍可能扫描大量结果。需要用真实数据查看 `EXPLAIN (ANALYZE, BUFFERS)`，再决定常用字段索引、搜索文本索引或游标分页；本次没有盲目添加生产索引。 -3. **历史配置与原始数据保留。** 新运行的执行配置快照已完成。旧运行仍无法还原未记录的当时配置；精简结果还原依赖原批次行，后续可为批次清理增加引用保护或保留策略。 -4. **重复键的业务语义。** 引擎按采集顺序配对同键行。如果业务应按集合判断，CLI 行顺序改变可能产生多条“不一致”。当前保留原有语义；可优先完善身份键，或另行实现并测试 Python/SQL 一致的集合匹配。 -5. **原表回查的数量上限。** 每侧候选命中最多 2000 条，完整分组补齐每侧预算 10000 条，最终展示最多 200 对；这不是全表匹配总数。界面会提示截断，大范围检索需要收窄条件。 diff --git a/docs/reviews/biz-state-review-2026-10-11.md b/docs/reviews/biz-state-review-2026-10-11.md deleted file mode 100644 index 7ca0e8e..0000000 --- a/docs/reviews/biz-state-review-2026-10-11.md +++ /dev/null @@ -1,54 +0,0 @@ -# 业务监控分析与优化记录 - -检查日期：2026-10-11。范围：网络管理 → 割接对比 → 业务监控，包含任务配置、周期采集状态、批次保护、工作簿、原始日志、筛选与导出。 - -本次修改项目代码并使用内存 SQLite 和固定浏览器 API 响应验证，没有连接设备执行采集，也没有修改生产数据库。未新增解析模板或数据库字段。前后端应一起更新，前端轮询使用新增的 `/v1/biz-state/tasks/{id}/progress` 接口。 - -## 已修复的问题 - -1. **旧请求覆盖当前内容。** 任务、用途筛选、批次工作簿、指标表、采集详情、参数发现和原始日志增加请求序号或取消保护。关闭弹窗、切换任务/表后，慢响应不能重新打开弹窗或覆盖新内容。原始日志下载保留其实际批次 ID，避免根据当前另一层弹窗推断来源。 -2. **后台采集不能及时反映、慢轮询叠加。** 同时观察服务端 `collect_running` 与当前页面启动的任务；周期任务和打开的任务保留低频查询，能够发现后台启动的采集。活动采集每轮完成后等待 4 秒，空闲等待 15 秒，再发起下一轮。跨 effect 重建共享在途标记，避免慢请求叠加。用途筛选隐藏的本地采集任务单独读取进度，避免误报采集结束。 -3. **刷新覆盖未保存配置、批次数突然缩减。** 轮询只合并运行标志和时间，不重新写入周期、保留策略或监控项。进度刷新和首次加载统一读取最多 200 个最近批次，界面明确显示已载入数量与上限。 -4. **加载失败看起来像空表或上一份数据。** 任务列表、任务配置、批次和指标表增加加载/错误/重试状态。指标查询失败清除旧行，不显示“本表无数据”；配置未就绪时操作区域不可交互。列表刷新失败保留当前行并显示错误。 -5. **正在采集的批次可能被删除或清理。** `queued/running` 批次加入 `active_collection` 保护，单条删除、批量删除、保留清理均保留它们；采集中的任务返回 409。前端同步禁用选择与删除，保留基线、业务对比及割接引用保护。 -6. **查询与导出边界错误。** `%`、`_`、反斜杠按字面值搜索；JSON 单列提取和 LLDP 分页排序更稳定。早于主命令保存的辅助命令失败保持可见。CSV 保留 `0`、`False`，正确引用逗号、双引号和 CR/LF；重复 `seq` 的导出不遗漏跨分块行，完整保留后续行才出现的列。 - -## 性能改动与验证范围 - -- 任务详情将逐监控项读取绑定改为一次批量读取。测试中 30 个监控项、60 个绑定由原来的 32 次查询降为 **3 次**；轻量进度接口为 **1 次**，不返回监控项和绑定。 -- 批次保护只读取引用 ID 列，按本次批次范围限制查询，不再加载所有对比/割接运行的配置和汇总 JSON。 -- 工作簿摘要不将原始 CLI 日志传输到 Python；只读取 SQL 计算的行数与是否存在日志。已有完整行数优先使用，原始日志在查看或导出时加载。回归使用 1 MB 日志，确认摘要结果集没有原文列。 -- 通用指标 CSV 先分块扫描列名，再按 `(seq, id)` 游标每块 1000 行写入 ZIP；LLDP 分批写入。减少指标表的 ORM 对象、字典和完整 CSV 同时驻留。**最终 ZIP 仍驻留内存，命令原文导出仍需要原文；这不是端到端流式下载。** 两次扫描保留完整列名，正在写入的批次仍不保证导出为一致性快照。 -- 同一厂商/型号的采集模板缓存 30 秒；轮询不重新读取模板。任务列表默认每页渲染 50 行，可选 20/50/100/200；列名去重使用 Set，命令搜索只计算一次。 -- 指标搜索先回到第一页并等待防抖值一致，避免先请求旧页码、旧关键字。 - -以上是查询次数、请求并发及读取方式的验证，没有测量真实设备或生产 PostgreSQL 的延迟与内存。任务列表 API 仍返回当前用途下的完整任务集合；客户端分页减少 DOM，不减少列表响应大小。JSON 全文模糊搜索、旧日志的 SQL 行数计算、非常大的 ZIP 内存仍是后续基准重点。 - -## 界面调整 - -统一深色工业风：连续指标栏、细边框、小圆角、等宽任务标识、紧凑表头及明确的异常提示。任务配置使用响应式网格；采集周期数值与单位并排；工作簿横向页签、过滤工具栏和表格支持窄屏滚动；底部按钮可换行，避免被裁切。任务和工作簿页签支持方向键、Home/End，过滤控件增加可访问名称。 - -“采集成功 / 部分完成 / 采集失败”描述采集流程，不能据此判断业务验收通过；业务状态需要查看数据表或业务对比结果。 - -- [任务列表](assets/biz-state-tasks.png) -- [任务配置](assets/biz-state-config.png) · [390px 配置](assets/biz-state-config-mobile.png) -- [采集批次](assets/biz-state-batches.png) -- [工作簿](assets/biz-state-workbook.png) · [390px 工作簿](assets/biz-state-workbook-mobile.png) -- [查询失败与重试](assets/biz-state-error.png) - -## 验证 - -- 后端相关回归 **124 项通过**：新增监控服务、保留策略、日志导入、命令导出，以及停止/恢复/收尾/认领采集、双车道、spool、解析池与业务对比相关回归。 -- `web/tests/bizState.browser.mjs`：**30 项回归通过**。固定 120 个任务、54 个批次、120 行指标，覆盖分页、配置保存、模板缓存、过期响应、搜索、重试、日志下载、后台采集和窄屏布局。4500ms 列表延迟下最大并发轮询为 1；轮询阶段任务完整详情仅首次打开请求 1 次，未保存的周期值保持不变。全部 `/v1` 请求被固定响应拦截。 -- `node node_modules/vite/bin/vite.js build` 通过。项目原有主包超过 500 kB 提示仍在。 -- `node node_modules/typescript/bin/tsc -b`：本页已有类型问题已修复；全项目仍被 `BizMigrationPage.tsx` 的 4 个既有错误阻断（两处未使用变量、两处字符串传给数值控件）。 -- 本页 ESLint 从 16 个错误降为 15 个，保留既有 `any` 和 effect 状态同步问题及 1 个依赖警告；不宣称全项目 lint 通过。 - -浏览器回归运行方法（先启动 Vite，可通过环境变量指定 URL、截图目录和浏览器）： - -```powershell -cd web -node node_modules/vite/bin/vite.js --host 127.0.0.1 --port 5179 -# 另一个终端；Playwright 已安装时直接运行，否则设置 NETX_PLAYWRIGHT_MODULE 为其模块路径。 -node tests/bizState.browser.mjs -``` diff --git a/docs/reviews/compare-templates-review-2026-10-11.md b/docs/reviews/compare-templates-review-2026-10-11.md deleted file mode 100644 index 6c62f0d..0000000 --- a/docs/reviews/compare-templates-review-2026-10-11.md +++ /dev/null @@ -1,46 +0,0 @@ -# 对比模板与业务对比评估、优化记录 - -日期：2026-10-11。范围：网络管理 → 割接对比 → 对比模板、业务对比。延续此前的业务对比和业务监控优化。本轮使用内存 SQLite、SQL 编译断言和本地浏览器固定 API 响应验证，没有连接真实设备，也没有改动生产数据库。 - -## 修复的结果准确性问题 - -1. **模板导入可能只保存有效部分。** 前后端检查全部检查项，空 Key、重复 ID、无效规则不再被静默丢弃。服务器在修改模板之前校验，错误响应包括配置路径和原因。执行时也检查历史模板的原始检查项，防止旧的无效配置被归一化后跳过。 -2. **错误过滤规则可能放行数据。** 新写入与执行入口拒绝未知运算符、混合分组/叶节点、空分组、无效正则及超过 12 层的嵌套。集合过滤要求非空数组；绝对/相对容差必须为有限非负数。无效归一模式、重复字段规则和接口别名同样报错。 -3. **复杂嵌套过滤被可视化编辑器截断。** 编辑器只接管能够完整表达的 OR/AND 结构，其余结构完整保留并显示 JSON。保存、导入、导出不删除嵌套条件。修改复杂结构可导出 JSON 后编辑并重新导入。 -4. **端口变化策略在 API 和导入导出中丢失。** `ignore_port_changes` 贯通请求模型、兼容单表配置、编辑、复制、保存、导入与导出。界面明确提供自动判断、接口必须匹配、允许端口变化三种选项；显式 `false` 不再丢失。 -5. **零、布尔和空值混淆。** Python 行键与过滤保留 `0`、`False`，布尔文本与 JSON 提取一致；数值比对不再把空值当成 0。Python/SQL 使用同一数值格式，支持 `+.5`、`1.`、科学计数法；解析不了的值按文本比较。 -6. **SQL 过滤与行键误匹配。** `contains` 的 `%`、`_`、反斜杠按字面值处理，空包含条件与 Python 一致。复合键改为 JSON 数组文本，避免 `('a|b','c')` 和 `('a','b|c')` 被拼成相同键。 -7. **任务范围失效导致隐式扩大或缩小对比。** 编辑任务时保留已移除的检查项 ID，显示错误并禁用保存、执行；用户可明确重新选择或启用当前全部检查项。服务器在保存和执行时都检查 ID，部分有效、部分失效的范围也会被拒绝。 -8. **不完整采集被解释成业务变化。** 比对入口只接受两侧状态为 `success` 的源批次；排队、执行中、失败、取消和部分完成批次返回 `source_batch_not_complete`。界面禁用这些选项。成功批次没有某指标或没有行的原有处理保持不变；本轮没有新增逐指标采集完备性判定。 -9. **删除破坏引用或在途对比。** 已被任务引用的模板不能删除；存在排队/执行中记录的任务及活动记录不能删除。模板页显示任务引用数量。这些是 API 状态检查，未新增数据库外键或锁，不宣称消除跨请求的全部并发窗口。 -10. **重新启用忽略字段仍然无效。** 勾选参与比对时清除遗留的 ignore/skip/off 规则。字段表保留模板配置的自定义字段；过滤专用字段也能显示。兼容数值、相对偏差和归一规则别名的显示。 - -## 性能与界面 - -- 检查项排序从逐表、逐批次查询改为 **一次按批次/metric 聚合**。相同 metric 的拆分表共享统计，SQL 不投影原始 CLI 日志；回归验证查询次数为 1。 -- `batch_metric_collect_ok` 只读取解析状态和行数，不加载命令原文。 -- 模板引用数量由一次 grouped 查询计算，避免每行请求。默认模板初始化/升级仍执行原来的查询。 -- 模板页面只请求模板和字段目录，省去业务监控任务、端口映射、对比任务三个列表；字段目录缓存 60 秒。业务对比与模板列表使用独立缓存键。 -- 两个列表默认每页渲染 50 行，可选 20/50/100/200；模板 ID 查找使用 Map。**API 仍返回完整列表，客户端分页减少 DOM，不减少响应体。** -- 模板编辑器使用深色平面面板、细边框、小圆角、蓝色选中态、等宽字段和固定表头。接口归一化折叠，检查项导航与规则区并排；窄屏导航横向滚动，操作按钮独立一行，过滤控件可换行，底部保存始终可见。 -- 导航和规则页签支持方向键、Home/End；规则控件增加可访问名称。保存期间编辑区不可交互，并阻止关闭后打开新模板造成保存响应串扰。 - -没有测量生产 PostgreSQL 的耗时或内存；本轮 SQL 变更由生成 SQL/绑定参数断言验证，未执行真实 PostgreSQL 集成回归。极大数值指数的 SQL 浮点转换、Python 与 PostgreSQL 正则方言差异、全文 JSON 搜索和大型 ZIP 内存仍需单独基准。既有回退机制不等于所有 SQL/Python 语义已完全一致。 - -## 验证与预览 - -- 后端相关回归：**120 项通过，1 项跳过**。跳过的是需显式开启的百万行压力测试。覆盖模板校验、原子更新、旧记录执行保护、端口策略、过滤、数值、复合键、查询次数、删除保护、范围失效、源批次状态，以及既有对比、搜索、配置快照、导出、恢复与内存相关回归。 -- Node 测试：**14 项通过**（8 项模板模型、6 项缓存）。 -- 浏览器：**53 个场景通过**（原业务对比 27 项、模板新增 26 项）。全部 `/v1` 请求使用固定响应，120 个模板列表验证默认 50 行上限；覆盖导入导出、复杂过滤、端口策略、负容差、忽略字段恢复、保存期间交互锁定、范围修复、键盘和 390px 布局。 -- Vite 生产构建通过；原有主包超过 500 kB 提示仍在。 -- 全项目 TypeScript 检查仍有 `BizMigrationPage.tsx` 的 4 个既有错误：2 个未使用变量、2 处字符串传给数值控件；本轮没有新增错误。 -- 新模板模型 ESLint 通过；业务对比页保留原来的 **23 个错误**（17 个 any、6 个 effect 状态同步），与修改前数量一致。不宣称全项目 lint 通过。 - -前后端应一起更新；新增 API 字段为模板引用数量与端口策略，无数据库迁移。原来使用标量 `in/not_in`、未知规则、负容差的模板需要修正再保存/执行；部分完成的源批次需要重新完成采集后再对比。 - -- [桌面模板编辑器](assets/compare-template-editor.png) -- [390px 模板编辑器](assets/compare-template-editor-mobile.png) -- [业务对比列表](assets/biz-compare-jobs.png) -- [此前业务对比记录](biz-compare-review-2026-10-10.md) - -复现：先在 `web` 启动 `node node_modules/vite/bin/vite.js --host 127.0.0.1 --port 5179`，另一个终端运行 `node tests/bizCompareTemplates.browser.mjs` 和 `node tests/bizCompare.browser.mjs`。可用 `NETX_PLAYWRIGHT_MODULE` 指定已有 Playwright，`NETX_TEST_URL` 指定本地地址，`NETX_TEST_OUTPUT` 指定截图目录。 diff --git a/docs/reviews/cutover-optimization-plan-2026-10-12.md b/docs/reviews/cutover-optimization-plan-2026-10-12.md deleted file mode 100644 index cebc1c7..0000000 --- a/docs/reviews/cutover-optimization-plan-2026-10-12.md +++ /dev/null @@ -1,477 +0,0 @@ -# 割接对比（业务对比 / 割接监控）问题分析与优化计划 - -日期：2026-10-12。范围：网络管理 → 割接对比下的 5 个子模块：业务监控（采集）、业务对比、割接监控、对比模板、监控模板。 -用途：交给 Cursor 按任务逐项实施；Claude 负责验收。 - -本文基于代码阅读和本地脚本验证，没有连接真实设备，也没有在真实 PostgreSQL 上执行。标了 **[已验证]** 的问题已经用脚本复现（脚本放在会话 scratchpad，结论见各条目）。基线：开始修改前，相关后端回归 **279 项通过**，`tsc -b` 通过，`cutoverDataCache` / `compareTemplateModel` Node 测试通过。 - ---- - -## 0. 结论摘要 - -1. **两种模式共用一套“归一化 + 映射”概念，但各个代码路径的实现不一致。** 割接监控会先对端口映射的键和值做接口归一化，业务对比不会（**[已验证]**：按设备原始名称填写映射，业务对比全部变成“删除 + 新增”）。行过滤用原始值，映射作用在归一化之后。键字段不受 `field_rules` 归一化影响。跨厂商割接最容易在这里产生误报。 -2. **目前只有两种映射：接口类型别名（模板级，两侧同一套规则）和端口映射（before→after，一对一）。** 没有分侧（按厂商）归一化，没有字段值映射（状态枚举、对端 IP、VRF 改名），也无法提前检查映射覆盖率。映射漏配会直接表现为业务“缺失”，用户很难判断是映射问题还是业务问题。 -3. **除 LLDP 外，状态采集解析只实现了中兴。** 华为、思科等的 parser 都是会抛出 `NotImplementedError` 的空壳，且没有注册。内置对比模板（`ZTE status default`）里的过滤和容差也都按中兴设定。所以现在的“跨厂商映射”只对 LLDP 真正有效，其余检查项在跨厂商场景下要么采集不到，要么全量误报。 -4. **判定语义有几处会把正常割接判成失败：** - - 端口预设要求旧端口 `admin=down 且 phy=down`。现场常见的做法是拔纤但不 shutdown，这种情况验收时会判为“未完成/红”（**[已验证]**）。 - - 两侧都是 0 行时通过率是 100%（空对空误判为通过）。 - - 解析不到数据会被当成业务缺失。 - - 新旧两侧的采集命令集不一致时，会被当成业务丢失。 - - 内置模板对 `output_bps` 用绝对值容差 5 bps，对 `rx_power`（dBm）用百分比容差。 -5. **割接监控的运行数据无限增长。** 每个 `BizMigrationRun` 或 `BizCompareRun` 引用的采集批次都会被永久保护，run 和 diff 本身也没有保留策略。60 秒一轮的高频监控跑几天，数据库和磁盘会持续膨胀。 -6. **性能：** 业务对比只要配置了端口映射，所有检查项（包括不含接口字段的 BGP 路由百万行表）都会退回 Python 引擎。割接验收是同步 HTTP 请求，会把路由表全量载入内存。 -7. **引用保护有漏洞：** 被监控模板引用的对比模板、被任务或割接项目引用的端口映射都能被删除。删除后，业务对比会静默地按“无映射”执行；割接监控会直接报错停摆。 - ---- - -## 1. 现状框架梳理（给 Cursor 的背景） - -### 1.1 数据流 - -``` -业务监控任务(biz_state_task, 每台设备) - └─ 采集批次 batch ──> 每条命令 batch_command(parse_status/row_count/raw_line_count) - └─ 解析(ParseProfile → parser → FieldDef) ──> biz_state_metric_row(data_json) / biz_state_lldp_neighbor - │ -对比模板(biz_compare_template.metrics_json = 多个 sheet) - sheet: metric_id, key_fields, iface_fields, compare_fields, display_fields, - row_filters, field_rules(normalize/compare/tolerance), ignore_port_changes - 模板级: iface_normalize_json（接口类型别名，两侧同一套） - │ -┌───────────── 业务对比（BizCompareJob/Run/Diff）─────────────┐ -│ before 批次 vs after 批次（两台设备或同一设备前后） │ -│ 行过滤(原始值) → 接口归一化(两侧) → 端口映射(仅 before) │ -│ → 按 match_key 分组 → 同键按采集顺序 zip 配对 │ -│ → added/removed/changed/unchanged → 通过率 │ -│ 引擎：PG + 可下推 → SQL FULL OUTER JOIN；否则 Python │ -└─────────────────────────────────────────────────────────────┘ -┌───────────── 割接监控（BizMigrationProject/Batch/Run/Diff）──┐ -│ 旧设备: 画像基线 vs 高频当前 → old_kind │ -│ 新设备: 画像基线 vs 高频当前 → new_kind │ -│ 预期集(expect_set: 本批要迁移的端口/对象) + 监控模板(成功/异常规则)│ -│ → dual_verdict: migrated/migrating/anomaly/lost/unfinished… │ -│ → 本批完成(finish_batch)=终验 → 红单(带红继续) │ -│ 高频采集：每个 metric 一对(新/旧)任务，同轮次配对采样 │ -└─────────────────────────────────────────────────────────────┘ -``` - -### 1.2 现有“映射/归一化”层与使用点（不一致的根源） - -| 层 | 归属 | 作用对象 | 业务对比 | 业务对比·实时检索/回填/导出 | 割接监控 | -|---|---|---|---|---|---| -| 接口类型别名 `iface_normalize_json` | 对比模板 | `iface_fields`，两侧同规则，只替换前缀 | 有 | 有（`_run_transform_config`） | 有 | -| 端口映射 `biz_port_mapping` | 任务 / 项目 | `iface_fields`，before→after，精确匹配 + 父接口继承 | **键/值不归一化** | 同左 | **键/值先归一化**（`service.py:1019`） | -| 行过滤 `row_filters` | sheet | **原始值**（归一化之前） | 有（PG 可下推） | 有 | 有（Python） | -| 字段规则 `field_rules.normalize` | sheet | **仅 compare 字段**，键字段不处理 | 有 | 有 | 仅完整性校验 | -| 监控模板 `sheet_overrides` | 监控模板 | 状态语义（up/down 值、成功/异常模式） | — | — | 有 | - -关键文件： -- 引擎：`netx_api/biz_state/compare_engine.py`、`compare_rules.py`、`iface_normalize.py`、`compare_sql.py` -- 业务对比服务：`netx_api/biz_state/compare_service.py`（约 4000 行） -- 割接监控：`netx_api/biz_migration/evaluate.py`、`service.py`、`monitor_templates.py`、`auto_monitor.py` -- 采集：`netx_api/biz_state/collect_runner.py`、`claim.py`、`profiles.py`、`parsers/*` -- 保留：`netx_api/biz_state/retention.py` -- 前端：`web/src/features/cutover/BizComparePage.tsx`、`BizMigrationPage.tsx`、`BizMonitorTemplatesPage.tsx`、`BizStatePage.tsx` -- 内置模板：`netx_api/biz_state/data/default_zte_status_template.json` - ---- - -## 2. 问题清单 - -严重度：**P0** 会产生错误结论或数据损坏；**P1** 高概率误报或明显性能/可用性问题；**P2** 边界问题或体验问题；**P3** 清理项。 - -### 2.1 业务对比（BizCompare） - -| ID | 严重度 | 问题 | 位置 | 影响 | -|---|---|---|---|---| -| B-01 | P0 **[已验证]** | 端口映射的键/值不按模板接口归一化规则转换，必须由用户手工填写“归一化后”的名称；割接监控却会自动归一化。两种模式行为不一致。 | `compare_engine.py:224-260`（`apply_port_map` 用原始 `pmap`），`compare_service.py:2461`、`_run_transform_config:2337` | 模板有 `GE→gei` 规则时，映射 `GE0/1/0/1→gei0/2/0/1` 完全不命中，结果 added=1 / removed=1、`map_ok=False` | -| B-02 | P1 | 任务里只要有端口映射，所有 sheet 都会跳过 SQL 引擎，包括没有接口字段的 BGP/路由表 | `compare_sql.py:165`（`if port_map: return "port_map"`） | 百万行 vpnv4 路由退回 Python，内存和耗时都大幅上升；割接任务几乎都会配映射 | -| B-03 | P1 | 端口映射被删除后，业务对比静默地按“无映射”执行（还会触发自动忽略端口的启发式），删除时也不检查引用 | `delete_mapping:1603`，`_port_map_dict:1612`，`_validate_compare_job:2220` | 结果语义悄悄改变 | -| B-04 | P1 | 对比模板删除只检查 `BizCompareJob` 的引用，不检查 `BizMonitorTemplate.compare_template_id` | `delete_template:1521`，`list_templates:1391` 的 `job_count` | 割接窗口中删掉模板后，`run_evaluate` 报 `compare_template_missing`，监控停摆 | -| B-05 | P1 | 创建向导或任务配置里编辑了映射文本但没点“保存映射”，创建/保存/执行任务时映射文本被丢弃，没有任何提示 | `BizComparePage.tsx:1814 jobConfigBody`、`1895 createJob`、`1997 saveJobConfig`、`2029 runNow` | 用户以为配了映射，实际没生效，产生大量假“缺失/新增” | -| B-06 | P2 | 映射文本解析只认 `, | Tab` 作分隔符；空格分隔或格式不对的行被静默丢弃，没有解析计数或错误提示 | `BizComparePage.tsx:1720 parseMapRows` | 映射少行而不自知 | -| B-07 | P1 | 映射校验只针对“第一个含接口字段的 sheet”，结果以原始 JSON 展示；只报告“映射键未命中”，不报告“before 侧端口没有映射且在 after 侧不存在”（漏配） | `validate_mapping:1810`，`mapping_stats`（`compare_engine.py:100`），`BizComparePage.tsx:2272` | 漏配映射最终表现为业务缺失 | -| B-08 | P1 | `row_filters` 作用在原始行（归一化、映射之前）；跨厂商时，同一条过滤在两侧含义不同 | `_load_metric_rows:1619`（SQL/Python 都在加载时过滤），`_run_sheet:2179` | 例如 `interface contains gei` 会把华为 `GE0/0/1` 整侧滤掉；ARP 的 `age_timer`（HH:MM:SS）只适用于中兴 | -| B-09 | P2 | 键字段不应用 `field_rules.normalize`（如 MAC 格式），跨厂商的 MAC/大小写差异导致配对失败 | `compare_engine.py:96 row_key`，`compare_sql.py _rk_sql` | 键里含 MAC 的 sheet（如 `l2vpn_mac`）全量误报 | -| B-10 | P1 | 两侧都是 0 行时通过率为 100%；解析不到数据（命令执行了但解析 0 行）被当成业务“缺失” | `_enrich_summary:3015`，`compute_pass_rate:210` | 空对空误判为通过；模板不匹配被当成业务丢失 | -| B-11 | P1 | 新旧两侧采集命令集不一致（例如旧设备采了 bgp ipv4 summary，新设备没采），对比结果是“全部缺失” | 没有任何检查 | 采集配置问题被误判为业务问题 | -| B-12 | P2 | 同键多行按采集顺序 zip 配对，CLI 输出顺序变化会产生假“不一致” | `compare_engine.py:331+`、`compare_sql.py` 中 `dup_rn ORDER BY seq` | ECMP/多路径场景误报 | -| B-13 | P2 | 只要提供了映射，`ignore_port_changes` 就被忽略；部分映射时，未映射端口既不按名称匹配也不忽略端口 | `compare_engine.py:224` | 部分映射时策略不可控 | -| B-14 | P2 | 结果表的键列只显示映射后的名称，看不到 before 侧原始端口名 | `BizComparePage.tsx:4161`（`d.key[k] ?? pre[k]`，`pre` 取 `mapped_before`） | 跨厂商时现场人员无法对照原端口 | -| B-15 | P2 | 判定没有方向：状态 down→up、前缀数增加也算失败；流量计数器被当成严格比较项 | `compare_rules.values_equal` | 割接后业务恢复或流量迁入被判为失败 | -| B-16 | P3 | `batch_metric_collect_ok` 是死代码；`_load_metric_rows` 中 LLDP 分支重复过滤；PG 下推时 `before_raw_count` 实际上是过滤后的数量 | `compare_service.py:78`、`1675-1688`、`2199` | 清理 | -| B-17 | P3 | 防止同一任务并发对比的锁是进程内的；`_job_has_active_run` 检查和插入不是原子操作。biz_state worker 多副本时，自动对比有极小概率并发 | `compare_service.py:57`、`2887` | 低概率 | - -### 2.2 割接监控（BizMigration） - -| ID | 严重度 | 问题 | 位置 | 影响 | -|---|---|---|---|---| -| M-01 | P0 **[已验证]** | 端口预设的成功条件要求旧侧 `removed` 或 `admin=down 且 phy=down`。旧端口拔纤但没有 shutdown（admin up / phy down）、新端口已 up 时：窗口内是 `migrating/黄`，**验收时是 `unfinished/红`** | `monitor_templates.py:255-275`（`_success_stateful` 的 `old_down_groups` 用的是 AND） | 正常割接被判为验收失败 | -| M-02 | P1 | 每个 run 引用的 HF 批次都会被永久保护，run 和 diff 本身也没有保留策略 | `retention.py:65-75` | 高频监控期间数据库无限增长 | -| M-03 | P1 | 验收（`finish_batch`）和手动评估是同步 HTTP 请求；路由类 sheet（`bgp_route`/`ip_route`）会把 4 个批次的全量行载入 Python | `biz_migration_router.py` 的 evaluate/finish，`service.py:1148-1163` | 超时、内存暴涨；而且这期间一直持有项目行锁（`with_for_update`，`service.py:940`） | -| M-04 | P1 | 默认监控模板会采集对比模板里的全部 metric，包括只能手动采集的路由表。没有手动采集时，该 sheet 是 `current_missing`，验收**必然**失败（`current_missing` / `coverage_incomplete`） | `resolve_collect_metric_ids:525`、`finish_batch:2358-2371`、`MANUAL_ROUTE_METRICS` | 默认配置跑不通验收 | -| M-05 | P1 | 只要任意 sheet 有重复键，验收就判失败（`duplicate_keys`，同时 `coverage_complete=False`），不管这个 sheet 是否在本批范围内 | `service.py:1299`、`2366` | 例如 ISIS L1/L2 两行同键，就永远过不了验收 | -| M-06 | P1 | 新旧两侧画像任务的命令集可能不同，HF 任务沿用各自画像的命令项，旧侧有、新侧没采的切片会被判为 `lost` | `_portrait_metric_items:1741`、`ensure_highfreq:1919` | 采集配置问题被当成业务丢失 | -| M-07 | P1 | 创建项目时不校验基线批次是否属于该侧画像任务（patch 时会校验） | `create_project:686-691`（对比 `patch_project:744-750`） | 可能拿错设备的基线；直到评估时才报 `baseline_task_mismatch` | -| M-08 | P2 | 不支持同设备割接（板卡更换、同框端口迁移）：`collect_pair_same_device` 直接报错 | `service.py:2016` | 常见场景无法使用 | -| M-09 | P2 | 每个 metric、每台设备一个 HF 任务；15 个检查项就是每分钟每台设备约 15 次 SSH 登录；没有单设备并发上限 | `ensure_highfreq:1943`，`claim.py` | VTY 耗尽、AAA 日志刷屏，进而采集失败，看板大量 `current_missing` | -| M-10 | P2 | 每轮自动评估都重新载入基线全量行（基线不会变） | `run_evaluate.metric_rows:1059` | 白白消耗 CPU/IO | -| M-11 | P2 | 双侧评估里，诊断索引的键串用 `str(v)` 构造，而其他地方用 `scalar_text().strip()`：`None` 变成 `"None"`，`True` 变成 `"True"`，空白不裁剪，预期键匹配不上 | `evaluate.py:759`、`compare_engine.py:273 _key_obj` | **[已验证]**：键字段为 `None` 时，存在且未变化的对象被判为 `unfinished` | -| M-12 | P2 | 用 metric 名称包含 `bgp` / `pw` 来决定是否用端口映射推导范围（字符串启发式） | `evaluate.py:804` | 新增 metric 时行为不可预期，也违反“引擎不写 metric 特例”的原则 | -| M-13 | P2 | 映射允许 N:1 吗？业务对比允许（结果是重复键），割接监控直接报 `mapping_target_ambiguous`；1:N（拆分 breakout）两边都不支持（`before_if` 唯一） | `models/biz_state.py BizPortMappingRow`，`service.py:991/1021` | 端口汇聚、拆分割接无法表达，报错也没说明是哪几行 | -| M-14 | P3 | `project_to_dict`（GET/列表）里会写库（`resolve_project_monitor_template` 持久化默认模板）；`list_projects` 存在 N+1 | `service.py:486-503`、`543` | 只读接口有副作用 | -| M-15 | P3 | 一个键在后续验收中已经变绿，对应红单仍然保持 open，只能人工关闭 | `_persist_red_tickets_from_run:2326` | 红单堆积 | - -### 2.3 内置模板与跨厂商能力 - -| ID | 严重度 | 问题 | 位置 | -|---|---|---|---| -| T-01 | P1 | `interface_detail.output_bps` 用 `numeric` 绝对容差 5（bps），而 `input_bps` 用 5%：相当于对出方向流量做严格比较 | `default_zte_status_template.json` | -| T-02 | P1 | `optical_brief.rx_power` 用 `percent 5`：dBm 是对数单位，-0.5 和 -0.6 已经差 20%。应改为绝对值 dB 容差 | 同上 | -| T-03 | P2 | 流量计数器（bps/util）默认参与比较；割接后流量迁移是预期行为，会产生大量噪声 | 同上 | -| T-04 | P2 | `l2vpn_mac` 键里的 `ac_port` 是接口，但没放进 `iface_fields`，端口映射不生效 | 同上 | -| T-05 | P1 | 华为、思科、华三、Juniper、诺基亚、爱立信的状态 parser 都是空壳（没有注册）；只有 LLDP 跨厂商可用 | `parsers/huawei/__init__.py` 等 | -| T-06 | P2 | 各厂商的状态值不统一（`Established`/`ESTAB`、`UP`/`up`/`*down`、`Full/DR`），而对比层只有 `lower`/`upper`/`mac` 几种归一化，没有枚举别名映射 | `compare_rules.normalize_value` | - ---- - -## 3. 优化方案（分阶段任务，按顺序执行） - -### 通用执行约束（Cursor 每个任务都要遵守） - -1. **引擎里不写 metric 特例**：所有 metric 相关的行为放进模板、sheet 或 profile 声明（遵循 `skills/biz-state/netx-biz-state/SKILL.md` 第 4、5 节）。 -2. **Python 和 SQL 必须一致**：凡是影响配对或判定的改动，要么两个引擎都实现，要么在 `sql_compare_skip_reason` 里让 SQL 自动降级到 Python，并补“同输入两引擎结果一致”的对照测试（SQL 侧至少断言生成的 SQL 和降级原因）。 -3. **历史 run 不变**：新增的执行配置写进 `config_snapshot`，并把 `version` 升到 2。`version==1` 和旧记录的回填、检索、导出保持原行为，不能因为新逻辑改变历史结果。 -4. **数据库结构**：尽量用 JSON 字段扩展，不加列。确实需要加列或表时，同时补 `alembic/versions/` 和 `netx_api/schema_patches.py`（参照现有做法），并写回归测试。 -5. **每个任务**：补单元测试；跑第 5 节的回归命令；前端改动跑 `tsc -b` 和 `vite build`，涉及交互的补 `web/tests/*.browser.mjs` 场景（全部 `/v1` 用固定响应）。 -6. 一个任务一个提交，提交信息写清楚任务 ID（例如 `fix(cutover): B-01 normalize port-map keys`）。 - ---- - -### 阶段 A：正确性快修（P0/P1，改动小，先做） - -#### A1 端口映射键/值统一归一化（B-01） -- 新增函数 `normalize_port_map(port_map, rules) -> (normalized_map, conflicts)`，放在 `iface_normalize.py`：键和值都用 `normalize_iface_name` 处理。归一化后键冲突（两个原始键归一后相同但目标不同）要作为 conflicts 返回。 -- 业务对比：在 `_run_transform_config`（`compare_service.py:2337`）统一处理，覆盖执行、实时检索、回填、导出所有路径。**只对 `config_snapshot.version>=2` 的 run 生效**；`version==1` 和旧记录保持原样。 -- 割接监控：`run_evaluate` 和 `list_baseline_expect_objects` 里的内联归一化改为调用这个函数，消除重复实现。 -- `mapping_stats` 的 `hint` 文案去掉“必须填写归一化后名称”。 -- **验收测试**：模板规则 `GE→gei`、映射写原始名 `GE0/1/0/1→gei0/2/0/1`，新 run 的结果是 `unchanged=1`；同样的数据在 `version=1` 的 run 上回填，结果不变。 - -#### A2 引用保护和缺失检测（B-03、B-04） -- `delete_mapping`：被 `BizCompareJob.mapping_id` 或 `BizMigrationProject.mapping_id` 引用时返回 409 `mapping_in_use`，detail 里带引用列表（任务名/项目名）。 -- `delete_template`：同时检查 `BizMonitorTemplate.compare_template_id`，返回 409 `template_in_use`（detail 区分 job/monitor）。`list_templates` 增加 `monitor_template_count`。 -- `_validate_compare_job`：`job.mapping_id` 非空但映射不存在时返回 400 `mapping_missing`。 -- 前端：删除失败时显示引用对象；模板列表显示两类引用数。 -- **验收测试**：上述 3 种 409/400 的单测；被引用的映射无法删除。 - -#### A3 映射文本不会丢失（B-05、B-06） -- `parseMapRows` 改为返回 `{rows, invalidLines[]}`：两段文本即使用空格分隔也接受（正好两个 token 时）；重复的 before 行要提示。 -- 映射文本相对于已保存版本“有修改”时：创建任务、保存配置、立即执行都要先拦截，提供“保存映射并继续 / 放弃修改”两个选项（或者自动先保存映射再保存任务，二选一，推荐自动保存并提示）。 -- 编辑区下方实时显示“已解析 N 对 / 无效 M 行（行号）”。 -- 保存一个被其他任务或项目共用的映射时，提示“该映射被 X 个任务 / Y 个割接项目使用”（后端 `list_mappings` 返回引用计数）。 -- **验收（浏览器脚本）**：在向导里输入映射后直接点“创建”，请求体里的 `mapping_id` 非空，或者被拦截并出现提示；空格分隔的行能被解析；无效行有提示。 - -#### A4 端口预设成功条件修正（M-01） -- `preset_override_for_metric("interface_brief")` 中旧侧的条件改为三个 OR 组：`[removed]`、`[phy in down]`、`[admin in down]`；新侧保持 `admin up 且 phy up`。 -- 已存在的监控模板：在 `ensure_default_monitor_templates` 中增加一个幂等升级，**只替换结构上和旧预设完全一致的 override**（逐字段对比），用户自定义过的不动，并在 `override_warnings` 里提示“端口成功条件为旧版 AND 语义”。 -- **验收测试**：新增 4 个用例——旧侧 admin up/phy down、admin down/phy up、两者都 down、端口已删除；在窗口内和验收时，新侧 up 都应判为 `migrated/green`。另外，新侧 phy down 时仍判为红。 - -#### A5 创建项目时校验基线归属（M-07） -- `create_project` 复用 `patch_project` 的校验：基线批次的 `task_id` 必须等于对应侧的画像任务，状态必须是 `success` 或 `partial`。抽成公共函数 `_validate_baseline_ownership`。 -- **验收测试**：拿其他设备的批次作为基线创建项目，返回 400 `old_baseline_invalid` / `new_baseline_invalid`。 - -#### A6 键串规范化统一（M-11） -- 在 `compare_engine.py` 新增 `key_tuple(row, fields)`，统一用 `scalar_text(...).strip()`。`_key_obj` 的输出键值和 `build_diff_index_from_compare` 都改用它（`evaluate.py:759` 不再用 `str(v)`）。 -- **验收测试**：键字段为 `None`、`True`、带首尾空格时，预期键能命中，判定为 `ok`/`migrated` 而不是 `unfinished`。 - -#### A7 内置模板参数修正（T-01、T-02、T-04） -- `default_zte_status_template.json`： - - `output_bps` 改为 `percent 5`（和 `input_bps` 对称）。 - - `rx_power` 改为 `numeric`，`tolerance: 2`（单位 dB）。 - - `l2vpn_mac.iface_fields` 加上 `ac_port`。 - - `interface_detail` 的 `input_bps/output_bps/in_util/out_util` 移出 `compare_fields`，只保留在 `display_fields`（见 V3 的方向性规则；在 V3 完成之前先不比较）。 -- `ensure_default_zte_status_template` 增量升级：**只修改值仍等于旧打包值的字段**，用户改过的保留。 -- **验收测试**：旧打包值的模板升级后变成新值；用户自定义过 `rx_power` 规则的模板升级后保持不变。 - ---- - -### 阶段 B：统一的“身份归一化流水线”（跨厂商核心） - -目标：业务对比执行、实时检索、回填、导出、映射校验、割接评估、预期集选择器这 7 条路径，**对同一行算出相同的 match key**，并支持按侧（按厂商）的规则和字段值映射。 - -#### B1 抽出 `IdentityPipeline`（解决“各路径各写一遍”） -- 新文件 `netx_api/biz_state/identity_pipeline.py`： - ```python - @dataclass(frozen=True) - class SideSpec: - iface_rules: list[dict] # 该侧接口归一化（已合并 both + side） - value_maps: dict[str, dict] # field -> {from_lower: to} - @dataclass(frozen=True) - class Pipeline: - before: SideSpec; after: SideSpec - port_map: dict[str, str] # 已归一化（A1） - key_fields, iface_fields, compare_fields, field_rules, row_filters, match_keys - def transform(self, row, side) -> dict # 归一化 → 值映射 →（before 侧）端口映射 - def match_key(self, row, side) -> tuple - def keep(self, row, side) -> bool # 行过滤（在 transform 之后求值，见 B4） - ``` -- 由 `build_pipeline(sheet, snapshot_or_template, port_map, before_vendor, after_vendor)` 统一构造；`compare_rows`、`_live_search_sheet_diffs`、`_load_search_key_groups`、`_hydrate_diff_rows`、`validate_mapping`、`evaluate_metric_dual`、`list_baseline_expect_objects` 全部改用它。 -- **验收测试**：参数化对照测试，用同一份 fixture 断言 7 条路径对同一行给出的 match key 相同（重点覆盖：映射、父接口继承、QinQ、值映射、归一化规则）。 - -#### B2 分侧接口归一化 + 厂商内置规范化 -- `iface_normalize_rules` 每条增加可选字段 `side: "both"|"before"|"after"`（默认 `both`）和 `vendor`（可选，按批次的厂商自动生效）。前端模板编辑器支持选择。 -- 内置“厂商内全称/缩写”规范化表（`iface_normalize.py` 中的 `VENDOR_CANONICAL`）： - - 华为：`GigabitEthernet`↔`GE`、`XGigabitEthernet`↔`XGE`、`100GE`、`Eth-Trunk` - - 思科：`GigabitEthernet`↔`Gi`、`TenGigE`↔`Te`、`HundredGigE`↔`Hu`、`Bundle-Ether`↔`BE` - - 华三：`GigabitEthernet`↔`GE`、`Ten-GigabitEthernet`↔`XGE`、`Bridge-Aggregation`↔`BAGG` - - 中兴：大小写 - - 统一去掉名称中的空格。 - - 只做厂商内同义统一，**不做跨厂商编号转换**（那是端口映射的职责）。模板 sheet 可以用 `builtin_vendor_canonical: false` 关闭。 -- 批次的厂商取 `BizStateBatch.vendor`，取不到再用任务的 vendor，经 `resolve_vendor_key` 解析。 -- `config_snapshot` v2 记录两侧最终生效的规则。 -- **验收测试**：华为 `GigabitEthernet0/0/1` 与 `GE0/0/1` 互认；before 华为、after 中兴，只对 before 生效的规则不影响 after。 - -#### B3 字段值映射（T-06、B-09） -- `field_rules[]` 新增可选字段： - - `value_map: {"estab": "established", "*down": "down", ...}`（大小写不敏感匹配，输出规范值） - - `side: both|before|after` - - `apply_to_key: true|false`（默认：字段在 key_fields 中时为 true） -- `normalize` 的处理顺序：`normalize`（strip/lower/mac…）→ `value_map`。**键字段也要应用**（解决 MAC 作为键时跨厂商对不上的问题）。 -- SQL：`value_map` 作用在键或比较字段上时，`sql_compare_skip_reason` 返回 `value_map`，降级到 Python（以后可以用 `CASE` 下推，本期不做）。 -- `compare_validation.py` 校验 `value_map` 必须是 `str→str` 映射，并且每个 sheet 最多 500 项。 -- 模板编辑器：字段规则行增加“值映射”编辑（每行一对，可以粘贴）。 -- **验收测试**：`state: Estab` 对 `established`，配置值映射后为 unchanged；MAC 作为键字段时，`00e0-fc12-3456` 与 `00e0.fc12.3456` 能够配对；Python 与 SQL 降级原因断言。 - -#### B4 行过滤在归一化之后求值（B-08） -- 新 run（snapshot v2）：先 `transform` 再 `row_filters`。 -- PG 下推只在“过滤涉及的字段都不在 `iface_fields` 中、也没有值映射”时启用；否则 Python 过滤。 -- 支持过滤条件带 `side`（只作用于某一侧），满足“旧侧只看 dynamic ARP”这类需求。 -- 模板编辑器提示“过滤在归一化之后执行，请按规范值书写”。 -- **验收测试**：规则 `GE→gei` 加过滤 `interface contains gei`，before 侧华为 `GE0/0/1` 不再被滤掉；`version=1` 的 run 回放结果不变。 - -#### B5 端口映射覆盖率预检（B-07、B-11、M-06 的基础） -- 新接口：`POST /v1/biz-state/compare/preflight`，请求体 `{template_id, enabled_sheet_ids, mapping_id | mapping_rows, before_batch_id, after_batch_id}`。割接监控复用同一个服务函数：`/v1/biz-migration/projects/{id}/preflight`，传入基线和最新 HF 批次。 -- 按 sheet 返回： - - `before_count / after_count`（过滤后）、`parse` 状态（见 V1） - - `mapped_exact[]`、`mapped_by_parent[]`、`identity_same_name[]`（未映射但两侧同名） - - **`suspect_unmapped[]`**：before 侧接口没有映射，after 侧也不存在 → 疑似漏配 - - `unused_map_keys[]`、`map_targets_missing_on_after[]`、`n_to_1_targets[]` - - `command_coverage`：两侧执行过的 profile_id 集合和差集（见 V2） -- 汇总：映射覆盖率 = 已映射 / (已映射 + 疑似漏配)，以及“可执行 / 有风险”的结论。 -- 业务对比 run 的 summary 也写入同结构的 `mapping_coverage`（按 sheet），结果页可以查看。 -- 前端：映射块里的“校验”按钮改为显示预检表格（取代原始 JSON）；疑似漏配可以一键加入映射编辑区（目标留空，待填）。 -- **验收测试**：构造 3 个端口，分别是精确映射、父接口继承、漏配，预检分类正确；`mapping_rows` 未保存时也能预检。 - -#### B6 SQL 引擎条件放宽（B-02） -- `sql_compare_skip_reason`：`port_map` 只在 `iface_fields ∩ (key_fields ∪ compare_fields)` 非空时才阻止 SQL。 -- **验收测试**：有端口映射的任务，`bgp_route` sheet（`iface_fields=[]`）走 SQL（断言 skip reason 为空），`interface_brief` 仍走 Python。 - -#### B7 映射语义：N:1 与部分映射（B-13、M-13） -- 决策见第 6 节，默认方案： - - **N:1**：允许，但预检和 summary 中必须列出；业务对比按重复键规则处理（V4）；割接监控继续拒绝，但错误信息里列出冲突行。 - - **1:N（breakout）**：本期不支持，在映射编辑器里检测到重复 before 时明确报错，不再只报 `duplicate_before_if`。 - - **部分映射**：新增 sheet 选项 `unmapped_port_policy: "by_name"|"ignore_port"`（默认 `by_name`，即当前行为），让 `ignore_port_changes` 与映射可以组合使用。 -- **验收测试**：N:1 在预检中出现在 `n_to_1_targets`；割接监控的报错包含冲突对。 - ---- - -### 阶段 C：判定语义与数据可信度 - -#### V1 sheet 级数据可信度（B-10） -- `_run_sheet` 和割接评估为每侧补充 `collect_quality`：该 metric 的命令数、`parse_status` 计数、`row_count`、`raw_line_count`、`declared_total`（只读这些列，不读 `raw_text`）。 -- 新增 sheet 状态： - - `no_data`：两侧过滤后都是 0 行，且命令都执行成功 → `pass_rate=null`，不计入总通过率，卡片显示“无数据”。 - - `suspect_parse`：某侧 `raw_line_count > 5` 但 `row_count == 0` → 卡片显示“疑似解析失败”，失败行数照常显示，但给出明确提示。 - - `not_collected`：某侧没有任何命令产生该 metric → 不做判定，`pass_rate=null`。 -- 任务级 `ok`：任一 sheet 处于 `not_collected` / `suspect_parse` 时为 `null`（未判定），不再是 `true`。 -- 前端卡片和 CSV `_summary.csv` 增加状态列。 -- **验收测试**：0 对 0 → `no_data`；before 1000 行、after 有原文但 0 行 → `suspect_parse`；after 侧无命令 → `not_collected`。 - -#### V2 采集命令集对齐（B-11、M-06） -- `ParseProfile` 增加可选字段 `slice: dict[str, str]`（例如 `zte.bgp_ipv4_summary` 的 slice 是 `{"afi": "ipv4"}`），声明这个 profile 产出哪个切片。先给中兴的 `bgp_peer` / `bgp_route` / `vrrp` / `isis` profile 补齐。 -- 判断一个 sheet 是否“被某侧覆盖”：该侧存在 `metric_id` 相同的成功命令，且（sheet 的 row_filters 里有 `eq` 条件时）存在 slice 与之相符的 profile。 -- 业务对比的预检（B5）和执行时都做这个检查：未覆盖的 sheet 标记 `not_collected`（V1）。 -- 割接监控：`ensure_highfreq` 生成 HF 任务项时，取**两侧画像命令项的并集**（按 profile 的 slice 转换成该侧厂商可用的 profile；找不到时返回明确错误 `no_profile_for_slice:::`）。评估时，未覆盖的 sheet 标记为 `not_collected`，不判红。 -- **验收测试**：旧画像有 ipv4 + vpnv4，新画像只有 vpnv4 → HF 任务两侧都有 ipv4 + vpnv4；如果强制只有一侧有，评估结果是 `not_collected`，而不是 `lost`。 - -#### V3 方向性判定（B-15、T-03） -- `field_rules.compare` 新增： - - `not_worse`：配合 `rank: ["up", "down"]`（排在前面表示更好）；after 不差于 before 即通过。 - - `numeric_min`：配合 `tolerance`（百分比），`after >= before*(1-tol%)` 即通过（适用于前缀数、流量“不应明显下降”）。 - - `numeric_max`：对称方向。 -- `explain_diff` 输出方向原因，例如 `worse: up→down`、`drop 37% > 10%`。 -- SQL：先降级（skip reason 为 `directional_compare`），后续再下推。 -- 内置模板：`interface_brief.prot`、`bgp_peer.state` 默认仍为 `eq`（保守）；在模板编辑器里提供这些新模式供选择；流量计数器若要比较，推荐用 `numeric_min`。 -- **验收测试**：`down→up` 用 `not_worse` 判为通过，`up→down` 判为失败；前缀数 1000→1200 用 `numeric_min 5%` 判为通过，1000→900 判为失败。 - -#### V4 重复键配对：先排序再配对（B-12） -- Python：同一个 match key 下，两侧各自按“归一化后的 compare 字段元组, 原始 seq”排序，然后 zip。 -- SQL：`dup_rn` 改为 `row_number() OVER (PARTITION BY rk ORDER BY , seq, id)`。 -- 用 sheet 选项 `duplicate_pairing: "sorted"|"ordered"` 控制，默认 `sorted`，只对 snapshot v2 生效；v1 回放保持 `ordered`。 -- **验收测试**：同键两行的值 `[a, b]` 对 `[b, a]` → 2 个 unchanged（原来是 2 个 changed）；Python 和 SQL 生成 SQL 的断言。 - -#### V5 原始身份展示（B-14） -- diff 增加 `raw_key_before`（before 原始键值，未归一化/映射）。实现方式：在 Python 路径生成；对精简行，在回填时由 `IdentityPipeline` 反推；SQL 路径没有映射，原始键等于 key。 -- 结果表：当原始值与映射后的值不同时，键单元格显示 `cgei-0/2/0/1 ← GE0/0/1`；CSV 增加 `__raw_before` 列（只针对接口键）。 -- **验收（浏览器脚本）**：映射行同时显示两侧端口名。 - ---- - -### 阶段 D：割接监控的健壮性与性能 - -#### H1 run、diff 保留策略 + 批次保护收敛（M-02） -- 新配置项：`biz_migration_auto_run_keep`（默认每个 batch 保留 300 个 auto run）、`biz_compare_auto_run_keep`（默认每个 job 保留 50 个 auto run）。 -- 清理规则： - - 永久保留：`purpose in (acceptance, manual)` 的 run，以及被红单 `run_id` 引用的 run。 - - 其余 auto run：超过数量上限的删除（连同其 diff）。 - - 保护范围：`retention.protected_batch_map` 只保护“仍然存在的 run”所引用的批次，并**加上 `summary.metric_batches` 里的所有批次**（目前只保护首个 binding 的批次）。 -- 清理挂在 `biz_state_scheduler._maybe_purge_retention` 上，每小时执行一次。 -- **验收测试**：插入 400 个 auto run 和 1 个验收 run，清理后剩 300 + 1；被删除 run 所引用的 HF 批次不再受保护，能按 `retention_days` 被清理；验收 run 引用的所有批次（包括 `metric_batches`）都受保护。 - -#### H2 评估和验收改为后台执行（M-03） -- 仿照业务对比的 `enqueue_compare`：`POST /batches/{id}/evaluate` 和 `/finish` 返回 `{run_id, status: "running"}`，在后台线程里执行，`BizMigrationRun.status` 依次经过 `running/success/failed`，`summary.progress` 记录进度。 -- 项目行锁只在“写 run 和 diff”这个短事务里持有，不再覆盖整个计算过程。 -- 重启恢复：参照 `recover_interrupted_compares_on_startup`，把遗留的 `running` 标记为 `cancelled`。 -- 前端：轮询 run 的状态；验收按钮在执行期间不可重复点击。 -- **验收测试**：评估请求立即返回；并发两次评估，第二次返回 409；模拟重启后状态被正确恢复。 - -#### H3 路由类 sheet 的处理（M-04） -- 默认：`MANUAL_ROUTE_METRICS` 对应的 sheet **不参与验收门槛**。没有在本批窗口内手动采集时，显示为 `manual_pending`（灰色），不计入 `current_missing` 或 `coverage_incomplete`；已采集时参与判定。 -- 路由 sheet 的双侧比对改为复用业务对比的 SQL 路径（旧基线 vs 旧当前、新基线 vs 新当前，各执行一次 `run_sql_sheet_compare`，只取计数和失败行），避免在 Python 里载入百万行。 -- **验收测试**：默认监控模板、没有采集路由时，验收不因路由失败；采集了路由时，判定会出现在结果里。 - -#### H4 基线行缓存（M-10） -- 进程内 LRU：键为 `(batch_id, metric_id)`，只缓存基线批次（基线不可变），总行数上限可配置（默认 50 万行）。基线变更或项目删除时失效。 -- **验收测试**：连续两次 auto 评估，第二次不再查询基线行（mock 计数）。 - -#### H5 重复键的处理口径（M-05） -- `duplicate_keys` 只在“本批范围内的键（expect）有重复”时作为验收失败原因；其他 sheet 的重复键作为警告，写进 `accept_summary.warnings`。 -- 卡片上展示具体是哪些重复键（已有 `duplicate_key_list`），并提示“完善 key_fields（例如 ISIS 加上 `lev`）或在 sheet 上启用 `duplicate_pairing: sorted`”。 -- **验收测试**：范围外的 sheet 有重复键时验收可以通过；范围内有重复时验收失败。 - -#### H6 HF 采集合并与单设备并发上限（M-09） -- 决策见第 6 节，默认：**同间隔、同 lane（light/heavy）的 metric 合并为一对任务**（一次 SSH 采多条命令）；heavy 和 manual 的路由表保持独立。 -- 新配置项 `biz_state_max_sessions_per_ne`（默认 2）：`claim_queued_batches` 认领时，同一个 `ne_id` 正在运行的批次达到上限就跳过（成对认领时要两侧都满足）。 -- `_hf_bindings` 的结构保持兼容（`metric_ids` 本来就是列表）。 -- **验收测试**：15 个 light metric、同一间隔 → 每侧 1 个 HF 任务；单设备并发上限生效（3 个同 NE 批次只认领 2 个）。 - -#### H7 同设备割接模式（M-08） -- 当 `old_ne == new_ne` 时允许创建项目：每个 metric 只建一个 HF 任务，新旧两侧共用同一批次（`collect_group_id` 只含一个任务，轮次天然对齐），映射表达同框端口迁移。 -- `evaluate_metric_dual` 不需要改（两侧是同一份行数据，靠映射区分新旧端口）。 -- `enqueue_collect_group` 允许单任务组（`mode=single_ne`）。 -- **验收测试**：同设备项目可以创建；端口 A 迁移到 B（A 变为 down、B 变为 up）判为 `migrated`。 - -#### H8 去掉 metric 名称启发式（M-12） -- `sheet_overrides` 增加 `scope_by_port_map: true|false`；不填时按当前规则推导，并把推导结果写进 override 的 `effective_*` 字段供界面展示。`_map_defines_iface_expect` 不再读取 metric 名称。 -- **验收测试**：一个名称不含 bgp 的 sheet 设置 `scope_by_port_map=false` 后，不会被映射推导出范围。 - -#### H9 其他（M-14、M-15） -- GET 接口不写库：`project_to_dict` 中的默认模板绑定改为只读解析，持久化只在 create/patch 时进行。 -- 红单：在后续验收中已转绿的键标记为 `resolvable=true`（不自动关闭），前端提供“一键关闭可关闭红单”。 - ---- - -### 阶段 E：跨厂商解析（需要你提供真实样本） - -#### P1 华为 VRP 首批状态解析（T-05） -- **前提**：每个命令提供至少 2 份真实输出样本，放到 `tests/fixtures/huawei/`，并做脱敏（不同版本、带子接口和 VRF 的各一份）。**没有样本时 Cursor 不得凭空编写 TextFSM。** -- 顺序：`interface_brief`（`display interface brief`）、`arp`（`display arp all`）、`bgp_peer`（`display bgp peer` / `display bgp vpnv4 all peer`）、`isis_adjacency`（`display isis peer`）、`ospf_neighbor`（`display ospf peer brief`）。 -- 每个 metric 输出**与中兴相同的字段名**（FieldDef 共用），状态值输出**规范枚举**： - - 接口 `admin/phy/prot ∈ {up, down}`；华为的 `*down` 输出为 `admin=down` - - BGP `state ∈ {established, idle, active, connect, opensent, openconfirm}` - - ARP `entry_type ∈ {dynamic, static, interface}` - - 这样对比层的过滤和预设不需要按厂商分叉。 -- 按 `skills/biz-state/netx-biz-state/SKILL.md` 第 2 节逐项完成：TextFSM、`prefer_fsm`、注册、profile（附 `slice`）、单测。 -- 新增“规范值契约”文档：`docs/biz-state-canonical-fields.md`，按 metric 列出字段、取值范围，以及各厂商原始值到规范值的对照。中兴 parser 也要对照这份契约补齐规范化（例如 `entry_type`）。 -- ARP 过滤：内置模板从 `age_timer` 改为 `entry_type eq dynamic`（依赖各厂商 parser 输出规范的 `entry_type`）；中兴 parser 在 `entry_type` 为空时，根据 age 推断出 `dynamic`。 -- **验收**：fixture 单测；中兴→华为的跨厂商对比用例（固定行数据）在配置映射和值映射后通过率为 100%。 - ---- - -### 阶段 F：前端体验（和上面各任务一起做） - -1. 映射编辑器升级为表格模式（保留文本模式）：两列可编辑，支持粘贴 CSV/Excel，逐行显示校验状态（重复、无效、归一化后的值）。“从基线生成”：左列 before 端口、右列 after 端口，可按“相同槽位/端口号”自动推荐。 -2. 预检结果表（B5）：疑似漏配、未使用的键、N:1、命令覆盖差集，每项都可以展开查看明细。 -3. sheet 卡片显示 `no_data` / `suspect_parse` / `not_collected` / `manual_pending` 状态和原因（V1、H3）。 -4. 结果表的键列显示原始端口名（V5）。 -5. 模板编辑器：接口归一化规则增加 side/vendor 选择；字段规则增加值映射和方向性比较；过滤条件可选 side；行过滤处提示“按归一化后的值书写”。 -6. 割接看板：重复键警告、未覆盖切片、预设语义说明（端口成功条件）。 - ---- - -## 4. 建议执行顺序 - -| 顺序 | 任务 | 说明 | -|---|---|---| -| 1 | A1–A7 | 正确性快修，相互独立，可以并行 | -| 2 | B1 → B2 → B3 → B4 | 流水线是 B2–B4 的基础，必须先做 | -| 3 | B5、B6、B7 | 预检依赖流水线 | -| 4 | V1 → V2 → V3 → V4 → V5 | V2 依赖 V1 的状态定义 | -| 5 | H1 → H2 → H3 → H4 → H5 → H6 → H7 → H8 → H9 | H1/H2 优先（生产风险最大） | -| 6 | 阶段 E | 等你提供样本 | -| 7 | 阶段 F | 随各任务交付；最后做一次整体走查 | - ---- - -## 5. 验收方式（Claude 验收时执行） - -### 5.1 回归命令 -```powershell -# 项目根目录：割接对比相关后端回归（基线 279 项通过） -.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare.py tests/test_biz_state_compare_sql.py tests/test_biz_state_compare_search.py tests/test_biz_state_compare_snapshot.py tests/test_biz_state_compare_export.py tests/test_biz_state_compare_recovery.py tests/test_biz_compare_templates.py tests/test_biz_migration_evaluate.py tests/test_biz_migration_evidence.py tests/test_biz_migration_hf_decouple.py tests/test_biz_migration_integrity.py tests/test_iface_normalize.py tests/test_biz_state_monitor_service.py tests/test_biz_state_paired_sampling.py tests/test_biz_state_retention.py -q -# 全量 -.venv\Scripts\python.exe -m pytest -q - -# web 目录 -node node_modules/typescript/bin/tsc -b -node node_modules/vite/bin/vite.js build -node --experimental-strip-types --test tests/cutoverDataCache.test.mjs tests/compareTemplateModel.test.mjs -# 浏览器（先启动 vite :5179） -node tests/bizCompare.browser.mjs -node tests/bizCompareTemplates.browser.mjs -node tests/bizMigration.browser.mjs -node tests/bizState.browser.mjs -``` - -### 5.2 必须新增的测试文件（便于验收定位） -- `tests/test_identity_pipeline.py`：B1 的 7 路径对照、B2、B3、B4 -- `tests/test_compare_preflight.py`：B5、B7、V2 -- `tests/test_compare_quality.py`：V1、V3、V4 -- `tests/test_biz_migration_presets.py`：A4 的 4 个端口场景、H5、H8 -- `tests/test_biz_migration_retention.py`：H1 -- `tests/test_biz_migration_async.py`：H2、H3 -- `tests/test_cutover_references.py`：A2、A5 -- 浏览器：`bizCompare.browser.mjs` 增加 A3、B5、V5 场景；`bizMigration.browser.mjs` 增加 H2、V1 卡片状态场景 - -### 5.3 我会逐条核对的场景 -1. 原始名称映射（A1）：`GE0/1/0/1→gei0/2/0/1` 加规则 `GE→gei`，结果为 unchanged。 -2. 拔纤但未 shutdown（A4）：验收判 `migrated`。 -3. 删除被引用的映射或模板（A2）：返回 409。 -4. 向导中映射未保存（A3）：不会被静默丢弃。 -5. 跨厂商过滤（B4）：华为侧不会被 `contains gei` 整侧滤掉。 -6. MAC 作为键（B3）：不同格式能配对。 -7. 空对空（V1）：显示 `no_data`，不是 100%。 -8. 命令集不一致（V2）：显示 `not_collected`，不是 `lost`。 -9. 同键乱序（V4）：不再出现假“不一致”。 -10. 有映射时 BGP 路由走 SQL（B6）。 -11. auto run 被清理、被引用批次的保护随之释放（H1）。 -12. 验收异步执行、不阻塞请求（H2）；默认模板不因路由未采集而验收失败（H3）。 -13. 历史 run（snapshot v1）的检索、回填、导出结果与改动前一致。 - ---- - -## 6. 需要你确认的决策（Cursor 按“默认”执行，除非你另行指定） - -| # | 决策 | 默认 | -|---|---|---| -| D1 | 业务对比是否允许 N:1 映射 | 允许，但在预检和结果中明确列出；割接监控继续拒绝并说明冲突行 | -| D2 | 1:N（端口拆分 breakout）本期是否支持 | 不支持，明确报错 | -| D3 | 空对空 / 疑似解析失败时，任务级“是否通过” | `null`（未判定），不计入通过率，不当作失败 | -| D4 | 路由表是否参与割接验收门槛 | 未采集时不参与（`manual_pending`），采集后参与 | -| D5 | HF 采集是否按“同间隔 + 同 lane”合并任务 | 合并；每台设备并发 SSH 上限默认 2 | -| D6 | 是否支持同设备割接 | 支持（H7） | -| D7 | auto run 保留数量 | 割接每批 300、业务对比每个任务 50；验收和手动 run 永久保留 | -| D8 | 流量计数器默认是否参与比较 | 不参与；需要时用 `numeric_min` | -| D9 | 华为样本由谁提供、首批覆盖哪些命令 | 由你提供；首批为接口、ARP、BGP、ISIS、OSPF | diff --git a/docs/reviews/netx-linux-release-plan-2026-10-14.md b/docs/reviews/netx-linux-release-plan-2026-10-14.md deleted file mode 100644 index 989ec0f..0000000 --- a/docs/reviews/netx-linux-release-plan-2026-10-14.md +++ /dev/null @@ -1,288 +0,0 @@ -# NetX Linux 发行版实施计划（2026-10-14） - -> 修订（同日）：用户确认**只出 x86_64**；**API/Web 端口也必须可自选**（服务器上服务多，避免冲突）。下文已按此调整；由 Claude 直接实现。 - -## 0. 目标与已确认的需求 - -| # | 需求（用户确认） | 落地方式 | -|---|---|---| -| 1 | 覆盖尽量多的 Linux 发行版 | 以 **glibc ≥ 2.17** 为基线（CentOS 7 级别），只出 **x86_64** 包 | -| 2 | tar.gz + install.sh；离线安装，目标机上**什么都不用装** | 包内自带 Python 3.12、全部依赖、PostgreSQL 16 和 web/dist；目标机只需要 bash、tar、gzip、coreutils 和 useradd，这些是任何 glibc 发行版的最小系统都有的 | -| 3 | 在线更新 | `netxctl update`，更新源与 Windows 版一致（GitHub / Forgejo / manifest），可选每日自动更新（systemd timer）；失败自动回滚程序 | -| 4a | API/Web 端口可配置 | 安装时 `--port` 或交互输入（检查占用，被占用时重新输入）；装好后用 `netxctl set-port`，失败自动回滚 | -| 4b | 内置 PG 端口可配置 | Linux：安装时 `--pg-port` 或交互输入，装好后用 `netxctl set-pg-port`；**Windows 同步补齐**：Setup 数据库页加端口框，静默参数 `/BundledPgPort=`，另提供改端口脚本 | - -现状：后端是纯 Python，依赖都有 manylinux2014 wheel，[host_metrics.py](../../netx_api/host_metrics.py) 已有 Linux 分支。所以 **`netx_api` 业务代码不需要改**，工作量都在 `packaging/linux/`、CI 和 Windows 的端口配置上。`scripts/start_netx.sh` / `stop_netx.sh` 保留，继续作为源码/开发模式的启动方式，不动。 - -### 支持范围 - -- **支持**（glibc ≥ 2.17，x86_64）：CentOS 7/8/Stream 9、RHEL 7–9、Rocky/Alma 8–9、Oracle Linux、Fedora、Ubuntu 18.04+、Debian 9+、openSUSE Leap/SLES 12+、openEuler 20.03+、麒麟 V10、统信 UOS 20、Anolis 等。 -- **不支持（第一期）**：ARM（鲲鹏/飞腾）、Alpine 等 musl 系统、32 位系统、CentOS 6（glibc 2.12）、龙芯 LoongArch / MIPS（没有现成的 standalone Python，后续单独评估）。 -- 有没有 systemd 都能用：有 systemd 就注册服务；没有（容器、精简系统）就用 `netxctl start/stop` 后台运行。 - ---- - -## 1. 产物与目录布局 - -### 1.1 发布产物（与 Windows Setup 挂在同一个 GitHub/Forgejo Release 下） - -``` -netx--linux-x86_64.tar.gz + .sha256 -``` - -解压后： - -``` -netx--linux-x86_64/ - install.sh # 入口：安装 / 更新 / 重装 - version.json # {"version": "...", "arch": "x86_64", "pg_major": 16, "python": "3.12.x"} - netx_api/ alembic/ alembic.ini pyproject.toml requirements.txt README.md LICENSE - web/dist/ # 由 API 托管，不需要 Node - python/ # python-build-standalone 3.12，依赖直接装在它的 site-packages 里（不用 venv） - pgsql/{bin,lib,share} # PostgreSQL 16 只保留运行需要的部分；rpath=$ORIGIN/../lib - packaging/linux/ # netxctl、netx_admin.py、systemd 单元模板、logrotate、README -``` - -### 1.2 安装后的目录 - -| 路径 | 内容 | 属主 / 权限 | -|---|---|---| -| `/opt/netx/releases//` | 某个版本的程序（只读） | root:root 0755 | -| `/opt/netx/current` → `releases/` | 当前版本的软链，切换是原子的 | | -| `/usr/bin/netxctl` → `/opt/netx/current/packaging/linux/netxctl` | 管理命令 | | -| `/var/lib/netx/` | `.env`、`pgdata/`、`data/…`（与 Windows 的 `Get-NetxDataEnvMap` 子目录一致）、`backups/`、`logs/` | netx:netx 0750，`.env` 0600 | -| `/etc/systemd/system/netx.service`、`netx-update.{service,timer}` | 服务单元 | | - -`--prefix` 和 `--data` 都可以改。始终只保留最近 2 个 release 目录，用于回滚。 - ---- - -## 2. 任务清单 - -### L1 构建：PostgreSQL 16 可移植二进制（glibc 2.17） - -- 新文件 `packaging/linux/build_pg.sh `：在 `quay.io/pypa/manylinux2014_` 容器里从源码编译 PostgreSQL 16.x，**和 Windows 用同一个小版本**（现在是 16.4，见 `packaging/cache/pgsql-16.4-1`）。 - - `./configure --prefix=/pgsql --without-readline --without-icu --with-zlib --without-openssl LDFLAGS='-Wl,-rpath,$ORIGIN/../lib'` - - 把 `libz.so.1` 复制进 `pgsql/lib/`，避免最小系统里没有 zlib。 - - 只保留运行需要的二进制：`postgres pg_ctl initdb psql pg_dump pg_dumpall pg_restore pg_isready`（与 [build_release.ps1:163](../../packaging/build_release.ps1:163) 一致），外加 `lib/`、`share/`。 - - 打包成 `pg-16.x-linux-.tar.gz`；CI 按版本+架构缓存，本地放 `packaging/cache/`。 -- 版本号和下载校验值统一写在 `packaging/linux/versions.env`（PG 源码 URL+sha256，python-build-standalone URL+sha256）。 - -**验收**：在 `centos:7` 容器里，`pgsql/bin/initdb` / `pg_ctl start` / `psql -c 'select version()'` 都能跑；`L4` 的检查脚本通过。 - -### L2 构建：Python 运行时 + 依赖 - -- `packaging/linux/build_python.sh `：下载 python-build-standalone `cpython-3.12.x+--unknown-linux-gnu-install_only_stripped.tar.gz`（pin 住版本并校验 sha256）。 -- **在 manylinux2014 容器里**执行 `python/bin/python3 -m pip install --only-binary=:all: -r requirements.txt`。容器的 glibc 是 2.17，pip 只会选兼容的 wheel；任何依赖要编译或要求更高的 glibc，构建就会直接失败。 -- 删掉 `test/`、`idlelib/`、`tkinter/`、`ensurepip/`，再执行 `compileall -q`，这样 `/opt` 只读也不会在运行时写 pyc。 -- 不用 venv（standalone Python 本身可以搬动，venv 的软链和绝对路径反而碍事）。所有启动都用 `python/bin/python3 -m …`，不依赖 `bin/` 下脚本的 shebang。 - -**验收**：在 `centos:7` 和 `ubuntu:24.04` 里，`python/bin/python3 -c "import netx_api.main, psycopg, cryptography, netmiko, bcrypt, psutil"` 都成功。 - -### L3 构建：打包脚本 `packaging/linux/build_linux_release.sh` - -- 参数：`--version`（默认读 pyproject）、`--web-dist `、`--skip-web-build`。 -- 流程：先检查 web/dist（没有就执行 `npm ci && npm run build:release`），调用 L1/L2，拷贝程序文件（清单与 [build_release.ps1:88-93](../../packaging/build_release.ps1:88) 一致），写 `version.json`，执行 `tar czf` 并生成 `.sha256`，最后调用 L4 检查。 -- 依赖 docker（manylinux 容器）。本机是 Windows，所以本地构建要在 WSL2 里的 Ubuntu+docker 上做，或者用一台 Linux 构建机。**CI 是主要的构建途径**。 -- 文件：`packaging/linux/README.md`，写清楚构建、安装、运维、支持范围。同时把 [packaging/README.md](../../packaging/README.md) 第 3 行"Linux is unchanged"改成指向 Linux 发行版的说明。 - -### L4 兼容性闸门 `packaging/linux/check_bundle.sh` - -- 用 `objdump -T` 扫描包里所有 ELF（`python/`、`site-packages/**/*.so`、`pgsql/`），引用的 `GLIBC_x.y` 最高不能超过 **2.17**。 -- 用 `readelf -d` 检查 NEEDED：除了 `libc/libm/libdl/libpthread/librt/libutil/libcrypt/ld-linux*/libgcc_s/libstdc++` 以外，都必须能在包内找到。 -- 不通过就让构建失败。 - -### L5 `install.sh`（安装 / 更新 / 重装） - -只依赖 bash、coreutils、tar、gzip、useradd（或 `adduser`）。凡是复杂逻辑（改 .env、检查端口、URL 编码、生成密钥）都交给**包里自带的 Python**（`packaging/linux/netx_admin.py`，只用标准库），不依赖 curl、ss、openssl 这些。 - -参数（交互模式下会逐项提问；中文还是英文按 `$LANG` 判断；`--yes` 为非交互）： - -``` ---mode install|update|reinstall # 已有安装时默认 update ---prefix /opt/netx --data /var/lib/netx ---db bundled|external ---pg-port 15432 # 内置 PG 端口（需求 4） ---pg-password # 不填则自动生成 ---db-url postgresql+psycopg://… # external 用；也可以拆成 --db-host/--db-port/--db-user/--db-password/--db-name ---bind 0.0.0.0 --port 8890 # Linux 一般是无桌面的服务器，默认绑 0.0.0.0（Windows 是 127.0.0.1） ---credential-key-file # 复用别的安装的 NE 密码加密密钥（对应 Windows 的 /CredentialSecretKey） ---no-service # 不注册 systemd 服务 ---auto-update # 打开每日自动更新（默认关闭，与 Windows 一致） ---open-firewall # 有 firewall-cmd / ufw 时放行 --port；默认只打印提示 ---no-backup # 更新前不做 pg_dump -``` - -预检（失败时给出中英文提示并退出，不留下半装的状态）： -1. 必须是 root； -2. 架构和包一致；`getconf GNU_LIBC_VERSION` ≥ 2.17；检测到 musl 就拒绝； -3. 磁盘空间够（≥ 1 GB）； -4. 用自带 Python 检查 `--port`、`--pg-port` 是否空闲（被占用时交互模式重新问，`--yes` 模式直接报 `port_in_use`）； -5. external 模式用 `psql SELECT 1` 测试连接（对应 Windows 的行为）。 - -**首次安装**的步骤： -1. 创建系统用户 `netx`（`useradd -r -s /sbin/nologin -d /var/lib/netx`；没有 useradd 就退到 `adduser -S`）； -2. 解压到 `releases/`，切换 `current` 软链，建立 `netxctl` 链接； -3. 创建数据目录、写 `.env`（键与 Windows [setup_first_run.ps1](../../packaging/setup_first_run.ps1) 一致：`NETX_DB_MODE`、`NETX_HOST`、`NETX_PORT`、`NETX_UI_DIST_DIR`、`NETX_BUNDLED_PG_PORT`、`NETX_DATABASE_URL`、`Get-NetxDataEnvMap` 里的路径、凭据密钥）； -4. 内置 PG：用 `netx` 用户执行 `initdb -U postgres -A scram-sha-256 -E UTF8 --locale=C`，在 `postgresql.conf` 写入 `port = ` 和 `listen_addresses = '127.0.0.1'`，`pg_hba` 只允许本机，然后创建 `netx` 角色和库（照搬 Windows 的 bootstrap SQL）； -5. 如果系统有 SELinux 并且有 `restorecon`，执行 `restorecon -R `； -6. 有 systemd（`[ -d /run/systemd/system ]`）就安装 `netx.service` 并 `enable --now`；没有就执行 `netxctl start`； -7. 等待 `/health` 返回正常（最多 90 秒），最后打印访问地址、首次登录说明和防火墙提示。 - -**更新**（`--mode update`，也是在线更新最后调用的入口）： -1. 解压到 `releases/`（版本已存在就覆盖）； -2. 内置 PG 时做 `pg_dump -Fc` 到 `backups/pre--.dump`，只保留最近 3 份； -3. 停服务，切换 `current` 软链，启动，等待 `/health`； -4. 如果没有起来：软链切回旧版本再启动，返回非 0，并提示备份文件的位置（数据库可能已经被 alembic 迁移过，所以只回滚程序，数据库靠这个 dump 恢复）； -5. 成功后清理旧 release，只保留 2 个； -6. 不会改动 `.env` 和 PG 端口；systemd 单元按新版本的模板重新生成。 - -`--mode reinstall`：重新走数据库配置，但**不删除** `/var/lib/netx`（和 Windows 一样）。 - -### L6 `netxctl` 管理命令 - -bash 写的薄入口，复杂的子命令转交 `netx_admin.py`。以 root 调用时，需要降权的操作用 `runuser -u netx --`（没有 runuser 就用 `su -s /bin/sh netx -c`）。 - -``` -netxctl run # 前台监督进程（systemd 的 ExecStart 就是它） -netxctl start|stop|restart|status -netxctl logs [-f] [api|worker|pg] -netxctl set-pg-port # 需求 4 -netxctl set-port / set-bind -netxctl reconfigure-db # 等于 install.sh --mode reinstall 里的数据库部分 -netxctl check-update / update [--yes] [--file ] -netxctl auto-update on|off -netxctl backup [] # pg_dump + data/auth 下的密钥，打成一个 tar.gz -netxctl uninstall [--purge] # --purge 删除数据，需要输入 "DELETE" 确认 -netxctl version -``` - -`netxctl run`（前台监督进程，对应 Windows 的 [service_run.ps1](../../packaging/service_run.ps1)）： -- 加载 `/var/lib/netx/.env`，并强制覆盖为数据目录下的路径；设置 `PYTHONPATH=current`、`NETX_UI_DIST_DIR`； -- 内置模式先执行 `pg_ctl -D pgdata -w start`。启动前检查 `pgdata/PG_VERSION` 的主版本与 `version.json` 里的 `pg_major` 是否一致，不一致就报 `pg_major_mismatch`，不擅自升级数据； -- 启动 API（`-m netx_api.main`）、worker（`-m netx_api.worker`）和 biz_state workers，副本数和参数照搬 [start_netx.sh:122-160](../../scripts/start_netx.sh:122)，`NETX_RUN_INLINE_SCHEDULERS=false`； -- `trap TERM INT`：先停所有子进程，再 `pg_ctl stop -m fast`； -- 用 `wait -n` 等待：任何一个子进程退出就整体退出并返回非 0，由 systemd `Restart=on-failure` 重启； -- 日志写到 `/var/lib/netx/logs/*.log`，并安装 `/etc/logrotate.d/netx`（copytruncate，7 份）。 - -没有 systemd 时：`netxctl start` 执行 `nohup netxctl run &`，pid 写到 `data/runtime/netx.pid`；`stop` 发 TERM，30 秒后还没退出就 KILL。 - -`netx.service`： - -```ini -[Unit] -Description=NetX -After=network-online.target -Wants=network-online.target -[Service] -Type=simple -User=netx -Group=netx -ExecStart=/opt/netx/current/packaging/linux/netxctl run -KillMode=mixed -TimeoutStopSec=60 -Restart=on-failure -RestartSec=5 -LimitNOFILE=65536 -[Install] -WantedBy=multi-user.target -``` - -### L7a API/Web 端口可配置 - -- `install.sh`：支持 `--port` 和 `--bind`；交互模式下默认 8890 / 0.0.0.0，会检查端口是否被占用（自带 Python 的 socket 检测），被占用时重新问，`--yes` 模式直接报 `port_in_use`。 -- `netxctl set-port ` / `set-bind `：校验 → 停服务 → 改 `.env` 里的 `NETX_PORT` / `NETX_HOST` → 启动 → 等 `/health`；失败就恢复 `.env` 并重新启动。改完后打印 firewalld/ufw 的放行命令。 -- 重装（`--mode reinstall`）时，默认值取当前配置，并允许沿用自己正在占用的端口。 - -### L7 内置 PG 端口可配置（Linux 部分） - -- `install.sh`：支持 `--pg-port`；交互模式默认值 15432，会检查是否被占用，被占用时重新问。 -- `netxctl set-pg-port `：校验范围在 1024–65535 且端口空闲 → 停服务 → 改 `postgresql.conf` 里的 `port` → 改 `.env` 里的 `NETX_BUNDLED_PG_PORT`，并把 `NETX_DATABASE_URL` 中的端口一并改掉（只换 host:port 部分，密码保持原样）→ 启动 → 等 `/health`；失败就把三处改回去。 -- external 模式下执行这个命令会报错，提示改用 `reconfigure-db`。 - -### L8 在线更新 `packaging/linux/netx_update.py`（只用标准库） - -- 更新源和配置项与 Windows [check_update.ps1](../../packaging/check_update.ps1) **完全一致**：`NETX_UPDATE_SOURCES`（默认 `github,forgejo`）、`NETX_UPDATE_GITHUB_REPO`、`NETX_UPDATE_FORGEJO_URL`、`NETX_UPDATE_URL`（manifest）、`NETX_UPDATE_TOKEN`；会探测所有源，选最新的那个；版本相同时优先 GitHub；token 只发给非 GitHub 的主机。 -- 资产匹配规则：`^netx-(?P[\d.]+)-linux-\.tar\.gz$`；sha256 优先取 `.sha256` 资产，其次取 manifest 里的值，**两者都没有就拒绝更新**。 -- 代理：遵循 `HTTPS_PROXY/NO_PROXY`（urllib 默认就支持）。 -- 流程：下载到 `/var/lib/netx/updates/` → 校验 → 解压到临时目录 → 执行**新包里的** `install.sh --mode update --yes` → 清理临时文件。 -- `netxctl update --file `：离线升级，跳过下载这一步。 -- 自动更新：`netx-update.timer`（`OnCalendar=*-*-* 03:30`，`RandomizedDelaySec=1h`，`Persistent=true`），执行 `netxctl update --yes --quiet`。没有 systemd 但有 crontab 时写到 `/etc/cron.d/netx-update`；两者都没有就提示不支持。 -- [manifest.example.json](../../packaging/manifest.example.json) 增加： - ```json - "linux": { - "x86_64": {"url": "...netx-0.4.19-linux-x86_64.tar.gz", "sha256": "", "size": 0}, - } - ``` -- Windows 这边 [check_update.ps1:115](../../packaging/check_update.ps1:115) 只匹配 `Setup-.*\.exe`，同一个 Release 里出现 Linux 资产不受影响。需要加一条回归验证。 - -### L9 CI：`.github/workflows/release-linux.yml` - -- 触发：`v*` tag（和 Windows 一致）；同样校验 tag 必须等于 pyproject 里的版本。 -- job `web`：Node 20，执行 `npm ci && npm run build:release`，上传 `web/dist` 作为 artifact。 -- job `build`：`ubuntu-latest` + manylinux2014_x86_64 容器 - - 步骤：缓存/构建 PG（L1），执行 L2、L3、L4，上传 tar.gz 和 sha256 artifact。 -- job `smoke`（必须通过才发布），x86_64 在下面这些镜像里各跑一遍（`docker run --network none`，同时验证离线安装）： - `centos:7`、`rockylinux:9`、`ubuntu:18.04`、`ubuntu:24.04`、`debian:12`、`opensuse/leap:15`、`openeuler/openeuler:22.03`。 - 每个镜像执行 `packaging/linux/smoke_test.sh`： - 1. `install.sh --yes --no-service --db bundled --pg-port 25432` - 2. `/health` 返回 200；用默认管理员登录拿到 token - 3. `netxctl set-pg-port 25433`，之后 `/health` 仍然是 200 - 4. `install.sh --mode update --yes`（用同一个包模拟升级）：`backups/` 里多出一份 dump，数据还在（登录前先建一条测试数据） - 5. `netxctl stop`，检查没有残留进程 - 6. `netxctl uninstall --purge`（用 `--yes` 跳过确认） -- job `release`：用 `softprops/action-gh-release@v2` 把 4 个文件上传到同一个 tag 的 Release（**不要**设 `generate_release_notes`，release notes 由 Windows 的 workflow 生成，避免互相覆盖）。 - -### L10 发布脚本 - -- [publish_forgejo_release.ps1](../../packaging/publish_forgejo_release.ps1)：`packaging/release/` 下如果有 `netx--linux-*.tar.gz(.sha256)`，就一起上传；Release 说明里加上 Linux 安装方式。 -- [publish_release.ps1](../../packaging/publish_release.ps1)：GitHub Release 说明模板（第 107 行附近）加一段 Linux 的说明。 -- 本地发布流程（写进 `packaging/linux/README.md`）：CI 发布 tag 后，用 `gh release download vX -p 'netx-*-linux-*'` 把 Linux 包取回 `packaging/release/`，再执行 `publish_forgejo_release.ps1` 同步到 Forgejo。 - -### W1 Windows：Setup 数据库页增加内置 PG 端口 - -- [netx.iss](../../packaging/installer/netx.iss)：在 `RbBundled`（约第 762 行）下面加一个"端口"输入框 `EdBundledPort`，默认 15432，只有选内置时才可用（`DbModeClick`，第 469 行）。翻页时校验它是 1024–65535 的数字，并且端口没有被占用（可以复用 `Test-NetxTcpPortFree`，或者在 Pascal 里用 `netstat -ano` 检查）。 -- 静默安装参数 `/BundledPgPort=25432`（加在第 644 行附近解析 `DbMode` 的地方）。 -- 执行 `setup_first_run.ps1` 时传入 `-BundledPort`（这个参数已经存在，见 [setup_first_run.ps1:11](../../packaging/setup_first_run.ps1:11)）。 -- **Update 模式**不显示、也不修改端口。 -- 中英文文案（`english.` / `chinesesimplified.`）。 - -### W2 Windows：安装后修改端口 - -- 新文件 `packaging/set_pg_port.ps1 -Port `（会自动提权）：做的事和 L7 一样：校验 → 通过 `runtime_ctl.ps1` 停服务 → 改 `postgresql.conf`、`.env` 里的 `NETX_BUNDLED_PG_PORT` 和 `NETX_DATABASE_URL` 的端口 → 启动 → 等 `/health`，失败就回滚。 -- 托盘的"设置"里加一项"内置数据库端口…"，调用这个脚本。 -- [build_release.ps1:121](../../packaging/build_release.ps1:121) 的拷贝清单里加上 `set_pg_port.ps1`。 - ---- - -## 3. 验收清单 - -| ID | 场景 | 通过标准 | -|---|---|---| -| A1 | CI 构建 | `release-linux.yml` 在测试 tag 上完整跑通；两个架构的 tar.gz 都 ≤ 150 MB；L4 检查通过 | -| A2 | 离线安装矩阵 | L9 的 smoke 在全部镜像里通过，并且都是 `--network none` | -| A3 | 真机 + systemd | 在一台真实的 Linux 服务器（例如 10.0.0.131 或一台 CentOS 7 / 麒麟虚拟机）上交互式安装：开机自启；`systemctl restart netx` 正常；`kill -9` API 进程后 5 秒内被拉起；浏览器能打开 `http://:8890/` 并登录 | -| A4 | PG 端口 | 安装时 `--pg-port 25432` 生效（`ss -lnt` 只看到 `127.0.0.1:25432`）；`netxctl set-pg-port 25433` 后服务正常；改成一个已被占用的端口会被拒绝，原配置不变 | -| A4b | API 端口 | 安装时 `--port 18890` 生效；`netxctl set-port 18891` 后新端口能访问、旧端口不再监听；设成一个已被占用的端口会被拒绝 | -| A5 | 在线更新 | 先装 vN，在 Release 上挂 vN+1，执行 `netxctl check-update` 能看到新版本；`netxctl update --yes` 成功，`current` 指向新版本，数据还在，`backups/` 里有 dump | -| A6 | 更新回滚 | 构造一个启动就失败的包（比如删掉 `netx_api/main.py`），`netxctl update --file` 之后 `current` 自动切回旧版本，服务正常，命令返回非 0 | -| A7 | 校验失败 | 篡改 `.sha256` 后执行 update，结果是拒绝更新，并且不会解压 | -| A8 | 自动更新 | `netxctl auto-update on` 后 `systemctl list-timers` 里能看到 `netx-update.timer`；`off` 后消失 | -| A9 | external 模式 | `--db external --db-url …` 连接到外部 PG 能正常工作；`set-pg-port` 会给出明确的报错 | -| A10 | 卸载 | `netxctl uninstall` 保留 `/var/lib/netx`；`--purge` 加确认后全部清掉；systemd 单元和 `netxctl` 链接都被删除 | -| A11 | Windows 端口 | Setup 选内置并把端口填成 25432：装好后 `.env` 和 `postgresql.conf` 一致，服务正常；静默 `/BundledPgPort=25432` 一样生效；Update 模式下端口不变；托盘改端口成功 | -| A12 | Windows 回归 | 现有的 Windows 沙箱离线验证（静默安装、启动、登录、停止、服务模式）全部通过；同一个 Release 里有 Linux 资产时，Windows 的 `check_update.ps1` 仍然只选 Setup.exe | - -## 4. 建议的实现顺序 - -1. L1 → L2 → L4（先把"能在 CentOS 7 上运行的运行时"做出来，这是风险最大的部分） -2. L6 `netxctl run`/start/stop → L5 install.sh → L7 -3. L3 + L9（CI + smoke 矩阵） -4. L8 在线更新 + L10 发布脚本 -5. W1、W2（Windows 端口，相对独立，可以并行做） - -## 5. 风险与说明 - -- **PG 大版本**：固定为 16。以后要升到 17 需要单独做 `pg_upgrade` 流程；第一期只检测、不处理（`pg_major_mismatch`）。 -- **API 默认绑 0.0.0.0**：Linux 服务器一般通过远程访问，所以默认对外。安装完成后会醒目提示修改默认管理员密码，并提醒放通防火墙端口。 -- **本地构建**：本机目前没有 docker，WSL 里也没有装发行版。要在本地构建 Linux 包，需要 `wsl --install -d Ubuntu` 再装 docker，或者在一台 Linux 构建机上执行。日常发布以 CI 为主。 diff --git a/docs/reviews/topology-optimization-plan-2026-10-12.md b/docs/reviews/topology-optimization-plan-2026-10-12.md deleted file mode 100644 index 4afd699..0000000 --- a/docs/reviews/topology-optimization-plan-2026-10-12.md +++ /dev/null @@ -1,295 +0,0 @@ -# 拓扑管理问题分析与优化计划 - -日期：2026-10-12。范围：拓扑管理画布（`web/src/features/topology/*`）及其视图接口（`netx_api/topology_views_graph.py`、`topology_router.py`、`topology_fabric_links.py`）。不含：UME World 平铺大图 LOD、LLDP 采集策略、分类规则、MCP 布局算法。 -用途：交给 Cursor 按任务逐项实施；Claude 负责验收。 - -本文结论来自代码阅读和本地脚本验证，没有在真实 PostgreSQL 或真实设备上执行。标了 **[已验证]** 的问题已经用脚本复现。 - -**基线（修改前）：** -- `pytest tests/test_topology.py tests/test_ume_topology_apply.py`：**69 项通过，4 项失败**（见 T13，修改前就失败，与本计划无关） -- `web`：`npx tsc -b` 通过 -- 拓扑前端目前没有任何单元测试 - -**实施进度（2026-10-12）：** -| 任务 | 状态 | -|---|---| -| T11 基线失败测试 | ✅ 完成 | -| T1 保存保留 lock | ✅ 完成 | -| T2 不固化设备名为 label | ✅ 完成 | -| T6 原子 commit + useTopologySave | ✅ 完成 | -| T3 脏状态添加确认 | ✅ 完成 | -| T4 手工边端口持久化 | ✅ 完成 | -| T5 边样式走保存模型 | ✅ 完成 | -| T7 N+1 / 截断确定性 | ✅ 完成 | -| T8 positions 跳过缺失节点 | ✅ 完成（MCP 不依赖 positions 顺便加点） | -| T9 点击不标脏 | ✅ 完成 | -| T10 拆分 TopologyPage | ✅ 完成（`TopologyPage.tsx` ≤1200；抽出 palette/edge/keyboard/find/export/nodeOps/moduleLinks + `TopologyPageView`） | - -回归：`pytest` 87 passed；`npx tsc -b` 通过；`topologySaveDiff.test.mjs` 4 passed。 - ---- - -## 0. 结论摘要 - -整体架构没有大问题（服务端图 + 本地编辑 + 一次保存）。主要问题集中在**“本地编辑状态”和“服务端状态”的同步边界**，有几处会静默丢数据或误删数据： - -1. **UI 保存会清掉 MCP 设置的节点锁定，并把设备名固化成标签。** **[已验证]** 保存后 `locked` 从 True 变为 False，节点被移走；之后设备改名，画布上仍显示旧名称。 -2. **有未保存修改时“添加网元/投影邻居”会静默丢弃这些修改，并留下隐形删除。** 被删除的连线会重新出现在画布上，但仍留在待删除队列里，下次保存时会被真正删掉。 -3. **修改已有连线的端口不会被保存。** 界面上可以改，也会显示新值，但不会标记为未保存，保存时也不会提交，刷新后就恢复原值。 -4. **连线样式绕过了“编辑→保存”模型。** 样式修改会立即写库，但撤销只回退本地。撤销之后，画布显示和数据库就不一致了。 -5. **保存不是原子的。** 一次保存会依次调用 N 次建边、1 次删边、1 次加点、1 次删点和 1 次写坐标，其中 3 次会返回完整的图数据。中途失败会留下“部分已保存”的状态。另外，同一套保存逻辑在 `purgePlaceholderById` 里重复实现了一遍。 -6. **后端有 N+1 查询和不确定的截断。** 写坐标时每个点各查一次；外部邻居查询的 `limit(5000)` 把视图内部的边也算进去了，边密集的视图可能找不到外部邻居；节点超过上限时截断没有排序。 -7. **`TopologyPage.tsx` 有 2509 行。** 保存、面板添加、连线编辑、快捷键等逻辑都堆在这一个组件里，所以上面这些问题很难被发现，也很难测试。 - ---- - -## 1. 现状：画布编辑模型（给 Cursor 的背景） - -``` -GET /views/{id} ──> graphQuery.data ──(graphToFlow)──> nodes / edges (本地 React state) - │ 拖动/改名/连线/删除 - ▼ - dirtyRef + historyRef(撤销栈, UNDO_MAX=40) - pendingEdgeCreatesRef / pendingEdgeDeletesRef - │ Ctrl+S / 保存 - ▼ - saveMut (TopologyPage.tsx:516) - 1. 每条待建边 POST /fabric/edges (逐条串行) - 2. POST /fabric/edges/delete (待删边) - 3. POST /views/{id}/nodes (服务端没有的本地点；返回整图) - 4. POST /views/{id}/nodes/remove (本地没有的服务端点；返回整图) - 5. PATCH /views/{id}/positions (全部点的 x/y/label；返回整图) -``` - -旁路（不走保存，直接写库）： -- 添加网元/拖入面板 `addPaletteItems` (`TopologyPage.tsx:621`)、投影外部邻居 `addOutsidePeers` (`:1047`)：写库后**用服务端图整体覆盖本地并 `clearDirty()`** -- 连线样式 `patchSelectedEdgeStyle` (`:765`)：立即 `PATCH /edge-style` -- 删除占位网元 `purgePlaceholderById` (`:1238`)：先把保存逻辑复制执行一遍，再清理 - -脏状态下的轮询 (`useTopologyCanvas.ts:306-318`)：只合并在线状态、层级等叠加字段，不动几何信息，这部分是对的。 - ---- - -## 2. 问题清单与任务 - -优先级：**P0** = 静默丢数据/误删，**P1** = 可靠性/性能，**P2** = 体验/可维护性。 - -### T1 [P0] UI 保存会清除节点锁定 **[已验证]** - -- `flowToPositions` (`web/src/features/topology/lib/graphFlow.ts:90`) 不发送 `locked` 字段。 -- `ViewNodeIn.locked` 的默认值是 `False` (`netx_api/topology_schemas.py:277`)。 -- `patch_view_positions` 只在“行已锁定且请求也带 locked=True”时才跳过 (`topology_views_graph.py:738`)，否则会覆盖坐标并写回 `locked=False`。 -- MCP 会用 `locked=true` 固定布局（`packages/netx-topology-mcp/.../http_tools.py:1985`）。用户在 UI 里随便拖一下再保存，这个锁就没了。 - -复现：MCP 把节点锁定在 (500,500)，UI 保存 (10,10)，结果节点变成 `locked=False x=10`。 - -**修改：** -1. 后端：把 `ViewNodeIn.locked` 改为 `bool | None = None`，`None` 表示保持原值。对已锁定的行，只有请求里明确带 `locked=False` 才能移动（解锁）；否则跳过并计入 `skipped_locked`。MCP 显式传 `locked` 的行为保持不变。 -2. 前端：在 `NeNodeData` 里加 `locked`（由 `graphToFlow` 从 `n.locked` 填入）。锁定的节点设 `draggable: false`，并显示一个锁形角标；`flowToPositions` 不再发送 `locked`。 -3. 保存结果里如果 `skipped_locked > 0`，用 toast 提示。 - -**验收：** -- 新增 `tests/test_topology.py::test_positions_preserve_lock_when_omitted`：锁定后用不带 locked 的 positions 请求，坐标不变，`locked` 仍为 True。 -- 新增 `test_positions_explicit_unlock_moves`：`locked=False` 可以移动并解锁。 -- 现有 MCP 相关测试全部通过。 - -### T2 [P0] 保存会把设备名固化成节点标签 **[已验证]** - -- 接口返回的 `label` 已经做过回退：`vn.label` 为空时用 `fn.name / ip` (`topology_views_graph.py:195-197`)。 -- 前端把它放进 `data.label`，保存时 `flowToPositions` 又原样写回 `vn.label` (`graphFlow.ts:100`)。 -- 所以保存过一次之后，所有节点的 `vn.label` 都是当时的设备名。以后设备改名（例如 UME/网管同步），画布不会跟着变。 - -复现：保存后把 fabric 节点改名为 `NEW-NAME`，画布仍显示 `OLD-NAME`。 - -**修改：** -1. 后端 `ViewNodeOut` 增加 `label_override: str`（原始 `vn.label`）；`label` 仍是回退后的显示值，保持兼容。`ViewNodeIn.label` 改为 `str | None = None`，`None` 表示不修改。 -2. 前端 `NeNodeData` 增加 `label_override`，只有用户“重命名节点”（`renameSelectedNode`，`TopologyPage.tsx:820`）时才写入。`flowToPositions` 只对 `label_override` 有变化的节点发送 `label`；重命名为空字符串表示清除覆盖，恢复跟随设备名。 -3. 数据修复脚本 `scripts/topology_clear_frozen_labels.py`，默认 dry-run：`vn.label` 等于当前 `fn.name` 或 `fn.ip` 的行，把 label 置空。输出受影响的行数，加 `--apply` 才真正写库。 - -**验收：** -- 新增 `test_positions_without_label_keeps_following_name`：保存不带 label，改名后画布 label 是新名称。 -- 新增 `test_label_override_survives_rename`：显式覆盖后改名，仍显示覆盖值。 -- 脚本 dry-run 在测试库上输出正确的计数，`--apply` 之后再跑一次计数为 0。 - -### T3 [P0] 有未保存修改时“添加网元 / 投影邻居”会丢弃修改并留下隐形删除 - -- `addPaletteItems` (`TopologyPage.tsx:681-686`) 和 `addOutsidePeers` (`:1096-1101`) 都是直接写库，然后 `applyViewGraph(refreshed)` 再 `clearDirty()`： - - 本地未保存的拖动、改名、删点**直接丢失**，也没有提示； - - `pendingEdgeDeletesRef` **没有清空**。被删除的边按服务端数据重新画回画布，但它的 id 还在待删队列里。用户以后再改任何东西并保存时，这条**看得见的边会被删掉**； - - 撤销栈也没有清空，撤销会回到和服务端不一致的快照。 -- 同类问题：`useTopologyCanvas.ts:320-355` 在非脏状态下整体刷新时，也不会清理撤销栈。 - -**修改（推荐“先保存再添加”，简单而且语义清楚）：** -1. 抽出 `commitCanvas()`（见 T6）。`addPaletteItems` / `addOutsidePeers` 开始时如果 `dirtyRef.current` 为 true，就弹窗“当前有未保存修改，保存后继续添加？”：确认则先 `commitCanvas()` 再添加，取消则中止。 -2. 新增一个统一的辅助函数 `resetLocalFromServer(graph)`，用于所有“用服务端图整体覆盖本地”的地方：同时清空两个 pending 集合、撤销栈和重做栈，然后 `clearDirty()`。目前 `purgePlaceholderById` (`:1289-1297`) 是手写的这套逻辑，其他地方都漏了。 - -**验收（手工，Cursor 需要在 PR 描述里写明步骤和结果）：** -1. 删除一条已有边 → 不保存 → 从面板拖入一个网元 → 应弹出确认框；选择保存后，该边被删除，新网元出现。 -2. 同上，但选择取消 → 不添加网元，删除操作仍处于未保存状态。 -3. 删除边 → 投影邻居（确认保存）→ 刷新页面，边确实被删除，没有“复活后又消失”。 -4. 用 grep 检查：所有 `applyViewGraph(` 调用点要么使用 `localPositions` 合并，要么经过 `resetLocalFromServer`。 - -### T4 [P0] 修改已有连线的端口不会被保存 - -- 右键菜单对任意单条边都显示端口输入框 (`TopologyCtxMenu.tsx:400-429`)。 -- `patchSelectedEdgeStyle` 修改本地端口后，只把样式字段发给 `PATCH /edge-style`（`ViewEdgeStylePatch` 本来就没有端口字段，`topology_schemas.py:450`），也没有 `markDirty()`。 -- 保存只处理 `pendingEdgeCreatesRef` 里的新边，已有边的端口修改永远不会提交。 - -**修改：** -1. 只有 `source === "manual"` 的边可以编辑端口；LLDP/UME 边的端口改为只读（这两类以采集结果为准，手工修改会在下次发现时被覆盖）。 -2. 未保存的新手工边：端口修改继续只改本地，保存时随建边一起提交（现有逻辑已支持）。 -3. 已保存的手工边：新增 `PATCH /v1/topology/fabric/edges/{id}`，body `{a_port, b_port}`，只允许 `source=manual`，否则 400 `edge_not_manual`；端口要做归一化，与已有边冲突（违反唯一约束）时返回 409。前端把修改记入 `pendingEdgePortEditsRef: Map` 并 `markDirty()`，在 `commitCanvas()` 里提交。 - -**验收：** -- 新增 `test_patch_manual_edge_ports`、`test_patch_non_manual_edge_ports_rejected`、`test_patch_manual_edge_ports_conflict_409`。 -- 手工测试：修改已保存手工边的端口 → 出现未保存标记 → 保存 → 刷新后仍是新端口；LLDP 边的端口输入框为只读。 - -### T5 [P1] 连线样式与撤销/保存模型不一致 - -- `patchSelectedEdgeStyle` (`TopologyPage.tsx:807-815`) 每次修改都立即发请求（颜色选择器拖动时会连续发很多次）。同时调用了 `pushHistory()`，但撤销只回退本地状态，所以撤销后画布和数据库不一致。 - -**修改：** 样式也改为走保存：记入 `pendingEdgeStyleRef: Map`，`markDirty()`，在 `commitCanvas()` 里批量提交（后端 `PATCH /edge-style` 支持 `items: []` 批量，旧的单条 body 保留兼容）。撤销快照要包含这个 Map。 - -**验收：** 改颜色 → 撤销 → 保存 → 刷新，颜色是修改前的值；改颜色 → 保存 → 刷新，颜色是修改后的值；拖动颜色选择器期间，网络面板里没有 `/edge-style` 请求。 - -### T6 [P1] 保存原子化，并去掉重复的保存逻辑 - -现状见第 1 节。中途失败时，前面已执行的步骤（例如已建的边、已加的点）不会回滚，`pending*` 集合也只清了一部分。建边是幂等 upsert（`topology_fabric_links.py:65`），重试不会建出重复边，但删点和写坐标的顺序问题依然存在。 - -**修改：** -1. 后端新增 `POST /v1/topology/views/{id}/commit`，在一个事务里依次处理： - ```json - { - "edge_creates": [{"client_id":"local-..","a_node_id":"","b_node_id":"","a_port":"","b_port":""}], - "edge_deletes": ["edge-id"], - "edge_port_edits": [{"id":"","a_port":"","b_port":""}], - "edge_styles": [{"fabric_edge_id":"","stroke_color":"","stroke_width":0,"line_style":""}], - "node_adds": ["fabric-id"], - "node_removes": ["fabric-id"], - "positions": [{"fabric_node_id":"","x":0,"y":0,"label":null}] - } - ``` - 任何一步失败都整体回滚，返回 4xx 和 `{"failed_step": "...", "detail": "..."}`。成功时只返回**一次**完整图数据，外加 `edge_id_map: {client_id: server_id}`。节点上限和冻结规则与 `add_nodes_to_view` 一致。 -2. 前端新增 `hooks/useTopologySave.ts`，导出 `commitCanvas()`：从本地状态计算差异（纯函数 `lib/saveDiff.ts`，可以单独测试），调用 `/commit`，成功后调用 `resetLocalFromServer`。`saveMut`、`purgePlaceholderById` (`:1248-1281`)、T3 的“先保存”都改用它。 -3. 旧的分步接口保留（MCP 在用）。 - -**验收：** -- 新增 `test_view_commit_atomic_rollback`：构造一个 `edge_creates` 自环（第一步就失败），断言 `node_adds` 和 `positions` 都没有落库。 -- 新增 `test_view_commit_happy_path`：一次提交包含建边、删边、加点、删点、写坐标，结果图正确，`edge_id_map` 正确。 -- `web/tests/topologySaveDiff.test.mjs`：覆盖“本地新增/删除节点”“region 节点不进入增删列表”“只发送有变化的 label”“删除后又撤销的边不出现在 edge_deletes 中”等情况。 -- 手工测试：一次保存只产生 **1 个** 写请求（网络面板截图附在 PR 描述里）。 - -### T7 [P1] 后端视图接口的 N+1 查询与不确定截断 - -| 位置 | 问题 | 修改 | -|---|---|---| -| `topology_views_graph.py:719` | 写坐标时每个点各执行一次 `db.get(TopoFabricNode)`，最多 2000 次 | 先用一次 `IN` 查询拿到存在的 id 集合 | -| `:160-166` | 节点超过 `VIEW_GRAPH_NODE_HARD_CAP` 时截断没有 `ORDER BY`，每次显示哪些节点不确定 | `order_by(TopoViewNode.created_at, TopoViewNode.id)` | -| `:354-363` | 外部邻居的 `limit(5000)` 把**两端都在视图内**的边也算进去了，边密集的视图会漏掉外部邻居 | SQL 里排除两端都在视图内的边（`~and_(a.in_, b.in_)`），再加 `limit` | -| `:375` | 每个外部邻居执行一次 `db.get` | 收集 id 后用一次 `IN` 查询 | -| `:178-179` | region 节点每个执行一次 `db.get(TopoFolder)` | 一次 `IN` 查询 | - -**验收：** -- 新增 `test_outside_peers_found_when_many_internal_edges`：视图内有 >5000 条内部边（测试中可以把上限改成参数，例如用 10 条模拟），外部邻居仍然能返回。 -- 新增 `test_view_truncation_is_deterministic`：两次 GET 返回同一批节点。 -- 用 SQLAlchemy `before_cursor_execute` 统计 2000 个点的 positions 请求，语句数 ≤ 10（写成测试）。 - -### T8 [P1] positions 接口可以绕过节点上限加点 - -`patch_view_positions` 遇到视图中没有的 `fabric_node_id` 会直接插入 `TopoViewNode`（`:720-733`），不检查 `max_nodes` 和冻结状态，而 `add_nodes_to_view` 会检查（`:778`）。 - -**修改：** 先确认 MCP 是否依赖“positions 顺便加点”（用 grep 查 `packages/netx-topology-mcp` 里的 `positions` 调用）。如果不依赖，就改为跳过不存在的节点，并在返回里计入 `skipped_missing`；如果依赖，就加上与 `add_nodes_to_view` 相同的上限和冻结检查。**这一项的结论需要写进 PR 描述。** - -**验收：** 新增 `test_positions_respects_max_nodes`。 - -### T9 [P2] 点击节点就会产生“未保存”和无效撤销记录 - -`onNodeDragStart` 会无条件 `pushHistory()`；`onNodesChange` 收到任何 `position` 变更都会 `markDirty()`，包括没有实际移动的点击 (`TopologyReactFlowView.tsx:442-449`)。后果：单击一个节点就出现“未保存”，并且触发脏锁，live sync 不再整体刷新。 - -**修改：** 在 `onNodeDragStart` 记录起点坐标；`onNodeDragStop` 时只有位移超过 1px 才把起始快照压入撤销栈并 `markDirty()`。`onNodesChange` 不再对 `position` 变更调用 `markDirty`（`remove`/`add` 保持不变）。 - -**验收：** 单击节点后不出现未保存标记，撤销按钮保持禁用；拖动 10px 后出现未保存标记，并且可以撤销。 - -### T10 [P2] 拆分 `TopologyPage.tsx`（在 T1–T6 完成后进行） - -目标是把主组件压到 ≤ 1200 行，**只搬代码、不改行为**： -- `hooks/useTopologySave.ts`（T6） -- `hooks/useTopologyPalette.ts`：`addPaletteItems`、`addOutsidePeers`、拖放、`neQuery`、`umeQuery` -- `hooks/useTopologyEdgeEditing.ts`：`patchSelectedEdgeStyle`、`queueDeleteEdges`、`removeSelected`、`onConnect`、`isValidConnection` -- `hooks/useTopologyKeyboard.ts`：全局快捷键（`:1130-1182`）。依赖改为通过 ref 读取，避免每次 `nodes`/`edges` 变化都重新绑定监听 -- `lib/moduleLinks.ts`：`openNeInventory`、`openWebcrtFor`、`openAlarmsFor`、`openPortTrafficForEdge` - -**验收：** `tsc -b` 通过；第 4 节的手工回归清单全部通过；`git diff --stat` 显示 TopologyPage.tsx 净减少 ≥ 1000 行。 - -### T11 [P2] 修复修改前就存在的失败测试 - -| 测试 | 原因 | 处理 | -|---|---|---| -| `test_ume_topology_apply.py::test_lldp_upgrades_bare_ume_ports_and_keeps_ume_primary` | **测试本身不稳定**：`rich_a/rich_b` 取自 `edge.a_port/b_port`（按节点 id 排序后的 A/B），但调用 upsert 时传的是 `a.id/b.id`（按 UME 名称取的节点）。uuid 排序不同时端口就配反了，结果约有一半概率失败。已用调试脚本确认这不是产品 bug。 | 改为传 `a_node_id=edge.a_node_id, b_node_id=edge.b_node_id`（`:358-365`）。`:297`、`:393` 有同样的写法，一起检查 | -| `test_topology.py::test_lldp_discover_writes_fabric_edge`、`test_lldp_unmatched_creates_ssh_placeholder_ne` | 发现任务一直是 `pending`，没有变成 `done`（`:1285`、`:1458`） | 排查后台 worker 在 SQLite 内存库下是否被启动或被跳过，修好测试的启动方式，或者改为同步执行 job | -| `test_topology.py::test_fabric_reconcile_scheduler_once_sweeps_orphans` | `0 not >= 1`（`:2287`） | 排查，可能和上一条同源 | - -**验收：** 两个测试文件 **0 失败**，并且连续跑 5 次都通过（用于检验不稳定性）。 - -### 产品决定（2026-10-12 已确认） - -- **Q1**：`isValidConnection` (`TopologyPage.tsx:568`) 不允许在同一对网元之间画第二条手工连线，但 LLDP/UME 支持并行链路，画布也支持聚合显示。是否应该允许手工建并行链路（端口不同即可）？ -- **Q2**：T3 选择了“先保存再添加”。如果希望“添加后保留本地未保存修改”，实现会复杂很多（需要把服务端新增部分合并进本地），请确认。 - -- **结论**：Q1 不允许手工画并行链路，保持 `isValidConnection`（现位于 `hooks/useTopologyEdgeEditing.ts`）现状；Q2 沿用“先保存再添加”。 - ---- - -## 3. 建议实施顺序 - -1. **T11**（先把基线修到全部通过，后面每一步的回归才有意义） -2. **T1 + T2**（后端 schema 改动，两项一起做，再加数据修复脚本） -3. **T6**（commit 接口 + `useTopologySave`），然后在它之上做 **T3、T4、T5** -4. **T7、T8** -5. **T9**，最后做 **T10**（纯重构，单独一个 PR） - -每一步单独提交；T10 必须单独成 PR。 - -## 4. 回归命令与手工清单 - -```bash -python -m pytest tests/test_topology.py tests/test_ume_topology_apply.py -q -``` - -```bash -cd web && npx tsc -b -``` - -```bash -cd web && node --experimental-strip-types --test tests/topologySaveDiff.test.mjs -``` - -> `lib/saveDiff.ts` 只放纯函数，除 `import type` 外不要有运行时 import（Node strip-types 不会解析不带扩展名的相对路径）。写法参考现有的 `web/tests/cutoverDataCache.test.mjs`。 - -手工回归（每个 PR 都要过一遍，结果写进 PR 描述）： -1. 拖动节点 → 保存 → 刷新，位置保持。 -2. 新建手工连线（带端口）→ 保存 → 刷新，连线和端口都在。 -3. 删除连线 → 撤销 → 保存，连线仍然存在。 -4. 删除节点 → 保存，节点从视图中移除，但 fabric 中仍存在（可以通过搜索找到）。 -5. 重命名节点 → 保存 → 刷新，显示新名称；再把名称清空 → 保存，恢复为设备名。 -6. 打开 live sync，另一个窗口用 MCP 移动节点，3 秒内本窗口同步；本窗口有未保存修改时不会被覆盖，只刷新在线状态。 -7. 只读用户（没有 `ne:write`）：不能拖动、连线、保存，右键菜单里没有写操作。 -8. 删除占位网元（有未保存修改时），修改先被保存，占位网元被清理。 -9. 导出 SVG/XML 正常。 - ---- - -## 5. 验收记录（2026-10-12） - -Cursor 完成 T1–T11 后由 Claude 验收并接手修复，结论：主体实现正确，另修 6 个问题（均补了回归测试）： - -1. `/commit` 同次 `node_adds` 的节点坐标被忽略（`autoflush=False` 下重查看不到新行）→ 先 `flush`；`_apply_positions` 的逐点 `db.get` 改为批量。 -2. T8 收紧后，无放置行的旧子区域图标拖动无法保存 → `add_missing_region_placement`（仅限本画布的手工子区域）。 -3. MCP `updateTopologyViewPositions` 总是发送 `label:""`、`locked:false`，会清掉 UI 重命名并解锁 → 未提供则不发送。 -4. 已改样式/端口的边被删除后保存 404 → 前端 `saveDiff` 与后端都跳过已删边；新建本地边的样式经 `edge_id_map` 映射。 -5. 框选拖动、方向键移动不标记未保存 → 补 `onSelectionDrag*` 和键盘移动判定。 -6. （改动前已存在）大视图打开空白、远处节点定位失效：xyflow `fitView` 等待节点测量，而 `onlyRenderVisibleElements` 下视口外节点永不测量 → `lib/fitNodes.ts` 按已知几何计算 `fitBounds`。 - -开发库数据修复：`topology_clear_frozen_labels.py --apply` 清理 516 条固化标签（执行前已备份原值）。 - -回归：拓扑后端 91 通过；Web Node 测试 30 通过；`tsc -b` 通过；MCP 包 238 通过 / 5 失败（与改动前同一批，既有问题）。 diff --git a/docs/reviews/workbench-custom-modules-plan-2026-10-13.md b/docs/reviews/workbench-custom-modules-plan-2026-10-13.md deleted file mode 100644 index 82f52f7..0000000 --- a/docs/reviews/workbench-custom-modules-plan-2026-10-13.md +++ /dev/null @@ -1,112 +0,0 @@ -# 工作台功能卡片自定义 — 实现方案（2026-10-13） - -## 目标 - -工作台功能区（UME 同步、网元管理、网络管理、拓扑、WebCRT…）以后会越来越多，每个人关心的不一样。 -让每个用户自己选择：**显示哪些卡片、按什么顺序**；设置跟着账号走（换电脑/换浏览器仍生效）。 - -## 现状 - -- 卡片注册表：`web/src/config/modules.ts:33` `MODULES`，按 `section` 分 `monitoring / operations / system`。 -- 过滤逻辑：`web/src/features/workbench/WorkbenchPage.tsx:323` `visibleBySection` —— 只按权限（`adminOnly` / `requiredScope`）和 `workbenchHidden` 过滤。 -- 渲染：`web/src/features/workbench/components/WorkbenchModules.tsx:23`，operations 是大磁贴网格，system 是底部一排小链接。 -- 后端无任何“用户偏好”存储；`AppUser`（`netx_api/models/auth.py:13`）只有账号字段。 - -## 设计决定 - -| 项 | 决定 | 理由 | -|---|---|---| -| 可自定义范围 | 只做 **operations 区的磁贴**（含 monitoring 合并进来的 UME 同步）。system 区链接行保持固定 | 系统管理入口少且是管理类，隐藏后反而找不到；需求说的是“功能卡片” | -| 存储位置 | **服务端，按用户**：`app_user.prefs` JSON 列 | “每个人不一样”= 按账号；localStorage 只按浏览器，换机就丢 | -| 本地缓存 | 同时写一份 localStorage（key 带 user id），首屏先用缓存渲染，避免卡片闪烁 | 首屏体验 | -| 数据结构 | `{ order: string[], hidden: string[] }`，存的是“隐藏了哪些”而不是“显示哪些” | 以后新增模块默认对所有人出现，不需要每人手动开启 | -| 新模块默认隐藏 | `ModuleDefinition` 增加可选 `defaultHidden?: boolean`；不在 `order` 里的这类模块视为隐藏 | 给以后“小众功能”留开关 | -| 权限 | 没权限的模块永远不出现在卡片区，也不出现在“可添加”列表；偏好里残留的 id 静默忽略 | 偏好不能绕过权限 | -| 交互 | 卡片区右上角一个小齿轮 → 进入**原地编辑模式**：已显示卡片可拖拽排序、右上角 × 移除；下方出现“可添加”托盘（灰显卡片，点击加入）；按钮只有「恢复默认」「完成」 | 和 iOS 主屏编辑一致，最少文案；不弹窗 | -| 拖拽实现 | HTML5 原生 drag & drop，不引入新依赖 | 卡片数量小（<20），原生够用；项目里没有 dnd 库 | -| 保存时机 | 点「完成」时一次 PUT；失败则 toast 并保留编辑状态 | 避免拖动过程中频繁请求 | -| 全部隐藏 | 允许，但功能区显示一个空态（只有齿轮 + 一句“添加功能”） | 不阻止用户，但要能找回 | - -## 任务 - -### 后端 - -**B1 — `app_user.prefs` 列** -- `netx_api/models/auth.py:13` `AppUser` 加 `prefs: Mapped[dict] = mapped_column(_JsonType, default=dict)`。 -- 新迁移 `alembic/versions/20261013_user_prefs.py`，`down_revision = "20260922_biz_state_queued"`，`add_column("app_user", sa.Column("prefs", JSON, nullable=False, server_default="{}"))`。 - -**B2 — 读写接口**（`netx_api/auth_router.py`，紧跟 `api_me` 之后，约 :273） -- `GET /v1/auth/me/prefs` → `{"prefs": {...}}` -- `PUT /v1/auth/me/prefs/{key}`，body 为该 key 的值；`key` 白名单，目前只允许 `workbench`。 -- 校验 `workbench`：`{order: list[str], hidden: list[str]}`，每个列表 ≤ 64 项、每个 id ≤ 64 字符、只允许 `[a-z0-9-]`；其他字段丢弃。超限 422。 -- 只改自己的偏好（`require_user`），不需要额外 scope；不写审计（纯 UI 偏好，避免刷屏）。 -- `/v1/auth/me` 响应里顺带返回 `prefs`，前端登录后一次拿到，不多一次请求。 - -### 前端 - -**F1 — 注册表扩展** `web/src/config/modules.ts` -- `ModuleDefinition` 加 `defaultHidden?: boolean`。 -- 新增 `export function customizableModules()`：operations + monitoring 中非 `workbenchHidden` 的模块（即当前 `displaySections` 里 operations 那组的来源）。 - -**F2 — 偏好 hook** 新建 `web/src/features/workbench/useWorkbenchLayout.ts` -- 输入：权限过滤后的可用模块列表；输出：`{ shown: ModuleDefinition[], available: ModuleDefinition[], save(order, hidden), reset() }`。 -- 解析规则（纯函数 `resolveLayout(all, prefs)`，单独导出便于测试）： - 1. 先按 `prefs.order` 排已知 id，丢弃不存在/无权限的 id； - 2. 剩余模块按注册表顺序追加到末尾； - 3. 在 `hidden` 中的、或 `defaultHidden` 且不在 `order` 中的，归入 `available`。 -- 数据来源：`AuthContext` 的 `me.prefs.workbench` → 没有则 localStorage 缓存 `netx.wb.layout.` → 默认。 -- 保存：PUT 成功后更新 AuthContext 中的 prefs 和 localStorage。localStorage 读写全部 try/catch。 - -**F3 — AuthContext** `web/src/features/auth/AuthContext.tsx:48` -- `refreshMe` / `login` 把响应里的 `prefs` 存进 state，并暴露 `prefs` 与 `setPrefs`。 - -**F4 — 编辑模式 UI** `web/src/features/workbench/components/WorkbenchModules.tsx` -- `WorkbenchPage.tsx:334` `displaySections` 的 operations 组改为用 `useWorkbenchLayout().shown`。 -- 卡片区右上角齿轮按钮（`ConsoleIcon` 加一个 `settings` 图标），`aria-label` = “自定义功能区”。 -- 编辑态： - - 磁贴轻微抖动或描边（遵循 glass 风格：**不发光**，只用 `--nx-line` 边框）、右上角 × 按钮； - - `draggable`，`onDragStart/onDragOver/onDrop` 重排本地草稿数组； - - 下方“可添加”托盘：灰显磁贴，点击加入末尾；托盘为空时不显示； - - 底部两个按钮：「恢复默认」（清空草稿为默认）、「完成」（保存）；Esc = 放弃草稿退出。 - - 编辑态下点击磁贴不打开模块。 -- 非编辑态 shown 为空：显示空态 + “添加功能”按钮（直接进入编辑态）。 -- 样式写在 `web/src/styles/pages/workbench.css`，颜色只用现有 token（蓝黑 200–215°，无紫色）。 - -**F5 — 文案** `web/src/i18n/locales/{zh,en}/workbench.ts` -- 新增：`customize`（自定义功能区）、`customizeDone`（完成）、`customizeReset`（恢复默认）、`customizeAvailable`（可添加）、`customizeEmpty`（添加功能）、`customizeSaveFailed`（保存失败）。只这些，不加说明文字。 - -## 验收 - -| ID | 场景 | 期望 | -|---|---|---| -| A1 | 新用户首次登录 | 卡片与改动前完全一致（顺序、数量） | -| A2 | 隐藏「拓扑」、把「网元管理」拖到第一位，点完成 | 立即生效；刷新、换浏览器登录同一账号仍是该布局 | -| A3 | 用户 A 定制后，用户 B 登录 | B 不受影响 | -| A4 | 在 `MODULES` 新增一个模块 | 已定制过的用户自动在末尾看到它；若设了 `defaultHidden` 则出现在“可添加”里 | -| A5 | 偏好里包含无权限模块 id（如去掉用户 `ne:read` 后） | 不显示、不出现在可添加；不报错 | -| A6 | 全部移除后完成 | 显示空态，点“添加功能”可恢复 | -| A7 | 编辑中按 Esc / 保存失败（断网） | Esc 恢复原布局；失败提示并停留在编辑态 | -| A8 | `PUT /v1/auth/me/prefs/workbench` 传非法 key、超长列表、非法字符 | 404 / 422 | -| A9 | 视觉 | 编辑态无发光、无紫色，与现有 glass 风格一致 | - -## 回归命令 - -```bash -alembic upgrade head -``` - -```bash -python -m pytest tests/test_auth.py tests/test_user_prefs.py -q -``` - -```bash -cd web && npm run lint && npm run build -``` - -- 新增 `tests/test_user_prefs.py`：覆盖 B2 的读写、白名单、校验、用户隔离（A3、A8）。 -- `resolveLayout` 纯函数若项目有前端单测则补上（A1、A4、A5）；没有则在验收时手测。 - -## 以后可扩展（本次不做） - -- 同一机制用 `prefs.workbench.panels` 控制右侧任务栏、网元/拓扑/调度三张统计卡的显隐。 -- 管理员给角色设默认布局（`role_default_prefs`），用户未定制时用角色默认。 diff --git a/netx_api/app_startup.py b/netx_api/app_startup.py index 23d8a0c..ea16cd6 100644 --- a/netx_api/app_startup.py +++ b/netx_api/app_startup.py @@ -15,7 +15,6 @@ from .schema_patches import ( apply_topology_schema_safety_net, run_alembic_upgrade_to_head, ) -from .ne_crypto import ensure_credential_secret_key from .security_bootstrap import assert_secure_defaults_or_exit from .ume_runtime import start_api_sideband_threads, start_device_schedulers import netx_api.ume_support as ume_support @@ -43,8 +42,6 @@ def _configure_ume_diag_logging() -> None: def run_api_startup() -> None: """Full API boot sequence previously inlined in ``main.on_startup``.""" assert_secure_defaults_or_exit() - # Persist Fernet key for managed-NE passwords (same idea as JWT secret file). - ensure_credential_secret_key() _configure_ume_diag_logging() _log.info( "startup: ne_exec_policy_enabled=%s", @@ -159,25 +156,6 @@ def run_api_startup() -> None: ) except Exception: _log.exception("startup: biz_state collect recovery failed") - try: - from .biz_state.compare_service import recover_interrupted_compares_on_startup - - cmp_rec = recover_interrupted_compares_on_startup(db) - if cmp_rec.get("runs"): - _log.info( - "startup: cancelled %s interrupted biz compare run(s)", - cmp_rec.get("runs"), - ) - except Exception: - _log.exception("startup: biz compare recovery failed") - try: - from .biz_migration.service import recover_interrupted_evaluations_on_startup - - n_eval = recover_interrupted_evaluations_on_startup(db) - if n_eval: - _log.info("startup: cancelled %s interrupted cutover evaluation(s)", n_eval) - except Exception: - _log.exception("startup: cutover evaluation recovery failed") try: from .port_traffic_migrate import backfill_port_traffic_series diff --git a/netx_api/auth_router.py b/netx_api/auth_router.py index 1ff052a..f6e0d0b 100644 --- a/netx_api/auth_router.py +++ b/netx_api/auth_router.py @@ -2,12 +2,10 @@ from __future__ import annotations -import re from typing import Annotated, Any -from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response +from fastapi import APIRouter, Depends, Query, Request, Response from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified from .auth_cookies import clear_auth_cookies, read_refresh_cookie, set_auth_cookies from .auth_deps import AuthContext, require_admin, require_user @@ -32,13 +30,11 @@ from .auth_service import ( list_users, login_issue_token, refresh_login_tokens, - delete_api_token, revoke_api_token, revoke_auth_session_for_user, revoke_auth_sessions, update_api_token, update_user, - user_prefs, user_public, write_audit, ) @@ -49,14 +45,9 @@ from .auth_rate_limit import ( ) from .client_ip import resolve_client_ip from .db import get_db -from .models import AppUser router = APIRouter(tags=["auth"]) -_PREF_KEYS = frozenset({"workbench"}) -_PREF_ID_RE = re.compile(r"^[a-z0-9-]{1,64}$") -_PREF_LIST_MAX = 64 - def _client_meta(request: Request) -> tuple[str, str]: ip = resolve_client_ip(request) @@ -64,34 +55,6 @@ def _client_meta(request: Request) -> tuple[str, str]: return ip, ua -def _validate_id_list(value: Any, *, field: str) -> list[str]: - if not isinstance(value, list): - raise HTTPException(status_code=422, detail=f"invalid_{field}") - if len(value) > _PREF_LIST_MAX: - raise HTTPException(status_code=422, detail=f"{field}_too_long") - out: list[str] = [] - for item in value: - if not isinstance(item, str) or not _PREF_ID_RE.fullmatch(item): - raise HTTPException(status_code=422, detail=f"invalid_{field}_item") - out.append(item) - return out - - -def _validate_workbench_prefs(value: Any) -> dict[str, list[str]]: - if not isinstance(value, dict): - raise HTTPException(status_code=422, detail="invalid_workbench_prefs") - return { - "order": _validate_id_list(value.get("order", []), field="order"), - "hidden": _validate_id_list(value.get("hidden", []), field="hidden"), - } - - -def _validate_pref_value(key: str, value: Any) -> Any: - if key == "workbench": - return _validate_workbench_prefs(value) - raise HTTPException(status_code=404, detail="unknown_pref_key") - - def _token_response(response: Response, request: Request, out: dict[str, Any]) -> dict[str, Any]: access = str(out.get("access_token") or "") refresh = str(out.get("refresh_token") or "") @@ -305,39 +268,9 @@ def api_me(ctx: Annotated[AuthContext, Depends(require_user)]) -> dict[str, Any] "auth_via": ctx.auth_via, "scopes": sorted(ctx.scopes), "all_scopes": sorted(ALL_SCOPES), - "prefs": user_prefs(ctx.user), } -@router.get("/v1/auth/me/prefs") -def api_get_prefs(ctx: Annotated[AuthContext, Depends(require_user)]) -> dict[str, Any]: - return {"prefs": user_prefs(ctx.user)} - - -@router.put("/v1/auth/me/prefs/{key}") -def api_put_pref( - key: str, - body: dict[str, Any], - ctx: Annotated[AuthContext, Depends(require_user)], - db: Session = Depends(get_db), -) -> dict[str, Any]: - if key not in _PREF_KEYS: - raise HTTPException(status_code=404, detail="unknown_pref_key") - validated = _validate_pref_value(key, body) - # Reload in this session: ctx.user may be the middleware's detached copy, or the - # synthetic "system" user when auth is disabled (no row — never insert it). - user = db.get(AppUser, ctx.user.id) - if user is None: - raise HTTPException(status_code=409, detail="prefs_unavailable") - prefs = user_prefs(user) - prefs[key] = validated - user.prefs = prefs - flag_modified(user, "prefs") - db.commit() - db.refresh(user) - return {"prefs": user_prefs(user)} - - @router.post("/v1/auth/change-password") def api_change_password( body: ChangePasswordRequest, @@ -604,27 +537,3 @@ def api_revoke_token( detail={"token_id": row.id, "name": row.name}, ) return {"ok": True, "id": row.id} - - -@router.delete("/v1/api-tokens/{token_id}/purge") -def api_purge_token( - token_id: str, - request: Request, - ctx: Annotated[AuthContext, Depends(require_user)], - db: Session = Depends(get_db), -) -> dict[str, Any]: - row = delete_api_token(db, token_id=token_id, actor=ctx.user) - ip, ua = _client_meta(request) - write_audit( - db, - action="api_tokens.purge", - actor_user_id=ctx.user.id, - actor_username=ctx.user.username, - method="DELETE", - path=f"/v1/api-tokens/{token_id}/purge", - status_code=200, - client_ip=ip, - user_agent=ua, - detail={"token_id": row.id, "name": row.name}, - ) - return {"ok": True, "id": row.id} diff --git a/netx_api/auth_service.py b/netx_api/auth_service.py index 464bd1d..5aa9577 100644 --- a/netx_api/auth_service.py +++ b/netx_api/auth_service.py @@ -76,11 +76,6 @@ def user_public(user: AppUser) -> dict[str, Any]: } -def user_prefs(user: AppUser) -> dict[str, Any]: - raw = getattr(user, "prefs", None) - return dict(raw) if isinstance(raw, dict) else {} - - def sanitize_detail(detail: Any) -> Any: """Recursively drop secret-looking keys from audit detail payloads.""" if isinstance(detail, dict): @@ -436,7 +431,6 @@ def login_issue_token( "expires_in": ttl, "refresh_expires_in": refresh_ttl, "user": user_public(user), - "prefs": user_prefs(user), } @@ -673,18 +667,6 @@ def revoke_api_token(db: Session, *, token_id: str, actor: AppUser) -> ApiToken: return row -def delete_api_token(db: Session, *, token_id: str, actor: AppUser) -> ApiToken: - """Permanently remove an API key row (revoked or active).""" - row = db.query(ApiToken).filter(ApiToken.id == str(token_id)).one_or_none() - if row is None: - raise HTTPException(status_code=404, detail="api_token_not_found") - if actor.role != "admin" and row.user_id != actor.id: - raise HTTPException(status_code=403, detail="forbidden") - db.delete(row) - db.commit() - return row - - def update_api_token( db: Session, *, diff --git a/netx_api/biz_migration/auto_monitor.py b/netx_api/biz_migration/auto_monitor.py deleted file mode 100644 index 4a0154c..0000000 --- a/netx_api/biz_migration/auto_monitor.py +++ /dev/null @@ -1,120 +0,0 @@ -"""Coalesced collect-completion events for live cutover evaluation. - -One worker per process bounds concurrency. Project row locks in run_evaluate -serialize persistence across processes. No device collection runs here. -""" -from __future__ import annotations - -import hashlib -import json -import logging -import threading - -from sqlalchemy.orm import Session - -from ..models import BizMigrationBatch, BizMigrationProject, BizMigrationRun, BizMonitorTemplate, BizStateBatch -from . import service - -_log = logging.getLogger(__name__) -_lock = threading.Lock() -_pending: set[str] = set() -_worker_running = False - - -def try_auto_monitor_for_task(db: Session, task_id: str) -> int: - count = 0 - # Bindings are JSON for multiple sampling intervals; match in Python after - # limiting to projects with an active operation batch. - projects = db.query(BizMigrationProject).filter( - BizMigrationProject.status != "done", - BizMigrationProject.id.in_(db.query(BizMigrationBatch.project_id).filter( - BizMigrationBatch.status == "active")), - ).all() - for candidate in projects: - tids = service._all_hf_task_ids(candidate, "old") + service._all_hf_task_ids(candidate, "new") - if task_id not in tids or not candidate.old_baseline_batch_id or not candidate.new_baseline_batch_id: - continue - # Lock before deduplication so two worker processes cannot save the same - # snapshot. run_evaluate holds the same lock through the transaction. - proj = db.query(BizMigrationProject).filter( - BizMigrationProject.id == candidate.id).with_for_update().populate_existing().first() - batch = db.query(BizMigrationBatch).filter( - BizMigrationBatch.project_id == proj.id, BizMigrationBatch.status == "active" - ).order_by(BizMigrationBatch.started_at.desc()).first() - if not batch or proj.status == "done": - db.rollback() - continue - if db.query(BizMigrationRun.id).filter( - BizMigrationRun.project_id == proj.id, BizMigrationRun.status == "running" - ).first(): - db.rollback() # a manual / acceptance evaluation owns the project right now - continue - mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None - if not mt: - db.rollback() - _log.error("cutover monitor template missing project=%s", proj.id) - continue - sheets, overrides, defaults, norms = service.resolve_evaluate_sheets(db, mt) - samples = [] - for tid in tids: - sample = db.query(BizStateBatch).filter( - BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued")) - ).order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() - samples.append([tid, sample.id if sample else "", sample.status if sample else ""]) - payload = [samples, proj.old_baseline_batch_id, proj.new_baseline_batch_id, - batch.expect_set_json, service._port_map_dict(db, proj.mapping_id), - sheets, overrides, defaults, norms, service.resolve_collect_metric_ids(db, proj)] - fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest() - last = db.query(BizMigrationRun).filter( - BizMigrationRun.batch_id == batch.id, BizMigrationRun.status == "success").order_by( - BizMigrationRun.created_at.desc()).first() - if last and (last.summary_json or {}).get("auto_fingerprint") == fingerprint: - db.rollback() - continue - previous_samples = {s[0]: (s[1], s[2]) for s in (last.summary_json or {}).get("auto_samples", [])} if last else {} - if last and not previous_samples: - for sample in (last.summary_json or {}).get("metric_batches", {}).values(): - for side in ("old", "new"): - previous_samples[sample.get(f"{side}_task_id", "")] = ( - sample.get(f"{side}_batch_id", ""), sample.get(f"{side}_status", "success")) - changed_tasks = {tid for tid, bid, status in samples if previous_samples.get(tid) != (bid, status)} - changed_metrics = {mid for side in ("old", "new") for binding in service._hf_bindings(proj, side) - if binding["task_id"] in changed_tasks for mid in binding.get("metric_ids") or payload[-1]} - try: - result = service.run_evaluate(db, batch_id=batch.id, purpose="auto", _commit=False, - _metric_ids=changed_metrics) - run = db.get(BizMigrationRun, result["id"]) - run.summary_json = {**run.summary_json, "auto_fingerprint": fingerprint, "auto_samples": samples} - db.commit() - count += 1 - except Exception: - db.rollback() - _log.exception("cutover monitor evaluate failed project=%s", proj.id) - return count - - -def schedule_auto_monitor_for_task(task_id: str) -> None: - global _worker_running - with _lock: - _pending.add(str(task_id)) - if _worker_running: - return - _worker_running = True - - def work() -> None: - global _worker_running - from ..db import SessionLocal - - while True: - with _lock: - if not _pending: - _worker_running = False - return - tid = _pending.pop() - try: - with SessionLocal() as db: - try_auto_monitor_for_task(db, tid) - except Exception: - _log.exception("cutover monitor worker failed task=%s", tid) - - threading.Thread(target=work, name="biz-cutover-monitor", daemon=True).start() diff --git a/netx_api/biz_migration/evaluate.py b/netx_api/biz_migration/evaluate.py index dd34fc0..eb85841 100644 --- a/netx_api/biz_migration/evaluate.py +++ b/netx_api/biz_migration/evaluate.py @@ -4,10 +4,7 @@ from __future__ import annotations from typing import Any -from ..biz_state.compare_engine import apply_port_map, compare_rows, key_tuple -from ..biz_state.compare_rules import ( - effective_compare_fields, field_rule_map, scalar_text, values_equal, -) +from ..biz_state.compare_engine import apply_port_map, compare_rows PORT_METRIC_ID = "interface_brief" PORT_STATUS_FIELDS = ("admin", "phy", "prot") @@ -25,7 +22,7 @@ def normalize_expect_key(raw: str) -> str: Accepts legacy ``a|b`` multi-field keys and already-normalized ``\\x1f`` keys. """ - s = scalar_text(raw).strip() + s = str(raw or "").strip() if not s: return "" if KEY_SEP in s: @@ -68,13 +65,13 @@ def classify_status(row: dict[str, Any] | None, sheet_override: dict[str, Any] | return "none" down_vals = {str(x).lower() for x in (ov.get("down_values") or ["down"])} up_vals = {str(x).lower() for x in (ov.get("up_values") or ["up"])} - vals = [scalar_text(row.get(f)).strip().lower() for f in fields] + vals = [str(row.get(f) or "").strip().lower() for f in fields] vals = [v for v in vals if v] if not vals: return "none" if any(v in down_vals for v in vals): return "down" - if len(vals) == len(fields) and all(v in up_vals for v in vals): + if vals and all(v in up_vals for v in vals): return "up" return "other" @@ -108,11 +105,11 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]: if it.get("key") is not None: k = it.get("key") if isinstance(k, (list, tuple)): - joined = KEY_SEP.join(scalar_text(p).strip() for p in k) - if joined.strip(KEY_SEP): + joined = KEY_SEP.join(str(p).strip() for p in k if str(p).strip()) + if joined: bucket.add(joined) else: - s = normalize_expect_key(scalar_text(k)) + s = normalize_expect_key(str(k or "")) if s: bucket.add(s) keys = it.get("keys") @@ -120,14 +117,14 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]: # Flat list of segments → one composite key; else each entry is a key # (string or nested list/tuple of segments). if keys and all(not isinstance(x, (list, tuple, dict)) for x in keys): - joined = KEY_SEP.join(scalar_text(x).strip() for x in keys) - if joined.strip(KEY_SEP): + joined = KEY_SEP.join(str(x).strip() for x in keys if str(x).strip()) + if joined: bucket.add(joined) else: for x in keys: if isinstance(x, (list, tuple)): - joined = KEY_SEP.join(scalar_text(p).strip() for p in x) - if joined.strip(KEY_SEP): + joined = KEY_SEP.join(str(p).strip() for p in x if str(p).strip()) + if joined: bucket.add(joined) else: s = normalize_expect_key(str(x or "")) @@ -222,7 +219,7 @@ def _field_tokens_from_row(row: dict[str, Any] | None, fields: set[str]) -> set[ return set() out: set[str] = set() for f in fields: - v = scalar_text(row.get(f)).strip() + v = str(row.get(f) or "").strip() if v: out.add(field_token(f, v)) return out @@ -248,8 +245,8 @@ def _side_tokens( def _norm_list_values(raw: Any) -> list[str]: if isinstance(raw, list): - return [scalar_text(x).strip().lower() for x in raw if scalar_text(x).strip()] - s = scalar_text(raw).strip() + return [str(x).strip().lower() for x in raw if str(x).strip()] + s = str(raw or "").strip() if not s: return [] if "," in s: @@ -269,22 +266,18 @@ def _eval_field_op( f = str(field or "").strip() if not f: return False - raw = "" if not row else scalar_text(row.get(f)).strip() + raw = "" if not row else str(row.get(f) or "").strip() val = raw.lower() o = str(op or "eq").strip().lower() - if row and f not in row: - # A missing parser field is not evidence for a negative condition such as - # state != down. Only an explicit empty check may match that absence. - return o == "empty" if o in ("changed", "diff"): - base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower() + base = "" if not base_row else str(base_row.get(f) or "").strip().lower() if not row: return False return base != val if o in ("unchanged", "same"): if not row: return False - base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower() + base = "" if not base_row else str(base_row.get(f) or "").strip().lower() return base == val if o == "empty": return not val @@ -575,7 +568,7 @@ def _match_field_token_rules( if not field or not allowed: continue row = old_row if side == "old" else new_row if side == "new" else None - val = scalar_text((row or {}).get(field)).strip().lower() + val = str((row or {}).get(field) or "").strip().lower() if val not in allowed: return False return True @@ -756,10 +749,10 @@ def build_diff_index_from_compare(result: dict[str, Any]) -> dict[str, dict[str, continue key = d.get("key") if isinstance(key, dict): - key_list = [scalar_text(v).strip() for v in key.values()] + key_list = [str(v) for v in key.values()] ks = _key_str(key_list) elif isinstance(key, (list, tuple)): - key_list = [scalar_text(x).strip() for x in key] + key_list = [str(x) for x in key] ks = _key_str(key_list) else: ks = str(key or "") @@ -801,18 +794,17 @@ def expect_keys_for_metric( return keys -def _map_defines_iface_expect( - iface_fields: list[str], port_map: dict[str, str], scope_by_port_map: bool | None = None, -) -> bool: - """Whether a non-empty port map is this sheet's expect scope. +def _map_defines_iface_expect(metric_id: str, iface_fields: list[str], port_map: dict[str, str]) -> bool: + """Iface sheets: a non-empty port map is the expect scope. - The sheet override decides (``scope_by_port_map``); the built-in presets set it to - false for BGP / pseudowire sheets, which keep the batch expect picker. Unset means - "an interface sheet with a map is scoped by the map". + BGP / PW keep the batch expect picker even if a column is marked iface. """ if not port_map or not iface_fields: return False - return True if scope_by_port_map is None else bool(scope_by_port_map) + mid = str(metric_id or "").strip().lower() + if "bgp" in mid or "pw" in mid or "pseudowire" in mid: + return False + return True def _iface_values(key_str: str, *, key_fields: list[str], iface_fields: list[str]) -> list[str]: @@ -866,14 +858,6 @@ def _port_identity(key_fields: list[str], iface_fields: list[str]) -> bool: return all(str(f) in iface_set for f in key_fields) -def _iface_selected(value: str, ports: set[str]) -> bool: - """Set lookup plus parent interface prefixes; independent of scope size.""" - if value in ports: - return True - parts = value.split(".") - return any(".".join(parts[:i]) in ports for i in range(len(parts) - 1, 0, -1)) - - def _unmapped_same_iface_anomaly( *, old_kind: str, @@ -908,8 +892,6 @@ def _remap_key_str( return key_str if not port_map: return key_str - if key_fields and not set(key_fields).intersection(iface_fields): - return key_str if (not key_fields or len(key_fields) == 1) and ( key_str in port_map or any( @@ -946,8 +928,6 @@ def _reverse_remap_key_str( return key_str if not rev_map: return key_str - if key_fields and not set(key_fields).intersection(iface_fields): - return key_str if (not key_fields or len(key_fields) == 1) and ( key_str in rev_map or any( @@ -1023,16 +1003,13 @@ def row_match_key( key_fields: list[str], iface_fields: list[str], rules: list[dict[str, str]] | None, - normalizer: Any = None, ) -> str: from ..biz_state.iface_normalize import apply_iface_normalize data = strip_netx(row) - if normalizer is not None: - data = normalizer(dict(data)) - elif iface_fields and rules: + if iface_fields and rules: data = apply_iface_normalize(dict(data), iface_fields=iface_fields, rules=rules) - return KEY_SEP.join(scalar_text(data.get(k)).strip() for k in key_fields) + return KEY_SEP.join(str(data.get(k) or "").strip() for k in key_fields) def index_raw_by_match_key( @@ -1041,14 +1018,11 @@ def index_raw_by_match_key( key_fields: list[str], iface_fields: list[str], rules: list[dict[str, str]] | None, - normalizer: Any = None, ) -> dict[str, dict[str, Any]]: """First raw row per internal match key (normalized iface, raw elsewhere).""" idx: dict[str, dict[str, Any]] = {} for raw in raw_rows: - ks = row_match_key( - raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules, normalizer=normalizer, - ) + ks = row_match_key(raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules) if ks and ks not in idx: idx[ks] = raw return idx @@ -1059,7 +1033,7 @@ def display_key_from_raw(raw: dict[str, Any] | None, key_fields: list[str]) -> s if not raw or not key_fields: return "" data = strip_netx(raw) - return "|".join(scalar_text(data.get(k)).strip() for k in key_fields) + return "|".join(str(data.get(k) or "").strip() for k in key_fields) def iface_lineage( @@ -1069,7 +1043,6 @@ def iface_lineage( rules: list[dict[str, str]] | None, port_map: dict[str, str] | None, apply_map: bool, - namer: Any = None, ) -> list[dict[str, Any]]: """Per iface field: raw → normalized → (optional) mapped.""" from ..biz_state.iface_normalize import normalize_iface_name, resolve_mapped_iface @@ -1080,8 +1053,8 @@ def iface_lineage( pmap = port_map or {} out: list[dict[str, Any]] = [] for f in iface_fields: - raw_v = scalar_text(data.get(f)).strip() - norm_v = namer(raw_v) if namer is not None else (normalize_iface_name(raw_v, rules) if rules else raw_v) + raw_v = str(data.get(f) or "").strip() + norm_v = normalize_iface_name(raw_v, rules) if rules else raw_v mapped = False map_to = "" if apply_map and pmap and norm_v: @@ -1119,9 +1092,6 @@ def evaluate_metric_dual( out_of_expect: str = "strict", sheet_id: str = "", iface_normalize_rules: list[dict[str, str]] | None = None, - previous_expect: dict[str, set[str]] | None = None, - allow_mapping_scope: bool = True, - pipeline: Any = None, ) -> dict[str, Any]: """Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge. @@ -1129,21 +1099,15 @@ def evaluate_metric_dual( Board / AI fields ``old_key`` / ``new_key`` and ``old`` / ``new`` stay **raw** device values (A/B), never the post-map BB form. """ - from ..biz_state.identity_pipeline import legacy_pipeline - - # Old device = "before" side, new device = "after" side of the identity pipeline. - pipe = pipeline or legacy_pipeline( - key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields, - field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules, + from ..biz_state.iface_normalize import ( + apply_iface_normalize_rows, + normalize_iface_rules, ) sid = str(sheet_id or "").strip() or str(metric_id or "").strip() expect_keys = expect_keys_for_metric( expect, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid ) - previous_keys = expect_keys_for_metric( - previous_expect or {}, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid - ) ov = sheet_override or {} if ov.get("skip_dual"): return { @@ -1170,44 +1134,39 @@ def evaluate_metric_dual( [dict(r) for r in new_baseline_rows] if new_baseline_rows is not None else None ) - def normalized_key(key: str) -> str: - parts = key.split(KEY_SEP) if KEY_SEP in key else ([key] if len(key_fields) == 1 else key.split("|")) - if len(parts) != len(key_fields): - return key - return KEY_SEP.join(pipe.normalize_name(p, "before") if f in iface_fields else p - for p, f in zip(parts, key_fields)) - - def norm_before(row: dict[str, Any]) -> dict[str, Any]: - return pipe.normalize(row, "before") - - def norm_after(row: dict[str, Any]) -> dict[str, Any]: - return pipe.normalize(row, "after") - - expect_keys = {normalized_key(key) for key in expect_keys} - previous_keys = {normalized_key(key) for key in previous_keys} - old_baseline_rows = [norm_before(strip_netx(r)) for r in raw_old_base] - old_current_rows = [norm_before(strip_netx(r)) for r in raw_old_cur] - new_current_rows = [norm_after(strip_netx(r)) for r in raw_new_cur] + norm_rules = normalize_iface_rules(iface_normalize_rules) + old_baseline_rows = apply_iface_normalize_rows( + [strip_netx(r) for r in raw_old_base], iface_fields=iface_fields, rules=norm_rules + ) + old_current_rows = apply_iface_normalize_rows( + [strip_netx(r) for r in raw_old_cur], iface_fields=iface_fields, rules=norm_rules + ) + new_current_rows = apply_iface_normalize_rows( + [strip_netx(r) for r in raw_new_cur], iface_fields=iface_fields, rules=norm_rules + ) if raw_new_base is not None: - new_baseline_rows = [norm_after(strip_netx(r)) for r in raw_new_base] + new_baseline_rows = apply_iface_normalize_rows( + [strip_netx(r) for r in raw_new_base], + iface_fields=iface_fields, + rules=norm_rules, + ) else: new_baseline_rows = None old_raw_cur_idx = index_raw_by_match_key( - raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before + raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules ) old_raw_base_idx = index_raw_by_match_key( - raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before + raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules ) new_raw_cur_idx = index_raw_by_match_key( - raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_after + raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules ) new_raw_base_idx = index_raw_by_match_key( raw_new_base or [], key_fields=key_fields, iface_fields=iface_fields, - rules=None, - normalizer=norm_after, + rules=norm_rules, ) old_cmp = compare_rows( @@ -1219,8 +1178,6 @@ def evaluate_metric_dual( port_map=None, field_rules=field_rules, iface_normalize_rules=None, # already applied - include_unchanged=True, - duplicate_pairing=pipe.duplicate_pairing, ) old_idx = build_diff_index_from_compare(old_cmp) @@ -1229,8 +1186,18 @@ def evaluate_metric_dual( if new_baseline_rows is not None: new_before = list(new_baseline_rows) new_baseline_mode = "provided" - # A successfully collected empty baseline is valid on a replacement NE. - # Collection completeness is checked by the service, not inferred from rows. + # Explicit empty batch is as unusable as "no baseline" for presence semantics. + if not new_before and (old_baseline_rows or new_current_rows): + if port_map and iface_fields: + new_before = [ + apply_port_map(r, iface_fields=iface_fields, port_map=port_map) + for r in old_baseline_rows + ] + new_baseline_mode = "port_mapped" + new_baseline_missing = False + else: + new_baseline_missing = True + new_baseline_mode = "empty" elif port_map and iface_fields: new_before = [ apply_port_map(r, iface_fields=iface_fields, port_map=port_map) @@ -1251,23 +1218,12 @@ def evaluate_metric_dual( compare_fields=compare_fields, port_map=None, field_rules=field_rules, - include_unchanged=True, - duplicate_pairing=pipe.duplicate_pairing, ) new_idx = build_diff_index_from_compare(new_cmp) rev_map = {v: k for k, v in port_map.items()} - map_scoped = _map_defines_iface_expect(iface_fields, port_map, ov.get("scope_by_port_map")) - # Mapping describes identity; an explicit batch selection defines scope. - # Port selection also covers business rows attached to those interfaces. - for scope, keys in ((expect, expect_keys), (previous_expect or {}, previous_keys)): - ports = {pipe.normalize_name(p, "before") for p in scope.get("_ports") or set()} - if ports and not keys and iface_fields: - for ks, diff in old_idx.items(): - row = diff.get("before") or diff.get("after") or {} - if any(_iface_selected(scalar_text(row.get(f)), ports) for f in iface_fields): - keys.add(ks) - if allow_mapping_scope and map_scoped and not any(expect.values()): + map_scoped = _map_defines_iface_expect(metric_id, iface_fields, port_map) + if map_scoped: scoped: set[str] = set() for ks in old_idx: if _key_in_port_map( @@ -1287,21 +1243,6 @@ def evaluate_metric_dual( scoped.add(canon) expect_keys = scoped - # Duplicate keys only block when they sit inside this wave's scope. - from collections import Counter - - def _dup_keys(rows: list[dict[str, Any]]) -> set[str]: - return {KEY_SEP.join(k) for k, n in Counter(key_tuple(r, key_fields) for r in rows).items() if n > 1} - - in_scope = expect_keys | previous_keys - new_dups = _dup_keys(new_current_rows) | _dup_keys(new_baseline_rows or []) - flagged = ( - _dup_keys(old_baseline_rows) | _dup_keys(old_current_rows) | new_dups - | {_reverse_remap_key_str(k, key_fields=key_fields, iface_fields=iface_fields, rev_map=rev_map) - for k in new_dups} - ) - duplicate_in_expect = sorted(flagged & in_scope)[:64] - canon_keys: list[str] = [] seen: set[str] = set() @@ -1312,8 +1253,6 @@ def evaluate_metric_dual( for ok in sorted(expect_keys): _add(ok) - for ok in sorted(previous_keys): - _add(ok) for ks in sorted(old_idx): _add(ks) for ks in sorted(new_idx): @@ -1331,11 +1270,6 @@ def evaluate_metric_dual( progress_total = 0 anomaly = 0 anomaly_in_expect = 0 - steady_outside = 0 - rules_by_field = field_rule_map(field_rules) - integrity_fields = effective_compare_fields(compare_fields, field_rules) - # Status transitions are interpreted by the monitor's correction rules. - integrity_fields = [f for f in integrity_fields if f not in (ov.get("status_fields") or [])] for old_ks in canon_keys: new_ks = _remap_key_str( @@ -1345,12 +1279,11 @@ def evaluate_metric_dual( port_map=port_map, ) in_exp = old_ks in expect_keys - previous = old_ks in previous_keys and not in_exp if in_exp: progress_total += 1 od = old_idx.get(old_ks) - nd = new_idx.get(new_ks) + nd = new_idx.get(new_ks) or new_idx.get(old_ks) old_kind = str((od or {}).get("kind") or "") new_kind = str((nd or {}).get("kind") or "") @@ -1368,7 +1301,11 @@ def evaluate_metric_dual( old_st = classify_status(old_cur, ov) new_st = classify_status(new_cur, ov) - if map_scoped and not (in_exp or previous) and ( + # Mapped iface rows are the expect set. Unmapped rows stay out of the + # migration compare, except same-name ports where old went down and the + # new device's same port is up — that is anomaly, not a successful cutover. + if map_scoped and not in_exp: + if not ( _port_identity(key_fields, iface_fields) and _unmapped_same_iface_anomaly( old_kind=old_kind, @@ -1377,63 +1314,26 @@ def evaluate_metric_dual( new_status=new_st, ) ): + continue verdict, color, rule_hit = "anomaly", "red", "unmapped_same_iface" else: verdict, color, rule_hit = dual_verdict_ex( old_kind=old_kind or "", new_kind=new_kind or "", - in_expect=in_exp or previous, + in_expect=in_exp, window_active=window_active, acceptance=acceptance, old_status=old_st, new_status=new_st, success_patterns=success_patterns or None, anomaly_patterns=anomaly_patterns or None, - # Business drift outside this batch must remain visible. - out_of_expect="strict", + out_of_expect=out_of_expect, old_row=old_cur or None, new_row=new_cur or None, old_base=old_base or None, new_base=new_base or None, sheet_override=ov, ) - integrity_changes: list[dict[str, Any]] = [] - if in_exp or previous: - old_departed = old_kind == "removed" or old_st == "down" - new_healthy = bool(new_cur) and ( - new_st == "up" if ov.get("status_fields") else True - ) - if old_departed and not new_healthy: - verdict, color, rule_hit = "lost", "red", "service_not_received" - elif verdict == "migrated": - if not new_healthy: - verdict, color, rule_hit = "anomaly", "red", "target_unhealthy" - elif not old_base: - verdict, color, rule_hit = "anomaly", "red", "old_service_not_in_baseline" - else: - reference = apply_port_map(old_base, iface_fields=iface_fields, port_map=port_map) - integrity_changes = [ - {"field": f, "before": reference.get(f), "after": new_cur.get(f)} - for f in integrity_fields - if not values_equal(reference.get(f), new_cur.get(f), rule=rules_by_field.get(f)) - ] - if integrity_changes: - verdict, color, rule_hit = "anomaly", "red", "service_content_changed" - elif not old_departed and not success_patterns: - verdict, color, rule_hit = ( - ("unfinished", "red", "old_service_still_present") if acceptance - else ("migrating", "yellow", "old_service_still_present") - ) - if previous: - if verdict == "migrated": - verdict, rule_hit = "migrated_previous", "previous_batch_received" - elif color != "red": - verdict, color, rule_hit = "unfinished_previous", "red", "previous_batch_not_received" - # Stable inventory is covered by the comparison and counted, but does - # not need a full persisted evidence card on every high-frequency run. - if not in_exp and verdict == "not_involved": - steady_outside += 1 - continue if in_exp and verdict == "migrated" and color == "green": progress_ok += 1 if color == "red": @@ -1444,9 +1344,9 @@ def evaluate_metric_dual( # Prefer live raw rows; fall back to baseline raw. Never use port-mapped # synthetic "before" as the new-side display row. raw_old = old_raw_cur_idx.get(old_ks) or old_raw_base_idx.get(old_ks) - raw_new = new_raw_cur_idx.get(new_ks) + raw_new = new_raw_cur_idx.get(new_ks) or new_raw_cur_idx.get(old_ks) if not raw_new: - raw_new = new_raw_base_idx.get(new_ks) + raw_new = new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks) display_old = display_key_from_raw(raw_old, key_fields) or old_ks display_new = display_key_from_raw(raw_new, key_fields) or ( @@ -1460,8 +1360,8 @@ def evaluate_metric_dual( new_disp = strip_netx(raw_new) if raw_new else {} old_base_raw = strip_netx(old_raw_base_idx.get(old_ks)) if old_raw_base_idx.get(old_ks) else {} new_base_raw = ( - strip_netx(new_raw_base_idx.get(new_ks)) - if new_raw_base_idx.get(new_ks) + strip_netx(new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks)) + if (new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks)) else {} ) @@ -1471,10 +1371,9 @@ def evaluate_metric_dual( "iface": iface_lineage( raw_old, iface_fields=iface_fields, - rules=None, + rules=norm_rules, port_map=port_map, apply_map=True, - namer=lambda v: pipe.normalize_name(v, "before"), ), }, "new": { @@ -1482,10 +1381,9 @@ def evaluate_metric_dual( "iface": iface_lineage( raw_new, iface_fields=iface_fields, - rules=None, + rules=norm_rules, port_map=None, apply_map=False, - namer=lambda v: pipe.normalize_name(v, "after"), ), }, "port_map": { @@ -1513,9 +1411,7 @@ def evaluate_metric_dual( "verdict": verdict, "color": color, "rule_hit": rule_hit, - "integrity_changes": integrity_changes, "in_expect": in_exp, - "previous_batch": previous, "old_kind": old_kind, "new_kind": new_kind, "old": old_disp, @@ -1548,14 +1444,12 @@ def evaluate_metric_dual( return { "metric_id": metric_id, - "effective_scope_by_port_map": map_scoped, "old_summary": old_cmp.get("summary") or {}, "new_summary": new_cmp.get("summary") or {}, "progress_ok": progress_ok, "progress_total": progress_total, "anomaly": anomaly, "anomaly_in_expect": anomaly_in_expect, - "steady_outside": steady_outside, "rows": rows_out, "new_baseline_mode": new_baseline_mode, "new_baseline_missing": new_baseline_missing, @@ -1563,7 +1457,6 @@ def evaluate_metric_dual( + int((new_cmp.get("summary") or {}).get("duplicate_keys_before") or 0), "duplicate_keys_after": int((old_cmp.get("summary") or {}).get("duplicate_keys_after") or 0) + int((new_cmp.get("summary") or {}).get("duplicate_keys_after") or 0), - "duplicate_in_expect": duplicate_in_expect, "duplicate_key_list": list( dict.fromkeys( list((old_cmp.get("summary") or {}).get("duplicate_key_list") or []) diff --git a/netx_api/biz_migration/monitor_templates.py b/netx_api/biz_migration/monitor_templates.py index bd72ee0..b55f487 100644 --- a/netx_api/biz_migration/monitor_templates.py +++ b/netx_api/biz_migration/monitor_templates.py @@ -8,8 +8,7 @@ from uuid import uuid4 from fastapi import HTTPException from sqlalchemy.orm import Session -from ..models import BizCompareTemplate, BizMigrationProject, BizMonitorTemplate -from .validation import validate_monitor_rules +from ..models import BizCompareTemplate, BizMonitorTemplate from ..timeutil import utcnow_naive from ..biz_state import compare_service as cmp_svc from ..biz_state.iface_normalize import default_zte_iface_normalize_rules @@ -34,8 +33,6 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None effective = seen if not effective: effective = [PORT_METRIC_ID] - ct = db.get(BizCompareTemplate, row.compare_template_id) if db and row.compare_template_id else None - available = list(dict.fromkeys(str(s.get("metric_id") or "") for s in cmp_svc.template_metrics(ct))) if ct else effective out: dict[str, Any] = { "id": row.id, "name": row.name, @@ -43,7 +40,6 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None "compare_template_name": compare_name, "collect_metric_ids": collect, "collect_metric_ids_effective": effective, - "available_metric_ids": [m for m in available if m], "defaults": dict(row.defaults_json or {}), "sheet_overrides": list(row.sheet_overrides_json or []), "note": row.note or "", @@ -61,18 +57,6 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None return out -def _has_legacy_port_and_group(override: dict[str, Any]) -> bool: - """True when a success rule's old side demands admin down AND phy down.""" - legacy = _port_old_down_groups(legacy_and=True)[0] - for rule in override.get("success") or []: - if not isinstance(rule, dict): - continue - for group in rule.get("old_groups") or []: - if group == legacy: - return True - return False - - def validate_sheet_overrides( db: Session, *, @@ -106,9 +90,6 @@ def validate_sheet_overrides( for ov in overrides: sid = str(ov.get("sheet_id") or "").strip() mid = str(ov.get("metric_id") or "").strip() - if _has_legacy_port_and_group(ov): - warnings.append({"sheet_id": sid, "metric_id": mid, - "reason": "port_success_legacy_and_semantics"}) if sid: if sid not in sheet_ids: warnings.append( @@ -243,29 +224,9 @@ def _success_stateful( } -_PORT_ADMIN_DOWN = {"type": "value", "field": "admin", "op": "in", "value": ["down"]} -_PORT_PHY_DOWN = {"type": "value", "field": "phy", "op": "in", "value": ["down"]} - - -def _port_old_down_groups(*, legacy_and: bool = False) -> list[list[dict[str, Any]]]: - """Old-side "port is out of service" groups (OR of groups, AND within one). - - A fibre pulled without ``shutdown`` leaves admin up / phy down, so either - signal alone counts. ``legacy_and`` is the pre-A4 form requiring both. - """ - if legacy_and: - return [[dict(_PORT_ADMIN_DOWN), dict(_PORT_PHY_DOWN)]] - return [[dict(_PORT_PHY_DOWN)], [dict(_PORT_ADMIN_DOWN)]] - - -def preset_override_for_metric( - metric_id: str, *, sheet_id: str = "", legacy_port_and: bool = False, -) -> dict[str, Any]: +def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[str, Any]: """Default dual-verdict overlay for a compare-template sheet. - ``legacy_port_and`` rebuilds the pre-A4 port preset (old side must be - admin down AND phy down) so stored copies can be recognised and upgraded. - ``metric_id`` is the collected source table; ``sheet_id`` identifies a filtered split (e.g. bgp_peer.vpnv4). Presets key off the source metric. """ @@ -279,8 +240,6 @@ def preset_override_for_metric( look = mid.split(".", 1)[0] if "." in mid else mid def _out(body: dict[str, Any]) -> dict[str, Any]: - if _scope_by_map_default_off(look): - body["scope_by_port_map"] = False body["metric_id"] = mid or look if sid and sid != (mid or look): body["sheet_id"] = sid @@ -298,7 +257,12 @@ def preset_override_for_metric( "up_values": ["up"], "success": [ _success_stateful( - old_down_groups=_port_old_down_groups(legacy_and=legacy_port_and), + old_down_groups=[ + [ + {"type": "value", "field": "admin", "op": "in", "value": ["down"]}, + {"type": "value", "field": "phy", "op": "in", "value": ["down"]}, + ] + ], new_up_conds=up, ) ], @@ -401,8 +365,6 @@ DEFAULT_MONITOR_TEMPLATE_NAME = "默认割接监控" def ensure_default_monitor_templates(db: Session) -> None: """Seed once when monitor-template table is empty — one template on the full status compare.""" if db.query(BizMonitorTemplate.id).limit(1).first(): - upgrade_legacy_port_overrides(db) - fill_scope_by_port_map(db) return cmp_svc.ensure_default_templates(db) zte = ( @@ -437,68 +399,9 @@ def ensure_default_monitor_templates(db: Session) -> None: db.commit() -def _scope_by_map_default_off(name: str) -> bool: - """BGP / pseudowire sheets keep the batch expect picker; a port map does not scope them.""" - low = str(name or "").lower() - return "bgp" in low or "pw" in low or "pseudowire" in low - - -def fill_scope_by_port_map(db: Session) -> int: - """Make the old name-based default explicit on stored overrides (one-off, idempotent).""" - changed = 0 - for row in db.query(BizMonitorTemplate).all(): - out = [] - touched = False - for ov in list(row.sheet_overrides_json or []): - if (isinstance(ov, dict) and "scope_by_port_map" not in ov - and _scope_by_map_default_off(f"{ov.get('metric_id') or ''} {ov.get('sheet_id') or ''}")): - ov = {**ov, "scope_by_port_map": False} - touched = True - out.append(ov) - if touched: - row.sheet_overrides_json = out - row.updated_at = _utcnow() - changed += 1 - if changed: - db.commit() - return changed - - -def upgrade_legacy_port_overrides(db: Session) -> int: - """Rewrite stored port overrides that are exactly the pre-A4 preset (AND semantics). - - Field-by-field equality only: anything a user customised is left alone and - surfaces ``port_success_legacy_and_semantics`` in ``override_warnings``. - """ - changed = 0 - for row in db.query(BizMonitorTemplate).all(): - overrides = list(row.sheet_overrides_json or []) - out: list[Any] = [] - touched = False - for ov in overrides: - if isinstance(ov, dict): - mid = str(ov.get("metric_id") or "").strip() - sid = str(ov.get("sheet_id") or "").strip() - if (mid or sid).split(".", 1)[0] == PORT_METRIC_ID and ov == preset_override_for_metric( - mid, sheet_id=sid, legacy_port_and=True, - ): - out.append(preset_override_for_metric(mid, sheet_id=sid)) - touched = True - continue - out.append(ov) - if touched: - row.sheet_overrides_json = out - row.updated_at = _utcnow() - changed += 1 - if changed: - db.commit() - return changed - - -def default_port_monitor_template_id(db: Session, *, seed: bool = True) -> str: - """Return the default monitor template id, seeding the built-ins first unless ``seed=False``.""" - if seed: - ensure_default_monitor_templates(db) +def default_port_monitor_template_id(db: Session) -> str: + """Ensure seeds exist and return the default monitor template id.""" + ensure_default_monitor_templates(db) for name in ( DEFAULT_MONITOR_TEMPLATE_NAME, "ZTE 状态割接监控", @@ -553,7 +456,6 @@ def create_monitor_template(db: Session, body: dict[str, Any]) -> dict[str, Any] overrides = body.get("sheet_overrides") if not isinstance(overrides, list): overrides = [] - validate_monitor_rules(defaults, overrides) row = BizMonitorTemplate( id=uuid4().hex, name=name[:256], @@ -577,10 +479,6 @@ def update_monitor_template(db: Session, template_id: str, body: dict[str, Any]) row = db.get(BizMonitorTemplate, template_id) if not row: raise HTTPException(status_code=404, detail="monitor_template_not_found") - validate_monitor_rules(body.get("defaults", row.defaults_json or {}), - body.get("sheet_overrides", row.sheet_overrides_json or [])) - if body.get("compare_template_id") and not db.get(BizCompareTemplate, body["compare_template_id"]): - raise HTTPException(status_code=404, detail="compare_template_not_found") if "name" in body and body["name"] is not None: name = str(body["name"] or "").strip() if not name: @@ -617,7 +515,5 @@ def delete_monitor_template(db: Session, template_id: str) -> None: row = db.get(BizMonitorTemplate, template_id) if not row: raise HTTPException(status_code=404, detail="monitor_template_not_found") - if db.query(BizMigrationProject.id).filter(BizMigrationProject.monitor_template_id == template_id).first(): - raise HTTPException(status_code=409, detail="monitor_template_in_use") db.delete(row) db.commit() diff --git a/netx_api/biz_migration/service.py b/netx_api/biz_migration/service.py index fc965c8..760a6c0 100644 --- a/netx_api/biz_migration/service.py +++ b/netx_api/biz_migration/service.py @@ -2,24 +2,18 @@ from __future__ import annotations -import hashlib -import logging -import threading -from collections import OrderedDict -from collections import Counter from datetime import datetime, timezone from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import and_, insert, or_ from sqlalchemy.orm import Session -from ..biz_state.compare_rules import apply_row_filters, scalar_text -from ..config import settings +from ..biz_state.compare_rules import apply_row_filters from ..biz_state.compare_service import ( _load_metric_rows, _port_map_dict, + batch_metric_collect_ok, sheet_key, sheet_title, template_metrics, @@ -34,7 +28,6 @@ from ..models import ( BizMonitorTemplate, BizPortMapping, BizStateBatch, - BizStateBatchCommand, BizStateTask, ) from ..timeutil import utcnow_naive @@ -48,45 +41,6 @@ from .evaluate import ( ) PURPOSE_CUTOVER_HF = "cutover_hf" -_log = logging.getLogger("netx.biz_migration") - - -class BaselineRowCache: - """Process-local LRU of immutable baseline rows, bounded by total row count.""" - - def __init__(self, max_rows: int = 500_000) -> None: - self.max_rows = max_rows - self._items: "OrderedDict[tuple[str, str], list[dict[str, Any]]]" = OrderedDict() - self._rows = 0 - self._lock = threading.Lock() - - def get_or_load(self, batch_id: str, metric_id: str, loader: Any) -> list[dict[str, Any]]: - key = (batch_id, metric_id) - with self._lock: - hit = self._items.get(key) - if hit is not None: - self._items.move_to_end(key) - return hit - rows = loader() - if self.max_rows > 0 and len(rows) <= self.max_rows: - with self._lock: - if key not in self._items: - self._items[key] = rows - self._rows += len(rows) - while self._rows > self.max_rows and self._items: - _, evicted = self._items.popitem(last=False) - self._rows -= len(evicted) - return rows - - def invalidate(self, batch_id: str = "") -> None: - with self._lock: - for key in [k for k in self._items if not batch_id or k[0] == batch_id]: - self._rows -= len(self._items.pop(key)) - - -BASELINE_ROWS = BaselineRowCache(int(getattr(settings, "biz_migration_baseline_cache_rows", 500_000))) - -MANUAL_ROUTE_METRICS = frozenset({"ip_route", "ipv6_route", "bgp_route"}) def _parse_dt(raw: Any) -> datetime | None: @@ -195,7 +149,7 @@ def hf_task_id(proj: BizMigrationProject, side: str) -> str: def current_task_id(proj: BizMigrationProject, side: str) -> str: """Evaluate current batches: prefer HF, fall back to portrait.""" - return hf_task_id(proj, side) + return hf_task_id(proj, side) or portrait_task_id(proj, side) def _metric_interval_map(proj: BizMigrationProject) -> dict[str, int]: @@ -262,9 +216,7 @@ def _hf_bindings(proj: BizMigrationProject, side: str) -> list[dict[str, Any]]: iv = max(60, int(row.get("interval_sec") or 60)) except (TypeError, ValueError): iv = 60 - out.append({"task_id": tid, "metric_ids": mids, "interval_sec": iv, - "collect_group_id": str(row.get("collect_group_id") or ""), - "manual_only": bool(row.get("manual_only"))}) + out.append({"task_id": tid, "metric_ids": mids, "interval_sec": iv}) if out: return out # Legacy single slot @@ -316,124 +268,15 @@ def _current_batch_for_metric( # Only honor pin when it belongs to this metric's HF task (multi-interval safe) if pinned and (not tid or str(pinned.task_id or "") == tid): # Pinned partial / failed metric → treat as missing (avoid false red) - if not _metric_sample_ok(db, pinned, metric_id): + if not batch_metric_collect_ok(db, pinned.id, metric_id): return None return pinned - batch = ( - db.query(BizStateBatch) - .filter(BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued"))) - .order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() - ) if tid else None - if batch and not _metric_sample_ok(db, batch, metric_id): + batch = _latest_success_batch(db, tid) if tid else None + if batch and not batch_metric_collect_ok(db, batch.id, metric_id): return None return batch -def _metric_sample_ok(db: Session, batch: BizStateBatch, metric_id: str) -> bool: - """An empty result requires a successful command receipt, just like a full one. - - Batch success alone cannot prove this metric was collected. Unmatched and - skipped parsers are uncertainty, never evidence that a business disappeared. - """ - if batch.status not in ("success", "partial"): - return False - statuses = [str(s or "").lower() for (s,) in db.query(BizStateBatchCommand.parse_status) - .filter(BizStateBatchCommand.batch_id == batch.id, - BizStateBatchCommand.metric_id == metric_id).all()] - return bool(statuses) and all(s in ("ok", "ok_aux") for s in statuses) - - -def _uncovered_slice_sides( - db: Session, sheet: dict[str, Any], metric_id: str, old_batch_id: str, new_batch_id: str, -) -> list[str]: - from ..biz_state.compare_quality import covers_slice, side_receipts, sheet_slice - - wanted = sheet_slice(sheet) - if not wanted: - return [] - out = [] - for side, bid in (("old", old_batch_id), ("new", new_batch_id)): - receipts = side_receipts(db, bid, metric_id) - if receipts and not covers_slice(receipts, wanted): - out.append(side) - return out - - -def _current_pair_for_metric(db: Session, proj: BizMigrationProject, mid: str, - pin_old: str = "", pin_new: str = "") -> tuple[BizStateBatch | None, BizStateBatch | None, dict[str, Any]]: - old_tid, new_tid = (_hf_task_id_for_metric(proj, side, mid) for side in ("old", "new")) - old_task = db.get(BizStateTask, old_tid) if old_tid else None - new_task = db.get(BizStateTask, new_tid) if new_tid else None - gid = old_task.collect_group_id if old_task else "" - meta: dict[str, Any] = {"collect_group_id": gid, "manual_only": bool(old_task and old_task.collect_manual_only)} - if gid and old_tid and old_tid == new_tid and not (pin_old or pin_new): - # Same device on both sides: one task, one batch, trivially the same round. - latest = db.query(BizStateBatch).filter( - BizStateBatch.collect_group_id == gid, BizStateBatch.task_id == old_tid).order_by( - BizStateBatch.queued_at.desc(), BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() - if latest is None: - return None, None, {**meta, "sample_issues": ["collect_pair_missing"]} - meta.update({"collect_round_id": latest.collect_round_id, "old_collect_status": latest.status, - "new_collect_status": latest.status}) - if latest.status in ("running", "queued"): - return None, None, {**meta, "sample_issues": ["collect_pair_pending"]} - if not _metric_sample_ok(db, latest, mid): - return None, None, {**meta, "sample_issues": ["collect_pair_failed"]} - return latest, latest, meta - if gid and not (pin_old or pin_new): - if not new_task or new_task.collect_group_id != gid: - return None, None, {**meta, "sample_issues": ["collect_pair_invalid"]} - latest = db.query(BizStateBatch).filter(BizStateBatch.collect_group_id == gid, - BizStateBatch.task_id.in_((old_tid, new_tid))).order_by( - BizStateBatch.queued_at.desc(), BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() - peers = db.query(BizStateBatch).filter(BizStateBatch.collect_round_id == latest.collect_round_id, - BizStateBatch.task_id.in_((old_tid, new_tid))).all() if latest and latest.collect_round_id else [] - meta.update({"collect_round_id": latest.collect_round_id if latest else "", - "old_collect_status": next((b.status for b in peers if b.task_id == old_tid), "missing"), - "new_collect_status": next((b.status for b in peers if b.task_id == new_tid), "missing")}) - if len(peers) != 2: - return None, None, {**meta, "sample_issues": ["collect_pair_missing"]} - if any(b.status in ("running", "queued") for b in peers): - return None, None, {**meta, "sample_issues": ["collect_pair_pending"]} - by_task = {b.task_id: b for b in peers} - if not all(_metric_sample_ok(db, b, mid) for b in peers): - return None, None, {**meta, "sample_issues": ["collect_pair_failed"]} - return by_task[old_tid], by_task[new_tid], meta - old = _current_batch_for_metric(db, proj, "old", mid, pinned_batch_id=pin_old) - new = _current_batch_for_metric(db, proj, "new", mid, pinned_batch_id=pin_new) - if old and new and (old.collect_round_id or new.collect_round_id) and old.collect_round_id != new.collect_round_id: - return None, None, {**meta, "sample_issues": ["collect_round_mismatch"]} - return old, new, meta - - -def _sample_timing(proj: BizMigrationProject, mid: str, old: BizStateBatch, - new: BizStateBatch, *, check_age: bool) -> dict[str, Any]: - interval = max(60, _metric_interval_map(proj).get(mid) or proj.hf_interval_sec or 60) - old_at, new_at = old.ended_at or old.started_at, new.ended_at or new.started_at - durations = [max(0, ((b.ended_at or b.started_at) - b.started_at).total_seconds()) for b in (old, new)] - age_limit, skew_limit = interval * 3 + max(durations), interval * 2 - reasons: list[str] = [] - if not old_at or not new_at: - reasons.append("sample_time_missing") - skew = None - else: - skew = abs((old_at - new_at).total_seconds()) - if skew > skew_limit: - reasons.append("sample_skew") - if check_age and max((utcnow_naive() - old_at).total_seconds(), - (utcnow_naive() - new_at).total_seconds()) > age_limit: - reasons.append("sample_stale") - start_skew = abs((old.started_at - new.started_at).total_seconds()) - if old.collect_round_id and start_skew > min(30, interval / 2): - reasons.append("sample_start_skew") - return {"old_collected_at": _dt_iso(old_at), "new_collected_at": _dt_iso(new_at), - "age_limit_sec": age_limit, "skew_limit_sec": skew_limit, - "skew_sec": skew, "start_skew_sec": start_skew, "sample_issues": reasons, - "old_duration_sec": durations[0], "new_duration_sec": durations[1], - "old_queue_sec": max(0, (old.started_at - (old.queued_at or old.started_at)).total_seconds()), - "new_queue_sec": max(0, (new.started_at - (new.queued_at or new.started_at)).total_seconds())} - - def find_portrait_task_for_ne(db: Session, *, source: str, ne_id: str) -> BizStateTask | None: """Best portrait (non-HF) biz_state task for an NE — for baseline binding.""" src = str(source or "managed").strip().lower() or "managed" @@ -555,26 +398,19 @@ def _monitor_template_brief(db: Session, template_id: str) -> dict[str, Any]: return mon_tpl._out(row, cmp_name, db=db) # noqa: SLF001 -def resolve_project_monitor_template( - db: Session, proj: BizMigrationProject, *, persist: bool = True, -) -> BizMonitorTemplate: - """Return monitor template for project; seed default port template if unbound. - - ``persist=False`` is for read-only callers (GET): it never seeds, upgrades or binds, - it only reports which template the project would use. - """ - if persist: - mon_tpl.ensure_default_monitor_templates(db) +def resolve_project_monitor_template(db: Session, proj: BizMigrationProject) -> BizMonitorTemplate: + """Return monitor template for project; seed default port template if unbound.""" + mon_tpl.ensure_default_monitor_templates(db) tid = str(getattr(proj, "monitor_template_id", None) or "").strip() row = db.get(BizMonitorTemplate, tid) if tid else None if row: return row - default_id = mon_tpl.default_port_monitor_template_id(db, seed=persist) + default_id = mon_tpl.default_port_monitor_template_id(db) row = db.get(BizMonitorTemplate, default_id) if default_id else None if not row: raise HTTPException(status_code=400, detail="monitor_template_required") # Persist default on first use so UI shows binding - if not tid and persist: + if not tid: proj.monitor_template_id = row.id proj.updated_at = utcnow_naive() db.commit() @@ -601,12 +437,12 @@ def resolve_evaluate_sheets( return [port_sheet_def()], overrides, defaults, [] -def resolve_collect_metric_ids(db: Session, proj: BizMigrationProject, *, persist: bool = True) -> list[str]: +def resolve_collect_metric_ids(db: Session, proj: BizMigrationProject) -> list[str]: """Project-level override wins; else monitor template; else sheet metric ids.""" override = _project_collect_override(proj) if override: return override - mt = resolve_project_monitor_template(db, proj, persist=persist) + mt = resolve_project_monitor_template(db, proj) collect = [str(x).strip() for x in (mt.collect_metric_ids_json or []) if str(x).strip()] if collect: return collect @@ -624,7 +460,7 @@ def project_to_dict(db: Session, p: BizMigrationProject) -> dict[str, Any]: collect_override = _project_collect_override(p) mt_brief = _monitor_template_brief(db, getattr(p, "monitor_template_id", None) or "") # Same resolution as ensure/evaluate (empty → compare sheets) - effective_collect = resolve_collect_metric_ids(db, p, persist=False) + effective_collect = resolve_collect_metric_ids(db, p) return { "id": p.id, "name": p.name, @@ -676,79 +512,11 @@ def batch_to_dict(b: BizMigrationBatch) -> dict[str, Any]: } -def _preload_project_refs(db: Session, rows: list[BizMigrationProject]) -> list[Any]: - """Load everything ``project_to_dict`` looks up with a handful of IN queries. - - ``Session.get`` answers from the identity map afterwards, so a list of N projects - costs a constant number of queries instead of ~10 per project. The identity map is - weak: the caller must keep the returned objects alive while it serializes. - """ - keep: list[Any] = [] - def ids(*attrs: str) -> set[str]: - out: set[str] = set() - for p in rows: - for a in attrs: - v = getattr(p, a, None) - if v: - out.add(str(v)) - return out - - task_ids = ids("old_task_id", "new_task_id", "old_hf_task_id", "new_hf_task_id") - for side in ("old", "new"): - for p in rows: - task_ids.update(b["task_id"] for b in _hf_bindings(p, side) if b.get("task_id")) - if task_ids: - keep.append(db.query(BizStateTask).filter(BizStateTask.id.in_(task_ids)).all()) - batch_ids = ids("old_baseline_batch_id", "new_baseline_batch_id") - if batch_ids: - keep.append(db.query(BizStateBatch).filter(BizStateBatch.id.in_(batch_ids)).all()) - template_ids = ids("monitor_template_id") - if template_ids: - monitors = db.query(BizMonitorTemplate).filter(BizMonitorTemplate.id.in_(template_ids)).all() - keep.append(monitors) - compare_ids = {m.compare_template_id for m in monitors if m.compare_template_id} - if compare_ids: - keep.append(db.query(BizCompareTemplate).filter(BizCompareTemplate.id.in_(compare_ids)).all()) - return keep - - def list_projects(db: Session) -> list[dict[str, Any]]: rows = db.query(BizMigrationProject).order_by(BizMigrationProject.created_at.desc()).all() - preloaded = _preload_project_refs(db, rows) # noqa: F841 — keeps the identity map populated return [project_to_dict(db, p) for p in rows] -def _reject_ambiguous_mapping( - port_map: dict[str, str], *, conflicts: list[dict[str, str]] | None = None, - raw: dict[str, str] | None = None, -) -> None: - """Cutover monitoring needs a 1:1 map. Say which rows collide, not just "ambiguous".""" - by_target: dict[str, list[str]] = {} - for before, after in port_map.items(): - by_target.setdefault(after, []).append(before) - n_to_1 = [{"target": t, "sources": sorted(src)} for t, src in sorted(by_target.items()) if len(src) > 1] - collided = bool(conflicts) or (raw is not None and len(port_map) != len(raw)) - if not n_to_1 and not collided: - return - parts = [f"{', '.join(c['sources'])} -> {c['target']}" for c in n_to_1] - parts += [f"{c['dropped_from']} vs {c['kept_from']}" for c in (conflicts or [])] - raise HTTPException(status_code=400, detail={ - "code": "mapping_target_ambiguous", - "message": "mapping_target_ambiguous: " + "; ".join(parts), - "n_to_1": n_to_1, "key_conflicts": list(conflicts or []), - }) - - -def _validate_baseline_ownership(db: Session, side: str, batch_id: str, task_id: str) -> None: - """A baseline batch must be a finished sample of that side's portrait task.""" - bid = str(batch_id or "").strip() - if not bid: - return - sample = db.get(BizStateBatch, bid) - if not sample or sample.task_id != str(task_id or "").strip() or sample.status not in ("success", "partial"): - raise HTTPException(status_code=400, detail=f"{side}_baseline_invalid") - - def create_project(db: Session, body: dict[str, Any]) -> dict[str, Any]: """One-shot cutover project: NE + monitor template + metrics + schedule + baseline + mapping. @@ -836,8 +604,6 @@ def create_project(db: Session, body: dict[str, Any]) -> dict[str, Any]: raise HTTPException(status_code=404, detail="old_baseline_not_found") if new_baseline and not db.get(BizStateBatch, new_baseline): raise HTTPException(status_code=404, detail="new_baseline_not_found") - _validate_baseline_ownership(db, "old", old_baseline, old_task_id) - _validate_baseline_ownership(db, "new", new_baseline, new_task_id) p = BizMigrationProject( id=uuid4().hex, @@ -889,11 +655,6 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st p = db.get(BizMigrationProject, project_id) if not p: raise HTTPException(status_code=404, detail="project_not_found") - # Validate incoming baseline ownership before mutating any project field. - for side in ("old", "new"): - bid = str(body.get(f"{side}_baseline_batch_id", getattr(p, f"{side}_baseline_batch_id")) or "").strip() - tid = str(body.get(f"{side}_task_id", getattr(p, f"{side}_task_id")) or "").strip() - _validate_baseline_ownership(db, side, bid, tid) if "name" in body and body["name"] is not None: p.name = str(body["name"]).strip() or p.name if "note" in body and body["note"] is not None: @@ -909,13 +670,11 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st bid = str(body["old_baseline_batch_id"] or "").strip() if bid and not db.get(BizStateBatch, bid): raise HTTPException(status_code=404, detail="batch_not_found") - BASELINE_ROWS.invalidate(p.old_baseline_batch_id) p.old_baseline_batch_id = bid if "new_baseline_batch_id" in body and body["new_baseline_batch_id"] is not None: bid = str(body["new_baseline_batch_id"] or "").strip() if bid and not db.get(BizStateBatch, bid): raise HTTPException(status_code=404, detail="batch_not_found") - BASELINE_ROWS.invalidate(p.new_baseline_batch_id) p.new_baseline_batch_id = bid if "monitor_template_id" in body and body["monitor_template_id"] is not None: mid = str(body["monitor_template_id"] or "").strip() @@ -955,19 +714,6 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st p.hf_start_at = _parse_dt(body.get("hf_start_at")) if "hf_end_at" in body: p.hf_end_at = _parse_dt(body.get("hf_end_at")) - sampling_changed = any(k in body for k in ("collect_metric_ids", "metric_interval_sec", "hf_interval_sec", "monitor_template_id")) - has_hf = bool(_all_hf_task_ids(p, "old") and _all_hf_task_ids(p, "new")) - if sampling_changed and has_hf: - # Validate newly selected commands before saving the project selection. - for side in ("old", "new"): - portrait = db.get(BizStateTask, portrait_task_id(p, side)) - existing = db.get(BizStateTask, hf_task_id(p, side)) - fields = _template_from_task_or_ne(db, portrait=portrait, existing_hf=existing, ne=None) - for mid in resolve_collect_metric_ids(db, p): - task = db.get(BizStateTask, _hf_task_id_for_metric(p, side, mid)) - if (task and mid in _enabled_metric_ids(db, task.id)) or _portrait_metric_items(db, portrait, mid): - continue - _catalog_item_for_metric(vendor=fields["vendor"], device_type=fields["device_type"], metric_id=mid) p.updated_at = utcnow_naive() db.commit() db.refresh(p) @@ -975,9 +721,6 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st if any(k in body for k in ("status", "hf_start_at", "hf_end_at")): _apply_hf_window_to_tasks(db, p) db.refresh(p) - if sampling_changed and has_hf: - ensure_highfreq(db, p.id, collect_now=False) - db.refresh(p) return project_to_dict(db, p) @@ -1005,8 +748,6 @@ def delete_project(db: Session, project_id: str) -> dict[str, Any]: db.delete(b) db.delete(p) db.commit() - BASELINE_ROWS.invalidate(p.old_baseline_batch_id) - BASELINE_ROWS.invalidate(p.new_baseline_batch_id) return {"ok": True} @@ -1027,13 +768,11 @@ def create_batch(db: Session, project_id: str, body: dict[str, Any]) -> dict[str raise HTTPException(status_code=404, detail="project_not_found") label = str(body.get("batch_label") or "").strip() or "batch" expect = body.get("expect_set") if isinstance(body.get("expect_set"), dict) else {} - status = str(body.get("status") or "pending").strip() - _validate_batch_status(db, project_id, "", status) b = BizMigrationBatch( id=uuid4().hex, project_id=project_id, batch_label=label, - status=status, + status=str(body.get("status") or "pending").strip() or "pending", expect_set_json=dict(expect or {}), note=str(body.get("note") or "")[:500], ) @@ -1080,61 +819,29 @@ def run_evaluate( new_batch_id: str = "", acceptance: bool = False, purpose: str = "manual", - _commit: bool = True, - _metric_ids: set[str] | None = None, - _run_id: str = "", ) -> dict[str, Any]: - """Evaluate a wave. - - Synchronous callers (auto monitor, tests) hold the project row lock for the whole - computation. With ``_run_id`` (background evaluate/finish) the caller already - claimed the project through a ``running`` run row, so the lock is only taken for the - short write at the end. - """ mb = db.get(BizMigrationBatch, batch_id) if not mb: raise HTTPException(status_code=404, detail="batch_not_found") - proj_query = db.query(BizMigrationProject).filter(BizMigrationProject.id == mb.project_id) - if not _run_id: - # PostgreSQL serializes evaluate/finish for this project across API workers. - proj_query = proj_query.with_for_update().populate_existing() - proj = proj_query.first() + proj = db.get(BizMigrationProject, mb.project_id) if not proj: raise HTTPException(status_code=404, detail="project_not_found") - db.refresh(mb) - if acceptance and mb.status not in ("active", "review"): - raise HTTPException(status_code=409, detail="monitor_window_closed") - if purpose == "auto" and (mb.status != "active" or proj.status == "done"): - raise HTTPException(status_code=409, detail="monitor_window_closed") if not proj.old_baseline_batch_id: raise HTTPException(status_code=400, detail="old_baseline_required") if not proj.new_baseline_batch_id: raise HTTPException(status_code=400, detail="new_baseline_required") - for side in ("old", "new"): - baseline = db.get(BizStateBatch, getattr(proj, f"{side}_baseline_batch_id")) - if not baseline or baseline.task_id != portrait_task_id(proj, side): - raise HTTPException(status_code=400, detail=f"{side}_baseline_task_mismatch") - if baseline.status not in ("success", "partial"): - raise HTTPException(status_code=400, detail=f"{side}_baseline_not_complete") - pin = (old_batch_id if side == "old" else new_batch_id).strip() - if pin: - pinned = db.get(BizStateBatch, pin) - if not pinned or pinned.task_id not in _all_hf_task_ids(proj, side): - raise HTTPException(status_code=400, detail=f"{side}_current_batch_task_mismatch") old_batch = db.get(BizStateBatch, old_batch_id.strip()) if old_batch_id.strip() else None new_batch = db.get(BizStateBatch, new_batch_id.strip()) if new_batch_id.strip() else None # Primary current batch ids (for run record / pin UX); per-metric may differ when multi-HF if not old_batch: for tid in _all_hf_task_ids(proj, "old") or [current_task_id(proj, "old")]: - old_batch = db.query(BizStateBatch).filter(BizStateBatch.task_id == tid).order_by( - BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() + old_batch = _latest_success_batch(db, tid) if old_batch: break if not new_batch: for tid in _all_hf_task_ids(proj, "new") or [current_task_id(proj, "new")]: - new_batch = db.query(BizStateBatch).filter(BizStateBatch.task_id == tid).order_by( - BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() + new_batch = _latest_success_batch(db, tid) if new_batch: break if not old_batch: @@ -1147,50 +854,11 @@ def run_evaluate( pin_new = new_batch_id.strip() port_map = _port_map_dict(db, proj.mapping_id) - if proj.mapping_id and not db.get(BizPortMapping, proj.mapping_id): - raise HTTPException(status_code=400, detail="mapping_missing") - _reject_ambiguous_mapping(port_map) expect = parse_expect_set(mb.expect_set_json if isinstance(mb.expect_set_json, dict) else {}) - previous_expect: dict[str, set[str]] = {} - for prior in db.query(BizMigrationBatch).filter( - BizMigrationBatch.project_id == proj.id, BizMigrationBatch.id != mb.id, - BizMigrationBatch.status.in_(("review", "done")), - BizMigrationBatch.created_at <= mb.created_at, - ).all(): - for mid, keys in parse_expect_set(prior.expect_set_json).items(): - previous_expect.setdefault(mid, set()).update(keys) # Final acceptance: window closed → unfinished expect = red window_active = (mb.status == "active") and (not acceptance) - # Evaluations never seed or commit default configuration inside the lock. - mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None - if not mt: - raise HTTPException(status_code=400, detail="monitor_template_missing") - if mt.compare_template_id and not db.get(BizCompareTemplate, mt.compare_template_id): - raise HTTPException(status_code=400, detail="compare_template_missing") + mt = resolve_project_monitor_template(db, proj) sheets, sheet_overrides, defaults, iface_norm = resolve_evaluate_sheets(db, mt) - from .validation import validate_monitor_rules - - validate_monitor_rules(defaults, sheet_overrides) - from ..biz_state.compare_validation import validate_template_body - - validate_template_body({"metrics": sheets, "iface_normalize_rules": iface_norm}) - from ..biz_state.compare_service import _batch_vendor_key - from ..biz_state.identity_pipeline import build_pipeline - from ..biz_state.iface_normalize import normalize_port_map - - raw_port_map = dict(port_map) - old_vendor = _batch_vendor_key(db, proj.old_baseline_batch_id) - new_vendor = _batch_vendor_key(db, proj.new_baseline_batch_id) - normalized_map, map_conflicts = normalize_port_map(port_map, iface_norm) - _reject_ambiguous_mapping(normalized_map, conflicts=map_conflicts, raw=port_map) - port_map = normalized_map - known_sheets = {sheet_key(s) for s in sheets} - known_metrics = {str(s.get("metric_id") or "") for s in sheets} - for override in sheet_overrides: - if (override.get("sheet_id") and override["sheet_id"] not in known_sheets) or ( - not override.get("sheet_id") and override.get("metric_id") not in known_metrics - ): - raise HTTPException(status_code=400, detail="monitor_override_sheet_missing") out_of_expect = str((defaults or {}).get("out_of_expect") or "strict").strip().lower() collect_ids = set(resolve_collect_metric_ids(db, proj)) @@ -1198,58 +866,20 @@ def run_evaluate( all_rows: list[dict[str, Any]] = [] metric_batches: dict[str, dict[str, str]] = {} seq = 0 - rows_cache: dict[tuple[str, str], list[dict[str, Any]]] = {} - command_cache: dict[str, dict[str, Any]] = {} - enabled_sheets = [s for s in sheets if str(s.get("metric_id") or "") in collect_ids and not - override_for_sheet(sheet_overrides, sheet_id=sheet_key(s), metric_id=s.get("metric_id", "")).get("skip_dual")] - scope_ids = {str(s.get("metric_id") or "") for s in enabled_sheets} | {sheet_key(s) for s in enabled_sheets} - if any(s.get("iface_fields") for s in enabled_sheets): - scope_ids.add("_ports") - uncovered_scope = sorted({k for scope in (expect, previous_expect) for k, values in scope.items() - if values and k not in scope_ids}) - config_snapshot = {"identity_version": 2, "vendors": {"old": old_vendor, "new": new_vendor}, - "sheets": sheets, "sheet_overrides": sheet_overrides, - "defaults": defaults, "iface_normalize": iface_norm, "port_map": port_map, - "expect_set": mb.expect_set_json, "previous_expect": {k: sorted(v) for k, v in previous_expect.items()}, - "baseline_ids": [proj.old_baseline_batch_id, proj.new_baseline_batch_id], - "collect_metric_ids": sorted(collect_ids), "window_active": window_active, - "bindings": [_hf_bindings(proj, "old"), _hf_bindings(proj, "new")]} - prior_run = db.query(BizMigrationRun).filter( - BizMigrationRun.batch_id == batch_id, BizMigrationRun.status == "success").order_by( - BizMigrationRun.created_at.desc()).first() if _metric_ids is not None else None - reusable = (prior_run.summary_json or {}).get("config_snapshot") == config_snapshot if prior_run else False - prior_cards = {sheet_key(c): c for c in (prior_run.summary_json or {}).get("sheet_cards", [])} if reusable else {} - pair_cache: dict[str, tuple[BizStateBatch | None, BizStateBatch | None, dict[str, Any]]] = {} + verdict_counts: dict[str, int] = {} - baseline_ids = {proj.old_baseline_batch_id, proj.new_baseline_batch_id} - - def metric_rows(bid: str, mid: str) -> list[dict[str, Any]]: - key = (bid, mid) - if key not in rows_cache: - if bid in baseline_ids: - # Baselines never change: share them across the minute-by-minute evaluations. - rows_cache[key] = BASELINE_ROWS.get_or_load( - bid, mid, lambda: _load_metric_rows(db, batch_id=bid, metric_id=mid)) - else: - rows_cache[key] = _load_metric_rows(db, batch_id=bid, metric_id=mid) - return rows_cache[key] - - for sheet_index, sheet in enumerate(sheets, start=1): + for sheet in sheets: mid = str(sheet.get("metric_id") or "").strip() sid = sheet_key(sheet) key_fields = list(sheet.get("key_fields") or []) if not mid or not key_fields: continue - if _run_id: - _report_eval_progress(_run_id, sheet_index, len(sheets), sheet_title(sheet)) iface_fields = list(sheet.get("iface_fields") or []) compare_fields = list(sheet.get("compare_fields") or []) + row_filters = list(sheet.get("row_filters") or []) field_rules = list(sheet.get("field_rules") or []) sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid) if sheet_ov.get("skip_dual"): - sheet_cards.append({"metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), - "progress_ok": 0, "progress_total": 0, "anomaly": 0, - "collect_skipped": True}) continue # Metric not in HF collect set → skip dual eval (avoid false red) @@ -1272,28 +902,9 @@ def run_evaluate( ) continue - if _metric_ids is not None and mid not in _metric_ids and sid in prior_cards: - cached = {**prior_cards[sid], "source_run_id": prior_cards[sid].get("source_run_id") or prior_run.id} - times = [_parse_dt(cached.get(k)) for k in ("old_collected_at", "new_collected_at")] - if any(at and (utcnow_naive() - at).total_seconds() > cached.get("age_limit_sec", 180) for at in times): - cached.update({"current_missing": True, "sample_issues": ["sample_stale"]}) - sheet_cards.append(cached) - metric_batches[mid] = (prior_run.summary_json or {}).get("metric_batches", {}).get(mid, {}) - continue - if mid not in pair_cache: - pair_cache[mid] = _current_pair_for_metric(db, proj, mid, pin_old, pin_new) - old_cur_batch, new_cur_batch, pair_meta = pair_cache[mid] + old_cur_batch = _current_batch_for_metric(db, proj, "old", mid, pinned_batch_id=pin_old) + new_cur_batch = _current_batch_for_metric(db, proj, "new", mid, pinned_batch_id=pin_new) # Do NOT fall back to another HF task's batch — empty/wrong metric → false red - if (not old_cur_batch or not new_cur_batch) and mid in MANUAL_ROUTE_METRICS: - # Route tables are collected by hand; until someone does, they are not part of the gate. - sheet_cards.append({ - "metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), - "progress_ok": 0, "progress_total": 0, "anomaly": 0, "anomaly_in_expect": 0, - "old_summary": {}, "new_summary": {}, "new_baseline_mode": "manual_pending", - "new_baseline_missing": False, "manual_pending": True, "collect_skipped": False, - "current_missing": False, **pair_meta, - }) - continue if not old_cur_batch or not new_cur_batch: sheet_cards.append( { @@ -1311,45 +922,28 @@ def run_evaluate( "current_missing": True, "collect_incomplete": True, "collect_skipped": False, - **pair_meta, } ) continue old_cur_mid = old_cur_batch.id new_cur_mid = new_cur_batch.id - # A slice (address family) one device never collected is a collection gap, not a lost business. - uncovered = _uncovered_slice_sides(db, sheet, mid, old_cur_mid, new_cur_mid) - if uncovered: - sheet_cards.append({ - "metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), - "progress_ok": 0, "progress_total": 0, "anomaly": 0, "anomaly_in_expect": 0, - "old_summary": {}, "new_summary": {}, "new_baseline_mode": "not_collected", - "new_baseline_missing": False, "not_collected": True, "data_state": "not_collected", - "data_state_reasons": [f"{side}:slice_not_collected" for side in uncovered], - "collect_skipped": False, "current_missing": False, - "old_batch_id": old_cur_mid, "new_batch_id": new_cur_mid, **pair_meta, - }) - continue - timing = {**pair_meta, **_sample_timing(proj, mid, old_cur_batch, new_cur_batch, - check_age=acceptance or not (pin_old and pin_new))} - baseline_missing = [side for side in ("old", "new") if not _metric_sample_ok( - db, db.get(BizStateBatch, getattr(proj, f"{side}_baseline_batch_id")), mid)] - if timing["sample_issues"] or baseline_missing: - sheet_cards.append({"metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), - "progress_ok": 0, "progress_total": 0, "anomaly": 0, - "current_missing": True, "collect_incomplete": True, - "baseline_missing_sides": baseline_missing, **timing}) - continue - # Old device = "before", new device = "after"; filters see normalized rows. - pipe = build_pipeline( - sheet, iface_rules=iface_norm, port_map=raw_port_map, - before_vendor=old_vendor, after_vendor=new_vendor, + old_base = apply_row_filters( + _load_metric_rows(db, batch_id=proj.old_baseline_batch_id, metric_id=mid), + row_filters, + ) + old_now = apply_row_filters( + _load_metric_rows(db, batch_id=old_cur_mid, metric_id=mid), + row_filters, + ) + new_base_rows = apply_row_filters( + _load_metric_rows(db, batch_id=proj.new_baseline_batch_id, metric_id=mid), + row_filters, + ) + new_now = apply_row_filters( + _load_metric_rows(db, batch_id=new_cur_mid, metric_id=mid), + row_filters, ) - old_base = [raw for raw, _ in pipe.prepare(metric_rows(proj.old_baseline_batch_id, mid), "before")] - old_now = [raw for raw, _ in pipe.prepare(metric_rows(old_cur_mid, mid), "before")] - new_base_rows = [raw for raw, _ in pipe.prepare(metric_rows(proj.new_baseline_batch_id, mid), "after")] - new_now = [raw for raw, _ in pipe.prepare(metric_rows(new_cur_mid, mid), "after")] one = evaluate_metric_dual( metric_id=mid, @@ -1361,7 +955,7 @@ def run_evaluate( old_current_rows=old_now, new_baseline_rows=new_base_rows, new_current_rows=new_now, - port_map=pipe.port_map if iface_fields else {}, + port_map=port_map if iface_fields else {}, expect=expect, window_active=window_active, acceptance=acceptance, @@ -1369,20 +963,7 @@ def run_evaluate( sheet_override=sheet_ov, out_of_expect=out_of_expect, iface_normalize_rules=iface_norm, - previous_expect=previous_expect, - allow_mapping_scope=False, - pipeline=pipe, ) - # A target snapshot taken before the source's disappearance cannot prove - # loss. Wait for the next target receipt; final acceptance stays blocked. - awaiting = 0 - if (new_cur_batch.ended_at or new_cur_batch.started_at) < old_cur_batch.started_at: - for row in one["rows"]: - if row["in_expect"] and row["rule_hit"] == "service_not_received": - row.update(verdict="awaiting_peer", color="gray", rule_hit="target_sample_predates_source") - one["anomaly"] -= 1 - one["anomaly_in_expect"] -= 1 - awaiting += 1 old_tid = _hf_task_id_for_metric(proj, "old", mid) or current_task_id(proj, "old") new_tid = _hf_task_id_for_metric(proj, "new", mid) or current_task_id(proj, "new") old_task_brief = _task_brief(db, old_tid) @@ -1392,9 +973,6 @@ def run_evaluate( "new_batch_id": new_cur_mid, "old_task_id": old_tid, "new_task_id": new_tid, - "old_status": old_cur_batch.status, - "new_status": new_cur_batch.status, - **timing, } sheet_cards.append( { @@ -1405,7 +983,6 @@ def run_evaluate( "progress_total": one["progress_total"], "anomaly": one["anomaly"], "anomaly_in_expect": int(one.get("anomaly_in_expect") or 0), - "steady_outside": int(one.get("steady_outside") or 0), "old_summary": one["old_summary"], "new_summary": one["new_summary"], "new_baseline_mode": one.get("new_baseline_mode") or "provided", @@ -1416,11 +993,6 @@ def run_evaluate( "duplicate_key_list": list(one.get("duplicate_key_list") or []), "duplicate_keys_before": int(one.get("duplicate_keys_before") or 0), "duplicate_keys_after": int(one.get("duplicate_keys_after") or 0), - "duplicate_in_expect": list(one.get("duplicate_in_expect") or []), - **timing, - "awaiting_peer": awaiting, - "verdict_counts": dict(Counter(r["verdict"] for r in one["rows"] if r.get("verdict"))), - "expected_count": sum(bool(r.get("in_expect")) for r in one["rows"]), } ) for r in one["rows"]: @@ -1431,25 +1003,20 @@ def run_evaluate( new_batch_id=new_cur_mid, old_task=old_task_brief, new_task=new_task_brief, - command_cache=command_cache, ) r["seq"] = seq r["sheet_id"] = sid seq += 1 all_rows.append(r) + v = str(r.get("verdict") or "") + if v: + verdict_counts[v] = verdict_counts.get(v, 0) + 1 - verdict_counts = {} - for card in sheet_cards: - if not card.get("current_missing"): - for verdict, count in card.get("verdict_counts", {}).items(): - verdict_counts[verdict] = verdict_counts.get(verdict, 0) + count active_cards = [ - c for c in sheet_cards - if not c.get("collect_skipped") and not c.get("current_missing") and not c.get("manual_pending") - and not c.get("not_collected") + c for c in sheet_cards if not c.get("collect_skipped") and not c.get("current_missing") ] run = BizMigrationRun( - id=_run_id or uuid4().hex, + id=uuid4().hex, project_id=proj.id, batch_id=mb.id, old_batch_id=old_cur, @@ -1485,34 +1052,6 @@ def run_evaluate( if c.get("collect_skipped") ], "verdict_counts": verdict_counts, - "anomaly_outside_expect": sum(int(c.get("anomaly") or 0) - int(c.get("anomaly_in_expect") or 0) for c in active_cards), - "steady_outside": sum(int(c.get("steady_outside") or 0) for c in active_cards), - # Only duplicate keys inside the wave's scope fail an acceptance; the rest are warnings. - "duplicate_keys": sum(len(c.get("duplicate_in_expect") or []) for c in active_cards), - "duplicate_warnings": [ - {"sheet_id": c.get("sheet_id"), "title": c.get("title"), - "count": int(c.get("duplicate_keys_before") or 0) + int(c.get("duplicate_keys_after") or 0), - "examples": list(c.get("duplicate_key_list") or [])[:5]} - for c in active_cards - if (c.get("duplicate_keys_before") or c.get("duplicate_keys_after")) - and not c.get("duplicate_in_expect") - ], - "manual_pending_metrics": [ - str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "") - for c in sheet_cards if c.get("manual_pending") - ], - "not_collected_metrics": [ - str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "") - for c in sheet_cards if c.get("not_collected") - ], - "uncovered_scope": uncovered_scope, - "coverage_complete": bool(enabled_sheets) and not uncovered_scope and not any( - c.get("current_missing") or c.get("awaiting_peer") - or c.get("duplicate_in_expect") - for c in sheet_cards - if not c.get("collect_skipped") and not c.get("manual_pending") and not c.get("not_collected") - ), - "config_snapshot": config_snapshot, "window_active": window_active, "expect_ports": sorted(expect.get("_ports") or ()), "old_current_task_id": current_task_id(proj, "old"), @@ -1520,20 +1059,8 @@ def run_evaluate( }, message="", ) - if _run_id: - # Claim the project only for the write; the computation above ran unlocked. - db.query(BizMigrationProject).filter(BizMigrationProject.id == proj.id).with_for_update().first() - existing = db.get(BizMigrationRun, _run_id) - if existing is None or existing.status != "running": - raise HTTPException(status_code=409, detail="evaluation_cancelled") - run = db.merge(run) - else: - db.add(run) + db.add(run) db.flush() - for card in sheet_cards: - card.setdefault("source_run_id", run.id) - run.summary_json = {**run.summary_json, "sheet_cards": sheet_cards} - pending_diffs: list[dict[str, Any]] = [] for r in all_rows: key_list = r.get("key") or [] ev = r.get("evidence") if isinstance(r.get("evidence"), dict) else {} @@ -1552,8 +1079,8 @@ def run_evaluate( str(r.get("metric_id") or ""), ] ) - pending_diffs.append( - dict( + db.add( + BizMigrationDiff( id=uuid4().hex, run_id=run.id, metric_id=str(r.get("metric_id") or ""), @@ -1571,8 +1098,6 @@ def run_evaluate( "old_status": r.get("old_status"), "new_status": r.get("new_status"), "rule_hit": r.get("rule_hit") or "", - "integrity_changes": r.get("integrity_changes") or [], - "previous_batch": bool(r.get("previous_batch")), "sheet_id": r.get("sheet_id") or "", "evidence": r.get("evidence") or {}, }, @@ -1584,15 +1109,8 @@ def run_evaluate( search_text=search[:2000], ) ) - if len(pending_diffs) >= 500: - db.execute(insert(BizMigrationDiff), pending_diffs) - pending_diffs.clear() - if pending_diffs: - db.execute(insert(BizMigrationDiff), pending_diffs) - db.flush() - if _commit: - db.commit() - db.refresh(run) + db.commit() + db.refresh(run) return run_to_dict(db, run) @@ -1629,12 +1147,7 @@ def _command_brief(db: Session, command_id: str) -> dict[str, Any]: cid = str(command_id or "").strip() if not cid: return {} - # Select metadata only: raw CLI output can be several MB per command. - c = db.query(BizStateBatchCommand.id, BizStateBatchCommand.raw_command, - BizStateBatchCommand.parse_status, BizStateBatchCommand.row_count, - BizStateBatchCommand.profile_id, BizStateBatchCommand.parser_id, - BizStateBatchCommand.metric_id, BizStateBatchCommand.message).filter( - BizStateBatchCommand.id == cid).first() + c = db.get(BizStateBatchCommand, cid) if not c: return {"command_id": cid} return { @@ -1657,7 +1170,6 @@ def _enrich_row_evidence( new_batch_id: str, old_task: dict[str, Any] | None, new_task: dict[str, Any] | None, - command_cache: dict[str, dict[str, Any]] | None = None, ) -> None: """Attach device / collect / show-command onto evaluate evidence (in-place).""" ev = dict(row.get("evidence") or {}) @@ -1667,11 +1179,7 @@ def _enrich_row_evidence( ): side_ev = dict(ev.get(side) or {}) netx = dict(side_ev.get("netx") or {}) - cid = str(netx.get("batch_command_id") or "") - cache = command_cache if command_cache is not None else {} - if cid not in cache: - cache[cid] = _command_brief(db, cid) - cmd = cache[cid] + cmd = _command_brief(db, str(netx.get("batch_command_id") or "")) side_ev["device"] = { "ne_id": str(brief.get("ne_id") or netx.get("ne_id") or ""), "ne_name": str(brief.get("ne_name") or ""), @@ -1713,8 +1221,6 @@ def diff_to_dict(d: BizMigrationDiff) -> dict[str, Any]: "old_status": kj.get("old_status") or "", "new_status": kj.get("new_status") or "", "rule_hit": kj.get("rule_hit") or "", - "integrity_changes": kj.get("integrity_changes") or [], - "previous_batch": bool(kj.get("previous_batch")), "evidence": dict(kj.get("evidence") or {}), "old_kind": d.old_kind, "new_kind": d.new_kind, @@ -1769,17 +1275,8 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] mt = resolve_project_monitor_template(db, p) sheets, sheet_overrides, _defaults, _norm = resolve_evaluate_sheets(db, mt) collect_ids = set(resolve_collect_metric_ids(db, p)) - raw_port_map = _port_map_dict(db, p.mapping_id) + port_map = _port_map_dict(db, p.mapping_id) out_sheets: list[dict[str, Any]] = [] - from ..biz_state.compare_service import _batch_vendor_key - from ..biz_state.identity_pipeline import build_pipeline - from ..biz_state.iface_normalize import normalize_port_map, resolve_mapped_iface - from .evaluate import KEY_SEP - - rows_cache: dict[str, list[dict[str, Any]]] = {} - old_vendor = _batch_vendor_key(db, p.old_baseline_batch_id) - new_vendor = _batch_vendor_key(db, p.new_baseline_batch_id) - port_map, _ = normalize_port_map(raw_port_map, _norm) for sheet in sheets: mid = str(sheet.get("metric_id") or "").strip() sid = sheet_key(sheet) @@ -1789,38 +1286,29 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] # Only offer expect keys for metrics that are actually HF-collected if collect_ids and mid not in collect_ids: continue - if mid in MANUAL_ROUTE_METRICS: - # Never enumerate million-row route tables into batch scope pickers. - continue sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid) if sheet_ov.get("skip_dual"): continue iface_fields = [str(k) for k in (sheet.get("iface_fields") or []) if str(k).strip()] - if mid not in rows_cache: - rows_cache[mid] = _load_metric_rows(db, batch_id=p.old_baseline_batch_id, metric_id=mid) - pipe = build_pipeline( - sheet, iface_rules=_norm, port_map=raw_port_map, - before_vendor=old_vendor, after_vendor=new_vendor, + rows_raw = apply_row_filters( + _load_metric_rows(db, batch_id=p.old_baseline_batch_id, metric_id=mid), + list(sheet.get("row_filters") or []), ) items: list[dict[str, Any]] = [] - seen_keys: set[str] = set() - for r, normalized in pipe.prepare(rows_cache[mid], "before"): - keys = [scalar_text(normalized.get(f)).strip() for f in key_fields] + for r in rows_raw: + keys = [str(r.get(f) or "").strip() for f in key_fields] if not any(keys): continue - key_str = KEY_SEP.join(keys) - if key_str in seen_keys: - continue - seen_keys.add(key_str) + key_str = "|".join(keys) mapped = "" if iface_fields and key_fields and key_fields[0] in iface_fields: - mapped = resolve_mapped_iface(keys[0], pipe.port_map) if pipe.port_map else "" + mapped = port_map.get(keys[0]) or "" items.append( { "key": key_str, "keys": keys, "mapped_to": mapped, - "label": "|".join(scalar_text(r.get(f)).strip() for f in key_fields), + "label": key_str, "row": {f: r.get(f) for f in list(dict.fromkeys([*key_fields, *iface_fields, "description", "admin", "phy", "prot"])) if f in r}, } ) @@ -1843,73 +1331,26 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] } -def preflight_project(db: Session, project_id: str) -> dict[str, Any]: - """Mapping coverage + baseline data quality of a cutover project, before the window. - - Judged on the two portrait baselines (full collections of both devices), so a - missing/ambiguous mapping row shows up before monitoring starts. - """ - from ..biz_state.compare_preflight import preflight - from ..biz_state.compare_service import _batch_vendor_key - from ..biz_state.iface_normalize import normalize_port_map - - p = db.get(BizMigrationProject, project_id) - if not p: - raise HTTPException(status_code=404, detail="project_not_found") - if not p.old_baseline_batch_id or not p.new_baseline_batch_id: - raise HTTPException(status_code=400, detail="baseline_required") - mt = resolve_project_monitor_template(db, p) - sheets, sheet_overrides, _defaults, iface_norm = resolve_evaluate_sheets(db, mt) - collect_ids = set(resolve_collect_metric_ids(db, p)) - scoped = [ - s for s in sheets - if str(s.get("metric_id") or "") and (not collect_ids or str(s.get("metric_id")) in collect_ids) - and not override_for_sheet(sheet_overrides, sheet_id=sheet_key(s), metric_id=str(s.get("metric_id"))).get("skip_dual") - ] - raw_map = _port_map_dict(db, p.mapping_id) - cache: dict[tuple[str, str], list[dict[str, Any]]] = {} - - def load(side: str, sheet: dict[str, Any]) -> list[dict[str, Any]]: - bid = p.old_baseline_batch_id if side == "before" else p.new_baseline_batch_id - key = (bid, str(sheet["metric_id"])) - if key not in cache: - cache[key] = _load_metric_rows(db, batch_id=bid, metric_id=key[1]) - return cache[key] - - report = preflight( - db, sheets=scoped, iface_rules=iface_norm, raw_port_map=raw_map, - before_batch_id=p.old_baseline_batch_id, after_batch_id=p.new_baseline_batch_id, - before_vendor=_batch_vendor_key(db, p.old_baseline_batch_id), - after_vendor=_batch_vendor_key(db, p.new_baseline_batch_id), - load_rows=load, - ) - normalized, _ = normalize_port_map(raw_map, iface_norm) - # Monitoring needs 1:1; the report lists the rows that collide. - if report["mapping"]["n_to_1_targets"] or report["mapping"]["conflicts"]: - report["risk_reasons"] = sorted({*report["risk_reasons"], "mapping_target_ambiguous"}) - report["verdict"] = "risk" - report["mapping"]["pairs_normalized"] = len(normalized) - return report - - def _catalog_item_for_metric(*, vendor: str, device_type: str, metric_id: str) -> dict[str, Any]: from ..biz_state.profiles import profiles_for_vendor from ..lldp_shared import resolve_vendor_key mid = str(metric_id or "").strip() vkey = resolve_vendor_key(vendor, device_type) - candidates = [p for p in profiles_for_vendor(vkey) if p.metric_id == mid and p.kind == "collect"] - for p in sorted(candidates, key=lambda p: bool(p.placeholders)): - if not any(getattr(ph, "required", True) for ph in (p.placeholders or [])): + for p in profiles_for_vendor(vkey): + if p.metric_id == mid and p.kind == "collect": + if getattr(p, "placeholders", None): + ph = ",".join(str(x.name) for x in (p.placeholders or []) if getattr(x, "name", None)) + raise HTTPException( + status_code=400, + detail=f"metric_needs_bindings:{mid}:{ph or 'required'}", + ) return { "source_profile_id": p.profile_id, "kind": "catalog", "enabled": True, "title": p.title or mid, } - if candidates: - ph = ",".join(x.name for x in candidates[0].placeholders if getattr(x, "required", True)) - raise HTTPException(status_code=400, detail=f"metric_needs_bindings:{mid}:{ph}") raise HTTPException( status_code=400, detail=f"no_profile_for_metric:{mid}:{vkey or vendor or 'unknown'}", @@ -2013,116 +1454,6 @@ def _template_from_task_or_ne( raise HTTPException(status_code=400, detail="old_new_ne_or_task_required") -def _portrait_metric_items(db: Session, portrait: BizStateTask | None, mid: str) -> list[dict[str, Any]]: - """Preserve CLI variants and bindings already used by the device baseline.""" - from ..biz_state.profiles import get_profile - from ..models import BizStateTaskItem, BizStateTaskItemBinding - - if not portrait: - return [] - out = [] - for item in db.query(BizStateTaskItem).filter(BizStateTaskItem.task_id == portrait.id, - BizStateTaskItem.enabled.is_(True)).order_by(BizStateTaskItem.sort_order).all(): - profile = get_profile(item.source_profile_id) if item.kind == "catalog" else None - if not profile or profile.metric_id != mid: - continue - bindings = db.query(BizStateTaskItemBinding).filter(BizStateTaskItemBinding.item_id == item.id).all() - out.append({"source_profile_id": item.source_profile_id, "kind": item.kind, - "enabled": True, "title": item.title, "command_override": item.command_override, - "bindings": [{"placeholder": b.placeholder, "value": b.value} for b in bindings]}) - return out - - -def _group_metrics(metric_ids: list[str], by_metric: dict[str, int], default_iv: int) -> list[tuple[int, list[str]]]: - """Collect groups: one task pair (one SSH login per device) per interval + lane. - - Manual route tables stay on their own; heavy dumps never share a task with light - status tables. - """ - from ..biz_state.profiles import metric_lane - - groups: list[tuple[int, list[str]]] = [] - buckets: dict[tuple[int, str], list[str]] = {} - for mid in metric_ids: - iv = max(60, by_metric.get(mid) or default_iv) - if mid in MANUAL_ROUTE_METRICS: - groups.append((iv, [mid])) - continue - key = (iv, metric_lane(mid)) - if key not in buckets: - buckets[key] = [] - groups.append((iv, buckets[key])) - buckets[key].append(mid) - return groups - - -def _group_id(project_id: str, metrics: list[str]) -> str: - return "cutover:" + project_id + ":" + hashlib.sha256("+".join(sorted(metrics)).encode()).hexdigest()[:16] - - -def _slice_key(item: dict[str, Any]) -> tuple[tuple[str, str], ...]: - from ..biz_state.profiles import get_profile - - prof = get_profile(str(item.get("source_profile_id") or "")) - return tuple(sorted((prof.slice or {}).items())) if prof is not None else () - - -def _union_slice_items( - metric_id: str, old_items: list[dict[str, Any]], new_items: list[dict[str, Any]], - old_vendor: str, new_vendor: str, -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Both devices must collect the same slices (afi / address family) of a metric. - - A slice one portrait collects and the other does not would otherwise look like - business loss. The missing side gets this vendor's profile for the slice, or a - clear ``no_profile_for_slice`` error. - """ - from ..biz_state.profiles import get_profile, profiles_for_vendor - from ..lldp_shared import resolve_vendor_key - - if not old_items or not new_items: - return old_items, new_items - - def fill(items: list[dict[str, Any]], other: list[dict[str, Any]], vendor: str) -> list[dict[str, Any]]: - have = {_slice_key(i) for i in items} - out = list(items) - vkey = resolve_vendor_key(vendor, "") or vendor - for want in [i for i in other if _slice_key(i) not in have and _slice_key(i)]: - wanted_slice = dict(_slice_key(want)) - profile = next((p for p in profiles_for_vendor(vkey, kind="collect") - if p.metric_id == metric_id and dict(p.slice or {}) == wanted_slice), None) - if profile is None: - label = ",".join(f"{k}={v}" for k, v in sorted(wanted_slice.items())) - raise HTTPException( - status_code=400, detail=f"no_profile_for_slice:{metric_id}:{label}:{vkey or vendor or 'unknown'}") - names = {ph.name for ph in profile.placeholders} - source = get_profile(str(want.get("source_profile_id") or "")) - copied = [b for b in want.get("bindings") or [] if b.get("placeholder") in names] - if any(getattr(ph, "required", True) for ph in profile.placeholders) and not copied: - raise HTTPException( - status_code=400, detail=f"metric_needs_bindings:{metric_id}:" + ",".join(sorted(names))) - out.append({"source_profile_id": profile.profile_id, "kind": "catalog", "enabled": True, - "title": profile.title or (source.title if source else metric_id), "bindings": copied}) - have.add(_slice_key(out[-1])) - return out - - return fill(old_items, new_items, old_vendor), fill(new_items, old_items, new_vendor) - - -def _group_items( - db: Session, portrait: BizStateTask | None, template_fields: dict[str, Any], metrics: list[str], - portrait_items: dict[str, list[dict[str, Any]]], -) -> list[dict[str, Any]]: - """Every metric of a merged task keeps its own portrait-configured commands.""" - out: list[dict[str, Any]] = [] - for mid in metrics: - items = portrait_items.get(mid) or [_catalog_item_for_metric( - vendor=str(template_fields.get("vendor") or ""), - device_type=str(template_fields.get("device_type") or ""), metric_id=mid)] - out.extend(items) - return out - - def _ensure_side_highfreq( db: Session, *, @@ -2133,7 +1464,6 @@ def _ensure_side_highfreq( retention_days: int, metric_ids: list[str], status: str = "running", - configured_items: list[dict[str, Any]] | None = None, ) -> tuple[BizStateTask, bool]: """Return (hf_task, created). Reuse existing HF if metrics match; never mutate portrait.""" from ..biz_state import service as biz_svc @@ -2141,9 +1471,6 @@ def _ensure_side_highfreq( want = {str(m).strip() for m in metric_ids if str(m).strip()} if not want: want = {PORT_METRIC_ID} - if existing_hf and existing_hf.collect_running and _enabled_metric_ids(db, existing_hf.id) != want: - # A running collection owns its item configuration until completion. - existing_hf = None if existing_hf and _is_highfreq_task(db, existing_hf, want): patch: dict[str, Any] = { @@ -2160,7 +1487,7 @@ def _ensure_side_highfreq( # Existing HF (purpose already cutover_hf) with wrong metrics → update items in place. # Never rewrite a task that only "looks like" HF by note — create a sibling instead. if existing_hf and str(getattr(existing_hf, "purpose", None) or "").strip() == PURPOSE_CUTOVER_HF: - items = configured_items or [ + items = [ _catalog_item_for_metric( vendor=str(template_fields.get("vendor") or existing_hf.vendor), device_type=str(template_fields.get("device_type") or existing_hf.device_type), @@ -2184,7 +1511,7 @@ def _ensure_side_highfreq( refreshed = db.get(BizStateTask, existing_hf.id) return refreshed or existing_hf, False - items = configured_items or [ + items = [ _catalog_item_for_metric( vendor=str(template_fields.get("vendor") or ""), device_type=str(template_fields.get("device_type") or ""), @@ -2238,12 +1565,11 @@ def _apply_hf_window_to_tasks(db: Session, proj: BizMigrationProject) -> None: from ..biz_state import service as biz_svc log = logging.getLogger("netx.biz_migration.hf_window") - window_status = _hf_window_status(proj) + want = _hf_window_status(proj) for tid in _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new"): task = db.get(BizStateTask, tid) if not task: continue - want = "paused" if task.collect_manual_only else window_status purpose = str(getattr(task, "purpose", None) or "").strip() if purpose and purpose != PURPOSE_CUTOVER_HF: continue @@ -2291,7 +1617,9 @@ def ensure_highfreq( old_ne: dict[str, Any] | None = None, new_ne: dict[str, Any] | None = None, ) -> dict[str, Any]: - """One task per metric/device, paired by project; route tables are manual-only.""" + """Create/bind HF tasks (one per interval group) — never overwrites portrait ids.""" + from ..biz_state.collect_runner import dispatch_collect + proj = db.get(BizMigrationProject, project_id) if not proj: raise HTTPException(status_code=404, detail="project_not_found") @@ -2305,26 +1633,33 @@ def ensure_highfreq( if not metric_ids: metric_ids = [PORT_METRIC_ID] by_metric = _metric_interval_map(proj) - groups = _group_metrics(metric_ids, by_metric, iv_default) + groups = _group_metrics_by_interval(metric_ids, default_interval=iv_default, by_metric=by_metric) want_status = _hf_window_status(proj) old_portrait = db.get(BizStateTask, proj.old_task_id) if proj.old_task_id else None new_portrait = db.get(BizStateTask, proj.new_task_id) if proj.new_task_id else None - # Reuse a single-metric binding; a legacy combined task can be assigned - # once, while the remaining metrics receive independent sibling tasks. - def _existing_by_metric(side: str) -> dict[str, BizStateTask]: - out: dict[str, BizStateTask] = {} - for binding in _hf_bindings(proj, side): - task = db.get(BizStateTask, binding["task_id"]) - if task: - for mid in binding.get("metric_ids") or metric_ids: - if mid in metric_ids: - out.setdefault(mid, task) + # Reuse existing bindings by interval when possible + def _existing_by_interval(side: str) -> dict[int, BizStateTask]: + out: dict[int, BizStateTask] = {} + for b in _hf_bindings(proj, side): + tid = str(b.get("task_id") or "").strip() + t = db.get(BizStateTask, tid) if tid else None + if not t: + continue + try: + iv = max(60, int(b.get("interval_sec") or t.interval_sec or 60)) + except (TypeError, ValueError): + iv = 60 + out[iv] = t + # legacy primary slot + primary = db.get(BizStateTask, hf_task_id(proj, side)) if hf_task_id(proj, side) else None + if primary and iv_default not in out: + out[iv_default] = primary return out - old_existing = _existing_by_metric("old") - new_existing = _existing_by_metric("new") + old_existing = _existing_by_interval("old") + new_existing = _existing_by_interval("new") old_prev_ids = set(_all_hf_task_ids(proj, "old")) new_prev_ids = set(_all_hf_task_ids(proj, "new")) @@ -2346,64 +1681,35 @@ def ensure_highfreq( old_any_created = False new_any_created = False - # Same physical device on both sides (board swap, same-chassis port move): one task - # per group serves as "old" and "new"; the port map tells the two ports apart. - same_device = (old_fields.get("source"), old_fields.get("ne_id")) == (new_fields.get("source"), new_fields.get("ne_id")) - for g_iv, g_metrics in groups: - manual_only = any(m in MANUAL_ROUTE_METRICS for m in g_metrics) - gid = _group_id(proj.id, g_metrics) - old_candidate = next((old_existing[m] for m in g_metrics if m in old_existing), None) - new_candidate = next((new_existing[m] for m in g_metrics if m in new_existing), None) - if old_candidate and old_candidate.collect_group_id not in ("", gid): - old_candidate = None - if new_candidate and new_candidate.collect_group_id not in ("", gid): - new_candidate = None - old_by_metric = {m: _portrait_metric_items(db, old_portrait, m) for m in g_metrics} - new_by_metric = {m: _portrait_metric_items(db, new_portrait, m) for m in g_metrics} - if not same_device: - for m in g_metrics: - old_by_metric[m], new_by_metric[m] = _union_slice_items( - m, old_by_metric[m], new_by_metric[m], - str(old_fields.get("vendor") or ""), str(new_fields.get("vendor") or "")) old_task, old_created = _ensure_side_highfreq( db, - existing_hf=old_candidate, + existing_hf=old_existing.get(g_iv), template_fields=old_fields, project_name=proj.name, interval_sec=g_iv, retention_days=ret, metric_ids=g_metrics, - status="paused", - configured_items=_group_items(db, old_portrait, old_fields, g_metrics, old_by_metric), + status=want_status, + ) + new_task, new_created = _ensure_side_highfreq( + db, + existing_hf=new_existing.get(g_iv), + template_fields=new_fields, + project_name=proj.name, + interval_sec=g_iv, + retention_days=ret, + metric_ids=g_metrics, + status=want_status, ) - if same_device: - new_task, new_created = old_task, False - else: - new_task, new_created = _ensure_side_highfreq( - db, - existing_hf=new_candidate, - template_fields=new_fields, - project_name=proj.name, - interval_sec=g_iv, - retention_days=ret, - metric_ids=g_metrics, - status="paused", - configured_items=_group_items(db, new_portrait, new_fields, g_metrics, new_by_metric), - ) - if not same_device and old_task.ne_id == new_task.ne_id: - raise HTTPException(status_code=400, detail="collect_pair_same_device") - for task in {old_task.id: old_task, new_task.id: new_task}.values(): - task.collect_group_id = gid - task.collect_manual_only = manual_only - task.status = "paused" if task.collect_manual_only else want_status - db.commit() old_any_created = old_any_created or old_created new_any_created = new_any_created or new_created - binding = {"metric_ids": list(g_metrics), "interval_sec": g_iv, "collect_group_id": gid, - "manual_only": manual_only} - old_bindings.append({"task_id": old_task.id, **binding}) - new_bindings.append({"task_id": new_task.id, **binding}) + old_bindings.append( + {"task_id": old_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv} + ) + new_bindings.append( + {"task_id": new_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv} + ) proj = db.get(BizMigrationProject, project_id) if not proj: @@ -2426,24 +1732,27 @@ def ensure_highfreq( continue if str(getattr(t, "purpose", None) or "").strip() not in ("", PURPOSE_CUTOVER_HF): continue - if t.collect_group_id and not t.collect_group_id.startswith("cutover:" + proj.id + ":"): - continue - t.collect_group_id = "" - t.collect_manual_only = False - db.commit() if t.status == "running": try: biz_svc.update_task(db, tid, {"status": "paused"}) except Exception: # noqa: BLE001 pass - if t.collect_running: - from ..biz_state.collect_stop import request_stop_collect - - request_stop_collect(tid) collect: dict[str, Any] = {"old": [], "new": []} - if collect_now: - collect = collect_project_now(db, project_id) + if collect_now and want_status == "running": + for side, binds in (("old", old_bindings), ("new", new_bindings)): + for b in binds: + tid = b["task_id"] + try: + dispatch_collect(tid) + collect[side].append({"ok": True, "task_id": tid}) + except Exception as exc: # noqa: BLE001 + collect[side].append({"ok": False, "task_id": tid, "error": str(exc)[:200]}) + elif collect_now and want_status != "running": + collect = { + "old": [{"ok": False, "error": "hf_window_inactive"}], + "new": [{"ok": False, "error": "hf_window_inactive"}], + } return { "project": project_to_dict(db, proj), @@ -2486,33 +1795,26 @@ def collect_project_now(db: Session, project_id: str) -> dict[str, Any]: } out: dict[str, Any] = {"old": [], "new": [], "hf_status": want_status} - old_bindings, new_bindings = _hf_bindings(proj, "old"), _hf_bindings(proj, "new") - handled: set[str] = set() - for old_binding in old_bindings: - gid = old_binding.get("collect_group_id") - if not gid or gid in handled: + for side in ("old", "new"): + tids = _all_hf_task_ids(proj, side) + if not tids: + out[side] = [{"ok": False, "error": "hf_task_missing"}] continue - handled.add(gid) - new_binding = next((b for b in new_bindings if b.get("collect_group_id") == gid), None) - if not new_binding: - result = {"ok": False, "error": "collect_pair_invalid"} - else: - result = dispatch_collect(old_binding["task_id"], manual=True) - result = {**result, "ok": bool(result.get("queued")), "error": result.get("reason", "")} - for side, binding in (("old", old_binding), ("new", new_binding)): - out[side].append({**result, "task_id": binding["task_id"] if binding else "", - "metric_ids": old_binding.get("metric_ids", [])}) - # Legacy projects remain operable until their bindings are upgraded. - if not handled: - for side in ("old", "new"): - tids = _all_hf_task_ids(proj, side) - if not tids: - out[side] = [{"ok": False, "error": "hf_task_missing"}] - for tid in tids: - result = dispatch_collect(tid, manual=True) - out[side].append({**result, "ok": bool(result.get("queued")), "task_id": tid, - "error": result.get("reason", "")}) - + for tid in tids: + task = db.get(BizStateTask, tid) + if not task: + out[side].append({"ok": False, "error": "task_not_found", "task_id": tid}) + continue + if bool(task.collect_running): + out[side].append( + {"ok": False, "error": "collect_already_running", "task_id": tid} + ) + continue + try: + dispatch_collect(tid) + out[side].append({"ok": True, "task_id": tid}) + except Exception as exc: # noqa: BLE001 + out[side].append({"ok": False, "task_id": tid, "error": str(exc)[:200]}) return out @@ -2539,7 +1841,6 @@ def _red_ticket_to_dict(t: BizMigrationRedTicket) -> dict[str, Any]: "new_status": t.new_status, "detail": detail, "evidence": dict(detail.get("evidence") or {}), - "resolvable": bool(detail.get("resolvable")), "status": t.status, "carried_to_batch_id": t.carried_to_batch_id, "note": t.note, @@ -2621,30 +1922,26 @@ def _persist_red_tickets_from_run( ) .all() ) - by_id: dict[tuple[str, str, str], BizMigrationRedTicket] = {} + by_id: dict[tuple[str, str], BizMigrationRedTicket] = {} for t in existing: - detail = t.detail_json if isinstance(t.detail_json, dict) else {} mk = str(getattr(t, "match_key_str", "") or "").strip() if not mk: detail = t.detail_json if isinstance(t.detail_json, dict) else {} mk = str(detail.get("match_old_key") or t.key_str or "").strip() if mk: - by_id[(str(t.metric_id or ""), str(detail.get("sheet_id") or t.metric_id or ""), - str(detail.get("match_old_key") or mk))] = t + by_id[(str(t.metric_id or ""), mk)] = t created_or_updated: list[BizMigrationRedTicket] = [] - seen: set[tuple[str, str, str]] = set() + seen: set[tuple[str, str]] = set() for d in diffs: kj = d.key_json if isinstance(d.key_json, dict) else {} mid = str(d.metric_id or "") match_key = str( kj.get("match_old_key") or kj.get("key_str") or kj.get("old_key") or "" - ).strip() + ).strip()[:256] if not match_key: match_key = str(kj.get("key_str") or "")[:256] - ident = (mid, str(kj.get("sheet_id") or mid), match_key) - from hashlib import sha256 - stored_key = match_key if len(match_key) <= 256 else "sha256:" + sha256(match_key.encode()).hexdigest() + ident = (mid, match_key) detail = { "old_kind": d.old_kind, "new_kind": d.new_kind, @@ -2655,8 +1952,6 @@ def _persist_red_tickets_from_run( "new_key": kj.get("new_key") or kj.get("new_key_str") or "", "match_old_key": kj.get("match_old_key") or match_key, "match_new_key": kj.get("match_new_key") or "", - "sheet_id": kj.get("sheet_id") or mid, - "integrity_changes": kj.get("integrity_changes") or [], "evidence": dict(kj.get("evidence") or {}), } prev = by_id.get(ident) @@ -2665,7 +1960,7 @@ def _persist_red_tickets_from_run( prev.run_id = run_id prev.key_str = str(kj.get("key_str") or "")[:256] prev.new_key_str = str(kj.get("new_key_str") or "")[:256] - prev.match_key_str = stored_key + prev.match_key_str = match_key prev.verdict = d.verdict prev.color = d.color or "red" prev.old_status = str(kj.get("old_status") or "")[:64] @@ -2685,7 +1980,7 @@ def _persist_red_tickets_from_run( metric_id=mid, key_str=str(kj.get("key_str") or "")[:256], new_key_str=str(kj.get("new_key_str") or "")[:256], - match_key_str=stored_key, + match_key_str=match_key, verdict=d.verdict, color=d.color or "red", old_status=str(kj.get("old_status") or "")[:64], @@ -2698,51 +1993,16 @@ def _persist_red_tickets_from_run( created_or_updated.append(t) seen.add(ident) - # Tickets whose key is now green stay open (an operator decides) but are flagged so the - # UI can offer a one-click close. Keys absent from this run are left alone. - pending = {ident: t for ident, t in by_id.items() if ident not in seen and t.status in ("open", "carried")} - if pending: - metrics = {ident[0] for ident in pending} - greens: dict[tuple[str, str, str], str] = {} - for g in db.query(BizMigrationDiff).filter( - BizMigrationDiff.run_id == run_id, BizMigrationDiff.color == "green", - BizMigrationDiff.metric_id.in_(metrics), - ): - gk = g.key_json if isinstance(g.key_json, dict) else {} - mk = str(gk.get("match_old_key") or gk.get("key_str") or gk.get("old_key") or "").strip() - greens[(str(g.metric_id or ""), str(gk.get("sheet_id") or g.metric_id or ""), mk)] = str(g.verdict or "") - for ident, t in pending.items(): - verdict = greens.get(ident) - if verdict is not None: - t.detail_json = {**dict(t.detail_json or {}), "resolvable": {"run_id": run_id, "verdict": verdict}} + # Tickets open on this batch but no longer red → leave as open (operator resolves). + # Carried tickets for keys not in this run stay carried. return created_or_updated -def resolve_resolvable_red_tickets(db: Session, project_id: str) -> dict[str, Any]: - """Close every ticket the latest acceptance saw turn green (explicit operator action).""" - if not db.get(BizMigrationProject, project_id): - raise HTTPException(status_code=404, detail="project_not_found") - closed = [] - for t in db.query(BizMigrationRedTicket).filter( - BizMigrationRedTicket.project_id == project_id, - BizMigrationRedTicket.status.in_(("open", "carried")), - ): - mark = (t.detail_json or {}).get("resolvable") if isinstance(t.detail_json, dict) else None - if mark: - t.status = "resolved" - t.resolved_at = utcnow_naive() - t.note = (t.note or f"auto-closed: green in run {mark.get('run_id', '')}")[:500] - closed.append(t.id) - db.commit() - return {"closed": len(closed), "ids": closed, "open_red_count": count_open_red_tickets(db, project_id)} - - def finish_batch( db: Session, batch_id: str, *, mark_done: bool = False, - _run_id: str = "", ) -> dict[str, Any]: """本批完成：关窗 → 终验 → 验收小结 → 红单留痕（不硬卡下一批）.""" mb = db.get(BizMigrationBatch, batch_id) @@ -2754,11 +2014,14 @@ def finish_batch( detail=f"batch_not_finishable:{mb.status}", ) - # acceptance=True closes the logical window. Persist batch closure only after - # evaluation succeeds, so an API/collection failure keeps the batch operable. - run = run_evaluate( - db, batch_id=batch_id, acceptance=True, purpose="acceptance", _commit=False, _run_id=_run_id, - ) + # Close window first so acceptance uses non-active rules + if mb.status == "active": + mb.ended_at = utcnow_naive() + mb.status = "done" if mark_done else "review" + mb.updated_at = utcnow_naive() + db.commit() + + run = run_evaluate(db, batch_id=batch_id, acceptance=True, purpose="acceptance") summary = dict(run.get("summary") or {}) progress = dict(summary.get("progress") or {}) anomaly = int(summary.get("anomaly") or 0) @@ -2773,10 +2036,6 @@ def finish_batch( fail_reasons.append("new_baseline_missing") if current_missing: fail_reasons.append("current_missing") - if not summary.get("coverage_complete", False): - fail_reasons.append("coverage_incomplete") - if summary.get("duplicate_keys"): - fail_reasons.append("duplicate_keys") if anomaly: fail_reasons.append("anomaly") if total > 0 and ok < total: @@ -2793,11 +2052,6 @@ def finish_batch( "anomaly": anomaly, "verdict_counts": dict(summary.get("verdict_counts") or {}), "expect_ports": list(summary.get("expect_ports") or []), - "warnings": { - "duplicate_keys_outside_scope": list(summary.get("duplicate_warnings") or []), - "manual_pending_metrics": list(summary.get("manual_pending_metrics") or []), - "not_collected_metrics": list(summary.get("not_collected_metrics") or []), - }, "run_id": run.get("id"), "old_batch_id": run.get("old_batch_id"), "new_batch_id": run.get("new_batch_id"), @@ -2807,9 +2061,6 @@ def finish_batch( mb = db.get(BizMigrationBatch, batch_id) if not mb: raise HTTPException(status_code=404, detail="batch_not_found") - if mb.status == "active": - mb.ended_at = utcnow_naive() - mb.status = "done" if mark_done else "review" mb.accept_run_id = str(run.get("id") or "") mb.accept_status = "passed" if passed else "failed" mb.accept_summary_json = accept_summary @@ -2835,139 +2086,10 @@ def finish_batch( } -_EVAL_THREADS_LOCK = threading.Lock() - - -def _report_eval_progress(run_id: str, index: int, total: int, title: str) -> None: - """Progress visible to pollers: separate short transaction, never fatal.""" - from ..db import SessionLocal - - try: - s = SessionLocal() - try: - run = s.get(BizMigrationRun, run_id) - if run is not None and run.status == "running": - summary = dict(run.summary_json or {}) - summary["progress"] = {"phase": "evaluating", "sheet_index": index, - "sheet_total": total, "sheet_title": title} - run.summary_json = summary - s.commit() - finally: - s.close() - except Exception: # noqa: BLE001 — progress is best effort - _log.debug("evaluate progress write failed run=%s", run_id, exc_info=True) - - -def _claim_evaluation( - db: Session, batch_id: str, *, acceptance: bool, purpose: str, -) -> BizMigrationRun: - """Create the ``running`` run that owns the project. 409 when one is already in flight.""" - mb = db.get(BizMigrationBatch, batch_id) - if not mb: - raise HTTPException(status_code=404, detail="batch_not_found") - proj = db.query(BizMigrationProject).filter(BizMigrationProject.id == mb.project_id).with_for_update().first() - if not proj: - raise HTTPException(status_code=404, detail="project_not_found") - db.refresh(mb) - if acceptance and mb.status not in ("active", "review"): - raise HTTPException(status_code=409, detail="monitor_window_closed") - if not proj.old_baseline_batch_id: - raise HTTPException(status_code=400, detail="old_baseline_required") - if not proj.new_baseline_batch_id: - raise HTTPException(status_code=400, detail="new_baseline_required") - busy = db.query(BizMigrationRun.id).filter( - BizMigrationRun.project_id == proj.id, BizMigrationRun.status == "running").first() - if busy: - raise HTTPException(status_code=409, detail="evaluation_in_progress") - run = BizMigrationRun( - id=uuid4().hex, project_id=proj.id, batch_id=batch_id, purpose=purpose, status="running", - summary_json={"progress": {"phase": "queued", "sheet_index": 0, "sheet_total": 0}}, - message="queued", - ) - db.add(run) - db.commit() - db.refresh(run) - return run - - -def _spawn_evaluation(run_id: str, work: Any, label: str) -> None: - from ..db import SessionLocal - - def _bg() -> None: - s = SessionLocal() - try: - work(s) - except Exception as exc: # noqa: BLE001 — recorded on the run for the poller - s.rollback() - detail = getattr(exc, "detail", None) - _log.warning("background %s failed run=%s: %s", label, run_id, detail or exc) - run = s.get(BizMigrationRun, run_id) - if run is not None and run.status == "running": - run.status = "failed" - run.message = str(detail if detail is not None else exc)[:1000] - s.commit() - finally: - s.close() - - threading.Thread(target=_bg, name=f"biz-mig-{label}-{run_id[:8]}", daemon=True).start() - - -def enqueue_evaluate( - db: Session, batch_id: str, *, old_batch_id: str = "", new_batch_id: str = "", -) -> dict[str, Any]: - """Start a manual evaluation in the background; poll ``GET /runs/{id}`` for the result.""" - run = _claim_evaluation(db, batch_id, acceptance=False, purpose="manual") - _spawn_evaluation( - run.id, - lambda s: run_evaluate(s, batch_id=batch_id, old_batch_id=old_batch_id, new_batch_id=new_batch_id, - purpose="manual", _run_id=run.id), - "evaluate", - ) - return run_to_dict(db, run) - - -def enqueue_finish(db: Session, batch_id: str, *, mark_done: bool = False) -> dict[str, Any]: - """Final acceptance in the background. The wave closes only if the evaluation succeeds.""" - mb = db.get(BizMigrationBatch, batch_id) - if not mb: - raise HTTPException(status_code=404, detail="batch_not_found") - if mb.status not in ("active", "review"): - raise HTTPException(status_code=400, detail=f"batch_not_finishable:{mb.status}") - run = _claim_evaluation(db, batch_id, acceptance=True, purpose="acceptance") - _spawn_evaluation( - run.id, lambda s: finish_batch(s, batch_id, mark_done=mark_done, _run_id=run.id), "finish", - ) - return {"run_id": run.id, "status": "running", "run": run_to_dict(db, run), - "batch": batch_to_dict(db.get(BizMigrationBatch, batch_id))} - - -def recover_interrupted_evaluations_on_startup(db: Session) -> int: - """A restart kills background evaluations; release the projects they were holding.""" - rows = db.query(BizMigrationRun).filter(BizMigrationRun.status == "running").all() - for r in rows: - r.status = "cancelled" - r.message = "interrupted_by_restart" - if rows: - db.commit() - return len(rows) - - -def _validate_batch_status(db: Session, project_id: str, batch_id: str, status: str) -> None: - if status not in ("pending", "active", "review", "done"): - raise HTTPException(status_code=400, detail="batch_status_invalid") - if status == "active": - db.query(BizMigrationProject).filter(BizMigrationProject.id == project_id).with_for_update().first() - if db.query(BizMigrationBatch.id).filter(BizMigrationBatch.project_id == project_id, - BizMigrationBatch.id != batch_id, BizMigrationBatch.status == "active").first(): - raise HTTPException(status_code=409, detail="another_batch_active") - - def patch_batch(db: Session, batch_id: str, body: dict[str, Any]) -> dict[str, Any]: b = db.get(BizMigrationBatch, batch_id) if not b: raise HTTPException(status_code=404, detail="batch_not_found") - if body.get("status") is not None: - _validate_batch_status(db, b.project_id, b.id, str(body["status"]).strip()) if "batch_label" in body and body["batch_label"] is not None: b.batch_label = str(body["batch_label"]).strip() or b.batch_label if "note" in body and body["note"] is not None: @@ -3013,44 +2135,30 @@ def list_run_diffs( run_id: str, *, metric_id: str = "", - sheet_id: str = "", - only_expect: bool = False, verdict: str = "", color: str = "", kw: str = "", offset: int = 0, limit: int = 100, ) -> dict[str, Any]: - run = db.get(BizMigrationRun, run_id) - if not run: + if not db.get(BizMigrationRun, run_id): raise HTTPException(status_code=404, detail="run_not_found") - sources = [and_(BizMigrationDiff.run_id == c["source_run_id"], - BizMigrationDiff.key_json["sheet_id"].as_string() == sheet_key(c)) - for c in (run.summary_json or {}).get("sheet_cards", []) - if c.get("source_run_id") and c["source_run_id"] != run_id - and not (only_expect and c.get("expected_count") == 0) - and (not metric_id or c.get("metric_id") == metric_id) - and (not sheet_id or sheet_key(c) == sheet_id)] - q = db.query(BizMigrationDiff).filter(or_(BizMigrationDiff.run_id == run_id, *sources)) + q = db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id == run_id) if metric_id: q = q.filter(BizMigrationDiff.metric_id == metric_id) - if sheet_id: - q = q.filter(BizMigrationDiff.key_json["sheet_id"].as_string() == sheet_id) - if only_expect: - q = q.filter(BizMigrationDiff.in_expect.is_(True)) if verdict: q = q.filter(BizMigrationDiff.verdict == verdict) if color: q = q.filter(BizMigrationDiff.color == color) if kw.strip(): - literal = kw.strip().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - q = q.filter(BizMigrationDiff.search_text.ilike(f"%{literal}%", escape="\\")) + like = f"%{kw.strip()}%" + q = q.filter(BizMigrationDiff.search_text.ilike(like)) total = q.count() - rows = q.order_by(BizMigrationDiff.metric_id.asc(), BizMigrationDiff.seq.asc(), BizMigrationDiff.id.asc()).offset(max(0, offset)).limit(min(500, max(1, limit))).all() + rows = q.order_by(BizMigrationDiff.seq.asc()).offset(max(0, offset)).limit(min(500, max(1, limit))).all() return { "total": total, - "offset": max(0, offset), - "limit": min(500, max(1, limit)), + "offset": offset, + "limit": limit, "items": [diff_to_dict(d) for d in rows], } @@ -3078,10 +2186,9 @@ def board(db: Session, batch_id: str, run_id: str = "") -> dict[str, Any]: if not run or run.batch_id != batch_id: raise HTTPException(status_code=404, detail="run_not_found") else: - # Latest *finished* run; a run still being computed has nothing to show yet. run = ( db.query(BizMigrationRun) - .filter(BizMigrationRun.batch_id == batch_id, BizMigrationRun.status == "success") + .filter(BizMigrationRun.batch_id == batch_id) .order_by(BizMigrationRun.created_at.desc()) .first() ) @@ -3089,35 +2196,9 @@ def board(db: Session, batch_id: str, run_id: str = "") -> dict[str, Any]: "project": proj, "batch": batch_to_dict(mb), "run": run_to_dict(db, run) if run else None, - "sampling": sampling_state(db, db.get(BizMigrationProject, mb.project_id)) if not run_id else [], } -def sampling_state(db: Session, proj: BizMigrationProject) -> list[dict[str, Any]]: - """Small live read independent of evaluation and route row counts.""" - out = [] - for binding in _hf_bindings(proj, "old"): - gid = binding.get("collect_group_id") - if not gid: - continue - latest = db.query(BizStateBatch).filter(BizStateBatch.collect_group_id == gid).order_by( - BizStateBatch.queued_at.desc(), BizStateBatch.started_at.desc()).first() - peers = db.query(BizStateBatch).filter(BizStateBatch.collect_round_id == latest.collect_round_id).all() if latest else [] - old = next((b for b in peers if b.task_id == binding["task_id"]), None) - new = next((b for b in peers if b.task_id != binding["task_id"]), None) - card: dict[str, Any] = {"metric_id": (binding.get("metric_ids") or [""])[0], - "collect_group_id": gid, "collect_round_id": latest.collect_round_id if latest else "", - "manual_only": binding.get("manual_only", False), "interval_sec": binding["interval_sec"]} - for side, batch in (("old", old), ("new", new)): - card[f"{side}_collect_status"] = batch.status if batch else "idle" - card[f"{side}_duration_sec"] = max(0, ((batch.ended_at or utcnow_naive()) - batch.started_at).total_seconds()) if batch and batch.status != "queued" else 0 - card[f"{side}_queue_sec"] = max(0, ((utcnow_naive() if batch.status == "queued" else batch.started_at) - (batch.queued_at or batch.started_at)).total_seconds()) if batch else 0 - if old and new and all(b.status != "queued" for b in (old, new)): - card["start_skew_sec"] = abs((old.started_at - new.started_at).total_seconds()) - out.append(card) - return out - - def get_monitor_context( db: Session, *, diff --git a/netx_api/biz_migration/validation.py b/netx_api/biz_migration/validation.py deleted file mode 100644 index 998761a..0000000 --- a/netx_api/biz_migration/validation.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Validate monitor correction rules before they can affect live verdicts.""" -from typing import Any - -from fastapi import HTTPException - -from .evaluate import _groups_for_side, _normalize_cond_type - - -def validate_monitor_rules(defaults: Any, overrides: Any) -> None: - def fail(path: str) -> None: - raise HTTPException(status_code=400, detail=f"monitor_rule_invalid:{path}") - - if not isinstance(defaults, dict) or not isinstance(overrides, list): - fail("defaults/sheet_overrides") - if defaults.get("out_of_expect", "strict") not in ("strict", "warn", "ignore"): - fail("defaults.out_of_expect") - seen = set() - for i, ov in enumerate(overrides): - path = f"sheet_overrides[{i}]" - if not isinstance(ov, dict) or not (ov.get("sheet_id") or ov.get("metric_id")): - fail(path) - ident = ov.get("sheet_id") or ov.get("metric_id") - if ident in seen: - fail(path + ".duplicate") - seen.add(ident) - if ov.get("scope_by_port_map") is not None and not isinstance(ov["scope_by_port_map"], bool): - fail(path + ".scope_by_port_map") - for key in ("status_fields", "up_values", "down_values"): - if key in ov and (not isinstance(ov[key], list) or any(not str(v).strip() for v in ov[key])): - fail(path + "." + key) - for key in ("success", "anomaly"): - patterns = ov.get(key, []) - if not isinstance(patterns, list): - fail(path + "." + key) - for j, pat in enumerate(patterns): - pp = f"{path}.{key}[{j}]" - if not isinstance(pat, dict): - fail(pp) - for side in ("old", "new"): - for attr in (side, side + "_conds", side + "_groups"): - if attr in pat and not isinstance(pat[attr], list): - fail(pp + "." + attr) - if any(isinstance(g, list) and not g for g in pat.get(side + "_groups", [])): - fail(pp + "." + side + "_groups.empty_group") - groups = _groups_for_side(pat, side) - if key == "success" and not groups: - fail(pp + "." + side + ".required") - for group in groups: - for cond in group: - cp = pp + "." + side - if isinstance(cond, str): - if cond not in ("added", "removed", "changed", "unchanged", "up", "down", "other", "none") and not (cond.startswith("field:") and cond.count(":") >= 2): - fail(cp + ".token") - elif isinstance(cond, dict): - typ = _normalize_cond_type(cond) - if typ not in ("presence", "status", "value"): - fail(cp + ".type") - val = cond.get("value", cond.get("kind", cond.get("status"))) - if typ == "presence" and val not in ("added", "removed", "changed", "unchanged"): - fail(cp + ".presence") - if typ == "status" and val not in ("up", "down", "other", "none"): - fail(cp + ".status") - if typ == "value": - op = cond.get("op", "eq") - if op not in ("eq", "ne", "in", "not_in", "empty", "not_empty", "changed", "unchanged", "same", "diff"): - fail(cp + ".op") - if op in ("eq", "ne", "in", "not_in") and (val is None or val == "" or val == []): - fail(cp + ".value") - else: - fail(cp + ".condition") - if key == "anomaly" and not any(_groups_for_side(pat, s) for s in ("old", "new")): - fail(pp + ".empty") diff --git a/netx_api/biz_migration_router.py b/netx_api/biz_migration_router.py index 8104a4c..29b4377 100644 --- a/netx_api/biz_migration_router.py +++ b/netx_api/biz_migration_router.py @@ -151,12 +151,6 @@ def api_baseline_ports(project_id: str, db: Session = Depends(get_db)): return svc.list_baseline_ports(db, project_id) -@router.get("/projects/{project_id}/preflight") -@router.post("/projects/{project_id}/preflight") -def api_project_preflight(project_id: str, db: Session = Depends(get_db)): - return svc.preflight_project(db, project_id) - - @router.get("/projects/{project_id}/baseline-expect") def api_baseline_expect(project_id: str, db: Session = Depends(get_db)): """Multi-metric baseline keys for expect-set picking (from monitor template sheets).""" @@ -228,13 +222,8 @@ def api_finish_batch( mark_done: bool = False, db: Session = Depends(get_db), ): - """本批完成：终验 + 小结 + 红单（允许带红继续）. Runs in the background; poll the run.""" - return svc.enqueue_finish(db, batch_id, mark_done=mark_done) - - -@router.post("/projects/{project_id}/red-tickets/resolve-resolvable") -def api_resolve_resolvable(project_id: str, db: Session = Depends(get_db)): - return svc.resolve_resolvable_red_tickets(db, project_id) + """本批完成：终验 + 小结 + 红单（允许带红继续）.""" + return svc.finish_batch(db, batch_id, mark_done=mark_done) @router.get("/projects/{project_id}/red-tickets") @@ -269,10 +258,9 @@ def api_patch_batch(batch_id: str, body: BatchPatchIn, db: Session = Depends(get @router.post("/batches/{batch_id}/evaluate") def api_evaluate(batch_id: str, body: EvaluateIn | None = None, db: Session = Depends(get_db)): payload = body.model_dump() if body else {} - # Returns the ``running`` run at once; poll GET /runs/{id} until ``status`` settles. - return svc.enqueue_evaluate( + return svc.run_evaluate( db, - batch_id, + batch_id=batch_id, old_batch_id=str(payload.get("old_batch_id") or ""), new_batch_id=str(payload.get("new_batch_id") or ""), ) @@ -297,8 +285,6 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)): def api_list_diffs( run_id: str, metric_id: str = "", - sheet_id: str = "", - only_expect: bool = False, verdict: str = "", color: str = "", kw: str = "", @@ -310,8 +296,6 @@ def api_list_diffs( db, run_id, metric_id=metric_id, - sheet_id=sheet_id, - only_expect=only_expect, verdict=verdict, color=color, kw=kw, diff --git a/netx_api/biz_state/claim.py b/netx_api/biz_state/claim.py index d45b6ec..a86674e 100644 --- a/netx_api/biz_state/claim.py +++ b/netx_api/biz_state/claim.py @@ -2,10 +2,8 @@ from __future__ import annotations -from collections import defaultdict - import logging -from datetime import datetime, timedelta +from datetime import datetime from typing import Any from uuid import uuid4 @@ -27,11 +25,6 @@ def max_concurrent_tasks() -> int: return max(1, int(getattr(settings, "biz_state_max_concurrent_tasks", 16) or 16)) -def max_sessions_per_ne() -> int: - """Concurrent SSH collections allowed per network element (>= 1).""" - return max(1, int(getattr(settings, "biz_state_max_sessions_per_ne", 2) or 2)) - - def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: """Create a queued batch and mark task collect_running. Does not open SSH. @@ -46,10 +39,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: task = db.get(BizStateTask, tid) if not task: return {"ok": False, "queued": False, "reason": "task_not_found", "task_id": tid} - if task.collect_group_id: - group_id = task.collect_group_id - db.close() - return enqueue_collect_group(group_id, manual=manual, requested_task_id=tid) if str(task.source or "").strip().lower() == "import": return { "ok": False, @@ -110,7 +99,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: vendor=task.vendor, status="queued", started_at=now, - queued_at=now, message="queued" if not manual else "queued_manual", ) db.add(batch) @@ -133,62 +121,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: db.close() -def enqueue_collect_group(group_id: str, *, manual: bool = False, - requested_task_id: str = "") -> dict[str, Any]: - """Atomically enqueue both devices with one round identity, or neither.""" - db = SessionLocal() - try: - tasks = db.query(BizStateTask).filter(BizStateTask.collect_group_id == group_id).order_by( - BizStateTask.id).with_for_update().populate_existing().all() - result: dict[str, Any] = {"ok": False, "queued": False, "task_id": requested_task_id, - "collect_group_id": group_id} - # One task = a single device used as both sides (same-device cutover); two tasks = a pair. - if len(tasks) not in (1, 2) or len({t.ne_id for t in tasks}) != len(tasks): - return {**result, "reason": "collect_pair_invalid"} - if max_concurrent_tasks() < len(tasks): - return {**result, "reason": "collect_pair_capacity"} - from ..cli_budget import clamp_cli_workers - - worker_cap = clamp_cli_workers(int(getattr(settings, "biz_state_worker_collect_threads", None) - or getattr(settings, "biz_state_dispatch_workers", 8) or 8)) - if worker_cap < len(tasks): - return {**result, "reason": "collect_pair_capacity"} - if any(t.collect_running for t in tasks): - return {**result, "reason": "collect_pair_busy"} - for task in tasks: - if task.source == "import" or (not manual and (task.status != "running" or task.collect_manual_only)): - return {**result, "reason": "not_scheduled"} - if task.status in ("", "deleted"): - return {**result, "reason": "bad_status"} - if not db.query(BizStateTaskItem.id).filter(BizStateTaskItem.task_id == task.id, - BizStateTaskItem.enabled.is_(True)).first(): - return {**result, "reason": "no_enabled_items"} - now, round_id = _utcnow(), uuid4().hex - batches = [] - for task in tasks: - task.collect_running = True - task.collect_queued_at = now - task.last_error = "" - task.updated_at = now - batch = BizStateBatch(id=uuid4().hex, task_id=task.id, source=task.source, - ne_id=task.ne_id, ne_name=task.ne_name, vendor=task.vendor, status="queued", - started_at=now, queued_at=now, collect_group_id=group_id, - collect_round_id=round_id, collect_priority=10 if task.collect_manual_only else 20, - message="queued_manual" if manual else "queued") - db.add(batch) - batches.append({"batch_id": batch.id, "task_id": task.id}) - db.commit() - selected = next((b for b in batches if b["task_id"] == requested_task_id), batches[0]) - return {**result, **selected, "ok": True, "queued": True, "collect_round_id": round_id, - "batches": batches, "manual": manual} - except Exception: - db.rollback() - _log.exception("enqueue collect pair failed group=%s", group_id) - return {"ok": False, "queued": False, "reason": "enqueue_error", "task_id": requested_task_id} - finally: - db.close() - - def _dialect_supports_skip_locked(db: Session) -> bool: try: bind = db.get_bind() @@ -219,16 +151,11 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: Global ceiling is always ``max_concurrent_tasks()``; ``limit`` only caps how many this caller wants in one call. """ - from sqlalchemy import and_, case, text + from sqlalchemy import text global_cap = max_concurrent_tasks() db = SessionLocal() try: - pg = _dialect_supports_skip_locked(db) - if pg: - # A short transaction lock makes capacity reservation and pair claims - # atomic across worker processes. Never held during device I/O. - db.execute(text("SELECT pg_advisory_xact_lock(hashtext('biz-state-claim-capacity'))")) running_n = ( db.query(BizStateBatch).filter(BizStateBatch.status == "running").count() ) @@ -238,67 +165,58 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: if slots <= 0: return [] - # Running collections per NE; an NE accepts new ones until it reaches its session cap. - per_ne_cap = max_sessions_per_ne() - ne_running: dict[str, int] = defaultdict(int) - for (running_ne,) in db.query(BizStateBatch.ne_id).filter(BizStateBatch.status == "running").all(): - if str(running_ne or "").strip(): - ne_running[str(running_ne).strip()] += 1 + busy_nes = { + str(r[0] or "").strip() + for r in db.query(BizStateBatch.ne_id) + .filter(BizStateBatch.status == "running") + .all() + if str(r[0] or "").strip() + } - # Manual route rounds yield to fresh monitoring, but must eventually - # run even when a busy device always has regular metrics queued. - priority = case((and_(BizStateBatch.collect_priority == 10, - BizStateBatch.queued_at <= _utcnow() - timedelta(seconds=300)), 30), - else_=BizStateBatch.collect_priority) q = ( db.query(BizStateBatch) .filter(BizStateBatch.status == "queued") - .order_by(priority.desc(), BizStateBatch.started_at.asc(), BizStateBatch.id.asc()) + .order_by(BizStateBatch.started_at.asc()) ) + pg = _dialect_supports_skip_locked(db) if pg: q = q.with_for_update(skip_locked=True) else: q = q.with_for_update() - candidates = q.limit(max(slots * 8, 64)).all() + candidates = q.limit(max(slots * 4, slots)).all() claimed_rows: list[BizStateBatch] = [] - visited: set[str] = set() for batch in candidates: - if batch.id in visited: - continue if len(claimed_rows) >= slots: break - group = [batch] - if batch.collect_round_id: - peers = db.query(BizStateBatch).filter( - BizStateBatch.collect_round_id == batch.collect_round_id, - ).with_for_update(skip_locked=pg).all() - expected = db.query(BizStateTask.id).filter( - BizStateTask.collect_group_id == batch.collect_group_id).count() if batch.collect_group_id else 2 - if len(peers) != max(1, expected) or any(p.status != "queued" for p in peers): + ne = str(batch.ne_id or "").strip() + if ne and ne in busy_nes: + continue + # Serialize claims per NE across workers (Postgres). + if ne and pg: + try: + db.execute( + text("SELECT pg_advisory_xact_lock(hashtext(:ne))"), + {"ne": ne}, + ) + except Exception: + _log.exception("advisory lock failed ne=%s", ne) + conflict = ( + db.query(BizStateBatch.id) + .filter( + BizStateBatch.status == "running", + BizStateBatch.ne_id == ne, + BizStateBatch.id != batch.id, + ) + .first() + ) + if conflict: continue - group = peers - visited.update(p.id for p in group) - if len(group) > slots - len(claimed_rows): - continue - wanted: dict[str, int] = defaultdict(int) - for p in group: - if str(p.ne_id or "").strip(): - wanted[str(p.ne_id).strip()] += 1 - if any(ne_running[ne] + n > per_ne_cap for ne, n in wanted.items()): - continue - now = _utcnow() - for member in group: - member.status = "running" - member.message = "" - if member.collect_round_id: - member.started_at = now - task = db.get(BizStateTask, member.task_id) - if task: - task.last_collect_started_at = now - claimed_rows.append(member) - for ne, n in wanted.items(): - ne_running[ne] += n + batch.status = "running" + batch.message = "" + claimed_rows.append(batch) + if ne: + busy_nes.add(ne) if not claimed_rows: db.rollback() @@ -313,13 +231,10 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: db.query(BizStateBatch).filter(BizStateBatch.status == "running").count() ) while n_running > global_cap and claimed_rows: - last = claimed_rows[-1] - demoted = [b for b in claimed_rows if b.collect_round_id == last.collect_round_id] if last.collect_round_id else [last] - for demote in demoted: - claimed_rows.remove(demote) - demote.status = "queued" - demote.message = "queued" - n_running -= 1 + demote = claimed_rows.pop() + demote.status = "queued" + demote.message = "queued" + n_running -= 1 if not claimed_rows: db.rollback() diff --git a/netx_api/biz_state/collect_runner.py b/netx_api/biz_state/collect_runner.py index 91797ce..ad04d77 100644 --- a/netx_api/biz_state/collect_runner.py +++ b/netx_api/biz_state/collect_runner.py @@ -784,12 +784,6 @@ def dispatch_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: result = enqueue_collect(task_id, manual=manual) if not result.get("queued"): return result - if result.get("collect_group_id"): - if _should_execute_inline(): - from ..biz_state_scheduler import _claim_and_dispatch - - _claim_and_dispatch() - return result batch_id = str(result.get("batch_id") or "") if not batch_id: return result @@ -876,11 +870,6 @@ def execute_claimed_batch( task = db.get(BizStateTask, task_id) if task_id else None if not batch: return - if batch.collect_round_id: - batch.started_at = _utcnow() - if task: - task.last_collect_started_at = batch.started_at - db.commit() if not task_id: task_id = str(batch.task_id or "") task = db.get(BizStateTask, task_id) if task_id else None @@ -1731,14 +1720,6 @@ def _finalize_batch_status( batch.message = "" status = str(batch.status or "") db.commit() - # Failed/partial receipts also update cutover coverage; never keep an old - # success looking healthy after the latest collection has failed. - try: - from ..biz_migration.auto_monitor import schedule_auto_monitor_for_task - - schedule_auto_monitor_for_task(task_id) - except Exception: - _log.exception("cutover monitor schedule failed task=%s", task_id) # Only full success triggers auto compare; never block the collect thread. if status == "success" and not stopped: try: diff --git a/netx_api/biz_state/collect_stop.py b/netx_api/biz_state/collect_stop.py index 1f27e31..8c92db0 100644 --- a/netx_api/biz_state/collect_stop.py +++ b/netx_api/biz_state/collect_stop.py @@ -131,7 +131,7 @@ def _force_close_holders(batch_id: str) -> int: def request_stop_collect(task_id: str) -> dict[str, Any]: - """Stop a task, including its paired collector, without stranding a half round.""" + """Cancel queued batches and signal running collect for this task to abort.""" tid = str(task_id or "").strip() if not tid: return {"ok": False, "reason": "missing_task_id"} @@ -141,37 +141,26 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: signaled_ids: list[str] = [] closed = 0 try: - from sqlalchemy import text - from .claim import _dialect_supports_skip_locked - - if _dialect_supports_skip_locked(db): - db.execute(text("SELECT pg_advisory_xact_lock(hashtext('biz-state-claim-capacity'))")) task = db.get(BizStateTask, tid) if not task: return {"ok": False, "reason": "task_not_found", "task_id": tid} - task_ids = [tid] - if task.collect_group_id: - task_ids = [r[0] for r in db.query(BizStateTask.id).filter( - BizStateTask.collect_group_id == task.collect_group_id).all()] active = ( db.query(BizStateBatch) .filter( - BizStateBatch.task_id.in_(task_ids), + BizStateBatch.task_id == tid, BizStateBatch.status.in_(("queued", "running")), ) - .order_by(BizStateBatch.id).with_for_update().all() + .all() ) - tasks = db.query(BizStateTask).filter(BizStateTask.id.in_(task_ids)).order_by( - BizStateTask.id).with_for_update().populate_existing().all() if not active: # Nothing to stop — clear sticky collect_running if orphaned. - for member in tasks: - if member.collect_running: - member.collect_running = False - member.collect_queued_at = None - member.updated_at = utcnow_naive() - db.commit() + if bool(task.collect_running): + task.collect_running = False + if hasattr(task, "collect_queued_at"): + task.collect_queued_at = None + task.updated_at = utcnow_naive() + db.commit() return { "ok": True, "task_id": tid, @@ -182,7 +171,7 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: } now = utcnow_naive() - running_tasks: set[str] = set() + still_running = False for batch in active: bid = str(batch.id) mark_stop_requested(bid) @@ -194,16 +183,16 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: else: batch.message = STOP_REQUEST_TOKEN signaled_ids.append(bid) - running_tasks.add(batch.task_id) + still_running = True closed += _force_close_holders(bid) - for member in tasks: - if member.id not in running_tasks: - member.collect_running = False - member.collect_queued_at = None - member.last_collect_ended_at = now - member.last_error = STOP_USER_MESSAGE - member.updated_at = now + if not still_running: + task.collect_running = False + if hasattr(task, "collect_queued_at"): + task.collect_queued_at = None + task.last_collect_ended_at = now + task.last_error = STOP_USER_MESSAGE + task.updated_at = now db.commit() _log.info( diff --git a/netx_api/biz_state/compare_engine.py b/netx_api/biz_state/compare_engine.py index 27953ff..b11cae8 100644 --- a/netx_api/biz_state/compare_engine.py +++ b/netx_api/biz_state/compare_engine.py @@ -2,81 +2,15 @@ from __future__ import annotations -from collections import defaultdict -from heapq import heappush, heapreplace -from itertools import chain -from typing import Any, Iterable, Mapping, Sequence +from typing import Any, Mapping, Sequence -from .compare_rules import field_rule_map, values_equal, explain_diff, scalar_text +from .compare_rules import field_rule_map, values_equal, explain_diff from .iface_normalize import ( apply_iface_normalize_rows, normalize_iface_rules, resolve_mapped_iface, ) -# Prefer these fields when stratifying success samples (BGP multipath / multi-cmd). -_STRATUM_FIELDS = ("neighbor", "direction", "afi", "vrf") - - -def stratum_key(row: Mapping[str, Any] | None) -> str: - """Bucket key for stratified unchanged sampling.""" - if not isinstance(row, Mapping): - return "_" - parts: list[str] = [] - for f in _STRATUM_FIELDS: - v = str(row.get(f) or "").strip() - if v: - parts.append(f"{f}={v}") - return "|".join(parts) if parts else "_" - - -class _StratifiedSample: - """Keep the first N round-robin ranks in O(N) space, even with many strata.""" - - def __init__(self, limit: int) -> None: - self.limit = max(0, int(limit)) - self.count = 0 - self.strata: dict[str, tuple[int, int]] = {} - self.heap: list[tuple[int, int, int, Any]] = [] - - def add(self, item: Any, key: str) -> None: - serial = self.count - self.count += 1 - if not self.limit: - return - state = self.strata.get(key) - if state is None: - # Later strata cannot beat the first item of N earlier strata. - if len(self.strata) >= self.limit: - return - index, round_n = len(self.strata), 0 - else: - index, round_n = state - self.strata[key] = (index, round_n + 1) - entry = (-round_n, -index, serial, item) - if len(self.heap) < self.limit: - heappush(self.heap, entry) - elif entry[:2] > self.heap[0][:2]: - heapreplace(self.heap, entry) - - def picked(self) -> list[Any]: - # The legacy helper preserves encounter order when no truncation occurs. - if self.count <= self.limit: - entries = sorted(self.heap, key=lambda e: e[2]) - else: - entries = sorted(self.heap, key=lambda e: (-e[0], -e[1])) - return [entry[3] for entry in entries] - - -def stratify_take(items: Iterable[Any], limit: int, *, key_fn) -> list[Any]: - """Round-robin across strata without retaining every candidate.""" - sample = _StratifiedSample(limit) - if not sample.limit: - return [] - for item in items: - sample.add(item, str(key_fn(item) or "_")) - return sample.picked() - def apply_port_map( row: dict[str, Any], @@ -93,17 +27,8 @@ def apply_port_map( return out -def key_tuple(row: Mapping[str, Any], fields: Sequence[str]) -> tuple[str, ...]: - """The one canonical match-key form: ``scalar_text(...).strip()`` per field. - - Every place that builds, stores or looks up a key string must go through - this (``None`` → ``""``, ``True`` → ``"true"``, surrounding blanks trimmed). - """ - return tuple(scalar_text(row.get(k)).strip() for k in fields) - - def row_key(row: dict[str, Any], key_fields: list[str]) -> tuple[str, ...]: - return key_tuple(row, key_fields) + return tuple(str(row.get(k) or "").strip() for k in key_fields) def mapping_stats( @@ -118,8 +43,8 @@ def mapping_stats( ``hit_before``: map key appears as a normalized before iface, or as the parent of a before subinterface (so main-port-only maps still validate). - Map keys are matched against **normalized** iface values; callers pass a - port map already normalized with the template iface rules. + Map keys are matched against **normalized** iface values (same pipeline as + compare). Enter mapping keys in normalized form. """ before_ifaces: set[str] = set() after_ifaces: set[str] = set() @@ -165,7 +90,7 @@ def mapping_stats( "miss_after": miss_after, "unused_before_keys": unused, "ok": not miss_before and not miss_after, - "hint": "map keys/values are normalized with the template iface rules before matching", + "hint": "map keys must match normalized iface names (post iface_normalize)", } @@ -180,32 +105,12 @@ def compare_rows( field_rules: Sequence[Mapping[str, Any]] | None = None, iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, ignore_port_changes: bool | None = None, - include_unchanged: bool = False, - unchanged_limit: int | None = None, - compact_unchanged: bool = False, - pipeline: Any = None, - match_keys_override: Sequence[str] | None = None, - prepared: tuple[list, list] | None = None, - duplicate_pairing: str = "ordered", ) -> dict[str, Any]: """Return summary + diffs list. - Diff kinds: added | removed | changed | unchanged + Diff kinds: added | removed | changed | unchanged | duplicate - Pipeline: iface normalize (both sides) → port map (before) → ordered - same-key pairing (load / list order within each match key). - - Same match key with N before and M after rows: zip by index - ``0..min(N,M)-1`` for field compare; extras become ``removed`` (before) - or ``added`` (after). Example: 5 vs 2 → 2 compared + 3 removed. - - ``include_unchanged``: when False, matching rows still increment - ``summary.unchanged`` but are omitted from ``diffs``. - - ``unchanged_limit``: max unchanged diffs to emit (None = no cap). Use with - large sheets so the UI can browse a sample without writing millions of rows. - - ``compact_unchanged``: emit key only (empty before/after) to cut storage. + Pipeline: iface normalize (both sides) → port map (before) → match. ``ignore_port_changes``: - ``None`` (default): auto — drop iface from match key only when remaining @@ -213,219 +118,175 @@ def compare_rows( - ``True``: force drop iface from match key when a non-empty candidate exists. - ``False``: never drop iface from match key. - ``pipeline`` (``identity_pipeline.Pipeline``) owns normalization, filtering, - port mapping and key building; when omitted a legacy pipeline is derived - from the loose arguments (callers have already filtered the rows). - ``match_keys_override`` pins the match key (re-running a stored result). - - ``summary.duplicate`` is always 0. ``duplicate_keys_*`` count match keys - that appear more than once on a side (diagnostic only). + Duplicate match keys are not silently discarded: extras become ``duplicate`` + diffs and ``summary.duplicate_key_list`` lists the colliding keys. ``field_rules`` drives normalize / numeric tolerance / per-field compare mode (template-driven; no metric-specific branches here). """ if not key_fields: raise ValueError("key_fields required") - if pipeline is None: - from .identity_pipeline import legacy_pipeline - - pipeline = legacy_pipeline( - key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields, - field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules, - ignore_port_changes=ignore_port_changes, duplicate_pairing=duplicate_pairing, - ) - pmap = dict(pipeline.port_map) - rules = field_rule_map(field_rules if field_rules is not None else pipeline.field_rules) - iface_list = list(pipeline.iface_fields) + pmap = dict(port_map or {}) + rules = field_rule_map(field_rules) + norm_rules = normalize_iface_rules(iface_normalize_rules) + iface_list = [str(f) for f in (iface_fields or []) if str(f).strip()] iface_set = set(iface_list) - if prepared is not None: - before_pairs, after_pairs = prepared - else: - before_pairs = pipeline.prepare(before_rows, "before") - after_pairs = pipeline.prepare(after_rows, "after") - before_rows = [orig for orig, _ in before_pairs] - after_rows = [orig for orig, _ in after_pairs] - before_norm = [norm for _, norm in before_pairs] - after_norm = [norm for _, norm in after_pairs] + before_norm = apply_iface_normalize_rows( + before_rows, iface_fields=iface_list, rules=norm_rules + ) + after_norm = apply_iface_normalize_rows( + after_rows, iface_fields=iface_list, rules=norm_rules + ) ignore_ports = False - if match_keys_override is not None: - match_keys = [str(k) for k in match_keys_override] - ignore_ports = match_keys != list(key_fields) + # No map → optionally ignore port renames by dropping iface from match key. + if not pmap and iface_set: + candidate = [k for k in key_fields if k not in iface_set] + if not candidate: + match_keys = list(key_fields) + elif ignore_port_changes is True: + match_keys = candidate + ignore_ports = True + elif ignore_port_changes is False: + match_keys = list(key_fields) + else: + # Auto heuristic (legacy default) + before_c = [row_key(r, candidate) for r in before_norm] + after_c = [row_key(r, candidate) for r in after_norm] + if len(before_c) == len(set(before_c)) and len(after_c) == len(set(after_c)): + match_keys = candidate + ignore_ports = True + else: + match_keys = list(key_fields) else: - # No map → optionally ignore port renames by dropping iface from match key. - policy = pipeline.ignore_port_changes - unique = None - if not pmap and iface_set and policy is None: - candidate = [k for k in key_fields if k not in iface_set] + match_keys = list(key_fields) - def unique_keys(rows: list[dict[str, Any]]) -> bool: - seen: set[tuple[str, ...]] = set() - for row in rows: - key = row_key(row, candidate) - if key in seen: - return False - seen.add(key) - return True + before_mapped: list[dict[str, Any]] = [ + apply_port_map(r, iface_fields=iface_list, port_map=pmap) for r in before_norm + ] - unique = bool(candidate) and unique_keys(before_norm) and unique_keys(after_norm) - match_keys = pipeline.effective_match_keys(unique_without_iface=unique) - ignore_ports = match_keys != list(key_fields) - - before_groups: dict[tuple[str, ...], list[tuple[dict[str, Any], dict[str, Any]]]] = ( - defaultdict(list) - ) - after_groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) - for orig, norm in zip(before_rows, before_norm): - mapped, k = pipeline.key_of(norm, "before", match_keys) - before_groups[k].append((orig, mapped)) - for norm in after_norm: - after_groups[pipeline.key_of(norm, "after", match_keys)[1]].append(norm) - sort_pairs = pipeline.duplicate_pairing == "sorted" + after_index: dict[tuple[str, ...], dict[str, Any]] = {} + after_dup = 0 + after_dup_keys: list[tuple[str, ...]] = [] + after_dup_rows: list[tuple[tuple[str, ...], dict[str, Any]]] = [] + for r in after_norm: + k = row_key(r, match_keys) + if k in after_index: + after_dup += 1 + after_dup_keys.append(k) + after_dup_rows.append((k, r)) + continue # first wins — do not overwrite + after_index[k] = r + before_keys: set[tuple[str, ...]] = set() + before_dup = 0 + before_dup_keys: list[tuple[str, ...]] = [] diffs: list[dict[str, Any]] = [] - added = removed = changed = unchanged = 0 - unchanged_listed = 0 - limit_n = None if unchanged_limit is None else max(0, int(unchanged_limit)) - multi_before_keys: list[tuple[str, ...]] = [] - multi_after_keys: list[tuple[str, ...]] = [] - unchanged_candidates: list[tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = [] - sample = _StratifiedSample(limit_n) if include_unchanged and limit_n is not None else None + added = removed = changed = unchanged = duplicate = 0 def _key_obj(row: dict[str, Any]) -> dict[str, Any]: - return dict(zip(key_fields, key_tuple(row, key_fields))) + return {f: row.get(f, "") for f in key_fields} - def _row_id(row: dict[str, Any] | None) -> str: - if not isinstance(row, dict): - return "" - netx = row.get("_netx") - if isinstance(netx, dict): - return str(netx.get("row_id") or "") - return "" - - def _append_unchanged_diff( - orig: dict[str, Any], mapped: dict[str, Any], after_row: dict[str, Any] - ) -> None: - nonlocal unchanged_listed - unchanged_listed += 1 - before_rid = _row_id(orig) - after_rid = _row_id(after_row) - if compact_unchanged: + for orig, mapped in zip(before_rows, before_mapped): + k = row_key(mapped, match_keys) + if k in before_keys: + before_dup += 1 + before_dup_keys.append(k) + duplicate += 1 diffs.append( { - "kind": "unchanged", + "kind": "duplicate", + "side": "before", "key": _key_obj(mapped), - "before": {}, - "after": {}, - "mapped_before": {}, + "before": orig, + "after": after_index.get(k), + "mapped_before": mapped, "changes": {}, - "compact": True, - "before_row_id": before_rid, - "after_row_id": after_rid, + } + ) + continue # only first before row participates in match + before_keys.add(k) + after = after_index.get(k) + if after is None: + removed += 1 + diffs.append( + { + "kind": "removed", + "key": _key_obj(mapped), + "before": orig, + "after": None, + "mapped_before": mapped, + "changes": {}, + } + ) + continue + # Empty compare_fields = presence-only: keyed rows that exist on both + # sides are unchanged (no value checks). + field_changes: dict[str, dict[str, Any]] = {} + for f in compare_fields: + bv = mapped.get(f, "") + av = after.get(f, "") + rule = rules.get(f) + if not values_equal(bv, av, rule=rule): + entry: dict[str, Any] = {"before": bv, "after": av} + reason = explain_diff(bv, av, rule=rule) + if reason: + entry["reason"] = reason + field_changes[f] = entry + if field_changes: + changed += 1 + diffs.append( + { + "kind": "changed", + "key": _key_obj(mapped), + "before": orig, + "after": after, + "mapped_before": mapped, + "changes": field_changes, } ) else: + unchanged += 1 diffs.append( { "kind": "unchanged", "key": _key_obj(mapped), "before": orig, - "after": after_row, + "after": after, "mapped_before": mapped, "changes": {}, - "before_row_id": before_rid, - "after_row_id": after_rid, } ) - # Stable key order: before encounter order, then after-only keys - ordered_keys = chain(before_groups, (k for k in after_groups if k not in before_groups)) - for k in ordered_keys: - b_list = before_groups.get(k) or [] - a_list = after_groups.get(k) or [] - if len(b_list) > 1: - multi_before_keys.append(k) - if len(a_list) > 1: - multi_after_keys.append(k) - if sort_pairs and (len(b_list) > 1 or len(a_list) > 1): - # Same-key rows pair by value, not by CLI output order (ECMP, multipath). - b_list = sorted(b_list, key=lambda pair: pipeline.pair_sort_key(pair[1])) - a_list = sorted(a_list, key=pipeline.pair_sort_key) - n = max(len(b_list), len(a_list)) - for i in range(n): - if i >= len(b_list): - after = a_list[i] - added += 1 - diffs.append( - { - "kind": "added", - "key": _key_obj(after), - "before": None, - "after": after, - "mapped_before": None, - "changes": {}, - "before_row_id": "", - "after_row_id": _row_id(after), - } - ) - continue - if i >= len(a_list): - orig, mapped = b_list[i] - removed += 1 - diffs.append( - { - "kind": "removed", - "key": _key_obj(mapped), - "before": orig, - "after": None, - "mapped_before": mapped, - "changes": {}, - "before_row_id": _row_id(orig), - "after_row_id": "", - } - ) - continue - orig, mapped = b_list[i] - after = a_list[i] - field_changes: dict[str, dict[str, Any]] = {} - for f in compare_fields: - bv = mapped.get(f, "") - av = after.get(f, "") - rule = rules.get(f) - if not values_equal(bv, av, rule=rule): - entry: dict[str, Any] = {"before": bv, "after": av} - reason = explain_diff(bv, av, rule=rule) - if reason: - entry["reason"] = reason - field_changes[f] = entry - if field_changes: - changed += 1 - diffs.append( - { - "kind": "changed", - "key": _key_obj(mapped), - "before": orig, - "after": after, - "mapped_before": mapped, - "changes": field_changes, - "before_row_id": _row_id(orig), - "after_row_id": _row_id(after), - } - ) - else: - unchanged += 1 - if include_unchanged: - if sample is not None: - if sample.limit: - sample.add((orig, mapped, after), stratum_key(mapped)) - else: - unchanged_candidates.append((orig, mapped, after)) + for k, after in after_index.items(): + if k in before_keys: + continue + added += 1 + diffs.append( + { + "kind": "added", + "key": _key_obj(after), + "before": None, + "after": after, + "mapped_before": None, + "changes": {}, + } + ) - if include_unchanged: - picked = sample.picked() if sample is not None else unchanged_candidates - for orig, mapped, after_row in picked: - _append_unchanged_diff(orig, mapped, after_row) + for k, after in after_dup_rows: + duplicate += 1 + diffs.append( + { + "kind": "duplicate", + "side": "after", + "key": _key_obj(after), + "before": None, + "after": after, + "mapped_before": None, + "changes": {}, + } + ) def _fmt_keys(keys: list[tuple[str, ...]]) -> list[str]: seen: set[str] = set() @@ -435,7 +296,7 @@ def compare_rows( if s not in seen: seen.add(s) out.append(s) - return out[:64] + return out stats = mapping_stats( before_rows=before_norm, @@ -453,21 +314,11 @@ def compare_rows( "removed": removed, "changed": changed, "unchanged": unchanged, - "duplicate": 0, + "duplicate": duplicate, "match_key_fields": match_keys, - "duplicate_keys_before": len(multi_before_keys), - "duplicate_keys_after": len(multi_after_keys), - "duplicate_key_list": _fmt_keys(multi_before_keys + multi_after_keys), - "unchanged_listed": unchanged_listed, - "unchanged_truncated": bool( - include_unchanged and limit_n is not None and unchanged > unchanged_listed - ), - "unchanged_compact": bool(compact_unchanged and unchanged_listed > 0), - "unchanged_sample_mode": ( - "stratified" - if include_unchanged and limit_n is not None and unchanged > unchanged_listed - else ("full" if include_unchanged else "none") - ), + "duplicate_keys_before": before_dup, + "duplicate_keys_after": after_dup, + "duplicate_key_list": _fmt_keys(before_dup_keys + after_dup_keys), }, "diffs": diffs, "mapping_stats": stats, diff --git a/netx_api/biz_state/compare_preflight.py b/netx_api/biz_state/compare_preflight.py deleted file mode 100644 index c9c6081..0000000 --- a/netx_api/biz_state/compare_preflight.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Pre-flight check of a port mapping against the two batches it will be applied to. - -The same function backs the compare "validate mapping" button and the cutover -project pre-flight, so both modes judge mapping coverage the same way. It answers -before anyone runs a comparison: - -- which *before* interfaces are mapped (exact / by parent), unmapped but present on - the *after* side by the same name, or **suspect** (not mapped, not on *after*) -- which map keys / targets match nothing, which targets receive several sources -- whether each sheet's two sides were actually collected (``data_state``) -""" - -from __future__ import annotations - -from typing import Any, Callable, Mapping, Sequence - -from sqlalchemy.orm import Session - -from .compare_quality import assess_sheet, command_coverage, collect_quality, side_receipts -from .identity_pipeline import build_pipeline -from .iface_normalize import normalize_port_map, resolve_mapped_iface - -LIST_CAP = 200 -# Million-row tables are never loaded just to look at interface names. -HEAVY_METRICS = frozenset({"bgp_route", "ip_route", "ipv6_route", "l2vpn_mac", "evpn_mac", "interface_detail"}) - -RowsLoader = Callable[[str, Mapping[str, Any]], list[dict[str, Any]]] - - -def _parents(name: str) -> list[str]: - parts = name.split(".") - return [".".join(parts[:i]) for i in range(len(parts) - 1, 0, -1)] - - -def _classify(name: str, pmap: Mapping[str, str], after: set[str]) -> str: - if name in pmap: - return "mapped_exact" - if any(p in pmap for p in _parents(name)): - return "mapped_by_parent" - if name in after: - return "identity_same_name" - return "suspect_unmapped" - - -def _present(name: str, names: set[str]) -> bool: - """``name`` exists, or is the parent of an existing sub-interface.""" - if name in names: - return True - prefix = name + "." - return any(n.startswith(prefix) for n in names) - - -def _top_level(names: Sequence[str]) -> list[str]: - """Drop sub-interfaces whose parent is already in the list (noise).""" - pool = set(names) - return sorted(n for n in pool if not any(p in pool for p in _parents(n))) - - -def sheet_mapping_coverage( - pipe: Any, - before_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]], - after_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]], -) -> dict[str, Any]: - """Classify every *before* interface of a sheet against the map and the *after* side. - - ``before_pairs`` / ``after_pairs`` are ``pipeline.prepare`` outputs. The result - carries two private sets (``_after_ifaces``, ``_used_keys``) for aggregation. - """ - b_if = {str(n.get(f) or "").strip() for _, n in before_pairs for f in pipe.iface_fields} - a_if = {str(n.get(f) or "").strip() for _, n in after_pairs for f in pipe.iface_fields} - b_if.discard("") - a_if.discard("") - buckets: dict[str, list[str]] = { - "mapped_exact": [], "mapped_by_parent": [], "identity_same_name": [], "suspect_unmapped": [], - } - for name in sorted(b_if): - buckets[_classify(name, pipe.port_map, a_if)].append(name) - used = {k for name in b_if for k in pipe.port_map if name == k or name.startswith(k + ".")} - missing_targets = sorted({ - resolve_mapped_iface(n, pipe.port_map) for n in buckets["mapped_exact"] + buckets["mapped_by_parent"] - if not _present(resolve_mapped_iface(n, pipe.port_map), a_if) - }) - suspects = _top_level(buckets["suspect_unmapped"]) - return { - "counts": {k: len(v) for k, v in buckets.items()}, - "mapped_exact": buckets["mapped_exact"][:LIST_CAP], - "mapped_by_parent": buckets["mapped_by_parent"][:LIST_CAP], - "identity_same_name": buckets["identity_same_name"][:LIST_CAP], - "suspect_unmapped": suspects[:LIST_CAP], - "suspect_unmapped_total": len(suspects), - "mapped_target_missing_on_after": missing_targets[:LIST_CAP], - "_after_ifaces": a_if, - "_used_keys": used, - } - - -def preflight( - db: Session, - *, - sheets: Sequence[Mapping[str, Any]], - iface_rules: Sequence[Mapping[str, Any]] | None, - raw_port_map: Mapping[str, str] | None, - before_batch_id: str, - after_batch_id: str, - before_vendor: str = "", - after_vendor: str = "", - load_rows: RowsLoader, -) -> dict[str, Any]: - """Build the pre-flight report. ``load_rows(side, sheet)`` returns raw rows.""" - raw_map = dict(raw_port_map or {}) - out_sheets: list[dict[str, Any]] = [] - used_keys: set[str] = set() - after_all: set[str] = set() - totals = {"mapped": 0, "suspect": 0, "same_name": 0} - risk_reasons: list[str] = [] - full_pmap: dict[str, str] = {} - conflicts: list[dict[str, str]] = [] - - for sheet in sheets: - sid = str(sheet.get("sheet_id") or sheet.get("metric_id") or "") - mid = str(sheet.get("metric_id") or "") - title = str(sheet.get("title") or sid) - pipe = build_pipeline( - sheet, iface_rules=iface_rules, port_map=raw_map, - before_vendor=before_vendor, after_vendor=after_vendor, - ) - if not full_pmap and pipe.port_map: - full_pmap = dict(pipe.port_map) - heavy = mid in HEAVY_METRICS - card: dict[str, Any] = {"sheet_id": sid, "metric_id": mid, "title": title} - if heavy: - b_n = collect_quality(side_receipts(db, before_batch_id, mid))["row_count"] - a_n = collect_quality(side_receipts(db, after_batch_id, mid))["row_count"] - card.update(skipped="heavy_table", before_count=b_n, after_count=a_n) - else: - b_pairs = pipe.prepare(load_rows("before", sheet), "before") - a_pairs = pipe.prepare(load_rows("after", sheet), "after") - card.update(before_count=len(b_pairs), after_count=len(a_pairs)) - if pipe.iface_fields: - cov = sheet_mapping_coverage(pipe, b_pairs, a_pairs) - after_all |= cov.pop("_after_ifaces") - used_keys |= cov.pop("_used_keys") - card.update(cov) - totals["mapped"] += cov["counts"]["mapped_exact"] + cov["counts"]["mapped_by_parent"] - totals["same_name"] += cov["counts"]["identity_same_name"] - totals["suspect"] += cov["suspect_unmapped_total"] - card.update(assess_sheet( - db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id, - before_rows=int(card.get("before_count") or 0), after_rows=int(card.get("after_count") or 0), - )) - if card["data_state"] in ("not_collected", "suspect_parse"): - risk_reasons.append(f"{sid}:{card['data_state']}") - out_sheets.append(card) - - if raw_map and not full_pmap: - full_pmap, conflicts = normalize_port_map(raw_map, iface_rules) - elif raw_map: - _, conflicts = normalize_port_map(raw_map, iface_rules) - sources: dict[str, list[str]] = {} - for k, v in full_pmap.items(): - sources.setdefault(v, []).append(k) - n_to_1 = [{"target": t, "sources": sorted(src)} for t, src in sorted(sources.items()) if len(src) > 1] - unused = sorted(k for k in full_pmap if k not in used_keys) - targets_missing = sorted(v for v in set(full_pmap.values()) if after_all and not _present(v, after_all)) - - judged = totals["mapped"] + totals["suspect"] - coverage = { - "mapped": totals["mapped"], "suspect_unmapped": totals["suspect"], - "identity_same_name": totals["same_name"], - "rate": round(totals["mapped"] / judged * 100, 1) if judged else None, - } - mapping = { - "pairs": len(full_pmap), "conflicts": conflicts[:LIST_CAP], - "unused_map_keys": unused[:LIST_CAP], "unused_map_keys_total": len(unused), - "map_targets_missing_on_after": targets_missing[:LIST_CAP], - "map_targets_missing_on_after_total": len(targets_missing), - "n_to_1_targets": n_to_1[:LIST_CAP], - } - if totals["suspect"]: - risk_reasons.append("suspect_unmapped") - if unused: - risk_reasons.append("unused_map_keys") - if targets_missing: - risk_reasons.append("map_targets_missing_on_after") - if conflicts: - risk_reasons.append("map_conflicts") - return { - "verdict": "risk" if risk_reasons else "ready", - "risk_reasons": sorted(set(risk_reasons)), - "coverage": coverage, - "mapping": mapping, - "sheets": out_sheets, - "command_coverage": command_coverage(db, before_batch_id=before_batch_id, after_batch_id=after_batch_id), - } diff --git a/netx_api/biz_state/compare_quality.py b/netx_api/biz_state/compare_quality.py deleted file mode 100644 index 30a3551..0000000 --- a/netx_api/biz_state/compare_quality.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Collection quality of a compare side: is a missing row a business loss or a data problem? - -Compare results are only as good as the two batches behind them. This module -reads *receipts* (``BizStateBatchCommand``: counts and status, never ``raw_text``) -to tell apart: - -- ``not_collected`` – a side never ran a command (or the slice) for the sheet -- ``suspect_parse`` – the device printed lines but the parser produced no rows -- ``no_data`` – both sides ran fine and are really empty -- ``ok`` – ordinary comparison -""" - -from __future__ import annotations - -from typing import Any, Mapping, Sequence - -from sqlalchemy.orm import Session - -from ..models import BizStateBatchCommand -from .compare_rules import scalar_text - -# More than this many CLI lines with zero parsed rows is not "an empty table". -SUSPECT_RAW_LINES = 5 -DATA_STATES = ("ok", "no_data", "suspect_parse", "not_collected") -UNJUDGED_STATES = ("no_data", "not_collected") - - -def _has_receipts(db: Session, batch_id: str) -> bool: - return bool(batch_id) and db.query(BizStateBatchCommand.id).filter( - BizStateBatchCommand.batch_id == batch_id).limit(1).first() is not None - - -def side_receipts(db: Session, batch_id: str, metric_id: str) -> list[dict[str, Any]]: - if not batch_id or not metric_id: - return [] - rows = ( - db.query( - BizStateBatchCommand.profile_id, BizStateBatchCommand.parse_status, - BizStateBatchCommand.row_count, BizStateBatchCommand.raw_line_count, - BizStateBatchCommand.declared_total, - ) - .filter(BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id == metric_id) - .all() - ) - return [ - { - "profile_id": str(r[0] or ""), "parse_status": str(r[1] or ""), - "row_count": int(r[2] or 0), "raw_line_count": int(r[3] or 0), - "declared_total": int(r[4] or 0), - } - for r in rows - ] - - -def collect_quality(receipts: Sequence[Mapping[str, Any]]) -> dict[str, Any]: - statuses: dict[str, int] = {} - for r in receipts: - key = str(r.get("parse_status") or "unknown") - statuses[key] = statuses.get(key, 0) + 1 - return { - "commands": len(receipts), - "parse_status": statuses, - "row_count": sum(int(r.get("row_count") or 0) for r in receipts), - "raw_line_count": sum(int(r.get("raw_line_count") or 0) for r in receipts), - "declared_total": sum(int(r.get("declared_total") or 0) for r in receipts), - "profile_ids": sorted({str(r.get("profile_id") or "") for r in receipts if r.get("profile_id")}), - } - - -def sheet_slice(sheet: Mapping[str, Any]) -> dict[str, str]: - """``{field: value}`` the sheet pins with top-level ``eq`` row filters.""" - out: dict[str, str] = {} - for f in sheet.get("row_filters") or []: - if not isinstance(f, Mapping) or "any" in f or "all" in f: - continue - if str(f.get("op") or "eq").strip().lower() not in ("eq", "==", "ci_eq"): - continue - field = str(f.get("field") or "").strip() - value = scalar_text(f.get("value")).strip().lower() - if field and value: - out[field] = value - return out - - -def covers_slice( - receipts: Sequence[Mapping[str, Any]], wanted: Mapping[str, str], *, profile_slice: Any = None, -) -> bool: - """True when at least one successful command could have produced ``wanted``. - - A command without a declared slice (or unknown profile) covers everything. - """ - if profile_slice is None: - from .profiles import get_profile - - def profile_slice(pid: str) -> dict[str, str]: # noqa: F811 - prof = get_profile(pid) if pid else None - return dict(prof.slice) if prof is not None and prof.slice else {} - - for r in receipts: - if str(r.get("parse_status") or "") in ("failed", "skipped_custom"): - continue - declared = profile_slice(str(r.get("profile_id") or "")) - if all( - f not in declared or str(declared[f]).strip().lower() == v - for f, v in wanted.items() - ): - return True - return False - - -def classify_sheet( - *, - before: Sequence[Mapping[str, Any]] | None, - after: Sequence[Mapping[str, Any]] | None, - before_rows: int, - after_rows: int, - wanted_slice: Mapping[str, str] | None = None, -) -> tuple[str, list[str]]: - """``(data_state, reasons)`` for one sheet; reasons name the offending side. - - ``None`` receipts mean the batch carries no receipts at all (imported or - legacy data): that side cannot be judged and is trusted as-is. - """ - wanted = dict(wanted_slice or {}) - reasons: list[str] = [] - for side, receipts in (("before", before), ("after", after)): - if receipts is None: - continue - if not receipts: - reasons.append(f"{side}:no_command") - elif wanted and not covers_slice(receipts, wanted): - reasons.append(f"{side}:slice_not_collected") - if reasons: - return "not_collected", reasons - for side, receipts in (("before", before), ("after", after)): - if receipts is None: - continue - q = collect_quality(receipts) - if q["row_count"] == 0 and q["raw_line_count"] > SUSPECT_RAW_LINES: - reasons.append(f"{side}:parsed_0_of_{q['raw_line_count']}_lines") - elif q["commands"] and q["parse_status"].get("failed", 0) == q["commands"]: - reasons.append(f"{side}:all_commands_failed") - if reasons: - return "suspect_parse", reasons - if before_rows == 0 and after_rows == 0: - return "no_data", ["both_sides_empty"] - return "ok", [] - - -def assess_sheet( - db: Session, sheet: Mapping[str, Any], *, before_batch_id: str, after_batch_id: str, - before_rows: int, after_rows: int, -) -> dict[str, Any]: - """Quality block stored on a sheet summary.""" - metric_id = str(sheet.get("metric_id") or "") - before = side_receipts(db, before_batch_id, metric_id) if _has_receipts(db, before_batch_id) else None - after = side_receipts(db, after_batch_id, metric_id) if _has_receipts(db, after_batch_id) else None - state, reasons = classify_sheet( - before=before, after=after, before_rows=before_rows, after_rows=after_rows, - wanted_slice=sheet_slice(sheet), - ) - return { - "data_state": state, - "data_state_reasons": reasons, - "collect_quality": { - "before": collect_quality(before or []), "after": collect_quality(after or []), - "unknown_sides": [n for n, r in (("before", before), ("after", after)) if r is None], - }, - } - - -def command_coverage( - db: Session, *, before_batch_id: str, after_batch_id: str, -) -> dict[str, Any]: - """Collected "metric{slice}" labels per side and their difference. - - Profile ids differ per vendor (``zte.x`` vs ``huawei.x``), so the comparison - is on what a command *produces*: its metric and slice. - """ - from .profiles import get_profile - - def labels(batch_id: str) -> set[str]: - if not batch_id: - return set() - rows = db.query(BizStateBatchCommand.metric_id, BizStateBatchCommand.profile_id).filter( - BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id != "", - ).distinct().all() - out: set[str] = set() - for metric_id, profile_id in rows: - prof = get_profile(str(profile_id or "")) if profile_id else None - sl = dict(prof.slice) if prof is not None and prof.slice else {} - suffix = ",".join(f"{k}={v}" for k, v in sorted(sl.items())) - out.add(f"{metric_id}{{{suffix}}}" if suffix else str(metric_id)) - return out - - b, a = labels(before_batch_id), labels(after_batch_id) - return {"before": sorted(b), "after": sorted(a), - "only_before": sorted(b - a), "only_after": sorted(a - b)} diff --git a/netx_api/biz_state/compare_rules.py b/netx_api/biz_state/compare_rules.py index 928fcdb..97e454b 100644 --- a/netx_api/biz_state/compare_rules.py +++ b/netx_api/biz_state/compare_rules.py @@ -7,21 +7,10 @@ All metric-specific compare behavior belongs in the sheet template from __future__ import annotations import re -import math from typing import Any, Mapping, Sequence _AGE_TIMER_RE = re.compile(r"^\d{1,2}:\d{2}:\d{2}$") -NUMBER_PATTERN = r"^[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?$" - - -def scalar_text(value: Any) -> str: - """Match JSON text extraction: only null is blank; keep zero and booleans.""" - if value is None: - return "" - if isinstance(value, bool): - return "true" if value else "false" - return str(value) def _as_list(raw: Any) -> list[Any]: @@ -33,7 +22,7 @@ def _as_list(raw: Any) -> list[Any]: def _field_val(row: Mapping[str, Any], field: str) -> str: - return scalar_text((row or {}).get(field)).strip() + return str((row or {}).get(field) or "").strip() def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: @@ -46,17 +35,17 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: expect = filt.get("value") if op in ("eq", "=="): - return raw.lower() == scalar_text(expect).strip().lower() + return raw.lower() == str(expect or "").strip().lower() if op in ("ne", "!="): - return raw.lower() != scalar_text(expect).strip().lower() + return raw.lower() != str(expect or "").strip().lower() if op == "in": - opts = {scalar_text(x).strip().lower() for x in _as_list(expect) if scalar_text(x).strip()} + opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()} return raw.lower() in opts if op in ("not_in", "nin"): - opts = {scalar_text(x).strip().lower() for x in _as_list(expect) if scalar_text(x).strip()} + opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()} return raw.lower() not in opts if op == "contains": - needle = scalar_text(expect).strip().lower() + needle = str(expect or "").strip().lower() return bool(needle) and needle in raw.lower() if op == "empty": return not raw @@ -74,7 +63,7 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: # HH:MM:SS dynamic ARP age return bool(_AGE_TIMER_RE.match(raw)) if op == "ci_eq": - return raw.lower() == scalar_text(expect).strip().lower() + return raw.lower() == str(expect or "").strip().lower() return True @@ -110,7 +99,7 @@ def apply_row_filters( def normalize_value(value: Any, how: str) -> str: - text = scalar_text(value).strip() + text = str(value if value is not None else "").strip() mode = str(how or "").strip().lower() if not mode or mode in ("none", "strip"): return text @@ -137,8 +126,6 @@ def field_rule_map(rules: Sequence[Mapping[str, Any]] | None) -> dict[str, dict[ name = str(raw.get("field") or "").strip() if not name: continue - if str(raw.get("side") or "").strip(): - continue # side-scoped rules feed the identity pipeline, not the compare out[name] = dict(raw) return out @@ -203,59 +190,12 @@ def effective_display_fields( def _parse_float(text: str) -> float | None: - if not re.fullmatch(NUMBER_PATTERN, text): - return None try: - value = float(text) - return value if math.isfinite(value) else None + return float(text) if text else 0.0 except ValueError: return None -DIRECTIONAL_MODES = ("not_worse", "numeric_min", "numeric_max") - - -def _rank_index(value: str, rank: Sequence[Any]) -> int | None: - wanted = value.strip().lower() - for i, item in enumerate(rank or []): - if scalar_text(item).strip().lower() == wanted: - return i - return None - - -def _tolerance(rule: Mapping[str, Any]) -> float: - try: - return float(rule.get("tolerance") or 0) - except (TypeError, ValueError): - return 0.0 - - -def _directional_verdict(bv: str, av: str, rule: Mapping[str, Any]) -> tuple[bool, str]: - """``(equal_or_better, reason)`` for not_worse / numeric_min / numeric_max.""" - mode = str(rule.get("compare") or "").strip().lower() - if mode == "not_worse": - rank = rule.get("rank") or [] - bi, ai = _rank_index(bv, rank), _rank_index(av, rank) - if bi is None or ai is None: - return bv == av, "neq" - return ai <= bi, (f"worse: {bv}→{av}" if ai > bi else "") - bn, an = _parse_float(bv), _parse_float(av) - if bn is None or an is None: - return bv == av, "neq" - tol = _tolerance(rule) - if mode == "numeric_min": - floor = bn * (1 - tol / 100.0) - if an >= floor: - return True, "" - drop = (bn - an) / abs(bn) * 100.0 if bn else 0.0 - return False, f"drop {drop:.0f}% > {tol:g}%" - ceil = bn * (1 + tol / 100.0) - if an <= ceil: - return True, "" - rise = (an - bn) / abs(bn) * 100.0 if bn else 0.0 - return False, f"rise {rise:.0f}% > {tol:g}%" - - def explain_diff( before: Any, after: Any, @@ -268,8 +208,6 @@ def explain_diff( bv = normalize_value(before, norm) av = normalize_value(after, norm) mode = str(rule.get("compare") or "eq").strip().lower() or "eq" - if mode in DIRECTIONAL_MODES: - return _directional_verdict(bv, av, rule)[1] if mode in ("percent", "pct", "rel"): bn = _parse_float(bv) an = _parse_float(av) @@ -310,8 +248,6 @@ def values_equal( mode = str(rule.get("compare") or "eq").strip().lower() or "eq" if mode in ("ignore", "skip", "off"): return True - if mode in DIRECTIONAL_MODES: - return _directional_verdict(bv, av, rule)[0] if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"): bn = _parse_float(bv) an = _parse_float(av) diff --git a/netx_api/biz_state/compare_service.py b/netx_api/biz_state/compare_service.py index d54cb35..195e752 100644 --- a/netx_api/biz_state/compare_service.py +++ b/netx_api/biz_state/compare_service.py @@ -6,24 +6,19 @@ import io import json import logging import threading -import time import zipfile -from copy import deepcopy from datetime import datetime -from typing import Any, Callable, Iterable, Iterator +from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import String, and_, cast, func, or_, tuple_ -from sqlalchemy.orm import Session, aliased +from sqlalchemy.orm import Session from ..models import ( BizCompareDiff, BizCompareJob, BizCompareRun, BizCompareTemplate, - BizMigrationProject, - BizMonitorTemplate, BizPortMapping, BizPortMappingRow, BizStateBatch, @@ -32,33 +27,23 @@ from ..models import ( BizStateTask, ) from ..timeutil import utcnow_naive -from .compare_engine import apply_port_map, compare_rows, mapping_stats, row_key +from .compare_engine import compare_rows, mapping_stats from .compare_rules import ( ROW_FILTER_PRESETS, apply_row_filters, - scalar_text, arp_dynamic_row_filters, effective_compare_fields, effective_display_fields, - row_matches_filter, ) from .iface_normalize import ( apply_iface_normalize_rows, default_zte_iface_normalize_rules, normalize_iface_rules, - normalize_port_map, ) -from .compare_quality import UNJUDGED_STATES, assess_sheet -from .identity_pipeline import Pipeline, build_pipeline, legacy_pipeline from .profiles import metric_field_map -from .compare_validation import validate_template_body _log = logging.getLogger("netx.biz_state.compare") -# v1: port-map keys/values must be written post-normalization. -# v2: port-map keys/values are normalized with the template iface rules (A1). -CONFIG_SNAPSHOT_VERSION = 2 - # Per-job mutex so auto + manual run_compare do not dual-write the same job. _auto_compare_locks: dict[str, threading.Lock] = {} _auto_compare_locks_guard = threading.Lock() @@ -103,7 +88,7 @@ def batch_metric_collect_ok(db: Session, batch_id: str, metric_id: str) -> bool: if status == "success": return True cmds = ( - db.query(BizStateBatchCommand.parse_status, BizStateBatchCommand.row_count) + db.query(BizStateBatchCommand) .filter( BizStateBatchCommand.batch_id == bid, BizStateBatchCommand.metric_id == mid, @@ -196,93 +181,10 @@ def _compare_side( } _DIFF_CHUNK = 2000 -_LOAD_YIELD_PER = 5000 _SEARCH_TEXT_MAX = 4000 -# Heartbeat while bulk-inserting large fail/ok diff sets (vpnv4-scale). -_PERSIST_PROGRESS_EVERY = 10_000 -# Above this, store fail diffs as key + row_id + changes (hydrate sides on read). -_FAIL_COMPACT_MIN = 50_000 -# Live search (kw): load at most this many matching rows per side, return ≤ this many pairs. -_LIVE_SEARCH_LOAD_CAP = 2_000 -_LIVE_SEARCH_RESULT_CAP = 200 -_LIVE_SEARCH_GROUP_LOAD_CAP = 10_000 -# Success-row persist policy (see resolve_unchanged_policy) -_STORE_UNCHANGED_MODES = frozenset({"auto", "always", "never", "sample", "keys"}) -_UNCHANGED_FULL_MAX = 20_000 -_UNCHANGED_SAMPLE_MAX = 5_000 -def normalize_store_unchanged(raw: Any) -> str: - m = str(raw or "auto").strip().lower() - return m if m in _STORE_UNCHANGED_MODES else "auto" - - -def compute_pass_rate(*, success: int, fail: int, empty_as: float = 100.0) -> float: - """Pass rate for cutover UI. - - With million-row sheets, 3 fails / 50k can round to 100.0% at 1 decimal — - never report 100 when any fail remains; add decimals until it stays < 100. - """ - s = max(0, int(success or 0)) - f = max(0, int(fail or 0)) - judged = s + f - if judged <= 0: - return float(empty_as) - rate = (s / judged) * 100.0 - if f <= 0: - return round(rate, 1) - for nd in (1, 2, 3, 4): - r = round(rate, nd) - if r < 100.0: - return r - return 99.9999 - - -def resolve_unchanged_policy( - mode: str, *, before_n: int, after_n: int -) -> dict[str, Any]: - """Decide whether / how many success rows to emit for one sheet. - - - always: full before/after for every match (slow on million-row sheets) - - never: count only - - sample: key + row_id sample (cap) — cutover spot-check default for large sheets - - keys: all success as key + row_id (full browse; write still heavy) - - auto: full when sheet is small; otherwise sample compact - """ - m = normalize_store_unchanged(mode) - n = max(int(before_n or 0), int(after_n or 0)) - if m == "never": - return {"mode": m, "include": False, "limit": None, "compact": False} - if m == "always": - return {"mode": m, "include": True, "limit": None, "compact": False} - if m == "keys": - return {"mode": m, "include": True, "limit": None, "compact": True} - if m == "sample": - return { - "mode": m, - "include": True, - "limit": _UNCHANGED_SAMPLE_MAX, - "compact": True, - } - # auto — cutover-oriented: large sheets sample, not full key dump - if n <= _UNCHANGED_FULL_MAX: - return {"mode": m, "include": True, "limit": None, "compact": False} - return { - "mode": m, - "include": True, - "limit": _UNCHANGED_SAMPLE_MAX, - "compact": True, - } - - -def _strip_netx(row: Any) -> dict[str, Any]: - """Drop collector provenance before persisting compare payloads.""" - if not isinstance(row, dict): - return {} - return {k: v for k, v in row.items() if k != "_netx"} - - -def _diff_search_text(d: dict[str, Any], *, limit: int | None = _SEARCH_TEXT_MAX) -> str: +def _diff_search_text(d: dict[str, Any]) -> str: parts = [str(d.get("kind") or "")] for key in ("key", "before", "after", "mapped_before", "changes"): val = d.get(key) @@ -291,8 +193,7 @@ def _diff_search_text(d: dict[str, Any], *, limit: int | None = _SEARCH_TEXT_MAX parts.append(json.dumps(val, ensure_ascii=False, default=str, separators=(",", ":"))) except Exception: parts.append(str(val)) - value = " ".join(parts) - return value if limit is None else value[:limit] + return " ".join(parts)[:_SEARCH_TEXT_MAX] def _top_changed_fields(diffs: list[dict[str, Any]], *, limit: int = 8) -> list[dict[str, Any]]: @@ -314,167 +215,30 @@ def _persist_sheet_diffs( run_id: str, metric_id: str, diffs: list[dict[str, Any]], - seq_start: int = 0, - on_progress: Callable[[int, int], None] | None = None, -) -> int: - """Bulk-insert diff rows already selected by the engine policy. - - Compact success rows carry key + before/after_row_id; JSON sides stay empty - and are hydrated from metric tables on read. - - ``on_progress(written, total)`` fires periodically so UI elapsed time moves - during multi-minute inserts (e.g. large vpnv4 fail sets). - """ +) -> None: + """Bulk-insert diff rows; avoids embedding million-row arrays in summary_json.""" buf: list[dict[str, Any]] = [] - seq = int(seq_start or 0) - written = 0 - total = len(diffs) - last_prog = 0 - last_prog_t = time.monotonic() - - def _maybe_prog(force: bool = False) -> None: - nonlocal last_prog, last_prog_t - if not on_progress: - return - now = time.monotonic() - if ( - not force - and written - last_prog < _PERSIST_PROGRESS_EVERY - and now - last_prog_t < 2.0 - ): - return - last_prog = written - last_prog_t = now - try: - on_progress(written, total) - except Exception: - _log.exception("persist progress callback failed run=%s metric=%s", run_id, metric_id) - - for d in diffs: - kind = str(d.get("kind") or "") - before = _strip_netx(d.get("before")) - after = _strip_netx(d.get("after")) - mapped = _strip_netx(d.get("mapped_before")) - payload = { - "kind": kind, - "key": dict(d.get("key") or {}), - "before": before, - "after": after, - "mapped_before": mapped, - "changes": dict(d.get("changes") or {}), - } - # Compact rows: search_text = kind + key (+ changes) only — no fat sides - search_src = ( - { - "kind": kind, - "key": payload["key"], - **({"changes": payload["changes"]} if payload["changes"] else {}), - } - if bool(d.get("compact")) - else payload - ) + for i, d in enumerate(diffs): buf.append( { "id": uuid4().hex, "run_id": run_id, "metric_id": metric_id, - "seq": seq, - "kind": kind, - "key_json": payload["key"], - "before_json": before, - "after_json": after, - "mapped_before_json": mapped, - "changes_json": payload["changes"], - "before_row_id": str(d.get("before_row_id") or "")[:64], - "after_row_id": str(d.get("after_row_id") or "")[:64], - "search_text": _diff_search_text(search_src), + "seq": i, + "kind": str(d.get("kind") or ""), + "key_json": dict(d.get("key") or {}), + "before_json": dict(d.get("before") or {}), + "after_json": dict(d.get("after") or {}), + "mapped_before_json": dict(d.get("mapped_before") or {}), + "changes_json": dict(d.get("changes") or {}), + "search_text": _diff_search_text(d), } ) - seq += 1 - written += 1 if len(buf) >= _DIFF_CHUNK: db.bulk_insert_mappings(BizCompareDiff, buf) buf.clear() - # Commit chunks so progress/UI can see mid-write fail rows and - # elapsed_ms advances (otherwise persisting_* looks frozen). - db.commit() - _maybe_prog() if buf: db.bulk_insert_mappings(BizCompareDiff, buf) - db.commit() - _maybe_prog(force=True) - return written - - -def _metric_rows_by_ids(db: Session, ids: list[str]) -> dict[str, dict[str, Any]]: - """Load BizStateMetricRow / LLDP rows by primary key for hydrate.""" - clean = [str(i).strip() for i in ids if str(i or "").strip()] - if not clean: - return {} - from ..models import BizStateMetricRow - - out: dict[str, dict[str, Any]] = {} - # Chunk IN lists for large pages - for i in range(0, len(clean), 500): - chunk = clean[i : i + 500] - for r in db.query(BizStateMetricRow).filter(BizStateMetricRow.id.in_(chunk)).all(): - out[str(r.id)] = dict(r.data_json or {}) - missing = [x for x in chunk if x not in out] - if missing: - for n in ( - db.query(BizStateLldpNeighbor) - .filter(BizStateLldpNeighbor.id.in_(missing)) - .all() - ): - out[str(n.id)] = { - "local_if": n.local_if, - "remote_sys": n.remote_sys, - "remote_if": n.remote_if, - "remote_ip": n.remote_ip, - "protocol": n.protocol, - } - return out - - -def _attach_raw_key_before(items: list[dict[str, Any]], key_fields: list[str]) -> list[dict[str, Any]]: - """``raw_key_before``: the before-side key as printed, only where it differs from the match key.""" - for it in items: - before = it.get("before") - if not before or it.get("raw_key_before"): - continue - shown = it.get("key") or {} - raw = {f: scalar_text(before.get(f)).strip() for f in key_fields} - if any(raw[f] != scalar_text(shown.get(f)).strip() for f in key_fields if f in shown): - it["raw_key_before"] = raw - return items - - -def _hydrate_diff_rows( - db: Session, items: list[dict[str, Any]], *, - run: BizCompareRun | None = None, sheet: dict[str, Any] | None = None, -) -> list[dict[str, Any]]: - """Fill empty before/after from metric tables when row_ids are present.""" - key_fields = list((sheet or {}).get("key_fields") or []) - need: list[str] = [] - for it in items: - if not it.get("before") and it.get("before_row_id"): - need.append(str(it["before_row_id"])) - if not it.get("after") and it.get("after_row_id"): - need.append(str(it["after_row_id"])) - if not need: - return _attach_raw_key_before(items, key_fields) - by_id = _metric_rows_by_ids(db, need) - pipeline = _run_pipeline(db, run, sheet or {}) - for it in items: - brid = str(it.get("before_row_id") or "") - arid = str(it.get("after_row_id") or "") - if not it.get("before") and brid and brid in by_id: - it["before"] = by_id[brid] - if not it.get("after") and arid and arid in by_id: - it["after"] = pipeline.normalize(by_id[arid], "after") - if not it.get("mapped_before") and it.get("before"): - it["mapped_before"] = pipeline.transform(it["before"], "before") - return _attach_raw_key_before(items, key_fields) def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]: @@ -485,8 +249,6 @@ def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]: "after": r.after_json or {}, "mapped_before": r.mapped_before_json or {}, "changes": r.changes_json or {}, - "before_row_id": getattr(r, "before_row_id", "") or "", - "after_row_id": getattr(r, "after_row_id", "") or "", } @@ -502,7 +264,6 @@ def _filter_inline_diffs( *, kind: str, kw: str, - field_q: dict[str, str] | None = None, ) -> list[dict[str, Any]]: kind_n = (kind or "diff").strip().lower() kw_n = (kw or "").strip().lower() @@ -516,70 +277,17 @@ def _filter_inline_diffs( elif kind_n != "all" and dk != kind_n: continue if kw_n: - blob = _diff_search_text(d, limit=None).lower() + blob = _diff_search_text(d).lower() if kw_n not in blob: continue - if field_q and not _diff_matches_search(d, kw="", field_q=field_q): - continue out.append(d) return out -def _diff_matches_search(d: dict[str, Any], *, kw: str, field_q: dict[str, str]) -> bool: - """Search a completed pair, never filter its two sides before pairing.""" - if kw and kw.lower() not in _diff_search_text(d, limit=None).lower(): - return False - return all( - any( - val.lower() in str((d.get(side) or {}).get(name, "")).lower() - for side in ("key", "before", "mapped_before", "after") - ) - for name, val in field_q.items() - ) - - -def _search_stored_diffs(q: Any, *, metric_id: str, kw: str, field_q: dict[str, str]) -> Any: - """Filter persisted verdicts in SQL; join source rows for compact payloads.""" - from ..models import BizStateMetricRow - - sides = [BizCompareDiff.key_json, BizCompareDiff.before_json, - BizCompareDiff.mapped_before_json, BizCompareDiff.after_json] - sources = [] - for row_id in (BizCompareDiff.before_row_id, BizCompareDiff.after_row_id): - model = BizStateLldpNeighbor if metric_id == "lldp_neighbor" else BizStateMetricRow - src = aliased(model) - q = q.outerjoin(src, src.id == row_id) - sources.append(src) - if kw: - expressions = [BizCompareDiff.search_text, *[cast(s, String) for s in sides]] - for src in sources: - if metric_id == "lldp_neighbor": - expressions.extend(getattr(src, f) for f in - ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")) - else: - expressions.append(cast(src.data_json, String)) - q = q.filter(or_(*[func.lower(e).contains(kw.lower(), autoescape=True) for e in expressions])) - for name, val in field_q.items(): - expressions = [s[name].as_string() for s in sides] - for src in sources: - if metric_id == "lldp_neighbor": - if name in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol"): - expressions.append(getattr(src, name)) - else: - expressions.append(src.data_json[name].as_string()) - q = q.filter(or_(*[func.lower(cast(e, String)).contains(val.lower(), autoescape=True) - for e in expressions])) - return q - - def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: Any) -> list[dict[str, Any]]: sheets = list(summary.get("sheets") or []) if sheets: return sheets - # Running / empty: do NOT invent a fake first-metric sheet (was showing - # isis 100% with 0→0 while BGP was still loading). - if str(run.status or "") in ("running", "queued"): - return [] return [ { "metric_id": run.metric_id, @@ -589,7 +297,6 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A "iface_fields": list((tpl.iface_fields if tpl else None) or []), "compare_fields": list((tpl.compare_fields if tpl else None) or []), "mode": "fields", - "status": "done", "summary": { k: summary.get(k, 0) for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count") @@ -599,62 +306,6 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A ] -def _order_sheets_small_first( - db: Session, - sheets: list[dict[str, Any]], - *, - before_batch_id: str, - after_batch_id: str, -) -> list[dict[str, Any]]: - """Run smaller metrics first so field engineers can review early results.""" - if len(sheets) <= 1: - return list(sheets) - # Split sheets share metrics. Aggregate all counters once, without raw CLI blobs. - counts: dict[str, int] = {} - for bid, mid, total in db.query( - BizStateBatchCommand.batch_id, BizStateBatchCommand.metric_id, - func.sum(BizStateBatchCommand.row_count), - ).filter( - BizStateBatchCommand.batch_id.in_([before_batch_id, after_batch_id]), - BizStateBatchCommand.metric_id.in_({str(s.get("metric_id") or "") for s in sheets}), - ).group_by(BizStateBatchCommand.batch_id, BizStateBatchCommand.metric_id): - counts[mid] = max(counts.get(mid, 0), int(total or 0)) - scored: list[tuple[int, int, dict[str, Any]]] = [] - for i, sheet in enumerate(sheets): - n = counts.get(str(sheet.get("metric_id") or ""), 0) - scored.append((n, i, sheet)) - scored.sort(key=lambda x: (x[0], x[1])) - return [s for _, _, s in scored] - - -def _pending_sheet_meta(sheet: dict[str, Any]) -> dict[str, Any]: - """Placeholder meta so the UI lists all check items while a run is in flight.""" - key_fields = list(sheet.get("key_fields") or []) - compare_fields = list(sheet.get("compare_fields") or []) - return { - "sheet_id": sheet_key(sheet), - "title": sheet_title(sheet), - "metric_id": sheet.get("metric_id") or "", - "key_fields": key_fields, - "iface_fields": list(sheet.get("iface_fields") or []), - "compare_fields": compare_fields, - "display_fields": list(sheet.get("display_fields") or []), - "field_rules": list(sheet.get("field_rules") or []), - "row_filters": list(sheet.get("row_filters") or []), - "ignore_port_changes": sheet.get("ignore_port_changes"), - "mode": "presence" if not compare_fields else "fields", - "status": "pending", - "summary": { - "added": 0, - "removed": 0, - "changed": 0, - "unchanged": 0, - "before_count": 0, - "after_count": 0, - }, - } - - def _str_list(raw: Any) -> list[str]: return [str(x).strip() for x in (raw or []) if str(x).strip()] @@ -669,41 +320,18 @@ def _normalize_row_filters(raw: Any) -> list[dict[str, Any]]: return out -# Optional sheet-level switches that must survive normalization. -_SHEET_OPTION_VALUES: dict[str, tuple[str, ...]] = { - "unmapped_port_policy": ("by_name", "ignore_port"), - "duplicate_pairing": ("sorted", "ordered"), -} - - -def _sheet_options(raw: dict[str, Any]) -> dict[str, Any]: - out: dict[str, Any] = {} - for key, allowed in _SHEET_OPTION_VALUES.items(): - val = str(raw.get(key) or "").strip().lower() - if val in allowed: - out[key] = val - if raw.get("builtin_vendor_canonical") is False: - out["builtin_vendor_canonical"] = False - if raw.get("scope_by_port_map") is not None: - out["scope_by_port_map"] = bool(raw.get("scope_by_port_map")) - return out - - def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]: if not isinstance(raw, list): return [] out: list[dict[str, Any]] = [] - seen: set[tuple[str, str]] = set() + seen: set[str] = set() for item in raw: if not isinstance(item, dict): continue name = str(item.get("field") or "").strip() - side = str(item.get("side") or "").strip().lower() - side = side if side in ("before", "after") else "" - # A field may carry one rule per side (e.g. a different value_map each). - if not name or (name, side) in seen: + if not name or name in seen: continue - seen.add((name, side)) + seen.add(name) rule: dict[str, Any] = {"field": name} compare = str(item.get("compare") or "").strip().lower() if compare: @@ -719,16 +347,6 @@ def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]: rule["tolerance"] = float(item.get("tolerance")) except (TypeError, ValueError): pass - vmap = item.get("value_map") - if isinstance(vmap, dict) and vmap: - rule["value_map"] = {str(k): str(v) for k, v in vmap.items()} - if side: - rule["side"] = side - if isinstance(item.get("apply_to_key"), bool): - rule["apply_to_key"] = item["apply_to_key"] - rank = item.get("rank") - if isinstance(rank, list) and rank: - rule["rank"] = [str(x) for x in rank] # Drop empty rules (only field name) if len(rule) > 1: out.append(rule) @@ -758,7 +376,6 @@ def _sheet_def( row_filters: list[dict[str, Any]] | None = None, field_rules: list[dict[str, Any]] | None = None, ignore_port_changes: bool | None = None, - options: dict[str, Any] | None = None, ) -> dict[str, Any]: mid = str(metric_id or "").strip() sid = str(sheet_id or "").strip() or mid @@ -799,7 +416,6 @@ def _sheet_def( } if ignore_port_changes is not None: sheet["ignore_port_changes"] = bool(ignore_port_changes) - sheet.update(options or {}) return sheet @@ -935,35 +551,7 @@ def _builtin_source_splits() -> dict[str, list[dict[str, Any]]]: } -def _packaged_zte_status_template_path(): - from pathlib import Path - - return Path(__file__).resolve().parent / "data" / "default_zte_status_template.json" - - -def _load_packaged_zte_status_template() -> dict[str, Any]: - """IOH CN migration sheet set shipped as the built-in status default.""" - path = _packaged_zte_status_template_path() - if not path.is_file(): - return {} - try: - return dict(json.loads(path.read_text(encoding="utf-8")) or {}) - except Exception: - _log.exception("failed to load packaged ZTE status template %s", path) - return {} - - def _default_zte_status_sheets() -> list[dict[str, Any]]: - """Built-in status sheets — prefer packaged IOH CN migration rules.""" - raw = _load_packaged_zte_status_template() - out: list[dict[str, Any]] = [] - for item in list(raw.get("metrics") or []): - sheet = _normalize_sheet(item) - if sheet: - out.append(sheet) - if out: - return out - # Fallback if package missing (tests / incomplete install) return [ *_isis_af_sheets(), _default_sheet_for_metric("interface_brief", compare_roles=("state",)), @@ -980,66 +568,6 @@ def _default_zte_status_sheets() -> list[dict[str, Any]]: ] -def _builtin_status_needs_packaged_upgrade(existing: list[dict[str, Any]]) -> bool: - """True when built-in template still lacks filtered BGP route sheets.""" - mids = {str(s.get("metric_id") or "") for s in existing} - if "bgp_route" not in mids and "l2vpn_mac" not in mids: - return True - has_filtered_route = any( - str(s.get("metric_id") or "") == "bgp_route" and list(s.get("row_filters") or []) - for s in existing - ) - if not has_filtered_route: - return True - packaged_keys = {sheet_key(s) for s in _default_zte_status_sheets()} - have_keys = {sheet_key(s) for s in existing} - # Missing several packaged sheet ids → sync to packaged default - return len(packaged_keys - have_keys) >= 3 - - -# A7: (metric_id, kind, field, old packaged value, new value). A stored sheet is -# only touched when its value still equals the *old* packaged one, so anything -# an operator edited survives the upgrade. -_INTERFACE_DETAIL_OLD_COMPARE = [ - "admin", "input_bps", "output_bps", "port_status", "in_util", "out_util", - "ip_mtu", "mtu", "mpls_mtu", "ipv6_mtu", "port_media", "negotiation", -] -_BUILTIN_STATUS_VALUE_UPGRADES: tuple[tuple[str, str, str, Any, Any], ...] = ( - ("interface_detail", "rule", "output_bps", - {"compare": "numeric", "tolerance": 5}, {"compare": "percent", "tolerance": 5}), - ("interface_detail", "list", "compare_fields", _INTERFACE_DETAIL_OLD_COMPARE, - [f for f in _INTERFACE_DETAIL_OLD_COMPARE if f not in ("input_bps", "output_bps", "in_util", "out_util")]), - ("optical_brief", "rule", "rx_power", - {"compare": "percent", "tolerance": 5}, {"compare": "numeric", "tolerance": 2}), - ("l2vpn_mac", "list", "iface_fields", [], ["ac_port"]), - # ARP: filter on the canonical ``entry_type`` (every vendor parser emits it); the - # age-timer clause stays only as a fallback for rows parsed before entry_type existed. - ("arp", "list", "row_filters", [{"op": "age_timer", "field": "age", "value": ""}], arp_dynamic_row_filters()), -) - - -def _upgrade_builtin_status_values(sheet: dict[str, Any]) -> tuple[dict[str, Any], bool]: - """Apply ``_BUILTIN_STATUS_VALUE_UPGRADES`` to one stored sheet.""" - mid = str(sheet.get("metric_id") or "") - out = deepcopy(sheet) - changed = False - for metric_id, kind, field, old, new in _BUILTIN_STATUS_VALUE_UPGRADES: - if mid != metric_id: - continue - if kind == "list": - if list(out.get(field) or []) == old: - out[field] = list(new) - changed = True - continue - for rule in out.get("field_rules") or []: - if isinstance(rule, dict) and rule.get("field") == field and all( - rule.get(k) == v for k, v in old.items() - ): - rule.update(new) - changed = True - return out, changed - - def _default_zte_config_sheets() -> list[dict[str, Any]]: """Config-intent metrics for cutover / intent-vs-intent compare.""" return [ @@ -1103,7 +631,6 @@ def _normalize_sheet(raw: Any) -> dict[str, Any] | None: row_filters=_normalize_row_filters(raw.get("row_filters")), field_rules=rules, ignore_port_changes=ignore_ports, - options=_sheet_options(raw), ) @@ -1209,7 +736,6 @@ def _parse_metrics_body(body: dict[str, Any]) -> list[dict[str, Any]]: display_fields=disp_arg, row_filters=_normalize_row_filters(body.get("row_filters")), field_rules=rules, - ignore_port_changes=body.get("ignore_port_changes"), ) ] @@ -1308,65 +834,52 @@ def ensure_default_lldp_template(db: Session) -> BizCompareTemplate: def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: name = "ZTE status default" row = db.query(BizCompareTemplate).filter(BizCompareTemplate.name == name).one_or_none() - packaged = _load_packaged_zte_status_template() sheets = _default_zte_status_sheets() - note = str( - packaged.get("note") - or "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP route AF sheets)" - )[:512] - iface_rules = list(packaged.get("iface_normalize_rules") or []) + # Volatile counters must not stay in compare_fields on upgraded installs. + _STRIP_COMPARE: dict[str, frozenset[str]] = { + "interface_detail": frozenset({"input_bps", "output_bps", "in_util", "out_util"}), + "optical_brief": frozenset({"rx_power", "tx_power"}), + "bgp_peer": frozenset({"pfx_rcd"}), + } if row: existing = template_metrics(row) - if _builtin_status_needs_packaged_upgrade(existing) and sheets: - _apply_sheets_to_row(row, sheets) - row.note = note - row.updated_at = _utcnow() - if iface_rules: - _set_template_iface_normalize(row, iface_rules) - elif not template_iface_normalize(row): - # Packaged IOH rules use empty normalize; leave empty when explicit - _set_template_iface_normalize(row, []) - db.commit() - db.refresh(row) - return row - # Incremental patches for already-upgraded installs - changed = False + want = {s["metric_id"] for s in sheets} + have = {s["metric_id"] for s in existing} + changed = bool(want - have) upgraded: list[dict[str, Any]] = [] - by_sid = {sheet_key(s): s for s in sheets} + by_want = {s["metric_id"]: s for s in sheets} for s in existing: cur = dict(s) mid = str(cur.get("metric_id") or "") - sid = sheet_key(cur) if mid == "arp" and not cur.get("row_filters"): - src = by_sid.get(sid) or next( - (x for x in sheets if x.get("metric_id") == "arp"), None - ) - cur["row_filters"] = list( - (src or {}).get("row_filters") or arp_dynamic_row_filters() - ) - if not cur.get("field_rules") and src and src.get("field_rules"): - cur["field_rules"] = list(src["field_rules"]) + cur["row_filters"] = list(by_want.get("arp", {}).get("row_filters") or arp_dynamic_row_filters()) + if not cur.get("field_rules") and by_want.get("arp", {}).get("field_rules"): + cur["field_rules"] = list(by_want["arp"]["field_rules"]) changed = True - cur, value_changed = _upgrade_builtin_status_values(cur) - changed = changed or value_changed + strip = _STRIP_COMPARE.get(mid) + if strip: + old_cmp = list(cur.get("compare_fields") or []) + new_cmp = [f for f in old_cmp if f not in strip] + if new_cmp != old_cmp: + cur["compare_fields"] = new_cmp + want_disp = list((by_want.get(mid) or {}).get("display_fields") or []) + if want_disp: + cur["display_fields"] = want_disp + changed = True upgraded.append(_normalize_sheet(cur) or cur) - have_mids = {str(s.get("metric_id") or "") for s in upgraded} - for s in sheets: - if str(s.get("metric_id") or "") not in have_mids: - # Only append wholly missing metrics (e.g. bgp_route family) - if str(s.get("metric_id") or "") == "bgp_route" and "bgp_route" not in have_mids: - upgraded.extend( - [x for x in sheets if x.get("metric_id") == "bgp_route"] - ) - have_mids.add("bgp_route") - changed = True - elif str(s.get("metric_id") or "") not in have_mids: + if want - have: + for s in sheets: + if s["metric_id"] not in have: upgraded.append(s) - have_mids.add(str(s.get("metric_id") or "")) - changed = True + changed = True if changed: _apply_sheets_to_row(row, upgraded if upgraded else sheets) - row.note = note + row.note = "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)" + row.updated_at = _utcnow() + db.commit() + db.refresh(row) + if not template_iface_normalize(row): + _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) row.updated_at = _utcnow() db.commit() db.refresh(row) @@ -1374,12 +887,12 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: row = BizCompareTemplate( id=uuid4().hex, name=name, - note=note, + note="Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)", created_at=_utcnow(), updated_at=_utcnow(), ) _apply_sheets_to_row(row, sheets) - _set_template_iface_normalize(row, iface_rules) + _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) db.add(row) db.commit() db.refresh(row) @@ -1435,10 +948,10 @@ def ensure_default_templates(db: Session) -> None: def upgrade_builtin_split_sheets(db: Session) -> None: - """Upgrade built-in ZTE status template to packaged AF / BGP route sheets. + """Split unfiltered whole-table sheets on the built-in ZTE template only. - Custom templates are left alone. Built-in is replaced wholesale when it - still lacks filtered ``bgp_route`` sheets (IOH CN migration default). + A sheet is replaced when its id is still the source metric and it has no + row filters. Custom templates and already-split sheets are left alone. """ row = ( db.query(BizCompareTemplate) @@ -1448,18 +961,6 @@ def upgrade_builtin_split_sheets(db: Session) -> None: if not row: return existing = template_metrics(row) - packaged_sheets = _default_zte_status_sheets() - if _builtin_status_needs_packaged_upgrade(existing) and packaged_sheets: - packaged = _load_packaged_zte_status_template() - _apply_sheets_to_row(row, packaged_sheets) - row.note = str( - packaged.get("note") - or "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP route AF sheets)" - )[:512] - _set_template_iface_normalize(row, list(packaged.get("iface_normalize_rules") or [])) - row.updated_at = _utcnow() - db.commit() - return splits = _builtin_source_splits() out: list[dict[str, Any]] = [] changed = False @@ -1485,17 +986,7 @@ def list_templates(db: Session) -> list[dict[str, Any]]: ensure_default_templates(db) upgrade_builtin_split_sheets(db) rows = db.query(BizCompareTemplate).order_by(BizCompareTemplate.name.asc()).all() - usage = dict(db.query(BizCompareJob.template_id, func.count(BizCompareJob.id)).group_by(BizCompareJob.template_id).all()) - monitor_usage = dict( - db.query(BizMonitorTemplate.compare_template_id, func.count(BizMonitorTemplate.id)) - .filter(BizMonitorTemplate.compare_template_id != "") - .group_by(BizMonitorTemplate.compare_template_id).all() - ) - return [ - {**_template_out(t), "job_count": usage.get(t.id, 0), - "monitor_template_count": monitor_usage.get(t.id, 0)} - for t in rows - ] + return [_template_out(t) for t in rows] def list_metric_schemas() -> list[dict[str, Any]]: @@ -1532,7 +1023,6 @@ def list_row_filter_presets() -> list[dict[str, Any]]: def create_template(db: Session, body: dict[str, Any]) -> dict[str, Any]: - validate_template_body(body) sheets = _parse_metrics_body(body) t = BizCompareTemplate( id=uuid4().hex, @@ -1555,7 +1045,6 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict t = db.get(BizCompareTemplate, template_id) if not t: raise HTTPException(status_code=404, detail="template_not_found") - validate_template_body(body, partial=True) if "name" in body: t.name = str(body.get("name") or "")[:256] if "note" in body: @@ -1577,7 +1066,6 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict "display_fields", "row_filters", "field_rules", - "ignore_port_changes", ) ): # Prefer explicit metrics; otherwise merge into current sheets from legacy keys @@ -1615,8 +1103,6 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict first["row_filters"] = _normalize_row_filters(body.get("row_filters")) if "field_rules" in body: first["field_rules"] = _normalize_field_rules(body.get("field_rules")) - if "ignore_port_changes" in body: - first["ignore_port_changes"] = body["ignore_port_changes"] sheets[0] = _normalize_sheet(first) or first _apply_sheets_to_row(t, sheets) t.updated_at = _utcnow() @@ -1628,19 +1114,6 @@ def delete_template(db: Session, template_id: str) -> None: t = db.get(BizCompareTemplate, template_id) if not t: raise HTTPException(status_code=404, detail="template_not_found") - jobs = [n or i for i, n in db.query(BizCompareJob.id, BizCompareJob.name) - .filter(BizCompareJob.template_id == template_id).all()] - monitors = [n or i for i, n in db.query(BizMonitorTemplate.id, BizMonitorTemplate.name) - .filter(BizMonitorTemplate.compare_template_id == template_id).all()] - if jobs or monitors: - raise HTTPException(status_code=409, detail={ - "code": "template_in_use", - "message": "template_in_use: " + "; ".join(filter(None, [ - f"jobs={', '.join(jobs)}" if jobs else "", - f"monitor_templates={', '.join(monitors)}" if monitors else "", - ])), - "jobs": jobs, "monitor_templates": monitors, - }) db.delete(t) db.commit() @@ -1661,30 +1134,9 @@ def _mapping_out(db: Session, m: BizPortMapping) -> dict[str, Any]: } -def _mapping_references(db: Session, mapping_id: str) -> tuple[list[str], list[str]]: - """(compare job names, cutover project names) that reference this mapping.""" - jobs = [n or i for i, n in db.query(BizCompareJob.id, BizCompareJob.name) - .filter(BizCompareJob.mapping_id == mapping_id).all()] - projects = [n or i for i, n in db.query(BizMigrationProject.id, BizMigrationProject.name) - .filter(BizMigrationProject.mapping_id == mapping_id).all()] - return jobs, projects - - def list_mappings(db: Session) -> list[dict[str, Any]]: rows = db.query(BizPortMapping).order_by(BizPortMapping.name.asc()).all() - job_counts = dict( - db.query(BizCompareJob.mapping_id, func.count(BizCompareJob.id)) - .filter(BizCompareJob.mapping_id != "").group_by(BizCompareJob.mapping_id).all() - ) - project_counts = dict( - db.query(BizMigrationProject.mapping_id, func.count(BizMigrationProject.id)) - .filter(BizMigrationProject.mapping_id != "").group_by(BizMigrationProject.mapping_id).all() - ) - return [ - {**_mapping_out(db, m), "job_count": job_counts.get(m.id, 0), - "project_count": project_counts.get(m.id, 0)} - for m in rows - ] + return [_mapping_out(db, m) for m in rows] def create_mapping(db: Session, body: dict[str, Any]) -> dict[str, Any]: @@ -1742,16 +1194,6 @@ def delete_mapping(db: Session, mapping_id: str) -> None: m = db.get(BizPortMapping, mapping_id) if not m: raise HTTPException(status_code=404, detail="mapping_not_found") - jobs, projects = _mapping_references(db, mapping_id) - if jobs or projects: - raise HTTPException(status_code=409, detail={ - "code": "mapping_in_use", - "message": "mapping_in_use: " + "; ".join(filter(None, [ - f"jobs={', '.join(jobs)}" if jobs else "", - f"projects={', '.join(projects)}" if projects else "", - ])), - "jobs": jobs, "projects": projects, - }) db.query(BizPortMappingRow).filter(BizPortMappingRow.mapping_id == mapping_id).delete() db.delete(m) db.commit() @@ -1764,191 +1206,61 @@ def _port_map_dict(db: Session, mapping_id: str) -> dict[str, str]: return {str(r.before_if): str(r.after_if) for r in rows if r.before_if and r.after_if} -def _load_metric_rows( - db: Session, - *, - batch_id: str, - metric_id: str, - on_chunk: Callable[[int], None] | None = None, - row_filters: list[dict[str, Any]] | None = None, -) -> list[dict[str, Any]]: - """Load metric rows in keyset chunks (stable on million-row sheets). - - When ``row_filters`` are SQL-pushdown-safe on PostgreSQL, they are applied in - the SELECT (critical for BGP afi/vrf sheet splits — avoids loading 1M+ then - discarding). Otherwise filters are applied in Python after each chunk. - """ - from .compare_sql import ( - _dialect_is_postgres, - _filters_sql_compatible, - compile_row_filters_sql, - ) - - filters = [f for f in (row_filters or []) if isinstance(f, dict)] - pushdown = bool( - filters and _dialect_is_postgres(db) and _filters_sql_compatible(filters) - ) - filter_sql, filter_params = ("TRUE", {}) - if pushdown: - filter_sql, filter_params = compile_row_filters_sql(filters) - +def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dict[str, Any]]: if metric_id == "lldp_neighbor": - out: list[dict[str, Any]] = [] - last_id = "" - while True: - q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id) - if last_id: - q = q.filter(BizStateLldpNeighbor.id > last_id) - chunk = q.order_by(BizStateLldpNeighbor.id.asc()).limit(_LOAD_YIELD_PER).all() - if not chunk: - break - for n in chunk: - row = { - "local_if": n.local_if, - "remote_sys": n.remote_sys, - "remote_if": n.remote_if, - "remote_ip": n.remote_ip, - "protocol": n.protocol, - "_netx": { - "batch_id": batch_id, - "batch_command_id": n.batch_command_id or "", - "task_id": n.task_id or "", - "ne_id": n.ne_id or "", - "collected_at": n.collected_at.isoformat() + "Z" - if n.collected_at - else None, - "row_id": n.id, - }, - } - if filters and not pushdown and not all( - row_matches_filter(row, f) for f in filters - ): - db.expunge(n) - continue - out.append(row) - db.expunge(n) - last_id = str(chunk[-1].id) - if on_chunk: - on_chunk(len(out)) - if len(chunk) < _LOAD_YIELD_PER: - break - if filters and not pushdown: - return apply_row_filters(out, filters) - return out - - # Generic tabular metrics — PG + pushdown uses SQL keyset with JSON filters - from ..models import BizStateMetricRow - from sqlalchemy import text as sql_text - - out: list[dict[str, Any]] = [] - last_seq = -1 - last_id = "" - while True: - if pushdown: - params = { - "bid": batch_id, - "mid": metric_id, - "last_seq": last_seq, - "last_id": last_id, - "lim": int(_LOAD_YIELD_PER), - **filter_params, + rows = ( + db.query(BizStateLldpNeighbor) + .filter(BizStateLldpNeighbor.batch_id == batch_id) + .all() + ) + return [ + { + "local_if": n.local_if, + "remote_sys": n.remote_sys, + "remote_if": n.remote_if, + "remote_ip": n.remote_ip, + "protocol": n.protocol, + "_netx": { + "batch_id": batch_id, + "batch_command_id": n.batch_command_id or "", + "task_id": n.task_id or "", + "ne_id": n.ne_id or "", + "collected_at": n.collected_at.isoformat() + "Z" if n.collected_at else None, + "row_id": n.id, + }, } - keyset = ( - "(seq > :last_seq OR (seq = :last_seq AND id > :last_id))" - if last_id - else "TRUE" - ) - rows = db.execute( - sql_text( - f""" - SELECT id, batch_command_id, task_id, ne_id, seq, data_json, collected_at - FROM biz_state_metric_row - WHERE batch_id = :bid - AND metric_id = :mid - AND ({filter_sql}) - AND ({keyset}) - ORDER BY seq ASC, id ASC - LIMIT :lim - """ - ), - params, - ).mappings().all() - if not rows: - break - for r in rows: - data = dict(r["data_json"] or {}) - collected = r["collected_at"] - out.append( - { - **data, - "_netx": { - "batch_id": batch_id, - "batch_command_id": str(r["batch_command_id"] or ""), - "task_id": str(r["task_id"] or ""), - "ne_id": str(r["ne_id"] or ""), - "collected_at": collected.isoformat() + "Z" - if collected is not None - else None, - "row_id": str(r["id"]), - }, - } - ) - last_seq = int(rows[-1]["seq"] or 0) - last_id = str(rows[-1]["id"]) - if on_chunk: - on_chunk(len(out)) - if len(rows) < _LOAD_YIELD_PER: - break - continue + for n in rows + ] + # Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …) + from ..models import BizStateMetricRow - q = db.query(BizStateMetricRow).filter( + rows = ( + db.query(BizStateMetricRow) + .filter( BizStateMetricRow.batch_id == batch_id, BizStateMetricRow.metric_id == metric_id, ) - if last_id: - q = q.filter( - or_( - BizStateMetricRow.seq > last_seq, - and_( - BizStateMetricRow.seq == last_seq, - BizStateMetricRow.id > last_id, - ), - ) - ) - chunk = ( - q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) - .limit(_LOAD_YIELD_PER) - .all() - ) - if not chunk: - break - for r in chunk: - row = { + .order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) + .all() + ) + if rows: + # Raw rows only — filtering belongs to the compare sheet template + # (``row_filters``), not metric-specific branches here. + # ``_netx`` is collector provenance (stripped before field compare). + return [ + { **dict(r.data_json or {}), "_netx": { "batch_id": batch_id, "batch_command_id": r.batch_command_id or "", "task_id": r.task_id or "", "ne_id": r.ne_id or "", - "collected_at": r.collected_at.isoformat() + "Z" - if r.collected_at - else None, + "collected_at": r.collected_at.isoformat() + "Z" if r.collected_at else None, "row_id": r.id, }, } - if filters and not all(row_matches_filter(row, f) for f in filters): - db.expunge(r) - continue - out.append(row) - db.expunge(r) - last_seq = int(chunk[-1].seq or 0) - last_id = str(chunk[-1].id) - if on_chunk: - on_chunk(len(out)) - if len(chunk) < _LOAD_YIELD_PER: - break - if out: - return out + for r in rows + ] # Known metric with zero rows is OK; unknown metric still errors if metric_id in metric_field_map(): return [] @@ -1970,11 +1282,11 @@ def validate_mapping( sheets = template_metrics(tpl) if not sheets: raise HTTPException(status_code=400, detail="template_has_no_metrics") + pmap = _port_map_dict(db, mapping_id) # Validate against first sheet that has iface fields (or first sheet) target = next((s for s in sheets if s.get("iface_fields")), sheets[0]) iface_fields = list(target.get("iface_fields") or []) norm = template_iface_normalize(tpl) - pmap, _ = normalize_port_map(_port_map_dict(db, mapping_id), norm) before = apply_iface_normalize_rows( _load_metric_rows(db, batch_id=before_batch_id, metric_id=target["metric_id"]), iface_fields=iface_fields, @@ -1993,66 +1305,6 @@ def validate_mapping( ) -def _mapping_rows_to_dict(rows: list[Any]) -> tuple[dict[str, str], list[dict[str, str]]]: - """Editor rows → raw map; a repeated before port is reported, first one wins.""" - out: dict[str, str] = {} - dups: list[dict[str, str]] = [] - for r in rows or []: - raw = r if isinstance(r, dict) else dict(r) - before = str(raw.get("before_if") or "").strip() - after = str(raw.get("after_if") or "").strip() - if not before or not after: - continue - if before in out: - dups.append({"before_if": before, "kept_to": out[before], "dropped_to": after}) - continue - out[before] = after - return out, dups - - -def preflight_compare(db: Session, body: dict[str, Any]) -> dict[str, Any]: - """Mapping coverage + data-quality report for a template and two batches.""" - from .compare_preflight import preflight - - ensure_default_templates(db) - template_id = str(body.get("template_id") or "") - tpl = db.get(BizCompareTemplate, template_id) if template_id else ensure_default_cutover_template(db) - if not tpl: - raise HTTPException(status_code=404, detail="template_not_found") - before_id = str(body.get("before_batch_id") or "") - after_id = str(body.get("after_batch_id") or "") - if not db.get(BizStateBatch, before_id) or not db.get(BizStateBatch, after_id): - raise HTTPException(status_code=404, detail="batch_not_found") - sheets = _filter_enabled_sheets(template_metrics(tpl), _str_list(body.get("enabled_sheet_ids"))) - if not sheets: - raise HTTPException(status_code=400, detail="no_enabled_sheets") - dups: list[dict[str, str]] = [] - if body.get("mapping_rows") is not None: - raw_map, dups = _mapping_rows_to_dict(list(body.get("mapping_rows") or [])) - else: - raw_map = _port_map_dict(db, str(body.get("mapping_id") or "")) - cache: dict[tuple[str, str], list[dict[str, Any]]] = {} - - def load(side: str, sheet: dict[str, Any]) -> list[dict[str, Any]]: - batch_id = before_id if side == "before" else after_id - key = (batch_id, str(sheet["metric_id"])) - if key not in cache: - cache[key] = _load_metric_rows(db, batch_id=batch_id, metric_id=key[1]) - return cache[key] - - report = preflight( - db, sheets=sheets, iface_rules=template_iface_normalize(tpl), raw_port_map=raw_map, - before_batch_id=before_id, after_batch_id=after_id, - before_vendor=_batch_vendor_key(db, before_id), after_vendor=_batch_vendor_key(db, after_id), - load_rows=load, - ) - if dups: - report["mapping"]["duplicate_before"] = dups - report["risk_reasons"] = sorted({*report["risk_reasons"], "duplicate_before"}) - report["verdict"] = "risk" - return report - - def _job_out(j: BizCompareJob) -> dict[str, Any]: return { "id": j.id, @@ -2066,7 +1318,6 @@ def _job_out(j: BizCompareJob) -> dict[str, Any]: "mode": j.mode, "status": j.status, "enabled_sheet_ids": _str_list(getattr(j, "enabled_sheet_ids", None)), - "store_unchanged": normalize_store_unchanged(getattr(j, "store_unchanged", None)), "note": j.note, "updated_at": j.updated_at.isoformat() + "Z" if j.updated_at else None, } @@ -2087,16 +1338,6 @@ def list_jobs(db: Session) -> list[dict[str, Any]]: return [_job_out(j) for j in rows] -def _validate_job_sheets(db: Session, template_id: str, enabled: list[str]) -> None: - tpl = db.get(BizCompareTemplate, template_id) - if not tpl: - raise HTTPException(status_code=404, detail="template_not_found") - sheets = {sheet_key(s) for s in template_metrics(tpl)} - unknown = sorted(set(enabled) - sheets) - if unknown: - raise HTTPException(status_code=400, detail={"error": "unknown_enabled_sheets", "sheet_ids": unknown}) - - def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: ensure_default_templates(db) template_id = str(body.get("template_id") or "").strip() @@ -2107,7 +1348,6 @@ def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: if not db.get(BizCompareTemplate, template_id): raise HTTPException(status_code=404, detail="template_not_found") enabled = _str_list(body.get("enabled_sheet_ids")) - _validate_job_sheets(db, template_id, enabled) j = BizCompareJob( id=uuid4().hex, name=str(body.get("name") or "compare")[:256], @@ -2120,7 +1360,6 @@ def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: mode=str(body.get("mode") or "manual")[:16], status="ready", enabled_sheet_ids=enabled, - store_unchanged=normalize_store_unchanged(body.get("store_unchanged")), note=str(body.get("note") or "")[:512], created_at=_utcnow(), updated_at=_utcnow(), @@ -2136,8 +1375,6 @@ def update_job(db: Session, job_id: str, body: dict[str, Any]) -> dict[str, Any] j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") - _validate_job_sheets(db, str(body.get("template_id", j.template_id) or ""), - _str_list(body.get("enabled_sheet_ids", j.enabled_sheet_ids))) for key in ( "name", "template_id", @@ -2154,8 +1391,6 @@ def update_job(db: Session, job_id: str, body: dict[str, Any]) -> dict[str, Any] setattr(j, key, str(body.get(key) or "")) if "enabled_sheet_ids" in body: j.enabled_sheet_ids = _str_list(body.get("enabled_sheet_ids")) - if "store_unchanged" in body: - j.store_unchanged = normalize_store_unchanged(body.get("store_unchanged")) j.updated_at = _utcnow() db.commit() return _job_out(j) @@ -2165,9 +1400,6 @@ def delete_job(db: Session, job_id: str) -> None: j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") - if db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id, - BizCompareRun.status.in_(("queued", "running"))).first(): - raise HTTPException(status_code=409, detail="compare_running") run_ids = [ rid for (rid,) in db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id).all() ] @@ -2185,8 +1417,6 @@ def delete_run(db: Session, run_id: str) -> dict[str, Any]: r = db.get(BizCompareRun, run_id) if not r: raise HTTPException(status_code=404, detail="run_not_found") - if r.status in ("queued", "running"): - raise HTTPException(status_code=409, detail="compare_running") job_id = str(r.job_id or "") db.query(BizCompareDiff).filter(BizCompareDiff.run_id == run_id).delete( synchronize_session=False @@ -2214,59 +1444,14 @@ def _resolve_after_batch(db: Session, job: BizCompareJob) -> str: return str(latest.id) if latest else "" -def _sheet_result_envelope( - sheet: dict[str, Any], - *, - key_fields: list[str], - iface_fields: list[str], - compare_fields: list[str], - display_fields: list[str], - row_filters: list[Any], - field_rules: list[Any], - ignore_ports: bool | None, - mode: str, - summary: dict[str, Any], - diffs: list[Any], - mapping_stats: dict[str, Any], -) -> dict[str, Any]: - return { - "sheet_id": sheet_key(sheet), - "title": sheet_title(sheet), - "metric_id": sheet["metric_id"], - "key_fields": key_fields, - "iface_fields": iface_fields, - "compare_fields": compare_fields, - "display_fields": display_fields, - "row_filters": row_filters, - "field_rules": field_rules, - "ignore_port_changes": ignore_ports, - "mode": mode, - "status": "done", - "summary": summary, - "diffs": diffs, - "mapping_stats": mapping_stats, - } - - -def _sheet_quality( - db: Session, sheet: dict[str, Any], before_batch_id: str, after_batch_id: str, - summary: dict[str, Any], -) -> dict[str, Any]: - return assess_sheet( - db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id, - before_rows=int(summary.get("before_count") or 0), after_rows=int(summary.get("after_count") or 0), - ) - - def _run_sheet( db: Session, *, sheet: dict[str, Any], before_batch_id: str, after_batch_id: str, - pipeline: Pipeline, - store_unchanged: str = "auto", - on_load_progress: Callable[..., None] | None = None, + port_map: dict[str, str], + iface_normalize_rules: list[dict[str, str]] | None = None, ) -> dict[str, Any]: key_fields = list(sheet.get("key_fields") or []) iface_fields = list(sheet.get("iface_fields") or []) @@ -2287,472 +1472,79 @@ def _run_sheet( ignore_ports = sheet.get("ignore_port_changes") if ignore_ports is not None: ignore_ports = bool(ignore_ports) - mid = sheet["metric_id"] - - def _emit_load(side: str, n: int, **meta: Any) -> None: - if not on_load_progress: - return - try: - on_load_progress(side, n, **meta) - except TypeError: - on_load_progress(side, n) - - # PostgreSQL path: pushdown-safe sheets join in-DB (BGP-scale). - from .compare_sql import SqlCompareSkip, run_sql_sheet_compare, sql_compare_skip_reason - - skip_reason = sql_compare_skip_reason(db, sheet, pipeline=pipeline) - if not skip_reason: - try: - result = run_sql_sheet_compare( - db, - sheet=sheet, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - store_unchanged=store_unchanged, - on_progress=_emit_load, - duplicate_pairing=pipeline.duplicate_pairing, - ) - summary = dict(result["summary"]) - # Engine already sets raw counts / policy; keep keys stable - if "unchanged_policy" not in summary: - summary["unchanged_policy"] = resolve_unchanged_policy( - store_unchanged, - before_n=int(summary.get("before_count") or 0), - after_n=int(summary.get("after_count") or 0), - ) - summary.update(_sheet_quality(db, sheet, before_batch_id, after_batch_id, summary)) - return _sheet_result_envelope( - sheet, - key_fields=key_fields, - iface_fields=iface_fields, - compare_fields=compare_fields, - display_fields=display_fields, - row_filters=row_filters, - field_rules=field_rules, - ignore_ports=ignore_ports, - mode=mode, - summary=summary, - diffs=list(result.get("diffs") or []), - mapping_stats=dict(result.get("mapping_stats") or {}), - ) - except SqlCompareSkip as skip: - skip_reason = skip.reason or "skip" - _log.info( - "python compare sheet=%s metric=%s skip_sql=%s", - sheet_key(sheet), - mid, - skip_reason, - ) - try: - db.rollback() - except Exception: - pass - except Exception: - _log.exception( - "sql compare fallback sheet=%s metric=%s — using Python engine", - sheet_key(sheet), - mid, - ) - skip_reason = "sql_error_fallback" - # Roll back aborted SQL transaction so Python path can use the session - try: - db.rollback() - except Exception: - pass - else: - _log.info( - "python compare sheet=%s metric=%s skip_sql=%s", - sheet_key(sheet), - mid, - skip_reason, - ) - - _emit_load("before", 0, engine="python", note=skip_reason or "python", phase="loading") - - def _before_chunk(n: int) -> None: - _emit_load("before", n, engine="python", note=skip_reason or "python", phase="loading") - - def _after_chunk(n: int) -> None: - _emit_load("after", n, engine="python", note=skip_reason or "python", phase="loading") - - # Only filters that read untransformed fields may run in SQL on the raw rows; - # the pipeline applies every filter (post-normalize for v2) in ``prepare``. - before_raw = _load_metric_rows( - db, - batch_id=before_batch_id, - metric_id=mid, - on_chunk=_before_chunk, - row_filters=pipeline.pushdown_filters("before"), - ) - after_raw = _load_metric_rows( - db, - batch_id=after_batch_id, - metric_id=mid, - on_chunk=_after_chunk, - row_filters=pipeline.pushdown_filters("after"), - ) - before_pairs = pipeline.prepare(before_raw, "before") - after_pairs = pipeline.prepare(after_raw, "after") - policy = resolve_unchanged_policy( - store_unchanged, before_n=len(before_pairs), after_n=len(after_pairs) - ) + before_raw = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"]) + after_raw = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"]) + before_rows = apply_row_filters(before_raw, row_filters) + after_rows = apply_row_filters(after_raw, row_filters) result = compare_rows( - before_rows=[], after_rows=[], + before_rows=before_rows, + after_rows=after_rows, key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields, + port_map=port_map, field_rules=field_rules, - include_unchanged=bool(policy["include"]), - unchanged_limit=policy.get("limit"), - compact_unchanged=bool(policy.get("compact")), - pipeline=pipeline, - prepared=(before_pairs, after_pairs), + iface_normalize_rules=iface_normalize_rules, + ignore_port_changes=ignore_ports, ) summary = dict(result["summary"]) summary["before_raw_count"] = len(before_raw) summary["after_raw_count"] = len(after_raw) summary["row_filters"] = len(row_filters) - summary["unchanged_policy"] = policy - summary.setdefault("engine", "python") - if pipeline.port_map and pipeline.iface_fields: - from .compare_preflight import sheet_mapping_coverage - - coverage = sheet_mapping_coverage(pipeline, before_pairs, after_pairs) - coverage.pop("_after_ifaces", None) - coverage.pop("_used_keys", None) - summary["mapping_coverage"] = coverage - summary.update(_sheet_quality(db, sheet, before_batch_id, after_batch_id, summary)) - return _sheet_result_envelope( - sheet, - key_fields=key_fields, - iface_fields=iface_fields, - compare_fields=compare_fields, - display_fields=display_fields, - row_filters=row_filters, - field_rules=field_rules, - ignore_ports=ignore_ports, - mode=mode, - summary=summary, - diffs=list(result.get("diffs") or []), - mapping_stats=dict(result.get("mapping_stats") or {}), - ) + return { + "sheet_id": sheet_key(sheet), + "title": sheet_title(sheet), + "metric_id": sheet["metric_id"], + "key_fields": key_fields, + "iface_fields": iface_fields, + "compare_fields": compare_fields, + "display_fields": display_fields, + "row_filters": row_filters, + "field_rules": field_rules, + "ignore_port_changes": ignore_ports, + "mode": mode, + "summary": summary, + "diffs": result["diffs"], + "mapping_stats": result["mapping_stats"], + } -def _validate_compare_job( +def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]: + lock = _job_compare_lock(job_id) + if not lock.acquire(blocking=False): + raise HTTPException(status_code=409, detail="compare_already_running") + try: + return _run_compare_unlocked(db, job_id, force_after_batch_id=force_after_batch_id) + finally: + lock.release() + + +def _run_compare_unlocked( db: Session, job_id: str, *, force_after_batch_id: str = "" ) -> dict[str, Any]: - """Resolve job/template/batches/sheets; raises HTTPException on bad input.""" j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") tpl = db.get(BizCompareTemplate, j.template_id) if not tpl: raise HTTPException(status_code=404, detail="template_not_found") - if j.mapping_id and not db.get(BizPortMapping, j.mapping_id): - raise HTTPException(status_code=400, detail="mapping_missing") before_batch_id = str(j.before_batch_id or "") after_batch_id = str(force_after_batch_id or "").strip() or _resolve_after_batch(db, j) if not before_batch_id or not after_batch_id: raise HTTPException(status_code=400, detail="before_and_after_batch_required") - source_batches = [db.get(BizStateBatch, bid) for bid in (before_batch_id, after_batch_id)] - if not all(source_batches): + if not db.get(BizStateBatch, before_batch_id) or not db.get(BizStateBatch, after_batch_id): raise HTTPException(status_code=404, detail="batch_not_found") - if any(b.status != "success" for b in source_batches): - raise HTTPException(status_code=409, detail="source_batch_not_complete") - # Existing records may predate write validation. Do not normalize invalid - # sheets away or run unknown rules as successful comparisons. - if tpl.metrics_json: - validate_template_body({"metrics": tpl.metrics_json, - "iface_normalize_rules": tpl.iface_normalize_json}) sheets_cfg = template_metrics(tpl) if not sheets_cfg: raise HTTPException(status_code=400, detail="template_has_no_metrics") - _validate_job_sheets(db, tpl.id, _str_list(j.enabled_sheet_ids)) sheets_cfg = _filter_enabled_sheets(sheets_cfg, getattr(j, "enabled_sheet_ids", None)) if not sheets_cfg: raise HTTPException(status_code=400, detail="no_enabled_sheets") - sheets_cfg = _order_sheets_small_first( - db, - sheets_cfg, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - ) - return { - "job": j, - "template": tpl, - "before_batch_id": before_batch_id, - "after_batch_id": after_batch_id, - "sheets_cfg": sheets_cfg, - } - -def _create_running_run( - db: Session, - *, - job: BizCompareJob, - tpl: BizCompareTemplate, - before_batch_id: str, - after_batch_id: str, - sheets_cfg: list[dict[str, Any]], - purpose: str = "manual", -) -> BizCompareRun: - first_metric = str(sheets_cfg[0].get("metric_id") or "") - pending_sheets = [_pending_sheet_meta(s) for s in sheets_cfg] - mapping = db.get(BizPortMapping, job.mapping_id) if job.mapping_id else None - vendors = { - "before": _batch_vendor_key(db, before_batch_id), - "after": _batch_vendor_key(db, after_batch_id), - } - raw_port_map = _port_map_dict(db, str(job.mapping_id or "")) - iface_rules = template_iface_normalize(tpl) - identity = { - sheet_key(sh): build_pipeline( - sh, iface_rules=iface_rules, port_map=raw_port_map, - before_vendor=vendors["before"], after_vendor=vendors["after"], - ).describe() - for sh in sheets_cfg - } - config_snapshot = deepcopy({ - "version": CONFIG_SNAPSHOT_VERSION, - "template": _template_out(tpl), - "sheets": sheets_cfg, - "port_map": raw_port_map, - "vendors": vendors, - "identity": identity, - "store_unchanged": normalize_store_unchanged(getattr(job, "store_unchanged", None)), - "job_name": job.name or "", - "before_task_id": job.before_task_id or "", - "after_task_id": job.after_task_id or "", - "mapping_name": (mapping.name if mapping else "") or "", - }) - run = BizCompareRun( - id=uuid4().hex, - job_id=job.id, - template_id=tpl.id, - mapping_id=job.mapping_id, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - metric_id=first_metric, - status="running", - summary_json={ - "purpose": purpose, - "config_snapshot": config_snapshot, - "progress": { - "phase": "queued", - "sheet_index": 0, - "sheet_total": len(sheets_cfg), - "sheet_id": "", - "sheet_title": "", - "elapsed_ms": 0, - }, - "sheet_count": len(sheets_cfg), - "added": 0, - "removed": 0, - "changed": 0, - "unchanged": 0, - "duplicate": 0, - "before_count": 0, - "after_count": 0, - "sheets": pending_sheets, - }, - diffs_json=[], - mapping_stats_json={}, - message="queued", - created_at=_utcnow(), - ) - db.add(run) - db.commit() - db.refresh(run) - return run - - -def _run_config_snapshot(run: BizCompareRun) -> dict[str, Any] | None: - snapshot = (run.summary_json or {}).get("config_snapshot") - if isinstance(snapshot, dict) and snapshot.get("version") in (1, 2): - return snapshot - return None - - -def _run_transform_config( - db: Session, run: BizCompareRun, -) -> tuple[dict[str, str], list[dict[str, str]]]: - snapshot = _run_config_snapshot(run) - if snapshot is not None: - rules = normalize_iface_rules( - (snapshot.get("template") or {}).get("iface_normalize_rules") or [], - ) - pmap = dict(snapshot.get("port_map") or {}) - if int(snapshot.get("version") or 1) >= 2: - # v2 runs: map keys/values are written as the device prints them. - pmap, _ = normalize_port_map(pmap, rules) - return pmap, rules - # Legacy runs did not record these settings; retain their fallback behavior. - tpl = db.get(BizCompareTemplate, run.template_id) if run.template_id else None - return _port_map_dict(db, str(run.mapping_id or "")), template_iface_normalize(tpl) - - -def _batch_vendor_key(db: Session, batch_id: str) -> str: - """Vendor key (``huawei``/``zte``/…) for a batch, '' when unknown.""" - from ..topology_lldp import resolve_vendor_key - - batch = db.get(BizStateBatch, batch_id) if batch_id else None - if batch is None: - return "" - task = db.get(BizStateTask, batch.task_id) if batch.task_id else None - key = resolve_vendor_key( - batch.vendor or (task.vendor if task else ""), (task.device_type if task else "") or "", - ) - return "" if key == "generic" else key - - -def _run_pipeline(db: Session, run: BizCompareRun | None, sheet: dict[str, Any]) -> Pipeline: - """The identity pipeline a run used (or would use) for one sheet. - - Snapshot v2 runs get the full pipeline; v1 and snapshot-less runs get the - legacy one so stored results replay exactly as they were produced. - """ - sid = sheet_key(sheet) - snapshot = _run_config_snapshot(run) if run is not None else None - if snapshot is None: - pmap, rules = _run_transform_config(db, run) if run is not None else ({}, []) - return legacy_pipeline( - key_fields=sheet.get("key_fields") or [], iface_fields=sheet.get("iface_fields") or [], - compare_fields=sheet.get("compare_fields") or [], field_rules=sheet.get("field_rules"), - port_map=pmap, iface_normalize_rules=rules, row_filters=sheet.get("row_filters"), - ignore_port_changes=sheet.get("ignore_port_changes"), - ) - authoritative = next( - (sh for sh in snapshot.get("sheets") or [] if isinstance(sh, dict) and sheet_key(sh) == sid), None, - ) - cfg = authoritative or sheet - version = int(snapshot.get("version") or 1) - vendors = snapshot.get("vendors") or {} - return build_pipeline( - cfg, - iface_rules=(snapshot.get("template") or {}).get("iface_normalize_rules") or [], - port_map=dict(snapshot.get("port_map") or {}), - before_vendor=str(vendors.get("before") or ""), - after_vendor=str(vendors.get("after") or ""), - legacy=version < 2, - normalize_map=version >= 2, - ) - - -def _set_run_progress( - db: Session, - run: BizCompareRun, - *, - phase: str, - sheet_index: int, - sheet_total: int, - sheet: dict[str, Any] | None, - started_mono: float, - extra: dict[str, Any] | None = None, - detach: bool = False, -) -> None: - """Update run progress. - - ``detach=True`` writes via a fresh session so SQL compare can keep an open - transaction (TEMP CTAS) without mid-flight commits on the worker ``db``. - """ - elapsed_ms = int((time.monotonic() - started_mono) * 1000) - progress = { - "phase": phase, - "sheet_index": sheet_index, - "sheet_total": sheet_total, - "sheet_id": sheet_key(sheet) if sheet else "", - "sheet_title": sheet_title(sheet) if sheet else "", - "elapsed_ms": elapsed_ms, - } - if extra: - progress.update(extra) - title = progress["sheet_title"] or progress["sheet_id"] or "" - message = ( - f"{phase} {sheet_index}/{sheet_total}" - + (f" · {title}" if title else "") - + f" · {elapsed_ms // 1000}s" - )[:1024] - - if detach: - from ..db import SessionLocal - - s = SessionLocal() - try: - r = s.get(BizCompareRun, str(run.id)) - if not r: - return - prev = dict(r.summary_json or {}) - prev["progress"] = progress - r.summary_json = prev - if str(r.status or "") != "cancelled": - r.status = "running" - r.message = message - s.commit() - # Mirror into worker instance for later in-memory reads (do not commit db) - prev_w = dict(run.summary_json or {}) - prev_w["progress"] = progress - run.summary_json = prev_w - if str(run.status or "") != "cancelled": - run.message = message - finally: - s.close() - return - - prev = dict(run.summary_json or {}) - prev["progress"] = progress - run.summary_json = prev - # Re-read status from DB — cancel may have been committed by another session - # (UI cancel / startup recovery) while this worker still holds a stale "running". - db.expire(run, ["status", "message"]) - if str(run.status or "") != "cancelled": - run.status = "running" - run.message = message - db.commit() - - -def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]: - """Run compare into an existing ``running`` BizCompareRun; persist sheet-by-sheet.""" - run = db.get(BizCompareRun, run_id) - if not run: - raise HTTPException(status_code=404, detail="run_not_found") - snapshot = _run_config_snapshot(run) - j = db.get(BizCompareJob, run.job_id) - if not j: - run.status = "failed" - run.message = "job_not_found" - db.commit() - raise HTTPException(status_code=404, detail="job_not_found") - tpl = db.get(BizCompareTemplate, run.template_id) if snapshot is None else None - if not tpl and snapshot is None: - run.status = "failed" - run.message = "template_not_found" - db.commit() - raise HTTPException(status_code=404, detail="template_not_found") - - before_batch_id = str(run.before_batch_id or "") - after_batch_id = str(run.after_batch_id or "") - if snapshot is not None: - sheets_cfg = deepcopy(snapshot.get("sheets") or []) - else: - sheets_cfg = _filter_enabled_sheets(template_metrics(tpl), getattr(j, "enabled_sheet_ids", None)) - if not sheets_cfg: - run.status = "failed" - run.message = "no_enabled_sheets" - db.commit() - raise HTTPException(status_code=400, detail="no_enabled_sheets") - if snapshot is None: - sheets_cfg = _order_sheets_small_first( - db, sheets_cfg, before_batch_id=before_batch_id, after_batch_id=after_batch_id, - ) - - started_mono = time.monotonic() - store_mode = normalize_store_unchanged( - snapshot.get("store_unchanged") if snapshot is not None else getattr(j, "store_unchanged", None), - ) - unchanged_listed_total = 0 - unchanged_truncated_any = False - unchanged_compact_any = False + pmap = _port_map_dict(db, j.mapping_id) + norm_rules = template_iface_normalize(tpl) + sheet_results: list[dict[str, Any]] = [] agg = { "before_count": 0, "after_count": 0, @@ -2763,488 +1555,87 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]: "duplicate": 0, } mapping_by_metric: dict[str, Any] = {} + for sheet in sheets_cfg: + one = _run_sheet( + db, + sheet=sheet, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + port_map=pmap, + iface_normalize_rules=norm_rules, + ) + sheet_results.append(one) + s = one["summary"] + for k in agg: + agg[k] += int(s.get(k) or 0) + mapping_by_metric[sheet_key(one)] = one["mapping_stats"] + + first = sheet_results[0] field_counts: dict[str, int] = {} - total = len(sheets_cfg) + for s in sheet_results: + for d in list(s.get("diffs") or []): + if str(d.get("kind") or "") != "changed": + continue + for fname in d.get("changes") or {}: + field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1 + top_fields = sorted( + [{"field": k, "count": v} for k, v in field_counts.items()], + key=lambda x: (-int(x["count"]), str(x["field"])), + )[:8] - # Prefer seeded pending sheets from create; realign to small-first order - prev_metas = list((run.summary_json or {}).get("sheets") or []) - by_key = {sheet_key(m): m for m in prev_metas} - sheet_metas = [ - by_key.get(sheet_key(s)) or _pending_sheet_meta(s) for s in sheets_cfg - ] - - def _publish_sheets() -> None: - prev = dict(run.summary_json or {}) - prev["sheets"] = list(sheet_metas) - prev.update({k: agg[k] for k in agg}) - run.summary_json = prev - db.expire(run, ["status"]) - # Never resurrect cancelled while publishing incremental sheet metas - if str(run.status or "") == "cancelled": - db.commit() - return - db.commit() - - try: - for idx, sheet in enumerate(sheets_cfg, start=1): - if _run_is_cancelled(db, run_id): - run = db.get(BizCompareRun, run_id) or run - return get_run(db, run.id) - sid = sheet_key(sheet) - # Mark current sheet running in the sidebar list - for meta in sheet_metas: - if sheet_key(meta) == sid: - meta["status"] = "running" - break - _publish_sheets() - - _load_pub = {"t": 0.0, "n": -1, "engine": ""} - - def _on_load( - side: str, - n: int, - *, - engine: str = "python", - note: str = "", - phase: str | None = None, - _idx: int = idx, - _sheet: dict = sheet, - ) -> None: - now = time.monotonic() - eng = str(engine or "python") - # SQL emits sparse updates; Python still throttle chunk spam - if eng != "sql": - if n - _load_pub["n"] < 25_000 and now - _load_pub["t"] < 2.0: - return - _load_pub["t"] = now - _load_pub["n"] = n - _load_pub["engine"] = eng - extra: dict[str, Any] = { - "load_side": side, - "rows_loaded": n, - "engine": eng, - } - if note: - extra["engine_note"] = str(note)[:128] - # SQL path: detach progress commits so TEMP CTAS stays in one txn - _set_run_progress( - db, - run, - phase=str(phase or ("loading" if eng != "sql" else "sql_count")), - sheet_index=_idx, - sheet_total=total, - sheet=_sheet, - started_mono=started_mono, - extra=extra, - detach=(eng == "sql"), - ) - - _set_run_progress( - db, - run, - phase="loading", - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={"engine": "", "engine_note": ""}, - ) - one = _run_sheet( - db, - sheet=sheet, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - pipeline=_run_pipeline(db, run, sheet), - store_unchanged=store_mode, - on_load_progress=_on_load, - ) - s = one["summary"] - listed = int(s.get("unchanged_listed") or 0) - unchanged_listed_total += listed - if s.get("unchanged_truncated"): - unchanged_truncated_any = True - if s.get("unchanged_compact"): - unchanged_compact_any = True - _set_run_progress( - db, - run, - phase="comparing", - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={ - "before_count": int(s.get("before_count") or 0), - "after_count": int(s.get("after_count") or 0), - "diff_rows": int(s.get("added") or 0) - + int(s.get("removed") or 0) - + int(s.get("changed") or 0) - + int(s.get("duplicate") or 0) - + listed, - }, - ) - diffs = list(one.get("diffs") or []) - for d in diffs: - if str(d.get("kind") or "") != "changed": - continue - for fname in d.get("changes") or {}: - field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1 - # Cutover-first: persist fails so the UI can open the fail tab ASAP, - # then write success slim keys (sample / keys mode). - fail_diffs = [d for d in diffs if str(d.get("kind") or "") != "unchanged"] - ok_diffs = [d for d in diffs if str(d.get("kind") or "") == "unchanged"] - mid = sheet_key(one) - # Million-row vpnv4 with many diffs: keep key/row_id/changes only. - if len(fail_diffs) >= _FAIL_COMPACT_MIN: - for d in fail_diffs: - d["before"] = {} - d["after"] = {} - d["mapped_before"] = {} - d["compact"] = True - - def _on_persist( - written: int, - total_n: int, - *, - phase: str, - kind_key: str, - ) -> None: - _set_run_progress( - db, - run, - phase=phase, - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={ - kind_key: total_n, - "persisted": written, - "persist_total": total_n, - }, - # Separate session so chunk commits in persist do not race - # with progress JSON writes on the worker session. - detach=True, - ) - - _set_run_progress( - db, - run, - phase="persisting_fail", - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={ - "fail_rows": len(fail_diffs), - "persisted": 0, - "persist_total": len(fail_diffs), - }, - ) - n_fail = _persist_sheet_diffs( - db, - run_id=run.id, - metric_id=mid, - diffs=fail_diffs, - seq_start=0, - on_progress=lambda w, n: _on_persist( - w, n, phase="persisting_fail", kind_key="fail_rows" - ), - ) - if ok_diffs: - _set_run_progress( - db, - run, - phase="persisting_ok", - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={ - "ok_rows": len(ok_diffs), - "persisted": 0, - "persist_total": len(ok_diffs), - }, - ) - _persist_sheet_diffs( - db, - run_id=run.id, - metric_id=mid, - diffs=ok_diffs, - seq_start=n_fail, - on_progress=lambda w, n: _on_persist( - w, n, phase="persisting_ok", kind_key="ok_rows" - ), - ) - for k in agg: - agg[k] += int(s.get(k) or 0) - mapping_by_metric[mid] = one["mapping_stats"] - done_meta = { - "sheet_id": one.get("sheet_id") or one["metric_id"], - "title": one.get("title") or one.get("sheet_id") or one["metric_id"], - "metric_id": one["metric_id"], - "key_fields": one["key_fields"], - "iface_fields": one["iface_fields"], - "compare_fields": one["compare_fields"], - "display_fields": one.get("display_fields") or [], - "field_rules": one.get("field_rules") or [], - "row_filters": one.get("row_filters") or [], - "ignore_port_changes": one.get("ignore_port_changes"), - "mode": one["mode"], - "status": "done", - "summary": one["summary"], + run_id = uuid4().hex + # Persist counts/meta only — diffs go to biz_compare_diff rows + summary_payload = { + **agg, + "sheet_count": len(sheet_results), + "top_changed_fields": top_fields, + "sheets": [ + { + "sheet_id": s.get("sheet_id") or s["metric_id"], + "title": s.get("title") or s.get("sheet_id") or s["metric_id"], + "metric_id": s["metric_id"], + "key_fields": s["key_fields"], + "iface_fields": s["iface_fields"], + "compare_fields": s["compare_fields"], + "display_fields": s.get("display_fields") or [], + "field_rules": s.get("field_rules") or [], + "ignore_port_changes": s.get("ignore_port_changes"), + "mode": s["mode"], + "summary": s["summary"], } - replaced = False - for i, meta in enumerate(sheet_metas): - if sheet_key(meta) == mid: - sheet_metas[i] = done_meta - replaced = True - break - if not replaced: - sheet_metas.append(done_meta) - _publish_sheets() - # Drop heavy diffs before next sheet - one.clear() - diffs.clear() - fail_diffs.clear() - ok_diffs.clear() - db.commit() - if _run_is_cancelled(db, run_id): - run = db.get(BizCompareRun, run_id) or run - return get_run(db, run.id) + for s in sheet_results + ], + } - if _run_is_cancelled(db, run_id): - run = db.get(BizCompareRun, run_id) or run - return get_run(db, run.id) - - duration_ms = int((time.monotonic() - started_mono) * 1000) - top_fields = sorted( - [{"field": k, "count": v} for k, v in field_counts.items()], - key=lambda x: (-int(x["count"]), str(x["field"])), - )[:8] - summary_payload = { - **agg, - "sheet_count": len(sheet_metas), - "top_changed_fields": top_fields, - "duration_ms": duration_ms, - "store_unchanged": store_mode, - "unchanged_stored": unchanged_listed_total > 0, - "unchanged_listed": unchanged_listed_total, - "unchanged_truncated": unchanged_truncated_any, - "unchanged_compact": unchanged_compact_any, - "progress": { - "phase": "done", - "sheet_index": total, - "sheet_total": total, - "sheet_id": "", - "sheet_title": "", - "elapsed_ms": duration_ms, - }, - "sheets": sheet_metas, - } - if snapshot is not None: - summary_payload["config_snapshot"] = snapshot - run = db.get(BizCompareRun, run_id) or run - summary_payload["purpose"] = (run.summary_json or {}).get("purpose") or "manual" - if str(run.status or "") == "cancelled": - return get_run(db, run.id) - run.status = "success" - run.summary_json = summary_payload - run.mapping_stats_json = mapping_by_metric - run.metric_id = str(sheet_metas[0]["metric_id"]) if sheet_metas else run.metric_id - run.message = f"done · {duration_ms // 1000}s" - run.diffs_json = [] - j.updated_at = _utcnow() - if j.mode == "manual": - j.after_batch_id = after_batch_id - db.commit() - return get_run(db, run.id) - except HTTPException as exc: - run = db.get(BizCompareRun, run_id) or run - if str(run.status or "") != "cancelled": - run.status = "failed" - run.message = str(getattr(exc, "detail", "") or exc)[:1024] - prev = dict(run.summary_json or {}) - prog = dict(prev.get("progress") or {}) - prog["phase"] = "failed" - prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000) - prev["progress"] = prog - prev["duration_ms"] = prog["elapsed_ms"] - run.summary_json = prev - db.commit() - raise - except Exception as exc: - _log.exception("compare run failed run=%s job=%s", run_id, j.id) - run = db.get(BizCompareRun, run_id) or run - if str(run.status or "") != "cancelled": - run.status = "failed" - run.message = str(exc)[:1024] - prev = dict(run.summary_json or {}) - prog = dict(prev.get("progress") or {}) - prog["phase"] = "failed" - prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000) - prev["progress"] = prog - prev["duration_ms"] = prog["elapsed_ms"] - run.summary_json = prev - db.commit() - raise - - -_INTERRUPT_MARK = "interrupted_by_restart" - - -def _job_has_active_run(db: Session, job_id: str) -> BizCompareRun | None: - return ( - db.query(BizCompareRun) - .filter( - BizCompareRun.job_id == str(job_id or ""), - BizCompareRun.status.in_(("running", "queued")), + run = BizCompareRun( + id=run_id, + job_id=j.id, + template_id=tpl.id, + mapping_id=j.mapping_id, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + metric_id=first["metric_id"], + status="success", + summary_json=summary_payload, + diffs_json=[], + mapping_stats_json=mapping_by_metric, + message="", + created_at=_utcnow(), + ) + db.add(run) + db.flush() + for s in sheet_results: + _persist_sheet_diffs( + db, + run_id=run_id, + metric_id=sheet_key(s), + diffs=list(s.get("diffs") or []), ) - .order_by(BizCompareRun.created_at.desc()) - .first() - ) - - -def _run_is_cancelled(db: Session, run_id: str) -> bool: - """Re-read status so user/startup cancel is visible to the worker thread.""" - db.expire_all() - r = db.get(BizCompareRun, run_id) - return bool(r and str(r.status or "") == "cancelled") - - -def recover_interrupted_compares_on_startup(db: Session) -> dict[str, Any]: - """Mark orphaned running/queued compare runs as cancelled after process restart. - - In-memory job locks die with the process; without this, the UI stays on - 「比对中」and blocks a new run. - """ - now = _utcnow() - rows = ( - db.query(BizCompareRun) - .filter(BizCompareRun.status.in_(("running", "queued"))) - .all() - ) - n = 0 - for r in rows: - r.status = "cancelled" - msg = str(r.message or "").strip() - if _INTERRUPT_MARK not in msg: - r.message = f"{msg} | {_INTERRUPT_MARK}".strip(" |")[:1024] - prev = dict(r.summary_json or {}) - prog = dict(prev.get("progress") or {}) - prog["phase"] = "cancelled" - prog["elapsed_ms"] = int(prog.get("elapsed_ms") or 0) - prev["progress"] = prog - # Mark in-flight sheet placeholders so UI does not show fake pass - sheets = list(prev.get("sheets") or []) - for sh in sheets: - st = str(sh.get("status") or "") - if st in ("pending", "running", "queued"): - sh["status"] = "cancelled" - prev["sheets"] = sheets - r.summary_json = prev - n += 1 - if n: - db.commit() - _log.info("startup: cancelled %s interrupted compare run(s)", n) - return {"runs": n, "at": now.isoformat() + "Z"} - - -def cancel_compare_run(db: Session, run_id: str) -> dict[str, Any]: - """Cancel a running/queued compare so a new run can start.""" - r = db.get(BizCompareRun, run_id) - if not r: - raise HTTPException(status_code=404, detail="run_not_found") - st = str(r.status or "") - if st not in ("running", "queued"): - return get_run(db, run_id) - r.status = "cancelled" - msg = str(r.message or "").strip() - r.message = f"{msg} | cancelled_by_user".strip(" |")[:1024] - prev = dict(r.summary_json or {}) - prog = dict(prev.get("progress") or {}) - prog["phase"] = "cancelled" - prev["progress"] = prog - for sh in list(prev.get("sheets") or []): - if str(sh.get("status") or "") in ("pending", "running", "queued"): - sh["status"] = "cancelled" - r.summary_json = prev + j.updated_at = _utcnow() + if j.mode == "manual": + j.after_batch_id = after_batch_id db.commit() - return get_run(db, run_id) - - -def run_compare( - db: Session, job_id: str, *, force_after_batch_id: str = "", purpose: str = "manual", -) -> dict[str, Any]: - """Synchronous compare (auto-compare / tests). Blocks the caller until done.""" - if _job_has_active_run(db, job_id): - raise HTTPException(status_code=409, detail="compare_already_running") - lock = _job_compare_lock(job_id) - if not lock.acquire(blocking=False): - raise HTTPException(status_code=409, detail="compare_already_running") - try: - ctx = _validate_compare_job(db, job_id, force_after_batch_id=force_after_batch_id) - run = _create_running_run( - db, - job=ctx["job"], - tpl=ctx["template"], - before_batch_id=ctx["before_batch_id"], - after_batch_id=ctx["after_batch_id"], - sheets_cfg=ctx["sheets_cfg"], - purpose=purpose, - ) - return _execute_compare_into_run(db, run.id) - finally: - lock.release() - - -def enqueue_compare( - db: Session, job_id: str, *, force_after_batch_id: str = "" -) -> dict[str, Any]: - """Create a ``running`` run and execute compare on a daemon thread. - - Returns immediately so the HTTP worker / UI stay responsive. Poll - ``GET /compare/runs/{id}`` for progress (``summary.progress``). - """ - if _job_has_active_run(db, job_id): - raise HTTPException(status_code=409, detail="compare_already_running") - lock = _job_compare_lock(job_id) - if not lock.acquire(blocking=False): - raise HTTPException(status_code=409, detail="compare_already_running") - run_id = "" - try: - ctx = _validate_compare_job(db, job_id, force_after_batch_id=force_after_batch_id) - run = _create_running_run( - db, - job=ctx["job"], - tpl=ctx["template"], - before_batch_id=ctx["before_batch_id"], - after_batch_id=ctx["after_batch_id"], - sheets_cfg=ctx["sheets_cfg"], - ) - run_id = run.id - except Exception: - lock.release() - raise - - def _bg() -> None: - from ..db import SessionLocal - - s = SessionLocal() - try: - _execute_compare_into_run(s, run_id) - except Exception: - _log.exception("bg compare failed job=%s run=%s", job_id, run_id) - finally: - s.close() - lock.release() - - threading.Thread( - target=_bg, - name=f"biz-cmp-{run_id[:8]}", - daemon=True, - ).start() - return get_run(db, run_id) + return get_run(db, run.id) def _csv_cell(v: Any) -> str: @@ -3254,7 +1645,7 @@ def _csv_cell(v: Any) -> str: return s -def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) -> Iterator[str]: +def _sheet_csv(sheet: dict[str, Any]) -> str: keys = list(sheet.get("key_fields") or []) key_set = set(keys) compare = [f for f in list(sheet.get("compare_fields") or []) if f not in key_set] @@ -3266,17 +1657,15 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) -> ) # Non-key columns already ordered Key→Compare→Display by effective_display_fields extra = [f for f in display if f not in key_set] - # Interface keys also get the name as the device printed it (before side). - raw_cols = [k for k in keys if k in set(sheet.get("iface_fields") or [])] - headers = ["kind", *keys, *[f"{k}__raw_before" for k in raw_cols]] + headers = ["kind", *keys] for f in extra: if f in compare_set: headers.append(f"{f}__pre") headers.append(f"{f}__post") else: headers.append(f) - yield ",".join(_csv_cell(h) for h in headers) - for d in diffs: + lines = [",".join(_csv_cell(h) for h in headers)] + for d in list(sheet.get("diffs") or []): kind = str(d.get("kind") or "") pre = dict(d.get("mapped_before") or d.get("before") or {}) post = dict(d.get("after") or {}) @@ -3284,9 +1673,6 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) -> row = [kind] for k in keys: row.append(key.get(k, pre.get(k, post.get(k, "")))) - raw_before = dict(d.get("raw_key_before") or {}) - for k in raw_cols: - row.append(raw_before.get(k, "")) for f in extra: if f in compare_set: if kind == "added": @@ -3304,23 +1690,8 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) -> row.append(pre.get(f, "")) else: row.append(post.get(f, pre.get(f, ""))) - yield ",".join(_csv_cell(x) for x in row) - - -def _sheet_csv(sheet: dict[str, Any]) -> str: - return "\ufeff" + "\n".join(_sheet_csv_lines(sheet, sheet.get("diffs") or [])) + "\n" - - -def _task_ok(sheets: list[dict[str, Any]], fail_count: int, any_pending: bool) -> bool | None: - """True/False once judged; None when data quality leaves the verdict open.""" - if any_pending: - return False - states = [str((sh.get("summary") or {}).get("data_state") or "ok") for sh in sheets] - if any(st in ("suspect_parse", "not_collected") for st in states): - return None - if states and all(st == "no_data" for st in states): - return None - return fail_count == 0 + lines.append(",".join(_csv_cell(x) for x in row)) + return "\ufeff" + "\n".join(lines) + "\n" def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> dict[str, Any]: @@ -3331,28 +1702,14 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di before_count = int(summary.get("before_count") or 0) after_count = int(summary.get("after_count") or 0) total = added + removed + changed + unchanged - - def _data_state(sh: dict[str, Any]) -> str: - return str((sh.get("summary") or {}).get("data_state") or "ok") - - # Sheets whose side was never collected cannot be judged: their "missing" - # rows are a collection gap, not a business loss, so they stay out of the totals. - dropped = [sh for sh in sheets if _data_state(sh) == "not_collected" - and str(sh.get("status") or "done") not in ("pending", "running", "queued")] - drop_fail = sum(int((sh.get("summary") or {}).get("removed") or 0) - + int((sh.get("summary") or {}).get("changed") or 0) for sh in dropped) - drop_ok = sum(int((sh.get("summary") or {}).get("unchanged") or 0) for sh in dropped) # Compare verdict: fail = missing + mismatch; success = match; added is special - fail_count = removed + changed - drop_fail - success_count = unchanged - drop_ok + fail_count = removed + changed + success_count = unchanged judged = fail_count + success_count matched = changed + unchanged diff_count = fail_count - # Nothing compared at all is "not judged" (None), never a 100% pass. pass_rate = ( - compute_pass_rate(success=success_count, fail=fail_count) - if judged - else (None if total == 0 else 0.0) + round((success_count / judged) * 100, 1) if judged else (100.0 if total == 0 else 0.0) ) diff_rate = round((fail_count / judged) * 100, 1) if judged else 0.0 @@ -3366,19 +1723,12 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di st = sa + sr + sc + su sf = sr + sc sj = sf + su - status = str(sh.get("status") or "done") - pending = status in ("pending", "running", "queued") - data_state = str(ss.get("data_state") or "ok") sheet_cards.append( { - "data_state": data_state, - "data_state_reasons": list(ss.get("data_state_reasons") or []), - "collect_quality": ss.get("collect_quality") or {}, "sheet_id": sheet_key(sh), "title": sheet_title(sh), "metric_id": sh.get("metric_id") or "", "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"), - "status": status, "added": sa, "removed": sr, "changed": sc, @@ -3388,10 +1738,7 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di "fail_count": sf, "success_count": su, "diff_count": sf, - # Pending sheets must not look like "100% pass" - "pass_rate": None - if pending or data_state in UNJUDGED_STATES - else (compute_pass_rate(success=su, fail=sf) if sj else (None if st == 0 else 0.0)), + "pass_rate": round((su / sj) * 100, 1) if sj else (100.0 if st == 0 else 0.0), } ) @@ -3410,9 +1757,6 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di key=lambda x: (-int(x["count"]), str(x["field"])), )[:8] - any_pending = any( - str(sh.get("status") or "") in ("pending", "running", "queued") for sh in sheets - ) return { "added": added, "removed": removed, @@ -3426,20 +1770,11 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di "fail_count": fail_count, "success_count": success_count, "diff_count": diff_count, - "pass_rate": None if any_pending else pass_rate, + "pass_rate": pass_rate, "diff_rate": diff_rate, - "ok": _task_ok(sheets, fail_count, any_pending), - "unjudged_sheets": sum(1 for sh in sheets if _data_state(sh) in UNJUDGED_STATES), - "suspect_sheets": sum(1 for sh in sheets if _data_state(sh) == "suspect_parse"), + "ok": fail_count == 0, "sheet_cards": sheet_cards, "top_changed_fields": top_fields, - "duration_ms": int(summary.get("duration_ms") or 0), - "store_unchanged": str(summary.get("store_unchanged") or ""), - "unchanged_stored": bool(summary.get("unchanged_stored", True)), - "unchanged_listed": int(summary.get("unchanged_listed") or 0), - "unchanged_truncated": bool(summary.get("unchanged_truncated")), - "unchanged_compact": bool(summary.get("unchanged_compact")), - "progress": dict(summary.get("progress") or {}), } @@ -3447,13 +1782,8 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: r = db.get(BizCompareRun, run_id) if not r: raise HTTPException(status_code=404, detail="run_not_found") + tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None summary = dict(r.summary_json or {}) - snapshot = _run_config_snapshot(r) - tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id and snapshot is None else None - template_out = ( - deepcopy(snapshot.get("template")) if snapshot is not None - else (_template_out(tpl) if tpl else None) - ) raw_sheets = _sheet_meta_from_summary(summary, r, tpl) # Never return full diffs in run detail (million-row safe) sheets = [ @@ -3467,38 +1797,28 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: "display_fields": list(sh.get("display_fields") or []), "field_rules": list(sh.get("field_rules") or []), "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"), - "status": str(sh.get("status") or "done"), "summary": dict(sh.get("summary") or {}), } for sh in raw_sheets ] enriched = _enrich_summary(summary, raw_sheets) stored = "rows" if _run_has_diff_rows(db, run_id) else "inline" - job = db.get(BizCompareJob, r.job_id) if r.job_id and snapshot is None else None - mapping = db.get(BizPortMapping, r.mapping_id) if r.mapping_id and snapshot is None else None + job = db.get(BizCompareJob, r.job_id) if r.job_id else None + mapping = db.get(BizPortMapping, r.mapping_id) if r.mapping_id else None before_side = _compare_side( - db, r.before_batch_id, fallback_task_id=( - snapshot.get("before_task_id", "") if snapshot is not None else (job.before_task_id if job else "") - ), + db, r.before_batch_id, fallback_task_id=(job.before_task_id if job else "") ) after_side = _compare_side( - db, r.after_batch_id, fallback_task_id=( - snapshot.get("after_task_id", "") if snapshot is not None else (job.after_task_id if job else "") - ), + db, r.after_batch_id, fallback_task_id=(job.after_task_id if job else "") ) return { "id": r.id, "job_id": r.job_id, - "job_name": ( - snapshot.get("job_name", "") if snapshot is not None else ((job.name if job else "") or "") - ), + "job_name": (job.name if job else "") or "", "template_id": r.template_id, - "template_name": (template_out or {}).get("name") or "", + "template_name": (tpl.name if tpl else "") or "", "mapping_id": r.mapping_id, - "mapping_name": ( - snapshot.get("mapping_name", "") if snapshot is not None else ((mapping.name if mapping else "") or "") - ), - "config_snapshot_version": snapshot["version"] if snapshot is not None else None, + "mapping_name": (mapping.name if mapping else "") or "", "before_batch_id": r.before_batch_id, "after_batch_id": r.after_batch_id, "before": before_side, @@ -3512,7 +1832,7 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: "mapping_stats": r.mapping_stats_json or {}, "message": r.message, "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, - "template": template_out, + "template": _template_out(tpl) if tpl else None, } @@ -3530,451 +1850,6 @@ def _lookup_sheet(sheets: list[dict[str, Any]], key: str) -> dict[str, Any] | No return None -def _kind_allows(kind_n: str, diff_kind: str) -> bool: - dk = str(diff_kind or "") - if kind_n == "all": - return True - if kind_n == "diff": - return dk in ("removed", "changed") - return dk == kind_n - - -def _parse_qf(raw: Any) -> dict[str, str]: - """Normalize field query map: {field: value} with safe names only.""" - from .compare_sql import _FIELD_RE - - if raw is None or raw == "": - return {} - obj: Any = raw - if isinstance(raw, str): - s = raw.strip() - if not s: - return {} - try: - obj = json.loads(s) - except Exception: - return {} - if not isinstance(obj, dict): - return {} - out: dict[str, str] = {} - for k, v in obj.items(): - name = str(k or "").strip() - val = str(v or "").strip() - if not name or not val or not _FIELD_RE.match(name): - continue - out[name] = val - return out - - -def _split_kw_and_field_tokens(kw: str) -> tuple[str, dict[str, str]]: - """Parse ``direction:out network:1.1.1.1 foo`` → free kw + field map.""" - from .compare_sql import _FIELD_RE - - free: list[str] = [] - fields: dict[str, str] = {} - for tok in str(kw or "").split(): - if ":" in tok: - name, _, val = tok.partition(":") - name = name.strip() - val = val.strip() - if name and val and _FIELD_RE.match(name): - fields[name] = val - continue - if tok.strip(): - free.append(tok.strip()) - return " ".join(free), fields - - -def _kw_match_sql(key_fields: list[str], *, param: str = "kw") -> str: - """OR of ILIKE on key fields (and data_json::text fallback).""" - from .compare_sql import _FIELD_RE, _safe_field - - parts: list[str] = [] - for f in key_fields: - name = str(f or "").strip() - if not name or not _FIELD_RE.match(name): - continue - sf = _safe_field(name) - parts.append(f"lower(trim(both from coalesce(data_json->>'{sf}', ''))) LIKE :{param}") - # Broad fallback so free-text still hits non-key columns (path, next_hop, …) - parts.append(f"lower(data_json::text) LIKE :{param}") - return "(" + " OR ".join(parts) + ")" if parts else f"(lower(data_json::text) LIKE :{param})" - - -def _field_qf_sql(field_q: dict[str, str], *, prefix: str = "qf") -> tuple[str, dict[str, Any]]: - """AND of ILIKE contains on each field.""" - from .compare_sql import _safe_field - - if not field_q: - return "TRUE", {} - parts: list[str] = [] - params: dict[str, Any] = {} - for i, (name, val) in enumerate(field_q.items()): - sf = _safe_field(name) - key = f"{prefix}_{i}" - params[key] = _literal_search_pattern(val) - parts.append( - f"lower(trim(both from coalesce(data_json->>'{sf}', ''))) LIKE :{key}" - ) - return "(" + " AND ".join(parts) + ")", params - - -def _literal_search_pattern(value: str) -> str: - return "%" + value.lower().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" - - -def _load_metric_rows_for_search( - db: Session, - *, - batch_id: str, - metric_id: str, - row_filters: list[dict[str, Any]] | None, - key_fields: list[str], - kw: str = "", - field_q: dict[str, str] | None = None, - cap: int = _LIVE_SEARCH_LOAD_CAP, -) -> tuple[list[dict[str, Any]], bool]: - """Load rows matching sheet filters + optional free kw / field_q. Returns (rows, truncated).""" - from .compare_sql import ( - _dialect_is_postgres, - _filters_sql_compatible, - compile_row_filters_sql, - ) - from ..models import BizStateMetricRow - from sqlalchemy import text as sql_text - - bid = str(batch_id or "").strip() - mid = str(metric_id or "").strip() - needle = str(kw or "").strip() - fq = {k: v for k, v in (field_q or {}).items() if str(v or "").strip()} - if not bid or not mid or (not needle and not fq): - return [], False - lim = max(1, min(int(cap), _LIVE_SEARCH_LOAD_CAP)) - filters = [f for f in (row_filters or []) if isinstance(f, dict)] - - if mid != "lldp_neighbor" and _dialect_is_postgres(db) and ( - not filters or _filters_sql_compatible(filters) - ): - filter_sql, filter_params = ("TRUE", {}) - if filters and _filters_sql_compatible(filters): - filter_sql, filter_params = compile_row_filters_sql(filters) - qf_sql, qf_params = _field_qf_sql(fq) - search_parts = [qf_sql] - params: dict[str, Any] = { - "bid": bid, - "mid": mid, - "lim": lim + 1, - **filter_params, - **qf_params, - } - if needle: - params["kw"] = _literal_search_pattern(needle) - search_parts.append(_kw_match_sql(key_fields)) - search_sql = " AND ".join(f"({p})" for p in search_parts if p and p != "TRUE") - if not search_sql: - search_sql = "TRUE" - rows = db.execute( - sql_text( - f""" - SELECT id, batch_command_id, task_id, ne_id, seq, data_json, collected_at - FROM biz_state_metric_row - WHERE batch_id = :bid - AND metric_id = :mid - AND ({filter_sql}) - AND ({search_sql}) - ORDER BY seq ASC, id ASC - LIMIT :lim - """ - ), - params, - ).mappings().all() - truncated = len(rows) > lim - rows = rows[:lim] - out: list[dict[str, Any]] = [] - for r in rows: - data = dict(r["data_json"] or {}) - collected = r["collected_at"] - out.append( - { - **data, - "_netx": { - "batch_id": bid, - "batch_command_id": str(r["batch_command_id"] or ""), - "task_id": str(r["task_id"] or ""), - "ne_id": str(r["ne_id"] or ""), - "collected_at": collected.isoformat() + "Z" - if collected is not None - else None, - "row_id": str(r["id"]), - }, - } - ) - return out, truncated - - # Non-PG / fallback: scan with early stop (OK for tests / small sheets) - model = BizStateLldpNeighbor if mid == "lldp_neighbor" else BizStateMetricRow - q = db.query(model).filter(model.batch_id == bid) - if mid == "lldp_neighbor": - q = q.order_by(model.id.asc()) - else: - q = q.filter(model.metric_id == mid).order_by(model.seq.asc(), model.id.asc()) - out = [] - truncated = False - needle_l = needle.lower() - key_set = [str(k).strip() for k in key_fields if str(k).strip()] - for r in q.yield_per(500): - data = ( - {f: getattr(r, f) for f in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")} - if mid == "lldp_neighbor" else dict(r.data_json or {}) - ) - row = { - **data, - "_netx": { - "batch_id": bid, - "batch_command_id": r.batch_command_id or "", - "task_id": r.task_id or "", - "ne_id": r.ne_id or "", - "collected_at": r.collected_at.isoformat() + "Z" - if r.collected_at - else None, - "row_id": r.id, - }, - } - if filters and not all(row_matches_filter(row, f) for f in filters): - db.expunge(r) - continue - if fq: - ok_f = True - for fname, fval in fq.items(): - if fval.lower() not in str(row.get(fname) or "").lower(): - ok_f = False - break - if not ok_f: - db.expunge(r) - continue - if needle_l: - hit = False - for kf in key_set: - if needle_l in str(row.get(kf) or "").lower(): - hit = True - break - if not hit: - blob = json.dumps(data, ensure_ascii=False, default=str).lower() - hit = needle_l in blob - if not hit: - db.expunge(r) - continue - db.expunge(r) - if len(out) >= lim: - truncated = True - break - out.append(row) - return out, truncated - - -def _load_search_key_groups( - db: Session, *, batch_id: str, metric_id: str, keys: set[tuple[str, ...]], - match_keys: list[str], pipeline: Pipeline, side: str, -) -> tuple[list[dict[str, Any]], set[tuple[str, ...]]]: - """Complete candidate key groups so a one-sided search cannot change verdicts. - - Identity rules (normalize / value map / port map) run in the pipeline, in - Python. Key fields they never touch narrow the SQL query. If the load budget - is exceeded, omit entire groups rather than compare partial groups and - manufacture additions/deletions. - """ - from ..models import BizStateMetricRow - - model = BizStateLldpNeighbor if metric_id == "lldp_neighbor" else BizStateMetricRow - # Narrowing compares raw SQL text, so only fields no rule rewrites may be used, - # and only when key tuples line up 1:1 with ``match_keys`` (no mapped/unmapped marker). - invariant = [] if pipeline.ignores_ports_by_policy() else [ - f for f in match_keys if f not in pipeline._transformed # noqa: SLF001 - ] - positions = [match_keys.index(f) for f in invariant] - coarse = sorted({tuple(k[i] for i in positions) for k in keys}) if invariant else [()] - out: list[dict[str, Any]] = [] - complete: set[tuple[str, ...]] = set() - # Bound memory even when a candidate identifies a very large duplicate group. - budget = _LIVE_SEARCH_GROUP_LOAD_CAP - for start in range(0, len(coarse), 100): - group = coarse[start:start + 100] - group_set = set(group) - target = {k for k in keys if tuple(k[i] for i in positions) in group_set} if invariant else set(keys) - q = db.query(model).filter(model.batch_id == batch_id) - if metric_id == "lldp_neighbor": - q = q.order_by(model.id.asc()) - else: - q = q.filter(model.metric_id == metric_id).order_by(model.seq.asc(), model.id.asc()) - if invariant: - exprs = [func.trim(func.coalesce( - cast(getattr(model, f) if metric_id == "lldp_neighbor" - else model.data_json[f].as_string(), String), "")) for f in invariant] - q = q.filter(tuple_(*exprs).in_(group)) - rows = q.limit(budget + 1).all() - if len(rows) > budget: - for r in rows: - db.expunge(r) - # No partial verdicts. Later groups may still fit the remaining budget. - continue - budget -= len(rows) - complete.update(target) - for r in rows: - data = ( - {f: getattr(r, f) for f in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")} - if metric_id == "lldp_neighbor" else dict(r.data_json or {}) - ) - row = {**data, "_netx": {"row_id": r.id}} - db.expunge(r) - norm = pipeline.normalize(row, side) - if not pipeline.accepts(row, norm, side): - continue - if pipeline.key_of(norm, side, match_keys)[1] in target: - out.append(row) - return out, complete - - -def _live_search_sheet_diffs( - db: Session, - run: BizCompareRun, - sheet: dict[str, Any], - *, - kind: str, - kw: str = "", - field_q: dict[str, str] | None = None, - page: int, - page_size: int, -) -> dict[str, Any]: - """Find candidate keys, complete both sides, then compare and filter pairs.""" - mid_src = str(sheet.get("metric_id") or "").strip() - sid = sheet_key(sheet) - key_fields = list(sheet.get("key_fields") or []) - iface_fields = list(sheet.get("iface_fields") or []) - field_rules = list(sheet.get("field_rules") or []) - compare_fields = effective_compare_fields( - list(sheet.get("compare_fields") or []), - field_rules, - ) - row_filters = list(sheet.get("row_filters") or []) - # Older runs may lack row_filters on sheet meta — fall back to template - if "row_filters" not in sheet and _run_config_snapshot(run) is None and run.template_id: - tpl = db.get(BizCompareTemplate, run.template_id) - if tpl: - for s in template_metrics(tpl): - if sheet_key(s) == sid: - row_filters = list(s.get("row_filters") or []) - if not key_fields: - key_fields = list(s.get("key_fields") or []) - if not field_rules: - field_rules = list(s.get("field_rules") or []) - compare_fields = effective_compare_fields( - list(s.get("compare_fields") or compare_fields), - field_rules, - ) - break - - free_kw, tok_fields = _split_kw_and_field_tokens(kw) - merged_q = {**tok_fields, **(field_q or {})} - # Drop empty - merged_q = {k: v for k, v in merged_q.items() if str(v or "").strip()} - - if not key_fields or not mid_src: - return { - "total": 0, - "page": page, - "page_size": page_size, - "metric_id": sid, - "items": [], - "source": "live", - "truncated": False, - } - - ignore_ports = sheet.get("ignore_port_changes") - if ignore_ports is not None: - ignore_ports = bool(ignore_ports) - pipeline = _run_pipeline(db, run, sheet) - if pipeline.legacy and not pipeline.row_filters and row_filters: - pipeline.row_filters = [dict(f) for f in row_filters if isinstance(f, dict)] - - before_rows, trunc_b = _load_metric_rows_for_search( - db, - batch_id=str(run.before_batch_id or ""), - metric_id=mid_src, - row_filters=pipeline.pushdown_filters("before"), - key_fields=key_fields, - kw=free_kw, - field_q=merged_q, - ) - after_rows, trunc_a = _load_metric_rows_for_search( - db, - batch_id=str(run.after_batch_id or ""), - metric_id=mid_src, - row_filters=pipeline.pushdown_filters("after"), - key_fields=key_fields, - kw=free_kw, - field_q=merged_q, - ) - match_keys = list((sheet.get("summary") or {}).get("match_key_fields") or key_fields) - if ignore_ports is True and not pipeline.port_map: - match_keys = [f for f in key_fields if f not in iface_fields] or key_fields - keys: set[tuple[str, ...]] = set() - for rows, side in ((before_rows, "before"), (after_rows, "after")): - for raw, norm in pipeline.prepare(rows, side): - keys.add(pipeline.key_of(norm, side, match_keys)[1]) - before_rows, complete_b = _load_search_key_groups( - db, batch_id=str(run.before_batch_id or ""), metric_id=mid_src, keys=keys, - match_keys=match_keys, pipeline=pipeline, side="before", - ) - after_rows, complete_a = _load_search_key_groups( - db, batch_id=str(run.after_batch_id or ""), metric_id=mid_src, keys=keys, - match_keys=match_keys, pipeline=pipeline, side="after", - ) - complete = complete_b & complete_a - - def keep_complete(rows: list[dict[str, Any]], side: str) -> list[dict[str, Any]]: - return [raw for raw, norm in pipeline.prepare(rows, side) - if pipeline.key_of(norm, side, match_keys)[1] in complete] - - before_rows = keep_complete(before_rows, "before") - after_rows = keep_complete(after_rows, "after") - result = compare_rows( - before_rows=before_rows, - after_rows=after_rows, - key_fields=key_fields, - iface_fields=iface_fields, - compare_fields=compare_fields, - field_rules=field_rules, - include_unchanged=True, - unchanged_limit=None, - compact_unchanged=False, - pipeline=pipeline, - match_keys_override=match_keys, - ) - kind_n = (kind or "diff").strip().lower() - filtered = [ - d for d in list(result.get("diffs") or []) - if _kind_allows(kind_n, str(d.get("kind") or "")) - and _diff_matches_search(d, kw=free_kw, field_q=merged_q) - ] - # Cap pairs returned to keep UI snappy - truncated = bool(trunc_b or trunc_a or complete != keys or len(filtered) > _LIVE_SEARCH_RESULT_CAP) - filtered = filtered[:_LIVE_SEARCH_RESULT_CAP] - total = len(filtered) - start = (page - 1) * page_size - page_items = _attach_raw_key_before(filtered[start : start + page_size], key_fields) - return { - "total": total, - "page": page, - "page_size": page_size, - "metric_id": sid, - "items": page_items, - "source": "live", - "truncated": truncated, - "live_before_matched": len(before_rows), - "live_after_matched": len(after_rows), - } - - def list_run_diffs( db: Session, run_id: str, @@ -3982,7 +1857,6 @@ def list_run_diffs( metric_id: str = "", kind: str = "diff", kw: str = "", - qf: Any = None, page: int = 1, page_size: int = 100, ) -> dict[str, Any]: @@ -3993,40 +1867,14 @@ def list_run_diffs( size_n = max(1, min(500, int(page_size or 100))) kind_n = (kind or "diff").strip().lower() kw_n = (kw or "").strip() - field_q = _parse_qf(qf) - # Also accept field:value tokens inside kw - free_from_kw, tok_fields = _split_kw_and_field_tokens(kw_n) - if tok_fields: - field_q = {**tok_fields, **field_q} - kw_n = free_from_kw summary = dict(r.summary_json or {}) tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None sheets = _sheet_meta_from_summary(summary, r, tpl) asked = (metric_id or "").strip() sheet = _lookup_sheet(sheets, asked) if asked else (sheets[0] if sheets else None) - if asked and sheet is None: - raise HTTPException(status_code=404, detail="sheet_not_found") mid = sheet_key(sheet) if sheet else (asked or str(r.metric_id or "")) - # Only reconstruct missing success rows. Persisted verdicts must stay stable - # while searching and avoid loading/recomparing source tables on every page. - sheet_summary = (sheet or {}).get("summary") or {} - success_missing = int(sheet_summary.get("unchanged") or 0) > int( - sheet_summary.get("unchanged_listed") or 0 - ) - if (kw_n or field_q) and sheet and kind_n in ("all", "unchanged") and success_missing: - return _live_search_sheet_diffs( - db, - r, - sheet, - kind=kind_n, - kw=kw_n, - field_q=field_q, - page=page_n, - page_size=size_n, - ) - if _run_has_diff_rows(db, run_id): q = db.query(BizCompareDiff).filter( BizCompareDiff.run_id == run_id, @@ -4034,15 +1882,10 @@ def list_run_diffs( ) if kind_n == "diff": q = q.filter(BizCompareDiff.kind.in_(("removed", "changed"))) - elif kind_n == "removed": - q = q.filter(BizCompareDiff.kind == "removed") - elif kind_n == "changed": - q = q.filter(BizCompareDiff.kind == "changed") elif kind_n != "all": q = q.filter(BizCompareDiff.kind == kind_n) - if kw_n or field_q: - q = _search_stored_diffs(q, metric_id=str((sheet or {}).get("metric_id") or mid), - kw=kw_n, field_q=field_q) + if kw_n: + q = q.filter(BizCompareDiff.search_text.ilike(f"%{kw_n}%")) total = q.count() rows = ( q.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc()) @@ -4050,15 +1893,12 @@ def list_run_diffs( .limit(size_n) .all() ) - items = _hydrate_diff_rows(db, [_diff_row_out(x) for x in rows], run=r, sheet=sheet) return { "total": total, "page": page_n, "page_size": size_n, "metric_id": mid, - "items": items, - "source": "stored", - "truncated": False, + "items": [_diff_row_out(x) for x in rows], } # Legacy: diffs embedded in summary_json / diffs_json @@ -4068,7 +1908,7 @@ def list_run_diffs( inline = list((sheet or {}).get("diffs") or []) if not inline and mid == r.metric_id: inline = list(r.diffs_json or []) - filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n, field_q=field_q) + filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n) total = len(filtered) start = (page_n - 1) * size_n page_items = filtered[start : start + size_n] @@ -4078,58 +1918,43 @@ def list_run_diffs( "page_size": size_n, "metric_id": mid, "items": page_items, - "source": "stored", - "truncated": False, } -def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> Iterator[dict[str, Any]]: - """Stream one sheet using a stable (seq, id) cursor; fall back to inline. - - Compact success rows are hydrated in chunks from metric tables. - """ - r = db.get(BizCompareRun, run_id) - if not r: - return - tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None - sheets = _sheet_meta_from_summary(dict(r.summary_json or {}), r, tpl) - sheet = next((s for s in sheets if sheet_key(s) == metric_id), None) +def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str, Any]]: + """Load all diffs for one sheet (export). Prefer row table; fall back to inline.""" if _run_has_diff_rows(db, run_id): - cursor: tuple[int, str] | None = None + out: list[dict[str, Any]] = [] + offset = 0 while True: - q = db.query(BizCompareDiff).filter( - BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id, - ) - if cursor is not None: - seq, diff_id = cursor - q = q.filter(or_(BizCompareDiff.seq > seq, and_( - BizCompareDiff.seq == seq, BizCompareDiff.id > diff_id, - ))) - rows = (q + rows = ( + db.query(BizCompareDiff) + .filter(BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id) .order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc()) + .offset(offset) .limit(_DIFF_CHUNK) .all() ) if not rows: break - chunk = _hydrate_diff_rows(db, [_diff_row_out(x) for x in rows], run=r, sheet=sheet) - cursor = (rows[-1].seq, rows[-1].id) - yield from chunk + out.extend(_diff_row_out(x) for x in rows) + offset += len(rows) if len(rows) < _DIFF_CHUNK: break - # Release the previous chunk before loading the next one. - del rows, chunk - return + return out + r = db.get(BizCompareRun, run_id) + if not r: + return [] summary = dict(r.summary_json or {}) sheets = list(summary.get("sheets") or []) for sh in sheets: if sheet_key(sh) == metric_id or ( str(sh.get("metric_id") or "") == metric_id and sheet_key(sh) == metric_id ): - yield from sh.get("diffs") or [] - return + return list(sh.get("diffs") or []) if metric_id == r.metric_id: - yield from r.diffs_json or [] + return list(r.diffs_json or []) + return [] def export_run_zip(db: Session, run_id: str) -> bytes: @@ -4158,18 +1983,15 @@ def export_run_zip(db: Session, run_id: str) -> bytes: f"+{card.get('added')}/-{card.get('removed')}/~{card.get('changed')}/= {card.get('unchanged')}" ) zf.writestr("manifest.txt", "\n".join(manifest) + "\n") - run = db.get(BizCompareRun, run_id) - snapshot = _run_config_snapshot(run) if run else None - if snapshot is not None: - zf.writestr("config_snapshot.json", json.dumps(snapshot, ensure_ascii=False, indent=2)) for sheet in list(detail.get("sheets") or []): sid = sheet_key(sheet) or "sheet" safe = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in sid)[:80] or "sheet" - with zf.open(f"tables/{safe}.csv", "w", force_zip64=True) as csv_file: - csv_file.write(b"\xef\xbb\xbf") - for line in _sheet_csv_lines(sheet, _iter_sheet_diffs(db, run_id, sid)): - csv_file.write((line + "\n").encode("utf-8")) - sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate,data_state"] + sheet_full = { + **sheet, + "diffs": _iter_sheet_diffs(db, run_id, sid), + } + zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet_full)) + sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"] for card in list(s.get("sheet_cards") or []): sum_lines.append( ",".join( @@ -4185,7 +2007,6 @@ def export_run_zip(db: Session, run_id: str) -> bytes: card.get("unchanged"), card.get("diff_count"), card.get("pass_rate"), - card.get("data_state") or "ok", ) ) ) @@ -4204,34 +2025,22 @@ def list_runs(db: Session, job_id: str, *, limit: int = 20) -> list[dict[str, An job = db.get(BizCompareJob, job_id) before_tid = str(job.before_task_id or "") if job else "" after_tid = str(job.after_task_id or "") if job else "" - out: list[dict[str, Any]] = [] - for r in rows: - summary = dict(r.summary_json or {}) - out.append( - { - "id": r.id, - "before_batch_id": r.before_batch_id, - "after_batch_id": r.after_batch_id, - "before": _compare_side(db, r.before_batch_id, fallback_task_id=before_tid), - "after": _compare_side(db, r.after_batch_id, fallback_task_id=after_tid), - "status": r.status, - "message": r.message or "", - "summary": { - k: summary.get(k, 0) - for k in ( - "added", - "removed", - "changed", - "unchanged", - "sheet_count", - "duration_ms", - ) - }, - "progress": dict(summary.get("progress") or {}), - "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, - } - ) - return out + return [ + { + "id": r.id, + "before_batch_id": r.before_batch_id, + "after_batch_id": r.after_batch_id, + "before": _compare_side(db, r.before_batch_id, fallback_task_id=before_tid), + "after": _compare_side(db, r.after_batch_id, fallback_task_id=after_tid), + "status": r.status, + "summary": { + k: (r.summary_json or {}).get(k, 0) + for k in ("added", "removed", "changed", "unchanged", "sheet_count") + }, + "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, + } + for r in rows + ] def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int: @@ -4253,7 +2062,7 @@ def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int: if not j.before_batch_id: continue try: - run_compare(db, j.id, force_after_batch_id=batch_id, purpose="auto") + run_compare(db, j.id, force_after_batch_id=batch_id) n += 1 except HTTPException as exc: if int(getattr(exc, "status_code", 0) or 0) == 409: diff --git a/netx_api/biz_state/compare_sql.py b/netx_api/biz_state/compare_sql.py deleted file mode 100644 index d417475..0000000 --- a/netx_api/biz_state/compare_sql.py +++ /dev/null @@ -1,801 +0,0 @@ -"""PostgreSQL-backed sheet compare (FULL OUTER JOIN) for pushdown-safe sheets. - -Falls back to the Python engine when port-map / complex rules cannot be expressed -in SQL. See ``can_sql_compare``. -""" - -from __future__ import annotations - -import logging -import re -from typing import Any, Callable, Mapping, Sequence -from uuid import uuid4 - -from sqlalchemy import text -from sqlalchemy.orm import Session - -from .compare_engine import key_tuple -from .compare_rules import effective_compare_fields, field_rule_map, NUMBER_PATTERN, scalar_text - -_log = logging.getLogger("netx.biz_state.compare_sql") - -# Below this, Python hash-join is faster and avoids TEMP CTAS / progress races. -_SQL_MIN_ROWS = 20_000 -# Cap a single SQL statement so a stuck planner/lock surfaces as fallback. -_SQL_STATEMENT_TIMEOUT_MS = 180_000 - -_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") - - -class SqlCompareSkip(Exception): - """Soft skip — caller should use the Python engine (not an error).""" - - def __init__(self, reason: str) -> None: - super().__init__(reason) - self.reason = str(reason or "skip") -_SQL_FILTER_OPS = frozenset( - {"eq", "==", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "ci_eq"} -) -_SQL_NORMALIZE = frozenset({"", "none", "strip", "lower", "upper", "empty_as_blank"}) -_SQL_COMPARE_MODES = frozenset( - { - "", - "eq", - "ignore", - "skip", - "off", - "numeric", - "number", - "int", - "float", - "percent", - "pct", - "rel", - } -) -_EMPTY_AS_BLANK = ("n/a", "na", "-", "--", "none", "null") -_NUM_RE_SQL = NUMBER_PATTERN - - -def _dialect_is_postgres(db: Session) -> bool: - bind = db.get_bind() - if bind is None: - return False - return str(getattr(bind.dialect, "name", "") or "").lower() in ("postgresql", "postgres") - - -def _safe_field(name: str) -> str: - f = str(name or "").strip() - if not f or not _FIELD_RE.match(f): - raise ValueError(f"unsafe_json_field:{name!r}") - return f - - -def _json_text_expr(alias: str, field: str) -> str: - """SQL expression: trimmed text from JSONB column ``alias.data`` / ``data_json``.""" - f = _safe_field(field) - # alias is caller-controlled (_b / data_json) — not user input - return f"trim(both from coalesce({alias}->>'{f}', ''))" - - -def _norm_expr(alias: str, field: str, normalize: str) -> str: - base = _json_text_expr(alias, field) - mode = str(normalize or "strip").strip().lower() or "strip" - if mode in ("", "none", "strip"): - return base - if mode == "lower": - return f"lower({base})" - if mode == "upper": - return f"upper({base})" - if mode == "empty_as_blank": - opts = ", ".join(f"'{x}'" for x in _EMPTY_AS_BLANK) - return ( - f"CASE WHEN lower({base}) IN ({opts}) THEN '' ELSE {base} END" - ) - raise ValueError(f"unsupported_normalize:{mode}") - - -def _filters_sql_compatible(filters: Sequence[Mapping[str, Any]] | None) -> bool: - fl = [f for f in (filters or []) if isinstance(f, dict)] - return all(_filter_node_compatible(f) for f in fl) - - -def _filter_node_compatible(filt: Mapping[str, Any]) -> bool: - if "any" in filt: - kids = filt.get("any") or [] - return isinstance(kids, list) and all( - isinstance(k, dict) and _filter_node_compatible(k) for k in kids - ) - if "all" in filt: - kids = filt.get("all") or [] - return isinstance(kids, list) and all( - isinstance(k, dict) and _filter_node_compatible(k) for k in kids - ) - field = str(filt.get("field") or "").strip() - op = str(filt.get("op") or "eq").strip().lower() - if op not in _SQL_FILTER_OPS: - return False - if op in ("empty", "not_empty", "nonempty"): - return bool(field) and bool(_FIELD_RE.match(field)) - if not field or not _FIELD_RE.match(field): - return False - if op in ("in", "not_in", "nin"): - vals = filt.get("value") - if vals is None: - return True - if not isinstance(vals, (list, tuple)): - vals = [vals] - return all(isinstance(x, (str, int, float, bool)) or x is None for x in vals) - return True - - -def _field_rules_sql_compatible(rules: Sequence[Mapping[str, Any]] | None) -> bool: - for raw in rules or []: - if not isinstance(raw, dict): - return False - name = str(raw.get("field") or "").strip() - if name and not _FIELD_RE.match(name): - return False - mode = str(raw.get("compare") or "eq").strip().lower() or "eq" - if mode not in _SQL_COMPARE_MODES: - return False - if raw.get("ignore") is True: - continue - if mode in ("ignore", "skip", "off"): - continue - norm = str(raw.get("normalize") or "strip").strip().lower() or "strip" - if norm not in _SQL_NORMALIZE: - return False - return True - - -def sql_compare_skip_reason( - db: Session, - sheet: Mapping[str, Any], - *, - port_map: Mapping[str, str] | None = None, - iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, - pipeline: Any = None, -) -> str: - """Empty string when SQL is allowed; otherwise a short reason for progress/logs. - - Simple ``row_filters`` (eq/in/contains/…) used for BGP sheet splits are fine — - they do **not** force Python by themselves. - - A port map only blocks SQL when the sheet actually reads an interface field - in its key or compare columns; a BGP route table with no interface fields is - unaffected by it. ``pipeline`` (v2 identity rules) adds its own blockers. - """ - if not _dialect_is_postgres(db): - return "not_postgres" - if pipeline is not None: - port_map = pipeline.port_map - if iface_normalize_rules is None and (pipeline.before.has_iface_rules or pipeline.after.has_iface_rules): - iface_normalize_rules = [{"from": "*", "to": "*"}] - key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] - if not key_fields: - return "no_key_fields" - if any(not _FIELD_RE.match(k) for k in key_fields): - return "unsafe_key_field" - iface_fields = [str(f).strip() for f in (sheet.get("iface_fields") or []) if str(f).strip()] - iface_set = set(iface_fields) - if port_map and iface_set: - early_cmp = effective_compare_fields( - list(sheet.get("compare_fields") or []), list(sheet.get("field_rules") or []), - ) - if iface_set & (set(key_fields) | {str(f).strip() for f in early_cmp}): - return "port_map" - ignore_ports = sheet.get("ignore_port_changes") - if ignore_ports is True: - return "ignore_port_changes" - if ignore_ports is None and iface_set and any(k in iface_set for k in key_fields): - return "auto_ignore_ports" - field_rules = list(sheet.get("field_rules") or []) - if any( - isinstance(r, dict) and str(r.get("compare") or "").strip().lower() in - ("not_worse", "numeric_min", "numeric_max") - for r in field_rules - ): - return "directional_compare" - if not _field_rules_sql_compatible(field_rules): - return "field_rules" - compare_fields = effective_compare_fields( - list(sheet.get("compare_fields") or []), - field_rules, - ) - if any(not _FIELD_RE.match(str(f).strip()) for f in compare_fields if str(f).strip()): - return "unsafe_compare_field" - used = set(key_fields) | {str(f).strip() for f in compare_fields if str(f).strip()} - if port_map and (used & iface_set): - return "port_map" - if iface_normalize_rules and (used & iface_set): - return "iface_normalize" - if pipeline is not None: - blocked = pipeline.sql_blockers(used) - if blocked: - return blocked - if not _filters_sql_compatible(list(sheet.get("row_filters") or [])): - return "row_filters" - if not str(sheet.get("metric_id") or "").strip(): - return "no_metric_id" - return "" - - -def can_sql_compare( - db: Session, - sheet: Mapping[str, Any], - *, - port_map: Mapping[str, str] | None = None, - iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, -) -> bool: - """True when this sheet can run entirely as a PostgreSQL JOIN.""" - return not bool( - sql_compare_skip_reason( - db, - sheet, - port_map=port_map, - iface_normalize_rules=iface_normalize_rules, - ) - ) - - -def compile_row_filters_sql( - filters: Sequence[Mapping[str, Any]] | None, - *, - json_col: str = "data_json", - param_prefix: str = "f", -) -> tuple[str, dict[str, Any]]: - """Compile template row_filters to SQL AND-clause + bind params. - - Returns ``(sql_fragment, params)``. Empty filters → ``(\"TRUE\", {})``. - ``json_col`` is the JSONB column/expression name (trusted). - """ - fl = [f for f in (filters or []) if isinstance(f, dict)] - if not fl: - return "TRUE", {} - params: dict[str, Any] = {} - counter = {"n": 0} - - def _next(name: str) -> str: - counter["n"] += 1 - return f"{param_prefix}_{counter['n']}_{name}" - - def _node(filt: Mapping[str, Any]) -> str: - if "any" in filt: - kids = [k for k in (filt.get("any") or []) if isinstance(k, dict)] - if not kids: - return "TRUE" - return "(" + " OR ".join(_node(k) for k in kids) + ")" - if "all" in filt: - kids = [k for k in (filt.get("all") or []) if isinstance(k, dict)] - if not kids: - return "TRUE" - return "(" + " AND ".join(_node(k) for k in kids) + ")" - field = _safe_field(str(filt.get("field") or "")) - op = str(filt.get("op") or "eq").strip().lower() - expr = _json_text_expr(json_col, field) - # _json_text_expr uses alias->> ; for bare column use data_json directly - if json_col == "data_json": - expr = f"trim(both from coalesce(data_json->>'{field}', ''))" - if op in ("empty",): - return f"({expr} = '')" - if op in ("not_empty", "nonempty"): - return f"({expr} <> '')" - expect = filt.get("value") - if op in ("eq", "==", "ci_eq"): - key = _next("v") - params[key] = scalar_text(expect).strip() - if op == "ci_eq" or op in ("eq", "=="): - # Python eq is case-insensitive via .lower() - params[key] = scalar_text(expect).strip().lower() - return f"(lower({expr}) = :{key})" - if op in ("ne", "!="): - key = _next("v") - params[key] = scalar_text(expect).strip().lower() - return f"(lower({expr}) <> :{key})" - if op == "contains": - key = _next("v") - needle = scalar_text(expect).strip().lower() - if not needle: - return "FALSE" - escaped = needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - params[key] = f"%{escaped}%" - return f"(lower({expr}) LIKE :{key} ESCAPE E'\\\\')" - if op in ("in", "not_in", "nin"): - vals = expect - if vals is None: - vals = [] - if not isinstance(vals, (list, tuple)): - vals = [vals] - clean = [scalar_text(x).strip().lower() for x in vals if scalar_text(x).strip()] - if not clean: - return "FALSE" if op == "in" else "TRUE" - keys = [] - for i, v in enumerate(clean): - k = _next(f"in{i}") - params[k] = v - keys.append(f":{k}") - inside = f"lower({expr}) IN ({', '.join(keys)})" - return f"({inside})" if op == "in" else f"(NOT {inside})" - raise ValueError(f"unsupported_filter_op:{op}") - - parts = [_node(f) for f in fl] - return "(" + " AND ".join(parts) + ")", params - - -def _rk_sql(key_fields: list[str], *, json_col: str = "data_json") -> str: - parts = [] - for k in key_fields: - f = _safe_field(k) - if json_col == "data_json": - parts.append(f"trim(both from coalesce(data_json->>'{f}', ''))") - else: - parts.append(f"trim(both from coalesce({json_col}->>'{f}', ''))") - if len(parts) == 1: - return parts[0] - # Delimiter concatenation conflates ('a|b', 'c') with ('a', 'b|c'). - return "jsonb_build_array(" + ", ".join(parts) + ")::text" - - -def _field_differs_sql(bv: str, av: str, rule: Mapping[str, Any]) -> str: - """SQL boolean: True when before/after values differ under the field rule.""" - mode = str(rule.get("compare") or "eq").strip().lower() or "eq" - if mode in ("ignore", "skip", "off") or rule.get("ignore") is True: - return "FALSE" - try: - tol = float(rule.get("tolerance") or 0) - except (TypeError, ValueError): - tol = 0.0 - if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"): - # Match Python values_equal: parseable → numeric compare; else string eq - num_b = f"(({bv}) ~ '{_NUM_RE_SQL}')" - num_a = f"(({av}) ~ '{_NUM_RE_SQL}')" - bn = f"({bv})::double precision" - an = f"({av})::double precision" - if mode in ("percent", "pct", "rel"): - # differ when relative % > tol (before==0 → after must be 0) - num_diff = ( - f"(CASE WHEN {bn} = 0 THEN {an} IS DISTINCT FROM 0 " - f"ELSE (abs({an} - {bn}) / abs({bn}) * 100.0) > {tol} END)" - ) - else: - num_diff = f"(abs({an} - {bn}) > {tol})" - return ( - f"(CASE WHEN {num_b} AND {num_a} THEN {num_diff} " - f"ELSE ({bv} IS DISTINCT FROM {av}) END)" - ) - return f"({bv} IS DISTINCT FROM {av})" - - -def _changed_predicate( - compare_fields: list[str], - rules: dict[str, dict[str, Any]], - *, - before_alias: str = "b", - after_alias: str = "a", -) -> str: - """SQL boolean: True when any compare field differs (IS DISTINCT FROM).""" - if not compare_fields: - return "FALSE" - clauses: list[str] = [] - for f in compare_fields: - name = str(f).strip() - if not name: - continue - rule = rules.get(name) or {} - mode = str(rule.get("compare") or "eq").strip().lower() or "eq" - if mode in ("ignore", "skip", "off") or rule.get("ignore") is True: - continue - norm = str(rule.get("normalize") or "strip").strip().lower() or "strip" - bv = _norm_expr(f"{before_alias}.data", name, norm) - av = _norm_expr(f"{after_alias}.data", name, norm) - clauses.append(_field_differs_sql(bv, av, rule)) - if not clauses: - return "FALSE" - return "(" + " OR ".join(clauses) + ")" - - -def _key_obj_from_data(data: dict[str, Any] | None, key_fields: list[str]) -> dict[str, Any]: - row = data if isinstance(data, dict) else {} - return dict(zip(key_fields, key_tuple(row, key_fields))) - - -def _changes_from_rows( - before: dict[str, Any] | None, - after: dict[str, Any] | None, - compare_fields: list[str], - rules: dict[str, dict[str, Any]], -) -> dict[str, dict[str, Any]]: - from .compare_rules import explain_diff, values_equal - - out: dict[str, dict[str, Any]] = {} - b = before or {} - a = after or {} - for f in compare_fields: - rule = rules.get(f) - bv = b.get(f, "") - av = a.get(f, "") - if values_equal(bv, av, rule=rule): - continue - entry: dict[str, Any] = {"before": bv, "after": av} - reason = explain_diff(bv, av, rule=rule) - if reason: - entry["reason"] = reason - out[f] = entry - return out - - -def dup_pairing_order_sql(compare_fields: Sequence[str], rules: Mapping[str, Mapping[str, Any]]) -> str: - """``ORDER BY`` prefix that pairs same-key rows by value (mirrors ``Pipeline.pair_sort_key``). - - Byte-order collation matches Python's code-point string order, so both - engines choose the same pairs. Returns '' (or a trailing-comma list). - """ - parts = [] - for f in compare_fields: - name = str(f).strip() - if not name: - continue - mode = str((rules.get(name) or {}).get("normalize") or "strip").strip().lower() or "strip" - parts.append(f'({_norm_expr("data_json", name, mode)}) COLLATE "C" ASC') - return ", ".join(parts) + ", " if parts else "" - - -def run_sql_sheet_compare( - db: Session, - *, - sheet: Mapping[str, Any], - before_batch_id: str, - after_batch_id: str, - store_unchanged: str = "auto", - on_progress: Callable[..., None] | None = None, - duplicate_pairing: str = "ordered", -) -> dict[str, Any]: - """Compare one sheet via PostgreSQL TEMP tables + FULL OUTER JOIN. - - Same-key multipath: rows get ``dup_rn`` by ``seq,id`` and join on - ``(rk, dup_rn)`` (ordered zip). Extras are added/removed, not duplicate. - - Returns the same ``{summary, diffs, mapping_stats}`` shape as ``compare_rows``. - ``on_progress(side, n, *, engine=\"sql\", note=..., phase=...)``. - """ - if not _dialect_is_postgres(db): - raise RuntimeError("sql_compare_requires_postgres") - - def _prog(side: str, n: int, *, phase: str = "sql_count", note: str = "") -> None: - if not on_progress: - return - try: - on_progress(side, n, engine="sql", note=note, phase=phase) - except TypeError: - on_progress(side, n) - - key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] - field_rules = list(sheet.get("field_rules") or []) - rules = field_rule_map(field_rules) - compare_fields = effective_compare_fields( - list(sheet.get("compare_fields") or []), - field_rules, - ) - row_filters = list(sheet.get("row_filters") or []) - mid = str(sheet.get("metric_id") or "").strip() - bid_b = str(before_batch_id or "").strip() - bid_a = str(after_batch_id or "").strip() - if not mid or not bid_b or not bid_a or not key_fields: - raise ValueError("sql_compare_missing_args") - - filter_sql, filter_params = compile_row_filters_sql(row_filters) - rk = _rk_sql(key_fields) - dup_order = dup_pairing_order_sql(compare_fields, rules) if duplicate_pairing == "sorted" else "" - tag = uuid4().hex[:8] - tb = f"_netx_cmp_b_{tag}" - ta = f"_netx_cmp_a_{tag}" - - # Keep worker transaction open across CTAS — progress must NOT commit this session. - db.execute(text(f"SET LOCAL statement_timeout = '{int(_SQL_STATEMENT_TIMEOUT_MS)}'")) - - _prog("before", 0, phase="sql_count", note="count") - - # Raw counts (no row_filters) - raw_b = int( - db.execute( - text( - "SELECT count(*) FROM biz_state_metric_row " - "WHERE batch_id = :bid AND metric_id = :mid" - ), - {"bid": bid_b, "mid": mid}, - ).scalar() - or 0 - ) - _prog("before", raw_b, phase="sql_count") - raw_a = int( - db.execute( - text( - "SELECT count(*) FROM biz_state_metric_row " - "WHERE batch_id = :bid AND metric_id = :mid" - ), - {"bid": bid_a, "mid": mid}, - ).scalar() - or 0 - ) - _prog("after", raw_a, phase="sql_count") - - if max(raw_b, raw_a) < int(_SQL_MIN_ROWS): - raise SqlCompareSkip("small_sheet") - - base_params = {"bid": bid_b, "mid": mid, **filter_params} - # Build TEMP sides (one transaction — no mid-flight commits on ``db``) - _prog("before", raw_b, phase="sql_project", note="temp_before") - for tname, batch_id, side, note in ( - (tb, bid_b, "before", "temp_before"), - (ta, bid_a, "after", "temp_after"), - ): - db.execute(text(f"DROP TABLE IF EXISTS {tname}")) - params = {**base_params, "bid": batch_id} - if side == "after": - _prog(side, raw_a, phase="sql_project", note=note) - db.execute( - text( - f""" - CREATE TEMP TABLE {tname} ON COMMIT PRESERVE ROWS AS - SELECT - id, - ({rk}) AS rk, - data_json AS data, - row_number() OVER ( - PARTITION BY ({rk}) - ORDER BY {dup_order}seq ASC, id ASC - ) AS dup_rn - FROM biz_state_metric_row - WHERE batch_id = :bid - AND metric_id = :mid - AND ({filter_sql}) - """ - ), - params, - ) - db.execute(text(f"CREATE INDEX IF NOT EXISTS {tname}_rk ON {tname} (rk, dup_rn)")) - - before_n = int( - db.execute(text(f"SELECT count(*) FROM {tb}")).scalar() or 0 - ) - after_n = int( - db.execute(text(f"SELECT count(*) FROM {ta}")).scalar() or 0 - ) - _prog("after", after_n, phase="sql_join", note="join") - - changed_pred = _changed_predicate(compare_fields, rules) - kind_expr = f""" - CASE - WHEN b.id IS NULL THEN 'added' - WHEN a.id IS NULL THEN 'removed' - WHEN {changed_pred} THEN 'changed' - ELSE 'unchanged' - END - """ - - # Ordered same-key pairing: join on (rk, dup_rn) so 5 vs 2 → 2 compared + 3 removed - agg_rows = db.execute( - text( - f""" - SELECT {kind_expr} AS kind, count(*)::bigint AS n - FROM {tb} b - FULL OUTER JOIN {ta} a - ON b.rk = a.rk AND b.dup_rn = a.dup_rn - GROUP BY 1 - """ - ) - ).mappings().all() - counts = {str(r["kind"]): int(r["n"] or 0) for r in agg_rows} - added = counts.get("added", 0) - removed = counts.get("removed", 0) - changed = counts.get("changed", 0) - unchanged = counts.get("unchanged", 0) - - # Diagnostic: count of match keys with >1 row (not fail rows) - multi_b = int( - db.execute( - text(f"SELECT count(*) FROM (SELECT rk FROM {tb} GROUP BY rk HAVING count(*) > 1) t") - ).scalar() - or 0 - ) - multi_a = int( - db.execute( - text(f"SELECT count(*) FROM (SELECT rk FROM {ta} GROUP BY rk HAVING count(*) > 1) t") - ).scalar() - or 0 - ) - - # Lazy import — avoid circular import with compare_service - from .compare_service import resolve_unchanged_policy - - policy = resolve_unchanged_policy( - store_unchanged, before_n=before_n, after_n=after_n - ) - diffs: list[dict[str, Any]] = [] - - # Fail rows (stream into Python — should be << million) - fail_sql = text( - f""" - SELECT - {kind_expr} AS kind, - b.id AS before_row_id, - a.id AS after_row_id, - b.data AS before_data, - a.data AS after_data, - COALESCE(b.rk, a.rk) AS rk - FROM {tb} b - FULL OUTER JOIN {ta} a - ON b.rk = a.rk AND b.dup_rn = a.dup_rn - WHERE {kind_expr} IN ('added', 'removed', 'changed') - """ - ) - fail_n = 0 - _prog("after", after_n, phase="sql_fail_fetch", note="fetch_fails") - for row in db.execute(fail_sql).mappings(): - kind = str(row["kind"] or "") - before = dict(row["before_data"] or {}) if row["before_data"] is not None else None - after = dict(row["after_data"] or {}) if row["after_data"] is not None else None - key_src = after if kind == "added" else (before or after or {}) - item: dict[str, Any] = { - "kind": kind, - "key": _key_obj_from_data(key_src, key_fields), - "before": before, - "after": after, - "mapped_before": before, - "changes": {}, - "before_row_id": str(row["before_row_id"] or ""), - "after_row_id": str(row["after_row_id"] or ""), - } - if kind == "changed": - item["changes"] = _changes_from_rows(before, after, compare_fields, rules) - diffs.append(item) - fail_n += 1 - if fail_n == 1 or fail_n % 25_000 == 0: - _prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails") - if fail_n: - _prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails") - - unchanged_listed = 0 - include_u = bool(policy.get("include")) - compact = bool(policy.get("compact")) - limit_n = policy.get("limit") - if include_u and unchanged > 0: - params_u: dict[str, Any] = {} - # Stratified sample: round-robin by neighbor|direction|afi|vrf so one - # BGP command cannot consume the whole success sample. - stratum_expr = """ - concat_ws('|', - coalesce(nullif(trim(both from coalesce(b.data->>'neighbor','')), ''), '_'), - coalesce(nullif(trim(both from coalesce(b.data->>'direction','')), ''), '_'), - coalesce(nullif(trim(both from coalesce(b.data->>'afi','')), ''), '_'), - coalesce(nullif(trim(both from coalesce(b.data->>'vrf','')), ''), '_') - ) - """ - if limit_n is not None: - params_u["lim"] = max(0, int(limit_n)) - u_sql = text( - f""" - WITH u AS ( - SELECT - b.id AS before_row_id, - a.id AS after_row_id, - b.data AS before_data, - a.data AS after_data, - {stratum_expr} AS stratum - FROM {tb} b - INNER JOIN {ta} a - ON b.rk = a.rk AND b.dup_rn = a.dup_rn - WHERE NOT ({changed_pred}) - ), - ranked AS ( - SELECT *, - row_number() OVER ( - PARTITION BY stratum ORDER BY before_row_id ASC - ) AS rn_in - FROM u - ), - picked AS ( - SELECT *, - row_number() OVER ( - ORDER BY rn_in ASC, stratum ASC, before_row_id ASC - ) AS pick_ord - FROM ranked - ) - SELECT before_row_id, after_row_id, before_data, after_data - FROM picked - WHERE pick_ord <= :lim - """ - ) - else: - u_sql = text( - f""" - SELECT - b.id AS before_row_id, - a.id AS after_row_id, - b.data AS before_data, - a.data AS after_data - FROM {tb} b - INNER JOIN {ta} a - ON b.rk = a.rk AND b.dup_rn = a.dup_rn - WHERE NOT ({changed_pred}) - """ - ) - for row in db.execute(u_sql, params_u).mappings(): - before = dict(row["before_data"] or {}) - after = dict(row["after_data"] or {}) - unchanged_listed += 1 - if compact: - diffs.append( - { - "kind": "unchanged", - "key": _key_obj_from_data(before, key_fields), - "before": {}, - "after": {}, - "mapped_before": {}, - "changes": {}, - "compact": True, - "before_row_id": str(row["before_row_id"] or ""), - "after_row_id": str(row["after_row_id"] or ""), - } - ) - else: - diffs.append( - { - "kind": "unchanged", - "key": _key_obj_from_data(before, key_fields), - "before": before, - "after": after, - "mapped_before": before, - "changes": {}, - "before_row_id": str(row["before_row_id"] or ""), - "after_row_id": str(row["after_row_id"] or ""), - } - ) - - # Cleanup (also ON COMMIT DROP) - db.execute(text(f"DROP TABLE IF EXISTS {tb}")) - db.execute(text(f"DROP TABLE IF EXISTS {ta}")) - - summary = { - "before_count": before_n, - "after_count": after_n, - "added": added, - "removed": removed, - "changed": changed, - "unchanged": unchanged, - "duplicate": 0, - "match_key_fields": list(key_fields), - "duplicate_keys_before": multi_b, - "duplicate_keys_after": multi_a, - "duplicate_key_list": [], - "unchanged_listed": unchanged_listed, - "unchanged_truncated": bool( - include_u and limit_n is not None and unchanged > unchanged_listed - ), - "unchanged_compact": bool(compact and unchanged_listed > 0), - "unchanged_sample_mode": ( - "stratified" - if include_u and limit_n is not None and unchanged > unchanged_listed - else ("full" if include_u else "none") - ), - "engine": "sql", - "before_raw_count": raw_b, - "after_raw_count": raw_a, - "row_filters": len(row_filters), - "unchanged_policy": policy, - } - mapping_stats = { - "before_iface_count": 0, - "after_iface_count": 0, - "map_pairs": 0, - "hit_before": [], - "miss_before": [], - "hit_after": [], - "miss_after": [], - "unused_before_keys": [], - "ok": True, - "ignore_port_changes": False, - "engine": "sql", - } - return {"summary": summary, "diffs": diffs, "mapping_stats": mapping_stats} diff --git a/netx_api/biz_state/compare_validation.py b/netx_api/biz_state/compare_validation.py deleted file mode 100644 index 3fba6c7..0000000 --- a/netx_api/biz_state/compare_validation.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Validate template writes without silently dropping parts of the comparison.""" -from __future__ import annotations - -import math -import re -from typing import Any - -from fastapi import HTTPException - -FILTER_OPS = {"eq", "==", "ci_eq", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "regex", "age_timer"} -COMPARE_MODES = {"", "eq", "ignore", "skip", "off", "numeric", "number", "int", "float", "percent", "pct", "rel", - "not_worse", "numeric_min", "numeric_max"} -SIDES = {"", "both", "before", "after"} -MAX_VALUE_MAP = 500 -NORMALIZE_MODES = {"", "none", "strip", "lower", "upper", "mac", "empty_as_blank"} - - -def invalid(path: str, reason: str) -> None: - raise HTTPException(status_code=400, detail={"error": "invalid_template", "path": path, "reason": reason}) - - -def _strings(value: Any, path: str, *, required: bool = False) -> None: - if not isinstance(value, list) or any(not isinstance(x, str) or not x.strip() for x in value): - invalid(path, "nonempty_strings_required") - if required and not value: - invalid(path, "key_fields_required") - if len({x.strip() for x in value}) != len(value): - invalid(path, "duplicate_field") - - -def validate_filters(value: Any, path: str, depth: int = 0) -> None: - if not isinstance(value, list): - invalid(path, "list_required") - if depth > 12: - invalid(path, "filter_nesting_too_deep") - for i, node in enumerate(value): - here = f"{path}[{i}]" - if not isinstance(node, dict) or not node: - invalid(here, "filter_required") - if depth == 0 and str(node.get("side") or "").strip().lower() not in SIDES: - invalid(here, "unknown_filter_side") - groups = [key for key in ("any", "all") if key in node] - if groups: - if len(groups) != 1 or any(node.get(key) is not None for key in ("field", "op", "value")): - invalid(here, "group_or_leaf_required") - kids = node[groups[0]] - if not isinstance(kids, list) or not kids: - invalid(here, "nonempty_group_required") - validate_filters(kids, f"{here}.{groups[0]}", depth + 1) - continue - if not isinstance(node.get("field"), str) or not node["field"].strip(): - invalid(here, "filter_field_required") - op = str(node.get("op") or "eq").strip().lower() - if op not in FILTER_OPS: - invalid(here, "unknown_filter_operator") - expect = node.get("value") - if op in ("in", "not_in", "nin"): - if not isinstance(expect, list) or not expect or any(not isinstance(x, (str, int, float, bool)) for x in expect): - invalid(here, "nonempty_value_list_required") - elif op not in ("empty", "not_empty", "nonempty", "age_timer"): - if not isinstance(expect, (str, int, float, bool)): - invalid(here, "scalar_value_required") - if op in ("contains", "regex") and not str(expect).strip(): - invalid(here, "value_required") - if op == "regex": - try: - re.compile(str(expect), re.I) - except re.error: - invalid(here, "invalid_regex") - - -def validate_sheet(sheet: Any, path: str, *, partial: bool = False) -> None: - if not isinstance(sheet, dict): - invalid(path, "sheet_required") - if not partial or "metric_id" in sheet: - if not isinstance(sheet.get("metric_id"), str) or not sheet["metric_id"].strip(): - invalid(path, "metric_id_required") - for key in ("key_fields", "iface_fields", "compare_fields", "display_fields", "ignore_fields"): - if key in sheet or (key == "key_fields" and not partial): - _strings(sheet.get(key), f"{path}.{key}", required=key == "key_fields") - for opt, allowed in (("unmapped_port_policy", {"by_name", "ignore_port"}), - ("duplicate_pairing", {"sorted", "ordered"})): - if str(sheet.get(opt) or "").strip() and str(sheet[opt]).strip().lower() not in allowed: - invalid(path, f"unknown_{opt}") - if sheet.get("builtin_vendor_canonical") is not None and not isinstance(sheet["builtin_vendor_canonical"], bool): - invalid(path, "boolean_builtin_vendor_canonical_required") - if "ignore_port_changes" in sheet and sheet["ignore_port_changes"] is not None and not isinstance(sheet["ignore_port_changes"], bool): - invalid(path, "boolean_port_policy_required") - if "row_filters" in sheet: - validate_filters(sheet["row_filters"], f"{path}.row_filters") - if "field_rules" in sheet: - rules = sheet["field_rules"] - if not isinstance(rules, list): - invalid(path, "field_rules_list_required") - seen = set() - for i, rule in enumerate(rules): - here = f"{path}.field_rules[{i}]" - if not isinstance(rule, dict) or not isinstance(rule.get("field"), str) or not rule["field"].strip(): - invalid(here, "rule_field_required") - field = rule["field"].strip() - rule_side = str(rule.get("side") or "").strip().lower() - if rule_side not in SIDES: - invalid(here, "unknown_rule_side") - marker = (field, "" if rule_side == "both" else rule_side) - if marker in seen: - invalid(here, "duplicate_field_rule") - seen.add(marker) - vmap = rule.get("value_map") - if vmap is not None: - if not isinstance(vmap, dict) or any( - not isinstance(k, str) or not isinstance(v, str) for k, v in vmap.items() - ): - invalid(here, "value_map_string_pairs_required") - if len(vmap) > MAX_VALUE_MAP: - invalid(here, "value_map_too_large") - if rule.get("apply_to_key") is not None and not isinstance(rule["apply_to_key"], bool): - invalid(here, "boolean_apply_to_key_required") - mode = str(rule.get("compare") or "").strip().lower() - if mode == "not_worse": - rank = rule.get("rank") - if not isinstance(rank, list) or len(rank) < 2 or any(not isinstance(x, str) or not x.strip() for x in rank): - invalid(here, "rank_list_required") - elif rule.get("rank") is not None: - invalid(here, "rank_only_for_not_worse") - if str(rule.get("compare") or "").strip().lower() not in COMPARE_MODES: - invalid(here, "unknown_compare_mode") - if str(rule.get("normalize") or "").strip().lower() not in NORMALIZE_MODES: - invalid(here, "unknown_normalize_mode") - tol = rule.get("tolerance") - if tol is not None: - if isinstance(tol, bool) or not isinstance(tol, (int, float)) or not math.isfinite(tol) or tol < 0: - invalid(here, "finite_nonnegative_tolerance_required") - - -def validate_template_body(body: dict[str, Any], *, partial: bool = False) -> None: - metrics = body.get("metrics") - if metrics is not None: - if not isinstance(metrics, list) or not metrics: - invalid("metrics", "metrics_required") - seen = set() - for i, sheet in enumerate(metrics): - path = f"metrics[{i}]" - validate_sheet(sheet, path) - sid = str(sheet.get("sheet_id") or sheet["metric_id"]).strip() - if sid in seen: - invalid(path, "duplicate_sheet_id") - seen.add(sid) - else: - validate_sheet(body, "template", partial=partial) - if "iface_normalize_rules" in body and body["iface_normalize_rules"] is not None: - rules = body["iface_normalize_rules"] - if not isinstance(rules, list): - invalid("iface_normalize_rules", "list_required") - seen = set() - for i, rule in enumerate(rules): - path = f"iface_normalize_rules[{i}]" - if not isinstance(rule, dict) or any(not isinstance(rule.get(k), str) or not rule[k].strip() for k in ("from", "to")): - invalid(path, "alias_pair_required") - rside = str(rule.get("side") or "").strip().lower() - if rside not in SIDES: - invalid(path, "unknown_alias_side") - if rule.get("vendor") is not None and not isinstance(rule["vendor"], str): - invalid(path, "alias_vendor_string_required") - key = (rule["from"].strip().lower(), "" if rside == "both" else rside, - str(rule.get("vendor") or "").strip().lower()) - if key in seen: - invalid(path, "duplicate_alias") - seen.add(key) diff --git a/netx_api/biz_state/data/default_zte_status_template.json b/netx_api/biz_state/data/default_zte_status_template.json deleted file mode 100644 index 7549a38..0000000 --- a/netx_api/biz_state/data/default_zte_status_template.json +++ /dev/null @@ -1,1065 +0,0 @@ -{ - "format": "netx.biz_compare_template", - "version": 1, - "name": "ZTE status default", - "note": "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP, address-family sheets)", - "iface_normalize_rules": [], - "metrics": [ - { - "sheet_id": "isis_adjacency.ipv4", - "title": "ISIS IPv4", - "metric_id": "isis_adjacency", - "key_fields": [ - "process_id", - "interface", - "system_id" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state", - "lev", - "af" - ], - "display_fields": [ - "process_id", - "interface", - "system_id", - "state", - "lev", - "af" - ], - "row_filters": [ - { - "op": "contains", - "field": "af", - "value": "IPv4" - } - ], - "field_rules": [] - }, - { - "sheet_id": "isis_adjacency.ipv6", - "title": "ISIS IPv6", - "metric_id": "isis_adjacency", - "key_fields": [ - "process_id", - "interface", - "system_id" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state", - "lev", - "af" - ], - "display_fields": [ - "process_id", - "interface", - "system_id", - "state", - "lev", - "af" - ], - "row_filters": [ - { - "op": "contains", - "field": "af", - "value": "IPv6" - } - ], - "field_rules": [] - }, - { - "sheet_id": "interface_brief", - "title": "interface_brief", - "metric_id": "interface_brief", - "key_fields": [ - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "prot" - ], - "display_fields": [ - "interface", - "prot", - "admin", - "phy" - ], - "row_filters": [ - { - "op": "regex", - "field": "interface", - "value": "^[^.]+$" - }, - { - "op": "contains", - "field": "interface", - "value": "gei" - } - ], - "field_rules": [] - }, - { - "sheet_id": "arp", - "title": "arp", - "metric_id": "arp", - "key_fields": [ - "ip", - "interface", - "vrf" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "mac" - ], - "display_fields": [ - "ip", - "interface", - "vrf", - "mac", - "age", - "entry_type" - ], - "row_filters": [ - { - "any": [ - { - "field": "entry_type", - "op": "eq", - "value": "dynamic" - }, - { - "all": [ - { - "field": "entry_type", - "op": "empty" - }, - { - "field": "age", - "op": "age_timer" - } - ] - } - ] - } - ], - "field_rules": [ - { - "field": "mac", - "normalize": "mac" - } - ] - }, - { - "sheet_id": "nd6_cache", - "title": "nd6_cache", - "metric_id": "nd6_cache", - "key_fields": [ - "address", - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "link_address", - "status" - ], - "display_fields": [ - "address", - "interface", - "link_address", - "status", - "type", - "age" - ], - "row_filters": [ - { - "op": "ne", - "field": "status", - "value": " Incomplete" - } - ], - "field_rules": [] - }, - { - "sheet_id": "bgp_peer.ipv4", - "title": "BGP IPv4", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "ipv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.ipv6", - "title": "BGP IPv6", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "ipv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.vpnv4", - "title": "BGP VPNv4", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpnv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.vpnv6", - "title": "BGP VPNv6", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpnv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.evpn", - "title": "BGP EVPN", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "evpn" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.vpls", - "title": "BGP VPLS", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpls" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "interface_detail", - "title": "interface_detail", - "metric_id": "interface_detail", - "key_fields": [ - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "admin", - "port_status", - "ip_mtu", - "mtu", - "mpls_mtu", - "ipv6_mtu", - "port_media", - "negotiation" - ], - "display_fields": [ - "interface", - "admin", - "input_bps", - "output_bps", - "port_status", - "in_util", - "out_util", - "ip_mtu", - "mtu", - "mpls_mtu", - "ipv6_mtu", - "port_media", - "negotiation", - "bw", - "description", - "rate_period" - ], - "row_filters": [], - "field_rules": [ - { - "field": "input_bps", - "compare": "percent", - "tolerance": 5 - }, - { - "field": "output_bps", - "compare": "percent", - "tolerance": 5 - }, - { - "field": "in_util", - "compare": "numeric", - "tolerance": 5 - }, - { - "field": "out_util", - "compare": "numeric", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "ospf_neighbor", - "title": "ospf_neighbor", - "metric_id": "ospf_neighbor", - "key_fields": [ - "neighbor_id", - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state" - ], - "display_fields": [ - "neighbor_id", - "interface", - "state", - "process_id", - "address" - ], - "row_filters": [], - "field_rules": [] - }, - { - "sheet_id": "vrrp.ipv4", - "title": "VRRP IPv4", - "metric_id": "vrrp", - "key_fields": [ - "af", - "interface", - "vr_id" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state", - "priority", - "master_addr", - "vrouter_addr" - ], - "display_fields": [ - "af", - "interface", - "vr_id", - "state", - "priority", - "master_addr", - "vrouter_addr" - ], - "row_filters": [ - { - "op": "eq", - "field": "af", - "value": "ipv4" - } - ], - "field_rules": [] - }, - { - "sheet_id": "vrrp.ipv6", - "title": "VRRP IPv6", - "metric_id": "vrrp", - "key_fields": [ - "af", - "interface", - "vr_id" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state", - "priority", - "master_addr", - "vrouter_addr" - ], - "display_fields": [ - "af", - "interface", - "vr_id", - "state", - "priority", - "master_addr", - "vrouter_addr" - ], - "row_filters": [ - { - "op": "eq", - "field": "af", - "value": "ipv6" - } - ], - "field_rules": [] - }, - { - "sheet_id": "optical_brief", - "title": "optical_brief", - "metric_id": "optical_brief", - "key_fields": [ - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "status", - "rx_power" - ], - "display_fields": [ - "interface", - "status", - "rx_power", - "tx_power", - "optic_type", - "wavelength", - "rx_threshold", - "tx_threshold" - ], - "row_filters": [], - "field_rules": [ - { - "field": "rx_power", - "compare": "numeric", - "tolerance": 2 - } - ] - }, - { - "sheet_id": "lldp_neighbor", - "title": "lldp_neighbor", - "metric_id": "lldp_neighbor", - "key_fields": [ - "local_if", - "remote_sys", - "remote_if" - ], - "iface_fields": [ - "local_if" - ], - "compare_fields": [], - "display_fields": [ - "local_if", - "remote_sys", - "remote_if", - "remote_ip", - "protocol" - ], - "row_filters": [], - "field_rules": [] - }, - { - "sheet_id": "l2vpn_pw_detail", - "title": "l2vpn_pw_detail", - "metric_id": "l2vpn_pw_detail", - "key_fields": [ - "peer", - "vcid", - "service_instance" - ], - "iface_fields": [], - "compare_fields": [ - "vc_status", - "remote_status", - "vc_type", - "control_word" - ], - "display_fields": [ - "peer", - "vcid", - "service_instance", - "vc_status", - "remote_status", - "vc_type", - "control_word", - "pw_name", - "activation_status", - "service_instance_type", - "conn_mode", - "signaling", - "tunnel_dest" - ], - "row_filters": [], - "field_rules": [] - }, - { - "sheet_id": "l2vpn_mac", - "title": "l2vpn_mac", - "metric_id": "l2vpn_mac", - "key_fields": [ - "mac", - "vpn", - "neighbor", - "ac_port", - "exter_vlan", - "vpn_sid", - "neighbor_sid" - ], - "iface_fields": [ - "ac_port" - ], - "compare_fields": [], - "display_fields": [ - "mac", - "vpn", - "neighbor", - "ac_port", - "exter_vlan", - "vpn_sid", - "neighbor_sid", - "vlan", - "pw", - "attribute" - ], - "row_filters": [], - "field_rules": [] - }, - { - "sheet_id": "bgp_route", - "title": "bgp_route_ipv4", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "empty", - "field": "vrf", - "value": "" - }, - { - "op": "eq", - "field": "afi", - "value": "ipv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.vpnv4", - "title": "bgp_route_vpnv4", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpnv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.ipv6", - "title": "bgp_route_ipv6", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "empty", - "field": "vrf", - "value": "" - }, - { - "op": "eq", - "field": "afi", - "value": "ipv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.vpnv6", - "title": "bgp_route_vpnv6", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpnv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.vpnv6_vrf", - "title": "bgp_route_vpnv6_vrf", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "not_empty", - "field": "vrf", - "value": "" - }, - { - "op": "eq", - "field": "afi", - "value": "ipv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.vpnv4_vrf", - "title": "bgp_route_vpnv4_vrf", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "not_empty", - "field": "vrf", - "value": "" - }, - { - "op": "eq", - "field": "afi", - "value": "ipv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - } - ] -} diff --git a/netx_api/biz_state/identity_pipeline.py b/netx_api/biz_state/identity_pipeline.py deleted file mode 100644 index d3f1b26..0000000 --- a/netx_api/biz_state/identity_pipeline.py +++ /dev/null @@ -1,454 +0,0 @@ -"""One identity pipeline for every compare path. - -A row's *match key* must come out identical whether it is computed while a run -executes, while its results are searched / hydrated / exported, while a mapping -is validated, or while a cutover monitor evaluates the same sheet. All of those -paths therefore build a :class:`Pipeline` and call the same methods instead of -re-implementing "normalize → map → key". - -Stages for one row on one side (``before`` or ``after``):: - - raw row ──normalize──▶ iface aliases (+vendor canonical) ▶ key normalize ▶ value_map - ──filter────▶ ``row_filters`` (v2: on the normalized row; legacy: on the raw row) - ──map───────▶ port map (before side only) - ──key───────▶ ``key_tuple`` over the match key fields - -``legacy=True`` reproduces the pre-pipeline behaviour (snapshot v1 runs): the -template alias rules are the only normalization, keys are never normalized, -there is no value map and the filters see the raw row. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Mapping, Sequence - -from .compare_rules import ( - normalize_value, - row_matches_filter, - scalar_text, -) -from .iface_normalize import ( - normalize_iface_name, - normalize_iface_rules, - normalize_port_map, - resolve_mapped_iface, - rules_for_side, - vendor_canonical_rules, -) - -SIDES = ("before", "after") -DIRECTIONAL_MODES = {"not_worse", "numeric_min", "numeric_max"} -UNMAPPED_POLICIES = ("by_name", "ignore_port") -DUPLICATE_PAIRING = ("sorted", "ordered") - - -def filter_fields(filt: Mapping[str, Any] | None) -> set[str]: - """Every field name a (possibly nested) row filter reads.""" - out: set[str] = set() - if not isinstance(filt, Mapping): - return out - for group in ("any", "all"): - if group in filt: - for kid in filt.get(group) or []: - out |= filter_fields(kid) - return out - name = str(filt.get("field") or "").strip() - if name: - out.add(name) - return out - - -def _vendor_key(vendor: str = "", device_type: str = "") -> str: - from ..topology_lldp import resolve_vendor_key - - key = resolve_vendor_key(vendor, device_type) - return "" if key == "generic" else key - - -@dataclass(eq=False) -class SideSpec: - side: str - vendor: str = "" - canonical: list[dict[str, str]] = field(default_factory=list) - alias_rules: list[dict[str, str]] = field(default_factory=list) - # field -> {lowercased source value: canonical value} - value_maps: dict[str, dict[str, str]] = field(default_factory=dict) - # key field -> normalize mode (only fields whose rule asks for one) - key_norms: dict[str, str] = field(default_factory=dict) - strip_inner_space: bool = True - - def iface(self, name: Any) -> str: - text = str(name or "").strip() - if not text: - return "" - if self.strip_inner_space: - text = "".join(text.split()) - if self.canonical: - text = normalize_iface_name(text, self.canonical) - if self.alias_rules: - text = normalize_iface_name(text, self.alias_rules) - return text - - @property - def has_iface_rules(self) -> bool: - return bool(self.canonical or self.alias_rules) - - def describe(self) -> dict[str, Any]: - """JSON-able record of the rules in force (stored in the run snapshot).""" - return { - "vendor": self.vendor, - "canonical": list(self.canonical), - "alias_rules": list(self.alias_rules), - "value_maps": {k: dict(v) for k, v in self.value_maps.items()}, - "key_norms": dict(self.key_norms), - } - - -@dataclass(eq=False) -class Pipeline: - key_fields: list[str] - iface_fields: list[str] - compare_fields: list[str] - field_rules: list[dict[str, Any]] - row_filters: list[dict[str, Any]] - port_map: dict[str, str] - before: SideSpec - after: SideSpec - legacy: bool = False - ignore_port_changes: bool | None = None - unmapped_port_policy: str = "by_name" - duplicate_pairing: str = "ordered" - _iface_set: set[str] = field(default_factory=set, repr=False) - _transformed: set[str] = field(default_factory=set, repr=False) - _targets: set[str] = field(default_factory=set, repr=False) - - def __post_init__(self) -> None: - self._iface_set = set(self.iface_fields) - self._transformed = set(self._iface_set) - for spec in (self.before, self.after): - self._transformed |= set(spec.value_maps) | set(spec.key_norms) - self._targets = set(self.port_map.values()) - - # -- side helpers ---------------------------------------------------- - def spec(self, side: str) -> SideSpec: - return self.before if side == "before" else self.after - - # -- stage 1: normalize ---------------------------------------------- - def normalize(self, row: Mapping[str, Any], side: str) -> dict[str, Any]: - out = dict(row) - spec = self.spec(side) - for f in self.iface_fields: - val = scalar_text(out.get(f)).strip() - if val: - out[f] = spec.iface(val) - if self.legacy: - return out - for f, mode in spec.key_norms.items(): - if f in out: - out[f] = normalize_value(out[f], mode) - for f, vmap in spec.value_maps.items(): - if f not in out: - continue - raw = scalar_text(out[f]).strip() - hit = vmap.get(raw.lower()) - if hit is not None: - out[f] = hit - return out - - def normalize_name(self, name: Any, side: str) -> str: - """Interface-name normalization only (port-map keys / values, selectors).""" - return self.spec(side).iface(name) - - # -- stage 2: filter ------------------------------------------------- - def filters_for(self, side: str) -> list[dict[str, Any]]: - out = [] - for f in self.row_filters: - fs = str(f.get("side") or "").strip().lower() - if not fs or fs == "both" or fs == side: - out.append(f) - return out - - def keep(self, row: Mapping[str, Any], side: str) -> bool: - """Evaluate row filters. v2 expects the *normalized* row, legacy the raw one.""" - return all(row_matches_filter(row, f) for f in self.filters_for(side)) - - def accepts(self, raw: Mapping[str, Any], norm: Mapping[str, Any], side: str) -> bool: - """Row-filter verdict: legacy reads the raw row, v2 the normalized one.""" - return self.keep(raw if self.legacy else norm, side) - - def pushdown_filters(self, side: str) -> list[dict[str, Any]]: - """Filters that read only untransformed fields, so they may run in SQL on raw rows.""" - if self.legacy: - return list(self.row_filters) - return [ - f for f in self.row_filters - if not str(f.get("side") or "").strip() - and not (filter_fields(f) & self._transformed) - ] - - def prepare(self, rows: Sequence[Mapping[str, Any]], side: str) -> list[tuple[dict[str, Any], dict[str, Any]]]: - """``[(original row, normalized row)]`` after filtering, in input order.""" - out: list[tuple[dict[str, Any], dict[str, Any]]] = [] - has_filters = bool(self.filters_for(side)) - for raw in rows: - if not isinstance(raw, dict): - continue - norm = self.normalize(raw, side) - if has_filters and not self.accepts(raw, norm, side): - continue - out.append((raw, norm)) - return out - - # -- stage 3: port map ----------------------------------------------- - def apply_port_map(self, norm_row: Mapping[str, Any]) -> dict[str, Any]: - out = dict(norm_row) - if not self.port_map: - return out - for f in self.iface_fields: - val = scalar_text(out.get(f)).strip() - if val: - out[f] = resolve_mapped_iface(val, self.port_map) - return out - - def is_mapped(self, norm_row: Mapping[str, Any], side: str) -> bool: - """True when every interface value of the row is covered by the port map.""" - if not self.iface_fields: - return True - values = [scalar_text(norm_row.get(f)).strip() for f in self.iface_fields] - values = [v for v in values if v] - if not values: - return False - for v in values: - if side == "before": - if resolve_mapped_iface(v, self.port_map) == v and v not in self.port_map: - return False - else: - if not self._after_value_is_target(v): - return False - return True - - def _after_value_is_target(self, value: str) -> bool: - if value in self._targets: - return True - parts = value.split(".") - return any(".".join(parts[:i]) in self._targets for i in range(len(parts) - 1, 0, -1)) - - def transform(self, row: Mapping[str, Any], side: str) -> dict[str, Any]: - norm = self.normalize(row, side) - return self.apply_port_map(norm) if side == "before" else norm - - # -- stage 4: key ---------------------------------------------------- - def effective_match_keys(self, *, unique_without_iface: bool | None = None) -> list[str]: - """Match-key fields once the port policy is resolved. - - With a port map the full key is used (the map aligns the names). Without - one, ``ignore_port_changes`` may drop the interface fields; ``None`` means - "auto", resolved by the caller via ``unique_without_iface``. - """ - keys = list(self.key_fields) - if self.port_map or not self._iface_set: - return keys - candidate = [k for k in keys if k not in self._iface_set] - if not candidate: - return keys - policy = self.ignore_port_changes - if policy is True: - return candidate - if policy is False: - return keys - return candidate if unique_without_iface else keys - - def ignores_ports_by_policy(self) -> bool: - return bool(self.port_map) and self.unmapped_port_policy == "ignore_port" and bool(self._iface_set) - - def key_of( - self, norm_row: Mapping[str, Any], side: str, match_keys: Sequence[str], - ) -> tuple[dict[str, Any], tuple[str, ...]]: - """``(row after port map, match key)`` for a *normalized* row.""" - row = self.apply_port_map(norm_row) if side == "before" else dict(norm_row) - if self.ignores_ports_by_policy(): - if not self.is_mapped(norm_row, side): - reduced = [k for k in match_keys if k not in self._iface_set] or list(match_keys) - return row, ("U",) + tuple(scalar_text(row.get(k)).strip() for k in reduced) - return row, ("M",) + tuple(scalar_text(row.get(k)).strip() for k in match_keys) - return row, tuple(scalar_text(row.get(k)).strip() for k in match_keys) - - def match_key( - self, row: Mapping[str, Any], side: str, match_keys: Sequence[str] | None = None, - ) -> tuple[str, ...]: - """Match key of a *raw* row (normalize → port map → key).""" - keys = list(match_keys) if match_keys is not None else self.effective_match_keys() - return self.key_of(self.normalize(row, side), side, keys)[1] - - # -- duplicate pairing ----------------------------------------------- - def pair_sort_key(self, norm_row: Mapping[str, Any]) -> tuple[str, ...]: - rules = {str(r.get("field")): r for r in self.field_rules if isinstance(r, dict)} - out = [] - for f in self.compare_fields: - mode = str((rules.get(f) or {}).get("normalize") or "strip") - out.append(normalize_value(norm_row.get(f), mode)) - return tuple(out) - - # -- SQL eligibility ------------------------------------------------- - def sql_blockers(self, used_fields: set[str]) -> str: - """Reason the SQL engine must not run this sheet ('' when it may).""" - if self.legacy: - return "" - for spec in (self.before, self.after): - if set(spec.value_maps) & used_fields: - return "value_map" - if set(spec.key_norms) & set(self.key_fields): - return "key_normalize" - if used_fields & self._iface_set and (self.before.has_iface_rules or self.after.has_iface_rules): - return "iface_normalize" - if any(str(f.get("side") or "").strip() for f in self.row_filters): - return "row_filters_side" - if any(filter_fields(f) & self._transformed for f in self.row_filters): - return "row_filters_post_normalize" - if any( - str(r.get("compare") or "").strip().lower() in DIRECTIONAL_MODES - for r in self.field_rules if isinstance(r, dict) - ): - return "directional_compare" - return "" - - def describe(self) -> dict[str, Any]: - return { - "legacy": self.legacy, - "before": self.before.describe(), - "after": self.after.describe(), - "unmapped_port_policy": self.unmapped_port_policy, - "duplicate_pairing": self.duplicate_pairing, - } - - -def _field_rule_specs( - field_rules: Sequence[Mapping[str, Any]], key_fields: Sequence[str], side: str, -) -> tuple[dict[str, dict[str, str]], dict[str, str]]: - value_maps: dict[str, dict[str, str]] = {} - key_norms: dict[str, str] = {} - key_set = set(key_fields) - for rule in field_rules or []: - if not isinstance(rule, Mapping): - continue - name = str(rule.get("field") or "").strip() - if not name: - continue - rs = str(rule.get("side") or "").strip().lower() - if rs and rs != "both" and rs != side: - continue - apply_to_key = rule.get("apply_to_key") - apply_to_key = (name in key_set) if apply_to_key is None else bool(apply_to_key) - is_key = name in key_set - if is_key and not apply_to_key: - continue - mode = str(rule.get("normalize") or "").strip().lower() - vmap = rule.get("value_map") - if is_key and mode and mode not in ("none", "strip"): - key_norms[name] = mode - if isinstance(vmap, Mapping) and vmap: - pre = mode if (mode and mode not in ("none", "strip")) else "strip" - built: dict[str, str] = {} - for src, dst in vmap.items(): - built[normalize_value(src, pre).lower()] = scalar_text(dst).strip() - value_maps[name] = built - return value_maps, key_norms - - -def build_pipeline( - sheet: Mapping[str, Any], - *, - iface_rules: Sequence[Mapping[str, Any]] | None, - port_map: Mapping[str, str] | None, - before_vendor: str = "", - after_vendor: str = "", - legacy: bool = False, - normalize_map: bool = True, -) -> Pipeline: - """Build the pipeline for one sheet. - - ``port_map`` is the map as the user wrote it. Unless ``normalize_map`` is - false (legacy runs, whose maps were written post-normalization) its keys are - normalized with the *before* side rules and its values with the *after* side - rules. - """ - key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] - iface_fields = [str(k).strip() for k in (sheet.get("iface_fields") or []) if str(k).strip()] - field_rules = [dict(r) for r in (sheet.get("field_rules") or []) if isinstance(r, Mapping)] - from .compare_rules import effective_compare_fields - - compare_fields = effective_compare_fields(list(sheet.get("compare_fields") or []), field_rules) - canonical_on = sheet.get("builtin_vendor_canonical", True) is not False and not legacy - all_rules = normalize_iface_rules(list(iface_rules or [])) - - specs: dict[str, SideSpec] = {} - for side, vendor_raw in (("before", before_vendor), ("after", after_vendor)): - vendor = _vendor_key(vendor_raw) if vendor_raw else "" - if legacy: - alias = normalize_iface_rules([{"from": r["from"], "to": r["to"]} for r in all_rules]) - value_maps, key_norms = {}, {} - else: - alias = rules_for_side(all_rules, side, vendor) - value_maps, key_norms = _field_rule_specs(field_rules, key_fields, side) - specs[side] = SideSpec( - side=side, vendor=vendor, - canonical=vendor_canonical_rules(vendor) if canonical_on else [], - alias_rules=alias, value_maps=value_maps, key_norms=key_norms, - strip_inner_space=not legacy, - ) - - pmap = dict(port_map or {}) - if pmap and normalize_map and not legacy: - pmap, _ = normalize_port_map( - pmap, None, - key_normalizer=specs["before"].iface, value_normalizer=specs["after"].iface, - ) - ignore_ports = sheet.get("ignore_port_changes") - policy = str(sheet.get("unmapped_port_policy") or "by_name").strip().lower() - pairing = str(sheet.get("duplicate_pairing") or ("ordered" if legacy else "sorted")).strip().lower() - return Pipeline( - key_fields=key_fields, - iface_fields=iface_fields, - compare_fields=compare_fields, - field_rules=field_rules, - row_filters=[dict(f) for f in (sheet.get("row_filters") or []) if isinstance(f, Mapping)], - port_map=pmap, - before=specs["before"], - after=specs["after"], - legacy=legacy, - ignore_port_changes=None if ignore_ports is None else bool(ignore_ports), - unmapped_port_policy=policy if policy in UNMAPPED_POLICIES else "by_name", - duplicate_pairing="ordered" if legacy else (pairing if pairing in DUPLICATE_PAIRING else "sorted"), - ) - - -def legacy_pipeline( - *, - key_fields: Sequence[str], - iface_fields: Sequence[str], - compare_fields: Sequence[str], - field_rules: Sequence[Mapping[str, Any]] | None = None, - port_map: Mapping[str, str] | None = None, - iface_normalize_rules: Sequence[Mapping[str, Any]] | None = None, - ignore_port_changes: bool | None = None, - row_filters: Sequence[Mapping[str, Any]] | None = None, - duplicate_pairing: str = "ordered", -) -> Pipeline: - """Pipeline with the pre-v2 semantics, for snapshot-v1 runs and bare callers.""" - alias = normalize_iface_rules( - [{"from": r.get("from"), "to": r.get("to")} for r in (iface_normalize_rules or [])] - ) - return Pipeline( - key_fields=[str(k) for k in key_fields], - iface_fields=[str(f) for f in iface_fields if str(f).strip()], - compare_fields=[str(f) for f in compare_fields], - field_rules=[dict(r) for r in (field_rules or []) if isinstance(r, Mapping)], - row_filters=[dict(f) for f in (row_filters or []) if isinstance(f, Mapping)], - port_map=dict(port_map or {}), - before=SideSpec(side="before", alias_rules=list(alias), strip_inner_space=False), - after=SideSpec(side="after", alias_rules=list(alias), strip_inner_space=False), - legacy=True, - ignore_port_changes=ignore_port_changes, - duplicate_pairing=duplicate_pairing if duplicate_pairing in DUPLICATE_PAIRING else "ordered", - ) diff --git a/netx_api/biz_state/iface_normalize.py b/netx_api/biz_state/iface_normalize.py index 7018d63..5fdb793 100644 --- a/netx_api/biz_state/iface_normalize.py +++ b/netx_api/biz_state/iface_normalize.py @@ -39,14 +39,7 @@ def normalize_iface_rules(raw: Any) -> list[dict[str, str]]: fr = str(item.get("from") or item.get("src") or "").strip() to = str(item.get("to") or item.get("dst") or "").strip() if fr and to: - rule = {"from": fr, "to": to} - side = str(item.get("side") or "").strip().lower() - if side in ("before", "after"): - rule["side"] = side - vendor = str(item.get("vendor") or "").strip().lower() - if vendor: - rule["vendor"] = vendor - out.append(rule) + out.append({"from": fr, "to": to}) # Longest prefix first so XXVGE wins over XGE / GE. out.sort(key=lambda r: len(r["from"]), reverse=True) return out @@ -114,47 +107,6 @@ def apply_iface_normalize_rows( ] -def normalize_port_map( - port_map: Mapping[str, str] | None, - rules: Sequence[Mapping[str, str]] | None, - value_rules: Sequence[Mapping[str, str]] | None = None, - *, - key_normalizer: Any = None, - value_normalizer: Any = None, -) -> tuple[dict[str, str], list[dict[str, str]]]: - """Normalize port-map keys and values with the same rules as iface fields. - - ``value_rules`` (default: ``rules``) normalizes the after-side values, for - per-side rules. ``key_normalizer`` / ``value_normalizer`` are callables that - replace the rule-based normalization (used by the identity pipeline). - - Returns ``(normalized_map, conflicts)``. A conflict is two raw keys that - normalize to the same key but point at different (normalized) targets; the - first one wins in ``normalized_map``. - """ - out: dict[str, str] = {} - origin: dict[str, str] = {} - conflicts: list[dict[str, str]] = [] - for raw_k, raw_v in (port_map or {}).items(): - k = key_normalizer(raw_k) if key_normalizer else normalize_iface_name(raw_k, rules) - v = ( - value_normalizer(raw_v) if value_normalizer - else normalize_iface_name(raw_v, rules if value_rules is None else value_rules) - ) - if not k or not v: - continue - if k in out: - if out[k] != v: - conflicts.append({ - "before_if": k, "kept_from": origin[k], "kept_to": out[k], - "dropped_from": str(raw_k), "dropped_to": v, - }) - continue - out[k] = v - origin[k] = str(raw_k) - return out, conflicts - - def resolve_mapped_iface(name: str, port_map: Mapping[str, str] | None) -> str: """Exact map hit, else longest mapped parent + remaining suffix (QinQ-safe). @@ -190,53 +142,3 @@ def default_zte_iface_normalize_rules() -> list[dict[str, str]]: {"from": "SG", "to": "smartgroup"}, ] ) - - -# --------------------------------------------------------------------------- -# Built-in per-vendor canonical names (same-vendor synonyms only; cross-vendor -# numbering is the port map's job). Applied before template alias rules. -# --------------------------------------------------------------------------- - -VENDOR_CANONICAL: dict[str, list[dict[str, str]]] = { - "huawei": [ - {"from": "XGigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"}, - {"from": "100GE", "to": "100GE"}, {"from": "40GE", "to": "40GE"}, {"from": "25GE", "to": "25GE"}, - {"from": "10GE", "to": "10GE"}, {"from": "Eth-Trunk", "to": "Eth-Trunk"}, - ], - "cisco": [ - {"from": "GigabitEthernet", "to": "Gi"}, {"from": "TenGigabitEthernet", "to": "Te"}, - {"from": "TenGigE", "to": "Te"}, {"from": "HundredGigE", "to": "Hu"}, {"from": "FortyGigE", "to": "Fo"}, - {"from": "TwentyFiveGigE", "to": "Twe"}, {"from": "FastEthernet", "to": "Fa"}, - {"from": "Bundle-Ether", "to": "BE"}, {"from": "Loopback", "to": "Lo"}, - ], - "h3c": [ - {"from": "Ten-GigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"}, - {"from": "FortyGigE", "to": "FGE"}, {"from": "HundredGigE", "to": "HGE"}, - {"from": "Bridge-Aggregation", "to": "BAGG"}, - ], - "zte": [ - {"from": "xxvgei", "to": "xxvgei"}, {"from": "xgei", "to": "xgei"}, {"from": "cgei", "to": "cgei"}, - {"from": "gei", "to": "gei"}, {"from": "smartgroup", "to": "smartgroup"}, - ], -} - - -def vendor_canonical_rules(vendor: str) -> list[dict[str, str]]: - return normalize_iface_rules(VENDOR_CANONICAL.get(str(vendor or "").strip().lower(), [])) - - -def rules_for_side( - rules: Sequence[Mapping[str, str]] | None, side: str, vendor: str = "", -) -> list[dict[str, str]]: - """Template alias rules that apply to ``side`` (``before``/``after``) and ``vendor``.""" - vendor_n = str(vendor or "").strip().lower() - out: list[dict[str, str]] = [] - for r in rules or []: - rs = str(r.get("side") or "").strip().lower() - if rs and rs != side: - continue - rv = str(r.get("vendor") or "").strip().lower() - if rv and rv != vendor_n: - continue - out.append({"from": str(r.get("from") or ""), "to": str(r.get("to") or "")}) - return normalize_iface_rules(out) diff --git a/netx_api/biz_state/profiles.py b/netx_api/biz_state/profiles.py index 70fcf88..996d46b 100644 --- a/netx_api/biz_state/profiles.py +++ b/netx_api/biz_state/profiles.py @@ -85,9 +85,6 @@ class ParseProfile: enrich_joins: list[EnrichJoin] = field(default_factory=list) # light: default shared SSH lane; heavy: dedicated long-timeout connection collect_lane: str = "light" - # Which slice of the metric this command produces, e.g. {"afi": "ipv4"}. - # Empty = the command covers the whole metric (or its slicing is unknown). - slice: dict[str, str] = field(default_factory=dict) _LLDP_FIELDS: list[FieldDef] = [ @@ -1654,11 +1651,6 @@ _HEAVY_LANE_METRICS = frozenset( ) -def metric_lane(metric_id: str) -> str: - """``heavy`` for huge CLI dumps (own SSH connection), else ``light``.""" - return "heavy" if str(metric_id or "").strip() in _HEAVY_LANE_METRICS else "light" - - def _apply_collect_lanes(profiles: list[ParseProfile]) -> list[ParseProfile]: for p in profiles: if str(p.metric_id or "").strip() in _HEAVY_LANE_METRICS: @@ -1760,41 +1752,12 @@ def _validate_profiles(profiles: list[ParseProfile]) -> None: ) -# profile_id -> slice. Compare sheets filter on these fields (``afi``/``af``/``direction``), -# so a missing command is detected as "slice not collected" instead of "business lost". -_PROFILE_SLICES: dict[str, dict[str, str]] = { - **{f"zte.bgp_{afi}_summary": {"afi": afi} for afi in ("vpnv4", "ipv4", "vpnv6", "ipv6", "evpn", "vpls")}, - "zte.bgp_vpnv4_vrf_summary": {"afi": "vpnv4"}, - "zte.bgp_vpnv6_vrf_summary": {"afi": "vpnv6"}, - "zte.vrrp_ipv4": {"af": "ipv4"}, - "zte.vrrp_ipv6": {"af": "ipv6"}, - **{ - f"zte.bgp_{afi}_neighbor_{direction}": {"afi": afi, "direction": direction} - for afi in ("vpnv4", "vpnv6", "ipv4", "ipv6") - for direction in ("in", "out") - }, - **{ - f"zte.bgp_{afi}_vrf_neighbor_{direction}": {"afi": afi, "direction": direction} - for afi in ("vpnv4", "vpnv6") - for direction in ("in", "out") - }, -} - - -def _apply_profile_slices(profiles: list[ParseProfile]) -> list[ParseProfile]: - for p in profiles: - declared = _PROFILE_SLICES.get(p.profile_id) - if declared and not p.slice: - p.slice = dict(declared) - return profiles - - def _finalize_profiles(profiles: list[ParseProfile]) -> list[ParseProfile]: """Normalize discover→aux, validate, apply collect lanes.""" for p in profiles: _normalize_discover_aux(p) _validate_profiles(profiles) - return _apply_profile_slices(_apply_collect_lanes(profiles)) + return _apply_collect_lanes(profiles) def all_profiles() -> list[ParseProfile]: diff --git a/netx_api/biz_state/retention.py b/netx_api/biz_state/retention.py index bd3ccf8..d14c554 100644 --- a/netx_api/biz_state/retention.py +++ b/netx_api/biz_state/retention.py @@ -6,17 +6,12 @@ from collections import defaultdict from datetime import datetime, timedelta from typing import Any -from sqlalchemy import func, or_, select from sqlalchemy.orm import Session from ..models import ( - BizCompareDiff, BizCompareJob, BizCompareRun, - BizMigrationBatch, - BizMigrationDiff, BizMigrationProject, - BizMigrationRedTicket, BizMigrationRun, BizStateBatch, BizStateBatchCommand, @@ -31,158 +26,48 @@ def _utcnow() -> datetime: return datetime.utcnow() -def protected_batch_map( - db: Session, *, task_id: str = "", batch_ids: list[str] | None = None, -) -> dict[str, list[str]]: - """Only read reference columns, scoped to the requested task/batches when given.""" +def protected_batch_map(db: Session, *, task_id: str = "") -> dict[str, list[str]]: + """batch_id → reason codes. Empty task_id = all tasks.""" out: dict[str, list[str]] = defaultdict(list) - wanted = set(batch_ids) if batch_ids is not None else None - if wanted == set(): - return {} - scope = select(BizStateBatch.id) - q = db.query(BizStateBatch.id, BizStateBatch.is_baseline, BizStateBatch.status) - if task_id: - scope = scope.where(BizStateBatch.task_id == task_id) - q = q.filter(BizStateBatch.task_id == task_id) - if wanted is not None: - scope = scope.where(BizStateBatch.id.in_(wanted)) - q = q.filter(BizStateBatch.id.in_(wanted)) - batch_rows = q.all() if task_id or wanted is not None else q.filter(or_( - BizStateBatch.is_baseline.is_(True), BizStateBatch.status.in_(("queued", "running")), - )).all() - if task_id: - wanted = {row.id for row in batch_rows} def add(bid: str, reason: str) -> None: b = str(bid or "").strip() - if not b or (wanted is not None and b not in wanted): + if not b: return if reason not in out[b]: out[b].append(reason) - for b in batch_rows: - if b.is_baseline: - add(b.id, "manual_baseline") - if b.status in ("queued", "running"): - add(b.id, "active_collection") + q = db.query(BizStateBatch) + if task_id: + q = q.filter(BizStateBatch.task_id == task_id) + for b in q.filter(BizStateBatch.is_baseline.is_(True)).all(): + add(b.id, "manual_baseline") - references = ( - (BizCompareJob.before_batch_id, BizCompareJob.after_batch_id, "compare_job_before", "compare_job_after"), - (BizCompareRun.before_batch_id, BizCompareRun.after_batch_id, "compare_run_before", "compare_run_after"), - (BizMigrationProject.old_baseline_batch_id, BizMigrationProject.new_baseline_batch_id, "migration_old_baseline", "migration_new_baseline"), - (BizMigrationRun.old_batch_id, BizMigrationRun.new_batch_id, "migration_run_old", "migration_run_new"), - ) - for before, after, before_reason, after_reason in references: - refs = db.query(before, after) - if task_id or batch_ids is not None: - refs = refs.filter(or_(before.in_(scope), after.in_(scope))) - for before_id, after_id in refs: - add(before_id, before_reason) - add(after_id, after_reason) + for j in db.query(BizCompareJob).all(): + add(j.before_batch_id, "compare_job_before") + add(j.after_batch_id, "compare_job_after") - # Every batch a *kept* acceptance/manual run (or the newest auto run of a wave) - # compared, not only the first binding. Older auto runs are bounded by - # ``purge_auto_runs`` and keep protecting just their primary pair above. - for run_batches in _kept_run_metric_batches(db): - for bid in run_batches: - add(bid, "migration_run_metric") + for r in db.query(BizCompareRun).all(): + add(r.before_batch_id, "compare_run_before") + add(r.after_batch_id, "compare_run_after") + + for p in db.query(BizMigrationProject).all(): + add(p.old_baseline_batch_id, "migration_old_baseline") + add(p.new_baseline_batch_id, "migration_new_baseline") + + for r in db.query(BizMigrationRun).all(): + add(r.old_batch_id, "migration_run_old") + add(r.new_batch_id, "migration_run_new") return dict(out) -def _kept_run_metric_batches(db: Session, *, limit: int = 500) -> list[set[str]]: - newest_auto = ( - db.query(func.max(BizMigrationRun.created_at).label("at"), BizMigrationRun.batch_id.label("bid")) - .filter(BizMigrationRun.purpose == "auto").group_by(BizMigrationRun.batch_id).subquery() - ) - latest_auto_ids = [ - r[0] for r in db.query(BizMigrationRun.id).join( - newest_auto, - (BizMigrationRun.batch_id == newest_auto.c.bid) & (BizMigrationRun.created_at == newest_auto.c.at), - ).limit(limit).all() - ] - kept_ids = [ - r[0] for r in db.query(BizMigrationRun.id).filter(BizMigrationRun.purpose != "auto") - .order_by(BizMigrationRun.created_at.desc()).limit(limit).all() - ] + latest_auto_ids - out: list[set[str]] = [] - for start in range(0, len(kept_ids), 100): - for (summary,) in db.query(BizMigrationRun.summary_json).filter( - BizMigrationRun.id.in_(kept_ids[start:start + 100]) - ): - batches: set[str] = set() - for item in ((summary or {}).get("metric_batches") or {}).values(): - if isinstance(item, dict): - batches.update(str(item.get(k) or "") for k in ("old_batch_id", "new_batch_id")) - batches.discard("") - if batches: - out.append(batches) - return out - - -def purge_auto_runs(db: Session, *, migration_keep: int = 300, compare_keep: int = 50) -> dict[str, int]: - """Bound auto-generated run history. - - Cutover monitoring writes a run (+ one diff row per object) every minute and - auto-compare writes one per new sample. Acceptance / manual runs and runs a - red ticket points at are kept forever; only the newest ``keep`` auto runs of - each wave / job survive. - """ - dropped = {"migration_runs": 0, "compare_runs": 0} - ticket_runs = {r[0] for r in db.query(BizMigrationRedTicket.run_id).filter( - BizMigrationRedTicket.run_id != "").distinct()} - accepted = {r[0] for r in db.query(BizMigrationBatch.accept_run_id).filter( - BizMigrationBatch.accept_run_id != "")} - keep_forever = ticket_runs | accepted - - def surplus(model, group_col, extra_filter, keep: int) -> list[str]: - over = [g for g, n in db.query(group_col, func.count(model.id)).filter(*extra_filter) - .group_by(group_col).all() if n > keep] - ids: list[str] = [] - for g in over: - rows = db.query(model.id).filter(group_col == g, *extra_filter).order_by( - model.created_at.desc(), model.id.desc()).offset(keep).all() - ids.extend(r[0] for r in rows if r[0] not in keep_forever) - return ids - - mig_ids = surplus(BizMigrationRun, BizMigrationRun.batch_id, [BizMigrationRun.purpose == "auto"], - max(1, int(migration_keep))) - for start in range(0, len(mig_ids), 500): - chunk = mig_ids[start:start + 500] - db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id.in_(chunk)).delete(synchronize_session=False) - db.query(BizMigrationRun).filter(BizMigrationRun.id.in_(chunk)).delete(synchronize_session=False) - dropped["migration_runs"] = len(mig_ids) - - # Auto-mode jobs only. A run's origin lives in summary_json["purpose"]; unmarked runs - # are manual and stay. Only the surplus beyond ``compare_keep`` is inspected. - cmp_ids: list[str] = [] - keep_n = max(1, int(compare_keep)) - for (job_id,) in db.query(BizCompareJob.id).filter(BizCompareJob.mode == "auto").all(): - surplus_ids = [r[0] for r in db.query(BizCompareRun.id).filter( - BizCompareRun.job_id == job_id, BizCompareRun.status.notin_(("queued", "running")), - ).order_by(BizCompareRun.created_at.desc(), BizCompareRun.id.desc()).offset(keep_n).all()] - for start in range(0, len(surplus_ids), 200): - for rid, summary in db.query(BizCompareRun.id, BizCompareRun.summary_json).filter( - BizCompareRun.id.in_(surplus_ids[start:start + 200]) - ): - if (summary or {}).get("purpose") == "auto": - cmp_ids.append(rid) - for start in range(0, len(cmp_ids), 500): - chunk = cmp_ids[start:start + 500] - db.query(BizCompareDiff).filter(BizCompareDiff.run_id.in_(chunk)).delete(synchronize_session=False) - db.query(BizCompareRun).filter(BizCompareRun.id.in_(chunk)).delete(synchronize_session=False) - dropped["compare_runs"] = len(cmp_ids) - if mig_ids or cmp_ids: - db.commit() - return dropped - - def protected_batch_ids(db: Session, *, task_id: str = "") -> set[str]: return set(protected_batch_map(db, task_id=task_id).keys()) def batch_protect_info(db: Session, batch_id: str) -> dict[str, Any]: - reasons = protected_batch_map(db, batch_ids=[batch_id]).get(batch_id, []) + reasons = protected_batch_map(db).get(batch_id, []) b = db.get(BizStateBatch, batch_id) if b and bool(getattr(b, "is_baseline", False)) and "manual_baseline" not in reasons: reasons = ["manual_baseline", *reasons] diff --git a/netx_api/biz_state/schema_ensure.py b/netx_api/biz_state/schema_ensure.py index 1c11933..6b1013f 100644 --- a/netx_api/biz_state/schema_ensure.py +++ b/netx_api/biz_state/schema_ensure.py @@ -23,7 +23,6 @@ def apply_biz_state_schema(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_command_batch_id ON biz_state_batch_command (batch_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_lldp_neighbor_batch_id ON biz_state_lldp_neighbor (batch_id)", "ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS enabled_sheet_ids JSON DEFAULT '[]'", - "ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS store_unchanged VARCHAR(16) DEFAULT 'auto'", "CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)", @@ -32,8 +31,6 @@ def apply_biz_state_schema(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_id ON biz_compare_diff (run_id)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_kind ON biz_compare_diff (run_id, metric_id, kind)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_seq ON biz_compare_diff (run_id, metric_id, seq)", - "ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS before_row_id VARCHAR(64) DEFAULT ''", - "ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS after_row_id VARCHAR(64) DEFAULT ''", "CREATE INDEX IF NOT EXISTS ix_biz_migration_project_status ON biz_migration_project (status)", "CREATE INDEX IF NOT EXISTS ix_biz_migration_batch_project_id ON biz_migration_batch (project_id)", "CREATE INDEX IF NOT EXISTS ix_biz_migration_run_batch_id ON biz_migration_run (batch_id)", @@ -82,15 +79,6 @@ def apply_biz_state_schema(conn: Connection) -> None: "ALTER TABLE biz_migration_project ADD COLUMN IF NOT EXISTS new_hf_bindings_json JSON DEFAULT '[]'", "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS purpose VARCHAR(32) DEFAULT ''", "CREATE INDEX IF NOT EXISTS ix_biz_state_task_purpose ON biz_state_task (purpose)", - "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS collect_group_id VARCHAR(128) DEFAULT ''", - "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS collect_manual_only BOOLEAN DEFAULT FALSE", - "CREATE INDEX IF NOT EXISTS ix_biz_state_task_collect_group_id ON biz_state_task (collect_group_id)", - "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_group_id VARCHAR(128) DEFAULT ''", - "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_round_id VARCHAR(64) DEFAULT ''", - "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_priority INTEGER DEFAULT 0", - "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS queued_at TIMESTAMP", - "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_collect_group_id ON biz_state_batch (collect_group_id)", - "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_collect_round_id ON biz_state_batch (collect_round_id)", "ALTER TABLE biz_state_batch_command ADD COLUMN IF NOT EXISTS raw_line_count INTEGER DEFAULT 0", "ALTER TABLE biz_state_batch_command ADD COLUMN IF NOT EXISTS declared_total INTEGER DEFAULT 0", "ALTER TABLE biz_migration_red_ticket ADD COLUMN IF NOT EXISTS match_key_str VARCHAR(256) DEFAULT ''", diff --git a/netx_api/biz_state/service.py b/netx_api/biz_state/service.py index 95e191f..f3ad60a 100644 --- a/netx_api/biz_state/service.py +++ b/netx_api/biz_state/service.py @@ -9,8 +9,8 @@ from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import String, case, cast, func, or_ -from sqlalchemy.orm import Session, defer +from sqlalchemy import String, cast, func, or_ +from sqlalchemy.orm import Session from ..lldp_shared import resolve_vendor_key from ..models import ( @@ -354,15 +354,13 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]: .order_by(BizStateTaskItem.sort_order.asc()) .all() ) - bindings_by_item: dict[str, list[Any]] = {} - if items: - for binding in db.query(BizStateTaskItemBinding).filter( - BizStateTaskItemBinding.item_id.in_([it.id for it in items]) - ).all(): - bindings_by_item.setdefault(binding.item_id, []).append(binding) item_out = [] for it in items: - binds = bindings_by_item.get(it.id, []) + binds = ( + db.query(BizStateTaskItemBinding) + .filter(BizStateTaskItemBinding.item_id == it.id) + .all() + ) item_out.append( { "id": it.id, @@ -375,11 +373,6 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]: "bindings": [{"placeholder": b.placeholder, "value": b.value} for b in binds], } ) - return {**_task_summary(task), "items": item_out} - - -def _task_summary(task: BizStateTask) -> dict[str, Any]: - """Task metadata without command items or bindings (safe for frequent polling).""" return { "id": task.id, "source": task.source, @@ -396,8 +389,6 @@ def _task_summary(task: BizStateTask) -> dict[str, Any]: "daily_keep_enabled": bool(getattr(task, "daily_keep_enabled", False)), "daily_keep_count": int(getattr(task, "daily_keep_count", None) or 10), "collect_running": bool(task.collect_running), - "collect_group_id": task.collect_group_id or "", - "collect_manual_only": bool(task.collect_manual_only), "last_collect_started_at": task.last_collect_started_at.isoformat() + "Z" if task.last_collect_started_at else None, @@ -405,16 +396,10 @@ def _task_summary(task: BizStateTask) -> dict[str, Any]: if task.last_collect_ended_at else None, "last_error": task.last_error, + "items": item_out, } -def get_task_progress(db: Session, task_id: str) -> dict[str, Any]: - task = db.get(BizStateTask, task_id) - if not task: - raise HTTPException(status_code=404, detail="task_not_found") - return _task_summary(task) - - def list_tasks(db: Session, *, purpose: str | None = None) -> list[dict[str, Any]]: from sqlalchemy import or_ @@ -446,8 +431,6 @@ def list_tasks(db: Session, *, purpose: str | None = None) -> list[dict[str, Any "status": t.status, "interval_sec": t.interval_sec, "collect_running": bool(t.collect_running), - "collect_group_id": t.collect_group_id or "", - "collect_manual_only": bool(t.collect_manual_only), "last_error": t.last_error, "last_collect_started_at": t.last_collect_started_at.isoformat() + "Z" if t.last_collect_started_at @@ -464,8 +447,6 @@ def delete_task(db: Session, task_id: str) -> None: task = db.get(BizStateTask, task_id) if not task: raise HTTPException(status_code=404, detail="task_not_found") - if task.collect_running: - raise HTTPException(status_code=409, detail="task_collecting") batches = db.query(BizStateBatch).filter(BizStateBatch.task_id == task_id).all() # Refuse if any batch is still referenced by compare/migration (manual baseline is OK to drop with task) pmap = protected_batch_map(db, task_id=task_id) @@ -519,7 +500,7 @@ def list_batches(db: Session, task_id: str, *, limit: int = 50) -> list[dict[str .limit(max(1, min(500, int(limit)))) .all() ) - pmap = protected_batch_map(db, batch_ids=[b.id for b in rows]) + pmap = protected_batch_map(db, task_id=task_id) out: list[dict[str, Any]] = [] for b in rows: reasons = list(pmap.get(b.id, [])) @@ -641,29 +622,12 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: b = db.get(BizStateBatch, batch_id) if not b: raise HTTPException(status_code=404, detail="batch_not_found") - raw = func.coalesce(BizStateBatchCommand.raw_text, "") - raw_len = func.length(raw) - # Compute legacy counts in SQL; workbook summaries never transfer raw CLI text. - line_count = case( - (BizStateBatchCommand.raw_line_count > 0, BizStateBatchCommand.raw_line_count), - (raw_len == 0, 0), - else_=raw_len - func.length(func.replace(raw, "\n", "")) - + case((func.substr(raw, raw_len, 1) == "\n", 0), else_=1), - ) - whitespace = " \t\r\n\v\f" - trimmed = (func.btrim(raw, whitespace) if db.get_bind().dialect.name == "postgresql" - else func.trim(raw, whitespace)) - cmd_raw_stats: dict[str, tuple[int, bool]] = {} - cmds = [] - for c, lines, has_raw in ( - db.query(BizStateBatchCommand, line_count, func.length(trimmed) > 0) - .options(defer(BizStateBatchCommand.raw_text)) + cmds = ( + db.query(BizStateBatchCommand) .filter(BizStateBatchCommand.batch_id == batch_id) - .order_by(BizStateBatchCommand.created_at.asc(), BizStateBatchCommand.id.asc()) + .order_by(BizStateBatchCommand.created_at.asc()) .all() - ): - cmds.append(c) - cmd_raw_stats[c.id] = (int(lines or 0), bool(has_raw)) + ) protect = batch_protect_info(db, batch_id) # Per-metric row counts (generic table) @@ -704,14 +668,11 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: # Prefer primary collect rows over aux / aux_cached for the same CLI sheet_cmds[id_].append(cmd_info) - # Aux failures must remain visible even if their primary command was stored later. - primary_cmds_by_cli = { - normalize_command(str(c.raw_command or "")) for c in cmds - if not str(c.parse_status or "").strip().lower().startswith("aux") - } + primary_cmds_by_cli: dict[str, dict[str, Any]] = {} for c in cmds: status = str(c.parse_status or "").strip().lower() is_aux = status.startswith("aux") + raw = c.raw_text or "" info = { "id": c.id, "profile_id": c.profile_id, @@ -721,13 +682,15 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: "params": c.params_json or {}, "parse_status": c.parse_status, "row_count": c.row_count, - "raw_line_count": cmd_raw_stats[c.id][0], + "raw_line_count": _cmd_raw_line_count(c), "declared_total": _cmd_declared_total(c), "message": c.message, - "has_raw": cmd_raw_stats[c.id][1], + "has_raw": bool(str(raw).strip()), "is_aux": is_aux, } cmd_n = normalize_command(str(c.raw_command or "")) + if not is_aux and cmd_n: + primary_cmds_by_cli.setdefault(cmd_n, info) # Commands sheet: hide successful aux when the same CLI already has a primary row; # keep failed/skipped aux visible so partial reasons are not hidden. if is_aux and cmd_n and cmd_n in primary_cmds_by_cli: @@ -783,10 +746,10 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: "params": c.params_json or {}, "parse_status": c.parse_status, "row_count": c.row_count, - "raw_line_count": cmd_raw_stats[c.id][0], + "raw_line_count": _cmd_raw_line_count(c), "declared_total": _cmd_declared_total(c), "message": c.message, - "has_raw": cmd_raw_stats[c.id][1], + "has_raw": bool(str(c.raw_text or "").strip()), "is_aux": True, } ) @@ -881,7 +844,6 @@ def list_batch_metric_rows( size_n = max(1, min(200, int(page_size or 50))) kw_n = str(kw or "").strip() col_n = str(column or "").strip() - like = "%" + kw_n.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" fields = metric_field_map().get(mid) or [] columns = [ @@ -897,30 +859,30 @@ def list_batch_metric_rows( if mid == "lldp_neighbor": q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id) if kw_n: + like = f"%{kw_n}%" if col_n == "local_if": - q = q.filter(BizStateLldpNeighbor.local_if.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.local_if.ilike(like)) elif col_n == "remote_sys": - q = q.filter(BizStateLldpNeighbor.remote_sys.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.remote_sys.ilike(like)) elif col_n == "remote_if": - q = q.filter(BizStateLldpNeighbor.remote_if.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.remote_if.ilike(like)) elif col_n == "remote_ip": - q = q.filter(BizStateLldpNeighbor.remote_ip.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.remote_ip.ilike(like)) elif col_n == "protocol": - q = q.filter(BizStateLldpNeighbor.protocol.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.protocol.ilike(like)) else: q = q.filter( or_( - BizStateLldpNeighbor.local_if.ilike(like, escape="\\"), - BizStateLldpNeighbor.remote_sys.ilike(like, escape="\\"), - BizStateLldpNeighbor.remote_if.ilike(like, escape="\\"), - BizStateLldpNeighbor.remote_ip.ilike(like, escape="\\"), - BizStateLldpNeighbor.protocol.ilike(like, escape="\\"), + BizStateLldpNeighbor.local_if.ilike(like), + BizStateLldpNeighbor.remote_sys.ilike(like), + BizStateLldpNeighbor.remote_if.ilike(like), + BizStateLldpNeighbor.remote_ip.ilike(like), + BizStateLldpNeighbor.protocol.ilike(like), ) ) total = int(q.count() or 0) rows_db = ( - q.order_by(BizStateLldpNeighbor.local_if.asc(), BizStateLldpNeighbor.remote_sys.asc(), - BizStateLldpNeighbor.remote_if.asc(), BizStateLldpNeighbor.id.asc()) + q.order_by(BizStateLldpNeighbor.local_if.asc()) .offset((page_n - 1) * size_n) .limit(size_n) .all() @@ -949,11 +911,12 @@ def list_batch_metric_rows( BizStateMetricRow.metric_id == mid, ) if kw_n: + like = f"%{kw_n}%" if col_n: # JSON path as text — works on Postgres JSONB and SQLite JSON - q = q.filter(cast(BizStateMetricRow.data_json[col_n].as_string(), String).ilike(like, escape="\\")) + q = q.filter(cast(BizStateMetricRow.data_json[col_n], String).ilike(like)) else: - q = q.filter(cast(BizStateMetricRow.data_json, String).ilike(like, escape="\\")) + q = q.filter(cast(BizStateMetricRow.data_json, String).ilike(like)) total = int(q.count() or 0) rows_db = ( q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) @@ -1056,64 +1019,56 @@ def export_batch_zip(db: Session, batch_id: str) -> bytes: db.query(BizStateLldpNeighbor) .filter(BizStateLldpNeighbor.batch_id == batch_id) .order_by(BizStateLldpNeighbor.local_if.asc()) - .yield_per(1000) + .all() ) - with zf.open("tables/lldp_neighbor.csv", "w") as dest: - dest.write(b"local_if,remote_sys,remote_if,remote_ip,protocol\n") - for n in neighbors: - line = ",".join(_csv(v) for v in ( - n.local_if, n.remote_sys, n.remote_if, n.remote_ip, n.protocol, - )) + "\n" - dest.write(line.encode("utf-8")) + csv_lines = ["local_if,remote_sys,remote_if,remote_ip,protocol"] + for n in neighbors: + csv_lines.append( + ",".join( + [ + _csv(n.local_if), + _csv(n.remote_sys), + _csv(n.remote_if), + _csv(n.remote_ip), + _csv(n.protocol), + ] + ) + ) + zf.writestr("tables/lldp_neighbor.csv", "\n".join(csv_lines) + "\n") # Generic metrics CSV (stream by metric_id) for sheet in detail.get("sheets") or []: mid = str(sheet.get("metric_id") or "").strip() if not mid or mid == "lldp_neighbor": continue - # First scan discovers the same complete, ordered header as the legacy export. - # Second scan writes rows without materializing the table or CSV. - col_keys: dict[str, None] = {} - has_rows = False - for rec in _iter_metric_export_rows(db, batch_id, mid): - has_rows = True - for k in rec.keys(): - col_keys.setdefault(str(k), None) - if not has_rows: + rows = ( + db.query(BizStateMetricRow) + .filter( + BizStateMetricRow.batch_id == batch_id, + BizStateMetricRow.metric_id == mid, + ) + .order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) + .all() + ) + if not rows: continue - cols = list(col_keys) + recs = [dict(r.data_json or {}) for r in rows] + cols: list[str] = [] + for rec in recs: + for k in rec.keys(): + if k not in cols: + cols.append(str(k)) + out_lines = [",".join(_csv(c) for c in cols)] + for rec in recs: + out_lines.append(",".join(_csv(str(rec.get(c, "") or "")) for c in cols)) safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "metric" - with zf.open(f"tables/{safe}.csv", "w") as dest: - dest.write((",".join(_csv(c) for c in cols) + "\n").encode("utf-8")) - for rec in _iter_metric_export_rows(db, batch_id, mid): - line = ",".join(_csv(rec.get(c)) for c in cols) + "\n" - dest.write(line.encode("utf-8")) + zf.writestr(f"tables/{safe}.csv", "\n".join(out_lines) + "\n") return buf.getvalue() -def _iter_metric_export_rows(db: Session, batch_id: str, metric_id: str, chunk_size: int = 1000): - cursor: tuple[int, str] | None = None - while True: - query = db.query(BizStateMetricRow.seq, BizStateMetricRow.id, BizStateMetricRow.data_json).filter( - BizStateMetricRow.batch_id == batch_id, BizStateMetricRow.metric_id == metric_id, - ) - if cursor is not None: - seq, row_id = cursor - query = query.filter(or_( - BizStateMetricRow.seq > seq, - (BizStateMetricRow.seq == seq) & (BizStateMetricRow.id > row_id), - )) - chunk = query.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()).limit(chunk_size).all() - if not chunk: - return - for row in chunk: - yield row.data_json or {} - cursor = (chunk[-1].seq, chunk[-1].id) - - -def _csv(v: Any) -> str: - s = "" if v is None else str(v) - if any(ch in s for ch in ",\"\r\n"): +def _csv(v: str) -> str: + s = str(v or "") + if any(ch in s for ch in ",\"\n"): return '"' + s.replace('"', '""') + '"' return s diff --git a/netx_api/biz_state_router.py b/netx_api/biz_state_router.py index 0714e1c..4a4f3ed 100644 --- a/netx_api/biz_state_router.py +++ b/netx_api/biz_state_router.py @@ -173,11 +173,6 @@ def api_get_task(task_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return svc.get_task(db, task_id) -@router.get("/tasks/{task_id}/progress") -def api_task_progress(task_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: - return svc.get_task_progress(db, task_id) - - @router.get("/tasks/{task_id}/commands") def api_plan_task_commands( task_id: str, @@ -549,12 +544,6 @@ class FieldRuleIn(BaseModel): normalize: str = "" ignore: bool = False tolerance: float | None = None - # Cross-vendor identity rules (see identity_pipeline). - value_map: dict[str, str] | None = None - side: str = "" - apply_to_key: bool | None = None - # Order for compare=not_worse: earlier entries are better. - rank: list[str] | None = None class RowFilterIn(BaseModel): @@ -580,10 +569,6 @@ class TemplateMetricIn(BaseModel): display_fields: list[str] = Field(default_factory=list) row_filters: list[dict[str, Any]] = Field(default_factory=list) field_rules: list[FieldRuleIn] = Field(default_factory=list) - ignore_port_changes: bool | None = None - unmapped_port_policy: str = "" - duplicate_pairing: str = "" - builtin_vendor_canonical: bool | None = None class TemplateIn(BaseModel): @@ -602,7 +587,6 @@ class TemplateIn(BaseModel): display_fields: list[str] = Field(default_factory=list) row_filters: list[dict[str, Any]] = Field(default_factory=list) field_rules: list[FieldRuleIn] = Field(default_factory=list) - ignore_port_changes: bool | None = None class TemplatePatchIn(BaseModel): @@ -618,7 +602,6 @@ class TemplatePatchIn(BaseModel): display_fields: list[str] | None = None row_filters: list[dict[str, Any]] | None = None field_rules: list[FieldRuleIn] | None = None - ignore_port_changes: bool | None = None class MappingRowIn(BaseModel): @@ -632,16 +615,6 @@ class MappingIn(BaseModel): rows: list[MappingRowIn] = Field(default_factory=list) -class PreflightIn(BaseModel): - template_id: str = "" - enabled_sheet_ids: list[str] = Field(default_factory=list) - mapping_id: str = "" - # Unsaved editor rows win over ``mapping_id`` when given. - mapping_rows: list[MappingRowIn] | None = None - before_batch_id: str - after_batch_id: str - - class ValidateMappingIn(BaseModel): mapping_id: str before_batch_id: str @@ -660,8 +633,6 @@ class CompareJobIn(BaseModel): mode: str = "manual" # Empty = all template sheets; non-empty = only these sheet_id values enabled_sheet_ids: list[str] = Field(default_factory=list) - # auto|always|never|sample|keys — how to persist matching (success) rows - store_unchanged: str = "auto" note: str = "" @@ -730,11 +701,6 @@ def api_validate_mapping(body: ValidateMappingIn, db: Session = Depends(get_db)) ) -@router.post("/compare/preflight") -def api_compare_preflight(body: PreflightIn, db: Session = Depends(get_db)) -> dict[str, Any]: - return cmp_svc.preflight_compare(db, body.model_dump()) - - @router.get("/compare/jobs") def api_list_jobs(db: Session = Depends(get_db)) -> dict[str, Any]: return {"items": cmp_svc.list_jobs(db)} @@ -757,15 +723,8 @@ def api_delete_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any] @router.post("/compare/jobs/{job_id}/run") -def api_run_job( - job_id: str, - sync: bool = Query(False, description="If true, block until compare finishes (tests/debug)"), - db: Session = Depends(get_db), -) -> dict[str, Any]: - """Start a compare run. Default is async (returns status=running immediately).""" - if sync: - return cmp_svc.run_compare(db, job_id) - return cmp_svc.enqueue_compare(db, job_id) +def api_run_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: + return cmp_svc.run_compare(db, job_id) @router.get("/compare/jobs/{job_id}/runs") @@ -778,12 +737,6 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return cmp_svc.get_run(db, run_id) -@router.post("/compare/runs/{run_id}/cancel") -def api_cancel_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: - """Cancel a stuck/running compare so a new run can start.""" - return cmp_svc.cancel_compare_run(db, run_id) - - @router.delete("/compare/runs/{run_id}") def api_delete_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return cmp_svc.delete_run(db, run_id) @@ -795,7 +748,6 @@ def api_list_run_diffs( metric_id: str = "", kind: str = "diff", kw: str = "", - qf: str = "", page: int = 1, page_size: int = 100, db: Session = Depends(get_db), @@ -806,7 +758,6 @@ def api_list_run_diffs( metric_id=metric_id, kind=kind, kw=kw, - qf=qf, page=page, page_size=page_size, ) diff --git a/netx_api/biz_state_scheduler.py b/netx_api/biz_state_scheduler.py index 77b6338..d44250f 100644 --- a/netx_api/biz_state_scheduler.py +++ b/netx_api/biz_state_scheduler.py @@ -47,15 +47,6 @@ def _maybe_purge_retention() -> None: from .biz_state.retention import purge_all_tasks info = purge_all_tasks(db) - from .biz_state.retention import purge_auto_runs - - runs = purge_auto_runs( - db, - migration_keep=int(getattr(settings, "biz_migration_auto_run_keep", 300) or 300), - compare_keep=int(getattr(settings, "biz_compare_auto_run_keep", 50) or 50), - ) - if any(runs.values()): - _log.info("biz_state periodic run purge %s", runs) if info.get("dropped"): _log.info( "biz_state periodic retention dropped=%s tasks=%s", @@ -119,21 +110,12 @@ def _sync_cutover_hf_windows() -> None: projects = db.query(BizMigrationProject).all() for proj in projects: - tids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new") - if tids and any((t := db.get(BizStateTask, tid)) and not t.collect_group_id for tid in tids): - try: - from .biz_migration.service import ensure_highfreq - - ensure_highfreq(db, proj.id, collect_now=False) - except Exception: - _log.exception("upgrade cutover collection pairs failed project=%s", proj.id) - window_status = _hf_window_status(proj) + want = _hf_window_status(proj) tids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new") for tid in tids: task = db.get(BizStateTask, tid) if not task: continue - want = "paused" if task.collect_manual_only else window_status purpose = str(getattr(task, "purpose", None) or "").strip() if purpose and purpose != PURPOSE_CUTOVER_HF: continue @@ -167,26 +149,12 @@ def _enqueue_due_tasks() -> int: .filter( BizStateTask.status == "running", BizStateTask.collect_running.is_(False), - BizStateTask.collect_manual_only.is_(False), ) .all() ) due_ids: list[str] = [] - seen_groups: set[str] = set() now = _utcnow() for task in tasks: - if task.collect_group_id: - if task.collect_group_id in seen_groups: - continue - seen_groups.add(task.collect_group_id) - peers = db.query(BizStateTask).filter(BizStateTask.collect_group_id == task.collect_group_id).all() - if len(peers) != 2 or any(p.collect_running or p.status != "running" or p.collect_manual_only for p in peers): - continue - anchors = [p.last_collect_started_at or p.last_collect_ended_at for p in peers] - anchor = max((a for a in anchors if a), default=None) - if anchor is None or (now - anchor).total_seconds() >= max(p.interval_sec or 60 for p in peers): - due_ids.append(task.id) - continue interval = max(60, int(task.interval_sec or 300)) # Prefer start-based interval to avoid drift when collect duration # approaches the interval (end-based: 40s collect + 60s → 100s cycle). diff --git a/netx_api/cli_router.py b/netx_api/cli_router.py index d871850..065063c 100644 --- a/netx_api/cli_router.py +++ b/netx_api/cli_router.py @@ -1,6 +1,6 @@ from __future__ import annotations -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session from .cli_schemas import ( @@ -15,14 +15,12 @@ from .cli_service import ( delete_cli_profile, get_cli_profile, get_ume_cli_override, - list_cli_target_filter_options, list_cli_profiles, list_cli_targets, set_default_cli_profile, update_cli_profile, upsert_ume_cli_override, ) -from .table_filters import TableFilterParams, table_filter_params from .db import get_db from .ne_connect import schedule_ume_connect_tests @@ -68,57 +66,11 @@ def api_delete_cli_profile(profile_id: str, db: Session = Depends(get_db)): def api_list_cli_targets( source: str = Query(default="all"), keyword: str | None = Query(default=None), - source_filter: list[str] | None = Query(default=None), - source_filter_mode: str | None = Query(default=None), - name_filter: list[str] | None = Query(default=None), - name_filter_mode: str | None = Query(default=None), - ip_filter: list[str] | None = Query(default=None), - ip_filter_mode: str | None = Query(default=None), - vendor_filter: list[str] | None = Query(default=None), - vendor_filter_mode: str | None = Query(default=None), - device_type_filter: list[str] | None = Query(default=None), - device_type_filter_mode: str | None = Query(default=None), - connect_status_filter: list[str] | None = Query(default=None), - connect_status_filter_mode: str | None = Query(default=None), page: int = Query(default=1, ge=1), page_size: int = Query(default=50, ge=1, le=500), - table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ): - if table_filters.field: - return list_cli_target_filter_options(db, field=table_filters.field, source=source, keyword=keyword) - filters = table_filters.filters - if set(filters) - {"source", "name", "ip", "vendor", "deviceType", "connectStatus"}: - raise HTTPException(status_code=400, detail="invalid_filter_field") - return list_cli_targets( - db, - source=source, - keyword=keyword, - source_filter=filters.get("source", {}).get("values", source_filter), - source_filter_mode=filters.get("source", {}).get("mode", source_filter_mode), - name_filter=filters.get("name", {}).get("values", name_filter), - name_filter_mode=filters.get("name", {}).get("mode", name_filter_mode), - ip_filter=filters.get("ip", {}).get("values", ip_filter), - ip_filter_mode=filters.get("ip", {}).get("mode", ip_filter_mode), - vendor_filter=filters.get("vendor", {}).get("values", vendor_filter), - vendor_filter_mode=filters.get("vendor", {}).get("mode", vendor_filter_mode), - device_type_filter=filters.get("deviceType", {}).get("values", device_type_filter), - device_type_filter_mode=filters.get("deviceType", {}).get("mode", device_type_filter_mode), - connect_status_filter=filters.get("connectStatus", {}).get("values", connect_status_filter), - connect_status_filter_mode=filters.get("connectStatus", {}).get("mode", connect_status_filter_mode), - page=page, - page_size=page_size, - ) - - -@router.get("/targets/filter-options") -def api_list_cli_target_filter_options( - field: str = Query(), - source: str = Query(default="inventory"), - keyword: str | None = Query(default=None), - db: Session = Depends(get_db), -): - return list_cli_target_filter_options(db, field=field, source=source, keyword=keyword) + return list_cli_targets(db, source=source, keyword=keyword, page=page, page_size=page_size) @router.get("/ume-overrides/{ume_ne_id}") diff --git a/netx_api/cli_service.py b/netx_api/cli_service.py index 439db50..82707c6 100644 --- a/netx_api/cli_service.py +++ b/netx_api/cli_service.py @@ -4,7 +4,6 @@ from datetime import datetime from typing import Any from fastapi import HTTPException -from sqlalchemy import func from sqlalchemy.orm import Session from .cli_schemas import ( @@ -280,66 +279,19 @@ def list_cli_targets( *, source: str = "all", keyword: str | None = None, - source_filter: list[str] | None = None, - source_filter_mode: str | None = None, - name_filter: list[str] | None = None, - name_filter_mode: str | None = None, - ip_filter: list[str] | None = None, - ip_filter_mode: str | None = None, - vendor_filter: list[str] | None = None, - vendor_filter_mode: str | None = None, - device_type_filter: list[str] | None = None, - device_type_filter_mode: str | None = None, - connect_status_filter: list[str] | None = None, - connect_status_filter_mode: str | None = None, page: int = 1, page_size: int = 50, ) -> dict[str, Any]: from .device_types import WEBCRT_NE_SOURCE src = str(source or "all").strip().lower() - if src not in ("managed", "ume", "webcrt", "inventory", "all"): + if src not in ("managed", "ume", "webcrt", "all"): raise HTTPException(status_code=400, detail="invalid_source") ready = cli_profile_ready(db) page = max(1, int(page or 1)) page_size = max(1, min(500, int(page_size or 50))) offset = (page - 1) * page_size kw = str(keyword or "").strip() - column_filters: dict[str, list[str] | None] = { - "source": source_filter, - "name": name_filter, - "ip": ip_filter, - "vendor": vendor_filter, - "device_type": device_type_filter, - "connect_status": connect_status_filter, - } - filter_modes = { - "source": str(source_filter_mode or ""), - "name": str(name_filter_mode or ""), - "ip": str(ip_filter_mode or ""), - "vendor": str(vendor_filter_mode or ""), - "device_type": str(device_type_filter_mode or ""), - "connect_status": str(connect_status_filter_mode or ""), - } - - def _source_allowed(row_source: str) -> bool: - mode = filter_modes["source"] - values = column_filters["source"] or [] - if mode == "include": - return row_source in values - if mode == "exclude": - return row_source not in values - return True - - def _apply_value_filter(stmt: Any, field: str, expression: Any) -> Any: - mode = filter_modes[field] - if mode not in ("include", "exclude"): - return stmt - values = column_filters[field] or [] - normalized = func.coalesce(expression, "") - if mode == "include": - return stmt.filter(normalized.in_(values)) if values else stmt.filter(False) - return stmt.filter(~normalized.in_(values)) if values else stmt default_profile = get_default_profile(db) default_hop = bool(getattr(default_profile, "hop_enabled", False)) if default_profile else False @@ -399,8 +351,7 @@ def list_cli_targets( return stmt like = f"%{kw}%" return stmt.filter( - ManagedNE.id.ilike(like) - | ManagedNE.name.ilike(like) + ManagedNE.name.ilike(like) | ManagedNE.ip_address.ilike(like) | ManagedNE.username.ilike(like) | ManagedNE.tags.ilike(like) @@ -411,42 +362,16 @@ def list_cli_targets( def _inventory_managed_query(): # Asset inventory for WebCRT: exclude Quick-Connect (source=webcrt) rows. stmt = db.query(ManagedNE).filter(ManagedNE.source != WEBCRT_NE_SOURCE) - stmt = _apply_kw(stmt) - if not _source_allowed("managed"): - stmt = stmt.filter(False) - stmt = _apply_value_filter( - stmt, - "name", - func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address), - ) - stmt = _apply_value_filter(stmt, "ip", ManagedNE.ip_address) - stmt = _apply_value_filter(stmt, "vendor", ManagedNE.vendor) - stmt = _apply_value_filter(stmt, "device_type", ManagedNE.device_type) - stmt = _apply_value_filter(stmt, "connect_status", ManagedNE.connect_status) - return stmt.order_by(ManagedNE.updated_at.desc()) + return _apply_kw(stmt).order_by(ManagedNE.updated_at.desc()) def _webcrt_query(): stmt = db.query(ManagedNE).filter(ManagedNE.source == WEBCRT_NE_SOURCE) - stmt = _apply_kw(stmt) - if not _source_allowed("webcrt"): - stmt = stmt.filter(False) - stmt = _apply_value_filter( - stmt, - "name", - func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address), - ) - stmt = _apply_value_filter(stmt, "ip", ManagedNE.ip_address) - stmt = _apply_value_filter(stmt, "vendor", ManagedNE.vendor) - stmt = _apply_value_filter(stmt, "device_type", ManagedNE.device_type) - stmt = _apply_value_filter(stmt, "connect_status", ManagedNE.connect_status) - return stmt.order_by(ManagedNE.updated_at.desc()) + return _apply_kw(stmt).order_by(ManagedNE.updated_at.desc()) def _ume_query(): stmt = db.query(UmeInventoryNE, UmeCliOverride).outerjoin( UmeCliOverride, UmeInventoryNE.ne_id == UmeCliOverride.ume_ne_id ) - if not _source_allowed("ume"): - stmt = stmt.filter(False) if kw: like = f"%{kw}%" stmt = stmt.filter( @@ -456,46 +381,8 @@ def list_cli_targets( | UmeInventoryNE.ip_address.ilike(like) | UmeInventoryNE.host_name.ilike(like) ) - ume_name = func.coalesce( - func.nullif(UmeInventoryNE.user_label, ""), - func.nullif(UmeInventoryNE.ne_name, ""), - func.nullif(UmeInventoryNE.host_name, ""), - func.nullif(UmeInventoryNE.ip_address, ""), - UmeInventoryNE.ne_id, - ) - stmt = _apply_value_filter(stmt, "name", ume_name) - stmt = _apply_value_filter(stmt, "ip", UmeInventoryNE.ip_address) - stmt = _apply_value_filter(stmt, "vendor", UmeInventoryNE.vendor) - stmt = _apply_value_filter(stmt, "device_type", UmeInventoryNE.ne_type) - stmt = _apply_value_filter( - stmt, - "connect_status", - func.coalesce(UmeCliOverride.connect_status, "unknown"), - ) return stmt.order_by(UmeInventoryNE.ne_id.asc()) - if src == "inventory": - # Network Management inventory includes managed assets and UME inventory, - # but not transient WebCRT quick-connect sessions. - mq = _inventory_managed_query() - uq = _ume_query() - m_total = int(mq.count()) - u_total = int(uq.count()) - total = m_total + u_total - items: list[dict[str, Any]] = [] - - if offset < m_total: - take = min(page_size, m_total - offset) - items.extend(_managed_item(row, list_source="managed") for row in mq.offset(offset).limit(take).all()) - remaining = page_size - take - if remaining > 0: - items.extend(_ume_item(inv, ov) for inv, ov in uq.offset(0).limit(remaining).all()) - else: - ume_offset = max(0, offset - m_total) - items.extend(_ume_item(inv, ov) for inv, ov in uq.offset(ume_offset).limit(page_size).all()) - - return {"items": items, "total": total, "page": page, "page_size": page_size} - if src == "managed": mq = _inventory_managed_query() total = int(mq.count()) @@ -556,118 +443,6 @@ def list_cli_targets( return {"items": items, "total": total, "page": page, "page_size": page_size} -def list_cli_target_filter_options( - db: Session, - *, - field: str, - source: str = "inventory", - keyword: str | None = None, -) -> dict[str, Any]: - from .device_types import WEBCRT_NE_SOURCE - - field_map = { - "source": "source", - "name": "name", - "ip": "ip", - "vendor": "vendor", - "deviceType": "device_type", - "connectStatus": "connect_status", - } - if field not in field_map: - raise HTTPException(status_code=400, detail="invalid_filter_field") - src = str(source or "inventory").strip().lower() - if src not in ("managed", "ume", "webcrt", "inventory", "all"): - raise HTTPException(status_code=400, detail="invalid_source") - - include_managed = src in ("managed", "inventory", "all") - include_webcrt = src in ("webcrt", "all") - include_ume = src in ("ume", "inventory", "all") - kw = str(keyword or "").strip() - - managed = db.query(ManagedNE) - if src in ("managed", "inventory"): - managed = managed.filter(ManagedNE.source != WEBCRT_NE_SOURCE) - elif src == "webcrt": - managed = managed.filter(ManagedNE.source == WEBCRT_NE_SOURCE) - elif src == "all": - managed = managed.filter(ManagedNE.source != WEBCRT_NE_SOURCE) - ume = db.query(UmeInventoryNE) - ume_status = db.query(UmeInventoryNE, UmeCliOverride).outerjoin( - UmeCliOverride, UmeInventoryNE.ne_id == UmeCliOverride.ume_ne_id - ) - webcrt = db.query(ManagedNE).filter(ManagedNE.source == WEBCRT_NE_SOURCE) - - def _apply_managed_keyword(query: Any) -> Any: - if not kw: - return query - like = f"%{kw}%" - return query.filter( - ManagedNE.id.ilike(like) - | ManagedNE.name.ilike(like) - | ManagedNE.ip_address.ilike(like) - | ManagedNE.username.ilike(like) - | ManagedNE.tags.ilike(like) - | ManagedNE.vendor.ilike(like) - | ManagedNE.device_type.ilike(like) - ) - - def _apply_ume_keyword(query: Any) -> Any: - if not kw: - return query - like = f"%{kw}%" - return query.filter( - UmeInventoryNE.ne_id.ilike(like) - | UmeInventoryNE.ne_name.ilike(like) - | UmeInventoryNE.user_label.ilike(like) - | UmeInventoryNE.ip_address.ilike(like) - | UmeInventoryNE.host_name.ilike(like) - ) - - managed = _apply_managed_keyword(managed) - webcrt = _apply_managed_keyword(webcrt) - ume = _apply_ume_keyword(ume) - ume_status = _apply_ume_keyword(ume_status) - - if field == "source": - values: set[str] = set() - if include_managed and managed.count(): - values.add("managed") - if include_webcrt and webcrt.count(): - values.add("webcrt") - if include_ume and ume.count(): - values.add("ume") - else: - managed_expr = { - "name": func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address), - "ip": ManagedNE.ip_address, - "vendor": ManagedNE.vendor, - "device_type": ManagedNE.device_type, - "connect_status": ManagedNE.connect_status, - }.get(field_map[field]) - ume_expr = { - "name": func.coalesce( - func.nullif(UmeInventoryNE.user_label, ""), - func.nullif(UmeInventoryNE.ne_name, ""), - func.nullif(UmeInventoryNE.host_name, ""), - func.nullif(UmeInventoryNE.ip_address, ""), - UmeInventoryNE.ne_id, - ), - "ip": UmeInventoryNE.ip_address, - "vendor": UmeInventoryNE.vendor, - "device_type": UmeInventoryNE.ne_type, - "connect_status": func.coalesce(UmeCliOverride.connect_status, "unknown"), - }.get(field_map[field]) - values = set() - if include_managed and managed_expr is not None: - values.update(str(row[0] or "") for row in managed.with_entities(managed_expr).distinct().all()) - if include_webcrt and managed_expr is not None: - values.update(str(row[0] or "") for row in webcrt.with_entities(managed_expr).distinct().all()) - if include_ume and ume_expr is not None: - values.update(str(row[0] or "") for row in ume_status.with_entities(ume_expr).distinct().all()) - - return {"field": field, "options": sorted(values, key=str.casefold)} - - def cli_meta(db: Session) -> dict[str, Any]: return { "credentials_configured": credentials_configured(), diff --git a/netx_api/config.py b/netx_api/config.py index 2ebe55f..1126fc5 100644 --- a/netx_api/config.py +++ b/netx_api/config.py @@ -80,9 +80,8 @@ class Settings(BaseSettings): ume_topo_nodes_path: str = "/restconf/data/zte-resources-module:TopoNodes" ume_topological_links_path: str = "/restconf/data/zte-resources-module:TopologicalLinks" ume_sync_alarms_history_every_hours: int = 24 - # Managed NE credentials (Fernet). Empty = auto-generate & persist to credential_secret_file. + # Managed NE credentials (Fernet key; generate with cryptography.fernet.Fernet.generate_key()) credential_secret_key: str = "" - credential_secret_file: str = "data/auth/credential_secret" # Shared-server worker caps (sized for multi-operator use; raise if bastion/DB allow). ne_connect_max_workers: int = 8 ne_connect_timeout_sec: int = 30 @@ -137,13 +136,6 @@ class Settings(BaseSettings): biz_state_dedicated_workers: bool = True # Global ceiling for simultaneous running batches (across all workers). biz_state_max_concurrent_tasks: int = 16 - # Auto-generated run history kept per cutover wave / auto compare job (acceptance + manual runs are never purged). - biz_migration_auto_run_keep: int = 300 - biz_compare_auto_run_keep: int = 50 - # Rows of immutable baseline batches cached across cutover evaluations (0 disables). - biz_migration_baseline_cache_rows: int = 500000 - # Concurrent SSH collections per network element (0 = unlimited). - biz_state_max_sessions_per_ne: int = 2 biz_state_worker_collect_threads: int = 8 biz_state_persist_workers: int = 4 # TextFSM / normalize workers (off SSH collect threads). Default 8. diff --git a/netx_api/config_sync_router.py b/netx_api/config_sync_router.py index 62a9a81..375f241 100644 --- a/netx_api/config_sync_router.py +++ b/netx_api/config_sync_router.py @@ -27,7 +27,6 @@ from .config_sync_service import ( stop_cycle, update_policy, ) -from .table_filters import TableFilterParams, table_filter_params from .db import get_db router = APIRouter(prefix="/v1/config-sync", tags=["config-sync"]) @@ -165,11 +164,10 @@ def api_list_snapshots( keyword: str = Query(default=""), source: str = Query(default=""), vendor: str = Query(default=""), - table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ): return list_snapshots( - db, page=page, page_size=page_size, keyword=keyword, source=source, vendor=vendor, table_filters=table_filters + db, page=page, page_size=page_size, keyword=keyword, source=source, vendor=vendor ) diff --git a/netx_api/config_sync_snapshots.py b/netx_api/config_sync_snapshots.py index ca8e582..97299af 100644 --- a/netx_api/config_sync_snapshots.py +++ b/netx_api/config_sync_snapshots.py @@ -7,7 +7,7 @@ import zipfile from typing import Any from fastapi import HTTPException -from sqlalchemy import func, or_ +from sqlalchemy import or_ from sqlalchemy.orm import Session from .config_sync_codec import decompress_text @@ -16,7 +16,6 @@ from .config_sync_schemas import ( NeConfigSnapshotDetailOut, NeConfigSnapshotMetaOut, ) -from .table_filters import TableFilterParams, filter_query from .models import NeConfigHistory, NeConfigSnapshot def _snap_meta(row: NeConfigSnapshot) -> NeConfigSnapshotMetaOut: @@ -49,7 +48,6 @@ def list_snapshots( keyword: str = "", source: str = "", vendor: str = "", - table_filters: TableFilterParams | None = None, ) -> dict[str, Any]: q = db.query(NeConfigSnapshot) src = str(source or "").strip().lower() @@ -68,14 +66,6 @@ def list_snapshots( NeConfigSnapshot.target_id.ilike(like), ) ) - q, options = filter_query(q, { - "name": func.coalesce(func.nullif(NeConfigSnapshot.ne_name, ""), NeConfigSnapshot.target_id), - "ip": NeConfigSnapshot.ne_ip, "vendor": NeConfigSnapshot.vendor, - "source": NeConfigSnapshot.source, "size": NeConfigSnapshot.plain_size, - "collected": NeConfigSnapshot.collected_at, - }, table_filters) - if options is not None: - return options total = int(q.count()) rows = q.order_by(NeConfigSnapshot.collected_at.desc()).offset((page - 1) * page_size).limit(page_size).all() return {"total": total, "page": page, "page_size": page_size, "items": [_snap_meta(r) for r in rows]} diff --git a/netx_api/importer.py b/netx_api/importer.py index 38364df..7c17905 100644 --- a/netx_api/importer.py +++ b/netx_api/importer.py @@ -1,34 +1,29 @@ from __future__ import annotations import json -import math from datetime import datetime, timezone +from io import BytesIO from typing import Any +import pandas as pd from sqlalchemy import func, select from sqlalchemy.orm import Session from .config import settings from .models import AlarmBatch, AlarmNorm, ImportErrorRow from .parser_config import ParserConfig -from .tabular import parse_datetime, read_xlsx - - -def _blank(v: Any) -> bool: - if v is None: - return True - if isinstance(v, float) and math.isnan(v): - return True - text = str(v).strip() - return not text or text.lower() == "nan" def _dt(v: Any) -> datetime | None: - if _blank(v): + if v is None: return None - dt = parse_datetime(v) - if dt is None: + text = str(v).strip() + if not text or text.lower() == "nan": return None + parsed = pd.to_datetime(text, errors="coerce") + if pd.isna(parsed): + return None + dt = parsed.to_pydatetime() # Convention: Excel source time is local timezone time. # Convert to UTC before persisting; keep DB as naive UTC for compatibility. if dt.tzinfo is None: @@ -40,15 +35,20 @@ def _dt(v: Any) -> datetime | None: def _str(v: Any) -> str: - if _blank(v): + if v is None: return "" - return str(v).strip() + text = str(v).strip() + if text.lower() == "nan": + return "" + return text def _int(v: Any) -> int: - if _blank(v): + if v is None: return 0 text = str(v).strip() + if not text or text.lower() == "nan": + return 0 try: return int(float(text)) except Exception: @@ -56,7 +56,8 @@ def _int(v: Any) -> int: def import_alarm_excel(db: Session, filename: str, content: bytes, parser: ParserConfig) -> AlarmBatch: - headers, rows = read_xlsx(content) + df = pd.read_excel(BytesIO(content)) + headers = [str(h) for h in df.columns] col_alarm_time = parser.resolve_col(headers, "alarm_time") col_clear_time = parser.resolve_col(headers, "clear_time") @@ -79,7 +80,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse source_file=filename, parser_version=parser.parser_version, dict_version=parser.dict_version, - total_rows=len(rows), + total_rows=int(len(df.index)), status="processing", ) db.add(batch) @@ -88,8 +89,8 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse success_rows = 0 failed_rows = 0 - for row_no, row in rows: # row_no = sheet row (header is row 1) - raw_json = json.dumps(row, ensure_ascii=False, default=str) + for idx, row in df.iterrows(): + row_no = int(idx) + 2 # Excel header is row 1 alarm_time = _dt(row.get(col_alarm_time)) if col_alarm_time else None raw_severity = _str(row.get(col_severity)) if col_severity else "" alarm_code = _str(row.get(col_alarm_code)) if col_alarm_code else "" @@ -110,7 +111,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse batch_id=batch.batch_id, row_no=row_no, reason="missing_or_invalid_alarm_time", - raw_json=raw_json, + raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str), ) ) continue @@ -121,7 +122,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse batch_id=batch.batch_id, row_no=row_no, reason="missing_ne_name_and_ne_id", - raw_json=raw_json, + raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str), ) ) continue @@ -150,7 +151,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse vendor=settings.vendor, source_type=settings.source_type, source_file=filename, - raw_json=raw_json, + raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str), ) ) success_rows += 1 diff --git a/netx_api/lldp_collect_router.py b/netx_api/lldp_collect_router.py index 4844e47..addfcf2 100644 --- a/netx_api/lldp_collect_router.py +++ b/netx_api/lldp_collect_router.py @@ -7,7 +7,6 @@ from typing import Any from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session -from .table_filters import TableFilterParams, table_filter_params from .db import get_db from .lldp_collect_schemas import LldpCollectPolicyUpdate, LldpCollectStartBody from .lldp_collect_service import ( @@ -70,10 +69,9 @@ def api_list_jobs( page_size: int = Query(default=20, ge=1, le=100), status: str = Query(default=""), keyword: str = Query(default=""), - table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: - return list_jobs(db, page=page, page_size=page_size, status=status, keyword=keyword, table_filters=table_filters) + return list_jobs(db, page=page, page_size=page_size, status=status, keyword=keyword) @router.get("/jobs/{job_id}") @@ -83,7 +81,6 @@ def api_get_job( page_size: int = Query(default=20, ge=1, le=100), status: str = Query(default=""), keyword: str = Query(default=""), - table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: return get_job_detail( @@ -93,5 +90,4 @@ def api_get_job( page_size=page_size, item_status=status, item_keyword=keyword, - table_filters=table_filters, ) diff --git a/netx_api/lldp_collect_service.py b/netx_api/lldp_collect_service.py index 4cd40f8..e5eb63b 100644 --- a/netx_api/lldp_collect_service.py +++ b/netx_api/lldp_collect_service.py @@ -5,11 +5,9 @@ from __future__ import annotations from datetime import datetime, timedelta from fastapi import HTTPException -from sqlalchemy import String, cast, or_ +from sqlalchemy import or_ from sqlalchemy.orm import Session -from .table_filters import TableFilterParams, filter_query -from .topology_discover_common import discover_item_expressions from .lldp_collect_schemas import ( LldpCollectDashboardOut, LldpCollectJobSummary, @@ -490,7 +488,6 @@ def list_jobs( page_size: int = 20, status: str = "", keyword: str = "", - table_filters: TableFilterParams | None = None, ) -> dict: page = max(1, int(page or 1)) page_size = max(1, min(100, int(page_size or 20))) @@ -509,15 +506,6 @@ def list_jobs( TopoDiscoverJob.trigger_mode.ilike(like), ) ) - job = TopoDiscoverJob - q, options = filter_query(q, { - "id": job.id, "trigger": job.trigger_mode, "scope": job.scope, "status": job.status, - "progress": cast(job.done, String) + "/" + cast(job.total, String), - "edges": cast(job.edges_added, String) + "/" + cast(job.edges_updated, String), - "missingDelta": job.edges_stale, "started": job.started_at, "ended": job.ended_at, - }, table_filters) - if options is not None: - return options q = q.order_by(TopoDiscoverJob.created_at.desc()) total = int(q.count()) rows = q.offset((page - 1) * page_size).limit(page_size).all() @@ -544,14 +532,7 @@ def get_job_detail( page_size: int | None = None, item_status: str = "", item_keyword: str = "", - table_filters: TableFilterParams | None = None, ) -> dict: - if table_filters and table_filters.field: - if db.get(TopoDiscoverJob, job_id) is None: - raise HTTPException(status_code=404, detail="discover_job_not_found") - q = db.query(TopoDiscoverJobItem).filter(TopoDiscoverJobItem.job_id == job_id) - _, options = filter_query(q, discover_item_expressions(), table_filters) - return options return get_discover_job( db, job_id, @@ -559,5 +540,4 @@ def get_job_detail( page_size=page_size, item_status=item_status, item_keyword=item_keyword, - table_filters=table_filters, ).model_dump() diff --git a/netx_api/managed_ne_router.py b/netx_api/managed_ne_router.py index 6c656d2..a90bf62 100644 --- a/netx_api/managed_ne_router.py +++ b/netx_api/managed_ne_router.py @@ -42,7 +42,6 @@ from .ne_service import ( update_managed_ne, ) from .models import ManagedNE -from .table_filters import TableFilterParams, table_filter_params router = APIRouter(prefix="/v1/managed-ne", tags=["managed-ne"]) @@ -64,7 +63,6 @@ def api_list_managed_ne( connect_status: str | None = Query(default=None), page: int = Query(default=1, ge=1), page_size: int = Query(default=50, ge=1, le=500), - table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ): return list_managed_ne( @@ -74,7 +72,6 @@ def api_list_managed_ne( connect_status=connect_status, page=page, page_size=page_size, - table_filters=table_filters, ) diff --git a/netx_api/models/auth.py b/netx_api/models/auth.py index ffdbc39..39947a0 100644 --- a/netx_api/models/auth.py +++ b/netx_api/models/auth.py @@ -21,8 +21,6 @@ class AppUser(Base): role: Mapped[str] = mapped_column(String(32), default="user", index=True) # admin | user # Optional capability override; empty => role defaults (see auth_scopes). scopes: Mapped[list] = mapped_column(_JsonType, default=list) - # Per-user UI preferences (e.g. workbench card layout). Pure client prefs; not audited. - prefs: Mapped[dict] = mapped_column(_JsonType, default=dict) is_active: Mapped[bool] = mapped_column(Boolean, default=True, index=True) must_change_password: Mapped[bool] = mapped_column(Boolean, default=False) created_by: Mapped[str] = mapped_column(String(64), default="") diff --git a/netx_api/models/biz_state.py b/netx_api/models/biz_state.py index 864f85d..bd8c311 100644 --- a/netx_api/models/biz_state.py +++ b/netx_api/models/biz_state.py @@ -32,9 +32,6 @@ class BizStateTask(Base): note: Mapped[str] = mapped_column(String(256), default="") # "" | portrait | cutover_hf — filter cutover HF in biz-state list purpose: Mapped[str] = mapped_column(String(32), default="", index=True) - # Optional paired sampling; ordinary per-device tasks remain independent. - collect_group_id: Mapped[str] = mapped_column(String(128), default="", index=True) - collect_manual_only: Mapped[bool] = mapped_column(Boolean, default=False) status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|running|paused|stopped interval_sec: Mapped[int] = mapped_column(Integer, default=300) # Keep snapshots for N calendar days (protected baselines/refs never auto-deleted) @@ -89,10 +86,6 @@ class BizStateBatch(Base): id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex) task_id: Mapped[str] = mapped_column(String(64), default="", index=True) - collect_group_id: Mapped[str] = mapped_column(String(128), default="", index=True) - collect_round_id: Mapped[str] = mapped_column(String(64), default="", index=True) - collect_priority: Mapped[int] = mapped_column(Integer, default=0) - queued_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) source: Mapped[str] = mapped_column(String(32), default="managed", index=True) ne_id: Mapped[str] = mapped_column(String(128), default="", index=True) ne_name: Mapped[str] = mapped_column(String(256), default="") @@ -310,8 +303,6 @@ class BizCompareJob(Base): status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|ready|auto # Empty = all template sheets; non-empty = only these sheet_id values enabled_sheet_ids: Mapped[list] = mapped_column(_JsonType, default=list) - # auto|always|never|sample — how to persist matching (success) rows - store_unchanged: Mapped[str] = mapped_column(String(16), default="auto") note: Mapped[str] = mapped_column(String(512), default="") created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) @@ -356,7 +347,4 @@ class BizCompareDiff(Base): after_json: Mapped[dict] = mapped_column(_JsonType, default=dict) mapped_before_json: Mapped[dict] = mapped_column(_JsonType, default=dict) changes_json: Mapped[dict] = mapped_column(_JsonType, default=dict) - # Pointers into BizStateMetricRow / LLDP tables for compact success hydrate - before_row_id: Mapped[str] = mapped_column(String(64), default="") - after_row_id: Mapped[str] = mapped_column(String(64), default="") search_text: Mapped[str] = mapped_column(Text, default="") diff --git a/netx_api/ne_cli_hop.py b/netx_api/ne_cli_hop.py index 83a45dd..807d9f8 100644 --- a/netx_api/ne_cli_hop.py +++ b/netx_api/ne_cli_hop.py @@ -10,9 +10,7 @@ from netmiko import ConnectHandler _CLI_HOP_NESTED_END_RE = re.compile( r"(?is)" r"(?:^|\n)\s*(?:" - # Huawei VRP: "Info: The connection was closed by the remote host." - r"(?:info:\s*)?(?:the\s+)?connection\s+was\s+closed\s+by\s+(?:the\s+)?(?:foreign|remote)\s+host" - r"|connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?" + r"connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?" r"|closed\s+by\s+foreign\s+host" r"|connection\s+to\s+\S+\s+closed" r"|%\s*connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?" diff --git a/netx_api/ne_crypto.py b/netx_api/ne_crypto.py index 7624c2a..4ba7617 100644 --- a/netx_api/ne_crypto.py +++ b/netx_api/ne_crypto.py @@ -1,73 +1,18 @@ from __future__ import annotations -import logging -from pathlib import Path - from cryptography.fernet import Fernet, InvalidToken from .config import settings -_log = logging.getLogger("netx.crypto") -_cached_credential_key: str | None = None - class CredentialCryptoError(RuntimeError): pass -def credential_secret_file_path() -> Path: - raw = str(getattr(settings, "credential_secret_file", None) or "data/auth/credential_secret").strip() - path = Path(raw) - if not path.is_absolute(): - path = Path.cwd() / path - return path - - -def ensure_credential_secret_key() -> str: - """Resolve Fernet key: env ``NETX_CREDENTIAL_SECRET_KEY`` > file > generate once. - - Packaged installs should also write the key into ``.env`` via setup_first_run; - this startup/path fallback covers upgrades and missed first-run keys. - """ - global _cached_credential_key - configured = str(settings.credential_secret_key or "").strip() - if configured: - return configured - if _cached_credential_key: - return _cached_credential_key - - path = credential_secret_file_path() - try: - if path.is_file(): - existing = path.read_text(encoding="utf-8").strip() - if existing: - _cached_credential_key = existing - settings.credential_secret_key = existing - return existing - except Exception: - _log.exception("read credential secret file failed path=%s", path) - - generated = Fernet.generate_key().decode("ascii") - try: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(generated + "\n", encoding="utf-8") - try: - path.chmod(0o600) - except Exception: - pass - _log.info("wrote per-install credential Fernet key to %s", path) - except Exception: - _log.exception( - "write credential secret file failed path=%s; using in-memory key only", - path, - ) - _cached_credential_key = generated - settings.credential_secret_key = generated - return generated - - def _fernet() -> Fernet: - key = ensure_credential_secret_key() + key = str(settings.credential_secret_key or "").strip() + if not key: + raise CredentialCryptoError("credential_secret_key_not_configured") try: return Fernet(key.encode("ascii")) except Exception as exc: @@ -92,7 +37,4 @@ def decrypt_secret(value: str) -> str: def credentials_configured() -> bool: - try: - return bool(ensure_credential_secret_key()) - except Exception: - return False + return bool(str(settings.credential_secret_key or "").strip()) diff --git a/netx_api/ne_netmiko.py b/netx_api/ne_netmiko.py index 5353af1..9b351e6 100644 --- a/netx_api/ne_netmiko.py +++ b/netx_api/ne_netmiko.py @@ -47,9 +47,6 @@ def paging_disable_commands(*, vendor: str = "", device_type: str = "") -> list[ dt = str(device_type or "").strip().lower() v = str(vendor or "").strip().lower() blob = f"{dt} {v}" - # RouterOS has no pager; Cisco's command leaves an extra error/prompt redraw. - if "mikrotik" in blob or "routeros" in blob or "switchos" in blob: - return [] if "huawei" in blob or "华为" in v: return ["screen-length 0 temporary"] if "hp_comware" in blob or "h3c" in blob or "comware" in blob: @@ -137,23 +134,6 @@ def drain_read_channel( def _send_command_expect_prompt(conn: Any, cmd: str, *, read_timeout: int) -> str: - device_type = getattr(conn, "device_type", "") - if isinstance(device_type, str) and device_type.lower().startswith("mikrotik"): - # find_prompt sends an extra Enter *after* our channel drain. RouterOS - # can repaint that old prompt after send_command starts, ending the read - # before the command's output arrives. Use its complete prompt directly. - # Keep echo verification off: long RouterOS scripts wrap at PTY width. - prompt_pattern = r"(?m)^\[[^\r\n\]]+\](?:[ \t]+/[^\r\n>]*)?[ \t]*>[ \t]*$" - return str( - conn.send_command( - command_string=cmd, - read_timeout=read_timeout, - auto_find_prompt=False, - expect_string=prompt_pattern, - cmd_verify=False, - ) - or "" - ) try: return str( conn.send_command( diff --git a/netx_api/ne_schemas.py b/netx_api/ne_schemas.py index 5bc2813..0611390 100644 --- a/netx_api/ne_schemas.py +++ b/netx_api/ne_schemas.py @@ -17,7 +17,7 @@ class ManagedNeCreate(BaseModel): vendor: str device_type: str ip_address: str - port: int = Field(default=22, ge=1, le=65535) + port: int = 22 protocol: str = "ssh" username: str password: str = "" @@ -27,7 +27,7 @@ class ManagedNeCreate(BaseModel): hop_enabled: bool = False hop_vendor: str = "zte" hop_host: str = "" - hop_port: int = Field(default=22, ge=1, le=65535) + hop_port: int = 22 hop_protocol: str = "ssh" hop_username: str = "" hop_password: str = "" @@ -63,7 +63,7 @@ class ManagedNeUpdate(BaseModel): vendor: str | None = None device_type: str | None = None ip_address: str | None = None - port: int | None = Field(default=None, ge=1, le=65535) + port: int | None = None protocol: str | None = None username: str | None = None password: str | None = None @@ -73,7 +73,7 @@ class ManagedNeUpdate(BaseModel): hop_enabled: bool | None = None hop_vendor: str | None = None hop_host: str | None = None - hop_port: int | None = Field(default=None, ge=1, le=65535) + hop_port: int | None = None hop_protocol: str | None = None hop_username: str | None = None hop_password: str | None = None @@ -194,7 +194,7 @@ class HopProxyConfig(BaseModel): hop_vendor: str = "zte" hop_host: str - hop_port: int = Field(default=22, ge=1, le=65535) + hop_port: int = 22 hop_protocol: str = "ssh" hop_username: str hop_password: str = "" diff --git a/netx_api/ne_service_common.py b/netx_api/ne_service_common.py index f76d6d7..9e74f0b 100644 --- a/netx_api/ne_service_common.py +++ b/netx_api/ne_service_common.py @@ -2,10 +2,10 @@ from __future__ import annotations from datetime import datetime -import math import re from typing import Any +import pandas as pd from fastapi import HTTPException from sqlalchemy.orm import Session @@ -60,63 +60,6 @@ def _normalize_ip(ip: str) -> str: return str(ip or "").strip() -_HOSTNAME_RE = re.compile(r"^(?=.{1,253}$)[A-Za-z0-9_](?:[A-Za-z0-9_-]{0,62})(?:\.[A-Za-z0-9_](?:[A-Za-z0-9_-]{0,62}))*\.?$") - - -def _is_valid_mgmt_address(ip: str) -> bool: - """Accept an IPv4/IPv6 literal or a DNS hostname (no spaces, ports or schemes).""" - import ipaddress - - text = str(ip or "").strip() - if not text: - return False - try: - ipaddress.ip_address(text) - return True - except ValueError: - pass - # Dotted-quad lookalikes (e.g. 10.0.0.300) are typos, not hostnames. - if re.fullmatch(r"[0-9.]+", text): - return False - return bool(_HOSTNAME_RE.match(text)) - - -def _require_mgmt_address(ip: str) -> str: - text = _normalize_ip(ip) - if not text: - raise HTTPException(status_code=400, detail="ip_address_required") - if not _is_valid_mgmt_address(text): - raise HTTPException(status_code=400, detail="invalid_ip_address") - return text - - -CONNECT_FIELDS = ( - "ip_address", "port", "protocol", "username", "password_enc", "device_type", - "hop_enabled", "hop_vendor", "hop_host", "hop_port", "hop_protocol", "hop_username", - "hop_password_enc", "hop_command_template", "hop_vrf", "hop_target_auth_mode", - "hop_enter_system_view", -) - - -def connect_snapshot(row: ManagedNE) -> tuple[Any, ...]: - return tuple(getattr(row, f, None) for f in CONNECT_FIELDS) - - -def reset_connect_status_if_changed(row: ManagedNE, before: tuple[Any, ...]) -> None: - """A previous pass/fail no longer describes the device once login settings change.""" - if connect_snapshot(row) != before and str(row.connect_status or "") != "unknown": - row.connect_status = "unknown" - row.connect_message = "" - - -def hop_password_optional(hop_vendor: str, hop_target_auth_mode: str) -> bool: - """Only a bastion that manages target auth may run without a stored hop password.""" - return ( - _normalize_hop_vendor(hop_vendor) == "bastion" - and _normalize_hop_target_auth_mode(hop_target_auth_mode) == "bastion_managed" - ) - - def _normalize_protocol(protocol: str) -> str: p = str(protocol or "ssh").strip().lower() return p if p in ("ssh", "telnet") else "ssh" @@ -199,7 +142,7 @@ def _parse_import_bool(value: Any) -> bool: def _import_cell_str(value: Any) -> str: - if value is None or (isinstance(value, float) and math.isnan(value)): + if value is None or (isinstance(value, float) and pd.isna(value)): return "" text = str(value).strip() return "" if text.lower() == "nan" else text @@ -270,8 +213,9 @@ def _apply_hop_update(row: ManagedNE, data: dict[str, Any]) -> None: raise HTTPException(status_code=400, detail="hop_host_required") if not str(row.hop_username or "").strip(): raise HTTPException(status_code=400, detail="hop_username_required") - if not str(row.hop_password_enc or "").strip() and not hop_password_optional( - str(row.hop_vendor or ""), str(row.hop_target_auth_mode or "") + if ( + not str(row.hop_password_enc or "").strip() + and _normalize_hop_target_auth_mode(row.hop_target_auth_mode) != "bastion_managed" ): raise HTTPException(status_code=400, detail="hop_password_required") diff --git a/netx_api/ne_service_crud.py b/netx_api/ne_service_crud.py index 5b168e7..e8b253c 100644 --- a/netx_api/ne_service_crud.py +++ b/netx_api/ne_service_crud.py @@ -5,10 +5,10 @@ from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import false, func, literal, or_ +from sqlalchemy import or_ from sqlalchemy.orm import Session -from .device_types import SUPPORTED_VENDORS, WEBCRT_DEVICE_TYPES, is_placeholder_ne_source +from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS, WEBCRT_DEVICE_TYPES, is_placeholder_ne_source from .models import ManagedNE from .ne_crypto import encrypt_secret from .ne_hop_templates import default_bastion_username_template, default_hop_command_template @@ -24,11 +24,8 @@ from .ne_service_common import ( _apply_hop_update, _normalize_hop_target_auth_mode, _normalize_hop_vendor, + _normalize_ip, _normalize_protocol, - _require_mgmt_address, - connect_snapshot, - hop_password_optional, - reset_connect_status_if_changed, _normalize_saved_hop_endpoint, _normalize_vendor, _now, @@ -36,43 +33,16 @@ from .ne_service_common import ( _validate_hop_on_create, row_to_out, ) -from .table_filters import TableFilterParams, filter_query from .ne_exec_guard import ( EXEC_POLICY_READONLY, allows_open_exec_policy, require_exec_policy_writable, ) -# Create and update share the form's device-type list, which includes linux/generic -# (Linux hosts and WebCRT/LLDP placeholders promoted to inventory). +# Inventory create stays strict; updates must also accept WebCRT/LLDP placeholder types +# (generic/linux) so operators can open the form and promote them to zte_zxros etc. _ALLOWED_UPDATE_DEVICE_TYPES = frozenset(WEBCRT_DEVICE_TYPES) -def managed_ne_filter_expressions() -> dict[str, Any]: - """Column filter fields for the NE management table (tags handled separately).""" - return { - "name": func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address), - "vendor": ManagedNE.vendor, - "deviceType": ManagedNE.device_type, - "source": func.coalesce(func.nullif(ManagedNE.source, ""), "manual"), - "ip": ManagedNE.ip_address, - "username": ManagedNE.username, - "connectStatus": ManagedNE.connect_status, - } - - -def _filter_by_tags(stmt: Any, value: dict[str, Any]) -> Any: - """Tags are space-separated; a row matches when any selected tag is present ("" = untagged).""" - tags_col = func.coalesce(ManagedNE.tags, "") - padded = literal(" ") + tags_col + literal(" ") - conds = [] - for tag in value["values"]: - tag = str(tag).strip() - escaped = tag.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - conds.append(func.trim(tags_col) == "" if not tag else padded.like(f"% {escaped} %", escape="\\")) - matched = or_(*conds) if conds else false() - return stmt.filter(matched if value["mode"] == "include" else ~matched) - - def list_managed_ne( db: Session, *, @@ -81,7 +51,6 @@ def list_managed_ne( connect_status: str | None = None, page: int = 1, page_size: int = 50, - table_filters: TableFilterParams | None = None, ) -> dict[str, Any]: stmt = db.query(ManagedNE) kw = str(keyword or "").strip() @@ -103,22 +72,6 @@ def list_managed_ne( stmt = stmt.filter(ManagedNE.vendor == v) if cs: stmt = stmt.filter(ManagedNE.connect_status == cs) - params = table_filters or TableFilterParams() - tags_filter = params.filters.get("tags") - if tags_filter and params.field != "tags": - stmt = _filter_by_tags(stmt, tags_filter) - if params.field == "tags": - stmt, _ = filter_query(stmt, managed_ne_filter_expressions(), TableFilterParams( - {k: v for k, v in params.filters.items() if k != "tags"})) - options: set[str] = set() - for (tags_val,) in stmt.with_entities(ManagedNE.tags).all(): - parts = str(tags_val or "").split() - options.update(parts if parts else [""]) - return {"field": "tags", "options": sorted(options)} - stmt, options = filter_query(stmt, managed_ne_filter_expressions(), TableFilterParams( - {k: v for k, v in params.filters.items() if k != "tags"}, params.field)) - if options is not None: - return options total = int(stmt.count()) rows = ( stmt.order_by(ManagedNE.updated_at.desc()) @@ -144,8 +97,10 @@ def get_managed_ne(db: Session, ne_id: str) -> ManagedNeOut: def create_managed_ne(db: Session, body: ManagedNeCreate) -> ManagedNeOut: _require_crypto() _validate_hop_on_create(body) - ip = _require_mgmt_address(body.ip_address) - if body.device_type not in _ALLOWED_UPDATE_DEVICE_TYPES: + ip = _normalize_ip(body.ip_address) + if not ip: + raise HTTPException(status_code=400, detail="ip_address_required") + if body.device_type not in SUPPORTED_DEVICE_TYPES: raise HTTPException(status_code=400, detail="unsupported_device_type") existing = db.query(ManagedNE).filter(ManagedNE.ip_address == ip).first() if existing: @@ -185,9 +140,10 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed if not row: raise HTTPException(status_code=404, detail="managed_ne_not_found") data = body.model_dump(exclude_unset=True) - before = connect_snapshot(row) if "ip_address" in data: - ip = _require_mgmt_address(data["ip_address"]) + ip = _normalize_ip(data["ip_address"]) + if not ip: + raise HTTPException(status_code=400, detail="ip_address_required") other = db.query(ManagedNE).filter(ManagedNE.ip_address == ip, ManagedNE.id != ne_id).first() if other: raise HTTPException(status_code=400, detail="ip_address_exists") @@ -201,7 +157,7 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed v = str(data["vendor"] or "").strip() row.vendor = v if v in SUPPORTED_VENDORS else "Other" if "name" in data: - row.name = str(data["name"] or "").strip() or str(row.ip_address or "") + row.name = str(data["name"] or "").strip() if "port" in data and data["port"] is not None: row.port = int(data["port"]) if "protocol" in data and data["protocol"] is not None: @@ -242,7 +198,6 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed if "ip_address" in data and str(row.ip_address or "").strip(): if is_placeholder_ne_source(row.source): row.source = "" - reset_connect_status_if_changed(row, before) row.updated_at = _now() # Keep linked fabric node identity in sync (name / IP / vendor). if any(k in data for k in ("ip_address", "name", "vendor", "device_type")): @@ -270,6 +225,8 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d if not hop_user: raise HTTPException(status_code=400, detail="hop_username_required") hop_auth_mode = _normalize_hop_target_auth_mode(hop.hop_target_auth_mode) + if not hop_pass and hop_auth_mode != "bastion_managed": + raise HTTPException(status_code=400, detail="hop_password_required") template = str(hop.hop_command_template or "").strip() if hop_vendor == "bastion" and not template: @@ -286,15 +243,9 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d missing = [x for x in ne_ids if x not in found_ids] if missing: raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing[:5])}") - # Blank password keeps each NE's stored hop password; NEs without one cannot hop. - if not hop_pass and not hop_password_optional(hop_vendor, hop_auth_mode): - lacking = [str(r.name or r.ip_address) for r in rows if not str(r.hop_password_enc or "").strip()] - if lacking: - raise HTTPException(status_code=400, detail=f"hop_password_required: {','.join(lacking[:5])}") now = _now() for row in rows: - before = connect_snapshot(row) row.hop_enabled = True row.hop_vendor = hop_vendor row.hop_host = hop_host @@ -307,7 +258,6 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d row.hop_vrf = str(hop.hop_vrf or "").strip() row.hop_target_auth_mode = hop_auth_mode row.hop_enter_system_view = bool(getattr(hop, "hop_enter_system_view", False)) - reset_connect_status_if_changed(row, before) row.updated_at = now db.commit() return {"ok": True, "updated": len(rows)} @@ -333,12 +283,10 @@ def batch_apply_account(db: Session, ids: list[str], account: BatchAccountConfig raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing[:5])}") now = _now() for row in rows: - before = connect_snapshot(row) if user: row.username = user if pwd: row.password_enc = pwd_enc - reset_connect_status_if_changed(row, before) row.updated_at = now db.commit() return {"ok": True, "updated": len(rows)} diff --git a/netx_api/ne_service_import.py b/netx_api/ne_service_import.py index ca006af..22415c8 100644 --- a/netx_api/ne_service_import.py +++ b/netx_api/ne_service_import.py @@ -1,15 +1,16 @@ """Managed NE Excel import and UME inventory sync into managed_ne.""" from __future__ import annotations +from io import BytesIO from typing import Any +import pandas as pd from fastapi import HTTPException from sqlalchemy.orm import Session from .models import ManagedNE, UmeInventoryNE -from .tabular import read_table, write_csv, write_xlsx from .ne_crypto import CredentialCryptoError, encrypt_secret -from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS, is_placeholder_ne_source +from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS from .ne_schemas import ( ImportFailure, ImportResult, @@ -22,9 +23,6 @@ from .ne_service_common import ( UME_SYNC_TAG, _import_cell_str, _infer_managed_ne_type_vendor, - _is_valid_mgmt_address, - connect_snapshot, - reset_connect_status_if_changed, _merge_tags, _normalize_ip, _normalize_protocol, @@ -150,74 +148,66 @@ def build_managed_ne_import_template(fmt: str = "xlsx") -> tuple[str, bytes, str "remark": "no direct password, use batch proxy", }, ] - headers = list(IMPORT_COLUMNS) - table = [[r.get(c, "") for c in headers] for r in rows] + df = pd.DataFrame(rows, columns=list(IMPORT_COLUMNS)) + buf = BytesIO() kind = str(fmt or "xlsx").strip().lower() if kind == "csv": + df.to_csv(buf, index=False, encoding="utf-8-sig") return ( "managed_ne_import_template.csv", - write_csv(headers, table), + buf.getvalue(), "text/csv; charset=utf-8", ) - payload = write_xlsx([ - ("import", headers, table), - ("device_type", ["device_type"], [[t] for t in SUPPORTED_DEVICE_TYPES]), - ]) + device_types_df = pd.DataFrame({"device_type": list(SUPPORTED_DEVICE_TYPES)}) + with pd.ExcelWriter(buf, engine="openpyxl") as writer: + df.to_excel(writer, sheet_name="import", index=False) + device_types_df.to_excel(writer, sheet_name="device_type", index=False) return ( "managed_ne_import_template.xlsx", - payload, + buf.getvalue(), "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) def import_managed_ne(db: Session, content: bytes, filename: str) -> ImportResult: _require_crypto() + name = str(filename or "").lower() try: - headers, raw_rows = read_table(content, filename) + if name.endswith(".csv"): + df = pd.read_csv(BytesIO(content)) + else: + df = pd.read_excel(BytesIO(content)) except Exception as exc: raise HTTPException(status_code=400, detail=f"import_parse_failed: {exc}") from exc - columns = [h.strip().lower() for h in headers] - missing = [c for c in _REQUIRED_IMPORT_COLUMNS if c not in columns] + df.columns = [str(c).strip().lower() for c in df.columns] + missing = [c for c in _REQUIRED_IMPORT_COLUMNS if c not in df.columns] if missing: raise HTTPException(status_code=400, detail=f"import_missing_columns: {','.join(missing)}") inserted = 0 updated = 0 failed: list[ImportFailure] = [] - for row_no, cells in raw_rows: - row = {k.strip().lower(): v for k, v in cells.items()} - # Savepoint per row: a failing row must not leave a half-written NE behind. - savepoint = db.begin_nested() + for idx, row in df.iterrows(): + row_no = int(idx) + 2 try: ip = _normalize_ip(_import_cell_str(row.get("ip", ""))) if not ip: failed.append(ImportFailure(row=row_no, reason="ip_required")) - savepoint.rollback() - continue - if not _is_valid_mgmt_address(ip): - failed.append(ImportFailure(row=row_no, reason="invalid_ip_address")) - savepoint.rollback() continue device_type = _import_cell_str(row.get("device_type", "")) if device_type not in SUPPORTED_DEVICE_TYPES: failed.append(ImportFailure(row=row_no, reason="unsupported_device_type")) - savepoint.rollback() continue username = _import_cell_str(row.get("username", "")) password = _import_cell_str(row.get("password", "")) if not username: failed.append(ImportFailure(row=row_no, reason="username_required")) - savepoint.rollback() continue - port_raw = row.get("port") + port_raw = row.get("port", 22) try: - port = int(float(port_raw)) if port_raw is not None else 22 - except (TypeError, ValueError, OverflowError): + port = int(port_raw) + except (TypeError, ValueError): port = 22 - if not 1 <= port <= 65535: - failed.append(ImportFailure(row=row_no, reason="invalid_port")) - savepoint.rollback() - continue - protocol = _normalize_protocol(_import_cell_str(row.get("protocol")) or "ssh") + protocol = _normalize_protocol(str(row.get("protocol", "ssh"))) display_name = _import_cell_str(row.get("name", "")) or ip vendor_raw = _import_cell_str(row.get("vendor", "")) or "Other" vendor = "Other" @@ -227,43 +217,29 @@ def import_managed_ne(db: Session, content: bytes, filename: str) -> ImportResul break existing = db.query(ManagedNE).filter(ManagedNE.ip_address == ip).first() now = _now() - is_new = existing is None if existing is None: - existing = ManagedNE(ip_address=ip, created_at=now, connect_status="unknown") + existing = ManagedNE(ip_address=ip, created_at=now) db.add(existing) - before = connect_snapshot(existing) + inserted += 1 + else: + updated += 1 existing.name = display_name existing.vendor = vendor existing.device_type = device_type existing.port = port existing.protocol = protocol existing.username = username - # Blank password on an existing NE keeps the stored one (template row 2 is blank). - if password: - existing.password_enc = encrypt_secret(password) - elif is_new: - existing.password_enc = "" + existing.password_enc = encrypt_secret(password) if password else "" tags_val = _import_cell_str(row.get("tags", "")) remark_val = _import_cell_str(row.get("remark", "")) if tags_val: existing.tags = tags_val if remark_val: existing.remark = remark_val - if is_placeholder_ne_source(existing.source): - existing.source = "" - if not is_new: - reset_connect_status_if_changed(existing, before) existing.updated_at = now - savepoint.commit() - if is_new: - inserted += 1 - else: - updated += 1 except CredentialCryptoError as exc: - savepoint.rollback() failed.append(ImportFailure(row=row_no, reason=str(exc))) except Exception as exc: - savepoint.rollback() failed.append(ImportFailure(row=row_no, reason=str(exc)[:200])) db.commit() return ImportResult(inserted=inserted, updated=updated, failed=failed) diff --git a/netx_api/ne_session_connect.py b/netx_api/ne_session_connect.py index 049a850..3813473 100644 --- a/netx_api/ne_session_connect.py +++ b/netx_api/ne_session_connect.py @@ -545,26 +545,18 @@ def _connect_direct( def _read_channel(conn: ConnectHandler, wait: float = 0.5, max_loops: int = 40) -> str: - """Collect device output. - - ``wait`` is the longest we park for the *first* bytes; once output has arrived the read - returns as soon as the channel is quiet again. (It used to sleep the whole ``wait`` and - then poll at 50-150ms, which added ~0.1-0.4s of dead time to every login-prompt step.) - """ - quiet = 0.04 if wait <= 0.15 else 0.12 - first_deadline = time.monotonic() + max(0.0, wait) - got_at: float | None = None + if wait > 0: + time.sleep(wait) chunks: list[str] = [] - reads = 0 - while reads < max_loops: + for _ in range(max_loops): try: part = conn.read_channel() except Exception: break if part is None or part is False: - part = "" + break # MagicMock truthy-but-empty: stop when unit tests leave read_channel unconfigured. - elif not isinstance(part, (str, bytes, bytearray)): + if not isinstance(part, (str, bytes, bytearray)): text = str(part) # unittest.mock default str looks like "" if text.startswith("= first_deadline: - break - elif now - got_at >= quiet: + if not part: break - time.sleep(0.01) + chunks.append(part) + time.sleep(0.05 if wait <= 0.15 else 0.15) return "".join(chunks) @@ -817,11 +801,11 @@ def _interactive_target_auth( return else: empty_after_pass = 0 - time.sleep(0.02) + time.sleep(0.12) continue if not buf.strip(): - time.sleep(0.02) + time.sleep(0.2) continue # Huawei stelnet often reprints the hop ``[sysname]`` after host-key trust and # *before* ``Enter password:``. Do not treat that as target login success. @@ -842,7 +826,7 @@ def _interactive_target_auth( if denied: raise paramiko.AuthenticationException(f"target_auth_rejected: {denied}") return - time.sleep(0.02) + time.sleep(0.15) denied = _target_auth_failure_snippet(acc, sent_pass=sent_pass) if denied: raise paramiko.AuthenticationException(f"target_auth_rejected: {denied}") @@ -895,7 +879,7 @@ def _huawei_enter_system_view( for _ in range(8): part = _read_channel(conn, wait=0.2, max_loops=10) if not part: - time.sleep(0.02) + time.sleep(0.15) continue acc += part if emit_raw: diff --git a/netx_api/schema_patches.py b/netx_api/schema_patches.py index 888d732..1dc4528 100644 --- a/netx_api/schema_patches.py +++ b/netx_api/schema_patches.py @@ -108,7 +108,6 @@ def apply_auth_schema_patches(conn: Connection) -> None: "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS must_change_password BOOLEAN DEFAULT FALSE", ) _run_sql(conn, "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS scopes JSON DEFAULT '[]'") - _run_sql(conn, "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'") _run_sql(conn, "ALTER TABLE api_token ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP") _run_sql(conn, "ALTER TABLE api_token ADD COLUMN IF NOT EXISTS scopes JSON DEFAULT '[]'") _run_sql( @@ -355,7 +354,6 @@ def apply_domain_schema_patches(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_ume_alarms_history_host_name ON ume_alarms_history (host_name)", "ALTER TABLE api_token ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP", "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS must_change_password BOOLEAN DEFAULT FALSE", - "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'", "ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_enabled BOOLEAN DEFAULT FALSE", "ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_vendor VARCHAR(32) DEFAULT 'zte'", "ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_host VARCHAR(128) DEFAULT ''", diff --git a/netx_api/table_filters.py b/netx_api/table_filters.py deleted file mode 100644 index c9bbd55..0000000 --- a/netx_api/table_filters.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Shared exact-value column filters, applied before pagination.""" -from __future__ import annotations - -import json -from dataclasses import dataclass, field -from typing import Any - -from fastapi import HTTPException, Query -from sqlalchemy import String, cast, func - - -@dataclass -class TableFilterParams: - filters: dict[str, dict[str, Any]] = field(default_factory=dict) - field: str = "" - - -def table_filter_params( - column_filters: str = Query(default=""), - filter_field: str = Query(default=""), -) -> TableFilterParams: - try: - filters = json.loads(column_filters) if column_filters else {} - if not isinstance(filters, dict): - raise ValueError() - for value in filters.values(): - if not isinstance(value, dict) or value.get("mode") not in ("include", "exclude"): - raise ValueError() - values = value.get("values") - if not isinstance(values, list) or any(not isinstance(v, str) for v in values): - raise ValueError() - except (ValueError, TypeError): - raise HTTPException(status_code=400, detail="invalid_column_filters") from None - return TableFilterParams(filters, filter_field) - - -def filter_query(query: Any, expressions: dict[str, Any], params: TableFilterParams | None): - """Return the filtered query and, when requested, a distinct-value response.""" - params = params or TableFilterParams() - if (set(params.filters) | ({params.field} if params.field else set())) - expressions.keys(): - raise HTTPException(status_code=400, detail="invalid_filter_field") - for key, value in params.filters.items(): - if key == params.field: - continue - expression = func.coalesce(cast(expressions[key], String), "") - values = value["values"] - if value["mode"] == "include": - query = query.filter(expression.in_(values)) if values else query.filter(False) - elif values: - query = query.filter(~expression.in_(values)) - if params.field: - expression = func.coalesce(cast(expressions[params.field], String), "") - rows = query.order_by(None).with_entities(expression).distinct().all() - return query, {"field": params.field, "options": sorted({str(row[0]) for row in rows})} - return query, None diff --git a/netx_api/tabular.py b/netx_api/tabular.py deleted file mode 100644 index c571456..0000000 --- a/netx_api/tabular.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Small xlsx/csv reader/writer for imports and templates (replaces pandas). - -Rows come back as ``(row_no, {header: value})`` where ``row_no`` is the 1-based -sheet/file row (header = row 1), so import errors point at the row users see. -Blank rows are skipped. Excel cells keep their openpyxl types (str / int / -float / datetime / bool / None); integral floats become ints. CSV cells are -strings, empty cells are None. -""" -from __future__ import annotations - -import csv -import io -import re -from collections.abc import Iterable, Iterator -from datetime import datetime -from typing import Any - -from openpyxl import Workbook, load_workbook - -Row = tuple[int, dict[str, Any]] - - -def _headers(raw: Iterable[Any]) -> list[str]: - out: list[str] = [] - seen: dict[str, int] = {} - for i, h in enumerate(raw): - name = "" if h is None else str(h).strip() - if not name: - name = f"Unnamed: {i}" - n = seen.get(name, 0) - seen[name] = n + 1 - out.append(name if n == 0 else f"{name}.{n}") - return out - - -def _cell(v: Any) -> Any: - if isinstance(v, str): - return v if v.strip() else None - if isinstance(v, float) and v.is_integer(): - return int(v) - return v - - -def _rows(records: Iterator[tuple[int, list[Any]]]) -> tuple[list[str], list[Row]]: - headers: list[str] | None = None - rows: list[Row] = [] - for row_no, values in records: - cells = [_cell(v) for v in values] - if all(c is None for c in cells): - continue - if headers is None: - headers = _headers(values) - continue - cells += [None] * (len(headers) - len(cells)) - rows.append((row_no, dict(zip(headers, cells)))) - return headers or [], rows - - -def _decode(content: bytes) -> str: - try: - return content.decode("utf-8-sig") - except UnicodeDecodeError: - # CSVs saved by Chinese Excel default to GBK. - return content.decode("gb18030") - - -def read_csv(content: bytes) -> tuple[list[str], list[Row]]: - reader = csv.reader(io.StringIO(_decode(content), newline="")) - return _rows((i, rec) for i, rec in enumerate(reader, start=1)) - - -def read_xlsx(content: bytes) -> tuple[list[str], list[Row]]: - wb = load_workbook(io.BytesIO(content), read_only=True, data_only=True) - try: - ws = wb.worksheets[0] - return _rows((i, list(rec)) for i, rec in enumerate(ws.iter_rows(values_only=True), start=1)) - finally: - wb.close() - - -def read_table(content: bytes, filename: str = "") -> tuple[list[str], list[Row]]: - if str(filename or "").lower().endswith(".csv"): - return read_csv(content) - return read_xlsx(content) - - -def write_csv(headers: list[str], rows: Iterable[Iterable[Any]]) -> bytes: - buf = io.StringIO() - writer = csv.writer(buf, lineterminator="\n") - writer.writerow(headers) - for r in rows: - writer.writerow(["" if v is None else v for v in r]) - # BOM so Excel opens UTF-8 (Chinese) correctly. - return buf.getvalue().encode("utf-8-sig") - - -def write_xlsx(sheets: list[tuple[str, list[str], Iterable[Iterable[Any]]]]) -> bytes: - wb = Workbook() - wb.remove(wb.active) - for title, headers, rows in sheets: - ws = wb.create_sheet(title) - ws.append(headers) - for r in rows: - ws.append(list(r)) - buf = io.BytesIO() - wb.save(buf) - return buf.getvalue() - - -_DATE_FORMATS = ( - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%d %H:%M", - "%Y-%m-%d", - "%Y%m%d%H%M%S", - "%Y%m%d", - "%m/%d/%Y %H:%M:%S", - "%m/%d/%Y %H:%M", - "%m/%d/%Y", - "%d-%b-%Y %H:%M:%S", - "%b %d, %Y %H:%M:%S", -) -_CN_DATE = re.compile(r"(\d{4})年(\d{1,2})月(\d{1,2})日") - - -def parse_datetime(v: Any) -> datetime | None: - """Best-effort timestamp parse for spreadsheet cells; None when unparseable.""" - if v is None: - return None - if isinstance(v, datetime): - return v - text = str(v).strip() - if not text or text.lower() in ("nan", "nat", "none", "null"): - return None - text = _CN_DATE.sub(r"\1-\2-\3", text) - # 2024/1/2 03:04:05 -> 2024-1-2 ... ; zero-pad so fromisoformat/strptime accept it. - if re.match(r"^\d{4}/\d{1,2}/\d{1,2}", text): - text = text.replace("/", "-", 2) - m = re.match(r"^(\d{4})-(\d{1,2})-(\d{1,2})(.*)$", text) - if m: - text = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}{m.group(4)}" - iso = text[:-1] + "+00:00" if text.endswith(("Z", "z")) else text - try: - return datetime.fromisoformat(iso) - except ValueError: - pass - for fmt in _DATE_FORMATS: - try: - return datetime.strptime(text, fmt) - except ValueError: - continue - return None diff --git a/netx_api/topology_discover_common.py b/netx_api/topology_discover_common.py index cb2adb0..9558318 100644 --- a/netx_api/topology_discover_common.py +++ b/netx_api/topology_discover_common.py @@ -4,10 +4,9 @@ from __future__ import annotations from typing import Any from fastapi import HTTPException -from sqlalchemy import String, case, cast, func, or_ +from sqlalchemy import or_ from sqlalchemy.orm import Session -from .table_filters import TableFilterParams, filter_query from .cli_resolve import get_default_profile, infer_device_type_vendor from .device_types import is_placeholder_ne_source from .models import ManagedNE, TopoDiscoverJob, TopoDiscoverJobItem, UmeInventoryNE @@ -43,17 +42,6 @@ def _managed_target_dict(ne: ManagedNE) -> dict[str, str]: } -def discover_item_expressions() -> dict[str, Any]: - item = TopoDiscoverJobItem - failed = or_(item.ok.is_(False), item.parser_stub.is_(True), item.error.in_(["parser_stub", "empty_cli_output", "vendor_or_device_type_required"])) - return { - "name": func.coalesce(func.nullif(item.ne_name, ""), item.ne_id), "ip": item.ne_ip, - "status": case((failed, "fail"), (item.unmatched_count > 0, "warn"), else_="ok"), - "neighbors": item.neighbors, "error": item.error, - "edges": cast(item.edges_added, String) + "/" + cast(item.edges_updated, String), - } - - def _job_out( db: Session, job: TopoDiscoverJob, @@ -63,7 +51,6 @@ def _job_out( page_size: int | None = None, item_status: str = "", item_keyword: str = "", - table_filters: TableFilterParams | None = None, ) -> FabricDiscoverJobOut: items_out: list[FabricDiscoverJobItemOut] = [] items_total = 0 @@ -106,7 +93,6 @@ def _job_out( TopoDiscoverJobItem.error.ilike(like), ) ) - q, _ = filter_query(q, discover_item_expressions(), table_filters) items_total = int(q.count()) if page is not None and page_size is not None: items_page = max(1, int(page or 1)) @@ -171,7 +157,6 @@ def get_discover_job( page_size: int | None = None, item_status: str = "", item_keyword: str = "", - table_filters: TableFilterParams | None = None, ) -> FabricDiscoverJobOut: job = db.get(TopoDiscoverJob, str(job_id or "").strip()) if job is None: @@ -183,7 +168,6 @@ def get_discover_job( page_size=page_size, item_status=item_status, item_keyword=item_keyword, - table_filters=table_filters, ) diff --git a/netx_api/topology_fabric_nodes.py b/netx_api/topology_fabric_nodes.py index 5164ff0..a42f946 100644 --- a/netx_api/topology_fabric_nodes.py +++ b/netx_api/topology_fabric_nodes.py @@ -8,11 +8,10 @@ from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import and_, case, func, or_, text +from sqlalchemy import and_, func, or_, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session -from .table_filters import TableFilterParams, filter_query from .cli_resolve import get_default_profile, infer_device_type_vendor from .device_types import LLDP_DISCOVERED_NE_SOURCE, WEBCRT_NE_SOURCE from .models import ( @@ -276,7 +275,6 @@ def list_fabric_nodes( region_folder_id: str = "", unmatched: str = "", link_status: str = "", - table_filters: TableFilterParams | None = None, page: int = 1, page_size: int = PAGE_DEFAULT, ) -> dict[str, Any]: @@ -377,16 +375,6 @@ def list_fabric_nodes( and_(TopoFabricNode.managed_ne_id.isnot(None), TopoFabricNode.managed_ne_id != ""), and_(TopoFabricNode.ume_ne_id.isnot(None), TopoFabricNode.ume_ne_id != ""), ) - has_managed = func.coalesce(TopoFabricNode.managed_ne_id, "") != "" - has_ume = func.coalesce(TopoFabricNode.ume_ne_id, "") != "" - q, options = filter_query(q, { - "name": func.coalesce(func.nullif(TopoFabricNode.name, ""), TopoFabricNode.id), - "ip": TopoFabricNode.ip, "level": TopoFabricNode.level, - "region": TopoFabricNode.region_folder_id, - "link": case((and_(has_managed, has_ume), "both"), (has_managed, "managed"), (has_ume, "ume"), else_="orphaned"), - }, table_filters) - if options is not None: - return options total = int(q.count()) rows = ( q.order_by(TopoFabricNode.name.asc()) @@ -411,7 +399,6 @@ def list_fabric_edges( status: str = "", source: str = "", keyword: str = "", - table_filters: TableFilterParams | None = None, page: int = 1, page_size: int = PAGE_DEFAULT, ) -> dict[str, Any]: @@ -451,23 +438,6 @@ def list_fabric_edges( TopoFabricEdge.b_node_id.in_(matched_ids), ) ) - from sqlalchemy.orm import aliased - a = aliased(TopoFabricNode) - b = aliased(TopoFabricNode) - q = q.outerjoin(a, TopoFabricEdge.a_node_id == a.id).outerjoin(b, TopoFabricEdge.b_node_id == b.id) - status_expr = case((TopoFabricEdge.status == "stale", "missing"), else_=TopoFabricEdge.status) - miss = func.coalesce(TopoFabricEdge.attrs["miss_count"].as_integer(), 0) - q, options = filter_query(q, { - "status": status_expr, - "aSide": func.coalesce(func.nullif(a.name, ""), func.nullif(a.ip, ""), func.substr(TopoFabricEdge.a_node_id, 1, 8)), - "bSide": func.coalesce(func.nullif(b.name, ""), func.nullif(b.ip, ""), func.substr(TopoFabricEdge.b_node_id, 1, 8)), - "aPort": TopoFabricEdge.a_port, "bPort": TopoFabricEdge.b_port, - "source": case((TopoFabricEdge.source == "stale", "lldp"), else_=func.coalesce(func.nullif(TopoFabricEdge.source, ""), "lldp")), - "lastSeen": TopoFabricEdge.last_seen_at, - "missCount": case((miss != 0, miss), (status_expr == "missing", 1), else_=0), - }, table_filters) - if options is not None: - return options total = int(q.count()) rows = ( q.order_by(TopoFabricEdge.updated_at.desc()) @@ -620,13 +590,9 @@ def find_fabric_paths( if len(edge_path) >= max_hops: continue _ensure_adj({node}) - # Parallel links (LAG members / dual fibers) between the same two NEs are one hop: - # expanding each would return near-identical paths that differ only by port. - seen_nbrs: set[str] = set() for nbr, eid in adj_cache.get(node, []): - if nbr in visited or nbr in seen_nbrs: + if nbr in visited: continue - seen_nbrs.add(nbr) explored += 1 new_path = edge_path + [eid] if nbr == to_id: @@ -654,15 +620,10 @@ def find_fabric_paths( "vendor": n.vendor or "", } - def _parallel_count(e: TopoFabricEdge) -> int: - # Only expanded nodes have adjacency loaded; the path's last node never is. - src, dst = (e.a_node_id, e.b_node_id) if e.a_node_id in adj_loaded else (e.b_node_id, e.a_node_id) - return sum(1 for nbr, _ in adj_cache.get(src, []) if nbr == dst) or 1 - def _edge_summary(e: TopoFabricEdge) -> dict[str, Any]: a_node = node_map.get(e.a_node_id) b_node = node_map.get(e.b_node_id) - out = { + return { "a_name": (a_node.name if a_node else "") or "", "b_name": (b_node.name if b_node else "") or "", "a_ip": (a_node.ip if a_node else "") or "", @@ -671,10 +632,6 @@ def find_fabric_paths( "b_port": e.b_port or "", "status": _normalize_edge_status(e.status or "active"), } - parallel = _parallel_count(e) - if parallel > 1: - out["parallel_links"] = parallel - return out def _path_node_ids(edge_ids: list[str]) -> list[str]: ids = [from_id] diff --git a/netx_api/topology_region_canvas.py b/netx_api/topology_region_canvas.py index ba5f38c..22f6be9 100644 --- a/netx_api/topology_region_canvas.py +++ b/netx_api/topology_region_canvas.py @@ -121,39 +121,6 @@ def place_child_region_on_parent_canvas( return row -def add_missing_region_placement( - db: Session, view: TopoView, fabric_node_id: str, *, x: float, y: float -) -> TopoViewNode | None: - """Persist a child-region icon that was only synthesized (no placement row yet). - - Returns None unless ``fabric_node_id`` is a manual child region of the view's folder. - """ - if not is_region_canvas_node_id(fabric_node_id) or _is_world_flat(view): - return None - folder = db.get(TopoFolder, region_folder_id_from_node(fabric_node_id)) - if ( - folder is None - or str(folder.kind or "") != "region" - or str(folder.parent_id or "") != str(view.folder_id or "") - or _is_ume_managed_folder(folder) - ): - return None - now = _utcnow() - row = TopoViewNode( - id=uuid4().hex, - view_id=view.id, - fabric_node_id=fabric_node_id, - x=float(x), - y=float(y), - label="", - locked=False, - created_at=now, - updated_at=now, - ) - db.add(row) - return row - - def remove_region_canvas_placements(db: Session, folder_id: str) -> int: """Drop this folder's icon from any parent canvas.""" nid = region_canvas_node_id(folder_id) @@ -172,7 +139,6 @@ def region_node_out( y: float, locked: bool = False, label: str = "", - label_override: str | None = None, ) -> ViewNodeOut: child_view = _child_primary_view(db, folder.id) kids = ( @@ -188,14 +154,12 @@ def region_node_out( .filter(TopoViewNode.view_id == child_view.id) .count() ) - raw = str(label_override if label_override is not None else label or "") - display = (raw or folder.name or folder.id)[:256] + display = (label or folder.name or folder.id)[:256] return ViewNodeOut( fabric_node_id=region_canvas_node_id(folder.id), managed_ne_id="", ume_ne_id="", label=display, - label_override=raw[:256], x=float(x), y=float(y), locked=bool(locked), diff --git a/netx_api/topology_router.py b/netx_api/topology_router.py index ef79c06..b245cc7 100644 --- a/netx_api/topology_router.py +++ b/netx_api/topology_router.py @@ -7,7 +7,6 @@ from typing import Any from fastapi import APIRouter, Body, Depends, HTTPException, Query from sqlalchemy.orm import Session -from .table_filters import TableFilterParams, table_filter_params from .db import get_db from .topology_classify import ( apply_classify, @@ -39,8 +38,6 @@ from .topology_schemas import ( TopologyPlaceholderCreate, TopologyViewCreate, TopologyViewUpdate, - FabricEdgePortsPatch, - ViewCommitIn, ViewEdgeStylePatch, ViewNodesAdd, ViewNodesRemove, @@ -64,7 +61,6 @@ from .topology_fabric import ( from .topology_views import ( add_nodes_to_view, bootstrap_topology_tree, - commit_view_changes, create_folder, create_topology_placeholder_on_view, create_view, @@ -73,7 +69,6 @@ from .topology_views import ( get_topology_tree, get_view_graph, list_views, - patch_manual_edge_ports, patch_view_edge_style, patch_view_positions, populate_view, @@ -108,7 +103,6 @@ def api_fabric_nodes( ), page: int = Query(default=1, ge=1), page_size: int = Query(default=100, ge=1, le=2000), - table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: return list_fabric_nodes( @@ -120,7 +114,6 @@ def api_fabric_nodes( region_folder_id=region_folder_id, unmatched=unmatched, link_status=link_status, - table_filters=table_filters, page=page, page_size=page_size, ) @@ -135,7 +128,6 @@ def api_fabric_edges( keyword: str = "", page: int = Query(default=1, ge=1), page_size: int = Query(default=100, ge=1, le=2000), - table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: return list_fabric_edges( @@ -145,7 +137,6 @@ def api_fabric_edges( status=status, source=source, keyword=keyword, - table_filters=table_filters, page=page, page_size=page_size, ) @@ -490,32 +481,6 @@ def api_edge_style( return patch_view_edge_style(db, view_id, body).model_dump() -@router.post("/views/{view_id}/commit") -def api_commit_view( - view_id: str, body: ViewCommitIn, db: Session = Depends(get_db) -) -> dict[str, Any]: - return commit_view_changes(db, view_id, body).model_dump() - - -@router.patch("/fabric/edges/{edge_id}") -def api_patch_fabric_edge_ports( - edge_id: str, body: FabricEdgePortsPatch, db: Session = Depends(get_db) -) -> dict[str, Any]: - edge = patch_manual_edge_ports( - db, edge_id, a_port=body.a_port, b_port=body.b_port, commit=True - ) - return { - "id": edge.id, - "a_node_id": edge.a_node_id, - "b_node_id": edge.b_node_id, - "a_port": edge.a_port or "", - "b_port": edge.b_port or "", - "source": edge.source or "manual", - "status": edge.status or "active", - "layer": edge.layer or "physical", - } - - # --- Classify rules / slices / search --------------------------------------- diff --git a/netx_api/topology_schemas.py b/netx_api/topology_schemas.py index 9ab3023..e42f6be 100644 --- a/netx_api/topology_schemas.py +++ b/netx_api/topology_schemas.py @@ -273,10 +273,8 @@ class ViewNodeIn(BaseModel): fabric_node_id: str = Field(min_length=1, max_length=64) x: float = 0.0 y: float = 0.0 - # None = leave existing override unchanged; "" = clear override (follow device name). - label: str | None = None - # None = leave lock unchanged; False unlocks (and allows move); True locks after move. - locked: bool | None = None + label: str = "" + locked: bool = False class ViewNodeOut(BaseModel): @@ -284,8 +282,6 @@ class ViewNodeOut(BaseModel): managed_ne_id: str = "" ume_ne_id: str = "" label: str = "" - # Raw placement override (empty = follow fabric name/ip). ``label`` is the display value. - label_override: str = "" x: float = 0.0 y: float = 0.0 locked: bool = False @@ -452,16 +448,6 @@ class ViewNodesRemove(BaseModel): class ViewEdgeStylePatch(BaseModel): - """Single-edge style patch (legacy). Prefer ``items`` for batch.""" - - fabric_edge_id: str = "" - stroke_color: str = "" - stroke_width: int = Field(default=0, ge=0, le=12) - line_style: str = Field(default="", max_length=16) - items: list["ViewEdgeStyleItem"] = Field(default_factory=list) - - -class ViewEdgeStyleItem(BaseModel): fabric_edge_id: str stroke_color: str = "" stroke_width: int = Field(default=0, ge=0, le=12) @@ -475,11 +461,6 @@ class FabricManualEdgeIn(BaseModel): b_port: str = "" -class FabricEdgePortsPatch(BaseModel): - a_port: str = "" - b_port: str = "" - - class FabricEdgesDeleteRequest(BaseModel): edge_ids: list[str] = Field(default_factory=list, min_length=1) @@ -488,40 +469,6 @@ class FabricEdgesDeleteOut(BaseModel): deleted: int = 0 -class ViewCommitEdgeCreate(BaseModel): - client_id: str = "" - a_node_id: str = Field(min_length=1, max_length=64) - b_node_id: str = Field(min_length=1, max_length=64) - a_port: str = "" - b_port: str = "" - - -class ViewCommitEdgePortEdit(BaseModel): - id: str = Field(min_length=1, max_length=64) - a_port: str = "" - b_port: str = "" - - -class ViewCommitIn(BaseModel): - """Atomic canvas save: all steps succeed or the whole request rolls back.""" - - edge_creates: list[ViewCommitEdgeCreate] = Field(default_factory=list) - edge_deletes: list[str] = Field(default_factory=list) - edge_port_edits: list[ViewCommitEdgePortEdit] = Field(default_factory=list) - edge_styles: list[ViewEdgeStyleItem] = Field(default_factory=list) - node_adds: list[str] = Field(default_factory=list) - node_removes: list[str] = Field(default_factory=list) - positions: list[ViewNodeIn] = Field(default_factory=list) - - -class ViewCommitOut(BaseModel): - ok: bool = True - view_id: str = "" - edge_id_map: dict[str, str] = Field(default_factory=dict) - skipped_locked: int = 0 - graph: TopologyViewGraphOut | None = None - - # --------------------------------------------------------------------------- # Classify rules + slices + search # --------------------------------------------------------------------------- diff --git a/netx_api/topology_service.py b/netx_api/topology_service.py index f65ac30..431bc12 100644 --- a/netx_api/topology_service.py +++ b/netx_api/topology_service.py @@ -39,7 +39,6 @@ from .topology_views import ( _place_fabric_ids_on_view, add_nodes_to_view, bootstrap_topology_tree, - commit_view_changes, create_folder, create_topology_placeholder_on_view, create_view, @@ -49,7 +48,6 @@ from .topology_views import ( get_topology_tree, get_view_graph, list_views, - patch_manual_edge_ports, patch_view_edge_style, patch_view_positions, populate_view, @@ -70,7 +68,6 @@ __all__ = [ "_place_fabric_ids_on_view", "add_nodes_to_view", "bootstrap_topology_tree", - "commit_view_changes", "create_folder", "create_topology_placeholder_on_view", "create_view", @@ -91,7 +88,6 @@ __all__ = [ "list_fabric_nodes", "list_views", "merge_duplicate_fabric_nodes", - "patch_manual_edge_ports", "patch_view_edge_style", "patch_view_positions", "pause_discover_job", diff --git a/netx_api/topology_views.py b/netx_api/topology_views.py index a8af872..36d8bfb 100644 --- a/netx_api/topology_views.py +++ b/netx_api/topology_views.py @@ -1,7 +1,6 @@ """Topology folder tree and leaf view operations (facade).""" from __future__ import annotations -from .topology_views_commit import commit_view_changes, patch_manual_edge_ports from .topology_views_graph import ( _place_fabric_ids_on_view, add_nodes_to_view, @@ -30,7 +29,6 @@ __all__ = [ "_place_fabric_ids_on_view", "add_nodes_to_view", "bootstrap_topology_tree", - "commit_view_changes", "create_folder", "create_topology_placeholder_on_view", "create_view", @@ -40,7 +38,6 @@ __all__ = [ "get_topology_tree", "get_view_graph", "list_views", - "patch_manual_edge_ports", "patch_view_edge_style", "patch_view_positions", "populate_view", diff --git a/netx_api/topology_views_commit.py b/netx_api/topology_views_commit.py deleted file mode 100644 index dfd5f88..0000000 --- a/netx_api/topology_views_commit.py +++ /dev/null @@ -1,444 +0,0 @@ -"""Atomic view canvas commit (single transaction).""" - -from __future__ import annotations - -import re -from typing import Any -from uuid import uuid4 - -from fastapi import HTTPException -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session - -from .models import ( - TopoFabricEdge, - TopoFabricNode, - TopoViewEdgeStyle, - TopoViewNode, -) -from .topology_common import VIEW_GRAPH_NODE_HARD_CAP, _utcnow -from .topology_fabric_links import upsert_fabric_edge -from .topology_fabric_nodes import _normalize_endpoints, refresh_fabric_stats -from .topology_region_canvas import add_missing_region_placement, is_region_canvas_node_id -from .topology_schemas import ( - TopologyViewGraphOut, - ViewCommitIn, - ViewCommitOut, - ViewEdgeStyleItem, - ViewNodeIn, -) -from .topology_views_graph import ( - _get_view_or_404, - _membership_for_view, - get_view_graph, -) -from .ume_topology_world import is_world_flat_view - -_HEX_COLOR_RE = re.compile(r"^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$") -_LINE_STYLES = {"", "solid", "dashed", "dotted"} - - -def _fail(step: str, detail: str, *, status: int = 400) -> None: - raise HTTPException( - status_code=status, - detail={"failed_step": step, "detail": detail}, - ) - - -def _apply_edge_creates( - db: Session, creates: list[Any], edge_id_map: dict[str, str] -) -> None: - for item in creates: - a = str(getattr(item, "a_node_id", "") or "").strip() - b = str(getattr(item, "b_node_id", "") or "").strip() - if not a or not b: - _fail("edge_creates", "edge_endpoints_required") - edge, action = upsert_fabric_edge( - db, - a_node_id=a, - b_node_id=b, - a_port=str(getattr(item, "a_port", "") or ""), - b_port=str(getattr(item, "b_port", "") or ""), - source="manual", - ) - if edge is None or action == "skipped_self_loop": - _fail("edge_creates", "edge_self_loop") - client_id = str(getattr(item, "client_id", "") or "").strip() - if client_id: - edge_id_map[client_id] = str(edge.id) - - -def _apply_edge_deletes(db: Session, edge_ids: list[str]) -> set[str]: - """Delete edges; return the ids that were requested (for skipping later edits).""" - ids = [str(x).strip() for x in (edge_ids or []) if str(x).strip()] - if not ids: - return set() - seen: set[str] = set() - ordered: list[str] = [] - for i in ids: - if i in seen: - continue - seen.add(i) - ordered.append(i) - rows = db.query(TopoFabricEdge).filter(TopoFabricEdge.id.in_(ordered)).all() - found_ids = [str(r.id) for r in rows] - if not found_ids: - return seen - db.query(TopoViewEdgeStyle).filter( - TopoViewEdgeStyle.fabric_edge_id.in_(found_ids) - ).delete(synchronize_session=False) - db.query(TopoFabricEdge).filter(TopoFabricEdge.id.in_(found_ids)).delete( - synchronize_session=False - ) - return seen - - -def patch_manual_edge_ports( - db: Session, - edge_id: str, - *, - a_port: str, - b_port: str, - commit: bool = True, -) -> TopoFabricEdge: - """Update ports on a manual fabric edge. Raises 400/404/409.""" - eid = str(edge_id or "").strip() - row = db.get(TopoFabricEdge, eid) if eid else None - if row is None: - raise HTTPException(status_code=404, detail="fabric_edge_not_found") - if str(row.source or "").strip().lower() != "manual": - raise HTTPException(status_code=400, detail="edge_not_manual") - _a, _b, ap, bp = _normalize_endpoints( - row.a_node_id, - row.b_node_id, - str(a_port or ""), - str(b_port or ""), - ) - conflict = ( - db.query(TopoFabricEdge) - .filter( - TopoFabricEdge.id != row.id, - TopoFabricEdge.layer == row.layer, - TopoFabricEdge.a_node_id == row.a_node_id, - TopoFabricEdge.b_node_id == row.b_node_id, - TopoFabricEdge.a_port == ap, - TopoFabricEdge.b_port == bp, - ) - .one_or_none() - ) - if conflict is not None: - raise HTTPException(status_code=409, detail="edge_port_conflict") - row.a_port = ap - row.b_port = bp - row.updated_at = _utcnow() - try: - db.flush() - except IntegrityError as exc: - raise HTTPException(status_code=409, detail="edge_port_conflict") from exc - if commit: - db.commit() - db.refresh(row) - return row - - -def _apply_edge_port_edits(db: Session, edits: list[Any], *, deleted: set[str]) -> None: - for item in edits: - eid = str(getattr(item, "id", "") or "").strip() - if not eid: - _fail("edge_port_edits", "edge_id_required") - if eid in deleted: - continue - try: - patch_manual_edge_ports( - db, - eid, - a_port=str(getattr(item, "a_port", "") or ""), - b_port=str(getattr(item, "b_port", "") or ""), - commit=False, - ) - except HTTPException as exc: - detail = exc.detail if isinstance(exc.detail, str) else "edge_port_edit_failed" - _fail("edge_port_edits", str(detail), status=int(exc.status_code or 400)) - - -def _upsert_edge_style( - db: Session, - view_id: str, - item: ViewEdgeStyleItem, - *, - now, -) -> None: - eid = str(item.fabric_edge_id or "").strip() - if not eid or db.get(TopoFabricEdge, eid) is None: - _fail("edge_styles", "fabric_edge_not_found", status=404) - color = str(item.stroke_color or "").strip() - if color and not _HEX_COLOR_RE.match(color): - _fail("edge_styles", "invalid_stroke_color") - width = int(item.stroke_width or 0) - if width < 0 or width > 12: - _fail("edge_styles", "invalid_stroke_width") - style = str(item.line_style or "").strip().lower() - if style not in _LINE_STYLES: - _fail("edge_styles", "invalid_line_style") - row = ( - db.query(TopoViewEdgeStyle) - .filter( - TopoViewEdgeStyle.view_id == view_id, - TopoViewEdgeStyle.fabric_edge_id == eid, - ) - .one_or_none() - ) - if row is None: - db.add( - TopoViewEdgeStyle( - id=uuid4().hex, - view_id=view_id, - fabric_edge_id=eid, - stroke_color=color, - stroke_width=width, - line_style=style, - created_at=now, - updated_at=now, - ) - ) - else: - row.stroke_color = color - row.stroke_width = width - row.line_style = style - row.updated_at = now - - -def _apply_edge_styles( - db: Session, - view_id: str, - styles: list[ViewEdgeStyleItem], - *, - edge_id_map: dict[str, str], - deleted: set[str], -) -> None: - now = _utcnow() - for item in styles: - eid = str(item.fabric_edge_id or "").strip() - if eid in deleted: - continue - # Styles on a just-created local edge are keyed by its client id. - if eid in edge_id_map: - item = item.model_copy(update={"fabric_edge_id": edge_id_map[eid]}) - _upsert_edge_style(db, view_id, item, now=now) - - -def _apply_node_adds( - db: Session, - view, - fabric_ids: list[str], - *, - max_nodes: int, - existing: dict[str, TopoViewNode], -) -> None: - if is_world_flat_view(view): - _fail("node_adds", "world_map_no_direct_nes") - ids = [str(x).strip() for x in (fabric_ids or []) if str(x).strip()] - if not ids: - return - if len(existing) >= max_nodes: - _fail("node_adds", "membership_max_nodes") - now = _utcnow() - room = max(0, max_nodes - len(existing)) - added = 0 - for fid in ids: - if fid in existing: - continue - if is_region_canvas_node_id(fid): - continue - if db.get(TopoFabricNode, fid) is None: - _fail("node_adds", f"fabric_node_not_found:{fid}") - if added >= room: - _fail("node_adds", "membership_max_nodes") - row = TopoViewNode( - id=uuid4().hex, - view_id=view.id, - fabric_node_id=fid, - x=40.0, - y=40.0, - label="", - locked=False, - created_at=now, - updated_at=now, - ) - db.add(row) - existing[fid] = row - added += 1 - if added: - view.updated_at = now - - -def _apply_node_removes( - db: Session, - view, - fabric_ids: list[str], - *, - existing: dict[str, TopoViewNode], -) -> None: - ids = [str(x).strip() for x in (fabric_ids or []) if str(x).strip()] - ids = [x for x in ids if x and not is_region_canvas_node_id(x)] - if not ids: - return - db.query(TopoViewNode).filter( - TopoViewNode.view_id == view.id, - TopoViewNode.fabric_node_id.in_(ids), - ).delete(synchronize_session=False) - for fid in ids: - existing.pop(fid, None) - view.updated_at = _utcnow() - - -def _apply_positions( - db: Session, - view, - positions: list[ViewNodeIn], - *, - existing: dict[str, TopoViewNode], -) -> int: - """Apply positions in-session. Returns skipped_locked count.""" - if len(positions) > VIEW_GRAPH_NODE_HARD_CAP: - _fail("positions", "too_many_positions") - now = _utcnow() - skipped_locked = 0 - updated = 0 - fabric_ids = [ - str(p.fabric_node_id or "").strip() - for p in positions - if str(p.fabric_node_id or "").strip() - and not is_region_canvas_node_id(str(p.fabric_node_id or "").strip()) - ] - alive = ( - {str(r[0]) for r in db.query(TopoFabricNode.id).filter(TopoFabricNode.id.in_(fabric_ids))} - if fabric_ids - else set() - ) - for p in positions: - fid = str(p.fabric_node_id or "").strip() - if not fid: - continue - if not is_region_canvas_node_id(fid) and fid not in alive and fid not in existing: - # Missing fabric: skip rather than fail mid-commit when node was just removed. - continue - row = existing.get(fid) - if row is None: - # Fabric NEs are placed via node_adds; only synthesized child-region icons are created here. - row = add_missing_region_placement(db, view, fid, x=p.x, y=p.y) - if row is None: - continue - existing[fid] = row - if row.locked and p.locked is not False: - skipped_locked += 1 - continue - row.x = float(p.x or 0) - row.y = float(p.y or 0) - if p.label is not None: - row.label = str(p.label or "")[:256] - if p.locked is not None: - row.locked = bool(p.locked) - row.updated_at = now - updated += 1 - if updated or skipped_locked: - view.updated_at = now - return skipped_locked - - -def commit_view_changes(db: Session, view_id: str, body: ViewCommitIn) -> ViewCommitOut: - """Apply canvas mutations atomically; return one graph + client→server edge map.""" - view = _get_view_or_404(db, view_id) - mem = _membership_for_view(view) - max_nodes = int(mem.get("max_nodes") or 300) - existing = { - vn.fabric_node_id: vn - for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all() - } - edge_id_map: dict[str, str] = {} - skipped_locked = 0 - - try: - _apply_edge_creates(db, list(body.edge_creates or []), edge_id_map) - deleted = _apply_edge_deletes(db, list(body.edge_deletes or [])) - _apply_edge_port_edits(db, list(body.edge_port_edits or []), deleted=deleted) - _apply_edge_styles( - db, - view.id, - list(body.edge_styles or []), - edge_id_map=edge_id_map, - deleted=deleted, - ) - _apply_node_adds( - db, - view, - list(body.node_adds or []), - max_nodes=max_nodes, - existing=existing, - ) - _apply_node_removes( - db, - view, - list(body.node_removes or []), - existing=existing, - ) - # Re-load placements after add/remove so positions see current rows. - # Flush first: sessions run with autoflush=False, so new placements would be invisible. - db.flush() - existing = { - vn.fabric_node_id: vn - for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all() - } - skipped_locked = _apply_positions( - db, - view, - list(body.positions or []), - existing=existing, - ) - db.commit() - except HTTPException: - db.rollback() - raise - except Exception as exc: # noqa: BLE001 - db.rollback() - _fail("commit", str(exc)[:500], status=500) - - try: - refresh_fabric_stats(db) - except Exception: # noqa: BLE001 - pass - - graph: TopologyViewGraphOut = get_view_graph(db, view.id) - return ViewCommitOut( - ok=True, - view_id=view.id, - edge_id_map=edge_id_map, - skipped_locked=skipped_locked, - graph=graph, - ) - - -def normalize_edge_style_items(body: Any) -> list[ViewEdgeStyleItem]: - """Accept legacy single-edge body or batch ``items``.""" - items = list(getattr(body, "items", None) or []) - if items: - return [ - ViewEdgeStyleItem( - fabric_edge_id=str(i.fabric_edge_id or ""), - stroke_color=str(i.stroke_color or ""), - stroke_width=int(i.stroke_width or 0), - line_style=str(i.line_style or ""), - ) - for i in items - ] - fid = str(getattr(body, "fabric_edge_id", "") or "").strip() - if not fid: - return [] - return [ - ViewEdgeStyleItem( - fabric_edge_id=fid, - stroke_color=str(getattr(body, "stroke_color", "") or ""), - stroke_width=int(getattr(body, "stroke_width", 0) or 0), - line_style=str(getattr(body, "line_style", "") or ""), - ) - ] diff --git a/netx_api/topology_views_graph.py b/netx_api/topology_views_graph.py index 6409861..41e692d 100644 --- a/netx_api/topology_views_graph.py +++ b/netx_api/topology_views_graph.py @@ -157,12 +157,7 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: # Save/mutate return_graph must use the same UME synthetic graph as GET. if is_ume_canvas_view(view): return get_ume_canvas_graph(db, view_id) - vnodes = ( - db.query(TopoViewNode) - .filter(TopoViewNode.view_id == view.id) - .order_by(TopoViewNode.created_at.asc(), TopoViewNode.id.asc()) - .all() - ) + vnodes = db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all() truncated = False reason = "" if len(vnodes) > VIEW_GRAPH_NODE_HARD_CAP: @@ -174,19 +169,6 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: n.id: n for n in db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(fids)).all() } if fids else {} managed_by_id, ume_by_id = _batch_ne_lookups(db, fabric_nodes) - region_folder_ids = [ - region_folder_id_from_node(vn.fabric_node_id) - for vn in vnodes - if is_region_canvas_node_id(vn.fabric_node_id) - ] - folders_by_id = { - f.id: f - for f in ( - db.query(TopoFolder).filter(TopoFolder.id.in_(region_folder_ids)).all() - if region_folder_ids - else [] - ) - } filt = dict(view.filter or {}) layer = str(filt.get("layer") or "physical").strip() or "physical" status = str(filt.get("status") or "").strip().lower() @@ -194,7 +176,7 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: region_ids_seen: set[str] = set() for vn in vnodes: if is_region_canvas_node_id(vn.fabric_node_id): - folder = folders_by_id.get(region_folder_id_from_node(vn.fabric_node_id)) + folder = db.get(TopoFolder, region_folder_id_from_node(vn.fabric_node_id)) if folder is None or str(folder.kind or "") != "region": continue region_ids_seen.add(folder.id) @@ -205,13 +187,12 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: x=float(vn.x or 0), y=float(vn.y or 0), locked=bool(vn.locked), - label_override=str(vn.label or ""), + label=str(vn.label or folder.name or ""), ) ) continue fn = fabric_nodes.get(vn.fabric_node_id) - label_override = str(vn.label or "") - label = label_override.strip() + label = (vn.label or "").strip() if not label and fn is not None: label = (fn.name or fn.ip or vn.fabric_node_id)[:256] connect_status = "" @@ -235,7 +216,6 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: managed_ne_id=(fn.managed_ne_id if fn else "") or "", ume_ne_id=(fn.ume_ne_id if fn else "") or "", label=label, - label_override=label_override[:256], x=float(vn.x or 0), y=float(vn.y or 0), locked=bool(vn.locked), @@ -368,31 +348,22 @@ def _outside_peers_for_view( member_ids: set[str], layer: str, limit: int = 50, - edge_scan_limit: int = 5000, ) -> list[dict[str, str]]: if not member_ids: return [] - members = list(member_ids) - # Exclude edges whose both ends are already on the view so dense canvases - # do not burn the scan budget before any external neighbor is found. - both_internal = and_( - TopoFabricEdge.a_node_id.in_(members), - TopoFabricEdge.b_node_id.in_(members), - ) edges = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == layer, or_( - TopoFabricEdge.a_node_id.in_(members), - TopoFabricEdge.b_node_id.in_(members), + TopoFabricEdge.a_node_id.in_(list(member_ids)), + TopoFabricEdge.b_node_id.in_(list(member_ids)), ), - ~both_internal, ) - .limit(max(1, int(edge_scan_limit or 5000))) + .limit(5000) .all() ) - peer_via: list[tuple[str, str]] = [] + out: list[dict[str, str]] = [] seen: set[str] = set() for e in edges: for peer_id, local_id in ((e.b_node_id, e.a_node_id), (e.a_node_id, e.b_node_id)): @@ -401,29 +372,17 @@ def _outside_peers_for_view( if peer_id in seen: continue seen.add(peer_id) - peer_via.append((peer_id, local_id)) - if len(peer_via) >= limit: - break - if len(peer_via) >= limit: - break - if not peer_via: - return [] - peer_ids = [p for p, _ in peer_via] - peers = { - n.id: n - for n in db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(peer_ids)).all() - } - out: list[dict[str, str]] = [] - for peer_id, local_id in peer_via: - fn = peers.get(peer_id) - out.append( - { - "fabric_node_id": peer_id, - "name": (fn.name if fn else "") or "", - "ip": (fn.ip if fn else "") or "", - "via_node_id": local_id, - } - ) + fn = db.get(TopoFabricNode, peer_id) + out.append( + { + "fabric_node_id": peer_id, + "name": (fn.name if fn else "") or "", + "ip": (fn.ip if fn else "") or "", + "via_node_id": local_id, + } + ) + if len(out) >= limit: + return out return out @@ -687,7 +646,6 @@ def patch_view_positions( layout = str(body.layout or "").strip().lower() updated = 0 skipped_locked = 0 - skipped_missing = 0 matched = 0 existing = { @@ -748,54 +706,45 @@ def patch_view_positions( row.updated_at = now updated += 1 else: - from .topology_region_canvas import add_missing_region_placement, is_region_canvas_node_id - positions = list(body.positions or []) if len(positions) > VIEW_GRAPH_NODE_HARD_CAP: raise HTTPException(status_code=400, detail="too_many_positions") matched = len(positions) - # One IN query instead of per-node db.get (N+1). - check_ids = [ - str(p.fabric_node_id or "").strip() - for p in positions - if str(p.fabric_node_id or "").strip() - and not is_region_canvas_node_id(str(p.fabric_node_id or "").strip()) - ] - fabric_alive: set[str] = set() - if check_ids: - fabric_alive = { - str(x[0]) - for x in db.query(TopoFabricNode.id) - .filter(TopoFabricNode.id.in_(check_ids)) - .all() - } for p in positions: fid = str(p.fabric_node_id or "").strip() if not fid: continue - if not is_region_canvas_node_id(fid) and fid not in fabric_alive: + from .topology_region_canvas import is_region_canvas_node_id + + if not is_region_canvas_node_id(fid) and db.get(TopoFabricNode, fid) is None: raise HTTPException(status_code=400, detail=f"fabric_node_not_found:{fid}") row = existing.get(fid) if row is None: - # Child-region icons may be synthesized without a placement row; persist those. - # Fabric NEs must not bypass membership caps / freeze (MCP adds via /nodes). - row = add_missing_region_placement(db, view, fid, x=p.x, y=p.y) - if row is None: - skipped_missing += 1 - continue + row = TopoViewNode( + id=uuid4().hex, + view_id=view.id, + fabric_node_id=fid, + x=float(p.x or 0), + y=float(p.y or 0), + label=str(p.label or "")[:256], + locked=bool(p.locked), + created_at=now, + updated_at=now, + ) + db.add(row) existing[fid] = row - # Locked rows stay put unless the client explicitly unlocks (locked=False). - if row.locked and p.locked is not False: - skipped_locked += 1 - continue - row.x = float(p.x or 0) - row.y = float(p.y or 0) - if p.label is not None: - row.label = str(p.label or "")[:256] - if p.locked is not None: + updated += 1 + else: + if row.locked and bool(p.locked): + skipped_locked += 1 + continue + row.x = float(p.x or 0) + row.y = float(p.y or 0) + if p.label is not None: + row.label = str(p.label or "")[:256] row.locked = bool(p.locked) - row.updated_at = now - updated += 1 + row.updated_at = now + updated += 1 view.updated_at = now db.commit() @@ -807,7 +756,6 @@ def patch_view_positions( matched=matched, updated=updated, skipped_locked=skipped_locked, - skipped_missing=skipped_missing, ) @@ -1405,48 +1353,42 @@ _LINE_STYLES = {"", "solid", "dashed", "dotted"} def patch_view_edge_style( db: Session, view_id: str, body: ViewEdgeStylePatch ) -> TopologyViewGraphOut: - from .topology_views_commit import normalize_edge_style_items - view = _get_view_or_404(db, view_id) - items = normalize_edge_style_items(body) - if not items: - raise HTTPException(status_code=400, detail="fabric_edge_id_required") + eid = str(body.fabric_edge_id or "").strip() + if not eid or db.get(TopoFabricEdge, eid) is None: + raise HTTPException(status_code=404, detail="fabric_edge_not_found") + color = str(body.stroke_color or "").strip() + if color and not _HEX_COLOR_RE.match(color): + raise HTTPException(status_code=400, detail="invalid_stroke_color") + width = int(body.stroke_width or 0) + if width < 0 or width > 12: + raise HTTPException(status_code=400, detail="invalid_stroke_width") + style = str(body.line_style or "").strip().lower() + if style not in _LINE_STYLES: + raise HTTPException(status_code=400, detail="invalid_line_style") now = _utcnow() - for item in items: - eid = str(item.fabric_edge_id or "").strip() - if not eid or db.get(TopoFabricEdge, eid) is None: - raise HTTPException(status_code=404, detail="fabric_edge_not_found") - color = str(item.stroke_color or "").strip() - if color and not _HEX_COLOR_RE.match(color): - raise HTTPException(status_code=400, detail="invalid_stroke_color") - width = int(item.stroke_width or 0) - if width < 0 or width > 12: - raise HTTPException(status_code=400, detail="invalid_stroke_width") - style = str(item.line_style or "").strip().lower() - if style not in _LINE_STYLES: - raise HTTPException(status_code=400, detail="invalid_line_style") - row = ( - db.query(TopoViewEdgeStyle) - .filter(TopoViewEdgeStyle.view_id == view.id, TopoViewEdgeStyle.fabric_edge_id == eid) - .one_or_none() + row = ( + db.query(TopoViewEdgeStyle) + .filter(TopoViewEdgeStyle.view_id == view.id, TopoViewEdgeStyle.fabric_edge_id == eid) + .one_or_none() + ) + if row is None: + row = TopoViewEdgeStyle( + id=uuid4().hex, + view_id=view.id, + fabric_edge_id=eid, + stroke_color=color, + stroke_width=width, + line_style=style, + created_at=now, + updated_at=now, ) - if row is None: - row = TopoViewEdgeStyle( - id=uuid4().hex, - view_id=view.id, - fabric_edge_id=eid, - stroke_color=color, - stroke_width=width, - line_style=style, - created_at=now, - updated_at=now, - ) - db.add(row) - else: - row.stroke_color = color - row.stroke_width = width - row.line_style = style - row.updated_at = now + db.add(row) + else: + row.stroke_color = color + row.stroke_width = width + row.line_style = style + row.updated_at = now view.updated_at = now db.commit() return get_view_graph(db, view.id) diff --git a/netx_api/ume_alarms_router.py b/netx_api/ume_alarms_router.py index 8f94be5..472d37b 100644 --- a/netx_api/ume_alarms_router.py +++ b/netx_api/ume_alarms_router.py @@ -10,8 +10,6 @@ from sqlalchemy import func, or_ from sqlalchemy.orm import Session from .config import settings -from .table_filters import TableFilterParams, filter_query -from .table_filters import table_filter_params from .db import get_db from .key_alert_config import ( get_key_alert_monitor_config, @@ -80,7 +78,6 @@ def ume_list_alarms( time_to: str | None = Query(default=None, description="Filter by last_seen_at <="), page: int = Query(default=1, ge=1), page_size: int = Query(default=50, ge=1, le=500), - table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: stmt = db.query(UmeAlarmCurrent, UmeInventoryNE).outerjoin( @@ -116,15 +113,6 @@ def ume_list_alarms( stmt = stmt.filter(UmeAlarmCurrent.last_seen_at >= dt_from.replace(tzinfo=None)) if dt_to: stmt = stmt.filter(UmeAlarmCurrent.last_seen_at <= dt_to.replace(tzinfo=None)) - stmt, options = filter_query(stmt, { - "time": UmeAlarmCurrent.time_created, "severity": UmeAlarmCurrent.perceived_severity, - "neId": UmeAlarmCurrent.ne_id, - "hostName": func.coalesce(func.nullif(func.trim(UmeAlarmCurrent.host_name), ""), func.trim(UmeInventoryNE.host_name)), - "neType": UmeInventoryNE.ne_type, "cause": UmeAlarmCurrent.native_probable_cause, - "cleared": UmeAlarmCurrent.is_cleared, - }, table_filters) - if options is not None: - return options total = int(stmt.count()) rows = ( stmt.order_by( diff --git a/netx_api/webcrt_channel.py b/netx_api/webcrt_channel.py index 46ae67a..3194464 100644 --- a/netx_api/webcrt_channel.py +++ b/netx_api/webcrt_channel.py @@ -8,7 +8,6 @@ import queue import re import threading import time -from collections import deque from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -45,18 +44,6 @@ def uses_network_cli_keymap(device_type: str = "", vendor: str = "") -> bool: return True -_NETWORK_PROMPT_RE = re.compile( - r"^(?:<[^<>\s]{1,63}>" - r"|\[[~*]?[\w.\-/:+]{1,80}\]" - r"|[A-Za-z0-9][\w.\-/:+]{0,62}(?:\([^()\s]{1,60}\))*[#>])$" -) - - -def looks_like_network_prompt(line: str) -> bool: - """```` / ``[~r1]`` / ``R2#`` / ``R2(config)#`` — never a Linux ``user@host:~$`` prompt.""" - return bool(_NETWORK_PROMPT_RE.match(str(line or "").strip())) - - def map_network_cli_keys( data: str, *, @@ -69,8 +56,6 @@ def map_network_cli_keys( text = str(data or "") if not text: return text - if "\x1b" not in text and "\x7f" not in text: - return text # plain printable typing: nothing to rewrite out: list[str] = [] i = 0 n = len(text) @@ -101,11 +86,13 @@ def channel_return(conn: ConnectHandler | None) -> str: def map_network_cli_enter(data: str, conn: ConnectHandler | None) -> str: """Map xterm Enter (\\r) to the device's Netmiko RETURN.""" text = str(data or "") - if not text or ("\r" not in text and "\n" not in text): + if not text: return text ret = channel_return(conn) - # Replace once: chained replacements expand the CR in Telnet CRLF again. - return re.sub(r"\r\n|\r|\n", lambda _match: ret, text) + if ret == "\r": + return text + # Prefer replacing CRLF first so Telnet RETURN \\r\\n does not double-expand. + return text.replace("\r\n", ret).replace("\r", ret) def _drain_channel(conn: ConnectHandler, *, rounds: int = 6, wait: float = 0.06) -> str: @@ -382,10 +369,10 @@ def _encode_text(text: str, encoding: str) -> bytes: class _BoundedByteQueue: - """Thread-safe chunk queue that drops the oldest chunks when full (backpressure).""" + """Thread-safe queue that drops oldest chunks when full (backpressure).""" def __init__(self, maxsize: int = 2000) -> None: - self._dq: deque[bytes | None] = deque() + self._q: queue.Queue[bytes | None] = queue.Queue() self._max = max(8, int(maxsize or 2000)) self._cond = threading.Condition() self.dropped = 0 @@ -393,12 +380,13 @@ class _BoundedByteQueue: def put(self, item: bytes | None) -> None: with self._cond: - dq = self._dq - if item is not None: - while len(dq) >= self._max: - dq.popleft() + while self._q.qsize() >= self._max: + try: + self._q.get_nowait() self.dropped += 1 - dq.append(item) + except queue.Empty: + break + self._q.put(item) self._cond.notify() def put_nowait(self, item: bytes | None) -> None: @@ -406,42 +394,22 @@ class _BoundedByteQueue: def get_nowait(self) -> bytes | None: with self._cond: - if not self._dq: - raise queue.Empty - return self._dq.popleft() + return self._q.get_nowait() def get(self, timeout: float = 0.25) -> bytes | None: """Block until a chunk is available or timeout (raises queue.Empty).""" - deadline = time.monotonic() + max(0.0, float(timeout)) + deadline = time.time() + max(0.0, float(timeout)) with self._cond: - while not self._dq: - remaining = deadline - time.monotonic() + while self._q.empty(): + remaining = deadline - time.time() if remaining <= 0: raise queue.Empty self._cond.wait(timeout=remaining) - return self._dq.popleft() - - def drain(self, max_bytes: int = 262144) -> tuple[list[bytes], bool]: - """Pop every queued chunk (up to ~max_bytes). Returns ``(chunks, saw_eof)``.""" - out: list[bytes] = [] - total = 0 - with self._cond: - dq = self._dq - while dq: - item = dq[0] - if item is None: - dq.popleft() - return out, True - if out and total + len(item) > max_bytes: - break - dq.popleft() - out.append(item) - total += len(item) - return out, False + return self._q.get_nowait() def qsize(self) -> int: with self._cond: - return len(self._dq) + return self._q.qsize() def take_drop_delta(self) -> int: """Return newly dropped chunk count since last call (for client notice).""" @@ -1288,9 +1256,6 @@ def feed_command_line_buffer(buf: str, data: str, *, max_line: int = 512) -> tup completed: list[str] = [] limit = max(64, min(int(max_line or 512), 4096)) raw = str(data or "") - if raw.isprintable(): - # Plain typing (no Enter / BS / ESC): just append. - return (cur + raw)[:limit] if len(cur) < limit else cur, completed i = 0 while i < len(raw): ch = raw[i] diff --git a/netx_api/webcrt_cmd_audit.py b/netx_api/webcrt_cmd_audit.py deleted file mode 100644 index 3580ca7..0000000 --- a/netx_api/webcrt_cmd_audit.py +++ /dev/null @@ -1,525 +0,0 @@ -"""Server-side command audit for WebCRT: track the device's own input row. - -Why: inferring commands from browser keystrokes / xterm snapshots races with the -device echo (fast typing, Tab, history recall, slow VMs) and records fragments such -as ``R2#s`` or ``cddontainer/r/``. The device itself always knows what it executes: -when it echoes the line feed for our Enter, the *current input row* is exactly the -prompt + command it is about to run. - -``LineScreen`` is a tiny single-row VT100 model (CR/BS/CSI cursor + erase). -``CommandTracker`` owns one ``LineScreen`` per session and resolves each Enter we -send against the first line feed the device echoes back. Everything here is pure -(no I/O) and cheap: output before the last line feed is skipped unless an Enter -is waiting for its echo. -""" - -from __future__ import annotations - -import re -import threading -import time -from collections import deque -from dataclasses import dataclass, field - -# --------------------------------------------------------------------------- # -# Single-row terminal model -# --------------------------------------------------------------------------- # - -_TOKEN_RE = re.compile( - r"(\x1b\[[0-?]*[ -/]*[@-~]" # CSI - r"|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)" # OSC - r"|\x1b[()][0-9A-Za-z]" # charset - r"|\x1b[@-Z\\-_0-9=>]" # 2-byte ESC - r"|[\x00-\x1f\x7f])" # C0 + DEL -) -# Incomplete escape at the end of a chunk — carry into the next feed. -_PARTIAL_ESC_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*|\][^\x07\x1b]*\x1b?|[()])?\Z") -_MAX_CARRY = 96 - - -class LineScreen: - """Current input row of a terminal (no scrolling / wrapping).""" - - __slots__ = ("cells", "cur", "_carry") - - def __init__(self) -> None: - self.cells: list[str] = [] - self.cur = 0 - self._carry = "" - - def reset_row(self) -> None: - self.cells = [] - self.cur = 0 - - def text(self) -> str: - return "".join(self.cells).rstrip() - - def _put(self, run: str) -> None: - if not run: - return - n = len(run) - cells = self.cells - cur = self.cur - if cur == len(cells): - cells.extend(run) - else: - if cur > len(cells): - cells.extend(" " * (cur - len(cells))) - cells[cur : cur + n] = run - self.cur = cur + n - - def apply_token(self, tok: str) -> None: - """Apply one control token (not LF — the tracker owns line feeds).""" - c0 = tok[0] - if c0 == "\r": - self.cur = 0 - elif c0 == "\b": - if self.cur > 0: - self.cur -= 1 - elif c0 == "\t": - self._put(" " * (8 - (self.cur % 8))) - elif c0 == "\x1b": - if len(tok) >= 3 and tok[1] == "[": - self._csi(tok[2:-1], tok[-1]) - elif tok == "\x1bc": - self.reset_row() - # every other control byte (BEL, NUL, DEL, SO/SI …) is ignored - - def _csi(self, params: str, final: str) -> None: - if final not in "DCKP@XGHfJ": - return - if params and params[0] in "?>=": - return - try: - first = params.split(";", 1)[0] - n = int(first) if first else 0 - except ValueError: - n = 0 - cells = self.cells - if final == "D": - self.cur = max(0, self.cur - (n or 1)) - elif final == "C": - self.cur += n or 1 - elif final == "K": - if n == 0: - del cells[self.cur :] - elif n == 1: - for i in range(min(self.cur + 1, len(cells))): - cells[i] = " " - else: - self.reset_row() - elif final == "P": - del cells[self.cur : self.cur + (n or 1)] - elif final == "@": - if self.cur > len(cells): - cells.extend(" " * (self.cur - len(cells))) - cells[self.cur : self.cur] = [" "] * (n or 1) - elif final == "X": - for i in range(self.cur, min(self.cur + (n or 1), len(cells))): - cells[i] = " " - elif final == "G": - self.cur = max(0, (n or 1) - 1) - elif final in "Hf": - parts = params.split(";") - try: - col = int(parts[1]) if len(parts) > 1 and parts[1] else 1 - except ValueError: - col = 1 - self.cur = max(0, col - 1) - elif final == "J": - if n in (2, 3): - self.reset_row() - elif n == 0: - del cells[self.cur :] - - def feed_row(self, text: str) -> None: - """Apply text that contains no line feed.""" - if not text: - return - if not _has_ctl(text): - self._put(text) - return - parts = _TOKEN_RE.split(text) - for i, part in enumerate(parts): - if not part: - continue - if i & 1: - self.apply_token(part) - else: - self._put(part) - - def take_carry(self, text: str) -> str: - """Prepend a carried partial escape; stash a new trailing partial.""" - if self._carry: - text = self._carry + text - self._carry = "" - if text and "\x1b" in text[-_MAX_CARRY:]: - m = _PARTIAL_ESC_RE.search(text, max(0, len(text) - _MAX_CARRY)) - if m and m.group(0): - self._carry = m.group(0) - text = text[: m.start()] - return text - - -def _has_ctl(text: str) -> bool: - # str.isprintable is a fast C scan; only fall back to token split when needed. - return not text.isprintable() - - -# --------------------------------------------------------------------------- # -# Prompt splitting -# --------------------------------------------------------------------------- # - -_PROMPT_RES: tuple[re.Pattern[str], ...] = ( - # Huawei / H3C / ZTE user & system view: [r1] [~r1] [*r1-GigabitEthernet0/0/1] - re.compile(r"^<[^<>\s]{1,63}>"), - re.compile(r"^\[[~*]?[A-Za-z0-9][\w.\-/:+]{0,79}\]"), - # Cisco / Arista / ZTE / Ruijie: R2# R2> R2(config-if)# SW-1(config)# - re.compile(r"^[A-Za-z0-9][\w.\-/:+]{0,62}(?:\([^()\s]{1,60}\))*[#>]"), - # MikroTik: [admin@MikroTik] > [admin@MikroTik] /ip> - re.compile(r"^\[[\w.\-]+@[\w.\-]+\][^\n]{0,40}?>\s?"), -) -_LINUX_PROMPT_RES: tuple[re.Pattern[str], ...] = ( - # [user@host dir]$ user@host:~/dir$ (venv) user@host:~# - re.compile(r"^(?:\([^()]{1,40}\)\s*)?\[[^\]]{1,80}\]\s?[$#%>]\s?"), - re.compile(r"^(?:\([^()]{1,40}\)\s*)?[\w.\-]+@[\w.\-]+(?::|\s)[^\s$#%>]{0,120}\s?[$#%>]\s?"), - re.compile(r"^[^\s$#%>]{1,80}[$#%]\s"), -) -_NOT_PROMPT_RE = re.compile(r"^\[[YyNn]/[YyNn]\]") -_PASSWORD_ROW_RE = re.compile(r"(?:password|passwd|passphrase|密码)\s*[:：>]\s*\**\s*$", re.IGNORECASE) -_PAGER_ROW_RE = re.compile(r"-{2,}\s*more\s*-{2,}|--\s*more\s*--|\(END\)|lines?\s+\d+-\d+", re.IGNORECASE) - - -def split_prompt(row: str, hint: str = "", *, linux_like: bool = False) -> tuple[str, str] | None: - """Split ``row`` into ``(prompt, command)``; None when no CLI prompt is recognised. - - ``prompt`` keeps the whitespace before the command, so ``prompt + command`` is the - row exactly as the device displayed it. - """ - s = row.rstrip() - if not s or _NOT_PROMPT_RE.match(s) or _PAGER_ROW_RE.search(s[:80]): - return None - h = hint.rstrip() - if h and s.startswith(h): - return _cut(s, len(h)) - pats = _LINUX_PROMPT_RES + _PROMPT_RES if linux_like else _PROMPT_RES + _LINUX_PROMPT_RES - for pat in pats: - m = pat.match(s) - if m: - return _cut(s, m.end()) - return None - - -def _cut(s: str, end: int) -> tuple[str, str]: - rest = s[end:] - cmd = rest.lstrip() - return s[: len(s) - len(cmd)], cmd.rstrip() - - -def looks_like_password_row(row: str) -> bool: - return bool(_PASSWORD_ROW_RE.search(row.rstrip())) - - -# --------------------------------------------------------------------------- # -# Command tracker -# --------------------------------------------------------------------------- # - - -@dataclass -class AuditCmd: - command: str # prompt + command as the device displayed it - prompt: str = "" - input: str = "" # command only - redacted: bool = False - source: str = "stdin" - via: str = "device_echo" - - -@dataclass -class _Pending: - t: float - typed: str - snapshot: str # input row at the time we sent Enter - prompt_hint: str - client_hints: list[str] - source: str - # "bare": Enter with nothing typed in this payload (history recall / empty Enter) — - # consumes the first line feed whatever it shows. - bare: bool = False - # Tab can make the device redraw the line after a CRLF before the Enter echo; keep the - # latest prompt+command row seen and settle once the real output starts. - has_tab: bool = False - cand: tuple[str, str] | None = None - cand_t: float = 0.0 - - -_TAB_SETTLE_SEC = 1.2 - - -@dataclass -class CommandTracker: - """Per-session: device input row + FIFO of Enters waiting for their echo.""" - - linux_like: bool = False - expire_sec: float = 3.0 - max_command: int = 512 - screen: LineScreen = field(default_factory=LineScreen) - lock: threading.Lock = field(default_factory=threading.Lock) - pending: deque[_Pending] = field(default_factory=deque) - prompt_hint: str = "" - line_started: bool = False - password_mode: bool = False - # Rows abandoned with Ctrl-C: some devices (IOS, bash) echo that row + CRLF, and that - # line feed must not be mistaken for the echo of the next Enter. - abandoned: list[tuple[str, float]] = field(default_factory=list) - _typed: str = "" - - # ----- device output -------------------------------------------------- # - def feed(self, text: str) -> list[AuditCmd]: - """Feed decoded device output; returns commands resolved by line-feed echoes.""" - out: list[AuditCmd] = [] - with self.lock: - scr = self.screen - text = scr.take_carry(text) - if not text: - return out - while text: - nl = text.find("\n") - if nl < 0: - scr.feed_row(text) - break - self.line_started = False - if self.pending: - scr.feed_row(text[:nl]) - self._resolve_one(out) - scr.reset_row() - text = text[nl + 1 :] - continue - # Nothing waiting: only the last row matters — skip the bulk. - last = text.rfind("\n") - scr.reset_row() - text = text[last + 1 :] - scr.feed_row(text) - break - self.password_mode = looks_like_password_row(scr.text()) - if self.line_started and not scr.text(): - # Row wiped by the device (screen clear / new prompt): start over. - self.line_started = False - return out - - def _resolve_one(self, out: list[AuditCmd]) -> None: - row = self.screen.text() - p = self.pending[0] - if self.abandoned and not p.bare: - now = time.monotonic() - self.abandoned = [(r, t) for r, t in self.abandoned if now - t < 5.0] - typed = p.typed.replace("\t", "").strip() - for i, (ab, _t) in enumerate(self.abandoned): - # The device reprints the aborted row; ``startswith`` covers a slow echo - # (row snapshot was taken before every typed char came back). A VRP that - # keeps the text yields ``garbage`` + our command, which contains ``typed``. - if row == ab or (row.startswith(ab) and typed and typed not in row): - del self.abandoned[i] - return # echo of the aborted line, not our Enter - parsed = split_prompt(row, p.prompt_hint, linux_like=self.linux_like) - has_cmd = bool(parsed and parsed[1]) - if p.bare: - # A bare Enter (empty line, or executing a recalled history line) is echoed by - # the first *prompt* row's line feed; output rows of a command that is still - # printing must not consume it (Enter typed ahead during output). - if parsed is None: - return - self.pending.popleft() - if has_cmd: - out.append(self._make(parsed[0], parsed[1], p.source, "device_echo")) # type: ignore[index] - # Enter pressed at a pager left an orphan bare Enter in front of the next - # typed line: that line feed belongs to both — do not record it twice. - nxt = self.pending[0] if self.pending else None - if nxt is not None and not nxt.bare: - typed = nxt.typed.replace("\t", "").replace(" ", "").lower() - if typed and parsed[1].replace(" ", "").lower().startswith(typed[:3]): - self.pending.popleft() - return - if p.has_tab: - if has_cmd: - p.cand = (parsed[0], parsed[1]) # type: ignore[index] - p.cand_t = time.monotonic() - elif p.cand is not None: - # Prompt / output row after the echo: the last candidate is the executed line. - self._finish_candidate(p, out) - return - if not has_cmd: - # Prompt-only / non-CLI row: earlier output, a prompt reprint — not our echo. - return - self.pending.popleft() - out.append(self._make(parsed[0], parsed[1], p.source, "device_echo")) # type: ignore[index] - - def _finish_candidate(self, p: _Pending, out: list[AuditCmd]) -> None: - if self.pending and self.pending[0] is p: - self.pending.popleft() - if p.cand is not None: - out.append(self._make(p.cand[0], p.cand[1], p.source, "device_echo")) - p.cand = None - - def _make(self, prompt: str, cmd: str, source: str, via: str) -> AuditCmd: - cmd = cmd[: self.max_command] - base = prompt.rstrip() - # Keep one space after user@host style prompts; network CLIs print none. - sep = " " if (len(prompt) > len(base) and (self.linux_like or "@" in base)) else "" - return AuditCmd( - command=(base + sep + cmd)[: self.max_command], - prompt=base, - input=cmd, - source=source, - via=via, - ) - - def expire(self, now: float | None = None) -> list[AuditCmd]: - """Settle Tab candidates and resolve Enters whose echo never arrived.""" - out: list[AuditCmd] = [] - if not self.pending: - return out - t = time.monotonic() if now is None else now - with self.lock: - while self.pending: - p = self.pending[0] - if p.cand is not None and (t - p.cand_t) >= _TAB_SETTLE_SEC: - self._finish_candidate(p, out) - continue - if (t - p.t) >= self.expire_sec: - self.pending.popleft() - cmd = self._fallback(p) - if cmd is not None: - out.append(cmd) - continue - break - return out - - def _fallback(self, p: _Pending) -> AuditCmd | None: - parsed = split_prompt(p.snapshot, p.prompt_hint, linux_like=self.linux_like) - if parsed and parsed[1]: - return self._make(parsed[0], parsed[1], p.source, "snapshot") - for hint in reversed(p.client_hints): - parsed = split_prompt(hint, p.prompt_hint, linux_like=self.linux_like) - if parsed and parsed[1]: - return self._make(parsed[0], parsed[1], p.source, "client_hint") - # Last resort: what the operator typed, under the prompt that was on screen. - typed = p.typed.replace("\t", "").strip() - if typed and p.prompt_hint: - return self._make(p.prompt_hint, typed, p.source, "typed") - return None - - def flush_all(self) -> list[AuditCmd]: - """Session closing: resolve every still-pending Enter with the best fallback.""" - out: list[AuditCmd] = [] - with self.lock: - while self.pending: - p = self.pending.popleft() - if p.cand is not None: - out.append(self._make(p.cand[0], p.cand[1], p.source, "device_echo")) - continue - cmd = self._fallback(p) - if cmd is not None: - out.append(cmd) - return out - - # ----- operator input -------------------------------------------------- # - def on_stdin( - self, - lines: list[str], - *, - saw_input: bool, - had_enter: bool, - client_hints: list[str] | None = None, - source: str = "stdin", - now: float | None = None, - abort: bool = False, - abort_text: str = "", - ) -> list[AuditCmd]: - """Register keystrokes already split into completed ``lines`` by the caller. - - ``saw_input``: payload had anything besides bare Enter (captures the prompt). - ``had_enter``: payload contained at least one line terminator. - Returns immediately-final records (password redaction); echo-resolved ones - come from ``feed``. - """ - out: list[AuditCmd] = [] - t = time.monotonic() if now is None else now - with self.lock: - row = self.screen.text() - if abort: - parsed_row = split_prompt(row, self.prompt_hint, linux_like=self.linux_like) - if parsed_row and parsed_row[1]: - self.abandoned.append((row, t)) - # What we sent before the Ctrl-C may not have been echoed yet. - if abort_text.strip() and self.prompt_hint: - self.abandoned.append((self.prompt_hint + abort_text.strip(), t)) - del self.abandoned[:-6] - if saw_input and not self.line_started: - # First keystroke of a line: the row is still just the prompt. Prefer the - # regex-detected prompt so typed echo never leaks into the hint. - if row and not looks_like_password_row(row): - known = split_prompt(row, "", linux_like=self.linux_like) - if known: - self.prompt_hint = known[0] - elif row[-1] in "#>$%]": - self.prompt_hint = row # exotic CLI prompt - else: - # "Username:" / "Continue? [y/n]:" … a dialog, not a CLI prompt. - self.prompt_hint = "" - self.line_started = True - if not had_enter: - return out - hints = [h for h in (client_hints or []) if h] - if self.password_mode or looks_like_password_row(row): - # Typed secret: never keep it, record a redacted marker. - n = max(1, len(lines) or 1) - out.extend(AuditCmd(command="***", redacted=True, source=source, via="password") for _ in range(n)) - self.password_mode = False - self.line_started = False - return out - # No completed text: a bare Enter, possibly executing a recalled history line - # whose redraw has not reached us yet. - queue_lines = list(lines) or [""] - for i, typed in enumerate(queue_lines): - self.pending.append( - _Pending( - t=t, - typed=typed, - snapshot=row if i == 0 else "", - prompt_hint=self.prompt_hint, - client_hints=hints[-1:] if i == len(queue_lines) - 1 else [], - source=source, - bare=not typed, - has_tab="\t" in typed, - ) - ) - self.line_started = False - # Bound runaway growth (device never echoing). - while len(self.pending) > 64: - self.pending.popleft() - return out - - def immediate(self, lines: list[str], source: str) -> list[AuditCmd]: - """Trusted non-interactive input (post-login commands): record as typed.""" - return [ - AuditCmd(command=ln[: self.max_command], input=ln[: self.max_command], source=source, via="trusted") - for ln in lines - if ln.strip() - ] - - def current_row(self) -> str: - with self.lock: - return self.screen.text() - - def seed(self, text: str) -> None: - """Prime the row from the login transcript (prompt printed before the reader).""" - self.feed(text) - - def set_row(self, text: str) -> None: - """Force the current input row (e.g. after a duplicate prompt was dropped).""" - with self.lock: - self.screen.reset_row() - self.screen._carry = "" - self.screen.feed_row(text) - self.line_started = False diff --git a/netx_api/webcrt_router.py b/netx_api/webcrt_router.py index 54e6dd4..74d02c8 100644 --- a/netx_api/webcrt_router.py +++ b/netx_api/webcrt_router.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import codecs import json import logging import time @@ -209,8 +208,9 @@ def api_quick_connect( user_override = str(body.username or "").strip() or None if str(body.password or "").strip(): pwd_override = str(body.password) - # Open the terminal immediately; WS reports auth progress and retryable failures. - async_connect = bool(body.async_connect) + # SSH with password: wait for auth so wrong credentials can re-prompt (SecureCRT-like). + wait_for_auth = proto == "ssh" and bool(pwd_override) + async_connect = bool(body.async_connect) and not wait_for_auth try: session = create_session( db, @@ -687,7 +687,6 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: replay_text = _decode_bytes(raw_boot, "utf-8") sess.bootstrap_replayed = True else: - sess.flush_session_log() replay_text = read_session_log_tail(session_id, max_bytes=49152) if replay_text: replay_bytes = _encode_text(replay_text, "utf-8") @@ -701,72 +700,59 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: _log.debug("webcrt bootstrap/replay send failed session=%s", session_id, exc_info=True) stop = asyncio.Event() - loop = asyncio.get_running_loop() - io_pool = webcrt_io_executor() - wake = asyncio.Event() - # Keystrokes are written as they arrive (no timer delay); whatever piled up while a - # write was in flight is coalesced into the next one. - stdin_q: asyncio.Queue[tuple[str, str | None]] = asyncio.Queue() + stdin_buf: list[str] = [] + stdin_audit_lines: list[str] = [] + stdin_flush_task: asyncio.Task[None] | None = None - async def stdin_writer() -> None: - while True: - data, audit = await stdin_q.get() - parts = [data] - audits = [audit] if audit else [] - taken = 1 - while True: - try: - d2, a2 = stdin_q.get_nowait() - except asyncio.QueueEmpty: - break - parts.append(d2) - taken += 1 - if a2: - audits.append(a2) - blob = "".join(parts) - try: - await loop.run_in_executor( - io_pool, - lambda b=blob, al=audits: sess.write_stdin( - b, - audit_lines=al or None, - audit_line=al[-1] if al else None, - ), - ) - except Exception as exc: - try: - await websocket.send_json( - {"type": "status", "state": "error", "message": f"write_failed:{exc}"} - ) - except Exception: - pass - stop.set() - wake.set() - try: - await websocket.close(code=4500) - except Exception: - pass - finally: - for _ in range(taken): - stdin_q.task_done() - - async def drain_stdin() -> None: + async def flush_stdin() -> None: + nonlocal stdin_buf, stdin_audit_lines + if not stdin_buf: + return + data = "".join(stdin_buf) + audit_lines = list(stdin_audit_lines) + stdin_buf = [] + stdin_audit_lines = [] try: - await asyncio.wait_for(stdin_q.join(), timeout=5.0) - except Exception: - pass + await asyncio.get_running_loop().run_in_executor( + webcrt_io_executor(), + lambda: sess.write_stdin( + data, + audit_lines=audit_lines or None, + audit_line=audit_lines[-1] if audit_lines else None, + ), + ) + except Exception as exc: + await websocket.send_json( + {"type": "status", "state": "error", "message": f"write_failed:{exc}"} + ) + stop.set() + + async def schedule_stdin_flush() -> None: + await asyncio.sleep(0.008) + await flush_stdin() async def pump_stdout() -> None: - """Event-driven stdout pump: the reader thread wakes us, we drain and send at once.""" - sess.set_notifier(loop, wake) - dec = None - if _normalize_encoding(sess.encoding) != "utf-8": - dec = codecs.getincrementaldecoder("gbk")(errors="replace") + loop = asyncio.get_running_loop() + pending: list[bytes] = [] + last_flush = loop.time() - def _to_browser(raw: bytes) -> bytes: - if dec is None: + def _to_browser_bytes(raw: bytes) -> bytes: + if _normalize_encoding(sess.encoding) == "utf-8": return raw - return dec.decode(raw).encode("utf-8", errors="replace") + return _decode_bytes(raw, sess.encoding).encode("utf-8", errors="replace") + + async def _flush_pending() -> bool: + nonlocal pending, last_flush + if not pending: + return True + blob = _to_browser_bytes(b"".join(pending)) + pending = [] + last_flush = loop.time() + try: + await websocket.send_bytes(blob) + return True + except Exception: + return False async def _notify_queue_drops() -> None: try: @@ -787,48 +773,48 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: except Exception: pass - wake.set() # output queued before the pump started - try: - while not stop.is_set(): - await wake.wait() - wake.clear() - while not stop.is_set(): - status, data = sess.drain_stdout(attach_gen) - if status == "stale": - return - if data: - try: - await websocket.send_bytes(_to_browser(data)) - except Exception: - stop.set() - return - await _notify_queue_drops() - if status == "eof": - await _notify_queue_drops() + while not stop.is_set(): + # Longer block is cheap now (Condition wait); cuts executor churn when idle. + chunk = await loop.run_in_executor( + webcrt_io_executor(), lambda: sess.take_stdout(attach_gen, timeout=0.2) + ) + if chunk == "stale": + break + if chunk == "empty": + if pending and (loop.time() - last_flush) >= 0.016: + if not await _flush_pending(): stop.set() - try: - await websocket.send_json( - { - "type": "status", - "state": "closed", - "message": sess.close_reason or "device_closed", - } - ) - except Exception: - pass - return - if status != "data": break - finally: - sess.clear_notifier(wake) + await _notify_queue_drops() + continue + if chunk is None: + await _flush_pending() + await _notify_queue_drops() + stop.set() + try: + await websocket.send_json( + { + "type": "status", + "state": "closed", + "message": sess.close_reason or "device_closed", + } + ) + except Exception: + pass + break + pending.append(chunk) + if sum(len(p) for p in pending) >= 8192 or (loop.time() - last_flush) >= 0.016: + if not await _flush_pending(): + stop.set() + break + await _notify_queue_drops() reader_task = asyncio.create_task(pump_stdout()) - writer_task = asyncio.create_task(stdin_writer()) if sess.needs_live_prompt: sess.needs_live_prompt = False try: - await loop.run_in_executor( - io_pool, + await asyncio.get_running_loop().run_in_executor( + webcrt_io_executor(), lambda: sess.write_stdin("\r", audit_source="prompt_sync"), ) except Exception: @@ -844,7 +830,9 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: text = _decode_bytes(bytes(msg_raw["bytes"]), sess.encoding) except Exception: continue - stdin_q.put_nowait((text, None)) + stdin_buf.append(text) + if stdin_flush_task is None or stdin_flush_task.done(): + stdin_flush_task = asyncio.create_task(schedule_stdin_flush()) continue raw = msg_raw.get("text") if raw is None: @@ -859,16 +847,23 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: if data is None: continue audit_raw = msg.get("audit_line") - audit = str(audit_raw).strip()[:512] if audit_raw is not None and str(audit_raw).strip() else None - stdin_q.put_nowait((str(data), audit)) + if audit_raw is not None and str(audit_raw).strip(): + stdin_audit_lines.append(str(audit_raw).strip()[:512]) + stdin_buf.append(str(data)) + # Coalesce high-frequency keystrokes briefly. + if len(stdin_buf) >= 8: + if stdin_flush_task and not stdin_flush_task.done(): + stdin_flush_task.cancel() + await flush_stdin() + elif stdin_flush_task is None or stdin_flush_task.done(): + stdin_flush_task = asyncio.create_task(schedule_stdin_flush()) elif mtype == "resize": cols = int(msg.get("cols") or sess.cols) rows = int(msg.get("rows") or sess.rows) - await loop.run_in_executor(io_pool, sess.resize, cols, rows) + await asyncio.get_running_loop().run_in_executor(webcrt_io_executor(), sess.resize, cols, rows) elif mtype == "break": - await drain_stdin() try: - await loop.run_in_executor(io_pool, sess.send_break) + await asyncio.get_running_loop().run_in_executor(webcrt_io_executor(), sess.send_break) except Exception as exc: await websocket.send_json( {"type": "status", "state": "error", "message": f"break_failed:{exc}"} @@ -878,7 +873,7 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: await websocket.send_json({"type": "pong"}) elif mtype == "close": stop.set() - await drain_stdin() + await flush_stdin() close_session( session_id, reason="client_close", @@ -891,18 +886,17 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: _log.exception("webcrt ws error session=%s", session_id) finally: stop.set() - wake.set() + if stdin_flush_task and not stdin_flush_task.done(): + stdin_flush_task.cancel() try: - await drain_stdin() + await flush_stdin() except Exception: pass - writer_task.cancel() reader_task.cancel() - for t in (writer_task, reader_task): - try: - await t - except BaseException: - pass + try: + await reader_task + except Exception: + pass # Keep device session so UI reconnect / remount can re-attach. if get_session(session_id) is not None: grace = float(getattr(settings, "webcrt_detach_grace_sec", 120) or 120) diff --git a/netx_api/webcrt_session_model.py b/netx_api/webcrt_session_model.py index 17b9c37..0879395 100644 --- a/netx_api/webcrt_session_model.py +++ b/netx_api/webcrt_session_model.py @@ -1,12 +1,8 @@ """WebCRT interactive session object (I/O, SFTP, close).""" from __future__ import annotations -import codecs import logging import queue -import re -import select -import socket import threading import time from dataclasses import dataclass, field @@ -20,10 +16,8 @@ from .ne_session_factory import ( extract_cli_prompt_marker, should_close_cli_hop_session, ) -from .webcrt_cmd_audit import AuditCmd, CommandTracker from .webcrt_channel import ( _BoundedByteQueue, - _normalize_encoding, _audit, _decode_bytes, _encode_text, @@ -37,9 +31,6 @@ from .webcrt_channel import ( _log = logging.getLogger("netx.webcrt") -# Operator typed an exit-style command (arms the hop prompt-return heuristic). -_HOP_EXIT_RE = re.compile(r"(?:^|[\r\n])\s*(?:quit|exit|logout|logoff|q|qu|qui|ex|exi)\s*[\r\n]", re.IGNORECASE) - @dataclass class WebcrtSession: session_id: str @@ -107,17 +98,6 @@ class WebcrtSession: _last_prompt_line: str = field(default="", repr=False) _last_audited_command: str = field(default="", repr=False) _last_audited_at: float = field(default=0.0, repr=False) - # Live audit: device-echo based command tracker (active while the reader runs). - _tracker: CommandTracker = field(default_factory=CommandTracker, repr=False) - # asyncio wake-up for the WS stdout pump: (loop, event) + "already scheduled" flag. - _notify: Any = field(default=None, repr=False) - _wake_flag: bool = field(default=False, repr=False) - _log_last_flush: float = field(default=0.0, repr=False) - _log_lock: threading.Lock = field(default_factory=threading.Lock, repr=False) - _hop_exit_until: float = field(default=0.0, repr=False) - - def __post_init__(self) -> None: - self._tracker.linux_like = not bool(self.cli_keymap) def touch(self) -> None: self.last_activity = time.time() @@ -276,55 +256,39 @@ class WebcrtSession: self._log_fh = None def append_session_log(self, text: str) -> None: - """Buffered transcript write; flushed by the reader every ~250ms (and on attach).""" - fh = self._log_fh - if not text or fh is None: + if not text or self._log_fh is None: return try: - with self._log_lock: - fh.write(text) - if time.monotonic() - self._log_last_flush >= 0.25: - self.flush_session_log() + self._log_fh.write(text) + self._log_fh.flush() + max_bytes = int(getattr(settings, "webcrt_session_log_max_bytes", 0) or 0) + if max_bytes > 0: + try: + pos = int(self._log_fh.tell()) + except Exception: # noqa: BLE001 + pos = 0 + if pos >= max_bytes: + self._log_fh.write( + f"\n# rotated at {pos} bytes (cap={max_bytes}) ts={_utc_iso()}\n" + ) + self._log_fh.flush() + self._log_fh.close() + self._log_fh = None + # Re-open fresh file (append mode continues same path; rotate by rename). + try: + path = _session_log_path(self.session_id) + rotated = path.with_suffix(path.suffix + f".{int(pos)}.old") + if path.exists(): + path.replace(rotated) + except Exception: # noqa: BLE001 + _log.debug("webcrt session log rotate rename failed", exc_info=True) + self.open_session_log() except Exception: _log.warning("webcrt session log append failed session=%s", self.session_id, exc_info=True) - def flush_session_log(self) -> None: - fh = self._log_fh - if fh is None: - return - try: - with self._log_lock: - fh.flush() - self._log_last_flush = time.monotonic() - max_bytes = int(getattr(settings, "webcrt_session_log_max_bytes", 0) or 0) - if max_bytes <= 0: - return - try: - pos = int(fh.tell()) - except Exception: # noqa: BLE001 - pos = 0 - if pos < max_bytes: - return - fh.write(f"\n# rotated at {pos} bytes (cap={max_bytes}) ts={_utc_iso()}\n") - fh.flush() - fh.close() - self._log_fh = None - # Re-open fresh file (rotate by rename). - try: - path = _session_log_path(self.session_id) - rotated = path.with_suffix(path.suffix + f".{int(pos)}.old") - if path.exists(): - path.replace(rotated) - except Exception: # noqa: BLE001 - _log.debug("webcrt session log rotate rename failed", exc_info=True) - self.open_session_log() - except Exception: - _log.warning("webcrt session log flush failed session=%s", self.session_id, exc_info=True) - def close_session_log(self) -> None: - with self._log_lock: - fh = self._log_fh - self._log_fh = None + fh = self._log_fh + self._log_fh = None if fh is None: return try: @@ -362,57 +326,6 @@ class WebcrtSession: return "stale" return chunk # bytes | None - # ---- asyncio hand-off (no polling thread per WebSocket) -------------------- # - def set_notifier(self, loop: Any, event: Any) -> None: - with self._stdout_lock: - self._notify = (loop, event) - self._wake_flag = False - - def clear_notifier(self, event: Any) -> None: - with self._stdout_lock: - n = self._notify - if n is not None and n[1] is event: - self._notify = None - - def bump_attach(self) -> int: - """New WebSocket owns stdout: invalidate the previous pump and wake it.""" - with self._stdout_lock: - self.attach_gen += 1 - gen = self.attach_gen - self._wake(force=True) - return gen - - def _wake(self, *, force: bool = False) -> None: - """Called from any thread after queue changes; schedules at most one loop wake-up.""" - n = self._notify - if n is None: - return - if self._wake_flag and not force: - return - self._wake_flag = True - try: - n[0].call_soon_threadsafe(n[1].set) - except Exception: - self._wake_flag = False - self._notify = None - - def drain_stdout(self, attach_gen: int, max_bytes: int = 262144) -> tuple[str, bytes]: - """Non-blocking take for the WS pump. - - Returns ``(status, data)``: ``stale`` (newer WS owns stdout), ``eof`` (reader - ended; ``data`` may still hold the final bytes), ``data`` or ``empty``. - """ - self._wake_flag = False - with self._stdout_lock: - if attach_gen != self.attach_gen: - return "stale", b"" - chunks, eof = self.out_queue.drain(max_bytes) - data = b"".join(chunks) - if eof: - return "eof", data - return ("data" if data else "empty"), data - - # ---- stdin ----------------------------------------------------------------- # def write_stdin( self, data: str, @@ -438,51 +351,24 @@ class WebcrtSession: text = map_network_cli_enter(text, self.conn) if not text: return - if self.cli_hop_guard and (_HOP_EXIT_RE.search(text) or "\x04" in text): - self._hop_exit_until = time.monotonic() + 8.0 - live_cmds: list[AuditCmd] = [] - if self._audit_live(): - # Register Enter first (cheap), send to the device, then publish: the audit - # row is resolved from the device echo, so nothing here delays the keystroke. - hints = list(audit_list or []) - if audit_override and (not hints or hints[-1] != audit_override): - hints.append(audit_override) - live_cmds = self._audit_stdin_live(text, source=str(audit_source or "stdin"), hints=hints) - else: - self._note_stdin_for_audit( - text, - source=audit_source, - audit_line=audit_override, - audit_lines=audit_list, - ) - self._send_raw(text) - if live_cmds: - self._emit_audit_cmds(live_cmds) - self.touch() - - def _send_raw(self, text: str) -> None: + self._note_stdin_for_audit( + text, + source=audit_source, + audit_line=audit_override, + audit_lines=audit_list, + ) with self._write_lock: # Prefer raw channel I/O for interactive typing (char echo / backspace). - conn = self.conn - if conn is None: - raise RuntimeError("session_closed") - channel = getattr(conn, "remote_conn", None) + channel = getattr(self.conn, "remote_conn", None) try: if channel is not None and hasattr(channel, "send") and callable(channel.send): payload = _encode_text(text, self.encoding) - # Paramiko may write partially when the window is full; the reader - # keeps a short channel timeout, so a timed-out send is simply retried. + # Paramiko may write partially when the window is full. view = memoryview(payload) - deadline = time.monotonic() + 30.0 while len(view): - try: - n = int(channel.send(view) or 0) - except socket.timeout: - if time.monotonic() > deadline or self.closed: - raise - continue + n = int(channel.send(view) or 0) if n <= 0: - time.sleep(0.005) + time.sleep(0.01) continue view = view[n:] self.bytes_in += len(payload) @@ -491,79 +377,12 @@ class WebcrtSession: channel.write(payload) self.bytes_in += len(payload) else: - conn.write_channel(text) + self.conn.write_channel(text) self.bytes_in += len(text) except Exception: - if self.closed or self.conn is None: - raise self.conn.write_channel(text) self.bytes_in += len(text) - - # ---- command audit ------------------------------------------------------- # - def _audit_live(self) -> bool: - """Device-echo audit is active only while the reader thread feeds the tracker.""" - t = self._reader - return t is not None and t.is_alive() - - def _audit_stdin_live(self, text: str, *, source: str, hints: list[str]) -> list[AuditCmd]: - if source == "prompt_sync": - return [] - had_enter = "\r" in text or "\n" in text - abort_text = "" - with self._cmd_buf_lock: - cut = text.find("\x03") - if cut >= 0: - abort_text, _ = feed_command_line_buffer(self._cmd_buf, text[:cut]) - self._cmd_buf, buf_lines = feed_command_line_buffer(self._cmd_buf, text) - if source == "post_login": - return self._tracker.immediate(buf_lines, source) if had_enter else [] - saw_input = bool(text.strip("\r\n")) - return self._tracker.on_stdin( - buf_lines, - saw_input=saw_input, - had_enter=had_enter, - client_hints=hints, - source=source, - abort="\x03" in text, - abort_text=abort_text, - ) - - def _emit_audit_cmds(self, cmds: list[AuditCmd]) -> None: - for c in cmds: - if not str(c.command).strip(): - continue - try: - _audit( - "command", - session_id=self.session_id, - ne_id=self.ne_id, - ne_name=self.ne_name, - ne_ip=self.ne_ip, - protocol=self.protocol, - owner_user_id=self.owner_user_id, - owner_username=self.owner_username, - command="***" if c.redacted else str(c.command)[:512], - redacted=bool(c.redacted), - source=str(c.source or "stdin")[:32], - via=c.via, - prompt=str(c.prompt or "")[:128], - ) - except Exception: - _log.debug("webcrt command audit failed session=%s", self.session_id, exc_info=True) - - def seed_audit_row(self, transcript: str) -> None: - """Prime the input row from the login transcript (prompt printed before the reader).""" - try: - self._tracker.set_row("") - self._tracker.seed(str(transcript or "")[-4000:]) - except Exception: - _log.debug("webcrt audit seed failed session=%s", self.session_id, exc_info=True) - - def reset_audit_row(self, row: str) -> None: - try: - self._tracker.set_row(row) - except Exception: - pass + self.touch() def _note_stdin_for_audit( self, @@ -642,26 +461,6 @@ class WebcrtSession: _log.debug("webcrt command audit failed session=%s", self.session_id, exc_info=True) def _note_stdout_for_audit(self, text: str) -> None: - """Feed device output to the audit tracker (live) or the legacy tail buffer.""" - chunk = str(text or "") - if not chunk: - return - if self._audit_live(): - cmds = self._tracker.feed(chunk) - self._password_mode = self._tracker.password_mode - if cmds: - self._emit_audit_cmds(cmds) - return - self._note_stdout_legacy(chunk) - - def _audit_housekeeping(self) -> None: - if self._tracker.pending: - cmds = self._tracker.expire() - if cmds: - self._emit_audit_cmds(cmds) - self.flush_session_log() - - def _note_stdout_legacy(self, text: str) -> None: """Track device prompts so the next typed line can be redacted if it is a password.""" chunk = str(text or "") if not chunk: @@ -735,74 +534,50 @@ class WebcrtSession: ) self._reader.start() - @staticmethod - def _wait_readable(channel: Any, timeout: float) -> None: - """Block until a telnet-style socket is readable (or timeout).""" - sock = getattr(channel, "sock", None) - try: - if sock is not None and sock.fileno() >= 0: - select.select([sock], [], [], timeout) - return - except Exception: - pass - time.sleep(min(timeout, 0.01)) - def _reader_loop(self) -> None: conn = self.conn if conn is None: self.out_queue.put(None) - self._wake() return channel = getattr(conn, "remote_conn", None) hop_return = False poll = max(0.002, float(getattr(settings, "webcrt_reader_poll_sec", 0.01) or 0.01)) - # Incremental decode: a multi-byte char split across recv() chunks stays intact. - dec = codecs.getincrementaldecoder("gbk" if _normalize_encoding(self.encoding) == "gbk" else "utf-8")( - errors="replace" - ) - # Paramiko: park in recv() with a short timeout (wakes the moment data arrives) - # instead of spin-polling recv_ready(). - blocking = False - if ( - channel is not None - and hasattr(channel, "recv") - and hasattr(channel, "recv_ready") - and hasattr(channel, "settimeout") - ): - try: - channel.settimeout(0.2) - blocking = True - except Exception: - blocking = False - last_house = time.monotonic() try: while not self.closed: chunk = b"" try: - if blocking: + if channel is not None and hasattr(channel, "recv_ready") and hasattr(channel, "recv"): + # Paramiko SSH: prefer short blocking recv over fixed spin-sleep. + ready = False try: - chunk = channel.recv(65536) - except socket.timeout: - chunk = b"" - else: - if not chunk: - break # EOF - if not chunk: - if channel.recv_ready(): - continue - if getattr(channel, "closed", False) or ( - hasattr(channel, "exit_status_ready") and channel.exit_status_ready() - ): - break - elif channel is not None and hasattr(channel, "recv_ready") and hasattr(channel, "recv"): - if channel.recv_ready(): - chunk = channel.recv(65536) + ready = bool(channel.recv_ready()) + except Exception: + ready = False + if ready: + chunk = channel.recv(16384) if not chunk: break elif hasattr(channel, "exit_status_ready") and channel.exit_status_ready(): break else: - time.sleep(poll) + # Brief block: settimeout + recv wakes sooner than sleep(0.04). + prev_timeout = None + try: + prev_timeout = channel.gettimeout() + except Exception: + prev_timeout = None + try: + channel.settimeout(poll) + chunk = channel.recv(16384) + except Exception: + chunk = b"" + finally: + try: + channel.settimeout(prev_timeout) + except Exception: + pass + if not chunk: + continue elif channel is not None and hasattr(channel, "read_very_eager"): # Telnet: do NOT use conn.read_channel() — Netmiko strips ANSI. data = channel.read_very_eager() @@ -813,52 +588,39 @@ class WebcrtSession: else _encode_text(str(data), self.encoding) ) else: - self._wait_readable(channel, 0.2) + time.sleep(poll) + continue else: text = conn.read_channel() if text: chunk = _encode_text(str(text), self.encoding) else: time.sleep(poll) - except EOFError: - break + continue except Exception as exc: if self.closed: break _log.debug("webcrt reader error session=%s: %s", self.session_id, exc) - if channel is not None and getattr(channel, "closed", False): - break time.sleep(0.05) continue - now = time.monotonic() if chunk: - self.last_activity = time.time() + self.touch() self.bytes_out += len(chunk) self.out_queue.put(chunk) - self._wake() - decoded = "" try: - decoded = dec.decode(bytes(chunk)) - if decoded: - self.append_session_log(decoded) - self._note_stdout_for_audit(decoded) + decoded = _decode_bytes(chunk, self.encoding) + self.append_session_log(decoded) + self._note_stdout_for_audit(decoded) except Exception: - _log.debug("webcrt reader side-effects failed session=%s", self.session_id, exc_info=True) - if self.cli_hop_guard and self._note_cli_hop_output(chunk, decoded): + pass + if self.cli_hop_guard and self._note_cli_hop_output(chunk): hop_return = True notice = ( "\r\n*** WebCRT: 目标会话已结束，已断开代理连接 " "(target session ended; closing hop proxy) ***\r\n" ) self.out_queue.put(notice.encode("utf-8", errors="replace")) - self._wake() break - if now - last_house >= 0.25: - last_house = now - try: - self._audit_housekeeping() - except Exception: - _log.debug("webcrt housekeeping failed session=%s", self.session_id, exc_info=True) finally: if hop_return and not self.closed: try: @@ -868,33 +630,22 @@ class WebcrtSession: except Exception: self.close("cli_hop_return") self.out_queue.put(None) - self._wake() - try: - self.flush_session_log() - except Exception: - pass - def _note_cli_hop_output(self, chunk: bytes, text: str | None = None) -> bool: + def _note_cli_hop_output(self, chunk: bytes) -> bool: """Accumulate stdout and return True when nested CLI hop has returned to proxy.""" - if text is None: - try: - text = _decode_bytes(chunk, self.encoding) - except Exception: - text = str(chunk) + try: + text = _decode_bytes(chunk, self.encoding) + except Exception: + text = str(chunk) self._hop_scan_buf = (self._hop_scan_buf + text)[-12000:] marker = str(self.cli_hop_prompt or "").strip() last = extract_cli_prompt_marker(self._hop_scan_buf) if last and (not marker or last != marker): self._cli_hop_seen_other_prompt = True - # Prompt-only "back on the hop" is ambiguous when hop and target share a hostname - # (target ``system-view`` prints the same ``[~r1]`` as the hop). Only trust it - # shortly after the operator typed an exit command; the nested "connection closed" - # message needs no such gate. - armed = self._cli_hop_seen_other_prompt and time.monotonic() <= self._hop_exit_until return should_close_cli_hop_session( self._hop_scan_buf, self.cli_hop_prompt, - seen_other_prompt=armed, + seen_other_prompt=self._cli_hop_seen_other_prompt, ) def run_post_login_commands(self) -> None: @@ -909,19 +660,6 @@ class WebcrtSession: _log.debug("post_login command failed session=%s", self.session_id, exc_info=True) break - def _teardown_conn(self, conn: Any) -> None: - self.close_sftp() - if conn is None: - return - for attr in ("remote_conn", "remote_conn_pre", "_netx_jump_client"): - obj = getattr(conn, attr, None) - if obj is None: - continue - try: - obj.close() - except Exception: - pass - def close(self, reason: str = "closed") -> None: if self.closed: return @@ -929,27 +667,14 @@ class WebcrtSession: self.state = "closed" self.close_reason = reason or "closed" self._ready_event.set() - # Enters still waiting for a device echo would be lost with the session. + self.close_sftp() try: - pending_cmds = self._tracker.flush_all() - if pending_cmds: - self._emit_audit_cmds(pending_cmds) + close_netmiko_connection(self.conn) except Exception: - _log.debug("webcrt audit flush on close failed session=%s", self.session_id, exc_info=True) - conn = self.conn + pass self.conn = None - # Tear the transport down off-thread: Netmiko's disconnect() runs vendor cleanup - # (prompt probing, several seconds on Huawei) which would stall the event loop / - # reaper, and keeps the device VTY occupied longer than needed. - threading.Thread( - target=self._teardown_conn, - args=(conn,), - name=f"webcrt-close-{self.session_id[:8]}", - daemon=True, - ).start() try: self.out_queue.put_nowait(None) except Exception: pass - self._wake(force=True) self.close_session_log() diff --git a/netx_api/webcrt_session_registry.py b/netx_api/webcrt_session_registry.py index 9794649..cc38cb0 100644 --- a/netx_api/webcrt_session_registry.py +++ b/netx_api/webcrt_session_registry.py @@ -34,7 +34,6 @@ from .webcrt_channel import ( _normalize_encoding, _prime_interactive_channel, _session_log_text, - looks_like_network_prompt, prepare_bootstrap_output, uses_network_cli_keymap, ) @@ -285,16 +284,14 @@ def _finish_connect( # Pull post-auth banner/MOTD from the PTY. With interactive no-op session_preparation # (generic_termserver), Netmiko session_log is often empty — do not discard these bytes. # Keep this short: live echo already streamed login; long settle blocks WS stdin. - # Login already ended on a CLI prompt: only a short look for trailing MOTD bytes. - prompted_early = _looks_like_cli_prompt(pre_log) try: - early = _capture_raw_channel(conn, duration=0.06 if prompted_early else 0.2) + early = _capture_raw_channel(conn, duration=0.2) except Exception: early = "" if early: sess.push_connect_echo(early) seed = f"{pre_log}{early}" - already_prompted = _looks_like_cli_prompt(seed) or prompted_early + already_prompted = _looks_like_cli_prompt(seed) or _looks_like_cli_prompt(pre_log) saw_password_change = bool( re.search( r"(?i)(change\s*now|please\s*choose|password\s+needs\s+to\s+be\s+changed).{0,80}:", @@ -359,7 +356,7 @@ def _finish_connect( primed = "" combined = f"{seed}{primed}" # Brief settle only — do not burn seconds here while the terminal shows a prompt. - settle_sec = 0.05 if already_prompted or _looks_like_cli_prompt(combined) else 0.3 + settle_sec = 0.12 if already_prompted or _looks_like_cli_prompt(combined) else 0.3 try: more = _capture_raw_channel(conn, duration=settle_sec) if more: @@ -398,7 +395,7 @@ def _finish_connect( if not echoed.strip(): bootstrap = prepare_bootstrap_output(combined) try: - leftover = _capture_raw_channel(conn, duration=0.05) + leftover = _capture_raw_channel(conn, duration=0.1) except Exception: leftover = "" if ( @@ -410,7 +407,7 @@ def _finish_connect( bootstrap = prepare_bootstrap_output(f"{bootstrap}{leftover}") else: try: - leftover = _capture_raw_channel(conn, duration=0.04) + leftover = _capture_raw_channel(conn, duration=0.08) except Exception: leftover = "" if ( @@ -424,19 +421,6 @@ def _finish_connect( sess.conn = conn sess.cli_hop_guard = bool(hop_guard) sess.cli_hop_prompt = str((hop_guard or {}).get("hop_prompt") or "") - # Inventory type "generic" (e.g. a quick-connect to a Cisco box) disables the network - # key map, so Home/Delete reach IOS/VRP as raw escape sequences. The login prompt tells - # us what it really is. - if not sess.cli_keymap and str(sess.device_type or "").strip().lower() in ( - "", - "generic", - "generic_termserver", - "generic_telnet", - ): - last_line = str(echoed or combined or "").replace("\r\n", "\n").replace("\r", "\n").rstrip(" \n").split("\n")[-1] - if looks_like_network_prompt(prompt_hint or last_line) or looks_like_network_prompt(last_line): - sess.cli_keymap = True - sess._tracker.linux_like = False sess.bootstrap_output = _encode_text(str(bootstrap or ""), sess.encoding) # Nudge Enter on WS attach only when we still need a shell prompt. # Never when already at CLI prompt or Username:/Password: (would empty-submit login). @@ -448,8 +432,6 @@ def _finish_connect( and not _looks_like_login_prompt(final_view) ) sess.open_session_log() - # Prime the audit tracker with the prompt printed during login (reader starts later). - sess.seed_audit_row(echoed or bootstrap or combined) log_seed = echoed or bootstrap if log_seed: sess.append_session_log(log_seed if str(log_seed).endswith("\n") else str(log_seed) + "\n") @@ -457,7 +439,7 @@ def _finish_connect( # Drop late prompt echoes that race into the queue right after reader start. if not prompt_hint and final_view: prompt_hint = str(final_view).replace("\r\n", "\n").replace("\r", "\n").strip().split("\n")[-1].strip() - settle_deadline = time.time() + (0.08 if already_prompted or saw_password_change else 0.35) + settle_deadline = time.time() + (0.2 if already_prompted or saw_password_change else 0.35) while time.time() < settle_deadline: try: chunk = sess.out_queue.get_nowait() @@ -471,14 +453,8 @@ def _finish_connect( text = _decode_bytes(chunk, sess.encoding) except Exception: text = "" - # A slow VM may emit its FIRST prompt after the reader starts. - # Only drop duplicates when a prompt was already delivered during login. - if _looks_like_cli_prompt(final_view) and _is_prompt_only_echo(text, prompt_hint): - # Dropped from the browser stream: keep the audit row in step with it. - sess.reset_audit_row(prompt_hint) + if _is_prompt_only_echo(text, prompt_hint): continue - if _looks_like_cli_prompt(text): - sess.needs_live_prompt = False # Non-prompt data: put back and stop settling. sess.out_queue.put(chunk) break @@ -716,7 +692,8 @@ def mark_attached(session_id: str) -> tuple[WebcrtSession, int]: raise HTTPException(status_code=404, detail="webcrt_session_not_found") # Allow re-attach after brief WS drop (React StrictMode remount / network blip). # Bump generation so the previous WS pump stops and does not steal echo bytes. - attach_gen = sess.bump_attach() + sess.attach_gen += 1 + attach_gen = sess.attach_gen sess.attached = True sess.detach_deadline = None sess.touch() diff --git a/packages/netx-mcp/README.md b/packages/netx-mcp/README.md index 009c1ec..72ce048 100644 --- a/packages/netx-mcp/README.md +++ b/packages/netx-mcp/README.md @@ -38,7 +38,7 @@ python -m netx_mcp [`mcp.json`](./mcp.json) — `command: python`，`args: ["-m", "netx_mcp"]`，`env` 见文件。 -## 工具（15）— skill 组 **ops**（`netx-ops`） +## 工具（14）— skill 组 **ops**（`netx-ops`） **NMS**（模型面通用名；当前适配器 zte-ume，REST 仍 `/v1/ume/*`）： `queryNmsAlarms`, `aggregateNmsAlarms`, `runNmsDiagnostics`, `queryNmsNeInventory`, `getNmsNe`, `queryNmsAlarmsRaw`, `aggregateNmsAlarmsRaw`, `listNmsAlarmFields`, `sqlQueryNms` @@ -46,8 +46,6 @@ python -m netx_mcp **Managed CLI + paths**（问「能否登录」必须走这里，不要只查 inventory）： `listManagedNe`, `getManagedNe`, `execManagedNe`, `getNeExecJob`, `listCliTargets`, `findTopologyPaths` -0.3.1：`getNmsNe` 可直接传 host_name/IP；输出统一 `nms_ne_id` / `source=nms`；参数类型错误返回工具错误而非协议崩溃。 - 参数优先 `nms_ne_id` / `nms_ne_ids`（保留 `ume_*` 别名）。 拓扑画布 / Fabric → skill **topology** / [`netx-topology-mcp`](../netx-topology-mcp)（见 [docs/MCP_TOPOLOGY.md](../../docs/MCP_TOPOLOGY.md)）。 diff --git a/packages/netx-mcp/pyproject.toml b/packages/netx-mcp/pyproject.toml index 090358e..56d3e79 100644 --- a/packages/netx-mcp/pyproject.toml +++ b/packages/netx-mcp/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netx-mcp" -version = "0.3.1" +version = "0.3.0" description = "stdio MCP server for netx REST API (NMS alarms, managed NE CLI)" readme = "README.md" requires-python = ">=3.11" diff --git a/packages/netx-mcp/src/netx_mcp/http_client.py b/packages/netx-mcp/src/netx_mcp/http_client.py index 3a6914e..60a153c 100644 --- a/packages/netx-mcp/src/netx_mcp/http_client.py +++ b/packages/netx-mcp/src/netx_mcp/http_client.py @@ -117,9 +117,7 @@ def mcp_text_result(payload: Any, *, is_error: bool = False) -> dict[str, Any]: def mcp_from_handler_result(result: dict[str, Any]) -> dict[str, Any]: - # API envelopes may nest a failure: {"ok": true, "data": {"ok": false, ...}} (e.g. job_not_found). - data = result.get("data") - if not result.get("ok") or (isinstance(data, dict) and data.get("ok") is False): + if not result.get("ok"): return mcp_text_result(result, is_error=True) return mcp_text_result(result) diff --git a/packages/netx-mcp/src/netx_mcp/http_tools.py b/packages/netx-mcp/src/netx_mcp/http_tools.py index e604cbf..65e3bcd 100644 --- a/packages/netx-mcp/src/netx_mcp/http_tools.py +++ b/packages/netx-mcp/src/netx_mcp/http_tools.py @@ -75,80 +75,6 @@ def _should_run_exec_async(args: dict[str, Any]) -> bool: return _count_exec_ne_targets(args) >= min_n -class ToolArgError(ValueError): - """Bad tool argument; surfaced to the model as a tool error (not a JSON-RPC crash).""" - - -def _int_arg(args: dict[str, Any], key: str, default: int, lo: int, hi: int) -> int: - raw = args.get(key) - if raw is None or (isinstance(raw, str) and not raw.strip()): - return default - try: - val = int(raw) - except (TypeError, ValueError): - raise ToolArgError(f"{key} must be an integer, got {raw!r}") from None - return max(lo, min(hi, val)) - - -def _severity(args: dict[str, Any]) -> str: - # Stored values are lowercase (critical/major/minor/warning); exact match server-side. - return str(args.get("severity") or "").strip().lower() - - -def _is_me_placeholder(ne_name: Any, ne_id: Any) -> bool: - return bool(ne_id) and str(ne_name or "") == f"ME{{{ne_id}}}" - - -def _compact_rows(out: dict[str, Any], *, name_key: str, id_key: str) -> dict[str, Any]: - """Drop ZTE ``ME{}`` ne_name placeholders: they repeat ne_id and carry no meaning.""" - data = out.get("data") if isinstance(out, dict) else None - items = data.get("items") if isinstance(data, dict) else None - if isinstance(items, list): - for row in items: - if isinstance(row, dict) and _is_me_placeholder(row.get(name_key), row.get(id_key)): - row.pop(name_key, None) - return out - - -def _strip_meta_field_lists(out: dict[str, Any]) -> dict[str, Any]: - """Raw endpoints echo ~40 field names per call; listNmsAlarmFields already serves them.""" - data = out.get("data") if isinstance(out, dict) else None - meta = data.get("meta") if isinstance(data, dict) else None - if isinstance(meta, dict): - for k in ("available_fields", "group_by_allowed", "order_by_allowed"): - meta.pop(k, None) - return out - - -def _nms_naming(row: dict[str, Any]) -> None: - """REST still speaks ume_*; model-facing params are nms_* -- keep output consistent.""" - if "ume_ne_id" in row: - row["nms_ne_id"] = row.pop("ume_ne_id") - if row.get("source") == "ume": - row["source"] = "nms" - - -_HOP_KEYS = ( - "hop_vendor", - "hop_host", - "hop_port", - "hop_protocol", - "hop_username", - "hop_command_template", - "hop_vrf", - "hop_target_auth_mode", - "hop_enter_system_view", -) - - -def _compact_managed_ne(row: dict[str, Any], *, keep_detail: bool) -> None: - if row.get("hop_enabled") is False: - for k in _HOP_KEYS: - row.pop(k, None) - if not keep_detail: - row.pop("connect_detail", None) - - UME_RAW_GROUP_FIELDS = [ "alarm_alarm_key", "alarm_host_name", @@ -195,6 +121,7 @@ _UME_RAW_FIELD_PRESETS: dict[str, list[str]] = { "alarm_last_seen_at", "ne_host_name", "ne_user_label", + "ne_ne_name", "ne_ip_address", "ne_exists", ], @@ -210,6 +137,7 @@ _UME_RAW_FIELD_PRESETS: dict[str, list[str]] = { "alarm_last_seen_at", "ne_host_name", "ne_user_label", + "ne_ne_name", "ne_ip_address", "ne_connection_status", "ne_exists", @@ -220,6 +148,7 @@ _UME_RAW_FIELD_PRESETS: dict[str, list[str]] = { "alarm_perceived_severity", "alarm_last_seen_at", "ne_user_label", + "ne_ne_name", "ne_ip_address", "ne_ipv6_address", "ne_device_level", @@ -234,14 +163,13 @@ _UME_RAW_FIELD_PRESETS: dict[str, list[str]] = { def _query_ume_alarms(args: dict[str, Any]) -> dict[str, Any]: - params: dict[str, Any] = { - "page": _int_arg(args, "page", 1, 1, 100000), - "page_size": _int_arg(args, "page_size", 50, 1, 500), - } - if _severity(args): - params["severity"] = _severity(args) - if str(args.get("is_cleared") or "").strip(): - params["is_cleared"] = str(args.get("is_cleared")).strip() + page = max(1, int(args.get("page") or 1)) + if page > 2: + page = 2 + page_size = min(500, max(1, int(args.get("page_size") or 50))) + params: dict[str, Any] = {"page": page, "page_size": page_size} + if str(args.get("severity") or "").strip(): + params["severity"] = str(args.get("severity")).strip() keyword = str(args.get("keyword") or "").strip() ne_name = str(args.get("ne_name") or "").strip() if keyword: @@ -255,7 +183,7 @@ def _query_ume_alarms(args: dict[str, Any]) -> dict[str, Any]: for k in ("time_from", "time_to"): if str(args.get(k) or "").strip(): params[k] = str(args.get(k)).strip() - return _compact_rows(http_json("GET", "/v1/ume/alarms", params=params), name_key="ne_name", id_key="ne_id") + return http_json("GET", "/v1/ume/alarms", params=params) def _aggregate_ume_alarms(args: dict[str, Any]) -> dict[str, Any]: @@ -264,12 +192,11 @@ def _aggregate_ume_alarms(args: dict[str, Any]) -> dict[str, Any]: if group_by: return _aggregate_ume_alarms_raw(args) # Default top 50 named NEs — missing host buckets reported separately. - params: dict[str, Any] = {"top_ne": _int_arg(args, "top_ne", 50, 0, 500)} + top_ne = max(0, min(500, int(args.get("top_ne") if args.get("top_ne") is not None else 50))) + params: dict[str, Any] = {"top_ne": top_ne} if "exclude_missing_host" in args: params["exclude_missing_host"] = bool(args.get("exclude_missing_host")) - if _severity(args): - params["severity"] = _severity(args) - for k in ("time_from", "time_to"): + for k in ("severity", "time_from", "time_to"): if str(args.get(k) or "").strip(): params[k] = str(args.get(k)).strip() return http_json("GET", "/v1/ume/alarms/aggregate", params=params) @@ -281,58 +208,26 @@ def _run_ume_diagnostics(args: dict[str, Any]) -> dict[str, Any]: def _query_ume_ne_inventory(args: dict[str, Any]) -> dict[str, Any]: - params: dict[str, Any] = { - "page": _int_arg(args, "page", 1, 1, 100000), - "page_size": _int_arg(args, "page_size", 50, 1, 500), - } + page = max(1, int(args.get("page") or 1)) + page_size = min(500, max(1, int(args.get("page_size") or 50))) + params: dict[str, Any] = {"page": page, "page_size": page_size} if str(args.get("keyword") or "").strip(): params["keyword"] = str(args.get("keyword")).strip() - return _compact_rows(http_json("GET", "/v1/ume/inventory/ne", params=params), name_key="ne_name", id_key="ne_id") - - -def _looks_like_uuid(text: str) -> bool: - parts = text.split("-") - return len(parts) == 5 and all(p and all(c in "0123456789abcdefABCDEF" for c in p) for p in parts) + return http_json("GET", "/v1/ume/inventory/ne", params=params) def _get_ume_ne(args: dict[str, Any]) -> dict[str, Any]: ne_id = str(args.get("ne_id") or "").strip() if not ne_id: return {"ok": False, "error": "ne_id_required", "error_code": "ne_id_required"} - out = http_json("GET", f"/v1/ume/inventory/ne/{quote_ne_id(ne_id)}", params=None) - if out.get("ok") or out.get("error") != "netx_http_404" or _looks_like_uuid(ne_id): - return out - # Agents usually know the host_name (alarms show it), not the UUID: resolve it. - found = http_json("GET", "/v1/ume/inventory/ne", params={"keyword": ne_id, "page": 1, "page_size": 20}) - items = ((found.get("data") or {}).get("items") or []) if found.get("ok") else [] - needle = ne_id.lower() - exact = [ - r - for r in items - if isinstance(r, dict) - and needle in {str(r.get(k) or "").lower() for k in ("host_name", "user_label", "ip_address", "ne_name")} - ] - if len(exact) == 1 and exact[0].get("ne_id"): - return http_json("GET", f"/v1/ume/inventory/ne/{quote_ne_id(str(exact[0]['ne_id']))}", params=None) - out = dict(out) - out["hint"] = "ne_id is the NMS UUID. Resolve host_name/IP via queryNmsNeInventory(keyword=...)." - if items: - out["candidates"] = [ - {"ne_id": r.get("ne_id"), "host_name": r.get("host_name"), "ip_address": r.get("ip_address")} - for r in items[:10] - if isinstance(r, dict) - ] - return out + return http_json("GET", f"/v1/ume/inventory/ne/{quote_ne_id(ne_id)}", params=None) def _query_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: - params: dict[str, Any] = { - "page": _int_arg(args, "page", 1, 1, 100000), - "page_size": _int_arg(args, "page_size", 50, 1, 500), - } - if _severity(args): - params["severity"] = _severity(args) - for k in ("is_cleared", "ne_id", "event_type", "keyword", "time_from", "time_to", "order_by", "order"): + page = max(1, int(args.get("page") or 1)) + page_size = min(500, max(1, int(args.get("page_size") or 50))) + params: dict[str, Any] = {"page": page, "page_size": page_size} + for k in ("severity", "is_cleared", "ne_id", "event_type", "keyword", "time_from", "time_to", "order_by", "order"): v = str(args.get(k) or "").strip() if v: params[k] = v @@ -345,15 +240,7 @@ def _query_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: fields = list(_UME_RAW_FIELD_PRESETS.get(preset) or []) if fields: params["select_fields"] = ",".join(fields) - out = http_json("GET", "/v1/ume/alarms/raw", params=params) - data = out.get("data") if isinstance(out.get("data"), dict) else None - if data and isinstance(data.get("items"), list): - for row in data["items"]: - if isinstance(row, dict) and _is_me_placeholder( - row.get("ne_ne_name"), row.get("ne_ne_id") or row.get("alarm_ne_id") - ): - row.pop("ne_ne_name", None) - return _strip_meta_field_lists(out) + return http_json("GET", "/v1/ume/alarms/raw", params=params) def _aggregate_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: @@ -361,6 +248,7 @@ def _aggregate_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: for k in ( "group_by", "group_by2", + "severity", "is_cleared", "ne_id", "event_type", @@ -375,13 +263,9 @@ def _aggregate_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: sv = str(v).strip() if sv: params[k] = sv - if _severity(args): - params["severity"] = _severity(args) - if "limit" in params: - params["limit"] = _int_arg(args, "limit", 200, 1, 2000) if "exclude_missing_host" in args: params["exclude_missing_host"] = bool(args.get("exclude_missing_host")) - return _strip_meta_field_lists(http_json("GET", "/v1/ume/alarms/aggregate/raw", params=params)) + return http_json("GET", "/v1/ume/alarms/aggregate/raw", params=params) def _list_ume_alarm_fields(args: dict[str, Any]) -> dict[str, Any]: @@ -391,8 +275,8 @@ def _list_ume_alarm_fields(args: dict[str, Any]) -> dict[str, Any]: def _sql_query_ume(args: dict[str, Any]) -> dict[str, Any]: sql = str(args.get("sql") or "").strip() - limit = _int_arg(args, "limit", 200, 1, 2000) - statement_timeout_ms = _int_arg(args, "statement_timeout_ms", 0, 0, 30000) + limit = max(1, min(2000, int(args.get("limit") or 200))) + statement_timeout_ms = max(0, min(30000, int(args.get("statement_timeout_ms") or 0))) if not sql: return {"ok": False, "error": "sql_required"} return http_post_json( @@ -403,18 +287,13 @@ def _sql_query_ume(args: dict[str, Any]) -> dict[str, Any]: def _list_managed_ne(args: dict[str, Any]) -> dict[str, Any]: - page = _int_arg(args, "page", 1, 1, 100000) - page_size = _int_arg(args, "page_size", 20, 1, 100) + page = max(1, int(args.get("page") or 1)) + page_size = min(100, max(1, int(args.get("page_size") or 20))) keyword = str(args.get("keyword") or "").strip() vendor = str(args.get("vendor") or "").strip() connect_status = str(args.get("connect_status") or "").strip() if not (keyword or vendor or connect_status): - return { - "ok": False, - "error": "managed_ne_filter_required", - "error_code": "managed_ne_filter_required", - "hint": "Pass keyword (name/IP, >=2 chars), vendor, or connect_status (e.g. pass).", - } + return {"ok": False, "error": "managed_ne_filter_required", "error_code": "managed_ne_filter_required"} if keyword and len(keyword) < 2: return {"ok": False, "error": "managed_ne_keyword_too_short", "error_code": "managed_ne_keyword_too_short"} params: dict[str, Any] = {"page": page, "page_size": page_size} @@ -424,13 +303,7 @@ def _list_managed_ne(args: dict[str, Any]) -> dict[str, Any]: params["vendor"] = vendor if connect_status: params["connect_status"] = connect_status - out = http_json("GET", "/v1/managed-ne", params=params) - items = (out.get("data") or {}).get("items") if out.get("ok") else None - if isinstance(items, list): - for row in items: - if isinstance(row, dict): - _compact_managed_ne(row, keep_detail=False) - return out + return http_json("GET", "/v1/managed-ne", params=params) def _first_str(*vals: Any) -> str: @@ -457,9 +330,7 @@ def _get_managed_ne(args: dict[str, Any]) -> dict[str, Any]: "example": {"ne_id": ""}, } # NMS ne_id is typically a UUID; managed NE may differ. Soft-guide when callers mix them. - out = http_json("GET", f"/v1/managed-ne/{quote_ne_id(ne_id)}", params=None) - if out.get("ok") and isinstance(out.get("data"), dict): - _compact_managed_ne(out["data"], keep_detail=True) + out = http_json("GET", f"/v1/managed-ne/{ne_id}", params=None) if isinstance(out, dict) and out.get("ok") is False: detail = str(out.get("detail") or out.get("error") or "") low = detail.lower() @@ -520,9 +391,11 @@ def _exec_managed_ne(args: dict[str, Any]) -> dict[str, Any]: if len(shared_commands) > exec_max_commands(): return {"ok": False, "error": "too_many_commands", "error_code": "too_many_commands"} body["commands"] = shared_commands - body["read_timeout_sec"] = _int_arg(args, "read_timeout_sec", 60, 10, 120) - if args.get("concurrency") is not None: - body["concurrency"] = _int_arg(args, "concurrency", 4, 1, 8) + rts = args.get("read_timeout_sec") + body["read_timeout_sec"] = int(rts) if rts is not None else 60 + conc = args.get("concurrency") + if conc is not None: + body["concurrency"] = int(conc) if want_async: out = http_post_json("/v1/managed-ne/exec-jobs", body, timeout=60.0) if not out.get("ok"): @@ -561,7 +434,8 @@ def _exec_managed_ne(args: dict[str, Any]) -> dict[str, Any]: if nms_ne_id: body["ume_ne_id"] = nms_ne_id # Default 60s matches netx API default; slow show commands often exceed 30s. - body["read_timeout_sec"] = _int_arg(args, "read_timeout_sec", 60, 10, 120) + rts = args.get("read_timeout_sec") + body["read_timeout_sec"] = int(rts) if rts is not None else 60 if want_async: out = http_post_json("/v1/managed-ne/exec-jobs", body, timeout=60.0) if not out.get("ok"): @@ -590,23 +464,16 @@ def _get_ne_exec_job(args: dict[str, Any]) -> dict[str, Any]: def _list_cli_targets(args: dict[str, Any]) -> dict[str, Any]: - params: dict[str, Any] = { - "page": _int_arg(args, "page", 1, 1, 100000), - "page_size": _int_arg(args, "page_size", 50, 1, 500), - } + page = max(1, int(args.get("page") or 1)) + page_size = min(500, max(1, int(args.get("page_size") or 50))) + params: dict[str, Any] = {"page": page, "page_size": page_size} source = str(args.get("source") or "").strip() if source: # API still uses source=ume; accept model-facing "nms". params["source"] = "ume" if source.lower() == "nms" else source if str(args.get("keyword") or "").strip(): params["keyword"] = str(args.get("keyword")).strip() - out = http_json("GET", "/v1/cli/targets", params=params) - items = (out.get("data") or {}).get("items") if out.get("ok") else None - if isinstance(items, list): - for row in items: - if isinstance(row, dict): - _nms_naming(row) - return out + return http_json("GET", "/v1/cli/targets", params=params) def _find_topology_paths(args: dict[str, Any]) -> dict[str, Any]: @@ -622,8 +489,8 @@ def _find_topology_paths(args: dict[str, Any]) -> dict[str, Any]: if detail not in {"summary", "full"}: detail = "summary" body: dict[str, Any] = { - "max_paths": _int_arg(args, "max_paths", 3, 1, 10), - "max_hops": _int_arg(args, "max_hops", 6, 1, 12), + "max_paths": max(1, min(10, int(args.get("max_paths") or 3))), + "max_hops": max(1, min(12, int(args.get("max_hops") or 6))), "layer": str(args.get("layer") or "physical").strip() or "physical", "detail": detail, } @@ -635,14 +502,7 @@ def _find_topology_paths(args: dict[str, Any]) -> dict[str, Any]: body["to_ume_ne_id"] = to_uid else: body["to_managed_ne_id"] = to_mid - out = http_post_json("/v1/topology/fabric/paths", body, timeout=30.0) - paths = (out.get("data") or {}).get("paths") if out.get("ok") else None - if isinstance(paths, list): - for p in paths: - for node in (p.get("nodes") or []) if isinstance(p, dict) else []: - if isinstance(node, dict): - _nms_naming(node) - return out + return http_post_json("/v1/topology/fabric/paths", body, timeout=30.0) HTTP_MCP_TOOLS: list[dict[str, Any]] = [ @@ -660,8 +520,7 @@ HTTP_MCP_TOOLS: list[dict[str, Any]] = [ "inputSchema": { "type": "object", "properties": { - "severity": {"type": "string", "description": "critical/major/minor/warning"}, - "is_cleared": {"type": "string", "description": "\"False\" = active only; \"True\" = cleared."}, + "severity": {"type": "string"}, "ne_id": {"type": "string", "description": "Filter only; do not show UUID to users"}, "host_name": {"type": "string", "description": "Filter by NE host_name"}, "ne_name": {"type": "string", "description": "Legacy alias mapped to keyword"}, @@ -767,10 +626,10 @@ HTTP_MCP_TOOLS: list[dict[str, Any]] = [ }, { "name": "getNmsNe", - "description": "Get single NMS NE detail by ne_id (UUID); exact host_name / user_label / IP also resolve.", + "description": "Get single NMS NE detail by ne_id (UUID).", "inputSchema": { "type": "object", - "properties": {"ne_id": {"type": "string", "description": "NMS UUID, or exact host_name / IP."}}, + "properties": {"ne_id": {"type": "string"}}, "required": ["ne_id"], "additionalProperties": False, }, @@ -1048,8 +907,7 @@ HTTP_MCP_TOOLS: list[dict[str, Any]] = [ { "name": "findTopologyPaths", "description": ( - "Find up to max_paths simple paths between two fabric nodes for troubleshooting " - "(parallel links between the same NEs count as one hop; see edges[].parallel_links). " + "Find up to max_paths simple paths between two fabric nodes for troubleshooting. " "For each endpoint provide exactly one of nms_ne_id (from NMS alarm ne_id; alias " "from_ume_ne_id) or managed_ne_id — resolved to fabric node internally. Returns " "shortest paths first with compact label + node/edge summary (detail=summary default). " @@ -1156,8 +1014,4 @@ def call_http_tool(name: str, args: dict[str, Any]) -> dict[str, Any]: fn = _HANDLERS.get(str(name or "").strip()) if not fn: raise ValueError(f"unknown tool: {name}") - try: - result = fn(dict(args or {})) - except ToolArgError as exc: - result = {"ok": False, "error": "invalid_argument", "error_code": "invalid_argument", "detail": str(exc)} - return mcp_from_handler_result(result) + return mcp_from_handler_result(fn(dict(args or {}))) diff --git a/packages/netx-mcp/src/netx_mcp/server.py b/packages/netx-mcp/src/netx_mcp/server.py index 16b13ac..9f4b9ae 100644 --- a/packages/netx-mcp/src/netx_mcp/server.py +++ b/packages/netx-mcp/src/netx_mcp/server.py @@ -43,15 +43,6 @@ def _err(rid: Any, code: int, message: str) -> None: _UNSET = object() -def _version() -> str: - try: - from importlib.metadata import version - - return version("netx-mcp") - except Exception: - return "0.0.0" - - def _fetch_scopes() -> list[str] | None: """Return granted scopes from /v1/auth/me, or None if the call fails (show all tools).""" try: @@ -90,7 +81,6 @@ def run_stdio_loop() -> None: except Exception: continue rid = req.get("id") - is_notification = "id" not in req method = str(req.get("method") or "") params = req.get("params") if isinstance(req.get("params"), dict) else {} @@ -101,15 +91,11 @@ def run_stdio_loop() -> None: { "protocolVersion": "2024-11-05", "capabilities": {"tools": {}}, - "serverInfo": {"name": "netx-mcp", "version": _version(), "mode": "http"}, + "serverInfo": {"name": "netx-mcp", "version": "0.3.0", "mode": "http"}, }, ) continue - if is_notification: - # JSON-RPC: notifications (initialized, cancelled, ...) never get a response. - continue - if method == "ping": - _ok(rid, {}) + if method == "notifications/initialized": continue if method == "tools/list": _ok(rid, {"tools": tools_for_scopes(scopes())}) diff --git a/packages/netx-mcp/tests/test_mcp_http.py b/packages/netx-mcp/tests/test_mcp_http.py index 5b80df7..50e5b44 100644 --- a/packages/netx-mcp/tests/test_mcp_http.py +++ b/packages/netx-mcp/tests/test_mcp_http.py @@ -74,7 +74,7 @@ def test_call_aggregate_nms_alarms_group_by_routes_to_raw() -> None: params = mock_http.call_args[1]["params"] assert params["group_by"] == "alarm_host_name" assert params["severity"] == "critical" - assert params["limit"] == 20 + assert params["limit"] == "20" payload = json.loads(out["content"][0]["text"]) assert payload["ok"] is True @@ -285,14 +285,6 @@ def test_stdio_initialize_and_tools_list() -> None: assert msg.get("id") == 1 assert "result" in msg - # Notifications must not be answered; the next line belongs to ping. - for note in ("notifications/initialized", "notifications/cancelled"): - proc.stdin.write(json.dumps({"jsonrpc": "2.0", "method": note, "params": {}}) + "\n") - proc.stdin.write(json.dumps({"jsonrpc": "2.0", "id": 9, "method": "ping"}) + "\n") - proc.stdin.flush() - pong = json.loads(proc.stdout.readline()) - assert pong == {"jsonrpc": "2.0", "id": 9, "result": {}} - proc.stdin.write(json.dumps({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}) + "\n") proc.stdin.flush() line2 = proc.stdout.readline() @@ -305,92 +297,3 @@ def test_stdio_initialize_and_tools_list() -> None: finally: proc.kill() proc.wait(timeout=5) - - -def test_bad_integer_arg_is_tool_error_not_crash() -> None: - with patch("netx_mcp.http_tools.http_json") as mock_http: - out = call_http_tool("queryNmsAlarms", {"page": "two"}) - mock_http.assert_not_called() - assert out["isError"] is True - payload = json.loads(out["content"][0]["text"]) - assert payload["error"] == "invalid_argument" - assert "page" in payload["detail"] - - -def test_query_nms_alarms_pages_past_two_and_normalizes_severity() -> None: - with patch("netx_mcp.http_tools.http_json") as mock_http: - mock_http.return_value = { - "ok": True, - "data": {"items": [{"ne_id": "u-1", "ne_name": "ME{u-1}", "host_name": "H1"}]}, - } - out = call_http_tool("queryNmsAlarms", {"page": 5, "severity": "Critical", "is_cleared": "False"}) - params = mock_http.call_args[1]["params"] - assert params["page"] == 5 - assert params["severity"] == "critical" - assert params["is_cleared"] == "False" - row = json.loads(out["content"][0]["text"])["data"]["items"][0] - assert "ne_name" not in row - assert row["host_name"] == "H1" - - -def test_nested_api_failure_is_marked_error() -> None: - with patch("netx_mcp.http_tools.http_json") as mock_http: - mock_http.return_value = {"ok": True, "data": {"ok": False, "error": "job_not_found"}} - out = call_http_tool("getNeExecJob", {"job_id": "nope"}) - assert out.get("isError") is True - - -def test_raw_query_strips_echoed_field_lists() -> None: - with patch("netx_mcp.http_tools.http_json") as mock_http: - mock_http.return_value = { - "ok": True, - "data": { - "items": [], - "meta": {"available_fields": ["a"], "order_by_allowed": ["b"], "time_filter_field": "last_seen_at"}, - }, - } - out = call_http_tool("queryNmsAlarmsRaw", {"field_preset": "brief"}) - meta = json.loads(out["content"][0]["text"])["data"]["meta"] - assert meta == {"time_filter_field": "last_seen_at"} - - -def test_get_nms_ne_resolves_host_name() -> None: - calls: list[str] = [] - - def fake(method: str, path: str, params: dict | None = None) -> dict: - calls.append(path) - if path == "/v1/ume/inventory/ne": - return {"ok": True, "data": {"items": [{"ne_id": "uuid-1", "host_name": "MDN-X-AN1"}]}} - if path.endswith("/uuid-1"): - return {"ok": True, "data": {"ne_id": "uuid-1"}} - return {"ok": False, "error": "netx_http_404", "detail": "ume_ne_not_found"} - - with patch("netx_mcp.http_tools.http_json", side_effect=fake): - out = call_http_tool("getNmsNe", {"ne_id": "mdn-x-an1"}) - assert json.loads(out["content"][0]["text"])["data"]["ne_id"] == "uuid-1" - assert calls[-1] == "/v1/ume/inventory/ne/uuid-1" - - -def test_cli_targets_and_paths_use_nms_naming() -> None: - with patch("netx_mcp.http_tools.http_json") as mock_http: - mock_http.return_value = {"ok": True, "data": {"items": [{"source": "ume", "ume_ne_id": "u-1"}]}} - out = call_http_tool("listCliTargets", {"source": "nms"}) - row = json.loads(out["content"][0]["text"])["data"]["items"][0] - assert row == {"source": "nms", "nms_ne_id": "u-1"} - - with patch("netx_mcp.http_tools.http_post_json") as mock_post: - mock_post.return_value = {"ok": True, "data": {"paths": [{"nodes": [{"name": "A", "ume_ne_id": "u-1"}]}]}} - out = call_http_tool("findTopologyPaths", {"from_nms_ne_id": "u-1", "to_nms_ne_id": "u-2"}) - node = json.loads(out["content"][0]["text"])["data"]["paths"][0]["nodes"][0] - assert node == {"name": "A", "nms_ne_id": "u-1"} - - -def test_managed_ne_list_drops_disabled_hop_noise() -> None: - with patch("netx_mcp.http_tools.http_json") as mock_http: - mock_http.return_value = { - "ok": True, - "data": {"items": [{"id": "m1", "hop_enabled": False, "hop_vrf": "", "connect_detail": "long"}]}, - } - out = call_http_tool("listManagedNe", {"connect_status": "pass"}) - row = json.loads(out["content"][0]["text"])["data"]["items"][0] - assert row == {"id": "m1", "hop_enabled": False} diff --git a/packages/netx-topology-mcp/src/netx_topology_mcp/http_tools.py b/packages/netx-topology-mcp/src/netx_topology_mcp/http_tools.py index 17dfe37..581510e 100644 --- a/packages/netx-topology-mcp/src/netx_topology_mcp/http_tools.py +++ b/packages/netx-topology-mcp/src/netx_topology_mcp/http_tools.py @@ -1976,18 +1976,15 @@ def _update_topology_view_positions(args: dict[str, Any]) -> dict[str, Any]: fid = str(p.get("fabric_node_id") or "").strip() if not fid: continue - row: dict[str, Any] = { - "fabric_node_id": fid, - "x": float(p.get("x") or 0), - "y": float(p.get("y") or 0), - } - # Omitted label/locked = keep server value (a UI rename or an existing lock). - # label "" clears the override; locked false explicitly unlocks. - if p.get("label") is not None: - row["label"] = str(p.get("label")) - if p.get("locked") is not None: - row["locked"] = bool(p.get("locked")) - cleaned.append(row) + cleaned.append( + { + "fabric_node_id": fid, + "x": float(p.get("x") or 0), + "y": float(p.get("y") or 0), + "label": str(p.get("label") or ""), + "locked": bool(p.get("locked") or False), + } + ) if not cleaned: return {"ok": False, "error": "positions_required"} parts: list[dict[str, Any]] = [] @@ -3730,14 +3727,8 @@ HTTP_MCP_TOOLS: list[dict[str, Any]] = [ "fabric_node_id": {"type": "string"}, "x": {"type": "number"}, "y": {"type": "number"}, - "label": { - "type": "string", - "description": "Omit to keep the current label; \"\" clears a rename (follow device name).", - }, - "locked": { - "type": "boolean", - "description": "Omit to keep the lock. Locked nodes are skipped unless locked=false (unlock + move).", - }, + "label": {"type": "string"}, + "locked": {"type": "boolean"}, }, "required": ["fabric_node_id"], "additionalProperties": False, diff --git a/packages/netx-topology-mcp/src/netx_topology_mcp/layout_ops/press_crossings.py b/packages/netx-topology-mcp/src/netx_topology_mcp/layout_ops/press_crossings.py index 9ff5504..aff469f 100644 --- a/packages/netx-topology-mcp/src/netx_topology_mcp/layout_ops/press_crossings.py +++ b/packages/netx-topology-mcp/src/netx_topology_mcp/layout_ops/press_crossings.py @@ -284,10 +284,9 @@ def press_hot_edges( tuple[int, tuple[float, float], tuple[float, float]] | None ) = None - def _try_cand(cand: tuple[float, float], disp_cap: float | None = None) -> None: + def _try_cand(cand: tuple[float, float]) -> None: nonlocal best - cap = max_disp if disp_cap is None else disp_cap - if math.hypot(cand[0] - leaf_xy[0], cand[1] - leaf_xy[1]) > cap: + if math.hypot(cand[0] - leaf_xy[0], cand[1] - leaf_xy[1]) > max_disp: return trial_pos = {**pos, leaf: cand} if _overlaps_any(leaf, trial_pos, names): @@ -305,11 +304,7 @@ def press_hot_edges( best = (c1, cand) # Reel-in along the spoke first (shorten without hunting angles). - # Pulling straight toward the hub cannot fling the leaf across the - # bbox, so allow it to travel all the way to the r_use target; - # max_disp alone stops short (e.g. r0=8000 → cap 4400 < 5800 needed). if r0 > r_use + 50.0: - reel_cap = max(max_disp, r0 - r_use) ux = (leaf_xy[0] - hub_xy[0]) / r0 uy = (leaf_xy[1] - hub_xy[1]) / r0 for rt in ( @@ -323,7 +318,7 @@ def press_hot_edges( rt = min(rt, r0 - 1.0) if rt < 120.0: continue - _try_cand((hub_xy[0] + ux * rt, hub_xy[1] + uy * rt), reel_cap) + _try_cand((hub_xy[0] + ux * rt, hub_xy[1] + uy * rt)) # Both ends walk toward the midpoint (metro bridges often need this). # Skip on giant E: each trial is a full O(E²) count and stalls MCP. diff --git a/packages/netx-topology-mcp/tests/test_dual_mass.py b/packages/netx-topology-mcp/tests/test_dual_mass.py index e3b353b..6b57e9a 100644 --- a/packages/netx-topology-mcp/tests/test_dual_mass.py +++ b/packages/netx-topology-mcp/tests/test_dual_mass.py @@ -80,7 +80,7 @@ def test_classify_petal_and_straight() -> None: assert classify_dual_unit(short) == "straight" -def test_beautify_petal_ellipse_bands() -> None: +def test_beautify_parallel_lanes() -> None: nodes, edges = _two_eye_graph() st = build_state_from_nodes_edges(nodes, edges) unit = DualUnit( @@ -89,18 +89,15 @@ def test_beautify_petal_ellipse_bands() -> None: paths=[["p1", "a1", "a2", "p2"], ["p1", "b1", "b2", "p2"]], ) pos = beautify_dual_unit_positions(st, unit, LayoutParams()) - # Portals sit on the x-axis, a left of b (the eye's chord). - assert abs(pos["p1"][1]) < 1e-6 and abs(pos["p2"][1]) < 1e-6 - assert pos["p1"][0] < pos["p2"][0] - # Each corridor is one band on its own side of the chord. + # Parallel H/V lanes: same y in a corridor; first/last share portal x. + assert abs(pos["a1"][1] - pos["a2"][1]) < 1e-6 + assert abs(pos["b1"][1] - pos["b2"][1]) < 1e-6 + assert abs(pos["a1"][1]) > 1.0 and abs(pos["b1"][1]) > 1.0 assert pos["a1"][1] * pos["b1"][1] < 0 - for x, y in (("a1", "a2"), ("b1", "b2")): - assert pos[x][1] * pos[y][1] > 0 - assert abs(pos[x][1]) > 1.0 - # Runs a→b, strictly inside the portals (hollow eye, no node on the chord ends). - assert pos["p1"][0] < pos[x][0] < pos[y][0] < pos["p2"][0] - links = [(e["a_node_id"], e["b_node_id"]) for e in edges if e["a_node_id"] in pos and e["b_node_id"] in pos] - assert count_edge_crossings(pos, links) == 0 + assert abs(pos["a1"][0] - pos["p1"][0]) < 1e-6 + assert abs(pos["a2"][0] - pos["p2"][0]) < 1e-6 + assert abs(pos["b1"][0] - pos["p1"][0]) < 1e-6 + assert abs(pos["b2"][0] - pos["p2"][0]) < 1e-6 def test_dual_mass_refine_reports_drift() -> None: diff --git a/packages/netx-topology-mcp/tests/test_layout_jobs.py b/packages/netx-topology-mcp/tests/test_layout_jobs.py index 0c5ca70..b76ac09 100644 --- a/packages/netx-topology-mcp/tests/test_layout_jobs.py +++ b/packages/netx-topology-mcp/tests/test_layout_jobs.py @@ -105,54 +105,26 @@ def test_slim_layout_payload_drops_guide() -> None: assert len(slim["crossing"]["top_nodes"]) <= 5 -def test_compose_views_is_not_public() -> None: - # compose_* was retired from the public surface (dual-unit drain is the main path). - out = _layout_topology_view( - { - "view_id": "full", - "action": "compose_views", - "mode": "apply", - "params": {"source_view_ids": [f"s{i:02d}" for i in range(12)]}, - } - ) - assert out.get("ok") is False - assert out.get("error") == "unknown_action:compose_views" - - -def test_orbit_background_returns_job_id() -> None: - graph = { - "ok": True, - "nodes": [ - {"fabric_node_id": f"n{i}", "x": float(i * 100), "y": 0.0, "label": f"N{i}"} - for i in range(6) - ], - "edges": [ - {"id": f"e{i}", "a_node_id": f"n{i}", "b_node_id": f"n{i + 1}"} for i in range(5) - ], - } +def test_compose_background_returns_job_id() -> None: + src = [f"s{i:02d}" for i in range(12)] def fake_start(**kwargs): - assert kwargs["action"] == "orbit_sweep" + assert kwargs["action"] == "compose_views" assert kwargs["view_id"] == "full" - tool_args = kwargs.get("tool_args") - assert isinstance(tool_args, dict) - # The worker re-runs the same call synchronously. - assert tool_args["params"].get("_force_sync") is True - assert "background" not in tool_args["params"] + assert isinstance(kwargs.get("tool_args"), dict) + assert kwargs["tool_args"].get("action") == "compose_views" return "abc123job" - with patch("netx_topology_mcp.http_tools.http_json", return_value={"ok": True, "data": graph}), patch( - "netx_topology_mcp.http_tools.start_job", side_effect=fake_start - ): + with patch("netx_topology_mcp.http_tools.start_job", side_effect=fake_start): out = _layout_topology_view( { "view_id": "full", - "action": "orbit_sweep", + "action": "compose_views", "mode": "apply", - "params": {"background": True}, + "params": {"source_view_ids": src}, } ) - assert out.get("ok") is True, out + assert out.get("ok") is True assert out.get("status") == "running" assert out.get("job_id") == "abc123job" assert out.get("applied") is False diff --git a/packages/netx-topology-mcp/tests/test_mcp_topology.py b/packages/netx-topology-mcp/tests/test_mcp_topology.py index 09547f8..ccaa466 100644 --- a/packages/netx-topology-mcp/tests/test_mcp_topology.py +++ b/packages/netx-topology-mcp/tests/test_mcp_topology.py @@ -18,10 +18,8 @@ from netx_topology_mcp.server import _fetch_scopes def test_tool_list_has_draw_and_query_tools() -> None: names = {str(t.get("name") or "") for t in HTTP_MCP_TOOLS} - assert len(names) == 16 + assert len(names) == 14 assert "sinkTopologyDualUnits" in names - assert "suggestSinkHubs" in names - assert "classifyTopologyFabricNodes" in names assert "copyTopologyViewNodes" in names assert "createTopologyFolder" in names assert "createTopologyView" not in names @@ -549,10 +547,8 @@ def test_stdio_initialize_and_tools_list() -> None: proc.stdin.flush() list_resp = json.loads(proc.stdout.readline()) tools = list_resp["result"]["tools"] - assert len(tools) == 16 + assert len(tools) == 14 names = {t["name"] for t in tools} - assert "suggestSinkHubs" in names - assert "classifyTopologyFabricNodes" in names assert "createTopologyView" not in names assert "listTopologyViews" not in names assert "queryTopologyFabricNodes" in names @@ -618,24 +614,3 @@ def test_analyze_topology_view_layout_single() -> None: assert "sparsity" in payload assert "verdict" in payload assert "total" in payload["verdict"] - - -def test_update_positions_omits_label_and_locked_unless_given() -> None: - from netx_topology_mcp.http_tools import _update_topology_view_positions - - with patch("netx_topology_mcp.http_tools.http_json") as mock_http: - mock_http.return_value = {"ok": True, "data": {"ok": True, "updated": 3}} - _update_topology_view_positions( - { - "view_id": "v1", - "positions": [ - {"fabric_node_id": "a", "x": 1, "y": 2}, - {"fabric_node_id": "b", "x": 3, "y": 4, "label": "", "locked": False}, - {"fabric_node_id": "c", "x": 5, "y": 6, "label": "Core-1", "locked": True}, - ], - } - ) - sent = mock_http.call_args[1]["body"]["positions"] - assert sent[0] == {"fabric_node_id": "a", "x": 1.0, "y": 2.0} - assert sent[1]["label"] == "" and sent[1]["locked"] is False - assert sent[2]["label"] == "Core-1" and sent[2]["locked"] is True diff --git a/packaging/README.md b/packaging/README.md index 6a58a7f..4f22d6f 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -1,45 +1,38 @@ -# NetX Windows packaging +# NetX Windows packaging -**Linux:** the offline `netx--linux-x86_64.tar.gz` package lives in [`linux/`](linux/README.md) (install.sh + netxctl). `scripts/start_netx.sh` stays for running from a source checkout. +**Linux is unchanged:** keep using your own PostgreSQL and `NETX_DATABASE_URL` with `scripts/start_netx.sh`. Nothing under this folder is required on Linux. This directory builds a Windows deliverable with: - Optional **bundled** portable PostgreSQL **or** **external** existing Postgres - API-hosted UI (`web/dist`) — no separate Vite process for end users - Program / data split so upgrades do not wipe the database -- Manual update script + check/apply updates (GitHub Releases or custom manifest) -- Optional system tray + start-at-logon +- Manual update script + auto-update **manifest contract** (fetch not implemented yet) ## What users get | Artifact | How | |----------|-----| -| `NetX-Setup-x.y.z.exe` | Stage with `build_release.ps1`, then compile `installer/netx.iss` with [Inno Setup](https://jrsoftware.org/isinfo.php) | - -Zip packages are **not** published. Releases ship **Setup.exe only**. - -**Installer:** English + 简体中文 (language dialog). Icons use `packaging/assets/netx.ico`. - -**Offline:** Setup ships portable PostgreSQL, **`python/runtime` + `.venv`** (portable; not tied to the build PC’s user profile), and WinSW. On upgrade, Setup removes the old runtime and virtual environment before copying the bundled pair, preventing stale standard-library files when Python versions have different layouts. **First install** shows the Database page (built-in or external PostgreSQL; external credentials validated with `psql SELECT 1`). **Re-running Setup when `%ProgramData%\NetX\.env` has `NETX_DB_MODE` or a non-empty legacy `NETX_DATABASE_URL`** skips the DB wizard by default. Update stops and resumes an installed Windows service or startup task while replacing files. Service install uses bundled WinSW. Auto-update still needs network later, and is unchecked by default. - -**OS:** Windows 10/11 or Windows Server **2016+** recommended. Packaging scripts are UTF-8 **with BOM** so Chinese UI works on Windows PowerShell 5.x. The Windows release workflow pins bundled Python to **3.12**, so local release builds should use the same version for consistent artifacts. +| `NetX-x.y.z-win64.zip` | `build_release.ps1` | +| `NetX-Setup-x.y.z.exe` | Compile `installer/netx.iss` with [Inno Setup](https://jrsoftware.org/isinfo.php) after staging | ## Build (developer machine) -Prerequisites: Python **3.12.x**, Node 20+, PowerShell, network (to download PostgreSQL binaries once). +Prerequisites: Python 3.11+, Node 20+, PowerShell, network (to download PG binaries once). ```powershell cd netx -# Required once per clean build machine: download portable PostgreSQL into packaging\postgres\pgsql +# Optional: download portable Postgres into packaging\postgres\pgsql powershell -ExecutionPolicy Bypass -File .\packaging\download_postgres.ps1 -# Build web + offline stage containing Python, .venv, PostgreSQL and WinSW. No zip by default. +# Build web + stage + zip (-CreateVenv ships a ready .venv; large) powershell -ExecutionPolicy Bypass -File .\packaging\build_release.ps1 -CreateVenv ``` Output: -- `packaging/release/netx-win64/` — stage tree (input to Inno Setup) +- `packaging/release/netx-win64/` — stage tree +- `packaging/release/NetX--win64.zip` Inno Setup: @@ -47,40 +40,29 @@ Inno Setup: ISCC.exe packaging\installer\netx.iss ``` -Override version in the `.iss` or edit `#define MyAppVersion`. -Optional local zip only: `build_release.ps1 -CreateZip` (not for release upload). - -### Local release without GitHub - -GitHub Actions (`.github/workflows/release-windows.yml`, on `v*` tags) runs the same scripts. To release without it: - -1. Install Python **3.12** (same as CI; side by side with other versions is fine, PATH not needed — the build picks it via `py -3.12` or `%LOCALAPPDATA%\Programs\Python\Python312`) and Inno Setup 6. -2. `powershell -ExecutionPolicy Bypass -File .\packaging\publish_release.ps1 -Version x.y.z -SkipGitHub` → `packaging\release\NetX-Setup-x.y.z.exe` + `.sha256`. PostgreSQL and WinSW come from `packaging\cache` (offline once cached). `npm ci` is skipped when `web\node_modules` already matches `package-lock.json`, so a running Vite dev server does not break the build. -3. Publish to Forgejo with `publish_forgejo_release.ps1`, or copy the Setup to offline machines; clients can update from Forgejo only via `NETX_UPDATE_SOURCES=forgejo`. +Override version in the `.iss` or edit `#define MyAppVersion`. ## End-user install ### Setup.exe 1. Run `NetX-Setup-x.y.z.exe` (admin). -2. **First install:** Database page — choose built-in (offline) or external PostgreSQL (host/port/user/password/db; connection must succeed to continue). -3. **Already installed:** if `%ProgramData%\NetX\.env` already has `NETX_DB_MODE`, Setup shows an **Install mode** page: - - **Update** (default) — skip Database page; stop NetX; replace program files; **keep** ProgramData / DB settings. - - **Reinstall** — same Database wizard as first install (`ApplyDatabaseConfig`); does **not** delete ProgramData (use Uninstall → delete data for a wipe). -4. Files → `%ProgramFiles%\NetX\` (program root). -5. Data → `%ProgramData%\NetX\` (`.env`, `pgdata`, spool, secrets). -6. **Tasks page — run mode (exclusive):** *Desktop tray* (default; optional *start when I sign in* → `\NetX\NetXTray` logon task for local Administrators, quiet start without a browser) or *Windows service* (WinSW, runs at boot). Optional *daily automatic updates* (`\NetX\NetXUpdate`). On upgrades the page pre-selects what is actually installed; silent installs leave these alone unless `/TASKS=` is passed (e.g. `/TASKS="runservice,autoupdate"`). -7. Start menu: **NetX** (tray) / **Start NetX** / **Stop NetX** / **Check for updates** / **Reconfigure database** (names follow the Setup language). Autostart and auto-update can also be toggled later from tray → Settings. +2. Files → `%ProgramFiles%\NetX\` (program root). +3. Data → `%ProgramData%\NetX\` (`.env`, `pgdata`, spool, secrets). +4. Optional post-install task runs `setup_first_run.ps1` (choose bundled vs external DB). +5. Start menu: **Start NetX** / **Stop NetX** / **Open NetX UI**. -Start/stop from the tray or shortcuts go through `packaging\runtime_ctl.ps1`, which drives the Windows service / SYSTEM task when one is installed (stopping its children directly would just get them restarted). Uninstall removes the service and all `\NetX\` scheduled tasks. +### Zip (portable) -**Elevation (required on fresh PCs):** all end-user entry points (`NetX-Start.cmd`, `NetX-Tray.cmd`, `NetX-Stop.cmd`, `NetX-FirstRun.cmd`, tray, start/stop/setup/update) **self-elevate via UAC**. Program Files installs cannot start reliably as a normal user (`.venv` repair / writes). Approve the UAC prompt once per launch (tray stays elevated afterward). +1. Unpack anywhere. +2. Marker `.portable` → data root is sibling `NetXData\`. +3. Target needs **Python 3.11+** on PATH unless the zip was built with `-CreateVenv`. +4. Run: -Silent first install (bundled default): `/SILENT /DbMode=bundled` -Silent external: `/SILENT /DbMode=external /DbHost=... /DbPort=5432 /DbUser=... /DbPassword=... /DbName=...` -Silent update over existing data: `/VERYSILENT /NORESTART /InstallMode=update` (also `/SkipDbPage=1`) -Silent reinstall (DB wizard via params): `/VERYSILENT /InstallMode=reinstall /DbMode=bundled` (or external `/DbHost`…) — also `/ForceDbPage=1` -Optional credential key (reuse encrypted NE passwords from another install): `/CredentialSecretKey=...` — omit to auto-generate. +```powershell +.\packaging\setup_first_run.ps1 +.\packaging\start_netx_app.ps1 +``` ## Database modes @@ -94,122 +76,15 @@ Existing Windows deploys that only set `NETX_DATABASE_URL` keep working: never s ## Manual update -Preferred: run a newer `NetX-Setup-*.exe` and choose **Update** (or silent `/InstallMode=update`) so ProgramData is kept. Setup (elevated) always relinks `.venv` → `python/runtime` after file copy; runtime scripts also self-elevate so Start/Tray work on locked-down fresh PCs. - -Legacy/dev zip (only if you built with `-CreateZip`): - ```powershell -.\packaging\update_netx.ps1 -PackagePath .\NetX-0.4.0-win64.zip +.\packaging\update_netx.ps1 -PackagePath .\NetX-0.3.0-win64.zip ``` Stops services, replaces program folders (`netx_api`, `web`, `packaging`, `postgres`, …), **keeps** the data root, restarts. Schema migrations still run via Alembic on API start. -## Check / apply updates +## Auto-update (reserved) -**Defaults (no `.env` needed):** probe **GitHub** (`hansjone/netx`) and Forgejo mirror (`https://git.avelo.top/hansjone/netx`), then use the **newest reachable** version. Same version → prefer GitHub. - -```env -# Optional overrides in ProgramData\NetX\.env -# NETX_UPDATE_SOURCES=github,forgejo -# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest -# NETX_UPDATE_GITHUB_REPO=hansjone/netx -# NETX_UPDATE_URL=https://cdn.example.com/netx/manifest.json -# NETX_UPDATE_TOKEN=****** -``` - -| Variable | Role | -|----------|------| -| `NETX_UPDATE_FORGEJO_URL` | Forgejo/Gitea `releases/latest` API (default: git.avelo.top mirror) | -| `NETX_UPDATE_GITHUB_REPO` | GitHub `owner/repo` (default `hansjone/netx`) | -| `NETX_UPDATE_SOURCES` | Probe order / tie-break order, e.g. `github,forgejo`. Tray → Settings → Update source sets it: *GitHub (default)* = empty (GitHub + git.avelo.top fallback), *git.avelo.top* = `forgejo` only | -| `NETX_UPDATE_URL` | Optional custom JSON manifest | - -```powershell -.\packaging\check_update.ps1 -.\packaging\check_update.ps1 -Apply -``` - -Both release hosts must publish the same **Setup.exe and `.sha256` sidecar**. `check_update.ps1` prefers `NetX-Setup-*.exe` (legacy `win64.zip` is still accepted); it verifies the Setup against a manifest hash or the adjacent checksum asset. Old releases without a sidecar remain supported with a warning. **Code mirror alone is not enough** — Forgejo pull-mirror syncs git/tags only; release assets are uploaded separately. - -### Maintainer release checklist (Windows) - -Release from a clean, version-matched checkout in this order: - -1. **Build and publish GitHub** — ` .\packaging\publish_release.ps1 -Version x.y.z` builds, verifies and uploads Setup.exe plus SHA-256. -2. **Forgejo** — publish the same Setup.exe and checksum sidecar (default: public domain `https://git.avelo.top`): - -```powershell -# One-time: User env var (never commit the token) -# Forgejo → Settings → Applications → Generate New Token (repo write) -[Environment]::SetEnvironmentVariable("NETX_FORGEJO_TOKEN", "your_token", "User") -# Restart Cursor / open a new terminal so the agent/scripts see it - -# Default: https://git.avelo.top -.\packaging\publish_forgejo_release.ps1 -Version 0.4.11 - -# Optional when LAN IP is reachable: -# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 -ForgejoBase "http://10.0.0.131:3000" -``` - -| Role | URL | -|------|-----| -| Publish default | `https://git.avelo.top` (`-ForgejoBase` default) | -| Optional LAN | `http://10.0.0.131:3000` | -| Update probe (default) | GitHub + `https://git.avelo.top/.../releases/latest` | - -Verify: -- Confirm each release has `NetX-Setup-x.y.z.exe` and `NetX-Setup-x.y.z.exe.sha256`. -- https://git.avelo.top/hansjone/netx/releases -- Probe: `.\packaging\check_update.ps1` → both `github` and `forgejo` reachable with the new version - -Token: `NETX_FORGEJO_TOKEN` (or `FORGEJO_TOKEN`). - -## Tray & autostart - -```powershell -# System tray: Start / Stop / Open UI / Check updates (self-elevates) -.\packaging\netx_tray.ps1 -StartOnLaunch - -# Start tray at Windows logon — Scheduled Task with RunLevel Highest -# (replaces legacy HKCU\Run which cannot elevate on fresh installs) -.\packaging\install_autostart.ps1 -# Remove: .\packaging\install_autostart.ps1 -Remove -``` - -## Windows Service (admin) - -Uses [WinSW](https://github.com/winsw/winsw) (downloaded on first install into `packaging/cache`). -`service_run.ps1` probes `/health` and restarts children after consecutive failures. - -```powershell -# Elevated PowerShell -.\packaging\install_service.ps1 -Start -# Uninstall: .\packaging\install_service.ps1 -Uninstall - -# Fallback without WinSW binary management: SYSTEM scheduled task at startup -.\packaging\install_service.ps1 -Mode task -Start -``` - -## Silent auto-update - -```powershell -# Writes NETX_UPDATE_AUTO=true and registers a daily task (default 03:30, RunLevel Highest) -.\packaging\install_update_task.ps1 - -# Manual silent path (only applies when NETX_UPDATE_AUTO=true; self-elevates) -.\packaging\check_update.ps1 -Apply -Quiet -AutoOnly -``` - -Tray also auto-applies on launch when `NETX_UPDATE_AUTO=true`. - -## Code signing (optional, publisher machine) - -```powershell -.\packaging\sign_release.ps1 -Files .\packaging\release\NetX-Setup-0.4.0.exe -Thumbprint -# or: -PfxPath .\certs\code.pfx -PfxPassword *** -``` - -Needs `signtool.exe` (Windows SDK) and a real code-signing certificate. Without a trusted cert, SmartScreen may still warn. +See `manifest.example.json`. Future: set `NETX_UPDATE_URL` + `NETX_UPDATE_CHANNEL`; a checker will download the zip and call `update_netx.ps1`. Not implemented in this phase. ## Layout reminder diff --git a/packaging/_common.ps1 b/packaging/_common.ps1 index 355593a..4a1ef66 100644 --- a/packaging/_common.ps1 +++ b/packaging/_common.ps1 @@ -1,4 +1,4 @@ -# Shared path helpers for Windows packaging scripts. +# Shared path helpers for Windows packaging scripts. # Dot-source: . "$PSScriptRoot\_common.ps1" $ErrorActionPreference = "Stop" @@ -114,33 +114,7 @@ function Write-DotEnvValue { if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } - $text = ($lines -join "`r`n") - try { - Set-Content -LiteralPath $Path -Value $text -Encoding utf8 - } catch { - # Admin-created .env is often Users:RX only — repair ACL then retry once. - $root = $dir - if ((Split-Path -Leaf $dir) -eq "NetX" -or $dir -match '\\NetX$') { - $root = $dir - } else { - $parent = Split-Path -Parent $dir - if ($parent -and ((Split-Path -Leaf $parent) -eq "NetX")) { $root = $parent } - } - $null = Ensure-NetxDataAcl -DataRoot $root -Quiet - try { - # Reset .env ACL explicitly if still blocked. - $icacls = Join-Path $env:SystemRoot "System32\icacls.exe" - if (Test-Path -LiteralPath $icacls) { - & $icacls $Path /grant "*S-1-5-32-545:M" /C /Q 2>$null | Out-Null - } - Set-Content -LiteralPath $Path -Value $text -Encoding utf8 - } catch { - throw (New-Object System.UnauthorizedAccessException( - ("access_denied: cannot write {0}. Run Start Menu → NetX → Reconfigure database as Administrator, or: icacls `"{1}`" /grant Users:(OI)(CI)M /T" -f $Path, $root), - $_.Exception - )) - } - } + Set-Content -Path $Path -Value ($lines -join "`r`n") -Encoding utf8 } function Get-DbMode { @@ -161,522 +135,3 @@ function Import-NetxEnvFile { } return $map } - -function ConvertTo-NetxFsPath([string]$Path) { - # Forward slashes work in .env and most Windows APIs via Python pathlib. - return ($Path -replace '\\', '/') -} - -function Get-NetxDataEnvMap { - param([string]$DataRoot) - $d = $DataRoot - return @{ - "NETX_AUTH_MCP_TOKEN_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\mcp_token")) - "NETX_AUTH_SECRET_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\jwt_secret")) - "NETX_CREDENTIAL_SECRET_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\credential_secret")) - "NETX_SCHEDULER_HEARTBEAT_PATH" = (ConvertTo-NetxFsPath (Join-Path $d "data\runtime\scheduler_heartbeat.json")) - "NETX_RUN_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\runtime")) - "NETX_BIZ_STATE_SPOOL_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\biz_state_spool")) - "NETX_NE_COLLECTION_DATA_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\ne_collections")) - "NETX_NE_EXEC_JOB_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\ne_exec_jobs")) - "NETX_WEBCRT_DATA_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\webcrt")) - } -} - -function New-NetxFernetKey { - # cryptography.fernet.Fernet.generate_key() == urlsafe_b64encode(os.urandom(32)) - $bytes = New-Object byte[] 32 - $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() - try { - $rng.GetBytes($bytes) - } finally { - $rng.Dispose() - } - return [Convert]::ToBase64String($bytes).Replace('+', '-').Replace('/', '_') -} - -function Ensure-NetxDataAcl { - param( - [Parameter(Mandatory = $true)][string]$DataRoot, - [switch]$Quiet = $false - ) - # Installer creates %ProgramData%\NetX as Administrators. Tray / normal users must - # be able to update .env and runtime files when reconfiguring DB. - if (-not (Test-Path -LiteralPath $DataRoot)) { return $false } - try { - $acl = Get-Acl -LiteralPath $DataRoot - $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( - "BUILTIN\Users", - "Modify", - "ContainerInherit,ObjectInherit", - "None", - "Allow" - ) - $acl.SetAccessRule($rule) - Set-Acl -LiteralPath $DataRoot -AclObject $acl - - # Also fix already-created files that only inherited RX for Users. - $icacls = Join-Path $env:SystemRoot "System32\icacls.exe" - if (Test-Path -LiteralPath $icacls) { - & $icacls $DataRoot /grant "*S-1-5-32-545:(OI)(CI)M" /T /C /Q 2>$null | Out-Null - } - if (-not $Quiet) { - Write-Host "==> Data ACL: BUILTIN\Users Modify on $DataRoot" -ForegroundColor DarkGray - } - return $true - } catch { - if (-not $Quiet) { - Write-Host "[WARN] Ensure-NetxDataAcl: $($_.Exception.Message)" -ForegroundColor Yellow - } - return $false - } -} - -function Ensure-NetxDataDirectories { - param([string]$DataRoot) - if (-not (Test-Path -LiteralPath $DataRoot)) { - New-Item -ItemType Directory -Path $DataRoot -Force | Out-Null - } - $subs = @( - "data", "data\auth", "data\runtime", "data\biz_state_spool", - "data\ne_collections", "data\ne_exec_jobs", "data\webcrt", - "backups", "pgdata" - ) - foreach ($sub in $subs) { - $p = Join-Path $DataRoot $sub - if (-not (Test-Path $p)) { - New-Item -ItemType Directory -Path $p -Force | Out-Null - } - } - # Best-effort; succeeds when running elevated (installer / first-run as admin). - $null = Ensure-NetxDataAcl -DataRoot $DataRoot -Quiet -} - -function Test-NetxTcpPortFree { - param([string]$HostName = "127.0.0.1", [int]$Port) - try { - $client = New-Object System.Net.Sockets.TcpClient - $iar = $client.BeginConnect($HostName, $Port, $null, $null) - $ok = $iar.AsyncWaitHandle.WaitOne(400) - if ($ok -and $client.Connected) { - $client.Close() - return $false - } - $client.Close() - return $true - } catch { - return $true - } -} - -function Test-NetxApiHealthy { - param([string]$HostName = "127.0.0.1", [int]$Port = 8890, [int]$TimeoutSec = 2) - try { - $url = "http://${HostName}:${Port}/health" - $r = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec $TimeoutSec -MaximumRedirection 0 - if ($r.StatusCode -ne 200) { return $false } - $body = $r.Content | ConvertFrom-Json -ErrorAction Stop - return ($body.status -eq "ok") - } catch { - return $false - } -} - -function Get-NetxClientHost { - param([string]$BindHost = "") - # A wildcard bind address is not connectable on Windows; probe/open via loopback. - $h = "$BindHost".Trim() - if (-not $h -or $h -in @("0.0.0.0", "::", "[::]", "*")) { return "127.0.0.1" } - return $h -} - -function Open-NetxUrl { - param([Parameter(Mandatory = $true)][string]$Url) - # Packaged scripts run elevated; Start-Process would launch the browser - # elevated too. explorer.exe hands the URL to the user's non-elevated shell. - try { - Start-Process -FilePath (Join-Path $env:SystemRoot "explorer.exe") -ArgumentList "`"$Url`"" | Out-Null - return - } catch {} - try { Start-Process $Url | Out-Null } catch {} -} - -function Get-NetxManagedRunner { - # "service" (WinSW) / "task" (SYSTEM startup task) own the runtime when present; - # killing their children directly would just make service_run.ps1 restart them. - $svc = Get-Service -Name "NetX" -ErrorAction SilentlyContinue - if ($svc -and [string]$svc.StartType -ne "Disabled") { return "service" } - try { - $t = Get-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -ErrorAction Stop - if ([string]$t.State -ne "Disabled") { return "task" } - } catch {} - return "" -} - -function Write-NetxServiceStopFlag { - param([Parameter(Mandatory = $true)][string]$DataRoot) - # service_run.ps1 (the WinSW supervisor) exits when this file appears. - $runtimeDir = Join-Path $DataRoot "data\runtime" - if (-not (Test-Path $runtimeDir)) { return } - Set-Content -Path (Join-Path $runtimeDir "service.stop") -Value (Get-Date -Format o) -Encoding ascii -ErrorAction SilentlyContinue -} - -function Wait-NetxApiHealthy { - param( - [string]$HostName = "127.0.0.1", - [int]$Port = 8890, - [int]$TimeoutSec = 180, - [int]$PollMs = 500 - ) - $deadline = (Get-Date).AddSeconds($TimeoutSec) - while ((Get-Date) -lt $deadline) { - if (Test-NetxApiHealthy -HostName $HostName -Port $Port -TimeoutSec 2) { - return $true - } - Start-Sleep -Milliseconds $PollMs - } - return $false -} - -function ConvertTo-NetxProcessArgument { - param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) - # Start-Process joins string[] args with spaces and does NOT quote values that - # contain spaces (e.g. C:\Program Files\NetX). Always emit one argv token. - if ($Value -match '[\s"]') { - return '"' + ($Value -replace '"', '\"') + '"' - } - return $Value -} - -function ConvertTo-NetxProcessArgumentString { - param([Parameter(Mandatory = $true)][string[]]$Arguments) - return (($Arguments | ForEach-Object { ConvertTo-NetxProcessArgument -Value "$_" }) -join " ") -} - -function Start-NetxPowerShell { - param( - [Parameter(Mandatory = $true)][string]$File, - [string[]]$Arguments = @(), - [string]$WorkingDirectory = "", - [ValidateSet("Hidden", "Normal", "Minimized")] - [string]$WindowStyle = "Hidden", - [switch]$Wait = $false, - [switch]$PassThru = $false, - [switch]$RunAs = $false - ) - $parts = @( - "-NoProfile", - "-WindowStyle", $WindowStyle, - "-ExecutionPolicy", "Bypass", - "-File", $File - ) + $Arguments - $sp = @{ - FilePath = "powershell.exe" - ArgumentList = (ConvertTo-NetxProcessArgumentString -Arguments $parts) - } - # -Verb RunAs implies UseShellExecute; cannot combine with -WindowStyle. - if (-not $RunAs) { - $sp.WindowStyle = $WindowStyle - } - if ($WorkingDirectory) { $sp.WorkingDirectory = $WorkingDirectory } - if ($PassThru -or $Wait) { $sp.PassThru = $true } - if ($RunAs) { $sp.Verb = "RunAs" } - $p = Start-Process @sp - if ($Wait -and $p) { Wait-NetxProcessExit -Process $p } - return $p -} - -function Wait-NetxProcessExit { - param([Parameter(Mandatory = $true)]$Process) - # Windows PowerShell 5.1 Start-Process -Wait also waits for every descendant; the - # postgres / NetX processes a start script leaves running would block it forever. - $null = $Process.Handle # keep a handle so ExitCode survives the exit - $Process.WaitForExit() -} - -function Test-NetxIsAdmin { - try { - $id = [Security.Principal.WindowsIdentity]::GetCurrent() - $p = New-Object Security.Principal.WindowsPrincipal($id) - return $p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) - } catch { - return $false - } -} - -function ConvertTo-NetxRelaunchArgumentList { - param([System.Collections.IDictionary]$BoundParameters) - $parts = [System.Collections.Generic.List[string]]::new() - if (-not $BoundParameters) { return @() } - foreach ($key in $BoundParameters.Keys) { - $val = $BoundParameters[$key] - if ($val -is [System.Management.Automation.SwitchParameter]) { - if ($val.IsPresent) { [void]$parts.Add("-$key") } - continue - } - if ($val -is [bool]) { - if ($val) { [void]$parts.Add("-$key") } - continue - } - if ($null -eq $val) { continue } - $s = [string]$val - if ($s -eq "") { continue } - [void]$parts.Add("-$key") - [void]$parts.Add($s) - } - return @($parts) -} - -function Assert-NetxAdminOrRelaunch { - <# - .SYNOPSIS - End-user packaged scripts must run elevated (Program Files + ProgramData). - If not admin, re-launch the same script via UAC and exit with child exit code. - #> - param( - [Parameter(Mandatory = $true)][string]$ScriptPath, - [System.Collections.IDictionary]$BoundParameters = @{}, - [string]$WorkingDirectory = "", - [ValidateSet("Hidden", "Normal", "Minimized")] - [string]$WindowStyle = "Normal" - ) - if (Test-NetxIsAdmin) { return } - - if ($env:NETX_ELEVATION_RELAUNCH -eq "1") { - throw "admin_required: still not elevated after UAC. Run NetX-Start / NetX-Tray as Administrator." - } - - if (-not (Test-Path -LiteralPath $ScriptPath)) { - throw "elevation_script_missing: $ScriptPath" - } - $wd = $WorkingDirectory - if (-not $wd) { $wd = Split-Path -Parent $ScriptPath } - - # Window style belongs on powershell.exe argv, not Start-Process (incompatible with -Verb RunAs). - $parts = @( - "-NoProfile", - "-WindowStyle", $WindowStyle, - "-ExecutionPolicy", "Bypass", - "-File", $ScriptPath - ) + (ConvertTo-NetxRelaunchArgumentList -BoundParameters $BoundParameters) - - $prevRel = $env:NETX_ELEVATION_RELAUNCH - $env:NETX_ELEVATION_RELAUNCH = "1" - try { - $p = Start-Process -FilePath "powershell.exe" ` - -ArgumentList (ConvertTo-NetxProcessArgumentString -Arguments $parts) ` - -WorkingDirectory $wd ` - -Verb RunAs -PassThru - if ($p) { Wait-NetxProcessExit -Process $p } - } catch { - if ($null -ne $prevRel) { $env:NETX_ELEVATION_RELAUNCH = $prevRel } else { Remove-Item Env:NETX_ELEVATION_RELAUNCH -ErrorAction SilentlyContinue } - throw ("admin_required: UAC cancelled or elevation failed. Run NetX as Administrator. {0}" -f $_.Exception.Message) - } - if ($null -ne $prevRel) { $env:NETX_ELEVATION_RELAUNCH = $prevRel } else { Remove-Item Env:NETX_ELEVATION_RELAUNCH -ErrorAction SilentlyContinue } - if ($null -eq $p) { - throw "admin_required: elevation failed (no process)" - } - $code = 0 - if ($null -ne $p.ExitCode) { $code = [int]$p.ExitCode } - exit $code -} - -function Get-NetxSystemPython { - # Prefer a real interpreter; skip the WindowsApps Store stub. - $candidates = @() - foreach ($cmd in @("python", "python3")) { - $c = Get-Command $cmd -ErrorAction SilentlyContinue - if ($c -and $c.Source -and ($c.Source -notmatch '\\WindowsApps\\')) { - $candidates += $c.Source - } - } - $pyLauncher = Get-Command "py" -ErrorAction SilentlyContinue - if ($pyLauncher -and $pyLauncher.Source -and ($pyLauncher.Source -notmatch '\\WindowsApps\\')) { - try { - $resolved = (& $pyLauncher.Source -3 -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() - if ($resolved -and (Test-Path -LiteralPath $resolved)) { - $candidates += $resolved - } - } catch {} - } - foreach ($p in @( - "$env:LOCALAPPDATA\Python\pythoncore-3.14-64\python.exe", - "$env:LOCALAPPDATA\Programs\Python\Python312\python.exe", - "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", - "$env:ProgramFiles\Python312\python.exe", - "$env:ProgramFiles\Python311\python.exe" - )) { - if ($p -and (Test-Path $p)) { $candidates += $p } - } - foreach ($p in ($candidates | Select-Object -Unique)) { - try { - $v = & $p -c "import sys; print('%d.%d'%sys.version_info[:2])" 2>$null - if ($v -match '^(3\.(1[1-9]|[2-9]\d))$') { return $p } - } catch {} - } - return $null -} - -function Get-NetxBundledPythonRoot { - param([Parameter(Mandatory = $true)][string]$ProgramRoot) - return Join-Path $ProgramRoot "python\runtime" -} - -function Repair-NetxShippedVenv { - param([Parameter(Mandatory = $true)][string]$ProgramRoot) - $cfgPath = Join-Path $ProgramRoot ".venv\pyvenv.cfg" - $rtRoot = Get-NetxBundledPythonRoot -ProgramRoot $ProgramRoot - $rtPy = Join-Path $rtRoot "python.exe" - if (-not (Test-Path -LiteralPath $rtPy)) { return $false } - if (-not (Test-Path -LiteralPath $cfgPath)) { return $false } - - $rtRootAbs = (Resolve-Path -LiteralPath $rtRoot).Path - $rtPyAbs = (Resolve-Path -LiteralPath $rtPy).Path - $ver = "3.14.0" - try { - $verOut = & $rtPyAbs -c "import sys; print('%d.%d.%d' % sys.version_info[:3])" 2>$null - if ($verOut) { $ver = ($verOut | Out-String).Trim() } - } catch {} - - $lines = @( - "home = $rtRootAbs", - "include-system-site-packages = false", - "version = $ver", - "executable = $rtPyAbs" - ) - $desired = ($lines -join "`r`n") - # Skip write when already correct — Users cannot modify Program Files after Setup. - try { - $cur = ([IO.File]::ReadAllText($cfgPath) -replace "`r`n", "`n" -replace "`r", "`n").Trim() - $want = ($desired -replace "`r`n", "`n" -replace "`r", "`n").Trim() - if ($cur -eq $want) { - return $true - } - } catch {} - - try { - Set-Content -LiteralPath $cfgPath -Value $desired -Encoding ascii -ErrorAction Stop - return $true - } catch { - # Non-elevated tray/start: leave cfg as-is; caller may still run if paths happen to work. - return $false - } -} - -function Test-NetxVenvRunnable { - param([Parameter(Mandatory = $true)][string]$VenvPython) - if (-not (Test-Path -LiteralPath $VenvPython)) { return $false } - - $outFile = Join-Path $env:TEMP ("netx-venv-out-" + [Guid]::NewGuid().ToString("n") + ".txt") - $errFile = Join-Path $env:TEMP ("netx-venv-err-" + [Guid]::NewGuid().ToString("n") + ".txt") - try { - $psi = New-Object System.Diagnostics.ProcessStartInfo - $psi.FileName = $VenvPython - $psi.Arguments = '-c "import encodings, sys; sys.exit(0 if sys.prefix else 1)"' - $psi.UseShellExecute = $false - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - $psi.CreateNoWindow = $true - $p = [Diagnostics.Process]::Start($psi) - $stderr = $p.StandardError.ReadToEnd() - $null = $p.StandardOutput.ReadToEnd() - $p.WaitForExit() - if ($stderr -match 'did not find executable|No Python at|Fatal Python error') { - return $false - } - if ($p.ExitCode -ne 0) { return $false } - return $true - } finally { - Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue - } -} - -function Ensure-NetxVenv { - param( - [Parameter(Mandatory = $true)][string]$ProgramRoot, - [switch]$ForcePip = $false - ) - $venvPy = Join-Path $ProgramRoot ".venv\Scripts\python.exe" - $req = Join-Path $ProgramRoot "requirements.txt" - $rtPy = Join-Path (Get-NetxBundledPythonRoot -ProgramRoot $ProgramRoot) "python.exe" - - if (Test-Path -LiteralPath $rtPy) { - try { - $null = Repair-NetxShippedVenv -ProgramRoot $ProgramRoot - } catch { - # Access denied under Program Files when not elevated — ignore if venv already runs. - } - } - - if ((Test-Path -LiteralPath $venvPy) -and -not $ForcePip) { - if (Test-NetxVenvRunnable -VenvPython $venvPy) { - return $venvPy - } - Write-Host "[WARN] Shipped .venv exists but Python cannot start (broken pyvenv.cfg or missing runtime)." -ForegroundColor Yellow - if (Test-Path -LiteralPath $rtPy) { - $repaired = $false - try { - $repaired = [bool](Repair-NetxShippedVenv -ProgramRoot $ProgramRoot) - } catch { - $repaired = $false - } - if ($repaired -and (Test-NetxVenvRunnable -VenvPython $venvPy)) { - Write-Host "==> Repaired .venv to use bundled python/runtime" -ForegroundColor Green - return $venvPy - } - if (-not $repaired) { - throw "venv_needs_admin_repair: pyvenv.cfg still points at the build PC. Re-run Setup (Update) as Administrator, or Start Menu → Reconfigure database once elevated." - } - } - if (-not $ForcePip) { - throw "venv_not_runnable: reinstall NetX Setup (ships python/runtime + .venv) or run repair as admin" - } - } - if (-not (Test-Path $venvPy)) { - $py = Get-NetxSystemPython - if (-not $py) { - throw "python_not_found: install Python 3.11+ (or ship Setup built with -CreateVenv)" - } - Write-Host "==> Creating .venv with $py" - $venvDir = Join-Path $ProgramRoot ".venv" - try { - & $py -m venv $venvDir - } catch { - throw "venv_create_failed: cannot write $venvDir (need admin / ship -CreateVenv). $($_.Exception.Message)" - } - if (-not (Test-Path $venvPy)) { - throw "venv_create_failed: missing $venvPy (Program Files may be read-only without elevation)" - } - $ForcePip = $true - } - if ($ForcePip) { - if (-not (Test-Path $req)) { throw "requirements_missing: $req" } - Write-Host "==> pip install -r requirements.txt" - & $venvPy -m pip install --upgrade pip - & $venvPy -m pip install -r $req - if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } - } - return $venvPy -} - -function Stop-NetxTrayProcesses { - param([string]$ProgramRoot = "") - $hits = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { - $_.CommandLine -and $_.CommandLine -match 'netx_tray\.ps1' - }) - if ($ProgramRoot) { - $esc = [regex]::Escape($ProgramRoot) - $hits = @($hits | Where-Object { $_.CommandLine -match $esc -or $_.CommandLine -match 'netx_tray\.ps1' }) - } - foreach ($p in $hits) { - try { - Stop-Process -Id $p.ProcessId -Force -ErrorAction Stop - Write-Host "Stopped NetX tray PID=$($p.ProcessId)" - } catch {} - } -} - -function ConvertTo-NetxSqlLiteral([string]$Value) { - # Single-quote escaping for PostgreSQL string literals. - return ($Value -replace "'", "''") -} diff --git a/packaging/assets/README.md b/packaging/assets/README.md deleted file mode 100644 index 27b599c..0000000 --- a/packaging/assets/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# NetX packaging assets - -- `netx.ico` 鈥?installer / shortcuts / tray (regenerate with `python gen_icon.py`) -- `netx-256.png` / `netx-64.png` 鈥?previews - -Mark matches `web/public/favicon.svg` (network 鈥淣鈥?nodes on navy). diff --git a/packaging/assets/gen_icon.py b/packaging/assets/gen_icon.py deleted file mode 100644 index e5b96d5..0000000 --- a/packaging/assets/gen_icon.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Generate NetX Windows .ico/.png from the brand mark. - -Mirrors web/public/favicon.svg and web/src/layout/BrandMark.tsx (24x24 viewBox): -blue gradient tile, an "X" of links, four nodes around a hub, one node lit cyan. -""" -from __future__ import annotations - -import os -from PIL import Image, ImageDraw - -OUT_DIR = os.path.dirname(os.path.abspath(__file__)) -GRAD_A = (0x3B, 0x82, 0xF6) # #3b82f6 (top-left) -GRAD_B = (0x1D, 0x4E, 0xD8) # #1d4ed8 (bottom-right) -WHITE = (255, 255, 255, 255) -LINK = (255, 255, 255, int(255 * 0.55)) -LIVE = (0x67, 0xE8, 0xF9, 255) # #67e8f9 -SS = 4 # supersample factor - - -def _gradient(px: int) -> Image.Image: - # Diagonal gradient along (0,0)->(24,24), like the SVG userSpaceOnUse gradient. - img = Image.new("RGBA", (px, px)) - pix = img.load() - span = 2 * (px - 1) or 1 - for y in range(px): - for x in range(px): - t = (x + y) / span - pix[x, y] = tuple(round(a + (b - a) * t) for a, b in zip(GRAD_A, GRAD_B)) + (255,) - return img - - -def make(size: int) -> Image.Image: - px = size * SS - u = px / 24.0 - - tile = _gradient(px) - mask = Image.new("L", (px, px), 0) - ImageDraw.Draw(mask).rounded_rectangle([0, 0, px - 1, px - 1], radius=6.5 * u, fill=255) - img = Image.new("RGBA", (px, px), (0, 0, 0, 0)) - img.paste(tile, (0, 0), mask) - - links = Image.new("RGBA", (px, px), (0, 0, 0, 0)) - ld = ImageDraw.Draw(links) - w = max(1, round(1.6 * u)) - for (x1, y1), (x2, y2) in (((7, 7), (17, 17)), ((17, 7), (7, 17))): - ld.line([(x1 * u, y1 * u), (x2 * u, y2 * u)], fill=LINK, width=w) - for cx, cy in ((x1, y1), (x2, y2)): # round caps - r = w / 2 - ld.ellipse([cx * u - r, cy * u - r, cx * u + r, cy * u + r], fill=LINK) - img = Image.alpha_composite(img, links) - - d = ImageDraw.Draw(img) - - def node(cx: float, cy: float, r: float, fill: tuple[int, int, int, int]) -> None: - d.ellipse([(cx - r) * u, (cy - r) * u, (cx + r) * u, (cy + r) * u], fill=fill) - - node(12, 12, 2.6, WHITE) - node(7, 7, 1.9, WHITE) - node(7, 17, 1.9, WHITE) - node(17, 17, 1.9, WHITE) - node(17, 7, 2.1, LIVE) - return img.resize((size, size), Image.LANCZOS) - - -WIZ_TOP = (0x0A, 0x12, 0x22) # near-black navy -WIZ_BOTTOM = (0x10, 0x2A, 0x55) # deep blue - - -def make_wizard(w: int, h: int) -> Image.Image: - """Inno Setup side image: dark navy gradient, faint link mesh, brand mark.""" - img = Image.new("RGBA", (w, h)) - pix = img.load() - for y in range(h): - t = y / max(1, h - 1) - row = tuple(round(a + (b - a) * t) for a, b in zip(WIZ_TOP, WIZ_BOTTOM)) + (255,) - for x in range(w): - pix[x, y] = row - - mesh = Image.new("RGBA", (w * SS, h * SS), (0, 0, 0, 0)) - md = ImageDraw.Draw(mesh) - pts = [(0.12, 0.62), (0.42, 0.70), (0.80, 0.60), (0.28, 0.86), (0.66, 0.90), (0.95, 0.80), (0.05, 0.95)] - pts = [(x * w * SS, y * h * SS) for x, y in pts] - line = (0x60, 0xA5, 0xFA, 46) - for a, b in ((0, 1), (1, 2), (0, 3), (1, 3), (1, 4), (2, 4), (2, 5), (4, 5), (3, 6)): - md.line([pts[a], pts[b]], fill=line, width=max(1, SS)) - for i, (x, y) in enumerate(pts): - r = (2.4 if i != 2 else 3.0) * SS * w / 164 - fill = (0x67, 0xE8, 0xF9, 150) if i == 2 else (0xBF, 0xDB, 0xFE, 90) - md.ellipse([x - r, y - r, x + r, y + r], fill=fill) - img = Image.alpha_composite(img, mesh.resize((w, h), Image.LANCZOS)) - - side = round(w * 0.42) - mark = make(side) - img.alpha_composite(mark, ((w - side) // 2, round(h * 0.26))) - return img.convert("RGB") - - -def main() -> None: - sizes = [16, 20, 24, 32, 40, 48, 64, 128, 256] - images = {s: make(s) for s in sizes} - ico_path = os.path.join(OUT_DIR, "netx.ico") - images[256].save(ico_path, format="ICO", sizes=[(s, s) for s in sizes], bitmap_format="bmp", - append_images=[images[s] for s in sizes[:-1]]) - images[256].save(os.path.join(OUT_DIR, "netx-256.png")) - images[64].save(os.path.join(OUT_DIR, "netx-64.png")) - # Inno Setup wizard images at 100/125/150/200% DPI (modern style base sizes). - for pct in (100, 125, 150, 200): - k = pct / 100 - make_wizard(round(164 * k), round(314 * k)).save(os.path.join(OUT_DIR, f"wizard-{pct}.png")) - make(round(55 * k)).save(os.path.join(OUT_DIR, f"wizard-small-{pct}.png")) - print(f"wrote {ico_path} ({os.path.getsize(ico_path)} bytes)") - - -if __name__ == "__main__": - main() diff --git a/packaging/assets/netx-256.png b/packaging/assets/netx-256.png deleted file mode 100644 index 073acea..0000000 Binary files a/packaging/assets/netx-256.png and /dev/null differ diff --git a/packaging/assets/netx-64.png b/packaging/assets/netx-64.png deleted file mode 100644 index 0bbcfd1..0000000 Binary files a/packaging/assets/netx-64.png and /dev/null differ diff --git a/packaging/assets/netx.ico b/packaging/assets/netx.ico deleted file mode 100644 index 295a96d..0000000 Binary files a/packaging/assets/netx.ico and /dev/null differ diff --git a/packaging/assets/wizard-100.png b/packaging/assets/wizard-100.png deleted file mode 100644 index 040de08..0000000 Binary files a/packaging/assets/wizard-100.png and /dev/null differ diff --git a/packaging/assets/wizard-125.png b/packaging/assets/wizard-125.png deleted file mode 100644 index f4e211f..0000000 Binary files a/packaging/assets/wizard-125.png and /dev/null differ diff --git a/packaging/assets/wizard-150.png b/packaging/assets/wizard-150.png deleted file mode 100644 index 94cc820..0000000 Binary files a/packaging/assets/wizard-150.png and /dev/null differ diff --git a/packaging/assets/wizard-200.png b/packaging/assets/wizard-200.png deleted file mode 100644 index a2da6b3..0000000 Binary files a/packaging/assets/wizard-200.png and /dev/null differ diff --git a/packaging/assets/wizard-small-100.png b/packaging/assets/wizard-small-100.png deleted file mode 100644 index 65ba4ea..0000000 Binary files a/packaging/assets/wizard-small-100.png and /dev/null differ diff --git a/packaging/assets/wizard-small-125.png b/packaging/assets/wizard-small-125.png deleted file mode 100644 index a67a6a2..0000000 Binary files a/packaging/assets/wizard-small-125.png and /dev/null differ diff --git a/packaging/assets/wizard-small-150.png b/packaging/assets/wizard-small-150.png deleted file mode 100644 index 531f9a1..0000000 Binary files a/packaging/assets/wizard-small-150.png and /dev/null differ diff --git a/packaging/assets/wizard-small-200.png b/packaging/assets/wizard-small-200.png deleted file mode 100644 index eb82868..0000000 Binary files a/packaging/assets/wizard-small-200.png and /dev/null differ diff --git a/packaging/build_release.ps1 b/packaging/build_release.ps1 index 443f5f4..a347f03 100644 --- a/packaging/build_release.ps1 +++ b/packaging/build_release.ps1 @@ -1,14 +1,10 @@ -param( +param( [string]$Version = "", [string]$OutDir = "", [switch]$SkipWebBuild = $false, [switch]$SkipPostgresDownload = $false, - # Zip is no longer published; stage + Setup.exe only. Opt-in for local/dev testing. - [switch]$CreateZip = $false, - # Deprecated no-op (zip is off by default). Kept so older scripts that pass -SkipZip still run. [switch]$SkipZip = $false, - # Default on: offline Setup must ship .venv so target PCs never pip-install. - [switch]$CreateVenv = $true + [switch]$CreateVenv = $false ) $ErrorActionPreference = "Stop" @@ -36,25 +32,13 @@ if (-not $SkipWebBuild) { throw "npm_not_found" } Write-Host "==> npm build" - Push-Location $webRoot - try { - # npm ci wipes node_modules first; on a dev box a running Vite locks native - # .node files and leaves it half-deleted. Skip when installed deps match the lockfile. - $installedLock = Join-Path $webRoot "node_modules\.package-lock.json" - $lock = Join-Path $webRoot "package-lock.json" - if ((Test-Path $installedLock) -and - (Get-Item $installedLock).LastWriteTimeUtc -ge (Get-Item $lock).LastWriteTimeUtc) { - Write-Host "==> node_modules up to date with package-lock.json - skip npm ci" - } else { - & npm.cmd ci - if ($LASTEXITCODE -ne 0) { throw "npm_install_failed" } - } - # Release packaging skips tsc (dev CI still uses `npm run build`). - & npm.cmd run build:release - if ($LASTEXITCODE -ne 0) { throw "web_build_failed" } - } finally { - Pop-Location + if (-not (Test-Path (Join-Path $webRoot "node_modules"))) { + & npm.cmd install --prefix $webRoot + if ($LASTEXITCODE -ne 0) { throw "npm_install_failed" } } + # Release packaging skips tsc (dev CI still uses `npm run build`). + & npm.cmd run build:release --prefix $webRoot + if ($LASTEXITCODE -ne 0) { throw "web_build_failed" } } if (-not (Test-Path (Join-Path $dist "index.html"))) { throw "web_dist_missing: build web/ or drop -SkipWebBuild" @@ -67,22 +51,6 @@ if (-not $SkipPostgresDownload) { } } -# Bundle WinSW so offline installs can register a Windows Service without GitHub. -# .NET Framework 4.6.1 build (~0.6MB; in-box on Win10 1607+ / Server 2016+) instead of -# the self-contained x64 build (~18MB). -$winswCache = Join-Path $PSScriptRoot "cache\WinSW.NET461.exe" -$winswShip = Join-Path $PSScriptRoot "winsw\WinSW.exe" -if (-not (Test-Path $winswShip)) { - if (-not (Test-Path $winswCache)) { - $winswUrl = "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW.NET461.exe" - Write-Host "==> Downloading WinSW for offline bundle: $winswUrl" - New-Item -ItemType Directory -Path (Split-Path $winswCache -Parent) -Force | Out-Null - Invoke-WebRequest -Uri $winswUrl -OutFile $winswCache -UseBasicParsing - } - New-Item -ItemType Directory -Path (Split-Path $winswShip -Parent) -Force | Out-Null - Copy-Item -Path $winswCache -Destination $winswShip -Force -} - # Flat layout = same as repo so scripts\start_netx.ps1 works unchanged. foreach ($d in @("netx_api", "alembic", "scripts")) { Copy-Item -Path (Join-Path $repo $d) -Destination (Join-Path $stage $d) -Recurse -Force @@ -98,90 +66,26 @@ $webOut = Join-Path $stage "web\dist" New-Item -ItemType Directory -Path (Split-Path $webOut -Parent) -Force | Out-Null Copy-Item -Path $dist -Destination $webOut -Recurse -Force -function Write-Utf8BomFile { - param([string]$Path) - # Windows PowerShell 4/5 (Server 2012+) mis-parses UTF-8 without BOM when scripts contain CJK. - $utf8Bom = New-Object System.Text.UTF8Encoding $true - $bytes = [IO.File]::ReadAllBytes($Path) - $hasBom = ($bytes.Length -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) - $text = if ($hasBom) { - [Text.Encoding]::UTF8.GetString($bytes, 3, $bytes.Length - 3) - } else { - [Text.Encoding]::UTF8.GetString($bytes) - } - $text = $text -replace "`r`n", "`n" -replace "`n", "`r`n" - if (-not $text.EndsWith("`r`n")) { $text += "`r`n" } - [IO.File]::WriteAllText($Path, $text, $utf8Bom) -} - $packOut = Join-Path $stage "packaging" New-Item -ItemType Directory -Path $packOut -Force | Out-Null Copy-Item -Path (Join-Path $PSScriptRoot "_common.ps1") -Destination $packOut -Force foreach ($name in @( "download_postgres.ps1", "setup_first_run.ps1", "start_netx_app.ps1", - "stop_netx_app.ps1", "update_netx.ps1", "check_update.ps1", - "launch_shortcut.ps1", "netx_tray.ps1", "install_autostart.ps1", "install_service.ps1", - "service_run.ps1", "service_lifecycle.ps1", "install_update_task.ps1", "repair_venv.ps1", "runtime_ctl.ps1", "set_pg_port.ps1", "uninstall_prepare.ps1", "uninstall_delete_data.ps1", "sign_release.ps1", - "build_release.ps1", "publish_release.ps1", "publish_forgejo_release.ps1", "README.md", "manifest.example.json" + "stop_netx_app.ps1", "update_netx.ps1", "build_release.ps1", + "README.md", "manifest.example.json" )) { $src = Join-Path $PSScriptRoot $name if (Test-Path $src) { Copy-Item $src (Join-Path $packOut $name) -Force } } -Get-ChildItem -Path $packOut -Filter *.ps1 -File | ForEach-Object { Write-Utf8BomFile -Path $_.FullName } -Get-ChildItem -Path (Join-Path $stage "scripts") -Filter *.ps1 -File -ErrorAction SilentlyContinue | - ForEach-Object { Write-Utf8BomFile -Path $_.FullName } Copy-Item -Path (Join-Path $PSScriptRoot "config") -Destination (Join-Path $packOut "config") -Recurse -Force Copy-Item -Path (Join-Path $PSScriptRoot "installer") -Destination (Join-Path $packOut "installer") -Recurse -Force -$assetsSrc = Join-Path $PSScriptRoot "assets" -if (Test-Path $assetsSrc) { - Copy-Item -Path $assetsSrc -Destination (Join-Path $packOut "assets") -Recurse -Force -} -$winswOut = Join-Path $packOut "winsw" -New-Item -ItemType Directory -Path $winswOut -Force | Out-Null -if (Test-Path $winswShip) { - Copy-Item -Path $winswShip -Destination (Join-Path $winswOut "WinSW.exe") -Force - Write-Host "==> Bundled WinSW for offline service install" -} else { - Write-Host "[WARN] WinSW missing - offline service install will fail" -ForegroundColor Yellow -} New-Item -ItemType Directory -Path (Join-Path $packOut "postgres") -Force | Out-Null Copy-Item -Path (Join-Path $PSScriptRoot "postgres\README.md") -Destination (Join-Path $packOut "postgres\README.md") -Force if (Test-Path (Join-Path $pgsql "bin\pg_ctl.exe")) { - Write-Host '==> Copying bundled PostgreSQL (bin/lib/share; skip doc/)' - $pgStage = Join-Path $stage "postgres\pgsql" - New-Item -ItemType Directory -Path $pgStage -Force | Out-Null - foreach ($sub in @("bin", "lib", "share")) { - $srcSub = Join-Path $pgsql $sub - if (Test-Path $srcSub) { - Copy-Item -Path $srcSub -Destination (Join-Path $pgStage $sub) -Recurse -Force - } - } - # Runtime-only subset: NetX uses postgres/pg_ctl/initdb/psql/pg_dump (+ restore/isready - # for manual ops). Drop StackBuilder (wx GUI), dev/test tools, static libs, pgxs and - # non-Chinese message catalogs. - $pgBinKeep = @("postgres", "pg_ctl", "initdb", "psql", "pg_dump", "pg_dumpall", "pg_restore", "pg_isready") - Get-ChildItem (Join-Path $pgStage "bin") -File | Where-Object { - ($_.Extension -eq ".exe" -and $pgBinKeep -notcontains $_.BaseName) -or $_.Name -like "wx*.dll" - } | Remove-Item -Force - Get-ChildItem (Join-Path $pgStage "lib") -File -Filter *.lib | Remove-Item -Force - Remove-Item -Recurse -Force (Join-Path $pgStage "lib\pgxs") -ErrorAction SilentlyContinue - Get-ChildItem (Join-Path $pgStage "share\locale") -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.Name -notlike "zh_CN*" } | Remove-Item -Recurse -Force - # The EDB binaries link the VC++ 2015-2022 runtime (vcruntime140, msvcp140 via ICU), which a - # clean Windows install does not have. Deploy it app-local (allowed by the VC++ redist terms) - # so the bundled PostgreSQL starts on fresh / offline machines without the redistributable. - $sys32 = Join-Path $env:SystemRoot "System32" - foreach ($dll in @("vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll")) { - $src = Join-Path $sys32 $dll - if (-not (Test-Path -LiteralPath $src)) { throw "vc_runtime_missing_on_build_machine: $src (install the VC++ 2015-2022 x64 redistributable)" } - Copy-Item -LiteralPath $src -Destination (Join-Path $pgStage "bin\$dll") -Force - } - Get-ChildItem -Path $pgsql -File -ErrorAction SilentlyContinue | ForEach-Object { - Copy-Item -Path $_.FullName -Destination (Join-Path $pgStage $_.Name) -Force - } -} else { - throw "bundled_postgres_missing: Setup.exe must ship postgres for offline install. Run download_postgres.ps1 (build machine only)." + Write-Host "==> Copying bundled PostgreSQL" + New-Item -ItemType Directory -Path (Join-Path $stage "postgres") -Force | Out-Null + Copy-Item -Path $pgsql -Destination (Join-Path $stage "postgres\pgsql") -Recurse -Force } $builtAt = (Get-Date).ToUniversalTime().ToString("o") @@ -195,122 +99,26 @@ $builtAt = (Get-Date).ToUniversalTime().ToString("o") Set-Content -Path (Join-Path $stage ".portable") -Value "1" -Encoding ascii -# Root .cmd launchers (Explorer / Start Menu friendly; ASCII-only). -$cmdSrc = Join-Path $PSScriptRoot "cmd" -foreach ($cmdName in @("NetX-FirstRun.cmd", "NetX-Start.cmd", "NetX-Tray.cmd", "NetX-Stop.cmd")) { - $c = Join-Path $cmdSrc $cmdName - if (Test-Path $c) { - Copy-Item -Path $c -Destination (Join-Path $stage $cmdName) -Force - } -} - if ($CreateVenv) { - Write-Host "==> Creating portable python/runtime + .venv (must not reference build-machine paths)" - # Releases pin 3.12 (same as CI); prefer it even when another version is first on PATH. - $pyPath = $null - $pyLauncher = Get-Command "py" -ErrorAction SilentlyContinue - if ($pyLauncher -and $pyLauncher.Source -notmatch '\\WindowsApps\\') { - try { - $resolved = (& $pyLauncher.Source -3.12 -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() - if ($resolved -and (Test-Path -LiteralPath $resolved)) { $pyPath = $resolved } - } catch {} - } - if (-not $pyPath) { - foreach ($p in @("$env:LOCALAPPDATA\Programs\Python\Python312\python.exe", "$env:ProgramFiles\Python312\python.exe")) { - if (Test-Path -LiteralPath $p) { $pyPath = $p; break } - } - } - if (-not $pyPath) { $pyPath = Get-NetxSystemPython } - if (-not $pyPath) { throw "python_not_found_for_venv: need Python 3.12.x (not WindowsApps stub)" } - $pyVersion = (& $pyPath -c "import sys; print('%d.%d' % sys.version_info[:2])" 2>$null | Out-String).Trim() - if ($pyVersion -ne '3.12') { throw "python_version_mismatch: Windows releases require Python 3.12.x for reproducible runtimes; found $pyVersion at $pyPath" } - Write-Host " Build Python: $pyPath" - - $prefix = (& $pyPath -c "import sys; print(sys.base_prefix)" 2>$null | Out-String).Trim() - if (-not $prefix -or -not (Test-Path -LiteralPath $prefix)) { - throw "python_base_prefix_not_found: $prefix" - } - $rtDir = Join-Path $stage "python\runtime" - if (Test-Path $rtDir) { Remove-Item -Recurse -Force $rtDir } - New-Item -ItemType Directory -Path $rtDir -Force | Out-Null - - Write-Host "==> Copying Python stdlib/runtime to $rtDir (exclude base site-packages)" - $spEx = Join-Path $prefix "Lib\site-packages" - $robocopy = Join-Path $env:SystemRoot "System32\robocopy.exe" - if (-not (Test-Path -LiteralPath $robocopy)) { throw "robocopy_not_found" } - # Headless server: drop docs, the stdlib test suite and Tk/IDLE (~100MB unpacked). - # Keep ensurepip - venv creation needs it. Same result on CI and local builds. - $xdRuntime = @("Doc", "tcl", "Tools", "Lib\test", "Lib\idlelib", "Lib\tkinter", "Lib\turtledemo") | - ForEach-Object { Join-Path $prefix $_ } - # python-*.exe: the GitHub runner's toolcache keeps the full Python installer (~25MB) - # in the prefix; _test*.pyd: stdlib test extensions. - $xfRuntime = @("_tkinter.pyd", "tcl86t.dll", "tk86t.dll", "*.pyc", "python-*.exe", "_test*.pyd", "_ctypes_test.pyd", "xxlimited*.pyd") - & $robocopy $prefix $rtDir /E /XD $spEx __pycache__ @xdRuntime /XF @xfRuntime /NFL /NDL /NJH /NJS /nc /ns /np | Out-Null - if ($LASTEXITCODE -ge 8) { throw "robocopy_python_runtime_failed: exit $LASTEXITCODE" } - - $rtPy = Join-Path $rtDir "python.exe" - if (-not (Test-Path -LiteralPath $rtPy)) { throw "python_runtime_missing: $rtPy" } - - $venvDir = Join-Path $stage ".venv" - if (Test-Path $venvDir) { Remove-Item -Recurse -Force $venvDir } - Write-Host '==> Creating .venv from shipped runtime (--copies)' - & $rtPy -m venv $venvDir --copies - if ($LASTEXITCODE -ne 0) { throw "venv_create_failed" } - - $venvPy = Join-Path $venvDir "Scripts\python.exe" - $null = Repair-NetxShippedVenv -ProgramRoot $stage - if (-not (Test-NetxVenvRunnable -VenvPython $venvPy)) { - throw "venv_not_runnable_after_build: check python/runtime copy" - } - & $venvPy -c "from pathlib import Path; import glob; print(Path.cwd()); print(glob.__file__)" - if ($LASTEXITCODE -ne 0) { throw "python_stdlib_smoke_failed: pathlib/glob import mismatch" } - - # --no-warn-script-location: the PATH check realpath()s every entry and aborts on a - # broken reparse point (e.g. a stale app folder on a detached volume). Irrelevant here. - & $venvPy -m pip install --upgrade pip --no-warn-script-location - & $venvPy -m pip install -r (Join-Path $stage "requirements.txt") --no-warn-script-location - if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } - - $null = Repair-NetxShippedVenv -ProgramRoot $stage - if (-not (Test-NetxVenvRunnable -VenvPython $venvPy)) { - throw "venv_not_runnable_after_pip" - } - & $venvPy -c "from pathlib import Path; import glob; print(Path.cwd()); print(glob.__file__)" - if ($LASTEXITCODE -ne 0) { throw "python_stdlib_smoke_failed_after_pip: pathlib/glob import mismatch" } - - Write-Host "==> Slimming .venv (drop tests / __pycache__ / *.pyc)" - $site = Join-Path $stage ".venv\Lib\site-packages" - if (Test-Path $site) { - Get-ChildItem -Path $site -Recurse -Directory -Force -ErrorAction SilentlyContinue | - Where-Object { - $_.Name -in @('__pycache__', 'tests', 'test', 'testing', 'Tests') -or - $_.FullName -match '\\pandas\\tests(\\|$)' -or - $_.FullName -match '\\numpy\\(_*tests|tests)(\\|$)' -or - $_.FullName -match '\\scipy\\(_*tests|tests)(\\|$)' - } | - Sort-Object { $_.FullName.Length } -Descending | - ForEach-Object { - Remove-Item -LiteralPath $_.FullName -Recurse -Force -ErrorAction SilentlyContinue - } - Get-ChildItem -Path $site -Recurse -Include *.pyc,*.pyo -Force -ErrorAction SilentlyContinue | - Remove-Item -Force -ErrorAction SilentlyContinue - } + Write-Host "==> Creating .venv in stage (requires Python 3.11+ on PATH)" + $py = (Get-Command python -ErrorAction SilentlyContinue) + if (-not $py) { throw "python_not_found_for_venv" } + & $py.Source -m venv (Join-Path $stage ".venv") + & (Join-Path $stage ".venv\Scripts\python.exe") -m pip install --upgrade pip + & (Join-Path $stage ".venv\Scripts\python.exe") -m pip install -r (Join-Path $stage "requirements.txt") } Write-Host "==> Stage ready: $stage" -ForegroundColor Green -if ($SkipZip -and $CreateZip) { - Write-Host "[WARN] -SkipZip ignored because -CreateZip was set" -ForegroundColor Yellow -} -if ($CreateZip) { +if (-not $SkipZip) { $zip = Join-Path (Split-Path -Parent $stage) "NetX-$Version-win64.zip" if (Test-Path $zip) { Remove-Item -Force $zip } Compress-Archive -Path $stage -DestinationPath $zip -Force - Write-Host "==> Zip (optional/dev): $zip" -ForegroundColor Yellow + Write-Host "==> Zip: $zip" -ForegroundColor Green } Write-Host "" -Write-Host "Ship:" +Write-Host "Ship options:" +Write-Host " Zip: user unpacks, runs packaging\setup_first_run.ps1 then start_netx_app.ps1" Write-Host " Exe: compile packaging\installer\netx.iss with Inno Setup (needs stage above)" -Write-Host " (Zip packages are not published; use -CreateZip only for local testing.)" -Write-Host "Python: bundled python/runtime + .venv; target PCs do not need Python installed" +Write-Host "Python: install 3.11+ on target, or rebuild with -CreateVenv and ship .venv" diff --git a/packaging/check_update.ps1 b/packaging/check_update.ps1 deleted file mode 100644 index c8ecae8..0000000 --- a/packaging/check_update.ps1 +++ /dev/null @@ -1,501 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [string]$UpdateUrl = "", - [string]$FallbackUrl = "", - [string]$ForgejoUrl = "", - [string]$GithubRepo = "", - [string]$Channel = "", - [string]$Sources = "", - [switch]$Apply = $false, - [switch]$Quiet = $false, - # Only apply when NETX_UPDATE_AUTO=true (scheduled silent updates). - [switch]$AutoOnly = $false -) - -# Check for a newer NetX Windows package. -# -# Default (no config needed): -# GitHub primary + Forgejo mirror fallback (git.avelo.top). -# Probe every reachable source and pick the highest version; -# on equal versions prefer GitHub (listed first). -# -# Optional overrides: -# NETX_UPDATE_SOURCES=github,forgejo -# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest -# NETX_UPDATE_GITHUB_REPO=hansjone/netx -# NETX_UPDATE_URL=... manifest JSON -# NETX_UPDATE_TOKEN=... private API token - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters ` - -WindowStyle $(if ($Quiet) { "Hidden" } else { "Normal" }) - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$envPath = Join-Path $data ".env" -$map = @{} -if (Test-Path $envPath) { $map = Read-DotEnv -Path $envPath } - -function Get-Cfg([string]$Key, [string]$ParamVal = "") { - if ($ParamVal) { return $ParamVal } - if ($map.ContainsKey($Key) -and $map[$Key]) { return [string]$map[$Key] } - $envVal = [Environment]::GetEnvironmentVariable($Key) - if ($envVal) { return $envVal } - return "" -} - -$UpdateUrl = Get-Cfg "NETX_UPDATE_URL" $UpdateUrl -$FallbackUrl = Get-Cfg "NETX_UPDATE_FALLBACK_URL" $FallbackUrl -$ForgejoUrl = Get-Cfg "NETX_UPDATE_FORGEJO_URL" $ForgejoUrl -if (-not $ForgejoUrl) { - $ForgejoUrl = "https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest" -} -$GithubRepo = Get-Cfg "NETX_UPDATE_GITHUB_REPO" $GithubRepo -if (-not $GithubRepo) { $GithubRepo = "hansjone/netx" } -$Channel = Get-Cfg "NETX_UPDATE_CHANNEL" $Channel -if (-not $Channel) { $Channel = "stable" } -$Sources = Get-Cfg "NETX_UPDATE_SOURCES" $Sources -$UpdateToken = Get-Cfg "NETX_UPDATE_TOKEN" "" - -$current = Get-NetxVersion -ProgramRoot $prog - -function Compare-SemVer { - param([string]$A, [string]$B) - $pa = @($A.TrimStart('v', 'V').Split('.') | ForEach-Object { try { [int]$_ } catch { 0 } }) - $pb = @($B.TrimStart('v', 'V').Split('.') | ForEach-Object { try { [int]$_ } catch { 0 } }) - while ($pa.Count -lt 3) { $pa += 0 } - while ($pb.Count -lt 3) { $pb += 0 } - for ($i = 0; $i -lt 3; $i++) { - if ($pa[$i] -lt $pb[$i]) { return -1 } - if ($pa[$i] -gt $pb[$i]) { return 1 } - } - return 0 -} - -function Get-AuthHeaders { - param([string]$Style = "bearer") - $h = @{ "User-Agent" = "NetX-UpdateCheck" } - if (-not $UpdateToken) { return $h } - if ($Style -eq "token") { - $h["Authorization"] = "token $UpdateToken" - } else { - $h["Authorization"] = "Bearer $UpdateToken" - } - return $h -} - -function Resolve-AssetUrl { - param($Asset, $Rel, [string]$SourceName, [string]$ApiLatestUrl = "") - $url = [string]$Asset.browser_download_url - if ($url) { return $url } - # Forgejo/Gitea sometimes omit browser_download_url; synthesize release download URL. - if ($SourceName -eq "forgejo" -and $ApiLatestUrl -and $Rel.tag_name -and $Asset.name) { - if ($ApiLatestUrl -match '^(https?://[^/]+)/api/v1/repos/([^/]+)/([^/]+)/releases/latest') { - $base = $Matches[1] - $owner = $Matches[2] - $repo = $Matches[3] - $tag = [string]$Rel.tag_name - $name = [uri]::EscapeDataString([string]$Asset.name).Replace('%2F', '/') - # EscapeDataString encodes too much; use raw name (assets shouldn't need query encoding). - $name = [string]$Asset.name - return "$base/$owner/$repo/releases/download/$tag/$name" - } - } - return "" -} - -function Convert-ReleaseToManifest { - param($Rel, [string]$SourceName, [string]$ApiLatestUrl = "") - $tag = [string]$Rel.tag_name - $ver = $tag.TrimStart('v', 'V') - $assets = @($Rel.assets) - # Prefer Setup.exe (current ship format). Legacy win64.zip still accepted. - $setupAsset = $assets | Where-Object { $_.name -match 'Setup-.*\.exe$' } | Select-Object -First 1 - $zipAsset = $assets | Where-Object { $_.name -match 'win64\.zip$' } | Select-Object -First 1 - if (-not $setupAsset -and -not $zipAsset) { - throw "${SourceName}_release_missing_setup_or_zip" - } - - $setupUrl = "" - if ($setupAsset) { - $setupUrl = Resolve-AssetUrl -Asset $setupAsset -Rel $Rel -SourceName $SourceName -ApiLatestUrl $ApiLatestUrl - if (-not $setupUrl) { throw "${SourceName}_setup_url_missing" } - } - $zipUrl = "" - if ($zipAsset) { - $zipUrl = Resolve-AssetUrl -Asset $zipAsset -Rel $Rel -SourceName $SourceName -ApiLatestUrl $ApiLatestUrl - } - - if ($setupUrl) { - $primaryUrl = $setupUrl - $packageKind = "setup" - $primarySize = [int64]$(if ($setupAsset.size) { $setupAsset.size } else { 0 }) - } else { - if (-not $zipUrl) { throw "${SourceName}_zip_url_missing" } - $primaryUrl = $zipUrl - $packageKind = "zip" - $primarySize = [int64]$(if ($zipAsset.size) { $zipAsset.size } else { 0 }) - } - - return [pscustomobject]@{ - channel = "stable" - latest = $ver - min_compatible = $ver - notes_url = [string]$Rel.html_url - source = $SourceName - windows = [pscustomobject]@{ - url = $primaryUrl - sha256 = "" - size = $primarySize - setup_url = $setupUrl - package = $packageKind - } - } -} - -function Get-ManifestFromUrl { - param([string]$Url) - $headers = Get-AuthHeaders -Style "bearer" - $resp = Invoke-WebRequest -Uri $Url -Headers $headers -UseBasicParsing -TimeoutSec 30 - $m = $resp.Content | ConvertFrom-Json - if (-not $m.source) { - $m | Add-Member -NotePropertyName source -NotePropertyValue "manifest" -Force - } - return $m -} - -function Get-FromGitHubReleases { - $api = "https://api.github.com/repos/$GithubRepo/releases/latest" - $headers = Get-AuthHeaders -Style "bearer" - $headers["Accept"] = "application/vnd.github+json" - $rel = Invoke-RestMethod -Uri $api -Headers $headers -TimeoutSec 30 - return Convert-ReleaseToManifest -Rel $rel -SourceName "github" -} - -function Get-FromForgejoReleases { - param([string]$ApiUrl) - if (-not $ApiUrl) { throw "forgejo_url_empty" } - $headers = Get-AuthHeaders -Style "token" - $headers["Accept"] = "application/json" - $rel = Invoke-RestMethod -Uri $ApiUrl -Headers $headers -TimeoutSec 30 - return Convert-ReleaseToManifest -Rel $rel -SourceName "forgejo" -ApiLatestUrl $ApiUrl -} - -function Get-UpdateSourceList { - if ($Sources) { - return @($Sources.Split(',') | ForEach-Object { $_.Trim().ToLowerInvariant() } | Where-Object { $_ }) - } - # Default: GitHub primary, Forgejo mirror backup. Optional manifests prepended if configured. - $list = [System.Collections.Generic.List[string]]::new() - if ($UpdateUrl) { [void]$list.Add("manifest") } - [void]$list.Add("github") - [void]$list.Add("forgejo") - if ($FallbackUrl) { [void]$list.Add("manifest_fallback") } - return @($list) -} - -function Get-ManifestFromSource([string]$src) { - switch ($src) { - "manifest" { - if (-not $UpdateUrl) { throw "NETX_UPDATE_URL empty" } - Write-Host "==> Probing manifest: $UpdateUrl" - return Get-ManifestFromUrl -Url $UpdateUrl - } - "manifest_fallback" { - if (-not $FallbackUrl) { throw "NETX_UPDATE_FALLBACK_URL empty" } - Write-Host "==> Probing fallback manifest: $FallbackUrl" - $m = Get-ManifestFromUrl -Url $FallbackUrl - if ($m -and -not $m.source) { - $m | Add-Member -NotePropertyName source -NotePropertyValue "manifest_fallback" -Force - } - return $m - } - "forgejo" { - Write-Host "==> Probing Forgejo: $ForgejoUrl" - return Get-FromForgejoReleases -ApiUrl $ForgejoUrl - } - "github" { - Write-Host "==> Probing GitHub: $GithubRepo" - return Get-FromGitHubReleases - } - default { throw "unknown_source: $src" } - } -} - -Write-Host "==> Current version: $current" -$sourceList = Get-UpdateSourceList -Write-Host "==> Probe sources (pick newest reachable; tie → earlier in list): $($sourceList -join ', ')" - -$candidates = @() -$errors = @() -foreach ($src in $sourceList) { - try { - $m = Get-ManifestFromSource -src $src - if ($m.channel -and $Channel -and ($m.channel -ne $Channel)) { - Write-Host "[WARN] $src channel=$($m.channel) requested=$Channel" - } - if (-not $m.windows -or -not $m.windows.url) { - throw "missing_windows_download_url" - } - Write-Host " OK $($m.source) latest=$($m.latest)" -ForegroundColor DarkGreen - $candidates += $m - } catch { - $msg = $_.Exception.Message - $errors += "${src}: $msg" - Write-Host "[WARN] source '$src' unreachable/failed: $msg" -ForegroundColor Yellow - } -} - -if ($candidates.Count -eq 0) { - $joined = ($errors -join "; ") - if ($Quiet) { exit 2 } - throw "update_check_failed: all sources failed ($joined)" -} - -# Prefer highest semver; on tie keep earlier source in $sourceList (GitHub before Forgejo by default). -$manifest = $candidates[0] -foreach ($c in $candidates) { - $cmpCand = Compare-SemVer -A ([string]$manifest.latest) -B ([string]$c.latest) - if ($cmpCand -lt 0) { - $manifest = $c - } -} - -Write-Host "==> Selected source=$($manifest.source) latest=$($manifest.latest) (from $($candidates.Count) reachable)" -ForegroundColor Green - -# Prefer Setup.exe when a manifest still lists a legacy zip as windows.url but also has setup_url. -if ($manifest.windows -and $manifest.windows.setup_url) { - $setupCandidate = [string]$manifest.windows.setup_url - $urlNow = [string]$manifest.windows.url - $pkgNow = if ($manifest.windows.package) { [string]$manifest.windows.package } else { "" } - if ($setupCandidate -and ($pkgNow -eq "zip" -or $urlNow -match '\.zip(\?|$)' -or -not $urlNow)) { - $manifest.windows | Add-Member -NotePropertyName url -NotePropertyValue $setupCandidate -Force - $manifest.windows | Add-Member -NotePropertyName package -NotePropertyValue "setup" -Force - } -} - -$latest = [string]$manifest.latest -$cmp = Compare-SemVer -A $current -B $latest -$packageKind = "setup" -if ($manifest.windows.package) { - $packageKind = [string]$manifest.windows.package -} elseif ([string]$manifest.windows.url -match '\.zip(\?|$)') { - $packageKind = "zip" -} - -$result = [pscustomobject]@{ - current = $current - latest = $latest - update_available = ($cmp -lt 0) - source = $(if ($manifest.source) { [string]$manifest.source } else { "unknown" }) - download_url = [string]$manifest.windows.url - setup_url = $(if ($manifest.windows.setup_url) { [string]$manifest.windows.setup_url } else { "" }) - package = $packageKind - notes_url = $(if ($manifest.notes_url) { [string]$manifest.notes_url } else { "" }) - sha256 = $(if ($manifest.windows.sha256) { [string]$manifest.windows.sha256 } else { "" }) - reachable = @($candidates | ForEach-Object { "$($_.source)=$($_.latest)" }) -} - -if (-not $result.update_available) { - Write-Host "==> Up to date (latest=$latest, source=$($result.source))" -ForegroundColor Green - if (-not $Quiet) { - $result | ConvertTo-Json -Compress | Write-Output - } - exit 0 -} - -Write-Host "==> Update available: $current -> $latest (source=$($result.source))" -ForegroundColor Cyan -if ($result.notes_url) { Write-Host " Notes: $($result.notes_url)" } - -$autoEnabled = $false -$autoVal = Get-Cfg "NETX_UPDATE_AUTO" "" -if ($autoVal -match '^(1|true|yes|on)$') { $autoEnabled = $true } - -if (-not $Apply) { - Write-Host " Download ($($result.package)): $($result.download_url)" - Write-Host " To apply: .\packaging\check_update.ps1 -Apply" - if (-not $Quiet) { - $result | ConvertTo-Json -Compress | Write-Output - } - exit 10 -} - -if ($AutoOnly -and -not $autoEnabled) { - Write-Host "==> Update available but NETX_UPDATE_AUTO is not enabled; skip apply" - if (-not $Quiet) { - $result | ConvertTo-Json -Compress | Write-Output - } - exit 10 -} - -$lockFile = Join-Path $data "data\runtime\update.lock" -$lockDir = Split-Path -Parent $lockFile -if (-not (Test-Path $lockDir)) { - New-Item -ItemType Directory -Path $lockDir -Force | Out-Null -} -if (Test-Path $lockFile) { - $ageHrs = ((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalHours - if ($ageHrs -lt 2) { - Write-Host "==> Another update appears in progress ($lockFile); abort" - exit 3 - } - Remove-Item -Force $lockFile -ErrorAction SilentlyContinue -} -Set-Content -Path $lockFile -Value (Get-Date).ToString("o") -Encoding ascii - -try { - $dlDir = Join-Path $data "backups\downloads" - if (-not (Test-Path $dlDir)) { - New-Item -ItemType Directory -Path $dlDir -Force | Out-Null - } - - $isSetup = ($result.package -eq "setup") -or ($result.download_url -match '\.exe(\?|$)') - if ($isSetup) { - $pkgName = "NetX-Setup-$latest.exe" - } else { - $pkgName = "NetX-$latest-win64.zip" - } - $pkgPath = Join-Path $dlDir $pkgName - - $needDownload = $true - if ((Test-Path -LiteralPath $pkgPath) -and ((Get-Item -LiteralPath $pkgPath).Length -gt 1MB)) { - Write-Host "==> Using existing download: $pkgPath ($([math]::Round((Get-Item $pkgPath).Length/1MB,1)) MB)" - $needDownload = $false - } - if ($needDownload) { - Write-Host "==> Downloading $pkgName from $($result.source) ..." - # Only attach token for non-GitHub hosts (Forgejo/private). Public GitHub assets need no auth; - # a Forgejo token would break anonymous GitHub downloads. - $dlHeaders = @{ "User-Agent" = "NetX-UpdateCheck" } - $dlUri = [uri]$result.download_url - $isGithub = ($dlUri.Host -match '(^|\.)github\.com$' -or $dlUri.Host -match '(^|\.)githubusercontent\.com$') - if ($UpdateToken -and -not $isGithub) { - $dlHeaders["Authorization"] = "token $UpdateToken" - } - Invoke-WebRequest -Uri $result.download_url -OutFile $pkgPath -Headers $dlHeaders -UseBasicParsing - } - $expectedHash = [string]$result.sha256 - if (-not $expectedHash -or $expectedHash -match 'REPLACE') { - $checksumUri = [uri]$result.download_url - $checksumBuilder = New-Object -TypeName System.UriBuilder -ArgumentList $checksumUri - $checksumBuilder.Path = $checksumBuilder.Path + '.sha256' - $checksumBuilder.Query = '' - $checksumUrl = $checksumBuilder.Uri.AbsoluteUri - $checksumHeaders = @{ 'User-Agent' = 'NetX-UpdateCheck' } - $isChecksumGithub = ($checksumUri.Host -match '(^|\.)github\.com$' -or $checksumUri.Host -match '(^|\.)githubusercontent\.com$') - if ($UpdateToken -and -not $isChecksumGithub) { $checksumHeaders['Authorization'] = "token $UpdateToken" } - $checksumFile = "$pkgPath.sha256" - try { - Invoke-WebRequest -Uri $checksumUrl -OutFile $checksumFile -Headers $checksumHeaders -UseBasicParsing - $expectedHash = (Get-Content -LiteralPath $checksumFile -Raw).Trim() -split '\s+' | Select-Object -First 1 - } catch { - $statusCode = 0 - if ($_.Exception.Response -and $_.Exception.Response.StatusCode) { - $statusCode = [int]$_.Exception.Response.StatusCode - } - if ($statusCode -eq 404) { - Write-Host "[WARN] Release has no SHA-256 sidecar; accepting legacy asset without checksum verification." -ForegroundColor Yellow - } else { - throw - } - } finally { - Remove-Item -LiteralPath $checksumFile -Force -ErrorAction SilentlyContinue - } - } - if ($expectedHash) { - $expectedHash = $expectedHash.Trim().ToLowerInvariant() - if ($expectedHash -notmatch '^[0-9a-f]{64}$') { throw 'invalid_sha256_metadata' } - $actualHash = (Get-FileHash -LiteralPath $pkgPath -Algorithm SHA256).Hash.ToLowerInvariant() - if ($actualHash -ne $expectedHash) { - Remove-Item -LiteralPath $pkgPath -Force -ErrorAction SilentlyContinue - throw "sha256_mismatch: got $actualHash expected $expectedHash" - } - Write-Host '==> SHA256 OK' - } - - # Program Files installs need admin; elevate once (avoid loop via NETX_UPDATE_ELEVATED). - $progWritable = $false - try { - $probe = Join-Path $prog (".netx_w_" + [guid]::NewGuid().ToString("n")) - [IO.File]::WriteAllText($probe, "x") - Remove-Item -LiteralPath $probe -Force -ErrorAction SilentlyContinue - $progWritable = $true - } catch { - $progWritable = $false - } - if (-not $progWritable -and -not $env:NETX_UPDATE_ELEVATED) { - Write-Host "==> Program directory is not writable; relaunching update as Administrator (UAC)..." -ForegroundColor Yellow - $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply" - $ep = Start-Process -FilePath "powershell.exe" -ArgumentList $arg -WorkingDirectory $prog ` - -Verb RunAs -PassThru - Wait-NetxProcessExit -Process $ep - if ($null -eq $ep.ExitCode) { exit 1 } - exit $ep.ExitCode - } - - $env:NETX_UPDATE_ELEVATED = "1" - if ($isSetup) { - # Silent Setup update mode: skip DB wizard, keep ProgramData - # (see installer/netx.iss /InstallMode=update). - Write-Host "==> Applying update via silent Setup.exe" - $setupArgs = "/VERYSILENT /NORESTART /SUPPRESSMSGBOXES /DIR=`"$prog`" /InstallMode=update" - # Setup stops the tray (StopForUpdate -KillTray); remember to bring it back. - $trayWasRunning = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { - $_.CommandLine -and $_.CommandLine -match 'netx_tray\.ps1' - }).Count -gt 0 - $sp = Start-Process -FilePath $pkgPath -ArgumentList $setupArgs -PassThru - Wait-NetxProcessExit -Process $sp - if ($null -eq $sp.ExitCode -or $sp.ExitCode -ne 0) { - $code = if ($null -eq $sp.ExitCode) { "null" } else { $sp.ExitCode } - throw "setup_update_failed: exit $code" - } - # Setup post-install also repairs; belt-and-suspenders when running elevated apply. - $repairPs1 = Join-Path $prog "packaging\repair_venv.ps1" - if (Test-Path -LiteralPath $repairPs1) { - Write-Host "==> Relinking .venv to bundled python/runtime" - & powershell -NoProfile -ExecutionPolicy Bypass -File $repairPs1 ` - -ProgramRoot $prog -DataRoot $data - } - $managedRunnerActive = $false - $netxService = Get-Service -Name "NetX" -ErrorAction SilentlyContinue - if ($netxService -and $netxService.Status -eq "Running") { $managedRunnerActive = $true } - if (-not $managedRunnerActive) { - try { - $netxTask = Get-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -ErrorAction Stop - $managedRunnerActive = ([string]$netxTask.State -eq "Running") - } catch {} - } - if ($managedRunnerActive) { - Write-Host "==> Waiting for the previously running NetX service/task" - $netxPort = 8890 - $netxEnv = Read-DotEnv -Path (Join-Path $data '.env') - if ($netxEnv['NETX_PORT']) { try { $netxPort = [int]$netxEnv['NETX_PORT'] } catch {} } - if (-not (Wait-NetxApiHealthy -HostName "127.0.0.1" -Port $netxPort -TimeoutSec 180)) { - throw "updated_service_unhealthy: NetX service/task was resumed but /health did not become ready" - } - } else { - Write-Host "==> Restarting NetX after Setup" - & (Join-Path $PSScriptRoot "start_netx_app.ps1") ` - -ProgramRoot $prog -DataRoot $data -SkipBrowser - if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - } - if ($trayWasRunning) { - $trayPs1 = Join-Path $prog "packaging\netx_tray.ps1" - if (Test-Path -LiteralPath $trayPs1) { - Write-Host "==> Restarting NetX tray" - Start-NetxPowerShell -File $trayPs1 -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-QuietStart") ` - -WorkingDirectory $prog -WindowStyle Hidden | Out-Null - } - } - } else { - Write-Host "==> Applying legacy zip update via update_netx.ps1" - & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "update_netx.ps1") ` - -PackagePath $pkgPath -ProgramRoot $prog -DataRoot $data - } - Write-Host "==> Update applied to $latest" -ForegroundColor Green -} finally { - Remove-Item -Force $lockFile -ErrorAction SilentlyContinue -} -exit 0 diff --git a/packaging/cmd/NetX-FirstRun.cmd b/packaging/cmd/NetX-FirstRun.cmd deleted file mode 100644 index e5f9887..0000000 --- a/packaging/cmd/NetX-FirstRun.cmd +++ /dev/null @@ -1,33 +0,0 @@ -@echo off -setlocal EnableDelayedExpansion -net session >nul 2>&1 -if not "%errorLevel%"=="0" ( - powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" - exit /b !ERRORLEVEL! -) -set "ROOT=%~dp0" -if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" -cd /d "%ROOT%" -title NetX - Reconfigure database -echo. -echo NetX database reconfigure / 重新配置数据库 -echo Prefer the installer wizard for first-time setup. -echo 首次安装请用安装向导选择内置或外置数据库。 -echo This window is for repair / reconfigure only. -echo 本窗口仅用于修复或重新配置。 -echo. -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\setup_first_run.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -set "EC=!ERRORLEVEL!" -if not "!EC!"=="0" ( - echo. - echo Setup failed / 配置失败 exit=!EC! - pause - exit /b !EC! -) -echo. -echo Starting NetX tray... / 正在启动托盘... -start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\netx_tray.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -StartOnLaunch -echo. -echo Done. You can close this window. / 完成，可关闭本窗口。 -pause -exit /b 0 diff --git a/packaging/cmd/NetX-Start.cmd b/packaging/cmd/NetX-Start.cmd deleted file mode 100644 index 8342f92..0000000 --- a/packaging/cmd/NetX-Start.cmd +++ /dev/null @@ -1,13 +0,0 @@ -@echo off -setlocal EnableDelayedExpansion -REM Always elevate: Program Files install needs admin to start API / repair venv. -net session >nul 2>&1 -if not "%errorLevel%"=="0" ( - powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" - exit /b !ERRORLEVEL! -) -set "ROOT=%~dp0" -if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" -cd /d "%ROOT%" -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action start -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -exit /b !ERRORLEVEL! diff --git a/packaging/cmd/NetX-Stop.cmd b/packaging/cmd/NetX-Stop.cmd deleted file mode 100644 index 8a225f2..0000000 --- a/packaging/cmd/NetX-Stop.cmd +++ /dev/null @@ -1,12 +0,0 @@ -@echo off -setlocal EnableDelayedExpansion -net session >nul 2>&1 -if not "%errorLevel%"=="0" ( - powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" - exit /b !ERRORLEVEL! -) -set "ROOT=%~dp0" -if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" -cd /d "%ROOT%" -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action stop -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -exit /b !ERRORLEVEL! diff --git a/packaging/cmd/NetX-Tray.cmd b/packaging/cmd/NetX-Tray.cmd deleted file mode 100644 index ac1f46c..0000000 --- a/packaging/cmd/NetX-Tray.cmd +++ /dev/null @@ -1,13 +0,0 @@ -@echo off -setlocal EnableDelayedExpansion -REM Elevate once, then start tray (inherits admin for Start/Stop/Update). -net session >nul 2>&1 -if not "%errorLevel%"=="0" ( - powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" - exit /b !ERRORLEVEL! -) -set "ROOT=%~dp0" -if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" -cd /d "%ROOT%" -start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\netx_tray.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -StartOnLaunch -exit /b 0 diff --git a/packaging/config/database.example.env b/packaging/config/database.example.env index f5e5555..a9d072a 100644 --- a/packaging/config/database.example.env +++ b/packaging/config/database.example.env @@ -5,11 +5,5 @@ # NETX_HOST=127.0.0.1 # NETX_PORT=8890 # NETX_UI_DIST_DIR=web/dist -# Defaults already: github + forgejo mirror git.avelo.top (newest reachable wins) -# NETX_UPDATE_SOURCES=github,forgejo -# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest -# NETX_UPDATE_GITHUB_REPO=hansjone/netx # NETX_UPDATE_URL= -# NETX_UPDATE_TOKEN= # NETX_UPDATE_CHANNEL=stable -# NETX_UPDATE_AUTO=false diff --git a/packaging/download_postgres.ps1 b/packaging/download_postgres.ps1 index 0a0fe29..38c7b41 100644 --- a/packaging/download_postgres.ps1 +++ b/packaging/download_postgres.ps1 @@ -1,4 +1,4 @@ -param( +param( [string]$Version = "16.4-1", [string]$OutDir = "" ) @@ -12,7 +12,6 @@ $url = "https://get.enterprisedb.com/postgresql/postgresql-$ver-windows-x64-bina $cache = if ($OutDir) { $OutDir } else { Join-Path $PSScriptRoot "cache" } $zipPath = Join-Path $cache "postgresql-$ver-windows-x64-binaries.zip" -$partialPath = "$zipPath.partial" $extractRoot = Join-Path $cache "pgsql-$ver" $destPgsql = Join-Path $PSScriptRoot "postgres\pgsql" @@ -23,31 +22,7 @@ if (-not (Test-Path $cache)) { Write-Host "==> PostgreSQL binaries URL: $url" if (-not (Test-Path $zipPath)) { Write-Host "==> Downloading (large; may take several minutes)..." - $curl = Get-Command curl.exe -ErrorAction SilentlyContinue - if ($curl) { - # PostgreSQL's archive is several hundred MB. Resume interrupted transfers - # instead of restarting the entire download when the CDN drops a connection. - & $curl.Source --fail --location --retry 8 --retry-all-errors --retry-delay 3 ` - --connect-timeout 30 --continue-at - --output $partialPath $url - if ($LASTEXITCODE -ne 0) { - throw "postgres_download_failed: curl exit $LASTEXITCODE (partial file kept at $partialPath)" - } - Move-Item -LiteralPath $partialPath -Destination $zipPath -Force - } else { - # Fallback for environments without curl; retry from scratch on transient errors. - $downloaded = $false - for ($attempt = 1; $attempt -le 3 -and -not $downloaded; $attempt++) { - try { - Invoke-WebRequest -Uri $url -OutFile $partialPath -UseBasicParsing - Move-Item -LiteralPath $partialPath -Destination $zipPath -Force - $downloaded = $true - } catch { - if ($attempt -eq 3) { throw } - Remove-Item -LiteralPath $partialPath -Force -ErrorAction SilentlyContinue - Start-Sleep -Seconds (5 * $attempt) - } - } - } + Invoke-WebRequest -Uri $url -OutFile $zipPath -UseBasicParsing } else { Write-Host "==> Using cached zip: $zipPath" } diff --git a/packaging/install_autostart.ps1 b/packaging/install_autostart.ps1 deleted file mode 100644 index b0ac9ef..0000000 --- a/packaging/install_autostart.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [switch]$Remove = $false -) - -# Register / unregister NetX tray at logon with highest privileges (UAC admin). -# HKCU\Run cannot elevate reliably on fresh PCs; use a Scheduled Task instead. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$tray = Join-Path $PSScriptRoot "netx_tray.ps1" -$runKey = "HKCU:\Software\Microsoft\Windows\CurrentVersion\Run" -$legacyName = "NetX" -$taskName = "NetXTray" -$taskPath = "\NetX\" - -# Always clear legacy per-user Run key (non-elevated). -Remove-ItemProperty -Path $runKey -Name $legacyName -ErrorAction SilentlyContinue - -if ($Remove) { - Unregister-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Confirm:$false -ErrorAction SilentlyContinue - Write-Host "==> Removed NetX tray autostart task" - exit 0 -} - -# -QuietStart: bring NetX up at logon without popping a browser window. -$arg = "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$tray`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -StartOnLaunch -QuietStart" -$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog -# Any user logon, run for the local Administrators group: Setup runs elevated -# (often as a different admin account), so $env:USERNAME is not the desktop user. -$trig = New-ScheduledTaskTrigger -AtLogOn -$trig.Delay = "PT10S" -$admins = (New-Object System.Security.Principal.SecurityIdentifier("S-1-5-32-544")).Translate( - [System.Security.Principal.NTAccount]).Value -$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable ` - -ExecutionTimeLimit ([TimeSpan]::Zero) -MultipleInstances Parallel -$principal = New-ScheduledTaskPrincipal -GroupId $admins -RunLevel Highest -Register-ScheduledTask -TaskName $taskName -TaskPath $taskPath ` - -Action $action -Trigger $trig -Settings $settings -Principal $principal -Force | Out-Null - -Write-Host "==> NetX tray will start at logon (Scheduled Task, RunLevel Highest)" -ForegroundColor Green -Write-Host " $taskPath$taskName" -Write-Host " $arg" diff --git a/packaging/install_service.ps1 b/packaging/install_service.ps1 deleted file mode 100644 index 6a46763..0000000 --- a/packaging/install_service.ps1 +++ /dev/null @@ -1,154 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [ValidateSet("winsw", "task")] - [string]$Mode = "winsw", - [switch]$Uninstall = $false, - [switch]$Start = $false, - # Offline-safe by default: use WinSW shipped in the package. Opt-in to download from GitHub. - [switch]$AllowDownload = $false -) - -# Install NetX as a Windows Service (WinSW) or as a SYSTEM startup Scheduled Task. -# Requires elevation for winsw / task modes that run as SYSTEM. -# WinSW binary is bundled at packaging\winsw\WinSW.exe by build_release.ps1 — no network needed. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$svcName = "NetX" -$winswDir = Join-Path $prog "packaging\winsw" -$winswExe = Join-Path $winswDir "NetX.exe" -$winswXml = Join-Path $winswDir "NetX.xml" -$cacheDir = Join-Path $PSScriptRoot "cache" - -function Test-IsAdmin { - return Test-NetxIsAdmin -} - -function ConvertTo-WinSWPath([string]$Path) { - return ($Path -replace '\\', '/') -} - -function Ensure-WinSW { - if (-not (Test-Path $winswDir)) { - New-Item -ItemType Directory -Path $winswDir -Force | Out-Null - } - if (-not (Test-Path $cacheDir)) { - New-Item -ItemType Directory -Path $cacheDir -Force | Out-Null - } - # WinSW.exe = .NET Framework build (current); WinSW-x64.exe = older self-contained bundles. - $bundled = Join-Path $PSScriptRoot "winsw\WinSW.exe" - if (-not (Test-Path $bundled)) { $bundled = Join-Path $PSScriptRoot "winsw\WinSW-x64.exe" } - $dl = Join-Path $cacheDir "WinSW.NET461.exe" - $src = $null - if (Test-Path $bundled) { - $src = $bundled - Write-Host "==> Using bundled WinSW: $bundled" - } elseif (Test-Path $dl) { - $src = $dl - Write-Host "==> Using cached WinSW: $dl" - } elseif ($AllowDownload) { - $url = "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW.NET461.exe" - Write-Host "==> Downloading WinSW: $url" - Invoke-WebRequest -Uri $url -OutFile $dl -UseBasicParsing - $src = $dl - } else { - throw "winsw_missing_offline: expected $bundled (rebuild with build_release.ps1). Or pass -AllowDownload when online." - } - Copy-Item -Path $src -Destination $winswExe -Force - - $runPs1 = ConvertTo-WinSWPath (Join-Path $PSScriptRoot "service_run.ps1") - $stopPs1 = ConvertTo-WinSWPath (Join-Path $PSScriptRoot "stop_netx_app.ps1") - $progFs = ConvertTo-WinSWPath $prog - $dataFs = ConvertTo-WinSWPath $data - # Own folder: WinSW writes .out.log / .err.log = NetX.out.log, which on Windows is - # the same file as start_netx.ps1's netx.out.log. WinSW keeps it open, so NetX could not - # truncate its log and every service start failed. - $svcLogDir = Join-Path $data "data\runtime\service" - $logPath = ConvertTo-WinSWPath $svcLogDir - if (-not (Test-Path $svcLogDir)) { - New-Item -ItemType Directory -Path $svcLogDir -Force | Out-Null - } - - # Stop hook via cmd: NUL stdin + output to a file. Launched by WinSW directly, - # powershell.exe never reached the script's first line (stuck on the inherited - # std handles), so the supervisor was never told to exit and stops hung. - $stopLog = Join-Path $svcLogDir "stopexec.log" - $stopCmd = "/d /s /c `"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$stopPs1`" -ProgramRoot `"$progFs`" -DataRoot `"$dataFs`" <NUL >>`"$stopLog`" 2>&1`"" - - $xml = @" - - $svcName - NetX Ops - NetX API, workers, and optional bundled PostgreSQL - powershell.exe - -NoProfile -ExecutionPolicy Bypass -File "$runPs1" -ProgramRoot "$progFs" -DataRoot "$dataFs" - $logPath - - 10240 - 4 - - - - 1 hour - 60sec - cmd.exe - $stopCmd - $progFs - -"@ - # WinSW expects UTF-8 without BOM issues; ASCII-compatible paths preferred. - [System.IO.File]::WriteAllText($winswXml, $xml) -} - -if ($Uninstall) { - if (-not (Test-IsAdmin)) { throw "admin_required_for_uninstall" } - if (Test-Path $winswExe) { - Write-Host "==> Stopping/uninstalling WinSW service" - & $winswExe stop 2>$null - & $winswExe uninstall 2>$null - } - Unregister-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -Confirm:$false -ErrorAction SilentlyContinue - Write-Host "==> Service/task removed" - exit 0 -} - -if (-not (Test-IsAdmin)) { - throw "admin_required: run elevated PowerShell to install the NetX service" -} - -if ($Mode -eq "winsw") { - Ensure-WinSW - Write-Host "==> Installing Windows Service via WinSW" - & $winswExe stop 2>$null - & $winswExe uninstall 2>$null - & $winswExe install - if ($LASTEXITCODE -ne 0) { throw "winsw_install_failed" } - if ($Start) { - & $winswExe start - Write-Host "==> Service started" -ForegroundColor Green - } else { - Write-Host "==> Installed. Start with: Start-Service NetX or `"$winswExe`" start" - } -} else { - Write-Host "==> Registering Scheduled Task NetX\NetXService (At startup, SYSTEM)" - $runPs1 = Join-Path $PSScriptRoot "service_run.ps1" - $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$runPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`"" - $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog - $trigger = New-ScheduledTaskTrigger -AtStartup - $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest - # ExecutionTimeLimit 0: the default (72h) would kill the long-running supervisor. - $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 1) ` - -ExecutionTimeLimit ([TimeSpan]::Zero) - Register-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Force | Out-Null - if ($Start) { - Start-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" - Write-Host "==> Task started" -ForegroundColor Green - } else { - Write-Host "==> Task registered. Start with: Start-ScheduledTask -TaskPath '\NetX\' -TaskName NetXService" - } -} diff --git a/packaging/install_update_task.ps1 b/packaging/install_update_task.ps1 deleted file mode 100644 index abdecce..0000000 --- a/packaging/install_update_task.ps1 +++ /dev/null @@ -1,64 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [ValidateSet("Daily", "AtLogOn", "Hourly")] - [string]$Trigger = "Daily", - [string]$At = "03:30", - [switch]$Remove = $false, - [switch]$EnableAutoEnv = $true -) - -# Schedule silent update checks. With NETX_UPDATE_AUTO=true, applies updates when available. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$taskName = "NetXUpdate" -$taskPath = "\NetX\" -$checkPs1 = Join-Path $PSScriptRoot "check_update.ps1" - -if ($Remove) { - Unregister-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Confirm:$false -ErrorAction SilentlyContinue - # Tray also applies updates on launch when NETX_UPDATE_AUTO=true; turn that off too. - $envFile = Join-Path $data ".env" - if (Test-Path $envFile) { - Write-DotEnvValue -Path $envFile -Values @{ "NETX_UPDATE_AUTO" = "false" } - } - Write-Host "==> Update task removed" - exit 0 -} - -if ($EnableAutoEnv) { - $envFile = Join-Path $data ".env" - if (-not (Test-Path $data)) { - New-Item -ItemType Directory -Path $data -Force | Out-Null - } - Write-DotEnvValue -Path $envFile -Values @{ "NETX_UPDATE_AUTO" = "true" } - Write-Host "==> Set NETX_UPDATE_AUTO=true in $envFile" -} - -$arg = "-NoProfile -ExecutionPolicy Bypass -File `"$checkPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply -Quiet -AutoOnly" -$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog - -switch ($Trigger) { - "Hourly" { - $trig = New-ScheduledTaskTrigger -Once -At (Get-Date).Date.AddHours((Get-Date).Hour + 1) ` - -RepetitionInterval (New-TimeSpan -Hours 1) -RepetitionDuration ([TimeSpan]::MaxValue) - } - "AtLogOn" { - $trig = New-ScheduledTaskTrigger -AtLogOn - } - default { - $trig = New-ScheduledTaskTrigger -Daily -At $At - } -} - -$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -StartWhenAvailable -# Highest: silent apply must write Program Files / run Setup. -$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive -RunLevel Highest -Register-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Action $action -Trigger $trig -Settings $settings -Principal $principal -Force | Out-Null -Write-Host "==> Scheduled update task: $taskPath$taskName ($Trigger, RunLevel Highest)" -ForegroundColor Green -Write-Host " Manual run: .\packaging\check_update.ps1 -Apply -Quiet -AutoOnly" diff --git a/packaging/installer/Languages/ChineseSimplified.isl b/packaging/installer/Languages/ChineseSimplified.isl deleted file mode 100644 index 30d9973..0000000 --- a/packaging/installer/Languages/ChineseSimplified.isl +++ /dev/null @@ -1,417 +0,0 @@ -; *** Inno Setup version 6.5.0+ Chinese Simplified messages *** -; -; To download user-contributed translations of this file, go to: -; https://jrsoftware.org/files/istrans/ -; -; Note: When translating this text, do not add periods (.) to the end of -; messages that didn't have them already, because on those messages Inno -; Setup adds the periods automatically (appending a period would result in -; two periods being displayed). -; -; Maintainer: Zhenghan Yang (Kira) -; Email: 847320916@QQ.com -; Github: https://github.com/kira-96/Inno-Setup-Chinese-Simplified-Translation -; Encoding: UTF-8 -; Translation based on network resource -; - -[LangOptions] -; The following three entries are very important. Be sure to read and -; understand the '[LangOptions] section' topic in the help file. -LanguageName=简体中文 -; About LanguageID, to reference link: -; https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c -LanguageID=$0804 -; LanguageCodePage should always be set if possible, even if this file is Unicode -; For English it's set to zero anyway because English only uses ASCII characters -LanguageCodePage=936 -; If the language you are translating to requires special font faces or -; sizes, uncomment any of the following entries and change them accordingly. -;DialogFontName= -;DialogFontSize=9 -;DialogFontBaseScaleWidth=7 -;DialogFontBaseScaleHeight=15 -;WelcomeFontName=Segoe UI -;WelcomeFontSize=14 - -[Messages] - -; *** Application titles -SetupAppTitle=安装 -SetupWindowTitle=安装 - %1 -UninstallAppTitle=卸载 -UninstallAppFullTitle=%1 卸载 - -; *** Misc. common -InformationTitle=信息 -ConfirmTitle=确认 -ErrorTitle=错误 - -; *** SetupLdr messages -SetupLdrStartupMessage=现在将安装 %1。您想要继续吗？ -LdrCannotCreateTemp=无法创建临时文件。安装程序已中止 -LdrCannotExecTemp=无法执行临时目录中的文件。安装程序已中止 -HelpTextNote= - -; *** Startup error messages -LastErrorMessage=%1。%n%n错误 %2: %3 -SetupFileMissing=安装目录中缺少文件 %1。请修正这个问题或者获取程序的新副本。 -SetupFileCorrupt=安装文件已损坏。请获取程序的新副本。 -SetupFileCorruptOrWrongVer=安装文件已损坏，或是与这个安装程序的版本不兼容。请修正这个问题或获取新的程序副本。 -InvalidParameter=无效的命令行参数：%n%n%1 -SetupAlreadyRunning=安装程序已在运行。 -WindowsVersionNotSupported=此程序不支持当前计算机运行的 Windows 版本。 -WindowsServicePackRequired=此程序需要 %1 服务包 %2 或更高版本。 -NotOnThisPlatform=此程序不能在 %1 上运行。 -OnlyOnThisPlatform=此程序只能在 %1 上运行。 -OnlyOnTheseArchitectures=此程序只能安装到为下列处理器架构设计的 Windows 版本中：%n%n%1 -WinVersionTooLowError=此程序需要 %1 版本 %2 或更高。 -WinVersionTooHighError=此程序不能安装于 %1 版本 %2 或更高。 -AdminPrivilegesRequired=在安装此程序时您必须以管理员身份登录。 -PowerUserPrivilegesRequired=在安装此程序时您必须以管理员身份或高级用户组身份登录。 -SetupAppRunningError=安装程序检测到 %1 当前正在运行。%n%n请先关闭正在运行的程序，然后点击“确定”继续，或点击“取消”退出。 -UninstallAppRunningError=卸载程序检测到 %1 当前正在运行。%n%n请先关闭正在运行的程序，然后点击“确定”继续，或点击“取消”退出。 - -; *** Startup questions -PrivilegesRequiredOverrideTitle=选择安装程序安装模式 -PrivilegesRequiredOverrideInstruction=选择安装模式 -PrivilegesRequiredOverrideText1=%1 可以为所有用户安装（需要管理员权限），或仅为您安装。 -PrivilegesRequiredOverrideText2=%1 可以仅为您安装，或为所有用户安装（需要管理员权限）。 -PrivilegesRequiredOverrideAllUsers=为所有用户安装(&A) -PrivilegesRequiredOverrideAllUsersRecommended=为所有用户安装(&A)（推荐） -PrivilegesRequiredOverrideCurrentUser=仅为我安装(&M) -PrivilegesRequiredOverrideCurrentUserRecommended=仅为我安装(&M)（推荐） - -; *** Misc. errors -ErrorCreatingDir=安装程序无法创建目录“%1” -ErrorTooManyFilesInDir=无法在目录“%1”中创建文件，因为里面包含太多文件。 - -; *** Setup common messages -ExitSetupTitle=退出安装程序 -ExitSetupMessage=安装程序尚未完成。如果现在退出，将不会安装该程序。%n%n您之后可以再次运行安装程序完成安装。%n%n现在退出安装程序吗？ -AboutSetupMenuItem=关于安装程序(&A)... -AboutSetupTitle=关于安装程序 -AboutSetupMessage=%1 版本 %2%n%3%n%n%1 主页：%n%4 -AboutSetupNote= -TranslatorNote=简体中文翻译由 Kira（847320916@qq.com）维护。项目地址：https://github.com/kira-96/Inno-Setup-Chinese-Simplified-Translation - -; *** Buttons -ButtonBack=< 上一步(&B) -ButtonNext=下一步(&N) > -ButtonInstall=安装(&I) -ButtonOK=确定 -ButtonCancel=取消 -ButtonYes=是(&Y) -ButtonYesToAll=全是(&A) -ButtonNo=否(&N) -ButtonNoToAll=全否(&O) -ButtonFinish=完成(&F) -ButtonBrowse=浏览(&B)... -ButtonWizardBrowse=浏览(&R)... -ButtonNewFolder=新建文件夹(&M) - -; *** "Select Language" dialog messages -SelectLanguageTitle=选择安装语言 -SelectLanguageLabel=选择安装时使用的语言。 - -; *** Common wizard text -ClickNext=点击“下一步”继续，或点击“取消”退出安装程序。 -BeveledLabel= -BrowseDialogTitle=浏览文件夹 -BrowseDialogLabel=在下面的列表中选择一个文件夹，然后点击“确定”。 -NewFolderName=新建文件夹 - -; *** "Welcome" wizard page -WelcomeLabel1=欢迎使用 [name] 安装向导 -WelcomeLabel2=即将在您的计算机上安装 [name/ver]。%n%n建议您在继续安装前关闭所有其他应用程序。 - -; *** "Password" wizard page -WizardPassword=密码 -PasswordLabel1=此安装程序需要密码验证。 -PasswordLabel3=请输入密码，然后点击“下一步”继续。密码区分大小写。 -PasswordEditLabel=密码(&P)： -IncorrectPassword=您输入的密码不正确，请重新输入。 - -; *** "License Agreement" wizard page -WizardLicense=许可协议 -LicenseLabel=请在继续安装前阅读以下重要信息。 -LicenseLabel3=请阅读下列许可协议。在继续安装前您必须同意这些协议条款。 -LicenseAccepted=我同意此协议(&A) -LicenseNotAccepted=我不同意此协议(&D) - -; *** "Information" wizard pages -WizardInfoBefore=信息 -InfoBeforeLabel=请在继续安装前阅读以下重要信息。 -InfoBeforeClickLabel=准备好继续安装后，点击“下一步”。 -WizardInfoAfter=信息 -InfoAfterLabel=请在继续安装前阅读以下重要信息。 -InfoAfterClickLabel=准备好继续安装后，点击“下一步”。 - -; *** "User Information" wizard page -WizardUserInfo=用户信息 -UserInfoDesc=请输入您的信息。 -UserInfoName=用户名(&U)： -UserInfoOrg=组织(&O)： -UserInfoSerial=序列号(&S)： -UserInfoNameRequired=请输入用户名。 - -; *** "Select Destination Location" wizard page -WizardSelectDir=选择目标位置 -SelectDirDesc=您想将 [name] 安装在哪里？ -SelectDirLabel3=安装程序将安装 [name] 到下面的文件夹中。 -SelectDirBrowseLabel=点击“下一步”继续。如果您想选择其他文件夹，点击“浏览”。 -DiskSpaceGBLabel=至少需要有 [gb] GB 的可用磁盘空间。 -DiskSpaceMBLabel=至少需要有 [mb] MB 的可用磁盘空间。 -CannotInstallToNetworkDrive=安装程序无法安装到一个网络驱动器。 -CannotInstallToUNCPath=安装程序无法安装到一个 UNC 路径。 -InvalidPath=您必须输入一个带驱动器盘符的完整路径，例如：%n%nC:\App%n%n或UNC路径：%n%n\\server\share -InvalidDrive=您选定的驱动器或 UNC 共享不存在或不能访问。请选择其他位置。 -DiskSpaceWarningTitle=磁盘空间不足 -DiskSpaceWarning=安装程序至少需要 %1 KB 的可用空间才能安装，但选定驱动器只有 %2 KB 的可用空间。%n%n您确定要继续吗？ -DirNameTooLong=文件夹名称或路径太长。 -InvalidDirName=文件夹名称无效。 -BadDirName32=文件夹名称不能包含下列任何字符：%n%n%1 -DirExistsTitle=文件夹已存在 -DirExists=文件夹：%n%n%1%n%n已经存在。您确定安装到这个文件夹中吗？ -DirDoesntExistTitle=文件夹不存在 -DirDoesntExist=文件夹：%n%n%1%n%n不存在。您想要创建此文件夹吗？ - -; *** "Select Components" wizard page -WizardSelectComponents=选择组件 -SelectComponentsDesc=您想安装哪些程序组件？ -SelectComponentsLabel2=选中您想安装的组件；取消您不想安装的组件。然后点击“下一步”继续。 -FullInstallation=完全安装 -; if possible don't translate 'Compact' as 'Minimal' (I mean 'Minimal' in your language) -CompactInstallation=简洁安装 -CustomInstallation=自定义安装 -NoUninstallWarningTitle=组件已存在 -NoUninstallWarning=安装程序检测到下列组件已安装在您的计算机中：%n%n%1%n%n取消选中这些组件不会卸载它们。%n%n您确定要继续吗？ -ComponentSize1=%1 KB -ComponentSize2=%1 MB -ComponentsDiskSpaceGBLabel=当前选择的组件需要至少 [gb] GB 的磁盘空间。 -ComponentsDiskSpaceMBLabel=当前选择的组件需要至少 [mb] MB 的磁盘空间。 - -; *** "Select Additional Tasks" wizard page -WizardSelectTasks=选择附加任务 -SelectTasksDesc=您想要安装程序执行哪些附加任务？ -SelectTasksLabel2=选择您想要安装程序在安装 [name] 时执行的附加任务，然后点击“下一步”。 - -; *** "Select Start Menu Folder" wizard page -WizardSelectProgramGroup=选择开始菜单文件夹 -SelectStartMenuFolderDesc=安装程序应该在哪里放置程序的快捷方式？ -SelectStartMenuFolderLabel3=安装程序将在下列“开始”菜单文件夹中创建程序的快捷方式。 -SelectStartMenuFolderBrowseLabel=点击“下一步”继续。如果您想选择其他文件夹，点击“浏览”。 -MustEnterGroupName=您必须输入一个文件夹名称。 -GroupNameTooLong=文件夹名称或路径太长。 -InvalidGroupName=文件夹名称无效。 -BadGroupName=文件夹名称不能包含下列任何字符：%n%n%1 -NoProgramGroupCheck2=不创建开始菜单文件夹(&D) - -; *** "Ready to Install" wizard page -WizardReady=准备安装 -ReadyLabel1=安装程序准备就绪，现在可以开始安装 [name] 到您的计算机。 -ReadyLabel2a=点击“安装”继续此安装程序。如果您想重新查看或修改任何设置，点击“上一步”。 -ReadyLabel2b=点击“安装”继续此安装程序。 -ReadyMemoUserInfo=用户信息： -ReadyMemoDir=目标位置： -ReadyMemoType=安装类型： -ReadyMemoComponents=已选择组件： -ReadyMemoGroup=开始菜单文件夹： -ReadyMemoTasks=附加任务： - -; *** TDownloadWizardPage wizard page and DownloadTemporaryFile -DownloadingLabel2=正在下载文件... -ButtonStopDownload=停止下载(&S) -StopDownload=您确定要停止下载吗？ -ErrorDownloadAborted=下载已中止。 -ErrorDownloadFailed=下载失败：%1 %2。 -ErrorDownloadSizeFailed=获取大小失败：%1 %2。 -ErrorProgress=无效的进度：%1 / %2。 -ErrorFileSize=文件大小错误：预期 %1，实际 %2。 - -; *** TExtractionWizardPage wizard page and ExtractArchive -ExtractingLabel=正在提取文件... -ButtonStopExtraction=停止提取(&S) -StopExtraction=您确定要停止提取吗？ -ErrorExtractionAborted=提取已中止。 -ErrorExtractionFailed=提取失败：%1 - -; *** Archive extraction failure details -ArchiveIncorrectPassword=密码不正确。 -ArchiveIsCorrupted=压缩包已损坏。 -ArchiveUnsupportedFormat=不支持的压缩包格式。 - -; *** "Preparing to Install" wizard page -WizardPreparing=正在准备安装 -PreparingDesc=安装程序正在准备安装 [name] 到您的计算机。 -PreviousInstallNotCompleted=先前的程序安装或卸载未完成，需要您重启计算机以完成该安装。%n%n在重启计算机后，再次运行安装程序以完成 [name] 的安装。 -CannotContinue=安装程序不能继续。请点击“取消”退出。 -ApplicationsFound=以下应用程序正在使用将由安装程序更新的文件。建议您允许安装程序自动关闭这些应用程序。 -ApplicationsFound2=以下应用程序正在使用将由安装程序更新的文件。建议您允许安装程序自动关闭这些应用程序。安装完成后，安装程序将尝试重新启动这些应用程序。 -CloseApplications=自动关闭应用程序(&A) -DontCloseApplications=不要关闭应用程序(&D) -ErrorCloseApplications=安装程序无法自动关闭所有应用程序。建议您在继续之前，关闭所有在使用需要由安装程序更新的文件的应用程序。 -PrepareToInstallNeedsRestart=安装程序必须重启您的计算机。计算机重启后，请再次运行安装程序以完成 [name] 的安装。%n%n要立即重启吗？ - -; *** "Installing" wizard page -WizardInstalling=正在安装 -InstallingLabel=安装程序正在安装 [name] 到您的计算机，请稍候。 - -; *** "Setup Completed" wizard page -FinishedHeadingLabel=完成 [name] 安装向导 -FinishedLabelNoIcons=安装程序已在您的计算机中安装了 [name]。 -FinishedLabel=安装程序已在您的计算机中安装了 [name]。您可以通过已安装的快捷方式运行此应用程序。 -ClickFinish=点击“完成”退出安装程序。 -FinishedRestartLabel=为完成 [name] 的安装，安装程序必须重新启动您的计算机。要立即重启吗？ -FinishedRestartMessage=为完成 [name] 的安装，安装程序必须重新启动您的计算机。%n%n要立即重启吗？ -ShowReadmeCheck=是，我想查阅自述文件 -YesRadio=是，立即重启计算机(&Y) -NoRadio=否，稍后重启计算机(&N) -; used for example as 'Run MyProg.exe' -RunEntryExec=运行 %1 -; used for example as 'View Readme.txt' -RunEntryShellExec=查阅 %1 - -; *** "Setup Needs the Next Disk" stuff -ChangeDiskTitle=安装程序需要下一张磁盘 -SelectDiskLabel2=请插入磁盘 %1 并点击“确定”。%n%n如果这个磁盘中的文件可以在下列文件夹之外的文件夹中找到，请输入正确的路径或点击“浏览”。 -PathLabel=路径(&P)： -FileNotInDir2=“%2”中找不到文件“%1”。请插入正确的磁盘或选择其他文件夹。 -SelectDirectoryLabel=请指定下一张磁盘的位置。 - -; *** Installation phase messages -SetupAborted=安装程序未完成安装。%n%n请修正这个问题并重新运行安装程序。 -AbortRetryIgnoreSelectAction=选择操作 -AbortRetryIgnoreRetry=重试(&T) -AbortRetryIgnoreIgnore=忽略错误并继续(&I) -AbortRetryIgnoreCancel=取消安装 -RetryCancelSelectAction=选择操作 -RetryCancelRetry=重试(&T) -RetryCancelCancel=取消 - -; *** Installation status messages -StatusClosingApplications=正在关闭应用程序... -StatusCreateDirs=正在创建目录... -StatusExtractFiles=正在提取文件... -StatusDownloadFiles=正在下载文件... -StatusCreateIcons=正在创建快捷方式... -StatusCreateIniEntries=正在创建 INI 条目... -StatusCreateRegistryEntries=正在创建注册表条目... -StatusRegisterFiles=正在注册文件... -StatusSavingUninstall=正在保存卸载信息... -StatusRunProgram=正在完成安装... -StatusRestartingApplications=正在重启应用程序... -StatusRollback=正在撤销更改... - -; *** Misc. errors -ErrorInternal2=内部错误：%1。 -ErrorFunctionFailedNoCode=%1 失败。 -ErrorFunctionFailed=%1 失败；错误代码 %2。 -ErrorFunctionFailedWithMessage=%1 失败；错误代码 %2。%n%3 -ErrorExecutingProgram=无法执行文件：%n%1 - -; *** Registry errors -ErrorRegOpenKey=打开注册表项时出错：%n%1\%2 -ErrorRegCreateKey=创建注册表项时出错：%n%1\%2 -ErrorRegWriteKey=写入注册表项时出错：%n%1\%2 - -; *** INI errors -ErrorIniEntry=在文件“%1”中创建 INI 条目时出错。 - -; *** File copying errors -FileAbortRetryIgnoreSkipNotRecommended=跳过此文件(&S)（不推荐） -FileAbortRetryIgnoreIgnoreNotRecommended=忽略错误并继续(&I)（不推荐） -SourceIsCorrupted=源文件已损坏。 -SourceDoesntExist=源文件“%1”不存在。 -SourceVerificationFailed=源文件验证失败：%1 -VerificationSignatureDoesntExist=签名文件“%1”不存在。 -VerificationSignatureInvalid=签名文件“%1”无效。 -VerificationKeyNotFound=签名文件“%1”使用了未知的密钥。 -VerificationFileNameIncorrect=文件名不正确。 -VerificationFileTagIncorrect=文件标签不正确。 -VerificationFileSizeIncorrect=文件大小不正确。 -VerificationFileHashIncorrect=文件哈希值不正确。 -ExistingFileReadOnly2=无法替换已存在的文件，它是只读的。 -ExistingFileReadOnlyRetry=移除只读属性并重试(&R) -ExistingFileReadOnlyKeepExisting=保留已存在的文件(&K) -ErrorReadingExistingDest=尝试读取已存在的文件时出错： -FileExistsSelectAction=选择操作 -FileExists2=文件已经存在。 -FileExistsOverwriteExisting=覆盖已存在的文件(&O) -FileExistsKeepExisting=保留已存在的文件(&K) -FileExistsOverwriteOrKeepAll=为接下来的冲突文件执行此操作(&D) -ExistingFileNewerSelectAction=选择操作 -ExistingFileNewer2=已存在的文件比安装程序将要安装的文件还要新。 -ExistingFileNewerOverwriteExisting=覆盖已存在的文件(&O) -ExistingFileNewerKeepExisting=保留已存在的文件(&K)（推荐） -ExistingFileNewerOverwriteOrKeepAll=为接下来的冲突文件执行此操作(&D) -ErrorChangingAttr=尝试更改下列已存在的文件属性时出错： -ErrorCreatingTemp=尝试在目标目录创建文件时出错： -ErrorReadingSource=尝试读取下列源文件时出错： -ErrorCopying=尝试复制下列文件时出错： -ErrorDownloading=尝试下载文件时出错： -ErrorExtracting=尝试提取压缩包时出错： -ErrorReplacingExistingFile=尝试替换已存在的文件时出错： -ErrorRestartReplace=重启并替换失败： -ErrorRenamingTemp=尝试重命名下列目标目录中的一个文件时出错： -ErrorRegisterServer=无法注册 DLL/OCX：%1 -ErrorRegSvr32Failed=RegSvr32 失败；退出代码 %1。 -ErrorRegisterTypeLib=无法注册类型库：%1 - -; *** Uninstall display name markings -; used for example as 'My Program (32-bit)' -UninstallDisplayNameMark=%1 (%2) -; used for example as 'My Program (32-bit, All users)' -UninstallDisplayNameMarks=%1 (%2, %3) -UninstallDisplayNameMark32Bit=32 位 -UninstallDisplayNameMark64Bit=64 位 -UninstallDisplayNameMarkAllUsers=所有用户 -UninstallDisplayNameMarkCurrentUser=当前用户 - -; *** Post-installation errors -ErrorOpeningReadme=尝试打开自述文件时出错。 -ErrorRestartingComputer=安装程序无法重启计算机，请手动重启。 - -; *** Uninstaller messages -UninstallNotFound=文件“%1”不存在。无法卸载。 -UninstallOpenError=文件“%1”不能被打开。无法卸载 -UninstallUnsupportedVer=此版本的卸载程序无法识别卸载日志文件“%1”的格式。无法卸载。 -UninstallUnknownEntry=卸载日志中遇到一个未知条目（%1）。 -ConfirmUninstall=您确认要完全移除 %1 及其所有组件吗？ -UninstallOnlyOnWin64=仅允许在 64 位 Windows 中卸载此程序。 -OnlyAdminCanUninstall=仅使用管理员权限的用户能完成此卸载。 -UninstallStatusLabel=正在从您的计算机中移除 %1，请稍候。 -UninstalledAll=已顺利从您的计算机中移除 %1。 -UninstalledMost=%1 卸载完成。%n%n有部分内容未能被删除，但您可以手动删除它们。 -UninstalledAndNeedsRestart=为完成 %1 的卸载，需要重启您的计算机。%n%n要立即重启吗？ -UninstallDataCorrupted=文件“%1”已损坏。无法卸载。 - -; *** Uninstallation phase messages -ConfirmDeleteSharedFileTitle=删除共享文件？ -ConfirmDeleteSharedFile2=系统表示下列共享文件已不再有任何程序使用。您希望卸载程序删除此共享文件吗？%n%n如果仍有程序正在使用此文件，删除后这些程序可能无法正常运行。如果您不能确定，请选择“否”，保留此文件在系统中不会造成任何损害。 -SharedFileNameLabel=文件名： -SharedFileLocationLabel=位置： -WizardUninstalling=卸载状态 -StatusUninstalling=正在卸载 %1... - -; *** Shutdown block reasons -ShutdownBlockReasonInstallingApp=正在安装 %1。 -ShutdownBlockReasonUninstallingApp=正在卸载 %1。 - -; The custom messages below aren't used by Setup itself, but if you make -; use of them in your scripts, you'll want to translate them. - -[CustomMessages] - -NameAndVersion=%1 版本 %2 -AdditionalIcons=附加快捷方式： -CreateDesktopIcon=创建桌面快捷方式(&D) -CreateQuickLaunchIcon=创建快速启动栏快捷方式(&Q) -ProgramOnTheWeb=%1 网站 -UninstallProgram=卸载 %1 -LaunchProgram=运行 %1 -AssocFileExtension=将 %2 文件扩展名与 %1 建立关联(&A) -AssocingFileExtension=正在将 %2 文件扩展名与 %1 建立关联... -AutoStartProgramGroupDescription=启动： -AutoStartProgram=自动启动 %1 -AddonHostProgramNotFound=您选择的文件夹中无法找到 %1。%n%n您确定要继续吗？ diff --git a/packaging/installer/netx.iss b/packaging/installer/netx.iss index 9234a35..55090ea 100644 --- a/packaging/installer/netx.iss +++ b/packaging/installer/netx.iss @@ -1,11 +1,10 @@ -; NetX Windows installer (Inno Setup 6+) +; NetX Windows installer (Inno Setup 6+) ; Compile after: packaging\build_release.ps1 ; ISCC.exe packaging\installer\netx.iss -; Encoding: UTF-8 with BOM (required for Chinese CustomMessages) #define MyAppName "NetX" #ifndef MyAppVersion - #define MyAppVersion "0.4.19" + #define MyAppVersion "0.3.0" #endif #define MyAppPublisher "NetX" #define MyAppURL "https://github.com/hansjone/netx" @@ -13,17 +12,11 @@ ; Stage directory produced by build_release.ps1 (relative to this .iss) #define StageDir "..\release\netx-win64" -#define IconFile "..\assets\netx.ico" -#define AssetsDir "..\assets" [Setup] AppId={{A7E3C2D1-9F40-4B8E-9C1A-NETXWIN64001} AppName={#MyAppName} AppVersion={#MyAppVersion} -AppVerName={#MyAppName} {#MyAppVersion} -VersionInfoVersion={#MyAppVersion} -VersionInfoProductName={#MyAppName} -UninstallDisplayName={#MyAppName} {#MyAppVersion} AppPublisher={#MyAppPublisher} AppPublisherURL={#MyAppURL} DefaultDirName={autopf}\NetX @@ -31,1345 +24,46 @@ DefaultGroupName=NetX DisableProgramGroupPage=yes LicenseFile={#StageDir}\LICENSE OutputDir=..\release -#ifndef OutputBaseFilename - #define OutputBaseFilename "NetX-Setup-" + MyAppVersion -#endif -OutputBaseFilename={#OutputBaseFilename} -SetupIconFile={#IconFile} -UninstallDisplayIcon={app}\packaging\assets\netx.ico +OutputBaseFilename=NetX-Setup-{#MyAppVersion} Compression=lzma2 SolidCompression=yes -; Follows the Windows light/dark setting; windows11 = flat Fluent-style controls. -WizardStyle=modern dynamic windows11 hidebevels -WizardImageFile={#AssetsDir}\wizard-100.png,{#AssetsDir}\wizard-125.png,{#AssetsDir}\wizard-150.png,{#AssetsDir}\wizard-200.png -WizardSmallImageFile={#AssetsDir}\wizard-small-100.png,{#AssetsDir}\wizard-small-125.png,{#AssetsDir}\wizard-small-150.png,{#AssetsDir}\wizard-small-200.png -WizardImageBackColor=#0a1222 +WizardStyle=modern ArchitecturesAllowed=x64compatible ArchitecturesInstallIn64BitMode=x64compatible PrivilegesRequired=admin -; Data lives under {commonappdata}\NetX - not overwritten by upgrades. -; Do NOT use CloseApplications=yes — it can freeze/beep on the Tasks/Ready -; pages while scanning locked NetX tray/service processes with no visible dialog. -CloseApplications=no -; Pick from the Windows UI language; no extra dialog. -ShowLanguageDialog=auto +; Data lives under {commonappdata}\NetX — not overwritten by upgrades +CloseApplications=yes [Languages] Name: "english"; MessagesFile: "compiler:Default.isl" -Name: "chinesesimplified"; MessagesFile: "Languages\ChineseSimplified.isl" - -[CustomMessages] -english.CreateDesktopIcon=Create a desktop shortcut -english.SetupOptions=After install: -english.StartTrayNow=Start NetX now -english.RunModeGroup=How NetX runs: -english.RunModeTray=Desktop tray (recommended) - control NetX from the notification area -english.EnableAutostart=Start automatically when I sign in to Windows -english.RunModeService=Windows service - runs at boot without anyone signed in (servers) -english.MaintenanceGroup=Maintenance: -english.EnableAutoUpdate=Install updates automatically every day (needs network) -english.ServiceInstallFailed=NetX was installed, but the Windows service could not be registered (exit %1).%n%nYou can still start NetX from the Start Menu. Log: %2 -english.IconStart=Start NetX -english.IconStop=Stop NetX -english.IconUpdate=Check for updates -english.UninstallDeleteData=Also delete NetX data (%ProgramData%\NetX)?%n%nYes = remove database, settings, and secrets%nNo = keep data for a later reinstall -english.DbPageCaption=Database -english.DbPageDescription=Choose built-in or external PostgreSQL. Optional credential key: paste from an existing install, or leave empty to auto-generate. -english.DbBundled=Built-in PostgreSQL (portable, offline) -english.DbBundledPort=Port -english.DbBundledPortInvalid=The built-in PostgreSQL port must be a number between 1024 and 65535. -english.DbBundledPortInUse=Port %1 is already in use on this computer.%n%nChoose another port for the built-in PostgreSQL. -english.DbExternal=External PostgreSQL (existing server) -english.DbHost=Host -english.DbPort=Port -english.DbUser=User -english.DbPassword=Password -english.DbName=Database -english.CredKey=Credential key -english.CredKeyHint=NETX_CREDENTIAL_SECRET_KEY (optional — empty = auto-generate) -english.DbFieldsRequired=Please fill in host, port, user, password, and database name. -english.DbTesting=Testing PostgreSQL connection, please wait… -english.DbTestFailed=Cannot connect to PostgreSQL with these settings.%n%n%1%n%nFix the settings and try again. -english.DbTestExtractFailed=Could not extract PostgreSQL client tools for connection test. -english.DbSilentExternalMissing=Silent install with external DB requires /DbHost /DbUser /DbPassword /DbName (optional /DbPort). -english.DbApplyFailed=Database configuration failed after file install (exit %1).%n%nLog: %2%n%nFix the problem, then use Start Menu → Reconfigure database — or reinstall. -english.DbApplyExecFailed=Could not start database configuration script. -english.DbBundledMissing=Built-in PostgreSQL files are missing from the install folder:%n%1%n%nThe Setup package is incomplete. Re-download NetX-Setup and install again. -english.DbEnvMissing=Database configuration did not write %ProgramData%\NetX\.env. -english.ReconfigureDb=Reconfigure database -english.UpgradeKeepDb=Existing NetX data found - database settings will be kept (no wizard). -english.ModePageCaption=Install mode -english.ModePageDescription=An existing NetX installation was found. Choose update or reinstall. -english.ModeUpdate=Update existing installation (keep database settings) -english.ModeReinstall=Reinstall and reconfigure database (same as first install) -english.ModeUpdateHint=Replaces program files only. ProgramData\.env and database data are kept. -english.ModeReinstallHint=Shows the Database page again. Does not delete ProgramData unless you uninstall and choose to. -chinesesimplified.CreateDesktopIcon=创建桌面快捷方式 -chinesesimplified.SetupOptions=安装完成后: -chinesesimplified.StartTrayNow=立即启动 NetX -chinesesimplified.RunModeGroup=运行方式： -chinesesimplified.RunModeTray=桌面托盘（推荐）— 在任务栏通知区域控制 NetX -chinesesimplified.EnableAutostart=登录 Windows 时自动启动 -chinesesimplified.RunModeService=Windows 服务 — 开机即运行，无需登录（适合服务器） -chinesesimplified.MaintenanceGroup=维护： -chinesesimplified.EnableAutoUpdate=每天自动安装更新（需要联网） -chinesesimplified.ServiceInstallFailed=NetX 已安装，但 Windows 服务注册失败（退出码 %1）。%n%n仍可从开始菜单启动 NetX。日志：%2 -chinesesimplified.IconStart=启动 NetX -chinesesimplified.IconStop=停止 NetX -chinesesimplified.IconUpdate=检查更新 -chinesesimplified.UninstallDeleteData=是否同时删除 NetX 数据（%ProgramData%\NetX）？%n%n是 = 删除数据库、配置和密钥%n否 = 保留数据以便以后重装 -chinesesimplified.DbPageCaption=数据库 -chinesesimplified.DbPageDescription=选择内置或外置 PostgreSQL。可选粘贴已有 NETX_CREDENTIAL_SECRET_KEY；留空则自动生成。 -chinesesimplified.DbBundled=内置 PostgreSQL（便携，可离线） -chinesesimplified.DbBundledPort=端口 -chinesesimplified.DbBundledPortInvalid=内置 PostgreSQL 端口必须是 1024–65535 之间的数字。 -chinesesimplified.DbBundledPortInUse=本机端口 %1 已被占用。%n%n请为内置 PostgreSQL 换一个端口。 -chinesesimplified.DbExternal=外置 PostgreSQL（已有服务器） -chinesesimplified.DbHost=主机 -chinesesimplified.DbPort=端口 -chinesesimplified.DbUser=用户 -chinesesimplified.DbPassword=密码 -chinesesimplified.DbName=数据库名 -chinesesimplified.CredKey=凭据密钥 -chinesesimplified.CredKeyHint=NETX_CREDENTIAL_SECRET_KEY（可选，留空=自动生成） -chinesesimplified.DbFieldsRequired=请填写主机、端口、用户、密码和数据库名。 -chinesesimplified.DbTesting=正在测试 PostgreSQL 连接，请稍候… -chinesesimplified.DbTestFailed=无法用当前设置连接 PostgreSQL。%n%n%1%n%n请改正后重试。 -chinesesimplified.DbTestExtractFailed=无法解压 PostgreSQL 客户端以测试连接。 -chinesesimplified.DbSilentExternalMissing=静默安装外置库需要参数 /DbHost /DbUser /DbPassword /DbName（可选 /DbPort）。 -chinesesimplified.DbApplyFailed=文件安装后数据库自动配置失败（退出码 %1）。%n%n日志: %2%n%n请处理后使用开始菜单 → 重新配置数据库，或重新安装。 -chinesesimplified.DbApplyExecFailed=无法启动数据库配置脚本。 -chinesesimplified.DbBundledMissing=安装目录中缺少内置 PostgreSQL 文件：%n%1%n%n安装包不完整，请重新下载 NetX-Setup 后再装。 -chinesesimplified.DbEnvMissing=数据库配置未写入 %ProgramData%\NetX\.env。 -chinesesimplified.ReconfigureDb=重新配置数据库 -chinesesimplified.UpgradeKeepDb=检测到已有 NetX 数据 - 将保留数据库配置（跳过向导）。 -chinesesimplified.ModePageCaption=安装模式 -chinesesimplified.ModePageDescription=检测到已有 NetX 安装。请选择更新或重装。 -chinesesimplified.ModeUpdate=更新现有安装（保留数据库配置） -chinesesimplified.ModeReinstall=重装并重新配置数据库（与首次安装相同） -chinesesimplified.ModeUpdateHint=仅替换程序文件，保留 ProgramData\.env 与数据库数据。 -chinesesimplified.ModeReinstallHint=再次显示数据库页。不会删除 ProgramData（除非卸载时选择删除）。 [Tasks] -; Run mode is exclusive: tray + service together would race for the same port. -; On upgrades CurPageChanged pre-selects whatever is actually installed. -Name: "runtray"; Description: "{cm:RunModeTray}"; GroupDescription: "{cm:RunModeGroup}"; Flags: exclusive -Name: "runtray\autostart"; Description: "{cm:EnableAutostart}" -Name: "runservice"; Description: "{cm:RunModeService}"; GroupDescription: "{cm:RunModeGroup}"; Flags: exclusive unchecked -Name: "autoupdate"; Description: "{cm:EnableAutoUpdate}"; GroupDescription: "{cm:MaintenanceGroup}"; Flags: unchecked -Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: checkedonce +Name: "desktopicon"; Description: "Create a desktop shortcut"; GroupDescription: "Additional icons:"; Flags: unchecked +Name: "firstrun"; Description: "Run first-time setup (database mode) after install"; GroupDescription: "Setup:"; Flags: checkedonce [Files] -; Entire release stage -> {app}. Exclude .portable so installed builds use ProgramData. +; Entire release stage → {app}. Exclude .portable so installed builds use ProgramData. Source: "{#StageDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs; Excludes: ".portable" -; Connection-test tools (wizard Next / silent PrepareToInstall only) -Source: "test_pg_conn.ps1"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\psql.exe"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libpq.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libssl-3-x64.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libcrypto-3-x64.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libintl-9.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libiconv-2.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libwinpthread-1.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\zlib1.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\liblz4.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libzstd.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libxml2.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libxslt.dll"; Flags: dontcopy -; VC++ runtime deployed app-local by build_release.ps1 (clean PCs lack the redistributable). -Source: "{#StageDir}\postgres\pgsql\bin\vcruntime140.dll"; Flags: dontcopy -; These helpers are extracted for PrepareToInstall, before the new program tree is copied. -Source: "{#StageDir}\packaging\_common.ps1"; Flags: dontcopy -Source: "{#StageDir}\packaging\service_lifecycle.ps1"; Flags: dontcopy [Dirs] -; users-modify so tray / non-admin reconfigure can update .env and runtime files. -Name: "{commonappdata}\NetX"; Permissions: users-modify -Name: "{commonappdata}\NetX\data"; Permissions: users-modify -Name: "{commonappdata}\NetX\data\auth"; Permissions: users-modify -Name: "{commonappdata}\NetX\data\runtime"; Permissions: users-modify -Name: "{commonappdata}\NetX\pgdata"; Permissions: users-modify -Name: "{commonappdata}\NetX\backups"; Permissions: users-modify - -[InstallDelete] -; Python minor releases can change stdlib layout (for example pathlib.py -> pathlib\\). -; Remove the previous runtime and venv before copying the new ones so an upgrade -; cannot leave orphaned modules that shadow files from the shipped Python version. -Type: filesandordirs; Name: "{app}\python\runtime" -Type: filesandordirs; Name: "{app}\.venv" -; Remove legacy desktop shortcuts from older Setup builds (keep a single NetX icon). -Type: files; Name: "{commondesktop}\NetX Start.lnk" -Type: files; Name: "{commondesktop}\NetX First-time setup.lnk" -Type: files; Name: "{commondesktop}\首次配置（内置或外置数据库）.lnk" -; Start Menu entries from <= 0.4.16 (English-only names; toggles moved into the tray / Tasks page). -Type: files; Name: "{group}\NetX Tray.lnk" -Type: files; Name: "{group}\Start NetX.lnk" -Type: files; Name: "{group}\Stop NetX.lnk" -Type: files; Name: "{group}\Check for updates.lnk" -Type: files; Name: "{group}\Open NetX UI.url" -Type: files; Name: "{group}\Install Windows Service (admin).lnk" -Type: files; Name: "{group}\Enable silent auto-update (daily).lnk" -Type: files; Name: "{group}\Enable start at logon.lnk" +Name: "{commonappdata}\NetX" +Name: "{commonappdata}\NetX\data" +Name: "{commonappdata}\NetX\data\auth" +Name: "{commonappdata}\NetX\data\runtime" +Name: "{commonappdata}\NetX\pgdata" +Name: "{commonappdata}\NetX\backups" [Icons] -; Prefer .cmd targets so Server 2012 / classic Start Menu shows normal program entries. -; Autostart / auto-update / service toggles live in the tray Settings menu and the Tasks page. -Name: "{group}\NetX"; Filename: "{app}\NetX-Tray.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\{cm:IconStart}"; Filename: "{app}\NetX-Start.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\{cm:IconStop}"; Filename: "{app}\NetX-Stop.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\{cm:IconUpdate}"; Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\launch_shortcut.ps1"" -Action update -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\{cm:ReconfigureDb}"; Filename: "{app}\NetX-FirstRun.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -; One desktop shortcut only (tray = start + tray UI). -Name: "{commondesktop}\NetX"; Filename: "{app}\NetX-Tray.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico"; Tasks: desktopicon +Name: "{group}\Start NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" +Name: "{group}\Stop NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\stop_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" +Name: "{group}\Open NetX UI"; Filename: "http://127.0.0.1:8890/" +Name: "{group}\First-time setup"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\setup_first_run.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" +Name: "{autodesktop}\Start NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Tasks: desktopicon [Run] -; Autostart / service / auto-update are applied from the Tasks page in CurStepChanged. -Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\netx_tray.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"" -StartOnLaunch"; WorkingDir: "{app}"; Description: "{cm:StartTrayNow}"; Flags: postinstall runhidden nowait skipifsilent; Check: NetxShouldStartTray - -; Stop runtime BEFORE files are deleted. Do not also Exec prepare from -; InitializeUninstall (that previously raced/killed unins000 or hung the UI). -[UninstallRun] -Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\uninstall_prepare.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; RunOnceId: "StopNetX"; Flags: runhidden waituntilterminated +Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\setup_first_run.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Flags: postinstall skipifsilent; Tasks: firstrun +Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Description: "Start NetX now"; Flags: postinstall nowait skipifsilent unchecked [UninstallDelete] -; Program files always removed. Data dir only when user chose Yes in InitializeUninstall. +; Do NOT delete {commonappdata}\NetX — preserves DB and secrets across reinstall Type: filesandordirs; Name: "{app}" - -[Code] -var - GDeleteNetxData: Boolean; - GTempDeleteDataScript: String; - DbPage: TWizardPage; - RbBundled: TNewRadioButton; - RbExternal: TNewRadioButton; - LblBundledPort: TNewStaticText; - EdBundledPort: TNewEdit; - GBundledPort: String; - LblHost: TNewStaticText; - LblPort: TNewStaticText; - LblUser: TNewStaticText; - LblPassword: TNewStaticText; - LblDbName: TNewStaticText; - EdHost: TNewEdit; - EdPort: TNewEdit; - EdUser: TNewEdit; - EdPassword: TNewEdit; - EdDbName: TNewEdit; - LblCredKey: TNewStaticText; - LblCredHint: TNewStaticText; - EdCredKey: TNewEdit; - GDbMode: String; - GDbUrl: String; - GDbHost: String; - GDbPort: String; - GDbUser: String; - GDbPassword: String; - GDbName: String; - GCredKey: String; - GSkipDbPage: Boolean; - GIsUpgrade: Boolean; - GInstallMode: String; - ModePage: TWizardPage; - RbModeUpdate: TNewRadioButton; - RbModeReinstall: TNewRadioButton; - LblModeHint: TNewStaticText; - GDbConfigured: Boolean; - GPgToolsReady: Boolean; - GDbConnOk: Boolean; - GNextBtnCaption: String; - GTasksSynced: Boolean; - -function EnvHasDbMode(): Boolean; -var - Lines: TArrayOfString; - i: Integer; - Line: String; -begin - Result := False; - if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then - Exit; - for i := 0 to GetArrayLength(Lines) - 1 do - begin - Line := Trim(Lines[i]); - if Pos('NETX_DB_MODE=', Line) = 1 then - begin - if Length(Line) > Length('NETX_DB_MODE=') then - begin - Result := True; - Exit; - end; - end; - end; -end; - -function ReadEnvValue(Key: String): String; -var - Lines: TArrayOfString; - i: Integer; - Line: String; -begin - Result := ''; - if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then - Exit; - for i := 0 to GetArrayLength(Lines) - 1 do - begin - Line := Trim(Lines[i]); - if Pos(Key + '=', Line) = 1 then - Result := Trim(Copy(Line, Length(Key) + 2, Length(Line))); - end; -end; - -function ValidBundledPort(Port: String; var ErrMsg: String): Boolean; -var - N: Integer; -begin - N := StrToIntDef(Trim(Port), -1); - Result := (N >= 1024) and (N <= 65535); - if not Result then - ErrMsg := ExpandConstant('{cm:DbBundledPortInvalid}'); -end; - -{ True when nothing listens on Port (loopback or any address). } -function TcpPortFree(Port: String): Boolean; -var - ResultCode: Integer; -begin - Result := True; - if Exec('powershell.exe', - '-NoProfile -ExecutionPolicy Bypass -Command "$p=' + Port + - '; foreach ($a in @([Net.IPAddress]::Loopback, [Net.IPAddress]::Any)) { try { $l = New-Object Net.Sockets.TcpListener($a, $p); $l.Start(); $l.Stop() } catch { exit 1 } }; exit 0"', - '', SW_HIDE, ewWaitUntilTerminated, ResultCode) then - Result := (ResultCode = 0); -end; - -function EnvModeIsExternal(): Boolean; -var - Lines: TArrayOfString; - i: Integer; - Line: String; -begin - Result := False; - if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then - Exit; - for i := 0 to GetArrayLength(Lines) - 1 do - begin - Line := Trim(Lines[i]); - if CompareText(Line, 'NETX_DB_MODE=external') = 0 then - begin - Result := True; - Exit; - end; - end; -end; - -function ParamIsTruthy(const ParamName: String): Boolean; -var - V: String; -begin - V := LowerCase(Trim(ExpandConstant('{param:' + ParamName + '|}'))); - Result := (V = '1') or (V = 'true') or (V = 'yes') or (V = 'on'); -end; - -function DetectExistingConfiguredInstall(): Boolean; -var - Lines: TArrayOfString; - i: Integer; - Line: String; -begin - { NETX_DATABASE_URL-only installs predate NETX_DB_MODE and use an external DB. } - Result := EnvHasDbMode(); - if Result then - Exit; - if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then - Exit; - for i := 0 to GetArrayLength(Lines) - 1 do - begin - Line := Trim(Lines[i]); - if (Pos('NETX_DATABASE_URL=', Line) = 1) and - (Length(Line) > Length('NETX_DATABASE_URL=')) then - begin - Result := True; - Exit; - end; - end; -end; - -procedure SyncSkipDbFromInstallMode(); -begin - GSkipDbPage := (GInstallMode = 'update'); -end; - -procedure ApplyInstallModeFromChoice(); -begin - if (RbModeReinstall <> nil) and RbModeReinstall.Checked then - GInstallMode := 'reinstall' - else if GIsUpgrade then - GInstallMode := 'update' - else - GInstallMode := 'fresh'; - SyncSkipDbFromInstallMode(); -end; - -procedure ModeChoiceClick(Sender: TObject); -begin - ApplyInstallModeFromChoice(); - if LblModeHint = nil then - Exit; - if GInstallMode = 'reinstall' then - LblModeHint.Caption := ExpandConstant('{cm:ModeReinstallHint}') - else - LblModeHint.Caption := ExpandConstant('{cm:ModeUpdateHint}'); -end; - -function ResolveInstallModeFromParams(): Boolean; -{ Returns True if a CLI param forced the mode. } -var - Mode: String; -begin - Result := False; - Mode := LowerCase(Trim(ExpandConstant('{param:InstallMode|}'))); - if (Mode = 'update') or (Mode = 'upgrade') then - begin - GInstallMode := 'update'; - Result := True; - end - else if (Mode = 'reinstall') or (Mode = 'fresh') then - begin - GInstallMode := 'reinstall'; - Result := True; - end - else if ParamIsTruthy('SkipDbPage') then - begin - GInstallMode := 'update'; - Result := True; - end - else if ParamIsTruthy('ForceDbPage') then - begin - GInstallMode := 'reinstall'; - Result := True; - end; - if Result then - SyncSkipDbFromInstallMode(); -end; - -procedure StopNetxBeforeFileReplace(); -var - ResultCode: Integer; - LifecycleScript: String; - Params: String; -begin - ExtractTemporaryFile('_common.ps1'); - ExtractTemporaryFile('service_lifecycle.ps1'); - LifecycleScript := ExpandConstant('{tmp}\service_lifecycle.ps1'); - Params := - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + LifecycleScript + '"' + - ' -Action StopForUpdate' + - ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + - ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; - if not Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode) then - RaiseException('Could not stop NetX before replacing program files.'); - if ResultCode <> 0 then - RaiseException('Could not stop NetX services/tasks before replacing program files. See ProgramData\\NetX\\data\\runtime\\service_lifecycle.log.'); -end; - -procedure ResumeNetxServiceAfterInstall(); -var - ResultCode: Integer; - LifecycleScript: String; - Params: String; -begin - LifecycleScript := ExpandConstant('{app}\packaging\service_lifecycle.ps1'); - if not FileExists(LifecycleScript) then - Exit; - Params := - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + LifecycleScript + '"' + - ' -Action ResumeAfterUpdate' + - ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + - ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; - if not Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode) then - RaiseException('NetX was updated, but its previously running service could not be restarted.'); - if ResultCode <> 0 then - RaiseException('NetX was updated, but its previously running service could not be restarted. See ProgramData\\NetX\\data\\runtime\\service_lifecycle.log.'); -end; - -function NetxShouldStartTray(): Boolean; -begin - Result := GDbConfigured or - FileExists(ExpandConstant('{commonappdata}\NetX\.env')); -end; - -procedure UpdateDbFieldState(); -var - Ext: Boolean; -begin - Ext := RbExternal.Checked; - LblHost.Enabled := Ext; - LblPort.Enabled := Ext; - LblUser.Enabled := Ext; - LblPassword.Enabled := Ext; - LblDbName.Enabled := Ext; - EdHost.Enabled := Ext; - EdPort.Enabled := Ext; - EdUser.Enabled := Ext; - EdPassword.Enabled := Ext; - EdDbName.Enabled := Ext; - LblBundledPort.Enabled := not Ext; - EdBundledPort.Enabled := not Ext; -end; - -procedure DbModeClick(Sender: TObject); -begin - UpdateDbFieldState(); -end; - -function ExtractPgClientTools(): Boolean; -begin - if GPgToolsReady and FileExists(ExpandConstant('{tmp}\psql.exe')) and - FileExists(ExpandConstant('{tmp}\test_pg_conn.ps1')) then - begin - Result := True; - Exit; - end; - Result := False; - try - ExtractTemporaryFile('test_pg_conn.ps1'); - ExtractTemporaryFile('psql.exe'); - ExtractTemporaryFile('libpq.dll'); - ExtractTemporaryFile('libssl-3-x64.dll'); - ExtractTemporaryFile('libcrypto-3-x64.dll'); - ExtractTemporaryFile('libintl-9.dll'); - ExtractTemporaryFile('libiconv-2.dll'); - ExtractTemporaryFile('libwinpthread-1.dll'); - ExtractTemporaryFile('zlib1.dll'); - ExtractTemporaryFile('liblz4.dll'); - ExtractTemporaryFile('libzstd.dll'); - ExtractTemporaryFile('libxml2.dll'); - ExtractTemporaryFile('libxslt.dll'); - ExtractTemporaryFile('vcruntime140.dll'); - Result := FileExists(ExpandConstant('{tmp}\psql.exe')) and - FileExists(ExpandConstant('{tmp}\test_pg_conn.ps1')); - GPgToolsReady := Result; - except - Result := False; - GPgToolsReady := False; - end; -end; - -procedure SetWizardBusy(const Busy: Boolean; const StatusText: String); -begin - if Busy then - begin - if GNextBtnCaption = '' then - GNextBtnCaption := WizardForm.NextButton.Caption; - WizardForm.NextButton.Caption := StatusText; - WizardForm.NextButton.Enabled := False; - WizardForm.BackButton.Enabled := False; - WizardForm.CancelButton.Enabled := False; - end - else - begin - if GNextBtnCaption <> '' then - WizardForm.NextButton.Caption := GNextBtnCaption; - WizardForm.NextButton.Enabled := True; - WizardForm.BackButton.Enabled := True; - WizardForm.CancelButton.Enabled := True; - end; - WizardForm.Refresh; -end; - -function TestExternalDb(const Host, Port, User, Password, DbName: String; var ErrMsg: String): Boolean; -var - ResultCode: Integer; - UrlFile: String; - ErrFile: String; - PwFile: String; - Params: String; - Lines: TArrayOfString; - i: Integer; -begin - Result := False; - ErrMsg := ''; - if not ExtractPgClientTools() then - begin - ErrMsg := ExpandConstant('{cm:DbTestExtractFailed}'); - Exit; - end; - - UrlFile := ExpandConstant('{tmp}\netx_db_url.txt'); - ErrFile := ExpandConstant('{tmp}\netx_db_test_err.txt'); - PwFile := ExpandConstant('{tmp}\netx_db_pw.txt'); - DeleteFile(UrlFile); - DeleteFile(ErrFile); - SaveStringToFile(PwFile, Password, False); - - Params := - '-NoProfile -ExecutionPolicy Bypass -File "' + ExpandConstant('{tmp}\test_pg_conn.ps1') + '"' + - ' -PgHost "' + Host + '"' + - ' -Port ' + Port + - ' -User "' + User + '"' + - ' -PasswordFile "' + PwFile + '"' + - ' -Database "' + DbName + '"' + - ' -PsqlPath "' + ExpandConstant('{tmp}\psql.exe') + '"' + - ' -OutUrlFile "' + UrlFile + '"' + - ' -OutErrFile "' + ErrFile + '"'; - - if not Exec('powershell.exe', Params, ExpandConstant('{tmp}'), SW_HIDE, ewWaitUntilTerminated, ResultCode) then - begin - ErrMsg := 'powershell_exec_failed'; - DeleteFile(PwFile); - Exit; - end; - DeleteFile(PwFile); - if ResultCode <> 0 then - begin - ErrMsg := 'psql/test exit ' + IntToStr(ResultCode); - if LoadStringsFromFile(ErrFile, Lines) then - begin - ErrMsg := ''; - for i := 0 to GetArrayLength(Lines) - 1 do - begin - if ErrMsg <> '' then - ErrMsg := ErrMsg + #13#10; - ErrMsg := ErrMsg + Lines[i]; - end; - if ErrMsg = '' then - ErrMsg := 'psql/test exit ' + IntToStr(ResultCode); - end; - Exit; - end; - if not LoadStringsFromFile(UrlFile, Lines) or (GetArrayLength(Lines) < 1) or (Trim(Lines[0]) = '') then - begin - ErrMsg := 'url_file_missing'; - Exit; - end; - GDbUrl := Trim(Lines[0]); - Result := True; -end; - -function SaveExternalFieldsFromControls(var ErrMsg: String): Boolean; -begin - Result := False; - GDbHost := Trim(EdHost.Text); - GDbPort := Trim(EdPort.Text); - GDbUser := Trim(EdUser.Text); - GDbPassword := EdPassword.Text; - GDbName := Trim(EdDbName.Text); - GDbConnOk := False; - GDbUrl := ''; - if (GDbHost = '') or (GDbPort = '') or (GDbUser = '') or - (GDbPassword = '') or (GDbName = '') then - begin - ErrMsg := ExpandConstant('{cm:DbFieldsRequired}'); - Exit; - end; - GDbMode := 'external'; - Result := True; -end; - -function ValidateExternalDbSaved(var ErrMsg: String): Boolean; -begin - Result := False; - if GDbConnOk and (GDbUrl <> '') then - begin - Result := True; - Exit; - end; - if not TestExternalDb(GDbHost, GDbPort, GDbUser, GDbPassword, GDbName, ErrMsg) then - begin - if ErrMsg = '' then - ErrMsg := 'connection_failed'; - ErrMsg := FmtMessage(ExpandConstant('{cm:DbTestFailed}'), [ErrMsg]); - GDbConnOk := False; - Exit; - end; - GDbConnOk := True; - Result := True; -end; - -function ApplySilentDbParams(var ErrMsg: String): Boolean; -var - Mode, Host, Port, User, Password, DbName: String; -begin - Result := False; - GCredKey := Trim(ExpandConstant('{param:CredentialSecretKey|}')); - Mode := LowerCase(ExpandConstant('{param:DbMode|bundled}')); - if Mode = '' then - Mode := 'bundled'; - if Mode = 'bundled' then - begin - GBundledPort := Trim(ExpandConstant('{param:BundledPgPort|}')); - if GBundledPort = '' then - GBundledPort := ReadEnvValue('NETX_BUNDLED_PG_PORT'); - if GBundledPort = '' then - GBundledPort := '15432'; - if not ValidBundledPort(GBundledPort, ErrMsg) then - Exit; - GDbMode := 'bundled'; - GDbUrl := ''; - Result := True; - Exit; - end; - if Mode <> 'external' then - begin - ErrMsg := 'invalid DbMode (use bundled or external)'; - Exit; - end; - Host := ExpandConstant('{param:DbHost|}'); - Port := ExpandConstant('{param:DbPort|5432}'); - User := ExpandConstant('{param:DbUser|}'); - Password := ExpandConstant('{param:DbPassword|}'); - DbName := ExpandConstant('{param:DbName|}'); - if (Host = '') or (User = '') or (Password = '') or (DbName = '') then - begin - ErrMsg := ExpandConstant('{cm:DbSilentExternalMissing}'); - Exit; - end; - if not TestExternalDb(Host, Port, User, Password, DbName, ErrMsg) then - begin - if ErrMsg = '' then - ErrMsg := 'connection_failed'; - ErrMsg := FmtMessage(ExpandConstant('{cm:DbTestFailed}'), [ErrMsg]); - Exit; - end; - GDbMode := 'external'; - Result := True; -end; - -procedure CaptureCredKeyFromControls(); -begin - if EdCredKey <> nil then - GCredKey := Trim(EdCredKey.Text) - else - GCredKey := ''; -end; - -procedure InitializeWizard(); -var - TopPos: Integer; -begin - GDbMode := 'bundled'; - GDbUrl := ''; - GDbHost := ''; - GDbPort := '5432'; - GDbUser := ''; - GDbPassword := ''; - GDbName := ''; - GCredKey := ''; - GBundledPort := ReadEnvValue('NETX_BUNDLED_PG_PORT'); - if GBundledPort = '' then - GBundledPort := '15432'; - GDbConfigured := False; - GPgToolsReady := False; - GDbConnOk := False; - { Existing install: interactive Mode page (Update vs Reinstall). - Silent: /InstallMode=update|reinstall (or /SkipDbPage=1 /ForceDbPage=1). } - GIsUpgrade := DetectExistingConfiguredInstall(); - ModePage := nil; - RbModeUpdate := nil; - RbModeReinstall := nil; - LblModeHint := nil; - if not ResolveInstallModeFromParams() then - begin - if GIsUpgrade then - GInstallMode := 'update' - else - GInstallMode := 'fresh'; - SyncSkipDbFromInstallMode(); - end; - - if GIsUpgrade and (not WizardSilent) then - begin - ModePage := CreateCustomPage(wpSelectDir, - ExpandConstant('{cm:ModePageCaption}'), - ExpandConstant('{cm:ModePageDescription}')); - - RbModeUpdate := TNewRadioButton.Create(ModePage); - RbModeUpdate.Parent := ModePage.Surface; - RbModeUpdate.Caption := ExpandConstant('{cm:ModeUpdate}'); - RbModeUpdate.Checked := (GInstallMode <> 'reinstall'); - RbModeUpdate.Top := ScaleY(8); - RbModeUpdate.Left := ScaleX(0); - RbModeUpdate.Width := ModePage.SurfaceWidth; - RbModeUpdate.OnClick := @ModeChoiceClick; - - RbModeReinstall := TNewRadioButton.Create(ModePage); - RbModeReinstall.Parent := ModePage.Surface; - RbModeReinstall.Caption := ExpandConstant('{cm:ModeReinstall}'); - RbModeReinstall.Checked := (GInstallMode = 'reinstall'); - RbModeReinstall.Top := RbModeUpdate.Top + ScaleY(28); - RbModeReinstall.Left := ScaleX(0); - RbModeReinstall.Width := ModePage.SurfaceWidth; - RbModeReinstall.OnClick := @ModeChoiceClick; - - LblModeHint := TNewStaticText.Create(ModePage); - LblModeHint.Parent := ModePage.Surface; - LblModeHint.Top := RbModeReinstall.Top + ScaleY(36); - LblModeHint.Left := ScaleX(0); - LblModeHint.Width := ModePage.SurfaceWidth; - LblModeHint.Height := ScaleY(60); - LblModeHint.AutoSize := False; - LblModeHint.WordWrap := True; - ModeChoiceClick(nil); - - DbPage := CreateCustomPage(ModePage.ID, - ExpandConstant('{cm:DbPageCaption}'), - ExpandConstant('{cm:DbPageDescription}')); - end - else - DbPage := CreateCustomPage(wpSelectDir, - ExpandConstant('{cm:DbPageCaption}'), - ExpandConstant('{cm:DbPageDescription}')); - - RbBundled := TNewRadioButton.Create(DbPage); - RbBundled.Parent := DbPage.Surface; - RbBundled.Caption := ExpandConstant('{cm:DbBundled}'); - RbBundled.Checked := True; - RbBundled.Top := ScaleY(8); - RbBundled.Left := ScaleX(0); - RbBundled.Width := DbPage.SurfaceWidth - ScaleX(150); - RbBundled.OnClick := @DbModeClick; - - { Bundled PostgreSQL port (127.0.0.1 only), on the same row as its radio button. } - LblBundledPort := TNewStaticText.Create(DbPage); - LblBundledPort.Parent := DbPage.Surface; - LblBundledPort.Caption := ExpandConstant('{cm:DbBundledPort}'); - LblBundledPort.Top := RbBundled.Top + ScaleY(2); - LblBundledPort.Left := DbPage.SurfaceWidth - ScaleX(140); - - EdBundledPort := TNewEdit.Create(DbPage); - EdBundledPort.Parent := DbPage.Surface; - EdBundledPort.Top := RbBundled.Top - ScaleY(2); - EdBundledPort.Left := DbPage.SurfaceWidth - ScaleX(80); - EdBundledPort.Width := ScaleX(80); - EdBundledPort.Text := GBundledPort; - - RbExternal := TNewRadioButton.Create(DbPage); - RbExternal.Parent := DbPage.Surface; - RbExternal.Caption := ExpandConstant('{cm:DbExternal}'); - RbExternal.Top := RbBundled.Top + ScaleY(28); - RbExternal.Left := ScaleX(0); - RbExternal.Width := DbPage.SurfaceWidth; - RbExternal.OnClick := @DbModeClick; - - TopPos := RbExternal.Top + ScaleY(36); - - LblHost := TNewStaticText.Create(DbPage); - LblHost.Parent := DbPage.Surface; - LblHost.Caption := ExpandConstant('{cm:DbHost}'); - LblHost.Top := TopPos; - LblHost.Left := ScaleX(16); - - EdHost := TNewEdit.Create(DbPage); - EdHost.Parent := DbPage.Surface; - EdHost.Top := TopPos; - EdHost.Left := ScaleX(120); - EdHost.Width := ScaleX(220); - EdHost.Text := '127.0.0.1'; - - TopPos := TopPos + ScaleY(28); - LblPort := TNewStaticText.Create(DbPage); - LblPort.Parent := DbPage.Surface; - LblPort.Caption := ExpandConstant('{cm:DbPort}'); - LblPort.Top := TopPos; - LblPort.Left := ScaleX(16); - - EdPort := TNewEdit.Create(DbPage); - EdPort.Parent := DbPage.Surface; - EdPort.Top := TopPos; - EdPort.Left := ScaleX(120); - EdPort.Width := ScaleX(80); - EdPort.Text := '5432'; - - TopPos := TopPos + ScaleY(28); - LblUser := TNewStaticText.Create(DbPage); - LblUser.Parent := DbPage.Surface; - LblUser.Caption := ExpandConstant('{cm:DbUser}'); - LblUser.Top := TopPos; - LblUser.Left := ScaleX(16); - - EdUser := TNewEdit.Create(DbPage); - EdUser.Parent := DbPage.Surface; - EdUser.Top := TopPos; - EdUser.Left := ScaleX(120); - EdUser.Width := ScaleX(220); - EdUser.Text := 'netx'; - - TopPos := TopPos + ScaleY(28); - LblPassword := TNewStaticText.Create(DbPage); - LblPassword.Parent := DbPage.Surface; - LblPassword.Caption := ExpandConstant('{cm:DbPassword}'); - LblPassword.Top := TopPos; - LblPassword.Left := ScaleX(16); - - EdPassword := TNewEdit.Create(DbPage); - EdPassword.Parent := DbPage.Surface; - EdPassword.Top := TopPos; - EdPassword.Left := ScaleX(120); - EdPassword.Width := ScaleX(220); - EdPassword.PasswordChar := '*'; - - TopPos := TopPos + ScaleY(28); - LblDbName := TNewStaticText.Create(DbPage); - LblDbName.Parent := DbPage.Surface; - LblDbName.Caption := ExpandConstant('{cm:DbName}'); - LblDbName.Top := TopPos; - LblDbName.Left := ScaleX(16); - - EdDbName := TNewEdit.Create(DbPage); - EdDbName.Parent := DbPage.Surface; - EdDbName.Top := TopPos; - EdDbName.Left := ScaleX(120); - EdDbName.Width := ScaleX(220); - EdDbName.Text := 'netx'; - - TopPos := TopPos + ScaleY(36); - LblCredKey := TNewStaticText.Create(DbPage); - LblCredKey.Parent := DbPage.Surface; - LblCredKey.Caption := ExpandConstant('{cm:CredKey}'); - LblCredKey.Top := TopPos; - LblCredKey.Left := ScaleX(0); - - EdCredKey := TNewEdit.Create(DbPage); - EdCredKey.Parent := DbPage.Surface; - EdCredKey.Top := TopPos; - EdCredKey.Left := ScaleX(120); - EdCredKey.Width := DbPage.SurfaceWidth - ScaleX(120); - EdCredKey.Text := ''; - EdCredKey.PasswordChar := '*'; - - TopPos := TopPos + ScaleY(24); - LblCredHint := TNewStaticText.Create(DbPage); - LblCredHint.Parent := DbPage.Surface; - LblCredHint.Caption := ExpandConstant('{cm:CredKeyHint}'); - LblCredHint.Top := TopPos; - LblCredHint.Left := ScaleX(120); - LblCredHint.Width := DbPage.SurfaceWidth - ScaleX(120); - LblCredHint.AutoSize := False; - - UpdateDbFieldState(); - - { Prefill from existing ProgramData\.env when present. } - if EnvModeIsExternal() then - begin - RbExternal.Checked := True; - UpdateDbFieldState(); - end - else if EnvHasDbMode() then - begin - RbBundled.Checked := True; - UpdateDbFieldState(); - end; - - if WizardSilent then - begin - { Params applied in PrepareToInstall } - end; -end; - -function ShouldSkipPage(PageID: Integer): Boolean; -begin - Result := False; - if (ModePage <> nil) and (PageID = ModePage.ID) then - begin - { Only interactive existing installs show the mode page. } - Result := (not GIsUpgrade) or WizardSilent; - Exit; - end; - if (DbPage <> nil) and (PageID = DbPage.ID) then - Result := GSkipDbPage; -end; - -function NextButtonClick(CurPageID: Integer): Boolean; -var - ErrMsg: String; -begin - Result := True; - if (ModePage <> nil) and (CurPageID = ModePage.ID) then - begin - ApplyInstallModeFromChoice(); - Exit; - end; - { Only the DB page runs validation. Other pages must always proceed - (Tasks/Ready used to appear "dead" when CloseApplications hung). } - if (DbPage = nil) or (CurPageID <> DbPage.ID) or GSkipDbPage then - Exit; - - CaptureCredKeyFromControls(); - if RbBundled.Checked then - begin - GBundledPort := Trim(EdBundledPort.Text); - if not ValidBundledPort(GBundledPort, ErrMsg) then - begin - MsgBox(ErrMsg, mbError, MB_OK); - Result := False; - Exit; - end; - { The current install's own PG may hold its port; only probe a changed port. } - if (GBundledPort <> ReadEnvValue('NETX_BUNDLED_PG_PORT')) and (not TcpPortFree(GBundledPort)) then - begin - MsgBox(FmtMessage(ExpandConstant('{cm:DbBundledPortInUse}'), [GBundledPort]), mbError, MB_OK); - Result := False; - Exit; - end; - GDbMode := 'bundled'; - GDbUrl := ''; - GDbConnOk := True; - Result := True; - Exit; - end; - - if not SaveExternalFieldsFromControls(ErrMsg) then - begin - MsgBox(ErrMsg, mbError, MB_OK); - Result := False; - Exit; - end; - - { Connection test here so bad settings never reach Tasks/Ready. - Show busy state — ExtractTemporaryFile + psql can take a few seconds. } - SetWizardBusy(True, ExpandConstant('{cm:DbTesting}')); - try - Result := ValidateExternalDbSaved(ErrMsg); - finally - SetWizardBusy(False, ''); - end; - if not Result then - MsgBox(ErrMsg, mbError, MB_OK); -end; - -function PrepareToInstall(var NeedsRestart: Boolean): String; -var - ErrMsg: String; -begin - Result := ''; - NeedsRestart := False; - - if (ModePage <> nil) and (not WizardSilent) then - ApplyInstallModeFromChoice(); - - { Stop tray / API / bundled PG so Program Files can be replaced safely. } - if GSkipDbPage or GIsUpgrade or DirExists(ExpandConstant('{app}\packaging')) then - StopNetxBeforeFileReplace(); - - if GSkipDbPage then - Exit; - - if WizardSilent then - begin - if not ApplySilentDbParams(ErrMsg) then - Result := ErrMsg; - Exit; - end; - - if GDbMode = '' then - GDbMode := 'bundled'; - - { Re-check only if user never got a successful test (should be rare). } - if (GDbMode = 'external') and (not GDbConnOk) then - begin - if not ValidateExternalDbSaved(ErrMsg) then - Result := ErrMsg; - end; -end; - -function ApplyDatabaseConfig(): Boolean; -var - ResultCode: Integer; - Params: String; - UrlFile: String; - KeyFile: String; - LogFile: String; - InitDb: String; - EnvFile: String; - Wrapper: String; - WrapBody: String; -begin - Result := False; - GDbConfigured := False; - if GDbMode = '' then - GDbMode := 'bundled'; - - InitDb := ExpandConstant('{app}\postgres\pgsql\bin\initdb.exe'); - if (GDbMode = 'bundled') and (not FileExists(InitDb)) then - begin - MsgBox(FmtMessage(ExpandConstant('{cm:DbBundledMissing}'), [InitDb]), mbError, MB_OK); - Exit; - end; - - ForceDirectories(ExpandConstant('{commonappdata}\NetX\data\runtime')); - LogFile := ExpandConstant('{commonappdata}\NetX\data\runtime\setup_first_run.log'); - Wrapper := ExpandConstant('{tmp}\netx_apply_db.ps1'); - - WrapBody := - '$ErrorActionPreference = ''Stop''' + #13#10 + - '$log = ''' + LogFile + '''' + #13#10 + - 'Start-Transcript -Path $log -Force | Out-Null' + #13#10 + - 'try {' + #13#10 + - ' & ''' + ExpandConstant('{app}\packaging\setup_first_run.ps1') + '''' + - ' -ProgramRoot ''' + ExpandConstant('{app}') + '''' + - ' -DataRoot ''' + ExpandConstant('{commonappdata}\NetX') + '''' + - ' -NonInteractive -DbMode ' + GDbMode; - - if (GDbMode = 'bundled') and (GBundledPort <> '') then - WrapBody := WrapBody + ' -BundledPort ' + GBundledPort; - - if GDbMode = 'external' then - begin - UrlFile := ExpandConstant('{tmp}\netx_db_url.txt'); - if GDbUrl <> '' then - SaveStringToFile(UrlFile, GDbUrl, False); - WrapBody := WrapBody + - ' -ExternalDatabaseUrlFile ''' + UrlFile + ''' -SkipExternalProbe'; - end; - - if GCredKey <> '' then - begin - KeyFile := ExpandConstant('{tmp}\netx_cred_key.txt'); - SaveStringToFile(KeyFile, GCredKey, False); - WrapBody := WrapBody + ' -CredentialSecretKeyFile ''' + KeyFile + ''''; - end; - - WrapBody := WrapBody + #13#10 + - ' if ($null -ne $LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE }' + #13#10 + - '} catch {' + #13#10 + - ' Write-Error $_' + #13#10 + - ' exit 1' + #13#10 + - '} finally {' + #13#10 + - ' Stop-Transcript | Out-Null' + #13#10 + - '}' + #13#10; - - SaveStringToFile(Wrapper, WrapBody, False); - Params := '-NoProfile -ExecutionPolicy Bypass -File "' + Wrapper + '"'; - - if not Exec( - 'powershell.exe', Params, ExpandConstant('{app}'), - SW_HIDE, ewWaitUntilTerminated, ResultCode) then - begin - MsgBox(ExpandConstant('{cm:DbApplyExecFailed}'), mbError, MB_OK); - Exit; - end; - if ResultCode <> 0 then - begin - MsgBox(FmtMessage(ExpandConstant('{cm:DbApplyFailed}'), [IntToStr(ResultCode), LogFile]), mbError, MB_OK); - Exit; - end; - - EnvFile := ExpandConstant('{commonappdata}\NetX\.env'); - if not FileExists(EnvFile) then - begin - MsgBox(ExpandConstant('{cm:DbEnvMissing}'), mbError, MB_OK); - Exit; - end; - - GDbConfigured := True; - Result := True; -end; - -function InitializeUninstall(): Boolean; -var - DeleteSrc: String; - LifecycleScript: String; - Params: String; - ResultCode: Integer; -begin - Result := True; - GDeleteNetxData := False; - GTempDeleteDataScript := ExpandConstant('{tmp}\netx_uninstall_delete_data.ps1'); - - // Copy data-wipe helper to TEMP before program files are removed. - DeleteSrc := ExpandConstant('{app}\packaging\uninstall_delete_data.ps1'); - if FileExists(DeleteSrc) then - CopyFile(DeleteSrc, GTempDeleteDataScript, False); - - if UninstallSilent then - begin - // Settings often uses QuietUninstallString; keep data by default. - GDeleteNetxData := False; - end - else - begin - GDeleteNetxData := - MsgBox(ExpandConstant('{cm:UninstallDeleteData}'), mbConfirmation, MB_YESNO) = IDYES; - end; - - LifecycleScript := ExpandConstant('{app}\packaging\service_lifecycle.ps1'); - if FileExists(LifecycleScript) then - begin - Params := - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + LifecycleScript + '"' + - ' -Action RemoveForUninstall' + - ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + - ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; - if (not Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode)) or - (ResultCode <> 0) then - begin - MsgBox('Could not stop/remove the NetX Windows service or startup task. Uninstall has been cancelled to protect the installation. See ProgramData\\NetX\\data\\runtime\\service_lifecycle.log.', mbError, MB_OK); - Result := False; - Exit; - end; - end; -end; - -procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep); -var - ResultCode: Integer; - DataRoot: String; -begin - if (CurUninstallStep = usPostUninstall) and GDeleteNetxData then - begin - DataRoot := ExpandConstant('{commonappdata}\NetX'); - if FileExists(GTempDeleteDataScript) then - begin - Exec( - 'powershell.exe', - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + - GTempDeleteDataScript + '" -DataRoot "' + DataRoot + '"', - ExpandConstant('{tmp}'), - SW_HIDE, - ewWaitUntilTerminated, - ResultCode - ); - end; - if DirExists(DataRoot) then - DelTree(DataRoot, True, True, True); - end; -end; - -procedure RepairNetxVenvAfterInstall(); -var - ResultCode: Integer; - RepairScript: String; - Params: String; -begin - RepairScript := ExpandConstant('{app}\packaging\repair_venv.ps1'); - if not FileExists(RepairScript) then - Exit; - Params := - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + RepairScript + '"' + - ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + - ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; - Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode); -end; - -function NetxTaskExists(const Name: String): Boolean; -var - RC: Integer; -begin - Result := Exec(ExpandConstant('{sys}\schtasks.exe'), '/Query /TN "\NetX\' + Name + '"', '', - SW_HIDE, ewWaitUntilTerminated, RC) and (RC = 0); -end; - -function NetxServiceExists(): Boolean; -var - RC: Integer; -begin - Result := Exec(ExpandConstant('{sys}\sc.exe'), 'query NetX', '', SW_HIDE, ewWaitUntilTerminated, RC) and (RC = 0); -end; - -procedure CurPageChanged(CurPageID: Integer); -begin - { Existing install: show the run mode / toggles that are really in place - (older versions set them from Finish-page checkboxes Inno never recorded). } - if (CurPageID = wpSelectTasks) and (not GTasksSynced) then - begin - GTasksSynced := True; - if FileExists(ExpandConstant('{app}\packaging\netx_tray.ps1')) then - begin - if NetxServiceExists() then - WizardSelectTasks('runservice') - else - WizardSelectTasks('runtray'); - if NetxTaskExists('NetXTray') then - WizardSelectTasks('runtray\autostart') - else - WizardSelectTasks('!runtray\autostart'); - if NetxTaskExists('NetXUpdate') then - WizardSelectTasks('autoupdate') - else - WizardSelectTasks('!autoupdate'); - end; - end; -end; - -function RunNetxScript(const Script, Extra: String): Integer; -var - Params: String; -begin - Params := - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + ExpandConstant('{app}\packaging\' + Script) + '"' + - ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + - ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"' + Extra; - if not Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, Result) then - Result := -1; -end; - -function TasksGivenOnCommandLine(): Boolean; -var - i: Integer; - P: String; -begin - Result := False; - for i := 1 to ParamCount do - begin - P := Uppercase(ParamStr(i)); - if (Pos('/TASKS=', P) = 1) or (Pos('/MERGETASKS=', P) = 1) then - begin - Result := True; - Exit; - end; - end; -end; - -procedure ApplyRunModeTasks(); -var - Code: Integer; -begin - { Silent auto-updates keep whatever is installed unless /TASKS is passed. } - if WizardSilent and (not TasksGivenOnCommandLine()) then - Exit; - if not FileExists(ExpandConstant('{commonappdata}\NetX\.env')) then - Exit; - - if WizardIsTaskSelected('runservice') then - begin - Code := RunNetxScript('install_service.ps1', ' -Start'); - if Code <> 0 then - SuppressibleMsgBox(FmtMessage(ExpandConstant('{cm:ServiceInstallFailed}'), [IntToStr(Code), - ExpandConstant('{commonappdata}\NetX\data\runtime')]), mbError, MB_OK, IDOK); - end - else if NetxServiceExists() then - RunNetxScript('install_service.ps1', ' -Uninstall'); - - if WizardIsTaskSelected('runtray\autostart') then - RunNetxScript('install_autostart.ps1', '') - else - RunNetxScript('install_autostart.ps1', ' -Remove'); - - if WizardIsTaskSelected('autoupdate') then - RunNetxScript('install_update_task.ps1', '') - else if NetxTaskExists('NetXUpdate') then - RunNetxScript('install_update_task.ps1', ' -Remove'); -end; - -procedure CurStepChanged(CurStep: TSetupStep); -var - UninstallKey: String; -begin - if CurStep = ssPostInstall then - begin - { Always relink .venv -> local python/runtime (Setup is elevated). } - RepairNetxVenvAfterInstall(); - - { Upgrade keeps existing .env / pgdata - do not re-run DB wizard. } - if not GSkipDbPage then - ApplyDatabaseConfig() - else - GDbConfigured := FileExists(ExpandConstant('{commonappdata}\NetX\.env')); - - UninstallKey := - 'Software\Microsoft\Windows\CurrentVersion\Uninstall\{A7E3C2D1-9F40-4B8E-9C1A-NETXWIN64001}_is1'; - RegDeleteValue(HKLM, UninstallKey, 'QuietUninstallString'); - ApplyRunModeTasks(); - ResumeNetxServiceAfterInstall(); - end; -end; diff --git a/packaging/installer/test_pg_conn.ps1 b/packaging/installer/test_pg_conn.ps1 deleted file mode 100644 index efc8323..0000000 --- a/packaging/installer/test_pg_conn.ps1 +++ /dev/null @@ -1,104 +0,0 @@ -param( - [Parameter(Mandatory = $true)][string]$PgHost, - [int]$Port = 5432, - [Parameter(Mandatory = $true)][string]$User, - [string]$Password = "", - [string]$PasswordFile = "", - [Parameter(Mandatory = $true)][string]$Database, - [Parameter(Mandatory = $true)][string]$PsqlPath, - [string]$OutUrlFile = "", - [string]$OutErrFile = "" -) - -$ErrorActionPreference = "Stop" - -function Write-ErrFile([string]$Message) { - if ($OutErrFile) { - Set-Content -LiteralPath $OutErrFile -Value $Message -Encoding utf8 - } - [Console]::Error.WriteLine($Message) -} - -if (-not (Test-Path -LiteralPath $PsqlPath)) { - Write-ErrFile "psql_not_found: $PsqlPath" - exit 2 -} -if ($PasswordFile) { - if (-not (Test-Path -LiteralPath $PasswordFile)) { - Write-ErrFile "password_file_missing" - exit 3 - } - $Password = [IO.File]::ReadAllText($PasswordFile).TrimEnd("`r", "`n") -} -if ([string]::IsNullOrWhiteSpace($PgHost)) { - Write-ErrFile "host_required" - exit 3 -} -if ([string]::IsNullOrWhiteSpace($User) -or [string]::IsNullOrWhiteSpace($Database)) { - Write-ErrFile "user_and_database_required" - exit 3 -} -if ([string]::IsNullOrWhiteSpace($Password)) { - Write-ErrFile "password_required" - exit 3 -} -if ($Port -lt 1 -or $Port -gt 65535) { - Write-ErrFile "invalid_port: $Port" - exit 3 -} - -$encUser = [uri]::EscapeDataString($User) -$encPw = [uri]::EscapeDataString($Password) -$encDb = [uri]::EscapeDataString($Database) -$url = "postgresql+psycopg://${encUser}:${encPw}@${PgHost}:${Port}/${encDb}" - -$psqlDir = Split-Path -Parent $PsqlPath -$prevPath = $env:PATH -$prevPw = $env:PGPASSWORD -try { - $env:PATH = "$psqlDir;$env:PATH" - $env:PGPASSWORD = $Password - $psi = New-Object System.Diagnostics.ProcessStartInfo - $psi.FileName = $PsqlPath - $psi.Arguments = "-h `"$PgHost`" -p $Port -U `"$User`" -d `"$Database`" -v ON_ERROR_STOP=1 -t -A -c `"SELECT 1`"" - $psi.UseShellExecute = $false - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - $psi.CreateNoWindow = $true - $psi.WorkingDirectory = $psqlDir - $p = [Diagnostics.Process]::Start($psi) - $stdout = $p.StandardOutput.ReadToEnd() - $stderr = $p.StandardError.ReadToEnd() - $p.WaitForExit() - if ($p.ExitCode -ne 0) { - $msg = (($stderr + "`n" + $stdout).Trim()) - if (-not $msg) { $msg = "psql_exit_$($p.ExitCode)" } - Write-ErrFile "connection_failed: $msg" - exit 1 - } - if (($stdout.Trim()) -notmatch '1') { - Write-ErrFile "unexpected_result: $($stdout.Trim())" - exit 1 - } -} catch { - Write-ErrFile ("connection_failed: " + $_.Exception.Message) - exit 1 -} finally { - $env:PATH = $prevPath - if ($null -eq $prevPw) { - Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue - } else { - $env:PGPASSWORD = $prevPw - } -} - -if ($OutUrlFile) { - $dir = Split-Path -Parent $OutUrlFile - if ($dir -and -not (Test-Path -LiteralPath $dir)) { - New-Item -ItemType Directory -Path $dir -Force | Out-Null - } - Set-Content -LiteralPath $OutUrlFile -Value $url -Encoding ascii -NoNewline -} - -Write-Output "ok" -exit 0 diff --git a/packaging/launch_shortcut.ps1 b/packaging/launch_shortcut.ps1 deleted file mode 100644 index fcc5fa2..0000000 --- a/packaging/launch_shortcut.ps1 +++ /dev/null @@ -1,129 +0,0 @@ -param( - [ValidateSet("tray", "start", "stop", "setup", "update")] - [string]$Action = "tray", - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [switch]$StartOnLaunch = $true -) - -# Visible entrypoint for Start Menu / desktop shortcuts. -# Nested PowerShell runs are hidden; only failures show a message box. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden - -Add-Type -AssemblyName System.Windows.Forms - -$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') -function T([string]$En, [string]$Zh) { - if ($zh) { return $Zh } else { return $En } -} - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$logDir = Join-Path $data "data\runtime" -if (-not (Test-Path $logDir)) { - New-Item -ItemType Directory -Path $logDir -Force | Out-Null -} -$log = Join-Path $logDir "launch.log" - -function Write-LaunchLog([string]$Msg) { - $line = "[{0}] {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Msg - Add-Content -Path $log -Value $line -Encoding utf8 -} - -function Show-NetxError([string]$Msg) { - Write-LaunchLog "ERROR $Msg" - [void][System.Windows.Forms.MessageBox]::Show( - $Msg, - "NetX", - [System.Windows.Forms.MessageBoxButtons]::OK, - [System.Windows.Forms.MessageBoxIcon]::Error - ) -} - -function Invoke-NetxHiddenPs1 { - param( - [string]$File, - [string[]]$ExtraArgs = @(), - [switch]$NoWait = $false - ) - return Start-NetxPowerShell -File $File ` - -Arguments (@("-ProgramRoot", $prog, "-DataRoot", $data) + $ExtraArgs) ` - -WorkingDirectory $prog -WindowStyle Hidden -PassThru -Wait:(-not $NoWait) -} - -function Ensure-NetxEnv { - $envPath = Join-Path $data ".env" - if (Test-Path $envPath) { return } - $cmd = Join-Path $prog "NetX-FirstRun.cmd" - throw (T ` - "NetX is not configured yet (missing $envPath).`nRe-run Setup and choose built-in or external DB,`nor Start Menu → Reconfigure database:`n$cmd" ` - "尚未完成数据库配置（缺少 $envPath）。`n请重新运行安装向导选择内置/外置库，`n或开始菜单 → 重新配置数据库：`n$cmd") -} - -try { - Write-LaunchLog "action=$Action prog=$prog data=$data" - switch ($Action) { - "setup" { - # Visible console so user can pick bundled vs external DB. - $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot "setup_first_run.ps1") ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` - -WorkingDirectory $prog -WindowStyle Normal -Wait -PassThru - if ($p.ExitCode -ne 0) { throw "setup_exit_$($p.ExitCode)" } - if (Test-Path (Join-Path $data ".env")) { - Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-StartOnLaunch") ` - -WorkingDirectory $prog -WindowStyle Hidden | Out-Null - } - } - "stop" { - $p = Invoke-NetxHiddenPs1 -File (Join-Path $PSScriptRoot "runtime_ctl.ps1") -ExtraArgs @("-Action", "stop") - if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0) { throw "stop_exit_$($p.ExitCode)" } - } - "update" { - Ensure-NetxEnv - # Update UI may need a visible window for prompts. - $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot "check_update.ps1") ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` - -WorkingDirectory $prog -WindowStyle Normal -Wait -PassThru - if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0 -and $p.ExitCode -ne 10) { - throw "update_exit_$($p.ExitCode)" - } - } - "start" { - Ensure-NetxEnv - $hostBind = "127.0.0.1" - $port = 8890 - $envPath = Join-Path $data ".env" - if (Test-Path $envPath) { - $map = Read-DotEnv -Path $envPath - if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } - if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } - } - $hostBind = Get-NetxClientHost -BindHost $hostBind - $p = Invoke-NetxHiddenPs1 -File (Join-Path $PSScriptRoot "runtime_ctl.ps1") -ExtraArgs @("-Action", "start") - if ($p.ExitCode -ne 0) { throw "start_exit_$($p.ExitCode)" } - if (Wait-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 180) { - Open-NetxUrl -Url "http://${hostBind}:${port}/" - } else { - throw (T ` - "NetX started but /health not ready within 180s.`nSee: $logDir\netx.err.log" ` - "NetX 已启动但 /health 在 180 秒内未就绪。`n请查看: $logDir\netx.err.log") - } - } - "tray" { - Ensure-NetxEnv - $extra = @("-ProgramRoot", $prog, "-DataRoot", $data) - if ($StartOnLaunch) { $extra += "-StartOnLaunch" } - # Detach tray; do not leave a success MessageBox (keeps a console open). - Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` - -Arguments $extra -WorkingDirectory $prog -WindowStyle Hidden | Out-Null - } - } - Write-LaunchLog "action=$Action ok" -} catch { - Show-NetxError ((T "NetX failed to start:`n$($_.Exception.Message)" "NetX 启动失败：`n$($_.Exception.Message)") + "`n`n$log") - exit 1 -} diff --git a/packaging/linux/README.md b/packaging/linux/README.md deleted file mode 100644 index c1801c2..0000000 --- a/packaging/linux/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# NetX Linux 发行版 / Linux release - -一个离线安装包 `netx-<版本>-linux-x86_64.tar.gz`：自带 Python 3.12、全部依赖、PostgreSQL 16 和网页，目标机器上**不需要预装任何软件**（只要 bash、tar、gzip、useradd 这些系统自带命令）。 - -## 支持的系统 - -x86_64、glibc ≥ 2.17：CentOS 7/8/Stream 9、RHEL 7–9、Rocky/Alma 8–9、Oracle Linux、Fedora、Ubuntu 18.04+、Debian 9+、openSUSE Leap / SLES 12+、openEuler 20.03+、麒麟 V10、统信 UOS 20、Anolis 等。 - -不支持：Alpine（musl）、32 位、ARM、CentOS 6。 - -## 安装 - -```bash -tar xzf netx-0.4.19-linux-x86_64.tar.gz -cd netx-0.4.19-linux-x86_64 -sudo ./install.sh -``` - -交互式安装会依次询问：监听地址（默认 `0.0.0.0`）、**Web/API 端口**（默认 8890，被占用会提示重新输入）、数据库（内置 / 外置）、**内置 PostgreSQL 端口**（默认 15432，只监听 127.0.0.1）。 - -非交互示例： - -```bash -sudo ./install.sh --yes --port 9890 --db bundled --pg-port 25432 -sudo ./install.sh --yes --db external --db-url 'postgresql+psycopg://netx:密码@10.0.0.8:5432/netx' -``` - -装好后访问 `http://<服务器IP>:<端口>/`，初始账号 `admin`，密码在安装结束时显示（随机生成，也可用 `--admin-password` 指定；同时保存在 `/var/lib/netx/.env` 的 `NETX_BOOTSTRAP_ADMIN_PASSWORD`），首次登录必须改密码。 - -> 对外监听（非 127.0.0.1）时 NetX 自身要求：管理员初始密码不能是 admin123，并且开启 UME 的 TLS 校验。安装程序会自动设置 `NETX_UME_VERIFY_TLS=true`；如果 UME 用的是自签名证书，需要配置证书，或者改成只监听本机再通过反向代理访问。 - -| 路径 | 内容 | -|---|---| -| `/opt/netx/current` → `releases/<版本>` | 程序（只读，保留最近 2 个版本用于回滚） | -| `/var/lib/netx` | `.env`、`pgdata/`、`data/`、`logs/`、`backups/` | -| `/usr/bin/netxctl` | 管理命令 | -| `netx.service` | systemd 服务，开机自启；没有 systemd 时用 `netxctl start` | - -服务以系统用户 `netx` 运行。 - -## 日常管理 - -```bash -sudo netxctl status -sudo netxctl stop | start | restart -sudo netxctl logs api -f # api / worker / biz0 / pg / netxctl -sudo netxctl set-port 9890 # 改 Web/API 端口（检查占用，失败自动回滚） -sudo netxctl set-bind 127.0.0.1 -sudo netxctl set-pg-port 25432 # 改内置 PostgreSQL 端口 -sudo netxctl reconfigure-db # 重新选择内置/外置数据库（不删数据） -sudo netxctl backup # pg_dump + .env + 密钥 → /var/lib/netx/backups/ -sudo netxctl uninstall # 保留数据；--purge 连数据一起删除 -``` - -## 更新 - -- 在线：`sudo netxctl check-update`、`sudo netxctl update`。更新源与 Windows 版一致（GitHub、Forgejo，可用 `.env` 里的 `NETX_UPDATE_*` 覆盖），下载后校验 sha256。 -- 离线：`sudo netxctl update --file netx-0.4.20-linux-x86_64.tar.gz`，或者解压新包后执行 `sudo ./install.sh`（自动识别为更新）。 -- 自动：`sudo netxctl auto-update on`（每天 03:30 左右；systemd timer，没有 systemd 时用 cron）。 - -更新会先停服务，把内置数据库备份到 `backups/pre-<版本>-*.dump`（保留 3 份），然后切换版本并启动。新版本没起来就自动切回旧版本。端口、数据库等配置保持不变。 - -## 构建（开发者） - -需要 docker，构建在 `manylinux2014` 容器里进行（CentOS 7 / glibc 2.17），保证产物在老系统上也能运行： - -```bash -packaging/linux/build_linux_release.sh # 产物在 packaging/release/ -packaging/linux/build_linux_release.sh --skip-web-build # 复用现成的 web/dist -``` - -- 第一次会从源码编译 PostgreSQL，结果缓存在 `packaging/cache/linux/`。 -- `check_bundle.sh` 检查所有 ELF：要求的 glibc 不超过 2.17，并且依赖的库都在包里。 -- 冒烟测试（断网，在一个裸发行版容器里完整走一遍安装、改端口、更新和卸载）： - -```bash -docker run --rm --network none -v "$PWD/packaging/release:/pkg:ro" \ - -v "$PWD/packaging/linux/smoke_test.sh:/smoke.sh:ro" centos:7 \ - bash /smoke.sh /pkg/netx-0.4.19-linux-x86_64.tar.gz -``` - -CI：`.github/workflows/release-linux.yml` 在 `v*` tag 上构建，在 7 个发行版里跑冒烟测试，全部通过后把 tar.gz 和 .sha256 挂到同一个 Release。挂到 Forgejo 用 `packaging/publish_forgejo_release.ps1`，它会自动带上 `packaging/release/` 下的 Linux 包。 diff --git a/packaging/linux/build_in_container.sh b/packaging/linux/build_in_container.sh deleted file mode 100755 index 8988722..0000000 --- a/packaging/linux/build_in_container.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -# Runs inside manylinux2014 (CentOS 7, glibc 2.17). Called by build_linux_release.sh. -# Mounts: /src (repo, ro) /webdist (ro) /cache (downloads + built PG) /out -set -euo pipefail -. /src/packaging/linux/versions.env -: "${VERSION:?}" -ARCH=x86_64 -NAME="netx-${VERSION}-linux-${ARCH}" -BUILD=/tmp/build -STAGE="${BUILD}/${NAME}" -JOBS="$(nproc)" - -rm -rf /tmp/build -mkdir -p "${STAGE}" - -# ------------------------------------------------------------ PostgreSQL (cached) -PG_TGZ="/cache/pgsql-${PG_VERSION}-linux-${ARCH}.tar.gz" -if [[ ! -f "${PG_TGZ}" ]]; then - echo "==> building PostgreSQL ${PG_VERSION}" - rm -rf /tmp/pgsrc /opt/pgsql - mkdir -p /tmp/pgsrc - tar xjf "/cache/$(basename "${PG_URL}")" -C /tmp/pgsrc - cd "/tmp/pgsrc/postgresql-${PG_VERSION}" - # No ICU/readline/OpenSSL: nothing for the target box to provide. SCRAM works - # without OpenSSL; the server only listens on 127.0.0.1. - ./configure --prefix=/opt/pgsql --without-icu --without-readline --without-openssl \ - --with-zlib --disable-rpath CFLAGS="-O2" >/tmp/pg-configure.log - make -j"${JOBS}" >/tmp/pg-make.log - make install >/tmp/pg-install.log - cd /opt/pgsql - keep=" postgres pg_ctl initdb psql pg_dump pg_dumpall pg_restore pg_isready pg_controldata pg_upgrade " - for f in bin/*; do - [[ "${keep}" == *" $(basename "${f}") "* ]] || rm -f "${f}" - done - rm -rf /opt/pgsql/include /opt/pgsql/lib/pgxs /opt/pgsql/lib/pkgconfig - rm -f /opt/pgsql/lib/*.a - cp -L "$(ldconfig -p | awk '/libz\.so\.1 .*x86-64/ {print $NF; exit}')" lib/libz.so.1 - for f in bin/*; do patchelf --set-rpath '$ORIGIN/../lib' "${f}"; done - for f in lib/*.so*; do - [[ -L "${f}" ]] || patchelf --set-rpath '$ORIGIN' "${f}" - done - strip --strip-unneeded bin/* lib/*.so* 2>/dev/null || true - tar czf "${PG_TGZ}" -C /opt pgsql -fi -tar xzf "${PG_TGZ}" -C "${STAGE}" - -# ------------------------------------------------------------ Python + deps -echo "==> Python ${PY_VERSION} + requirements" -tar xzf "/cache/$(basename "${PY_URL}")" -C "${STAGE}" # -> python/ -PY="${STAGE}/python/bin/python3" -export PIP_DISABLE_PIP_VERSION_CHECK=1 -[[ -n "${PIP_INDEX_URL:-}" ]] || unset PIP_INDEX_URL -mkdir -p /cache/pip -# Running on glibc 2.17 makes pip pick manylinux2014-compatible wheels only. -"${PY}" -m pip install --cache-dir /cache/pip --no-warn-script-location --only-binary=:all: \ - -r /src/requirements.txt - -echo "==> slimming runtime" -PYLIB="${STAGE:?}/python/lib/python3.12" -for d in test idlelib tkinter turtledemo ensurepip turtle.py; do - rm -rf "${PYLIB:?}/${d}" -done -rm -f "${PYLIB:?}"/lib-dynload/_tkinter* "${PYLIB:?}"/lib-dynload/_test* "${PYLIB:?}"/lib-dynload/xxlimited* -for d in "${STAGE:?}"/python/lib/tcl* "${STAGE:?}"/python/lib/tk* "${STAGE:?}"/python/lib/itcl* \ - "${STAGE:?}"/python/lib/thread* "${STAGE:?}"/python/lib/libtcl* "${STAGE:?}"/python/lib/libtk* \ - "${STAGE:?}/python/share"; do - [[ -e "${d}" ]] && rm -rf "${d}" -done -find "${PYLIB}/site-packages" -depth -type d \( -name tests -o -name test -o -name testing \) \ - -exec rm -rf {} + 2>/dev/null || true -find "${STAGE}/python" -name __pycache__ -type d -prune -exec rm -rf {} + -find "${STAGE}/python" -name '*.pyc' -delete - -# ------------------------------------------------------------ app files -echo "==> app files" -cp -r /src/netx_api /src/alembic "${STAGE}/" -for f in pyproject.toml requirements.txt alembic.ini README.md LICENSE SECURITY.md; do - if [[ -f "/src/${f}" ]]; then cp "/src/${f}" "${STAGE}/"; fi -done -find "${STAGE}/netx_api" "${STAGE}/alembic" -name __pycache__ -type d -prune -exec rm -rf {} + -rm -rf "${STAGE:?}/netx_api/tests" -mkdir -p "${STAGE}/web" -cp -r /webdist "${STAGE}/web/dist" -mkdir -p "${STAGE}/packaging/linux" -for f in lib.sh netxctl netx_admin.py systemd logrotate.netx README.md; do - cp -r "/src/packaging/linux/${f}" "${STAGE}/packaging/linux/" -done -cp /src/packaging/linux/install.sh "${STAGE}/install.sh" -# Sources may come from a Windows checkout: force LF + exec bits. -sed -i 's/\r$//' "${STAGE}/install.sh" "${STAGE}"/packaging/linux/{lib.sh,netxctl,netx_admin.py,logrotate.netx} \ - "${STAGE}"/packaging/linux/systemd/* -chmod 0755 "${STAGE}/install.sh" "${STAGE}/packaging/linux/netxctl" -cat >"${STAGE}/version.json" < archive" -tar --owner=0 --group=0 --numeric-owner -czf "/out/${NAME}.tar.gz" -C "${BUILD}" "${NAME}" -(cd /out && sha256sum "${NAME}.tar.gz" >"${NAME}.tar.gz.sha256") -chown "${HOST_UID:-0}:${HOST_GID:-0}" "/out/${NAME}.tar.gz" "/out/${NAME}.tar.gz.sha256" "${PG_TGZ}" 2>/dev/null || true -chown -R "${HOST_UID:-0}:${HOST_GID:-0}" /cache/pip 2>/dev/null || true -du -sh "${STAGE}" "/out/${NAME}.tar.gz" diff --git a/packaging/linux/build_linux_release.sh b/packaging/linux/build_linux_release.sh deleted file mode 100755 index e0b787c..0000000 --- a/packaging/linux/build_linux_release.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash -# Build netx--linux-x86_64.tar.gz (+ .sha256) into packaging/release/. -# -# packaging/linux/build_linux_release.sh [--version X.Y.Z] [--web-dist DIR] [--skip-web-build] -# -# Needs docker (the build runs inside manylinux2014 so the result works on glibc >= 2.17). -# Node 20+ is needed only when web/dist must be (re)built. -set -euo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO="$(cd "${HERE}/../.." && pwd)" -# shellcheck source=versions.env -. "${HERE}/versions.env" - -VERSION="" -WEB_DIST="" -SKIP_WEB_BUILD=0 -OUT="${REPO}/packaging/release" -CACHE="${REPO}/packaging/cache/linux" - -while [[ $# -gt 0 ]]; do - case "$1" in - --version) VERSION="$2"; shift 2 ;; - --web-dist) WEB_DIST="$2"; shift 2 ;; - --skip-web-build) SKIP_WEB_BUILD=1; shift ;; - --out) OUT="$2"; shift 2 ;; - --cache) CACHE="$2"; shift 2 ;; - -h|--help) sed -n '2,8p' "$0"; exit 0 ;; - *) echo "unknown option: $1" >&2; exit 2 ;; - esac -done - -if [[ -z "${VERSION}" ]]; then - VERSION="$(sed -n 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "${REPO}/pyproject.toml" | head -n1)" -fi -[[ -n "${VERSION}" ]] || { echo "cannot read version from pyproject.toml" >&2; exit 1; } -command -v docker >/dev/null || { echo "docker is required" >&2; exit 1; } - -if [[ -z "${WEB_DIST}" ]]; then - WEB_DIST="${REPO}/web/dist" - if [[ "${SKIP_WEB_BUILD}" != "1" ]]; then - echo "==> npm build (web/dist)" - (cd "${REPO}/web" && npm ci && npm run build:release) - fi -fi -[[ -f "${WEB_DIST}/index.html" ]] || { echo "web dist missing: ${WEB_DIST}/index.html" >&2; exit 1; } -WEB_DIST="$(cd "${WEB_DIST}" && pwd)" - -mkdir -p "${OUT}" "${CACHE}" -fetch() { # fetch url sha256 - local f - f="${CACHE}/$(basename "$1")" - if [[ ! -f "${f}" ]] || ! echo "$2 ${f}" | sha256sum -c --status; then - echo "==> download $1" - curl -fSL --retry 3 -o "${f}.part" "$1" - mv "${f}.part" "${f}" - fi - echo "$2 ${f}" | sha256sum -c --status || { echo "sha256 mismatch: ${f}" >&2; exit 1; } -} -fetch "${PG_URL}" "${PG_SHA256}" -fetch "${PY_URL}" "${PY_SHA256}" - -echo "==> build in ${BUILD_IMAGE} (version ${VERSION})" -docker run --rm \ - -v "${REPO}:/src:ro" \ - -v "${WEB_DIST}:/webdist:ro" \ - -v "${CACHE}:/cache" \ - -v "${OUT}:/out" \ - -e VERSION="${VERSION}" \ - -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ - -e PIP_INDEX_URL="${PIP_INDEX_URL:-}" \ - -e HTTP_PROXY="${HTTP_PROXY:-}" -e HTTPS_PROXY="${HTTPS_PROXY:-}" -e NO_PROXY="${NO_PROXY:-}" \ - -e http_proxy="${http_proxy:-}" -e https_proxy="${https_proxy:-}" -e no_proxy="${no_proxy:-}" \ - "${BUILD_IMAGE}" bash /src/packaging/linux/build_in_container.sh - -echo "==> done:" -ls -la "${OUT}/netx-${VERSION}-linux-x86_64.tar.gz" "${OUT}/netx-${VERSION}-linux-x86_64.tar.gz.sha256" diff --git a/packaging/linux/check_bundle.sh b/packaging/linux/check_bundle.sh deleted file mode 100755 index 49fb764..0000000 --- a/packaging/linux/check_bundle.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# Fail when any ELF in the stage needs glibc newer than MAX or a shared library -# that is neither in the bundle nor on the base-system allowlist. -# check_bundle.sh [max-glibc=2.17] -set -euo pipefail -STAGE="$1" -MAX="${2:-2.17}" -# Present on every glibc distro (libcrypt.so.1 only backs the unused _crypt module). -ALLOW=" libc.so.6 libm.so.6 libdl.so.2 libpthread.so.0 librt.so.1 libutil.so.1 libcrypt.so.1 libnsl.so.1 libresolv.so.2 ld-linux-x86-64.so.2 libgcc_s.so.1 libstdc++.so.6 " - -is_elf() { [[ "$(head -c4 "$1" 2>/dev/null | od -An -c | tr -d ' ')" == '177ELF' ]]; } - -declare -A HAVE=() -ELFS=() -while IFS= read -r -d '' f; do - is_elf "${f}" && ELFS+=("${f}") -done < <(find "${STAGE}" -type f \( -name '*.so*' -o -perm -u+x \) -print0) -for f in "${ELFS[@]}"; do HAVE["$(basename "${f}")"]=1; done -while IFS= read -r -d '' l; do HAVE["$(basename "${l}")"]=1; done < <(find "${STAGE}" -type l -name '*.so*' -print0) - -bad=0 -for f in "${ELFS[@]}"; do - top="$(objdump -T "${f}" 2>/dev/null | grep -o 'GLIBC_[0-9][0-9.]*' | sed 's/GLIBC_//' | sort -V | tail -n1 || true)" - if [[ -n "${top}" && "$(printf '%s\n%s\n' "${top}" "${MAX}" | sort -V | tail -n1)" != "${MAX}" ]]; then - echo "[FAIL] ${f#"${STAGE}"/} needs GLIBC_${top} (> ${MAX})" - bad=1 - fi - while IFS= read -r need; do - need="${need##*/}" # $ORIGIN/../lib/libfoo.so -> libfoo.so - [[ -z "${need}" || "${ALLOW}" == *" ${need} "* ]] && continue - [[ -n "${HAVE[${need}]:-}" ]] && continue - echo "[FAIL] ${f#"${STAGE}"/} needs ${need} (not bundled)" - bad=1 - done < <(readelf -d "${f}" 2>/dev/null | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p') -done -echo "==> checked ${#ELFS[@]} ELF files (max GLIBC ${MAX})" -exit "${bad}" diff --git a/packaging/linux/install.sh b/packaging/linux/install.sh deleted file mode 100755 index c3f9db1..0000000 --- a/packaging/linux/install.sh +++ /dev/null @@ -1,614 +0,0 @@ -#!/usr/bin/env bash -# NetX Linux installer / updater. Ships at the root of netx--linux-x86_64.tar.gz. -# -# sudo ./install.sh # interactive install (or update when already installed) -# sudo ./install.sh --yes --db bundled --port 8890 --pg-port 15432 -# sudo ./install.sh --mode update --yes # what `netxctl update` runs -# -# Needs only bash, coreutils and useradd (tar to unpack it): Python, PostgreSQL -# and the web UI are inside the package. -set -euo pipefail - -PKG="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" -# shellcheck source=lib.sh -. "${PKG}/packaging/linux/lib.sh" - -NETX_PY="${PKG}/python/bin/python3" -NETX_ADMIN_PY="${PKG}/packaging/linux/netx_admin.py" - -PREFIX="/opt/netx" -DATA="/var/lib/netx" -MODE="" -DB_MODE="" -API_PORT="" -BIND="" -PG_PORT="" -PG_PASSWORD="" -DB_URL="" -DB_HOST="" DB_PORT_EXT="" DB_USER="" DB_PASS="" DB_NAME="" -CRED_KEY_FILE="" -ADMIN_PASSWORD="" -ADMIN_PW_GENERATED=0 -NO_SERVICE=0 -AUTO_UPDATE=0 -OPEN_FIREWALL=0 -NO_BACKUP=0 -YES=0 - -usage() { - cat < echo answer - local a - read -r -p "$1 [$2]: " a || true - echo "${a:-$2}" -} - -# ------------------------------------------------------------------ preflight - -preflight() { - is_root || die "$(t "请用 root 运行：sudo ./install.sh" "run as root: sudo ./install.sh")" - local arch - arch="$(uname -m)" - [[ "${arch}" == "x86_64" || "${arch}" == "amd64" ]] || die "$(t "本安装包仅支持 x86_64，当前" "this package is x86_64 only; this machine is") ${arch}" - local glibc - glibc="$(getconf GNU_LIBC_VERSION 2>/dev/null | awk '{print $2}')" || true - [[ -n "${glibc}" ]] || die "$(t "未检测到 glibc（不支持 Alpine/musl）" "glibc not found (Alpine/musl is not supported)")" - if [[ "$(printf '%s\n2.17\n' "${glibc}" | sort -V | head -n1)" != "2.17" ]]; then - die "$(t "glibc 版本过低" "glibc too old"): ${glibc} (>= 2.17)" - fi - "${NETX_PY}" -c "import sys" 2>/dev/null || die "$(t "内置 Python 无法运行，安装包可能损坏" "bundled Python does not run; package may be corrupt")" - [[ -f "${PKG}/version.json" ]] || die "version.json missing in ${PKG}" - local parent avail - parent="$(dirname "${PREFIX}")" - mkdir -p "${parent}" - avail="$(df -Pk "${parent}" | awk 'NR==2 {print $4}')" - if [[ -n "${avail}" && "${avail}" -lt 1048576 ]]; then - die "$(t "磁盘空间不足（至少 1GB）" "not enough disk space (need 1GB)"): ${parent}" - fi -} - -ensure_user() { - if id "${NETX_USER}" >/dev/null 2>&1; then return 0; fi - info "$(t "创建系统用户" "creating system user") ${NETX_USER}" - local nologin=/sbin/nologin - [[ -x "${nologin}" ]] || nologin=/usr/sbin/nologin - [[ -x "${nologin}" ]] || nologin=/bin/false - if command -v useradd >/dev/null 2>&1; then - useradd -r -U -d "${DATA}" -M -s "${nologin}" "${NETX_USER}" 2>/dev/null || - useradd -r -d "${DATA}" -M -s "${nologin}" "${NETX_USER}" - elif command -v adduser >/dev/null 2>&1; then - adduser -S -D -H -h "${DATA}" -s "${nologin}" "${NETX_USER}" 2>/dev/null || - adduser --system --group --no-create-home --home "${DATA}" "${NETX_USER}" - else - die "$(t "缺少 useradd/adduser，无法创建用户" "useradd/adduser not found")" - fi - # Some useradd builds do not create a same-named group with -r. - getent group "${NETX_USER}" >/dev/null 2>&1 || groupadd -r "${NETX_USER}" 2>/dev/null || true -} - -# ------------------------------------------------------------------ prompts - -pick_port() { # pick_port