diff --git a/.env.example b/.env.example index 7ede38d..23d7cfd 100644 --- a/.env.example +++ b/.env.example @@ -120,8 +120,16 @@ NETX_UME_NOTIFICATION_TOPIC=ALARM # bundled | external; unset = external (compatible with existing deploys) # NETX_BUNDLED_PG_PORT=15432 # NETX_BUNDLED_PG_DATA_DIR= +# Update: GitHub primary + Forgejo mirror (git.avelo.top); pick newest reachable +# NETX_UPDATE_SOURCES=github,forgejo +# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest +# NETX_UPDATE_GITHUB_REPO=hansjone/netx # NETX_UPDATE_URL= +# NETX_UPDATE_FALLBACK_URL= +# NETX_UPDATE_TOKEN= # NETX_UPDATE_CHANNEL=stable +# NETX_UPDATE_AUTO=false +# When true: tray/scheduled task may download+apply zip updates silently. # Heavier fleets: raise CLI/DB together; also ensure Postgres max_connections and bastion session limits. # One-click start (recommended): .\scripts\start_netx.ps1 -Background -WithWeb # -> API + netx_api.worker + N× netx_api.biz_state_worker + optional Vite diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index 12e2fa9..12b777e 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -22,10 +22,11 @@ jobs: with: python-version: "3.12" - - name: Build release zip + - name: Build release stage shell: pwsh run: | - powershell -ExecutionPolicy Bypass -File ./packaging/build_release.ps1 -CreateVenv + # Use pwsh -File (UTF-8). Nested Windows PowerShell 5.1 mis-parses UTF-8 scripts without BOM. + & ./packaging/build_release.ps1 -CreateVenv - name: Install Inno Setup shell: pwsh @@ -37,12 +38,18 @@ jobs: shell: pwsh run: | $ver = (Get-Content pyproject.toml | Select-String '^\s*version\s*=').Line -replace '.*"(.+)"','$1' + if ($env:GITHUB_REF_NAME -ne "v$ver") { throw "Release tag $env:GITHUB_REF_NAME does not match pyproject version $ver" } ISCC.exe "/DMyAppVersion=$ver" packaging/installer/netx.iss + if ($LASTEXITCODE -ne 0) { throw "Inno Setup failed with exit code $LASTEXITCODE" } + $setup = "packaging/release/NetX-Setup-$ver.exe" + if (-not (Test-Path $setup) -or (Get-Item $setup).Length -lt 1MB) { throw "Missing or invalid installer: $setup" } + $hash = (Get-FileHash $setup -Algorithm SHA256).Hash.ToLowerInvariant() + Set-Content "$setup.sha256" -Value $hash -Encoding ascii -NoNewline - name: Upload to GitHub Release uses: softprops/action-gh-release@v2 with: files: | - packaging/release/NetX-*-win64.zip packaging/release/NetX-Setup-*.exe + packaging/release/NetX-Setup-*.exe.sha256 generate_release_notes: true diff --git a/.gitignore b/.gitignore index 92197b4..c72cfd0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,16 +8,25 @@ build/ dist/ scripts/.run/ # Local one-off probes / staging dumps (not for commit) +artifacts/ scripts/_* scripts/archive/ packages/netx-topology-mcp/tests/_* -ume/ +_tmp_* +.idea/ +/ume/ data/ne_collections/ data/webcrt/ data/auth/ data/runtime/ +data/ne_exec_jobs/ # Windows packaging artifacts (binaries / release trees) packaging/cache/ packaging/release/ packaging/postgres/pgsql/ +packaging/winsw/ *.exe +!packaging/installer/*.iss + +# Claude Code personal overrides +.claude/settings.local.json diff --git a/README.md b/README.md index 2ce7550..8a4d5e9 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,9 @@ API base: `http://127.0.0.1:8890/` After `npm run build` in `web/`, the API can also serve the UI at `http://127.0.0.1:8890/` (same origin). See `NETX_UI_DIST_DIR` in `.env.example`. -### Windows installer / portable package (optional) +### Windows installer (optional) -Fool-proof Windows delivery (bundled or external Postgres, program/data split, manual update) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`. +Fool-proof Windows delivery via `NetX-Setup-*.exe` (bundled or external Postgres, program/data split, offline upgrade over existing installs) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`. ### 6) MCP（Cursor / oclaw / Claude） diff --git a/alembic/versions/20261013_user_prefs.py b/alembic/versions/20261013_user_prefs.py new file mode 100644 index 0000000..abd5d96 --- /dev/null +++ b/alembic/versions/20261013_user_prefs.py @@ -0,0 +1,52 @@ +"""Add app_user.prefs JSON for per-user UI preferences. + +Revision ID: 20261013_user_prefs +Revises: 20260922_biz_state_queued +Create Date: 2026-10-13 +""" + +from __future__ import annotations + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "20261013_user_prefs" +down_revision: Union[str, Sequence[str], None] = "20260922_biz_state_queued" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + bind = op.get_bind() + dialect = str(getattr(bind.dialect, "name", "") or "").lower() + if dialect.startswith("postgres"): + op.execute( + "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'::json NOT NULL" + ) + # Ensure NOT NULL + default for brownfield rows that already had a nullable column. + op.execute("UPDATE app_user SET prefs = '{}'::json WHERE prefs IS NULL") + op.execute("ALTER TABLE app_user ALTER COLUMN prefs SET DEFAULT '{}'::json") + op.execute("ALTER TABLE app_user ALTER COLUMN prefs SET NOT NULL") + else: + insp = sa.inspect(bind) + if "app_user" in insp.get_table_names(): + cols = {c["name"] for c in insp.get_columns("app_user")} + if "prefs" not in cols: + op.add_column( + "app_user", + sa.Column("prefs", sa.JSON(), nullable=False, server_default="{}"), + ) + + +def downgrade() -> None: + bind = op.get_bind() + dialect = str(getattr(bind.dialect, "name", "") or "").lower() + if dialect.startswith("postgres"): + op.execute("ALTER TABLE app_user DROP COLUMN IF EXISTS prefs") + else: + try: + op.drop_column("app_user", "prefs") + except Exception: + pass diff --git a/docs/biz-state-canonical-fields.md b/docs/biz-state-canonical-fields.md new file mode 100644 index 0000000..3670f0a --- /dev/null +++ b/docs/biz-state-canonical-fields.md @@ -0,0 +1,68 @@ +# 割接对比：规范值契约 + +适用范围：业务对比（`BizCompare*`）和割接监控（`BizMigration*`）。两种模式用同一条身份流水线（`netx_api/biz_state/identity_pipeline.py`），跨厂商对比靠它把两侧设备的原始输出对齐到同一套“规范值”。 + +## 1. 流水线 + +同一行在某一侧（`before` / `after`）依次经过： + +1. **归一化**：接口字段去内部空格 → 厂商内规范名（`VENDOR_CANONICAL`，如华为 `GigabitEthernet`→`GE`）→ 模板别名规则（`iface_normalize_rules`，如 `GE`→`gei`）；键字段按 `field_rules[].normalize`（`lower`/`upper`/`mac`…）；再按 `field_rules[].value_map` 改写状态值。 +2. **行过滤**：`row_filters` 在归一化之后求值，所以按规范值书写；带 `side` 的过滤只作用于该侧。 +3. **端口映射**：只作用于 `before` 侧，键用 before 侧规则归一化、值用 after 侧规则归一化。 +4. **取键**：`scalar_text(value).strip()` 逐字段取值，得到 match key。 + +快照版本 `config_snapshot.version >= 2` 的 run 走完整流水线；`version == 1` 的历史 run 保持旧语义（过滤看原始行、键不归一化、同键按采集顺序配对），回填、检索、导出结果不变。 + +## 2. 规范值 + +比对层只认下面这些取值。设备原始值不同的，用 `value_map` 或 parser 输出规范值解决，**引擎里不写厂商特例**。 + +| metric | 字段 | 规范取值 | +|---|---|---| +| `interface_brief` | `admin` / `phy` / `prot` | `up` / `down`（华为 `*down` 记 `admin=down`） | +| `bgp_peer` | `state` | `established` / `idle` / `active` / `connect` / `opensent` / `openconfirm` | +| `bgp_peer` | `afi` | `ipv4` / `ipv6` / `vpnv4` / `vpnv6` / `evpn` / `vpls` | +| `arp` | `entry_type` | `dynamic` / `static` / `interface` | +| `isis_adjacency` / `ospf_neighbor` | `state` | `up` / `full` / `2way` / `down` / `init` / `idle` | +| `vrrp` | `af` | `ipv4` / `ipv6` | + +### 中兴（已实现）现状 + +中兴 parser 输出设备打印的原值，没有改写： + +- `interface_brief.admin/phy/prot`：`up` / `down`，已是规范值。 +- `arp.entry_type`：由 `age` 推断，`HH:MM:SS` 为 `dynamic`，否则 `static`。 +- `bgp_peer.state`：设备原值，如 `Established`（首字母大写）。与中兴自身对比时一致；与其他厂商对比时在模板里加值映射： + + ```json + {"field": "state", "value_map": {"established": "established", "estab": "established", "*down": "down"}} + ``` + + `value_map` 大小写不敏感匹配，输出为右侧的规范值；两侧都经过同一张表。 + +> 没有改动中兴 parser 的输出值：已采集批次里的值是原值，直接改成小写会让“旧批次 vs 新批次”出现假差异。需要统一时在模板里加 `value_map`。 + +### 其他厂商（待办） + +华为 VRP 的状态 parser（`interface_brief`、`arp`、`bgp_peer`、`isis_adjacency`、`ospf_neighbor`）**尚未实现**：仓库里没有真实输出样本，按规则不凭空编写 TextFSM。收到样本后按 `skills/biz-state/netx-biz-state/SKILL.md` 第 2 节逐项完成，输出与中兴相同的字段名，状态值输出上表的规范值，并给 profile 补 `slice`。 + +目前只有 LLDP 在跨厂商场景下有真实 parser。 + +## 3. 命令切片（`ParseProfile.slice`） + +一个采集命令产出 metric 的哪一部分，由 profile 的 `slice` 声明，例如 `zte.bgp_ipv4_summary` → `{"afi": "ipv4"}`、`zte.vrrp_ipv6` → `{"af": "ipv6"}`。比对表单（sheet）用顶层 `eq` 行过滤声明自己要的切片。 + +- 一侧没有任何命令覆盖该切片 → 该 sheet 状态 `not_collected`，不判业务缺失、不计入通过率。 +- 割接监控创建 HF 采集任务时，取两侧画像命令切片的并集；某侧找不到对应切片的 profile 时返回 `no_profile_for_slice:::`。 +- 没声明 `slice` 的命令视为覆盖整个 metric。 + +## 4. sheet 数据状态 + +| 状态 | 条件 | 通过率 | +|---|---|---| +| `ok` | 两侧采集正常 | 正常计算 | +| `no_data` | 两侧过滤后都是 0 行，命令都成功 | `null`，不计入总通过率 | +| `suspect_parse` | 某侧 `raw_line_count > 5` 但 `row_count == 0`，或全部命令失败 | 失败行照常显示，任务级 `ok = null` | +| `not_collected` | 某侧没有任何命令（或切片）产出该 metric | `null`，不计入总通过率 | + +没有任何命令回执的批次（导入或历史数据）无法判断，按正常处理。 diff --git a/docs/reviews/assets/biz-compare-config-mobile.png b/docs/reviews/assets/biz-compare-config-mobile.png new file mode 100644 index 0000000..29539dc Binary files /dev/null and b/docs/reviews/assets/biz-compare-config-mobile.png differ diff --git a/docs/reviews/assets/biz-compare-config.png b/docs/reviews/assets/biz-compare-config.png new file mode 100644 index 0000000..98fe764 Binary files /dev/null and b/docs/reviews/assets/biz-compare-config.png differ diff --git a/docs/reviews/assets/biz-compare-desktop.png b/docs/reviews/assets/biz-compare-desktop.png new file mode 100644 index 0000000..20904ac Binary files /dev/null and b/docs/reviews/assets/biz-compare-desktop.png differ diff --git a/docs/reviews/assets/biz-compare-error.png b/docs/reviews/assets/biz-compare-error.png new file mode 100644 index 0000000..4396388 Binary files /dev/null and b/docs/reviews/assets/biz-compare-error.png differ diff --git a/docs/reviews/assets/biz-compare-jobs.png b/docs/reviews/assets/biz-compare-jobs.png new file mode 100644 index 0000000..ace1467 Binary files /dev/null and b/docs/reviews/assets/biz-compare-jobs.png differ diff --git a/docs/reviews/assets/biz-compare-legacy.png b/docs/reviews/assets/biz-compare-legacy.png new file mode 100644 index 0000000..a6c03ba Binary files /dev/null and b/docs/reviews/assets/biz-compare-legacy.png differ diff --git a/docs/reviews/assets/biz-compare-memory-benchmark.json b/docs/reviews/assets/biz-compare-memory-benchmark.json new file mode 100644 index 0000000..70290b7 --- /dev/null +++ b/docs/reviews/assets/biz-compare-memory-benchmark.json @@ -0,0 +1,29 @@ +{ + "date": "2026-10-10", + "baseline_ref": "afa8e06b2de64100f963ae30b3babfcb5d7009f4", + "python": "3.14.3", + "dataset": { + "rows_per_side": 1000000, + "fields": ["id", "neighbor", "state", "afi"], + "strata": 50, + "unchanged_limit": 5000, + "compact_unchanged": true, + "interface_transforms": false, + "all_rows_unchanged": true + }, + "measurement": "tracemalloc peak allocated during compare_rows; excludes input allocations, DB loading and process overhead", + "results": [ + { + "engine": "baseline", + "peak_engine_mib": 1144.43, + "seconds_with_tracemalloc": 92.4 + }, + { + "engine": "optimized", + "peak_engine_mib": 420.49, + "seconds_with_tracemalloc": 63.681 + } + ], + "results_and_sample_order_identical": true, + "test_result": "1 passed in 157.88s" +} diff --git a/docs/reviews/assets/biz-compare-narrow.png b/docs/reviews/assets/biz-compare-narrow.png new file mode 100644 index 0000000..bfec41c Binary files /dev/null and b/docs/reviews/assets/biz-compare-narrow.png differ diff --git a/docs/reviews/assets/biz-compare-runs-mobile.png b/docs/reviews/assets/biz-compare-runs-mobile.png new file mode 100644 index 0000000..9e53762 Binary files /dev/null and b/docs/reviews/assets/biz-compare-runs-mobile.png differ diff --git a/docs/reviews/assets/biz-compare-runs.png b/docs/reviews/assets/biz-compare-runs.png new file mode 100644 index 0000000..9042eca Binary files /dev/null and b/docs/reviews/assets/biz-compare-runs.png differ diff --git a/docs/reviews/assets/biz-state-batches.png b/docs/reviews/assets/biz-state-batches.png new file mode 100644 index 0000000..e6b7248 Binary files /dev/null and b/docs/reviews/assets/biz-state-batches.png differ diff --git a/docs/reviews/assets/biz-state-config-mobile.png b/docs/reviews/assets/biz-state-config-mobile.png new file mode 100644 index 0000000..19244fc Binary files /dev/null and b/docs/reviews/assets/biz-state-config-mobile.png differ diff --git a/docs/reviews/assets/biz-state-config.png b/docs/reviews/assets/biz-state-config.png new file mode 100644 index 0000000..a4eb2e0 Binary files /dev/null and b/docs/reviews/assets/biz-state-config.png differ diff --git a/docs/reviews/assets/biz-state-error.png b/docs/reviews/assets/biz-state-error.png new file mode 100644 index 0000000..62cd975 Binary files /dev/null and b/docs/reviews/assets/biz-state-error.png differ diff --git a/docs/reviews/assets/biz-state-tasks.png b/docs/reviews/assets/biz-state-tasks.png new file mode 100644 index 0000000..be971dc Binary files /dev/null and b/docs/reviews/assets/biz-state-tasks.png differ diff --git a/docs/reviews/assets/biz-state-workbook-mobile.png b/docs/reviews/assets/biz-state-workbook-mobile.png new file mode 100644 index 0000000..5b2a506 Binary files /dev/null and b/docs/reviews/assets/biz-state-workbook-mobile.png differ diff --git a/docs/reviews/assets/biz-state-workbook.png b/docs/reviews/assets/biz-state-workbook.png new file mode 100644 index 0000000..cad4dee Binary files /dev/null and b/docs/reviews/assets/biz-state-workbook.png differ diff --git a/docs/reviews/assets/compare-template-editor-mobile.png b/docs/reviews/assets/compare-template-editor-mobile.png new file mode 100644 index 0000000..3dc4212 Binary files /dev/null and b/docs/reviews/assets/compare-template-editor-mobile.png differ diff --git a/docs/reviews/assets/compare-template-editor.png b/docs/reviews/assets/compare-template-editor.png new file mode 100644 index 0000000..a6419a9 Binary files /dev/null and b/docs/reviews/assets/compare-template-editor.png differ diff --git a/docs/reviews/assets/cutover-monitor-board.png b/docs/reviews/assets/cutover-monitor-board.png new file mode 100644 index 0000000..9fe1dcf Binary files /dev/null and b/docs/reviews/assets/cutover-monitor-board.png differ diff --git a/docs/reviews/assets/cutover-monitor-mobile.png b/docs/reviews/assets/cutover-monitor-mobile.png new file mode 100644 index 0000000..e9e2c1b Binary files /dev/null and b/docs/reviews/assets/cutover-monitor-mobile.png differ diff --git a/docs/reviews/assets/cutover-monitor-wall.png b/docs/reviews/assets/cutover-monitor-wall.png new file mode 100644 index 0000000..f35d5db Binary files /dev/null and b/docs/reviews/assets/cutover-monitor-wall.png differ diff --git a/docs/reviews/biz-compare-review-2026-10-10.md b/docs/reviews/biz-compare-review-2026-10-10.md new file mode 100644 index 0000000..65e342e --- /dev/null +++ b/docs/reviews/biz-compare-review-2026-10-10.md @@ -0,0 +1,131 @@ +# 业务对比检查与优化记录 + +检查日期：2026-10-10。范围：网络管理 → 割接对比 → 业务对比，包括前端任务/批次/结果界面、Python 与 PostgreSQL 对比路径、结果分页与搜索、短期列表缓存。 + +本次直接修改项目代码并添加回归测试，没有连接设备执行采集，也没有修改生产数据库。浏览器截图与交互测试使用固定 API 响应。 + +## 已修复的问题 + +### 1. 搜索会改变业务判定（P1） + +原实现分别按搜索条件过滤割接前、后的原表，再重新对比。例如 `state=up → down`，搜索 `up` 后只剩前侧，原本的“变化”就可能变成“删除”。重复身份键的业务还可能在筛选后错位配对。 + +现在：已存储的判定直接在结果表分页筛选；需要回查未保存全量明细的成功行时，先找到候选身份键，再补齐两侧的完整分组，按原有配对规则对比，最后筛选。达到读取上限的分组整体跳过，并返回截断标记，避免产生虚假的新增或删除。 + +### 2. PostgreSQL 搜索漏用部分模板过滤条件（P1） + +原搜索只下推 SQL 支持的过滤规则，对 `age_timer` 等不支持的规则直接忽略，可能将静态 ARP 等不属于模板范围的数据纳入结果。现在不支持下推的规则走流式读取并在 Python 中完整过滤。 + +### 3. 快速切换时旧请求覆盖当前任务（P1） + +前后采集任务的批次列表、打开任务、加载历史结果以及轮询缺少完整的过期保护。现在对批次请求使用清理标记，对任务与结果使用请求序号；关闭弹窗、切换路由或打开其他任务后，旧响应不能覆盖当前界面。 + +### 4. 结果加载错误容易被误读（P2） + +原查询失败时保留上一组明细，界面已经显示新筛选条件；切到未完成的表后，加载状态也可能无法结束。现在加载和失败都有独立状态，错误时清除旧明细并提供重试入口；切换表/页签后清理加载状态。 + +### 5. LLDP 搜索漏查（P2） + +LLDP 数据存于 `biz_state_lldp_neighbor`，原搜索只读 `biz_state_metric_row`。现在原表回查与精简结果的字段搜索都支持 LLDP 表。 + +### 6. 其他已修复边界 + +- 恰好命中读取上限时，原非 PostgreSQL 路径错误地报告“已截断”；现在多读一条判断。 +- 无效或有歧义的表标识可能落到其他表；现在返回 `sheet_not_found`。 +- `%`、`_` 等搜索字符按字面值匹配，避免不同查询路径含义不一致。 +- 关闭最后一项检查后，空数组被解释为“启用全部”；现在阻止关闭最后一项并提示至少保留一项。 +- 两个 HeroUI Input 使用了 `isDisabled`，实际需要 `disabled`；过滤值与容差输入的禁用状态已修正。 +- 列表缓存缺少并发合并，慢请求可以覆盖强制刷新结果或在失效后重新填充缓存；现在共享请求、保护失效状态并抑制过期的后台通知。 +- 清理中英文文案中的重复键，解决相关类型检查错误。 +- 操作后任务选择“同操作前”时原来没有加载批次列表；现在按实际使用的任务读取并共享缓存。切换采集任务时清除原批次选择，避免把上一个任务的批次带入配置。 +- 原通用下拉框嵌套了两个 label，标签不能正确关联控件；现在使用明确的 `htmlFor/id`，并关联提示文本。 + +## 性能改动与实际收益范围 + +- 合并两套轮询，下一轮在本轮请求完成后启动。模拟浏览器测试中，运行任务在 5.2 秒内历史列表和详情各请求 2–3 次（包含可能尚未完成的初始加载），没有重复轮询，也不会因慢请求持续叠加。 +- 已保存的失败、新增、变化等结果在数据库中筛选并分页，避免每次翻页重新加载和对比原始批次。回归测试明确验证这一查询路径不调用 `compare_rows`。 +- 同一前后采集任务的批次请求共用短期缓存，列表后台刷新合并同一请求；6 项缓存测试覆盖并发、强制刷新、失败重试与失效竞态。 +- 模板查找使用 Map，避免列表搜索与绘制逐项线性查找模板。 +- 修改筛选后先回到第一页，避免先请求旧页码再重复请求第一页。 + +这些改动减少了重复工作。以下后续基准使用合成数据，没有测量生产环境延迟。 + +## 后续优化（2026-10-10） + +### 历史执行配置与精简行还原 + +新运行在创建时将模板、启用的检查项、字段规则/过滤、接口归一化、端口映射、成功行保存策略和名称存入 `summary_json.config_snapshot`。后台执行、成功行回查、精简行还原以及导出共用该配置；排队之后修改配置，甚至删除原模板/映射，也不会改变这一运行使用的配置。无需数据库结构迁移。 + +修复了精简存储明细还原时将 `mapped_before` 直接设为原始 `before` 的问题。现在前侧依次归一化和映射，后侧归一化，保持与完整结果的显示及 CSV 一致。ZIP 新增 `config_snapshot.json`，便于核对当时配置；详情接口只返回快照模板与版本，不将完整端口映射加入轮询响应。 + +页面显示“执行配置已保存”；旧记录显示“旧记录：未保存执行配置”。旧运行无法可靠补造当时的配置，仍保留兼容查询，成功行回查及精简行还原会使用当前配置。快照也不保存原始采集行的副本，若原始批次数据被清理，依赖行 ID 的还原仍会缺少内容。 + +### Python 大表内存 + +成功样本按“层内序号、层首次出现顺序”排序，只保留样本上限内的候选和分层状态，避免先积累全部成功行。保留原分层轮询顺序；未截断的小表仍保持采集顺序。不需要接口变换时复用输入行；有变换时创建新行，不修改输入数据。稳定身份键顺序直接使用分组字典，省去再次计算全表键、额外键列表与集合。 + +每侧 1,000,000 行、50 个邻居分层、全部成功、保存 5,000 条精简样本的测试：优化前引擎新增分配峰值 **1144.43 MiB**，优化后 **420.49 MiB**，下降约 **63.3%**。完整结果与样本顺序的 SHA-256 一致。开启 `tracemalloc` 时耗时为 92.400 秒和 63.681 秒；该计时含内存追踪开销，不代表生产耗时。输入行在追踪开始前创建，峰值不包含输入行、数据库加载或 Python 进程的总内存。 + +基线为 `afa8e06b2de64100f963ae30b3babfcb5d7009f4` 中的对比引擎。原始指标见 [百万行测试结果](assets/biz-compare-memory-benchmark.json)。身份键分组仍需要随原表行数增长的内存；这次不是完全流式对比。 + +### 大表导出 + +导出改为按 `(seq, id)` 游标分批查询、分批还原精简行、逐行将 UTF-8 CSV 写入 ZIP，避免将全部明细与完整 CSV 同时保存在内存。相同 `seq` 的多条结果按 `id` 连续读取，不漏掉跨批次的同行序号记录。保留 BOM、字段顺序及逗号/引号/换行转义。最终压缩包仍由现有接口在内存中返回，压缩文件本身的大小仍计入内存。 + +## UI 改动 + +任务列表、创建向导、任务配置、对比批次和结果区统一使用深色平面、细边框、小圆角、低饱和蓝色和等宽数字。任务配置按作业参数、前后数据源、检查范围、端口映射四个区块组织；数据源左右配对，显示设备地址和采集数量。保存配置作为配置页主操作，删除操作降低视觉权重。 + +批次页增加最近 50 条记录的统计、刷新、加载、错误重试；刷新失败保留已载入记录。已完成只表示执行结束，业务失败/成功/新增数量单独显示。排队与运行使用不同文案，未知进度使用不确定进度条；时间、批次标识和前后采集信息便于核查。页签支持方向键及 Home/End，窄屏配置改为单列，批次表在独立容器中横向滚动。 + +结果区分为当前检查项/全表通过率、结果类型、搜索与辅助操作三层。增加明确的分页范围、清空筛选、加载状态、错误重试和搜索无结果提示。筛选按钮增加按下状态，字段筛选增加展开状态与键盘焦点提示。后续增加了执行配置保存状态及旧记录限制提示。 + +窄屏检查项导航改为紧凑的横向排列，避免一个检查项占据整行的大块区域。浏览器检查覆盖 1440px、768px、390px；筛选栏没有横向溢出。结果表保留必要的横向滚动。 + +- [任务列表](assets/biz-compare-jobs.png) +- [桌面任务配置](assets/biz-compare-config.png) +- [390px 任务配置](assets/biz-compare-config-mobile.png) +- [桌面对比批次](assets/biz-compare-runs.png) +- [390px 对比批次](assets/biz-compare-runs-mobile.png) +- [桌面结果区](assets/biz-compare-desktop.png) +- [768px 结果区](assets/biz-compare-narrow.png) +- [加载失败与重试](assets/biz-compare-error.png) +- [旧记录执行配置提示](assets/biz-compare-legacy.png) + +## 验证 + +- Python：87 项通过，1 项百万行基准默认跳过；单独启用百万行基准后通过。新增测试覆盖搜索、历史配置快照、有界抽样与游标导出。 +- 缓存：6 项 Node 测试通过。 +- 浏览器：27 个场景通过，包括任务列表文案、配置保存数据、同任务批次加载与切换清理、自动模式、检查项开关、创建向导、键盘页签、批次统计/状态/刷新/错误重试、配置保存状态、旧记录提示、结果布局、筛选、窄屏布局、旧任务响应保护、单套轮询、无运行时异常。 +- Vite 生产打包成功。 +- 本次业务对比页面的类型错误已经修复。全项目 `tsc -b` 仍被 `BizStatePage.tsx`、`BizMigrationPage.tsx` 中原有的类型/未使用变量错误阻塞。 +- 页面 ESLint 原有 34 个错误，主要是 `any` 与现有 React Hook 写法；批次列表增加类型后减少了 `any`，本次未增加错误。结果请求依赖警告已修正。通用下拉框和缓存模块没有 lint 错误。 +- PostgreSQL 路径验证了 SQL 编译与规则降级分支，没有在真实 PostgreSQL 数据库上执行查询或测量执行计划。 + +可重复执行的检查： + +```powershell +# 项目根目录 +.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare_export.py tests/test_biz_state_compare_snapshot.py tests/test_biz_state_compare_memory.py tests/test_biz_state_compare_search.py tests/test_biz_state_compare.py tests/test_biz_state_compare_sql.py tests/test_biz_state_compare_recovery.py -q + +# 可选：百万行基准，耗时约数分钟，内存占用较大 +$env:NETX_BENCH_ROWS = '1000000' +$env:NETX_BENCH_REF = 'afa8e06b2de64100f963ae30b3babfcb5d7009f4' +.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare_memory.py::test_memory_benchmark_against_git_baseline -q -s +Remove-Item Env:NETX_BENCH_ROWS, Env:NETX_BENCH_REF + +# web 目录 +node --experimental-strip-types --test tests/cutoverDataCache.test.mjs +node node_modules/vite/bin/vite.js build +node node_modules/typescript/bin/tsc -b +``` + +浏览器脚本为 `web/tests/bizCompare.browser.mjs`。先启动本地 Vite 服务，再设置 `NETX_PLAYWRIGHT_MODULE` 为可用的 Playwright 包路径；`NETX_TEST_URL` 默认是 `http://127.0.0.1:5179`，浏览器默认使用已安装的 Chrome。所有 `/v1` 请求由脚本提供固定响应，不会访问实际设备或数据库。 + +## 后续值得优化的部分 + +1. **Python 大表分组。** 有界抽样和冗余复制优化已完成，百万行基准下降约 63% 的引擎新增分配峰值。分组与输入仍随原表规模增长；进一步降低内存可研究排序流式配对或分区执行，并保持重复键配对语义。 +2. **PostgreSQL 搜索与深分页。** JSON 字段包含搜索和精确总数统计仍可能扫描大量结果。需要用真实数据查看 `EXPLAIN (ANALYZE, BUFFERS)`，再决定常用字段索引、搜索文本索引或游标分页；本次没有盲目添加生产索引。 +3. **历史配置与原始数据保留。** 新运行的执行配置快照已完成。旧运行仍无法还原未记录的当时配置；精简结果还原依赖原批次行，后续可为批次清理增加引用保护或保留策略。 +4. **重复键的业务语义。** 引擎按采集顺序配对同键行。如果业务应按集合判断，CLI 行顺序改变可能产生多条“不一致”。当前保留原有语义；可优先完善身份键，或另行实现并测试 Python/SQL 一致的集合匹配。 +5. **原表回查的数量上限。** 每侧候选命中最多 2000 条，完整分组补齐每侧预算 10000 条，最终展示最多 200 对；这不是全表匹配总数。界面会提示截断，大范围检索需要收窄条件。 diff --git a/docs/reviews/biz-state-review-2026-10-11.md b/docs/reviews/biz-state-review-2026-10-11.md new file mode 100644 index 0000000..7ca0e8e --- /dev/null +++ b/docs/reviews/biz-state-review-2026-10-11.md @@ -0,0 +1,54 @@ +# 业务监控分析与优化记录 + +检查日期：2026-10-11。范围：网络管理 → 割接对比 → 业务监控，包含任务配置、周期采集状态、批次保护、工作簿、原始日志、筛选与导出。 + +本次修改项目代码并使用内存 SQLite 和固定浏览器 API 响应验证，没有连接设备执行采集，也没有修改生产数据库。未新增解析模板或数据库字段。前后端应一起更新，前端轮询使用新增的 `/v1/biz-state/tasks/{id}/progress` 接口。 + +## 已修复的问题 + +1. **旧请求覆盖当前内容。** 任务、用途筛选、批次工作簿、指标表、采集详情、参数发现和原始日志增加请求序号或取消保护。关闭弹窗、切换任务/表后，慢响应不能重新打开弹窗或覆盖新内容。原始日志下载保留其实际批次 ID，避免根据当前另一层弹窗推断来源。 +2. **后台采集不能及时反映、慢轮询叠加。** 同时观察服务端 `collect_running` 与当前页面启动的任务；周期任务和打开的任务保留低频查询，能够发现后台启动的采集。活动采集每轮完成后等待 4 秒，空闲等待 15 秒，再发起下一轮。跨 effect 重建共享在途标记，避免慢请求叠加。用途筛选隐藏的本地采集任务单独读取进度，避免误报采集结束。 +3. **刷新覆盖未保存配置、批次数突然缩减。** 轮询只合并运行标志和时间，不重新写入周期、保留策略或监控项。进度刷新和首次加载统一读取最多 200 个最近批次，界面明确显示已载入数量与上限。 +4. **加载失败看起来像空表或上一份数据。** 任务列表、任务配置、批次和指标表增加加载/错误/重试状态。指标查询失败清除旧行，不显示“本表无数据”；配置未就绪时操作区域不可交互。列表刷新失败保留当前行并显示错误。 +5. **正在采集的批次可能被删除或清理。** `queued/running` 批次加入 `active_collection` 保护，单条删除、批量删除、保留清理均保留它们；采集中的任务返回 409。前端同步禁用选择与删除，保留基线、业务对比及割接引用保护。 +6. **查询与导出边界错误。** `%`、`_`、反斜杠按字面值搜索；JSON 单列提取和 LLDP 分页排序更稳定。早于主命令保存的辅助命令失败保持可见。CSV 保留 `0`、`False`，正确引用逗号、双引号和 CR/LF；重复 `seq` 的导出不遗漏跨分块行，完整保留后续行才出现的列。 + +## 性能改动与验证范围 + +- 任务详情将逐监控项读取绑定改为一次批量读取。测试中 30 个监控项、60 个绑定由原来的 32 次查询降为 **3 次**；轻量进度接口为 **1 次**，不返回监控项和绑定。 +- 批次保护只读取引用 ID 列，按本次批次范围限制查询，不再加载所有对比/割接运行的配置和汇总 JSON。 +- 工作簿摘要不将原始 CLI 日志传输到 Python；只读取 SQL 计算的行数与是否存在日志。已有完整行数优先使用，原始日志在查看或导出时加载。回归使用 1 MB 日志，确认摘要结果集没有原文列。 +- 通用指标 CSV 先分块扫描列名，再按 `(seq, id)` 游标每块 1000 行写入 ZIP；LLDP 分批写入。减少指标表的 ORM 对象、字典和完整 CSV 同时驻留。**最终 ZIP 仍驻留内存，命令原文导出仍需要原文；这不是端到端流式下载。** 两次扫描保留完整列名，正在写入的批次仍不保证导出为一致性快照。 +- 同一厂商/型号的采集模板缓存 30 秒；轮询不重新读取模板。任务列表默认每页渲染 50 行，可选 20/50/100/200；列名去重使用 Set，命令搜索只计算一次。 +- 指标搜索先回到第一页并等待防抖值一致，避免先请求旧页码、旧关键字。 + +以上是查询次数、请求并发及读取方式的验证，没有测量真实设备或生产 PostgreSQL 的延迟与内存。任务列表 API 仍返回当前用途下的完整任务集合；客户端分页减少 DOM，不减少列表响应大小。JSON 全文模糊搜索、旧日志的 SQL 行数计算、非常大的 ZIP 内存仍是后续基准重点。 + +## 界面调整 + +统一深色工业风：连续指标栏、细边框、小圆角、等宽任务标识、紧凑表头及明确的异常提示。任务配置使用响应式网格；采集周期数值与单位并排；工作簿横向页签、过滤工具栏和表格支持窄屏滚动；底部按钮可换行，避免被裁切。任务和工作簿页签支持方向键、Home/End，过滤控件增加可访问名称。 + +“采集成功 / 部分完成 / 采集失败”描述采集流程，不能据此判断业务验收通过；业务状态需要查看数据表或业务对比结果。 + +- [任务列表](assets/biz-state-tasks.png) +- [任务配置](assets/biz-state-config.png) · [390px 配置](assets/biz-state-config-mobile.png) +- [采集批次](assets/biz-state-batches.png) +- [工作簿](assets/biz-state-workbook.png) · [390px 工作簿](assets/biz-state-workbook-mobile.png) +- [查询失败与重试](assets/biz-state-error.png) + +## 验证 + +- 后端相关回归 **124 项通过**：新增监控服务、保留策略、日志导入、命令导出，以及停止/恢复/收尾/认领采集、双车道、spool、解析池与业务对比相关回归。 +- `web/tests/bizState.browser.mjs`：**30 项回归通过**。固定 120 个任务、54 个批次、120 行指标，覆盖分页、配置保存、模板缓存、过期响应、搜索、重试、日志下载、后台采集和窄屏布局。4500ms 列表延迟下最大并发轮询为 1；轮询阶段任务完整详情仅首次打开请求 1 次，未保存的周期值保持不变。全部 `/v1` 请求被固定响应拦截。 +- `node node_modules/vite/bin/vite.js build` 通过。项目原有主包超过 500 kB 提示仍在。 +- `node node_modules/typescript/bin/tsc -b`：本页已有类型问题已修复；全项目仍被 `BizMigrationPage.tsx` 的 4 个既有错误阻断（两处未使用变量、两处字符串传给数值控件）。 +- 本页 ESLint 从 16 个错误降为 15 个，保留既有 `any` 和 effect 状态同步问题及 1 个依赖警告；不宣称全项目 lint 通过。 + +浏览器回归运行方法（先启动 Vite，可通过环境变量指定 URL、截图目录和浏览器）： + +```powershell +cd web +node node_modules/vite/bin/vite.js --host 127.0.0.1 --port 5179 +# 另一个终端；Playwright 已安装时直接运行，否则设置 NETX_PLAYWRIGHT_MODULE 为其模块路径。 +node tests/bizState.browser.mjs +``` diff --git a/docs/reviews/compare-templates-review-2026-10-11.md b/docs/reviews/compare-templates-review-2026-10-11.md new file mode 100644 index 0000000..6c62f0d --- /dev/null +++ b/docs/reviews/compare-templates-review-2026-10-11.md @@ -0,0 +1,46 @@ +# 对比模板与业务对比评估、优化记录 + +日期：2026-10-11。范围：网络管理 → 割接对比 → 对比模板、业务对比。延续此前的业务对比和业务监控优化。本轮使用内存 SQLite、SQL 编译断言和本地浏览器固定 API 响应验证，没有连接真实设备，也没有改动生产数据库。 + +## 修复的结果准确性问题 + +1. **模板导入可能只保存有效部分。** 前后端检查全部检查项，空 Key、重复 ID、无效规则不再被静默丢弃。服务器在修改模板之前校验，错误响应包括配置路径和原因。执行时也检查历史模板的原始检查项，防止旧的无效配置被归一化后跳过。 +2. **错误过滤规则可能放行数据。** 新写入与执行入口拒绝未知运算符、混合分组/叶节点、空分组、无效正则及超过 12 层的嵌套。集合过滤要求非空数组；绝对/相对容差必须为有限非负数。无效归一模式、重复字段规则和接口别名同样报错。 +3. **复杂嵌套过滤被可视化编辑器截断。** 编辑器只接管能够完整表达的 OR/AND 结构，其余结构完整保留并显示 JSON。保存、导入、导出不删除嵌套条件。修改复杂结构可导出 JSON 后编辑并重新导入。 +4. **端口变化策略在 API 和导入导出中丢失。** `ignore_port_changes` 贯通请求模型、兼容单表配置、编辑、复制、保存、导入与导出。界面明确提供自动判断、接口必须匹配、允许端口变化三种选项；显式 `false` 不再丢失。 +5. **零、布尔和空值混淆。** Python 行键与过滤保留 `0`、`False`，布尔文本与 JSON 提取一致；数值比对不再把空值当成 0。Python/SQL 使用同一数值格式，支持 `+.5`、`1.`、科学计数法；解析不了的值按文本比较。 +6. **SQL 过滤与行键误匹配。** `contains` 的 `%`、`_`、反斜杠按字面值处理，空包含条件与 Python 一致。复合键改为 JSON 数组文本，避免 `('a|b','c')` 和 `('a','b|c')` 被拼成相同键。 +7. **任务范围失效导致隐式扩大或缩小对比。** 编辑任务时保留已移除的检查项 ID，显示错误并禁用保存、执行；用户可明确重新选择或启用当前全部检查项。服务器在保存和执行时都检查 ID，部分有效、部分失效的范围也会被拒绝。 +8. **不完整采集被解释成业务变化。** 比对入口只接受两侧状态为 `success` 的源批次；排队、执行中、失败、取消和部分完成批次返回 `source_batch_not_complete`。界面禁用这些选项。成功批次没有某指标或没有行的原有处理保持不变；本轮没有新增逐指标采集完备性判定。 +9. **删除破坏引用或在途对比。** 已被任务引用的模板不能删除；存在排队/执行中记录的任务及活动记录不能删除。模板页显示任务引用数量。这些是 API 状态检查，未新增数据库外键或锁，不宣称消除跨请求的全部并发窗口。 +10. **重新启用忽略字段仍然无效。** 勾选参与比对时清除遗留的 ignore/skip/off 规则。字段表保留模板配置的自定义字段；过滤专用字段也能显示。兼容数值、相对偏差和归一规则别名的显示。 + +## 性能与界面 + +- 检查项排序从逐表、逐批次查询改为 **一次按批次/metric 聚合**。相同 metric 的拆分表共享统计，SQL 不投影原始 CLI 日志；回归验证查询次数为 1。 +- `batch_metric_collect_ok` 只读取解析状态和行数，不加载命令原文。 +- 模板引用数量由一次 grouped 查询计算，避免每行请求。默认模板初始化/升级仍执行原来的查询。 +- 模板页面只请求模板和字段目录，省去业务监控任务、端口映射、对比任务三个列表；字段目录缓存 60 秒。业务对比与模板列表使用独立缓存键。 +- 两个列表默认每页渲染 50 行，可选 20/50/100/200；模板 ID 查找使用 Map。**API 仍返回完整列表，客户端分页减少 DOM，不减少响应体。** +- 模板编辑器使用深色平面面板、细边框、小圆角、蓝色选中态、等宽字段和固定表头。接口归一化折叠，检查项导航与规则区并排；窄屏导航横向滚动，操作按钮独立一行，过滤控件可换行，底部保存始终可见。 +- 导航和规则页签支持方向键、Home/End；规则控件增加可访问名称。保存期间编辑区不可交互，并阻止关闭后打开新模板造成保存响应串扰。 + +没有测量生产 PostgreSQL 的耗时或内存；本轮 SQL 变更由生成 SQL/绑定参数断言验证，未执行真实 PostgreSQL 集成回归。极大数值指数的 SQL 浮点转换、Python 与 PostgreSQL 正则方言差异、全文 JSON 搜索和大型 ZIP 内存仍需单独基准。既有回退机制不等于所有 SQL/Python 语义已完全一致。 + +## 验证与预览 + +- 后端相关回归：**120 项通过，1 项跳过**。跳过的是需显式开启的百万行压力测试。覆盖模板校验、原子更新、旧记录执行保护、端口策略、过滤、数值、复合键、查询次数、删除保护、范围失效、源批次状态，以及既有对比、搜索、配置快照、导出、恢复与内存相关回归。 +- Node 测试：**14 项通过**（8 项模板模型、6 项缓存）。 +- 浏览器：**53 个场景通过**（原业务对比 27 项、模板新增 26 项）。全部 `/v1` 请求使用固定响应，120 个模板列表验证默认 50 行上限；覆盖导入导出、复杂过滤、端口策略、负容差、忽略字段恢复、保存期间交互锁定、范围修复、键盘和 390px 布局。 +- Vite 生产构建通过；原有主包超过 500 kB 提示仍在。 +- 全项目 TypeScript 检查仍有 `BizMigrationPage.tsx` 的 4 个既有错误：2 个未使用变量、2 处字符串传给数值控件；本轮没有新增错误。 +- 新模板模型 ESLint 通过；业务对比页保留原来的 **23 个错误**（17 个 any、6 个 effect 状态同步），与修改前数量一致。不宣称全项目 lint 通过。 + +前后端应一起更新；新增 API 字段为模板引用数量与端口策略，无数据库迁移。原来使用标量 `in/not_in`、未知规则、负容差的模板需要修正再保存/执行；部分完成的源批次需要重新完成采集后再对比。 + +- [桌面模板编辑器](assets/compare-template-editor.png) +- [390px 模板编辑器](assets/compare-template-editor-mobile.png) +- [业务对比列表](assets/biz-compare-jobs.png) +- [此前业务对比记录](biz-compare-review-2026-10-10.md) + +复现：先在 `web` 启动 `node node_modules/vite/bin/vite.js --host 127.0.0.1 --port 5179`，另一个终端运行 `node tests/bizCompareTemplates.browser.mjs` 和 `node tests/bizCompare.browser.mjs`。可用 `NETX_PLAYWRIGHT_MODULE` 指定已有 Playwright，`NETX_TEST_URL` 指定本地地址，`NETX_TEST_OUTPUT` 指定截图目录。 diff --git a/docs/reviews/cutover-optimization-plan-2026-10-12.md b/docs/reviews/cutover-optimization-plan-2026-10-12.md new file mode 100644 index 0000000..cebc1c7 --- /dev/null +++ b/docs/reviews/cutover-optimization-plan-2026-10-12.md @@ -0,0 +1,477 @@ +# 割接对比（业务对比 / 割接监控）问题分析与优化计划 + +日期：2026-10-12。范围：网络管理 → 割接对比下的 5 个子模块：业务监控（采集）、业务对比、割接监控、对比模板、监控模板。 +用途：交给 Cursor 按任务逐项实施；Claude 负责验收。 + +本文基于代码阅读和本地脚本验证，没有连接真实设备，也没有在真实 PostgreSQL 上执行。标了 **[已验证]** 的问题已经用脚本复现（脚本放在会话 scratchpad，结论见各条目）。基线：开始修改前，相关后端回归 **279 项通过**，`tsc -b` 通过，`cutoverDataCache` / `compareTemplateModel` Node 测试通过。 + +--- + +## 0. 结论摘要 + +1. **两种模式共用一套“归一化 + 映射”概念，但各个代码路径的实现不一致。** 割接监控会先对端口映射的键和值做接口归一化，业务对比不会（**[已验证]**：按设备原始名称填写映射，业务对比全部变成“删除 + 新增”）。行过滤用原始值，映射作用在归一化之后。键字段不受 `field_rules` 归一化影响。跨厂商割接最容易在这里产生误报。 +2. **目前只有两种映射：接口类型别名（模板级，两侧同一套规则）和端口映射（before→after，一对一）。** 没有分侧（按厂商）归一化，没有字段值映射（状态枚举、对端 IP、VRF 改名），也无法提前检查映射覆盖率。映射漏配会直接表现为业务“缺失”，用户很难判断是映射问题还是业务问题。 +3. **除 LLDP 外，状态采集解析只实现了中兴。** 华为、思科等的 parser 都是会抛出 `NotImplementedError` 的空壳，且没有注册。内置对比模板（`ZTE status default`）里的过滤和容差也都按中兴设定。所以现在的“跨厂商映射”只对 LLDP 真正有效，其余检查项在跨厂商场景下要么采集不到，要么全量误报。 +4. **判定语义有几处会把正常割接判成失败：** + - 端口预设要求旧端口 `admin=down 且 phy=down`。现场常见的做法是拔纤但不 shutdown，这种情况验收时会判为“未完成/红”（**[已验证]**）。 + - 两侧都是 0 行时通过率是 100%（空对空误判为通过）。 + - 解析不到数据会被当成业务缺失。 + - 新旧两侧的采集命令集不一致时，会被当成业务丢失。 + - 内置模板对 `output_bps` 用绝对值容差 5 bps，对 `rx_power`（dBm）用百分比容差。 +5. **割接监控的运行数据无限增长。** 每个 `BizMigrationRun` 或 `BizCompareRun` 引用的采集批次都会被永久保护，run 和 diff 本身也没有保留策略。60 秒一轮的高频监控跑几天，数据库和磁盘会持续膨胀。 +6. **性能：** 业务对比只要配置了端口映射，所有检查项（包括不含接口字段的 BGP 路由百万行表）都会退回 Python 引擎。割接验收是同步 HTTP 请求，会把路由表全量载入内存。 +7. **引用保护有漏洞：** 被监控模板引用的对比模板、被任务或割接项目引用的端口映射都能被删除。删除后，业务对比会静默地按“无映射”执行；割接监控会直接报错停摆。 + +--- + +## 1. 现状框架梳理（给 Cursor 的背景） + +### 1.1 数据流 + +``` +业务监控任务(biz_state_task, 每台设备) + └─ 采集批次 batch ──> 每条命令 batch_command(parse_status/row_count/raw_line_count) + └─ 解析(ParseProfile → parser → FieldDef) ──> biz_state_metric_row(data_json) / biz_state_lldp_neighbor + │ +对比模板(biz_compare_template.metrics_json = 多个 sheet) + sheet: metric_id, key_fields, iface_fields, compare_fields, display_fields, + row_filters, field_rules(normalize/compare/tolerance), ignore_port_changes + 模板级: iface_normalize_json（接口类型别名，两侧同一套） + │ +┌───────────── 业务对比（BizCompareJob/Run/Diff）─────────────┐ +│ before 批次 vs after 批次（两台设备或同一设备前后） │ +│ 行过滤(原始值) → 接口归一化(两侧) → 端口映射(仅 before) │ +│ → 按 match_key 分组 → 同键按采集顺序 zip 配对 │ +│ → added/removed/changed/unchanged → 通过率 │ +│ 引擎：PG + 可下推 → SQL FULL OUTER JOIN；否则 Python │ +└─────────────────────────────────────────────────────────────┘ +┌───────────── 割接监控（BizMigrationProject/Batch/Run/Diff）──┐ +│ 旧设备: 画像基线 vs 高频当前 → old_kind │ +│ 新设备: 画像基线 vs 高频当前 → new_kind │ +│ 预期集(expect_set: 本批要迁移的端口/对象) + 监控模板(成功/异常规则)│ +│ → dual_verdict: migrated/migrating/anomaly/lost/unfinished… │ +│ → 本批完成(finish_batch)=终验 → 红单(带红继续) │ +│ 高频采集：每个 metric 一对(新/旧)任务，同轮次配对采样 │ +└─────────────────────────────────────────────────────────────┘ +``` + +### 1.2 现有“映射/归一化”层与使用点（不一致的根源） + +| 层 | 归属 | 作用对象 | 业务对比 | 业务对比·实时检索/回填/导出 | 割接监控 | +|---|---|---|---|---|---| +| 接口类型别名 `iface_normalize_json` | 对比模板 | `iface_fields`，两侧同规则，只替换前缀 | 有 | 有（`_run_transform_config`） | 有 | +| 端口映射 `biz_port_mapping` | 任务 / 项目 | `iface_fields`，before→after，精确匹配 + 父接口继承 | **键/值不归一化** | 同左 | **键/值先归一化**（`service.py:1019`） | +| 行过滤 `row_filters` | sheet | **原始值**（归一化之前） | 有（PG 可下推） | 有 | 有（Python） | +| 字段规则 `field_rules.normalize` | sheet | **仅 compare 字段**，键字段不处理 | 有 | 有 | 仅完整性校验 | +| 监控模板 `sheet_overrides` | 监控模板 | 状态语义（up/down 值、成功/异常模式） | — | — | 有 | + +关键文件： +- 引擎：`netx_api/biz_state/compare_engine.py`、`compare_rules.py`、`iface_normalize.py`、`compare_sql.py` +- 业务对比服务：`netx_api/biz_state/compare_service.py`（约 4000 行） +- 割接监控：`netx_api/biz_migration/evaluate.py`、`service.py`、`monitor_templates.py`、`auto_monitor.py` +- 采集：`netx_api/biz_state/collect_runner.py`、`claim.py`、`profiles.py`、`parsers/*` +- 保留：`netx_api/biz_state/retention.py` +- 前端：`web/src/features/cutover/BizComparePage.tsx`、`BizMigrationPage.tsx`、`BizMonitorTemplatesPage.tsx`、`BizStatePage.tsx` +- 内置模板：`netx_api/biz_state/data/default_zte_status_template.json` + +--- + +## 2. 问题清单 + +严重度：**P0** 会产生错误结论或数据损坏；**P1** 高概率误报或明显性能/可用性问题；**P2** 边界问题或体验问题；**P3** 清理项。 + +### 2.1 业务对比（BizCompare） + +| ID | 严重度 | 问题 | 位置 | 影响 | +|---|---|---|---|---| +| B-01 | P0 **[已验证]** | 端口映射的键/值不按模板接口归一化规则转换，必须由用户手工填写“归一化后”的名称；割接监控却会自动归一化。两种模式行为不一致。 | `compare_engine.py:224-260`（`apply_port_map` 用原始 `pmap`），`compare_service.py:2461`、`_run_transform_config:2337` | 模板有 `GE→gei` 规则时，映射 `GE0/1/0/1→gei0/2/0/1` 完全不命中，结果 added=1 / removed=1、`map_ok=False` | +| B-02 | P1 | 任务里只要有端口映射，所有 sheet 都会跳过 SQL 引擎，包括没有接口字段的 BGP/路由表 | `compare_sql.py:165`（`if port_map: return "port_map"`） | 百万行 vpnv4 路由退回 Python，内存和耗时都大幅上升；割接任务几乎都会配映射 | +| B-03 | P1 | 端口映射被删除后，业务对比静默地按“无映射”执行（还会触发自动忽略端口的启发式），删除时也不检查引用 | `delete_mapping:1603`，`_port_map_dict:1612`，`_validate_compare_job:2220` | 结果语义悄悄改变 | +| B-04 | P1 | 对比模板删除只检查 `BizCompareJob` 的引用，不检查 `BizMonitorTemplate.compare_template_id` | `delete_template:1521`，`list_templates:1391` 的 `job_count` | 割接窗口中删掉模板后，`run_evaluate` 报 `compare_template_missing`，监控停摆 | +| B-05 | P1 | 创建向导或任务配置里编辑了映射文本但没点“保存映射”，创建/保存/执行任务时映射文本被丢弃，没有任何提示 | `BizComparePage.tsx:1814 jobConfigBody`、`1895 createJob`、`1997 saveJobConfig`、`2029 runNow` | 用户以为配了映射，实际没生效，产生大量假“缺失/新增” | +| B-06 | P2 | 映射文本解析只认 `, | Tab` 作分隔符；空格分隔或格式不对的行被静默丢弃，没有解析计数或错误提示 | `BizComparePage.tsx:1720 parseMapRows` | 映射少行而不自知 | +| B-07 | P1 | 映射校验只针对“第一个含接口字段的 sheet”，结果以原始 JSON 展示；只报告“映射键未命中”，不报告“before 侧端口没有映射且在 after 侧不存在”（漏配） | `validate_mapping:1810`，`mapping_stats`（`compare_engine.py:100`），`BizComparePage.tsx:2272` | 漏配映射最终表现为业务缺失 | +| B-08 | P1 | `row_filters` 作用在原始行（归一化、映射之前）；跨厂商时，同一条过滤在两侧含义不同 | `_load_metric_rows:1619`（SQL/Python 都在加载时过滤），`_run_sheet:2179` | 例如 `interface contains gei` 会把华为 `GE0/0/1` 整侧滤掉；ARP 的 `age_timer`（HH:MM:SS）只适用于中兴 | +| B-09 | P2 | 键字段不应用 `field_rules.normalize`（如 MAC 格式），跨厂商的 MAC/大小写差异导致配对失败 | `compare_engine.py:96 row_key`，`compare_sql.py _rk_sql` | 键里含 MAC 的 sheet（如 `l2vpn_mac`）全量误报 | +| B-10 | P1 | 两侧都是 0 行时通过率为 100%；解析不到数据（命令执行了但解析 0 行）被当成业务“缺失” | `_enrich_summary:3015`，`compute_pass_rate:210` | 空对空误判为通过；模板不匹配被当成业务丢失 | +| B-11 | P1 | 新旧两侧采集命令集不一致（例如旧设备采了 bgp ipv4 summary，新设备没采），对比结果是“全部缺失” | 没有任何检查 | 采集配置问题被误判为业务问题 | +| B-12 | P2 | 同键多行按采集顺序 zip 配对，CLI 输出顺序变化会产生假“不一致” | `compare_engine.py:331+`、`compare_sql.py` 中 `dup_rn ORDER BY seq` | ECMP/多路径场景误报 | +| B-13 | P2 | 只要提供了映射，`ignore_port_changes` 就被忽略；部分映射时，未映射端口既不按名称匹配也不忽略端口 | `compare_engine.py:224` | 部分映射时策略不可控 | +| B-14 | P2 | 结果表的键列只显示映射后的名称，看不到 before 侧原始端口名 | `BizComparePage.tsx:4161`（`d.key[k] ?? pre[k]`，`pre` 取 `mapped_before`） | 跨厂商时现场人员无法对照原端口 | +| B-15 | P2 | 判定没有方向：状态 down→up、前缀数增加也算失败；流量计数器被当成严格比较项 | `compare_rules.values_equal` | 割接后业务恢复或流量迁入被判为失败 | +| B-16 | P3 | `batch_metric_collect_ok` 是死代码；`_load_metric_rows` 中 LLDP 分支重复过滤；PG 下推时 `before_raw_count` 实际上是过滤后的数量 | `compare_service.py:78`、`1675-1688`、`2199` | 清理 | +| B-17 | P3 | 防止同一任务并发对比的锁是进程内的；`_job_has_active_run` 检查和插入不是原子操作。biz_state worker 多副本时，自动对比有极小概率并发 | `compare_service.py:57`、`2887` | 低概率 | + +### 2.2 割接监控（BizMigration） + +| ID | 严重度 | 问题 | 位置 | 影响 | +|---|---|---|---|---| +| M-01 | P0 **[已验证]** | 端口预设的成功条件要求旧侧 `removed` 或 `admin=down 且 phy=down`。旧端口拔纤但没有 shutdown（admin up / phy down）、新端口已 up 时：窗口内是 `migrating/黄`，**验收时是 `unfinished/红`** | `monitor_templates.py:255-275`（`_success_stateful` 的 `old_down_groups` 用的是 AND） | 正常割接被判为验收失败 | +| M-02 | P1 | 每个 run 引用的 HF 批次都会被永久保护，run 和 diff 本身也没有保留策略 | `retention.py:65-75` | 高频监控期间数据库无限增长 | +| M-03 | P1 | 验收（`finish_batch`）和手动评估是同步 HTTP 请求；路由类 sheet（`bgp_route`/`ip_route`）会把 4 个批次的全量行载入 Python | `biz_migration_router.py` 的 evaluate/finish，`service.py:1148-1163` | 超时、内存暴涨；而且这期间一直持有项目行锁（`with_for_update`，`service.py:940`） | +| M-04 | P1 | 默认监控模板会采集对比模板里的全部 metric，包括只能手动采集的路由表。没有手动采集时，该 sheet 是 `current_missing`，验收**必然**失败（`current_missing` / `coverage_incomplete`） | `resolve_collect_metric_ids:525`、`finish_batch:2358-2371`、`MANUAL_ROUTE_METRICS` | 默认配置跑不通验收 | +| M-05 | P1 | 只要任意 sheet 有重复键，验收就判失败（`duplicate_keys`，同时 `coverage_complete=False`），不管这个 sheet 是否在本批范围内 | `service.py:1299`、`2366` | 例如 ISIS L1/L2 两行同键，就永远过不了验收 | +| M-06 | P1 | 新旧两侧画像任务的命令集可能不同，HF 任务沿用各自画像的命令项，旧侧有、新侧没采的切片会被判为 `lost` | `_portrait_metric_items:1741`、`ensure_highfreq:1919` | 采集配置问题被当成业务丢失 | +| M-07 | P1 | 创建项目时不校验基线批次是否属于该侧画像任务（patch 时会校验） | `create_project:686-691`（对比 `patch_project:744-750`） | 可能拿错设备的基线；直到评估时才报 `baseline_task_mismatch` | +| M-08 | P2 | 不支持同设备割接（板卡更换、同框端口迁移）：`collect_pair_same_device` 直接报错 | `service.py:2016` | 常见场景无法使用 | +| M-09 | P2 | 每个 metric、每台设备一个 HF 任务；15 个检查项就是每分钟每台设备约 15 次 SSH 登录；没有单设备并发上限 | `ensure_highfreq:1943`，`claim.py` | VTY 耗尽、AAA 日志刷屏，进而采集失败，看板大量 `current_missing` | +| M-10 | P2 | 每轮自动评估都重新载入基线全量行（基线不会变） | `run_evaluate.metric_rows:1059` | 白白消耗 CPU/IO | +| M-11 | P2 | 双侧评估里，诊断索引的键串用 `str(v)` 构造，而其他地方用 `scalar_text().strip()`：`None` 变成 `"None"`，`True` 变成 `"True"`，空白不裁剪，预期键匹配不上 | `evaluate.py:759`、`compare_engine.py:273 _key_obj` | **[已验证]**：键字段为 `None` 时，存在且未变化的对象被判为 `unfinished` | +| M-12 | P2 | 用 metric 名称包含 `bgp` / `pw` 来决定是否用端口映射推导范围（字符串启发式） | `evaluate.py:804` | 新增 metric 时行为不可预期，也违反“引擎不写 metric 特例”的原则 | +| M-13 | P2 | 映射允许 N:1 吗？业务对比允许（结果是重复键），割接监控直接报 `mapping_target_ambiguous`；1:N（拆分 breakout）两边都不支持（`before_if` 唯一） | `models/biz_state.py BizPortMappingRow`，`service.py:991/1021` | 端口汇聚、拆分割接无法表达，报错也没说明是哪几行 | +| M-14 | P3 | `project_to_dict`（GET/列表）里会写库（`resolve_project_monitor_template` 持久化默认模板）；`list_projects` 存在 N+1 | `service.py:486-503`、`543` | 只读接口有副作用 | +| M-15 | P3 | 一个键在后续验收中已经变绿，对应红单仍然保持 open，只能人工关闭 | `_persist_red_tickets_from_run:2326` | 红单堆积 | + +### 2.3 内置模板与跨厂商能力 + +| ID | 严重度 | 问题 | 位置 | +|---|---|---|---| +| T-01 | P1 | `interface_detail.output_bps` 用 `numeric` 绝对容差 5（bps），而 `input_bps` 用 5%：相当于对出方向流量做严格比较 | `default_zte_status_template.json` | +| T-02 | P1 | `optical_brief.rx_power` 用 `percent 5`：dBm 是对数单位，-0.5 和 -0.6 已经差 20%。应改为绝对值 dB 容差 | 同上 | +| T-03 | P2 | 流量计数器（bps/util）默认参与比较；割接后流量迁移是预期行为，会产生大量噪声 | 同上 | +| T-04 | P2 | `l2vpn_mac` 键里的 `ac_port` 是接口，但没放进 `iface_fields`，端口映射不生效 | 同上 | +| T-05 | P1 | 华为、思科、华三、Juniper、诺基亚、爱立信的状态 parser 都是空壳（没有注册）；只有 LLDP 跨厂商可用 | `parsers/huawei/__init__.py` 等 | +| T-06 | P2 | 各厂商的状态值不统一（`Established`/`ESTAB`、`UP`/`up`/`*down`、`Full/DR`），而对比层只有 `lower`/`upper`/`mac` 几种归一化，没有枚举别名映射 | `compare_rules.normalize_value` | + +--- + +## 3. 优化方案（分阶段任务，按顺序执行） + +### 通用执行约束（Cursor 每个任务都要遵守） + +1. **引擎里不写 metric 特例**：所有 metric 相关的行为放进模板、sheet 或 profile 声明（遵循 `skills/biz-state/netx-biz-state/SKILL.md` 第 4、5 节）。 +2. **Python 和 SQL 必须一致**：凡是影响配对或判定的改动，要么两个引擎都实现，要么在 `sql_compare_skip_reason` 里让 SQL 自动降级到 Python，并补“同输入两引擎结果一致”的对照测试（SQL 侧至少断言生成的 SQL 和降级原因）。 +3. **历史 run 不变**：新增的执行配置写进 `config_snapshot`，并把 `version` 升到 2。`version==1` 和旧记录的回填、检索、导出保持原行为，不能因为新逻辑改变历史结果。 +4. **数据库结构**：尽量用 JSON 字段扩展，不加列。确实需要加列或表时，同时补 `alembic/versions/` 和 `netx_api/schema_patches.py`（参照现有做法），并写回归测试。 +5. **每个任务**：补单元测试；跑第 5 节的回归命令；前端改动跑 `tsc -b` 和 `vite build`，涉及交互的补 `web/tests/*.browser.mjs` 场景（全部 `/v1` 用固定响应）。 +6. 一个任务一个提交，提交信息写清楚任务 ID（例如 `fix(cutover): B-01 normalize port-map keys`）。 + +--- + +### 阶段 A：正确性快修（P0/P1，改动小，先做） + +#### A1 端口映射键/值统一归一化（B-01） +- 新增函数 `normalize_port_map(port_map, rules) -> (normalized_map, conflicts)`，放在 `iface_normalize.py`：键和值都用 `normalize_iface_name` 处理。归一化后键冲突（两个原始键归一后相同但目标不同）要作为 conflicts 返回。 +- 业务对比：在 `_run_transform_config`（`compare_service.py:2337`）统一处理，覆盖执行、实时检索、回填、导出所有路径。**只对 `config_snapshot.version>=2` 的 run 生效**；`version==1` 和旧记录保持原样。 +- 割接监控：`run_evaluate` 和 `list_baseline_expect_objects` 里的内联归一化改为调用这个函数，消除重复实现。 +- `mapping_stats` 的 `hint` 文案去掉“必须填写归一化后名称”。 +- **验收测试**：模板规则 `GE→gei`、映射写原始名 `GE0/1/0/1→gei0/2/0/1`，新 run 的结果是 `unchanged=1`；同样的数据在 `version=1` 的 run 上回填，结果不变。 + +#### A2 引用保护和缺失检测（B-03、B-04） +- `delete_mapping`：被 `BizCompareJob.mapping_id` 或 `BizMigrationProject.mapping_id` 引用时返回 409 `mapping_in_use`，detail 里带引用列表（任务名/项目名）。 +- `delete_template`：同时检查 `BizMonitorTemplate.compare_template_id`，返回 409 `template_in_use`（detail 区分 job/monitor）。`list_templates` 增加 `monitor_template_count`。 +- `_validate_compare_job`：`job.mapping_id` 非空但映射不存在时返回 400 `mapping_missing`。 +- 前端：删除失败时显示引用对象；模板列表显示两类引用数。 +- **验收测试**：上述 3 种 409/400 的单测；被引用的映射无法删除。 + +#### A3 映射文本不会丢失（B-05、B-06） +- `parseMapRows` 改为返回 `{rows, invalidLines[]}`：两段文本即使用空格分隔也接受（正好两个 token 时）；重复的 before 行要提示。 +- 映射文本相对于已保存版本“有修改”时：创建任务、保存配置、立即执行都要先拦截，提供“保存映射并继续 / 放弃修改”两个选项（或者自动先保存映射再保存任务，二选一，推荐自动保存并提示）。 +- 编辑区下方实时显示“已解析 N 对 / 无效 M 行（行号）”。 +- 保存一个被其他任务或项目共用的映射时，提示“该映射被 X 个任务 / Y 个割接项目使用”（后端 `list_mappings` 返回引用计数）。 +- **验收（浏览器脚本）**：在向导里输入映射后直接点“创建”，请求体里的 `mapping_id` 非空，或者被拦截并出现提示；空格分隔的行能被解析；无效行有提示。 + +#### A4 端口预设成功条件修正（M-01） +- `preset_override_for_metric("interface_brief")` 中旧侧的条件改为三个 OR 组：`[removed]`、`[phy in down]`、`[admin in down]`；新侧保持 `admin up 且 phy up`。 +- 已存在的监控模板：在 `ensure_default_monitor_templates` 中增加一个幂等升级，**只替换结构上和旧预设完全一致的 override**（逐字段对比），用户自定义过的不动，并在 `override_warnings` 里提示“端口成功条件为旧版 AND 语义”。 +- **验收测试**：新增 4 个用例——旧侧 admin up/phy down、admin down/phy up、两者都 down、端口已删除；在窗口内和验收时，新侧 up 都应判为 `migrated/green`。另外，新侧 phy down 时仍判为红。 + +#### A5 创建项目时校验基线归属（M-07） +- `create_project` 复用 `patch_project` 的校验：基线批次的 `task_id` 必须等于对应侧的画像任务，状态必须是 `success` 或 `partial`。抽成公共函数 `_validate_baseline_ownership`。 +- **验收测试**：拿其他设备的批次作为基线创建项目，返回 400 `old_baseline_invalid` / `new_baseline_invalid`。 + +#### A6 键串规范化统一（M-11） +- 在 `compare_engine.py` 新增 `key_tuple(row, fields)`，统一用 `scalar_text(...).strip()`。`_key_obj` 的输出键值和 `build_diff_index_from_compare` 都改用它（`evaluate.py:759` 不再用 `str(v)`）。 +- **验收测试**：键字段为 `None`、`True`、带首尾空格时，预期键能命中，判定为 `ok`/`migrated` 而不是 `unfinished`。 + +#### A7 内置模板参数修正（T-01、T-02、T-04） +- `default_zte_status_template.json`： + - `output_bps` 改为 `percent 5`（和 `input_bps` 对称）。 + - `rx_power` 改为 `numeric`，`tolerance: 2`（单位 dB）。 + - `l2vpn_mac.iface_fields` 加上 `ac_port`。 + - `interface_detail` 的 `input_bps/output_bps/in_util/out_util` 移出 `compare_fields`，只保留在 `display_fields`（见 V3 的方向性规则；在 V3 完成之前先不比较）。 +- `ensure_default_zte_status_template` 增量升级：**只修改值仍等于旧打包值的字段**，用户改过的保留。 +- **验收测试**：旧打包值的模板升级后变成新值；用户自定义过 `rx_power` 规则的模板升级后保持不变。 + +--- + +### 阶段 B：统一的“身份归一化流水线”（跨厂商核心） + +目标：业务对比执行、实时检索、回填、导出、映射校验、割接评估、预期集选择器这 7 条路径，**对同一行算出相同的 match key**，并支持按侧（按厂商）的规则和字段值映射。 + +#### B1 抽出 `IdentityPipeline`（解决“各路径各写一遍”） +- 新文件 `netx_api/biz_state/identity_pipeline.py`： + ```python + @dataclass(frozen=True) + class SideSpec: + iface_rules: list[dict] # 该侧接口归一化（已合并 both + side） + value_maps: dict[str, dict] # field -> {from_lower: to} + @dataclass(frozen=True) + class Pipeline: + before: SideSpec; after: SideSpec + port_map: dict[str, str] # 已归一化（A1） + key_fields, iface_fields, compare_fields, field_rules, row_filters, match_keys + def transform(self, row, side) -> dict # 归一化 → 值映射 →（before 侧）端口映射 + def match_key(self, row, side) -> tuple + def keep(self, row, side) -> bool # 行过滤（在 transform 之后求值，见 B4） + ``` +- 由 `build_pipeline(sheet, snapshot_or_template, port_map, before_vendor, after_vendor)` 统一构造；`compare_rows`、`_live_search_sheet_diffs`、`_load_search_key_groups`、`_hydrate_diff_rows`、`validate_mapping`、`evaluate_metric_dual`、`list_baseline_expect_objects` 全部改用它。 +- **验收测试**：参数化对照测试，用同一份 fixture 断言 7 条路径对同一行给出的 match key 相同（重点覆盖：映射、父接口继承、QinQ、值映射、归一化规则）。 + +#### B2 分侧接口归一化 + 厂商内置规范化 +- `iface_normalize_rules` 每条增加可选字段 `side: "both"|"before"|"after"`（默认 `both`）和 `vendor`（可选，按批次的厂商自动生效）。前端模板编辑器支持选择。 +- 内置“厂商内全称/缩写”规范化表（`iface_normalize.py` 中的 `VENDOR_CANONICAL`）： + - 华为：`GigabitEthernet`↔`GE`、`XGigabitEthernet`↔`XGE`、`100GE`、`Eth-Trunk` + - 思科：`GigabitEthernet`↔`Gi`、`TenGigE`↔`Te`、`HundredGigE`↔`Hu`、`Bundle-Ether`↔`BE` + - 华三：`GigabitEthernet`↔`GE`、`Ten-GigabitEthernet`↔`XGE`、`Bridge-Aggregation`↔`BAGG` + - 中兴：大小写 + - 统一去掉名称中的空格。 + + 只做厂商内同义统一，**不做跨厂商编号转换**（那是端口映射的职责）。模板 sheet 可以用 `builtin_vendor_canonical: false` 关闭。 +- 批次的厂商取 `BizStateBatch.vendor`，取不到再用任务的 vendor，经 `resolve_vendor_key` 解析。 +- `config_snapshot` v2 记录两侧最终生效的规则。 +- **验收测试**：华为 `GigabitEthernet0/0/1` 与 `GE0/0/1` 互认；before 华为、after 中兴，只对 before 生效的规则不影响 after。 + +#### B3 字段值映射（T-06、B-09） +- `field_rules[]` 新增可选字段： + - `value_map: {"estab": "established", "*down": "down", ...}`（大小写不敏感匹配，输出规范值） + - `side: both|before|after` + - `apply_to_key: true|false`（默认：字段在 key_fields 中时为 true） +- `normalize` 的处理顺序：`normalize`（strip/lower/mac…）→ `value_map`。**键字段也要应用**（解决 MAC 作为键时跨厂商对不上的问题）。 +- SQL：`value_map` 作用在键或比较字段上时，`sql_compare_skip_reason` 返回 `value_map`，降级到 Python（以后可以用 `CASE` 下推，本期不做）。 +- `compare_validation.py` 校验 `value_map` 必须是 `str→str` 映射，并且每个 sheet 最多 500 项。 +- 模板编辑器：字段规则行增加“值映射”编辑（每行一对，可以粘贴）。 +- **验收测试**：`state: Estab` 对 `established`，配置值映射后为 unchanged；MAC 作为键字段时，`00e0-fc12-3456` 与 `00e0.fc12.3456` 能够配对；Python 与 SQL 降级原因断言。 + +#### B4 行过滤在归一化之后求值（B-08） +- 新 run（snapshot v2）：先 `transform` 再 `row_filters`。 +- PG 下推只在“过滤涉及的字段都不在 `iface_fields` 中、也没有值映射”时启用；否则 Python 过滤。 +- 支持过滤条件带 `side`（只作用于某一侧），满足“旧侧只看 dynamic ARP”这类需求。 +- 模板编辑器提示“过滤在归一化之后执行，请按规范值书写”。 +- **验收测试**：规则 `GE→gei` 加过滤 `interface contains gei`，before 侧华为 `GE0/0/1` 不再被滤掉；`version=1` 的 run 回放结果不变。 + +#### B5 端口映射覆盖率预检（B-07、B-11、M-06 的基础） +- 新接口：`POST /v1/biz-state/compare/preflight`，请求体 `{template_id, enabled_sheet_ids, mapping_id | mapping_rows, before_batch_id, after_batch_id}`。割接监控复用同一个服务函数：`/v1/biz-migration/projects/{id}/preflight`，传入基线和最新 HF 批次。 +- 按 sheet 返回： + - `before_count / after_count`（过滤后）、`parse` 状态（见 V1） + - `mapped_exact[]`、`mapped_by_parent[]`、`identity_same_name[]`（未映射但两侧同名） + - **`suspect_unmapped[]`**：before 侧接口没有映射，after 侧也不存在 → 疑似漏配 + - `unused_map_keys[]`、`map_targets_missing_on_after[]`、`n_to_1_targets[]` + - `command_coverage`：两侧执行过的 profile_id 集合和差集（见 V2） +- 汇总：映射覆盖率 = 已映射 / (已映射 + 疑似漏配)，以及“可执行 / 有风险”的结论。 +- 业务对比 run 的 summary 也写入同结构的 `mapping_coverage`（按 sheet），结果页可以查看。 +- 前端：映射块里的“校验”按钮改为显示预检表格（取代原始 JSON）；疑似漏配可以一键加入映射编辑区（目标留空，待填）。 +- **验收测试**：构造 3 个端口，分别是精确映射、父接口继承、漏配，预检分类正确；`mapping_rows` 未保存时也能预检。 + +#### B6 SQL 引擎条件放宽（B-02） +- `sql_compare_skip_reason`：`port_map` 只在 `iface_fields ∩ (key_fields ∪ compare_fields)` 非空时才阻止 SQL。 +- **验收测试**：有端口映射的任务，`bgp_route` sheet（`iface_fields=[]`）走 SQL（断言 skip reason 为空），`interface_brief` 仍走 Python。 + +#### B7 映射语义：N:1 与部分映射（B-13、M-13） +- 决策见第 6 节，默认方案： + - **N:1**：允许，但预检和 summary 中必须列出；业务对比按重复键规则处理（V4）；割接监控继续拒绝，但错误信息里列出冲突行。 + - **1:N（breakout）**：本期不支持，在映射编辑器里检测到重复 before 时明确报错，不再只报 `duplicate_before_if`。 + - **部分映射**：新增 sheet 选项 `unmapped_port_policy: "by_name"|"ignore_port"`（默认 `by_name`，即当前行为），让 `ignore_port_changes` 与映射可以组合使用。 +- **验收测试**：N:1 在预检中出现在 `n_to_1_targets`；割接监控的报错包含冲突对。 + +--- + +### 阶段 C：判定语义与数据可信度 + +#### V1 sheet 级数据可信度（B-10） +- `_run_sheet` 和割接评估为每侧补充 `collect_quality`：该 metric 的命令数、`parse_status` 计数、`row_count`、`raw_line_count`、`declared_total`（只读这些列，不读 `raw_text`）。 +- 新增 sheet 状态： + - `no_data`：两侧过滤后都是 0 行，且命令都执行成功 → `pass_rate=null`，不计入总通过率，卡片显示“无数据”。 + - `suspect_parse`：某侧 `raw_line_count > 5` 但 `row_count == 0` → 卡片显示“疑似解析失败”，失败行数照常显示，但给出明确提示。 + - `not_collected`：某侧没有任何命令产生该 metric → 不做判定，`pass_rate=null`。 +- 任务级 `ok`：任一 sheet 处于 `not_collected` / `suspect_parse` 时为 `null`（未判定），不再是 `true`。 +- 前端卡片和 CSV `_summary.csv` 增加状态列。 +- **验收测试**：0 对 0 → `no_data`；before 1000 行、after 有原文但 0 行 → `suspect_parse`；after 侧无命令 → `not_collected`。 + +#### V2 采集命令集对齐（B-11、M-06） +- `ParseProfile` 增加可选字段 `slice: dict[str, str]`（例如 `zte.bgp_ipv4_summary` 的 slice 是 `{"afi": "ipv4"}`），声明这个 profile 产出哪个切片。先给中兴的 `bgp_peer` / `bgp_route` / `vrrp` / `isis` profile 补齐。 +- 判断一个 sheet 是否“被某侧覆盖”：该侧存在 `metric_id` 相同的成功命令，且（sheet 的 row_filters 里有 `eq` 条件时）存在 slice 与之相符的 profile。 +- 业务对比的预检（B5）和执行时都做这个检查：未覆盖的 sheet 标记 `not_collected`（V1）。 +- 割接监控：`ensure_highfreq` 生成 HF 任务项时，取**两侧画像命令项的并集**（按 profile 的 slice 转换成该侧厂商可用的 profile；找不到时返回明确错误 `no_profile_for_slice:::`）。评估时，未覆盖的 sheet 标记为 `not_collected`，不判红。 +- **验收测试**：旧画像有 ipv4 + vpnv4，新画像只有 vpnv4 → HF 任务两侧都有 ipv4 + vpnv4；如果强制只有一侧有，评估结果是 `not_collected`，而不是 `lost`。 + +#### V3 方向性判定（B-15、T-03） +- `field_rules.compare` 新增： + - `not_worse`：配合 `rank: ["up", "down"]`（排在前面表示更好）；after 不差于 before 即通过。 + - `numeric_min`：配合 `tolerance`（百分比），`after >= before*(1-tol%)` 即通过（适用于前缀数、流量“不应明显下降”）。 + - `numeric_max`：对称方向。 +- `explain_diff` 输出方向原因，例如 `worse: up→down`、`drop 37% > 10%`。 +- SQL：先降级（skip reason 为 `directional_compare`），后续再下推。 +- 内置模板：`interface_brief.prot`、`bgp_peer.state` 默认仍为 `eq`（保守）；在模板编辑器里提供这些新模式供选择；流量计数器若要比较，推荐用 `numeric_min`。 +- **验收测试**：`down→up` 用 `not_worse` 判为通过，`up→down` 判为失败；前缀数 1000→1200 用 `numeric_min 5%` 判为通过，1000→900 判为失败。 + +#### V4 重复键配对：先排序再配对（B-12） +- Python：同一个 match key 下，两侧各自按“归一化后的 compare 字段元组, 原始 seq”排序，然后 zip。 +- SQL：`dup_rn` 改为 `row_number() OVER (PARTITION BY rk ORDER BY , seq, id)`。 +- 用 sheet 选项 `duplicate_pairing: "sorted"|"ordered"` 控制，默认 `sorted`，只对 snapshot v2 生效；v1 回放保持 `ordered`。 +- **验收测试**：同键两行的值 `[a, b]` 对 `[b, a]` → 2 个 unchanged（原来是 2 个 changed）；Python 和 SQL 生成 SQL 的断言。 + +#### V5 原始身份展示（B-14） +- diff 增加 `raw_key_before`（before 原始键值，未归一化/映射）。实现方式：在 Python 路径生成；对精简行，在回填时由 `IdentityPipeline` 反推；SQL 路径没有映射，原始键等于 key。 +- 结果表：当原始值与映射后的值不同时，键单元格显示 `cgei-0/2/0/1 ← GE0/0/1`；CSV 增加 `__raw_before` 列（只针对接口键）。 +- **验收（浏览器脚本）**：映射行同时显示两侧端口名。 + +--- + +### 阶段 D：割接监控的健壮性与性能 + +#### H1 run、diff 保留策略 + 批次保护收敛（M-02） +- 新配置项：`biz_migration_auto_run_keep`（默认每个 batch 保留 300 个 auto run）、`biz_compare_auto_run_keep`（默认每个 job 保留 50 个 auto run）。 +- 清理规则： + - 永久保留：`purpose in (acceptance, manual)` 的 run，以及被红单 `run_id` 引用的 run。 + - 其余 auto run：超过数量上限的删除（连同其 diff）。 + - 保护范围：`retention.protected_batch_map` 只保护“仍然存在的 run”所引用的批次，并**加上 `summary.metric_batches` 里的所有批次**（目前只保护首个 binding 的批次）。 +- 清理挂在 `biz_state_scheduler._maybe_purge_retention` 上，每小时执行一次。 +- **验收测试**：插入 400 个 auto run 和 1 个验收 run，清理后剩 300 + 1；被删除 run 所引用的 HF 批次不再受保护，能按 `retention_days` 被清理；验收 run 引用的所有批次（包括 `metric_batches`）都受保护。 + +#### H2 评估和验收改为后台执行（M-03） +- 仿照业务对比的 `enqueue_compare`：`POST /batches/{id}/evaluate` 和 `/finish` 返回 `{run_id, status: "running"}`，在后台线程里执行，`BizMigrationRun.status` 依次经过 `running/success/failed`，`summary.progress` 记录进度。 +- 项目行锁只在“写 run 和 diff”这个短事务里持有，不再覆盖整个计算过程。 +- 重启恢复：参照 `recover_interrupted_compares_on_startup`，把遗留的 `running` 标记为 `cancelled`。 +- 前端：轮询 run 的状态；验收按钮在执行期间不可重复点击。 +- **验收测试**：评估请求立即返回；并发两次评估，第二次返回 409；模拟重启后状态被正确恢复。 + +#### H3 路由类 sheet 的处理（M-04） +- 默认：`MANUAL_ROUTE_METRICS` 对应的 sheet **不参与验收门槛**。没有在本批窗口内手动采集时，显示为 `manual_pending`（灰色），不计入 `current_missing` 或 `coverage_incomplete`；已采集时参与判定。 +- 路由 sheet 的双侧比对改为复用业务对比的 SQL 路径（旧基线 vs 旧当前、新基线 vs 新当前，各执行一次 `run_sql_sheet_compare`，只取计数和失败行），避免在 Python 里载入百万行。 +- **验收测试**：默认监控模板、没有采集路由时，验收不因路由失败；采集了路由时，判定会出现在结果里。 + +#### H4 基线行缓存（M-10） +- 进程内 LRU：键为 `(batch_id, metric_id)`，只缓存基线批次（基线不可变），总行数上限可配置（默认 50 万行）。基线变更或项目删除时失效。 +- **验收测试**：连续两次 auto 评估，第二次不再查询基线行（mock 计数）。 + +#### H5 重复键的处理口径（M-05） +- `duplicate_keys` 只在“本批范围内的键（expect）有重复”时作为验收失败原因；其他 sheet 的重复键作为警告，写进 `accept_summary.warnings`。 +- 卡片上展示具体是哪些重复键（已有 `duplicate_key_list`），并提示“完善 key_fields（例如 ISIS 加上 `lev`）或在 sheet 上启用 `duplicate_pairing: sorted`”。 +- **验收测试**：范围外的 sheet 有重复键时验收可以通过；范围内有重复时验收失败。 + +#### H6 HF 采集合并与单设备并发上限（M-09） +- 决策见第 6 节，默认：**同间隔、同 lane（light/heavy）的 metric 合并为一对任务**（一次 SSH 采多条命令）；heavy 和 manual 的路由表保持独立。 +- 新配置项 `biz_state_max_sessions_per_ne`（默认 2）：`claim_queued_batches` 认领时，同一个 `ne_id` 正在运行的批次达到上限就跳过（成对认领时要两侧都满足）。 +- `_hf_bindings` 的结构保持兼容（`metric_ids` 本来就是列表）。 +- **验收测试**：15 个 light metric、同一间隔 → 每侧 1 个 HF 任务；单设备并发上限生效（3 个同 NE 批次只认领 2 个）。 + +#### H7 同设备割接模式（M-08） +- 当 `old_ne == new_ne` 时允许创建项目：每个 metric 只建一个 HF 任务，新旧两侧共用同一批次（`collect_group_id` 只含一个任务，轮次天然对齐），映射表达同框端口迁移。 +- `evaluate_metric_dual` 不需要改（两侧是同一份行数据，靠映射区分新旧端口）。 +- `enqueue_collect_group` 允许单任务组（`mode=single_ne`）。 +- **验收测试**：同设备项目可以创建；端口 A 迁移到 B（A 变为 down、B 变为 up）判为 `migrated`。 + +#### H8 去掉 metric 名称启发式（M-12） +- `sheet_overrides` 增加 `scope_by_port_map: true|false`；不填时按当前规则推导，并把推导结果写进 override 的 `effective_*` 字段供界面展示。`_map_defines_iface_expect` 不再读取 metric 名称。 +- **验收测试**：一个名称不含 bgp 的 sheet 设置 `scope_by_port_map=false` 后，不会被映射推导出范围。 + +#### H9 其他（M-14、M-15） +- GET 接口不写库：`project_to_dict` 中的默认模板绑定改为只读解析，持久化只在 create/patch 时进行。 +- 红单：在后续验收中已转绿的键标记为 `resolvable=true`（不自动关闭），前端提供“一键关闭可关闭红单”。 + +--- + +### 阶段 E：跨厂商解析（需要你提供真实样本） + +#### P1 华为 VRP 首批状态解析（T-05） +- **前提**：每个命令提供至少 2 份真实输出样本，放到 `tests/fixtures/huawei/`，并做脱敏（不同版本、带子接口和 VRF 的各一份）。**没有样本时 Cursor 不得凭空编写 TextFSM。** +- 顺序：`interface_brief`（`display interface brief`）、`arp`（`display arp all`）、`bgp_peer`（`display bgp peer` / `display bgp vpnv4 all peer`）、`isis_adjacency`（`display isis peer`）、`ospf_neighbor`（`display ospf peer brief`）。 +- 每个 metric 输出**与中兴相同的字段名**（FieldDef 共用），状态值输出**规范枚举**： + - 接口 `admin/phy/prot ∈ {up, down}`；华为的 `*down` 输出为 `admin=down` + - BGP `state ∈ {established, idle, active, connect, opensent, openconfirm}` + - ARP `entry_type ∈ {dynamic, static, interface}` + + 这样对比层的过滤和预设不需要按厂商分叉。 +- 按 `skills/biz-state/netx-biz-state/SKILL.md` 第 2 节逐项完成：TextFSM、`prefer_fsm`、注册、profile（附 `slice`）、单测。 +- 新增“规范值契约”文档：`docs/biz-state-canonical-fields.md`，按 metric 列出字段、取值范围，以及各厂商原始值到规范值的对照。中兴 parser 也要对照这份契约补齐规范化（例如 `entry_type`）。 +- ARP 过滤：内置模板从 `age_timer` 改为 `entry_type eq dynamic`（依赖各厂商 parser 输出规范的 `entry_type`）；中兴 parser 在 `entry_type` 为空时，根据 age 推断出 `dynamic`。 +- **验收**：fixture 单测；中兴→华为的跨厂商对比用例（固定行数据）在配置映射和值映射后通过率为 100%。 + +--- + +### 阶段 F：前端体验（和上面各任务一起做） + +1. 映射编辑器升级为表格模式（保留文本模式）：两列可编辑，支持粘贴 CSV/Excel，逐行显示校验状态（重复、无效、归一化后的值）。“从基线生成”：左列 before 端口、右列 after 端口，可按“相同槽位/端口号”自动推荐。 +2. 预检结果表（B5）：疑似漏配、未使用的键、N:1、命令覆盖差集，每项都可以展开查看明细。 +3. sheet 卡片显示 `no_data` / `suspect_parse` / `not_collected` / `manual_pending` 状态和原因（V1、H3）。 +4. 结果表的键列显示原始端口名（V5）。 +5. 模板编辑器：接口归一化规则增加 side/vendor 选择；字段规则增加值映射和方向性比较；过滤条件可选 side；行过滤处提示“按归一化后的值书写”。 +6. 割接看板：重复键警告、未覆盖切片、预设语义说明（端口成功条件）。 + +--- + +## 4. 建议执行顺序 + +| 顺序 | 任务 | 说明 | +|---|---|---| +| 1 | A1–A7 | 正确性快修，相互独立，可以并行 | +| 2 | B1 → B2 → B3 → B4 | 流水线是 B2–B4 的基础，必须先做 | +| 3 | B5、B6、B7 | 预检依赖流水线 | +| 4 | V1 → V2 → V3 → V4 → V5 | V2 依赖 V1 的状态定义 | +| 5 | H1 → H2 → H3 → H4 → H5 → H6 → H7 → H8 → H9 | H1/H2 优先（生产风险最大） | +| 6 | 阶段 E | 等你提供样本 | +| 7 | 阶段 F | 随各任务交付；最后做一次整体走查 | + +--- + +## 5. 验收方式（Claude 验收时执行） + +### 5.1 回归命令 +```powershell +# 项目根目录：割接对比相关后端回归（基线 279 项通过） +.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare.py tests/test_biz_state_compare_sql.py tests/test_biz_state_compare_search.py tests/test_biz_state_compare_snapshot.py tests/test_biz_state_compare_export.py tests/test_biz_state_compare_recovery.py tests/test_biz_compare_templates.py tests/test_biz_migration_evaluate.py tests/test_biz_migration_evidence.py tests/test_biz_migration_hf_decouple.py tests/test_biz_migration_integrity.py tests/test_iface_normalize.py tests/test_biz_state_monitor_service.py tests/test_biz_state_paired_sampling.py tests/test_biz_state_retention.py -q +# 全量 +.venv\Scripts\python.exe -m pytest -q + +# web 目录 +node node_modules/typescript/bin/tsc -b +node node_modules/vite/bin/vite.js build +node --experimental-strip-types --test tests/cutoverDataCache.test.mjs tests/compareTemplateModel.test.mjs +# 浏览器（先启动 vite :5179） +node tests/bizCompare.browser.mjs +node tests/bizCompareTemplates.browser.mjs +node tests/bizMigration.browser.mjs +node tests/bizState.browser.mjs +``` + +### 5.2 必须新增的测试文件（便于验收定位） +- `tests/test_identity_pipeline.py`：B1 的 7 路径对照、B2、B3、B4 +- `tests/test_compare_preflight.py`：B5、B7、V2 +- `tests/test_compare_quality.py`：V1、V3、V4 +- `tests/test_biz_migration_presets.py`：A4 的 4 个端口场景、H5、H8 +- `tests/test_biz_migration_retention.py`：H1 +- `tests/test_biz_migration_async.py`：H2、H3 +- `tests/test_cutover_references.py`：A2、A5 +- 浏览器：`bizCompare.browser.mjs` 增加 A3、B5、V5 场景；`bizMigration.browser.mjs` 增加 H2、V1 卡片状态场景 + +### 5.3 我会逐条核对的场景 +1. 原始名称映射（A1）：`GE0/1/0/1→gei0/2/0/1` 加规则 `GE→gei`，结果为 unchanged。 +2. 拔纤但未 shutdown（A4）：验收判 `migrated`。 +3. 删除被引用的映射或模板（A2）：返回 409。 +4. 向导中映射未保存（A3）：不会被静默丢弃。 +5. 跨厂商过滤（B4）：华为侧不会被 `contains gei` 整侧滤掉。 +6. MAC 作为键（B3）：不同格式能配对。 +7. 空对空（V1）：显示 `no_data`，不是 100%。 +8. 命令集不一致（V2）：显示 `not_collected`，不是 `lost`。 +9. 同键乱序（V4）：不再出现假“不一致”。 +10. 有映射时 BGP 路由走 SQL（B6）。 +11. auto run 被清理、被引用批次的保护随之释放（H1）。 +12. 验收异步执行、不阻塞请求（H2）；默认模板不因路由未采集而验收失败（H3）。 +13. 历史 run（snapshot v1）的检索、回填、导出结果与改动前一致。 + +--- + +## 6. 需要你确认的决策（Cursor 按“默认”执行，除非你另行指定） + +| # | 决策 | 默认 | +|---|---|---| +| D1 | 业务对比是否允许 N:1 映射 | 允许，但在预检和结果中明确列出；割接监控继续拒绝并说明冲突行 | +| D2 | 1:N（端口拆分 breakout）本期是否支持 | 不支持，明确报错 | +| D3 | 空对空 / 疑似解析失败时，任务级“是否通过” | `null`（未判定），不计入通过率，不当作失败 | +| D4 | 路由表是否参与割接验收门槛 | 未采集时不参与（`manual_pending`），采集后参与 | +| D5 | HF 采集是否按“同间隔 + 同 lane”合并任务 | 合并；每台设备并发 SSH 上限默认 2 | +| D6 | 是否支持同设备割接 | 支持（H7） | +| D7 | auto run 保留数量 | 割接每批 300、业务对比每个任务 50；验收和手动 run 永久保留 | +| D8 | 流量计数器默认是否参与比较 | 不参与；需要时用 `numeric_min` | +| D9 | 华为样本由谁提供、首批覆盖哪些命令 | 由你提供；首批为接口、ARP、BGP、ISIS、OSPF | diff --git a/docs/reviews/topology-optimization-plan-2026-10-12.md b/docs/reviews/topology-optimization-plan-2026-10-12.md new file mode 100644 index 0000000..4afd699 --- /dev/null +++ b/docs/reviews/topology-optimization-plan-2026-10-12.md @@ -0,0 +1,295 @@ +# 拓扑管理问题分析与优化计划 + +日期：2026-10-12。范围：拓扑管理画布（`web/src/features/topology/*`）及其视图接口（`netx_api/topology_views_graph.py`、`topology_router.py`、`topology_fabric_links.py`）。不含：UME World 平铺大图 LOD、LLDP 采集策略、分类规则、MCP 布局算法。 +用途：交给 Cursor 按任务逐项实施；Claude 负责验收。 + +本文结论来自代码阅读和本地脚本验证，没有在真实 PostgreSQL 或真实设备上执行。标了 **[已验证]** 的问题已经用脚本复现。 + +**基线（修改前）：** +- `pytest tests/test_topology.py tests/test_ume_topology_apply.py`：**69 项通过，4 项失败**（见 T13，修改前就失败，与本计划无关） +- `web`：`npx tsc -b` 通过 +- 拓扑前端目前没有任何单元测试 + +**实施进度（2026-10-12）：** +| 任务 | 状态 | +|---|---| +| T11 基线失败测试 | ✅ 完成 | +| T1 保存保留 lock | ✅ 完成 | +| T2 不固化设备名为 label | ✅ 完成 | +| T6 原子 commit + useTopologySave | ✅ 完成 | +| T3 脏状态添加确认 | ✅ 完成 | +| T4 手工边端口持久化 | ✅ 完成 | +| T5 边样式走保存模型 | ✅ 完成 | +| T7 N+1 / 截断确定性 | ✅ 完成 | +| T8 positions 跳过缺失节点 | ✅ 完成（MCP 不依赖 positions 顺便加点） | +| T9 点击不标脏 | ✅ 完成 | +| T10 拆分 TopologyPage | ✅ 完成（`TopologyPage.tsx` ≤1200；抽出 palette/edge/keyboard/find/export/nodeOps/moduleLinks + `TopologyPageView`） | + +回归：`pytest` 87 passed；`npx tsc -b` 通过；`topologySaveDiff.test.mjs` 4 passed。 + +--- + +## 0. 结论摘要 + +整体架构没有大问题（服务端图 + 本地编辑 + 一次保存）。主要问题集中在**“本地编辑状态”和“服务端状态”的同步边界**，有几处会静默丢数据或误删数据： + +1. **UI 保存会清掉 MCP 设置的节点锁定，并把设备名固化成标签。** **[已验证]** 保存后 `locked` 从 True 变为 False，节点被移走；之后设备改名，画布上仍显示旧名称。 +2. **有未保存修改时“添加网元/投影邻居”会静默丢弃这些修改，并留下隐形删除。** 被删除的连线会重新出现在画布上，但仍留在待删除队列里，下次保存时会被真正删掉。 +3. **修改已有连线的端口不会被保存。** 界面上可以改，也会显示新值，但不会标记为未保存，保存时也不会提交，刷新后就恢复原值。 +4. **连线样式绕过了“编辑→保存”模型。** 样式修改会立即写库，但撤销只回退本地。撤销之后，画布显示和数据库就不一致了。 +5. **保存不是原子的。** 一次保存会依次调用 N 次建边、1 次删边、1 次加点、1 次删点和 1 次写坐标，其中 3 次会返回完整的图数据。中途失败会留下“部分已保存”的状态。另外，同一套保存逻辑在 `purgePlaceholderById` 里重复实现了一遍。 +6. **后端有 N+1 查询和不确定的截断。** 写坐标时每个点各查一次；外部邻居查询的 `limit(5000)` 把视图内部的边也算进去了，边密集的视图可能找不到外部邻居；节点超过上限时截断没有排序。 +7. **`TopologyPage.tsx` 有 2509 行。** 保存、面板添加、连线编辑、快捷键等逻辑都堆在这一个组件里，所以上面这些问题很难被发现，也很难测试。 + +--- + +## 1. 现状：画布编辑模型（给 Cursor 的背景） + +``` +GET /views/{id} ──> graphQuery.data ──(graphToFlow)──> nodes / edges (本地 React state) + │ 拖动/改名/连线/删除 + ▼ + dirtyRef + historyRef(撤销栈, UNDO_MAX=40) + pendingEdgeCreatesRef / pendingEdgeDeletesRef + │ Ctrl+S / 保存 + ▼ + saveMut (TopologyPage.tsx:516) + 1. 每条待建边 POST /fabric/edges (逐条串行) + 2. POST /fabric/edges/delete (待删边) + 3. POST /views/{id}/nodes (服务端没有的本地点；返回整图) + 4. POST /views/{id}/nodes/remove (本地没有的服务端点；返回整图) + 5. PATCH /views/{id}/positions (全部点的 x/y/label；返回整图) +``` + +旁路（不走保存，直接写库）： +- 添加网元/拖入面板 `addPaletteItems` (`TopologyPage.tsx:621`)、投影外部邻居 `addOutsidePeers` (`:1047`)：写库后**用服务端图整体覆盖本地并 `clearDirty()`** +- 连线样式 `patchSelectedEdgeStyle` (`:765`)：立即 `PATCH /edge-style` +- 删除占位网元 `purgePlaceholderById` (`:1238`)：先把保存逻辑复制执行一遍，再清理 + +脏状态下的轮询 (`useTopologyCanvas.ts:306-318`)：只合并在线状态、层级等叠加字段，不动几何信息，这部分是对的。 + +--- + +## 2. 问题清单与任务 + +优先级：**P0** = 静默丢数据/误删，**P1** = 可靠性/性能，**P2** = 体验/可维护性。 + +### T1 [P0] UI 保存会清除节点锁定 **[已验证]** + +- `flowToPositions` (`web/src/features/topology/lib/graphFlow.ts:90`) 不发送 `locked` 字段。 +- `ViewNodeIn.locked` 的默认值是 `False` (`netx_api/topology_schemas.py:277`)。 +- `patch_view_positions` 只在“行已锁定且请求也带 locked=True”时才跳过 (`topology_views_graph.py:738`)，否则会覆盖坐标并写回 `locked=False`。 +- MCP 会用 `locked=true` 固定布局（`packages/netx-topology-mcp/.../http_tools.py:1985`）。用户在 UI 里随便拖一下再保存，这个锁就没了。 + +复现：MCP 把节点锁定在 (500,500)，UI 保存 (10,10)，结果节点变成 `locked=False x=10`。 + +**修改：** +1. 后端：把 `ViewNodeIn.locked` 改为 `bool | None = None`，`None` 表示保持原值。对已锁定的行，只有请求里明确带 `locked=False` 才能移动（解锁）；否则跳过并计入 `skipped_locked`。MCP 显式传 `locked` 的行为保持不变。 +2. 前端：在 `NeNodeData` 里加 `locked`（由 `graphToFlow` 从 `n.locked` 填入）。锁定的节点设 `draggable: false`，并显示一个锁形角标；`flowToPositions` 不再发送 `locked`。 +3. 保存结果里如果 `skipped_locked > 0`，用 toast 提示。 + +**验收：** +- 新增 `tests/test_topology.py::test_positions_preserve_lock_when_omitted`：锁定后用不带 locked 的 positions 请求，坐标不变，`locked` 仍为 True。 +- 新增 `test_positions_explicit_unlock_moves`：`locked=False` 可以移动并解锁。 +- 现有 MCP 相关测试全部通过。 + +### T2 [P0] 保存会把设备名固化成节点标签 **[已验证]** + +- 接口返回的 `label` 已经做过回退：`vn.label` 为空时用 `fn.name / ip` (`topology_views_graph.py:195-197`)。 +- 前端把它放进 `data.label`，保存时 `flowToPositions` 又原样写回 `vn.label` (`graphFlow.ts:100`)。 +- 所以保存过一次之后，所有节点的 `vn.label` 都是当时的设备名。以后设备改名（例如 UME/网管同步），画布不会跟着变。 + +复现：保存后把 fabric 节点改名为 `NEW-NAME`，画布仍显示 `OLD-NAME`。 + +**修改：** +1. 后端 `ViewNodeOut` 增加 `label_override: str`（原始 `vn.label`）；`label` 仍是回退后的显示值，保持兼容。`ViewNodeIn.label` 改为 `str | None = None`，`None` 表示不修改。 +2. 前端 `NeNodeData` 增加 `label_override`，只有用户“重命名节点”（`renameSelectedNode`，`TopologyPage.tsx:820`）时才写入。`flowToPositions` 只对 `label_override` 有变化的节点发送 `label`；重命名为空字符串表示清除覆盖，恢复跟随设备名。 +3. 数据修复脚本 `scripts/topology_clear_frozen_labels.py`，默认 dry-run：`vn.label` 等于当前 `fn.name` 或 `fn.ip` 的行，把 label 置空。输出受影响的行数，加 `--apply` 才真正写库。 + +**验收：** +- 新增 `test_positions_without_label_keeps_following_name`：保存不带 label，改名后画布 label 是新名称。 +- 新增 `test_label_override_survives_rename`：显式覆盖后改名，仍显示覆盖值。 +- 脚本 dry-run 在测试库上输出正确的计数，`--apply` 之后再跑一次计数为 0。 + +### T3 [P0] 有未保存修改时“添加网元 / 投影邻居”会丢弃修改并留下隐形删除 + +- `addPaletteItems` (`TopologyPage.tsx:681-686`) 和 `addOutsidePeers` (`:1096-1101`) 都是直接写库，然后 `applyViewGraph(refreshed)` 再 `clearDirty()`： + - 本地未保存的拖动、改名、删点**直接丢失**，也没有提示； + - `pendingEdgeDeletesRef` **没有清空**。被删除的边按服务端数据重新画回画布，但它的 id 还在待删队列里。用户以后再改任何东西并保存时，这条**看得见的边会被删掉**； + - 撤销栈也没有清空，撤销会回到和服务端不一致的快照。 +- 同类问题：`useTopologyCanvas.ts:320-355` 在非脏状态下整体刷新时，也不会清理撤销栈。 + +**修改（推荐“先保存再添加”，简单而且语义清楚）：** +1. 抽出 `commitCanvas()`（见 T6）。`addPaletteItems` / `addOutsidePeers` 开始时如果 `dirtyRef.current` 为 true，就弹窗“当前有未保存修改，保存后继续添加？”：确认则先 `commitCanvas()` 再添加，取消则中止。 +2. 新增一个统一的辅助函数 `resetLocalFromServer(graph)`，用于所有“用服务端图整体覆盖本地”的地方：同时清空两个 pending 集合、撤销栈和重做栈，然后 `clearDirty()`。目前 `purgePlaceholderById` (`:1289-1297`) 是手写的这套逻辑，其他地方都漏了。 + +**验收（手工，Cursor 需要在 PR 描述里写明步骤和结果）：** +1. 删除一条已有边 → 不保存 → 从面板拖入一个网元 → 应弹出确认框；选择保存后，该边被删除，新网元出现。 +2. 同上，但选择取消 → 不添加网元，删除操作仍处于未保存状态。 +3. 删除边 → 投影邻居（确认保存）→ 刷新页面，边确实被删除，没有“复活后又消失”。 +4. 用 grep 检查：所有 `applyViewGraph(` 调用点要么使用 `localPositions` 合并，要么经过 `resetLocalFromServer`。 + +### T4 [P0] 修改已有连线的端口不会被保存 + +- 右键菜单对任意单条边都显示端口输入框 (`TopologyCtxMenu.tsx:400-429`)。 +- `patchSelectedEdgeStyle` 修改本地端口后，只把样式字段发给 `PATCH /edge-style`（`ViewEdgeStylePatch` 本来就没有端口字段，`topology_schemas.py:450`），也没有 `markDirty()`。 +- 保存只处理 `pendingEdgeCreatesRef` 里的新边，已有边的端口修改永远不会提交。 + +**修改：** +1. 只有 `source === "manual"` 的边可以编辑端口；LLDP/UME 边的端口改为只读（这两类以采集结果为准，手工修改会在下次发现时被覆盖）。 +2. 未保存的新手工边：端口修改继续只改本地，保存时随建边一起提交（现有逻辑已支持）。 +3. 已保存的手工边：新增 `PATCH /v1/topology/fabric/edges/{id}`，body `{a_port, b_port}`，只允许 `source=manual`，否则 400 `edge_not_manual`；端口要做归一化，与已有边冲突（违反唯一约束）时返回 409。前端把修改记入 `pendingEdgePortEditsRef: Map` 并 `markDirty()`，在 `commitCanvas()` 里提交。 + +**验收：** +- 新增 `test_patch_manual_edge_ports`、`test_patch_non_manual_edge_ports_rejected`、`test_patch_manual_edge_ports_conflict_409`。 +- 手工测试：修改已保存手工边的端口 → 出现未保存标记 → 保存 → 刷新后仍是新端口；LLDP 边的端口输入框为只读。 + +### T5 [P1] 连线样式与撤销/保存模型不一致 + +- `patchSelectedEdgeStyle` (`TopologyPage.tsx:807-815`) 每次修改都立即发请求（颜色选择器拖动时会连续发很多次）。同时调用了 `pushHistory()`，但撤销只回退本地状态，所以撤销后画布和数据库不一致。 + +**修改：** 样式也改为走保存：记入 `pendingEdgeStyleRef: Map`，`markDirty()`，在 `commitCanvas()` 里批量提交（后端 `PATCH /edge-style` 支持 `items: []` 批量，旧的单条 body 保留兼容）。撤销快照要包含这个 Map。 + +**验收：** 改颜色 → 撤销 → 保存 → 刷新，颜色是修改前的值；改颜色 → 保存 → 刷新，颜色是修改后的值；拖动颜色选择器期间，网络面板里没有 `/edge-style` 请求。 + +### T6 [P1] 保存原子化，并去掉重复的保存逻辑 + +现状见第 1 节。中途失败时，前面已执行的步骤（例如已建的边、已加的点）不会回滚，`pending*` 集合也只清了一部分。建边是幂等 upsert（`topology_fabric_links.py:65`），重试不会建出重复边，但删点和写坐标的顺序问题依然存在。 + +**修改：** +1. 后端新增 `POST /v1/topology/views/{id}/commit`，在一个事务里依次处理： + ```json + { + "edge_creates": [{"client_id":"local-..","a_node_id":"","b_node_id":"","a_port":"","b_port":""}], + "edge_deletes": ["edge-id"], + "edge_port_edits": [{"id":"","a_port":"","b_port":""}], + "edge_styles": [{"fabric_edge_id":"","stroke_color":"","stroke_width":0,"line_style":""}], + "node_adds": ["fabric-id"], + "node_removes": ["fabric-id"], + "positions": [{"fabric_node_id":"","x":0,"y":0,"label":null}] + } + ``` + 任何一步失败都整体回滚，返回 4xx 和 `{"failed_step": "...", "detail": "..."}`。成功时只返回**一次**完整图数据，外加 `edge_id_map: {client_id: server_id}`。节点上限和冻结规则与 `add_nodes_to_view` 一致。 +2. 前端新增 `hooks/useTopologySave.ts`，导出 `commitCanvas()`：从本地状态计算差异（纯函数 `lib/saveDiff.ts`，可以单独测试），调用 `/commit`，成功后调用 `resetLocalFromServer`。`saveMut`、`purgePlaceholderById` (`:1248-1281`)、T3 的“先保存”都改用它。 +3. 旧的分步接口保留（MCP 在用）。 + +**验收：** +- 新增 `test_view_commit_atomic_rollback`：构造一个 `edge_creates` 自环（第一步就失败），断言 `node_adds` 和 `positions` 都没有落库。 +- 新增 `test_view_commit_happy_path`：一次提交包含建边、删边、加点、删点、写坐标，结果图正确，`edge_id_map` 正确。 +- `web/tests/topologySaveDiff.test.mjs`：覆盖“本地新增/删除节点”“region 节点不进入增删列表”“只发送有变化的 label”“删除后又撤销的边不出现在 edge_deletes 中”等情况。 +- 手工测试：一次保存只产生 **1 个** 写请求（网络面板截图附在 PR 描述里）。 + +### T7 [P1] 后端视图接口的 N+1 查询与不确定截断 + +| 位置 | 问题 | 修改 | +|---|---|---| +| `topology_views_graph.py:719` | 写坐标时每个点各执行一次 `db.get(TopoFabricNode)`，最多 2000 次 | 先用一次 `IN` 查询拿到存在的 id 集合 | +| `:160-166` | 节点超过 `VIEW_GRAPH_NODE_HARD_CAP` 时截断没有 `ORDER BY`，每次显示哪些节点不确定 | `order_by(TopoViewNode.created_at, TopoViewNode.id)` | +| `:354-363` | 外部邻居的 `limit(5000)` 把**两端都在视图内**的边也算进去了，边密集的视图会漏掉外部邻居 | SQL 里排除两端都在视图内的边（`~and_(a.in_, b.in_)`），再加 `limit` | +| `:375` | 每个外部邻居执行一次 `db.get` | 收集 id 后用一次 `IN` 查询 | +| `:178-179` | region 节点每个执行一次 `db.get(TopoFolder)` | 一次 `IN` 查询 | + +**验收：** +- 新增 `test_outside_peers_found_when_many_internal_edges`：视图内有 >5000 条内部边（测试中可以把上限改成参数，例如用 10 条模拟），外部邻居仍然能返回。 +- 新增 `test_view_truncation_is_deterministic`：两次 GET 返回同一批节点。 +- 用 SQLAlchemy `before_cursor_execute` 统计 2000 个点的 positions 请求，语句数 ≤ 10（写成测试）。 + +### T8 [P1] positions 接口可以绕过节点上限加点 + +`patch_view_positions` 遇到视图中没有的 `fabric_node_id` 会直接插入 `TopoViewNode`（`:720-733`），不检查 `max_nodes` 和冻结状态，而 `add_nodes_to_view` 会检查（`:778`）。 + +**修改：** 先确认 MCP 是否依赖“positions 顺便加点”（用 grep 查 `packages/netx-topology-mcp` 里的 `positions` 调用）。如果不依赖，就改为跳过不存在的节点，并在返回里计入 `skipped_missing`；如果依赖，就加上与 `add_nodes_to_view` 相同的上限和冻结检查。**这一项的结论需要写进 PR 描述。** + +**验收：** 新增 `test_positions_respects_max_nodes`。 + +### T9 [P2] 点击节点就会产生“未保存”和无效撤销记录 + +`onNodeDragStart` 会无条件 `pushHistory()`；`onNodesChange` 收到任何 `position` 变更都会 `markDirty()`，包括没有实际移动的点击 (`TopologyReactFlowView.tsx:442-449`)。后果：单击一个节点就出现“未保存”，并且触发脏锁，live sync 不再整体刷新。 + +**修改：** 在 `onNodeDragStart` 记录起点坐标；`onNodeDragStop` 时只有位移超过 1px 才把起始快照压入撤销栈并 `markDirty()`。`onNodesChange` 不再对 `position` 变更调用 `markDirty`（`remove`/`add` 保持不变）。 + +**验收：** 单击节点后不出现未保存标记，撤销按钮保持禁用；拖动 10px 后出现未保存标记，并且可以撤销。 + +### T10 [P2] 拆分 `TopologyPage.tsx`（在 T1–T6 完成后进行） + +目标是把主组件压到 ≤ 1200 行，**只搬代码、不改行为**： +- `hooks/useTopologySave.ts`（T6） +- `hooks/useTopologyPalette.ts`：`addPaletteItems`、`addOutsidePeers`、拖放、`neQuery`、`umeQuery` +- `hooks/useTopologyEdgeEditing.ts`：`patchSelectedEdgeStyle`、`queueDeleteEdges`、`removeSelected`、`onConnect`、`isValidConnection` +- `hooks/useTopologyKeyboard.ts`：全局快捷键（`:1130-1182`）。依赖改为通过 ref 读取，避免每次 `nodes`/`edges` 变化都重新绑定监听 +- `lib/moduleLinks.ts`：`openNeInventory`、`openWebcrtFor`、`openAlarmsFor`、`openPortTrafficForEdge` + +**验收：** `tsc -b` 通过；第 4 节的手工回归清单全部通过；`git diff --stat` 显示 TopologyPage.tsx 净减少 ≥ 1000 行。 + +### T11 [P2] 修复修改前就存在的失败测试 + +| 测试 | 原因 | 处理 | +|---|---|---| +| `test_ume_topology_apply.py::test_lldp_upgrades_bare_ume_ports_and_keeps_ume_primary` | **测试本身不稳定**：`rich_a/rich_b` 取自 `edge.a_port/b_port`（按节点 id 排序后的 A/B），但调用 upsert 时传的是 `a.id/b.id`（按 UME 名称取的节点）。uuid 排序不同时端口就配反了，结果约有一半概率失败。已用调试脚本确认这不是产品 bug。 | 改为传 `a_node_id=edge.a_node_id, b_node_id=edge.b_node_id`（`:358-365`）。`:297`、`:393` 有同样的写法，一起检查 | +| `test_topology.py::test_lldp_discover_writes_fabric_edge`、`test_lldp_unmatched_creates_ssh_placeholder_ne` | 发现任务一直是 `pending`，没有变成 `done`（`:1285`、`:1458`） | 排查后台 worker 在 SQLite 内存库下是否被启动或被跳过，修好测试的启动方式，或者改为同步执行 job | +| `test_topology.py::test_fabric_reconcile_scheduler_once_sweeps_orphans` | `0 not >= 1`（`:2287`） | 排查，可能和上一条同源 | + +**验收：** 两个测试文件 **0 失败**，并且连续跑 5 次都通过（用于检验不稳定性）。 + +### 产品决定（2026-10-12 已确认） + +- **Q1**：`isValidConnection` (`TopologyPage.tsx:568`) 不允许在同一对网元之间画第二条手工连线，但 LLDP/UME 支持并行链路，画布也支持聚合显示。是否应该允许手工建并行链路（端口不同即可）？ +- **Q2**：T3 选择了“先保存再添加”。如果希望“添加后保留本地未保存修改”，实现会复杂很多（需要把服务端新增部分合并进本地），请确认。 + +- **结论**：Q1 不允许手工画并行链路，保持 `isValidConnection`（现位于 `hooks/useTopologyEdgeEditing.ts`）现状；Q2 沿用“先保存再添加”。 + +--- + +## 3. 建议实施顺序 + +1. **T11**（先把基线修到全部通过，后面每一步的回归才有意义） +2. **T1 + T2**（后端 schema 改动，两项一起做，再加数据修复脚本） +3. **T6**（commit 接口 + `useTopologySave`），然后在它之上做 **T3、T4、T5** +4. **T7、T8** +5. **T9**，最后做 **T10**（纯重构，单独一个 PR） + +每一步单独提交；T10 必须单独成 PR。 + +## 4. 回归命令与手工清单 + +```bash +python -m pytest tests/test_topology.py tests/test_ume_topology_apply.py -q +``` + +```bash +cd web && npx tsc -b +``` + +```bash +cd web && node --experimental-strip-types --test tests/topologySaveDiff.test.mjs +``` + +> `lib/saveDiff.ts` 只放纯函数，除 `import type` 外不要有运行时 import（Node strip-types 不会解析不带扩展名的相对路径）。写法参考现有的 `web/tests/cutoverDataCache.test.mjs`。 + +手工回归（每个 PR 都要过一遍，结果写进 PR 描述）： +1. 拖动节点 → 保存 → 刷新，位置保持。 +2. 新建手工连线（带端口）→ 保存 → 刷新，连线和端口都在。 +3. 删除连线 → 撤销 → 保存，连线仍然存在。 +4. 删除节点 → 保存，节点从视图中移除，但 fabric 中仍存在（可以通过搜索找到）。 +5. 重命名节点 → 保存 → 刷新，显示新名称；再把名称清空 → 保存，恢复为设备名。 +6. 打开 live sync，另一个窗口用 MCP 移动节点，3 秒内本窗口同步；本窗口有未保存修改时不会被覆盖，只刷新在线状态。 +7. 只读用户（没有 `ne:write`）：不能拖动、连线、保存，右键菜单里没有写操作。 +8. 删除占位网元（有未保存修改时），修改先被保存，占位网元被清理。 +9. 导出 SVG/XML 正常。 + +--- + +## 5. 验收记录（2026-10-12） + +Cursor 完成 T1–T11 后由 Claude 验收并接手修复，结论：主体实现正确，另修 6 个问题（均补了回归测试）： + +1. `/commit` 同次 `node_adds` 的节点坐标被忽略（`autoflush=False` 下重查看不到新行）→ 先 `flush`；`_apply_positions` 的逐点 `db.get` 改为批量。 +2. T8 收紧后，无放置行的旧子区域图标拖动无法保存 → `add_missing_region_placement`（仅限本画布的手工子区域）。 +3. MCP `updateTopologyViewPositions` 总是发送 `label:""`、`locked:false`，会清掉 UI 重命名并解锁 → 未提供则不发送。 +4. 已改样式/端口的边被删除后保存 404 → 前端 `saveDiff` 与后端都跳过已删边；新建本地边的样式经 `edge_id_map` 映射。 +5. 框选拖动、方向键移动不标记未保存 → 补 `onSelectionDrag*` 和键盘移动判定。 +6. （改动前已存在）大视图打开空白、远处节点定位失效：xyflow `fitView` 等待节点测量，而 `onlyRenderVisibleElements` 下视口外节点永不测量 → `lib/fitNodes.ts` 按已知几何计算 `fitBounds`。 + +开发库数据修复：`topology_clear_frozen_labels.py --apply` 清理 516 条固化标签（执行前已备份原值）。 + +回归：拓扑后端 91 通过；Web Node 测试 30 通过；`tsc -b` 通过；MCP 包 238 通过 / 5 失败（与改动前同一批，既有问题）。 diff --git a/docs/reviews/workbench-custom-modules-plan-2026-10-13.md b/docs/reviews/workbench-custom-modules-plan-2026-10-13.md new file mode 100644 index 0000000..82f52f7 --- /dev/null +++ b/docs/reviews/workbench-custom-modules-plan-2026-10-13.md @@ -0,0 +1,112 @@ +# 工作台功能卡片自定义 — 实现方案（2026-10-13） + +## 目标 + +工作台功能区（UME 同步、网元管理、网络管理、拓扑、WebCRT…）以后会越来越多，每个人关心的不一样。 +让每个用户自己选择：**显示哪些卡片、按什么顺序**；设置跟着账号走（换电脑/换浏览器仍生效）。 + +## 现状 + +- 卡片注册表：`web/src/config/modules.ts:33` `MODULES`，按 `section` 分 `monitoring / operations / system`。 +- 过滤逻辑：`web/src/features/workbench/WorkbenchPage.tsx:323` `visibleBySection` —— 只按权限（`adminOnly` / `requiredScope`）和 `workbenchHidden` 过滤。 +- 渲染：`web/src/features/workbench/components/WorkbenchModules.tsx:23`，operations 是大磁贴网格，system 是底部一排小链接。 +- 后端无任何“用户偏好”存储；`AppUser`（`netx_api/models/auth.py:13`）只有账号字段。 + +## 设计决定 + +| 项 | 决定 | 理由 | +|---|---|---| +| 可自定义范围 | 只做 **operations 区的磁贴**（含 monitoring 合并进来的 UME 同步）。system 区链接行保持固定 | 系统管理入口少且是管理类，隐藏后反而找不到；需求说的是“功能卡片” | +| 存储位置 | **服务端，按用户**：`app_user.prefs` JSON 列 | “每个人不一样”= 按账号；localStorage 只按浏览器，换机就丢 | +| 本地缓存 | 同时写一份 localStorage（key 带 user id），首屏先用缓存渲染，避免卡片闪烁 | 首屏体验 | +| 数据结构 | `{ order: string[], hidden: string[] }`，存的是“隐藏了哪些”而不是“显示哪些” | 以后新增模块默认对所有人出现，不需要每人手动开启 | +| 新模块默认隐藏 | `ModuleDefinition` 增加可选 `defaultHidden?: boolean`；不在 `order` 里的这类模块视为隐藏 | 给以后“小众功能”留开关 | +| 权限 | 没权限的模块永远不出现在卡片区，也不出现在“可添加”列表；偏好里残留的 id 静默忽略 | 偏好不能绕过权限 | +| 交互 | 卡片区右上角一个小齿轮 → 进入**原地编辑模式**：已显示卡片可拖拽排序、右上角 × 移除；下方出现“可添加”托盘（灰显卡片，点击加入）；按钮只有「恢复默认」「完成」 | 和 iOS 主屏编辑一致，最少文案；不弹窗 | +| 拖拽实现 | HTML5 原生 drag & drop，不引入新依赖 | 卡片数量小（<20），原生够用；项目里没有 dnd 库 | +| 保存时机 | 点「完成」时一次 PUT；失败则 toast 并保留编辑状态 | 避免拖动过程中频繁请求 | +| 全部隐藏 | 允许，但功能区显示一个空态（只有齿轮 + 一句“添加功能”） | 不阻止用户，但要能找回 | + +## 任务 + +### 后端 + +**B1 — `app_user.prefs` 列** +- `netx_api/models/auth.py:13` `AppUser` 加 `prefs: Mapped[dict] = mapped_column(_JsonType, default=dict)`。 +- 新迁移 `alembic/versions/20261013_user_prefs.py`，`down_revision = "20260922_biz_state_queued"`，`add_column("app_user", sa.Column("prefs", JSON, nullable=False, server_default="{}"))`。 + +**B2 — 读写接口**（`netx_api/auth_router.py`，紧跟 `api_me` 之后，约 :273） +- `GET /v1/auth/me/prefs` → `{"prefs": {...}}` +- `PUT /v1/auth/me/prefs/{key}`，body 为该 key 的值；`key` 白名单，目前只允许 `workbench`。 +- 校验 `workbench`：`{order: list[str], hidden: list[str]}`，每个列表 ≤ 64 项、每个 id ≤ 64 字符、只允许 `[a-z0-9-]`；其他字段丢弃。超限 422。 +- 只改自己的偏好（`require_user`），不需要额外 scope；不写审计（纯 UI 偏好，避免刷屏）。 +- `/v1/auth/me` 响应里顺带返回 `prefs`，前端登录后一次拿到，不多一次请求。 + +### 前端 + +**F1 — 注册表扩展** `web/src/config/modules.ts` +- `ModuleDefinition` 加 `defaultHidden?: boolean`。 +- 新增 `export function customizableModules()`：operations + monitoring 中非 `workbenchHidden` 的模块（即当前 `displaySections` 里 operations 那组的来源）。 + +**F2 — 偏好 hook** 新建 `web/src/features/workbench/useWorkbenchLayout.ts` +- 输入：权限过滤后的可用模块列表；输出：`{ shown: ModuleDefinition[], available: ModuleDefinition[], save(order, hidden), reset() }`。 +- 解析规则（纯函数 `resolveLayout(all, prefs)`，单独导出便于测试）： + 1. 先按 `prefs.order` 排已知 id，丢弃不存在/无权限的 id； + 2. 剩余模块按注册表顺序追加到末尾； + 3. 在 `hidden` 中的、或 `defaultHidden` 且不在 `order` 中的，归入 `available`。 +- 数据来源：`AuthContext` 的 `me.prefs.workbench` → 没有则 localStorage 缓存 `netx.wb.layout.` → 默认。 +- 保存：PUT 成功后更新 AuthContext 中的 prefs 和 localStorage。localStorage 读写全部 try/catch。 + +**F3 — AuthContext** `web/src/features/auth/AuthContext.tsx:48` +- `refreshMe` / `login` 把响应里的 `prefs` 存进 state，并暴露 `prefs` 与 `setPrefs`。 + +**F4 — 编辑模式 UI** `web/src/features/workbench/components/WorkbenchModules.tsx` +- `WorkbenchPage.tsx:334` `displaySections` 的 operations 组改为用 `useWorkbenchLayout().shown`。 +- 卡片区右上角齿轮按钮（`ConsoleIcon` 加一个 `settings` 图标），`aria-label` = “自定义功能区”。 +- 编辑态： + - 磁贴轻微抖动或描边（遵循 glass 风格：**不发光**，只用 `--nx-line` 边框）、右上角 × 按钮； + - `draggable`，`onDragStart/onDragOver/onDrop` 重排本地草稿数组； + - 下方“可添加”托盘：灰显磁贴，点击加入末尾；托盘为空时不显示； + - 底部两个按钮：「恢复默认」（清空草稿为默认）、「完成」（保存）；Esc = 放弃草稿退出。 + - 编辑态下点击磁贴不打开模块。 +- 非编辑态 shown 为空：显示空态 + “添加功能”按钮（直接进入编辑态）。 +- 样式写在 `web/src/styles/pages/workbench.css`，颜色只用现有 token（蓝黑 200–215°，无紫色）。 + +**F5 — 文案** `web/src/i18n/locales/{zh,en}/workbench.ts` +- 新增：`customize`（自定义功能区）、`customizeDone`（完成）、`customizeReset`（恢复默认）、`customizeAvailable`（可添加）、`customizeEmpty`（添加功能）、`customizeSaveFailed`（保存失败）。只这些，不加说明文字。 + +## 验收 + +| ID | 场景 | 期望 | +|---|---|---| +| A1 | 新用户首次登录 | 卡片与改动前完全一致（顺序、数量） | +| A2 | 隐藏「拓扑」、把「网元管理」拖到第一位，点完成 | 立即生效；刷新、换浏览器登录同一账号仍是该布局 | +| A3 | 用户 A 定制后，用户 B 登录 | B 不受影响 | +| A4 | 在 `MODULES` 新增一个模块 | 已定制过的用户自动在末尾看到它；若设了 `defaultHidden` 则出现在“可添加”里 | +| A5 | 偏好里包含无权限模块 id（如去掉用户 `ne:read` 后） | 不显示、不出现在可添加；不报错 | +| A6 | 全部移除后完成 | 显示空态，点“添加功能”可恢复 | +| A7 | 编辑中按 Esc / 保存失败（断网） | Esc 恢复原布局；失败提示并停留在编辑态 | +| A8 | `PUT /v1/auth/me/prefs/workbench` 传非法 key、超长列表、非法字符 | 404 / 422 | +| A9 | 视觉 | 编辑态无发光、无紫色，与现有 glass 风格一致 | + +## 回归命令 + +```bash +alembic upgrade head +``` + +```bash +python -m pytest tests/test_auth.py tests/test_user_prefs.py -q +``` + +```bash +cd web && npm run lint && npm run build +``` + +- 新增 `tests/test_user_prefs.py`：覆盖 B2 的读写、白名单、校验、用户隔离（A3、A8）。 +- `resolveLayout` 纯函数若项目有前端单测则补上（A1、A4、A5）；没有则在验收时手测。 + +## 以后可扩展（本次不做） + +- 同一机制用 `prefs.workbench.panels` 控制右侧任务栏、网元/拓扑/调度三张统计卡的显隐。 +- 管理员给角色设默认布局（`role_default_prefs`），用户未定制时用角色默认。 diff --git a/netx_api/app_startup.py b/netx_api/app_startup.py index ea16cd6..23d8a0c 100644 --- a/netx_api/app_startup.py +++ b/netx_api/app_startup.py @@ -15,6 +15,7 @@ from .schema_patches import ( apply_topology_schema_safety_net, run_alembic_upgrade_to_head, ) +from .ne_crypto import ensure_credential_secret_key from .security_bootstrap import assert_secure_defaults_or_exit from .ume_runtime import start_api_sideband_threads, start_device_schedulers import netx_api.ume_support as ume_support @@ -42,6 +43,8 @@ def _configure_ume_diag_logging() -> None: def run_api_startup() -> None: """Full API boot sequence previously inlined in ``main.on_startup``.""" assert_secure_defaults_or_exit() + # Persist Fernet key for managed-NE passwords (same idea as JWT secret file). + ensure_credential_secret_key() _configure_ume_diag_logging() _log.info( "startup: ne_exec_policy_enabled=%s", @@ -156,6 +159,25 @@ def run_api_startup() -> None: ) except Exception: _log.exception("startup: biz_state collect recovery failed") + try: + from .biz_state.compare_service import recover_interrupted_compares_on_startup + + cmp_rec = recover_interrupted_compares_on_startup(db) + if cmp_rec.get("runs"): + _log.info( + "startup: cancelled %s interrupted biz compare run(s)", + cmp_rec.get("runs"), + ) + except Exception: + _log.exception("startup: biz compare recovery failed") + try: + from .biz_migration.service import recover_interrupted_evaluations_on_startup + + n_eval = recover_interrupted_evaluations_on_startup(db) + if n_eval: + _log.info("startup: cancelled %s interrupted cutover evaluation(s)", n_eval) + except Exception: + _log.exception("startup: cutover evaluation recovery failed") try: from .port_traffic_migrate import backfill_port_traffic_series diff --git a/netx_api/auth_router.py b/netx_api/auth_router.py index f6e0d0b..1ff052a 100644 --- a/netx_api/auth_router.py +++ b/netx_api/auth_router.py @@ -2,10 +2,12 @@ from __future__ import annotations +import re from typing import Annotated, Any -from fastapi import APIRouter, Depends, Query, Request, Response +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified from .auth_cookies import clear_auth_cookies, read_refresh_cookie, set_auth_cookies from .auth_deps import AuthContext, require_admin, require_user @@ -30,11 +32,13 @@ from .auth_service import ( list_users, login_issue_token, refresh_login_tokens, + delete_api_token, revoke_api_token, revoke_auth_session_for_user, revoke_auth_sessions, update_api_token, update_user, + user_prefs, user_public, write_audit, ) @@ -45,9 +49,14 @@ from .auth_rate_limit import ( ) from .client_ip import resolve_client_ip from .db import get_db +from .models import AppUser router = APIRouter(tags=["auth"]) +_PREF_KEYS = frozenset({"workbench"}) +_PREF_ID_RE = re.compile(r"^[a-z0-9-]{1,64}$") +_PREF_LIST_MAX = 64 + def _client_meta(request: Request) -> tuple[str, str]: ip = resolve_client_ip(request) @@ -55,6 +64,34 @@ def _client_meta(request: Request) -> tuple[str, str]: return ip, ua +def _validate_id_list(value: Any, *, field: str) -> list[str]: + if not isinstance(value, list): + raise HTTPException(status_code=422, detail=f"invalid_{field}") + if len(value) > _PREF_LIST_MAX: + raise HTTPException(status_code=422, detail=f"{field}_too_long") + out: list[str] = [] + for item in value: + if not isinstance(item, str) or not _PREF_ID_RE.fullmatch(item): + raise HTTPException(status_code=422, detail=f"invalid_{field}_item") + out.append(item) + return out + + +def _validate_workbench_prefs(value: Any) -> dict[str, list[str]]: + if not isinstance(value, dict): + raise HTTPException(status_code=422, detail="invalid_workbench_prefs") + return { + "order": _validate_id_list(value.get("order", []), field="order"), + "hidden": _validate_id_list(value.get("hidden", []), field="hidden"), + } + + +def _validate_pref_value(key: str, value: Any) -> Any: + if key == "workbench": + return _validate_workbench_prefs(value) + raise HTTPException(status_code=404, detail="unknown_pref_key") + + def _token_response(response: Response, request: Request, out: dict[str, Any]) -> dict[str, Any]: access = str(out.get("access_token") or "") refresh = str(out.get("refresh_token") or "") @@ -268,9 +305,39 @@ def api_me(ctx: Annotated[AuthContext, Depends(require_user)]) -> dict[str, Any] "auth_via": ctx.auth_via, "scopes": sorted(ctx.scopes), "all_scopes": sorted(ALL_SCOPES), + "prefs": user_prefs(ctx.user), } +@router.get("/v1/auth/me/prefs") +def api_get_prefs(ctx: Annotated[AuthContext, Depends(require_user)]) -> dict[str, Any]: + return {"prefs": user_prefs(ctx.user)} + + +@router.put("/v1/auth/me/prefs/{key}") +def api_put_pref( + key: str, + body: dict[str, Any], + ctx: Annotated[AuthContext, Depends(require_user)], + db: Session = Depends(get_db), +) -> dict[str, Any]: + if key not in _PREF_KEYS: + raise HTTPException(status_code=404, detail="unknown_pref_key") + validated = _validate_pref_value(key, body) + # Reload in this session: ctx.user may be the middleware's detached copy, or the + # synthetic "system" user when auth is disabled (no row — never insert it). + user = db.get(AppUser, ctx.user.id) + if user is None: + raise HTTPException(status_code=409, detail="prefs_unavailable") + prefs = user_prefs(user) + prefs[key] = validated + user.prefs = prefs + flag_modified(user, "prefs") + db.commit() + db.refresh(user) + return {"prefs": user_prefs(user)} + + @router.post("/v1/auth/change-password") def api_change_password( body: ChangePasswordRequest, @@ -537,3 +604,27 @@ def api_revoke_token( detail={"token_id": row.id, "name": row.name}, ) return {"ok": True, "id": row.id} + + +@router.delete("/v1/api-tokens/{token_id}/purge") +def api_purge_token( + token_id: str, + request: Request, + ctx: Annotated[AuthContext, Depends(require_user)], + db: Session = Depends(get_db), +) -> dict[str, Any]: + row = delete_api_token(db, token_id=token_id, actor=ctx.user) + ip, ua = _client_meta(request) + write_audit( + db, + action="api_tokens.purge", + actor_user_id=ctx.user.id, + actor_username=ctx.user.username, + method="DELETE", + path=f"/v1/api-tokens/{token_id}/purge", + status_code=200, + client_ip=ip, + user_agent=ua, + detail={"token_id": row.id, "name": row.name}, + ) + return {"ok": True, "id": row.id} diff --git a/netx_api/auth_service.py b/netx_api/auth_service.py index 5aa9577..464bd1d 100644 --- a/netx_api/auth_service.py +++ b/netx_api/auth_service.py @@ -76,6 +76,11 @@ def user_public(user: AppUser) -> dict[str, Any]: } +def user_prefs(user: AppUser) -> dict[str, Any]: + raw = getattr(user, "prefs", None) + return dict(raw) if isinstance(raw, dict) else {} + + def sanitize_detail(detail: Any) -> Any: """Recursively drop secret-looking keys from audit detail payloads.""" if isinstance(detail, dict): @@ -431,6 +436,7 @@ def login_issue_token( "expires_in": ttl, "refresh_expires_in": refresh_ttl, "user": user_public(user), + "prefs": user_prefs(user), } @@ -667,6 +673,18 @@ def revoke_api_token(db: Session, *, token_id: str, actor: AppUser) -> ApiToken: return row +def delete_api_token(db: Session, *, token_id: str, actor: AppUser) -> ApiToken: + """Permanently remove an API key row (revoked or active).""" + row = db.query(ApiToken).filter(ApiToken.id == str(token_id)).one_or_none() + if row is None: + raise HTTPException(status_code=404, detail="api_token_not_found") + if actor.role != "admin" and row.user_id != actor.id: + raise HTTPException(status_code=403, detail="forbidden") + db.delete(row) + db.commit() + return row + + def update_api_token( db: Session, *, diff --git a/netx_api/biz_migration/auto_monitor.py b/netx_api/biz_migration/auto_monitor.py new file mode 100644 index 0000000..4a0154c --- /dev/null +++ b/netx_api/biz_migration/auto_monitor.py @@ -0,0 +1,120 @@ +"""Coalesced collect-completion events for live cutover evaluation. + +One worker per process bounds concurrency. Project row locks in run_evaluate +serialize persistence across processes. No device collection runs here. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import threading + +from sqlalchemy.orm import Session + +from ..models import BizMigrationBatch, BizMigrationProject, BizMigrationRun, BizMonitorTemplate, BizStateBatch +from . import service + +_log = logging.getLogger(__name__) +_lock = threading.Lock() +_pending: set[str] = set() +_worker_running = False + + +def try_auto_monitor_for_task(db: Session, task_id: str) -> int: + count = 0 + # Bindings are JSON for multiple sampling intervals; match in Python after + # limiting to projects with an active operation batch. + projects = db.query(BizMigrationProject).filter( + BizMigrationProject.status != "done", + BizMigrationProject.id.in_(db.query(BizMigrationBatch.project_id).filter( + BizMigrationBatch.status == "active")), + ).all() + for candidate in projects: + tids = service._all_hf_task_ids(candidate, "old") + service._all_hf_task_ids(candidate, "new") + if task_id not in tids or not candidate.old_baseline_batch_id or not candidate.new_baseline_batch_id: + continue + # Lock before deduplication so two worker processes cannot save the same + # snapshot. run_evaluate holds the same lock through the transaction. + proj = db.query(BizMigrationProject).filter( + BizMigrationProject.id == candidate.id).with_for_update().populate_existing().first() + batch = db.query(BizMigrationBatch).filter( + BizMigrationBatch.project_id == proj.id, BizMigrationBatch.status == "active" + ).order_by(BizMigrationBatch.started_at.desc()).first() + if not batch or proj.status == "done": + db.rollback() + continue + if db.query(BizMigrationRun.id).filter( + BizMigrationRun.project_id == proj.id, BizMigrationRun.status == "running" + ).first(): + db.rollback() # a manual / acceptance evaluation owns the project right now + continue + mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None + if not mt: + db.rollback() + _log.error("cutover monitor template missing project=%s", proj.id) + continue + sheets, overrides, defaults, norms = service.resolve_evaluate_sheets(db, mt) + samples = [] + for tid in tids: + sample = db.query(BizStateBatch).filter( + BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued")) + ).order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() + samples.append([tid, sample.id if sample else "", sample.status if sample else ""]) + payload = [samples, proj.old_baseline_batch_id, proj.new_baseline_batch_id, + batch.expect_set_json, service._port_map_dict(db, proj.mapping_id), + sheets, overrides, defaults, norms, service.resolve_collect_metric_ids(db, proj)] + fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + last = db.query(BizMigrationRun).filter( + BizMigrationRun.batch_id == batch.id, BizMigrationRun.status == "success").order_by( + BizMigrationRun.created_at.desc()).first() + if last and (last.summary_json or {}).get("auto_fingerprint") == fingerprint: + db.rollback() + continue + previous_samples = {s[0]: (s[1], s[2]) for s in (last.summary_json or {}).get("auto_samples", [])} if last else {} + if last and not previous_samples: + for sample in (last.summary_json or {}).get("metric_batches", {}).values(): + for side in ("old", "new"): + previous_samples[sample.get(f"{side}_task_id", "")] = ( + sample.get(f"{side}_batch_id", ""), sample.get(f"{side}_status", "success")) + changed_tasks = {tid for tid, bid, status in samples if previous_samples.get(tid) != (bid, status)} + changed_metrics = {mid for side in ("old", "new") for binding in service._hf_bindings(proj, side) + if binding["task_id"] in changed_tasks for mid in binding.get("metric_ids") or payload[-1]} + try: + result = service.run_evaluate(db, batch_id=batch.id, purpose="auto", _commit=False, + _metric_ids=changed_metrics) + run = db.get(BizMigrationRun, result["id"]) + run.summary_json = {**run.summary_json, "auto_fingerprint": fingerprint, "auto_samples": samples} + db.commit() + count += 1 + except Exception: + db.rollback() + _log.exception("cutover monitor evaluate failed project=%s", proj.id) + return count + + +def schedule_auto_monitor_for_task(task_id: str) -> None: + global _worker_running + with _lock: + _pending.add(str(task_id)) + if _worker_running: + return + _worker_running = True + + def work() -> None: + global _worker_running + from ..db import SessionLocal + + while True: + with _lock: + if not _pending: + _worker_running = False + return + tid = _pending.pop() + try: + with SessionLocal() as db: + try_auto_monitor_for_task(db, tid) + except Exception: + _log.exception("cutover monitor worker failed task=%s", tid) + + threading.Thread(target=work, name="biz-cutover-monitor", daemon=True).start() diff --git a/netx_api/biz_migration/evaluate.py b/netx_api/biz_migration/evaluate.py index eb85841..dd34fc0 100644 --- a/netx_api/biz_migration/evaluate.py +++ b/netx_api/biz_migration/evaluate.py @@ -4,7 +4,10 @@ from __future__ import annotations from typing import Any -from ..biz_state.compare_engine import apply_port_map, compare_rows +from ..biz_state.compare_engine import apply_port_map, compare_rows, key_tuple +from ..biz_state.compare_rules import ( + effective_compare_fields, field_rule_map, scalar_text, values_equal, +) PORT_METRIC_ID = "interface_brief" PORT_STATUS_FIELDS = ("admin", "phy", "prot") @@ -22,7 +25,7 @@ def normalize_expect_key(raw: str) -> str: Accepts legacy ``a|b`` multi-field keys and already-normalized ``\\x1f`` keys. """ - s = str(raw or "").strip() + s = scalar_text(raw).strip() if not s: return "" if KEY_SEP in s: @@ -65,13 +68,13 @@ def classify_status(row: dict[str, Any] | None, sheet_override: dict[str, Any] | return "none" down_vals = {str(x).lower() for x in (ov.get("down_values") or ["down"])} up_vals = {str(x).lower() for x in (ov.get("up_values") or ["up"])} - vals = [str(row.get(f) or "").strip().lower() for f in fields] + vals = [scalar_text(row.get(f)).strip().lower() for f in fields] vals = [v for v in vals if v] if not vals: return "none" if any(v in down_vals for v in vals): return "down" - if vals and all(v in up_vals for v in vals): + if len(vals) == len(fields) and all(v in up_vals for v in vals): return "up" return "other" @@ -105,11 +108,11 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]: if it.get("key") is not None: k = it.get("key") if isinstance(k, (list, tuple)): - joined = KEY_SEP.join(str(p).strip() for p in k if str(p).strip()) - if joined: + joined = KEY_SEP.join(scalar_text(p).strip() for p in k) + if joined.strip(KEY_SEP): bucket.add(joined) else: - s = normalize_expect_key(str(k or "")) + s = normalize_expect_key(scalar_text(k)) if s: bucket.add(s) keys = it.get("keys") @@ -117,14 +120,14 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]: # Flat list of segments → one composite key; else each entry is a key # (string or nested list/tuple of segments). if keys and all(not isinstance(x, (list, tuple, dict)) for x in keys): - joined = KEY_SEP.join(str(x).strip() for x in keys if str(x).strip()) - if joined: + joined = KEY_SEP.join(scalar_text(x).strip() for x in keys) + if joined.strip(KEY_SEP): bucket.add(joined) else: for x in keys: if isinstance(x, (list, tuple)): - joined = KEY_SEP.join(str(p).strip() for p in x if str(p).strip()) - if joined: + joined = KEY_SEP.join(scalar_text(p).strip() for p in x) + if joined.strip(KEY_SEP): bucket.add(joined) else: s = normalize_expect_key(str(x or "")) @@ -219,7 +222,7 @@ def _field_tokens_from_row(row: dict[str, Any] | None, fields: set[str]) -> set[ return set() out: set[str] = set() for f in fields: - v = str(row.get(f) or "").strip() + v = scalar_text(row.get(f)).strip() if v: out.add(field_token(f, v)) return out @@ -245,8 +248,8 @@ def _side_tokens( def _norm_list_values(raw: Any) -> list[str]: if isinstance(raw, list): - return [str(x).strip().lower() for x in raw if str(x).strip()] - s = str(raw or "").strip() + return [scalar_text(x).strip().lower() for x in raw if scalar_text(x).strip()] + s = scalar_text(raw).strip() if not s: return [] if "," in s: @@ -266,18 +269,22 @@ def _eval_field_op( f = str(field or "").strip() if not f: return False - raw = "" if not row else str(row.get(f) or "").strip() + raw = "" if not row else scalar_text(row.get(f)).strip() val = raw.lower() o = str(op or "eq").strip().lower() + if row and f not in row: + # A missing parser field is not evidence for a negative condition such as + # state != down. Only an explicit empty check may match that absence. + return o == "empty" if o in ("changed", "diff"): - base = "" if not base_row else str(base_row.get(f) or "").strip().lower() + base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower() if not row: return False return base != val if o in ("unchanged", "same"): if not row: return False - base = "" if not base_row else str(base_row.get(f) or "").strip().lower() + base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower() return base == val if o == "empty": return not val @@ -568,7 +575,7 @@ def _match_field_token_rules( if not field or not allowed: continue row = old_row if side == "old" else new_row if side == "new" else None - val = str((row or {}).get(field) or "").strip().lower() + val = scalar_text((row or {}).get(field)).strip().lower() if val not in allowed: return False return True @@ -749,10 +756,10 @@ def build_diff_index_from_compare(result: dict[str, Any]) -> dict[str, dict[str, continue key = d.get("key") if isinstance(key, dict): - key_list = [str(v) for v in key.values()] + key_list = [scalar_text(v).strip() for v in key.values()] ks = _key_str(key_list) elif isinstance(key, (list, tuple)): - key_list = [str(x) for x in key] + key_list = [scalar_text(x).strip() for x in key] ks = _key_str(key_list) else: ks = str(key or "") @@ -794,17 +801,18 @@ def expect_keys_for_metric( return keys -def _map_defines_iface_expect(metric_id: str, iface_fields: list[str], port_map: dict[str, str]) -> bool: - """Iface sheets: a non-empty port map is the expect scope. +def _map_defines_iface_expect( + iface_fields: list[str], port_map: dict[str, str], scope_by_port_map: bool | None = None, +) -> bool: + """Whether a non-empty port map is this sheet's expect scope. - BGP / PW keep the batch expect picker even if a column is marked iface. + The sheet override decides (``scope_by_port_map``); the built-in presets set it to + false for BGP / pseudowire sheets, which keep the batch expect picker. Unset means + "an interface sheet with a map is scoped by the map". """ if not port_map or not iface_fields: return False - mid = str(metric_id or "").strip().lower() - if "bgp" in mid or "pw" in mid or "pseudowire" in mid: - return False - return True + return True if scope_by_port_map is None else bool(scope_by_port_map) def _iface_values(key_str: str, *, key_fields: list[str], iface_fields: list[str]) -> list[str]: @@ -858,6 +866,14 @@ def _port_identity(key_fields: list[str], iface_fields: list[str]) -> bool: return all(str(f) in iface_set for f in key_fields) +def _iface_selected(value: str, ports: set[str]) -> bool: + """Set lookup plus parent interface prefixes; independent of scope size.""" + if value in ports: + return True + parts = value.split(".") + return any(".".join(parts[:i]) in ports for i in range(len(parts) - 1, 0, -1)) + + def _unmapped_same_iface_anomaly( *, old_kind: str, @@ -892,6 +908,8 @@ def _remap_key_str( return key_str if not port_map: return key_str + if key_fields and not set(key_fields).intersection(iface_fields): + return key_str if (not key_fields or len(key_fields) == 1) and ( key_str in port_map or any( @@ -928,6 +946,8 @@ def _reverse_remap_key_str( return key_str if not rev_map: return key_str + if key_fields and not set(key_fields).intersection(iface_fields): + return key_str if (not key_fields or len(key_fields) == 1) and ( key_str in rev_map or any( @@ -1003,13 +1023,16 @@ def row_match_key( key_fields: list[str], iface_fields: list[str], rules: list[dict[str, str]] | None, + normalizer: Any = None, ) -> str: from ..biz_state.iface_normalize import apply_iface_normalize data = strip_netx(row) - if iface_fields and rules: + if normalizer is not None: + data = normalizer(dict(data)) + elif iface_fields and rules: data = apply_iface_normalize(dict(data), iface_fields=iface_fields, rules=rules) - return KEY_SEP.join(str(data.get(k) or "").strip() for k in key_fields) + return KEY_SEP.join(scalar_text(data.get(k)).strip() for k in key_fields) def index_raw_by_match_key( @@ -1018,11 +1041,14 @@ def index_raw_by_match_key( key_fields: list[str], iface_fields: list[str], rules: list[dict[str, str]] | None, + normalizer: Any = None, ) -> dict[str, dict[str, Any]]: """First raw row per internal match key (normalized iface, raw elsewhere).""" idx: dict[str, dict[str, Any]] = {} for raw in raw_rows: - ks = row_match_key(raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules) + ks = row_match_key( + raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules, normalizer=normalizer, + ) if ks and ks not in idx: idx[ks] = raw return idx @@ -1033,7 +1059,7 @@ def display_key_from_raw(raw: dict[str, Any] | None, key_fields: list[str]) -> s if not raw or not key_fields: return "" data = strip_netx(raw) - return "|".join(str(data.get(k) or "").strip() for k in key_fields) + return "|".join(scalar_text(data.get(k)).strip() for k in key_fields) def iface_lineage( @@ -1043,6 +1069,7 @@ def iface_lineage( rules: list[dict[str, str]] | None, port_map: dict[str, str] | None, apply_map: bool, + namer: Any = None, ) -> list[dict[str, Any]]: """Per iface field: raw → normalized → (optional) mapped.""" from ..biz_state.iface_normalize import normalize_iface_name, resolve_mapped_iface @@ -1053,8 +1080,8 @@ def iface_lineage( pmap = port_map or {} out: list[dict[str, Any]] = [] for f in iface_fields: - raw_v = str(data.get(f) or "").strip() - norm_v = normalize_iface_name(raw_v, rules) if rules else raw_v + raw_v = scalar_text(data.get(f)).strip() + norm_v = namer(raw_v) if namer is not None else (normalize_iface_name(raw_v, rules) if rules else raw_v) mapped = False map_to = "" if apply_map and pmap and norm_v: @@ -1092,6 +1119,9 @@ def evaluate_metric_dual( out_of_expect: str = "strict", sheet_id: str = "", iface_normalize_rules: list[dict[str, str]] | None = None, + previous_expect: dict[str, set[str]] | None = None, + allow_mapping_scope: bool = True, + pipeline: Any = None, ) -> dict[str, Any]: """Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge. @@ -1099,15 +1129,21 @@ def evaluate_metric_dual( Board / AI fields ``old_key`` / ``new_key`` and ``old`` / ``new`` stay **raw** device values (A/B), never the post-map BB form. """ - from ..biz_state.iface_normalize import ( - apply_iface_normalize_rows, - normalize_iface_rules, + from ..biz_state.identity_pipeline import legacy_pipeline + + # Old device = "before" side, new device = "after" side of the identity pipeline. + pipe = pipeline or legacy_pipeline( + key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields, + field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules, ) sid = str(sheet_id or "").strip() or str(metric_id or "").strip() expect_keys = expect_keys_for_metric( expect, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid ) + previous_keys = expect_keys_for_metric( + previous_expect or {}, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid + ) ov = sheet_override or {} if ov.get("skip_dual"): return { @@ -1134,39 +1170,44 @@ def evaluate_metric_dual( [dict(r) for r in new_baseline_rows] if new_baseline_rows is not None else None ) - norm_rules = normalize_iface_rules(iface_normalize_rules) - old_baseline_rows = apply_iface_normalize_rows( - [strip_netx(r) for r in raw_old_base], iface_fields=iface_fields, rules=norm_rules - ) - old_current_rows = apply_iface_normalize_rows( - [strip_netx(r) for r in raw_old_cur], iface_fields=iface_fields, rules=norm_rules - ) - new_current_rows = apply_iface_normalize_rows( - [strip_netx(r) for r in raw_new_cur], iface_fields=iface_fields, rules=norm_rules - ) + def normalized_key(key: str) -> str: + parts = key.split(KEY_SEP) if KEY_SEP in key else ([key] if len(key_fields) == 1 else key.split("|")) + if len(parts) != len(key_fields): + return key + return KEY_SEP.join(pipe.normalize_name(p, "before") if f in iface_fields else p + for p, f in zip(parts, key_fields)) + + def norm_before(row: dict[str, Any]) -> dict[str, Any]: + return pipe.normalize(row, "before") + + def norm_after(row: dict[str, Any]) -> dict[str, Any]: + return pipe.normalize(row, "after") + + expect_keys = {normalized_key(key) for key in expect_keys} + previous_keys = {normalized_key(key) for key in previous_keys} + old_baseline_rows = [norm_before(strip_netx(r)) for r in raw_old_base] + old_current_rows = [norm_before(strip_netx(r)) for r in raw_old_cur] + new_current_rows = [norm_after(strip_netx(r)) for r in raw_new_cur] if raw_new_base is not None: - new_baseline_rows = apply_iface_normalize_rows( - [strip_netx(r) for r in raw_new_base], - iface_fields=iface_fields, - rules=norm_rules, - ) + new_baseline_rows = [norm_after(strip_netx(r)) for r in raw_new_base] else: new_baseline_rows = None old_raw_cur_idx = index_raw_by_match_key( - raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules + raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before ) old_raw_base_idx = index_raw_by_match_key( - raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules + raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before ) new_raw_cur_idx = index_raw_by_match_key( - raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules + raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_after ) new_raw_base_idx = index_raw_by_match_key( raw_new_base or [], key_fields=key_fields, iface_fields=iface_fields, - rules=norm_rules, + rules=None, + normalizer=norm_after, ) old_cmp = compare_rows( @@ -1178,6 +1219,8 @@ def evaluate_metric_dual( port_map=None, field_rules=field_rules, iface_normalize_rules=None, # already applied + include_unchanged=True, + duplicate_pairing=pipe.duplicate_pairing, ) old_idx = build_diff_index_from_compare(old_cmp) @@ -1186,18 +1229,8 @@ def evaluate_metric_dual( if new_baseline_rows is not None: new_before = list(new_baseline_rows) new_baseline_mode = "provided" - # Explicit empty batch is as unusable as "no baseline" for presence semantics. - if not new_before and (old_baseline_rows or new_current_rows): - if port_map and iface_fields: - new_before = [ - apply_port_map(r, iface_fields=iface_fields, port_map=port_map) - for r in old_baseline_rows - ] - new_baseline_mode = "port_mapped" - new_baseline_missing = False - else: - new_baseline_missing = True - new_baseline_mode = "empty" + # A successfully collected empty baseline is valid on a replacement NE. + # Collection completeness is checked by the service, not inferred from rows. elif port_map and iface_fields: new_before = [ apply_port_map(r, iface_fields=iface_fields, port_map=port_map) @@ -1218,12 +1251,23 @@ def evaluate_metric_dual( compare_fields=compare_fields, port_map=None, field_rules=field_rules, + include_unchanged=True, + duplicate_pairing=pipe.duplicate_pairing, ) new_idx = build_diff_index_from_compare(new_cmp) rev_map = {v: k for k, v in port_map.items()} - map_scoped = _map_defines_iface_expect(metric_id, iface_fields, port_map) - if map_scoped: + map_scoped = _map_defines_iface_expect(iface_fields, port_map, ov.get("scope_by_port_map")) + # Mapping describes identity; an explicit batch selection defines scope. + # Port selection also covers business rows attached to those interfaces. + for scope, keys in ((expect, expect_keys), (previous_expect or {}, previous_keys)): + ports = {pipe.normalize_name(p, "before") for p in scope.get("_ports") or set()} + if ports and not keys and iface_fields: + for ks, diff in old_idx.items(): + row = diff.get("before") or diff.get("after") or {} + if any(_iface_selected(scalar_text(row.get(f)), ports) for f in iface_fields): + keys.add(ks) + if allow_mapping_scope and map_scoped and not any(expect.values()): scoped: set[str] = set() for ks in old_idx: if _key_in_port_map( @@ -1243,6 +1287,21 @@ def evaluate_metric_dual( scoped.add(canon) expect_keys = scoped + # Duplicate keys only block when they sit inside this wave's scope. + from collections import Counter + + def _dup_keys(rows: list[dict[str, Any]]) -> set[str]: + return {KEY_SEP.join(k) for k, n in Counter(key_tuple(r, key_fields) for r in rows).items() if n > 1} + + in_scope = expect_keys | previous_keys + new_dups = _dup_keys(new_current_rows) | _dup_keys(new_baseline_rows or []) + flagged = ( + _dup_keys(old_baseline_rows) | _dup_keys(old_current_rows) | new_dups + | {_reverse_remap_key_str(k, key_fields=key_fields, iface_fields=iface_fields, rev_map=rev_map) + for k in new_dups} + ) + duplicate_in_expect = sorted(flagged & in_scope)[:64] + canon_keys: list[str] = [] seen: set[str] = set() @@ -1253,6 +1312,8 @@ def evaluate_metric_dual( for ok in sorted(expect_keys): _add(ok) + for ok in sorted(previous_keys): + _add(ok) for ks in sorted(old_idx): _add(ks) for ks in sorted(new_idx): @@ -1270,6 +1331,11 @@ def evaluate_metric_dual( progress_total = 0 anomaly = 0 anomaly_in_expect = 0 + steady_outside = 0 + rules_by_field = field_rule_map(field_rules) + integrity_fields = effective_compare_fields(compare_fields, field_rules) + # Status transitions are interpreted by the monitor's correction rules. + integrity_fields = [f for f in integrity_fields if f not in (ov.get("status_fields") or [])] for old_ks in canon_keys: new_ks = _remap_key_str( @@ -1279,11 +1345,12 @@ def evaluate_metric_dual( port_map=port_map, ) in_exp = old_ks in expect_keys + previous = old_ks in previous_keys and not in_exp if in_exp: progress_total += 1 od = old_idx.get(old_ks) - nd = new_idx.get(new_ks) or new_idx.get(old_ks) + nd = new_idx.get(new_ks) old_kind = str((od or {}).get("kind") or "") new_kind = str((nd or {}).get("kind") or "") @@ -1301,11 +1368,7 @@ def evaluate_metric_dual( old_st = classify_status(old_cur, ov) new_st = classify_status(new_cur, ov) - # Mapped iface rows are the expect set. Unmapped rows stay out of the - # migration compare, except same-name ports where old went down and the - # new device's same port is up — that is anomaly, not a successful cutover. - if map_scoped and not in_exp: - if not ( + if map_scoped and not (in_exp or previous) and ( _port_identity(key_fields, iface_fields) and _unmapped_same_iface_anomaly( old_kind=old_kind, @@ -1314,26 +1377,63 @@ def evaluate_metric_dual( new_status=new_st, ) ): - continue verdict, color, rule_hit = "anomaly", "red", "unmapped_same_iface" else: verdict, color, rule_hit = dual_verdict_ex( old_kind=old_kind or "", new_kind=new_kind or "", - in_expect=in_exp, + in_expect=in_exp or previous, window_active=window_active, acceptance=acceptance, old_status=old_st, new_status=new_st, success_patterns=success_patterns or None, anomaly_patterns=anomaly_patterns or None, - out_of_expect=out_of_expect, + # Business drift outside this batch must remain visible. + out_of_expect="strict", old_row=old_cur or None, new_row=new_cur or None, old_base=old_base or None, new_base=new_base or None, sheet_override=ov, ) + integrity_changes: list[dict[str, Any]] = [] + if in_exp or previous: + old_departed = old_kind == "removed" or old_st == "down" + new_healthy = bool(new_cur) and ( + new_st == "up" if ov.get("status_fields") else True + ) + if old_departed and not new_healthy: + verdict, color, rule_hit = "lost", "red", "service_not_received" + elif verdict == "migrated": + if not new_healthy: + verdict, color, rule_hit = "anomaly", "red", "target_unhealthy" + elif not old_base: + verdict, color, rule_hit = "anomaly", "red", "old_service_not_in_baseline" + else: + reference = apply_port_map(old_base, iface_fields=iface_fields, port_map=port_map) + integrity_changes = [ + {"field": f, "before": reference.get(f), "after": new_cur.get(f)} + for f in integrity_fields + if not values_equal(reference.get(f), new_cur.get(f), rule=rules_by_field.get(f)) + ] + if integrity_changes: + verdict, color, rule_hit = "anomaly", "red", "service_content_changed" + elif not old_departed and not success_patterns: + verdict, color, rule_hit = ( + ("unfinished", "red", "old_service_still_present") if acceptance + else ("migrating", "yellow", "old_service_still_present") + ) + if previous: + if verdict == "migrated": + verdict, rule_hit = "migrated_previous", "previous_batch_received" + elif color != "red": + verdict, color, rule_hit = "unfinished_previous", "red", "previous_batch_not_received" + # Stable inventory is covered by the comparison and counted, but does + # not need a full persisted evidence card on every high-frequency run. + if not in_exp and verdict == "not_involved": + steady_outside += 1 + continue if in_exp and verdict == "migrated" and color == "green": progress_ok += 1 if color == "red": @@ -1344,9 +1444,9 @@ def evaluate_metric_dual( # Prefer live raw rows; fall back to baseline raw. Never use port-mapped # synthetic "before" as the new-side display row. raw_old = old_raw_cur_idx.get(old_ks) or old_raw_base_idx.get(old_ks) - raw_new = new_raw_cur_idx.get(new_ks) or new_raw_cur_idx.get(old_ks) + raw_new = new_raw_cur_idx.get(new_ks) if not raw_new: - raw_new = new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks) + raw_new = new_raw_base_idx.get(new_ks) display_old = display_key_from_raw(raw_old, key_fields) or old_ks display_new = display_key_from_raw(raw_new, key_fields) or ( @@ -1360,8 +1460,8 @@ def evaluate_metric_dual( new_disp = strip_netx(raw_new) if raw_new else {} old_base_raw = strip_netx(old_raw_base_idx.get(old_ks)) if old_raw_base_idx.get(old_ks) else {} new_base_raw = ( - strip_netx(new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks)) - if (new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks)) + strip_netx(new_raw_base_idx.get(new_ks)) + if new_raw_base_idx.get(new_ks) else {} ) @@ -1371,9 +1471,10 @@ def evaluate_metric_dual( "iface": iface_lineage( raw_old, iface_fields=iface_fields, - rules=norm_rules, + rules=None, port_map=port_map, apply_map=True, + namer=lambda v: pipe.normalize_name(v, "before"), ), }, "new": { @@ -1381,9 +1482,10 @@ def evaluate_metric_dual( "iface": iface_lineage( raw_new, iface_fields=iface_fields, - rules=norm_rules, + rules=None, port_map=None, apply_map=False, + namer=lambda v: pipe.normalize_name(v, "after"), ), }, "port_map": { @@ -1411,7 +1513,9 @@ def evaluate_metric_dual( "verdict": verdict, "color": color, "rule_hit": rule_hit, + "integrity_changes": integrity_changes, "in_expect": in_exp, + "previous_batch": previous, "old_kind": old_kind, "new_kind": new_kind, "old": old_disp, @@ -1444,12 +1548,14 @@ def evaluate_metric_dual( return { "metric_id": metric_id, + "effective_scope_by_port_map": map_scoped, "old_summary": old_cmp.get("summary") or {}, "new_summary": new_cmp.get("summary") or {}, "progress_ok": progress_ok, "progress_total": progress_total, "anomaly": anomaly, "anomaly_in_expect": anomaly_in_expect, + "steady_outside": steady_outside, "rows": rows_out, "new_baseline_mode": new_baseline_mode, "new_baseline_missing": new_baseline_missing, @@ -1457,6 +1563,7 @@ def evaluate_metric_dual( + int((new_cmp.get("summary") or {}).get("duplicate_keys_before") or 0), "duplicate_keys_after": int((old_cmp.get("summary") or {}).get("duplicate_keys_after") or 0) + int((new_cmp.get("summary") or {}).get("duplicate_keys_after") or 0), + "duplicate_in_expect": duplicate_in_expect, "duplicate_key_list": list( dict.fromkeys( list((old_cmp.get("summary") or {}).get("duplicate_key_list") or []) diff --git a/netx_api/biz_migration/monitor_templates.py b/netx_api/biz_migration/monitor_templates.py index b55f487..bd72ee0 100644 --- a/netx_api/biz_migration/monitor_templates.py +++ b/netx_api/biz_migration/monitor_templates.py @@ -8,7 +8,8 @@ from uuid import uuid4 from fastapi import HTTPException from sqlalchemy.orm import Session -from ..models import BizCompareTemplate, BizMonitorTemplate +from ..models import BizCompareTemplate, BizMigrationProject, BizMonitorTemplate +from .validation import validate_monitor_rules from ..timeutil import utcnow_naive from ..biz_state import compare_service as cmp_svc from ..biz_state.iface_normalize import default_zte_iface_normalize_rules @@ -33,6 +34,8 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None effective = seen if not effective: effective = [PORT_METRIC_ID] + ct = db.get(BizCompareTemplate, row.compare_template_id) if db and row.compare_template_id else None + available = list(dict.fromkeys(str(s.get("metric_id") or "") for s in cmp_svc.template_metrics(ct))) if ct else effective out: dict[str, Any] = { "id": row.id, "name": row.name, @@ -40,6 +43,7 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None "compare_template_name": compare_name, "collect_metric_ids": collect, "collect_metric_ids_effective": effective, + "available_metric_ids": [m for m in available if m], "defaults": dict(row.defaults_json or {}), "sheet_overrides": list(row.sheet_overrides_json or []), "note": row.note or "", @@ -57,6 +61,18 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None return out +def _has_legacy_port_and_group(override: dict[str, Any]) -> bool: + """True when a success rule's old side demands admin down AND phy down.""" + legacy = _port_old_down_groups(legacy_and=True)[0] + for rule in override.get("success") or []: + if not isinstance(rule, dict): + continue + for group in rule.get("old_groups") or []: + if group == legacy: + return True + return False + + def validate_sheet_overrides( db: Session, *, @@ -90,6 +106,9 @@ def validate_sheet_overrides( for ov in overrides: sid = str(ov.get("sheet_id") or "").strip() mid = str(ov.get("metric_id") or "").strip() + if _has_legacy_port_and_group(ov): + warnings.append({"sheet_id": sid, "metric_id": mid, + "reason": "port_success_legacy_and_semantics"}) if sid: if sid not in sheet_ids: warnings.append( @@ -224,9 +243,29 @@ def _success_stateful( } -def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[str, Any]: +_PORT_ADMIN_DOWN = {"type": "value", "field": "admin", "op": "in", "value": ["down"]} +_PORT_PHY_DOWN = {"type": "value", "field": "phy", "op": "in", "value": ["down"]} + + +def _port_old_down_groups(*, legacy_and: bool = False) -> list[list[dict[str, Any]]]: + """Old-side "port is out of service" groups (OR of groups, AND within one). + + A fibre pulled without ``shutdown`` leaves admin up / phy down, so either + signal alone counts. ``legacy_and`` is the pre-A4 form requiring both. + """ + if legacy_and: + return [[dict(_PORT_ADMIN_DOWN), dict(_PORT_PHY_DOWN)]] + return [[dict(_PORT_PHY_DOWN)], [dict(_PORT_ADMIN_DOWN)]] + + +def preset_override_for_metric( + metric_id: str, *, sheet_id: str = "", legacy_port_and: bool = False, +) -> dict[str, Any]: """Default dual-verdict overlay for a compare-template sheet. + ``legacy_port_and`` rebuilds the pre-A4 port preset (old side must be + admin down AND phy down) so stored copies can be recognised and upgraded. + ``metric_id`` is the collected source table; ``sheet_id`` identifies a filtered split (e.g. bgp_peer.vpnv4). Presets key off the source metric. """ @@ -240,6 +279,8 @@ def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[st look = mid.split(".", 1)[0] if "." in mid else mid def _out(body: dict[str, Any]) -> dict[str, Any]: + if _scope_by_map_default_off(look): + body["scope_by_port_map"] = False body["metric_id"] = mid or look if sid and sid != (mid or look): body["sheet_id"] = sid @@ -257,12 +298,7 @@ def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[st "up_values": ["up"], "success": [ _success_stateful( - old_down_groups=[ - [ - {"type": "value", "field": "admin", "op": "in", "value": ["down"]}, - {"type": "value", "field": "phy", "op": "in", "value": ["down"]}, - ] - ], + old_down_groups=_port_old_down_groups(legacy_and=legacy_port_and), new_up_conds=up, ) ], @@ -365,6 +401,8 @@ DEFAULT_MONITOR_TEMPLATE_NAME = "默认割接监控" def ensure_default_monitor_templates(db: Session) -> None: """Seed once when monitor-template table is empty — one template on the full status compare.""" if db.query(BizMonitorTemplate.id).limit(1).first(): + upgrade_legacy_port_overrides(db) + fill_scope_by_port_map(db) return cmp_svc.ensure_default_templates(db) zte = ( @@ -399,9 +437,68 @@ def ensure_default_monitor_templates(db: Session) -> None: db.commit() -def default_port_monitor_template_id(db: Session) -> str: - """Ensure seeds exist and return the default monitor template id.""" - ensure_default_monitor_templates(db) +def _scope_by_map_default_off(name: str) -> bool: + """BGP / pseudowire sheets keep the batch expect picker; a port map does not scope them.""" + low = str(name or "").lower() + return "bgp" in low or "pw" in low or "pseudowire" in low + + +def fill_scope_by_port_map(db: Session) -> int: + """Make the old name-based default explicit on stored overrides (one-off, idempotent).""" + changed = 0 + for row in db.query(BizMonitorTemplate).all(): + out = [] + touched = False + for ov in list(row.sheet_overrides_json or []): + if (isinstance(ov, dict) and "scope_by_port_map" not in ov + and _scope_by_map_default_off(f"{ov.get('metric_id') or ''} {ov.get('sheet_id') or ''}")): + ov = {**ov, "scope_by_port_map": False} + touched = True + out.append(ov) + if touched: + row.sheet_overrides_json = out + row.updated_at = _utcnow() + changed += 1 + if changed: + db.commit() + return changed + + +def upgrade_legacy_port_overrides(db: Session) -> int: + """Rewrite stored port overrides that are exactly the pre-A4 preset (AND semantics). + + Field-by-field equality only: anything a user customised is left alone and + surfaces ``port_success_legacy_and_semantics`` in ``override_warnings``. + """ + changed = 0 + for row in db.query(BizMonitorTemplate).all(): + overrides = list(row.sheet_overrides_json or []) + out: list[Any] = [] + touched = False + for ov in overrides: + if isinstance(ov, dict): + mid = str(ov.get("metric_id") or "").strip() + sid = str(ov.get("sheet_id") or "").strip() + if (mid or sid).split(".", 1)[0] == PORT_METRIC_ID and ov == preset_override_for_metric( + mid, sheet_id=sid, legacy_port_and=True, + ): + out.append(preset_override_for_metric(mid, sheet_id=sid)) + touched = True + continue + out.append(ov) + if touched: + row.sheet_overrides_json = out + row.updated_at = _utcnow() + changed += 1 + if changed: + db.commit() + return changed + + +def default_port_monitor_template_id(db: Session, *, seed: bool = True) -> str: + """Return the default monitor template id, seeding the built-ins first unless ``seed=False``.""" + if seed: + ensure_default_monitor_templates(db) for name in ( DEFAULT_MONITOR_TEMPLATE_NAME, "ZTE 状态割接监控", @@ -456,6 +553,7 @@ def create_monitor_template(db: Session, body: dict[str, Any]) -> dict[str, Any] overrides = body.get("sheet_overrides") if not isinstance(overrides, list): overrides = [] + validate_monitor_rules(defaults, overrides) row = BizMonitorTemplate( id=uuid4().hex, name=name[:256], @@ -479,6 +577,10 @@ def update_monitor_template(db: Session, template_id: str, body: dict[str, Any]) row = db.get(BizMonitorTemplate, template_id) if not row: raise HTTPException(status_code=404, detail="monitor_template_not_found") + validate_monitor_rules(body.get("defaults", row.defaults_json or {}), + body.get("sheet_overrides", row.sheet_overrides_json or [])) + if body.get("compare_template_id") and not db.get(BizCompareTemplate, body["compare_template_id"]): + raise HTTPException(status_code=404, detail="compare_template_not_found") if "name" in body and body["name"] is not None: name = str(body["name"] or "").strip() if not name: @@ -515,5 +617,7 @@ def delete_monitor_template(db: Session, template_id: str) -> None: row = db.get(BizMonitorTemplate, template_id) if not row: raise HTTPException(status_code=404, detail="monitor_template_not_found") + if db.query(BizMigrationProject.id).filter(BizMigrationProject.monitor_template_id == template_id).first(): + raise HTTPException(status_code=409, detail="monitor_template_in_use") db.delete(row) db.commit() diff --git a/netx_api/biz_migration/service.py b/netx_api/biz_migration/service.py index 760a6c0..fc965c8 100644 --- a/netx_api/biz_migration/service.py +++ b/netx_api/biz_migration/service.py @@ -2,18 +2,24 @@ from __future__ import annotations +import hashlib +import logging +import threading +from collections import OrderedDict +from collections import Counter from datetime import datetime, timezone from typing import Any from uuid import uuid4 from fastapi import HTTPException +from sqlalchemy import and_, insert, or_ from sqlalchemy.orm import Session -from ..biz_state.compare_rules import apply_row_filters +from ..biz_state.compare_rules import apply_row_filters, scalar_text +from ..config import settings from ..biz_state.compare_service import ( _load_metric_rows, _port_map_dict, - batch_metric_collect_ok, sheet_key, sheet_title, template_metrics, @@ -28,6 +34,7 @@ from ..models import ( BizMonitorTemplate, BizPortMapping, BizStateBatch, + BizStateBatchCommand, BizStateTask, ) from ..timeutil import utcnow_naive @@ -41,6 +48,45 @@ from .evaluate import ( ) PURPOSE_CUTOVER_HF = "cutover_hf" +_log = logging.getLogger("netx.biz_migration") + + +class BaselineRowCache: + """Process-local LRU of immutable baseline rows, bounded by total row count.""" + + def __init__(self, max_rows: int = 500_000) -> None: + self.max_rows = max_rows + self._items: "OrderedDict[tuple[str, str], list[dict[str, Any]]]" = OrderedDict() + self._rows = 0 + self._lock = threading.Lock() + + def get_or_load(self, batch_id: str, metric_id: str, loader: Any) -> list[dict[str, Any]]: + key = (batch_id, metric_id) + with self._lock: + hit = self._items.get(key) + if hit is not None: + self._items.move_to_end(key) + return hit + rows = loader() + if self.max_rows > 0 and len(rows) <= self.max_rows: + with self._lock: + if key not in self._items: + self._items[key] = rows + self._rows += len(rows) + while self._rows > self.max_rows and self._items: + _, evicted = self._items.popitem(last=False) + self._rows -= len(evicted) + return rows + + def invalidate(self, batch_id: str = "") -> None: + with self._lock: + for key in [k for k in self._items if not batch_id or k[0] == batch_id]: + self._rows -= len(self._items.pop(key)) + + +BASELINE_ROWS = BaselineRowCache(int(getattr(settings, "biz_migration_baseline_cache_rows", 500_000))) + +MANUAL_ROUTE_METRICS = frozenset({"ip_route", "ipv6_route", "bgp_route"}) def _parse_dt(raw: Any) -> datetime | None: @@ -149,7 +195,7 @@ def hf_task_id(proj: BizMigrationProject, side: str) -> str: def current_task_id(proj: BizMigrationProject, side: str) -> str: """Evaluate current batches: prefer HF, fall back to portrait.""" - return hf_task_id(proj, side) or portrait_task_id(proj, side) + return hf_task_id(proj, side) def _metric_interval_map(proj: BizMigrationProject) -> dict[str, int]: @@ -216,7 +262,9 @@ def _hf_bindings(proj: BizMigrationProject, side: str) -> list[dict[str, Any]]: iv = max(60, int(row.get("interval_sec") or 60)) except (TypeError, ValueError): iv = 60 - out.append({"task_id": tid, "metric_ids": mids, "interval_sec": iv}) + out.append({"task_id": tid, "metric_ids": mids, "interval_sec": iv, + "collect_group_id": str(row.get("collect_group_id") or ""), + "manual_only": bool(row.get("manual_only"))}) if out: return out # Legacy single slot @@ -268,15 +316,124 @@ def _current_batch_for_metric( # Only honor pin when it belongs to this metric's HF task (multi-interval safe) if pinned and (not tid or str(pinned.task_id or "") == tid): # Pinned partial / failed metric → treat as missing (avoid false red) - if not batch_metric_collect_ok(db, pinned.id, metric_id): + if not _metric_sample_ok(db, pinned, metric_id): return None return pinned - batch = _latest_success_batch(db, tid) if tid else None - if batch and not batch_metric_collect_ok(db, batch.id, metric_id): + batch = ( + db.query(BizStateBatch) + .filter(BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued"))) + .order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() + ) if tid else None + if batch and not _metric_sample_ok(db, batch, metric_id): return None return batch +def _metric_sample_ok(db: Session, batch: BizStateBatch, metric_id: str) -> bool: + """An empty result requires a successful command receipt, just like a full one. + + Batch success alone cannot prove this metric was collected. Unmatched and + skipped parsers are uncertainty, never evidence that a business disappeared. + """ + if batch.status not in ("success", "partial"): + return False + statuses = [str(s or "").lower() for (s,) in db.query(BizStateBatchCommand.parse_status) + .filter(BizStateBatchCommand.batch_id == batch.id, + BizStateBatchCommand.metric_id == metric_id).all()] + return bool(statuses) and all(s in ("ok", "ok_aux") for s in statuses) + + +def _uncovered_slice_sides( + db: Session, sheet: dict[str, Any], metric_id: str, old_batch_id: str, new_batch_id: str, +) -> list[str]: + from ..biz_state.compare_quality import covers_slice, side_receipts, sheet_slice + + wanted = sheet_slice(sheet) + if not wanted: + return [] + out = [] + for side, bid in (("old", old_batch_id), ("new", new_batch_id)): + receipts = side_receipts(db, bid, metric_id) + if receipts and not covers_slice(receipts, wanted): + out.append(side) + return out + + +def _current_pair_for_metric(db: Session, proj: BizMigrationProject, mid: str, + pin_old: str = "", pin_new: str = "") -> tuple[BizStateBatch | None, BizStateBatch | None, dict[str, Any]]: + old_tid, new_tid = (_hf_task_id_for_metric(proj, side, mid) for side in ("old", "new")) + old_task = db.get(BizStateTask, old_tid) if old_tid else None + new_task = db.get(BizStateTask, new_tid) if new_tid else None + gid = old_task.collect_group_id if old_task else "" + meta: dict[str, Any] = {"collect_group_id": gid, "manual_only": bool(old_task and old_task.collect_manual_only)} + if gid and old_tid and old_tid == new_tid and not (pin_old or pin_new): + # Same device on both sides: one task, one batch, trivially the same round. + latest = db.query(BizStateBatch).filter( + BizStateBatch.collect_group_id == gid, BizStateBatch.task_id == old_tid).order_by( + BizStateBatch.queued_at.desc(), BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() + if latest is None: + return None, None, {**meta, "sample_issues": ["collect_pair_missing"]} + meta.update({"collect_round_id": latest.collect_round_id, "old_collect_status": latest.status, + "new_collect_status": latest.status}) + if latest.status in ("running", "queued"): + return None, None, {**meta, "sample_issues": ["collect_pair_pending"]} + if not _metric_sample_ok(db, latest, mid): + return None, None, {**meta, "sample_issues": ["collect_pair_failed"]} + return latest, latest, meta + if gid and not (pin_old or pin_new): + if not new_task or new_task.collect_group_id != gid: + return None, None, {**meta, "sample_issues": ["collect_pair_invalid"]} + latest = db.query(BizStateBatch).filter(BizStateBatch.collect_group_id == gid, + BizStateBatch.task_id.in_((old_tid, new_tid))).order_by( + BizStateBatch.queued_at.desc(), BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() + peers = db.query(BizStateBatch).filter(BizStateBatch.collect_round_id == latest.collect_round_id, + BizStateBatch.task_id.in_((old_tid, new_tid))).all() if latest and latest.collect_round_id else [] + meta.update({"collect_round_id": latest.collect_round_id if latest else "", + "old_collect_status": next((b.status for b in peers if b.task_id == old_tid), "missing"), + "new_collect_status": next((b.status for b in peers if b.task_id == new_tid), "missing")}) + if len(peers) != 2: + return None, None, {**meta, "sample_issues": ["collect_pair_missing"]} + if any(b.status in ("running", "queued") for b in peers): + return None, None, {**meta, "sample_issues": ["collect_pair_pending"]} + by_task = {b.task_id: b for b in peers} + if not all(_metric_sample_ok(db, b, mid) for b in peers): + return None, None, {**meta, "sample_issues": ["collect_pair_failed"]} + return by_task[old_tid], by_task[new_tid], meta + old = _current_batch_for_metric(db, proj, "old", mid, pinned_batch_id=pin_old) + new = _current_batch_for_metric(db, proj, "new", mid, pinned_batch_id=pin_new) + if old and new and (old.collect_round_id or new.collect_round_id) and old.collect_round_id != new.collect_round_id: + return None, None, {**meta, "sample_issues": ["collect_round_mismatch"]} + return old, new, meta + + +def _sample_timing(proj: BizMigrationProject, mid: str, old: BizStateBatch, + new: BizStateBatch, *, check_age: bool) -> dict[str, Any]: + interval = max(60, _metric_interval_map(proj).get(mid) or proj.hf_interval_sec or 60) + old_at, new_at = old.ended_at or old.started_at, new.ended_at or new.started_at + durations = [max(0, ((b.ended_at or b.started_at) - b.started_at).total_seconds()) for b in (old, new)] + age_limit, skew_limit = interval * 3 + max(durations), interval * 2 + reasons: list[str] = [] + if not old_at or not new_at: + reasons.append("sample_time_missing") + skew = None + else: + skew = abs((old_at - new_at).total_seconds()) + if skew > skew_limit: + reasons.append("sample_skew") + if check_age and max((utcnow_naive() - old_at).total_seconds(), + (utcnow_naive() - new_at).total_seconds()) > age_limit: + reasons.append("sample_stale") + start_skew = abs((old.started_at - new.started_at).total_seconds()) + if old.collect_round_id and start_skew > min(30, interval / 2): + reasons.append("sample_start_skew") + return {"old_collected_at": _dt_iso(old_at), "new_collected_at": _dt_iso(new_at), + "age_limit_sec": age_limit, "skew_limit_sec": skew_limit, + "skew_sec": skew, "start_skew_sec": start_skew, "sample_issues": reasons, + "old_duration_sec": durations[0], "new_duration_sec": durations[1], + "old_queue_sec": max(0, (old.started_at - (old.queued_at or old.started_at)).total_seconds()), + "new_queue_sec": max(0, (new.started_at - (new.queued_at or new.started_at)).total_seconds())} + + def find_portrait_task_for_ne(db: Session, *, source: str, ne_id: str) -> BizStateTask | None: """Best portrait (non-HF) biz_state task for an NE — for baseline binding.""" src = str(source or "managed").strip().lower() or "managed" @@ -398,19 +555,26 @@ def _monitor_template_brief(db: Session, template_id: str) -> dict[str, Any]: return mon_tpl._out(row, cmp_name, db=db) # noqa: SLF001 -def resolve_project_monitor_template(db: Session, proj: BizMigrationProject) -> BizMonitorTemplate: - """Return monitor template for project; seed default port template if unbound.""" - mon_tpl.ensure_default_monitor_templates(db) +def resolve_project_monitor_template( + db: Session, proj: BizMigrationProject, *, persist: bool = True, +) -> BizMonitorTemplate: + """Return monitor template for project; seed default port template if unbound. + + ``persist=False`` is for read-only callers (GET): it never seeds, upgrades or binds, + it only reports which template the project would use. + """ + if persist: + mon_tpl.ensure_default_monitor_templates(db) tid = str(getattr(proj, "monitor_template_id", None) or "").strip() row = db.get(BizMonitorTemplate, tid) if tid else None if row: return row - default_id = mon_tpl.default_port_monitor_template_id(db) + default_id = mon_tpl.default_port_monitor_template_id(db, seed=persist) row = db.get(BizMonitorTemplate, default_id) if default_id else None if not row: raise HTTPException(status_code=400, detail="monitor_template_required") # Persist default on first use so UI shows binding - if not tid: + if not tid and persist: proj.monitor_template_id = row.id proj.updated_at = utcnow_naive() db.commit() @@ -437,12 +601,12 @@ def resolve_evaluate_sheets( return [port_sheet_def()], overrides, defaults, [] -def resolve_collect_metric_ids(db: Session, proj: BizMigrationProject) -> list[str]: +def resolve_collect_metric_ids(db: Session, proj: BizMigrationProject, *, persist: bool = True) -> list[str]: """Project-level override wins; else monitor template; else sheet metric ids.""" override = _project_collect_override(proj) if override: return override - mt = resolve_project_monitor_template(db, proj) + mt = resolve_project_monitor_template(db, proj, persist=persist) collect = [str(x).strip() for x in (mt.collect_metric_ids_json or []) if str(x).strip()] if collect: return collect @@ -460,7 +624,7 @@ def project_to_dict(db: Session, p: BizMigrationProject) -> dict[str, Any]: collect_override = _project_collect_override(p) mt_brief = _monitor_template_brief(db, getattr(p, "monitor_template_id", None) or "") # Same resolution as ensure/evaluate (empty → compare sheets) - effective_collect = resolve_collect_metric_ids(db, p) + effective_collect = resolve_collect_metric_ids(db, p, persist=False) return { "id": p.id, "name": p.name, @@ -512,11 +676,79 @@ def batch_to_dict(b: BizMigrationBatch) -> dict[str, Any]: } +def _preload_project_refs(db: Session, rows: list[BizMigrationProject]) -> list[Any]: + """Load everything ``project_to_dict`` looks up with a handful of IN queries. + + ``Session.get`` answers from the identity map afterwards, so a list of N projects + costs a constant number of queries instead of ~10 per project. The identity map is + weak: the caller must keep the returned objects alive while it serializes. + """ + keep: list[Any] = [] + def ids(*attrs: str) -> set[str]: + out: set[str] = set() + for p in rows: + for a in attrs: + v = getattr(p, a, None) + if v: + out.add(str(v)) + return out + + task_ids = ids("old_task_id", "new_task_id", "old_hf_task_id", "new_hf_task_id") + for side in ("old", "new"): + for p in rows: + task_ids.update(b["task_id"] for b in _hf_bindings(p, side) if b.get("task_id")) + if task_ids: + keep.append(db.query(BizStateTask).filter(BizStateTask.id.in_(task_ids)).all()) + batch_ids = ids("old_baseline_batch_id", "new_baseline_batch_id") + if batch_ids: + keep.append(db.query(BizStateBatch).filter(BizStateBatch.id.in_(batch_ids)).all()) + template_ids = ids("monitor_template_id") + if template_ids: + monitors = db.query(BizMonitorTemplate).filter(BizMonitorTemplate.id.in_(template_ids)).all() + keep.append(monitors) + compare_ids = {m.compare_template_id for m in monitors if m.compare_template_id} + if compare_ids: + keep.append(db.query(BizCompareTemplate).filter(BizCompareTemplate.id.in_(compare_ids)).all()) + return keep + + def list_projects(db: Session) -> list[dict[str, Any]]: rows = db.query(BizMigrationProject).order_by(BizMigrationProject.created_at.desc()).all() + preloaded = _preload_project_refs(db, rows) # noqa: F841 — keeps the identity map populated return [project_to_dict(db, p) for p in rows] +def _reject_ambiguous_mapping( + port_map: dict[str, str], *, conflicts: list[dict[str, str]] | None = None, + raw: dict[str, str] | None = None, +) -> None: + """Cutover monitoring needs a 1:1 map. Say which rows collide, not just "ambiguous".""" + by_target: dict[str, list[str]] = {} + for before, after in port_map.items(): + by_target.setdefault(after, []).append(before) + n_to_1 = [{"target": t, "sources": sorted(src)} for t, src in sorted(by_target.items()) if len(src) > 1] + collided = bool(conflicts) or (raw is not None and len(port_map) != len(raw)) + if not n_to_1 and not collided: + return + parts = [f"{', '.join(c['sources'])} -> {c['target']}" for c in n_to_1] + parts += [f"{c['dropped_from']} vs {c['kept_from']}" for c in (conflicts or [])] + raise HTTPException(status_code=400, detail={ + "code": "mapping_target_ambiguous", + "message": "mapping_target_ambiguous: " + "; ".join(parts), + "n_to_1": n_to_1, "key_conflicts": list(conflicts or []), + }) + + +def _validate_baseline_ownership(db: Session, side: str, batch_id: str, task_id: str) -> None: + """A baseline batch must be a finished sample of that side's portrait task.""" + bid = str(batch_id or "").strip() + if not bid: + return + sample = db.get(BizStateBatch, bid) + if not sample or sample.task_id != str(task_id or "").strip() or sample.status not in ("success", "partial"): + raise HTTPException(status_code=400, detail=f"{side}_baseline_invalid") + + def create_project(db: Session, body: dict[str, Any]) -> dict[str, Any]: """One-shot cutover project: NE + monitor template + metrics + schedule + baseline + mapping. @@ -604,6 +836,8 @@ def create_project(db: Session, body: dict[str, Any]) -> dict[str, Any]: raise HTTPException(status_code=404, detail="old_baseline_not_found") if new_baseline and not db.get(BizStateBatch, new_baseline): raise HTTPException(status_code=404, detail="new_baseline_not_found") + _validate_baseline_ownership(db, "old", old_baseline, old_task_id) + _validate_baseline_ownership(db, "new", new_baseline, new_task_id) p = BizMigrationProject( id=uuid4().hex, @@ -655,6 +889,11 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st p = db.get(BizMigrationProject, project_id) if not p: raise HTTPException(status_code=404, detail="project_not_found") + # Validate incoming baseline ownership before mutating any project field. + for side in ("old", "new"): + bid = str(body.get(f"{side}_baseline_batch_id", getattr(p, f"{side}_baseline_batch_id")) or "").strip() + tid = str(body.get(f"{side}_task_id", getattr(p, f"{side}_task_id")) or "").strip() + _validate_baseline_ownership(db, side, bid, tid) if "name" in body and body["name"] is not None: p.name = str(body["name"]).strip() or p.name if "note" in body and body["note"] is not None: @@ -670,11 +909,13 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st bid = str(body["old_baseline_batch_id"] or "").strip() if bid and not db.get(BizStateBatch, bid): raise HTTPException(status_code=404, detail="batch_not_found") + BASELINE_ROWS.invalidate(p.old_baseline_batch_id) p.old_baseline_batch_id = bid if "new_baseline_batch_id" in body and body["new_baseline_batch_id"] is not None: bid = str(body["new_baseline_batch_id"] or "").strip() if bid and not db.get(BizStateBatch, bid): raise HTTPException(status_code=404, detail="batch_not_found") + BASELINE_ROWS.invalidate(p.new_baseline_batch_id) p.new_baseline_batch_id = bid if "monitor_template_id" in body and body["monitor_template_id"] is not None: mid = str(body["monitor_template_id"] or "").strip() @@ -714,6 +955,19 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st p.hf_start_at = _parse_dt(body.get("hf_start_at")) if "hf_end_at" in body: p.hf_end_at = _parse_dt(body.get("hf_end_at")) + sampling_changed = any(k in body for k in ("collect_metric_ids", "metric_interval_sec", "hf_interval_sec", "monitor_template_id")) + has_hf = bool(_all_hf_task_ids(p, "old") and _all_hf_task_ids(p, "new")) + if sampling_changed and has_hf: + # Validate newly selected commands before saving the project selection. + for side in ("old", "new"): + portrait = db.get(BizStateTask, portrait_task_id(p, side)) + existing = db.get(BizStateTask, hf_task_id(p, side)) + fields = _template_from_task_or_ne(db, portrait=portrait, existing_hf=existing, ne=None) + for mid in resolve_collect_metric_ids(db, p): + task = db.get(BizStateTask, _hf_task_id_for_metric(p, side, mid)) + if (task and mid in _enabled_metric_ids(db, task.id)) or _portrait_metric_items(db, portrait, mid): + continue + _catalog_item_for_metric(vendor=fields["vendor"], device_type=fields["device_type"], metric_id=mid) p.updated_at = utcnow_naive() db.commit() db.refresh(p) @@ -721,6 +975,9 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st if any(k in body for k in ("status", "hf_start_at", "hf_end_at")): _apply_hf_window_to_tasks(db, p) db.refresh(p) + if sampling_changed and has_hf: + ensure_highfreq(db, p.id, collect_now=False) + db.refresh(p) return project_to_dict(db, p) @@ -748,6 +1005,8 @@ def delete_project(db: Session, project_id: str) -> dict[str, Any]: db.delete(b) db.delete(p) db.commit() + BASELINE_ROWS.invalidate(p.old_baseline_batch_id) + BASELINE_ROWS.invalidate(p.new_baseline_batch_id) return {"ok": True} @@ -768,11 +1027,13 @@ def create_batch(db: Session, project_id: str, body: dict[str, Any]) -> dict[str raise HTTPException(status_code=404, detail="project_not_found") label = str(body.get("batch_label") or "").strip() or "batch" expect = body.get("expect_set") if isinstance(body.get("expect_set"), dict) else {} + status = str(body.get("status") or "pending").strip() + _validate_batch_status(db, project_id, "", status) b = BizMigrationBatch( id=uuid4().hex, project_id=project_id, batch_label=label, - status=str(body.get("status") or "pending").strip() or "pending", + status=status, expect_set_json=dict(expect or {}), note=str(body.get("note") or "")[:500], ) @@ -819,29 +1080,61 @@ def run_evaluate( new_batch_id: str = "", acceptance: bool = False, purpose: str = "manual", + _commit: bool = True, + _metric_ids: set[str] | None = None, + _run_id: str = "", ) -> dict[str, Any]: + """Evaluate a wave. + + Synchronous callers (auto monitor, tests) hold the project row lock for the whole + computation. With ``_run_id`` (background evaluate/finish) the caller already + claimed the project through a ``running`` run row, so the lock is only taken for the + short write at the end. + """ mb = db.get(BizMigrationBatch, batch_id) if not mb: raise HTTPException(status_code=404, detail="batch_not_found") - proj = db.get(BizMigrationProject, mb.project_id) + proj_query = db.query(BizMigrationProject).filter(BizMigrationProject.id == mb.project_id) + if not _run_id: + # PostgreSQL serializes evaluate/finish for this project across API workers. + proj_query = proj_query.with_for_update().populate_existing() + proj = proj_query.first() if not proj: raise HTTPException(status_code=404, detail="project_not_found") + db.refresh(mb) + if acceptance and mb.status not in ("active", "review"): + raise HTTPException(status_code=409, detail="monitor_window_closed") + if purpose == "auto" and (mb.status != "active" or proj.status == "done"): + raise HTTPException(status_code=409, detail="monitor_window_closed") if not proj.old_baseline_batch_id: raise HTTPException(status_code=400, detail="old_baseline_required") if not proj.new_baseline_batch_id: raise HTTPException(status_code=400, detail="new_baseline_required") + for side in ("old", "new"): + baseline = db.get(BizStateBatch, getattr(proj, f"{side}_baseline_batch_id")) + if not baseline or baseline.task_id != portrait_task_id(proj, side): + raise HTTPException(status_code=400, detail=f"{side}_baseline_task_mismatch") + if baseline.status not in ("success", "partial"): + raise HTTPException(status_code=400, detail=f"{side}_baseline_not_complete") + pin = (old_batch_id if side == "old" else new_batch_id).strip() + if pin: + pinned = db.get(BizStateBatch, pin) + if not pinned or pinned.task_id not in _all_hf_task_ids(proj, side): + raise HTTPException(status_code=400, detail=f"{side}_current_batch_task_mismatch") old_batch = db.get(BizStateBatch, old_batch_id.strip()) if old_batch_id.strip() else None new_batch = db.get(BizStateBatch, new_batch_id.strip()) if new_batch_id.strip() else None # Primary current batch ids (for run record / pin UX); per-metric may differ when multi-HF if not old_batch: for tid in _all_hf_task_ids(proj, "old") or [current_task_id(proj, "old")]: - old_batch = _latest_success_batch(db, tid) + old_batch = db.query(BizStateBatch).filter(BizStateBatch.task_id == tid).order_by( + BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() if old_batch: break if not new_batch: for tid in _all_hf_task_ids(proj, "new") or [current_task_id(proj, "new")]: - new_batch = _latest_success_batch(db, tid) + new_batch = db.query(BizStateBatch).filter(BizStateBatch.task_id == tid).order_by( + BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() if new_batch: break if not old_batch: @@ -854,11 +1147,50 @@ def run_evaluate( pin_new = new_batch_id.strip() port_map = _port_map_dict(db, proj.mapping_id) + if proj.mapping_id and not db.get(BizPortMapping, proj.mapping_id): + raise HTTPException(status_code=400, detail="mapping_missing") + _reject_ambiguous_mapping(port_map) expect = parse_expect_set(mb.expect_set_json if isinstance(mb.expect_set_json, dict) else {}) + previous_expect: dict[str, set[str]] = {} + for prior in db.query(BizMigrationBatch).filter( + BizMigrationBatch.project_id == proj.id, BizMigrationBatch.id != mb.id, + BizMigrationBatch.status.in_(("review", "done")), + BizMigrationBatch.created_at <= mb.created_at, + ).all(): + for mid, keys in parse_expect_set(prior.expect_set_json).items(): + previous_expect.setdefault(mid, set()).update(keys) # Final acceptance: window closed → unfinished expect = red window_active = (mb.status == "active") and (not acceptance) - mt = resolve_project_monitor_template(db, proj) + # Evaluations never seed or commit default configuration inside the lock. + mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None + if not mt: + raise HTTPException(status_code=400, detail="monitor_template_missing") + if mt.compare_template_id and not db.get(BizCompareTemplate, mt.compare_template_id): + raise HTTPException(status_code=400, detail="compare_template_missing") sheets, sheet_overrides, defaults, iface_norm = resolve_evaluate_sheets(db, mt) + from .validation import validate_monitor_rules + + validate_monitor_rules(defaults, sheet_overrides) + from ..biz_state.compare_validation import validate_template_body + + validate_template_body({"metrics": sheets, "iface_normalize_rules": iface_norm}) + from ..biz_state.compare_service import _batch_vendor_key + from ..biz_state.identity_pipeline import build_pipeline + from ..biz_state.iface_normalize import normalize_port_map + + raw_port_map = dict(port_map) + old_vendor = _batch_vendor_key(db, proj.old_baseline_batch_id) + new_vendor = _batch_vendor_key(db, proj.new_baseline_batch_id) + normalized_map, map_conflicts = normalize_port_map(port_map, iface_norm) + _reject_ambiguous_mapping(normalized_map, conflicts=map_conflicts, raw=port_map) + port_map = normalized_map + known_sheets = {sheet_key(s) for s in sheets} + known_metrics = {str(s.get("metric_id") or "") for s in sheets} + for override in sheet_overrides: + if (override.get("sheet_id") and override["sheet_id"] not in known_sheets) or ( + not override.get("sheet_id") and override.get("metric_id") not in known_metrics + ): + raise HTTPException(status_code=400, detail="monitor_override_sheet_missing") out_of_expect = str((defaults or {}).get("out_of_expect") or "strict").strip().lower() collect_ids = set(resolve_collect_metric_ids(db, proj)) @@ -866,20 +1198,58 @@ def run_evaluate( all_rows: list[dict[str, Any]] = [] metric_batches: dict[str, dict[str, str]] = {} seq = 0 - verdict_counts: dict[str, int] = {} + rows_cache: dict[tuple[str, str], list[dict[str, Any]]] = {} + command_cache: dict[str, dict[str, Any]] = {} + enabled_sheets = [s for s in sheets if str(s.get("metric_id") or "") in collect_ids and not + override_for_sheet(sheet_overrides, sheet_id=sheet_key(s), metric_id=s.get("metric_id", "")).get("skip_dual")] + scope_ids = {str(s.get("metric_id") or "") for s in enabled_sheets} | {sheet_key(s) for s in enabled_sheets} + if any(s.get("iface_fields") for s in enabled_sheets): + scope_ids.add("_ports") + uncovered_scope = sorted({k for scope in (expect, previous_expect) for k, values in scope.items() + if values and k not in scope_ids}) + config_snapshot = {"identity_version": 2, "vendors": {"old": old_vendor, "new": new_vendor}, + "sheets": sheets, "sheet_overrides": sheet_overrides, + "defaults": defaults, "iface_normalize": iface_norm, "port_map": port_map, + "expect_set": mb.expect_set_json, "previous_expect": {k: sorted(v) for k, v in previous_expect.items()}, + "baseline_ids": [proj.old_baseline_batch_id, proj.new_baseline_batch_id], + "collect_metric_ids": sorted(collect_ids), "window_active": window_active, + "bindings": [_hf_bindings(proj, "old"), _hf_bindings(proj, "new")]} + prior_run = db.query(BizMigrationRun).filter( + BizMigrationRun.batch_id == batch_id, BizMigrationRun.status == "success").order_by( + BizMigrationRun.created_at.desc()).first() if _metric_ids is not None else None + reusable = (prior_run.summary_json or {}).get("config_snapshot") == config_snapshot if prior_run else False + prior_cards = {sheet_key(c): c for c in (prior_run.summary_json or {}).get("sheet_cards", [])} if reusable else {} + pair_cache: dict[str, tuple[BizStateBatch | None, BizStateBatch | None, dict[str, Any]]] = {} - for sheet in sheets: + baseline_ids = {proj.old_baseline_batch_id, proj.new_baseline_batch_id} + + def metric_rows(bid: str, mid: str) -> list[dict[str, Any]]: + key = (bid, mid) + if key not in rows_cache: + if bid in baseline_ids: + # Baselines never change: share them across the minute-by-minute evaluations. + rows_cache[key] = BASELINE_ROWS.get_or_load( + bid, mid, lambda: _load_metric_rows(db, batch_id=bid, metric_id=mid)) + else: + rows_cache[key] = _load_metric_rows(db, batch_id=bid, metric_id=mid) + return rows_cache[key] + + for sheet_index, sheet in enumerate(sheets, start=1): mid = str(sheet.get("metric_id") or "").strip() sid = sheet_key(sheet) key_fields = list(sheet.get("key_fields") or []) if not mid or not key_fields: continue + if _run_id: + _report_eval_progress(_run_id, sheet_index, len(sheets), sheet_title(sheet)) iface_fields = list(sheet.get("iface_fields") or []) compare_fields = list(sheet.get("compare_fields") or []) - row_filters = list(sheet.get("row_filters") or []) field_rules = list(sheet.get("field_rules") or []) sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid) if sheet_ov.get("skip_dual"): + sheet_cards.append({"metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), + "progress_ok": 0, "progress_total": 0, "anomaly": 0, + "collect_skipped": True}) continue # Metric not in HF collect set → skip dual eval (avoid false red) @@ -902,9 +1272,28 @@ def run_evaluate( ) continue - old_cur_batch = _current_batch_for_metric(db, proj, "old", mid, pinned_batch_id=pin_old) - new_cur_batch = _current_batch_for_metric(db, proj, "new", mid, pinned_batch_id=pin_new) + if _metric_ids is not None and mid not in _metric_ids and sid in prior_cards: + cached = {**prior_cards[sid], "source_run_id": prior_cards[sid].get("source_run_id") or prior_run.id} + times = [_parse_dt(cached.get(k)) for k in ("old_collected_at", "new_collected_at")] + if any(at and (utcnow_naive() - at).total_seconds() > cached.get("age_limit_sec", 180) for at in times): + cached.update({"current_missing": True, "sample_issues": ["sample_stale"]}) + sheet_cards.append(cached) + metric_batches[mid] = (prior_run.summary_json or {}).get("metric_batches", {}).get(mid, {}) + continue + if mid not in pair_cache: + pair_cache[mid] = _current_pair_for_metric(db, proj, mid, pin_old, pin_new) + old_cur_batch, new_cur_batch, pair_meta = pair_cache[mid] # Do NOT fall back to another HF task's batch — empty/wrong metric → false red + if (not old_cur_batch or not new_cur_batch) and mid in MANUAL_ROUTE_METRICS: + # Route tables are collected by hand; until someone does, they are not part of the gate. + sheet_cards.append({ + "metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), + "progress_ok": 0, "progress_total": 0, "anomaly": 0, "anomaly_in_expect": 0, + "old_summary": {}, "new_summary": {}, "new_baseline_mode": "manual_pending", + "new_baseline_missing": False, "manual_pending": True, "collect_skipped": False, + "current_missing": False, **pair_meta, + }) + continue if not old_cur_batch or not new_cur_batch: sheet_cards.append( { @@ -922,28 +1311,45 @@ def run_evaluate( "current_missing": True, "collect_incomplete": True, "collect_skipped": False, + **pair_meta, } ) continue old_cur_mid = old_cur_batch.id new_cur_mid = new_cur_batch.id + # A slice (address family) one device never collected is a collection gap, not a lost business. + uncovered = _uncovered_slice_sides(db, sheet, mid, old_cur_mid, new_cur_mid) + if uncovered: + sheet_cards.append({ + "metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), + "progress_ok": 0, "progress_total": 0, "anomaly": 0, "anomaly_in_expect": 0, + "old_summary": {}, "new_summary": {}, "new_baseline_mode": "not_collected", + "new_baseline_missing": False, "not_collected": True, "data_state": "not_collected", + "data_state_reasons": [f"{side}:slice_not_collected" for side in uncovered], + "collect_skipped": False, "current_missing": False, + "old_batch_id": old_cur_mid, "new_batch_id": new_cur_mid, **pair_meta, + }) + continue + timing = {**pair_meta, **_sample_timing(proj, mid, old_cur_batch, new_cur_batch, + check_age=acceptance or not (pin_old and pin_new))} + baseline_missing = [side for side in ("old", "new") if not _metric_sample_ok( + db, db.get(BizStateBatch, getattr(proj, f"{side}_baseline_batch_id")), mid)] + if timing["sample_issues"] or baseline_missing: + sheet_cards.append({"metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), + "progress_ok": 0, "progress_total": 0, "anomaly": 0, + "current_missing": True, "collect_incomplete": True, + "baseline_missing_sides": baseline_missing, **timing}) + continue - old_base = apply_row_filters( - _load_metric_rows(db, batch_id=proj.old_baseline_batch_id, metric_id=mid), - row_filters, - ) - old_now = apply_row_filters( - _load_metric_rows(db, batch_id=old_cur_mid, metric_id=mid), - row_filters, - ) - new_base_rows = apply_row_filters( - _load_metric_rows(db, batch_id=proj.new_baseline_batch_id, metric_id=mid), - row_filters, - ) - new_now = apply_row_filters( - _load_metric_rows(db, batch_id=new_cur_mid, metric_id=mid), - row_filters, + # Old device = "before", new device = "after"; filters see normalized rows. + pipe = build_pipeline( + sheet, iface_rules=iface_norm, port_map=raw_port_map, + before_vendor=old_vendor, after_vendor=new_vendor, ) + old_base = [raw for raw, _ in pipe.prepare(metric_rows(proj.old_baseline_batch_id, mid), "before")] + old_now = [raw for raw, _ in pipe.prepare(metric_rows(old_cur_mid, mid), "before")] + new_base_rows = [raw for raw, _ in pipe.prepare(metric_rows(proj.new_baseline_batch_id, mid), "after")] + new_now = [raw for raw, _ in pipe.prepare(metric_rows(new_cur_mid, mid), "after")] one = evaluate_metric_dual( metric_id=mid, @@ -955,7 +1361,7 @@ def run_evaluate( old_current_rows=old_now, new_baseline_rows=new_base_rows, new_current_rows=new_now, - port_map=port_map if iface_fields else {}, + port_map=pipe.port_map if iface_fields else {}, expect=expect, window_active=window_active, acceptance=acceptance, @@ -963,7 +1369,20 @@ def run_evaluate( sheet_override=sheet_ov, out_of_expect=out_of_expect, iface_normalize_rules=iface_norm, + previous_expect=previous_expect, + allow_mapping_scope=False, + pipeline=pipe, ) + # A target snapshot taken before the source's disappearance cannot prove + # loss. Wait for the next target receipt; final acceptance stays blocked. + awaiting = 0 + if (new_cur_batch.ended_at or new_cur_batch.started_at) < old_cur_batch.started_at: + for row in one["rows"]: + if row["in_expect"] and row["rule_hit"] == "service_not_received": + row.update(verdict="awaiting_peer", color="gray", rule_hit="target_sample_predates_source") + one["anomaly"] -= 1 + one["anomaly_in_expect"] -= 1 + awaiting += 1 old_tid = _hf_task_id_for_metric(proj, "old", mid) or current_task_id(proj, "old") new_tid = _hf_task_id_for_metric(proj, "new", mid) or current_task_id(proj, "new") old_task_brief = _task_brief(db, old_tid) @@ -973,6 +1392,9 @@ def run_evaluate( "new_batch_id": new_cur_mid, "old_task_id": old_tid, "new_task_id": new_tid, + "old_status": old_cur_batch.status, + "new_status": new_cur_batch.status, + **timing, } sheet_cards.append( { @@ -983,6 +1405,7 @@ def run_evaluate( "progress_total": one["progress_total"], "anomaly": one["anomaly"], "anomaly_in_expect": int(one.get("anomaly_in_expect") or 0), + "steady_outside": int(one.get("steady_outside") or 0), "old_summary": one["old_summary"], "new_summary": one["new_summary"], "new_baseline_mode": one.get("new_baseline_mode") or "provided", @@ -993,6 +1416,11 @@ def run_evaluate( "duplicate_key_list": list(one.get("duplicate_key_list") or []), "duplicate_keys_before": int(one.get("duplicate_keys_before") or 0), "duplicate_keys_after": int(one.get("duplicate_keys_after") or 0), + "duplicate_in_expect": list(one.get("duplicate_in_expect") or []), + **timing, + "awaiting_peer": awaiting, + "verdict_counts": dict(Counter(r["verdict"] for r in one["rows"] if r.get("verdict"))), + "expected_count": sum(bool(r.get("in_expect")) for r in one["rows"]), } ) for r in one["rows"]: @@ -1003,20 +1431,25 @@ def run_evaluate( new_batch_id=new_cur_mid, old_task=old_task_brief, new_task=new_task_brief, + command_cache=command_cache, ) r["seq"] = seq r["sheet_id"] = sid seq += 1 all_rows.append(r) - v = str(r.get("verdict") or "") - if v: - verdict_counts[v] = verdict_counts.get(v, 0) + 1 + verdict_counts = {} + for card in sheet_cards: + if not card.get("current_missing"): + for verdict, count in card.get("verdict_counts", {}).items(): + verdict_counts[verdict] = verdict_counts.get(verdict, 0) + count active_cards = [ - c for c in sheet_cards if not c.get("collect_skipped") and not c.get("current_missing") + c for c in sheet_cards + if not c.get("collect_skipped") and not c.get("current_missing") and not c.get("manual_pending") + and not c.get("not_collected") ] run = BizMigrationRun( - id=uuid4().hex, + id=_run_id or uuid4().hex, project_id=proj.id, batch_id=mb.id, old_batch_id=old_cur, @@ -1052,6 +1485,34 @@ def run_evaluate( if c.get("collect_skipped") ], "verdict_counts": verdict_counts, + "anomaly_outside_expect": sum(int(c.get("anomaly") or 0) - int(c.get("anomaly_in_expect") or 0) for c in active_cards), + "steady_outside": sum(int(c.get("steady_outside") or 0) for c in active_cards), + # Only duplicate keys inside the wave's scope fail an acceptance; the rest are warnings. + "duplicate_keys": sum(len(c.get("duplicate_in_expect") or []) for c in active_cards), + "duplicate_warnings": [ + {"sheet_id": c.get("sheet_id"), "title": c.get("title"), + "count": int(c.get("duplicate_keys_before") or 0) + int(c.get("duplicate_keys_after") or 0), + "examples": list(c.get("duplicate_key_list") or [])[:5]} + for c in active_cards + if (c.get("duplicate_keys_before") or c.get("duplicate_keys_after")) + and not c.get("duplicate_in_expect") + ], + "manual_pending_metrics": [ + str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "") + for c in sheet_cards if c.get("manual_pending") + ], + "not_collected_metrics": [ + str(c.get("title") or c.get("sheet_id") or c.get("metric_id") or "") + for c in sheet_cards if c.get("not_collected") + ], + "uncovered_scope": uncovered_scope, + "coverage_complete": bool(enabled_sheets) and not uncovered_scope and not any( + c.get("current_missing") or c.get("awaiting_peer") + or c.get("duplicate_in_expect") + for c in sheet_cards + if not c.get("collect_skipped") and not c.get("manual_pending") and not c.get("not_collected") + ), + "config_snapshot": config_snapshot, "window_active": window_active, "expect_ports": sorted(expect.get("_ports") or ()), "old_current_task_id": current_task_id(proj, "old"), @@ -1059,8 +1520,20 @@ def run_evaluate( }, message="", ) - db.add(run) + if _run_id: + # Claim the project only for the write; the computation above ran unlocked. + db.query(BizMigrationProject).filter(BizMigrationProject.id == proj.id).with_for_update().first() + existing = db.get(BizMigrationRun, _run_id) + if existing is None or existing.status != "running": + raise HTTPException(status_code=409, detail="evaluation_cancelled") + run = db.merge(run) + else: + db.add(run) db.flush() + for card in sheet_cards: + card.setdefault("source_run_id", run.id) + run.summary_json = {**run.summary_json, "sheet_cards": sheet_cards} + pending_diffs: list[dict[str, Any]] = [] for r in all_rows: key_list = r.get("key") or [] ev = r.get("evidence") if isinstance(r.get("evidence"), dict) else {} @@ -1079,8 +1552,8 @@ def run_evaluate( str(r.get("metric_id") or ""), ] ) - db.add( - BizMigrationDiff( + pending_diffs.append( + dict( id=uuid4().hex, run_id=run.id, metric_id=str(r.get("metric_id") or ""), @@ -1098,6 +1571,8 @@ def run_evaluate( "old_status": r.get("old_status"), "new_status": r.get("new_status"), "rule_hit": r.get("rule_hit") or "", + "integrity_changes": r.get("integrity_changes") or [], + "previous_batch": bool(r.get("previous_batch")), "sheet_id": r.get("sheet_id") or "", "evidence": r.get("evidence") or {}, }, @@ -1109,8 +1584,15 @@ def run_evaluate( search_text=search[:2000], ) ) - db.commit() - db.refresh(run) + if len(pending_diffs) >= 500: + db.execute(insert(BizMigrationDiff), pending_diffs) + pending_diffs.clear() + if pending_diffs: + db.execute(insert(BizMigrationDiff), pending_diffs) + db.flush() + if _commit: + db.commit() + db.refresh(run) return run_to_dict(db, run) @@ -1147,7 +1629,12 @@ def _command_brief(db: Session, command_id: str) -> dict[str, Any]: cid = str(command_id or "").strip() if not cid: return {} - c = db.get(BizStateBatchCommand, cid) + # Select metadata only: raw CLI output can be several MB per command. + c = db.query(BizStateBatchCommand.id, BizStateBatchCommand.raw_command, + BizStateBatchCommand.parse_status, BizStateBatchCommand.row_count, + BizStateBatchCommand.profile_id, BizStateBatchCommand.parser_id, + BizStateBatchCommand.metric_id, BizStateBatchCommand.message).filter( + BizStateBatchCommand.id == cid).first() if not c: return {"command_id": cid} return { @@ -1170,6 +1657,7 @@ def _enrich_row_evidence( new_batch_id: str, old_task: dict[str, Any] | None, new_task: dict[str, Any] | None, + command_cache: dict[str, dict[str, Any]] | None = None, ) -> None: """Attach device / collect / show-command onto evaluate evidence (in-place).""" ev = dict(row.get("evidence") or {}) @@ -1179,7 +1667,11 @@ def _enrich_row_evidence( ): side_ev = dict(ev.get(side) or {}) netx = dict(side_ev.get("netx") or {}) - cmd = _command_brief(db, str(netx.get("batch_command_id") or "")) + cid = str(netx.get("batch_command_id") or "") + cache = command_cache if command_cache is not None else {} + if cid not in cache: + cache[cid] = _command_brief(db, cid) + cmd = cache[cid] side_ev["device"] = { "ne_id": str(brief.get("ne_id") or netx.get("ne_id") or ""), "ne_name": str(brief.get("ne_name") or ""), @@ -1221,6 +1713,8 @@ def diff_to_dict(d: BizMigrationDiff) -> dict[str, Any]: "old_status": kj.get("old_status") or "", "new_status": kj.get("new_status") or "", "rule_hit": kj.get("rule_hit") or "", + "integrity_changes": kj.get("integrity_changes") or [], + "previous_batch": bool(kj.get("previous_batch")), "evidence": dict(kj.get("evidence") or {}), "old_kind": d.old_kind, "new_kind": d.new_kind, @@ -1275,8 +1769,17 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] mt = resolve_project_monitor_template(db, p) sheets, sheet_overrides, _defaults, _norm = resolve_evaluate_sheets(db, mt) collect_ids = set(resolve_collect_metric_ids(db, p)) - port_map = _port_map_dict(db, p.mapping_id) + raw_port_map = _port_map_dict(db, p.mapping_id) out_sheets: list[dict[str, Any]] = [] + from ..biz_state.compare_service import _batch_vendor_key + from ..biz_state.identity_pipeline import build_pipeline + from ..biz_state.iface_normalize import normalize_port_map, resolve_mapped_iface + from .evaluate import KEY_SEP + + rows_cache: dict[str, list[dict[str, Any]]] = {} + old_vendor = _batch_vendor_key(db, p.old_baseline_batch_id) + new_vendor = _batch_vendor_key(db, p.new_baseline_batch_id) + port_map, _ = normalize_port_map(raw_port_map, _norm) for sheet in sheets: mid = str(sheet.get("metric_id") or "").strip() sid = sheet_key(sheet) @@ -1286,29 +1789,38 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] # Only offer expect keys for metrics that are actually HF-collected if collect_ids and mid not in collect_ids: continue + if mid in MANUAL_ROUTE_METRICS: + # Never enumerate million-row route tables into batch scope pickers. + continue sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid) if sheet_ov.get("skip_dual"): continue iface_fields = [str(k) for k in (sheet.get("iface_fields") or []) if str(k).strip()] - rows_raw = apply_row_filters( - _load_metric_rows(db, batch_id=p.old_baseline_batch_id, metric_id=mid), - list(sheet.get("row_filters") or []), + if mid not in rows_cache: + rows_cache[mid] = _load_metric_rows(db, batch_id=p.old_baseline_batch_id, metric_id=mid) + pipe = build_pipeline( + sheet, iface_rules=_norm, port_map=raw_port_map, + before_vendor=old_vendor, after_vendor=new_vendor, ) items: list[dict[str, Any]] = [] - for r in rows_raw: - keys = [str(r.get(f) or "").strip() for f in key_fields] + seen_keys: set[str] = set() + for r, normalized in pipe.prepare(rows_cache[mid], "before"): + keys = [scalar_text(normalized.get(f)).strip() for f in key_fields] if not any(keys): continue - key_str = "|".join(keys) + key_str = KEY_SEP.join(keys) + if key_str in seen_keys: + continue + seen_keys.add(key_str) mapped = "" if iface_fields and key_fields and key_fields[0] in iface_fields: - mapped = port_map.get(keys[0]) or "" + mapped = resolve_mapped_iface(keys[0], pipe.port_map) if pipe.port_map else "" items.append( { "key": key_str, "keys": keys, "mapped_to": mapped, - "label": key_str, + "label": "|".join(scalar_text(r.get(f)).strip() for f in key_fields), "row": {f: r.get(f) for f in list(dict.fromkeys([*key_fields, *iface_fields, "description", "admin", "phy", "prot"])) if f in r}, } ) @@ -1331,26 +1843,73 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] } +def preflight_project(db: Session, project_id: str) -> dict[str, Any]: + """Mapping coverage + baseline data quality of a cutover project, before the window. + + Judged on the two portrait baselines (full collections of both devices), so a + missing/ambiguous mapping row shows up before monitoring starts. + """ + from ..biz_state.compare_preflight import preflight + from ..biz_state.compare_service import _batch_vendor_key + from ..biz_state.iface_normalize import normalize_port_map + + p = db.get(BizMigrationProject, project_id) + if not p: + raise HTTPException(status_code=404, detail="project_not_found") + if not p.old_baseline_batch_id or not p.new_baseline_batch_id: + raise HTTPException(status_code=400, detail="baseline_required") + mt = resolve_project_monitor_template(db, p) + sheets, sheet_overrides, _defaults, iface_norm = resolve_evaluate_sheets(db, mt) + collect_ids = set(resolve_collect_metric_ids(db, p)) + scoped = [ + s for s in sheets + if str(s.get("metric_id") or "") and (not collect_ids or str(s.get("metric_id")) in collect_ids) + and not override_for_sheet(sheet_overrides, sheet_id=sheet_key(s), metric_id=str(s.get("metric_id"))).get("skip_dual") + ] + raw_map = _port_map_dict(db, p.mapping_id) + cache: dict[tuple[str, str], list[dict[str, Any]]] = {} + + def load(side: str, sheet: dict[str, Any]) -> list[dict[str, Any]]: + bid = p.old_baseline_batch_id if side == "before" else p.new_baseline_batch_id + key = (bid, str(sheet["metric_id"])) + if key not in cache: + cache[key] = _load_metric_rows(db, batch_id=bid, metric_id=key[1]) + return cache[key] + + report = preflight( + db, sheets=scoped, iface_rules=iface_norm, raw_port_map=raw_map, + before_batch_id=p.old_baseline_batch_id, after_batch_id=p.new_baseline_batch_id, + before_vendor=_batch_vendor_key(db, p.old_baseline_batch_id), + after_vendor=_batch_vendor_key(db, p.new_baseline_batch_id), + load_rows=load, + ) + normalized, _ = normalize_port_map(raw_map, iface_norm) + # Monitoring needs 1:1; the report lists the rows that collide. + if report["mapping"]["n_to_1_targets"] or report["mapping"]["conflicts"]: + report["risk_reasons"] = sorted({*report["risk_reasons"], "mapping_target_ambiguous"}) + report["verdict"] = "risk" + report["mapping"]["pairs_normalized"] = len(normalized) + return report + + def _catalog_item_for_metric(*, vendor: str, device_type: str, metric_id: str) -> dict[str, Any]: from ..biz_state.profiles import profiles_for_vendor from ..lldp_shared import resolve_vendor_key mid = str(metric_id or "").strip() vkey = resolve_vendor_key(vendor, device_type) - for p in profiles_for_vendor(vkey): - if p.metric_id == mid and p.kind == "collect": - if getattr(p, "placeholders", None): - ph = ",".join(str(x.name) for x in (p.placeholders or []) if getattr(x, "name", None)) - raise HTTPException( - status_code=400, - detail=f"metric_needs_bindings:{mid}:{ph or 'required'}", - ) + candidates = [p for p in profiles_for_vendor(vkey) if p.metric_id == mid and p.kind == "collect"] + for p in sorted(candidates, key=lambda p: bool(p.placeholders)): + if not any(getattr(ph, "required", True) for ph in (p.placeholders or [])): return { "source_profile_id": p.profile_id, "kind": "catalog", "enabled": True, "title": p.title or mid, } + if candidates: + ph = ",".join(x.name for x in candidates[0].placeholders if getattr(x, "required", True)) + raise HTTPException(status_code=400, detail=f"metric_needs_bindings:{mid}:{ph}") raise HTTPException( status_code=400, detail=f"no_profile_for_metric:{mid}:{vkey or vendor or 'unknown'}", @@ -1454,6 +2013,116 @@ def _template_from_task_or_ne( raise HTTPException(status_code=400, detail="old_new_ne_or_task_required") +def _portrait_metric_items(db: Session, portrait: BizStateTask | None, mid: str) -> list[dict[str, Any]]: + """Preserve CLI variants and bindings already used by the device baseline.""" + from ..biz_state.profiles import get_profile + from ..models import BizStateTaskItem, BizStateTaskItemBinding + + if not portrait: + return [] + out = [] + for item in db.query(BizStateTaskItem).filter(BizStateTaskItem.task_id == portrait.id, + BizStateTaskItem.enabled.is_(True)).order_by(BizStateTaskItem.sort_order).all(): + profile = get_profile(item.source_profile_id) if item.kind == "catalog" else None + if not profile or profile.metric_id != mid: + continue + bindings = db.query(BizStateTaskItemBinding).filter(BizStateTaskItemBinding.item_id == item.id).all() + out.append({"source_profile_id": item.source_profile_id, "kind": item.kind, + "enabled": True, "title": item.title, "command_override": item.command_override, + "bindings": [{"placeholder": b.placeholder, "value": b.value} for b in bindings]}) + return out + + +def _group_metrics(metric_ids: list[str], by_metric: dict[str, int], default_iv: int) -> list[tuple[int, list[str]]]: + """Collect groups: one task pair (one SSH login per device) per interval + lane. + + Manual route tables stay on their own; heavy dumps never share a task with light + status tables. + """ + from ..biz_state.profiles import metric_lane + + groups: list[tuple[int, list[str]]] = [] + buckets: dict[tuple[int, str], list[str]] = {} + for mid in metric_ids: + iv = max(60, by_metric.get(mid) or default_iv) + if mid in MANUAL_ROUTE_METRICS: + groups.append((iv, [mid])) + continue + key = (iv, metric_lane(mid)) + if key not in buckets: + buckets[key] = [] + groups.append((iv, buckets[key])) + buckets[key].append(mid) + return groups + + +def _group_id(project_id: str, metrics: list[str]) -> str: + return "cutover:" + project_id + ":" + hashlib.sha256("+".join(sorted(metrics)).encode()).hexdigest()[:16] + + +def _slice_key(item: dict[str, Any]) -> tuple[tuple[str, str], ...]: + from ..biz_state.profiles import get_profile + + prof = get_profile(str(item.get("source_profile_id") or "")) + return tuple(sorted((prof.slice or {}).items())) if prof is not None else () + + +def _union_slice_items( + metric_id: str, old_items: list[dict[str, Any]], new_items: list[dict[str, Any]], + old_vendor: str, new_vendor: str, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Both devices must collect the same slices (afi / address family) of a metric. + + A slice one portrait collects and the other does not would otherwise look like + business loss. The missing side gets this vendor's profile for the slice, or a + clear ``no_profile_for_slice`` error. + """ + from ..biz_state.profiles import get_profile, profiles_for_vendor + from ..lldp_shared import resolve_vendor_key + + if not old_items or not new_items: + return old_items, new_items + + def fill(items: list[dict[str, Any]], other: list[dict[str, Any]], vendor: str) -> list[dict[str, Any]]: + have = {_slice_key(i) for i in items} + out = list(items) + vkey = resolve_vendor_key(vendor, "") or vendor + for want in [i for i in other if _slice_key(i) not in have and _slice_key(i)]: + wanted_slice = dict(_slice_key(want)) + profile = next((p for p in profiles_for_vendor(vkey, kind="collect") + if p.metric_id == metric_id and dict(p.slice or {}) == wanted_slice), None) + if profile is None: + label = ",".join(f"{k}={v}" for k, v in sorted(wanted_slice.items())) + raise HTTPException( + status_code=400, detail=f"no_profile_for_slice:{metric_id}:{label}:{vkey or vendor or 'unknown'}") + names = {ph.name for ph in profile.placeholders} + source = get_profile(str(want.get("source_profile_id") or "")) + copied = [b for b in want.get("bindings") or [] if b.get("placeholder") in names] + if any(getattr(ph, "required", True) for ph in profile.placeholders) and not copied: + raise HTTPException( + status_code=400, detail=f"metric_needs_bindings:{metric_id}:" + ",".join(sorted(names))) + out.append({"source_profile_id": profile.profile_id, "kind": "catalog", "enabled": True, + "title": profile.title or (source.title if source else metric_id), "bindings": copied}) + have.add(_slice_key(out[-1])) + return out + + return fill(old_items, new_items, old_vendor), fill(new_items, old_items, new_vendor) + + +def _group_items( + db: Session, portrait: BizStateTask | None, template_fields: dict[str, Any], metrics: list[str], + portrait_items: dict[str, list[dict[str, Any]]], +) -> list[dict[str, Any]]: + """Every metric of a merged task keeps its own portrait-configured commands.""" + out: list[dict[str, Any]] = [] + for mid in metrics: + items = portrait_items.get(mid) or [_catalog_item_for_metric( + vendor=str(template_fields.get("vendor") or ""), + device_type=str(template_fields.get("device_type") or ""), metric_id=mid)] + out.extend(items) + return out + + def _ensure_side_highfreq( db: Session, *, @@ -1464,6 +2133,7 @@ def _ensure_side_highfreq( retention_days: int, metric_ids: list[str], status: str = "running", + configured_items: list[dict[str, Any]] | None = None, ) -> tuple[BizStateTask, bool]: """Return (hf_task, created). Reuse existing HF if metrics match; never mutate portrait.""" from ..biz_state import service as biz_svc @@ -1471,6 +2141,9 @@ def _ensure_side_highfreq( want = {str(m).strip() for m in metric_ids if str(m).strip()} if not want: want = {PORT_METRIC_ID} + if existing_hf and existing_hf.collect_running and _enabled_metric_ids(db, existing_hf.id) != want: + # A running collection owns its item configuration until completion. + existing_hf = None if existing_hf and _is_highfreq_task(db, existing_hf, want): patch: dict[str, Any] = { @@ -1487,7 +2160,7 @@ def _ensure_side_highfreq( # Existing HF (purpose already cutover_hf) with wrong metrics → update items in place. # Never rewrite a task that only "looks like" HF by note — create a sibling instead. if existing_hf and str(getattr(existing_hf, "purpose", None) or "").strip() == PURPOSE_CUTOVER_HF: - items = [ + items = configured_items or [ _catalog_item_for_metric( vendor=str(template_fields.get("vendor") or existing_hf.vendor), device_type=str(template_fields.get("device_type") or existing_hf.device_type), @@ -1511,7 +2184,7 @@ def _ensure_side_highfreq( refreshed = db.get(BizStateTask, existing_hf.id) return refreshed or existing_hf, False - items = [ + items = configured_items or [ _catalog_item_for_metric( vendor=str(template_fields.get("vendor") or ""), device_type=str(template_fields.get("device_type") or ""), @@ -1565,11 +2238,12 @@ def _apply_hf_window_to_tasks(db: Session, proj: BizMigrationProject) -> None: from ..biz_state import service as biz_svc log = logging.getLogger("netx.biz_migration.hf_window") - want = _hf_window_status(proj) + window_status = _hf_window_status(proj) for tid in _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new"): task = db.get(BizStateTask, tid) if not task: continue + want = "paused" if task.collect_manual_only else window_status purpose = str(getattr(task, "purpose", None) or "").strip() if purpose and purpose != PURPOSE_CUTOVER_HF: continue @@ -1617,9 +2291,7 @@ def ensure_highfreq( old_ne: dict[str, Any] | None = None, new_ne: dict[str, Any] | None = None, ) -> dict[str, Any]: - """Create/bind HF tasks (one per interval group) — never overwrites portrait ids.""" - from ..biz_state.collect_runner import dispatch_collect - + """One task per metric/device, paired by project; route tables are manual-only.""" proj = db.get(BizMigrationProject, project_id) if not proj: raise HTTPException(status_code=404, detail="project_not_found") @@ -1633,33 +2305,26 @@ def ensure_highfreq( if not metric_ids: metric_ids = [PORT_METRIC_ID] by_metric = _metric_interval_map(proj) - groups = _group_metrics_by_interval(metric_ids, default_interval=iv_default, by_metric=by_metric) + groups = _group_metrics(metric_ids, by_metric, iv_default) want_status = _hf_window_status(proj) old_portrait = db.get(BizStateTask, proj.old_task_id) if proj.old_task_id else None new_portrait = db.get(BizStateTask, proj.new_task_id) if proj.new_task_id else None - # Reuse existing bindings by interval when possible - def _existing_by_interval(side: str) -> dict[int, BizStateTask]: - out: dict[int, BizStateTask] = {} - for b in _hf_bindings(proj, side): - tid = str(b.get("task_id") or "").strip() - t = db.get(BizStateTask, tid) if tid else None - if not t: - continue - try: - iv = max(60, int(b.get("interval_sec") or t.interval_sec or 60)) - except (TypeError, ValueError): - iv = 60 - out[iv] = t - # legacy primary slot - primary = db.get(BizStateTask, hf_task_id(proj, side)) if hf_task_id(proj, side) else None - if primary and iv_default not in out: - out[iv_default] = primary + # Reuse a single-metric binding; a legacy combined task can be assigned + # once, while the remaining metrics receive independent sibling tasks. + def _existing_by_metric(side: str) -> dict[str, BizStateTask]: + out: dict[str, BizStateTask] = {} + for binding in _hf_bindings(proj, side): + task = db.get(BizStateTask, binding["task_id"]) + if task: + for mid in binding.get("metric_ids") or metric_ids: + if mid in metric_ids: + out.setdefault(mid, task) return out - old_existing = _existing_by_interval("old") - new_existing = _existing_by_interval("new") + old_existing = _existing_by_metric("old") + new_existing = _existing_by_metric("new") old_prev_ids = set(_all_hf_task_ids(proj, "old")) new_prev_ids = set(_all_hf_task_ids(proj, "new")) @@ -1681,35 +2346,64 @@ def ensure_highfreq( old_any_created = False new_any_created = False + # Same physical device on both sides (board swap, same-chassis port move): one task + # per group serves as "old" and "new"; the port map tells the two ports apart. + same_device = (old_fields.get("source"), old_fields.get("ne_id")) == (new_fields.get("source"), new_fields.get("ne_id")) + for g_iv, g_metrics in groups: + manual_only = any(m in MANUAL_ROUTE_METRICS for m in g_metrics) + gid = _group_id(proj.id, g_metrics) + old_candidate = next((old_existing[m] for m in g_metrics if m in old_existing), None) + new_candidate = next((new_existing[m] for m in g_metrics if m in new_existing), None) + if old_candidate and old_candidate.collect_group_id not in ("", gid): + old_candidate = None + if new_candidate and new_candidate.collect_group_id not in ("", gid): + new_candidate = None + old_by_metric = {m: _portrait_metric_items(db, old_portrait, m) for m in g_metrics} + new_by_metric = {m: _portrait_metric_items(db, new_portrait, m) for m in g_metrics} + if not same_device: + for m in g_metrics: + old_by_metric[m], new_by_metric[m] = _union_slice_items( + m, old_by_metric[m], new_by_metric[m], + str(old_fields.get("vendor") or ""), str(new_fields.get("vendor") or "")) old_task, old_created = _ensure_side_highfreq( db, - existing_hf=old_existing.get(g_iv), + existing_hf=old_candidate, template_fields=old_fields, project_name=proj.name, interval_sec=g_iv, retention_days=ret, metric_ids=g_metrics, - status=want_status, - ) - new_task, new_created = _ensure_side_highfreq( - db, - existing_hf=new_existing.get(g_iv), - template_fields=new_fields, - project_name=proj.name, - interval_sec=g_iv, - retention_days=ret, - metric_ids=g_metrics, - status=want_status, + status="paused", + configured_items=_group_items(db, old_portrait, old_fields, g_metrics, old_by_metric), ) + if same_device: + new_task, new_created = old_task, False + else: + new_task, new_created = _ensure_side_highfreq( + db, + existing_hf=new_candidate, + template_fields=new_fields, + project_name=proj.name, + interval_sec=g_iv, + retention_days=ret, + metric_ids=g_metrics, + status="paused", + configured_items=_group_items(db, new_portrait, new_fields, g_metrics, new_by_metric), + ) + if not same_device and old_task.ne_id == new_task.ne_id: + raise HTTPException(status_code=400, detail="collect_pair_same_device") + for task in {old_task.id: old_task, new_task.id: new_task}.values(): + task.collect_group_id = gid + task.collect_manual_only = manual_only + task.status = "paused" if task.collect_manual_only else want_status + db.commit() old_any_created = old_any_created or old_created new_any_created = new_any_created or new_created - old_bindings.append( - {"task_id": old_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv} - ) - new_bindings.append( - {"task_id": new_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv} - ) + binding = {"metric_ids": list(g_metrics), "interval_sec": g_iv, "collect_group_id": gid, + "manual_only": manual_only} + old_bindings.append({"task_id": old_task.id, **binding}) + new_bindings.append({"task_id": new_task.id, **binding}) proj = db.get(BizMigrationProject, project_id) if not proj: @@ -1732,27 +2426,24 @@ def ensure_highfreq( continue if str(getattr(t, "purpose", None) or "").strip() not in ("", PURPOSE_CUTOVER_HF): continue + if t.collect_group_id and not t.collect_group_id.startswith("cutover:" + proj.id + ":"): + continue + t.collect_group_id = "" + t.collect_manual_only = False + db.commit() if t.status == "running": try: biz_svc.update_task(db, tid, {"status": "paused"}) except Exception: # noqa: BLE001 pass + if t.collect_running: + from ..biz_state.collect_stop import request_stop_collect + + request_stop_collect(tid) collect: dict[str, Any] = {"old": [], "new": []} - if collect_now and want_status == "running": - for side, binds in (("old", old_bindings), ("new", new_bindings)): - for b in binds: - tid = b["task_id"] - try: - dispatch_collect(tid) - collect[side].append({"ok": True, "task_id": tid}) - except Exception as exc: # noqa: BLE001 - collect[side].append({"ok": False, "task_id": tid, "error": str(exc)[:200]}) - elif collect_now and want_status != "running": - collect = { - "old": [{"ok": False, "error": "hf_window_inactive"}], - "new": [{"ok": False, "error": "hf_window_inactive"}], - } + if collect_now: + collect = collect_project_now(db, project_id) return { "project": project_to_dict(db, proj), @@ -1795,26 +2486,33 @@ def collect_project_now(db: Session, project_id: str) -> dict[str, Any]: } out: dict[str, Any] = {"old": [], "new": [], "hf_status": want_status} - for side in ("old", "new"): - tids = _all_hf_task_ids(proj, side) - if not tids: - out[side] = [{"ok": False, "error": "hf_task_missing"}] + old_bindings, new_bindings = _hf_bindings(proj, "old"), _hf_bindings(proj, "new") + handled: set[str] = set() + for old_binding in old_bindings: + gid = old_binding.get("collect_group_id") + if not gid or gid in handled: continue - for tid in tids: - task = db.get(BizStateTask, tid) - if not task: - out[side].append({"ok": False, "error": "task_not_found", "task_id": tid}) - continue - if bool(task.collect_running): - out[side].append( - {"ok": False, "error": "collect_already_running", "task_id": tid} - ) - continue - try: - dispatch_collect(tid) - out[side].append({"ok": True, "task_id": tid}) - except Exception as exc: # noqa: BLE001 - out[side].append({"ok": False, "task_id": tid, "error": str(exc)[:200]}) + handled.add(gid) + new_binding = next((b for b in new_bindings if b.get("collect_group_id") == gid), None) + if not new_binding: + result = {"ok": False, "error": "collect_pair_invalid"} + else: + result = dispatch_collect(old_binding["task_id"], manual=True) + result = {**result, "ok": bool(result.get("queued")), "error": result.get("reason", "")} + for side, binding in (("old", old_binding), ("new", new_binding)): + out[side].append({**result, "task_id": binding["task_id"] if binding else "", + "metric_ids": old_binding.get("metric_ids", [])}) + # Legacy projects remain operable until their bindings are upgraded. + if not handled: + for side in ("old", "new"): + tids = _all_hf_task_ids(proj, side) + if not tids: + out[side] = [{"ok": False, "error": "hf_task_missing"}] + for tid in tids: + result = dispatch_collect(tid, manual=True) + out[side].append({**result, "ok": bool(result.get("queued")), "task_id": tid, + "error": result.get("reason", "")}) + return out @@ -1841,6 +2539,7 @@ def _red_ticket_to_dict(t: BizMigrationRedTicket) -> dict[str, Any]: "new_status": t.new_status, "detail": detail, "evidence": dict(detail.get("evidence") or {}), + "resolvable": bool(detail.get("resolvable")), "status": t.status, "carried_to_batch_id": t.carried_to_batch_id, "note": t.note, @@ -1922,26 +2621,30 @@ def _persist_red_tickets_from_run( ) .all() ) - by_id: dict[tuple[str, str], BizMigrationRedTicket] = {} + by_id: dict[tuple[str, str, str], BizMigrationRedTicket] = {} for t in existing: + detail = t.detail_json if isinstance(t.detail_json, dict) else {} mk = str(getattr(t, "match_key_str", "") or "").strip() if not mk: detail = t.detail_json if isinstance(t.detail_json, dict) else {} mk = str(detail.get("match_old_key") or t.key_str or "").strip() if mk: - by_id[(str(t.metric_id or ""), mk)] = t + by_id[(str(t.metric_id or ""), str(detail.get("sheet_id") or t.metric_id or ""), + str(detail.get("match_old_key") or mk))] = t created_or_updated: list[BizMigrationRedTicket] = [] - seen: set[tuple[str, str]] = set() + seen: set[tuple[str, str, str]] = set() for d in diffs: kj = d.key_json if isinstance(d.key_json, dict) else {} mid = str(d.metric_id or "") match_key = str( kj.get("match_old_key") or kj.get("key_str") or kj.get("old_key") or "" - ).strip()[:256] + ).strip() if not match_key: match_key = str(kj.get("key_str") or "")[:256] - ident = (mid, match_key) + ident = (mid, str(kj.get("sheet_id") or mid), match_key) + from hashlib import sha256 + stored_key = match_key if len(match_key) <= 256 else "sha256:" + sha256(match_key.encode()).hexdigest() detail = { "old_kind": d.old_kind, "new_kind": d.new_kind, @@ -1952,6 +2655,8 @@ def _persist_red_tickets_from_run( "new_key": kj.get("new_key") or kj.get("new_key_str") or "", "match_old_key": kj.get("match_old_key") or match_key, "match_new_key": kj.get("match_new_key") or "", + "sheet_id": kj.get("sheet_id") or mid, + "integrity_changes": kj.get("integrity_changes") or [], "evidence": dict(kj.get("evidence") or {}), } prev = by_id.get(ident) @@ -1960,7 +2665,7 @@ def _persist_red_tickets_from_run( prev.run_id = run_id prev.key_str = str(kj.get("key_str") or "")[:256] prev.new_key_str = str(kj.get("new_key_str") or "")[:256] - prev.match_key_str = match_key + prev.match_key_str = stored_key prev.verdict = d.verdict prev.color = d.color or "red" prev.old_status = str(kj.get("old_status") or "")[:64] @@ -1980,7 +2685,7 @@ def _persist_red_tickets_from_run( metric_id=mid, key_str=str(kj.get("key_str") or "")[:256], new_key_str=str(kj.get("new_key_str") or "")[:256], - match_key_str=match_key, + match_key_str=stored_key, verdict=d.verdict, color=d.color or "red", old_status=str(kj.get("old_status") or "")[:64], @@ -1993,16 +2698,51 @@ def _persist_red_tickets_from_run( created_or_updated.append(t) seen.add(ident) - # Tickets open on this batch but no longer red → leave as open (operator resolves). - # Carried tickets for keys not in this run stay carried. + # Tickets whose key is now green stay open (an operator decides) but are flagged so the + # UI can offer a one-click close. Keys absent from this run are left alone. + pending = {ident: t for ident, t in by_id.items() if ident not in seen and t.status in ("open", "carried")} + if pending: + metrics = {ident[0] for ident in pending} + greens: dict[tuple[str, str, str], str] = {} + for g in db.query(BizMigrationDiff).filter( + BizMigrationDiff.run_id == run_id, BizMigrationDiff.color == "green", + BizMigrationDiff.metric_id.in_(metrics), + ): + gk = g.key_json if isinstance(g.key_json, dict) else {} + mk = str(gk.get("match_old_key") or gk.get("key_str") or gk.get("old_key") or "").strip() + greens[(str(g.metric_id or ""), str(gk.get("sheet_id") or g.metric_id or ""), mk)] = str(g.verdict or "") + for ident, t in pending.items(): + verdict = greens.get(ident) + if verdict is not None: + t.detail_json = {**dict(t.detail_json or {}), "resolvable": {"run_id": run_id, "verdict": verdict}} return created_or_updated +def resolve_resolvable_red_tickets(db: Session, project_id: str) -> dict[str, Any]: + """Close every ticket the latest acceptance saw turn green (explicit operator action).""" + if not db.get(BizMigrationProject, project_id): + raise HTTPException(status_code=404, detail="project_not_found") + closed = [] + for t in db.query(BizMigrationRedTicket).filter( + BizMigrationRedTicket.project_id == project_id, + BizMigrationRedTicket.status.in_(("open", "carried")), + ): + mark = (t.detail_json or {}).get("resolvable") if isinstance(t.detail_json, dict) else None + if mark: + t.status = "resolved" + t.resolved_at = utcnow_naive() + t.note = (t.note or f"auto-closed: green in run {mark.get('run_id', '')}")[:500] + closed.append(t.id) + db.commit() + return {"closed": len(closed), "ids": closed, "open_red_count": count_open_red_tickets(db, project_id)} + + def finish_batch( db: Session, batch_id: str, *, mark_done: bool = False, + _run_id: str = "", ) -> dict[str, Any]: """本批完成：关窗 → 终验 → 验收小结 → 红单留痕（不硬卡下一批）.""" mb = db.get(BizMigrationBatch, batch_id) @@ -2014,14 +2754,11 @@ def finish_batch( detail=f"batch_not_finishable:{mb.status}", ) - # Close window first so acceptance uses non-active rules - if mb.status == "active": - mb.ended_at = utcnow_naive() - mb.status = "done" if mark_done else "review" - mb.updated_at = utcnow_naive() - db.commit() - - run = run_evaluate(db, batch_id=batch_id, acceptance=True, purpose="acceptance") + # acceptance=True closes the logical window. Persist batch closure only after + # evaluation succeeds, so an API/collection failure keeps the batch operable. + run = run_evaluate( + db, batch_id=batch_id, acceptance=True, purpose="acceptance", _commit=False, _run_id=_run_id, + ) summary = dict(run.get("summary") or {}) progress = dict(summary.get("progress") or {}) anomaly = int(summary.get("anomaly") or 0) @@ -2036,6 +2773,10 @@ def finish_batch( fail_reasons.append("new_baseline_missing") if current_missing: fail_reasons.append("current_missing") + if not summary.get("coverage_complete", False): + fail_reasons.append("coverage_incomplete") + if summary.get("duplicate_keys"): + fail_reasons.append("duplicate_keys") if anomaly: fail_reasons.append("anomaly") if total > 0 and ok < total: @@ -2052,6 +2793,11 @@ def finish_batch( "anomaly": anomaly, "verdict_counts": dict(summary.get("verdict_counts") or {}), "expect_ports": list(summary.get("expect_ports") or []), + "warnings": { + "duplicate_keys_outside_scope": list(summary.get("duplicate_warnings") or []), + "manual_pending_metrics": list(summary.get("manual_pending_metrics") or []), + "not_collected_metrics": list(summary.get("not_collected_metrics") or []), + }, "run_id": run.get("id"), "old_batch_id": run.get("old_batch_id"), "new_batch_id": run.get("new_batch_id"), @@ -2061,6 +2807,9 @@ def finish_batch( mb = db.get(BizMigrationBatch, batch_id) if not mb: raise HTTPException(status_code=404, detail="batch_not_found") + if mb.status == "active": + mb.ended_at = utcnow_naive() + mb.status = "done" if mark_done else "review" mb.accept_run_id = str(run.get("id") or "") mb.accept_status = "passed" if passed else "failed" mb.accept_summary_json = accept_summary @@ -2086,10 +2835,139 @@ def finish_batch( } +_EVAL_THREADS_LOCK = threading.Lock() + + +def _report_eval_progress(run_id: str, index: int, total: int, title: str) -> None: + """Progress visible to pollers: separate short transaction, never fatal.""" + from ..db import SessionLocal + + try: + s = SessionLocal() + try: + run = s.get(BizMigrationRun, run_id) + if run is not None and run.status == "running": + summary = dict(run.summary_json or {}) + summary["progress"] = {"phase": "evaluating", "sheet_index": index, + "sheet_total": total, "sheet_title": title} + run.summary_json = summary + s.commit() + finally: + s.close() + except Exception: # noqa: BLE001 — progress is best effort + _log.debug("evaluate progress write failed run=%s", run_id, exc_info=True) + + +def _claim_evaluation( + db: Session, batch_id: str, *, acceptance: bool, purpose: str, +) -> BizMigrationRun: + """Create the ``running`` run that owns the project. 409 when one is already in flight.""" + mb = db.get(BizMigrationBatch, batch_id) + if not mb: + raise HTTPException(status_code=404, detail="batch_not_found") + proj = db.query(BizMigrationProject).filter(BizMigrationProject.id == mb.project_id).with_for_update().first() + if not proj: + raise HTTPException(status_code=404, detail="project_not_found") + db.refresh(mb) + if acceptance and mb.status not in ("active", "review"): + raise HTTPException(status_code=409, detail="monitor_window_closed") + if not proj.old_baseline_batch_id: + raise HTTPException(status_code=400, detail="old_baseline_required") + if not proj.new_baseline_batch_id: + raise HTTPException(status_code=400, detail="new_baseline_required") + busy = db.query(BizMigrationRun.id).filter( + BizMigrationRun.project_id == proj.id, BizMigrationRun.status == "running").first() + if busy: + raise HTTPException(status_code=409, detail="evaluation_in_progress") + run = BizMigrationRun( + id=uuid4().hex, project_id=proj.id, batch_id=batch_id, purpose=purpose, status="running", + summary_json={"progress": {"phase": "queued", "sheet_index": 0, "sheet_total": 0}}, + message="queued", + ) + db.add(run) + db.commit() + db.refresh(run) + return run + + +def _spawn_evaluation(run_id: str, work: Any, label: str) -> None: + from ..db import SessionLocal + + def _bg() -> None: + s = SessionLocal() + try: + work(s) + except Exception as exc: # noqa: BLE001 — recorded on the run for the poller + s.rollback() + detail = getattr(exc, "detail", None) + _log.warning("background %s failed run=%s: %s", label, run_id, detail or exc) + run = s.get(BizMigrationRun, run_id) + if run is not None and run.status == "running": + run.status = "failed" + run.message = str(detail if detail is not None else exc)[:1000] + s.commit() + finally: + s.close() + + threading.Thread(target=_bg, name=f"biz-mig-{label}-{run_id[:8]}", daemon=True).start() + + +def enqueue_evaluate( + db: Session, batch_id: str, *, old_batch_id: str = "", new_batch_id: str = "", +) -> dict[str, Any]: + """Start a manual evaluation in the background; poll ``GET /runs/{id}`` for the result.""" + run = _claim_evaluation(db, batch_id, acceptance=False, purpose="manual") + _spawn_evaluation( + run.id, + lambda s: run_evaluate(s, batch_id=batch_id, old_batch_id=old_batch_id, new_batch_id=new_batch_id, + purpose="manual", _run_id=run.id), + "evaluate", + ) + return run_to_dict(db, run) + + +def enqueue_finish(db: Session, batch_id: str, *, mark_done: bool = False) -> dict[str, Any]: + """Final acceptance in the background. The wave closes only if the evaluation succeeds.""" + mb = db.get(BizMigrationBatch, batch_id) + if not mb: + raise HTTPException(status_code=404, detail="batch_not_found") + if mb.status not in ("active", "review"): + raise HTTPException(status_code=400, detail=f"batch_not_finishable:{mb.status}") + run = _claim_evaluation(db, batch_id, acceptance=True, purpose="acceptance") + _spawn_evaluation( + run.id, lambda s: finish_batch(s, batch_id, mark_done=mark_done, _run_id=run.id), "finish", + ) + return {"run_id": run.id, "status": "running", "run": run_to_dict(db, run), + "batch": batch_to_dict(db.get(BizMigrationBatch, batch_id))} + + +def recover_interrupted_evaluations_on_startup(db: Session) -> int: + """A restart kills background evaluations; release the projects they were holding.""" + rows = db.query(BizMigrationRun).filter(BizMigrationRun.status == "running").all() + for r in rows: + r.status = "cancelled" + r.message = "interrupted_by_restart" + if rows: + db.commit() + return len(rows) + + +def _validate_batch_status(db: Session, project_id: str, batch_id: str, status: str) -> None: + if status not in ("pending", "active", "review", "done"): + raise HTTPException(status_code=400, detail="batch_status_invalid") + if status == "active": + db.query(BizMigrationProject).filter(BizMigrationProject.id == project_id).with_for_update().first() + if db.query(BizMigrationBatch.id).filter(BizMigrationBatch.project_id == project_id, + BizMigrationBatch.id != batch_id, BizMigrationBatch.status == "active").first(): + raise HTTPException(status_code=409, detail="another_batch_active") + + def patch_batch(db: Session, batch_id: str, body: dict[str, Any]) -> dict[str, Any]: b = db.get(BizMigrationBatch, batch_id) if not b: raise HTTPException(status_code=404, detail="batch_not_found") + if body.get("status") is not None: + _validate_batch_status(db, b.project_id, b.id, str(body["status"]).strip()) if "batch_label" in body and body["batch_label"] is not None: b.batch_label = str(body["batch_label"]).strip() or b.batch_label if "note" in body and body["note"] is not None: @@ -2135,30 +3013,44 @@ def list_run_diffs( run_id: str, *, metric_id: str = "", + sheet_id: str = "", + only_expect: bool = False, verdict: str = "", color: str = "", kw: str = "", offset: int = 0, limit: int = 100, ) -> dict[str, Any]: - if not db.get(BizMigrationRun, run_id): + run = db.get(BizMigrationRun, run_id) + if not run: raise HTTPException(status_code=404, detail="run_not_found") - q = db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id == run_id) + sources = [and_(BizMigrationDiff.run_id == c["source_run_id"], + BizMigrationDiff.key_json["sheet_id"].as_string() == sheet_key(c)) + for c in (run.summary_json or {}).get("sheet_cards", []) + if c.get("source_run_id") and c["source_run_id"] != run_id + and not (only_expect and c.get("expected_count") == 0) + and (not metric_id or c.get("metric_id") == metric_id) + and (not sheet_id or sheet_key(c) == sheet_id)] + q = db.query(BizMigrationDiff).filter(or_(BizMigrationDiff.run_id == run_id, *sources)) if metric_id: q = q.filter(BizMigrationDiff.metric_id == metric_id) + if sheet_id: + q = q.filter(BizMigrationDiff.key_json["sheet_id"].as_string() == sheet_id) + if only_expect: + q = q.filter(BizMigrationDiff.in_expect.is_(True)) if verdict: q = q.filter(BizMigrationDiff.verdict == verdict) if color: q = q.filter(BizMigrationDiff.color == color) if kw.strip(): - like = f"%{kw.strip()}%" - q = q.filter(BizMigrationDiff.search_text.ilike(like)) + literal = kw.strip().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + q = q.filter(BizMigrationDiff.search_text.ilike(f"%{literal}%", escape="\\")) total = q.count() - rows = q.order_by(BizMigrationDiff.seq.asc()).offset(max(0, offset)).limit(min(500, max(1, limit))).all() + rows = q.order_by(BizMigrationDiff.metric_id.asc(), BizMigrationDiff.seq.asc(), BizMigrationDiff.id.asc()).offset(max(0, offset)).limit(min(500, max(1, limit))).all() return { "total": total, - "offset": offset, - "limit": limit, + "offset": max(0, offset), + "limit": min(500, max(1, limit)), "items": [diff_to_dict(d) for d in rows], } @@ -2186,9 +3078,10 @@ def board(db: Session, batch_id: str, run_id: str = "") -> dict[str, Any]: if not run or run.batch_id != batch_id: raise HTTPException(status_code=404, detail="run_not_found") else: + # Latest *finished* run; a run still being computed has nothing to show yet. run = ( db.query(BizMigrationRun) - .filter(BizMigrationRun.batch_id == batch_id) + .filter(BizMigrationRun.batch_id == batch_id, BizMigrationRun.status == "success") .order_by(BizMigrationRun.created_at.desc()) .first() ) @@ -2196,9 +3089,35 @@ def board(db: Session, batch_id: str, run_id: str = "") -> dict[str, Any]: "project": proj, "batch": batch_to_dict(mb), "run": run_to_dict(db, run) if run else None, + "sampling": sampling_state(db, db.get(BizMigrationProject, mb.project_id)) if not run_id else [], } +def sampling_state(db: Session, proj: BizMigrationProject) -> list[dict[str, Any]]: + """Small live read independent of evaluation and route row counts.""" + out = [] + for binding in _hf_bindings(proj, "old"): + gid = binding.get("collect_group_id") + if not gid: + continue + latest = db.query(BizStateBatch).filter(BizStateBatch.collect_group_id == gid).order_by( + BizStateBatch.queued_at.desc(), BizStateBatch.started_at.desc()).first() + peers = db.query(BizStateBatch).filter(BizStateBatch.collect_round_id == latest.collect_round_id).all() if latest else [] + old = next((b for b in peers if b.task_id == binding["task_id"]), None) + new = next((b for b in peers if b.task_id != binding["task_id"]), None) + card: dict[str, Any] = {"metric_id": (binding.get("metric_ids") or [""])[0], + "collect_group_id": gid, "collect_round_id": latest.collect_round_id if latest else "", + "manual_only": binding.get("manual_only", False), "interval_sec": binding["interval_sec"]} + for side, batch in (("old", old), ("new", new)): + card[f"{side}_collect_status"] = batch.status if batch else "idle" + card[f"{side}_duration_sec"] = max(0, ((batch.ended_at or utcnow_naive()) - batch.started_at).total_seconds()) if batch and batch.status != "queued" else 0 + card[f"{side}_queue_sec"] = max(0, ((utcnow_naive() if batch.status == "queued" else batch.started_at) - (batch.queued_at or batch.started_at)).total_seconds()) if batch else 0 + if old and new and all(b.status != "queued" for b in (old, new)): + card["start_skew_sec"] = abs((old.started_at - new.started_at).total_seconds()) + out.append(card) + return out + + def get_monitor_context( db: Session, *, diff --git a/netx_api/biz_migration/validation.py b/netx_api/biz_migration/validation.py new file mode 100644 index 0000000..998761a --- /dev/null +++ b/netx_api/biz_migration/validation.py @@ -0,0 +1,72 @@ +"""Validate monitor correction rules before they can affect live verdicts.""" +from typing import Any + +from fastapi import HTTPException + +from .evaluate import _groups_for_side, _normalize_cond_type + + +def validate_monitor_rules(defaults: Any, overrides: Any) -> None: + def fail(path: str) -> None: + raise HTTPException(status_code=400, detail=f"monitor_rule_invalid:{path}") + + if not isinstance(defaults, dict) or not isinstance(overrides, list): + fail("defaults/sheet_overrides") + if defaults.get("out_of_expect", "strict") not in ("strict", "warn", "ignore"): + fail("defaults.out_of_expect") + seen = set() + for i, ov in enumerate(overrides): + path = f"sheet_overrides[{i}]" + if not isinstance(ov, dict) or not (ov.get("sheet_id") or ov.get("metric_id")): + fail(path) + ident = ov.get("sheet_id") or ov.get("metric_id") + if ident in seen: + fail(path + ".duplicate") + seen.add(ident) + if ov.get("scope_by_port_map") is not None and not isinstance(ov["scope_by_port_map"], bool): + fail(path + ".scope_by_port_map") + for key in ("status_fields", "up_values", "down_values"): + if key in ov and (not isinstance(ov[key], list) or any(not str(v).strip() for v in ov[key])): + fail(path + "." + key) + for key in ("success", "anomaly"): + patterns = ov.get(key, []) + if not isinstance(patterns, list): + fail(path + "." + key) + for j, pat in enumerate(patterns): + pp = f"{path}.{key}[{j}]" + if not isinstance(pat, dict): + fail(pp) + for side in ("old", "new"): + for attr in (side, side + "_conds", side + "_groups"): + if attr in pat and not isinstance(pat[attr], list): + fail(pp + "." + attr) + if any(isinstance(g, list) and not g for g in pat.get(side + "_groups", [])): + fail(pp + "." + side + "_groups.empty_group") + groups = _groups_for_side(pat, side) + if key == "success" and not groups: + fail(pp + "." + side + ".required") + for group in groups: + for cond in group: + cp = pp + "." + side + if isinstance(cond, str): + if cond not in ("added", "removed", "changed", "unchanged", "up", "down", "other", "none") and not (cond.startswith("field:") and cond.count(":") >= 2): + fail(cp + ".token") + elif isinstance(cond, dict): + typ = _normalize_cond_type(cond) + if typ not in ("presence", "status", "value"): + fail(cp + ".type") + val = cond.get("value", cond.get("kind", cond.get("status"))) + if typ == "presence" and val not in ("added", "removed", "changed", "unchanged"): + fail(cp + ".presence") + if typ == "status" and val not in ("up", "down", "other", "none"): + fail(cp + ".status") + if typ == "value": + op = cond.get("op", "eq") + if op not in ("eq", "ne", "in", "not_in", "empty", "not_empty", "changed", "unchanged", "same", "diff"): + fail(cp + ".op") + if op in ("eq", "ne", "in", "not_in") and (val is None or val == "" or val == []): + fail(cp + ".value") + else: + fail(cp + ".condition") + if key == "anomaly" and not any(_groups_for_side(pat, s) for s in ("old", "new")): + fail(pp + ".empty") diff --git a/netx_api/biz_migration_router.py b/netx_api/biz_migration_router.py index 29b4377..8104a4c 100644 --- a/netx_api/biz_migration_router.py +++ b/netx_api/biz_migration_router.py @@ -151,6 +151,12 @@ def api_baseline_ports(project_id: str, db: Session = Depends(get_db)): return svc.list_baseline_ports(db, project_id) +@router.get("/projects/{project_id}/preflight") +@router.post("/projects/{project_id}/preflight") +def api_project_preflight(project_id: str, db: Session = Depends(get_db)): + return svc.preflight_project(db, project_id) + + @router.get("/projects/{project_id}/baseline-expect") def api_baseline_expect(project_id: str, db: Session = Depends(get_db)): """Multi-metric baseline keys for expect-set picking (from monitor template sheets).""" @@ -222,8 +228,13 @@ def api_finish_batch( mark_done: bool = False, db: Session = Depends(get_db), ): - """本批完成：终验 + 小结 + 红单（允许带红继续）.""" - return svc.finish_batch(db, batch_id, mark_done=mark_done) + """本批完成：终验 + 小结 + 红单（允许带红继续）. Runs in the background; poll the run.""" + return svc.enqueue_finish(db, batch_id, mark_done=mark_done) + + +@router.post("/projects/{project_id}/red-tickets/resolve-resolvable") +def api_resolve_resolvable(project_id: str, db: Session = Depends(get_db)): + return svc.resolve_resolvable_red_tickets(db, project_id) @router.get("/projects/{project_id}/red-tickets") @@ -258,9 +269,10 @@ def api_patch_batch(batch_id: str, body: BatchPatchIn, db: Session = Depends(get @router.post("/batches/{batch_id}/evaluate") def api_evaluate(batch_id: str, body: EvaluateIn | None = None, db: Session = Depends(get_db)): payload = body.model_dump() if body else {} - return svc.run_evaluate( + # Returns the ``running`` run at once; poll GET /runs/{id} until ``status`` settles. + return svc.enqueue_evaluate( db, - batch_id=batch_id, + batch_id, old_batch_id=str(payload.get("old_batch_id") or ""), new_batch_id=str(payload.get("new_batch_id") or ""), ) @@ -285,6 +297,8 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)): def api_list_diffs( run_id: str, metric_id: str = "", + sheet_id: str = "", + only_expect: bool = False, verdict: str = "", color: str = "", kw: str = "", @@ -296,6 +310,8 @@ def api_list_diffs( db, run_id, metric_id=metric_id, + sheet_id=sheet_id, + only_expect=only_expect, verdict=verdict, color=color, kw=kw, diff --git a/netx_api/biz_state/claim.py b/netx_api/biz_state/claim.py index a86674e..d45b6ec 100644 --- a/netx_api/biz_state/claim.py +++ b/netx_api/biz_state/claim.py @@ -2,8 +2,10 @@ from __future__ import annotations +from collections import defaultdict + import logging -from datetime import datetime +from datetime import datetime, timedelta from typing import Any from uuid import uuid4 @@ -25,6 +27,11 @@ def max_concurrent_tasks() -> int: return max(1, int(getattr(settings, "biz_state_max_concurrent_tasks", 16) or 16)) +def max_sessions_per_ne() -> int: + """Concurrent SSH collections allowed per network element (>= 1).""" + return max(1, int(getattr(settings, "biz_state_max_sessions_per_ne", 2) or 2)) + + def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: """Create a queued batch and mark task collect_running. Does not open SSH. @@ -39,6 +46,10 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: task = db.get(BizStateTask, tid) if not task: return {"ok": False, "queued": False, "reason": "task_not_found", "task_id": tid} + if task.collect_group_id: + group_id = task.collect_group_id + db.close() + return enqueue_collect_group(group_id, manual=manual, requested_task_id=tid) if str(task.source or "").strip().lower() == "import": return { "ok": False, @@ -99,6 +110,7 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: vendor=task.vendor, status="queued", started_at=now, + queued_at=now, message="queued" if not manual else "queued_manual", ) db.add(batch) @@ -121,6 +133,62 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: db.close() +def enqueue_collect_group(group_id: str, *, manual: bool = False, + requested_task_id: str = "") -> dict[str, Any]: + """Atomically enqueue both devices with one round identity, or neither.""" + db = SessionLocal() + try: + tasks = db.query(BizStateTask).filter(BizStateTask.collect_group_id == group_id).order_by( + BizStateTask.id).with_for_update().populate_existing().all() + result: dict[str, Any] = {"ok": False, "queued": False, "task_id": requested_task_id, + "collect_group_id": group_id} + # One task = a single device used as both sides (same-device cutover); two tasks = a pair. + if len(tasks) not in (1, 2) or len({t.ne_id for t in tasks}) != len(tasks): + return {**result, "reason": "collect_pair_invalid"} + if max_concurrent_tasks() < len(tasks): + return {**result, "reason": "collect_pair_capacity"} + from ..cli_budget import clamp_cli_workers + + worker_cap = clamp_cli_workers(int(getattr(settings, "biz_state_worker_collect_threads", None) + or getattr(settings, "biz_state_dispatch_workers", 8) or 8)) + if worker_cap < len(tasks): + return {**result, "reason": "collect_pair_capacity"} + if any(t.collect_running for t in tasks): + return {**result, "reason": "collect_pair_busy"} + for task in tasks: + if task.source == "import" or (not manual and (task.status != "running" or task.collect_manual_only)): + return {**result, "reason": "not_scheduled"} + if task.status in ("", "deleted"): + return {**result, "reason": "bad_status"} + if not db.query(BizStateTaskItem.id).filter(BizStateTaskItem.task_id == task.id, + BizStateTaskItem.enabled.is_(True)).first(): + return {**result, "reason": "no_enabled_items"} + now, round_id = _utcnow(), uuid4().hex + batches = [] + for task in tasks: + task.collect_running = True + task.collect_queued_at = now + task.last_error = "" + task.updated_at = now + batch = BizStateBatch(id=uuid4().hex, task_id=task.id, source=task.source, + ne_id=task.ne_id, ne_name=task.ne_name, vendor=task.vendor, status="queued", + started_at=now, queued_at=now, collect_group_id=group_id, + collect_round_id=round_id, collect_priority=10 if task.collect_manual_only else 20, + message="queued_manual" if manual else "queued") + db.add(batch) + batches.append({"batch_id": batch.id, "task_id": task.id}) + db.commit() + selected = next((b for b in batches if b["task_id"] == requested_task_id), batches[0]) + return {**result, **selected, "ok": True, "queued": True, "collect_round_id": round_id, + "batches": batches, "manual": manual} + except Exception: + db.rollback() + _log.exception("enqueue collect pair failed group=%s", group_id) + return {"ok": False, "queued": False, "reason": "enqueue_error", "task_id": requested_task_id} + finally: + db.close() + + def _dialect_supports_skip_locked(db: Session) -> bool: try: bind = db.get_bind() @@ -151,11 +219,16 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: Global ceiling is always ``max_concurrent_tasks()``; ``limit`` only caps how many this caller wants in one call. """ - from sqlalchemy import text + from sqlalchemy import and_, case, text global_cap = max_concurrent_tasks() db = SessionLocal() try: + pg = _dialect_supports_skip_locked(db) + if pg: + # A short transaction lock makes capacity reservation and pair claims + # atomic across worker processes. Never held during device I/O. + db.execute(text("SELECT pg_advisory_xact_lock(hashtext('biz-state-claim-capacity'))")) running_n = ( db.query(BizStateBatch).filter(BizStateBatch.status == "running").count() ) @@ -165,58 +238,67 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: if slots <= 0: return [] - busy_nes = { - str(r[0] or "").strip() - for r in db.query(BizStateBatch.ne_id) - .filter(BizStateBatch.status == "running") - .all() - if str(r[0] or "").strip() - } + # Running collections per NE; an NE accepts new ones until it reaches its session cap. + per_ne_cap = max_sessions_per_ne() + ne_running: dict[str, int] = defaultdict(int) + for (running_ne,) in db.query(BizStateBatch.ne_id).filter(BizStateBatch.status == "running").all(): + if str(running_ne or "").strip(): + ne_running[str(running_ne).strip()] += 1 + # Manual route rounds yield to fresh monitoring, but must eventually + # run even when a busy device always has regular metrics queued. + priority = case((and_(BizStateBatch.collect_priority == 10, + BizStateBatch.queued_at <= _utcnow() - timedelta(seconds=300)), 30), + else_=BizStateBatch.collect_priority) q = ( db.query(BizStateBatch) .filter(BizStateBatch.status == "queued") - .order_by(BizStateBatch.started_at.asc()) + .order_by(priority.desc(), BizStateBatch.started_at.asc(), BizStateBatch.id.asc()) ) - pg = _dialect_supports_skip_locked(db) if pg: q = q.with_for_update(skip_locked=True) else: q = q.with_for_update() - candidates = q.limit(max(slots * 4, slots)).all() + candidates = q.limit(max(slots * 8, 64)).all() claimed_rows: list[BizStateBatch] = [] + visited: set[str] = set() for batch in candidates: + if batch.id in visited: + continue if len(claimed_rows) >= slots: break - ne = str(batch.ne_id or "").strip() - if ne and ne in busy_nes: - continue - # Serialize claims per NE across workers (Postgres). - if ne and pg: - try: - db.execute( - text("SELECT pg_advisory_xact_lock(hashtext(:ne))"), - {"ne": ne}, - ) - except Exception: - _log.exception("advisory lock failed ne=%s", ne) - conflict = ( - db.query(BizStateBatch.id) - .filter( - BizStateBatch.status == "running", - BizStateBatch.ne_id == ne, - BizStateBatch.id != batch.id, - ) - .first() - ) - if conflict: + group = [batch] + if batch.collect_round_id: + peers = db.query(BizStateBatch).filter( + BizStateBatch.collect_round_id == batch.collect_round_id, + ).with_for_update(skip_locked=pg).all() + expected = db.query(BizStateTask.id).filter( + BizStateTask.collect_group_id == batch.collect_group_id).count() if batch.collect_group_id else 2 + if len(peers) != max(1, expected) or any(p.status != "queued" for p in peers): continue - batch.status = "running" - batch.message = "" - claimed_rows.append(batch) - if ne: - busy_nes.add(ne) + group = peers + visited.update(p.id for p in group) + if len(group) > slots - len(claimed_rows): + continue + wanted: dict[str, int] = defaultdict(int) + for p in group: + if str(p.ne_id or "").strip(): + wanted[str(p.ne_id).strip()] += 1 + if any(ne_running[ne] + n > per_ne_cap for ne, n in wanted.items()): + continue + now = _utcnow() + for member in group: + member.status = "running" + member.message = "" + if member.collect_round_id: + member.started_at = now + task = db.get(BizStateTask, member.task_id) + if task: + task.last_collect_started_at = now + claimed_rows.append(member) + for ne, n in wanted.items(): + ne_running[ne] += n if not claimed_rows: db.rollback() @@ -231,10 +313,13 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: db.query(BizStateBatch).filter(BizStateBatch.status == "running").count() ) while n_running > global_cap and claimed_rows: - demote = claimed_rows.pop() - demote.status = "queued" - demote.message = "queued" - n_running -= 1 + last = claimed_rows[-1] + demoted = [b for b in claimed_rows if b.collect_round_id == last.collect_round_id] if last.collect_round_id else [last] + for demote in demoted: + claimed_rows.remove(demote) + demote.status = "queued" + demote.message = "queued" + n_running -= 1 if not claimed_rows: db.rollback() diff --git a/netx_api/biz_state/collect_runner.py b/netx_api/biz_state/collect_runner.py index ad04d77..91797ce 100644 --- a/netx_api/biz_state/collect_runner.py +++ b/netx_api/biz_state/collect_runner.py @@ -784,6 +784,12 @@ def dispatch_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: result = enqueue_collect(task_id, manual=manual) if not result.get("queued"): return result + if result.get("collect_group_id"): + if _should_execute_inline(): + from ..biz_state_scheduler import _claim_and_dispatch + + _claim_and_dispatch() + return result batch_id = str(result.get("batch_id") or "") if not batch_id: return result @@ -870,6 +876,11 @@ def execute_claimed_batch( task = db.get(BizStateTask, task_id) if task_id else None if not batch: return + if batch.collect_round_id: + batch.started_at = _utcnow() + if task: + task.last_collect_started_at = batch.started_at + db.commit() if not task_id: task_id = str(batch.task_id or "") task = db.get(BizStateTask, task_id) if task_id else None @@ -1720,6 +1731,14 @@ def _finalize_batch_status( batch.message = "" status = str(batch.status or "") db.commit() + # Failed/partial receipts also update cutover coverage; never keep an old + # success looking healthy after the latest collection has failed. + try: + from ..biz_migration.auto_monitor import schedule_auto_monitor_for_task + + schedule_auto_monitor_for_task(task_id) + except Exception: + _log.exception("cutover monitor schedule failed task=%s", task_id) # Only full success triggers auto compare; never block the collect thread. if status == "success" and not stopped: try: diff --git a/netx_api/biz_state/collect_stop.py b/netx_api/biz_state/collect_stop.py index 8c92db0..1f27e31 100644 --- a/netx_api/biz_state/collect_stop.py +++ b/netx_api/biz_state/collect_stop.py @@ -131,7 +131,7 @@ def _force_close_holders(batch_id: str) -> int: def request_stop_collect(task_id: str) -> dict[str, Any]: - """Cancel queued batches and signal running collect for this task to abort.""" + """Stop a task, including its paired collector, without stranding a half round.""" tid = str(task_id or "").strip() if not tid: return {"ok": False, "reason": "missing_task_id"} @@ -141,26 +141,37 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: signaled_ids: list[str] = [] closed = 0 try: + from sqlalchemy import text + from .claim import _dialect_supports_skip_locked + + if _dialect_supports_skip_locked(db): + db.execute(text("SELECT pg_advisory_xact_lock(hashtext('biz-state-claim-capacity'))")) task = db.get(BizStateTask, tid) if not task: return {"ok": False, "reason": "task_not_found", "task_id": tid} + task_ids = [tid] + if task.collect_group_id: + task_ids = [r[0] for r in db.query(BizStateTask.id).filter( + BizStateTask.collect_group_id == task.collect_group_id).all()] active = ( db.query(BizStateBatch) .filter( - BizStateBatch.task_id == tid, + BizStateBatch.task_id.in_(task_ids), BizStateBatch.status.in_(("queued", "running")), ) - .all() + .order_by(BizStateBatch.id).with_for_update().all() ) + tasks = db.query(BizStateTask).filter(BizStateTask.id.in_(task_ids)).order_by( + BizStateTask.id).with_for_update().populate_existing().all() if not active: # Nothing to stop — clear sticky collect_running if orphaned. - if bool(task.collect_running): - task.collect_running = False - if hasattr(task, "collect_queued_at"): - task.collect_queued_at = None - task.updated_at = utcnow_naive() - db.commit() + for member in tasks: + if member.collect_running: + member.collect_running = False + member.collect_queued_at = None + member.updated_at = utcnow_naive() + db.commit() return { "ok": True, "task_id": tid, @@ -171,7 +182,7 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: } now = utcnow_naive() - still_running = False + running_tasks: set[str] = set() for batch in active: bid = str(batch.id) mark_stop_requested(bid) @@ -183,16 +194,16 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: else: batch.message = STOP_REQUEST_TOKEN signaled_ids.append(bid) - still_running = True + running_tasks.add(batch.task_id) closed += _force_close_holders(bid) - if not still_running: - task.collect_running = False - if hasattr(task, "collect_queued_at"): - task.collect_queued_at = None - task.last_collect_ended_at = now - task.last_error = STOP_USER_MESSAGE - task.updated_at = now + for member in tasks: + if member.id not in running_tasks: + member.collect_running = False + member.collect_queued_at = None + member.last_collect_ended_at = now + member.last_error = STOP_USER_MESSAGE + member.updated_at = now db.commit() _log.info( diff --git a/netx_api/biz_state/compare_engine.py b/netx_api/biz_state/compare_engine.py index b11cae8..27953ff 100644 --- a/netx_api/biz_state/compare_engine.py +++ b/netx_api/biz_state/compare_engine.py @@ -2,15 +2,81 @@ from __future__ import annotations -from typing import Any, Mapping, Sequence +from collections import defaultdict +from heapq import heappush, heapreplace +from itertools import chain +from typing import Any, Iterable, Mapping, Sequence -from .compare_rules import field_rule_map, values_equal, explain_diff +from .compare_rules import field_rule_map, values_equal, explain_diff, scalar_text from .iface_normalize import ( apply_iface_normalize_rows, normalize_iface_rules, resolve_mapped_iface, ) +# Prefer these fields when stratifying success samples (BGP multipath / multi-cmd). +_STRATUM_FIELDS = ("neighbor", "direction", "afi", "vrf") + + +def stratum_key(row: Mapping[str, Any] | None) -> str: + """Bucket key for stratified unchanged sampling.""" + if not isinstance(row, Mapping): + return "_" + parts: list[str] = [] + for f in _STRATUM_FIELDS: + v = str(row.get(f) or "").strip() + if v: + parts.append(f"{f}={v}") + return "|".join(parts) if parts else "_" + + +class _StratifiedSample: + """Keep the first N round-robin ranks in O(N) space, even with many strata.""" + + def __init__(self, limit: int) -> None: + self.limit = max(0, int(limit)) + self.count = 0 + self.strata: dict[str, tuple[int, int]] = {} + self.heap: list[tuple[int, int, int, Any]] = [] + + def add(self, item: Any, key: str) -> None: + serial = self.count + self.count += 1 + if not self.limit: + return + state = self.strata.get(key) + if state is None: + # Later strata cannot beat the first item of N earlier strata. + if len(self.strata) >= self.limit: + return + index, round_n = len(self.strata), 0 + else: + index, round_n = state + self.strata[key] = (index, round_n + 1) + entry = (-round_n, -index, serial, item) + if len(self.heap) < self.limit: + heappush(self.heap, entry) + elif entry[:2] > self.heap[0][:2]: + heapreplace(self.heap, entry) + + def picked(self) -> list[Any]: + # The legacy helper preserves encounter order when no truncation occurs. + if self.count <= self.limit: + entries = sorted(self.heap, key=lambda e: e[2]) + else: + entries = sorted(self.heap, key=lambda e: (-e[0], -e[1])) + return [entry[3] for entry in entries] + + +def stratify_take(items: Iterable[Any], limit: int, *, key_fn) -> list[Any]: + """Round-robin across strata without retaining every candidate.""" + sample = _StratifiedSample(limit) + if not sample.limit: + return [] + for item in items: + sample.add(item, str(key_fn(item) or "_")) + return sample.picked() + def apply_port_map( row: dict[str, Any], @@ -27,8 +93,17 @@ def apply_port_map( return out +def key_tuple(row: Mapping[str, Any], fields: Sequence[str]) -> tuple[str, ...]: + """The one canonical match-key form: ``scalar_text(...).strip()`` per field. + + Every place that builds, stores or looks up a key string must go through + this (``None`` → ``""``, ``True`` → ``"true"``, surrounding blanks trimmed). + """ + return tuple(scalar_text(row.get(k)).strip() for k in fields) + + def row_key(row: dict[str, Any], key_fields: list[str]) -> tuple[str, ...]: - return tuple(str(row.get(k) or "").strip() for k in key_fields) + return key_tuple(row, key_fields) def mapping_stats( @@ -43,8 +118,8 @@ def mapping_stats( ``hit_before``: map key appears as a normalized before iface, or as the parent of a before subinterface (so main-port-only maps still validate). - Map keys are matched against **normalized** iface values (same pipeline as - compare). Enter mapping keys in normalized form. + Map keys are matched against **normalized** iface values; callers pass a + port map already normalized with the template iface rules. """ before_ifaces: set[str] = set() after_ifaces: set[str] = set() @@ -90,7 +165,7 @@ def mapping_stats( "miss_after": miss_after, "unused_before_keys": unused, "ok": not miss_before and not miss_after, - "hint": "map keys must match normalized iface names (post iface_normalize)", + "hint": "map keys/values are normalized with the template iface rules before matching", } @@ -105,12 +180,32 @@ def compare_rows( field_rules: Sequence[Mapping[str, Any]] | None = None, iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, ignore_port_changes: bool | None = None, + include_unchanged: bool = False, + unchanged_limit: int | None = None, + compact_unchanged: bool = False, + pipeline: Any = None, + match_keys_override: Sequence[str] | None = None, + prepared: tuple[list, list] | None = None, + duplicate_pairing: str = "ordered", ) -> dict[str, Any]: """Return summary + diffs list. - Diff kinds: added | removed | changed | unchanged | duplicate + Diff kinds: added | removed | changed | unchanged - Pipeline: iface normalize (both sides) → port map (before) → match. + Pipeline: iface normalize (both sides) → port map (before) → ordered + same-key pairing (load / list order within each match key). + + Same match key with N before and M after rows: zip by index + ``0..min(N,M)-1`` for field compare; extras become ``removed`` (before) + or ``added`` (after). Example: 5 vs 2 → 2 compared + 3 removed. + + ``include_unchanged``: when False, matching rows still increment + ``summary.unchanged`` but are omitted from ``diffs``. + + ``unchanged_limit``: max unchanged diffs to emit (None = no cap). Use with + large sheets so the UI can browse a sample without writing millions of rows. + + ``compact_unchanged``: emit key only (empty before/after) to cut storage. ``ignore_port_changes``: - ``None`` (default): auto — drop iface from match key only when remaining @@ -118,175 +213,219 @@ def compare_rows( - ``True``: force drop iface from match key when a non-empty candidate exists. - ``False``: never drop iface from match key. - Duplicate match keys are not silently discarded: extras become ``duplicate`` - diffs and ``summary.duplicate_key_list`` lists the colliding keys. + ``pipeline`` (``identity_pipeline.Pipeline``) owns normalization, filtering, + port mapping and key building; when omitted a legacy pipeline is derived + from the loose arguments (callers have already filtered the rows). + ``match_keys_override`` pins the match key (re-running a stored result). + + ``summary.duplicate`` is always 0. ``duplicate_keys_*`` count match keys + that appear more than once on a side (diagnostic only). ``field_rules`` drives normalize / numeric tolerance / per-field compare mode (template-driven; no metric-specific branches here). """ if not key_fields: raise ValueError("key_fields required") - pmap = dict(port_map or {}) - rules = field_rule_map(field_rules) - norm_rules = normalize_iface_rules(iface_normalize_rules) - iface_list = [str(f) for f in (iface_fields or []) if str(f).strip()] + if pipeline is None: + from .identity_pipeline import legacy_pipeline + + pipeline = legacy_pipeline( + key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields, + field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules, + ignore_port_changes=ignore_port_changes, duplicate_pairing=duplicate_pairing, + ) + pmap = dict(pipeline.port_map) + rules = field_rule_map(field_rules if field_rules is not None else pipeline.field_rules) + iface_list = list(pipeline.iface_fields) iface_set = set(iface_list) - before_norm = apply_iface_normalize_rows( - before_rows, iface_fields=iface_list, rules=norm_rules - ) - after_norm = apply_iface_normalize_rows( - after_rows, iface_fields=iface_list, rules=norm_rules - ) + if prepared is not None: + before_pairs, after_pairs = prepared + else: + before_pairs = pipeline.prepare(before_rows, "before") + after_pairs = pipeline.prepare(after_rows, "after") + before_rows = [orig for orig, _ in before_pairs] + after_rows = [orig for orig, _ in after_pairs] + before_norm = [norm for _, norm in before_pairs] + after_norm = [norm for _, norm in after_pairs] ignore_ports = False - # No map → optionally ignore port renames by dropping iface from match key. - if not pmap and iface_set: - candidate = [k for k in key_fields if k not in iface_set] - if not candidate: - match_keys = list(key_fields) - elif ignore_port_changes is True: - match_keys = candidate - ignore_ports = True - elif ignore_port_changes is False: - match_keys = list(key_fields) - else: - # Auto heuristic (legacy default) - before_c = [row_key(r, candidate) for r in before_norm] - after_c = [row_key(r, candidate) for r in after_norm] - if len(before_c) == len(set(before_c)) and len(after_c) == len(set(after_c)): - match_keys = candidate - ignore_ports = True - else: - match_keys = list(key_fields) + if match_keys_override is not None: + match_keys = [str(k) for k in match_keys_override] + ignore_ports = match_keys != list(key_fields) else: - match_keys = list(key_fields) + # No map → optionally ignore port renames by dropping iface from match key. + policy = pipeline.ignore_port_changes + unique = None + if not pmap and iface_set and policy is None: + candidate = [k for k in key_fields if k not in iface_set] - before_mapped: list[dict[str, Any]] = [ - apply_port_map(r, iface_fields=iface_list, port_map=pmap) for r in before_norm - ] + def unique_keys(rows: list[dict[str, Any]]) -> bool: + seen: set[tuple[str, ...]] = set() + for row in rows: + key = row_key(row, candidate) + if key in seen: + return False + seen.add(key) + return True - after_index: dict[tuple[str, ...], dict[str, Any]] = {} - after_dup = 0 - after_dup_keys: list[tuple[str, ...]] = [] - after_dup_rows: list[tuple[tuple[str, ...], dict[str, Any]]] = [] - for r in after_norm: - k = row_key(r, match_keys) - if k in after_index: - after_dup += 1 - after_dup_keys.append(k) - after_dup_rows.append((k, r)) - continue # first wins — do not overwrite - after_index[k] = r + unique = bool(candidate) and unique_keys(before_norm) and unique_keys(after_norm) + match_keys = pipeline.effective_match_keys(unique_without_iface=unique) + ignore_ports = match_keys != list(key_fields) + + before_groups: dict[tuple[str, ...], list[tuple[dict[str, Any], dict[str, Any]]]] = ( + defaultdict(list) + ) + after_groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) + for orig, norm in zip(before_rows, before_norm): + mapped, k = pipeline.key_of(norm, "before", match_keys) + before_groups[k].append((orig, mapped)) + for norm in after_norm: + after_groups[pipeline.key_of(norm, "after", match_keys)[1]].append(norm) + sort_pairs = pipeline.duplicate_pairing == "sorted" - before_keys: set[tuple[str, ...]] = set() - before_dup = 0 - before_dup_keys: list[tuple[str, ...]] = [] diffs: list[dict[str, Any]] = [] - added = removed = changed = unchanged = duplicate = 0 + added = removed = changed = unchanged = 0 + unchanged_listed = 0 + limit_n = None if unchanged_limit is None else max(0, int(unchanged_limit)) + multi_before_keys: list[tuple[str, ...]] = [] + multi_after_keys: list[tuple[str, ...]] = [] + unchanged_candidates: list[tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = [] + sample = _StratifiedSample(limit_n) if include_unchanged and limit_n is not None else None def _key_obj(row: dict[str, Any]) -> dict[str, Any]: - return {f: row.get(f, "") for f in key_fields} + return dict(zip(key_fields, key_tuple(row, key_fields))) - for orig, mapped in zip(before_rows, before_mapped): - k = row_key(mapped, match_keys) - if k in before_keys: - before_dup += 1 - before_dup_keys.append(k) - duplicate += 1 + def _row_id(row: dict[str, Any] | None) -> str: + if not isinstance(row, dict): + return "" + netx = row.get("_netx") + if isinstance(netx, dict): + return str(netx.get("row_id") or "") + return "" + + def _append_unchanged_diff( + orig: dict[str, Any], mapped: dict[str, Any], after_row: dict[str, Any] + ) -> None: + nonlocal unchanged_listed + unchanged_listed += 1 + before_rid = _row_id(orig) + after_rid = _row_id(after_row) + if compact_unchanged: diffs.append( { - "kind": "duplicate", - "side": "before", + "kind": "unchanged", "key": _key_obj(mapped), - "before": orig, - "after": after_index.get(k), - "mapped_before": mapped, + "before": {}, + "after": {}, + "mapped_before": {}, "changes": {}, - } - ) - continue # only first before row participates in match - before_keys.add(k) - after = after_index.get(k) - if after is None: - removed += 1 - diffs.append( - { - "kind": "removed", - "key": _key_obj(mapped), - "before": orig, - "after": None, - "mapped_before": mapped, - "changes": {}, - } - ) - continue - # Empty compare_fields = presence-only: keyed rows that exist on both - # sides are unchanged (no value checks). - field_changes: dict[str, dict[str, Any]] = {} - for f in compare_fields: - bv = mapped.get(f, "") - av = after.get(f, "") - rule = rules.get(f) - if not values_equal(bv, av, rule=rule): - entry: dict[str, Any] = {"before": bv, "after": av} - reason = explain_diff(bv, av, rule=rule) - if reason: - entry["reason"] = reason - field_changes[f] = entry - if field_changes: - changed += 1 - diffs.append( - { - "kind": "changed", - "key": _key_obj(mapped), - "before": orig, - "after": after, - "mapped_before": mapped, - "changes": field_changes, + "compact": True, + "before_row_id": before_rid, + "after_row_id": after_rid, } ) else: - unchanged += 1 diffs.append( { "kind": "unchanged", "key": _key_obj(mapped), "before": orig, - "after": after, + "after": after_row, "mapped_before": mapped, "changes": {}, + "before_row_id": before_rid, + "after_row_id": after_rid, } ) - for k, after in after_index.items(): - if k in before_keys: - continue - added += 1 - diffs.append( - { - "kind": "added", - "key": _key_obj(after), - "before": None, - "after": after, - "mapped_before": None, - "changes": {}, - } - ) + # Stable key order: before encounter order, then after-only keys + ordered_keys = chain(before_groups, (k for k in after_groups if k not in before_groups)) + for k in ordered_keys: + b_list = before_groups.get(k) or [] + a_list = after_groups.get(k) or [] + if len(b_list) > 1: + multi_before_keys.append(k) + if len(a_list) > 1: + multi_after_keys.append(k) + if sort_pairs and (len(b_list) > 1 or len(a_list) > 1): + # Same-key rows pair by value, not by CLI output order (ECMP, multipath). + b_list = sorted(b_list, key=lambda pair: pipeline.pair_sort_key(pair[1])) + a_list = sorted(a_list, key=pipeline.pair_sort_key) + n = max(len(b_list), len(a_list)) + for i in range(n): + if i >= len(b_list): + after = a_list[i] + added += 1 + diffs.append( + { + "kind": "added", + "key": _key_obj(after), + "before": None, + "after": after, + "mapped_before": None, + "changes": {}, + "before_row_id": "", + "after_row_id": _row_id(after), + } + ) + continue + if i >= len(a_list): + orig, mapped = b_list[i] + removed += 1 + diffs.append( + { + "kind": "removed", + "key": _key_obj(mapped), + "before": orig, + "after": None, + "mapped_before": mapped, + "changes": {}, + "before_row_id": _row_id(orig), + "after_row_id": "", + } + ) + continue + orig, mapped = b_list[i] + after = a_list[i] + field_changes: dict[str, dict[str, Any]] = {} + for f in compare_fields: + bv = mapped.get(f, "") + av = after.get(f, "") + rule = rules.get(f) + if not values_equal(bv, av, rule=rule): + entry: dict[str, Any] = {"before": bv, "after": av} + reason = explain_diff(bv, av, rule=rule) + if reason: + entry["reason"] = reason + field_changes[f] = entry + if field_changes: + changed += 1 + diffs.append( + { + "kind": "changed", + "key": _key_obj(mapped), + "before": orig, + "after": after, + "mapped_before": mapped, + "changes": field_changes, + "before_row_id": _row_id(orig), + "after_row_id": _row_id(after), + } + ) + else: + unchanged += 1 + if include_unchanged: + if sample is not None: + if sample.limit: + sample.add((orig, mapped, after), stratum_key(mapped)) + else: + unchanged_candidates.append((orig, mapped, after)) - for k, after in after_dup_rows: - duplicate += 1 - diffs.append( - { - "kind": "duplicate", - "side": "after", - "key": _key_obj(after), - "before": None, - "after": after, - "mapped_before": None, - "changes": {}, - } - ) + if include_unchanged: + picked = sample.picked() if sample is not None else unchanged_candidates + for orig, mapped, after_row in picked: + _append_unchanged_diff(orig, mapped, after_row) def _fmt_keys(keys: list[tuple[str, ...]]) -> list[str]: seen: set[str] = set() @@ -296,7 +435,7 @@ def compare_rows( if s not in seen: seen.add(s) out.append(s) - return out + return out[:64] stats = mapping_stats( before_rows=before_norm, @@ -314,11 +453,21 @@ def compare_rows( "removed": removed, "changed": changed, "unchanged": unchanged, - "duplicate": duplicate, + "duplicate": 0, "match_key_fields": match_keys, - "duplicate_keys_before": before_dup, - "duplicate_keys_after": after_dup, - "duplicate_key_list": _fmt_keys(before_dup_keys + after_dup_keys), + "duplicate_keys_before": len(multi_before_keys), + "duplicate_keys_after": len(multi_after_keys), + "duplicate_key_list": _fmt_keys(multi_before_keys + multi_after_keys), + "unchanged_listed": unchanged_listed, + "unchanged_truncated": bool( + include_unchanged and limit_n is not None and unchanged > unchanged_listed + ), + "unchanged_compact": bool(compact_unchanged and unchanged_listed > 0), + "unchanged_sample_mode": ( + "stratified" + if include_unchanged and limit_n is not None and unchanged > unchanged_listed + else ("full" if include_unchanged else "none") + ), }, "diffs": diffs, "mapping_stats": stats, diff --git a/netx_api/biz_state/compare_preflight.py b/netx_api/biz_state/compare_preflight.py new file mode 100644 index 0000000..c9c6081 --- /dev/null +++ b/netx_api/biz_state/compare_preflight.py @@ -0,0 +1,194 @@ +"""Pre-flight check of a port mapping against the two batches it will be applied to. + +The same function backs the compare "validate mapping" button and the cutover +project pre-flight, so both modes judge mapping coverage the same way. It answers +before anyone runs a comparison: + +- which *before* interfaces are mapped (exact / by parent), unmapped but present on + the *after* side by the same name, or **suspect** (not mapped, not on *after*) +- which map keys / targets match nothing, which targets receive several sources +- whether each sheet's two sides were actually collected (``data_state``) +""" + +from __future__ import annotations + +from typing import Any, Callable, Mapping, Sequence + +from sqlalchemy.orm import Session + +from .compare_quality import assess_sheet, command_coverage, collect_quality, side_receipts +from .identity_pipeline import build_pipeline +from .iface_normalize import normalize_port_map, resolve_mapped_iface + +LIST_CAP = 200 +# Million-row tables are never loaded just to look at interface names. +HEAVY_METRICS = frozenset({"bgp_route", "ip_route", "ipv6_route", "l2vpn_mac", "evpn_mac", "interface_detail"}) + +RowsLoader = Callable[[str, Mapping[str, Any]], list[dict[str, Any]]] + + +def _parents(name: str) -> list[str]: + parts = name.split(".") + return [".".join(parts[:i]) for i in range(len(parts) - 1, 0, -1)] + + +def _classify(name: str, pmap: Mapping[str, str], after: set[str]) -> str: + if name in pmap: + return "mapped_exact" + if any(p in pmap for p in _parents(name)): + return "mapped_by_parent" + if name in after: + return "identity_same_name" + return "suspect_unmapped" + + +def _present(name: str, names: set[str]) -> bool: + """``name`` exists, or is the parent of an existing sub-interface.""" + if name in names: + return True + prefix = name + "." + return any(n.startswith(prefix) for n in names) + + +def _top_level(names: Sequence[str]) -> list[str]: + """Drop sub-interfaces whose parent is already in the list (noise).""" + pool = set(names) + return sorted(n for n in pool if not any(p in pool for p in _parents(n))) + + +def sheet_mapping_coverage( + pipe: Any, + before_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]], + after_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]], +) -> dict[str, Any]: + """Classify every *before* interface of a sheet against the map and the *after* side. + + ``before_pairs`` / ``after_pairs`` are ``pipeline.prepare`` outputs. The result + carries two private sets (``_after_ifaces``, ``_used_keys``) for aggregation. + """ + b_if = {str(n.get(f) or "").strip() for _, n in before_pairs for f in pipe.iface_fields} + a_if = {str(n.get(f) or "").strip() for _, n in after_pairs for f in pipe.iface_fields} + b_if.discard("") + a_if.discard("") + buckets: dict[str, list[str]] = { + "mapped_exact": [], "mapped_by_parent": [], "identity_same_name": [], "suspect_unmapped": [], + } + for name in sorted(b_if): + buckets[_classify(name, pipe.port_map, a_if)].append(name) + used = {k for name in b_if for k in pipe.port_map if name == k or name.startswith(k + ".")} + missing_targets = sorted({ + resolve_mapped_iface(n, pipe.port_map) for n in buckets["mapped_exact"] + buckets["mapped_by_parent"] + if not _present(resolve_mapped_iface(n, pipe.port_map), a_if) + }) + suspects = _top_level(buckets["suspect_unmapped"]) + return { + "counts": {k: len(v) for k, v in buckets.items()}, + "mapped_exact": buckets["mapped_exact"][:LIST_CAP], + "mapped_by_parent": buckets["mapped_by_parent"][:LIST_CAP], + "identity_same_name": buckets["identity_same_name"][:LIST_CAP], + "suspect_unmapped": suspects[:LIST_CAP], + "suspect_unmapped_total": len(suspects), + "mapped_target_missing_on_after": missing_targets[:LIST_CAP], + "_after_ifaces": a_if, + "_used_keys": used, + } + + +def preflight( + db: Session, + *, + sheets: Sequence[Mapping[str, Any]], + iface_rules: Sequence[Mapping[str, Any]] | None, + raw_port_map: Mapping[str, str] | None, + before_batch_id: str, + after_batch_id: str, + before_vendor: str = "", + after_vendor: str = "", + load_rows: RowsLoader, +) -> dict[str, Any]: + """Build the pre-flight report. ``load_rows(side, sheet)`` returns raw rows.""" + raw_map = dict(raw_port_map or {}) + out_sheets: list[dict[str, Any]] = [] + used_keys: set[str] = set() + after_all: set[str] = set() + totals = {"mapped": 0, "suspect": 0, "same_name": 0} + risk_reasons: list[str] = [] + full_pmap: dict[str, str] = {} + conflicts: list[dict[str, str]] = [] + + for sheet in sheets: + sid = str(sheet.get("sheet_id") or sheet.get("metric_id") or "") + mid = str(sheet.get("metric_id") or "") + title = str(sheet.get("title") or sid) + pipe = build_pipeline( + sheet, iface_rules=iface_rules, port_map=raw_map, + before_vendor=before_vendor, after_vendor=after_vendor, + ) + if not full_pmap and pipe.port_map: + full_pmap = dict(pipe.port_map) + heavy = mid in HEAVY_METRICS + card: dict[str, Any] = {"sheet_id": sid, "metric_id": mid, "title": title} + if heavy: + b_n = collect_quality(side_receipts(db, before_batch_id, mid))["row_count"] + a_n = collect_quality(side_receipts(db, after_batch_id, mid))["row_count"] + card.update(skipped="heavy_table", before_count=b_n, after_count=a_n) + else: + b_pairs = pipe.prepare(load_rows("before", sheet), "before") + a_pairs = pipe.prepare(load_rows("after", sheet), "after") + card.update(before_count=len(b_pairs), after_count=len(a_pairs)) + if pipe.iface_fields: + cov = sheet_mapping_coverage(pipe, b_pairs, a_pairs) + after_all |= cov.pop("_after_ifaces") + used_keys |= cov.pop("_used_keys") + card.update(cov) + totals["mapped"] += cov["counts"]["mapped_exact"] + cov["counts"]["mapped_by_parent"] + totals["same_name"] += cov["counts"]["identity_same_name"] + totals["suspect"] += cov["suspect_unmapped_total"] + card.update(assess_sheet( + db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id, + before_rows=int(card.get("before_count") or 0), after_rows=int(card.get("after_count") or 0), + )) + if card["data_state"] in ("not_collected", "suspect_parse"): + risk_reasons.append(f"{sid}:{card['data_state']}") + out_sheets.append(card) + + if raw_map and not full_pmap: + full_pmap, conflicts = normalize_port_map(raw_map, iface_rules) + elif raw_map: + _, conflicts = normalize_port_map(raw_map, iface_rules) + sources: dict[str, list[str]] = {} + for k, v in full_pmap.items(): + sources.setdefault(v, []).append(k) + n_to_1 = [{"target": t, "sources": sorted(src)} for t, src in sorted(sources.items()) if len(src) > 1] + unused = sorted(k for k in full_pmap if k not in used_keys) + targets_missing = sorted(v for v in set(full_pmap.values()) if after_all and not _present(v, after_all)) + + judged = totals["mapped"] + totals["suspect"] + coverage = { + "mapped": totals["mapped"], "suspect_unmapped": totals["suspect"], + "identity_same_name": totals["same_name"], + "rate": round(totals["mapped"] / judged * 100, 1) if judged else None, + } + mapping = { + "pairs": len(full_pmap), "conflicts": conflicts[:LIST_CAP], + "unused_map_keys": unused[:LIST_CAP], "unused_map_keys_total": len(unused), + "map_targets_missing_on_after": targets_missing[:LIST_CAP], + "map_targets_missing_on_after_total": len(targets_missing), + "n_to_1_targets": n_to_1[:LIST_CAP], + } + if totals["suspect"]: + risk_reasons.append("suspect_unmapped") + if unused: + risk_reasons.append("unused_map_keys") + if targets_missing: + risk_reasons.append("map_targets_missing_on_after") + if conflicts: + risk_reasons.append("map_conflicts") + return { + "verdict": "risk" if risk_reasons else "ready", + "risk_reasons": sorted(set(risk_reasons)), + "coverage": coverage, + "mapping": mapping, + "sheets": out_sheets, + "command_coverage": command_coverage(db, before_batch_id=before_batch_id, after_batch_id=after_batch_id), + } diff --git a/netx_api/biz_state/compare_quality.py b/netx_api/biz_state/compare_quality.py new file mode 100644 index 0000000..30a3551 --- /dev/null +++ b/netx_api/biz_state/compare_quality.py @@ -0,0 +1,198 @@ +"""Collection quality of a compare side: is a missing row a business loss or a data problem? + +Compare results are only as good as the two batches behind them. This module +reads *receipts* (``BizStateBatchCommand``: counts and status, never ``raw_text``) +to tell apart: + +- ``not_collected`` – a side never ran a command (or the slice) for the sheet +- ``suspect_parse`` – the device printed lines but the parser produced no rows +- ``no_data`` – both sides ran fine and are really empty +- ``ok`` – ordinary comparison +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +from sqlalchemy.orm import Session + +from ..models import BizStateBatchCommand +from .compare_rules import scalar_text + +# More than this many CLI lines with zero parsed rows is not "an empty table". +SUSPECT_RAW_LINES = 5 +DATA_STATES = ("ok", "no_data", "suspect_parse", "not_collected") +UNJUDGED_STATES = ("no_data", "not_collected") + + +def _has_receipts(db: Session, batch_id: str) -> bool: + return bool(batch_id) and db.query(BizStateBatchCommand.id).filter( + BizStateBatchCommand.batch_id == batch_id).limit(1).first() is not None + + +def side_receipts(db: Session, batch_id: str, metric_id: str) -> list[dict[str, Any]]: + if not batch_id or not metric_id: + return [] + rows = ( + db.query( + BizStateBatchCommand.profile_id, BizStateBatchCommand.parse_status, + BizStateBatchCommand.row_count, BizStateBatchCommand.raw_line_count, + BizStateBatchCommand.declared_total, + ) + .filter(BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id == metric_id) + .all() + ) + return [ + { + "profile_id": str(r[0] or ""), "parse_status": str(r[1] or ""), + "row_count": int(r[2] or 0), "raw_line_count": int(r[3] or 0), + "declared_total": int(r[4] or 0), + } + for r in rows + ] + + +def collect_quality(receipts: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + statuses: dict[str, int] = {} + for r in receipts: + key = str(r.get("parse_status") or "unknown") + statuses[key] = statuses.get(key, 0) + 1 + return { + "commands": len(receipts), + "parse_status": statuses, + "row_count": sum(int(r.get("row_count") or 0) for r in receipts), + "raw_line_count": sum(int(r.get("raw_line_count") or 0) for r in receipts), + "declared_total": sum(int(r.get("declared_total") or 0) for r in receipts), + "profile_ids": sorted({str(r.get("profile_id") or "") for r in receipts if r.get("profile_id")}), + } + + +def sheet_slice(sheet: Mapping[str, Any]) -> dict[str, str]: + """``{field: value}`` the sheet pins with top-level ``eq`` row filters.""" + out: dict[str, str] = {} + for f in sheet.get("row_filters") or []: + if not isinstance(f, Mapping) or "any" in f or "all" in f: + continue + if str(f.get("op") or "eq").strip().lower() not in ("eq", "==", "ci_eq"): + continue + field = str(f.get("field") or "").strip() + value = scalar_text(f.get("value")).strip().lower() + if field and value: + out[field] = value + return out + + +def covers_slice( + receipts: Sequence[Mapping[str, Any]], wanted: Mapping[str, str], *, profile_slice: Any = None, +) -> bool: + """True when at least one successful command could have produced ``wanted``. + + A command without a declared slice (or unknown profile) covers everything. + """ + if profile_slice is None: + from .profiles import get_profile + + def profile_slice(pid: str) -> dict[str, str]: # noqa: F811 + prof = get_profile(pid) if pid else None + return dict(prof.slice) if prof is not None and prof.slice else {} + + for r in receipts: + if str(r.get("parse_status") or "") in ("failed", "skipped_custom"): + continue + declared = profile_slice(str(r.get("profile_id") or "")) + if all( + f not in declared or str(declared[f]).strip().lower() == v + for f, v in wanted.items() + ): + return True + return False + + +def classify_sheet( + *, + before: Sequence[Mapping[str, Any]] | None, + after: Sequence[Mapping[str, Any]] | None, + before_rows: int, + after_rows: int, + wanted_slice: Mapping[str, str] | None = None, +) -> tuple[str, list[str]]: + """``(data_state, reasons)`` for one sheet; reasons name the offending side. + + ``None`` receipts mean the batch carries no receipts at all (imported or + legacy data): that side cannot be judged and is trusted as-is. + """ + wanted = dict(wanted_slice or {}) + reasons: list[str] = [] + for side, receipts in (("before", before), ("after", after)): + if receipts is None: + continue + if not receipts: + reasons.append(f"{side}:no_command") + elif wanted and not covers_slice(receipts, wanted): + reasons.append(f"{side}:slice_not_collected") + if reasons: + return "not_collected", reasons + for side, receipts in (("before", before), ("after", after)): + if receipts is None: + continue + q = collect_quality(receipts) + if q["row_count"] == 0 and q["raw_line_count"] > SUSPECT_RAW_LINES: + reasons.append(f"{side}:parsed_0_of_{q['raw_line_count']}_lines") + elif q["commands"] and q["parse_status"].get("failed", 0) == q["commands"]: + reasons.append(f"{side}:all_commands_failed") + if reasons: + return "suspect_parse", reasons + if before_rows == 0 and after_rows == 0: + return "no_data", ["both_sides_empty"] + return "ok", [] + + +def assess_sheet( + db: Session, sheet: Mapping[str, Any], *, before_batch_id: str, after_batch_id: str, + before_rows: int, after_rows: int, +) -> dict[str, Any]: + """Quality block stored on a sheet summary.""" + metric_id = str(sheet.get("metric_id") or "") + before = side_receipts(db, before_batch_id, metric_id) if _has_receipts(db, before_batch_id) else None + after = side_receipts(db, after_batch_id, metric_id) if _has_receipts(db, after_batch_id) else None + state, reasons = classify_sheet( + before=before, after=after, before_rows=before_rows, after_rows=after_rows, + wanted_slice=sheet_slice(sheet), + ) + return { + "data_state": state, + "data_state_reasons": reasons, + "collect_quality": { + "before": collect_quality(before or []), "after": collect_quality(after or []), + "unknown_sides": [n for n, r in (("before", before), ("after", after)) if r is None], + }, + } + + +def command_coverage( + db: Session, *, before_batch_id: str, after_batch_id: str, +) -> dict[str, Any]: + """Collected "metric{slice}" labels per side and their difference. + + Profile ids differ per vendor (``zte.x`` vs ``huawei.x``), so the comparison + is on what a command *produces*: its metric and slice. + """ + from .profiles import get_profile + + def labels(batch_id: str) -> set[str]: + if not batch_id: + return set() + rows = db.query(BizStateBatchCommand.metric_id, BizStateBatchCommand.profile_id).filter( + BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id != "", + ).distinct().all() + out: set[str] = set() + for metric_id, profile_id in rows: + prof = get_profile(str(profile_id or "")) if profile_id else None + sl = dict(prof.slice) if prof is not None and prof.slice else {} + suffix = ",".join(f"{k}={v}" for k, v in sorted(sl.items())) + out.add(f"{metric_id}{{{suffix}}}" if suffix else str(metric_id)) + return out + + b, a = labels(before_batch_id), labels(after_batch_id) + return {"before": sorted(b), "after": sorted(a), + "only_before": sorted(b - a), "only_after": sorted(a - b)} diff --git a/netx_api/biz_state/compare_rules.py b/netx_api/biz_state/compare_rules.py index 97e454b..928fcdb 100644 --- a/netx_api/biz_state/compare_rules.py +++ b/netx_api/biz_state/compare_rules.py @@ -7,10 +7,21 @@ All metric-specific compare behavior belongs in the sheet template from __future__ import annotations import re +import math from typing import Any, Mapping, Sequence _AGE_TIMER_RE = re.compile(r"^\d{1,2}:\d{2}:\d{2}$") +NUMBER_PATTERN = r"^[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?$" + + +def scalar_text(value: Any) -> str: + """Match JSON text extraction: only null is blank; keep zero and booleans.""" + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) def _as_list(raw: Any) -> list[Any]: @@ -22,7 +33,7 @@ def _as_list(raw: Any) -> list[Any]: def _field_val(row: Mapping[str, Any], field: str) -> str: - return str((row or {}).get(field) or "").strip() + return scalar_text((row or {}).get(field)).strip() def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: @@ -35,17 +46,17 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: expect = filt.get("value") if op in ("eq", "=="): - return raw.lower() == str(expect or "").strip().lower() + return raw.lower() == scalar_text(expect).strip().lower() if op in ("ne", "!="): - return raw.lower() != str(expect or "").strip().lower() + return raw.lower() != scalar_text(expect).strip().lower() if op == "in": - opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()} + opts = {scalar_text(x).strip().lower() for x in _as_list(expect) if scalar_text(x).strip()} return raw.lower() in opts if op in ("not_in", "nin"): - opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()} + opts = {scalar_text(x).strip().lower() for x in _as_list(expect) if scalar_text(x).strip()} return raw.lower() not in opts if op == "contains": - needle = str(expect or "").strip().lower() + needle = scalar_text(expect).strip().lower() return bool(needle) and needle in raw.lower() if op == "empty": return not raw @@ -63,7 +74,7 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: # HH:MM:SS dynamic ARP age return bool(_AGE_TIMER_RE.match(raw)) if op == "ci_eq": - return raw.lower() == str(expect or "").strip().lower() + return raw.lower() == scalar_text(expect).strip().lower() return True @@ -99,7 +110,7 @@ def apply_row_filters( def normalize_value(value: Any, how: str) -> str: - text = str(value if value is not None else "").strip() + text = scalar_text(value).strip() mode = str(how or "").strip().lower() if not mode or mode in ("none", "strip"): return text @@ -126,6 +137,8 @@ def field_rule_map(rules: Sequence[Mapping[str, Any]] | None) -> dict[str, dict[ name = str(raw.get("field") or "").strip() if not name: continue + if str(raw.get("side") or "").strip(): + continue # side-scoped rules feed the identity pipeline, not the compare out[name] = dict(raw) return out @@ -190,12 +203,59 @@ def effective_display_fields( def _parse_float(text: str) -> float | None: + if not re.fullmatch(NUMBER_PATTERN, text): + return None try: - return float(text) if text else 0.0 + value = float(text) + return value if math.isfinite(value) else None except ValueError: return None +DIRECTIONAL_MODES = ("not_worse", "numeric_min", "numeric_max") + + +def _rank_index(value: str, rank: Sequence[Any]) -> int | None: + wanted = value.strip().lower() + for i, item in enumerate(rank or []): + if scalar_text(item).strip().lower() == wanted: + return i + return None + + +def _tolerance(rule: Mapping[str, Any]) -> float: + try: + return float(rule.get("tolerance") or 0) + except (TypeError, ValueError): + return 0.0 + + +def _directional_verdict(bv: str, av: str, rule: Mapping[str, Any]) -> tuple[bool, str]: + """``(equal_or_better, reason)`` for not_worse / numeric_min / numeric_max.""" + mode = str(rule.get("compare") or "").strip().lower() + if mode == "not_worse": + rank = rule.get("rank") or [] + bi, ai = _rank_index(bv, rank), _rank_index(av, rank) + if bi is None or ai is None: + return bv == av, "neq" + return ai <= bi, (f"worse: {bv}→{av}" if ai > bi else "") + bn, an = _parse_float(bv), _parse_float(av) + if bn is None or an is None: + return bv == av, "neq" + tol = _tolerance(rule) + if mode == "numeric_min": + floor = bn * (1 - tol / 100.0) + if an >= floor: + return True, "" + drop = (bn - an) / abs(bn) * 100.0 if bn else 0.0 + return False, f"drop {drop:.0f}% > {tol:g}%" + ceil = bn * (1 + tol / 100.0) + if an <= ceil: + return True, "" + rise = (an - bn) / abs(bn) * 100.0 if bn else 0.0 + return False, f"rise {rise:.0f}% > {tol:g}%" + + def explain_diff( before: Any, after: Any, @@ -208,6 +268,8 @@ def explain_diff( bv = normalize_value(before, norm) av = normalize_value(after, norm) mode = str(rule.get("compare") or "eq").strip().lower() or "eq" + if mode in DIRECTIONAL_MODES: + return _directional_verdict(bv, av, rule)[1] if mode in ("percent", "pct", "rel"): bn = _parse_float(bv) an = _parse_float(av) @@ -248,6 +310,8 @@ def values_equal( mode = str(rule.get("compare") or "eq").strip().lower() or "eq" if mode in ("ignore", "skip", "off"): return True + if mode in DIRECTIONAL_MODES: + return _directional_verdict(bv, av, rule)[0] if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"): bn = _parse_float(bv) an = _parse_float(av) diff --git a/netx_api/biz_state/compare_service.py b/netx_api/biz_state/compare_service.py index 195e752..d54cb35 100644 --- a/netx_api/biz_state/compare_service.py +++ b/netx_api/biz_state/compare_service.py @@ -6,19 +6,24 @@ import io import json import logging import threading +import time import zipfile +from copy import deepcopy from datetime import datetime -from typing import Any +from typing import Any, Callable, Iterable, Iterator from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy.orm import Session +from sqlalchemy import String, and_, cast, func, or_, tuple_ +from sqlalchemy.orm import Session, aliased from ..models import ( BizCompareDiff, BizCompareJob, BizCompareRun, BizCompareTemplate, + BizMigrationProject, + BizMonitorTemplate, BizPortMapping, BizPortMappingRow, BizStateBatch, @@ -27,23 +32,33 @@ from ..models import ( BizStateTask, ) from ..timeutil import utcnow_naive -from .compare_engine import compare_rows, mapping_stats +from .compare_engine import apply_port_map, compare_rows, mapping_stats, row_key from .compare_rules import ( ROW_FILTER_PRESETS, apply_row_filters, + scalar_text, arp_dynamic_row_filters, effective_compare_fields, effective_display_fields, + row_matches_filter, ) from .iface_normalize import ( apply_iface_normalize_rows, default_zte_iface_normalize_rules, normalize_iface_rules, + normalize_port_map, ) +from .compare_quality import UNJUDGED_STATES, assess_sheet +from .identity_pipeline import Pipeline, build_pipeline, legacy_pipeline from .profiles import metric_field_map +from .compare_validation import validate_template_body _log = logging.getLogger("netx.biz_state.compare") +# v1: port-map keys/values must be written post-normalization. +# v2: port-map keys/values are normalized with the template iface rules (A1). +CONFIG_SNAPSHOT_VERSION = 2 + # Per-job mutex so auto + manual run_compare do not dual-write the same job. _auto_compare_locks: dict[str, threading.Lock] = {} _auto_compare_locks_guard = threading.Lock() @@ -88,7 +103,7 @@ def batch_metric_collect_ok(db: Session, batch_id: str, metric_id: str) -> bool: if status == "success": return True cmds = ( - db.query(BizStateBatchCommand) + db.query(BizStateBatchCommand.parse_status, BizStateBatchCommand.row_count) .filter( BizStateBatchCommand.batch_id == bid, BizStateBatchCommand.metric_id == mid, @@ -181,10 +196,93 @@ def _compare_side( } _DIFF_CHUNK = 2000 +_LOAD_YIELD_PER = 5000 _SEARCH_TEXT_MAX = 4000 +# Heartbeat while bulk-inserting large fail/ok diff sets (vpnv4-scale). +_PERSIST_PROGRESS_EVERY = 10_000 +# Above this, store fail diffs as key + row_id + changes (hydrate sides on read). +_FAIL_COMPACT_MIN = 50_000 +# Live search (kw): load at most this many matching rows per side, return ≤ this many pairs. +_LIVE_SEARCH_LOAD_CAP = 2_000 +_LIVE_SEARCH_RESULT_CAP = 200 +_LIVE_SEARCH_GROUP_LOAD_CAP = 10_000 +# Success-row persist policy (see resolve_unchanged_policy) +_STORE_UNCHANGED_MODES = frozenset({"auto", "always", "never", "sample", "keys"}) +_UNCHANGED_FULL_MAX = 20_000 +_UNCHANGED_SAMPLE_MAX = 5_000 -def _diff_search_text(d: dict[str, Any]) -> str: +def normalize_store_unchanged(raw: Any) -> str: + m = str(raw or "auto").strip().lower() + return m if m in _STORE_UNCHANGED_MODES else "auto" + + +def compute_pass_rate(*, success: int, fail: int, empty_as: float = 100.0) -> float: + """Pass rate for cutover UI. + + With million-row sheets, 3 fails / 50k can round to 100.0% at 1 decimal — + never report 100 when any fail remains; add decimals until it stays < 100. + """ + s = max(0, int(success or 0)) + f = max(0, int(fail or 0)) + judged = s + f + if judged <= 0: + return float(empty_as) + rate = (s / judged) * 100.0 + if f <= 0: + return round(rate, 1) + for nd in (1, 2, 3, 4): + r = round(rate, nd) + if r < 100.0: + return r + return 99.9999 + + +def resolve_unchanged_policy( + mode: str, *, before_n: int, after_n: int +) -> dict[str, Any]: + """Decide whether / how many success rows to emit for one sheet. + + - always: full before/after for every match (slow on million-row sheets) + - never: count only + - sample: key + row_id sample (cap) — cutover spot-check default for large sheets + - keys: all success as key + row_id (full browse; write still heavy) + - auto: full when sheet is small; otherwise sample compact + """ + m = normalize_store_unchanged(mode) + n = max(int(before_n or 0), int(after_n or 0)) + if m == "never": + return {"mode": m, "include": False, "limit": None, "compact": False} + if m == "always": + return {"mode": m, "include": True, "limit": None, "compact": False} + if m == "keys": + return {"mode": m, "include": True, "limit": None, "compact": True} + if m == "sample": + return { + "mode": m, + "include": True, + "limit": _UNCHANGED_SAMPLE_MAX, + "compact": True, + } + # auto — cutover-oriented: large sheets sample, not full key dump + if n <= _UNCHANGED_FULL_MAX: + return {"mode": m, "include": True, "limit": None, "compact": False} + return { + "mode": m, + "include": True, + "limit": _UNCHANGED_SAMPLE_MAX, + "compact": True, + } + + +def _strip_netx(row: Any) -> dict[str, Any]: + """Drop collector provenance before persisting compare payloads.""" + if not isinstance(row, dict): + return {} + return {k: v for k, v in row.items() if k != "_netx"} + + +def _diff_search_text(d: dict[str, Any], *, limit: int | None = _SEARCH_TEXT_MAX) -> str: parts = [str(d.get("kind") or "")] for key in ("key", "before", "after", "mapped_before", "changes"): val = d.get(key) @@ -193,7 +291,8 @@ def _diff_search_text(d: dict[str, Any]) -> str: parts.append(json.dumps(val, ensure_ascii=False, default=str, separators=(",", ":"))) except Exception: parts.append(str(val)) - return " ".join(parts)[:_SEARCH_TEXT_MAX] + value = " ".join(parts) + return value if limit is None else value[:limit] def _top_changed_fields(diffs: list[dict[str, Any]], *, limit: int = 8) -> list[dict[str, Any]]: @@ -215,30 +314,167 @@ def _persist_sheet_diffs( run_id: str, metric_id: str, diffs: list[dict[str, Any]], -) -> None: - """Bulk-insert diff rows; avoids embedding million-row arrays in summary_json.""" + seq_start: int = 0, + on_progress: Callable[[int, int], None] | None = None, +) -> int: + """Bulk-insert diff rows already selected by the engine policy. + + Compact success rows carry key + before/after_row_id; JSON sides stay empty + and are hydrated from metric tables on read. + + ``on_progress(written, total)`` fires periodically so UI elapsed time moves + during multi-minute inserts (e.g. large vpnv4 fail sets). + """ buf: list[dict[str, Any]] = [] - for i, d in enumerate(diffs): + seq = int(seq_start or 0) + written = 0 + total = len(diffs) + last_prog = 0 + last_prog_t = time.monotonic() + + def _maybe_prog(force: bool = False) -> None: + nonlocal last_prog, last_prog_t + if not on_progress: + return + now = time.monotonic() + if ( + not force + and written - last_prog < _PERSIST_PROGRESS_EVERY + and now - last_prog_t < 2.0 + ): + return + last_prog = written + last_prog_t = now + try: + on_progress(written, total) + except Exception: + _log.exception("persist progress callback failed run=%s metric=%s", run_id, metric_id) + + for d in diffs: + kind = str(d.get("kind") or "") + before = _strip_netx(d.get("before")) + after = _strip_netx(d.get("after")) + mapped = _strip_netx(d.get("mapped_before")) + payload = { + "kind": kind, + "key": dict(d.get("key") or {}), + "before": before, + "after": after, + "mapped_before": mapped, + "changes": dict(d.get("changes") or {}), + } + # Compact rows: search_text = kind + key (+ changes) only — no fat sides + search_src = ( + { + "kind": kind, + "key": payload["key"], + **({"changes": payload["changes"]} if payload["changes"] else {}), + } + if bool(d.get("compact")) + else payload + ) buf.append( { "id": uuid4().hex, "run_id": run_id, "metric_id": metric_id, - "seq": i, - "kind": str(d.get("kind") or ""), - "key_json": dict(d.get("key") or {}), - "before_json": dict(d.get("before") or {}), - "after_json": dict(d.get("after") or {}), - "mapped_before_json": dict(d.get("mapped_before") or {}), - "changes_json": dict(d.get("changes") or {}), - "search_text": _diff_search_text(d), + "seq": seq, + "kind": kind, + "key_json": payload["key"], + "before_json": before, + "after_json": after, + "mapped_before_json": mapped, + "changes_json": payload["changes"], + "before_row_id": str(d.get("before_row_id") or "")[:64], + "after_row_id": str(d.get("after_row_id") or "")[:64], + "search_text": _diff_search_text(search_src), } ) + seq += 1 + written += 1 if len(buf) >= _DIFF_CHUNK: db.bulk_insert_mappings(BizCompareDiff, buf) buf.clear() + # Commit chunks so progress/UI can see mid-write fail rows and + # elapsed_ms advances (otherwise persisting_* looks frozen). + db.commit() + _maybe_prog() if buf: db.bulk_insert_mappings(BizCompareDiff, buf) + db.commit() + _maybe_prog(force=True) + return written + + +def _metric_rows_by_ids(db: Session, ids: list[str]) -> dict[str, dict[str, Any]]: + """Load BizStateMetricRow / LLDP rows by primary key for hydrate.""" + clean = [str(i).strip() for i in ids if str(i or "").strip()] + if not clean: + return {} + from ..models import BizStateMetricRow + + out: dict[str, dict[str, Any]] = {} + # Chunk IN lists for large pages + for i in range(0, len(clean), 500): + chunk = clean[i : i + 500] + for r in db.query(BizStateMetricRow).filter(BizStateMetricRow.id.in_(chunk)).all(): + out[str(r.id)] = dict(r.data_json or {}) + missing = [x for x in chunk if x not in out] + if missing: + for n in ( + db.query(BizStateLldpNeighbor) + .filter(BizStateLldpNeighbor.id.in_(missing)) + .all() + ): + out[str(n.id)] = { + "local_if": n.local_if, + "remote_sys": n.remote_sys, + "remote_if": n.remote_if, + "remote_ip": n.remote_ip, + "protocol": n.protocol, + } + return out + + +def _attach_raw_key_before(items: list[dict[str, Any]], key_fields: list[str]) -> list[dict[str, Any]]: + """``raw_key_before``: the before-side key as printed, only where it differs from the match key.""" + for it in items: + before = it.get("before") + if not before or it.get("raw_key_before"): + continue + shown = it.get("key") or {} + raw = {f: scalar_text(before.get(f)).strip() for f in key_fields} + if any(raw[f] != scalar_text(shown.get(f)).strip() for f in key_fields if f in shown): + it["raw_key_before"] = raw + return items + + +def _hydrate_diff_rows( + db: Session, items: list[dict[str, Any]], *, + run: BizCompareRun | None = None, sheet: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + """Fill empty before/after from metric tables when row_ids are present.""" + key_fields = list((sheet or {}).get("key_fields") or []) + need: list[str] = [] + for it in items: + if not it.get("before") and it.get("before_row_id"): + need.append(str(it["before_row_id"])) + if not it.get("after") and it.get("after_row_id"): + need.append(str(it["after_row_id"])) + if not need: + return _attach_raw_key_before(items, key_fields) + by_id = _metric_rows_by_ids(db, need) + pipeline = _run_pipeline(db, run, sheet or {}) + for it in items: + brid = str(it.get("before_row_id") or "") + arid = str(it.get("after_row_id") or "") + if not it.get("before") and brid and brid in by_id: + it["before"] = by_id[brid] + if not it.get("after") and arid and arid in by_id: + it["after"] = pipeline.normalize(by_id[arid], "after") + if not it.get("mapped_before") and it.get("before"): + it["mapped_before"] = pipeline.transform(it["before"], "before") + return _attach_raw_key_before(items, key_fields) def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]: @@ -249,6 +485,8 @@ def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]: "after": r.after_json or {}, "mapped_before": r.mapped_before_json or {}, "changes": r.changes_json or {}, + "before_row_id": getattr(r, "before_row_id", "") or "", + "after_row_id": getattr(r, "after_row_id", "") or "", } @@ -264,6 +502,7 @@ def _filter_inline_diffs( *, kind: str, kw: str, + field_q: dict[str, str] | None = None, ) -> list[dict[str, Any]]: kind_n = (kind or "diff").strip().lower() kw_n = (kw or "").strip().lower() @@ -277,17 +516,70 @@ def _filter_inline_diffs( elif kind_n != "all" and dk != kind_n: continue if kw_n: - blob = _diff_search_text(d).lower() + blob = _diff_search_text(d, limit=None).lower() if kw_n not in blob: continue + if field_q and not _diff_matches_search(d, kw="", field_q=field_q): + continue out.append(d) return out +def _diff_matches_search(d: dict[str, Any], *, kw: str, field_q: dict[str, str]) -> bool: + """Search a completed pair, never filter its two sides before pairing.""" + if kw and kw.lower() not in _diff_search_text(d, limit=None).lower(): + return False + return all( + any( + val.lower() in str((d.get(side) or {}).get(name, "")).lower() + for side in ("key", "before", "mapped_before", "after") + ) + for name, val in field_q.items() + ) + + +def _search_stored_diffs(q: Any, *, metric_id: str, kw: str, field_q: dict[str, str]) -> Any: + """Filter persisted verdicts in SQL; join source rows for compact payloads.""" + from ..models import BizStateMetricRow + + sides = [BizCompareDiff.key_json, BizCompareDiff.before_json, + BizCompareDiff.mapped_before_json, BizCompareDiff.after_json] + sources = [] + for row_id in (BizCompareDiff.before_row_id, BizCompareDiff.after_row_id): + model = BizStateLldpNeighbor if metric_id == "lldp_neighbor" else BizStateMetricRow + src = aliased(model) + q = q.outerjoin(src, src.id == row_id) + sources.append(src) + if kw: + expressions = [BizCompareDiff.search_text, *[cast(s, String) for s in sides]] + for src in sources: + if metric_id == "lldp_neighbor": + expressions.extend(getattr(src, f) for f in + ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")) + else: + expressions.append(cast(src.data_json, String)) + q = q.filter(or_(*[func.lower(e).contains(kw.lower(), autoescape=True) for e in expressions])) + for name, val in field_q.items(): + expressions = [s[name].as_string() for s in sides] + for src in sources: + if metric_id == "lldp_neighbor": + if name in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol"): + expressions.append(getattr(src, name)) + else: + expressions.append(src.data_json[name].as_string()) + q = q.filter(or_(*[func.lower(cast(e, String)).contains(val.lower(), autoescape=True) + for e in expressions])) + return q + + def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: Any) -> list[dict[str, Any]]: sheets = list(summary.get("sheets") or []) if sheets: return sheets + # Running / empty: do NOT invent a fake first-metric sheet (was showing + # isis 100% with 0→0 while BGP was still loading). + if str(run.status or "") in ("running", "queued"): + return [] return [ { "metric_id": run.metric_id, @@ -297,6 +589,7 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A "iface_fields": list((tpl.iface_fields if tpl else None) or []), "compare_fields": list((tpl.compare_fields if tpl else None) or []), "mode": "fields", + "status": "done", "summary": { k: summary.get(k, 0) for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count") @@ -306,6 +599,62 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A ] +def _order_sheets_small_first( + db: Session, + sheets: list[dict[str, Any]], + *, + before_batch_id: str, + after_batch_id: str, +) -> list[dict[str, Any]]: + """Run smaller metrics first so field engineers can review early results.""" + if len(sheets) <= 1: + return list(sheets) + # Split sheets share metrics. Aggregate all counters once, without raw CLI blobs. + counts: dict[str, int] = {} + for bid, mid, total in db.query( + BizStateBatchCommand.batch_id, BizStateBatchCommand.metric_id, + func.sum(BizStateBatchCommand.row_count), + ).filter( + BizStateBatchCommand.batch_id.in_([before_batch_id, after_batch_id]), + BizStateBatchCommand.metric_id.in_({str(s.get("metric_id") or "") for s in sheets}), + ).group_by(BizStateBatchCommand.batch_id, BizStateBatchCommand.metric_id): + counts[mid] = max(counts.get(mid, 0), int(total or 0)) + scored: list[tuple[int, int, dict[str, Any]]] = [] + for i, sheet in enumerate(sheets): + n = counts.get(str(sheet.get("metric_id") or ""), 0) + scored.append((n, i, sheet)) + scored.sort(key=lambda x: (x[0], x[1])) + return [s for _, _, s in scored] + + +def _pending_sheet_meta(sheet: dict[str, Any]) -> dict[str, Any]: + """Placeholder meta so the UI lists all check items while a run is in flight.""" + key_fields = list(sheet.get("key_fields") or []) + compare_fields = list(sheet.get("compare_fields") or []) + return { + "sheet_id": sheet_key(sheet), + "title": sheet_title(sheet), + "metric_id": sheet.get("metric_id") or "", + "key_fields": key_fields, + "iface_fields": list(sheet.get("iface_fields") or []), + "compare_fields": compare_fields, + "display_fields": list(sheet.get("display_fields") or []), + "field_rules": list(sheet.get("field_rules") or []), + "row_filters": list(sheet.get("row_filters") or []), + "ignore_port_changes": sheet.get("ignore_port_changes"), + "mode": "presence" if not compare_fields else "fields", + "status": "pending", + "summary": { + "added": 0, + "removed": 0, + "changed": 0, + "unchanged": 0, + "before_count": 0, + "after_count": 0, + }, + } + + def _str_list(raw: Any) -> list[str]: return [str(x).strip() for x in (raw or []) if str(x).strip()] @@ -320,18 +669,41 @@ def _normalize_row_filters(raw: Any) -> list[dict[str, Any]]: return out +# Optional sheet-level switches that must survive normalization. +_SHEET_OPTION_VALUES: dict[str, tuple[str, ...]] = { + "unmapped_port_policy": ("by_name", "ignore_port"), + "duplicate_pairing": ("sorted", "ordered"), +} + + +def _sheet_options(raw: dict[str, Any]) -> dict[str, Any]: + out: dict[str, Any] = {} + for key, allowed in _SHEET_OPTION_VALUES.items(): + val = str(raw.get(key) or "").strip().lower() + if val in allowed: + out[key] = val + if raw.get("builtin_vendor_canonical") is False: + out["builtin_vendor_canonical"] = False + if raw.get("scope_by_port_map") is not None: + out["scope_by_port_map"] = bool(raw.get("scope_by_port_map")) + return out + + def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]: if not isinstance(raw, list): return [] out: list[dict[str, Any]] = [] - seen: set[str] = set() + seen: set[tuple[str, str]] = set() for item in raw: if not isinstance(item, dict): continue name = str(item.get("field") or "").strip() - if not name or name in seen: + side = str(item.get("side") or "").strip().lower() + side = side if side in ("before", "after") else "" + # A field may carry one rule per side (e.g. a different value_map each). + if not name or (name, side) in seen: continue - seen.add(name) + seen.add((name, side)) rule: dict[str, Any] = {"field": name} compare = str(item.get("compare") or "").strip().lower() if compare: @@ -347,6 +719,16 @@ def _normalize_field_rules(raw: Any) -> list[dict[str, Any]]: rule["tolerance"] = float(item.get("tolerance")) except (TypeError, ValueError): pass + vmap = item.get("value_map") + if isinstance(vmap, dict) and vmap: + rule["value_map"] = {str(k): str(v) for k, v in vmap.items()} + if side: + rule["side"] = side + if isinstance(item.get("apply_to_key"), bool): + rule["apply_to_key"] = item["apply_to_key"] + rank = item.get("rank") + if isinstance(rank, list) and rank: + rule["rank"] = [str(x) for x in rank] # Drop empty rules (only field name) if len(rule) > 1: out.append(rule) @@ -376,6 +758,7 @@ def _sheet_def( row_filters: list[dict[str, Any]] | None = None, field_rules: list[dict[str, Any]] | None = None, ignore_port_changes: bool | None = None, + options: dict[str, Any] | None = None, ) -> dict[str, Any]: mid = str(metric_id or "").strip() sid = str(sheet_id or "").strip() or mid @@ -416,6 +799,7 @@ def _sheet_def( } if ignore_port_changes is not None: sheet["ignore_port_changes"] = bool(ignore_port_changes) + sheet.update(options or {}) return sheet @@ -551,7 +935,35 @@ def _builtin_source_splits() -> dict[str, list[dict[str, Any]]]: } +def _packaged_zte_status_template_path(): + from pathlib import Path + + return Path(__file__).resolve().parent / "data" / "default_zte_status_template.json" + + +def _load_packaged_zte_status_template() -> dict[str, Any]: + """IOH CN migration sheet set shipped as the built-in status default.""" + path = _packaged_zte_status_template_path() + if not path.is_file(): + return {} + try: + return dict(json.loads(path.read_text(encoding="utf-8")) or {}) + except Exception: + _log.exception("failed to load packaged ZTE status template %s", path) + return {} + + def _default_zte_status_sheets() -> list[dict[str, Any]]: + """Built-in status sheets — prefer packaged IOH CN migration rules.""" + raw = _load_packaged_zte_status_template() + out: list[dict[str, Any]] = [] + for item in list(raw.get("metrics") or []): + sheet = _normalize_sheet(item) + if sheet: + out.append(sheet) + if out: + return out + # Fallback if package missing (tests / incomplete install) return [ *_isis_af_sheets(), _default_sheet_for_metric("interface_brief", compare_roles=("state",)), @@ -568,6 +980,66 @@ def _default_zte_status_sheets() -> list[dict[str, Any]]: ] +def _builtin_status_needs_packaged_upgrade(existing: list[dict[str, Any]]) -> bool: + """True when built-in template still lacks filtered BGP route sheets.""" + mids = {str(s.get("metric_id") or "") for s in existing} + if "bgp_route" not in mids and "l2vpn_mac" not in mids: + return True + has_filtered_route = any( + str(s.get("metric_id") or "") == "bgp_route" and list(s.get("row_filters") or []) + for s in existing + ) + if not has_filtered_route: + return True + packaged_keys = {sheet_key(s) for s in _default_zte_status_sheets()} + have_keys = {sheet_key(s) for s in existing} + # Missing several packaged sheet ids → sync to packaged default + return len(packaged_keys - have_keys) >= 3 + + +# A7: (metric_id, kind, field, old packaged value, new value). A stored sheet is +# only touched when its value still equals the *old* packaged one, so anything +# an operator edited survives the upgrade. +_INTERFACE_DETAIL_OLD_COMPARE = [ + "admin", "input_bps", "output_bps", "port_status", "in_util", "out_util", + "ip_mtu", "mtu", "mpls_mtu", "ipv6_mtu", "port_media", "negotiation", +] +_BUILTIN_STATUS_VALUE_UPGRADES: tuple[tuple[str, str, str, Any, Any], ...] = ( + ("interface_detail", "rule", "output_bps", + {"compare": "numeric", "tolerance": 5}, {"compare": "percent", "tolerance": 5}), + ("interface_detail", "list", "compare_fields", _INTERFACE_DETAIL_OLD_COMPARE, + [f for f in _INTERFACE_DETAIL_OLD_COMPARE if f not in ("input_bps", "output_bps", "in_util", "out_util")]), + ("optical_brief", "rule", "rx_power", + {"compare": "percent", "tolerance": 5}, {"compare": "numeric", "tolerance": 2}), + ("l2vpn_mac", "list", "iface_fields", [], ["ac_port"]), + # ARP: filter on the canonical ``entry_type`` (every vendor parser emits it); the + # age-timer clause stays only as a fallback for rows parsed before entry_type existed. + ("arp", "list", "row_filters", [{"op": "age_timer", "field": "age", "value": ""}], arp_dynamic_row_filters()), +) + + +def _upgrade_builtin_status_values(sheet: dict[str, Any]) -> tuple[dict[str, Any], bool]: + """Apply ``_BUILTIN_STATUS_VALUE_UPGRADES`` to one stored sheet.""" + mid = str(sheet.get("metric_id") or "") + out = deepcopy(sheet) + changed = False + for metric_id, kind, field, old, new in _BUILTIN_STATUS_VALUE_UPGRADES: + if mid != metric_id: + continue + if kind == "list": + if list(out.get(field) or []) == old: + out[field] = list(new) + changed = True + continue + for rule in out.get("field_rules") or []: + if isinstance(rule, dict) and rule.get("field") == field and all( + rule.get(k) == v for k, v in old.items() + ): + rule.update(new) + changed = True + return out, changed + + def _default_zte_config_sheets() -> list[dict[str, Any]]: """Config-intent metrics for cutover / intent-vs-intent compare.""" return [ @@ -631,6 +1103,7 @@ def _normalize_sheet(raw: Any) -> dict[str, Any] | None: row_filters=_normalize_row_filters(raw.get("row_filters")), field_rules=rules, ignore_port_changes=ignore_ports, + options=_sheet_options(raw), ) @@ -736,6 +1209,7 @@ def _parse_metrics_body(body: dict[str, Any]) -> list[dict[str, Any]]: display_fields=disp_arg, row_filters=_normalize_row_filters(body.get("row_filters")), field_rules=rules, + ignore_port_changes=body.get("ignore_port_changes"), ) ] @@ -834,52 +1308,65 @@ def ensure_default_lldp_template(db: Session) -> BizCompareTemplate: def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: name = "ZTE status default" row = db.query(BizCompareTemplate).filter(BizCompareTemplate.name == name).one_or_none() + packaged = _load_packaged_zte_status_template() sheets = _default_zte_status_sheets() - # Volatile counters must not stay in compare_fields on upgraded installs. - _STRIP_COMPARE: dict[str, frozenset[str]] = { - "interface_detail": frozenset({"input_bps", "output_bps", "in_util", "out_util"}), - "optical_brief": frozenset({"rx_power", "tx_power"}), - "bgp_peer": frozenset({"pfx_rcd"}), - } + note = str( + packaged.get("note") + or "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP route AF sheets)" + )[:512] + iface_rules = list(packaged.get("iface_normalize_rules") or []) if row: existing = template_metrics(row) - want = {s["metric_id"] for s in sheets} - have = {s["metric_id"] for s in existing} - changed = bool(want - have) + if _builtin_status_needs_packaged_upgrade(existing) and sheets: + _apply_sheets_to_row(row, sheets) + row.note = note + row.updated_at = _utcnow() + if iface_rules: + _set_template_iface_normalize(row, iface_rules) + elif not template_iface_normalize(row): + # Packaged IOH rules use empty normalize; leave empty when explicit + _set_template_iface_normalize(row, []) + db.commit() + db.refresh(row) + return row + # Incremental patches for already-upgraded installs + changed = False upgraded: list[dict[str, Any]] = [] - by_want = {s["metric_id"]: s for s in sheets} + by_sid = {sheet_key(s): s for s in sheets} for s in existing: cur = dict(s) mid = str(cur.get("metric_id") or "") + sid = sheet_key(cur) if mid == "arp" and not cur.get("row_filters"): - cur["row_filters"] = list(by_want.get("arp", {}).get("row_filters") or arp_dynamic_row_filters()) - if not cur.get("field_rules") and by_want.get("arp", {}).get("field_rules"): - cur["field_rules"] = list(by_want["arp"]["field_rules"]) + src = by_sid.get(sid) or next( + (x for x in sheets if x.get("metric_id") == "arp"), None + ) + cur["row_filters"] = list( + (src or {}).get("row_filters") or arp_dynamic_row_filters() + ) + if not cur.get("field_rules") and src and src.get("field_rules"): + cur["field_rules"] = list(src["field_rules"]) changed = True - strip = _STRIP_COMPARE.get(mid) - if strip: - old_cmp = list(cur.get("compare_fields") or []) - new_cmp = [f for f in old_cmp if f not in strip] - if new_cmp != old_cmp: - cur["compare_fields"] = new_cmp - want_disp = list((by_want.get(mid) or {}).get("display_fields") or []) - if want_disp: - cur["display_fields"] = want_disp - changed = True + cur, value_changed = _upgrade_builtin_status_values(cur) + changed = changed or value_changed upgraded.append(_normalize_sheet(cur) or cur) - if want - have: - for s in sheets: - if s["metric_id"] not in have: + have_mids = {str(s.get("metric_id") or "") for s in upgraded} + for s in sheets: + if str(s.get("metric_id") or "") not in have_mids: + # Only append wholly missing metrics (e.g. bgp_route family) + if str(s.get("metric_id") or "") == "bgp_route" and "bgp_route" not in have_mids: + upgraded.extend( + [x for x in sheets if x.get("metric_id") == "bgp_route"] + ) + have_mids.add("bgp_route") + changed = True + elif str(s.get("metric_id") or "") not in have_mids: upgraded.append(s) - changed = True + have_mids.add(str(s.get("metric_id") or "")) + changed = True if changed: _apply_sheets_to_row(row, upgraded if upgraded else sheets) - row.note = "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)" - row.updated_at = _utcnow() - db.commit() - db.refresh(row) - if not template_iface_normalize(row): - _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) + row.note = note row.updated_at = _utcnow() db.commit() db.refresh(row) @@ -887,12 +1374,12 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: row = BizCompareTemplate( id=uuid4().hex, name=name, - note="Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)", + note=note, created_at=_utcnow(), updated_at=_utcnow(), ) _apply_sheets_to_row(row, sheets) - _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) + _set_template_iface_normalize(row, iface_rules) db.add(row) db.commit() db.refresh(row) @@ -948,10 +1435,10 @@ def ensure_default_templates(db: Session) -> None: def upgrade_builtin_split_sheets(db: Session) -> None: - """Split unfiltered whole-table sheets on the built-in ZTE template only. + """Upgrade built-in ZTE status template to packaged AF / BGP route sheets. - A sheet is replaced when its id is still the source metric and it has no - row filters. Custom templates and already-split sheets are left alone. + Custom templates are left alone. Built-in is replaced wholesale when it + still lacks filtered ``bgp_route`` sheets (IOH CN migration default). """ row = ( db.query(BizCompareTemplate) @@ -961,6 +1448,18 @@ def upgrade_builtin_split_sheets(db: Session) -> None: if not row: return existing = template_metrics(row) + packaged_sheets = _default_zte_status_sheets() + if _builtin_status_needs_packaged_upgrade(existing) and packaged_sheets: + packaged = _load_packaged_zte_status_template() + _apply_sheets_to_row(row, packaged_sheets) + row.note = str( + packaged.get("note") + or "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP route AF sheets)" + )[:512] + _set_template_iface_normalize(row, list(packaged.get("iface_normalize_rules") or [])) + row.updated_at = _utcnow() + db.commit() + return splits = _builtin_source_splits() out: list[dict[str, Any]] = [] changed = False @@ -986,7 +1485,17 @@ def list_templates(db: Session) -> list[dict[str, Any]]: ensure_default_templates(db) upgrade_builtin_split_sheets(db) rows = db.query(BizCompareTemplate).order_by(BizCompareTemplate.name.asc()).all() - return [_template_out(t) for t in rows] + usage = dict(db.query(BizCompareJob.template_id, func.count(BizCompareJob.id)).group_by(BizCompareJob.template_id).all()) + monitor_usage = dict( + db.query(BizMonitorTemplate.compare_template_id, func.count(BizMonitorTemplate.id)) + .filter(BizMonitorTemplate.compare_template_id != "") + .group_by(BizMonitorTemplate.compare_template_id).all() + ) + return [ + {**_template_out(t), "job_count": usage.get(t.id, 0), + "monitor_template_count": monitor_usage.get(t.id, 0)} + for t in rows + ] def list_metric_schemas() -> list[dict[str, Any]]: @@ -1023,6 +1532,7 @@ def list_row_filter_presets() -> list[dict[str, Any]]: def create_template(db: Session, body: dict[str, Any]) -> dict[str, Any]: + validate_template_body(body) sheets = _parse_metrics_body(body) t = BizCompareTemplate( id=uuid4().hex, @@ -1045,6 +1555,7 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict t = db.get(BizCompareTemplate, template_id) if not t: raise HTTPException(status_code=404, detail="template_not_found") + validate_template_body(body, partial=True) if "name" in body: t.name = str(body.get("name") or "")[:256] if "note" in body: @@ -1066,6 +1577,7 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict "display_fields", "row_filters", "field_rules", + "ignore_port_changes", ) ): # Prefer explicit metrics; otherwise merge into current sheets from legacy keys @@ -1103,6 +1615,8 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict first["row_filters"] = _normalize_row_filters(body.get("row_filters")) if "field_rules" in body: first["field_rules"] = _normalize_field_rules(body.get("field_rules")) + if "ignore_port_changes" in body: + first["ignore_port_changes"] = body["ignore_port_changes"] sheets[0] = _normalize_sheet(first) or first _apply_sheets_to_row(t, sheets) t.updated_at = _utcnow() @@ -1114,6 +1628,19 @@ def delete_template(db: Session, template_id: str) -> None: t = db.get(BizCompareTemplate, template_id) if not t: raise HTTPException(status_code=404, detail="template_not_found") + jobs = [n or i for i, n in db.query(BizCompareJob.id, BizCompareJob.name) + .filter(BizCompareJob.template_id == template_id).all()] + monitors = [n or i for i, n in db.query(BizMonitorTemplate.id, BizMonitorTemplate.name) + .filter(BizMonitorTemplate.compare_template_id == template_id).all()] + if jobs or monitors: + raise HTTPException(status_code=409, detail={ + "code": "template_in_use", + "message": "template_in_use: " + "; ".join(filter(None, [ + f"jobs={', '.join(jobs)}" if jobs else "", + f"monitor_templates={', '.join(monitors)}" if monitors else "", + ])), + "jobs": jobs, "monitor_templates": monitors, + }) db.delete(t) db.commit() @@ -1134,9 +1661,30 @@ def _mapping_out(db: Session, m: BizPortMapping) -> dict[str, Any]: } +def _mapping_references(db: Session, mapping_id: str) -> tuple[list[str], list[str]]: + """(compare job names, cutover project names) that reference this mapping.""" + jobs = [n or i for i, n in db.query(BizCompareJob.id, BizCompareJob.name) + .filter(BizCompareJob.mapping_id == mapping_id).all()] + projects = [n or i for i, n in db.query(BizMigrationProject.id, BizMigrationProject.name) + .filter(BizMigrationProject.mapping_id == mapping_id).all()] + return jobs, projects + + def list_mappings(db: Session) -> list[dict[str, Any]]: rows = db.query(BizPortMapping).order_by(BizPortMapping.name.asc()).all() - return [_mapping_out(db, m) for m in rows] + job_counts = dict( + db.query(BizCompareJob.mapping_id, func.count(BizCompareJob.id)) + .filter(BizCompareJob.mapping_id != "").group_by(BizCompareJob.mapping_id).all() + ) + project_counts = dict( + db.query(BizMigrationProject.mapping_id, func.count(BizMigrationProject.id)) + .filter(BizMigrationProject.mapping_id != "").group_by(BizMigrationProject.mapping_id).all() + ) + return [ + {**_mapping_out(db, m), "job_count": job_counts.get(m.id, 0), + "project_count": project_counts.get(m.id, 0)} + for m in rows + ] def create_mapping(db: Session, body: dict[str, Any]) -> dict[str, Any]: @@ -1194,6 +1742,16 @@ def delete_mapping(db: Session, mapping_id: str) -> None: m = db.get(BizPortMapping, mapping_id) if not m: raise HTTPException(status_code=404, detail="mapping_not_found") + jobs, projects = _mapping_references(db, mapping_id) + if jobs or projects: + raise HTTPException(status_code=409, detail={ + "code": "mapping_in_use", + "message": "mapping_in_use: " + "; ".join(filter(None, [ + f"jobs={', '.join(jobs)}" if jobs else "", + f"projects={', '.join(projects)}" if projects else "", + ])), + "jobs": jobs, "projects": projects, + }) db.query(BizPortMappingRow).filter(BizPortMappingRow.mapping_id == mapping_id).delete() db.delete(m) db.commit() @@ -1206,61 +1764,191 @@ def _port_map_dict(db: Session, mapping_id: str) -> dict[str, str]: return {str(r.before_if): str(r.after_if) for r in rows if r.before_if and r.after_if} -def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dict[str, Any]]: - if metric_id == "lldp_neighbor": - rows = ( - db.query(BizStateLldpNeighbor) - .filter(BizStateLldpNeighbor.batch_id == batch_id) - .all() - ) - return [ - { - "local_if": n.local_if, - "remote_sys": n.remote_sys, - "remote_if": n.remote_if, - "remote_ip": n.remote_ip, - "protocol": n.protocol, - "_netx": { - "batch_id": batch_id, - "batch_command_id": n.batch_command_id or "", - "task_id": n.task_id or "", - "ne_id": n.ne_id or "", - "collected_at": n.collected_at.isoformat() + "Z" if n.collected_at else None, - "row_id": n.id, - }, - } - for n in rows - ] - # Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …) - from ..models import BizStateMetricRow +def _load_metric_rows( + db: Session, + *, + batch_id: str, + metric_id: str, + on_chunk: Callable[[int], None] | None = None, + row_filters: list[dict[str, Any]] | None = None, +) -> list[dict[str, Any]]: + """Load metric rows in keyset chunks (stable on million-row sheets). - rows = ( - db.query(BizStateMetricRow) - .filter( + When ``row_filters`` are SQL-pushdown-safe on PostgreSQL, they are applied in + the SELECT (critical for BGP afi/vrf sheet splits — avoids loading 1M+ then + discarding). Otherwise filters are applied in Python after each chunk. + """ + from .compare_sql import ( + _dialect_is_postgres, + _filters_sql_compatible, + compile_row_filters_sql, + ) + + filters = [f for f in (row_filters or []) if isinstance(f, dict)] + pushdown = bool( + filters and _dialect_is_postgres(db) and _filters_sql_compatible(filters) + ) + filter_sql, filter_params = ("TRUE", {}) + if pushdown: + filter_sql, filter_params = compile_row_filters_sql(filters) + + if metric_id == "lldp_neighbor": + out: list[dict[str, Any]] = [] + last_id = "" + while True: + q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id) + if last_id: + q = q.filter(BizStateLldpNeighbor.id > last_id) + chunk = q.order_by(BizStateLldpNeighbor.id.asc()).limit(_LOAD_YIELD_PER).all() + if not chunk: + break + for n in chunk: + row = { + "local_if": n.local_if, + "remote_sys": n.remote_sys, + "remote_if": n.remote_if, + "remote_ip": n.remote_ip, + "protocol": n.protocol, + "_netx": { + "batch_id": batch_id, + "batch_command_id": n.batch_command_id or "", + "task_id": n.task_id or "", + "ne_id": n.ne_id or "", + "collected_at": n.collected_at.isoformat() + "Z" + if n.collected_at + else None, + "row_id": n.id, + }, + } + if filters and not pushdown and not all( + row_matches_filter(row, f) for f in filters + ): + db.expunge(n) + continue + out.append(row) + db.expunge(n) + last_id = str(chunk[-1].id) + if on_chunk: + on_chunk(len(out)) + if len(chunk) < _LOAD_YIELD_PER: + break + if filters and not pushdown: + return apply_row_filters(out, filters) + return out + + # Generic tabular metrics — PG + pushdown uses SQL keyset with JSON filters + from ..models import BizStateMetricRow + from sqlalchemy import text as sql_text + + out: list[dict[str, Any]] = [] + last_seq = -1 + last_id = "" + while True: + if pushdown: + params = { + "bid": batch_id, + "mid": metric_id, + "last_seq": last_seq, + "last_id": last_id, + "lim": int(_LOAD_YIELD_PER), + **filter_params, + } + keyset = ( + "(seq > :last_seq OR (seq = :last_seq AND id > :last_id))" + if last_id + else "TRUE" + ) + rows = db.execute( + sql_text( + f""" + SELECT id, batch_command_id, task_id, ne_id, seq, data_json, collected_at + FROM biz_state_metric_row + WHERE batch_id = :bid + AND metric_id = :mid + AND ({filter_sql}) + AND ({keyset}) + ORDER BY seq ASC, id ASC + LIMIT :lim + """ + ), + params, + ).mappings().all() + if not rows: + break + for r in rows: + data = dict(r["data_json"] or {}) + collected = r["collected_at"] + out.append( + { + **data, + "_netx": { + "batch_id": batch_id, + "batch_command_id": str(r["batch_command_id"] or ""), + "task_id": str(r["task_id"] or ""), + "ne_id": str(r["ne_id"] or ""), + "collected_at": collected.isoformat() + "Z" + if collected is not None + else None, + "row_id": str(r["id"]), + }, + } + ) + last_seq = int(rows[-1]["seq"] or 0) + last_id = str(rows[-1]["id"]) + if on_chunk: + on_chunk(len(out)) + if len(rows) < _LOAD_YIELD_PER: + break + continue + + q = db.query(BizStateMetricRow).filter( BizStateMetricRow.batch_id == batch_id, BizStateMetricRow.metric_id == metric_id, ) - .order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) - .all() - ) - if rows: - # Raw rows only — filtering belongs to the compare sheet template - # (``row_filters``), not metric-specific branches here. - # ``_netx`` is collector provenance (stripped before field compare). - return [ - { + if last_id: + q = q.filter( + or_( + BizStateMetricRow.seq > last_seq, + and_( + BizStateMetricRow.seq == last_seq, + BizStateMetricRow.id > last_id, + ), + ) + ) + chunk = ( + q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) + .limit(_LOAD_YIELD_PER) + .all() + ) + if not chunk: + break + for r in chunk: + row = { **dict(r.data_json or {}), "_netx": { "batch_id": batch_id, "batch_command_id": r.batch_command_id or "", "task_id": r.task_id or "", "ne_id": r.ne_id or "", - "collected_at": r.collected_at.isoformat() + "Z" if r.collected_at else None, + "collected_at": r.collected_at.isoformat() + "Z" + if r.collected_at + else None, "row_id": r.id, }, } - for r in rows - ] + if filters and not all(row_matches_filter(row, f) for f in filters): + db.expunge(r) + continue + out.append(row) + db.expunge(r) + last_seq = int(chunk[-1].seq or 0) + last_id = str(chunk[-1].id) + if on_chunk: + on_chunk(len(out)) + if len(chunk) < _LOAD_YIELD_PER: + break + if out: + return out # Known metric with zero rows is OK; unknown metric still errors if metric_id in metric_field_map(): return [] @@ -1282,11 +1970,11 @@ def validate_mapping( sheets = template_metrics(tpl) if not sheets: raise HTTPException(status_code=400, detail="template_has_no_metrics") - pmap = _port_map_dict(db, mapping_id) # Validate against first sheet that has iface fields (or first sheet) target = next((s for s in sheets if s.get("iface_fields")), sheets[0]) iface_fields = list(target.get("iface_fields") or []) norm = template_iface_normalize(tpl) + pmap, _ = normalize_port_map(_port_map_dict(db, mapping_id), norm) before = apply_iface_normalize_rows( _load_metric_rows(db, batch_id=before_batch_id, metric_id=target["metric_id"]), iface_fields=iface_fields, @@ -1305,6 +1993,66 @@ def validate_mapping( ) +def _mapping_rows_to_dict(rows: list[Any]) -> tuple[dict[str, str], list[dict[str, str]]]: + """Editor rows → raw map; a repeated before port is reported, first one wins.""" + out: dict[str, str] = {} + dups: list[dict[str, str]] = [] + for r in rows or []: + raw = r if isinstance(r, dict) else dict(r) + before = str(raw.get("before_if") or "").strip() + after = str(raw.get("after_if") or "").strip() + if not before or not after: + continue + if before in out: + dups.append({"before_if": before, "kept_to": out[before], "dropped_to": after}) + continue + out[before] = after + return out, dups + + +def preflight_compare(db: Session, body: dict[str, Any]) -> dict[str, Any]: + """Mapping coverage + data-quality report for a template and two batches.""" + from .compare_preflight import preflight + + ensure_default_templates(db) + template_id = str(body.get("template_id") or "") + tpl = db.get(BizCompareTemplate, template_id) if template_id else ensure_default_cutover_template(db) + if not tpl: + raise HTTPException(status_code=404, detail="template_not_found") + before_id = str(body.get("before_batch_id") or "") + after_id = str(body.get("after_batch_id") or "") + if not db.get(BizStateBatch, before_id) or not db.get(BizStateBatch, after_id): + raise HTTPException(status_code=404, detail="batch_not_found") + sheets = _filter_enabled_sheets(template_metrics(tpl), _str_list(body.get("enabled_sheet_ids"))) + if not sheets: + raise HTTPException(status_code=400, detail="no_enabled_sheets") + dups: list[dict[str, str]] = [] + if body.get("mapping_rows") is not None: + raw_map, dups = _mapping_rows_to_dict(list(body.get("mapping_rows") or [])) + else: + raw_map = _port_map_dict(db, str(body.get("mapping_id") or "")) + cache: dict[tuple[str, str], list[dict[str, Any]]] = {} + + def load(side: str, sheet: dict[str, Any]) -> list[dict[str, Any]]: + batch_id = before_id if side == "before" else after_id + key = (batch_id, str(sheet["metric_id"])) + if key not in cache: + cache[key] = _load_metric_rows(db, batch_id=batch_id, metric_id=key[1]) + return cache[key] + + report = preflight( + db, sheets=sheets, iface_rules=template_iface_normalize(tpl), raw_port_map=raw_map, + before_batch_id=before_id, after_batch_id=after_id, + before_vendor=_batch_vendor_key(db, before_id), after_vendor=_batch_vendor_key(db, after_id), + load_rows=load, + ) + if dups: + report["mapping"]["duplicate_before"] = dups + report["risk_reasons"] = sorted({*report["risk_reasons"], "duplicate_before"}) + report["verdict"] = "risk" + return report + + def _job_out(j: BizCompareJob) -> dict[str, Any]: return { "id": j.id, @@ -1318,6 +2066,7 @@ def _job_out(j: BizCompareJob) -> dict[str, Any]: "mode": j.mode, "status": j.status, "enabled_sheet_ids": _str_list(getattr(j, "enabled_sheet_ids", None)), + "store_unchanged": normalize_store_unchanged(getattr(j, "store_unchanged", None)), "note": j.note, "updated_at": j.updated_at.isoformat() + "Z" if j.updated_at else None, } @@ -1338,6 +2087,16 @@ def list_jobs(db: Session) -> list[dict[str, Any]]: return [_job_out(j) for j in rows] +def _validate_job_sheets(db: Session, template_id: str, enabled: list[str]) -> None: + tpl = db.get(BizCompareTemplate, template_id) + if not tpl: + raise HTTPException(status_code=404, detail="template_not_found") + sheets = {sheet_key(s) for s in template_metrics(tpl)} + unknown = sorted(set(enabled) - sheets) + if unknown: + raise HTTPException(status_code=400, detail={"error": "unknown_enabled_sheets", "sheet_ids": unknown}) + + def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: ensure_default_templates(db) template_id = str(body.get("template_id") or "").strip() @@ -1348,6 +2107,7 @@ def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: if not db.get(BizCompareTemplate, template_id): raise HTTPException(status_code=404, detail="template_not_found") enabled = _str_list(body.get("enabled_sheet_ids")) + _validate_job_sheets(db, template_id, enabled) j = BizCompareJob( id=uuid4().hex, name=str(body.get("name") or "compare")[:256], @@ -1360,6 +2120,7 @@ def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: mode=str(body.get("mode") or "manual")[:16], status="ready", enabled_sheet_ids=enabled, + store_unchanged=normalize_store_unchanged(body.get("store_unchanged")), note=str(body.get("note") or "")[:512], created_at=_utcnow(), updated_at=_utcnow(), @@ -1375,6 +2136,8 @@ def update_job(db: Session, job_id: str, body: dict[str, Any]) -> dict[str, Any] j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") + _validate_job_sheets(db, str(body.get("template_id", j.template_id) or ""), + _str_list(body.get("enabled_sheet_ids", j.enabled_sheet_ids))) for key in ( "name", "template_id", @@ -1391,6 +2154,8 @@ def update_job(db: Session, job_id: str, body: dict[str, Any]) -> dict[str, Any] setattr(j, key, str(body.get(key) or "")) if "enabled_sheet_ids" in body: j.enabled_sheet_ids = _str_list(body.get("enabled_sheet_ids")) + if "store_unchanged" in body: + j.store_unchanged = normalize_store_unchanged(body.get("store_unchanged")) j.updated_at = _utcnow() db.commit() return _job_out(j) @@ -1400,6 +2165,9 @@ def delete_job(db: Session, job_id: str) -> None: j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") + if db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id, + BizCompareRun.status.in_(("queued", "running"))).first(): + raise HTTPException(status_code=409, detail="compare_running") run_ids = [ rid for (rid,) in db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id).all() ] @@ -1417,6 +2185,8 @@ def delete_run(db: Session, run_id: str) -> dict[str, Any]: r = db.get(BizCompareRun, run_id) if not r: raise HTTPException(status_code=404, detail="run_not_found") + if r.status in ("queued", "running"): + raise HTTPException(status_code=409, detail="compare_running") job_id = str(r.job_id or "") db.query(BizCompareDiff).filter(BizCompareDiff.run_id == run_id).delete( synchronize_session=False @@ -1444,14 +2214,59 @@ def _resolve_after_batch(db: Session, job: BizCompareJob) -> str: return str(latest.id) if latest else "" +def _sheet_result_envelope( + sheet: dict[str, Any], + *, + key_fields: list[str], + iface_fields: list[str], + compare_fields: list[str], + display_fields: list[str], + row_filters: list[Any], + field_rules: list[Any], + ignore_ports: bool | None, + mode: str, + summary: dict[str, Any], + diffs: list[Any], + mapping_stats: dict[str, Any], +) -> dict[str, Any]: + return { + "sheet_id": sheet_key(sheet), + "title": sheet_title(sheet), + "metric_id": sheet["metric_id"], + "key_fields": key_fields, + "iface_fields": iface_fields, + "compare_fields": compare_fields, + "display_fields": display_fields, + "row_filters": row_filters, + "field_rules": field_rules, + "ignore_port_changes": ignore_ports, + "mode": mode, + "status": "done", + "summary": summary, + "diffs": diffs, + "mapping_stats": mapping_stats, + } + + +def _sheet_quality( + db: Session, sheet: dict[str, Any], before_batch_id: str, after_batch_id: str, + summary: dict[str, Any], +) -> dict[str, Any]: + return assess_sheet( + db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id, + before_rows=int(summary.get("before_count") or 0), after_rows=int(summary.get("after_count") or 0), + ) + + def _run_sheet( db: Session, *, sheet: dict[str, Any], before_batch_id: str, after_batch_id: str, - port_map: dict[str, str], - iface_normalize_rules: list[dict[str, str]] | None = None, + pipeline: Pipeline, + store_unchanged: str = "auto", + on_load_progress: Callable[..., None] | None = None, ) -> dict[str, Any]: key_fields = list(sheet.get("key_fields") or []) iface_fields = list(sheet.get("iface_fields") or []) @@ -1472,79 +2287,472 @@ def _run_sheet( ignore_ports = sheet.get("ignore_port_changes") if ignore_ports is not None: ignore_ports = bool(ignore_ports) - before_raw = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"]) - after_raw = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"]) - before_rows = apply_row_filters(before_raw, row_filters) - after_rows = apply_row_filters(after_raw, row_filters) + mid = sheet["metric_id"] + + def _emit_load(side: str, n: int, **meta: Any) -> None: + if not on_load_progress: + return + try: + on_load_progress(side, n, **meta) + except TypeError: + on_load_progress(side, n) + + # PostgreSQL path: pushdown-safe sheets join in-DB (BGP-scale). + from .compare_sql import SqlCompareSkip, run_sql_sheet_compare, sql_compare_skip_reason + + skip_reason = sql_compare_skip_reason(db, sheet, pipeline=pipeline) + if not skip_reason: + try: + result = run_sql_sheet_compare( + db, + sheet=sheet, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + store_unchanged=store_unchanged, + on_progress=_emit_load, + duplicate_pairing=pipeline.duplicate_pairing, + ) + summary = dict(result["summary"]) + # Engine already sets raw counts / policy; keep keys stable + if "unchanged_policy" not in summary: + summary["unchanged_policy"] = resolve_unchanged_policy( + store_unchanged, + before_n=int(summary.get("before_count") or 0), + after_n=int(summary.get("after_count") or 0), + ) + summary.update(_sheet_quality(db, sheet, before_batch_id, after_batch_id, summary)) + return _sheet_result_envelope( + sheet, + key_fields=key_fields, + iface_fields=iface_fields, + compare_fields=compare_fields, + display_fields=display_fields, + row_filters=row_filters, + field_rules=field_rules, + ignore_ports=ignore_ports, + mode=mode, + summary=summary, + diffs=list(result.get("diffs") or []), + mapping_stats=dict(result.get("mapping_stats") or {}), + ) + except SqlCompareSkip as skip: + skip_reason = skip.reason or "skip" + _log.info( + "python compare sheet=%s metric=%s skip_sql=%s", + sheet_key(sheet), + mid, + skip_reason, + ) + try: + db.rollback() + except Exception: + pass + except Exception: + _log.exception( + "sql compare fallback sheet=%s metric=%s — using Python engine", + sheet_key(sheet), + mid, + ) + skip_reason = "sql_error_fallback" + # Roll back aborted SQL transaction so Python path can use the session + try: + db.rollback() + except Exception: + pass + else: + _log.info( + "python compare sheet=%s metric=%s skip_sql=%s", + sheet_key(sheet), + mid, + skip_reason, + ) + + _emit_load("before", 0, engine="python", note=skip_reason or "python", phase="loading") + + def _before_chunk(n: int) -> None: + _emit_load("before", n, engine="python", note=skip_reason or "python", phase="loading") + + def _after_chunk(n: int) -> None: + _emit_load("after", n, engine="python", note=skip_reason or "python", phase="loading") + + # Only filters that read untransformed fields may run in SQL on the raw rows; + # the pipeline applies every filter (post-normalize for v2) in ``prepare``. + before_raw = _load_metric_rows( + db, + batch_id=before_batch_id, + metric_id=mid, + on_chunk=_before_chunk, + row_filters=pipeline.pushdown_filters("before"), + ) + after_raw = _load_metric_rows( + db, + batch_id=after_batch_id, + metric_id=mid, + on_chunk=_after_chunk, + row_filters=pipeline.pushdown_filters("after"), + ) + before_pairs = pipeline.prepare(before_raw, "before") + after_pairs = pipeline.prepare(after_raw, "after") + policy = resolve_unchanged_policy( + store_unchanged, before_n=len(before_pairs), after_n=len(after_pairs) + ) result = compare_rows( - before_rows=before_rows, - after_rows=after_rows, + before_rows=[], after_rows=[], key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields, - port_map=port_map, field_rules=field_rules, - iface_normalize_rules=iface_normalize_rules, - ignore_port_changes=ignore_ports, + include_unchanged=bool(policy["include"]), + unchanged_limit=policy.get("limit"), + compact_unchanged=bool(policy.get("compact")), + pipeline=pipeline, + prepared=(before_pairs, after_pairs), ) summary = dict(result["summary"]) summary["before_raw_count"] = len(before_raw) summary["after_raw_count"] = len(after_raw) summary["row_filters"] = len(row_filters) - return { - "sheet_id": sheet_key(sheet), - "title": sheet_title(sheet), - "metric_id": sheet["metric_id"], - "key_fields": key_fields, - "iface_fields": iface_fields, - "compare_fields": compare_fields, - "display_fields": display_fields, - "row_filters": row_filters, - "field_rules": field_rules, - "ignore_port_changes": ignore_ports, - "mode": mode, - "summary": summary, - "diffs": result["diffs"], - "mapping_stats": result["mapping_stats"], - } + summary["unchanged_policy"] = policy + summary.setdefault("engine", "python") + if pipeline.port_map and pipeline.iface_fields: + from .compare_preflight import sheet_mapping_coverage + + coverage = sheet_mapping_coverage(pipeline, before_pairs, after_pairs) + coverage.pop("_after_ifaces", None) + coverage.pop("_used_keys", None) + summary["mapping_coverage"] = coverage + summary.update(_sheet_quality(db, sheet, before_batch_id, after_batch_id, summary)) + return _sheet_result_envelope( + sheet, + key_fields=key_fields, + iface_fields=iface_fields, + compare_fields=compare_fields, + display_fields=display_fields, + row_filters=row_filters, + field_rules=field_rules, + ignore_ports=ignore_ports, + mode=mode, + summary=summary, + diffs=list(result.get("diffs") or []), + mapping_stats=dict(result.get("mapping_stats") or {}), + ) -def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]: - lock = _job_compare_lock(job_id) - if not lock.acquire(blocking=False): - raise HTTPException(status_code=409, detail="compare_already_running") - try: - return _run_compare_unlocked(db, job_id, force_after_batch_id=force_after_batch_id) - finally: - lock.release() - - -def _run_compare_unlocked( +def _validate_compare_job( db: Session, job_id: str, *, force_after_batch_id: str = "" ) -> dict[str, Any]: + """Resolve job/template/batches/sheets; raises HTTPException on bad input.""" j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") tpl = db.get(BizCompareTemplate, j.template_id) if not tpl: raise HTTPException(status_code=404, detail="template_not_found") + if j.mapping_id and not db.get(BizPortMapping, j.mapping_id): + raise HTTPException(status_code=400, detail="mapping_missing") before_batch_id = str(j.before_batch_id or "") after_batch_id = str(force_after_batch_id or "").strip() or _resolve_after_batch(db, j) if not before_batch_id or not after_batch_id: raise HTTPException(status_code=400, detail="before_and_after_batch_required") - if not db.get(BizStateBatch, before_batch_id) or not db.get(BizStateBatch, after_batch_id): + source_batches = [db.get(BizStateBatch, bid) for bid in (before_batch_id, after_batch_id)] + if not all(source_batches): raise HTTPException(status_code=404, detail="batch_not_found") + if any(b.status != "success" for b in source_batches): + raise HTTPException(status_code=409, detail="source_batch_not_complete") + # Existing records may predate write validation. Do not normalize invalid + # sheets away or run unknown rules as successful comparisons. + if tpl.metrics_json: + validate_template_body({"metrics": tpl.metrics_json, + "iface_normalize_rules": tpl.iface_normalize_json}) sheets_cfg = template_metrics(tpl) if not sheets_cfg: raise HTTPException(status_code=400, detail="template_has_no_metrics") + _validate_job_sheets(db, tpl.id, _str_list(j.enabled_sheet_ids)) sheets_cfg = _filter_enabled_sheets(sheets_cfg, getattr(j, "enabled_sheet_ids", None)) if not sheets_cfg: raise HTTPException(status_code=400, detail="no_enabled_sheets") + sheets_cfg = _order_sheets_small_first( + db, + sheets_cfg, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + ) + return { + "job": j, + "template": tpl, + "before_batch_id": before_batch_id, + "after_batch_id": after_batch_id, + "sheets_cfg": sheets_cfg, + } - pmap = _port_map_dict(db, j.mapping_id) - norm_rules = template_iface_normalize(tpl) - sheet_results: list[dict[str, Any]] = [] + +def _create_running_run( + db: Session, + *, + job: BizCompareJob, + tpl: BizCompareTemplate, + before_batch_id: str, + after_batch_id: str, + sheets_cfg: list[dict[str, Any]], + purpose: str = "manual", +) -> BizCompareRun: + first_metric = str(sheets_cfg[0].get("metric_id") or "") + pending_sheets = [_pending_sheet_meta(s) for s in sheets_cfg] + mapping = db.get(BizPortMapping, job.mapping_id) if job.mapping_id else None + vendors = { + "before": _batch_vendor_key(db, before_batch_id), + "after": _batch_vendor_key(db, after_batch_id), + } + raw_port_map = _port_map_dict(db, str(job.mapping_id or "")) + iface_rules = template_iface_normalize(tpl) + identity = { + sheet_key(sh): build_pipeline( + sh, iface_rules=iface_rules, port_map=raw_port_map, + before_vendor=vendors["before"], after_vendor=vendors["after"], + ).describe() + for sh in sheets_cfg + } + config_snapshot = deepcopy({ + "version": CONFIG_SNAPSHOT_VERSION, + "template": _template_out(tpl), + "sheets": sheets_cfg, + "port_map": raw_port_map, + "vendors": vendors, + "identity": identity, + "store_unchanged": normalize_store_unchanged(getattr(job, "store_unchanged", None)), + "job_name": job.name or "", + "before_task_id": job.before_task_id or "", + "after_task_id": job.after_task_id or "", + "mapping_name": (mapping.name if mapping else "") or "", + }) + run = BizCompareRun( + id=uuid4().hex, + job_id=job.id, + template_id=tpl.id, + mapping_id=job.mapping_id, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + metric_id=first_metric, + status="running", + summary_json={ + "purpose": purpose, + "config_snapshot": config_snapshot, + "progress": { + "phase": "queued", + "sheet_index": 0, + "sheet_total": len(sheets_cfg), + "sheet_id": "", + "sheet_title": "", + "elapsed_ms": 0, + }, + "sheet_count": len(sheets_cfg), + "added": 0, + "removed": 0, + "changed": 0, + "unchanged": 0, + "duplicate": 0, + "before_count": 0, + "after_count": 0, + "sheets": pending_sheets, + }, + diffs_json=[], + mapping_stats_json={}, + message="queued", + created_at=_utcnow(), + ) + db.add(run) + db.commit() + db.refresh(run) + return run + + +def _run_config_snapshot(run: BizCompareRun) -> dict[str, Any] | None: + snapshot = (run.summary_json or {}).get("config_snapshot") + if isinstance(snapshot, dict) and snapshot.get("version") in (1, 2): + return snapshot + return None + + +def _run_transform_config( + db: Session, run: BizCompareRun, +) -> tuple[dict[str, str], list[dict[str, str]]]: + snapshot = _run_config_snapshot(run) + if snapshot is not None: + rules = normalize_iface_rules( + (snapshot.get("template") or {}).get("iface_normalize_rules") or [], + ) + pmap = dict(snapshot.get("port_map") or {}) + if int(snapshot.get("version") or 1) >= 2: + # v2 runs: map keys/values are written as the device prints them. + pmap, _ = normalize_port_map(pmap, rules) + return pmap, rules + # Legacy runs did not record these settings; retain their fallback behavior. + tpl = db.get(BizCompareTemplate, run.template_id) if run.template_id else None + return _port_map_dict(db, str(run.mapping_id or "")), template_iface_normalize(tpl) + + +def _batch_vendor_key(db: Session, batch_id: str) -> str: + """Vendor key (``huawei``/``zte``/…) for a batch, '' when unknown.""" + from ..topology_lldp import resolve_vendor_key + + batch = db.get(BizStateBatch, batch_id) if batch_id else None + if batch is None: + return "" + task = db.get(BizStateTask, batch.task_id) if batch.task_id else None + key = resolve_vendor_key( + batch.vendor or (task.vendor if task else ""), (task.device_type if task else "") or "", + ) + return "" if key == "generic" else key + + +def _run_pipeline(db: Session, run: BizCompareRun | None, sheet: dict[str, Any]) -> Pipeline: + """The identity pipeline a run used (or would use) for one sheet. + + Snapshot v2 runs get the full pipeline; v1 and snapshot-less runs get the + legacy one so stored results replay exactly as they were produced. + """ + sid = sheet_key(sheet) + snapshot = _run_config_snapshot(run) if run is not None else None + if snapshot is None: + pmap, rules = _run_transform_config(db, run) if run is not None else ({}, []) + return legacy_pipeline( + key_fields=sheet.get("key_fields") or [], iface_fields=sheet.get("iface_fields") or [], + compare_fields=sheet.get("compare_fields") or [], field_rules=sheet.get("field_rules"), + port_map=pmap, iface_normalize_rules=rules, row_filters=sheet.get("row_filters"), + ignore_port_changes=sheet.get("ignore_port_changes"), + ) + authoritative = next( + (sh for sh in snapshot.get("sheets") or [] if isinstance(sh, dict) and sheet_key(sh) == sid), None, + ) + cfg = authoritative or sheet + version = int(snapshot.get("version") or 1) + vendors = snapshot.get("vendors") or {} + return build_pipeline( + cfg, + iface_rules=(snapshot.get("template") or {}).get("iface_normalize_rules") or [], + port_map=dict(snapshot.get("port_map") or {}), + before_vendor=str(vendors.get("before") or ""), + after_vendor=str(vendors.get("after") or ""), + legacy=version < 2, + normalize_map=version >= 2, + ) + + +def _set_run_progress( + db: Session, + run: BizCompareRun, + *, + phase: str, + sheet_index: int, + sheet_total: int, + sheet: dict[str, Any] | None, + started_mono: float, + extra: dict[str, Any] | None = None, + detach: bool = False, +) -> None: + """Update run progress. + + ``detach=True`` writes via a fresh session so SQL compare can keep an open + transaction (TEMP CTAS) without mid-flight commits on the worker ``db``. + """ + elapsed_ms = int((time.monotonic() - started_mono) * 1000) + progress = { + "phase": phase, + "sheet_index": sheet_index, + "sheet_total": sheet_total, + "sheet_id": sheet_key(sheet) if sheet else "", + "sheet_title": sheet_title(sheet) if sheet else "", + "elapsed_ms": elapsed_ms, + } + if extra: + progress.update(extra) + title = progress["sheet_title"] or progress["sheet_id"] or "" + message = ( + f"{phase} {sheet_index}/{sheet_total}" + + (f" · {title}" if title else "") + + f" · {elapsed_ms // 1000}s" + )[:1024] + + if detach: + from ..db import SessionLocal + + s = SessionLocal() + try: + r = s.get(BizCompareRun, str(run.id)) + if not r: + return + prev = dict(r.summary_json or {}) + prev["progress"] = progress + r.summary_json = prev + if str(r.status or "") != "cancelled": + r.status = "running" + r.message = message + s.commit() + # Mirror into worker instance for later in-memory reads (do not commit db) + prev_w = dict(run.summary_json or {}) + prev_w["progress"] = progress + run.summary_json = prev_w + if str(run.status or "") != "cancelled": + run.message = message + finally: + s.close() + return + + prev = dict(run.summary_json or {}) + prev["progress"] = progress + run.summary_json = prev + # Re-read status from DB — cancel may have been committed by another session + # (UI cancel / startup recovery) while this worker still holds a stale "running". + db.expire(run, ["status", "message"]) + if str(run.status or "") != "cancelled": + run.status = "running" + run.message = message + db.commit() + + +def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]: + """Run compare into an existing ``running`` BizCompareRun; persist sheet-by-sheet.""" + run = db.get(BizCompareRun, run_id) + if not run: + raise HTTPException(status_code=404, detail="run_not_found") + snapshot = _run_config_snapshot(run) + j = db.get(BizCompareJob, run.job_id) + if not j: + run.status = "failed" + run.message = "job_not_found" + db.commit() + raise HTTPException(status_code=404, detail="job_not_found") + tpl = db.get(BizCompareTemplate, run.template_id) if snapshot is None else None + if not tpl and snapshot is None: + run.status = "failed" + run.message = "template_not_found" + db.commit() + raise HTTPException(status_code=404, detail="template_not_found") + + before_batch_id = str(run.before_batch_id or "") + after_batch_id = str(run.after_batch_id or "") + if snapshot is not None: + sheets_cfg = deepcopy(snapshot.get("sheets") or []) + else: + sheets_cfg = _filter_enabled_sheets(template_metrics(tpl), getattr(j, "enabled_sheet_ids", None)) + if not sheets_cfg: + run.status = "failed" + run.message = "no_enabled_sheets" + db.commit() + raise HTTPException(status_code=400, detail="no_enabled_sheets") + if snapshot is None: + sheets_cfg = _order_sheets_small_first( + db, sheets_cfg, before_batch_id=before_batch_id, after_batch_id=after_batch_id, + ) + + started_mono = time.monotonic() + store_mode = normalize_store_unchanged( + snapshot.get("store_unchanged") if snapshot is not None else getattr(j, "store_unchanged", None), + ) + unchanged_listed_total = 0 + unchanged_truncated_any = False + unchanged_compact_any = False agg = { "before_count": 0, "after_count": 0, @@ -1555,87 +2763,488 @@ def _run_compare_unlocked( "duplicate": 0, } mapping_by_metric: dict[str, Any] = {} - for sheet in sheets_cfg: - one = _run_sheet( - db, - sheet=sheet, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - port_map=pmap, - iface_normalize_rules=norm_rules, - ) - sheet_results.append(one) - s = one["summary"] - for k in agg: - agg[k] += int(s.get(k) or 0) - mapping_by_metric[sheet_key(one)] = one["mapping_stats"] - - first = sheet_results[0] field_counts: dict[str, int] = {} - for s in sheet_results: - for d in list(s.get("diffs") or []): - if str(d.get("kind") or "") != "changed": - continue - for fname in d.get("changes") or {}: - field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1 - top_fields = sorted( - [{"field": k, "count": v} for k, v in field_counts.items()], - key=lambda x: (-int(x["count"]), str(x["field"])), - )[:8] + total = len(sheets_cfg) - run_id = uuid4().hex - # Persist counts/meta only — diffs go to biz_compare_diff rows - summary_payload = { - **agg, - "sheet_count": len(sheet_results), - "top_changed_fields": top_fields, - "sheets": [ - { - "sheet_id": s.get("sheet_id") or s["metric_id"], - "title": s.get("title") or s.get("sheet_id") or s["metric_id"], - "metric_id": s["metric_id"], - "key_fields": s["key_fields"], - "iface_fields": s["iface_fields"], - "compare_fields": s["compare_fields"], - "display_fields": s.get("display_fields") or [], - "field_rules": s.get("field_rules") or [], - "ignore_port_changes": s.get("ignore_port_changes"), - "mode": s["mode"], - "summary": s["summary"], + # Prefer seeded pending sheets from create; realign to small-first order + prev_metas = list((run.summary_json or {}).get("sheets") or []) + by_key = {sheet_key(m): m for m in prev_metas} + sheet_metas = [ + by_key.get(sheet_key(s)) or _pending_sheet_meta(s) for s in sheets_cfg + ] + + def _publish_sheets() -> None: + prev = dict(run.summary_json or {}) + prev["sheets"] = list(sheet_metas) + prev.update({k: agg[k] for k in agg}) + run.summary_json = prev + db.expire(run, ["status"]) + # Never resurrect cancelled while publishing incremental sheet metas + if str(run.status or "") == "cancelled": + db.commit() + return + db.commit() + + try: + for idx, sheet in enumerate(sheets_cfg, start=1): + if _run_is_cancelled(db, run_id): + run = db.get(BizCompareRun, run_id) or run + return get_run(db, run.id) + sid = sheet_key(sheet) + # Mark current sheet running in the sidebar list + for meta in sheet_metas: + if sheet_key(meta) == sid: + meta["status"] = "running" + break + _publish_sheets() + + _load_pub = {"t": 0.0, "n": -1, "engine": ""} + + def _on_load( + side: str, + n: int, + *, + engine: str = "python", + note: str = "", + phase: str | None = None, + _idx: int = idx, + _sheet: dict = sheet, + ) -> None: + now = time.monotonic() + eng = str(engine or "python") + # SQL emits sparse updates; Python still throttle chunk spam + if eng != "sql": + if n - _load_pub["n"] < 25_000 and now - _load_pub["t"] < 2.0: + return + _load_pub["t"] = now + _load_pub["n"] = n + _load_pub["engine"] = eng + extra: dict[str, Any] = { + "load_side": side, + "rows_loaded": n, + "engine": eng, + } + if note: + extra["engine_note"] = str(note)[:128] + # SQL path: detach progress commits so TEMP CTAS stays in one txn + _set_run_progress( + db, + run, + phase=str(phase or ("loading" if eng != "sql" else "sql_count")), + sheet_index=_idx, + sheet_total=total, + sheet=_sheet, + started_mono=started_mono, + extra=extra, + detach=(eng == "sql"), + ) + + _set_run_progress( + db, + run, + phase="loading", + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={"engine": "", "engine_note": ""}, + ) + one = _run_sheet( + db, + sheet=sheet, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + pipeline=_run_pipeline(db, run, sheet), + store_unchanged=store_mode, + on_load_progress=_on_load, + ) + s = one["summary"] + listed = int(s.get("unchanged_listed") or 0) + unchanged_listed_total += listed + if s.get("unchanged_truncated"): + unchanged_truncated_any = True + if s.get("unchanged_compact"): + unchanged_compact_any = True + _set_run_progress( + db, + run, + phase="comparing", + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={ + "before_count": int(s.get("before_count") or 0), + "after_count": int(s.get("after_count") or 0), + "diff_rows": int(s.get("added") or 0) + + int(s.get("removed") or 0) + + int(s.get("changed") or 0) + + int(s.get("duplicate") or 0) + + listed, + }, + ) + diffs = list(one.get("diffs") or []) + for d in diffs: + if str(d.get("kind") or "") != "changed": + continue + for fname in d.get("changes") or {}: + field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1 + # Cutover-first: persist fails so the UI can open the fail tab ASAP, + # then write success slim keys (sample / keys mode). + fail_diffs = [d for d in diffs if str(d.get("kind") or "") != "unchanged"] + ok_diffs = [d for d in diffs if str(d.get("kind") or "") == "unchanged"] + mid = sheet_key(one) + # Million-row vpnv4 with many diffs: keep key/row_id/changes only. + if len(fail_diffs) >= _FAIL_COMPACT_MIN: + for d in fail_diffs: + d["before"] = {} + d["after"] = {} + d["mapped_before"] = {} + d["compact"] = True + + def _on_persist( + written: int, + total_n: int, + *, + phase: str, + kind_key: str, + ) -> None: + _set_run_progress( + db, + run, + phase=phase, + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={ + kind_key: total_n, + "persisted": written, + "persist_total": total_n, + }, + # Separate session so chunk commits in persist do not race + # with progress JSON writes on the worker session. + detach=True, + ) + + _set_run_progress( + db, + run, + phase="persisting_fail", + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={ + "fail_rows": len(fail_diffs), + "persisted": 0, + "persist_total": len(fail_diffs), + }, + ) + n_fail = _persist_sheet_diffs( + db, + run_id=run.id, + metric_id=mid, + diffs=fail_diffs, + seq_start=0, + on_progress=lambda w, n: _on_persist( + w, n, phase="persisting_fail", kind_key="fail_rows" + ), + ) + if ok_diffs: + _set_run_progress( + db, + run, + phase="persisting_ok", + sheet_index=idx, + sheet_total=total, + sheet=sheet, + started_mono=started_mono, + extra={ + "ok_rows": len(ok_diffs), + "persisted": 0, + "persist_total": len(ok_diffs), + }, + ) + _persist_sheet_diffs( + db, + run_id=run.id, + metric_id=mid, + diffs=ok_diffs, + seq_start=n_fail, + on_progress=lambda w, n: _on_persist( + w, n, phase="persisting_ok", kind_key="ok_rows" + ), + ) + for k in agg: + agg[k] += int(s.get(k) or 0) + mapping_by_metric[mid] = one["mapping_stats"] + done_meta = { + "sheet_id": one.get("sheet_id") or one["metric_id"], + "title": one.get("title") or one.get("sheet_id") or one["metric_id"], + "metric_id": one["metric_id"], + "key_fields": one["key_fields"], + "iface_fields": one["iface_fields"], + "compare_fields": one["compare_fields"], + "display_fields": one.get("display_fields") or [], + "field_rules": one.get("field_rules") or [], + "row_filters": one.get("row_filters") or [], + "ignore_port_changes": one.get("ignore_port_changes"), + "mode": one["mode"], + "status": "done", + "summary": one["summary"], } - for s in sheet_results - ], - } + replaced = False + for i, meta in enumerate(sheet_metas): + if sheet_key(meta) == mid: + sheet_metas[i] = done_meta + replaced = True + break + if not replaced: + sheet_metas.append(done_meta) + _publish_sheets() + # Drop heavy diffs before next sheet + one.clear() + diffs.clear() + fail_diffs.clear() + ok_diffs.clear() + db.commit() + if _run_is_cancelled(db, run_id): + run = db.get(BizCompareRun, run_id) or run + return get_run(db, run.id) - run = BizCompareRun( - id=run_id, - job_id=j.id, - template_id=tpl.id, - mapping_id=j.mapping_id, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - metric_id=first["metric_id"], - status="success", - summary_json=summary_payload, - diffs_json=[], - mapping_stats_json=mapping_by_metric, - message="", - created_at=_utcnow(), - ) - db.add(run) - db.flush() - for s in sheet_results: - _persist_sheet_diffs( - db, - run_id=run_id, - metric_id=sheet_key(s), - diffs=list(s.get("diffs") or []), + if _run_is_cancelled(db, run_id): + run = db.get(BizCompareRun, run_id) or run + return get_run(db, run.id) + + duration_ms = int((time.monotonic() - started_mono) * 1000) + top_fields = sorted( + [{"field": k, "count": v} for k, v in field_counts.items()], + key=lambda x: (-int(x["count"]), str(x["field"])), + )[:8] + summary_payload = { + **agg, + "sheet_count": len(sheet_metas), + "top_changed_fields": top_fields, + "duration_ms": duration_ms, + "store_unchanged": store_mode, + "unchanged_stored": unchanged_listed_total > 0, + "unchanged_listed": unchanged_listed_total, + "unchanged_truncated": unchanged_truncated_any, + "unchanged_compact": unchanged_compact_any, + "progress": { + "phase": "done", + "sheet_index": total, + "sheet_total": total, + "sheet_id": "", + "sheet_title": "", + "elapsed_ms": duration_ms, + }, + "sheets": sheet_metas, + } + if snapshot is not None: + summary_payload["config_snapshot"] = snapshot + run = db.get(BizCompareRun, run_id) or run + summary_payload["purpose"] = (run.summary_json or {}).get("purpose") or "manual" + if str(run.status or "") == "cancelled": + return get_run(db, run.id) + run.status = "success" + run.summary_json = summary_payload + run.mapping_stats_json = mapping_by_metric + run.metric_id = str(sheet_metas[0]["metric_id"]) if sheet_metas else run.metric_id + run.message = f"done · {duration_ms // 1000}s" + run.diffs_json = [] + j.updated_at = _utcnow() + if j.mode == "manual": + j.after_batch_id = after_batch_id + db.commit() + return get_run(db, run.id) + except HTTPException as exc: + run = db.get(BizCompareRun, run_id) or run + if str(run.status or "") != "cancelled": + run.status = "failed" + run.message = str(getattr(exc, "detail", "") or exc)[:1024] + prev = dict(run.summary_json or {}) + prog = dict(prev.get("progress") or {}) + prog["phase"] = "failed" + prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000) + prev["progress"] = prog + prev["duration_ms"] = prog["elapsed_ms"] + run.summary_json = prev + db.commit() + raise + except Exception as exc: + _log.exception("compare run failed run=%s job=%s", run_id, j.id) + run = db.get(BizCompareRun, run_id) or run + if str(run.status or "") != "cancelled": + run.status = "failed" + run.message = str(exc)[:1024] + prev = dict(run.summary_json or {}) + prog = dict(prev.get("progress") or {}) + prog["phase"] = "failed" + prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000) + prev["progress"] = prog + prev["duration_ms"] = prog["elapsed_ms"] + run.summary_json = prev + db.commit() + raise + + +_INTERRUPT_MARK = "interrupted_by_restart" + + +def _job_has_active_run(db: Session, job_id: str) -> BizCompareRun | None: + return ( + db.query(BizCompareRun) + .filter( + BizCompareRun.job_id == str(job_id or ""), + BizCompareRun.status.in_(("running", "queued")), ) - j.updated_at = _utcnow() - if j.mode == "manual": - j.after_batch_id = after_batch_id + .order_by(BizCompareRun.created_at.desc()) + .first() + ) + + +def _run_is_cancelled(db: Session, run_id: str) -> bool: + """Re-read status so user/startup cancel is visible to the worker thread.""" + db.expire_all() + r = db.get(BizCompareRun, run_id) + return bool(r and str(r.status or "") == "cancelled") + + +def recover_interrupted_compares_on_startup(db: Session) -> dict[str, Any]: + """Mark orphaned running/queued compare runs as cancelled after process restart. + + In-memory job locks die with the process; without this, the UI stays on + 「比对中」and blocks a new run. + """ + now = _utcnow() + rows = ( + db.query(BizCompareRun) + .filter(BizCompareRun.status.in_(("running", "queued"))) + .all() + ) + n = 0 + for r in rows: + r.status = "cancelled" + msg = str(r.message or "").strip() + if _INTERRUPT_MARK not in msg: + r.message = f"{msg} | {_INTERRUPT_MARK}".strip(" |")[:1024] + prev = dict(r.summary_json or {}) + prog = dict(prev.get("progress") or {}) + prog["phase"] = "cancelled" + prog["elapsed_ms"] = int(prog.get("elapsed_ms") or 0) + prev["progress"] = prog + # Mark in-flight sheet placeholders so UI does not show fake pass + sheets = list(prev.get("sheets") or []) + for sh in sheets: + st = str(sh.get("status") or "") + if st in ("pending", "running", "queued"): + sh["status"] = "cancelled" + prev["sheets"] = sheets + r.summary_json = prev + n += 1 + if n: + db.commit() + _log.info("startup: cancelled %s interrupted compare run(s)", n) + return {"runs": n, "at": now.isoformat() + "Z"} + + +def cancel_compare_run(db: Session, run_id: str) -> dict[str, Any]: + """Cancel a running/queued compare so a new run can start.""" + r = db.get(BizCompareRun, run_id) + if not r: + raise HTTPException(status_code=404, detail="run_not_found") + st = str(r.status or "") + if st not in ("running", "queued"): + return get_run(db, run_id) + r.status = "cancelled" + msg = str(r.message or "").strip() + r.message = f"{msg} | cancelled_by_user".strip(" |")[:1024] + prev = dict(r.summary_json or {}) + prog = dict(prev.get("progress") or {}) + prog["phase"] = "cancelled" + prev["progress"] = prog + for sh in list(prev.get("sheets") or []): + if str(sh.get("status") or "") in ("pending", "running", "queued"): + sh["status"] = "cancelled" + r.summary_json = prev db.commit() - return get_run(db, run.id) + return get_run(db, run_id) + + +def run_compare( + db: Session, job_id: str, *, force_after_batch_id: str = "", purpose: str = "manual", +) -> dict[str, Any]: + """Synchronous compare (auto-compare / tests). Blocks the caller until done.""" + if _job_has_active_run(db, job_id): + raise HTTPException(status_code=409, detail="compare_already_running") + lock = _job_compare_lock(job_id) + if not lock.acquire(blocking=False): + raise HTTPException(status_code=409, detail="compare_already_running") + try: + ctx = _validate_compare_job(db, job_id, force_after_batch_id=force_after_batch_id) + run = _create_running_run( + db, + job=ctx["job"], + tpl=ctx["template"], + before_batch_id=ctx["before_batch_id"], + after_batch_id=ctx["after_batch_id"], + sheets_cfg=ctx["sheets_cfg"], + purpose=purpose, + ) + return _execute_compare_into_run(db, run.id) + finally: + lock.release() + + +def enqueue_compare( + db: Session, job_id: str, *, force_after_batch_id: str = "" +) -> dict[str, Any]: + """Create a ``running`` run and execute compare on a daemon thread. + + Returns immediately so the HTTP worker / UI stay responsive. Poll + ``GET /compare/runs/{id}`` for progress (``summary.progress``). + """ + if _job_has_active_run(db, job_id): + raise HTTPException(status_code=409, detail="compare_already_running") + lock = _job_compare_lock(job_id) + if not lock.acquire(blocking=False): + raise HTTPException(status_code=409, detail="compare_already_running") + run_id = "" + try: + ctx = _validate_compare_job(db, job_id, force_after_batch_id=force_after_batch_id) + run = _create_running_run( + db, + job=ctx["job"], + tpl=ctx["template"], + before_batch_id=ctx["before_batch_id"], + after_batch_id=ctx["after_batch_id"], + sheets_cfg=ctx["sheets_cfg"], + ) + run_id = run.id + except Exception: + lock.release() + raise + + def _bg() -> None: + from ..db import SessionLocal + + s = SessionLocal() + try: + _execute_compare_into_run(s, run_id) + except Exception: + _log.exception("bg compare failed job=%s run=%s", job_id, run_id) + finally: + s.close() + lock.release() + + threading.Thread( + target=_bg, + name=f"biz-cmp-{run_id[:8]}", + daemon=True, + ).start() + return get_run(db, run_id) def _csv_cell(v: Any) -> str: @@ -1645,7 +3254,7 @@ def _csv_cell(v: Any) -> str: return s -def _sheet_csv(sheet: dict[str, Any]) -> str: +def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) -> Iterator[str]: keys = list(sheet.get("key_fields") or []) key_set = set(keys) compare = [f for f in list(sheet.get("compare_fields") or []) if f not in key_set] @@ -1657,15 +3266,17 @@ def _sheet_csv(sheet: dict[str, Any]) -> str: ) # Non-key columns already ordered Key→Compare→Display by effective_display_fields extra = [f for f in display if f not in key_set] - headers = ["kind", *keys] + # Interface keys also get the name as the device printed it (before side). + raw_cols = [k for k in keys if k in set(sheet.get("iface_fields") or [])] + headers = ["kind", *keys, *[f"{k}__raw_before" for k in raw_cols]] for f in extra: if f in compare_set: headers.append(f"{f}__pre") headers.append(f"{f}__post") else: headers.append(f) - lines = [",".join(_csv_cell(h) for h in headers)] - for d in list(sheet.get("diffs") or []): + yield ",".join(_csv_cell(h) for h in headers) + for d in diffs: kind = str(d.get("kind") or "") pre = dict(d.get("mapped_before") or d.get("before") or {}) post = dict(d.get("after") or {}) @@ -1673,6 +3284,9 @@ def _sheet_csv(sheet: dict[str, Any]) -> str: row = [kind] for k in keys: row.append(key.get(k, pre.get(k, post.get(k, "")))) + raw_before = dict(d.get("raw_key_before") or {}) + for k in raw_cols: + row.append(raw_before.get(k, "")) for f in extra: if f in compare_set: if kind == "added": @@ -1690,8 +3304,23 @@ def _sheet_csv(sheet: dict[str, Any]) -> str: row.append(pre.get(f, "")) else: row.append(post.get(f, pre.get(f, ""))) - lines.append(",".join(_csv_cell(x) for x in row)) - return "\ufeff" + "\n".join(lines) + "\n" + yield ",".join(_csv_cell(x) for x in row) + + +def _sheet_csv(sheet: dict[str, Any]) -> str: + return "\ufeff" + "\n".join(_sheet_csv_lines(sheet, sheet.get("diffs") or [])) + "\n" + + +def _task_ok(sheets: list[dict[str, Any]], fail_count: int, any_pending: bool) -> bool | None: + """True/False once judged; None when data quality leaves the verdict open.""" + if any_pending: + return False + states = [str((sh.get("summary") or {}).get("data_state") or "ok") for sh in sheets] + if any(st in ("suspect_parse", "not_collected") for st in states): + return None + if states and all(st == "no_data" for st in states): + return None + return fail_count == 0 def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> dict[str, Any]: @@ -1702,14 +3331,28 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di before_count = int(summary.get("before_count") or 0) after_count = int(summary.get("after_count") or 0) total = added + removed + changed + unchanged + + def _data_state(sh: dict[str, Any]) -> str: + return str((sh.get("summary") or {}).get("data_state") or "ok") + + # Sheets whose side was never collected cannot be judged: their "missing" + # rows are a collection gap, not a business loss, so they stay out of the totals. + dropped = [sh for sh in sheets if _data_state(sh) == "not_collected" + and str(sh.get("status") or "done") not in ("pending", "running", "queued")] + drop_fail = sum(int((sh.get("summary") or {}).get("removed") or 0) + + int((sh.get("summary") or {}).get("changed") or 0) for sh in dropped) + drop_ok = sum(int((sh.get("summary") or {}).get("unchanged") or 0) for sh in dropped) # Compare verdict: fail = missing + mismatch; success = match; added is special - fail_count = removed + changed - success_count = unchanged + fail_count = removed + changed - drop_fail + success_count = unchanged - drop_ok judged = fail_count + success_count matched = changed + unchanged diff_count = fail_count + # Nothing compared at all is "not judged" (None), never a 100% pass. pass_rate = ( - round((success_count / judged) * 100, 1) if judged else (100.0 if total == 0 else 0.0) + compute_pass_rate(success=success_count, fail=fail_count) + if judged + else (None if total == 0 else 0.0) ) diff_rate = round((fail_count / judged) * 100, 1) if judged else 0.0 @@ -1723,12 +3366,19 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di st = sa + sr + sc + su sf = sr + sc sj = sf + su + status = str(sh.get("status") or "done") + pending = status in ("pending", "running", "queued") + data_state = str(ss.get("data_state") or "ok") sheet_cards.append( { + "data_state": data_state, + "data_state_reasons": list(ss.get("data_state_reasons") or []), + "collect_quality": ss.get("collect_quality") or {}, "sheet_id": sheet_key(sh), "title": sheet_title(sh), "metric_id": sh.get("metric_id") or "", "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"), + "status": status, "added": sa, "removed": sr, "changed": sc, @@ -1738,7 +3388,10 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di "fail_count": sf, "success_count": su, "diff_count": sf, - "pass_rate": round((su / sj) * 100, 1) if sj else (100.0 if st == 0 else 0.0), + # Pending sheets must not look like "100% pass" + "pass_rate": None + if pending or data_state in UNJUDGED_STATES + else (compute_pass_rate(success=su, fail=sf) if sj else (None if st == 0 else 0.0)), } ) @@ -1757,6 +3410,9 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di key=lambda x: (-int(x["count"]), str(x["field"])), )[:8] + any_pending = any( + str(sh.get("status") or "") in ("pending", "running", "queued") for sh in sheets + ) return { "added": added, "removed": removed, @@ -1770,11 +3426,20 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di "fail_count": fail_count, "success_count": success_count, "diff_count": diff_count, - "pass_rate": pass_rate, + "pass_rate": None if any_pending else pass_rate, "diff_rate": diff_rate, - "ok": fail_count == 0, + "ok": _task_ok(sheets, fail_count, any_pending), + "unjudged_sheets": sum(1 for sh in sheets if _data_state(sh) in UNJUDGED_STATES), + "suspect_sheets": sum(1 for sh in sheets if _data_state(sh) == "suspect_parse"), "sheet_cards": sheet_cards, "top_changed_fields": top_fields, + "duration_ms": int(summary.get("duration_ms") or 0), + "store_unchanged": str(summary.get("store_unchanged") or ""), + "unchanged_stored": bool(summary.get("unchanged_stored", True)), + "unchanged_listed": int(summary.get("unchanged_listed") or 0), + "unchanged_truncated": bool(summary.get("unchanged_truncated")), + "unchanged_compact": bool(summary.get("unchanged_compact")), + "progress": dict(summary.get("progress") or {}), } @@ -1782,8 +3447,13 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: r = db.get(BizCompareRun, run_id) if not r: raise HTTPException(status_code=404, detail="run_not_found") - tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None summary = dict(r.summary_json or {}) + snapshot = _run_config_snapshot(r) + tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id and snapshot is None else None + template_out = ( + deepcopy(snapshot.get("template")) if snapshot is not None + else (_template_out(tpl) if tpl else None) + ) raw_sheets = _sheet_meta_from_summary(summary, r, tpl) # Never return full diffs in run detail (million-row safe) sheets = [ @@ -1797,28 +3467,38 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: "display_fields": list(sh.get("display_fields") or []), "field_rules": list(sh.get("field_rules") or []), "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"), + "status": str(sh.get("status") or "done"), "summary": dict(sh.get("summary") or {}), } for sh in raw_sheets ] enriched = _enrich_summary(summary, raw_sheets) stored = "rows" if _run_has_diff_rows(db, run_id) else "inline" - job = db.get(BizCompareJob, r.job_id) if r.job_id else None - mapping = db.get(BizPortMapping, r.mapping_id) if r.mapping_id else None + job = db.get(BizCompareJob, r.job_id) if r.job_id and snapshot is None else None + mapping = db.get(BizPortMapping, r.mapping_id) if r.mapping_id and snapshot is None else None before_side = _compare_side( - db, r.before_batch_id, fallback_task_id=(job.before_task_id if job else "") + db, r.before_batch_id, fallback_task_id=( + snapshot.get("before_task_id", "") if snapshot is not None else (job.before_task_id if job else "") + ), ) after_side = _compare_side( - db, r.after_batch_id, fallback_task_id=(job.after_task_id if job else "") + db, r.after_batch_id, fallback_task_id=( + snapshot.get("after_task_id", "") if snapshot is not None else (job.after_task_id if job else "") + ), ) return { "id": r.id, "job_id": r.job_id, - "job_name": (job.name if job else "") or "", + "job_name": ( + snapshot.get("job_name", "") if snapshot is not None else ((job.name if job else "") or "") + ), "template_id": r.template_id, - "template_name": (tpl.name if tpl else "") or "", + "template_name": (template_out or {}).get("name") or "", "mapping_id": r.mapping_id, - "mapping_name": (mapping.name if mapping else "") or "", + "mapping_name": ( + snapshot.get("mapping_name", "") if snapshot is not None else ((mapping.name if mapping else "") or "") + ), + "config_snapshot_version": snapshot["version"] if snapshot is not None else None, "before_batch_id": r.before_batch_id, "after_batch_id": r.after_batch_id, "before": before_side, @@ -1832,7 +3512,7 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: "mapping_stats": r.mapping_stats_json or {}, "message": r.message, "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, - "template": _template_out(tpl) if tpl else None, + "template": template_out, } @@ -1850,6 +3530,451 @@ def _lookup_sheet(sheets: list[dict[str, Any]], key: str) -> dict[str, Any] | No return None +def _kind_allows(kind_n: str, diff_kind: str) -> bool: + dk = str(diff_kind or "") + if kind_n == "all": + return True + if kind_n == "diff": + return dk in ("removed", "changed") + return dk == kind_n + + +def _parse_qf(raw: Any) -> dict[str, str]: + """Normalize field query map: {field: value} with safe names only.""" + from .compare_sql import _FIELD_RE + + if raw is None or raw == "": + return {} + obj: Any = raw + if isinstance(raw, str): + s = raw.strip() + if not s: + return {} + try: + obj = json.loads(s) + except Exception: + return {} + if not isinstance(obj, dict): + return {} + out: dict[str, str] = {} + for k, v in obj.items(): + name = str(k or "").strip() + val = str(v or "").strip() + if not name or not val or not _FIELD_RE.match(name): + continue + out[name] = val + return out + + +def _split_kw_and_field_tokens(kw: str) -> tuple[str, dict[str, str]]: + """Parse ``direction:out network:1.1.1.1 foo`` → free kw + field map.""" + from .compare_sql import _FIELD_RE + + free: list[str] = [] + fields: dict[str, str] = {} + for tok in str(kw or "").split(): + if ":" in tok: + name, _, val = tok.partition(":") + name = name.strip() + val = val.strip() + if name and val and _FIELD_RE.match(name): + fields[name] = val + continue + if tok.strip(): + free.append(tok.strip()) + return " ".join(free), fields + + +def _kw_match_sql(key_fields: list[str], *, param: str = "kw") -> str: + """OR of ILIKE on key fields (and data_json::text fallback).""" + from .compare_sql import _FIELD_RE, _safe_field + + parts: list[str] = [] + for f in key_fields: + name = str(f or "").strip() + if not name or not _FIELD_RE.match(name): + continue + sf = _safe_field(name) + parts.append(f"lower(trim(both from coalesce(data_json->>'{sf}', ''))) LIKE :{param}") + # Broad fallback so free-text still hits non-key columns (path, next_hop, …) + parts.append(f"lower(data_json::text) LIKE :{param}") + return "(" + " OR ".join(parts) + ")" if parts else f"(lower(data_json::text) LIKE :{param})" + + +def _field_qf_sql(field_q: dict[str, str], *, prefix: str = "qf") -> tuple[str, dict[str, Any]]: + """AND of ILIKE contains on each field.""" + from .compare_sql import _safe_field + + if not field_q: + return "TRUE", {} + parts: list[str] = [] + params: dict[str, Any] = {} + for i, (name, val) in enumerate(field_q.items()): + sf = _safe_field(name) + key = f"{prefix}_{i}" + params[key] = _literal_search_pattern(val) + parts.append( + f"lower(trim(both from coalesce(data_json->>'{sf}', ''))) LIKE :{key}" + ) + return "(" + " AND ".join(parts) + ")", params + + +def _literal_search_pattern(value: str) -> str: + return "%" + value.lower().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" + + +def _load_metric_rows_for_search( + db: Session, + *, + batch_id: str, + metric_id: str, + row_filters: list[dict[str, Any]] | None, + key_fields: list[str], + kw: str = "", + field_q: dict[str, str] | None = None, + cap: int = _LIVE_SEARCH_LOAD_CAP, +) -> tuple[list[dict[str, Any]], bool]: + """Load rows matching sheet filters + optional free kw / field_q. Returns (rows, truncated).""" + from .compare_sql import ( + _dialect_is_postgres, + _filters_sql_compatible, + compile_row_filters_sql, + ) + from ..models import BizStateMetricRow + from sqlalchemy import text as sql_text + + bid = str(batch_id or "").strip() + mid = str(metric_id or "").strip() + needle = str(kw or "").strip() + fq = {k: v for k, v in (field_q or {}).items() if str(v or "").strip()} + if not bid or not mid or (not needle and not fq): + return [], False + lim = max(1, min(int(cap), _LIVE_SEARCH_LOAD_CAP)) + filters = [f for f in (row_filters or []) if isinstance(f, dict)] + + if mid != "lldp_neighbor" and _dialect_is_postgres(db) and ( + not filters or _filters_sql_compatible(filters) + ): + filter_sql, filter_params = ("TRUE", {}) + if filters and _filters_sql_compatible(filters): + filter_sql, filter_params = compile_row_filters_sql(filters) + qf_sql, qf_params = _field_qf_sql(fq) + search_parts = [qf_sql] + params: dict[str, Any] = { + "bid": bid, + "mid": mid, + "lim": lim + 1, + **filter_params, + **qf_params, + } + if needle: + params["kw"] = _literal_search_pattern(needle) + search_parts.append(_kw_match_sql(key_fields)) + search_sql = " AND ".join(f"({p})" for p in search_parts if p and p != "TRUE") + if not search_sql: + search_sql = "TRUE" + rows = db.execute( + sql_text( + f""" + SELECT id, batch_command_id, task_id, ne_id, seq, data_json, collected_at + FROM biz_state_metric_row + WHERE batch_id = :bid + AND metric_id = :mid + AND ({filter_sql}) + AND ({search_sql}) + ORDER BY seq ASC, id ASC + LIMIT :lim + """ + ), + params, + ).mappings().all() + truncated = len(rows) > lim + rows = rows[:lim] + out: list[dict[str, Any]] = [] + for r in rows: + data = dict(r["data_json"] or {}) + collected = r["collected_at"] + out.append( + { + **data, + "_netx": { + "batch_id": bid, + "batch_command_id": str(r["batch_command_id"] or ""), + "task_id": str(r["task_id"] or ""), + "ne_id": str(r["ne_id"] or ""), + "collected_at": collected.isoformat() + "Z" + if collected is not None + else None, + "row_id": str(r["id"]), + }, + } + ) + return out, truncated + + # Non-PG / fallback: scan with early stop (OK for tests / small sheets) + model = BizStateLldpNeighbor if mid == "lldp_neighbor" else BizStateMetricRow + q = db.query(model).filter(model.batch_id == bid) + if mid == "lldp_neighbor": + q = q.order_by(model.id.asc()) + else: + q = q.filter(model.metric_id == mid).order_by(model.seq.asc(), model.id.asc()) + out = [] + truncated = False + needle_l = needle.lower() + key_set = [str(k).strip() for k in key_fields if str(k).strip()] + for r in q.yield_per(500): + data = ( + {f: getattr(r, f) for f in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")} + if mid == "lldp_neighbor" else dict(r.data_json or {}) + ) + row = { + **data, + "_netx": { + "batch_id": bid, + "batch_command_id": r.batch_command_id or "", + "task_id": r.task_id or "", + "ne_id": r.ne_id or "", + "collected_at": r.collected_at.isoformat() + "Z" + if r.collected_at + else None, + "row_id": r.id, + }, + } + if filters and not all(row_matches_filter(row, f) for f in filters): + db.expunge(r) + continue + if fq: + ok_f = True + for fname, fval in fq.items(): + if fval.lower() not in str(row.get(fname) or "").lower(): + ok_f = False + break + if not ok_f: + db.expunge(r) + continue + if needle_l: + hit = False + for kf in key_set: + if needle_l in str(row.get(kf) or "").lower(): + hit = True + break + if not hit: + blob = json.dumps(data, ensure_ascii=False, default=str).lower() + hit = needle_l in blob + if not hit: + db.expunge(r) + continue + db.expunge(r) + if len(out) >= lim: + truncated = True + break + out.append(row) + return out, truncated + + +def _load_search_key_groups( + db: Session, *, batch_id: str, metric_id: str, keys: set[tuple[str, ...]], + match_keys: list[str], pipeline: Pipeline, side: str, +) -> tuple[list[dict[str, Any]], set[tuple[str, ...]]]: + """Complete candidate key groups so a one-sided search cannot change verdicts. + + Identity rules (normalize / value map / port map) run in the pipeline, in + Python. Key fields they never touch narrow the SQL query. If the load budget + is exceeded, omit entire groups rather than compare partial groups and + manufacture additions/deletions. + """ + from ..models import BizStateMetricRow + + model = BizStateLldpNeighbor if metric_id == "lldp_neighbor" else BizStateMetricRow + # Narrowing compares raw SQL text, so only fields no rule rewrites may be used, + # and only when key tuples line up 1:1 with ``match_keys`` (no mapped/unmapped marker). + invariant = [] if pipeline.ignores_ports_by_policy() else [ + f for f in match_keys if f not in pipeline._transformed # noqa: SLF001 + ] + positions = [match_keys.index(f) for f in invariant] + coarse = sorted({tuple(k[i] for i in positions) for k in keys}) if invariant else [()] + out: list[dict[str, Any]] = [] + complete: set[tuple[str, ...]] = set() + # Bound memory even when a candidate identifies a very large duplicate group. + budget = _LIVE_SEARCH_GROUP_LOAD_CAP + for start in range(0, len(coarse), 100): + group = coarse[start:start + 100] + group_set = set(group) + target = {k for k in keys if tuple(k[i] for i in positions) in group_set} if invariant else set(keys) + q = db.query(model).filter(model.batch_id == batch_id) + if metric_id == "lldp_neighbor": + q = q.order_by(model.id.asc()) + else: + q = q.filter(model.metric_id == metric_id).order_by(model.seq.asc(), model.id.asc()) + if invariant: + exprs = [func.trim(func.coalesce( + cast(getattr(model, f) if metric_id == "lldp_neighbor" + else model.data_json[f].as_string(), String), "")) for f in invariant] + q = q.filter(tuple_(*exprs).in_(group)) + rows = q.limit(budget + 1).all() + if len(rows) > budget: + for r in rows: + db.expunge(r) + # No partial verdicts. Later groups may still fit the remaining budget. + continue + budget -= len(rows) + complete.update(target) + for r in rows: + data = ( + {f: getattr(r, f) for f in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")} + if metric_id == "lldp_neighbor" else dict(r.data_json or {}) + ) + row = {**data, "_netx": {"row_id": r.id}} + db.expunge(r) + norm = pipeline.normalize(row, side) + if not pipeline.accepts(row, norm, side): + continue + if pipeline.key_of(norm, side, match_keys)[1] in target: + out.append(row) + return out, complete + + +def _live_search_sheet_diffs( + db: Session, + run: BizCompareRun, + sheet: dict[str, Any], + *, + kind: str, + kw: str = "", + field_q: dict[str, str] | None = None, + page: int, + page_size: int, +) -> dict[str, Any]: + """Find candidate keys, complete both sides, then compare and filter pairs.""" + mid_src = str(sheet.get("metric_id") or "").strip() + sid = sheet_key(sheet) + key_fields = list(sheet.get("key_fields") or []) + iface_fields = list(sheet.get("iface_fields") or []) + field_rules = list(sheet.get("field_rules") or []) + compare_fields = effective_compare_fields( + list(sheet.get("compare_fields") or []), + field_rules, + ) + row_filters = list(sheet.get("row_filters") or []) + # Older runs may lack row_filters on sheet meta — fall back to template + if "row_filters" not in sheet and _run_config_snapshot(run) is None and run.template_id: + tpl = db.get(BizCompareTemplate, run.template_id) + if tpl: + for s in template_metrics(tpl): + if sheet_key(s) == sid: + row_filters = list(s.get("row_filters") or []) + if not key_fields: + key_fields = list(s.get("key_fields") or []) + if not field_rules: + field_rules = list(s.get("field_rules") or []) + compare_fields = effective_compare_fields( + list(s.get("compare_fields") or compare_fields), + field_rules, + ) + break + + free_kw, tok_fields = _split_kw_and_field_tokens(kw) + merged_q = {**tok_fields, **(field_q or {})} + # Drop empty + merged_q = {k: v for k, v in merged_q.items() if str(v or "").strip()} + + if not key_fields or not mid_src: + return { + "total": 0, + "page": page, + "page_size": page_size, + "metric_id": sid, + "items": [], + "source": "live", + "truncated": False, + } + + ignore_ports = sheet.get("ignore_port_changes") + if ignore_ports is not None: + ignore_ports = bool(ignore_ports) + pipeline = _run_pipeline(db, run, sheet) + if pipeline.legacy and not pipeline.row_filters and row_filters: + pipeline.row_filters = [dict(f) for f in row_filters if isinstance(f, dict)] + + before_rows, trunc_b = _load_metric_rows_for_search( + db, + batch_id=str(run.before_batch_id or ""), + metric_id=mid_src, + row_filters=pipeline.pushdown_filters("before"), + key_fields=key_fields, + kw=free_kw, + field_q=merged_q, + ) + after_rows, trunc_a = _load_metric_rows_for_search( + db, + batch_id=str(run.after_batch_id or ""), + metric_id=mid_src, + row_filters=pipeline.pushdown_filters("after"), + key_fields=key_fields, + kw=free_kw, + field_q=merged_q, + ) + match_keys = list((sheet.get("summary") or {}).get("match_key_fields") or key_fields) + if ignore_ports is True and not pipeline.port_map: + match_keys = [f for f in key_fields if f not in iface_fields] or key_fields + keys: set[tuple[str, ...]] = set() + for rows, side in ((before_rows, "before"), (after_rows, "after")): + for raw, norm in pipeline.prepare(rows, side): + keys.add(pipeline.key_of(norm, side, match_keys)[1]) + before_rows, complete_b = _load_search_key_groups( + db, batch_id=str(run.before_batch_id or ""), metric_id=mid_src, keys=keys, + match_keys=match_keys, pipeline=pipeline, side="before", + ) + after_rows, complete_a = _load_search_key_groups( + db, batch_id=str(run.after_batch_id or ""), metric_id=mid_src, keys=keys, + match_keys=match_keys, pipeline=pipeline, side="after", + ) + complete = complete_b & complete_a + + def keep_complete(rows: list[dict[str, Any]], side: str) -> list[dict[str, Any]]: + return [raw for raw, norm in pipeline.prepare(rows, side) + if pipeline.key_of(norm, side, match_keys)[1] in complete] + + before_rows = keep_complete(before_rows, "before") + after_rows = keep_complete(after_rows, "after") + result = compare_rows( + before_rows=before_rows, + after_rows=after_rows, + key_fields=key_fields, + iface_fields=iface_fields, + compare_fields=compare_fields, + field_rules=field_rules, + include_unchanged=True, + unchanged_limit=None, + compact_unchanged=False, + pipeline=pipeline, + match_keys_override=match_keys, + ) + kind_n = (kind or "diff").strip().lower() + filtered = [ + d for d in list(result.get("diffs") or []) + if _kind_allows(kind_n, str(d.get("kind") or "")) + and _diff_matches_search(d, kw=free_kw, field_q=merged_q) + ] + # Cap pairs returned to keep UI snappy + truncated = bool(trunc_b or trunc_a or complete != keys or len(filtered) > _LIVE_SEARCH_RESULT_CAP) + filtered = filtered[:_LIVE_SEARCH_RESULT_CAP] + total = len(filtered) + start = (page - 1) * page_size + page_items = _attach_raw_key_before(filtered[start : start + page_size], key_fields) + return { + "total": total, + "page": page, + "page_size": page_size, + "metric_id": sid, + "items": page_items, + "source": "live", + "truncated": truncated, + "live_before_matched": len(before_rows), + "live_after_matched": len(after_rows), + } + + def list_run_diffs( db: Session, run_id: str, @@ -1857,6 +3982,7 @@ def list_run_diffs( metric_id: str = "", kind: str = "diff", kw: str = "", + qf: Any = None, page: int = 1, page_size: int = 100, ) -> dict[str, Any]: @@ -1867,14 +3993,40 @@ def list_run_diffs( size_n = max(1, min(500, int(page_size or 100))) kind_n = (kind or "diff").strip().lower() kw_n = (kw or "").strip() + field_q = _parse_qf(qf) + # Also accept field:value tokens inside kw + free_from_kw, tok_fields = _split_kw_and_field_tokens(kw_n) + if tok_fields: + field_q = {**tok_fields, **field_q} + kw_n = free_from_kw summary = dict(r.summary_json or {}) tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None sheets = _sheet_meta_from_summary(summary, r, tpl) asked = (metric_id or "").strip() sheet = _lookup_sheet(sheets, asked) if asked else (sheets[0] if sheets else None) + if asked and sheet is None: + raise HTTPException(status_code=404, detail="sheet_not_found") mid = sheet_key(sheet) if sheet else (asked or str(r.metric_id or "")) + # Only reconstruct missing success rows. Persisted verdicts must stay stable + # while searching and avoid loading/recomparing source tables on every page. + sheet_summary = (sheet or {}).get("summary") or {} + success_missing = int(sheet_summary.get("unchanged") or 0) > int( + sheet_summary.get("unchanged_listed") or 0 + ) + if (kw_n or field_q) and sheet and kind_n in ("all", "unchanged") and success_missing: + return _live_search_sheet_diffs( + db, + r, + sheet, + kind=kind_n, + kw=kw_n, + field_q=field_q, + page=page_n, + page_size=size_n, + ) + if _run_has_diff_rows(db, run_id): q = db.query(BizCompareDiff).filter( BizCompareDiff.run_id == run_id, @@ -1882,10 +4034,15 @@ def list_run_diffs( ) if kind_n == "diff": q = q.filter(BizCompareDiff.kind.in_(("removed", "changed"))) + elif kind_n == "removed": + q = q.filter(BizCompareDiff.kind == "removed") + elif kind_n == "changed": + q = q.filter(BizCompareDiff.kind == "changed") elif kind_n != "all": q = q.filter(BizCompareDiff.kind == kind_n) - if kw_n: - q = q.filter(BizCompareDiff.search_text.ilike(f"%{kw_n}%")) + if kw_n or field_q: + q = _search_stored_diffs(q, metric_id=str((sheet or {}).get("metric_id") or mid), + kw=kw_n, field_q=field_q) total = q.count() rows = ( q.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc()) @@ -1893,12 +4050,15 @@ def list_run_diffs( .limit(size_n) .all() ) + items = _hydrate_diff_rows(db, [_diff_row_out(x) for x in rows], run=r, sheet=sheet) return { "total": total, "page": page_n, "page_size": size_n, "metric_id": mid, - "items": [_diff_row_out(x) for x in rows], + "items": items, + "source": "stored", + "truncated": False, } # Legacy: diffs embedded in summary_json / diffs_json @@ -1908,7 +4068,7 @@ def list_run_diffs( inline = list((sheet or {}).get("diffs") or []) if not inline and mid == r.metric_id: inline = list(r.diffs_json or []) - filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n) + filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n, field_q=field_q) total = len(filtered) start = (page_n - 1) * size_n page_items = filtered[start : start + size_n] @@ -1918,43 +4078,58 @@ def list_run_diffs( "page_size": size_n, "metric_id": mid, "items": page_items, + "source": "stored", + "truncated": False, } -def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str, Any]]: - """Load all diffs for one sheet (export). Prefer row table; fall back to inline.""" +def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> Iterator[dict[str, Any]]: + """Stream one sheet using a stable (seq, id) cursor; fall back to inline. + + Compact success rows are hydrated in chunks from metric tables. + """ + r = db.get(BizCompareRun, run_id) + if not r: + return + tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None + sheets = _sheet_meta_from_summary(dict(r.summary_json or {}), r, tpl) + sheet = next((s for s in sheets if sheet_key(s) == metric_id), None) if _run_has_diff_rows(db, run_id): - out: list[dict[str, Any]] = [] - offset = 0 + cursor: tuple[int, str] | None = None while True: - rows = ( - db.query(BizCompareDiff) - .filter(BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id) + q = db.query(BizCompareDiff).filter( + BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id, + ) + if cursor is not None: + seq, diff_id = cursor + q = q.filter(or_(BizCompareDiff.seq > seq, and_( + BizCompareDiff.seq == seq, BizCompareDiff.id > diff_id, + ))) + rows = (q .order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc()) - .offset(offset) .limit(_DIFF_CHUNK) .all() ) if not rows: break - out.extend(_diff_row_out(x) for x in rows) - offset += len(rows) + chunk = _hydrate_diff_rows(db, [_diff_row_out(x) for x in rows], run=r, sheet=sheet) + cursor = (rows[-1].seq, rows[-1].id) + yield from chunk if len(rows) < _DIFF_CHUNK: break - return out - r = db.get(BizCompareRun, run_id) - if not r: - return [] + # Release the previous chunk before loading the next one. + del rows, chunk + return summary = dict(r.summary_json or {}) sheets = list(summary.get("sheets") or []) for sh in sheets: if sheet_key(sh) == metric_id or ( str(sh.get("metric_id") or "") == metric_id and sheet_key(sh) == metric_id ): - return list(sh.get("diffs") or []) + yield from sh.get("diffs") or [] + return if metric_id == r.metric_id: - return list(r.diffs_json or []) - return [] + yield from r.diffs_json or [] def export_run_zip(db: Session, run_id: str) -> bytes: @@ -1983,15 +4158,18 @@ def export_run_zip(db: Session, run_id: str) -> bytes: f"+{card.get('added')}/-{card.get('removed')}/~{card.get('changed')}/= {card.get('unchanged')}" ) zf.writestr("manifest.txt", "\n".join(manifest) + "\n") + run = db.get(BizCompareRun, run_id) + snapshot = _run_config_snapshot(run) if run else None + if snapshot is not None: + zf.writestr("config_snapshot.json", json.dumps(snapshot, ensure_ascii=False, indent=2)) for sheet in list(detail.get("sheets") or []): sid = sheet_key(sheet) or "sheet" safe = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in sid)[:80] or "sheet" - sheet_full = { - **sheet, - "diffs": _iter_sheet_diffs(db, run_id, sid), - } - zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet_full)) - sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"] + with zf.open(f"tables/{safe}.csv", "w", force_zip64=True) as csv_file: + csv_file.write(b"\xef\xbb\xbf") + for line in _sheet_csv_lines(sheet, _iter_sheet_diffs(db, run_id, sid)): + csv_file.write((line + "\n").encode("utf-8")) + sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate,data_state"] for card in list(s.get("sheet_cards") or []): sum_lines.append( ",".join( @@ -2007,6 +4185,7 @@ def export_run_zip(db: Session, run_id: str) -> bytes: card.get("unchanged"), card.get("diff_count"), card.get("pass_rate"), + card.get("data_state") or "ok", ) ) ) @@ -2025,22 +4204,34 @@ def list_runs(db: Session, job_id: str, *, limit: int = 20) -> list[dict[str, An job = db.get(BizCompareJob, job_id) before_tid = str(job.before_task_id or "") if job else "" after_tid = str(job.after_task_id or "") if job else "" - return [ - { - "id": r.id, - "before_batch_id": r.before_batch_id, - "after_batch_id": r.after_batch_id, - "before": _compare_side(db, r.before_batch_id, fallback_task_id=before_tid), - "after": _compare_side(db, r.after_batch_id, fallback_task_id=after_tid), - "status": r.status, - "summary": { - k: (r.summary_json or {}).get(k, 0) - for k in ("added", "removed", "changed", "unchanged", "sheet_count") - }, - "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, - } - for r in rows - ] + out: list[dict[str, Any]] = [] + for r in rows: + summary = dict(r.summary_json or {}) + out.append( + { + "id": r.id, + "before_batch_id": r.before_batch_id, + "after_batch_id": r.after_batch_id, + "before": _compare_side(db, r.before_batch_id, fallback_task_id=before_tid), + "after": _compare_side(db, r.after_batch_id, fallback_task_id=after_tid), + "status": r.status, + "message": r.message or "", + "summary": { + k: summary.get(k, 0) + for k in ( + "added", + "removed", + "changed", + "unchanged", + "sheet_count", + "duration_ms", + ) + }, + "progress": dict(summary.get("progress") or {}), + "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, + } + ) + return out def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int: @@ -2062,7 +4253,7 @@ def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int: if not j.before_batch_id: continue try: - run_compare(db, j.id, force_after_batch_id=batch_id) + run_compare(db, j.id, force_after_batch_id=batch_id, purpose="auto") n += 1 except HTTPException as exc: if int(getattr(exc, "status_code", 0) or 0) == 409: diff --git a/netx_api/biz_state/compare_sql.py b/netx_api/biz_state/compare_sql.py new file mode 100644 index 0000000..d417475 --- /dev/null +++ b/netx_api/biz_state/compare_sql.py @@ -0,0 +1,801 @@ +"""PostgreSQL-backed sheet compare (FULL OUTER JOIN) for pushdown-safe sheets. + +Falls back to the Python engine when port-map / complex rules cannot be expressed +in SQL. See ``can_sql_compare``. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, Callable, Mapping, Sequence +from uuid import uuid4 + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from .compare_engine import key_tuple +from .compare_rules import effective_compare_fields, field_rule_map, NUMBER_PATTERN, scalar_text + +_log = logging.getLogger("netx.biz_state.compare_sql") + +# Below this, Python hash-join is faster and avoids TEMP CTAS / progress races. +_SQL_MIN_ROWS = 20_000 +# Cap a single SQL statement so a stuck planner/lock surfaces as fallback. +_SQL_STATEMENT_TIMEOUT_MS = 180_000 + +_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class SqlCompareSkip(Exception): + """Soft skip — caller should use the Python engine (not an error).""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = str(reason or "skip") +_SQL_FILTER_OPS = frozenset( + {"eq", "==", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "ci_eq"} +) +_SQL_NORMALIZE = frozenset({"", "none", "strip", "lower", "upper", "empty_as_blank"}) +_SQL_COMPARE_MODES = frozenset( + { + "", + "eq", + "ignore", + "skip", + "off", + "numeric", + "number", + "int", + "float", + "percent", + "pct", + "rel", + } +) +_EMPTY_AS_BLANK = ("n/a", "na", "-", "--", "none", "null") +_NUM_RE_SQL = NUMBER_PATTERN + + +def _dialect_is_postgres(db: Session) -> bool: + bind = db.get_bind() + if bind is None: + return False + return str(getattr(bind.dialect, "name", "") or "").lower() in ("postgresql", "postgres") + + +def _safe_field(name: str) -> str: + f = str(name or "").strip() + if not f or not _FIELD_RE.match(f): + raise ValueError(f"unsafe_json_field:{name!r}") + return f + + +def _json_text_expr(alias: str, field: str) -> str: + """SQL expression: trimmed text from JSONB column ``alias.data`` / ``data_json``.""" + f = _safe_field(field) + # alias is caller-controlled (_b / data_json) — not user input + return f"trim(both from coalesce({alias}->>'{f}', ''))" + + +def _norm_expr(alias: str, field: str, normalize: str) -> str: + base = _json_text_expr(alias, field) + mode = str(normalize or "strip").strip().lower() or "strip" + if mode in ("", "none", "strip"): + return base + if mode == "lower": + return f"lower({base})" + if mode == "upper": + return f"upper({base})" + if mode == "empty_as_blank": + opts = ", ".join(f"'{x}'" for x in _EMPTY_AS_BLANK) + return ( + f"CASE WHEN lower({base}) IN ({opts}) THEN '' ELSE {base} END" + ) + raise ValueError(f"unsupported_normalize:{mode}") + + +def _filters_sql_compatible(filters: Sequence[Mapping[str, Any]] | None) -> bool: + fl = [f for f in (filters or []) if isinstance(f, dict)] + return all(_filter_node_compatible(f) for f in fl) + + +def _filter_node_compatible(filt: Mapping[str, Any]) -> bool: + if "any" in filt: + kids = filt.get("any") or [] + return isinstance(kids, list) and all( + isinstance(k, dict) and _filter_node_compatible(k) for k in kids + ) + if "all" in filt: + kids = filt.get("all") or [] + return isinstance(kids, list) and all( + isinstance(k, dict) and _filter_node_compatible(k) for k in kids + ) + field = str(filt.get("field") or "").strip() + op = str(filt.get("op") or "eq").strip().lower() + if op not in _SQL_FILTER_OPS: + return False + if op in ("empty", "not_empty", "nonempty"): + return bool(field) and bool(_FIELD_RE.match(field)) + if not field or not _FIELD_RE.match(field): + return False + if op in ("in", "not_in", "nin"): + vals = filt.get("value") + if vals is None: + return True + if not isinstance(vals, (list, tuple)): + vals = [vals] + return all(isinstance(x, (str, int, float, bool)) or x is None for x in vals) + return True + + +def _field_rules_sql_compatible(rules: Sequence[Mapping[str, Any]] | None) -> bool: + for raw in rules or []: + if not isinstance(raw, dict): + return False + name = str(raw.get("field") or "").strip() + if name and not _FIELD_RE.match(name): + return False + mode = str(raw.get("compare") or "eq").strip().lower() or "eq" + if mode not in _SQL_COMPARE_MODES: + return False + if raw.get("ignore") is True: + continue + if mode in ("ignore", "skip", "off"): + continue + norm = str(raw.get("normalize") or "strip").strip().lower() or "strip" + if norm not in _SQL_NORMALIZE: + return False + return True + + +def sql_compare_skip_reason( + db: Session, + sheet: Mapping[str, Any], + *, + port_map: Mapping[str, str] | None = None, + iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, + pipeline: Any = None, +) -> str: + """Empty string when SQL is allowed; otherwise a short reason for progress/logs. + + Simple ``row_filters`` (eq/in/contains/…) used for BGP sheet splits are fine — + they do **not** force Python by themselves. + + A port map only blocks SQL when the sheet actually reads an interface field + in its key or compare columns; a BGP route table with no interface fields is + unaffected by it. ``pipeline`` (v2 identity rules) adds its own blockers. + """ + if not _dialect_is_postgres(db): + return "not_postgres" + if pipeline is not None: + port_map = pipeline.port_map + if iface_normalize_rules is None and (pipeline.before.has_iface_rules or pipeline.after.has_iface_rules): + iface_normalize_rules = [{"from": "*", "to": "*"}] + key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] + if not key_fields: + return "no_key_fields" + if any(not _FIELD_RE.match(k) for k in key_fields): + return "unsafe_key_field" + iface_fields = [str(f).strip() for f in (sheet.get("iface_fields") or []) if str(f).strip()] + iface_set = set(iface_fields) + if port_map and iface_set: + early_cmp = effective_compare_fields( + list(sheet.get("compare_fields") or []), list(sheet.get("field_rules") or []), + ) + if iface_set & (set(key_fields) | {str(f).strip() for f in early_cmp}): + return "port_map" + ignore_ports = sheet.get("ignore_port_changes") + if ignore_ports is True: + return "ignore_port_changes" + if ignore_ports is None and iface_set and any(k in iface_set for k in key_fields): + return "auto_ignore_ports" + field_rules = list(sheet.get("field_rules") or []) + if any( + isinstance(r, dict) and str(r.get("compare") or "").strip().lower() in + ("not_worse", "numeric_min", "numeric_max") + for r in field_rules + ): + return "directional_compare" + if not _field_rules_sql_compatible(field_rules): + return "field_rules" + compare_fields = effective_compare_fields( + list(sheet.get("compare_fields") or []), + field_rules, + ) + if any(not _FIELD_RE.match(str(f).strip()) for f in compare_fields if str(f).strip()): + return "unsafe_compare_field" + used = set(key_fields) | {str(f).strip() for f in compare_fields if str(f).strip()} + if port_map and (used & iface_set): + return "port_map" + if iface_normalize_rules and (used & iface_set): + return "iface_normalize" + if pipeline is not None: + blocked = pipeline.sql_blockers(used) + if blocked: + return blocked + if not _filters_sql_compatible(list(sheet.get("row_filters") or [])): + return "row_filters" + if not str(sheet.get("metric_id") or "").strip(): + return "no_metric_id" + return "" + + +def can_sql_compare( + db: Session, + sheet: Mapping[str, Any], + *, + port_map: Mapping[str, str] | None = None, + iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, +) -> bool: + """True when this sheet can run entirely as a PostgreSQL JOIN.""" + return not bool( + sql_compare_skip_reason( + db, + sheet, + port_map=port_map, + iface_normalize_rules=iface_normalize_rules, + ) + ) + + +def compile_row_filters_sql( + filters: Sequence[Mapping[str, Any]] | None, + *, + json_col: str = "data_json", + param_prefix: str = "f", +) -> tuple[str, dict[str, Any]]: + """Compile template row_filters to SQL AND-clause + bind params. + + Returns ``(sql_fragment, params)``. Empty filters → ``(\"TRUE\", {})``. + ``json_col`` is the JSONB column/expression name (trusted). + """ + fl = [f for f in (filters or []) if isinstance(f, dict)] + if not fl: + return "TRUE", {} + params: dict[str, Any] = {} + counter = {"n": 0} + + def _next(name: str) -> str: + counter["n"] += 1 + return f"{param_prefix}_{counter['n']}_{name}" + + def _node(filt: Mapping[str, Any]) -> str: + if "any" in filt: + kids = [k for k in (filt.get("any") or []) if isinstance(k, dict)] + if not kids: + return "TRUE" + return "(" + " OR ".join(_node(k) for k in kids) + ")" + if "all" in filt: + kids = [k for k in (filt.get("all") or []) if isinstance(k, dict)] + if not kids: + return "TRUE" + return "(" + " AND ".join(_node(k) for k in kids) + ")" + field = _safe_field(str(filt.get("field") or "")) + op = str(filt.get("op") or "eq").strip().lower() + expr = _json_text_expr(json_col, field) + # _json_text_expr uses alias->> ; for bare column use data_json directly + if json_col == "data_json": + expr = f"trim(both from coalesce(data_json->>'{field}', ''))" + if op in ("empty",): + return f"({expr} = '')" + if op in ("not_empty", "nonempty"): + return f"({expr} <> '')" + expect = filt.get("value") + if op in ("eq", "==", "ci_eq"): + key = _next("v") + params[key] = scalar_text(expect).strip() + if op == "ci_eq" or op in ("eq", "=="): + # Python eq is case-insensitive via .lower() + params[key] = scalar_text(expect).strip().lower() + return f"(lower({expr}) = :{key})" + if op in ("ne", "!="): + key = _next("v") + params[key] = scalar_text(expect).strip().lower() + return f"(lower({expr}) <> :{key})" + if op == "contains": + key = _next("v") + needle = scalar_text(expect).strip().lower() + if not needle: + return "FALSE" + escaped = needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + params[key] = f"%{escaped}%" + return f"(lower({expr}) LIKE :{key} ESCAPE E'\\\\')" + if op in ("in", "not_in", "nin"): + vals = expect + if vals is None: + vals = [] + if not isinstance(vals, (list, tuple)): + vals = [vals] + clean = [scalar_text(x).strip().lower() for x in vals if scalar_text(x).strip()] + if not clean: + return "FALSE" if op == "in" else "TRUE" + keys = [] + for i, v in enumerate(clean): + k = _next(f"in{i}") + params[k] = v + keys.append(f":{k}") + inside = f"lower({expr}) IN ({', '.join(keys)})" + return f"({inside})" if op == "in" else f"(NOT {inside})" + raise ValueError(f"unsupported_filter_op:{op}") + + parts = [_node(f) for f in fl] + return "(" + " AND ".join(parts) + ")", params + + +def _rk_sql(key_fields: list[str], *, json_col: str = "data_json") -> str: + parts = [] + for k in key_fields: + f = _safe_field(k) + if json_col == "data_json": + parts.append(f"trim(both from coalesce(data_json->>'{f}', ''))") + else: + parts.append(f"trim(both from coalesce({json_col}->>'{f}', ''))") + if len(parts) == 1: + return parts[0] + # Delimiter concatenation conflates ('a|b', 'c') with ('a', 'b|c'). + return "jsonb_build_array(" + ", ".join(parts) + ")::text" + + +def _field_differs_sql(bv: str, av: str, rule: Mapping[str, Any]) -> str: + """SQL boolean: True when before/after values differ under the field rule.""" + mode = str(rule.get("compare") or "eq").strip().lower() or "eq" + if mode in ("ignore", "skip", "off") or rule.get("ignore") is True: + return "FALSE" + try: + tol = float(rule.get("tolerance") or 0) + except (TypeError, ValueError): + tol = 0.0 + if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"): + # Match Python values_equal: parseable → numeric compare; else string eq + num_b = f"(({bv}) ~ '{_NUM_RE_SQL}')" + num_a = f"(({av}) ~ '{_NUM_RE_SQL}')" + bn = f"({bv})::double precision" + an = f"({av})::double precision" + if mode in ("percent", "pct", "rel"): + # differ when relative % > tol (before==0 → after must be 0) + num_diff = ( + f"(CASE WHEN {bn} = 0 THEN {an} IS DISTINCT FROM 0 " + f"ELSE (abs({an} - {bn}) / abs({bn}) * 100.0) > {tol} END)" + ) + else: + num_diff = f"(abs({an} - {bn}) > {tol})" + return ( + f"(CASE WHEN {num_b} AND {num_a} THEN {num_diff} " + f"ELSE ({bv} IS DISTINCT FROM {av}) END)" + ) + return f"({bv} IS DISTINCT FROM {av})" + + +def _changed_predicate( + compare_fields: list[str], + rules: dict[str, dict[str, Any]], + *, + before_alias: str = "b", + after_alias: str = "a", +) -> str: + """SQL boolean: True when any compare field differs (IS DISTINCT FROM).""" + if not compare_fields: + return "FALSE" + clauses: list[str] = [] + for f in compare_fields: + name = str(f).strip() + if not name: + continue + rule = rules.get(name) or {} + mode = str(rule.get("compare") or "eq").strip().lower() or "eq" + if mode in ("ignore", "skip", "off") or rule.get("ignore") is True: + continue + norm = str(rule.get("normalize") or "strip").strip().lower() or "strip" + bv = _norm_expr(f"{before_alias}.data", name, norm) + av = _norm_expr(f"{after_alias}.data", name, norm) + clauses.append(_field_differs_sql(bv, av, rule)) + if not clauses: + return "FALSE" + return "(" + " OR ".join(clauses) + ")" + + +def _key_obj_from_data(data: dict[str, Any] | None, key_fields: list[str]) -> dict[str, Any]: + row = data if isinstance(data, dict) else {} + return dict(zip(key_fields, key_tuple(row, key_fields))) + + +def _changes_from_rows( + before: dict[str, Any] | None, + after: dict[str, Any] | None, + compare_fields: list[str], + rules: dict[str, dict[str, Any]], +) -> dict[str, dict[str, Any]]: + from .compare_rules import explain_diff, values_equal + + out: dict[str, dict[str, Any]] = {} + b = before or {} + a = after or {} + for f in compare_fields: + rule = rules.get(f) + bv = b.get(f, "") + av = a.get(f, "") + if values_equal(bv, av, rule=rule): + continue + entry: dict[str, Any] = {"before": bv, "after": av} + reason = explain_diff(bv, av, rule=rule) + if reason: + entry["reason"] = reason + out[f] = entry + return out + + +def dup_pairing_order_sql(compare_fields: Sequence[str], rules: Mapping[str, Mapping[str, Any]]) -> str: + """``ORDER BY`` prefix that pairs same-key rows by value (mirrors ``Pipeline.pair_sort_key``). + + Byte-order collation matches Python's code-point string order, so both + engines choose the same pairs. Returns '' (or a trailing-comma list). + """ + parts = [] + for f in compare_fields: + name = str(f).strip() + if not name: + continue + mode = str((rules.get(name) or {}).get("normalize") or "strip").strip().lower() or "strip" + parts.append(f'({_norm_expr("data_json", name, mode)}) COLLATE "C" ASC') + return ", ".join(parts) + ", " if parts else "" + + +def run_sql_sheet_compare( + db: Session, + *, + sheet: Mapping[str, Any], + before_batch_id: str, + after_batch_id: str, + store_unchanged: str = "auto", + on_progress: Callable[..., None] | None = None, + duplicate_pairing: str = "ordered", +) -> dict[str, Any]: + """Compare one sheet via PostgreSQL TEMP tables + FULL OUTER JOIN. + + Same-key multipath: rows get ``dup_rn`` by ``seq,id`` and join on + ``(rk, dup_rn)`` (ordered zip). Extras are added/removed, not duplicate. + + Returns the same ``{summary, diffs, mapping_stats}`` shape as ``compare_rows``. + ``on_progress(side, n, *, engine=\"sql\", note=..., phase=...)``. + """ + if not _dialect_is_postgres(db): + raise RuntimeError("sql_compare_requires_postgres") + + def _prog(side: str, n: int, *, phase: str = "sql_count", note: str = "") -> None: + if not on_progress: + return + try: + on_progress(side, n, engine="sql", note=note, phase=phase) + except TypeError: + on_progress(side, n) + + key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] + field_rules = list(sheet.get("field_rules") or []) + rules = field_rule_map(field_rules) + compare_fields = effective_compare_fields( + list(sheet.get("compare_fields") or []), + field_rules, + ) + row_filters = list(sheet.get("row_filters") or []) + mid = str(sheet.get("metric_id") or "").strip() + bid_b = str(before_batch_id or "").strip() + bid_a = str(after_batch_id or "").strip() + if not mid or not bid_b or not bid_a or not key_fields: + raise ValueError("sql_compare_missing_args") + + filter_sql, filter_params = compile_row_filters_sql(row_filters) + rk = _rk_sql(key_fields) + dup_order = dup_pairing_order_sql(compare_fields, rules) if duplicate_pairing == "sorted" else "" + tag = uuid4().hex[:8] + tb = f"_netx_cmp_b_{tag}" + ta = f"_netx_cmp_a_{tag}" + + # Keep worker transaction open across CTAS — progress must NOT commit this session. + db.execute(text(f"SET LOCAL statement_timeout = '{int(_SQL_STATEMENT_TIMEOUT_MS)}'")) + + _prog("before", 0, phase="sql_count", note="count") + + # Raw counts (no row_filters) + raw_b = int( + db.execute( + text( + "SELECT count(*) FROM biz_state_metric_row " + "WHERE batch_id = :bid AND metric_id = :mid" + ), + {"bid": bid_b, "mid": mid}, + ).scalar() + or 0 + ) + _prog("before", raw_b, phase="sql_count") + raw_a = int( + db.execute( + text( + "SELECT count(*) FROM biz_state_metric_row " + "WHERE batch_id = :bid AND metric_id = :mid" + ), + {"bid": bid_a, "mid": mid}, + ).scalar() + or 0 + ) + _prog("after", raw_a, phase="sql_count") + + if max(raw_b, raw_a) < int(_SQL_MIN_ROWS): + raise SqlCompareSkip("small_sheet") + + base_params = {"bid": bid_b, "mid": mid, **filter_params} + # Build TEMP sides (one transaction — no mid-flight commits on ``db``) + _prog("before", raw_b, phase="sql_project", note="temp_before") + for tname, batch_id, side, note in ( + (tb, bid_b, "before", "temp_before"), + (ta, bid_a, "after", "temp_after"), + ): + db.execute(text(f"DROP TABLE IF EXISTS {tname}")) + params = {**base_params, "bid": batch_id} + if side == "after": + _prog(side, raw_a, phase="sql_project", note=note) + db.execute( + text( + f""" + CREATE TEMP TABLE {tname} ON COMMIT PRESERVE ROWS AS + SELECT + id, + ({rk}) AS rk, + data_json AS data, + row_number() OVER ( + PARTITION BY ({rk}) + ORDER BY {dup_order}seq ASC, id ASC + ) AS dup_rn + FROM biz_state_metric_row + WHERE batch_id = :bid + AND metric_id = :mid + AND ({filter_sql}) + """ + ), + params, + ) + db.execute(text(f"CREATE INDEX IF NOT EXISTS {tname}_rk ON {tname} (rk, dup_rn)")) + + before_n = int( + db.execute(text(f"SELECT count(*) FROM {tb}")).scalar() or 0 + ) + after_n = int( + db.execute(text(f"SELECT count(*) FROM {ta}")).scalar() or 0 + ) + _prog("after", after_n, phase="sql_join", note="join") + + changed_pred = _changed_predicate(compare_fields, rules) + kind_expr = f""" + CASE + WHEN b.id IS NULL THEN 'added' + WHEN a.id IS NULL THEN 'removed' + WHEN {changed_pred} THEN 'changed' + ELSE 'unchanged' + END + """ + + # Ordered same-key pairing: join on (rk, dup_rn) so 5 vs 2 → 2 compared + 3 removed + agg_rows = db.execute( + text( + f""" + SELECT {kind_expr} AS kind, count(*)::bigint AS n + FROM {tb} b + FULL OUTER JOIN {ta} a + ON b.rk = a.rk AND b.dup_rn = a.dup_rn + GROUP BY 1 + """ + ) + ).mappings().all() + counts = {str(r["kind"]): int(r["n"] or 0) for r in agg_rows} + added = counts.get("added", 0) + removed = counts.get("removed", 0) + changed = counts.get("changed", 0) + unchanged = counts.get("unchanged", 0) + + # Diagnostic: count of match keys with >1 row (not fail rows) + multi_b = int( + db.execute( + text(f"SELECT count(*) FROM (SELECT rk FROM {tb} GROUP BY rk HAVING count(*) > 1) t") + ).scalar() + or 0 + ) + multi_a = int( + db.execute( + text(f"SELECT count(*) FROM (SELECT rk FROM {ta} GROUP BY rk HAVING count(*) > 1) t") + ).scalar() + or 0 + ) + + # Lazy import — avoid circular import with compare_service + from .compare_service import resolve_unchanged_policy + + policy = resolve_unchanged_policy( + store_unchanged, before_n=before_n, after_n=after_n + ) + diffs: list[dict[str, Any]] = [] + + # Fail rows (stream into Python — should be << million) + fail_sql = text( + f""" + SELECT + {kind_expr} AS kind, + b.id AS before_row_id, + a.id AS after_row_id, + b.data AS before_data, + a.data AS after_data, + COALESCE(b.rk, a.rk) AS rk + FROM {tb} b + FULL OUTER JOIN {ta} a + ON b.rk = a.rk AND b.dup_rn = a.dup_rn + WHERE {kind_expr} IN ('added', 'removed', 'changed') + """ + ) + fail_n = 0 + _prog("after", after_n, phase="sql_fail_fetch", note="fetch_fails") + for row in db.execute(fail_sql).mappings(): + kind = str(row["kind"] or "") + before = dict(row["before_data"] or {}) if row["before_data"] is not None else None + after = dict(row["after_data"] or {}) if row["after_data"] is not None else None + key_src = after if kind == "added" else (before or after or {}) + item: dict[str, Any] = { + "kind": kind, + "key": _key_obj_from_data(key_src, key_fields), + "before": before, + "after": after, + "mapped_before": before, + "changes": {}, + "before_row_id": str(row["before_row_id"] or ""), + "after_row_id": str(row["after_row_id"] or ""), + } + if kind == "changed": + item["changes"] = _changes_from_rows(before, after, compare_fields, rules) + diffs.append(item) + fail_n += 1 + if fail_n == 1 or fail_n % 25_000 == 0: + _prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails") + if fail_n: + _prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails") + + unchanged_listed = 0 + include_u = bool(policy.get("include")) + compact = bool(policy.get("compact")) + limit_n = policy.get("limit") + if include_u and unchanged > 0: + params_u: dict[str, Any] = {} + # Stratified sample: round-robin by neighbor|direction|afi|vrf so one + # BGP command cannot consume the whole success sample. + stratum_expr = """ + concat_ws('|', + coalesce(nullif(trim(both from coalesce(b.data->>'neighbor','')), ''), '_'), + coalesce(nullif(trim(both from coalesce(b.data->>'direction','')), ''), '_'), + coalesce(nullif(trim(both from coalesce(b.data->>'afi','')), ''), '_'), + coalesce(nullif(trim(both from coalesce(b.data->>'vrf','')), ''), '_') + ) + """ + if limit_n is not None: + params_u["lim"] = max(0, int(limit_n)) + u_sql = text( + f""" + WITH u AS ( + SELECT + b.id AS before_row_id, + a.id AS after_row_id, + b.data AS before_data, + a.data AS after_data, + {stratum_expr} AS stratum + FROM {tb} b + INNER JOIN {ta} a + ON b.rk = a.rk AND b.dup_rn = a.dup_rn + WHERE NOT ({changed_pred}) + ), + ranked AS ( + SELECT *, + row_number() OVER ( + PARTITION BY stratum ORDER BY before_row_id ASC + ) AS rn_in + FROM u + ), + picked AS ( + SELECT *, + row_number() OVER ( + ORDER BY rn_in ASC, stratum ASC, before_row_id ASC + ) AS pick_ord + FROM ranked + ) + SELECT before_row_id, after_row_id, before_data, after_data + FROM picked + WHERE pick_ord <= :lim + """ + ) + else: + u_sql = text( + f""" + SELECT + b.id AS before_row_id, + a.id AS after_row_id, + b.data AS before_data, + a.data AS after_data + FROM {tb} b + INNER JOIN {ta} a + ON b.rk = a.rk AND b.dup_rn = a.dup_rn + WHERE NOT ({changed_pred}) + """ + ) + for row in db.execute(u_sql, params_u).mappings(): + before = dict(row["before_data"] or {}) + after = dict(row["after_data"] or {}) + unchanged_listed += 1 + if compact: + diffs.append( + { + "kind": "unchanged", + "key": _key_obj_from_data(before, key_fields), + "before": {}, + "after": {}, + "mapped_before": {}, + "changes": {}, + "compact": True, + "before_row_id": str(row["before_row_id"] or ""), + "after_row_id": str(row["after_row_id"] or ""), + } + ) + else: + diffs.append( + { + "kind": "unchanged", + "key": _key_obj_from_data(before, key_fields), + "before": before, + "after": after, + "mapped_before": before, + "changes": {}, + "before_row_id": str(row["before_row_id"] or ""), + "after_row_id": str(row["after_row_id"] or ""), + } + ) + + # Cleanup (also ON COMMIT DROP) + db.execute(text(f"DROP TABLE IF EXISTS {tb}")) + db.execute(text(f"DROP TABLE IF EXISTS {ta}")) + + summary = { + "before_count": before_n, + "after_count": after_n, + "added": added, + "removed": removed, + "changed": changed, + "unchanged": unchanged, + "duplicate": 0, + "match_key_fields": list(key_fields), + "duplicate_keys_before": multi_b, + "duplicate_keys_after": multi_a, + "duplicate_key_list": [], + "unchanged_listed": unchanged_listed, + "unchanged_truncated": bool( + include_u and limit_n is not None and unchanged > unchanged_listed + ), + "unchanged_compact": bool(compact and unchanged_listed > 0), + "unchanged_sample_mode": ( + "stratified" + if include_u and limit_n is not None and unchanged > unchanged_listed + else ("full" if include_u else "none") + ), + "engine": "sql", + "before_raw_count": raw_b, + "after_raw_count": raw_a, + "row_filters": len(row_filters), + "unchanged_policy": policy, + } + mapping_stats = { + "before_iface_count": 0, + "after_iface_count": 0, + "map_pairs": 0, + "hit_before": [], + "miss_before": [], + "hit_after": [], + "miss_after": [], + "unused_before_keys": [], + "ok": True, + "ignore_port_changes": False, + "engine": "sql", + } + return {"summary": summary, "diffs": diffs, "mapping_stats": mapping_stats} diff --git a/netx_api/biz_state/compare_validation.py b/netx_api/biz_state/compare_validation.py new file mode 100644 index 0000000..3fba6c7 --- /dev/null +++ b/netx_api/biz_state/compare_validation.py @@ -0,0 +1,168 @@ +"""Validate template writes without silently dropping parts of the comparison.""" +from __future__ import annotations + +import math +import re +from typing import Any + +from fastapi import HTTPException + +FILTER_OPS = {"eq", "==", "ci_eq", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "regex", "age_timer"} +COMPARE_MODES = {"", "eq", "ignore", "skip", "off", "numeric", "number", "int", "float", "percent", "pct", "rel", + "not_worse", "numeric_min", "numeric_max"} +SIDES = {"", "both", "before", "after"} +MAX_VALUE_MAP = 500 +NORMALIZE_MODES = {"", "none", "strip", "lower", "upper", "mac", "empty_as_blank"} + + +def invalid(path: str, reason: str) -> None: + raise HTTPException(status_code=400, detail={"error": "invalid_template", "path": path, "reason": reason}) + + +def _strings(value: Any, path: str, *, required: bool = False) -> None: + if not isinstance(value, list) or any(not isinstance(x, str) or not x.strip() for x in value): + invalid(path, "nonempty_strings_required") + if required and not value: + invalid(path, "key_fields_required") + if len({x.strip() for x in value}) != len(value): + invalid(path, "duplicate_field") + + +def validate_filters(value: Any, path: str, depth: int = 0) -> None: + if not isinstance(value, list): + invalid(path, "list_required") + if depth > 12: + invalid(path, "filter_nesting_too_deep") + for i, node in enumerate(value): + here = f"{path}[{i}]" + if not isinstance(node, dict) or not node: + invalid(here, "filter_required") + if depth == 0 and str(node.get("side") or "").strip().lower() not in SIDES: + invalid(here, "unknown_filter_side") + groups = [key for key in ("any", "all") if key in node] + if groups: + if len(groups) != 1 or any(node.get(key) is not None for key in ("field", "op", "value")): + invalid(here, "group_or_leaf_required") + kids = node[groups[0]] + if not isinstance(kids, list) or not kids: + invalid(here, "nonempty_group_required") + validate_filters(kids, f"{here}.{groups[0]}", depth + 1) + continue + if not isinstance(node.get("field"), str) or not node["field"].strip(): + invalid(here, "filter_field_required") + op = str(node.get("op") or "eq").strip().lower() + if op not in FILTER_OPS: + invalid(here, "unknown_filter_operator") + expect = node.get("value") + if op in ("in", "not_in", "nin"): + if not isinstance(expect, list) or not expect or any(not isinstance(x, (str, int, float, bool)) for x in expect): + invalid(here, "nonempty_value_list_required") + elif op not in ("empty", "not_empty", "nonempty", "age_timer"): + if not isinstance(expect, (str, int, float, bool)): + invalid(here, "scalar_value_required") + if op in ("contains", "regex") and not str(expect).strip(): + invalid(here, "value_required") + if op == "regex": + try: + re.compile(str(expect), re.I) + except re.error: + invalid(here, "invalid_regex") + + +def validate_sheet(sheet: Any, path: str, *, partial: bool = False) -> None: + if not isinstance(sheet, dict): + invalid(path, "sheet_required") + if not partial or "metric_id" in sheet: + if not isinstance(sheet.get("metric_id"), str) or not sheet["metric_id"].strip(): + invalid(path, "metric_id_required") + for key in ("key_fields", "iface_fields", "compare_fields", "display_fields", "ignore_fields"): + if key in sheet or (key == "key_fields" and not partial): + _strings(sheet.get(key), f"{path}.{key}", required=key == "key_fields") + for opt, allowed in (("unmapped_port_policy", {"by_name", "ignore_port"}), + ("duplicate_pairing", {"sorted", "ordered"})): + if str(sheet.get(opt) or "").strip() and str(sheet[opt]).strip().lower() not in allowed: + invalid(path, f"unknown_{opt}") + if sheet.get("builtin_vendor_canonical") is not None and not isinstance(sheet["builtin_vendor_canonical"], bool): + invalid(path, "boolean_builtin_vendor_canonical_required") + if "ignore_port_changes" in sheet and sheet["ignore_port_changes"] is not None and not isinstance(sheet["ignore_port_changes"], bool): + invalid(path, "boolean_port_policy_required") + if "row_filters" in sheet: + validate_filters(sheet["row_filters"], f"{path}.row_filters") + if "field_rules" in sheet: + rules = sheet["field_rules"] + if not isinstance(rules, list): + invalid(path, "field_rules_list_required") + seen = set() + for i, rule in enumerate(rules): + here = f"{path}.field_rules[{i}]" + if not isinstance(rule, dict) or not isinstance(rule.get("field"), str) or not rule["field"].strip(): + invalid(here, "rule_field_required") + field = rule["field"].strip() + rule_side = str(rule.get("side") or "").strip().lower() + if rule_side not in SIDES: + invalid(here, "unknown_rule_side") + marker = (field, "" if rule_side == "both" else rule_side) + if marker in seen: + invalid(here, "duplicate_field_rule") + seen.add(marker) + vmap = rule.get("value_map") + if vmap is not None: + if not isinstance(vmap, dict) or any( + not isinstance(k, str) or not isinstance(v, str) for k, v in vmap.items() + ): + invalid(here, "value_map_string_pairs_required") + if len(vmap) > MAX_VALUE_MAP: + invalid(here, "value_map_too_large") + if rule.get("apply_to_key") is not None and not isinstance(rule["apply_to_key"], bool): + invalid(here, "boolean_apply_to_key_required") + mode = str(rule.get("compare") or "").strip().lower() + if mode == "not_worse": + rank = rule.get("rank") + if not isinstance(rank, list) or len(rank) < 2 or any(not isinstance(x, str) or not x.strip() for x in rank): + invalid(here, "rank_list_required") + elif rule.get("rank") is not None: + invalid(here, "rank_only_for_not_worse") + if str(rule.get("compare") or "").strip().lower() not in COMPARE_MODES: + invalid(here, "unknown_compare_mode") + if str(rule.get("normalize") or "").strip().lower() not in NORMALIZE_MODES: + invalid(here, "unknown_normalize_mode") + tol = rule.get("tolerance") + if tol is not None: + if isinstance(tol, bool) or not isinstance(tol, (int, float)) or not math.isfinite(tol) or tol < 0: + invalid(here, "finite_nonnegative_tolerance_required") + + +def validate_template_body(body: dict[str, Any], *, partial: bool = False) -> None: + metrics = body.get("metrics") + if metrics is not None: + if not isinstance(metrics, list) or not metrics: + invalid("metrics", "metrics_required") + seen = set() + for i, sheet in enumerate(metrics): + path = f"metrics[{i}]" + validate_sheet(sheet, path) + sid = str(sheet.get("sheet_id") or sheet["metric_id"]).strip() + if sid in seen: + invalid(path, "duplicate_sheet_id") + seen.add(sid) + else: + validate_sheet(body, "template", partial=partial) + if "iface_normalize_rules" in body and body["iface_normalize_rules"] is not None: + rules = body["iface_normalize_rules"] + if not isinstance(rules, list): + invalid("iface_normalize_rules", "list_required") + seen = set() + for i, rule in enumerate(rules): + path = f"iface_normalize_rules[{i}]" + if not isinstance(rule, dict) or any(not isinstance(rule.get(k), str) or not rule[k].strip() for k in ("from", "to")): + invalid(path, "alias_pair_required") + rside = str(rule.get("side") or "").strip().lower() + if rside not in SIDES: + invalid(path, "unknown_alias_side") + if rule.get("vendor") is not None and not isinstance(rule["vendor"], str): + invalid(path, "alias_vendor_string_required") + key = (rule["from"].strip().lower(), "" if rside == "both" else rside, + str(rule.get("vendor") or "").strip().lower()) + if key in seen: + invalid(path, "duplicate_alias") + seen.add(key) diff --git a/netx_api/biz_state/data/default_zte_status_template.json b/netx_api/biz_state/data/default_zte_status_template.json new file mode 100644 index 0000000..7549a38 --- /dev/null +++ b/netx_api/biz_state/data/default_zte_status_template.json @@ -0,0 +1,1065 @@ +{ + "format": "netx.biz_compare_template", + "version": 1, + "name": "ZTE status default", + "note": "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP, address-family sheets)", + "iface_normalize_rules": [], + "metrics": [ + { + "sheet_id": "isis_adjacency.ipv4", + "title": "ISIS IPv4", + "metric_id": "isis_adjacency", + "key_fields": [ + "process_id", + "interface", + "system_id" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state", + "lev", + "af" + ], + "display_fields": [ + "process_id", + "interface", + "system_id", + "state", + "lev", + "af" + ], + "row_filters": [ + { + "op": "contains", + "field": "af", + "value": "IPv4" + } + ], + "field_rules": [] + }, + { + "sheet_id": "isis_adjacency.ipv6", + "title": "ISIS IPv6", + "metric_id": "isis_adjacency", + "key_fields": [ + "process_id", + "interface", + "system_id" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state", + "lev", + "af" + ], + "display_fields": [ + "process_id", + "interface", + "system_id", + "state", + "lev", + "af" + ], + "row_filters": [ + { + "op": "contains", + "field": "af", + "value": "IPv6" + } + ], + "field_rules": [] + }, + { + "sheet_id": "interface_brief", + "title": "interface_brief", + "metric_id": "interface_brief", + "key_fields": [ + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "prot" + ], + "display_fields": [ + "interface", + "prot", + "admin", + "phy" + ], + "row_filters": [ + { + "op": "regex", + "field": "interface", + "value": "^[^.]+$" + }, + { + "op": "contains", + "field": "interface", + "value": "gei" + } + ], + "field_rules": [] + }, + { + "sheet_id": "arp", + "title": "arp", + "metric_id": "arp", + "key_fields": [ + "ip", + "interface", + "vrf" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "mac" + ], + "display_fields": [ + "ip", + "interface", + "vrf", + "mac", + "age", + "entry_type" + ], + "row_filters": [ + { + "any": [ + { + "field": "entry_type", + "op": "eq", + "value": "dynamic" + }, + { + "all": [ + { + "field": "entry_type", + "op": "empty" + }, + { + "field": "age", + "op": "age_timer" + } + ] + } + ] + } + ], + "field_rules": [ + { + "field": "mac", + "normalize": "mac" + } + ] + }, + { + "sheet_id": "nd6_cache", + "title": "nd6_cache", + "metric_id": "nd6_cache", + "key_fields": [ + "address", + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "link_address", + "status" + ], + "display_fields": [ + "address", + "interface", + "link_address", + "status", + "type", + "age" + ], + "row_filters": [ + { + "op": "ne", + "field": "status", + "value": " Incomplete" + } + ], + "field_rules": [] + }, + { + "sheet_id": "bgp_peer.ipv4", + "title": "BGP IPv4", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "ipv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.ipv6", + "title": "BGP IPv6", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "ipv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.vpnv4", + "title": "BGP VPNv4", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpnv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.vpnv6", + "title": "BGP VPNv6", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpnv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.evpn", + "title": "BGP EVPN", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "evpn" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_peer.vpls", + "title": "BGP VPLS", + "metric_id": "bgp_peer", + "key_fields": [ + "afi", + "vrf", + "neighbor", + "as_num" + ], + "iface_fields": [], + "compare_fields": [ + "state", + "pfx_rcd" + ], + "display_fields": [ + "afi", + "vrf", + "neighbor", + "as_num", + "state", + "pfx_rcd" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpls" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "interface_detail", + "title": "interface_detail", + "metric_id": "interface_detail", + "key_fields": [ + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "admin", + "port_status", + "ip_mtu", + "mtu", + "mpls_mtu", + "ipv6_mtu", + "port_media", + "negotiation" + ], + "display_fields": [ + "interface", + "admin", + "input_bps", + "output_bps", + "port_status", + "in_util", + "out_util", + "ip_mtu", + "mtu", + "mpls_mtu", + "ipv6_mtu", + "port_media", + "negotiation", + "bw", + "description", + "rate_period" + ], + "row_filters": [], + "field_rules": [ + { + "field": "input_bps", + "compare": "percent", + "tolerance": 5 + }, + { + "field": "output_bps", + "compare": "percent", + "tolerance": 5 + }, + { + "field": "in_util", + "compare": "numeric", + "tolerance": 5 + }, + { + "field": "out_util", + "compare": "numeric", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "ospf_neighbor", + "title": "ospf_neighbor", + "metric_id": "ospf_neighbor", + "key_fields": [ + "neighbor_id", + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state" + ], + "display_fields": [ + "neighbor_id", + "interface", + "state", + "process_id", + "address" + ], + "row_filters": [], + "field_rules": [] + }, + { + "sheet_id": "vrrp.ipv4", + "title": "VRRP IPv4", + "metric_id": "vrrp", + "key_fields": [ + "af", + "interface", + "vr_id" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state", + "priority", + "master_addr", + "vrouter_addr" + ], + "display_fields": [ + "af", + "interface", + "vr_id", + "state", + "priority", + "master_addr", + "vrouter_addr" + ], + "row_filters": [ + { + "op": "eq", + "field": "af", + "value": "ipv4" + } + ], + "field_rules": [] + }, + { + "sheet_id": "vrrp.ipv6", + "title": "VRRP IPv6", + "metric_id": "vrrp", + "key_fields": [ + "af", + "interface", + "vr_id" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "state", + "priority", + "master_addr", + "vrouter_addr" + ], + "display_fields": [ + "af", + "interface", + "vr_id", + "state", + "priority", + "master_addr", + "vrouter_addr" + ], + "row_filters": [ + { + "op": "eq", + "field": "af", + "value": "ipv6" + } + ], + "field_rules": [] + }, + { + "sheet_id": "optical_brief", + "title": "optical_brief", + "metric_id": "optical_brief", + "key_fields": [ + "interface" + ], + "iface_fields": [ + "interface" + ], + "compare_fields": [ + "status", + "rx_power" + ], + "display_fields": [ + "interface", + "status", + "rx_power", + "tx_power", + "optic_type", + "wavelength", + "rx_threshold", + "tx_threshold" + ], + "row_filters": [], + "field_rules": [ + { + "field": "rx_power", + "compare": "numeric", + "tolerance": 2 + } + ] + }, + { + "sheet_id": "lldp_neighbor", + "title": "lldp_neighbor", + "metric_id": "lldp_neighbor", + "key_fields": [ + "local_if", + "remote_sys", + "remote_if" + ], + "iface_fields": [ + "local_if" + ], + "compare_fields": [], + "display_fields": [ + "local_if", + "remote_sys", + "remote_if", + "remote_ip", + "protocol" + ], + "row_filters": [], + "field_rules": [] + }, + { + "sheet_id": "l2vpn_pw_detail", + "title": "l2vpn_pw_detail", + "metric_id": "l2vpn_pw_detail", + "key_fields": [ + "peer", + "vcid", + "service_instance" + ], + "iface_fields": [], + "compare_fields": [ + "vc_status", + "remote_status", + "vc_type", + "control_word" + ], + "display_fields": [ + "peer", + "vcid", + "service_instance", + "vc_status", + "remote_status", + "vc_type", + "control_word", + "pw_name", + "activation_status", + "service_instance_type", + "conn_mode", + "signaling", + "tunnel_dest" + ], + "row_filters": [], + "field_rules": [] + }, + { + "sheet_id": "l2vpn_mac", + "title": "l2vpn_mac", + "metric_id": "l2vpn_mac", + "key_fields": [ + "mac", + "vpn", + "neighbor", + "ac_port", + "exter_vlan", + "vpn_sid", + "neighbor_sid" + ], + "iface_fields": [ + "ac_port" + ], + "compare_fields": [], + "display_fields": [ + "mac", + "vpn", + "neighbor", + "ac_port", + "exter_vlan", + "vpn_sid", + "neighbor_sid", + "vlan", + "pw", + "attribute" + ], + "row_filters": [], + "field_rules": [] + }, + { + "sheet_id": "bgp_route", + "title": "bgp_route_ipv4", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "empty", + "field": "vrf", + "value": "" + }, + { + "op": "eq", + "field": "afi", + "value": "ipv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.vpnv4", + "title": "bgp_route_vpnv4", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpnv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.ipv6", + "title": "bgp_route_ipv6", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "empty", + "field": "vrf", + "value": "" + }, + { + "op": "eq", + "field": "afi", + "value": "ipv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.vpnv6", + "title": "bgp_route_vpnv6", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "eq", + "field": "afi", + "value": "vpnv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.vpnv6_vrf", + "title": "bgp_route_vpnv6_vrf", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "not_empty", + "field": "vrf", + "value": "" + }, + { + "op": "eq", + "field": "afi", + "value": "ipv6" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + }, + { + "sheet_id": "bgp_route.vpnv4_vrf", + "title": "bgp_route_vpnv4_vrf", + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf" + ], + "iface_fields": [], + "compare_fields": [ + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as" + ], + "display_fields": [ + "local_as", + "afi", + "neighbor", + "direction", + "rd", + "network", + "vrf", + "metric", + "loc_prf", + "path", + "as_num", + "state", + "pfx_rcd", + "remote_as", + "next_hop", + "tag", + "status_codes" + ], + "row_filters": [ + { + "op": "not_empty", + "field": "vrf", + "value": "" + }, + { + "op": "eq", + "field": "afi", + "value": "ipv4" + } + ], + "field_rules": [ + { + "field": "pfx_rcd", + "compare": "percent", + "tolerance": 5 + } + ] + } + ] +} diff --git a/netx_api/biz_state/identity_pipeline.py b/netx_api/biz_state/identity_pipeline.py new file mode 100644 index 0000000..d3f1b26 --- /dev/null +++ b/netx_api/biz_state/identity_pipeline.py @@ -0,0 +1,454 @@ +"""One identity pipeline for every compare path. + +A row's *match key* must come out identical whether it is computed while a run +executes, while its results are searched / hydrated / exported, while a mapping +is validated, or while a cutover monitor evaluates the same sheet. All of those +paths therefore build a :class:`Pipeline` and call the same methods instead of +re-implementing "normalize → map → key". + +Stages for one row on one side (``before`` or ``after``):: + + raw row ──normalize──▶ iface aliases (+vendor canonical) ▶ key normalize ▶ value_map + ──filter────▶ ``row_filters`` (v2: on the normalized row; legacy: on the raw row) + ──map───────▶ port map (before side only) + ──key───────▶ ``key_tuple`` over the match key fields + +``legacy=True`` reproduces the pre-pipeline behaviour (snapshot v1 runs): the +template alias rules are the only normalization, keys are never normalized, +there is no value map and the filters see the raw row. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping, Sequence + +from .compare_rules import ( + normalize_value, + row_matches_filter, + scalar_text, +) +from .iface_normalize import ( + normalize_iface_name, + normalize_iface_rules, + normalize_port_map, + resolve_mapped_iface, + rules_for_side, + vendor_canonical_rules, +) + +SIDES = ("before", "after") +DIRECTIONAL_MODES = {"not_worse", "numeric_min", "numeric_max"} +UNMAPPED_POLICIES = ("by_name", "ignore_port") +DUPLICATE_PAIRING = ("sorted", "ordered") + + +def filter_fields(filt: Mapping[str, Any] | None) -> set[str]: + """Every field name a (possibly nested) row filter reads.""" + out: set[str] = set() + if not isinstance(filt, Mapping): + return out + for group in ("any", "all"): + if group in filt: + for kid in filt.get(group) or []: + out |= filter_fields(kid) + return out + name = str(filt.get("field") or "").strip() + if name: + out.add(name) + return out + + +def _vendor_key(vendor: str = "", device_type: str = "") -> str: + from ..topology_lldp import resolve_vendor_key + + key = resolve_vendor_key(vendor, device_type) + return "" if key == "generic" else key + + +@dataclass(eq=False) +class SideSpec: + side: str + vendor: str = "" + canonical: list[dict[str, str]] = field(default_factory=list) + alias_rules: list[dict[str, str]] = field(default_factory=list) + # field -> {lowercased source value: canonical value} + value_maps: dict[str, dict[str, str]] = field(default_factory=dict) + # key field -> normalize mode (only fields whose rule asks for one) + key_norms: dict[str, str] = field(default_factory=dict) + strip_inner_space: bool = True + + def iface(self, name: Any) -> str: + text = str(name or "").strip() + if not text: + return "" + if self.strip_inner_space: + text = "".join(text.split()) + if self.canonical: + text = normalize_iface_name(text, self.canonical) + if self.alias_rules: + text = normalize_iface_name(text, self.alias_rules) + return text + + @property + def has_iface_rules(self) -> bool: + return bool(self.canonical or self.alias_rules) + + def describe(self) -> dict[str, Any]: + """JSON-able record of the rules in force (stored in the run snapshot).""" + return { + "vendor": self.vendor, + "canonical": list(self.canonical), + "alias_rules": list(self.alias_rules), + "value_maps": {k: dict(v) for k, v in self.value_maps.items()}, + "key_norms": dict(self.key_norms), + } + + +@dataclass(eq=False) +class Pipeline: + key_fields: list[str] + iface_fields: list[str] + compare_fields: list[str] + field_rules: list[dict[str, Any]] + row_filters: list[dict[str, Any]] + port_map: dict[str, str] + before: SideSpec + after: SideSpec + legacy: bool = False + ignore_port_changes: bool | None = None + unmapped_port_policy: str = "by_name" + duplicate_pairing: str = "ordered" + _iface_set: set[str] = field(default_factory=set, repr=False) + _transformed: set[str] = field(default_factory=set, repr=False) + _targets: set[str] = field(default_factory=set, repr=False) + + def __post_init__(self) -> None: + self._iface_set = set(self.iface_fields) + self._transformed = set(self._iface_set) + for spec in (self.before, self.after): + self._transformed |= set(spec.value_maps) | set(spec.key_norms) + self._targets = set(self.port_map.values()) + + # -- side helpers ---------------------------------------------------- + def spec(self, side: str) -> SideSpec: + return self.before if side == "before" else self.after + + # -- stage 1: normalize ---------------------------------------------- + def normalize(self, row: Mapping[str, Any], side: str) -> dict[str, Any]: + out = dict(row) + spec = self.spec(side) + for f in self.iface_fields: + val = scalar_text(out.get(f)).strip() + if val: + out[f] = spec.iface(val) + if self.legacy: + return out + for f, mode in spec.key_norms.items(): + if f in out: + out[f] = normalize_value(out[f], mode) + for f, vmap in spec.value_maps.items(): + if f not in out: + continue + raw = scalar_text(out[f]).strip() + hit = vmap.get(raw.lower()) + if hit is not None: + out[f] = hit + return out + + def normalize_name(self, name: Any, side: str) -> str: + """Interface-name normalization only (port-map keys / values, selectors).""" + return self.spec(side).iface(name) + + # -- stage 2: filter ------------------------------------------------- + def filters_for(self, side: str) -> list[dict[str, Any]]: + out = [] + for f in self.row_filters: + fs = str(f.get("side") or "").strip().lower() + if not fs or fs == "both" or fs == side: + out.append(f) + return out + + def keep(self, row: Mapping[str, Any], side: str) -> bool: + """Evaluate row filters. v2 expects the *normalized* row, legacy the raw one.""" + return all(row_matches_filter(row, f) for f in self.filters_for(side)) + + def accepts(self, raw: Mapping[str, Any], norm: Mapping[str, Any], side: str) -> bool: + """Row-filter verdict: legacy reads the raw row, v2 the normalized one.""" + return self.keep(raw if self.legacy else norm, side) + + def pushdown_filters(self, side: str) -> list[dict[str, Any]]: + """Filters that read only untransformed fields, so they may run in SQL on raw rows.""" + if self.legacy: + return list(self.row_filters) + return [ + f for f in self.row_filters + if not str(f.get("side") or "").strip() + and not (filter_fields(f) & self._transformed) + ] + + def prepare(self, rows: Sequence[Mapping[str, Any]], side: str) -> list[tuple[dict[str, Any], dict[str, Any]]]: + """``[(original row, normalized row)]`` after filtering, in input order.""" + out: list[tuple[dict[str, Any], dict[str, Any]]] = [] + has_filters = bool(self.filters_for(side)) + for raw in rows: + if not isinstance(raw, dict): + continue + norm = self.normalize(raw, side) + if has_filters and not self.accepts(raw, norm, side): + continue + out.append((raw, norm)) + return out + + # -- stage 3: port map ----------------------------------------------- + def apply_port_map(self, norm_row: Mapping[str, Any]) -> dict[str, Any]: + out = dict(norm_row) + if not self.port_map: + return out + for f in self.iface_fields: + val = scalar_text(out.get(f)).strip() + if val: + out[f] = resolve_mapped_iface(val, self.port_map) + return out + + def is_mapped(self, norm_row: Mapping[str, Any], side: str) -> bool: + """True when every interface value of the row is covered by the port map.""" + if not self.iface_fields: + return True + values = [scalar_text(norm_row.get(f)).strip() for f in self.iface_fields] + values = [v for v in values if v] + if not values: + return False + for v in values: + if side == "before": + if resolve_mapped_iface(v, self.port_map) == v and v not in self.port_map: + return False + else: + if not self._after_value_is_target(v): + return False + return True + + def _after_value_is_target(self, value: str) -> bool: + if value in self._targets: + return True + parts = value.split(".") + return any(".".join(parts[:i]) in self._targets for i in range(len(parts) - 1, 0, -1)) + + def transform(self, row: Mapping[str, Any], side: str) -> dict[str, Any]: + norm = self.normalize(row, side) + return self.apply_port_map(norm) if side == "before" else norm + + # -- stage 4: key ---------------------------------------------------- + def effective_match_keys(self, *, unique_without_iface: bool | None = None) -> list[str]: + """Match-key fields once the port policy is resolved. + + With a port map the full key is used (the map aligns the names). Without + one, ``ignore_port_changes`` may drop the interface fields; ``None`` means + "auto", resolved by the caller via ``unique_without_iface``. + """ + keys = list(self.key_fields) + if self.port_map or not self._iface_set: + return keys + candidate = [k for k in keys if k not in self._iface_set] + if not candidate: + return keys + policy = self.ignore_port_changes + if policy is True: + return candidate + if policy is False: + return keys + return candidate if unique_without_iface else keys + + def ignores_ports_by_policy(self) -> bool: + return bool(self.port_map) and self.unmapped_port_policy == "ignore_port" and bool(self._iface_set) + + def key_of( + self, norm_row: Mapping[str, Any], side: str, match_keys: Sequence[str], + ) -> tuple[dict[str, Any], tuple[str, ...]]: + """``(row after port map, match key)`` for a *normalized* row.""" + row = self.apply_port_map(norm_row) if side == "before" else dict(norm_row) + if self.ignores_ports_by_policy(): + if not self.is_mapped(norm_row, side): + reduced = [k for k in match_keys if k not in self._iface_set] or list(match_keys) + return row, ("U",) + tuple(scalar_text(row.get(k)).strip() for k in reduced) + return row, ("M",) + tuple(scalar_text(row.get(k)).strip() for k in match_keys) + return row, tuple(scalar_text(row.get(k)).strip() for k in match_keys) + + def match_key( + self, row: Mapping[str, Any], side: str, match_keys: Sequence[str] | None = None, + ) -> tuple[str, ...]: + """Match key of a *raw* row (normalize → port map → key).""" + keys = list(match_keys) if match_keys is not None else self.effective_match_keys() + return self.key_of(self.normalize(row, side), side, keys)[1] + + # -- duplicate pairing ----------------------------------------------- + def pair_sort_key(self, norm_row: Mapping[str, Any]) -> tuple[str, ...]: + rules = {str(r.get("field")): r for r in self.field_rules if isinstance(r, dict)} + out = [] + for f in self.compare_fields: + mode = str((rules.get(f) or {}).get("normalize") or "strip") + out.append(normalize_value(norm_row.get(f), mode)) + return tuple(out) + + # -- SQL eligibility ------------------------------------------------- + def sql_blockers(self, used_fields: set[str]) -> str: + """Reason the SQL engine must not run this sheet ('' when it may).""" + if self.legacy: + return "" + for spec in (self.before, self.after): + if set(spec.value_maps) & used_fields: + return "value_map" + if set(spec.key_norms) & set(self.key_fields): + return "key_normalize" + if used_fields & self._iface_set and (self.before.has_iface_rules or self.after.has_iface_rules): + return "iface_normalize" + if any(str(f.get("side") or "").strip() for f in self.row_filters): + return "row_filters_side" + if any(filter_fields(f) & self._transformed for f in self.row_filters): + return "row_filters_post_normalize" + if any( + str(r.get("compare") or "").strip().lower() in DIRECTIONAL_MODES + for r in self.field_rules if isinstance(r, dict) + ): + return "directional_compare" + return "" + + def describe(self) -> dict[str, Any]: + return { + "legacy": self.legacy, + "before": self.before.describe(), + "after": self.after.describe(), + "unmapped_port_policy": self.unmapped_port_policy, + "duplicate_pairing": self.duplicate_pairing, + } + + +def _field_rule_specs( + field_rules: Sequence[Mapping[str, Any]], key_fields: Sequence[str], side: str, +) -> tuple[dict[str, dict[str, str]], dict[str, str]]: + value_maps: dict[str, dict[str, str]] = {} + key_norms: dict[str, str] = {} + key_set = set(key_fields) + for rule in field_rules or []: + if not isinstance(rule, Mapping): + continue + name = str(rule.get("field") or "").strip() + if not name: + continue + rs = str(rule.get("side") or "").strip().lower() + if rs and rs != "both" and rs != side: + continue + apply_to_key = rule.get("apply_to_key") + apply_to_key = (name in key_set) if apply_to_key is None else bool(apply_to_key) + is_key = name in key_set + if is_key and not apply_to_key: + continue + mode = str(rule.get("normalize") or "").strip().lower() + vmap = rule.get("value_map") + if is_key and mode and mode not in ("none", "strip"): + key_norms[name] = mode + if isinstance(vmap, Mapping) and vmap: + pre = mode if (mode and mode not in ("none", "strip")) else "strip" + built: dict[str, str] = {} + for src, dst in vmap.items(): + built[normalize_value(src, pre).lower()] = scalar_text(dst).strip() + value_maps[name] = built + return value_maps, key_norms + + +def build_pipeline( + sheet: Mapping[str, Any], + *, + iface_rules: Sequence[Mapping[str, Any]] | None, + port_map: Mapping[str, str] | None, + before_vendor: str = "", + after_vendor: str = "", + legacy: bool = False, + normalize_map: bool = True, +) -> Pipeline: + """Build the pipeline for one sheet. + + ``port_map`` is the map as the user wrote it. Unless ``normalize_map`` is + false (legacy runs, whose maps were written post-normalization) its keys are + normalized with the *before* side rules and its values with the *after* side + rules. + """ + key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] + iface_fields = [str(k).strip() for k in (sheet.get("iface_fields") or []) if str(k).strip()] + field_rules = [dict(r) for r in (sheet.get("field_rules") or []) if isinstance(r, Mapping)] + from .compare_rules import effective_compare_fields + + compare_fields = effective_compare_fields(list(sheet.get("compare_fields") or []), field_rules) + canonical_on = sheet.get("builtin_vendor_canonical", True) is not False and not legacy + all_rules = normalize_iface_rules(list(iface_rules or [])) + + specs: dict[str, SideSpec] = {} + for side, vendor_raw in (("before", before_vendor), ("after", after_vendor)): + vendor = _vendor_key(vendor_raw) if vendor_raw else "" + if legacy: + alias = normalize_iface_rules([{"from": r["from"], "to": r["to"]} for r in all_rules]) + value_maps, key_norms = {}, {} + else: + alias = rules_for_side(all_rules, side, vendor) + value_maps, key_norms = _field_rule_specs(field_rules, key_fields, side) + specs[side] = SideSpec( + side=side, vendor=vendor, + canonical=vendor_canonical_rules(vendor) if canonical_on else [], + alias_rules=alias, value_maps=value_maps, key_norms=key_norms, + strip_inner_space=not legacy, + ) + + pmap = dict(port_map or {}) + if pmap and normalize_map and not legacy: + pmap, _ = normalize_port_map( + pmap, None, + key_normalizer=specs["before"].iface, value_normalizer=specs["after"].iface, + ) + ignore_ports = sheet.get("ignore_port_changes") + policy = str(sheet.get("unmapped_port_policy") or "by_name").strip().lower() + pairing = str(sheet.get("duplicate_pairing") or ("ordered" if legacy else "sorted")).strip().lower() + return Pipeline( + key_fields=key_fields, + iface_fields=iface_fields, + compare_fields=compare_fields, + field_rules=field_rules, + row_filters=[dict(f) for f in (sheet.get("row_filters") or []) if isinstance(f, Mapping)], + port_map=pmap, + before=specs["before"], + after=specs["after"], + legacy=legacy, + ignore_port_changes=None if ignore_ports is None else bool(ignore_ports), + unmapped_port_policy=policy if policy in UNMAPPED_POLICIES else "by_name", + duplicate_pairing="ordered" if legacy else (pairing if pairing in DUPLICATE_PAIRING else "sorted"), + ) + + +def legacy_pipeline( + *, + key_fields: Sequence[str], + iface_fields: Sequence[str], + compare_fields: Sequence[str], + field_rules: Sequence[Mapping[str, Any]] | None = None, + port_map: Mapping[str, str] | None = None, + iface_normalize_rules: Sequence[Mapping[str, Any]] | None = None, + ignore_port_changes: bool | None = None, + row_filters: Sequence[Mapping[str, Any]] | None = None, + duplicate_pairing: str = "ordered", +) -> Pipeline: + """Pipeline with the pre-v2 semantics, for snapshot-v1 runs and bare callers.""" + alias = normalize_iface_rules( + [{"from": r.get("from"), "to": r.get("to")} for r in (iface_normalize_rules or [])] + ) + return Pipeline( + key_fields=[str(k) for k in key_fields], + iface_fields=[str(f) for f in iface_fields if str(f).strip()], + compare_fields=[str(f) for f in compare_fields], + field_rules=[dict(r) for r in (field_rules or []) if isinstance(r, Mapping)], + row_filters=[dict(f) for f in (row_filters or []) if isinstance(f, Mapping)], + port_map=dict(port_map or {}), + before=SideSpec(side="before", alias_rules=list(alias), strip_inner_space=False), + after=SideSpec(side="after", alias_rules=list(alias), strip_inner_space=False), + legacy=True, + ignore_port_changes=ignore_port_changes, + duplicate_pairing=duplicate_pairing if duplicate_pairing in DUPLICATE_PAIRING else "ordered", + ) diff --git a/netx_api/biz_state/iface_normalize.py b/netx_api/biz_state/iface_normalize.py index 5fdb793..7018d63 100644 --- a/netx_api/biz_state/iface_normalize.py +++ b/netx_api/biz_state/iface_normalize.py @@ -39,7 +39,14 @@ def normalize_iface_rules(raw: Any) -> list[dict[str, str]]: fr = str(item.get("from") or item.get("src") or "").strip() to = str(item.get("to") or item.get("dst") or "").strip() if fr and to: - out.append({"from": fr, "to": to}) + rule = {"from": fr, "to": to} + side = str(item.get("side") or "").strip().lower() + if side in ("before", "after"): + rule["side"] = side + vendor = str(item.get("vendor") or "").strip().lower() + if vendor: + rule["vendor"] = vendor + out.append(rule) # Longest prefix first so XXVGE wins over XGE / GE. out.sort(key=lambda r: len(r["from"]), reverse=True) return out @@ -107,6 +114,47 @@ def apply_iface_normalize_rows( ] +def normalize_port_map( + port_map: Mapping[str, str] | None, + rules: Sequence[Mapping[str, str]] | None, + value_rules: Sequence[Mapping[str, str]] | None = None, + *, + key_normalizer: Any = None, + value_normalizer: Any = None, +) -> tuple[dict[str, str], list[dict[str, str]]]: + """Normalize port-map keys and values with the same rules as iface fields. + + ``value_rules`` (default: ``rules``) normalizes the after-side values, for + per-side rules. ``key_normalizer`` / ``value_normalizer`` are callables that + replace the rule-based normalization (used by the identity pipeline). + + Returns ``(normalized_map, conflicts)``. A conflict is two raw keys that + normalize to the same key but point at different (normalized) targets; the + first one wins in ``normalized_map``. + """ + out: dict[str, str] = {} + origin: dict[str, str] = {} + conflicts: list[dict[str, str]] = [] + for raw_k, raw_v in (port_map or {}).items(): + k = key_normalizer(raw_k) if key_normalizer else normalize_iface_name(raw_k, rules) + v = ( + value_normalizer(raw_v) if value_normalizer + else normalize_iface_name(raw_v, rules if value_rules is None else value_rules) + ) + if not k or not v: + continue + if k in out: + if out[k] != v: + conflicts.append({ + "before_if": k, "kept_from": origin[k], "kept_to": out[k], + "dropped_from": str(raw_k), "dropped_to": v, + }) + continue + out[k] = v + origin[k] = str(raw_k) + return out, conflicts + + def resolve_mapped_iface(name: str, port_map: Mapping[str, str] | None) -> str: """Exact map hit, else longest mapped parent + remaining suffix (QinQ-safe). @@ -142,3 +190,53 @@ def default_zte_iface_normalize_rules() -> list[dict[str, str]]: {"from": "SG", "to": "smartgroup"}, ] ) + + +# --------------------------------------------------------------------------- +# Built-in per-vendor canonical names (same-vendor synonyms only; cross-vendor +# numbering is the port map's job). Applied before template alias rules. +# --------------------------------------------------------------------------- + +VENDOR_CANONICAL: dict[str, list[dict[str, str]]] = { + "huawei": [ + {"from": "XGigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"}, + {"from": "100GE", "to": "100GE"}, {"from": "40GE", "to": "40GE"}, {"from": "25GE", "to": "25GE"}, + {"from": "10GE", "to": "10GE"}, {"from": "Eth-Trunk", "to": "Eth-Trunk"}, + ], + "cisco": [ + {"from": "GigabitEthernet", "to": "Gi"}, {"from": "TenGigabitEthernet", "to": "Te"}, + {"from": "TenGigE", "to": "Te"}, {"from": "HundredGigE", "to": "Hu"}, {"from": "FortyGigE", "to": "Fo"}, + {"from": "TwentyFiveGigE", "to": "Twe"}, {"from": "FastEthernet", "to": "Fa"}, + {"from": "Bundle-Ether", "to": "BE"}, {"from": "Loopback", "to": "Lo"}, + ], + "h3c": [ + {"from": "Ten-GigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"}, + {"from": "FortyGigE", "to": "FGE"}, {"from": "HundredGigE", "to": "HGE"}, + {"from": "Bridge-Aggregation", "to": "BAGG"}, + ], + "zte": [ + {"from": "xxvgei", "to": "xxvgei"}, {"from": "xgei", "to": "xgei"}, {"from": "cgei", "to": "cgei"}, + {"from": "gei", "to": "gei"}, {"from": "smartgroup", "to": "smartgroup"}, + ], +} + + +def vendor_canonical_rules(vendor: str) -> list[dict[str, str]]: + return normalize_iface_rules(VENDOR_CANONICAL.get(str(vendor or "").strip().lower(), [])) + + +def rules_for_side( + rules: Sequence[Mapping[str, str]] | None, side: str, vendor: str = "", +) -> list[dict[str, str]]: + """Template alias rules that apply to ``side`` (``before``/``after``) and ``vendor``.""" + vendor_n = str(vendor or "").strip().lower() + out: list[dict[str, str]] = [] + for r in rules or []: + rs = str(r.get("side") or "").strip().lower() + if rs and rs != side: + continue + rv = str(r.get("vendor") or "").strip().lower() + if rv and rv != vendor_n: + continue + out.append({"from": str(r.get("from") or ""), "to": str(r.get("to") or "")}) + return normalize_iface_rules(out) diff --git a/netx_api/biz_state/profiles.py b/netx_api/biz_state/profiles.py index 996d46b..70fcf88 100644 --- a/netx_api/biz_state/profiles.py +++ b/netx_api/biz_state/profiles.py @@ -85,6 +85,9 @@ class ParseProfile: enrich_joins: list[EnrichJoin] = field(default_factory=list) # light: default shared SSH lane; heavy: dedicated long-timeout connection collect_lane: str = "light" + # Which slice of the metric this command produces, e.g. {"afi": "ipv4"}. + # Empty = the command covers the whole metric (or its slicing is unknown). + slice: dict[str, str] = field(default_factory=dict) _LLDP_FIELDS: list[FieldDef] = [ @@ -1651,6 +1654,11 @@ _HEAVY_LANE_METRICS = frozenset( ) +def metric_lane(metric_id: str) -> str: + """``heavy`` for huge CLI dumps (own SSH connection), else ``light``.""" + return "heavy" if str(metric_id or "").strip() in _HEAVY_LANE_METRICS else "light" + + def _apply_collect_lanes(profiles: list[ParseProfile]) -> list[ParseProfile]: for p in profiles: if str(p.metric_id or "").strip() in _HEAVY_LANE_METRICS: @@ -1752,12 +1760,41 @@ def _validate_profiles(profiles: list[ParseProfile]) -> None: ) +# profile_id -> slice. Compare sheets filter on these fields (``afi``/``af``/``direction``), +# so a missing command is detected as "slice not collected" instead of "business lost". +_PROFILE_SLICES: dict[str, dict[str, str]] = { + **{f"zte.bgp_{afi}_summary": {"afi": afi} for afi in ("vpnv4", "ipv4", "vpnv6", "ipv6", "evpn", "vpls")}, + "zte.bgp_vpnv4_vrf_summary": {"afi": "vpnv4"}, + "zte.bgp_vpnv6_vrf_summary": {"afi": "vpnv6"}, + "zte.vrrp_ipv4": {"af": "ipv4"}, + "zte.vrrp_ipv6": {"af": "ipv6"}, + **{ + f"zte.bgp_{afi}_neighbor_{direction}": {"afi": afi, "direction": direction} + for afi in ("vpnv4", "vpnv6", "ipv4", "ipv6") + for direction in ("in", "out") + }, + **{ + f"zte.bgp_{afi}_vrf_neighbor_{direction}": {"afi": afi, "direction": direction} + for afi in ("vpnv4", "vpnv6") + for direction in ("in", "out") + }, +} + + +def _apply_profile_slices(profiles: list[ParseProfile]) -> list[ParseProfile]: + for p in profiles: + declared = _PROFILE_SLICES.get(p.profile_id) + if declared and not p.slice: + p.slice = dict(declared) + return profiles + + def _finalize_profiles(profiles: list[ParseProfile]) -> list[ParseProfile]: """Normalize discover→aux, validate, apply collect lanes.""" for p in profiles: _normalize_discover_aux(p) _validate_profiles(profiles) - return _apply_collect_lanes(profiles) + return _apply_profile_slices(_apply_collect_lanes(profiles)) def all_profiles() -> list[ParseProfile]: diff --git a/netx_api/biz_state/retention.py b/netx_api/biz_state/retention.py index d14c554..bd3ccf8 100644 --- a/netx_api/biz_state/retention.py +++ b/netx_api/biz_state/retention.py @@ -6,12 +6,17 @@ from collections import defaultdict from datetime import datetime, timedelta from typing import Any +from sqlalchemy import func, or_, select from sqlalchemy.orm import Session from ..models import ( + BizCompareDiff, BizCompareJob, BizCompareRun, + BizMigrationBatch, + BizMigrationDiff, BizMigrationProject, + BizMigrationRedTicket, BizMigrationRun, BizStateBatch, BizStateBatchCommand, @@ -26,48 +31,158 @@ def _utcnow() -> datetime: return datetime.utcnow() -def protected_batch_map(db: Session, *, task_id: str = "") -> dict[str, list[str]]: - """batch_id → reason codes. Empty task_id = all tasks.""" +def protected_batch_map( + db: Session, *, task_id: str = "", batch_ids: list[str] | None = None, +) -> dict[str, list[str]]: + """Only read reference columns, scoped to the requested task/batches when given.""" out: dict[str, list[str]] = defaultdict(list) + wanted = set(batch_ids) if batch_ids is not None else None + if wanted == set(): + return {} + scope = select(BizStateBatch.id) + q = db.query(BizStateBatch.id, BizStateBatch.is_baseline, BizStateBatch.status) + if task_id: + scope = scope.where(BizStateBatch.task_id == task_id) + q = q.filter(BizStateBatch.task_id == task_id) + if wanted is not None: + scope = scope.where(BizStateBatch.id.in_(wanted)) + q = q.filter(BizStateBatch.id.in_(wanted)) + batch_rows = q.all() if task_id or wanted is not None else q.filter(or_( + BizStateBatch.is_baseline.is_(True), BizStateBatch.status.in_(("queued", "running")), + )).all() + if task_id: + wanted = {row.id for row in batch_rows} def add(bid: str, reason: str) -> None: b = str(bid or "").strip() - if not b: + if not b or (wanted is not None and b not in wanted): return if reason not in out[b]: out[b].append(reason) - q = db.query(BizStateBatch) - if task_id: - q = q.filter(BizStateBatch.task_id == task_id) - for b in q.filter(BizStateBatch.is_baseline.is_(True)).all(): - add(b.id, "manual_baseline") + for b in batch_rows: + if b.is_baseline: + add(b.id, "manual_baseline") + if b.status in ("queued", "running"): + add(b.id, "active_collection") - for j in db.query(BizCompareJob).all(): - add(j.before_batch_id, "compare_job_before") - add(j.after_batch_id, "compare_job_after") + references = ( + (BizCompareJob.before_batch_id, BizCompareJob.after_batch_id, "compare_job_before", "compare_job_after"), + (BizCompareRun.before_batch_id, BizCompareRun.after_batch_id, "compare_run_before", "compare_run_after"), + (BizMigrationProject.old_baseline_batch_id, BizMigrationProject.new_baseline_batch_id, "migration_old_baseline", "migration_new_baseline"), + (BizMigrationRun.old_batch_id, BizMigrationRun.new_batch_id, "migration_run_old", "migration_run_new"), + ) + for before, after, before_reason, after_reason in references: + refs = db.query(before, after) + if task_id or batch_ids is not None: + refs = refs.filter(or_(before.in_(scope), after.in_(scope))) + for before_id, after_id in refs: + add(before_id, before_reason) + add(after_id, after_reason) - for r in db.query(BizCompareRun).all(): - add(r.before_batch_id, "compare_run_before") - add(r.after_batch_id, "compare_run_after") - - for p in db.query(BizMigrationProject).all(): - add(p.old_baseline_batch_id, "migration_old_baseline") - add(p.new_baseline_batch_id, "migration_new_baseline") - - for r in db.query(BizMigrationRun).all(): - add(r.old_batch_id, "migration_run_old") - add(r.new_batch_id, "migration_run_new") + # Every batch a *kept* acceptance/manual run (or the newest auto run of a wave) + # compared, not only the first binding. Older auto runs are bounded by + # ``purge_auto_runs`` and keep protecting just their primary pair above. + for run_batches in _kept_run_metric_batches(db): + for bid in run_batches: + add(bid, "migration_run_metric") return dict(out) +def _kept_run_metric_batches(db: Session, *, limit: int = 500) -> list[set[str]]: + newest_auto = ( + db.query(func.max(BizMigrationRun.created_at).label("at"), BizMigrationRun.batch_id.label("bid")) + .filter(BizMigrationRun.purpose == "auto").group_by(BizMigrationRun.batch_id).subquery() + ) + latest_auto_ids = [ + r[0] for r in db.query(BizMigrationRun.id).join( + newest_auto, + (BizMigrationRun.batch_id == newest_auto.c.bid) & (BizMigrationRun.created_at == newest_auto.c.at), + ).limit(limit).all() + ] + kept_ids = [ + r[0] for r in db.query(BizMigrationRun.id).filter(BizMigrationRun.purpose != "auto") + .order_by(BizMigrationRun.created_at.desc()).limit(limit).all() + ] + latest_auto_ids + out: list[set[str]] = [] + for start in range(0, len(kept_ids), 100): + for (summary,) in db.query(BizMigrationRun.summary_json).filter( + BizMigrationRun.id.in_(kept_ids[start:start + 100]) + ): + batches: set[str] = set() + for item in ((summary or {}).get("metric_batches") or {}).values(): + if isinstance(item, dict): + batches.update(str(item.get(k) or "") for k in ("old_batch_id", "new_batch_id")) + batches.discard("") + if batches: + out.append(batches) + return out + + +def purge_auto_runs(db: Session, *, migration_keep: int = 300, compare_keep: int = 50) -> dict[str, int]: + """Bound auto-generated run history. + + Cutover monitoring writes a run (+ one diff row per object) every minute and + auto-compare writes one per new sample. Acceptance / manual runs and runs a + red ticket points at are kept forever; only the newest ``keep`` auto runs of + each wave / job survive. + """ + dropped = {"migration_runs": 0, "compare_runs": 0} + ticket_runs = {r[0] for r in db.query(BizMigrationRedTicket.run_id).filter( + BizMigrationRedTicket.run_id != "").distinct()} + accepted = {r[0] for r in db.query(BizMigrationBatch.accept_run_id).filter( + BizMigrationBatch.accept_run_id != "")} + keep_forever = ticket_runs | accepted + + def surplus(model, group_col, extra_filter, keep: int) -> list[str]: + over = [g for g, n in db.query(group_col, func.count(model.id)).filter(*extra_filter) + .group_by(group_col).all() if n > keep] + ids: list[str] = [] + for g in over: + rows = db.query(model.id).filter(group_col == g, *extra_filter).order_by( + model.created_at.desc(), model.id.desc()).offset(keep).all() + ids.extend(r[0] for r in rows if r[0] not in keep_forever) + return ids + + mig_ids = surplus(BizMigrationRun, BizMigrationRun.batch_id, [BizMigrationRun.purpose == "auto"], + max(1, int(migration_keep))) + for start in range(0, len(mig_ids), 500): + chunk = mig_ids[start:start + 500] + db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id.in_(chunk)).delete(synchronize_session=False) + db.query(BizMigrationRun).filter(BizMigrationRun.id.in_(chunk)).delete(synchronize_session=False) + dropped["migration_runs"] = len(mig_ids) + + # Auto-mode jobs only. A run's origin lives in summary_json["purpose"]; unmarked runs + # are manual and stay. Only the surplus beyond ``compare_keep`` is inspected. + cmp_ids: list[str] = [] + keep_n = max(1, int(compare_keep)) + for (job_id,) in db.query(BizCompareJob.id).filter(BizCompareJob.mode == "auto").all(): + surplus_ids = [r[0] for r in db.query(BizCompareRun.id).filter( + BizCompareRun.job_id == job_id, BizCompareRun.status.notin_(("queued", "running")), + ).order_by(BizCompareRun.created_at.desc(), BizCompareRun.id.desc()).offset(keep_n).all()] + for start in range(0, len(surplus_ids), 200): + for rid, summary in db.query(BizCompareRun.id, BizCompareRun.summary_json).filter( + BizCompareRun.id.in_(surplus_ids[start:start + 200]) + ): + if (summary or {}).get("purpose") == "auto": + cmp_ids.append(rid) + for start in range(0, len(cmp_ids), 500): + chunk = cmp_ids[start:start + 500] + db.query(BizCompareDiff).filter(BizCompareDiff.run_id.in_(chunk)).delete(synchronize_session=False) + db.query(BizCompareRun).filter(BizCompareRun.id.in_(chunk)).delete(synchronize_session=False) + dropped["compare_runs"] = len(cmp_ids) + if mig_ids or cmp_ids: + db.commit() + return dropped + + def protected_batch_ids(db: Session, *, task_id: str = "") -> set[str]: return set(protected_batch_map(db, task_id=task_id).keys()) def batch_protect_info(db: Session, batch_id: str) -> dict[str, Any]: - reasons = protected_batch_map(db).get(batch_id, []) + reasons = protected_batch_map(db, batch_ids=[batch_id]).get(batch_id, []) b = db.get(BizStateBatch, batch_id) if b and bool(getattr(b, "is_baseline", False)) and "manual_baseline" not in reasons: reasons = ["manual_baseline", *reasons] diff --git a/netx_api/biz_state/schema_ensure.py b/netx_api/biz_state/schema_ensure.py index 6b1013f..1c11933 100644 --- a/netx_api/biz_state/schema_ensure.py +++ b/netx_api/biz_state/schema_ensure.py @@ -23,6 +23,7 @@ def apply_biz_state_schema(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_command_batch_id ON biz_state_batch_command (batch_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_lldp_neighbor_batch_id ON biz_state_lldp_neighbor (batch_id)", "ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS enabled_sheet_ids JSON DEFAULT '[]'", + "ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS store_unchanged VARCHAR(16) DEFAULT 'auto'", "CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)", @@ -31,6 +32,8 @@ def apply_biz_state_schema(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_id ON biz_compare_diff (run_id)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_kind ON biz_compare_diff (run_id, metric_id, kind)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_seq ON biz_compare_diff (run_id, metric_id, seq)", + "ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS before_row_id VARCHAR(64) DEFAULT ''", + "ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS after_row_id VARCHAR(64) DEFAULT ''", "CREATE INDEX IF NOT EXISTS ix_biz_migration_project_status ON biz_migration_project (status)", "CREATE INDEX IF NOT EXISTS ix_biz_migration_batch_project_id ON biz_migration_batch (project_id)", "CREATE INDEX IF NOT EXISTS ix_biz_migration_run_batch_id ON biz_migration_run (batch_id)", @@ -79,6 +82,15 @@ def apply_biz_state_schema(conn: Connection) -> None: "ALTER TABLE biz_migration_project ADD COLUMN IF NOT EXISTS new_hf_bindings_json JSON DEFAULT '[]'", "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS purpose VARCHAR(32) DEFAULT ''", "CREATE INDEX IF NOT EXISTS ix_biz_state_task_purpose ON biz_state_task (purpose)", + "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS collect_group_id VARCHAR(128) DEFAULT ''", + "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS collect_manual_only BOOLEAN DEFAULT FALSE", + "CREATE INDEX IF NOT EXISTS ix_biz_state_task_collect_group_id ON biz_state_task (collect_group_id)", + "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_group_id VARCHAR(128) DEFAULT ''", + "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_round_id VARCHAR(64) DEFAULT ''", + "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_priority INTEGER DEFAULT 0", + "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS queued_at TIMESTAMP", + "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_collect_group_id ON biz_state_batch (collect_group_id)", + "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_collect_round_id ON biz_state_batch (collect_round_id)", "ALTER TABLE biz_state_batch_command ADD COLUMN IF NOT EXISTS raw_line_count INTEGER DEFAULT 0", "ALTER TABLE biz_state_batch_command ADD COLUMN IF NOT EXISTS declared_total INTEGER DEFAULT 0", "ALTER TABLE biz_migration_red_ticket ADD COLUMN IF NOT EXISTS match_key_str VARCHAR(256) DEFAULT ''", diff --git a/netx_api/biz_state/service.py b/netx_api/biz_state/service.py index f3ad60a..95e191f 100644 --- a/netx_api/biz_state/service.py +++ b/netx_api/biz_state/service.py @@ -9,8 +9,8 @@ from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import String, cast, func, or_ -from sqlalchemy.orm import Session +from sqlalchemy import String, case, cast, func, or_ +from sqlalchemy.orm import Session, defer from ..lldp_shared import resolve_vendor_key from ..models import ( @@ -354,13 +354,15 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]: .order_by(BizStateTaskItem.sort_order.asc()) .all() ) + bindings_by_item: dict[str, list[Any]] = {} + if items: + for binding in db.query(BizStateTaskItemBinding).filter( + BizStateTaskItemBinding.item_id.in_([it.id for it in items]) + ).all(): + bindings_by_item.setdefault(binding.item_id, []).append(binding) item_out = [] for it in items: - binds = ( - db.query(BizStateTaskItemBinding) - .filter(BizStateTaskItemBinding.item_id == it.id) - .all() - ) + binds = bindings_by_item.get(it.id, []) item_out.append( { "id": it.id, @@ -373,6 +375,11 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]: "bindings": [{"placeholder": b.placeholder, "value": b.value} for b in binds], } ) + return {**_task_summary(task), "items": item_out} + + +def _task_summary(task: BizStateTask) -> dict[str, Any]: + """Task metadata without command items or bindings (safe for frequent polling).""" return { "id": task.id, "source": task.source, @@ -389,6 +396,8 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]: "daily_keep_enabled": bool(getattr(task, "daily_keep_enabled", False)), "daily_keep_count": int(getattr(task, "daily_keep_count", None) or 10), "collect_running": bool(task.collect_running), + "collect_group_id": task.collect_group_id or "", + "collect_manual_only": bool(task.collect_manual_only), "last_collect_started_at": task.last_collect_started_at.isoformat() + "Z" if task.last_collect_started_at else None, @@ -396,10 +405,16 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]: if task.last_collect_ended_at else None, "last_error": task.last_error, - "items": item_out, } +def get_task_progress(db: Session, task_id: str) -> dict[str, Any]: + task = db.get(BizStateTask, task_id) + if not task: + raise HTTPException(status_code=404, detail="task_not_found") + return _task_summary(task) + + def list_tasks(db: Session, *, purpose: str | None = None) -> list[dict[str, Any]]: from sqlalchemy import or_ @@ -431,6 +446,8 @@ def list_tasks(db: Session, *, purpose: str | None = None) -> list[dict[str, Any "status": t.status, "interval_sec": t.interval_sec, "collect_running": bool(t.collect_running), + "collect_group_id": t.collect_group_id or "", + "collect_manual_only": bool(t.collect_manual_only), "last_error": t.last_error, "last_collect_started_at": t.last_collect_started_at.isoformat() + "Z" if t.last_collect_started_at @@ -447,6 +464,8 @@ def delete_task(db: Session, task_id: str) -> None: task = db.get(BizStateTask, task_id) if not task: raise HTTPException(status_code=404, detail="task_not_found") + if task.collect_running: + raise HTTPException(status_code=409, detail="task_collecting") batches = db.query(BizStateBatch).filter(BizStateBatch.task_id == task_id).all() # Refuse if any batch is still referenced by compare/migration (manual baseline is OK to drop with task) pmap = protected_batch_map(db, task_id=task_id) @@ -500,7 +519,7 @@ def list_batches(db: Session, task_id: str, *, limit: int = 50) -> list[dict[str .limit(max(1, min(500, int(limit)))) .all() ) - pmap = protected_batch_map(db, task_id=task_id) + pmap = protected_batch_map(db, batch_ids=[b.id for b in rows]) out: list[dict[str, Any]] = [] for b in rows: reasons = list(pmap.get(b.id, [])) @@ -622,12 +641,29 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: b = db.get(BizStateBatch, batch_id) if not b: raise HTTPException(status_code=404, detail="batch_not_found") - cmds = ( - db.query(BizStateBatchCommand) - .filter(BizStateBatchCommand.batch_id == batch_id) - .order_by(BizStateBatchCommand.created_at.asc()) - .all() + raw = func.coalesce(BizStateBatchCommand.raw_text, "") + raw_len = func.length(raw) + # Compute legacy counts in SQL; workbook summaries never transfer raw CLI text. + line_count = case( + (BizStateBatchCommand.raw_line_count > 0, BizStateBatchCommand.raw_line_count), + (raw_len == 0, 0), + else_=raw_len - func.length(func.replace(raw, "\n", "")) + + case((func.substr(raw, raw_len, 1) == "\n", 0), else_=1), ) + whitespace = " \t\r\n\v\f" + trimmed = (func.btrim(raw, whitespace) if db.get_bind().dialect.name == "postgresql" + else func.trim(raw, whitespace)) + cmd_raw_stats: dict[str, tuple[int, bool]] = {} + cmds = [] + for c, lines, has_raw in ( + db.query(BizStateBatchCommand, line_count, func.length(trimmed) > 0) + .options(defer(BizStateBatchCommand.raw_text)) + .filter(BizStateBatchCommand.batch_id == batch_id) + .order_by(BizStateBatchCommand.created_at.asc(), BizStateBatchCommand.id.asc()) + .all() + ): + cmds.append(c) + cmd_raw_stats[c.id] = (int(lines or 0), bool(has_raw)) protect = batch_protect_info(db, batch_id) # Per-metric row counts (generic table) @@ -668,11 +704,14 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: # Prefer primary collect rows over aux / aux_cached for the same CLI sheet_cmds[id_].append(cmd_info) - primary_cmds_by_cli: dict[str, dict[str, Any]] = {} + # Aux failures must remain visible even if their primary command was stored later. + primary_cmds_by_cli = { + normalize_command(str(c.raw_command or "")) for c in cmds + if not str(c.parse_status or "").strip().lower().startswith("aux") + } for c in cmds: status = str(c.parse_status or "").strip().lower() is_aux = status.startswith("aux") - raw = c.raw_text or "" info = { "id": c.id, "profile_id": c.profile_id, @@ -682,15 +721,13 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: "params": c.params_json or {}, "parse_status": c.parse_status, "row_count": c.row_count, - "raw_line_count": _cmd_raw_line_count(c), + "raw_line_count": cmd_raw_stats[c.id][0], "declared_total": _cmd_declared_total(c), "message": c.message, - "has_raw": bool(str(raw).strip()), + "has_raw": cmd_raw_stats[c.id][1], "is_aux": is_aux, } cmd_n = normalize_command(str(c.raw_command or "")) - if not is_aux and cmd_n: - primary_cmds_by_cli.setdefault(cmd_n, info) # Commands sheet: hide successful aux when the same CLI already has a primary row; # keep failed/skipped aux visible so partial reasons are not hidden. if is_aux and cmd_n and cmd_n in primary_cmds_by_cli: @@ -746,10 +783,10 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: "params": c.params_json or {}, "parse_status": c.parse_status, "row_count": c.row_count, - "raw_line_count": _cmd_raw_line_count(c), + "raw_line_count": cmd_raw_stats[c.id][0], "declared_total": _cmd_declared_total(c), "message": c.message, - "has_raw": bool(str(c.raw_text or "").strip()), + "has_raw": cmd_raw_stats[c.id][1], "is_aux": True, } ) @@ -844,6 +881,7 @@ def list_batch_metric_rows( size_n = max(1, min(200, int(page_size or 50))) kw_n = str(kw or "").strip() col_n = str(column or "").strip() + like = "%" + kw_n.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" fields = metric_field_map().get(mid) or [] columns = [ @@ -859,30 +897,30 @@ def list_batch_metric_rows( if mid == "lldp_neighbor": q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id) if kw_n: - like = f"%{kw_n}%" if col_n == "local_if": - q = q.filter(BizStateLldpNeighbor.local_if.ilike(like)) + q = q.filter(BizStateLldpNeighbor.local_if.ilike(like, escape="\\")) elif col_n == "remote_sys": - q = q.filter(BizStateLldpNeighbor.remote_sys.ilike(like)) + q = q.filter(BizStateLldpNeighbor.remote_sys.ilike(like, escape="\\")) elif col_n == "remote_if": - q = q.filter(BizStateLldpNeighbor.remote_if.ilike(like)) + q = q.filter(BizStateLldpNeighbor.remote_if.ilike(like, escape="\\")) elif col_n == "remote_ip": - q = q.filter(BizStateLldpNeighbor.remote_ip.ilike(like)) + q = q.filter(BizStateLldpNeighbor.remote_ip.ilike(like, escape="\\")) elif col_n == "protocol": - q = q.filter(BizStateLldpNeighbor.protocol.ilike(like)) + q = q.filter(BizStateLldpNeighbor.protocol.ilike(like, escape="\\")) else: q = q.filter( or_( - BizStateLldpNeighbor.local_if.ilike(like), - BizStateLldpNeighbor.remote_sys.ilike(like), - BizStateLldpNeighbor.remote_if.ilike(like), - BizStateLldpNeighbor.remote_ip.ilike(like), - BizStateLldpNeighbor.protocol.ilike(like), + BizStateLldpNeighbor.local_if.ilike(like, escape="\\"), + BizStateLldpNeighbor.remote_sys.ilike(like, escape="\\"), + BizStateLldpNeighbor.remote_if.ilike(like, escape="\\"), + BizStateLldpNeighbor.remote_ip.ilike(like, escape="\\"), + BizStateLldpNeighbor.protocol.ilike(like, escape="\\"), ) ) total = int(q.count() or 0) rows_db = ( - q.order_by(BizStateLldpNeighbor.local_if.asc()) + q.order_by(BizStateLldpNeighbor.local_if.asc(), BizStateLldpNeighbor.remote_sys.asc(), + BizStateLldpNeighbor.remote_if.asc(), BizStateLldpNeighbor.id.asc()) .offset((page_n - 1) * size_n) .limit(size_n) .all() @@ -911,12 +949,11 @@ def list_batch_metric_rows( BizStateMetricRow.metric_id == mid, ) if kw_n: - like = f"%{kw_n}%" if col_n: # JSON path as text — works on Postgres JSONB and SQLite JSON - q = q.filter(cast(BizStateMetricRow.data_json[col_n], String).ilike(like)) + q = q.filter(cast(BizStateMetricRow.data_json[col_n].as_string(), String).ilike(like, escape="\\")) else: - q = q.filter(cast(BizStateMetricRow.data_json, String).ilike(like)) + q = q.filter(cast(BizStateMetricRow.data_json, String).ilike(like, escape="\\")) total = int(q.count() or 0) rows_db = ( q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) @@ -1019,56 +1056,64 @@ def export_batch_zip(db: Session, batch_id: str) -> bytes: db.query(BizStateLldpNeighbor) .filter(BizStateLldpNeighbor.batch_id == batch_id) .order_by(BizStateLldpNeighbor.local_if.asc()) - .all() + .yield_per(1000) ) - csv_lines = ["local_if,remote_sys,remote_if,remote_ip,protocol"] - for n in neighbors: - csv_lines.append( - ",".join( - [ - _csv(n.local_if), - _csv(n.remote_sys), - _csv(n.remote_if), - _csv(n.remote_ip), - _csv(n.protocol), - ] - ) - ) - zf.writestr("tables/lldp_neighbor.csv", "\n".join(csv_lines) + "\n") + with zf.open("tables/lldp_neighbor.csv", "w") as dest: + dest.write(b"local_if,remote_sys,remote_if,remote_ip,protocol\n") + for n in neighbors: + line = ",".join(_csv(v) for v in ( + n.local_if, n.remote_sys, n.remote_if, n.remote_ip, n.protocol, + )) + "\n" + dest.write(line.encode("utf-8")) # Generic metrics CSV (stream by metric_id) for sheet in detail.get("sheets") or []: mid = str(sheet.get("metric_id") or "").strip() if not mid or mid == "lldp_neighbor": continue - rows = ( - db.query(BizStateMetricRow) - .filter( - BizStateMetricRow.batch_id == batch_id, - BizStateMetricRow.metric_id == mid, - ) - .order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) - .all() - ) - if not rows: - continue - recs = [dict(r.data_json or {}) for r in rows] - cols: list[str] = [] - for rec in recs: + # First scan discovers the same complete, ordered header as the legacy export. + # Second scan writes rows without materializing the table or CSV. + col_keys: dict[str, None] = {} + has_rows = False + for rec in _iter_metric_export_rows(db, batch_id, mid): + has_rows = True for k in rec.keys(): - if k not in cols: - cols.append(str(k)) - out_lines = [",".join(_csv(c) for c in cols)] - for rec in recs: - out_lines.append(",".join(_csv(str(rec.get(c, "") or "")) for c in cols)) + col_keys.setdefault(str(k), None) + if not has_rows: + continue + cols = list(col_keys) safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "metric" - zf.writestr(f"tables/{safe}.csv", "\n".join(out_lines) + "\n") + with zf.open(f"tables/{safe}.csv", "w") as dest: + dest.write((",".join(_csv(c) for c in cols) + "\n").encode("utf-8")) + for rec in _iter_metric_export_rows(db, batch_id, mid): + line = ",".join(_csv(rec.get(c)) for c in cols) + "\n" + dest.write(line.encode("utf-8")) return buf.getvalue() -def _csv(v: str) -> str: - s = str(v or "") - if any(ch in s for ch in ",\"\n"): +def _iter_metric_export_rows(db: Session, batch_id: str, metric_id: str, chunk_size: int = 1000): + cursor: tuple[int, str] | None = None + while True: + query = db.query(BizStateMetricRow.seq, BizStateMetricRow.id, BizStateMetricRow.data_json).filter( + BizStateMetricRow.batch_id == batch_id, BizStateMetricRow.metric_id == metric_id, + ) + if cursor is not None: + seq, row_id = cursor + query = query.filter(or_( + BizStateMetricRow.seq > seq, + (BizStateMetricRow.seq == seq) & (BizStateMetricRow.id > row_id), + )) + chunk = query.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()).limit(chunk_size).all() + if not chunk: + return + for row in chunk: + yield row.data_json or {} + cursor = (chunk[-1].seq, chunk[-1].id) + + +def _csv(v: Any) -> str: + s = "" if v is None else str(v) + if any(ch in s for ch in ",\"\r\n"): return '"' + s.replace('"', '""') + '"' return s diff --git a/netx_api/biz_state_router.py b/netx_api/biz_state_router.py index 4a4f3ed..0714e1c 100644 --- a/netx_api/biz_state_router.py +++ b/netx_api/biz_state_router.py @@ -173,6 +173,11 @@ def api_get_task(task_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return svc.get_task(db, task_id) +@router.get("/tasks/{task_id}/progress") +def api_task_progress(task_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: + return svc.get_task_progress(db, task_id) + + @router.get("/tasks/{task_id}/commands") def api_plan_task_commands( task_id: str, @@ -544,6 +549,12 @@ class FieldRuleIn(BaseModel): normalize: str = "" ignore: bool = False tolerance: float | None = None + # Cross-vendor identity rules (see identity_pipeline). + value_map: dict[str, str] | None = None + side: str = "" + apply_to_key: bool | None = None + # Order for compare=not_worse: earlier entries are better. + rank: list[str] | None = None class RowFilterIn(BaseModel): @@ -569,6 +580,10 @@ class TemplateMetricIn(BaseModel): display_fields: list[str] = Field(default_factory=list) row_filters: list[dict[str, Any]] = Field(default_factory=list) field_rules: list[FieldRuleIn] = Field(default_factory=list) + ignore_port_changes: bool | None = None + unmapped_port_policy: str = "" + duplicate_pairing: str = "" + builtin_vendor_canonical: bool | None = None class TemplateIn(BaseModel): @@ -587,6 +602,7 @@ class TemplateIn(BaseModel): display_fields: list[str] = Field(default_factory=list) row_filters: list[dict[str, Any]] = Field(default_factory=list) field_rules: list[FieldRuleIn] = Field(default_factory=list) + ignore_port_changes: bool | None = None class TemplatePatchIn(BaseModel): @@ -602,6 +618,7 @@ class TemplatePatchIn(BaseModel): display_fields: list[str] | None = None row_filters: list[dict[str, Any]] | None = None field_rules: list[FieldRuleIn] | None = None + ignore_port_changes: bool | None = None class MappingRowIn(BaseModel): @@ -615,6 +632,16 @@ class MappingIn(BaseModel): rows: list[MappingRowIn] = Field(default_factory=list) +class PreflightIn(BaseModel): + template_id: str = "" + enabled_sheet_ids: list[str] = Field(default_factory=list) + mapping_id: str = "" + # Unsaved editor rows win over ``mapping_id`` when given. + mapping_rows: list[MappingRowIn] | None = None + before_batch_id: str + after_batch_id: str + + class ValidateMappingIn(BaseModel): mapping_id: str before_batch_id: str @@ -633,6 +660,8 @@ class CompareJobIn(BaseModel): mode: str = "manual" # Empty = all template sheets; non-empty = only these sheet_id values enabled_sheet_ids: list[str] = Field(default_factory=list) + # auto|always|never|sample|keys — how to persist matching (success) rows + store_unchanged: str = "auto" note: str = "" @@ -701,6 +730,11 @@ def api_validate_mapping(body: ValidateMappingIn, db: Session = Depends(get_db)) ) +@router.post("/compare/preflight") +def api_compare_preflight(body: PreflightIn, db: Session = Depends(get_db)) -> dict[str, Any]: + return cmp_svc.preflight_compare(db, body.model_dump()) + + @router.get("/compare/jobs") def api_list_jobs(db: Session = Depends(get_db)) -> dict[str, Any]: return {"items": cmp_svc.list_jobs(db)} @@ -723,8 +757,15 @@ def api_delete_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any] @router.post("/compare/jobs/{job_id}/run") -def api_run_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: - return cmp_svc.run_compare(db, job_id) +def api_run_job( + job_id: str, + sync: bool = Query(False, description="If true, block until compare finishes (tests/debug)"), + db: Session = Depends(get_db), +) -> dict[str, Any]: + """Start a compare run. Default is async (returns status=running immediately).""" + if sync: + return cmp_svc.run_compare(db, job_id) + return cmp_svc.enqueue_compare(db, job_id) @router.get("/compare/jobs/{job_id}/runs") @@ -737,6 +778,12 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return cmp_svc.get_run(db, run_id) +@router.post("/compare/runs/{run_id}/cancel") +def api_cancel_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: + """Cancel a stuck/running compare so a new run can start.""" + return cmp_svc.cancel_compare_run(db, run_id) + + @router.delete("/compare/runs/{run_id}") def api_delete_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return cmp_svc.delete_run(db, run_id) @@ -748,6 +795,7 @@ def api_list_run_diffs( metric_id: str = "", kind: str = "diff", kw: str = "", + qf: str = "", page: int = 1, page_size: int = 100, db: Session = Depends(get_db), @@ -758,6 +806,7 @@ def api_list_run_diffs( metric_id=metric_id, kind=kind, kw=kw, + qf=qf, page=page, page_size=page_size, ) diff --git a/netx_api/biz_state_scheduler.py b/netx_api/biz_state_scheduler.py index d44250f..77b6338 100644 --- a/netx_api/biz_state_scheduler.py +++ b/netx_api/biz_state_scheduler.py @@ -47,6 +47,15 @@ def _maybe_purge_retention() -> None: from .biz_state.retention import purge_all_tasks info = purge_all_tasks(db) + from .biz_state.retention import purge_auto_runs + + runs = purge_auto_runs( + db, + migration_keep=int(getattr(settings, "biz_migration_auto_run_keep", 300) or 300), + compare_keep=int(getattr(settings, "biz_compare_auto_run_keep", 50) or 50), + ) + if any(runs.values()): + _log.info("biz_state periodic run purge %s", runs) if info.get("dropped"): _log.info( "biz_state periodic retention dropped=%s tasks=%s", @@ -110,12 +119,21 @@ def _sync_cutover_hf_windows() -> None: projects = db.query(BizMigrationProject).all() for proj in projects: - want = _hf_window_status(proj) + tids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new") + if tids and any((t := db.get(BizStateTask, tid)) and not t.collect_group_id for tid in tids): + try: + from .biz_migration.service import ensure_highfreq + + ensure_highfreq(db, proj.id, collect_now=False) + except Exception: + _log.exception("upgrade cutover collection pairs failed project=%s", proj.id) + window_status = _hf_window_status(proj) tids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new") for tid in tids: task = db.get(BizStateTask, tid) if not task: continue + want = "paused" if task.collect_manual_only else window_status purpose = str(getattr(task, "purpose", None) or "").strip() if purpose and purpose != PURPOSE_CUTOVER_HF: continue @@ -149,12 +167,26 @@ def _enqueue_due_tasks() -> int: .filter( BizStateTask.status == "running", BizStateTask.collect_running.is_(False), + BizStateTask.collect_manual_only.is_(False), ) .all() ) due_ids: list[str] = [] + seen_groups: set[str] = set() now = _utcnow() for task in tasks: + if task.collect_group_id: + if task.collect_group_id in seen_groups: + continue + seen_groups.add(task.collect_group_id) + peers = db.query(BizStateTask).filter(BizStateTask.collect_group_id == task.collect_group_id).all() + if len(peers) != 2 or any(p.collect_running or p.status != "running" or p.collect_manual_only for p in peers): + continue + anchors = [p.last_collect_started_at or p.last_collect_ended_at for p in peers] + anchor = max((a for a in anchors if a), default=None) + if anchor is None or (now - anchor).total_seconds() >= max(p.interval_sec or 60 for p in peers): + due_ids.append(task.id) + continue interval = max(60, int(task.interval_sec or 300)) # Prefer start-based interval to avoid drift when collect duration # approaches the interval (end-based: 40s collect + 60s → 100s cycle). diff --git a/netx_api/cli_router.py b/netx_api/cli_router.py index 065063c..d871850 100644 --- a/netx_api/cli_router.py +++ b/netx_api/cli_router.py @@ -1,6 +1,6 @@ from __future__ import annotations -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session from .cli_schemas import ( @@ -15,12 +15,14 @@ from .cli_service import ( delete_cli_profile, get_cli_profile, get_ume_cli_override, + list_cli_target_filter_options, list_cli_profiles, list_cli_targets, set_default_cli_profile, update_cli_profile, upsert_ume_cli_override, ) +from .table_filters import TableFilterParams, table_filter_params from .db import get_db from .ne_connect import schedule_ume_connect_tests @@ -66,11 +68,57 @@ def api_delete_cli_profile(profile_id: str, db: Session = Depends(get_db)): def api_list_cli_targets( source: str = Query(default="all"), keyword: str | None = Query(default=None), + source_filter: list[str] | None = Query(default=None), + source_filter_mode: str | None = Query(default=None), + name_filter: list[str] | None = Query(default=None), + name_filter_mode: str | None = Query(default=None), + ip_filter: list[str] | None = Query(default=None), + ip_filter_mode: str | None = Query(default=None), + vendor_filter: list[str] | None = Query(default=None), + vendor_filter_mode: str | None = Query(default=None), + device_type_filter: list[str] | None = Query(default=None), + device_type_filter_mode: str | None = Query(default=None), + connect_status_filter: list[str] | None = Query(default=None), + connect_status_filter_mode: str | None = Query(default=None), page: int = Query(default=1, ge=1), page_size: int = Query(default=50, ge=1, le=500), + table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ): - return list_cli_targets(db, source=source, keyword=keyword, page=page, page_size=page_size) + if table_filters.field: + return list_cli_target_filter_options(db, field=table_filters.field, source=source, keyword=keyword) + filters = table_filters.filters + if set(filters) - {"source", "name", "ip", "vendor", "deviceType", "connectStatus"}: + raise HTTPException(status_code=400, detail="invalid_filter_field") + return list_cli_targets( + db, + source=source, + keyword=keyword, + source_filter=filters.get("source", {}).get("values", source_filter), + source_filter_mode=filters.get("source", {}).get("mode", source_filter_mode), + name_filter=filters.get("name", {}).get("values", name_filter), + name_filter_mode=filters.get("name", {}).get("mode", name_filter_mode), + ip_filter=filters.get("ip", {}).get("values", ip_filter), + ip_filter_mode=filters.get("ip", {}).get("mode", ip_filter_mode), + vendor_filter=filters.get("vendor", {}).get("values", vendor_filter), + vendor_filter_mode=filters.get("vendor", {}).get("mode", vendor_filter_mode), + device_type_filter=filters.get("deviceType", {}).get("values", device_type_filter), + device_type_filter_mode=filters.get("deviceType", {}).get("mode", device_type_filter_mode), + connect_status_filter=filters.get("connectStatus", {}).get("values", connect_status_filter), + connect_status_filter_mode=filters.get("connectStatus", {}).get("mode", connect_status_filter_mode), + page=page, + page_size=page_size, + ) + + +@router.get("/targets/filter-options") +def api_list_cli_target_filter_options( + field: str = Query(), + source: str = Query(default="inventory"), + keyword: str | None = Query(default=None), + db: Session = Depends(get_db), +): + return list_cli_target_filter_options(db, field=field, source=source, keyword=keyword) @router.get("/ume-overrides/{ume_ne_id}") diff --git a/netx_api/cli_service.py b/netx_api/cli_service.py index 82707c6..439db50 100644 --- a/netx_api/cli_service.py +++ b/netx_api/cli_service.py @@ -4,6 +4,7 @@ from datetime import datetime from typing import Any from fastapi import HTTPException +from sqlalchemy import func from sqlalchemy.orm import Session from .cli_schemas import ( @@ -279,19 +280,66 @@ def list_cli_targets( *, source: str = "all", keyword: str | None = None, + source_filter: list[str] | None = None, + source_filter_mode: str | None = None, + name_filter: list[str] | None = None, + name_filter_mode: str | None = None, + ip_filter: list[str] | None = None, + ip_filter_mode: str | None = None, + vendor_filter: list[str] | None = None, + vendor_filter_mode: str | None = None, + device_type_filter: list[str] | None = None, + device_type_filter_mode: str | None = None, + connect_status_filter: list[str] | None = None, + connect_status_filter_mode: str | None = None, page: int = 1, page_size: int = 50, ) -> dict[str, Any]: from .device_types import WEBCRT_NE_SOURCE src = str(source or "all").strip().lower() - if src not in ("managed", "ume", "webcrt", "all"): + if src not in ("managed", "ume", "webcrt", "inventory", "all"): raise HTTPException(status_code=400, detail="invalid_source") ready = cli_profile_ready(db) page = max(1, int(page or 1)) page_size = max(1, min(500, int(page_size or 50))) offset = (page - 1) * page_size kw = str(keyword or "").strip() + column_filters: dict[str, list[str] | None] = { + "source": source_filter, + "name": name_filter, + "ip": ip_filter, + "vendor": vendor_filter, + "device_type": device_type_filter, + "connect_status": connect_status_filter, + } + filter_modes = { + "source": str(source_filter_mode or ""), + "name": str(name_filter_mode or ""), + "ip": str(ip_filter_mode or ""), + "vendor": str(vendor_filter_mode or ""), + "device_type": str(device_type_filter_mode or ""), + "connect_status": str(connect_status_filter_mode or ""), + } + + def _source_allowed(row_source: str) -> bool: + mode = filter_modes["source"] + values = column_filters["source"] or [] + if mode == "include": + return row_source in values + if mode == "exclude": + return row_source not in values + return True + + def _apply_value_filter(stmt: Any, field: str, expression: Any) -> Any: + mode = filter_modes[field] + if mode not in ("include", "exclude"): + return stmt + values = column_filters[field] or [] + normalized = func.coalesce(expression, "") + if mode == "include": + return stmt.filter(normalized.in_(values)) if values else stmt.filter(False) + return stmt.filter(~normalized.in_(values)) if values else stmt default_profile = get_default_profile(db) default_hop = bool(getattr(default_profile, "hop_enabled", False)) if default_profile else False @@ -351,7 +399,8 @@ def list_cli_targets( return stmt like = f"%{kw}%" return stmt.filter( - ManagedNE.name.ilike(like) + ManagedNE.id.ilike(like) + | ManagedNE.name.ilike(like) | ManagedNE.ip_address.ilike(like) | ManagedNE.username.ilike(like) | ManagedNE.tags.ilike(like) @@ -362,16 +411,42 @@ def list_cli_targets( def _inventory_managed_query(): # Asset inventory for WebCRT: exclude Quick-Connect (source=webcrt) rows. stmt = db.query(ManagedNE).filter(ManagedNE.source != WEBCRT_NE_SOURCE) - return _apply_kw(stmt).order_by(ManagedNE.updated_at.desc()) + stmt = _apply_kw(stmt) + if not _source_allowed("managed"): + stmt = stmt.filter(False) + stmt = _apply_value_filter( + stmt, + "name", + func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address), + ) + stmt = _apply_value_filter(stmt, "ip", ManagedNE.ip_address) + stmt = _apply_value_filter(stmt, "vendor", ManagedNE.vendor) + stmt = _apply_value_filter(stmt, "device_type", ManagedNE.device_type) + stmt = _apply_value_filter(stmt, "connect_status", ManagedNE.connect_status) + return stmt.order_by(ManagedNE.updated_at.desc()) def _webcrt_query(): stmt = db.query(ManagedNE).filter(ManagedNE.source == WEBCRT_NE_SOURCE) - return _apply_kw(stmt).order_by(ManagedNE.updated_at.desc()) + stmt = _apply_kw(stmt) + if not _source_allowed("webcrt"): + stmt = stmt.filter(False) + stmt = _apply_value_filter( + stmt, + "name", + func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address), + ) + stmt = _apply_value_filter(stmt, "ip", ManagedNE.ip_address) + stmt = _apply_value_filter(stmt, "vendor", ManagedNE.vendor) + stmt = _apply_value_filter(stmt, "device_type", ManagedNE.device_type) + stmt = _apply_value_filter(stmt, "connect_status", ManagedNE.connect_status) + return stmt.order_by(ManagedNE.updated_at.desc()) def _ume_query(): stmt = db.query(UmeInventoryNE, UmeCliOverride).outerjoin( UmeCliOverride, UmeInventoryNE.ne_id == UmeCliOverride.ume_ne_id ) + if not _source_allowed("ume"): + stmt = stmt.filter(False) if kw: like = f"%{kw}%" stmt = stmt.filter( @@ -381,8 +456,46 @@ def list_cli_targets( | UmeInventoryNE.ip_address.ilike(like) | UmeInventoryNE.host_name.ilike(like) ) + ume_name = func.coalesce( + func.nullif(UmeInventoryNE.user_label, ""), + func.nullif(UmeInventoryNE.ne_name, ""), + func.nullif(UmeInventoryNE.host_name, ""), + func.nullif(UmeInventoryNE.ip_address, ""), + UmeInventoryNE.ne_id, + ) + stmt = _apply_value_filter(stmt, "name", ume_name) + stmt = _apply_value_filter(stmt, "ip", UmeInventoryNE.ip_address) + stmt = _apply_value_filter(stmt, "vendor", UmeInventoryNE.vendor) + stmt = _apply_value_filter(stmt, "device_type", UmeInventoryNE.ne_type) + stmt = _apply_value_filter( + stmt, + "connect_status", + func.coalesce(UmeCliOverride.connect_status, "unknown"), + ) return stmt.order_by(UmeInventoryNE.ne_id.asc()) + if src == "inventory": + # Network Management inventory includes managed assets and UME inventory, + # but not transient WebCRT quick-connect sessions. + mq = _inventory_managed_query() + uq = _ume_query() + m_total = int(mq.count()) + u_total = int(uq.count()) + total = m_total + u_total + items: list[dict[str, Any]] = [] + + if offset < m_total: + take = min(page_size, m_total - offset) + items.extend(_managed_item(row, list_source="managed") for row in mq.offset(offset).limit(take).all()) + remaining = page_size - take + if remaining > 0: + items.extend(_ume_item(inv, ov) for inv, ov in uq.offset(0).limit(remaining).all()) + else: + ume_offset = max(0, offset - m_total) + items.extend(_ume_item(inv, ov) for inv, ov in uq.offset(ume_offset).limit(page_size).all()) + + return {"items": items, "total": total, "page": page, "page_size": page_size} + if src == "managed": mq = _inventory_managed_query() total = int(mq.count()) @@ -443,6 +556,118 @@ def list_cli_targets( return {"items": items, "total": total, "page": page, "page_size": page_size} +def list_cli_target_filter_options( + db: Session, + *, + field: str, + source: str = "inventory", + keyword: str | None = None, +) -> dict[str, Any]: + from .device_types import WEBCRT_NE_SOURCE + + field_map = { + "source": "source", + "name": "name", + "ip": "ip", + "vendor": "vendor", + "deviceType": "device_type", + "connectStatus": "connect_status", + } + if field not in field_map: + raise HTTPException(status_code=400, detail="invalid_filter_field") + src = str(source or "inventory").strip().lower() + if src not in ("managed", "ume", "webcrt", "inventory", "all"): + raise HTTPException(status_code=400, detail="invalid_source") + + include_managed = src in ("managed", "inventory", "all") + include_webcrt = src in ("webcrt", "all") + include_ume = src in ("ume", "inventory", "all") + kw = str(keyword or "").strip() + + managed = db.query(ManagedNE) + if src in ("managed", "inventory"): + managed = managed.filter(ManagedNE.source != WEBCRT_NE_SOURCE) + elif src == "webcrt": + managed = managed.filter(ManagedNE.source == WEBCRT_NE_SOURCE) + elif src == "all": + managed = managed.filter(ManagedNE.source != WEBCRT_NE_SOURCE) + ume = db.query(UmeInventoryNE) + ume_status = db.query(UmeInventoryNE, UmeCliOverride).outerjoin( + UmeCliOverride, UmeInventoryNE.ne_id == UmeCliOverride.ume_ne_id + ) + webcrt = db.query(ManagedNE).filter(ManagedNE.source == WEBCRT_NE_SOURCE) + + def _apply_managed_keyword(query: Any) -> Any: + if not kw: + return query + like = f"%{kw}%" + return query.filter( + ManagedNE.id.ilike(like) + | ManagedNE.name.ilike(like) + | ManagedNE.ip_address.ilike(like) + | ManagedNE.username.ilike(like) + | ManagedNE.tags.ilike(like) + | ManagedNE.vendor.ilike(like) + | ManagedNE.device_type.ilike(like) + ) + + def _apply_ume_keyword(query: Any) -> Any: + if not kw: + return query + like = f"%{kw}%" + return query.filter( + UmeInventoryNE.ne_id.ilike(like) + | UmeInventoryNE.ne_name.ilike(like) + | UmeInventoryNE.user_label.ilike(like) + | UmeInventoryNE.ip_address.ilike(like) + | UmeInventoryNE.host_name.ilike(like) + ) + + managed = _apply_managed_keyword(managed) + webcrt = _apply_managed_keyword(webcrt) + ume = _apply_ume_keyword(ume) + ume_status = _apply_ume_keyword(ume_status) + + if field == "source": + values: set[str] = set() + if include_managed and managed.count(): + values.add("managed") + if include_webcrt and webcrt.count(): + values.add("webcrt") + if include_ume and ume.count(): + values.add("ume") + else: + managed_expr = { + "name": func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address), + "ip": ManagedNE.ip_address, + "vendor": ManagedNE.vendor, + "device_type": ManagedNE.device_type, + "connect_status": ManagedNE.connect_status, + }.get(field_map[field]) + ume_expr = { + "name": func.coalesce( + func.nullif(UmeInventoryNE.user_label, ""), + func.nullif(UmeInventoryNE.ne_name, ""), + func.nullif(UmeInventoryNE.host_name, ""), + func.nullif(UmeInventoryNE.ip_address, ""), + UmeInventoryNE.ne_id, + ), + "ip": UmeInventoryNE.ip_address, + "vendor": UmeInventoryNE.vendor, + "device_type": UmeInventoryNE.ne_type, + "connect_status": func.coalesce(UmeCliOverride.connect_status, "unknown"), + }.get(field_map[field]) + values = set() + if include_managed and managed_expr is not None: + values.update(str(row[0] or "") for row in managed.with_entities(managed_expr).distinct().all()) + if include_webcrt and managed_expr is not None: + values.update(str(row[0] or "") for row in webcrt.with_entities(managed_expr).distinct().all()) + if include_ume and ume_expr is not None: + values.update(str(row[0] or "") for row in ume_status.with_entities(ume_expr).distinct().all()) + + return {"field": field, "options": sorted(values, key=str.casefold)} + + def cli_meta(db: Session) -> dict[str, Any]: return { "credentials_configured": credentials_configured(), diff --git a/netx_api/config.py b/netx_api/config.py index 1126fc5..2ebe55f 100644 --- a/netx_api/config.py +++ b/netx_api/config.py @@ -80,8 +80,9 @@ class Settings(BaseSettings): ume_topo_nodes_path: str = "/restconf/data/zte-resources-module:TopoNodes" ume_topological_links_path: str = "/restconf/data/zte-resources-module:TopologicalLinks" ume_sync_alarms_history_every_hours: int = 24 - # Managed NE credentials (Fernet key; generate with cryptography.fernet.Fernet.generate_key()) + # Managed NE credentials (Fernet). Empty = auto-generate & persist to credential_secret_file. credential_secret_key: str = "" + credential_secret_file: str = "data/auth/credential_secret" # Shared-server worker caps (sized for multi-operator use; raise if bastion/DB allow). ne_connect_max_workers: int = 8 ne_connect_timeout_sec: int = 30 @@ -136,6 +137,13 @@ class Settings(BaseSettings): biz_state_dedicated_workers: bool = True # Global ceiling for simultaneous running batches (across all workers). biz_state_max_concurrent_tasks: int = 16 + # Auto-generated run history kept per cutover wave / auto compare job (acceptance + manual runs are never purged). + biz_migration_auto_run_keep: int = 300 + biz_compare_auto_run_keep: int = 50 + # Rows of immutable baseline batches cached across cutover evaluations (0 disables). + biz_migration_baseline_cache_rows: int = 500000 + # Concurrent SSH collections per network element (0 = unlimited). + biz_state_max_sessions_per_ne: int = 2 biz_state_worker_collect_threads: int = 8 biz_state_persist_workers: int = 4 # TextFSM / normalize workers (off SSH collect threads). Default 8. diff --git a/netx_api/config_sync_router.py b/netx_api/config_sync_router.py index 375f241..62a9a81 100644 --- a/netx_api/config_sync_router.py +++ b/netx_api/config_sync_router.py @@ -27,6 +27,7 @@ from .config_sync_service import ( stop_cycle, update_policy, ) +from .table_filters import TableFilterParams, table_filter_params from .db import get_db router = APIRouter(prefix="/v1/config-sync", tags=["config-sync"]) @@ -164,10 +165,11 @@ def api_list_snapshots( keyword: str = Query(default=""), source: str = Query(default=""), vendor: str = Query(default=""), + table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ): return list_snapshots( - db, page=page, page_size=page_size, keyword=keyword, source=source, vendor=vendor + db, page=page, page_size=page_size, keyword=keyword, source=source, vendor=vendor, table_filters=table_filters ) diff --git a/netx_api/config_sync_snapshots.py b/netx_api/config_sync_snapshots.py index 97299af..ca8e582 100644 --- a/netx_api/config_sync_snapshots.py +++ b/netx_api/config_sync_snapshots.py @@ -7,7 +7,7 @@ import zipfile from typing import Any from fastapi import HTTPException -from sqlalchemy import or_ +from sqlalchemy import func, or_ from sqlalchemy.orm import Session from .config_sync_codec import decompress_text @@ -16,6 +16,7 @@ from .config_sync_schemas import ( NeConfigSnapshotDetailOut, NeConfigSnapshotMetaOut, ) +from .table_filters import TableFilterParams, filter_query from .models import NeConfigHistory, NeConfigSnapshot def _snap_meta(row: NeConfigSnapshot) -> NeConfigSnapshotMetaOut: @@ -48,6 +49,7 @@ def list_snapshots( keyword: str = "", source: str = "", vendor: str = "", + table_filters: TableFilterParams | None = None, ) -> dict[str, Any]: q = db.query(NeConfigSnapshot) src = str(source or "").strip().lower() @@ -66,6 +68,14 @@ def list_snapshots( NeConfigSnapshot.target_id.ilike(like), ) ) + q, options = filter_query(q, { + "name": func.coalesce(func.nullif(NeConfigSnapshot.ne_name, ""), NeConfigSnapshot.target_id), + "ip": NeConfigSnapshot.ne_ip, "vendor": NeConfigSnapshot.vendor, + "source": NeConfigSnapshot.source, "size": NeConfigSnapshot.plain_size, + "collected": NeConfigSnapshot.collected_at, + }, table_filters) + if options is not None: + return options total = int(q.count()) rows = q.order_by(NeConfigSnapshot.collected_at.desc()).offset((page - 1) * page_size).limit(page_size).all() return {"total": total, "page": page, "page_size": page_size, "items": [_snap_meta(r) for r in rows]} diff --git a/netx_api/importer.py b/netx_api/importer.py index 7c17905..38364df 100644 --- a/netx_api/importer.py +++ b/netx_api/importer.py @@ -1,29 +1,34 @@ from __future__ import annotations import json +import math from datetime import datetime, timezone -from io import BytesIO from typing import Any -import pandas as pd from sqlalchemy import func, select from sqlalchemy.orm import Session from .config import settings from .models import AlarmBatch, AlarmNorm, ImportErrorRow from .parser_config import ParserConfig +from .tabular import parse_datetime, read_xlsx + + +def _blank(v: Any) -> bool: + if v is None: + return True + if isinstance(v, float) and math.isnan(v): + return True + text = str(v).strip() + return not text or text.lower() == "nan" def _dt(v: Any) -> datetime | None: - if v is None: + if _blank(v): return None - text = str(v).strip() - if not text or text.lower() == "nan": + dt = parse_datetime(v) + if dt is None: return None - parsed = pd.to_datetime(text, errors="coerce") - if pd.isna(parsed): - return None - dt = parsed.to_pydatetime() # Convention: Excel source time is local timezone time. # Convert to UTC before persisting; keep DB as naive UTC for compatibility. if dt.tzinfo is None: @@ -35,20 +40,15 @@ def _dt(v: Any) -> datetime | None: def _str(v: Any) -> str: - if v is None: + if _blank(v): return "" - text = str(v).strip() - if text.lower() == "nan": - return "" - return text + return str(v).strip() def _int(v: Any) -> int: - if v is None: + if _blank(v): return 0 text = str(v).strip() - if not text or text.lower() == "nan": - return 0 try: return int(float(text)) except Exception: @@ -56,8 +56,7 @@ def _int(v: Any) -> int: def import_alarm_excel(db: Session, filename: str, content: bytes, parser: ParserConfig) -> AlarmBatch: - df = pd.read_excel(BytesIO(content)) - headers = [str(h) for h in df.columns] + headers, rows = read_xlsx(content) col_alarm_time = parser.resolve_col(headers, "alarm_time") col_clear_time = parser.resolve_col(headers, "clear_time") @@ -80,7 +79,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse source_file=filename, parser_version=parser.parser_version, dict_version=parser.dict_version, - total_rows=int(len(df.index)), + total_rows=len(rows), status="processing", ) db.add(batch) @@ -89,8 +88,8 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse success_rows = 0 failed_rows = 0 - for idx, row in df.iterrows(): - row_no = int(idx) + 2 # Excel header is row 1 + for row_no, row in rows: # row_no = sheet row (header is row 1) + raw_json = json.dumps(row, ensure_ascii=False, default=str) alarm_time = _dt(row.get(col_alarm_time)) if col_alarm_time else None raw_severity = _str(row.get(col_severity)) if col_severity else "" alarm_code = _str(row.get(col_alarm_code)) if col_alarm_code else "" @@ -111,7 +110,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse batch_id=batch.batch_id, row_no=row_no, reason="missing_or_invalid_alarm_time", - raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str), + raw_json=raw_json, ) ) continue @@ -122,7 +121,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse batch_id=batch.batch_id, row_no=row_no, reason="missing_ne_name_and_ne_id", - raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str), + raw_json=raw_json, ) ) continue @@ -151,7 +150,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse vendor=settings.vendor, source_type=settings.source_type, source_file=filename, - raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str), + raw_json=raw_json, ) ) success_rows += 1 diff --git a/netx_api/lldp_collect_router.py b/netx_api/lldp_collect_router.py index addfcf2..4844e47 100644 --- a/netx_api/lldp_collect_router.py +++ b/netx_api/lldp_collect_router.py @@ -7,6 +7,7 @@ from typing import Any from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session +from .table_filters import TableFilterParams, table_filter_params from .db import get_db from .lldp_collect_schemas import LldpCollectPolicyUpdate, LldpCollectStartBody from .lldp_collect_service import ( @@ -69,9 +70,10 @@ def api_list_jobs( page_size: int = Query(default=20, ge=1, le=100), status: str = Query(default=""), keyword: str = Query(default=""), + table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: - return list_jobs(db, page=page, page_size=page_size, status=status, keyword=keyword) + return list_jobs(db, page=page, page_size=page_size, status=status, keyword=keyword, table_filters=table_filters) @router.get("/jobs/{job_id}") @@ -81,6 +83,7 @@ def api_get_job( page_size: int = Query(default=20, ge=1, le=100), status: str = Query(default=""), keyword: str = Query(default=""), + table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: return get_job_detail( @@ -90,4 +93,5 @@ def api_get_job( page_size=page_size, item_status=status, item_keyword=keyword, + table_filters=table_filters, ) diff --git a/netx_api/lldp_collect_service.py b/netx_api/lldp_collect_service.py index e5eb63b..4cd40f8 100644 --- a/netx_api/lldp_collect_service.py +++ b/netx_api/lldp_collect_service.py @@ -5,9 +5,11 @@ from __future__ import annotations from datetime import datetime, timedelta from fastapi import HTTPException -from sqlalchemy import or_ +from sqlalchemy import String, cast, or_ from sqlalchemy.orm import Session +from .table_filters import TableFilterParams, filter_query +from .topology_discover_common import discover_item_expressions from .lldp_collect_schemas import ( LldpCollectDashboardOut, LldpCollectJobSummary, @@ -488,6 +490,7 @@ def list_jobs( page_size: int = 20, status: str = "", keyword: str = "", + table_filters: TableFilterParams | None = None, ) -> dict: page = max(1, int(page or 1)) page_size = max(1, min(100, int(page_size or 20))) @@ -506,6 +509,15 @@ def list_jobs( TopoDiscoverJob.trigger_mode.ilike(like), ) ) + job = TopoDiscoverJob + q, options = filter_query(q, { + "id": job.id, "trigger": job.trigger_mode, "scope": job.scope, "status": job.status, + "progress": cast(job.done, String) + "/" + cast(job.total, String), + "edges": cast(job.edges_added, String) + "/" + cast(job.edges_updated, String), + "missingDelta": job.edges_stale, "started": job.started_at, "ended": job.ended_at, + }, table_filters) + if options is not None: + return options q = q.order_by(TopoDiscoverJob.created_at.desc()) total = int(q.count()) rows = q.offset((page - 1) * page_size).limit(page_size).all() @@ -532,7 +544,14 @@ def get_job_detail( page_size: int | None = None, item_status: str = "", item_keyword: str = "", + table_filters: TableFilterParams | None = None, ) -> dict: + if table_filters and table_filters.field: + if db.get(TopoDiscoverJob, job_id) is None: + raise HTTPException(status_code=404, detail="discover_job_not_found") + q = db.query(TopoDiscoverJobItem).filter(TopoDiscoverJobItem.job_id == job_id) + _, options = filter_query(q, discover_item_expressions(), table_filters) + return options return get_discover_job( db, job_id, @@ -540,4 +559,5 @@ def get_job_detail( page_size=page_size, item_status=item_status, item_keyword=item_keyword, + table_filters=table_filters, ).model_dump() diff --git a/netx_api/managed_ne_router.py b/netx_api/managed_ne_router.py index a90bf62..6c656d2 100644 --- a/netx_api/managed_ne_router.py +++ b/netx_api/managed_ne_router.py @@ -42,6 +42,7 @@ from .ne_service import ( update_managed_ne, ) from .models import ManagedNE +from .table_filters import TableFilterParams, table_filter_params router = APIRouter(prefix="/v1/managed-ne", tags=["managed-ne"]) @@ -63,6 +64,7 @@ def api_list_managed_ne( connect_status: str | None = Query(default=None), page: int = Query(default=1, ge=1), page_size: int = Query(default=50, ge=1, le=500), + table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ): return list_managed_ne( @@ -72,6 +74,7 @@ def api_list_managed_ne( connect_status=connect_status, page=page, page_size=page_size, + table_filters=table_filters, ) diff --git a/netx_api/models/auth.py b/netx_api/models/auth.py index 39947a0..ffdbc39 100644 --- a/netx_api/models/auth.py +++ b/netx_api/models/auth.py @@ -21,6 +21,8 @@ class AppUser(Base): role: Mapped[str] = mapped_column(String(32), default="user", index=True) # admin | user # Optional capability override; empty => role defaults (see auth_scopes). scopes: Mapped[list] = mapped_column(_JsonType, default=list) + # Per-user UI preferences (e.g. workbench card layout). Pure client prefs; not audited. + prefs: Mapped[dict] = mapped_column(_JsonType, default=dict) is_active: Mapped[bool] = mapped_column(Boolean, default=True, index=True) must_change_password: Mapped[bool] = mapped_column(Boolean, default=False) created_by: Mapped[str] = mapped_column(String(64), default="") diff --git a/netx_api/models/biz_state.py b/netx_api/models/biz_state.py index bd8c311..864f85d 100644 --- a/netx_api/models/biz_state.py +++ b/netx_api/models/biz_state.py @@ -32,6 +32,9 @@ class BizStateTask(Base): note: Mapped[str] = mapped_column(String(256), default="") # "" | portrait | cutover_hf — filter cutover HF in biz-state list purpose: Mapped[str] = mapped_column(String(32), default="", index=True) + # Optional paired sampling; ordinary per-device tasks remain independent. + collect_group_id: Mapped[str] = mapped_column(String(128), default="", index=True) + collect_manual_only: Mapped[bool] = mapped_column(Boolean, default=False) status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|running|paused|stopped interval_sec: Mapped[int] = mapped_column(Integer, default=300) # Keep snapshots for N calendar days (protected baselines/refs never auto-deleted) @@ -86,6 +89,10 @@ class BizStateBatch(Base): id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex) task_id: Mapped[str] = mapped_column(String(64), default="", index=True) + collect_group_id: Mapped[str] = mapped_column(String(128), default="", index=True) + collect_round_id: Mapped[str] = mapped_column(String(64), default="", index=True) + collect_priority: Mapped[int] = mapped_column(Integer, default=0) + queued_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) source: Mapped[str] = mapped_column(String(32), default="managed", index=True) ne_id: Mapped[str] = mapped_column(String(128), default="", index=True) ne_name: Mapped[str] = mapped_column(String(256), default="") @@ -303,6 +310,8 @@ class BizCompareJob(Base): status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|ready|auto # Empty = all template sheets; non-empty = only these sheet_id values enabled_sheet_ids: Mapped[list] = mapped_column(_JsonType, default=list) + # auto|always|never|sample — how to persist matching (success) rows + store_unchanged: Mapped[str] = mapped_column(String(16), default="auto") note: Mapped[str] = mapped_column(String(512), default="") created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) @@ -347,4 +356,7 @@ class BizCompareDiff(Base): after_json: Mapped[dict] = mapped_column(_JsonType, default=dict) mapped_before_json: Mapped[dict] = mapped_column(_JsonType, default=dict) changes_json: Mapped[dict] = mapped_column(_JsonType, default=dict) + # Pointers into BizStateMetricRow / LLDP tables for compact success hydrate + before_row_id: Mapped[str] = mapped_column(String(64), default="") + after_row_id: Mapped[str] = mapped_column(String(64), default="") search_text: Mapped[str] = mapped_column(Text, default="") diff --git a/netx_api/ne_cli_hop.py b/netx_api/ne_cli_hop.py index 807d9f8..83a45dd 100644 --- a/netx_api/ne_cli_hop.py +++ b/netx_api/ne_cli_hop.py @@ -10,7 +10,9 @@ from netmiko import ConnectHandler _CLI_HOP_NESTED_END_RE = re.compile( r"(?is)" r"(?:^|\n)\s*(?:" - r"connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?" + # Huawei VRP: "Info: The connection was closed by the remote host." + r"(?:info:\s*)?(?:the\s+)?connection\s+was\s+closed\s+by\s+(?:the\s+)?(?:foreign|remote)\s+host" + r"|connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?" r"|closed\s+by\s+foreign\s+host" r"|connection\s+to\s+\S+\s+closed" r"|%\s*connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?" diff --git a/netx_api/ne_crypto.py b/netx_api/ne_crypto.py index 4ba7617..7624c2a 100644 --- a/netx_api/ne_crypto.py +++ b/netx_api/ne_crypto.py @@ -1,18 +1,73 @@ from __future__ import annotations +import logging +from pathlib import Path + from cryptography.fernet import Fernet, InvalidToken from .config import settings +_log = logging.getLogger("netx.crypto") +_cached_credential_key: str | None = None + class CredentialCryptoError(RuntimeError): pass +def credential_secret_file_path() -> Path: + raw = str(getattr(settings, "credential_secret_file", None) or "data/auth/credential_secret").strip() + path = Path(raw) + if not path.is_absolute(): + path = Path.cwd() / path + return path + + +def ensure_credential_secret_key() -> str: + """Resolve Fernet key: env ``NETX_CREDENTIAL_SECRET_KEY`` > file > generate once. + + Packaged installs should also write the key into ``.env`` via setup_first_run; + this startup/path fallback covers upgrades and missed first-run keys. + """ + global _cached_credential_key + configured = str(settings.credential_secret_key or "").strip() + if configured: + return configured + if _cached_credential_key: + return _cached_credential_key + + path = credential_secret_file_path() + try: + if path.is_file(): + existing = path.read_text(encoding="utf-8").strip() + if existing: + _cached_credential_key = existing + settings.credential_secret_key = existing + return existing + except Exception: + _log.exception("read credential secret file failed path=%s", path) + + generated = Fernet.generate_key().decode("ascii") + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(generated + "\n", encoding="utf-8") + try: + path.chmod(0o600) + except Exception: + pass + _log.info("wrote per-install credential Fernet key to %s", path) + except Exception: + _log.exception( + "write credential secret file failed path=%s; using in-memory key only", + path, + ) + _cached_credential_key = generated + settings.credential_secret_key = generated + return generated + + def _fernet() -> Fernet: - key = str(settings.credential_secret_key or "").strip() - if not key: - raise CredentialCryptoError("credential_secret_key_not_configured") + key = ensure_credential_secret_key() try: return Fernet(key.encode("ascii")) except Exception as exc: @@ -37,4 +92,7 @@ def decrypt_secret(value: str) -> str: def credentials_configured() -> bool: - return bool(str(settings.credential_secret_key or "").strip()) + try: + return bool(ensure_credential_secret_key()) + except Exception: + return False diff --git a/netx_api/ne_netmiko.py b/netx_api/ne_netmiko.py index 9b351e6..5353af1 100644 --- a/netx_api/ne_netmiko.py +++ b/netx_api/ne_netmiko.py @@ -47,6 +47,9 @@ def paging_disable_commands(*, vendor: str = "", device_type: str = "") -> list[ dt = str(device_type or "").strip().lower() v = str(vendor or "").strip().lower() blob = f"{dt} {v}" + # RouterOS has no pager; Cisco's command leaves an extra error/prompt redraw. + if "mikrotik" in blob or "routeros" in blob or "switchos" in blob: + return [] if "huawei" in blob or "华为" in v: return ["screen-length 0 temporary"] if "hp_comware" in blob or "h3c" in blob or "comware" in blob: @@ -134,6 +137,23 @@ def drain_read_channel( def _send_command_expect_prompt(conn: Any, cmd: str, *, read_timeout: int) -> str: + device_type = getattr(conn, "device_type", "") + if isinstance(device_type, str) and device_type.lower().startswith("mikrotik"): + # find_prompt sends an extra Enter *after* our channel drain. RouterOS + # can repaint that old prompt after send_command starts, ending the read + # before the command's output arrives. Use its complete prompt directly. + # Keep echo verification off: long RouterOS scripts wrap at PTY width. + prompt_pattern = r"(?m)^\[[^\r\n\]]+\](?:[ \t]+/[^\r\n>]*)?[ \t]*>[ \t]*$" + return str( + conn.send_command( + command_string=cmd, + read_timeout=read_timeout, + auto_find_prompt=False, + expect_string=prompt_pattern, + cmd_verify=False, + ) + or "" + ) try: return str( conn.send_command( diff --git a/netx_api/ne_schemas.py b/netx_api/ne_schemas.py index 0611390..5bc2813 100644 --- a/netx_api/ne_schemas.py +++ b/netx_api/ne_schemas.py @@ -17,7 +17,7 @@ class ManagedNeCreate(BaseModel): vendor: str device_type: str ip_address: str - port: int = 22 + port: int = Field(default=22, ge=1, le=65535) protocol: str = "ssh" username: str password: str = "" @@ -27,7 +27,7 @@ class ManagedNeCreate(BaseModel): hop_enabled: bool = False hop_vendor: str = "zte" hop_host: str = "" - hop_port: int = 22 + hop_port: int = Field(default=22, ge=1, le=65535) hop_protocol: str = "ssh" hop_username: str = "" hop_password: str = "" @@ -63,7 +63,7 @@ class ManagedNeUpdate(BaseModel): vendor: str | None = None device_type: str | None = None ip_address: str | None = None - port: int | None = None + port: int | None = Field(default=None, ge=1, le=65535) protocol: str | None = None username: str | None = None password: str | None = None @@ -73,7 +73,7 @@ class ManagedNeUpdate(BaseModel): hop_enabled: bool | None = None hop_vendor: str | None = None hop_host: str | None = None - hop_port: int | None = None + hop_port: int | None = Field(default=None, ge=1, le=65535) hop_protocol: str | None = None hop_username: str | None = None hop_password: str | None = None @@ -194,7 +194,7 @@ class HopProxyConfig(BaseModel): hop_vendor: str = "zte" hop_host: str - hop_port: int = 22 + hop_port: int = Field(default=22, ge=1, le=65535) hop_protocol: str = "ssh" hop_username: str hop_password: str = "" diff --git a/netx_api/ne_service_common.py b/netx_api/ne_service_common.py index 9e74f0b..f76d6d7 100644 --- a/netx_api/ne_service_common.py +++ b/netx_api/ne_service_common.py @@ -2,10 +2,10 @@ from __future__ import annotations from datetime import datetime +import math import re from typing import Any -import pandas as pd from fastapi import HTTPException from sqlalchemy.orm import Session @@ -60,6 +60,63 @@ def _normalize_ip(ip: str) -> str: return str(ip or "").strip() +_HOSTNAME_RE = re.compile(r"^(?=.{1,253}$)[A-Za-z0-9_](?:[A-Za-z0-9_-]{0,62})(?:\.[A-Za-z0-9_](?:[A-Za-z0-9_-]{0,62}))*\.?$") + + +def _is_valid_mgmt_address(ip: str) -> bool: + """Accept an IPv4/IPv6 literal or a DNS hostname (no spaces, ports or schemes).""" + import ipaddress + + text = str(ip or "").strip() + if not text: + return False + try: + ipaddress.ip_address(text) + return True + except ValueError: + pass + # Dotted-quad lookalikes (e.g. 10.0.0.300) are typos, not hostnames. + if re.fullmatch(r"[0-9.]+", text): + return False + return bool(_HOSTNAME_RE.match(text)) + + +def _require_mgmt_address(ip: str) -> str: + text = _normalize_ip(ip) + if not text: + raise HTTPException(status_code=400, detail="ip_address_required") + if not _is_valid_mgmt_address(text): + raise HTTPException(status_code=400, detail="invalid_ip_address") + return text + + +CONNECT_FIELDS = ( + "ip_address", "port", "protocol", "username", "password_enc", "device_type", + "hop_enabled", "hop_vendor", "hop_host", "hop_port", "hop_protocol", "hop_username", + "hop_password_enc", "hop_command_template", "hop_vrf", "hop_target_auth_mode", + "hop_enter_system_view", +) + + +def connect_snapshot(row: ManagedNE) -> tuple[Any, ...]: + return tuple(getattr(row, f, None) for f in CONNECT_FIELDS) + + +def reset_connect_status_if_changed(row: ManagedNE, before: tuple[Any, ...]) -> None: + """A previous pass/fail no longer describes the device once login settings change.""" + if connect_snapshot(row) != before and str(row.connect_status or "") != "unknown": + row.connect_status = "unknown" + row.connect_message = "" + + +def hop_password_optional(hop_vendor: str, hop_target_auth_mode: str) -> bool: + """Only a bastion that manages target auth may run without a stored hop password.""" + return ( + _normalize_hop_vendor(hop_vendor) == "bastion" + and _normalize_hop_target_auth_mode(hop_target_auth_mode) == "bastion_managed" + ) + + def _normalize_protocol(protocol: str) -> str: p = str(protocol or "ssh").strip().lower() return p if p in ("ssh", "telnet") else "ssh" @@ -142,7 +199,7 @@ def _parse_import_bool(value: Any) -> bool: def _import_cell_str(value: Any) -> str: - if value is None or (isinstance(value, float) and pd.isna(value)): + if value is None or (isinstance(value, float) and math.isnan(value)): return "" text = str(value).strip() return "" if text.lower() == "nan" else text @@ -213,9 +270,8 @@ def _apply_hop_update(row: ManagedNE, data: dict[str, Any]) -> None: raise HTTPException(status_code=400, detail="hop_host_required") if not str(row.hop_username or "").strip(): raise HTTPException(status_code=400, detail="hop_username_required") - if ( - not str(row.hop_password_enc or "").strip() - and _normalize_hop_target_auth_mode(row.hop_target_auth_mode) != "bastion_managed" + if not str(row.hop_password_enc or "").strip() and not hop_password_optional( + str(row.hop_vendor or ""), str(row.hop_target_auth_mode or "") ): raise HTTPException(status_code=400, detail="hop_password_required") diff --git a/netx_api/ne_service_crud.py b/netx_api/ne_service_crud.py index e8b253c..5b168e7 100644 --- a/netx_api/ne_service_crud.py +++ b/netx_api/ne_service_crud.py @@ -5,10 +5,10 @@ from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import or_ +from sqlalchemy import false, func, literal, or_ from sqlalchemy.orm import Session -from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS, WEBCRT_DEVICE_TYPES, is_placeholder_ne_source +from .device_types import SUPPORTED_VENDORS, WEBCRT_DEVICE_TYPES, is_placeholder_ne_source from .models import ManagedNE from .ne_crypto import encrypt_secret from .ne_hop_templates import default_bastion_username_template, default_hop_command_template @@ -24,8 +24,11 @@ from .ne_service_common import ( _apply_hop_update, _normalize_hop_target_auth_mode, _normalize_hop_vendor, - _normalize_ip, _normalize_protocol, + _require_mgmt_address, + connect_snapshot, + hop_password_optional, + reset_connect_status_if_changed, _normalize_saved_hop_endpoint, _normalize_vendor, _now, @@ -33,16 +36,43 @@ from .ne_service_common import ( _validate_hop_on_create, row_to_out, ) +from .table_filters import TableFilterParams, filter_query from .ne_exec_guard import ( EXEC_POLICY_READONLY, allows_open_exec_policy, require_exec_policy_writable, ) -# Inventory create stays strict; updates must also accept WebCRT/LLDP placeholder types -# (generic/linux) so operators can open the form and promote them to zte_zxros etc. +# Create and update share the form's device-type list, which includes linux/generic +# (Linux hosts and WebCRT/LLDP placeholders promoted to inventory). _ALLOWED_UPDATE_DEVICE_TYPES = frozenset(WEBCRT_DEVICE_TYPES) +def managed_ne_filter_expressions() -> dict[str, Any]: + """Column filter fields for the NE management table (tags handled separately).""" + return { + "name": func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address), + "vendor": ManagedNE.vendor, + "deviceType": ManagedNE.device_type, + "source": func.coalesce(func.nullif(ManagedNE.source, ""), "manual"), + "ip": ManagedNE.ip_address, + "username": ManagedNE.username, + "connectStatus": ManagedNE.connect_status, + } + + +def _filter_by_tags(stmt: Any, value: dict[str, Any]) -> Any: + """Tags are space-separated; a row matches when any selected tag is present ("" = untagged).""" + tags_col = func.coalesce(ManagedNE.tags, "") + padded = literal(" ") + tags_col + literal(" ") + conds = [] + for tag in value["values"]: + tag = str(tag).strip() + escaped = tag.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + conds.append(func.trim(tags_col) == "" if not tag else padded.like(f"% {escaped} %", escape="\\")) + matched = or_(*conds) if conds else false() + return stmt.filter(matched if value["mode"] == "include" else ~matched) + + def list_managed_ne( db: Session, *, @@ -51,6 +81,7 @@ def list_managed_ne( connect_status: str | None = None, page: int = 1, page_size: int = 50, + table_filters: TableFilterParams | None = None, ) -> dict[str, Any]: stmt = db.query(ManagedNE) kw = str(keyword or "").strip() @@ -72,6 +103,22 @@ def list_managed_ne( stmt = stmt.filter(ManagedNE.vendor == v) if cs: stmt = stmt.filter(ManagedNE.connect_status == cs) + params = table_filters or TableFilterParams() + tags_filter = params.filters.get("tags") + if tags_filter and params.field != "tags": + stmt = _filter_by_tags(stmt, tags_filter) + if params.field == "tags": + stmt, _ = filter_query(stmt, managed_ne_filter_expressions(), TableFilterParams( + {k: v for k, v in params.filters.items() if k != "tags"})) + options: set[str] = set() + for (tags_val,) in stmt.with_entities(ManagedNE.tags).all(): + parts = str(tags_val or "").split() + options.update(parts if parts else [""]) + return {"field": "tags", "options": sorted(options)} + stmt, options = filter_query(stmt, managed_ne_filter_expressions(), TableFilterParams( + {k: v for k, v in params.filters.items() if k != "tags"}, params.field)) + if options is not None: + return options total = int(stmt.count()) rows = ( stmt.order_by(ManagedNE.updated_at.desc()) @@ -97,10 +144,8 @@ def get_managed_ne(db: Session, ne_id: str) -> ManagedNeOut: def create_managed_ne(db: Session, body: ManagedNeCreate) -> ManagedNeOut: _require_crypto() _validate_hop_on_create(body) - ip = _normalize_ip(body.ip_address) - if not ip: - raise HTTPException(status_code=400, detail="ip_address_required") - if body.device_type not in SUPPORTED_DEVICE_TYPES: + ip = _require_mgmt_address(body.ip_address) + if body.device_type not in _ALLOWED_UPDATE_DEVICE_TYPES: raise HTTPException(status_code=400, detail="unsupported_device_type") existing = db.query(ManagedNE).filter(ManagedNE.ip_address == ip).first() if existing: @@ -140,10 +185,9 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed if not row: raise HTTPException(status_code=404, detail="managed_ne_not_found") data = body.model_dump(exclude_unset=True) + before = connect_snapshot(row) if "ip_address" in data: - ip = _normalize_ip(data["ip_address"]) - if not ip: - raise HTTPException(status_code=400, detail="ip_address_required") + ip = _require_mgmt_address(data["ip_address"]) other = db.query(ManagedNE).filter(ManagedNE.ip_address == ip, ManagedNE.id != ne_id).first() if other: raise HTTPException(status_code=400, detail="ip_address_exists") @@ -157,7 +201,7 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed v = str(data["vendor"] or "").strip() row.vendor = v if v in SUPPORTED_VENDORS else "Other" if "name" in data: - row.name = str(data["name"] or "").strip() + row.name = str(data["name"] or "").strip() or str(row.ip_address or "") if "port" in data and data["port"] is not None: row.port = int(data["port"]) if "protocol" in data and data["protocol"] is not None: @@ -198,6 +242,7 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed if "ip_address" in data and str(row.ip_address or "").strip(): if is_placeholder_ne_source(row.source): row.source = "" + reset_connect_status_if_changed(row, before) row.updated_at = _now() # Keep linked fabric node identity in sync (name / IP / vendor). if any(k in data for k in ("ip_address", "name", "vendor", "device_type")): @@ -225,8 +270,6 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d if not hop_user: raise HTTPException(status_code=400, detail="hop_username_required") hop_auth_mode = _normalize_hop_target_auth_mode(hop.hop_target_auth_mode) - if not hop_pass and hop_auth_mode != "bastion_managed": - raise HTTPException(status_code=400, detail="hop_password_required") template = str(hop.hop_command_template or "").strip() if hop_vendor == "bastion" and not template: @@ -243,9 +286,15 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d missing = [x for x in ne_ids if x not in found_ids] if missing: raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing[:5])}") + # Blank password keeps each NE's stored hop password; NEs without one cannot hop. + if not hop_pass and not hop_password_optional(hop_vendor, hop_auth_mode): + lacking = [str(r.name or r.ip_address) for r in rows if not str(r.hop_password_enc or "").strip()] + if lacking: + raise HTTPException(status_code=400, detail=f"hop_password_required: {','.join(lacking[:5])}") now = _now() for row in rows: + before = connect_snapshot(row) row.hop_enabled = True row.hop_vendor = hop_vendor row.hop_host = hop_host @@ -258,6 +307,7 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d row.hop_vrf = str(hop.hop_vrf or "").strip() row.hop_target_auth_mode = hop_auth_mode row.hop_enter_system_view = bool(getattr(hop, "hop_enter_system_view", False)) + reset_connect_status_if_changed(row, before) row.updated_at = now db.commit() return {"ok": True, "updated": len(rows)} @@ -283,10 +333,12 @@ def batch_apply_account(db: Session, ids: list[str], account: BatchAccountConfig raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing[:5])}") now = _now() for row in rows: + before = connect_snapshot(row) if user: row.username = user if pwd: row.password_enc = pwd_enc + reset_connect_status_if_changed(row, before) row.updated_at = now db.commit() return {"ok": True, "updated": len(rows)} diff --git a/netx_api/ne_service_import.py b/netx_api/ne_service_import.py index 22415c8..ca006af 100644 --- a/netx_api/ne_service_import.py +++ b/netx_api/ne_service_import.py @@ -1,16 +1,15 @@ """Managed NE Excel import and UME inventory sync into managed_ne.""" from __future__ import annotations -from io import BytesIO from typing import Any -import pandas as pd from fastapi import HTTPException from sqlalchemy.orm import Session from .models import ManagedNE, UmeInventoryNE +from .tabular import read_table, write_csv, write_xlsx from .ne_crypto import CredentialCryptoError, encrypt_secret -from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS +from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS, is_placeholder_ne_source from .ne_schemas import ( ImportFailure, ImportResult, @@ -23,6 +22,9 @@ from .ne_service_common import ( UME_SYNC_TAG, _import_cell_str, _infer_managed_ne_type_vendor, + _is_valid_mgmt_address, + connect_snapshot, + reset_connect_status_if_changed, _merge_tags, _normalize_ip, _normalize_protocol, @@ -148,66 +150,74 @@ def build_managed_ne_import_template(fmt: str = "xlsx") -> tuple[str, bytes, str "remark": "no direct password, use batch proxy", }, ] - df = pd.DataFrame(rows, columns=list(IMPORT_COLUMNS)) - buf = BytesIO() + headers = list(IMPORT_COLUMNS) + table = [[r.get(c, "") for c in headers] for r in rows] kind = str(fmt or "xlsx").strip().lower() if kind == "csv": - df.to_csv(buf, index=False, encoding="utf-8-sig") return ( "managed_ne_import_template.csv", - buf.getvalue(), + write_csv(headers, table), "text/csv; charset=utf-8", ) - device_types_df = pd.DataFrame({"device_type": list(SUPPORTED_DEVICE_TYPES)}) - with pd.ExcelWriter(buf, engine="openpyxl") as writer: - df.to_excel(writer, sheet_name="import", index=False) - device_types_df.to_excel(writer, sheet_name="device_type", index=False) + payload = write_xlsx([ + ("import", headers, table), + ("device_type", ["device_type"], [[t] for t in SUPPORTED_DEVICE_TYPES]), + ]) return ( "managed_ne_import_template.xlsx", - buf.getvalue(), + payload, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) def import_managed_ne(db: Session, content: bytes, filename: str) -> ImportResult: _require_crypto() - name = str(filename or "").lower() try: - if name.endswith(".csv"): - df = pd.read_csv(BytesIO(content)) - else: - df = pd.read_excel(BytesIO(content)) + headers, raw_rows = read_table(content, filename) except Exception as exc: raise HTTPException(status_code=400, detail=f"import_parse_failed: {exc}") from exc - df.columns = [str(c).strip().lower() for c in df.columns] - missing = [c for c in _REQUIRED_IMPORT_COLUMNS if c not in df.columns] + columns = [h.strip().lower() for h in headers] + missing = [c for c in _REQUIRED_IMPORT_COLUMNS if c not in columns] if missing: raise HTTPException(status_code=400, detail=f"import_missing_columns: {','.join(missing)}") inserted = 0 updated = 0 failed: list[ImportFailure] = [] - for idx, row in df.iterrows(): - row_no = int(idx) + 2 + for row_no, cells in raw_rows: + row = {k.strip().lower(): v for k, v in cells.items()} + # Savepoint per row: a failing row must not leave a half-written NE behind. + savepoint = db.begin_nested() try: ip = _normalize_ip(_import_cell_str(row.get("ip", ""))) if not ip: failed.append(ImportFailure(row=row_no, reason="ip_required")) + savepoint.rollback() + continue + if not _is_valid_mgmt_address(ip): + failed.append(ImportFailure(row=row_no, reason="invalid_ip_address")) + savepoint.rollback() continue device_type = _import_cell_str(row.get("device_type", "")) if device_type not in SUPPORTED_DEVICE_TYPES: failed.append(ImportFailure(row=row_no, reason="unsupported_device_type")) + savepoint.rollback() continue username = _import_cell_str(row.get("username", "")) password = _import_cell_str(row.get("password", "")) if not username: failed.append(ImportFailure(row=row_no, reason="username_required")) + savepoint.rollback() continue - port_raw = row.get("port", 22) + port_raw = row.get("port") try: - port = int(port_raw) - except (TypeError, ValueError): + port = int(float(port_raw)) if port_raw is not None else 22 + except (TypeError, ValueError, OverflowError): port = 22 - protocol = _normalize_protocol(str(row.get("protocol", "ssh"))) + if not 1 <= port <= 65535: + failed.append(ImportFailure(row=row_no, reason="invalid_port")) + savepoint.rollback() + continue + protocol = _normalize_protocol(_import_cell_str(row.get("protocol")) or "ssh") display_name = _import_cell_str(row.get("name", "")) or ip vendor_raw = _import_cell_str(row.get("vendor", "")) or "Other" vendor = "Other" @@ -217,29 +227,43 @@ def import_managed_ne(db: Session, content: bytes, filename: str) -> ImportResul break existing = db.query(ManagedNE).filter(ManagedNE.ip_address == ip).first() now = _now() + is_new = existing is None if existing is None: - existing = ManagedNE(ip_address=ip, created_at=now) + existing = ManagedNE(ip_address=ip, created_at=now, connect_status="unknown") db.add(existing) - inserted += 1 - else: - updated += 1 + before = connect_snapshot(existing) existing.name = display_name existing.vendor = vendor existing.device_type = device_type existing.port = port existing.protocol = protocol existing.username = username - existing.password_enc = encrypt_secret(password) if password else "" + # Blank password on an existing NE keeps the stored one (template row 2 is blank). + if password: + existing.password_enc = encrypt_secret(password) + elif is_new: + existing.password_enc = "" tags_val = _import_cell_str(row.get("tags", "")) remark_val = _import_cell_str(row.get("remark", "")) if tags_val: existing.tags = tags_val if remark_val: existing.remark = remark_val + if is_placeholder_ne_source(existing.source): + existing.source = "" + if not is_new: + reset_connect_status_if_changed(existing, before) existing.updated_at = now + savepoint.commit() + if is_new: + inserted += 1 + else: + updated += 1 except CredentialCryptoError as exc: + savepoint.rollback() failed.append(ImportFailure(row=row_no, reason=str(exc))) except Exception as exc: + savepoint.rollback() failed.append(ImportFailure(row=row_no, reason=str(exc)[:200])) db.commit() return ImportResult(inserted=inserted, updated=updated, failed=failed) diff --git a/netx_api/ne_session_connect.py b/netx_api/ne_session_connect.py index 3813473..049a850 100644 --- a/netx_api/ne_session_connect.py +++ b/netx_api/ne_session_connect.py @@ -545,18 +545,26 @@ def _connect_direct( def _read_channel(conn: ConnectHandler, wait: float = 0.5, max_loops: int = 40) -> str: - if wait > 0: - time.sleep(wait) + """Collect device output. + + ``wait`` is the longest we park for the *first* bytes; once output has arrived the read + returns as soon as the channel is quiet again. (It used to sleep the whole ``wait`` and + then poll at 50-150ms, which added ~0.1-0.4s of dead time to every login-prompt step.) + """ + quiet = 0.04 if wait <= 0.15 else 0.12 + first_deadline = time.monotonic() + max(0.0, wait) + got_at: float | None = None chunks: list[str] = [] - for _ in range(max_loops): + reads = 0 + while reads < max_loops: try: part = conn.read_channel() except Exception: break if part is None or part is False: - break + part = "" # MagicMock truthy-but-empty: stop when unit tests leave read_channel unconfigured. - if not isinstance(part, (str, bytes, bytearray)): + elif not isinstance(part, (str, bytes, bytearray)): text = str(part) # unittest.mock default str looks like "" if text.startswith("= first_deadline: + break + elif now - got_at >= quiet: break - chunks.append(part) - time.sleep(0.05 if wait <= 0.15 else 0.15) + time.sleep(0.01) return "".join(chunks) @@ -801,11 +817,11 @@ def _interactive_target_auth( return else: empty_after_pass = 0 - time.sleep(0.12) + time.sleep(0.02) continue if not buf.strip(): - time.sleep(0.2) + time.sleep(0.02) continue # Huawei stelnet often reprints the hop ``[sysname]`` after host-key trust and # *before* ``Enter password:``. Do not treat that as target login success. @@ -826,7 +842,7 @@ def _interactive_target_auth( if denied: raise paramiko.AuthenticationException(f"target_auth_rejected: {denied}") return - time.sleep(0.15) + time.sleep(0.02) denied = _target_auth_failure_snippet(acc, sent_pass=sent_pass) if denied: raise paramiko.AuthenticationException(f"target_auth_rejected: {denied}") @@ -879,7 +895,7 @@ def _huawei_enter_system_view( for _ in range(8): part = _read_channel(conn, wait=0.2, max_loops=10) if not part: - time.sleep(0.15) + time.sleep(0.02) continue acc += part if emit_raw: diff --git a/netx_api/schema_patches.py b/netx_api/schema_patches.py index 1dc4528..888d732 100644 --- a/netx_api/schema_patches.py +++ b/netx_api/schema_patches.py @@ -108,6 +108,7 @@ def apply_auth_schema_patches(conn: Connection) -> None: "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS must_change_password BOOLEAN DEFAULT FALSE", ) _run_sql(conn, "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS scopes JSON DEFAULT '[]'") + _run_sql(conn, "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'") _run_sql(conn, "ALTER TABLE api_token ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP") _run_sql(conn, "ALTER TABLE api_token ADD COLUMN IF NOT EXISTS scopes JSON DEFAULT '[]'") _run_sql( @@ -354,6 +355,7 @@ def apply_domain_schema_patches(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_ume_alarms_history_host_name ON ume_alarms_history (host_name)", "ALTER TABLE api_token ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP", "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS must_change_password BOOLEAN DEFAULT FALSE", + "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'", "ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_enabled BOOLEAN DEFAULT FALSE", "ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_vendor VARCHAR(32) DEFAULT 'zte'", "ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_host VARCHAR(128) DEFAULT ''", diff --git a/netx_api/table_filters.py b/netx_api/table_filters.py new file mode 100644 index 0000000..c9bbd55 --- /dev/null +++ b/netx_api/table_filters.py @@ -0,0 +1,55 @@ +"""Shared exact-value column filters, applied before pagination.""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any + +from fastapi import HTTPException, Query +from sqlalchemy import String, cast, func + + +@dataclass +class TableFilterParams: + filters: dict[str, dict[str, Any]] = field(default_factory=dict) + field: str = "" + + +def table_filter_params( + column_filters: str = Query(default=""), + filter_field: str = Query(default=""), +) -> TableFilterParams: + try: + filters = json.loads(column_filters) if column_filters else {} + if not isinstance(filters, dict): + raise ValueError() + for value in filters.values(): + if not isinstance(value, dict) or value.get("mode") not in ("include", "exclude"): + raise ValueError() + values = value.get("values") + if not isinstance(values, list) or any(not isinstance(v, str) for v in values): + raise ValueError() + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="invalid_column_filters") from None + return TableFilterParams(filters, filter_field) + + +def filter_query(query: Any, expressions: dict[str, Any], params: TableFilterParams | None): + """Return the filtered query and, when requested, a distinct-value response.""" + params = params or TableFilterParams() + if (set(params.filters) | ({params.field} if params.field else set())) - expressions.keys(): + raise HTTPException(status_code=400, detail="invalid_filter_field") + for key, value in params.filters.items(): + if key == params.field: + continue + expression = func.coalesce(cast(expressions[key], String), "") + values = value["values"] + if value["mode"] == "include": + query = query.filter(expression.in_(values)) if values else query.filter(False) + elif values: + query = query.filter(~expression.in_(values)) + if params.field: + expression = func.coalesce(cast(expressions[params.field], String), "") + rows = query.order_by(None).with_entities(expression).distinct().all() + return query, {"field": params.field, "options": sorted({str(row[0]) for row in rows})} + return query, None diff --git a/netx_api/tabular.py b/netx_api/tabular.py new file mode 100644 index 0000000..c571456 --- /dev/null +++ b/netx_api/tabular.py @@ -0,0 +1,151 @@ +"""Small xlsx/csv reader/writer for imports and templates (replaces pandas). + +Rows come back as ``(row_no, {header: value})`` where ``row_no`` is the 1-based +sheet/file row (header = row 1), so import errors point at the row users see. +Blank rows are skipped. Excel cells keep their openpyxl types (str / int / +float / datetime / bool / None); integral floats become ints. CSV cells are +strings, empty cells are None. +""" +from __future__ import annotations + +import csv +import io +import re +from collections.abc import Iterable, Iterator +from datetime import datetime +from typing import Any + +from openpyxl import Workbook, load_workbook + +Row = tuple[int, dict[str, Any]] + + +def _headers(raw: Iterable[Any]) -> list[str]: + out: list[str] = [] + seen: dict[str, int] = {} + for i, h in enumerate(raw): + name = "" if h is None else str(h).strip() + if not name: + name = f"Unnamed: {i}" + n = seen.get(name, 0) + seen[name] = n + 1 + out.append(name if n == 0 else f"{name}.{n}") + return out + + +def _cell(v: Any) -> Any: + if isinstance(v, str): + return v if v.strip() else None + if isinstance(v, float) and v.is_integer(): + return int(v) + return v + + +def _rows(records: Iterator[tuple[int, list[Any]]]) -> tuple[list[str], list[Row]]: + headers: list[str] | None = None + rows: list[Row] = [] + for row_no, values in records: + cells = [_cell(v) for v in values] + if all(c is None for c in cells): + continue + if headers is None: + headers = _headers(values) + continue + cells += [None] * (len(headers) - len(cells)) + rows.append((row_no, dict(zip(headers, cells)))) + return headers or [], rows + + +def _decode(content: bytes) -> str: + try: + return content.decode("utf-8-sig") + except UnicodeDecodeError: + # CSVs saved by Chinese Excel default to GBK. + return content.decode("gb18030") + + +def read_csv(content: bytes) -> tuple[list[str], list[Row]]: + reader = csv.reader(io.StringIO(_decode(content), newline="")) + return _rows((i, rec) for i, rec in enumerate(reader, start=1)) + + +def read_xlsx(content: bytes) -> tuple[list[str], list[Row]]: + wb = load_workbook(io.BytesIO(content), read_only=True, data_only=True) + try: + ws = wb.worksheets[0] + return _rows((i, list(rec)) for i, rec in enumerate(ws.iter_rows(values_only=True), start=1)) + finally: + wb.close() + + +def read_table(content: bytes, filename: str = "") -> tuple[list[str], list[Row]]: + if str(filename or "").lower().endswith(".csv"): + return read_csv(content) + return read_xlsx(content) + + +def write_csv(headers: list[str], rows: Iterable[Iterable[Any]]) -> bytes: + buf = io.StringIO() + writer = csv.writer(buf, lineterminator="\n") + writer.writerow(headers) + for r in rows: + writer.writerow(["" if v is None else v for v in r]) + # BOM so Excel opens UTF-8 (Chinese) correctly. + return buf.getvalue().encode("utf-8-sig") + + +def write_xlsx(sheets: list[tuple[str, list[str], Iterable[Iterable[Any]]]]) -> bytes: + wb = Workbook() + wb.remove(wb.active) + for title, headers, rows in sheets: + ws = wb.create_sheet(title) + ws.append(headers) + for r in rows: + ws.append(list(r)) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +_DATE_FORMATS = ( + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M", + "%Y-%m-%d", + "%Y%m%d%H%M%S", + "%Y%m%d", + "%m/%d/%Y %H:%M:%S", + "%m/%d/%Y %H:%M", + "%m/%d/%Y", + "%d-%b-%Y %H:%M:%S", + "%b %d, %Y %H:%M:%S", +) +_CN_DATE = re.compile(r"(\d{4})年(\d{1,2})月(\d{1,2})日") + + +def parse_datetime(v: Any) -> datetime | None: + """Best-effort timestamp parse for spreadsheet cells; None when unparseable.""" + if v is None: + return None + if isinstance(v, datetime): + return v + text = str(v).strip() + if not text or text.lower() in ("nan", "nat", "none", "null"): + return None + text = _CN_DATE.sub(r"\1-\2-\3", text) + # 2024/1/2 03:04:05 -> 2024-1-2 ... ; zero-pad so fromisoformat/strptime accept it. + if re.match(r"^\d{4}/\d{1,2}/\d{1,2}", text): + text = text.replace("/", "-", 2) + m = re.match(r"^(\d{4})-(\d{1,2})-(\d{1,2})(.*)$", text) + if m: + text = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}{m.group(4)}" + iso = text[:-1] + "+00:00" if text.endswith(("Z", "z")) else text + try: + return datetime.fromisoformat(iso) + except ValueError: + pass + for fmt in _DATE_FORMATS: + try: + return datetime.strptime(text, fmt) + except ValueError: + continue + return None diff --git a/netx_api/topology_discover_common.py b/netx_api/topology_discover_common.py index 9558318..cb2adb0 100644 --- a/netx_api/topology_discover_common.py +++ b/netx_api/topology_discover_common.py @@ -4,9 +4,10 @@ from __future__ import annotations from typing import Any from fastapi import HTTPException -from sqlalchemy import or_ +from sqlalchemy import String, case, cast, func, or_ from sqlalchemy.orm import Session +from .table_filters import TableFilterParams, filter_query from .cli_resolve import get_default_profile, infer_device_type_vendor from .device_types import is_placeholder_ne_source from .models import ManagedNE, TopoDiscoverJob, TopoDiscoverJobItem, UmeInventoryNE @@ -42,6 +43,17 @@ def _managed_target_dict(ne: ManagedNE) -> dict[str, str]: } +def discover_item_expressions() -> dict[str, Any]: + item = TopoDiscoverJobItem + failed = or_(item.ok.is_(False), item.parser_stub.is_(True), item.error.in_(["parser_stub", "empty_cli_output", "vendor_or_device_type_required"])) + return { + "name": func.coalesce(func.nullif(item.ne_name, ""), item.ne_id), "ip": item.ne_ip, + "status": case((failed, "fail"), (item.unmatched_count > 0, "warn"), else_="ok"), + "neighbors": item.neighbors, "error": item.error, + "edges": cast(item.edges_added, String) + "/" + cast(item.edges_updated, String), + } + + def _job_out( db: Session, job: TopoDiscoverJob, @@ -51,6 +63,7 @@ def _job_out( page_size: int | None = None, item_status: str = "", item_keyword: str = "", + table_filters: TableFilterParams | None = None, ) -> FabricDiscoverJobOut: items_out: list[FabricDiscoverJobItemOut] = [] items_total = 0 @@ -93,6 +106,7 @@ def _job_out( TopoDiscoverJobItem.error.ilike(like), ) ) + q, _ = filter_query(q, discover_item_expressions(), table_filters) items_total = int(q.count()) if page is not None and page_size is not None: items_page = max(1, int(page or 1)) @@ -157,6 +171,7 @@ def get_discover_job( page_size: int | None = None, item_status: str = "", item_keyword: str = "", + table_filters: TableFilterParams | None = None, ) -> FabricDiscoverJobOut: job = db.get(TopoDiscoverJob, str(job_id or "").strip()) if job is None: @@ -168,6 +183,7 @@ def get_discover_job( page_size=page_size, item_status=item_status, item_keyword=item_keyword, + table_filters=table_filters, ) diff --git a/netx_api/topology_fabric_nodes.py b/netx_api/topology_fabric_nodes.py index a42f946..5164ff0 100644 --- a/netx_api/topology_fabric_nodes.py +++ b/netx_api/topology_fabric_nodes.py @@ -8,10 +8,11 @@ from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import and_, func, or_, text +from sqlalchemy import and_, case, func, or_, text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session +from .table_filters import TableFilterParams, filter_query from .cli_resolve import get_default_profile, infer_device_type_vendor from .device_types import LLDP_DISCOVERED_NE_SOURCE, WEBCRT_NE_SOURCE from .models import ( @@ -275,6 +276,7 @@ def list_fabric_nodes( region_folder_id: str = "", unmatched: str = "", link_status: str = "", + table_filters: TableFilterParams | None = None, page: int = 1, page_size: int = PAGE_DEFAULT, ) -> dict[str, Any]: @@ -375,6 +377,16 @@ def list_fabric_nodes( and_(TopoFabricNode.managed_ne_id.isnot(None), TopoFabricNode.managed_ne_id != ""), and_(TopoFabricNode.ume_ne_id.isnot(None), TopoFabricNode.ume_ne_id != ""), ) + has_managed = func.coalesce(TopoFabricNode.managed_ne_id, "") != "" + has_ume = func.coalesce(TopoFabricNode.ume_ne_id, "") != "" + q, options = filter_query(q, { + "name": func.coalesce(func.nullif(TopoFabricNode.name, ""), TopoFabricNode.id), + "ip": TopoFabricNode.ip, "level": TopoFabricNode.level, + "region": TopoFabricNode.region_folder_id, + "link": case((and_(has_managed, has_ume), "both"), (has_managed, "managed"), (has_ume, "ume"), else_="orphaned"), + }, table_filters) + if options is not None: + return options total = int(q.count()) rows = ( q.order_by(TopoFabricNode.name.asc()) @@ -399,6 +411,7 @@ def list_fabric_edges( status: str = "", source: str = "", keyword: str = "", + table_filters: TableFilterParams | None = None, page: int = 1, page_size: int = PAGE_DEFAULT, ) -> dict[str, Any]: @@ -438,6 +451,23 @@ def list_fabric_edges( TopoFabricEdge.b_node_id.in_(matched_ids), ) ) + from sqlalchemy.orm import aliased + a = aliased(TopoFabricNode) + b = aliased(TopoFabricNode) + q = q.outerjoin(a, TopoFabricEdge.a_node_id == a.id).outerjoin(b, TopoFabricEdge.b_node_id == b.id) + status_expr = case((TopoFabricEdge.status == "stale", "missing"), else_=TopoFabricEdge.status) + miss = func.coalesce(TopoFabricEdge.attrs["miss_count"].as_integer(), 0) + q, options = filter_query(q, { + "status": status_expr, + "aSide": func.coalesce(func.nullif(a.name, ""), func.nullif(a.ip, ""), func.substr(TopoFabricEdge.a_node_id, 1, 8)), + "bSide": func.coalesce(func.nullif(b.name, ""), func.nullif(b.ip, ""), func.substr(TopoFabricEdge.b_node_id, 1, 8)), + "aPort": TopoFabricEdge.a_port, "bPort": TopoFabricEdge.b_port, + "source": case((TopoFabricEdge.source == "stale", "lldp"), else_=func.coalesce(func.nullif(TopoFabricEdge.source, ""), "lldp")), + "lastSeen": TopoFabricEdge.last_seen_at, + "missCount": case((miss != 0, miss), (status_expr == "missing", 1), else_=0), + }, table_filters) + if options is not None: + return options total = int(q.count()) rows = ( q.order_by(TopoFabricEdge.updated_at.desc()) @@ -590,9 +620,13 @@ def find_fabric_paths( if len(edge_path) >= max_hops: continue _ensure_adj({node}) + # Parallel links (LAG members / dual fibers) between the same two NEs are one hop: + # expanding each would return near-identical paths that differ only by port. + seen_nbrs: set[str] = set() for nbr, eid in adj_cache.get(node, []): - if nbr in visited: + if nbr in visited or nbr in seen_nbrs: continue + seen_nbrs.add(nbr) explored += 1 new_path = edge_path + [eid] if nbr == to_id: @@ -620,10 +654,15 @@ def find_fabric_paths( "vendor": n.vendor or "", } + def _parallel_count(e: TopoFabricEdge) -> int: + # Only expanded nodes have adjacency loaded; the path's last node never is. + src, dst = (e.a_node_id, e.b_node_id) if e.a_node_id in adj_loaded else (e.b_node_id, e.a_node_id) + return sum(1 for nbr, _ in adj_cache.get(src, []) if nbr == dst) or 1 + def _edge_summary(e: TopoFabricEdge) -> dict[str, Any]: a_node = node_map.get(e.a_node_id) b_node = node_map.get(e.b_node_id) - return { + out = { "a_name": (a_node.name if a_node else "") or "", "b_name": (b_node.name if b_node else "") or "", "a_ip": (a_node.ip if a_node else "") or "", @@ -632,6 +671,10 @@ def find_fabric_paths( "b_port": e.b_port or "", "status": _normalize_edge_status(e.status or "active"), } + parallel = _parallel_count(e) + if parallel > 1: + out["parallel_links"] = parallel + return out def _path_node_ids(edge_ids: list[str]) -> list[str]: ids = [from_id] diff --git a/netx_api/topology_region_canvas.py b/netx_api/topology_region_canvas.py index 22f6be9..ba5f38c 100644 --- a/netx_api/topology_region_canvas.py +++ b/netx_api/topology_region_canvas.py @@ -121,6 +121,39 @@ def place_child_region_on_parent_canvas( return row +def add_missing_region_placement( + db: Session, view: TopoView, fabric_node_id: str, *, x: float, y: float +) -> TopoViewNode | None: + """Persist a child-region icon that was only synthesized (no placement row yet). + + Returns None unless ``fabric_node_id`` is a manual child region of the view's folder. + """ + if not is_region_canvas_node_id(fabric_node_id) or _is_world_flat(view): + return None + folder = db.get(TopoFolder, region_folder_id_from_node(fabric_node_id)) + if ( + folder is None + or str(folder.kind or "") != "region" + or str(folder.parent_id or "") != str(view.folder_id or "") + or _is_ume_managed_folder(folder) + ): + return None + now = _utcnow() + row = TopoViewNode( + id=uuid4().hex, + view_id=view.id, + fabric_node_id=fabric_node_id, + x=float(x), + y=float(y), + label="", + locked=False, + created_at=now, + updated_at=now, + ) + db.add(row) + return row + + def remove_region_canvas_placements(db: Session, folder_id: str) -> int: """Drop this folder's icon from any parent canvas.""" nid = region_canvas_node_id(folder_id) @@ -139,6 +172,7 @@ def region_node_out( y: float, locked: bool = False, label: str = "", + label_override: str | None = None, ) -> ViewNodeOut: child_view = _child_primary_view(db, folder.id) kids = ( @@ -154,12 +188,14 @@ def region_node_out( .filter(TopoViewNode.view_id == child_view.id) .count() ) - display = (label or folder.name or folder.id)[:256] + raw = str(label_override if label_override is not None else label or "") + display = (raw or folder.name or folder.id)[:256] return ViewNodeOut( fabric_node_id=region_canvas_node_id(folder.id), managed_ne_id="", ume_ne_id="", label=display, + label_override=raw[:256], x=float(x), y=float(y), locked=bool(locked), diff --git a/netx_api/topology_router.py b/netx_api/topology_router.py index b245cc7..ef79c06 100644 --- a/netx_api/topology_router.py +++ b/netx_api/topology_router.py @@ -7,6 +7,7 @@ from typing import Any from fastapi import APIRouter, Body, Depends, HTTPException, Query from sqlalchemy.orm import Session +from .table_filters import TableFilterParams, table_filter_params from .db import get_db from .topology_classify import ( apply_classify, @@ -38,6 +39,8 @@ from .topology_schemas import ( TopologyPlaceholderCreate, TopologyViewCreate, TopologyViewUpdate, + FabricEdgePortsPatch, + ViewCommitIn, ViewEdgeStylePatch, ViewNodesAdd, ViewNodesRemove, @@ -61,6 +64,7 @@ from .topology_fabric import ( from .topology_views import ( add_nodes_to_view, bootstrap_topology_tree, + commit_view_changes, create_folder, create_topology_placeholder_on_view, create_view, @@ -69,6 +73,7 @@ from .topology_views import ( get_topology_tree, get_view_graph, list_views, + patch_manual_edge_ports, patch_view_edge_style, patch_view_positions, populate_view, @@ -103,6 +108,7 @@ def api_fabric_nodes( ), page: int = Query(default=1, ge=1), page_size: int = Query(default=100, ge=1, le=2000), + table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: return list_fabric_nodes( @@ -114,6 +120,7 @@ def api_fabric_nodes( region_folder_id=region_folder_id, unmatched=unmatched, link_status=link_status, + table_filters=table_filters, page=page, page_size=page_size, ) @@ -128,6 +135,7 @@ def api_fabric_edges( keyword: str = "", page: int = Query(default=1, ge=1), page_size: int = Query(default=100, ge=1, le=2000), + table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: return list_fabric_edges( @@ -137,6 +145,7 @@ def api_fabric_edges( status=status, source=source, keyword=keyword, + table_filters=table_filters, page=page, page_size=page_size, ) @@ -481,6 +490,32 @@ def api_edge_style( return patch_view_edge_style(db, view_id, body).model_dump() +@router.post("/views/{view_id}/commit") +def api_commit_view( + view_id: str, body: ViewCommitIn, db: Session = Depends(get_db) +) -> dict[str, Any]: + return commit_view_changes(db, view_id, body).model_dump() + + +@router.patch("/fabric/edges/{edge_id}") +def api_patch_fabric_edge_ports( + edge_id: str, body: FabricEdgePortsPatch, db: Session = Depends(get_db) +) -> dict[str, Any]: + edge = patch_manual_edge_ports( + db, edge_id, a_port=body.a_port, b_port=body.b_port, commit=True + ) + return { + "id": edge.id, + "a_node_id": edge.a_node_id, + "b_node_id": edge.b_node_id, + "a_port": edge.a_port or "", + "b_port": edge.b_port or "", + "source": edge.source or "manual", + "status": edge.status or "active", + "layer": edge.layer or "physical", + } + + # --- Classify rules / slices / search --------------------------------------- diff --git a/netx_api/topology_schemas.py b/netx_api/topology_schemas.py index e42f6be..9ab3023 100644 --- a/netx_api/topology_schemas.py +++ b/netx_api/topology_schemas.py @@ -273,8 +273,10 @@ class ViewNodeIn(BaseModel): fabric_node_id: str = Field(min_length=1, max_length=64) x: float = 0.0 y: float = 0.0 - label: str = "" - locked: bool = False + # None = leave existing override unchanged; "" = clear override (follow device name). + label: str | None = None + # None = leave lock unchanged; False unlocks (and allows move); True locks after move. + locked: bool | None = None class ViewNodeOut(BaseModel): @@ -282,6 +284,8 @@ class ViewNodeOut(BaseModel): managed_ne_id: str = "" ume_ne_id: str = "" label: str = "" + # Raw placement override (empty = follow fabric name/ip). ``label`` is the display value. + label_override: str = "" x: float = 0.0 y: float = 0.0 locked: bool = False @@ -448,6 +452,16 @@ class ViewNodesRemove(BaseModel): class ViewEdgeStylePatch(BaseModel): + """Single-edge style patch (legacy). Prefer ``items`` for batch.""" + + fabric_edge_id: str = "" + stroke_color: str = "" + stroke_width: int = Field(default=0, ge=0, le=12) + line_style: str = Field(default="", max_length=16) + items: list["ViewEdgeStyleItem"] = Field(default_factory=list) + + +class ViewEdgeStyleItem(BaseModel): fabric_edge_id: str stroke_color: str = "" stroke_width: int = Field(default=0, ge=0, le=12) @@ -461,6 +475,11 @@ class FabricManualEdgeIn(BaseModel): b_port: str = "" +class FabricEdgePortsPatch(BaseModel): + a_port: str = "" + b_port: str = "" + + class FabricEdgesDeleteRequest(BaseModel): edge_ids: list[str] = Field(default_factory=list, min_length=1) @@ -469,6 +488,40 @@ class FabricEdgesDeleteOut(BaseModel): deleted: int = 0 +class ViewCommitEdgeCreate(BaseModel): + client_id: str = "" + a_node_id: str = Field(min_length=1, max_length=64) + b_node_id: str = Field(min_length=1, max_length=64) + a_port: str = "" + b_port: str = "" + + +class ViewCommitEdgePortEdit(BaseModel): + id: str = Field(min_length=1, max_length=64) + a_port: str = "" + b_port: str = "" + + +class ViewCommitIn(BaseModel): + """Atomic canvas save: all steps succeed or the whole request rolls back.""" + + edge_creates: list[ViewCommitEdgeCreate] = Field(default_factory=list) + edge_deletes: list[str] = Field(default_factory=list) + edge_port_edits: list[ViewCommitEdgePortEdit] = Field(default_factory=list) + edge_styles: list[ViewEdgeStyleItem] = Field(default_factory=list) + node_adds: list[str] = Field(default_factory=list) + node_removes: list[str] = Field(default_factory=list) + positions: list[ViewNodeIn] = Field(default_factory=list) + + +class ViewCommitOut(BaseModel): + ok: bool = True + view_id: str = "" + edge_id_map: dict[str, str] = Field(default_factory=dict) + skipped_locked: int = 0 + graph: TopologyViewGraphOut | None = None + + # --------------------------------------------------------------------------- # Classify rules + slices + search # --------------------------------------------------------------------------- diff --git a/netx_api/topology_service.py b/netx_api/topology_service.py index 431bc12..f65ac30 100644 --- a/netx_api/topology_service.py +++ b/netx_api/topology_service.py @@ -39,6 +39,7 @@ from .topology_views import ( _place_fabric_ids_on_view, add_nodes_to_view, bootstrap_topology_tree, + commit_view_changes, create_folder, create_topology_placeholder_on_view, create_view, @@ -48,6 +49,7 @@ from .topology_views import ( get_topology_tree, get_view_graph, list_views, + patch_manual_edge_ports, patch_view_edge_style, patch_view_positions, populate_view, @@ -68,6 +70,7 @@ __all__ = [ "_place_fabric_ids_on_view", "add_nodes_to_view", "bootstrap_topology_tree", + "commit_view_changes", "create_folder", "create_topology_placeholder_on_view", "create_view", @@ -88,6 +91,7 @@ __all__ = [ "list_fabric_nodes", "list_views", "merge_duplicate_fabric_nodes", + "patch_manual_edge_ports", "patch_view_edge_style", "patch_view_positions", "pause_discover_job", diff --git a/netx_api/topology_views.py b/netx_api/topology_views.py index 36d8bfb..a8af872 100644 --- a/netx_api/topology_views.py +++ b/netx_api/topology_views.py @@ -1,6 +1,7 @@ """Topology folder tree and leaf view operations (facade).""" from __future__ import annotations +from .topology_views_commit import commit_view_changes, patch_manual_edge_ports from .topology_views_graph import ( _place_fabric_ids_on_view, add_nodes_to_view, @@ -29,6 +30,7 @@ __all__ = [ "_place_fabric_ids_on_view", "add_nodes_to_view", "bootstrap_topology_tree", + "commit_view_changes", "create_folder", "create_topology_placeholder_on_view", "create_view", @@ -38,6 +40,7 @@ __all__ = [ "get_topology_tree", "get_view_graph", "list_views", + "patch_manual_edge_ports", "patch_view_edge_style", "patch_view_positions", "populate_view", diff --git a/netx_api/topology_views_commit.py b/netx_api/topology_views_commit.py new file mode 100644 index 0000000..dfd5f88 --- /dev/null +++ b/netx_api/topology_views_commit.py @@ -0,0 +1,444 @@ +"""Atomic view canvas commit (single transaction).""" + +from __future__ import annotations + +import re +from typing import Any +from uuid import uuid4 + +from fastapi import HTTPException +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from .models import ( + TopoFabricEdge, + TopoFabricNode, + TopoViewEdgeStyle, + TopoViewNode, +) +from .topology_common import VIEW_GRAPH_NODE_HARD_CAP, _utcnow +from .topology_fabric_links import upsert_fabric_edge +from .topology_fabric_nodes import _normalize_endpoints, refresh_fabric_stats +from .topology_region_canvas import add_missing_region_placement, is_region_canvas_node_id +from .topology_schemas import ( + TopologyViewGraphOut, + ViewCommitIn, + ViewCommitOut, + ViewEdgeStyleItem, + ViewNodeIn, +) +from .topology_views_graph import ( + _get_view_or_404, + _membership_for_view, + get_view_graph, +) +from .ume_topology_world import is_world_flat_view + +_HEX_COLOR_RE = re.compile(r"^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$") +_LINE_STYLES = {"", "solid", "dashed", "dotted"} + + +def _fail(step: str, detail: str, *, status: int = 400) -> None: + raise HTTPException( + status_code=status, + detail={"failed_step": step, "detail": detail}, + ) + + +def _apply_edge_creates( + db: Session, creates: list[Any], edge_id_map: dict[str, str] +) -> None: + for item in creates: + a = str(getattr(item, "a_node_id", "") or "").strip() + b = str(getattr(item, "b_node_id", "") or "").strip() + if not a or not b: + _fail("edge_creates", "edge_endpoints_required") + edge, action = upsert_fabric_edge( + db, + a_node_id=a, + b_node_id=b, + a_port=str(getattr(item, "a_port", "") or ""), + b_port=str(getattr(item, "b_port", "") or ""), + source="manual", + ) + if edge is None or action == "skipped_self_loop": + _fail("edge_creates", "edge_self_loop") + client_id = str(getattr(item, "client_id", "") or "").strip() + if client_id: + edge_id_map[client_id] = str(edge.id) + + +def _apply_edge_deletes(db: Session, edge_ids: list[str]) -> set[str]: + """Delete edges; return the ids that were requested (for skipping later edits).""" + ids = [str(x).strip() for x in (edge_ids or []) if str(x).strip()] + if not ids: + return set() + seen: set[str] = set() + ordered: list[str] = [] + for i in ids: + if i in seen: + continue + seen.add(i) + ordered.append(i) + rows = db.query(TopoFabricEdge).filter(TopoFabricEdge.id.in_(ordered)).all() + found_ids = [str(r.id) for r in rows] + if not found_ids: + return seen + db.query(TopoViewEdgeStyle).filter( + TopoViewEdgeStyle.fabric_edge_id.in_(found_ids) + ).delete(synchronize_session=False) + db.query(TopoFabricEdge).filter(TopoFabricEdge.id.in_(found_ids)).delete( + synchronize_session=False + ) + return seen + + +def patch_manual_edge_ports( + db: Session, + edge_id: str, + *, + a_port: str, + b_port: str, + commit: bool = True, +) -> TopoFabricEdge: + """Update ports on a manual fabric edge. Raises 400/404/409.""" + eid = str(edge_id or "").strip() + row = db.get(TopoFabricEdge, eid) if eid else None + if row is None: + raise HTTPException(status_code=404, detail="fabric_edge_not_found") + if str(row.source or "").strip().lower() != "manual": + raise HTTPException(status_code=400, detail="edge_not_manual") + _a, _b, ap, bp = _normalize_endpoints( + row.a_node_id, + row.b_node_id, + str(a_port or ""), + str(b_port or ""), + ) + conflict = ( + db.query(TopoFabricEdge) + .filter( + TopoFabricEdge.id != row.id, + TopoFabricEdge.layer == row.layer, + TopoFabricEdge.a_node_id == row.a_node_id, + TopoFabricEdge.b_node_id == row.b_node_id, + TopoFabricEdge.a_port == ap, + TopoFabricEdge.b_port == bp, + ) + .one_or_none() + ) + if conflict is not None: + raise HTTPException(status_code=409, detail="edge_port_conflict") + row.a_port = ap + row.b_port = bp + row.updated_at = _utcnow() + try: + db.flush() + except IntegrityError as exc: + raise HTTPException(status_code=409, detail="edge_port_conflict") from exc + if commit: + db.commit() + db.refresh(row) + return row + + +def _apply_edge_port_edits(db: Session, edits: list[Any], *, deleted: set[str]) -> None: + for item in edits: + eid = str(getattr(item, "id", "") or "").strip() + if not eid: + _fail("edge_port_edits", "edge_id_required") + if eid in deleted: + continue + try: + patch_manual_edge_ports( + db, + eid, + a_port=str(getattr(item, "a_port", "") or ""), + b_port=str(getattr(item, "b_port", "") or ""), + commit=False, + ) + except HTTPException as exc: + detail = exc.detail if isinstance(exc.detail, str) else "edge_port_edit_failed" + _fail("edge_port_edits", str(detail), status=int(exc.status_code or 400)) + + +def _upsert_edge_style( + db: Session, + view_id: str, + item: ViewEdgeStyleItem, + *, + now, +) -> None: + eid = str(item.fabric_edge_id or "").strip() + if not eid or db.get(TopoFabricEdge, eid) is None: + _fail("edge_styles", "fabric_edge_not_found", status=404) + color = str(item.stroke_color or "").strip() + if color and not _HEX_COLOR_RE.match(color): + _fail("edge_styles", "invalid_stroke_color") + width = int(item.stroke_width or 0) + if width < 0 or width > 12: + _fail("edge_styles", "invalid_stroke_width") + style = str(item.line_style or "").strip().lower() + if style not in _LINE_STYLES: + _fail("edge_styles", "invalid_line_style") + row = ( + db.query(TopoViewEdgeStyle) + .filter( + TopoViewEdgeStyle.view_id == view_id, + TopoViewEdgeStyle.fabric_edge_id == eid, + ) + .one_or_none() + ) + if row is None: + db.add( + TopoViewEdgeStyle( + id=uuid4().hex, + view_id=view_id, + fabric_edge_id=eid, + stroke_color=color, + stroke_width=width, + line_style=style, + created_at=now, + updated_at=now, + ) + ) + else: + row.stroke_color = color + row.stroke_width = width + row.line_style = style + row.updated_at = now + + +def _apply_edge_styles( + db: Session, + view_id: str, + styles: list[ViewEdgeStyleItem], + *, + edge_id_map: dict[str, str], + deleted: set[str], +) -> None: + now = _utcnow() + for item in styles: + eid = str(item.fabric_edge_id or "").strip() + if eid in deleted: + continue + # Styles on a just-created local edge are keyed by its client id. + if eid in edge_id_map: + item = item.model_copy(update={"fabric_edge_id": edge_id_map[eid]}) + _upsert_edge_style(db, view_id, item, now=now) + + +def _apply_node_adds( + db: Session, + view, + fabric_ids: list[str], + *, + max_nodes: int, + existing: dict[str, TopoViewNode], +) -> None: + if is_world_flat_view(view): + _fail("node_adds", "world_map_no_direct_nes") + ids = [str(x).strip() for x in (fabric_ids or []) if str(x).strip()] + if not ids: + return + if len(existing) >= max_nodes: + _fail("node_adds", "membership_max_nodes") + now = _utcnow() + room = max(0, max_nodes - len(existing)) + added = 0 + for fid in ids: + if fid in existing: + continue + if is_region_canvas_node_id(fid): + continue + if db.get(TopoFabricNode, fid) is None: + _fail("node_adds", f"fabric_node_not_found:{fid}") + if added >= room: + _fail("node_adds", "membership_max_nodes") + row = TopoViewNode( + id=uuid4().hex, + view_id=view.id, + fabric_node_id=fid, + x=40.0, + y=40.0, + label="", + locked=False, + created_at=now, + updated_at=now, + ) + db.add(row) + existing[fid] = row + added += 1 + if added: + view.updated_at = now + + +def _apply_node_removes( + db: Session, + view, + fabric_ids: list[str], + *, + existing: dict[str, TopoViewNode], +) -> None: + ids = [str(x).strip() for x in (fabric_ids or []) if str(x).strip()] + ids = [x for x in ids if x and not is_region_canvas_node_id(x)] + if not ids: + return + db.query(TopoViewNode).filter( + TopoViewNode.view_id == view.id, + TopoViewNode.fabric_node_id.in_(ids), + ).delete(synchronize_session=False) + for fid in ids: + existing.pop(fid, None) + view.updated_at = _utcnow() + + +def _apply_positions( + db: Session, + view, + positions: list[ViewNodeIn], + *, + existing: dict[str, TopoViewNode], +) -> int: + """Apply positions in-session. Returns skipped_locked count.""" + if len(positions) > VIEW_GRAPH_NODE_HARD_CAP: + _fail("positions", "too_many_positions") + now = _utcnow() + skipped_locked = 0 + updated = 0 + fabric_ids = [ + str(p.fabric_node_id or "").strip() + for p in positions + if str(p.fabric_node_id or "").strip() + and not is_region_canvas_node_id(str(p.fabric_node_id or "").strip()) + ] + alive = ( + {str(r[0]) for r in db.query(TopoFabricNode.id).filter(TopoFabricNode.id.in_(fabric_ids))} + if fabric_ids + else set() + ) + for p in positions: + fid = str(p.fabric_node_id or "").strip() + if not fid: + continue + if not is_region_canvas_node_id(fid) and fid not in alive and fid not in existing: + # Missing fabric: skip rather than fail mid-commit when node was just removed. + continue + row = existing.get(fid) + if row is None: + # Fabric NEs are placed via node_adds; only synthesized child-region icons are created here. + row = add_missing_region_placement(db, view, fid, x=p.x, y=p.y) + if row is None: + continue + existing[fid] = row + if row.locked and p.locked is not False: + skipped_locked += 1 + continue + row.x = float(p.x or 0) + row.y = float(p.y or 0) + if p.label is not None: + row.label = str(p.label or "")[:256] + if p.locked is not None: + row.locked = bool(p.locked) + row.updated_at = now + updated += 1 + if updated or skipped_locked: + view.updated_at = now + return skipped_locked + + +def commit_view_changes(db: Session, view_id: str, body: ViewCommitIn) -> ViewCommitOut: + """Apply canvas mutations atomically; return one graph + client→server edge map.""" + view = _get_view_or_404(db, view_id) + mem = _membership_for_view(view) + max_nodes = int(mem.get("max_nodes") or 300) + existing = { + vn.fabric_node_id: vn + for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all() + } + edge_id_map: dict[str, str] = {} + skipped_locked = 0 + + try: + _apply_edge_creates(db, list(body.edge_creates or []), edge_id_map) + deleted = _apply_edge_deletes(db, list(body.edge_deletes or [])) + _apply_edge_port_edits(db, list(body.edge_port_edits or []), deleted=deleted) + _apply_edge_styles( + db, + view.id, + list(body.edge_styles or []), + edge_id_map=edge_id_map, + deleted=deleted, + ) + _apply_node_adds( + db, + view, + list(body.node_adds or []), + max_nodes=max_nodes, + existing=existing, + ) + _apply_node_removes( + db, + view, + list(body.node_removes or []), + existing=existing, + ) + # Re-load placements after add/remove so positions see current rows. + # Flush first: sessions run with autoflush=False, so new placements would be invisible. + db.flush() + existing = { + vn.fabric_node_id: vn + for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all() + } + skipped_locked = _apply_positions( + db, + view, + list(body.positions or []), + existing=existing, + ) + db.commit() + except HTTPException: + db.rollback() + raise + except Exception as exc: # noqa: BLE001 + db.rollback() + _fail("commit", str(exc)[:500], status=500) + + try: + refresh_fabric_stats(db) + except Exception: # noqa: BLE001 + pass + + graph: TopologyViewGraphOut = get_view_graph(db, view.id) + return ViewCommitOut( + ok=True, + view_id=view.id, + edge_id_map=edge_id_map, + skipped_locked=skipped_locked, + graph=graph, + ) + + +def normalize_edge_style_items(body: Any) -> list[ViewEdgeStyleItem]: + """Accept legacy single-edge body or batch ``items``.""" + items = list(getattr(body, "items", None) or []) + if items: + return [ + ViewEdgeStyleItem( + fabric_edge_id=str(i.fabric_edge_id or ""), + stroke_color=str(i.stroke_color or ""), + stroke_width=int(i.stroke_width or 0), + line_style=str(i.line_style or ""), + ) + for i in items + ] + fid = str(getattr(body, "fabric_edge_id", "") or "").strip() + if not fid: + return [] + return [ + ViewEdgeStyleItem( + fabric_edge_id=fid, + stroke_color=str(getattr(body, "stroke_color", "") or ""), + stroke_width=int(getattr(body, "stroke_width", 0) or 0), + line_style=str(getattr(body, "line_style", "") or ""), + ) + ] diff --git a/netx_api/topology_views_graph.py b/netx_api/topology_views_graph.py index 41e692d..6409861 100644 --- a/netx_api/topology_views_graph.py +++ b/netx_api/topology_views_graph.py @@ -157,7 +157,12 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: # Save/mutate return_graph must use the same UME synthetic graph as GET. if is_ume_canvas_view(view): return get_ume_canvas_graph(db, view_id) - vnodes = db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all() + vnodes = ( + db.query(TopoViewNode) + .filter(TopoViewNode.view_id == view.id) + .order_by(TopoViewNode.created_at.asc(), TopoViewNode.id.asc()) + .all() + ) truncated = False reason = "" if len(vnodes) > VIEW_GRAPH_NODE_HARD_CAP: @@ -169,6 +174,19 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: n.id: n for n in db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(fids)).all() } if fids else {} managed_by_id, ume_by_id = _batch_ne_lookups(db, fabric_nodes) + region_folder_ids = [ + region_folder_id_from_node(vn.fabric_node_id) + for vn in vnodes + if is_region_canvas_node_id(vn.fabric_node_id) + ] + folders_by_id = { + f.id: f + for f in ( + db.query(TopoFolder).filter(TopoFolder.id.in_(region_folder_ids)).all() + if region_folder_ids + else [] + ) + } filt = dict(view.filter or {}) layer = str(filt.get("layer") or "physical").strip() or "physical" status = str(filt.get("status") or "").strip().lower() @@ -176,7 +194,7 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: region_ids_seen: set[str] = set() for vn in vnodes: if is_region_canvas_node_id(vn.fabric_node_id): - folder = db.get(TopoFolder, region_folder_id_from_node(vn.fabric_node_id)) + folder = folders_by_id.get(region_folder_id_from_node(vn.fabric_node_id)) if folder is None or str(folder.kind or "") != "region": continue region_ids_seen.add(folder.id) @@ -187,12 +205,13 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: x=float(vn.x or 0), y=float(vn.y or 0), locked=bool(vn.locked), - label=str(vn.label or folder.name or ""), + label_override=str(vn.label or ""), ) ) continue fn = fabric_nodes.get(vn.fabric_node_id) - label = (vn.label or "").strip() + label_override = str(vn.label or "") + label = label_override.strip() if not label and fn is not None: label = (fn.name or fn.ip or vn.fabric_node_id)[:256] connect_status = "" @@ -216,6 +235,7 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut: managed_ne_id=(fn.managed_ne_id if fn else "") or "", ume_ne_id=(fn.ume_ne_id if fn else "") or "", label=label, + label_override=label_override[:256], x=float(vn.x or 0), y=float(vn.y or 0), locked=bool(vn.locked), @@ -348,22 +368,31 @@ def _outside_peers_for_view( member_ids: set[str], layer: str, limit: int = 50, + edge_scan_limit: int = 5000, ) -> list[dict[str, str]]: if not member_ids: return [] + members = list(member_ids) + # Exclude edges whose both ends are already on the view so dense canvases + # do not burn the scan budget before any external neighbor is found. + both_internal = and_( + TopoFabricEdge.a_node_id.in_(members), + TopoFabricEdge.b_node_id.in_(members), + ) edges = ( db.query(TopoFabricEdge) .filter( TopoFabricEdge.layer == layer, or_( - TopoFabricEdge.a_node_id.in_(list(member_ids)), - TopoFabricEdge.b_node_id.in_(list(member_ids)), + TopoFabricEdge.a_node_id.in_(members), + TopoFabricEdge.b_node_id.in_(members), ), + ~both_internal, ) - .limit(5000) + .limit(max(1, int(edge_scan_limit or 5000))) .all() ) - out: list[dict[str, str]] = [] + peer_via: list[tuple[str, str]] = [] seen: set[str] = set() for e in edges: for peer_id, local_id in ((e.b_node_id, e.a_node_id), (e.a_node_id, e.b_node_id)): @@ -372,17 +401,29 @@ def _outside_peers_for_view( if peer_id in seen: continue seen.add(peer_id) - fn = db.get(TopoFabricNode, peer_id) - out.append( - { - "fabric_node_id": peer_id, - "name": (fn.name if fn else "") or "", - "ip": (fn.ip if fn else "") or "", - "via_node_id": local_id, - } - ) - if len(out) >= limit: - return out + peer_via.append((peer_id, local_id)) + if len(peer_via) >= limit: + break + if len(peer_via) >= limit: + break + if not peer_via: + return [] + peer_ids = [p for p, _ in peer_via] + peers = { + n.id: n + for n in db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(peer_ids)).all() + } + out: list[dict[str, str]] = [] + for peer_id, local_id in peer_via: + fn = peers.get(peer_id) + out.append( + { + "fabric_node_id": peer_id, + "name": (fn.name if fn else "") or "", + "ip": (fn.ip if fn else "") or "", + "via_node_id": local_id, + } + ) return out @@ -646,6 +687,7 @@ def patch_view_positions( layout = str(body.layout or "").strip().lower() updated = 0 skipped_locked = 0 + skipped_missing = 0 matched = 0 existing = { @@ -706,45 +748,54 @@ def patch_view_positions( row.updated_at = now updated += 1 else: + from .topology_region_canvas import add_missing_region_placement, is_region_canvas_node_id + positions = list(body.positions or []) if len(positions) > VIEW_GRAPH_NODE_HARD_CAP: raise HTTPException(status_code=400, detail="too_many_positions") matched = len(positions) + # One IN query instead of per-node db.get (N+1). + check_ids = [ + str(p.fabric_node_id or "").strip() + for p in positions + if str(p.fabric_node_id or "").strip() + and not is_region_canvas_node_id(str(p.fabric_node_id or "").strip()) + ] + fabric_alive: set[str] = set() + if check_ids: + fabric_alive = { + str(x[0]) + for x in db.query(TopoFabricNode.id) + .filter(TopoFabricNode.id.in_(check_ids)) + .all() + } for p in positions: fid = str(p.fabric_node_id or "").strip() if not fid: continue - from .topology_region_canvas import is_region_canvas_node_id - - if not is_region_canvas_node_id(fid) and db.get(TopoFabricNode, fid) is None: + if not is_region_canvas_node_id(fid) and fid not in fabric_alive: raise HTTPException(status_code=400, detail=f"fabric_node_not_found:{fid}") row = existing.get(fid) if row is None: - row = TopoViewNode( - id=uuid4().hex, - view_id=view.id, - fabric_node_id=fid, - x=float(p.x or 0), - y=float(p.y or 0), - label=str(p.label or "")[:256], - locked=bool(p.locked), - created_at=now, - updated_at=now, - ) - db.add(row) - existing[fid] = row - updated += 1 - else: - if row.locked and bool(p.locked): - skipped_locked += 1 + # Child-region icons may be synthesized without a placement row; persist those. + # Fabric NEs must not bypass membership caps / freeze (MCP adds via /nodes). + row = add_missing_region_placement(db, view, fid, x=p.x, y=p.y) + if row is None: + skipped_missing += 1 continue - row.x = float(p.x or 0) - row.y = float(p.y or 0) - if p.label is not None: - row.label = str(p.label or "")[:256] + existing[fid] = row + # Locked rows stay put unless the client explicitly unlocks (locked=False). + if row.locked and p.locked is not False: + skipped_locked += 1 + continue + row.x = float(p.x or 0) + row.y = float(p.y or 0) + if p.label is not None: + row.label = str(p.label or "")[:256] + if p.locked is not None: row.locked = bool(p.locked) - row.updated_at = now - updated += 1 + row.updated_at = now + updated += 1 view.updated_at = now db.commit() @@ -756,6 +807,7 @@ def patch_view_positions( matched=matched, updated=updated, skipped_locked=skipped_locked, + skipped_missing=skipped_missing, ) @@ -1353,42 +1405,48 @@ _LINE_STYLES = {"", "solid", "dashed", "dotted"} def patch_view_edge_style( db: Session, view_id: str, body: ViewEdgeStylePatch ) -> TopologyViewGraphOut: + from .topology_views_commit import normalize_edge_style_items + view = _get_view_or_404(db, view_id) - eid = str(body.fabric_edge_id or "").strip() - if not eid or db.get(TopoFabricEdge, eid) is None: - raise HTTPException(status_code=404, detail="fabric_edge_not_found") - color = str(body.stroke_color or "").strip() - if color and not _HEX_COLOR_RE.match(color): - raise HTTPException(status_code=400, detail="invalid_stroke_color") - width = int(body.stroke_width or 0) - if width < 0 or width > 12: - raise HTTPException(status_code=400, detail="invalid_stroke_width") - style = str(body.line_style or "").strip().lower() - if style not in _LINE_STYLES: - raise HTTPException(status_code=400, detail="invalid_line_style") + items = normalize_edge_style_items(body) + if not items: + raise HTTPException(status_code=400, detail="fabric_edge_id_required") now = _utcnow() - row = ( - db.query(TopoViewEdgeStyle) - .filter(TopoViewEdgeStyle.view_id == view.id, TopoViewEdgeStyle.fabric_edge_id == eid) - .one_or_none() - ) - if row is None: - row = TopoViewEdgeStyle( - id=uuid4().hex, - view_id=view.id, - fabric_edge_id=eid, - stroke_color=color, - stroke_width=width, - line_style=style, - created_at=now, - updated_at=now, + for item in items: + eid = str(item.fabric_edge_id or "").strip() + if not eid or db.get(TopoFabricEdge, eid) is None: + raise HTTPException(status_code=404, detail="fabric_edge_not_found") + color = str(item.stroke_color or "").strip() + if color and not _HEX_COLOR_RE.match(color): + raise HTTPException(status_code=400, detail="invalid_stroke_color") + width = int(item.stroke_width or 0) + if width < 0 or width > 12: + raise HTTPException(status_code=400, detail="invalid_stroke_width") + style = str(item.line_style or "").strip().lower() + if style not in _LINE_STYLES: + raise HTTPException(status_code=400, detail="invalid_line_style") + row = ( + db.query(TopoViewEdgeStyle) + .filter(TopoViewEdgeStyle.view_id == view.id, TopoViewEdgeStyle.fabric_edge_id == eid) + .one_or_none() ) - db.add(row) - else: - row.stroke_color = color - row.stroke_width = width - row.line_style = style - row.updated_at = now + if row is None: + row = TopoViewEdgeStyle( + id=uuid4().hex, + view_id=view.id, + fabric_edge_id=eid, + stroke_color=color, + stroke_width=width, + line_style=style, + created_at=now, + updated_at=now, + ) + db.add(row) + else: + row.stroke_color = color + row.stroke_width = width + row.line_style = style + row.updated_at = now view.updated_at = now db.commit() return get_view_graph(db, view.id) diff --git a/netx_api/ume_alarms_router.py b/netx_api/ume_alarms_router.py index 472d37b..8f94be5 100644 --- a/netx_api/ume_alarms_router.py +++ b/netx_api/ume_alarms_router.py @@ -10,6 +10,8 @@ from sqlalchemy import func, or_ from sqlalchemy.orm import Session from .config import settings +from .table_filters import TableFilterParams, filter_query +from .table_filters import table_filter_params from .db import get_db from .key_alert_config import ( get_key_alert_monitor_config, @@ -78,6 +80,7 @@ def ume_list_alarms( time_to: str | None = Query(default=None, description="Filter by last_seen_at <="), page: int = Query(default=1, ge=1), page_size: int = Query(default=50, ge=1, le=500), + table_filters: TableFilterParams = Depends(table_filter_params), db: Session = Depends(get_db), ) -> dict[str, Any]: stmt = db.query(UmeAlarmCurrent, UmeInventoryNE).outerjoin( @@ -113,6 +116,15 @@ def ume_list_alarms( stmt = stmt.filter(UmeAlarmCurrent.last_seen_at >= dt_from.replace(tzinfo=None)) if dt_to: stmt = stmt.filter(UmeAlarmCurrent.last_seen_at <= dt_to.replace(tzinfo=None)) + stmt, options = filter_query(stmt, { + "time": UmeAlarmCurrent.time_created, "severity": UmeAlarmCurrent.perceived_severity, + "neId": UmeAlarmCurrent.ne_id, + "hostName": func.coalesce(func.nullif(func.trim(UmeAlarmCurrent.host_name), ""), func.trim(UmeInventoryNE.host_name)), + "neType": UmeInventoryNE.ne_type, "cause": UmeAlarmCurrent.native_probable_cause, + "cleared": UmeAlarmCurrent.is_cleared, + }, table_filters) + if options is not None: + return options total = int(stmt.count()) rows = ( stmt.order_by( diff --git a/netx_api/webcrt_channel.py b/netx_api/webcrt_channel.py index 3194464..46ae67a 100644 --- a/netx_api/webcrt_channel.py +++ b/netx_api/webcrt_channel.py @@ -8,6 +8,7 @@ import queue import re import threading import time +from collections import deque from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -44,6 +45,18 @@ def uses_network_cli_keymap(device_type: str = "", vendor: str = "") -> bool: return True +_NETWORK_PROMPT_RE = re.compile( + r"^(?:<[^<>\s]{1,63}>" + r"|\[[~*]?[\w.\-/:+]{1,80}\]" + r"|[A-Za-z0-9][\w.\-/:+]{0,62}(?:\([^()\s]{1,60}\))*[#>])$" +) + + +def looks_like_network_prompt(line: str) -> bool: + """```` / ``[~r1]`` / ``R2#`` / ``R2(config)#`` — never a Linux ``user@host:~$`` prompt.""" + return bool(_NETWORK_PROMPT_RE.match(str(line or "").strip())) + + def map_network_cli_keys( data: str, *, @@ -56,6 +69,8 @@ def map_network_cli_keys( text = str(data or "") if not text: return text + if "\x1b" not in text and "\x7f" not in text: + return text # plain printable typing: nothing to rewrite out: list[str] = [] i = 0 n = len(text) @@ -86,13 +101,11 @@ def channel_return(conn: ConnectHandler | None) -> str: def map_network_cli_enter(data: str, conn: ConnectHandler | None) -> str: """Map xterm Enter (\\r) to the device's Netmiko RETURN.""" text = str(data or "") - if not text: + if not text or ("\r" not in text and "\n" not in text): return text ret = channel_return(conn) - if ret == "\r": - return text - # Prefer replacing CRLF first so Telnet RETURN \\r\\n does not double-expand. - return text.replace("\r\n", ret).replace("\r", ret) + # Replace once: chained replacements expand the CR in Telnet CRLF again. + return re.sub(r"\r\n|\r|\n", lambda _match: ret, text) def _drain_channel(conn: ConnectHandler, *, rounds: int = 6, wait: float = 0.06) -> str: @@ -369,10 +382,10 @@ def _encode_text(text: str, encoding: str) -> bytes: class _BoundedByteQueue: - """Thread-safe queue that drops oldest chunks when full (backpressure).""" + """Thread-safe chunk queue that drops the oldest chunks when full (backpressure).""" def __init__(self, maxsize: int = 2000) -> None: - self._q: queue.Queue[bytes | None] = queue.Queue() + self._dq: deque[bytes | None] = deque() self._max = max(8, int(maxsize or 2000)) self._cond = threading.Condition() self.dropped = 0 @@ -380,13 +393,12 @@ class _BoundedByteQueue: def put(self, item: bytes | None) -> None: with self._cond: - while self._q.qsize() >= self._max: - try: - self._q.get_nowait() + dq = self._dq + if item is not None: + while len(dq) >= self._max: + dq.popleft() self.dropped += 1 - except queue.Empty: - break - self._q.put(item) + dq.append(item) self._cond.notify() def put_nowait(self, item: bytes | None) -> None: @@ -394,22 +406,42 @@ class _BoundedByteQueue: def get_nowait(self) -> bytes | None: with self._cond: - return self._q.get_nowait() + if not self._dq: + raise queue.Empty + return self._dq.popleft() def get(self, timeout: float = 0.25) -> bytes | None: """Block until a chunk is available or timeout (raises queue.Empty).""" - deadline = time.time() + max(0.0, float(timeout)) + deadline = time.monotonic() + max(0.0, float(timeout)) with self._cond: - while self._q.empty(): - remaining = deadline - time.time() + while not self._dq: + remaining = deadline - time.monotonic() if remaining <= 0: raise queue.Empty self._cond.wait(timeout=remaining) - return self._q.get_nowait() + return self._dq.popleft() + + def drain(self, max_bytes: int = 262144) -> tuple[list[bytes], bool]: + """Pop every queued chunk (up to ~max_bytes). Returns ``(chunks, saw_eof)``.""" + out: list[bytes] = [] + total = 0 + with self._cond: + dq = self._dq + while dq: + item = dq[0] + if item is None: + dq.popleft() + return out, True + if out and total + len(item) > max_bytes: + break + dq.popleft() + out.append(item) + total += len(item) + return out, False def qsize(self) -> int: with self._cond: - return self._q.qsize() + return len(self._dq) def take_drop_delta(self) -> int: """Return newly dropped chunk count since last call (for client notice).""" @@ -1256,6 +1288,9 @@ def feed_command_line_buffer(buf: str, data: str, *, max_line: int = 512) -> tup completed: list[str] = [] limit = max(64, min(int(max_line or 512), 4096)) raw = str(data or "") + if raw.isprintable(): + # Plain typing (no Enter / BS / ESC): just append. + return (cur + raw)[:limit] if len(cur) < limit else cur, completed i = 0 while i < len(raw): ch = raw[i] diff --git a/netx_api/webcrt_cmd_audit.py b/netx_api/webcrt_cmd_audit.py new file mode 100644 index 0000000..3580ca7 --- /dev/null +++ b/netx_api/webcrt_cmd_audit.py @@ -0,0 +1,525 @@ +"""Server-side command audit for WebCRT: track the device's own input row. + +Why: inferring commands from browser keystrokes / xterm snapshots races with the +device echo (fast typing, Tab, history recall, slow VMs) and records fragments such +as ``R2#s`` or ``cddontainer/r/``. The device itself always knows what it executes: +when it echoes the line feed for our Enter, the *current input row* is exactly the +prompt + command it is about to run. + +``LineScreen`` is a tiny single-row VT100 model (CR/BS/CSI cursor + erase). +``CommandTracker`` owns one ``LineScreen`` per session and resolves each Enter we +send against the first line feed the device echoes back. Everything here is pure +(no I/O) and cheap: output before the last line feed is skipped unless an Enter +is waiting for its echo. +""" + +from __future__ import annotations + +import re +import threading +import time +from collections import deque +from dataclasses import dataclass, field + +# --------------------------------------------------------------------------- # +# Single-row terminal model +# --------------------------------------------------------------------------- # + +_TOKEN_RE = re.compile( + r"(\x1b\[[0-?]*[ -/]*[@-~]" # CSI + r"|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)" # OSC + r"|\x1b[()][0-9A-Za-z]" # charset + r"|\x1b[@-Z\\-_0-9=>]" # 2-byte ESC + r"|[\x00-\x1f\x7f])" # C0 + DEL +) +# Incomplete escape at the end of a chunk — carry into the next feed. +_PARTIAL_ESC_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*|\][^\x07\x1b]*\x1b?|[()])?\Z") +_MAX_CARRY = 96 + + +class LineScreen: + """Current input row of a terminal (no scrolling / wrapping).""" + + __slots__ = ("cells", "cur", "_carry") + + def __init__(self) -> None: + self.cells: list[str] = [] + self.cur = 0 + self._carry = "" + + def reset_row(self) -> None: + self.cells = [] + self.cur = 0 + + def text(self) -> str: + return "".join(self.cells).rstrip() + + def _put(self, run: str) -> None: + if not run: + return + n = len(run) + cells = self.cells + cur = self.cur + if cur == len(cells): + cells.extend(run) + else: + if cur > len(cells): + cells.extend(" " * (cur - len(cells))) + cells[cur : cur + n] = run + self.cur = cur + n + + def apply_token(self, tok: str) -> None: + """Apply one control token (not LF — the tracker owns line feeds).""" + c0 = tok[0] + if c0 == "\r": + self.cur = 0 + elif c0 == "\b": + if self.cur > 0: + self.cur -= 1 + elif c0 == "\t": + self._put(" " * (8 - (self.cur % 8))) + elif c0 == "\x1b": + if len(tok) >= 3 and tok[1] == "[": + self._csi(tok[2:-1], tok[-1]) + elif tok == "\x1bc": + self.reset_row() + # every other control byte (BEL, NUL, DEL, SO/SI …) is ignored + + def _csi(self, params: str, final: str) -> None: + if final not in "DCKP@XGHfJ": + return + if params and params[0] in "?>=": + return + try: + first = params.split(";", 1)[0] + n = int(first) if first else 0 + except ValueError: + n = 0 + cells = self.cells + if final == "D": + self.cur = max(0, self.cur - (n or 1)) + elif final == "C": + self.cur += n or 1 + elif final == "K": + if n == 0: + del cells[self.cur :] + elif n == 1: + for i in range(min(self.cur + 1, len(cells))): + cells[i] = " " + else: + self.reset_row() + elif final == "P": + del cells[self.cur : self.cur + (n or 1)] + elif final == "@": + if self.cur > len(cells): + cells.extend(" " * (self.cur - len(cells))) + cells[self.cur : self.cur] = [" "] * (n or 1) + elif final == "X": + for i in range(self.cur, min(self.cur + (n or 1), len(cells))): + cells[i] = " " + elif final == "G": + self.cur = max(0, (n or 1) - 1) + elif final in "Hf": + parts = params.split(";") + try: + col = int(parts[1]) if len(parts) > 1 and parts[1] else 1 + except ValueError: + col = 1 + self.cur = max(0, col - 1) + elif final == "J": + if n in (2, 3): + self.reset_row() + elif n == 0: + del cells[self.cur :] + + def feed_row(self, text: str) -> None: + """Apply text that contains no line feed.""" + if not text: + return + if not _has_ctl(text): + self._put(text) + return + parts = _TOKEN_RE.split(text) + for i, part in enumerate(parts): + if not part: + continue + if i & 1: + self.apply_token(part) + else: + self._put(part) + + def take_carry(self, text: str) -> str: + """Prepend a carried partial escape; stash a new trailing partial.""" + if self._carry: + text = self._carry + text + self._carry = "" + if text and "\x1b" in text[-_MAX_CARRY:]: + m = _PARTIAL_ESC_RE.search(text, max(0, len(text) - _MAX_CARRY)) + if m and m.group(0): + self._carry = m.group(0) + text = text[: m.start()] + return text + + +def _has_ctl(text: str) -> bool: + # str.isprintable is a fast C scan; only fall back to token split when needed. + return not text.isprintable() + + +# --------------------------------------------------------------------------- # +# Prompt splitting +# --------------------------------------------------------------------------- # + +_PROMPT_RES: tuple[re.Pattern[str], ...] = ( + # Huawei / H3C / ZTE user & system view: [r1] [~r1] [*r1-GigabitEthernet0/0/1] + re.compile(r"^<[^<>\s]{1,63}>"), + re.compile(r"^\[[~*]?[A-Za-z0-9][\w.\-/:+]{0,79}\]"), + # Cisco / Arista / ZTE / Ruijie: R2# R2> R2(config-if)# SW-1(config)# + re.compile(r"^[A-Za-z0-9][\w.\-/:+]{0,62}(?:\([^()\s]{1,60}\))*[#>]"), + # MikroTik: [admin@MikroTik] > [admin@MikroTik] /ip> + re.compile(r"^\[[\w.\-]+@[\w.\-]+\][^\n]{0,40}?>\s?"), +) +_LINUX_PROMPT_RES: tuple[re.Pattern[str], ...] = ( + # [user@host dir]$ user@host:~/dir$ (venv) user@host:~# + re.compile(r"^(?:\([^()]{1,40}\)\s*)?\[[^\]]{1,80}\]\s?[$#%>]\s?"), + re.compile(r"^(?:\([^()]{1,40}\)\s*)?[\w.\-]+@[\w.\-]+(?::|\s)[^\s$#%>]{0,120}\s?[$#%>]\s?"), + re.compile(r"^[^\s$#%>]{1,80}[$#%]\s"), +) +_NOT_PROMPT_RE = re.compile(r"^\[[YyNn]/[YyNn]\]") +_PASSWORD_ROW_RE = re.compile(r"(?:password|passwd|passphrase|密码)\s*[:：>]\s*\**\s*$", re.IGNORECASE) +_PAGER_ROW_RE = re.compile(r"-{2,}\s*more\s*-{2,}|--\s*more\s*--|\(END\)|lines?\s+\d+-\d+", re.IGNORECASE) + + +def split_prompt(row: str, hint: str = "", *, linux_like: bool = False) -> tuple[str, str] | None: + """Split ``row`` into ``(prompt, command)``; None when no CLI prompt is recognised. + + ``prompt`` keeps the whitespace before the command, so ``prompt + command`` is the + row exactly as the device displayed it. + """ + s = row.rstrip() + if not s or _NOT_PROMPT_RE.match(s) or _PAGER_ROW_RE.search(s[:80]): + return None + h = hint.rstrip() + if h and s.startswith(h): + return _cut(s, len(h)) + pats = _LINUX_PROMPT_RES + _PROMPT_RES if linux_like else _PROMPT_RES + _LINUX_PROMPT_RES + for pat in pats: + m = pat.match(s) + if m: + return _cut(s, m.end()) + return None + + +def _cut(s: str, end: int) -> tuple[str, str]: + rest = s[end:] + cmd = rest.lstrip() + return s[: len(s) - len(cmd)], cmd.rstrip() + + +def looks_like_password_row(row: str) -> bool: + return bool(_PASSWORD_ROW_RE.search(row.rstrip())) + + +# --------------------------------------------------------------------------- # +# Command tracker +# --------------------------------------------------------------------------- # + + +@dataclass +class AuditCmd: + command: str # prompt + command as the device displayed it + prompt: str = "" + input: str = "" # command only + redacted: bool = False + source: str = "stdin" + via: str = "device_echo" + + +@dataclass +class _Pending: + t: float + typed: str + snapshot: str # input row at the time we sent Enter + prompt_hint: str + client_hints: list[str] + source: str + # "bare": Enter with nothing typed in this payload (history recall / empty Enter) — + # consumes the first line feed whatever it shows. + bare: bool = False + # Tab can make the device redraw the line after a CRLF before the Enter echo; keep the + # latest prompt+command row seen and settle once the real output starts. + has_tab: bool = False + cand: tuple[str, str] | None = None + cand_t: float = 0.0 + + +_TAB_SETTLE_SEC = 1.2 + + +@dataclass +class CommandTracker: + """Per-session: device input row + FIFO of Enters waiting for their echo.""" + + linux_like: bool = False + expire_sec: float = 3.0 + max_command: int = 512 + screen: LineScreen = field(default_factory=LineScreen) + lock: threading.Lock = field(default_factory=threading.Lock) + pending: deque[_Pending] = field(default_factory=deque) + prompt_hint: str = "" + line_started: bool = False + password_mode: bool = False + # Rows abandoned with Ctrl-C: some devices (IOS, bash) echo that row + CRLF, and that + # line feed must not be mistaken for the echo of the next Enter. + abandoned: list[tuple[str, float]] = field(default_factory=list) + _typed: str = "" + + # ----- device output -------------------------------------------------- # + def feed(self, text: str) -> list[AuditCmd]: + """Feed decoded device output; returns commands resolved by line-feed echoes.""" + out: list[AuditCmd] = [] + with self.lock: + scr = self.screen + text = scr.take_carry(text) + if not text: + return out + while text: + nl = text.find("\n") + if nl < 0: + scr.feed_row(text) + break + self.line_started = False + if self.pending: + scr.feed_row(text[:nl]) + self._resolve_one(out) + scr.reset_row() + text = text[nl + 1 :] + continue + # Nothing waiting: only the last row matters — skip the bulk. + last = text.rfind("\n") + scr.reset_row() + text = text[last + 1 :] + scr.feed_row(text) + break + self.password_mode = looks_like_password_row(scr.text()) + if self.line_started and not scr.text(): + # Row wiped by the device (screen clear / new prompt): start over. + self.line_started = False + return out + + def _resolve_one(self, out: list[AuditCmd]) -> None: + row = self.screen.text() + p = self.pending[0] + if self.abandoned and not p.bare: + now = time.monotonic() + self.abandoned = [(r, t) for r, t in self.abandoned if now - t < 5.0] + typed = p.typed.replace("\t", "").strip() + for i, (ab, _t) in enumerate(self.abandoned): + # The device reprints the aborted row; ``startswith`` covers a slow echo + # (row snapshot was taken before every typed char came back). A VRP that + # keeps the text yields ``garbage`` + our command, which contains ``typed``. + if row == ab or (row.startswith(ab) and typed and typed not in row): + del self.abandoned[i] + return # echo of the aborted line, not our Enter + parsed = split_prompt(row, p.prompt_hint, linux_like=self.linux_like) + has_cmd = bool(parsed and parsed[1]) + if p.bare: + # A bare Enter (empty line, or executing a recalled history line) is echoed by + # the first *prompt* row's line feed; output rows of a command that is still + # printing must not consume it (Enter typed ahead during output). + if parsed is None: + return + self.pending.popleft() + if has_cmd: + out.append(self._make(parsed[0], parsed[1], p.source, "device_echo")) # type: ignore[index] + # Enter pressed at a pager left an orphan bare Enter in front of the next + # typed line: that line feed belongs to both — do not record it twice. + nxt = self.pending[0] if self.pending else None + if nxt is not None and not nxt.bare: + typed = nxt.typed.replace("\t", "").replace(" ", "").lower() + if typed and parsed[1].replace(" ", "").lower().startswith(typed[:3]): + self.pending.popleft() + return + if p.has_tab: + if has_cmd: + p.cand = (parsed[0], parsed[1]) # type: ignore[index] + p.cand_t = time.monotonic() + elif p.cand is not None: + # Prompt / output row after the echo: the last candidate is the executed line. + self._finish_candidate(p, out) + return + if not has_cmd: + # Prompt-only / non-CLI row: earlier output, a prompt reprint — not our echo. + return + self.pending.popleft() + out.append(self._make(parsed[0], parsed[1], p.source, "device_echo")) # type: ignore[index] + + def _finish_candidate(self, p: _Pending, out: list[AuditCmd]) -> None: + if self.pending and self.pending[0] is p: + self.pending.popleft() + if p.cand is not None: + out.append(self._make(p.cand[0], p.cand[1], p.source, "device_echo")) + p.cand = None + + def _make(self, prompt: str, cmd: str, source: str, via: str) -> AuditCmd: + cmd = cmd[: self.max_command] + base = prompt.rstrip() + # Keep one space after user@host style prompts; network CLIs print none. + sep = " " if (len(prompt) > len(base) and (self.linux_like or "@" in base)) else "" + return AuditCmd( + command=(base + sep + cmd)[: self.max_command], + prompt=base, + input=cmd, + source=source, + via=via, + ) + + def expire(self, now: float | None = None) -> list[AuditCmd]: + """Settle Tab candidates and resolve Enters whose echo never arrived.""" + out: list[AuditCmd] = [] + if not self.pending: + return out + t = time.monotonic() if now is None else now + with self.lock: + while self.pending: + p = self.pending[0] + if p.cand is not None and (t - p.cand_t) >= _TAB_SETTLE_SEC: + self._finish_candidate(p, out) + continue + if (t - p.t) >= self.expire_sec: + self.pending.popleft() + cmd = self._fallback(p) + if cmd is not None: + out.append(cmd) + continue + break + return out + + def _fallback(self, p: _Pending) -> AuditCmd | None: + parsed = split_prompt(p.snapshot, p.prompt_hint, linux_like=self.linux_like) + if parsed and parsed[1]: + return self._make(parsed[0], parsed[1], p.source, "snapshot") + for hint in reversed(p.client_hints): + parsed = split_prompt(hint, p.prompt_hint, linux_like=self.linux_like) + if parsed and parsed[1]: + return self._make(parsed[0], parsed[1], p.source, "client_hint") + # Last resort: what the operator typed, under the prompt that was on screen. + typed = p.typed.replace("\t", "").strip() + if typed and p.prompt_hint: + return self._make(p.prompt_hint, typed, p.source, "typed") + return None + + def flush_all(self) -> list[AuditCmd]: + """Session closing: resolve every still-pending Enter with the best fallback.""" + out: list[AuditCmd] = [] + with self.lock: + while self.pending: + p = self.pending.popleft() + if p.cand is not None: + out.append(self._make(p.cand[0], p.cand[1], p.source, "device_echo")) + continue + cmd = self._fallback(p) + if cmd is not None: + out.append(cmd) + return out + + # ----- operator input -------------------------------------------------- # + def on_stdin( + self, + lines: list[str], + *, + saw_input: bool, + had_enter: bool, + client_hints: list[str] | None = None, + source: str = "stdin", + now: float | None = None, + abort: bool = False, + abort_text: str = "", + ) -> list[AuditCmd]: + """Register keystrokes already split into completed ``lines`` by the caller. + + ``saw_input``: payload had anything besides bare Enter (captures the prompt). + ``had_enter``: payload contained at least one line terminator. + Returns immediately-final records (password redaction); echo-resolved ones + come from ``feed``. + """ + out: list[AuditCmd] = [] + t = time.monotonic() if now is None else now + with self.lock: + row = self.screen.text() + if abort: + parsed_row = split_prompt(row, self.prompt_hint, linux_like=self.linux_like) + if parsed_row and parsed_row[1]: + self.abandoned.append((row, t)) + # What we sent before the Ctrl-C may not have been echoed yet. + if abort_text.strip() and self.prompt_hint: + self.abandoned.append((self.prompt_hint + abort_text.strip(), t)) + del self.abandoned[:-6] + if saw_input and not self.line_started: + # First keystroke of a line: the row is still just the prompt. Prefer the + # regex-detected prompt so typed echo never leaks into the hint. + if row and not looks_like_password_row(row): + known = split_prompt(row, "", linux_like=self.linux_like) + if known: + self.prompt_hint = known[0] + elif row[-1] in "#>$%]": + self.prompt_hint = row # exotic CLI prompt + else: + # "Username:" / "Continue? [y/n]:" … a dialog, not a CLI prompt. + self.prompt_hint = "" + self.line_started = True + if not had_enter: + return out + hints = [h for h in (client_hints or []) if h] + if self.password_mode or looks_like_password_row(row): + # Typed secret: never keep it, record a redacted marker. + n = max(1, len(lines) or 1) + out.extend(AuditCmd(command="***", redacted=True, source=source, via="password") for _ in range(n)) + self.password_mode = False + self.line_started = False + return out + # No completed text: a bare Enter, possibly executing a recalled history line + # whose redraw has not reached us yet. + queue_lines = list(lines) or [""] + for i, typed in enumerate(queue_lines): + self.pending.append( + _Pending( + t=t, + typed=typed, + snapshot=row if i == 0 else "", + prompt_hint=self.prompt_hint, + client_hints=hints[-1:] if i == len(queue_lines) - 1 else [], + source=source, + bare=not typed, + has_tab="\t" in typed, + ) + ) + self.line_started = False + # Bound runaway growth (device never echoing). + while len(self.pending) > 64: + self.pending.popleft() + return out + + def immediate(self, lines: list[str], source: str) -> list[AuditCmd]: + """Trusted non-interactive input (post-login commands): record as typed.""" + return [ + AuditCmd(command=ln[: self.max_command], input=ln[: self.max_command], source=source, via="trusted") + for ln in lines + if ln.strip() + ] + + def current_row(self) -> str: + with self.lock: + return self.screen.text() + + def seed(self, text: str) -> None: + """Prime the row from the login transcript (prompt printed before the reader).""" + self.feed(text) + + def set_row(self, text: str) -> None: + """Force the current input row (e.g. after a duplicate prompt was dropped).""" + with self.lock: + self.screen.reset_row() + self.screen._carry = "" + self.screen.feed_row(text) + self.line_started = False diff --git a/netx_api/webcrt_router.py b/netx_api/webcrt_router.py index 74d02c8..54e6dd4 100644 --- a/netx_api/webcrt_router.py +++ b/netx_api/webcrt_router.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import codecs import json import logging import time @@ -208,9 +209,8 @@ def api_quick_connect( user_override = str(body.username or "").strip() or None if str(body.password or "").strip(): pwd_override = str(body.password) - # SSH with password: wait for auth so wrong credentials can re-prompt (SecureCRT-like). - wait_for_auth = proto == "ssh" and bool(pwd_override) - async_connect = bool(body.async_connect) and not wait_for_auth + # Open the terminal immediately; WS reports auth progress and retryable failures. + async_connect = bool(body.async_connect) try: session = create_session( db, @@ -687,6 +687,7 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: replay_text = _decode_bytes(raw_boot, "utf-8") sess.bootstrap_replayed = True else: + sess.flush_session_log() replay_text = read_session_log_tail(session_id, max_bytes=49152) if replay_text: replay_bytes = _encode_text(replay_text, "utf-8") @@ -700,59 +701,72 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: _log.debug("webcrt bootstrap/replay send failed session=%s", session_id, exc_info=True) stop = asyncio.Event() - stdin_buf: list[str] = [] - stdin_audit_lines: list[str] = [] - stdin_flush_task: asyncio.Task[None] | None = None + loop = asyncio.get_running_loop() + io_pool = webcrt_io_executor() + wake = asyncio.Event() + # Keystrokes are written as they arrive (no timer delay); whatever piled up while a + # write was in flight is coalesced into the next one. + stdin_q: asyncio.Queue[tuple[str, str | None]] = asyncio.Queue() - async def flush_stdin() -> None: - nonlocal stdin_buf, stdin_audit_lines - if not stdin_buf: - return - data = "".join(stdin_buf) - audit_lines = list(stdin_audit_lines) - stdin_buf = [] - stdin_audit_lines = [] + async def stdin_writer() -> None: + while True: + data, audit = await stdin_q.get() + parts = [data] + audits = [audit] if audit else [] + taken = 1 + while True: + try: + d2, a2 = stdin_q.get_nowait() + except asyncio.QueueEmpty: + break + parts.append(d2) + taken += 1 + if a2: + audits.append(a2) + blob = "".join(parts) + try: + await loop.run_in_executor( + io_pool, + lambda b=blob, al=audits: sess.write_stdin( + b, + audit_lines=al or None, + audit_line=al[-1] if al else None, + ), + ) + except Exception as exc: + try: + await websocket.send_json( + {"type": "status", "state": "error", "message": f"write_failed:{exc}"} + ) + except Exception: + pass + stop.set() + wake.set() + try: + await websocket.close(code=4500) + except Exception: + pass + finally: + for _ in range(taken): + stdin_q.task_done() + + async def drain_stdin() -> None: try: - await asyncio.get_running_loop().run_in_executor( - webcrt_io_executor(), - lambda: sess.write_stdin( - data, - audit_lines=audit_lines or None, - audit_line=audit_lines[-1] if audit_lines else None, - ), - ) - except Exception as exc: - await websocket.send_json( - {"type": "status", "state": "error", "message": f"write_failed:{exc}"} - ) - stop.set() - - async def schedule_stdin_flush() -> None: - await asyncio.sleep(0.008) - await flush_stdin() + await asyncio.wait_for(stdin_q.join(), timeout=5.0) + except Exception: + pass async def pump_stdout() -> None: - loop = asyncio.get_running_loop() - pending: list[bytes] = [] - last_flush = loop.time() + """Event-driven stdout pump: the reader thread wakes us, we drain and send at once.""" + sess.set_notifier(loop, wake) + dec = None + if _normalize_encoding(sess.encoding) != "utf-8": + dec = codecs.getincrementaldecoder("gbk")(errors="replace") - def _to_browser_bytes(raw: bytes) -> bytes: - if _normalize_encoding(sess.encoding) == "utf-8": + def _to_browser(raw: bytes) -> bytes: + if dec is None: return raw - return _decode_bytes(raw, sess.encoding).encode("utf-8", errors="replace") - - async def _flush_pending() -> bool: - nonlocal pending, last_flush - if not pending: - return True - blob = _to_browser_bytes(b"".join(pending)) - pending = [] - last_flush = loop.time() - try: - await websocket.send_bytes(blob) - return True - except Exception: - return False + return dec.decode(raw).encode("utf-8", errors="replace") async def _notify_queue_drops() -> None: try: @@ -773,48 +787,48 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: except Exception: pass - while not stop.is_set(): - # Longer block is cheap now (Condition wait); cuts executor churn when idle. - chunk = await loop.run_in_executor( - webcrt_io_executor(), lambda: sess.take_stdout(attach_gen, timeout=0.2) - ) - if chunk == "stale": - break - if chunk == "empty": - if pending and (loop.time() - last_flush) >= 0.016: - if not await _flush_pending(): + wake.set() # output queued before the pump started + try: + while not stop.is_set(): + await wake.wait() + wake.clear() + while not stop.is_set(): + status, data = sess.drain_stdout(attach_gen) + if status == "stale": + return + if data: + try: + await websocket.send_bytes(_to_browser(data)) + except Exception: + stop.set() + return + await _notify_queue_drops() + if status == "eof": + await _notify_queue_drops() stop.set() + try: + await websocket.send_json( + { + "type": "status", + "state": "closed", + "message": sess.close_reason or "device_closed", + } + ) + except Exception: + pass + return + if status != "data": break - await _notify_queue_drops() - continue - if chunk is None: - await _flush_pending() - await _notify_queue_drops() - stop.set() - try: - await websocket.send_json( - { - "type": "status", - "state": "closed", - "message": sess.close_reason or "device_closed", - } - ) - except Exception: - pass - break - pending.append(chunk) - if sum(len(p) for p in pending) >= 8192 or (loop.time() - last_flush) >= 0.016: - if not await _flush_pending(): - stop.set() - break - await _notify_queue_drops() + finally: + sess.clear_notifier(wake) reader_task = asyncio.create_task(pump_stdout()) + writer_task = asyncio.create_task(stdin_writer()) if sess.needs_live_prompt: sess.needs_live_prompt = False try: - await asyncio.get_running_loop().run_in_executor( - webcrt_io_executor(), + await loop.run_in_executor( + io_pool, lambda: sess.write_stdin("\r", audit_source="prompt_sync"), ) except Exception: @@ -830,9 +844,7 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: text = _decode_bytes(bytes(msg_raw["bytes"]), sess.encoding) except Exception: continue - stdin_buf.append(text) - if stdin_flush_task is None or stdin_flush_task.done(): - stdin_flush_task = asyncio.create_task(schedule_stdin_flush()) + stdin_q.put_nowait((text, None)) continue raw = msg_raw.get("text") if raw is None: @@ -847,23 +859,16 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: if data is None: continue audit_raw = msg.get("audit_line") - if audit_raw is not None and str(audit_raw).strip(): - stdin_audit_lines.append(str(audit_raw).strip()[:512]) - stdin_buf.append(str(data)) - # Coalesce high-frequency keystrokes briefly. - if len(stdin_buf) >= 8: - if stdin_flush_task and not stdin_flush_task.done(): - stdin_flush_task.cancel() - await flush_stdin() - elif stdin_flush_task is None or stdin_flush_task.done(): - stdin_flush_task = asyncio.create_task(schedule_stdin_flush()) + audit = str(audit_raw).strip()[:512] if audit_raw is not None and str(audit_raw).strip() else None + stdin_q.put_nowait((str(data), audit)) elif mtype == "resize": cols = int(msg.get("cols") or sess.cols) rows = int(msg.get("rows") or sess.rows) - await asyncio.get_running_loop().run_in_executor(webcrt_io_executor(), sess.resize, cols, rows) + await loop.run_in_executor(io_pool, sess.resize, cols, rows) elif mtype == "break": + await drain_stdin() try: - await asyncio.get_running_loop().run_in_executor(webcrt_io_executor(), sess.send_break) + await loop.run_in_executor(io_pool, sess.send_break) except Exception as exc: await websocket.send_json( {"type": "status", "state": "error", "message": f"break_failed:{exc}"} @@ -873,7 +878,7 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: await websocket.send_json({"type": "pong"}) elif mtype == "close": stop.set() - await flush_stdin() + await drain_stdin() close_session( session_id, reason="client_close", @@ -886,17 +891,18 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None: _log.exception("webcrt ws error session=%s", session_id) finally: stop.set() - if stdin_flush_task and not stdin_flush_task.done(): - stdin_flush_task.cancel() + wake.set() try: - await flush_stdin() + await drain_stdin() except Exception: pass + writer_task.cancel() reader_task.cancel() - try: - await reader_task - except Exception: - pass + for t in (writer_task, reader_task): + try: + await t + except BaseException: + pass # Keep device session so UI reconnect / remount can re-attach. if get_session(session_id) is not None: grace = float(getattr(settings, "webcrt_detach_grace_sec", 120) or 120) diff --git a/netx_api/webcrt_session_model.py b/netx_api/webcrt_session_model.py index 0879395..17b9c37 100644 --- a/netx_api/webcrt_session_model.py +++ b/netx_api/webcrt_session_model.py @@ -1,8 +1,12 @@ """WebCRT interactive session object (I/O, SFTP, close).""" from __future__ import annotations +import codecs import logging import queue +import re +import select +import socket import threading import time from dataclasses import dataclass, field @@ -16,8 +20,10 @@ from .ne_session_factory import ( extract_cli_prompt_marker, should_close_cli_hop_session, ) +from .webcrt_cmd_audit import AuditCmd, CommandTracker from .webcrt_channel import ( _BoundedByteQueue, + _normalize_encoding, _audit, _decode_bytes, _encode_text, @@ -31,6 +37,9 @@ from .webcrt_channel import ( _log = logging.getLogger("netx.webcrt") +# Operator typed an exit-style command (arms the hop prompt-return heuristic). +_HOP_EXIT_RE = re.compile(r"(?:^|[\r\n])\s*(?:quit|exit|logout|logoff|q|qu|qui|ex|exi)\s*[\r\n]", re.IGNORECASE) + @dataclass class WebcrtSession: session_id: str @@ -98,6 +107,17 @@ class WebcrtSession: _last_prompt_line: str = field(default="", repr=False) _last_audited_command: str = field(default="", repr=False) _last_audited_at: float = field(default=0.0, repr=False) + # Live audit: device-echo based command tracker (active while the reader runs). + _tracker: CommandTracker = field(default_factory=CommandTracker, repr=False) + # asyncio wake-up for the WS stdout pump: (loop, event) + "already scheduled" flag. + _notify: Any = field(default=None, repr=False) + _wake_flag: bool = field(default=False, repr=False) + _log_last_flush: float = field(default=0.0, repr=False) + _log_lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + _hop_exit_until: float = field(default=0.0, repr=False) + + def __post_init__(self) -> None: + self._tracker.linux_like = not bool(self.cli_keymap) def touch(self) -> None: self.last_activity = time.time() @@ -256,39 +276,55 @@ class WebcrtSession: self._log_fh = None def append_session_log(self, text: str) -> None: - if not text or self._log_fh is None: + """Buffered transcript write; flushed by the reader every ~250ms (and on attach).""" + fh = self._log_fh + if not text or fh is None: return try: - self._log_fh.write(text) - self._log_fh.flush() - max_bytes = int(getattr(settings, "webcrt_session_log_max_bytes", 0) or 0) - if max_bytes > 0: - try: - pos = int(self._log_fh.tell()) - except Exception: # noqa: BLE001 - pos = 0 - if pos >= max_bytes: - self._log_fh.write( - f"\n# rotated at {pos} bytes (cap={max_bytes}) ts={_utc_iso()}\n" - ) - self._log_fh.flush() - self._log_fh.close() - self._log_fh = None - # Re-open fresh file (append mode continues same path; rotate by rename). - try: - path = _session_log_path(self.session_id) - rotated = path.with_suffix(path.suffix + f".{int(pos)}.old") - if path.exists(): - path.replace(rotated) - except Exception: # noqa: BLE001 - _log.debug("webcrt session log rotate rename failed", exc_info=True) - self.open_session_log() + with self._log_lock: + fh.write(text) + if time.monotonic() - self._log_last_flush >= 0.25: + self.flush_session_log() except Exception: _log.warning("webcrt session log append failed session=%s", self.session_id, exc_info=True) - def close_session_log(self) -> None: + def flush_session_log(self) -> None: fh = self._log_fh - self._log_fh = None + if fh is None: + return + try: + with self._log_lock: + fh.flush() + self._log_last_flush = time.monotonic() + max_bytes = int(getattr(settings, "webcrt_session_log_max_bytes", 0) or 0) + if max_bytes <= 0: + return + try: + pos = int(fh.tell()) + except Exception: # noqa: BLE001 + pos = 0 + if pos < max_bytes: + return + fh.write(f"\n# rotated at {pos} bytes (cap={max_bytes}) ts={_utc_iso()}\n") + fh.flush() + fh.close() + self._log_fh = None + # Re-open fresh file (rotate by rename). + try: + path = _session_log_path(self.session_id) + rotated = path.with_suffix(path.suffix + f".{int(pos)}.old") + if path.exists(): + path.replace(rotated) + except Exception: # noqa: BLE001 + _log.debug("webcrt session log rotate rename failed", exc_info=True) + self.open_session_log() + except Exception: + _log.warning("webcrt session log flush failed session=%s", self.session_id, exc_info=True) + + def close_session_log(self) -> None: + with self._log_lock: + fh = self._log_fh + self._log_fh = None if fh is None: return try: @@ -326,6 +362,57 @@ class WebcrtSession: return "stale" return chunk # bytes | None + # ---- asyncio hand-off (no polling thread per WebSocket) -------------------- # + def set_notifier(self, loop: Any, event: Any) -> None: + with self._stdout_lock: + self._notify = (loop, event) + self._wake_flag = False + + def clear_notifier(self, event: Any) -> None: + with self._stdout_lock: + n = self._notify + if n is not None and n[1] is event: + self._notify = None + + def bump_attach(self) -> int: + """New WebSocket owns stdout: invalidate the previous pump and wake it.""" + with self._stdout_lock: + self.attach_gen += 1 + gen = self.attach_gen + self._wake(force=True) + return gen + + def _wake(self, *, force: bool = False) -> None: + """Called from any thread after queue changes; schedules at most one loop wake-up.""" + n = self._notify + if n is None: + return + if self._wake_flag and not force: + return + self._wake_flag = True + try: + n[0].call_soon_threadsafe(n[1].set) + except Exception: + self._wake_flag = False + self._notify = None + + def drain_stdout(self, attach_gen: int, max_bytes: int = 262144) -> tuple[str, bytes]: + """Non-blocking take for the WS pump. + + Returns ``(status, data)``: ``stale`` (newer WS owns stdout), ``eof`` (reader + ended; ``data`` may still hold the final bytes), ``data`` or ``empty``. + """ + self._wake_flag = False + with self._stdout_lock: + if attach_gen != self.attach_gen: + return "stale", b"" + chunks, eof = self.out_queue.drain(max_bytes) + data = b"".join(chunks) + if eof: + return "eof", data + return ("data" if data else "empty"), data + + # ---- stdin ----------------------------------------------------------------- # def write_stdin( self, data: str, @@ -351,24 +438,51 @@ class WebcrtSession: text = map_network_cli_enter(text, self.conn) if not text: return - self._note_stdin_for_audit( - text, - source=audit_source, - audit_line=audit_override, - audit_lines=audit_list, - ) + if self.cli_hop_guard and (_HOP_EXIT_RE.search(text) or "\x04" in text): + self._hop_exit_until = time.monotonic() + 8.0 + live_cmds: list[AuditCmd] = [] + if self._audit_live(): + # Register Enter first (cheap), send to the device, then publish: the audit + # row is resolved from the device echo, so nothing here delays the keystroke. + hints = list(audit_list or []) + if audit_override and (not hints or hints[-1] != audit_override): + hints.append(audit_override) + live_cmds = self._audit_stdin_live(text, source=str(audit_source or "stdin"), hints=hints) + else: + self._note_stdin_for_audit( + text, + source=audit_source, + audit_line=audit_override, + audit_lines=audit_list, + ) + self._send_raw(text) + if live_cmds: + self._emit_audit_cmds(live_cmds) + self.touch() + + def _send_raw(self, text: str) -> None: with self._write_lock: # Prefer raw channel I/O for interactive typing (char echo / backspace). - channel = getattr(self.conn, "remote_conn", None) + conn = self.conn + if conn is None: + raise RuntimeError("session_closed") + channel = getattr(conn, "remote_conn", None) try: if channel is not None and hasattr(channel, "send") and callable(channel.send): payload = _encode_text(text, self.encoding) - # Paramiko may write partially when the window is full. + # Paramiko may write partially when the window is full; the reader + # keeps a short channel timeout, so a timed-out send is simply retried. view = memoryview(payload) + deadline = time.monotonic() + 30.0 while len(view): - n = int(channel.send(view) or 0) + try: + n = int(channel.send(view) or 0) + except socket.timeout: + if time.monotonic() > deadline or self.closed: + raise + continue if n <= 0: - time.sleep(0.01) + time.sleep(0.005) continue view = view[n:] self.bytes_in += len(payload) @@ -377,12 +491,79 @@ class WebcrtSession: channel.write(payload) self.bytes_in += len(payload) else: - self.conn.write_channel(text) + conn.write_channel(text) self.bytes_in += len(text) except Exception: + if self.closed or self.conn is None: + raise self.conn.write_channel(text) self.bytes_in += len(text) - self.touch() + + # ---- command audit ------------------------------------------------------- # + def _audit_live(self) -> bool: + """Device-echo audit is active only while the reader thread feeds the tracker.""" + t = self._reader + return t is not None and t.is_alive() + + def _audit_stdin_live(self, text: str, *, source: str, hints: list[str]) -> list[AuditCmd]: + if source == "prompt_sync": + return [] + had_enter = "\r" in text or "\n" in text + abort_text = "" + with self._cmd_buf_lock: + cut = text.find("\x03") + if cut >= 0: + abort_text, _ = feed_command_line_buffer(self._cmd_buf, text[:cut]) + self._cmd_buf, buf_lines = feed_command_line_buffer(self._cmd_buf, text) + if source == "post_login": + return self._tracker.immediate(buf_lines, source) if had_enter else [] + saw_input = bool(text.strip("\r\n")) + return self._tracker.on_stdin( + buf_lines, + saw_input=saw_input, + had_enter=had_enter, + client_hints=hints, + source=source, + abort="\x03" in text, + abort_text=abort_text, + ) + + def _emit_audit_cmds(self, cmds: list[AuditCmd]) -> None: + for c in cmds: + if not str(c.command).strip(): + continue + try: + _audit( + "command", + session_id=self.session_id, + ne_id=self.ne_id, + ne_name=self.ne_name, + ne_ip=self.ne_ip, + protocol=self.protocol, + owner_user_id=self.owner_user_id, + owner_username=self.owner_username, + command="***" if c.redacted else str(c.command)[:512], + redacted=bool(c.redacted), + source=str(c.source or "stdin")[:32], + via=c.via, + prompt=str(c.prompt or "")[:128], + ) + except Exception: + _log.debug("webcrt command audit failed session=%s", self.session_id, exc_info=True) + + def seed_audit_row(self, transcript: str) -> None: + """Prime the input row from the login transcript (prompt printed before the reader).""" + try: + self._tracker.set_row("") + self._tracker.seed(str(transcript or "")[-4000:]) + except Exception: + _log.debug("webcrt audit seed failed session=%s", self.session_id, exc_info=True) + + def reset_audit_row(self, row: str) -> None: + try: + self._tracker.set_row(row) + except Exception: + pass def _note_stdin_for_audit( self, @@ -461,6 +642,26 @@ class WebcrtSession: _log.debug("webcrt command audit failed session=%s", self.session_id, exc_info=True) def _note_stdout_for_audit(self, text: str) -> None: + """Feed device output to the audit tracker (live) or the legacy tail buffer.""" + chunk = str(text or "") + if not chunk: + return + if self._audit_live(): + cmds = self._tracker.feed(chunk) + self._password_mode = self._tracker.password_mode + if cmds: + self._emit_audit_cmds(cmds) + return + self._note_stdout_legacy(chunk) + + def _audit_housekeeping(self) -> None: + if self._tracker.pending: + cmds = self._tracker.expire() + if cmds: + self._emit_audit_cmds(cmds) + self.flush_session_log() + + def _note_stdout_legacy(self, text: str) -> None: """Track device prompts so the next typed line can be redacted if it is a password.""" chunk = str(text or "") if not chunk: @@ -534,50 +735,74 @@ class WebcrtSession: ) self._reader.start() + @staticmethod + def _wait_readable(channel: Any, timeout: float) -> None: + """Block until a telnet-style socket is readable (or timeout).""" + sock = getattr(channel, "sock", None) + try: + if sock is not None and sock.fileno() >= 0: + select.select([sock], [], [], timeout) + return + except Exception: + pass + time.sleep(min(timeout, 0.01)) + def _reader_loop(self) -> None: conn = self.conn if conn is None: self.out_queue.put(None) + self._wake() return channel = getattr(conn, "remote_conn", None) hop_return = False poll = max(0.002, float(getattr(settings, "webcrt_reader_poll_sec", 0.01) or 0.01)) + # Incremental decode: a multi-byte char split across recv() chunks stays intact. + dec = codecs.getincrementaldecoder("gbk" if _normalize_encoding(self.encoding) == "gbk" else "utf-8")( + errors="replace" + ) + # Paramiko: park in recv() with a short timeout (wakes the moment data arrives) + # instead of spin-polling recv_ready(). + blocking = False + if ( + channel is not None + and hasattr(channel, "recv") + and hasattr(channel, "recv_ready") + and hasattr(channel, "settimeout") + ): + try: + channel.settimeout(0.2) + blocking = True + except Exception: + blocking = False + last_house = time.monotonic() try: while not self.closed: chunk = b"" try: - if channel is not None and hasattr(channel, "recv_ready") and hasattr(channel, "recv"): - # Paramiko SSH: prefer short blocking recv over fixed spin-sleep. - ready = False + if blocking: try: - ready = bool(channel.recv_ready()) - except Exception: - ready = False - if ready: - chunk = channel.recv(16384) + chunk = channel.recv(65536) + except socket.timeout: + chunk = b"" + else: + if not chunk: + break # EOF + if not chunk: + if channel.recv_ready(): + continue + if getattr(channel, "closed", False) or ( + hasattr(channel, "exit_status_ready") and channel.exit_status_ready() + ): + break + elif channel is not None and hasattr(channel, "recv_ready") and hasattr(channel, "recv"): + if channel.recv_ready(): + chunk = channel.recv(65536) if not chunk: break elif hasattr(channel, "exit_status_ready") and channel.exit_status_ready(): break else: - # Brief block: settimeout + recv wakes sooner than sleep(0.04). - prev_timeout = None - try: - prev_timeout = channel.gettimeout() - except Exception: - prev_timeout = None - try: - channel.settimeout(poll) - chunk = channel.recv(16384) - except Exception: - chunk = b"" - finally: - try: - channel.settimeout(prev_timeout) - except Exception: - pass - if not chunk: - continue + time.sleep(poll) elif channel is not None and hasattr(channel, "read_very_eager"): # Telnet: do NOT use conn.read_channel() — Netmiko strips ANSI. data = channel.read_very_eager() @@ -588,39 +813,52 @@ class WebcrtSession: else _encode_text(str(data), self.encoding) ) else: - time.sleep(poll) - continue + self._wait_readable(channel, 0.2) else: text = conn.read_channel() if text: chunk = _encode_text(str(text), self.encoding) else: time.sleep(poll) - continue + except EOFError: + break except Exception as exc: if self.closed: break _log.debug("webcrt reader error session=%s: %s", self.session_id, exc) + if channel is not None and getattr(channel, "closed", False): + break time.sleep(0.05) continue + now = time.monotonic() if chunk: - self.touch() + self.last_activity = time.time() self.bytes_out += len(chunk) self.out_queue.put(chunk) + self._wake() + decoded = "" try: - decoded = _decode_bytes(chunk, self.encoding) - self.append_session_log(decoded) - self._note_stdout_for_audit(decoded) + decoded = dec.decode(bytes(chunk)) + if decoded: + self.append_session_log(decoded) + self._note_stdout_for_audit(decoded) except Exception: - pass - if self.cli_hop_guard and self._note_cli_hop_output(chunk): + _log.debug("webcrt reader side-effects failed session=%s", self.session_id, exc_info=True) + if self.cli_hop_guard and self._note_cli_hop_output(chunk, decoded): hop_return = True notice = ( "\r\n*** WebCRT: 目标会话已结束，已断开代理连接 " "(target session ended; closing hop proxy) ***\r\n" ) self.out_queue.put(notice.encode("utf-8", errors="replace")) + self._wake() break + if now - last_house >= 0.25: + last_house = now + try: + self._audit_housekeeping() + except Exception: + _log.debug("webcrt housekeeping failed session=%s", self.session_id, exc_info=True) finally: if hop_return and not self.closed: try: @@ -630,22 +868,33 @@ class WebcrtSession: except Exception: self.close("cli_hop_return") self.out_queue.put(None) + self._wake() + try: + self.flush_session_log() + except Exception: + pass - def _note_cli_hop_output(self, chunk: bytes) -> bool: + def _note_cli_hop_output(self, chunk: bytes, text: str | None = None) -> bool: """Accumulate stdout and return True when nested CLI hop has returned to proxy.""" - try: - text = _decode_bytes(chunk, self.encoding) - except Exception: - text = str(chunk) + if text is None: + try: + text = _decode_bytes(chunk, self.encoding) + except Exception: + text = str(chunk) self._hop_scan_buf = (self._hop_scan_buf + text)[-12000:] marker = str(self.cli_hop_prompt or "").strip() last = extract_cli_prompt_marker(self._hop_scan_buf) if last and (not marker or last != marker): self._cli_hop_seen_other_prompt = True + # Prompt-only "back on the hop" is ambiguous when hop and target share a hostname + # (target ``system-view`` prints the same ``[~r1]`` as the hop). Only trust it + # shortly after the operator typed an exit command; the nested "connection closed" + # message needs no such gate. + armed = self._cli_hop_seen_other_prompt and time.monotonic() <= self._hop_exit_until return should_close_cli_hop_session( self._hop_scan_buf, self.cli_hop_prompt, - seen_other_prompt=self._cli_hop_seen_other_prompt, + seen_other_prompt=armed, ) def run_post_login_commands(self) -> None: @@ -660,6 +909,19 @@ class WebcrtSession: _log.debug("post_login command failed session=%s", self.session_id, exc_info=True) break + def _teardown_conn(self, conn: Any) -> None: + self.close_sftp() + if conn is None: + return + for attr in ("remote_conn", "remote_conn_pre", "_netx_jump_client"): + obj = getattr(conn, attr, None) + if obj is None: + continue + try: + obj.close() + except Exception: + pass + def close(self, reason: str = "closed") -> None: if self.closed: return @@ -667,14 +929,27 @@ class WebcrtSession: self.state = "closed" self.close_reason = reason or "closed" self._ready_event.set() - self.close_sftp() + # Enters still waiting for a device echo would be lost with the session. try: - close_netmiko_connection(self.conn) + pending_cmds = self._tracker.flush_all() + if pending_cmds: + self._emit_audit_cmds(pending_cmds) except Exception: - pass + _log.debug("webcrt audit flush on close failed session=%s", self.session_id, exc_info=True) + conn = self.conn self.conn = None + # Tear the transport down off-thread: Netmiko's disconnect() runs vendor cleanup + # (prompt probing, several seconds on Huawei) which would stall the event loop / + # reaper, and keeps the device VTY occupied longer than needed. + threading.Thread( + target=self._teardown_conn, + args=(conn,), + name=f"webcrt-close-{self.session_id[:8]}", + daemon=True, + ).start() try: self.out_queue.put_nowait(None) except Exception: pass + self._wake(force=True) self.close_session_log() diff --git a/netx_api/webcrt_session_registry.py b/netx_api/webcrt_session_registry.py index cc38cb0..9794649 100644 --- a/netx_api/webcrt_session_registry.py +++ b/netx_api/webcrt_session_registry.py @@ -34,6 +34,7 @@ from .webcrt_channel import ( _normalize_encoding, _prime_interactive_channel, _session_log_text, + looks_like_network_prompt, prepare_bootstrap_output, uses_network_cli_keymap, ) @@ -284,14 +285,16 @@ def _finish_connect( # Pull post-auth banner/MOTD from the PTY. With interactive no-op session_preparation # (generic_termserver), Netmiko session_log is often empty — do not discard these bytes. # Keep this short: live echo already streamed login; long settle blocks WS stdin. + # Login already ended on a CLI prompt: only a short look for trailing MOTD bytes. + prompted_early = _looks_like_cli_prompt(pre_log) try: - early = _capture_raw_channel(conn, duration=0.2) + early = _capture_raw_channel(conn, duration=0.06 if prompted_early else 0.2) except Exception: early = "" if early: sess.push_connect_echo(early) seed = f"{pre_log}{early}" - already_prompted = _looks_like_cli_prompt(seed) or _looks_like_cli_prompt(pre_log) + already_prompted = _looks_like_cli_prompt(seed) or prompted_early saw_password_change = bool( re.search( r"(?i)(change\s*now|please\s*choose|password\s+needs\s+to\s+be\s+changed).{0,80}:", @@ -356,7 +359,7 @@ def _finish_connect( primed = "" combined = f"{seed}{primed}" # Brief settle only — do not burn seconds here while the terminal shows a prompt. - settle_sec = 0.12 if already_prompted or _looks_like_cli_prompt(combined) else 0.3 + settle_sec = 0.05 if already_prompted or _looks_like_cli_prompt(combined) else 0.3 try: more = _capture_raw_channel(conn, duration=settle_sec) if more: @@ -395,7 +398,7 @@ def _finish_connect( if not echoed.strip(): bootstrap = prepare_bootstrap_output(combined) try: - leftover = _capture_raw_channel(conn, duration=0.1) + leftover = _capture_raw_channel(conn, duration=0.05) except Exception: leftover = "" if ( @@ -407,7 +410,7 @@ def _finish_connect( bootstrap = prepare_bootstrap_output(f"{bootstrap}{leftover}") else: try: - leftover = _capture_raw_channel(conn, duration=0.08) + leftover = _capture_raw_channel(conn, duration=0.04) except Exception: leftover = "" if ( @@ -421,6 +424,19 @@ def _finish_connect( sess.conn = conn sess.cli_hop_guard = bool(hop_guard) sess.cli_hop_prompt = str((hop_guard or {}).get("hop_prompt") or "") + # Inventory type "generic" (e.g. a quick-connect to a Cisco box) disables the network + # key map, so Home/Delete reach IOS/VRP as raw escape sequences. The login prompt tells + # us what it really is. + if not sess.cli_keymap and str(sess.device_type or "").strip().lower() in ( + "", + "generic", + "generic_termserver", + "generic_telnet", + ): + last_line = str(echoed or combined or "").replace("\r\n", "\n").replace("\r", "\n").rstrip(" \n").split("\n")[-1] + if looks_like_network_prompt(prompt_hint or last_line) or looks_like_network_prompt(last_line): + sess.cli_keymap = True + sess._tracker.linux_like = False sess.bootstrap_output = _encode_text(str(bootstrap or ""), sess.encoding) # Nudge Enter on WS attach only when we still need a shell prompt. # Never when already at CLI prompt or Username:/Password: (would empty-submit login). @@ -432,6 +448,8 @@ def _finish_connect( and not _looks_like_login_prompt(final_view) ) sess.open_session_log() + # Prime the audit tracker with the prompt printed during login (reader starts later). + sess.seed_audit_row(echoed or bootstrap or combined) log_seed = echoed or bootstrap if log_seed: sess.append_session_log(log_seed if str(log_seed).endswith("\n") else str(log_seed) + "\n") @@ -439,7 +457,7 @@ def _finish_connect( # Drop late prompt echoes that race into the queue right after reader start. if not prompt_hint and final_view: prompt_hint = str(final_view).replace("\r\n", "\n").replace("\r", "\n").strip().split("\n")[-1].strip() - settle_deadline = time.time() + (0.2 if already_prompted or saw_password_change else 0.35) + settle_deadline = time.time() + (0.08 if already_prompted or saw_password_change else 0.35) while time.time() < settle_deadline: try: chunk = sess.out_queue.get_nowait() @@ -453,8 +471,14 @@ def _finish_connect( text = _decode_bytes(chunk, sess.encoding) except Exception: text = "" - if _is_prompt_only_echo(text, prompt_hint): + # A slow VM may emit its FIRST prompt after the reader starts. + # Only drop duplicates when a prompt was already delivered during login. + if _looks_like_cli_prompt(final_view) and _is_prompt_only_echo(text, prompt_hint): + # Dropped from the browser stream: keep the audit row in step with it. + sess.reset_audit_row(prompt_hint) continue + if _looks_like_cli_prompt(text): + sess.needs_live_prompt = False # Non-prompt data: put back and stop settling. sess.out_queue.put(chunk) break @@ -692,8 +716,7 @@ def mark_attached(session_id: str) -> tuple[WebcrtSession, int]: raise HTTPException(status_code=404, detail="webcrt_session_not_found") # Allow re-attach after brief WS drop (React StrictMode remount / network blip). # Bump generation so the previous WS pump stops and does not steal echo bytes. - sess.attach_gen += 1 - attach_gen = sess.attach_gen + attach_gen = sess.bump_attach() sess.attached = True sess.detach_deadline = None sess.touch() diff --git a/packages/netx-mcp/README.md b/packages/netx-mcp/README.md index 72ce048..009c1ec 100644 --- a/packages/netx-mcp/README.md +++ b/packages/netx-mcp/README.md @@ -38,7 +38,7 @@ python -m netx_mcp [`mcp.json`](./mcp.json) — `command: python`，`args: ["-m", "netx_mcp"]`，`env` 见文件。 -## 工具（14）— skill 组 **ops**（`netx-ops`） +## 工具（15）— skill 组 **ops**（`netx-ops`） **NMS**（模型面通用名；当前适配器 zte-ume，REST 仍 `/v1/ume/*`）： `queryNmsAlarms`, `aggregateNmsAlarms`, `runNmsDiagnostics`, `queryNmsNeInventory`, `getNmsNe`, `queryNmsAlarmsRaw`, `aggregateNmsAlarmsRaw`, `listNmsAlarmFields`, `sqlQueryNms` @@ -46,6 +46,8 @@ python -m netx_mcp **Managed CLI + paths**（问「能否登录」必须走这里，不要只查 inventory）： `listManagedNe`, `getManagedNe`, `execManagedNe`, `getNeExecJob`, `listCliTargets`, `findTopologyPaths` +0.3.1：`getNmsNe` 可直接传 host_name/IP；输出统一 `nms_ne_id` / `source=nms`；参数类型错误返回工具错误而非协议崩溃。 + 参数优先 `nms_ne_id` / `nms_ne_ids`（保留 `ume_*` 别名）。 拓扑画布 / Fabric → skill **topology** / [`netx-topology-mcp`](../netx-topology-mcp)（见 [docs/MCP_TOPOLOGY.md](../../docs/MCP_TOPOLOGY.md)）。 diff --git a/packages/netx-mcp/pyproject.toml b/packages/netx-mcp/pyproject.toml index 56d3e79..090358e 100644 --- a/packages/netx-mcp/pyproject.toml +++ b/packages/netx-mcp/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netx-mcp" -version = "0.3.0" +version = "0.3.1" description = "stdio MCP server for netx REST API (NMS alarms, managed NE CLI)" readme = "README.md" requires-python = ">=3.11" diff --git a/packages/netx-mcp/src/netx_mcp/http_client.py b/packages/netx-mcp/src/netx_mcp/http_client.py index 60a153c..3a6914e 100644 --- a/packages/netx-mcp/src/netx_mcp/http_client.py +++ b/packages/netx-mcp/src/netx_mcp/http_client.py @@ -117,7 +117,9 @@ def mcp_text_result(payload: Any, *, is_error: bool = False) -> dict[str, Any]: def mcp_from_handler_result(result: dict[str, Any]) -> dict[str, Any]: - if not result.get("ok"): + # API envelopes may nest a failure: {"ok": true, "data": {"ok": false, ...}} (e.g. job_not_found). + data = result.get("data") + if not result.get("ok") or (isinstance(data, dict) and data.get("ok") is False): return mcp_text_result(result, is_error=True) return mcp_text_result(result) diff --git a/packages/netx-mcp/src/netx_mcp/http_tools.py b/packages/netx-mcp/src/netx_mcp/http_tools.py index 65e3bcd..e604cbf 100644 --- a/packages/netx-mcp/src/netx_mcp/http_tools.py +++ b/packages/netx-mcp/src/netx_mcp/http_tools.py @@ -75,6 +75,80 @@ def _should_run_exec_async(args: dict[str, Any]) -> bool: return _count_exec_ne_targets(args) >= min_n +class ToolArgError(ValueError): + """Bad tool argument; surfaced to the model as a tool error (not a JSON-RPC crash).""" + + +def _int_arg(args: dict[str, Any], key: str, default: int, lo: int, hi: int) -> int: + raw = args.get(key) + if raw is None or (isinstance(raw, str) and not raw.strip()): + return default + try: + val = int(raw) + except (TypeError, ValueError): + raise ToolArgError(f"{key} must be an integer, got {raw!r}") from None + return max(lo, min(hi, val)) + + +def _severity(args: dict[str, Any]) -> str: + # Stored values are lowercase (critical/major/minor/warning); exact match server-side. + return str(args.get("severity") or "").strip().lower() + + +def _is_me_placeholder(ne_name: Any, ne_id: Any) -> bool: + return bool(ne_id) and str(ne_name or "") == f"ME{{{ne_id}}}" + + +def _compact_rows(out: dict[str, Any], *, name_key: str, id_key: str) -> dict[str, Any]: + """Drop ZTE ``ME{}`` ne_name placeholders: they repeat ne_id and carry no meaning.""" + data = out.get("data") if isinstance(out, dict) else None + items = data.get("items") if isinstance(data, dict) else None + if isinstance(items, list): + for row in items: + if isinstance(row, dict) and _is_me_placeholder(row.get(name_key), row.get(id_key)): + row.pop(name_key, None) + return out + + +def _strip_meta_field_lists(out: dict[str, Any]) -> dict[str, Any]: + """Raw endpoints echo ~40 field names per call; listNmsAlarmFields already serves them.""" + data = out.get("data") if isinstance(out, dict) else None + meta = data.get("meta") if isinstance(data, dict) else None + if isinstance(meta, dict): + for k in ("available_fields", "group_by_allowed", "order_by_allowed"): + meta.pop(k, None) + return out + + +def _nms_naming(row: dict[str, Any]) -> None: + """REST still speaks ume_*; model-facing params are nms_* -- keep output consistent.""" + if "ume_ne_id" in row: + row["nms_ne_id"] = row.pop("ume_ne_id") + if row.get("source") == "ume": + row["source"] = "nms" + + +_HOP_KEYS = ( + "hop_vendor", + "hop_host", + "hop_port", + "hop_protocol", + "hop_username", + "hop_command_template", + "hop_vrf", + "hop_target_auth_mode", + "hop_enter_system_view", +) + + +def _compact_managed_ne(row: dict[str, Any], *, keep_detail: bool) -> None: + if row.get("hop_enabled") is False: + for k in _HOP_KEYS: + row.pop(k, None) + if not keep_detail: + row.pop("connect_detail", None) + + UME_RAW_GROUP_FIELDS = [ "alarm_alarm_key", "alarm_host_name", @@ -121,7 +195,6 @@ _UME_RAW_FIELD_PRESETS: dict[str, list[str]] = { "alarm_last_seen_at", "ne_host_name", "ne_user_label", - "ne_ne_name", "ne_ip_address", "ne_exists", ], @@ -137,7 +210,6 @@ _UME_RAW_FIELD_PRESETS: dict[str, list[str]] = { "alarm_last_seen_at", "ne_host_name", "ne_user_label", - "ne_ne_name", "ne_ip_address", "ne_connection_status", "ne_exists", @@ -148,7 +220,6 @@ _UME_RAW_FIELD_PRESETS: dict[str, list[str]] = { "alarm_perceived_severity", "alarm_last_seen_at", "ne_user_label", - "ne_ne_name", "ne_ip_address", "ne_ipv6_address", "ne_device_level", @@ -163,13 +234,14 @@ _UME_RAW_FIELD_PRESETS: dict[str, list[str]] = { def _query_ume_alarms(args: dict[str, Any]) -> dict[str, Any]: - page = max(1, int(args.get("page") or 1)) - if page > 2: - page = 2 - page_size = min(500, max(1, int(args.get("page_size") or 50))) - params: dict[str, Any] = {"page": page, "page_size": page_size} - if str(args.get("severity") or "").strip(): - params["severity"] = str(args.get("severity")).strip() + params: dict[str, Any] = { + "page": _int_arg(args, "page", 1, 1, 100000), + "page_size": _int_arg(args, "page_size", 50, 1, 500), + } + if _severity(args): + params["severity"] = _severity(args) + if str(args.get("is_cleared") or "").strip(): + params["is_cleared"] = str(args.get("is_cleared")).strip() keyword = str(args.get("keyword") or "").strip() ne_name = str(args.get("ne_name") or "").strip() if keyword: @@ -183,7 +255,7 @@ def _query_ume_alarms(args: dict[str, Any]) -> dict[str, Any]: for k in ("time_from", "time_to"): if str(args.get(k) or "").strip(): params[k] = str(args.get(k)).strip() - return http_json("GET", "/v1/ume/alarms", params=params) + return _compact_rows(http_json("GET", "/v1/ume/alarms", params=params), name_key="ne_name", id_key="ne_id") def _aggregate_ume_alarms(args: dict[str, Any]) -> dict[str, Any]: @@ -192,11 +264,12 @@ def _aggregate_ume_alarms(args: dict[str, Any]) -> dict[str, Any]: if group_by: return _aggregate_ume_alarms_raw(args) # Default top 50 named NEs — missing host buckets reported separately. - top_ne = max(0, min(500, int(args.get("top_ne") if args.get("top_ne") is not None else 50))) - params: dict[str, Any] = {"top_ne": top_ne} + params: dict[str, Any] = {"top_ne": _int_arg(args, "top_ne", 50, 0, 500)} if "exclude_missing_host" in args: params["exclude_missing_host"] = bool(args.get("exclude_missing_host")) - for k in ("severity", "time_from", "time_to"): + if _severity(args): + params["severity"] = _severity(args) + for k in ("time_from", "time_to"): if str(args.get(k) or "").strip(): params[k] = str(args.get(k)).strip() return http_json("GET", "/v1/ume/alarms/aggregate", params=params) @@ -208,26 +281,58 @@ def _run_ume_diagnostics(args: dict[str, Any]) -> dict[str, Any]: def _query_ume_ne_inventory(args: dict[str, Any]) -> dict[str, Any]: - page = max(1, int(args.get("page") or 1)) - page_size = min(500, max(1, int(args.get("page_size") or 50))) - params: dict[str, Any] = {"page": page, "page_size": page_size} + params: dict[str, Any] = { + "page": _int_arg(args, "page", 1, 1, 100000), + "page_size": _int_arg(args, "page_size", 50, 1, 500), + } if str(args.get("keyword") or "").strip(): params["keyword"] = str(args.get("keyword")).strip() - return http_json("GET", "/v1/ume/inventory/ne", params=params) + return _compact_rows(http_json("GET", "/v1/ume/inventory/ne", params=params), name_key="ne_name", id_key="ne_id") + + +def _looks_like_uuid(text: str) -> bool: + parts = text.split("-") + return len(parts) == 5 and all(p and all(c in "0123456789abcdefABCDEF" for c in p) for p in parts) def _get_ume_ne(args: dict[str, Any]) -> dict[str, Any]: ne_id = str(args.get("ne_id") or "").strip() if not ne_id: return {"ok": False, "error": "ne_id_required", "error_code": "ne_id_required"} - return http_json("GET", f"/v1/ume/inventory/ne/{quote_ne_id(ne_id)}", params=None) + out = http_json("GET", f"/v1/ume/inventory/ne/{quote_ne_id(ne_id)}", params=None) + if out.get("ok") or out.get("error") != "netx_http_404" or _looks_like_uuid(ne_id): + return out + # Agents usually know the host_name (alarms show it), not the UUID: resolve it. + found = http_json("GET", "/v1/ume/inventory/ne", params={"keyword": ne_id, "page": 1, "page_size": 20}) + items = ((found.get("data") or {}).get("items") or []) if found.get("ok") else [] + needle = ne_id.lower() + exact = [ + r + for r in items + if isinstance(r, dict) + and needle in {str(r.get(k) or "").lower() for k in ("host_name", "user_label", "ip_address", "ne_name")} + ] + if len(exact) == 1 and exact[0].get("ne_id"): + return http_json("GET", f"/v1/ume/inventory/ne/{quote_ne_id(str(exact[0]['ne_id']))}", params=None) + out = dict(out) + out["hint"] = "ne_id is the NMS UUID. Resolve host_name/IP via queryNmsNeInventory(keyword=...)." + if items: + out["candidates"] = [ + {"ne_id": r.get("ne_id"), "host_name": r.get("host_name"), "ip_address": r.get("ip_address")} + for r in items[:10] + if isinstance(r, dict) + ] + return out def _query_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: - page = max(1, int(args.get("page") or 1)) - page_size = min(500, max(1, int(args.get("page_size") or 50))) - params: dict[str, Any] = {"page": page, "page_size": page_size} - for k in ("severity", "is_cleared", "ne_id", "event_type", "keyword", "time_from", "time_to", "order_by", "order"): + params: dict[str, Any] = { + "page": _int_arg(args, "page", 1, 1, 100000), + "page_size": _int_arg(args, "page_size", 50, 1, 500), + } + if _severity(args): + params["severity"] = _severity(args) + for k in ("is_cleared", "ne_id", "event_type", "keyword", "time_from", "time_to", "order_by", "order"): v = str(args.get(k) or "").strip() if v: params[k] = v @@ -240,7 +345,15 @@ def _query_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: fields = list(_UME_RAW_FIELD_PRESETS.get(preset) or []) if fields: params["select_fields"] = ",".join(fields) - return http_json("GET", "/v1/ume/alarms/raw", params=params) + out = http_json("GET", "/v1/ume/alarms/raw", params=params) + data = out.get("data") if isinstance(out.get("data"), dict) else None + if data and isinstance(data.get("items"), list): + for row in data["items"]: + if isinstance(row, dict) and _is_me_placeholder( + row.get("ne_ne_name"), row.get("ne_ne_id") or row.get("alarm_ne_id") + ): + row.pop("ne_ne_name", None) + return _strip_meta_field_lists(out) def _aggregate_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: @@ -248,7 +361,6 @@ def _aggregate_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: for k in ( "group_by", "group_by2", - "severity", "is_cleared", "ne_id", "event_type", @@ -263,9 +375,13 @@ def _aggregate_ume_alarms_raw(args: dict[str, Any]) -> dict[str, Any]: sv = str(v).strip() if sv: params[k] = sv + if _severity(args): + params["severity"] = _severity(args) + if "limit" in params: + params["limit"] = _int_arg(args, "limit", 200, 1, 2000) if "exclude_missing_host" in args: params["exclude_missing_host"] = bool(args.get("exclude_missing_host")) - return http_json("GET", "/v1/ume/alarms/aggregate/raw", params=params) + return _strip_meta_field_lists(http_json("GET", "/v1/ume/alarms/aggregate/raw", params=params)) def _list_ume_alarm_fields(args: dict[str, Any]) -> dict[str, Any]: @@ -275,8 +391,8 @@ def _list_ume_alarm_fields(args: dict[str, Any]) -> dict[str, Any]: def _sql_query_ume(args: dict[str, Any]) -> dict[str, Any]: sql = str(args.get("sql") or "").strip() - limit = max(1, min(2000, int(args.get("limit") or 200))) - statement_timeout_ms = max(0, min(30000, int(args.get("statement_timeout_ms") or 0))) + limit = _int_arg(args, "limit", 200, 1, 2000) + statement_timeout_ms = _int_arg(args, "statement_timeout_ms", 0, 0, 30000) if not sql: return {"ok": False, "error": "sql_required"} return http_post_json( @@ -287,13 +403,18 @@ def _sql_query_ume(args: dict[str, Any]) -> dict[str, Any]: def _list_managed_ne(args: dict[str, Any]) -> dict[str, Any]: - page = max(1, int(args.get("page") or 1)) - page_size = min(100, max(1, int(args.get("page_size") or 20))) + page = _int_arg(args, "page", 1, 1, 100000) + page_size = _int_arg(args, "page_size", 20, 1, 100) keyword = str(args.get("keyword") or "").strip() vendor = str(args.get("vendor") or "").strip() connect_status = str(args.get("connect_status") or "").strip() if not (keyword or vendor or connect_status): - return {"ok": False, "error": "managed_ne_filter_required", "error_code": "managed_ne_filter_required"} + return { + "ok": False, + "error": "managed_ne_filter_required", + "error_code": "managed_ne_filter_required", + "hint": "Pass keyword (name/IP, >=2 chars), vendor, or connect_status (e.g. pass).", + } if keyword and len(keyword) < 2: return {"ok": False, "error": "managed_ne_keyword_too_short", "error_code": "managed_ne_keyword_too_short"} params: dict[str, Any] = {"page": page, "page_size": page_size} @@ -303,7 +424,13 @@ def _list_managed_ne(args: dict[str, Any]) -> dict[str, Any]: params["vendor"] = vendor if connect_status: params["connect_status"] = connect_status - return http_json("GET", "/v1/managed-ne", params=params) + out = http_json("GET", "/v1/managed-ne", params=params) + items = (out.get("data") or {}).get("items") if out.get("ok") else None + if isinstance(items, list): + for row in items: + if isinstance(row, dict): + _compact_managed_ne(row, keep_detail=False) + return out def _first_str(*vals: Any) -> str: @@ -330,7 +457,9 @@ def _get_managed_ne(args: dict[str, Any]) -> dict[str, Any]: "example": {"ne_id": ""}, } # NMS ne_id is typically a UUID; managed NE may differ. Soft-guide when callers mix them. - out = http_json("GET", f"/v1/managed-ne/{ne_id}", params=None) + out = http_json("GET", f"/v1/managed-ne/{quote_ne_id(ne_id)}", params=None) + if out.get("ok") and isinstance(out.get("data"), dict): + _compact_managed_ne(out["data"], keep_detail=True) if isinstance(out, dict) and out.get("ok") is False: detail = str(out.get("detail") or out.get("error") or "") low = detail.lower() @@ -391,11 +520,9 @@ def _exec_managed_ne(args: dict[str, Any]) -> dict[str, Any]: if len(shared_commands) > exec_max_commands(): return {"ok": False, "error": "too_many_commands", "error_code": "too_many_commands"} body["commands"] = shared_commands - rts = args.get("read_timeout_sec") - body["read_timeout_sec"] = int(rts) if rts is not None else 60 - conc = args.get("concurrency") - if conc is not None: - body["concurrency"] = int(conc) + body["read_timeout_sec"] = _int_arg(args, "read_timeout_sec", 60, 10, 120) + if args.get("concurrency") is not None: + body["concurrency"] = _int_arg(args, "concurrency", 4, 1, 8) if want_async: out = http_post_json("/v1/managed-ne/exec-jobs", body, timeout=60.0) if not out.get("ok"): @@ -434,8 +561,7 @@ def _exec_managed_ne(args: dict[str, Any]) -> dict[str, Any]: if nms_ne_id: body["ume_ne_id"] = nms_ne_id # Default 60s matches netx API default; slow show commands often exceed 30s. - rts = args.get("read_timeout_sec") - body["read_timeout_sec"] = int(rts) if rts is not None else 60 + body["read_timeout_sec"] = _int_arg(args, "read_timeout_sec", 60, 10, 120) if want_async: out = http_post_json("/v1/managed-ne/exec-jobs", body, timeout=60.0) if not out.get("ok"): @@ -464,16 +590,23 @@ def _get_ne_exec_job(args: dict[str, Any]) -> dict[str, Any]: def _list_cli_targets(args: dict[str, Any]) -> dict[str, Any]: - page = max(1, int(args.get("page") or 1)) - page_size = min(500, max(1, int(args.get("page_size") or 50))) - params: dict[str, Any] = {"page": page, "page_size": page_size} + params: dict[str, Any] = { + "page": _int_arg(args, "page", 1, 1, 100000), + "page_size": _int_arg(args, "page_size", 50, 1, 500), + } source = str(args.get("source") or "").strip() if source: # API still uses source=ume; accept model-facing "nms". params["source"] = "ume" if source.lower() == "nms" else source if str(args.get("keyword") or "").strip(): params["keyword"] = str(args.get("keyword")).strip() - return http_json("GET", "/v1/cli/targets", params=params) + out = http_json("GET", "/v1/cli/targets", params=params) + items = (out.get("data") or {}).get("items") if out.get("ok") else None + if isinstance(items, list): + for row in items: + if isinstance(row, dict): + _nms_naming(row) + return out def _find_topology_paths(args: dict[str, Any]) -> dict[str, Any]: @@ -489,8 +622,8 @@ def _find_topology_paths(args: dict[str, Any]) -> dict[str, Any]: if detail not in {"summary", "full"}: detail = "summary" body: dict[str, Any] = { - "max_paths": max(1, min(10, int(args.get("max_paths") or 3))), - "max_hops": max(1, min(12, int(args.get("max_hops") or 6))), + "max_paths": _int_arg(args, "max_paths", 3, 1, 10), + "max_hops": _int_arg(args, "max_hops", 6, 1, 12), "layer": str(args.get("layer") or "physical").strip() or "physical", "detail": detail, } @@ -502,7 +635,14 @@ def _find_topology_paths(args: dict[str, Any]) -> dict[str, Any]: body["to_ume_ne_id"] = to_uid else: body["to_managed_ne_id"] = to_mid - return http_post_json("/v1/topology/fabric/paths", body, timeout=30.0) + out = http_post_json("/v1/topology/fabric/paths", body, timeout=30.0) + paths = (out.get("data") or {}).get("paths") if out.get("ok") else None + if isinstance(paths, list): + for p in paths: + for node in (p.get("nodes") or []) if isinstance(p, dict) else []: + if isinstance(node, dict): + _nms_naming(node) + return out HTTP_MCP_TOOLS: list[dict[str, Any]] = [ @@ -520,7 +660,8 @@ HTTP_MCP_TOOLS: list[dict[str, Any]] = [ "inputSchema": { "type": "object", "properties": { - "severity": {"type": "string"}, + "severity": {"type": "string", "description": "critical/major/minor/warning"}, + "is_cleared": {"type": "string", "description": "\"False\" = active only; \"True\" = cleared."}, "ne_id": {"type": "string", "description": "Filter only; do not show UUID to users"}, "host_name": {"type": "string", "description": "Filter by NE host_name"}, "ne_name": {"type": "string", "description": "Legacy alias mapped to keyword"}, @@ -626,10 +767,10 @@ HTTP_MCP_TOOLS: list[dict[str, Any]] = [ }, { "name": "getNmsNe", - "description": "Get single NMS NE detail by ne_id (UUID).", + "description": "Get single NMS NE detail by ne_id (UUID); exact host_name / user_label / IP also resolve.", "inputSchema": { "type": "object", - "properties": {"ne_id": {"type": "string"}}, + "properties": {"ne_id": {"type": "string", "description": "NMS UUID, or exact host_name / IP."}}, "required": ["ne_id"], "additionalProperties": False, }, @@ -907,7 +1048,8 @@ HTTP_MCP_TOOLS: list[dict[str, Any]] = [ { "name": "findTopologyPaths", "description": ( - "Find up to max_paths simple paths between two fabric nodes for troubleshooting. " + "Find up to max_paths simple paths between two fabric nodes for troubleshooting " + "(parallel links between the same NEs count as one hop; see edges[].parallel_links). " "For each endpoint provide exactly one of nms_ne_id (from NMS alarm ne_id; alias " "from_ume_ne_id) or managed_ne_id — resolved to fabric node internally. Returns " "shortest paths first with compact label + node/edge summary (detail=summary default). " @@ -1014,4 +1156,8 @@ def call_http_tool(name: str, args: dict[str, Any]) -> dict[str, Any]: fn = _HANDLERS.get(str(name or "").strip()) if not fn: raise ValueError(f"unknown tool: {name}") - return mcp_from_handler_result(fn(dict(args or {}))) + try: + result = fn(dict(args or {})) + except ToolArgError as exc: + result = {"ok": False, "error": "invalid_argument", "error_code": "invalid_argument", "detail": str(exc)} + return mcp_from_handler_result(result) diff --git a/packages/netx-mcp/src/netx_mcp/server.py b/packages/netx-mcp/src/netx_mcp/server.py index 9f4b9ae..16b13ac 100644 --- a/packages/netx-mcp/src/netx_mcp/server.py +++ b/packages/netx-mcp/src/netx_mcp/server.py @@ -43,6 +43,15 @@ def _err(rid: Any, code: int, message: str) -> None: _UNSET = object() +def _version() -> str: + try: + from importlib.metadata import version + + return version("netx-mcp") + except Exception: + return "0.0.0" + + def _fetch_scopes() -> list[str] | None: """Return granted scopes from /v1/auth/me, or None if the call fails (show all tools).""" try: @@ -81,6 +90,7 @@ def run_stdio_loop() -> None: except Exception: continue rid = req.get("id") + is_notification = "id" not in req method = str(req.get("method") or "") params = req.get("params") if isinstance(req.get("params"), dict) else {} @@ -91,11 +101,15 @@ def run_stdio_loop() -> None: { "protocolVersion": "2024-11-05", "capabilities": {"tools": {}}, - "serverInfo": {"name": "netx-mcp", "version": "0.3.0", "mode": "http"}, + "serverInfo": {"name": "netx-mcp", "version": _version(), "mode": "http"}, }, ) continue - if method == "notifications/initialized": + if is_notification: + # JSON-RPC: notifications (initialized, cancelled, ...) never get a response. + continue + if method == "ping": + _ok(rid, {}) continue if method == "tools/list": _ok(rid, {"tools": tools_for_scopes(scopes())}) diff --git a/packages/netx-mcp/tests/test_mcp_http.py b/packages/netx-mcp/tests/test_mcp_http.py index 50e5b44..5b80df7 100644 --- a/packages/netx-mcp/tests/test_mcp_http.py +++ b/packages/netx-mcp/tests/test_mcp_http.py @@ -74,7 +74,7 @@ def test_call_aggregate_nms_alarms_group_by_routes_to_raw() -> None: params = mock_http.call_args[1]["params"] assert params["group_by"] == "alarm_host_name" assert params["severity"] == "critical" - assert params["limit"] == "20" + assert params["limit"] == 20 payload = json.loads(out["content"][0]["text"]) assert payload["ok"] is True @@ -285,6 +285,14 @@ def test_stdio_initialize_and_tools_list() -> None: assert msg.get("id") == 1 assert "result" in msg + # Notifications must not be answered; the next line belongs to ping. + for note in ("notifications/initialized", "notifications/cancelled"): + proc.stdin.write(json.dumps({"jsonrpc": "2.0", "method": note, "params": {}}) + "\n") + proc.stdin.write(json.dumps({"jsonrpc": "2.0", "id": 9, "method": "ping"}) + "\n") + proc.stdin.flush() + pong = json.loads(proc.stdout.readline()) + assert pong == {"jsonrpc": "2.0", "id": 9, "result": {}} + proc.stdin.write(json.dumps({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}) + "\n") proc.stdin.flush() line2 = proc.stdout.readline() @@ -297,3 +305,92 @@ def test_stdio_initialize_and_tools_list() -> None: finally: proc.kill() proc.wait(timeout=5) + + +def test_bad_integer_arg_is_tool_error_not_crash() -> None: + with patch("netx_mcp.http_tools.http_json") as mock_http: + out = call_http_tool("queryNmsAlarms", {"page": "two"}) + mock_http.assert_not_called() + assert out["isError"] is True + payload = json.loads(out["content"][0]["text"]) + assert payload["error"] == "invalid_argument" + assert "page" in payload["detail"] + + +def test_query_nms_alarms_pages_past_two_and_normalizes_severity() -> None: + with patch("netx_mcp.http_tools.http_json") as mock_http: + mock_http.return_value = { + "ok": True, + "data": {"items": [{"ne_id": "u-1", "ne_name": "ME{u-1}", "host_name": "H1"}]}, + } + out = call_http_tool("queryNmsAlarms", {"page": 5, "severity": "Critical", "is_cleared": "False"}) + params = mock_http.call_args[1]["params"] + assert params["page"] == 5 + assert params["severity"] == "critical" + assert params["is_cleared"] == "False" + row = json.loads(out["content"][0]["text"])["data"]["items"][0] + assert "ne_name" not in row + assert row["host_name"] == "H1" + + +def test_nested_api_failure_is_marked_error() -> None: + with patch("netx_mcp.http_tools.http_json") as mock_http: + mock_http.return_value = {"ok": True, "data": {"ok": False, "error": "job_not_found"}} + out = call_http_tool("getNeExecJob", {"job_id": "nope"}) + assert out.get("isError") is True + + +def test_raw_query_strips_echoed_field_lists() -> None: + with patch("netx_mcp.http_tools.http_json") as mock_http: + mock_http.return_value = { + "ok": True, + "data": { + "items": [], + "meta": {"available_fields": ["a"], "order_by_allowed": ["b"], "time_filter_field": "last_seen_at"}, + }, + } + out = call_http_tool("queryNmsAlarmsRaw", {"field_preset": "brief"}) + meta = json.loads(out["content"][0]["text"])["data"]["meta"] + assert meta == {"time_filter_field": "last_seen_at"} + + +def test_get_nms_ne_resolves_host_name() -> None: + calls: list[str] = [] + + def fake(method: str, path: str, params: dict | None = None) -> dict: + calls.append(path) + if path == "/v1/ume/inventory/ne": + return {"ok": True, "data": {"items": [{"ne_id": "uuid-1", "host_name": "MDN-X-AN1"}]}} + if path.endswith("/uuid-1"): + return {"ok": True, "data": {"ne_id": "uuid-1"}} + return {"ok": False, "error": "netx_http_404", "detail": "ume_ne_not_found"} + + with patch("netx_mcp.http_tools.http_json", side_effect=fake): + out = call_http_tool("getNmsNe", {"ne_id": "mdn-x-an1"}) + assert json.loads(out["content"][0]["text"])["data"]["ne_id"] == "uuid-1" + assert calls[-1] == "/v1/ume/inventory/ne/uuid-1" + + +def test_cli_targets_and_paths_use_nms_naming() -> None: + with patch("netx_mcp.http_tools.http_json") as mock_http: + mock_http.return_value = {"ok": True, "data": {"items": [{"source": "ume", "ume_ne_id": "u-1"}]}} + out = call_http_tool("listCliTargets", {"source": "nms"}) + row = json.loads(out["content"][0]["text"])["data"]["items"][0] + assert row == {"source": "nms", "nms_ne_id": "u-1"} + + with patch("netx_mcp.http_tools.http_post_json") as mock_post: + mock_post.return_value = {"ok": True, "data": {"paths": [{"nodes": [{"name": "A", "ume_ne_id": "u-1"}]}]}} + out = call_http_tool("findTopologyPaths", {"from_nms_ne_id": "u-1", "to_nms_ne_id": "u-2"}) + node = json.loads(out["content"][0]["text"])["data"]["paths"][0]["nodes"][0] + assert node == {"name": "A", "nms_ne_id": "u-1"} + + +def test_managed_ne_list_drops_disabled_hop_noise() -> None: + with patch("netx_mcp.http_tools.http_json") as mock_http: + mock_http.return_value = { + "ok": True, + "data": {"items": [{"id": "m1", "hop_enabled": False, "hop_vrf": "", "connect_detail": "long"}]}, + } + out = call_http_tool("listManagedNe", {"connect_status": "pass"}) + row = json.loads(out["content"][0]["text"])["data"]["items"][0] + assert row == {"id": "m1", "hop_enabled": False} diff --git a/packages/netx-topology-mcp/src/netx_topology_mcp/http_tools.py b/packages/netx-topology-mcp/src/netx_topology_mcp/http_tools.py index 581510e..17dfe37 100644 --- a/packages/netx-topology-mcp/src/netx_topology_mcp/http_tools.py +++ b/packages/netx-topology-mcp/src/netx_topology_mcp/http_tools.py @@ -1976,15 +1976,18 @@ def _update_topology_view_positions(args: dict[str, Any]) -> dict[str, Any]: fid = str(p.get("fabric_node_id") or "").strip() if not fid: continue - cleaned.append( - { - "fabric_node_id": fid, - "x": float(p.get("x") or 0), - "y": float(p.get("y") or 0), - "label": str(p.get("label") or ""), - "locked": bool(p.get("locked") or False), - } - ) + row: dict[str, Any] = { + "fabric_node_id": fid, + "x": float(p.get("x") or 0), + "y": float(p.get("y") or 0), + } + # Omitted label/locked = keep server value (a UI rename or an existing lock). + # label "" clears the override; locked false explicitly unlocks. + if p.get("label") is not None: + row["label"] = str(p.get("label")) + if p.get("locked") is not None: + row["locked"] = bool(p.get("locked")) + cleaned.append(row) if not cleaned: return {"ok": False, "error": "positions_required"} parts: list[dict[str, Any]] = [] @@ -3727,8 +3730,14 @@ HTTP_MCP_TOOLS: list[dict[str, Any]] = [ "fabric_node_id": {"type": "string"}, "x": {"type": "number"}, "y": {"type": "number"}, - "label": {"type": "string"}, - "locked": {"type": "boolean"}, + "label": { + "type": "string", + "description": "Omit to keep the current label; \"\" clears a rename (follow device name).", + }, + "locked": { + "type": "boolean", + "description": "Omit to keep the lock. Locked nodes are skipped unless locked=false (unlock + move).", + }, }, "required": ["fabric_node_id"], "additionalProperties": False, diff --git a/packages/netx-topology-mcp/src/netx_topology_mcp/layout_ops/press_crossings.py b/packages/netx-topology-mcp/src/netx_topology_mcp/layout_ops/press_crossings.py index aff469f..9ff5504 100644 --- a/packages/netx-topology-mcp/src/netx_topology_mcp/layout_ops/press_crossings.py +++ b/packages/netx-topology-mcp/src/netx_topology_mcp/layout_ops/press_crossings.py @@ -284,9 +284,10 @@ def press_hot_edges( tuple[int, tuple[float, float], tuple[float, float]] | None ) = None - def _try_cand(cand: tuple[float, float]) -> None: + def _try_cand(cand: tuple[float, float], disp_cap: float | None = None) -> None: nonlocal best - if math.hypot(cand[0] - leaf_xy[0], cand[1] - leaf_xy[1]) > max_disp: + cap = max_disp if disp_cap is None else disp_cap + if math.hypot(cand[0] - leaf_xy[0], cand[1] - leaf_xy[1]) > cap: return trial_pos = {**pos, leaf: cand} if _overlaps_any(leaf, trial_pos, names): @@ -304,7 +305,11 @@ def press_hot_edges( best = (c1, cand) # Reel-in along the spoke first (shorten without hunting angles). + # Pulling straight toward the hub cannot fling the leaf across the + # bbox, so allow it to travel all the way to the r_use target; + # max_disp alone stops short (e.g. r0=8000 → cap 4400 < 5800 needed). if r0 > r_use + 50.0: + reel_cap = max(max_disp, r0 - r_use) ux = (leaf_xy[0] - hub_xy[0]) / r0 uy = (leaf_xy[1] - hub_xy[1]) / r0 for rt in ( @@ -318,7 +323,7 @@ def press_hot_edges( rt = min(rt, r0 - 1.0) if rt < 120.0: continue - _try_cand((hub_xy[0] + ux * rt, hub_xy[1] + uy * rt)) + _try_cand((hub_xy[0] + ux * rt, hub_xy[1] + uy * rt), reel_cap) # Both ends walk toward the midpoint (metro bridges often need this). # Skip on giant E: each trial is a full O(E²) count and stalls MCP. diff --git a/packages/netx-topology-mcp/tests/test_dual_mass.py b/packages/netx-topology-mcp/tests/test_dual_mass.py index 6b57e9a..e3b353b 100644 --- a/packages/netx-topology-mcp/tests/test_dual_mass.py +++ b/packages/netx-topology-mcp/tests/test_dual_mass.py @@ -80,7 +80,7 @@ def test_classify_petal_and_straight() -> None: assert classify_dual_unit(short) == "straight" -def test_beautify_parallel_lanes() -> None: +def test_beautify_petal_ellipse_bands() -> None: nodes, edges = _two_eye_graph() st = build_state_from_nodes_edges(nodes, edges) unit = DualUnit( @@ -89,15 +89,18 @@ def test_beautify_parallel_lanes() -> None: paths=[["p1", "a1", "a2", "p2"], ["p1", "b1", "b2", "p2"]], ) pos = beautify_dual_unit_positions(st, unit, LayoutParams()) - # Parallel H/V lanes: same y in a corridor; first/last share portal x. - assert abs(pos["a1"][1] - pos["a2"][1]) < 1e-6 - assert abs(pos["b1"][1] - pos["b2"][1]) < 1e-6 - assert abs(pos["a1"][1]) > 1.0 and abs(pos["b1"][1]) > 1.0 + # Portals sit on the x-axis, a left of b (the eye's chord). + assert abs(pos["p1"][1]) < 1e-6 and abs(pos["p2"][1]) < 1e-6 + assert pos["p1"][0] < pos["p2"][0] + # Each corridor is one band on its own side of the chord. assert pos["a1"][1] * pos["b1"][1] < 0 - assert abs(pos["a1"][0] - pos["p1"][0]) < 1e-6 - assert abs(pos["a2"][0] - pos["p2"][0]) < 1e-6 - assert abs(pos["b1"][0] - pos["p1"][0]) < 1e-6 - assert abs(pos["b2"][0] - pos["p2"][0]) < 1e-6 + for x, y in (("a1", "a2"), ("b1", "b2")): + assert pos[x][1] * pos[y][1] > 0 + assert abs(pos[x][1]) > 1.0 + # Runs a→b, strictly inside the portals (hollow eye, no node on the chord ends). + assert pos["p1"][0] < pos[x][0] < pos[y][0] < pos["p2"][0] + links = [(e["a_node_id"], e["b_node_id"]) for e in edges if e["a_node_id"] in pos and e["b_node_id"] in pos] + assert count_edge_crossings(pos, links) == 0 def test_dual_mass_refine_reports_drift() -> None: diff --git a/packages/netx-topology-mcp/tests/test_layout_jobs.py b/packages/netx-topology-mcp/tests/test_layout_jobs.py index b76ac09..0c5ca70 100644 --- a/packages/netx-topology-mcp/tests/test_layout_jobs.py +++ b/packages/netx-topology-mcp/tests/test_layout_jobs.py @@ -105,26 +105,54 @@ def test_slim_layout_payload_drops_guide() -> None: assert len(slim["crossing"]["top_nodes"]) <= 5 -def test_compose_background_returns_job_id() -> None: - src = [f"s{i:02d}" for i in range(12)] +def test_compose_views_is_not_public() -> None: + # compose_* was retired from the public surface (dual-unit drain is the main path). + out = _layout_topology_view( + { + "view_id": "full", + "action": "compose_views", + "mode": "apply", + "params": {"source_view_ids": [f"s{i:02d}" for i in range(12)]}, + } + ) + assert out.get("ok") is False + assert out.get("error") == "unknown_action:compose_views" + + +def test_orbit_background_returns_job_id() -> None: + graph = { + "ok": True, + "nodes": [ + {"fabric_node_id": f"n{i}", "x": float(i * 100), "y": 0.0, "label": f"N{i}"} + for i in range(6) + ], + "edges": [ + {"id": f"e{i}", "a_node_id": f"n{i}", "b_node_id": f"n{i + 1}"} for i in range(5) + ], + } def fake_start(**kwargs): - assert kwargs["action"] == "compose_views" + assert kwargs["action"] == "orbit_sweep" assert kwargs["view_id"] == "full" - assert isinstance(kwargs.get("tool_args"), dict) - assert kwargs["tool_args"].get("action") == "compose_views" + tool_args = kwargs.get("tool_args") + assert isinstance(tool_args, dict) + # The worker re-runs the same call synchronously. + assert tool_args["params"].get("_force_sync") is True + assert "background" not in tool_args["params"] return "abc123job" - with patch("netx_topology_mcp.http_tools.start_job", side_effect=fake_start): + with patch("netx_topology_mcp.http_tools.http_json", return_value={"ok": True, "data": graph}), patch( + "netx_topology_mcp.http_tools.start_job", side_effect=fake_start + ): out = _layout_topology_view( { "view_id": "full", - "action": "compose_views", + "action": "orbit_sweep", "mode": "apply", - "params": {"source_view_ids": src}, + "params": {"background": True}, } ) - assert out.get("ok") is True + assert out.get("ok") is True, out assert out.get("status") == "running" assert out.get("job_id") == "abc123job" assert out.get("applied") is False diff --git a/packages/netx-topology-mcp/tests/test_mcp_topology.py b/packages/netx-topology-mcp/tests/test_mcp_topology.py index ccaa466..09547f8 100644 --- a/packages/netx-topology-mcp/tests/test_mcp_topology.py +++ b/packages/netx-topology-mcp/tests/test_mcp_topology.py @@ -18,8 +18,10 @@ from netx_topology_mcp.server import _fetch_scopes def test_tool_list_has_draw_and_query_tools() -> None: names = {str(t.get("name") or "") for t in HTTP_MCP_TOOLS} - assert len(names) == 14 + assert len(names) == 16 assert "sinkTopologyDualUnits" in names + assert "suggestSinkHubs" in names + assert "classifyTopologyFabricNodes" in names assert "copyTopologyViewNodes" in names assert "createTopologyFolder" in names assert "createTopologyView" not in names @@ -547,8 +549,10 @@ def test_stdio_initialize_and_tools_list() -> None: proc.stdin.flush() list_resp = json.loads(proc.stdout.readline()) tools = list_resp["result"]["tools"] - assert len(tools) == 14 + assert len(tools) == 16 names = {t["name"] for t in tools} + assert "suggestSinkHubs" in names + assert "classifyTopologyFabricNodes" in names assert "createTopologyView" not in names assert "listTopologyViews" not in names assert "queryTopologyFabricNodes" in names @@ -614,3 +618,24 @@ def test_analyze_topology_view_layout_single() -> None: assert "sparsity" in payload assert "verdict" in payload assert "total" in payload["verdict"] + + +def test_update_positions_omits_label_and_locked_unless_given() -> None: + from netx_topology_mcp.http_tools import _update_topology_view_positions + + with patch("netx_topology_mcp.http_tools.http_json") as mock_http: + mock_http.return_value = {"ok": True, "data": {"ok": True, "updated": 3}} + _update_topology_view_positions( + { + "view_id": "v1", + "positions": [ + {"fabric_node_id": "a", "x": 1, "y": 2}, + {"fabric_node_id": "b", "x": 3, "y": 4, "label": "", "locked": False}, + {"fabric_node_id": "c", "x": 5, "y": 6, "label": "Core-1", "locked": True}, + ], + } + ) + sent = mock_http.call_args[1]["body"]["positions"] + assert sent[0] == {"fabric_node_id": "a", "x": 1.0, "y": 2.0} + assert sent[1]["label"] == "" and sent[1]["locked"] is False + assert sent[2]["label"] == "Core-1" and sent[2]["locked"] is True diff --git a/packaging/README.md b/packaging/README.md index 4f22d6f..b0dfae3 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -1,4 +1,4 @@ -# NetX Windows packaging +# NetX Windows packaging **Linux is unchanged:** keep using your own PostgreSQL and `NETX_DATABASE_URL` with `scripts/start_netx.sh`. Nothing under this folder is required on Linux. @@ -7,32 +7,39 @@ This directory builds a Windows deliverable with: - Optional **bundled** portable PostgreSQL **or** **external** existing Postgres - API-hosted UI (`web/dist`) — no separate Vite process for end users - Program / data split so upgrades do not wipe the database -- Manual update script + auto-update **manifest contract** (fetch not implemented yet) +- Manual update script + check/apply updates (GitHub Releases or custom manifest) +- Optional system tray + start-at-logon ## What users get | Artifact | How | |----------|-----| -| `NetX-x.y.z-win64.zip` | `build_release.ps1` | -| `NetX-Setup-x.y.z.exe` | Compile `installer/netx.iss` with [Inno Setup](https://jrsoftware.org/isinfo.php) after staging | +| `NetX-Setup-x.y.z.exe` | Stage with `build_release.ps1`, then compile `installer/netx.iss` with [Inno Setup](https://jrsoftware.org/isinfo.php) | + +Zip packages are **not** published. Releases ship **Setup.exe only**. + +**Installer:** English + 简体中文 (language dialog). Icons use `packaging/assets/netx.ico`. + +**Offline:** Setup ships portable PostgreSQL, **`python/runtime` + `.venv`** (portable; not tied to the build PC’s user profile), and WinSW. On upgrade, Setup removes the old runtime and virtual environment before copying the bundled pair, preventing stale standard-library files when Python versions have different layouts. **First install** shows the Database page (built-in or external PostgreSQL; external credentials validated with `psql SELECT 1`). **Re-running Setup when `%ProgramData%\NetX\.env` has `NETX_DB_MODE` or a non-empty legacy `NETX_DATABASE_URL`** skips the DB wizard by default. Update stops and resumes an installed Windows service or startup task while replacing files. Service install uses bundled WinSW. Auto-update still needs network later, and is unchecked by default. + +**OS:** Windows 10/11 or Windows Server **2016+** recommended. Packaging scripts are UTF-8 **with BOM** so Chinese UI works on Windows PowerShell 5.x. The Windows release workflow pins bundled Python to **3.12**, so local release builds should use the same version for consistent artifacts. ## Build (developer machine) -Prerequisites: Python 3.11+, Node 20+, PowerShell, network (to download PG binaries once). +Prerequisites: Python **3.12.x**, Node 20+, PowerShell, network (to download PostgreSQL binaries once). ```powershell cd netx -# Optional: download portable Postgres into packaging\postgres\pgsql +# Required once per clean build machine: download portable PostgreSQL into packaging\postgres\pgsql powershell -ExecutionPolicy Bypass -File .\packaging\download_postgres.ps1 -# Build web + stage + zip (-CreateVenv ships a ready .venv; large) +# Build web + offline stage containing Python, .venv, PostgreSQL and WinSW. No zip by default. powershell -ExecutionPolicy Bypass -File .\packaging\build_release.ps1 -CreateVenv ``` Output: -- `packaging/release/netx-win64/` — stage tree -- `packaging/release/NetX--win64.zip` +- `packaging/release/netx-win64/` — stage tree (input to Inno Setup) Inno Setup: @@ -40,29 +47,40 @@ Inno Setup: ISCC.exe packaging\installer\netx.iss ``` -Override version in the `.iss` or edit `#define MyAppVersion`. +Override version in the `.iss` or edit `#define MyAppVersion`. +Optional local zip only: `build_release.ps1 -CreateZip` (not for release upload). + +### Local release without GitHub + +GitHub Actions (`.github/workflows/release-windows.yml`, on `v*` tags) runs the same scripts. To release without it: + +1. Install Python **3.12** (same as CI; side by side with other versions is fine, PATH not needed — the build picks it via `py -3.12` or `%LOCALAPPDATA%\Programs\Python\Python312`) and Inno Setup 6. +2. `powershell -ExecutionPolicy Bypass -File .\packaging\publish_release.ps1 -Version x.y.z -SkipGitHub` → `packaging\release\NetX-Setup-x.y.z.exe` + `.sha256`. PostgreSQL and WinSW come from `packaging\cache` (offline once cached). `npm ci` is skipped when `web\node_modules` already matches `package-lock.json`, so a running Vite dev server does not break the build. +3. Publish to Forgejo with `publish_forgejo_release.ps1`, or copy the Setup to offline machines; clients can update from Forgejo only via `NETX_UPDATE_SOURCES=forgejo`. ## End-user install ### Setup.exe 1. Run `NetX-Setup-x.y.z.exe` (admin). -2. Files → `%ProgramFiles%\NetX\` (program root). -3. Data → `%ProgramData%\NetX\` (`.env`, `pgdata`, spool, secrets). -4. Optional post-install task runs `setup_first_run.ps1` (choose bundled vs external DB). -5. Start menu: **Start NetX** / **Stop NetX** / **Open NetX UI**. +2. **First install:** Database page — choose built-in (offline) or external PostgreSQL (host/port/user/password/db; connection must succeed to continue). +3. **Already installed:** if `%ProgramData%\NetX\.env` already has `NETX_DB_MODE`, Setup shows an **Install mode** page: + - **Update** (default) — skip Database page; stop NetX; replace program files; **keep** ProgramData / DB settings. + - **Reinstall** — same Database wizard as first install (`ApplyDatabaseConfig`); does **not** delete ProgramData (use Uninstall → delete data for a wipe). +4. Files → `%ProgramFiles%\NetX\` (program root). +5. Data → `%ProgramData%\NetX\` (`.env`, `pgdata`, spool, secrets). +6. **Tasks page — run mode (exclusive):** *Desktop tray* (default; optional *start when I sign in* → `\NetX\NetXTray` logon task for local Administrators, quiet start without a browser) or *Windows service* (WinSW, runs at boot). Optional *daily automatic updates* (`\NetX\NetXUpdate`). On upgrades the page pre-selects what is actually installed; silent installs leave these alone unless `/TASKS=` is passed (e.g. `/TASKS="runservice,autoupdate"`). +7. Start menu: **NetX** (tray) / **Start NetX** / **Stop NetX** / **Check for updates** / **Reconfigure database** (names follow the Setup language). Autostart and auto-update can also be toggled later from tray → Settings. -### Zip (portable) +Start/stop from the tray or shortcuts go through `packaging\runtime_ctl.ps1`, which drives the Windows service / SYSTEM task when one is installed (stopping its children directly would just get them restarted). Uninstall removes the service and all `\NetX\` scheduled tasks. -1. Unpack anywhere. -2. Marker `.portable` → data root is sibling `NetXData\`. -3. Target needs **Python 3.11+** on PATH unless the zip was built with `-CreateVenv`. -4. Run: +**Elevation (required on fresh PCs):** all end-user entry points (`NetX-Start.cmd`, `NetX-Tray.cmd`, `NetX-Stop.cmd`, `NetX-FirstRun.cmd`, tray, start/stop/setup/update) **self-elevate via UAC**. Program Files installs cannot start reliably as a normal user (`.venv` repair / writes). Approve the UAC prompt once per launch (tray stays elevated afterward). -```powershell -.\packaging\setup_first_run.ps1 -.\packaging\start_netx_app.ps1 -``` +Silent first install (bundled default): `/SILENT /DbMode=bundled` +Silent external: `/SILENT /DbMode=external /DbHost=... /DbPort=5432 /DbUser=... /DbPassword=... /DbName=...` +Silent update over existing data: `/VERYSILENT /NORESTART /InstallMode=update` (also `/SkipDbPage=1`) +Silent reinstall (DB wizard via params): `/VERYSILENT /InstallMode=reinstall /DbMode=bundled` (or external `/DbHost`…) — also `/ForceDbPage=1` +Optional credential key (reuse encrypted NE passwords from another install): `/CredentialSecretKey=...` — omit to auto-generate. ## Database modes @@ -76,15 +94,122 @@ Existing Windows deploys that only set `NETX_DATABASE_URL` keep working: never s ## Manual update +Preferred: run a newer `NetX-Setup-*.exe` and choose **Update** (or silent `/InstallMode=update`) so ProgramData is kept. Setup (elevated) always relinks `.venv` → `python/runtime` after file copy; runtime scripts also self-elevate so Start/Tray work on locked-down fresh PCs. + +Legacy/dev zip (only if you built with `-CreateZip`): + ```powershell -.\packaging\update_netx.ps1 -PackagePath .\NetX-0.3.0-win64.zip +.\packaging\update_netx.ps1 -PackagePath .\NetX-0.4.0-win64.zip ``` Stops services, replaces program folders (`netx_api`, `web`, `packaging`, `postgres`, …), **keeps** the data root, restarts. Schema migrations still run via Alembic on API start. -## Auto-update (reserved) +## Check / apply updates -See `manifest.example.json`. Future: set `NETX_UPDATE_URL` + `NETX_UPDATE_CHANNEL`; a checker will download the zip and call `update_netx.ps1`. Not implemented in this phase. +**Defaults (no `.env` needed):** probe **GitHub** (`hansjone/netx`) and Forgejo mirror (`https://git.avelo.top/hansjone/netx`), then use the **newest reachable** version. Same version → prefer GitHub. + +```env +# Optional overrides in ProgramData\NetX\.env +# NETX_UPDATE_SOURCES=github,forgejo +# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest +# NETX_UPDATE_GITHUB_REPO=hansjone/netx +# NETX_UPDATE_URL=https://cdn.example.com/netx/manifest.json +# NETX_UPDATE_TOKEN=****** +``` + +| Variable | Role | +|----------|------| +| `NETX_UPDATE_FORGEJO_URL` | Forgejo/Gitea `releases/latest` API (default: git.avelo.top mirror) | +| `NETX_UPDATE_GITHUB_REPO` | GitHub `owner/repo` (default `hansjone/netx`) | +| `NETX_UPDATE_SOURCES` | Probe order / tie-break order, e.g. `github,forgejo`. Tray → Settings → Update source sets it: *GitHub (default)* = empty (GitHub + git.avelo.top fallback), *git.avelo.top* = `forgejo` only | +| `NETX_UPDATE_URL` | Optional custom JSON manifest | + +```powershell +.\packaging\check_update.ps1 +.\packaging\check_update.ps1 -Apply +``` + +Both release hosts must publish the same **Setup.exe and `.sha256` sidecar**. `check_update.ps1` prefers `NetX-Setup-*.exe` (legacy `win64.zip` is still accepted); it verifies the Setup against a manifest hash or the adjacent checksum asset. Old releases without a sidecar remain supported with a warning. **Code mirror alone is not enough** — Forgejo pull-mirror syncs git/tags only; release assets are uploaded separately. + +### Maintainer release checklist (Windows) + +Release from a clean, version-matched checkout in this order: + +1. **Build and publish GitHub** — ` .\packaging\publish_release.ps1 -Version x.y.z` builds, verifies and uploads Setup.exe plus SHA-256. +2. **Forgejo** — publish the same Setup.exe and checksum sidecar (default: public domain `https://git.avelo.top`): + +```powershell +# One-time: User env var (never commit the token) +# Forgejo → Settings → Applications → Generate New Token (repo write) +[Environment]::SetEnvironmentVariable("NETX_FORGEJO_TOKEN", "your_token", "User") +# Restart Cursor / open a new terminal so the agent/scripts see it + +# Default: https://git.avelo.top +.\packaging\publish_forgejo_release.ps1 -Version 0.4.11 + +# Optional when LAN IP is reachable: +# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 -ForgejoBase "http://10.0.0.131:3000" +``` + +| Role | URL | +|------|-----| +| Publish default | `https://git.avelo.top` (`-ForgejoBase` default) | +| Optional LAN | `http://10.0.0.131:3000` | +| Update probe (default) | GitHub + `https://git.avelo.top/.../releases/latest` | + +Verify: +- Confirm each release has `NetX-Setup-x.y.z.exe` and `NetX-Setup-x.y.z.exe.sha256`. +- https://git.avelo.top/hansjone/netx/releases +- Probe: `.\packaging\check_update.ps1` → both `github` and `forgejo` reachable with the new version + +Token: `NETX_FORGEJO_TOKEN` (or `FORGEJO_TOKEN`). + +## Tray & autostart + +```powershell +# System tray: Start / Stop / Open UI / Check updates (self-elevates) +.\packaging\netx_tray.ps1 -StartOnLaunch + +# Start tray at Windows logon — Scheduled Task with RunLevel Highest +# (replaces legacy HKCU\Run which cannot elevate on fresh installs) +.\packaging\install_autostart.ps1 +# Remove: .\packaging\install_autostart.ps1 -Remove +``` + +## Windows Service (admin) + +Uses [WinSW](https://github.com/winsw/winsw) (downloaded on first install into `packaging/cache`). +`service_run.ps1` probes `/health` and restarts children after consecutive failures. + +```powershell +# Elevated PowerShell +.\packaging\install_service.ps1 -Start +# Uninstall: .\packaging\install_service.ps1 -Uninstall + +# Fallback without WinSW binary management: SYSTEM scheduled task at startup +.\packaging\install_service.ps1 -Mode task -Start +``` + +## Silent auto-update + +```powershell +# Writes NETX_UPDATE_AUTO=true and registers a daily task (default 03:30, RunLevel Highest) +.\packaging\install_update_task.ps1 + +# Manual silent path (only applies when NETX_UPDATE_AUTO=true; self-elevates) +.\packaging\check_update.ps1 -Apply -Quiet -AutoOnly +``` + +Tray also auto-applies on launch when `NETX_UPDATE_AUTO=true`. + +## Code signing (optional, publisher machine) + +```powershell +.\packaging\sign_release.ps1 -Files .\packaging\release\NetX-Setup-0.4.0.exe -Thumbprint +# or: -PfxPath .\certs\code.pfx -PfxPassword *** +``` + +Needs `signtool.exe` (Windows SDK) and a real code-signing certificate. Without a trusted cert, SmartScreen may still warn. ## Layout reminder diff --git a/packaging/_common.ps1 b/packaging/_common.ps1 index 4a1ef66..355593a 100644 --- a/packaging/_common.ps1 +++ b/packaging/_common.ps1 @@ -1,4 +1,4 @@ -# Shared path helpers for Windows packaging scripts. +# Shared path helpers for Windows packaging scripts. # Dot-source: . "$PSScriptRoot\_common.ps1" $ErrorActionPreference = "Stop" @@ -114,7 +114,33 @@ function Write-DotEnvValue { if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } - Set-Content -Path $Path -Value ($lines -join "`r`n") -Encoding utf8 + $text = ($lines -join "`r`n") + try { + Set-Content -LiteralPath $Path -Value $text -Encoding utf8 + } catch { + # Admin-created .env is often Users:RX only — repair ACL then retry once. + $root = $dir + if ((Split-Path -Leaf $dir) -eq "NetX" -or $dir -match '\\NetX$') { + $root = $dir + } else { + $parent = Split-Path -Parent $dir + if ($parent -and ((Split-Path -Leaf $parent) -eq "NetX")) { $root = $parent } + } + $null = Ensure-NetxDataAcl -DataRoot $root -Quiet + try { + # Reset .env ACL explicitly if still blocked. + $icacls = Join-Path $env:SystemRoot "System32\icacls.exe" + if (Test-Path -LiteralPath $icacls) { + & $icacls $Path /grant "*S-1-5-32-545:M" /C /Q 2>$null | Out-Null + } + Set-Content -LiteralPath $Path -Value $text -Encoding utf8 + } catch { + throw (New-Object System.UnauthorizedAccessException( + ("access_denied: cannot write {0}. Run Start Menu → NetX → Reconfigure database as Administrator, or: icacls `"{1}`" /grant Users:(OI)(CI)M /T" -f $Path, $root), + $_.Exception + )) + } + } } function Get-DbMode { @@ -135,3 +161,522 @@ function Import-NetxEnvFile { } return $map } + +function ConvertTo-NetxFsPath([string]$Path) { + # Forward slashes work in .env and most Windows APIs via Python pathlib. + return ($Path -replace '\\', '/') +} + +function Get-NetxDataEnvMap { + param([string]$DataRoot) + $d = $DataRoot + return @{ + "NETX_AUTH_MCP_TOKEN_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\mcp_token")) + "NETX_AUTH_SECRET_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\jwt_secret")) + "NETX_CREDENTIAL_SECRET_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\credential_secret")) + "NETX_SCHEDULER_HEARTBEAT_PATH" = (ConvertTo-NetxFsPath (Join-Path $d "data\runtime\scheduler_heartbeat.json")) + "NETX_RUN_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\runtime")) + "NETX_BIZ_STATE_SPOOL_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\biz_state_spool")) + "NETX_NE_COLLECTION_DATA_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\ne_collections")) + "NETX_NE_EXEC_JOB_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\ne_exec_jobs")) + "NETX_WEBCRT_DATA_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\webcrt")) + } +} + +function New-NetxFernetKey { + # cryptography.fernet.Fernet.generate_key() == urlsafe_b64encode(os.urandom(32)) + $bytes = New-Object byte[] 32 + $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() + try { + $rng.GetBytes($bytes) + } finally { + $rng.Dispose() + } + return [Convert]::ToBase64String($bytes).Replace('+', '-').Replace('/', '_') +} + +function Ensure-NetxDataAcl { + param( + [Parameter(Mandatory = $true)][string]$DataRoot, + [switch]$Quiet = $false + ) + # Installer creates %ProgramData%\NetX as Administrators. Tray / normal users must + # be able to update .env and runtime files when reconfiguring DB. + if (-not (Test-Path -LiteralPath $DataRoot)) { return $false } + try { + $acl = Get-Acl -LiteralPath $DataRoot + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( + "BUILTIN\Users", + "Modify", + "ContainerInherit,ObjectInherit", + "None", + "Allow" + ) + $acl.SetAccessRule($rule) + Set-Acl -LiteralPath $DataRoot -AclObject $acl + + # Also fix already-created files that only inherited RX for Users. + $icacls = Join-Path $env:SystemRoot "System32\icacls.exe" + if (Test-Path -LiteralPath $icacls) { + & $icacls $DataRoot /grant "*S-1-5-32-545:(OI)(CI)M" /T /C /Q 2>$null | Out-Null + } + if (-not $Quiet) { + Write-Host "==> Data ACL: BUILTIN\Users Modify on $DataRoot" -ForegroundColor DarkGray + } + return $true + } catch { + if (-not $Quiet) { + Write-Host "[WARN] Ensure-NetxDataAcl: $($_.Exception.Message)" -ForegroundColor Yellow + } + return $false + } +} + +function Ensure-NetxDataDirectories { + param([string]$DataRoot) + if (-not (Test-Path -LiteralPath $DataRoot)) { + New-Item -ItemType Directory -Path $DataRoot -Force | Out-Null + } + $subs = @( + "data", "data\auth", "data\runtime", "data\biz_state_spool", + "data\ne_collections", "data\ne_exec_jobs", "data\webcrt", + "backups", "pgdata" + ) + foreach ($sub in $subs) { + $p = Join-Path $DataRoot $sub + if (-not (Test-Path $p)) { + New-Item -ItemType Directory -Path $p -Force | Out-Null + } + } + # Best-effort; succeeds when running elevated (installer / first-run as admin). + $null = Ensure-NetxDataAcl -DataRoot $DataRoot -Quiet +} + +function Test-NetxTcpPortFree { + param([string]$HostName = "127.0.0.1", [int]$Port) + try { + $client = New-Object System.Net.Sockets.TcpClient + $iar = $client.BeginConnect($HostName, $Port, $null, $null) + $ok = $iar.AsyncWaitHandle.WaitOne(400) + if ($ok -and $client.Connected) { + $client.Close() + return $false + } + $client.Close() + return $true + } catch { + return $true + } +} + +function Test-NetxApiHealthy { + param([string]$HostName = "127.0.0.1", [int]$Port = 8890, [int]$TimeoutSec = 2) + try { + $url = "http://${HostName}:${Port}/health" + $r = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec $TimeoutSec -MaximumRedirection 0 + if ($r.StatusCode -ne 200) { return $false } + $body = $r.Content | ConvertFrom-Json -ErrorAction Stop + return ($body.status -eq "ok") + } catch { + return $false + } +} + +function Get-NetxClientHost { + param([string]$BindHost = "") + # A wildcard bind address is not connectable on Windows; probe/open via loopback. + $h = "$BindHost".Trim() + if (-not $h -or $h -in @("0.0.0.0", "::", "[::]", "*")) { return "127.0.0.1" } + return $h +} + +function Open-NetxUrl { + param([Parameter(Mandatory = $true)][string]$Url) + # Packaged scripts run elevated; Start-Process would launch the browser + # elevated too. explorer.exe hands the URL to the user's non-elevated shell. + try { + Start-Process -FilePath (Join-Path $env:SystemRoot "explorer.exe") -ArgumentList "`"$Url`"" | Out-Null + return + } catch {} + try { Start-Process $Url | Out-Null } catch {} +} + +function Get-NetxManagedRunner { + # "service" (WinSW) / "task" (SYSTEM startup task) own the runtime when present; + # killing their children directly would just make service_run.ps1 restart them. + $svc = Get-Service -Name "NetX" -ErrorAction SilentlyContinue + if ($svc -and [string]$svc.StartType -ne "Disabled") { return "service" } + try { + $t = Get-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -ErrorAction Stop + if ([string]$t.State -ne "Disabled") { return "task" } + } catch {} + return "" +} + +function Write-NetxServiceStopFlag { + param([Parameter(Mandatory = $true)][string]$DataRoot) + # service_run.ps1 (the WinSW supervisor) exits when this file appears. + $runtimeDir = Join-Path $DataRoot "data\runtime" + if (-not (Test-Path $runtimeDir)) { return } + Set-Content -Path (Join-Path $runtimeDir "service.stop") -Value (Get-Date -Format o) -Encoding ascii -ErrorAction SilentlyContinue +} + +function Wait-NetxApiHealthy { + param( + [string]$HostName = "127.0.0.1", + [int]$Port = 8890, + [int]$TimeoutSec = 180, + [int]$PollMs = 500 + ) + $deadline = (Get-Date).AddSeconds($TimeoutSec) + while ((Get-Date) -lt $deadline) { + if (Test-NetxApiHealthy -HostName $HostName -Port $Port -TimeoutSec 2) { + return $true + } + Start-Sleep -Milliseconds $PollMs + } + return $false +} + +function ConvertTo-NetxProcessArgument { + param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) + # Start-Process joins string[] args with spaces and does NOT quote values that + # contain spaces (e.g. C:\Program Files\NetX). Always emit one argv token. + if ($Value -match '[\s"]') { + return '"' + ($Value -replace '"', '\"') + '"' + } + return $Value +} + +function ConvertTo-NetxProcessArgumentString { + param([Parameter(Mandatory = $true)][string[]]$Arguments) + return (($Arguments | ForEach-Object { ConvertTo-NetxProcessArgument -Value "$_" }) -join " ") +} + +function Start-NetxPowerShell { + param( + [Parameter(Mandatory = $true)][string]$File, + [string[]]$Arguments = @(), + [string]$WorkingDirectory = "", + [ValidateSet("Hidden", "Normal", "Minimized")] + [string]$WindowStyle = "Hidden", + [switch]$Wait = $false, + [switch]$PassThru = $false, + [switch]$RunAs = $false + ) + $parts = @( + "-NoProfile", + "-WindowStyle", $WindowStyle, + "-ExecutionPolicy", "Bypass", + "-File", $File + ) + $Arguments + $sp = @{ + FilePath = "powershell.exe" + ArgumentList = (ConvertTo-NetxProcessArgumentString -Arguments $parts) + } + # -Verb RunAs implies UseShellExecute; cannot combine with -WindowStyle. + if (-not $RunAs) { + $sp.WindowStyle = $WindowStyle + } + if ($WorkingDirectory) { $sp.WorkingDirectory = $WorkingDirectory } + if ($PassThru -or $Wait) { $sp.PassThru = $true } + if ($RunAs) { $sp.Verb = "RunAs" } + $p = Start-Process @sp + if ($Wait -and $p) { Wait-NetxProcessExit -Process $p } + return $p +} + +function Wait-NetxProcessExit { + param([Parameter(Mandatory = $true)]$Process) + # Windows PowerShell 5.1 Start-Process -Wait also waits for every descendant; the + # postgres / NetX processes a start script leaves running would block it forever. + $null = $Process.Handle # keep a handle so ExitCode survives the exit + $Process.WaitForExit() +} + +function Test-NetxIsAdmin { + try { + $id = [Security.Principal.WindowsIdentity]::GetCurrent() + $p = New-Object Security.Principal.WindowsPrincipal($id) + return $p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) + } catch { + return $false + } +} + +function ConvertTo-NetxRelaunchArgumentList { + param([System.Collections.IDictionary]$BoundParameters) + $parts = [System.Collections.Generic.List[string]]::new() + if (-not $BoundParameters) { return @() } + foreach ($key in $BoundParameters.Keys) { + $val = $BoundParameters[$key] + if ($val -is [System.Management.Automation.SwitchParameter]) { + if ($val.IsPresent) { [void]$parts.Add("-$key") } + continue + } + if ($val -is [bool]) { + if ($val) { [void]$parts.Add("-$key") } + continue + } + if ($null -eq $val) { continue } + $s = [string]$val + if ($s -eq "") { continue } + [void]$parts.Add("-$key") + [void]$parts.Add($s) + } + return @($parts) +} + +function Assert-NetxAdminOrRelaunch { + <# + .SYNOPSIS + End-user packaged scripts must run elevated (Program Files + ProgramData). + If not admin, re-launch the same script via UAC and exit with child exit code. + #> + param( + [Parameter(Mandatory = $true)][string]$ScriptPath, + [System.Collections.IDictionary]$BoundParameters = @{}, + [string]$WorkingDirectory = "", + [ValidateSet("Hidden", "Normal", "Minimized")] + [string]$WindowStyle = "Normal" + ) + if (Test-NetxIsAdmin) { return } + + if ($env:NETX_ELEVATION_RELAUNCH -eq "1") { + throw "admin_required: still not elevated after UAC. Run NetX-Start / NetX-Tray as Administrator." + } + + if (-not (Test-Path -LiteralPath $ScriptPath)) { + throw "elevation_script_missing: $ScriptPath" + } + $wd = $WorkingDirectory + if (-not $wd) { $wd = Split-Path -Parent $ScriptPath } + + # Window style belongs on powershell.exe argv, not Start-Process (incompatible with -Verb RunAs). + $parts = @( + "-NoProfile", + "-WindowStyle", $WindowStyle, + "-ExecutionPolicy", "Bypass", + "-File", $ScriptPath + ) + (ConvertTo-NetxRelaunchArgumentList -BoundParameters $BoundParameters) + + $prevRel = $env:NETX_ELEVATION_RELAUNCH + $env:NETX_ELEVATION_RELAUNCH = "1" + try { + $p = Start-Process -FilePath "powershell.exe" ` + -ArgumentList (ConvertTo-NetxProcessArgumentString -Arguments $parts) ` + -WorkingDirectory $wd ` + -Verb RunAs -PassThru + if ($p) { Wait-NetxProcessExit -Process $p } + } catch { + if ($null -ne $prevRel) { $env:NETX_ELEVATION_RELAUNCH = $prevRel } else { Remove-Item Env:NETX_ELEVATION_RELAUNCH -ErrorAction SilentlyContinue } + throw ("admin_required: UAC cancelled or elevation failed. Run NetX as Administrator. {0}" -f $_.Exception.Message) + } + if ($null -ne $prevRel) { $env:NETX_ELEVATION_RELAUNCH = $prevRel } else { Remove-Item Env:NETX_ELEVATION_RELAUNCH -ErrorAction SilentlyContinue } + if ($null -eq $p) { + throw "admin_required: elevation failed (no process)" + } + $code = 0 + if ($null -ne $p.ExitCode) { $code = [int]$p.ExitCode } + exit $code +} + +function Get-NetxSystemPython { + # Prefer a real interpreter; skip the WindowsApps Store stub. + $candidates = @() + foreach ($cmd in @("python", "python3")) { + $c = Get-Command $cmd -ErrorAction SilentlyContinue + if ($c -and $c.Source -and ($c.Source -notmatch '\\WindowsApps\\')) { + $candidates += $c.Source + } + } + $pyLauncher = Get-Command "py" -ErrorAction SilentlyContinue + if ($pyLauncher -and $pyLauncher.Source -and ($pyLauncher.Source -notmatch '\\WindowsApps\\')) { + try { + $resolved = (& $pyLauncher.Source -3 -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() + if ($resolved -and (Test-Path -LiteralPath $resolved)) { + $candidates += $resolved + } + } catch {} + } + foreach ($p in @( + "$env:LOCALAPPDATA\Python\pythoncore-3.14-64\python.exe", + "$env:LOCALAPPDATA\Programs\Python\Python312\python.exe", + "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", + "$env:ProgramFiles\Python312\python.exe", + "$env:ProgramFiles\Python311\python.exe" + )) { + if ($p -and (Test-Path $p)) { $candidates += $p } + } + foreach ($p in ($candidates | Select-Object -Unique)) { + try { + $v = & $p -c "import sys; print('%d.%d'%sys.version_info[:2])" 2>$null + if ($v -match '^(3\.(1[1-9]|[2-9]\d))$') { return $p } + } catch {} + } + return $null +} + +function Get-NetxBundledPythonRoot { + param([Parameter(Mandatory = $true)][string]$ProgramRoot) + return Join-Path $ProgramRoot "python\runtime" +} + +function Repair-NetxShippedVenv { + param([Parameter(Mandatory = $true)][string]$ProgramRoot) + $cfgPath = Join-Path $ProgramRoot ".venv\pyvenv.cfg" + $rtRoot = Get-NetxBundledPythonRoot -ProgramRoot $ProgramRoot + $rtPy = Join-Path $rtRoot "python.exe" + if (-not (Test-Path -LiteralPath $rtPy)) { return $false } + if (-not (Test-Path -LiteralPath $cfgPath)) { return $false } + + $rtRootAbs = (Resolve-Path -LiteralPath $rtRoot).Path + $rtPyAbs = (Resolve-Path -LiteralPath $rtPy).Path + $ver = "3.14.0" + try { + $verOut = & $rtPyAbs -c "import sys; print('%d.%d.%d' % sys.version_info[:3])" 2>$null + if ($verOut) { $ver = ($verOut | Out-String).Trim() } + } catch {} + + $lines = @( + "home = $rtRootAbs", + "include-system-site-packages = false", + "version = $ver", + "executable = $rtPyAbs" + ) + $desired = ($lines -join "`r`n") + # Skip write when already correct — Users cannot modify Program Files after Setup. + try { + $cur = ([IO.File]::ReadAllText($cfgPath) -replace "`r`n", "`n" -replace "`r", "`n").Trim() + $want = ($desired -replace "`r`n", "`n" -replace "`r", "`n").Trim() + if ($cur -eq $want) { + return $true + } + } catch {} + + try { + Set-Content -LiteralPath $cfgPath -Value $desired -Encoding ascii -ErrorAction Stop + return $true + } catch { + # Non-elevated tray/start: leave cfg as-is; caller may still run if paths happen to work. + return $false + } +} + +function Test-NetxVenvRunnable { + param([Parameter(Mandatory = $true)][string]$VenvPython) + if (-not (Test-Path -LiteralPath $VenvPython)) { return $false } + + $outFile = Join-Path $env:TEMP ("netx-venv-out-" + [Guid]::NewGuid().ToString("n") + ".txt") + $errFile = Join-Path $env:TEMP ("netx-venv-err-" + [Guid]::NewGuid().ToString("n") + ".txt") + try { + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $VenvPython + $psi.Arguments = '-c "import encodings, sys; sys.exit(0 if sys.prefix else 1)"' + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + $p = [Diagnostics.Process]::Start($psi) + $stderr = $p.StandardError.ReadToEnd() + $null = $p.StandardOutput.ReadToEnd() + $p.WaitForExit() + if ($stderr -match 'did not find executable|No Python at|Fatal Python error') { + return $false + } + if ($p.ExitCode -ne 0) { return $false } + return $true + } finally { + Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue + } +} + +function Ensure-NetxVenv { + param( + [Parameter(Mandatory = $true)][string]$ProgramRoot, + [switch]$ForcePip = $false + ) + $venvPy = Join-Path $ProgramRoot ".venv\Scripts\python.exe" + $req = Join-Path $ProgramRoot "requirements.txt" + $rtPy = Join-Path (Get-NetxBundledPythonRoot -ProgramRoot $ProgramRoot) "python.exe" + + if (Test-Path -LiteralPath $rtPy) { + try { + $null = Repair-NetxShippedVenv -ProgramRoot $ProgramRoot + } catch { + # Access denied under Program Files when not elevated — ignore if venv already runs. + } + } + + if ((Test-Path -LiteralPath $venvPy) -and -not $ForcePip) { + if (Test-NetxVenvRunnable -VenvPython $venvPy) { + return $venvPy + } + Write-Host "[WARN] Shipped .venv exists but Python cannot start (broken pyvenv.cfg or missing runtime)." -ForegroundColor Yellow + if (Test-Path -LiteralPath $rtPy) { + $repaired = $false + try { + $repaired = [bool](Repair-NetxShippedVenv -ProgramRoot $ProgramRoot) + } catch { + $repaired = $false + } + if ($repaired -and (Test-NetxVenvRunnable -VenvPython $venvPy)) { + Write-Host "==> Repaired .venv to use bundled python/runtime" -ForegroundColor Green + return $venvPy + } + if (-not $repaired) { + throw "venv_needs_admin_repair: pyvenv.cfg still points at the build PC. Re-run Setup (Update) as Administrator, or Start Menu → Reconfigure database once elevated." + } + } + if (-not $ForcePip) { + throw "venv_not_runnable: reinstall NetX Setup (ships python/runtime + .venv) or run repair as admin" + } + } + if (-not (Test-Path $venvPy)) { + $py = Get-NetxSystemPython + if (-not $py) { + throw "python_not_found: install Python 3.11+ (or ship Setup built with -CreateVenv)" + } + Write-Host "==> Creating .venv with $py" + $venvDir = Join-Path $ProgramRoot ".venv" + try { + & $py -m venv $venvDir + } catch { + throw "venv_create_failed: cannot write $venvDir (need admin / ship -CreateVenv). $($_.Exception.Message)" + } + if (-not (Test-Path $venvPy)) { + throw "venv_create_failed: missing $venvPy (Program Files may be read-only without elevation)" + } + $ForcePip = $true + } + if ($ForcePip) { + if (-not (Test-Path $req)) { throw "requirements_missing: $req" } + Write-Host "==> pip install -r requirements.txt" + & $venvPy -m pip install --upgrade pip + & $venvPy -m pip install -r $req + if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } + } + return $venvPy +} + +function Stop-NetxTrayProcesses { + param([string]$ProgramRoot = "") + $hits = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { + $_.CommandLine -and $_.CommandLine -match 'netx_tray\.ps1' + }) + if ($ProgramRoot) { + $esc = [regex]::Escape($ProgramRoot) + $hits = @($hits | Where-Object { $_.CommandLine -match $esc -or $_.CommandLine -match 'netx_tray\.ps1' }) + } + foreach ($p in $hits) { + try { + Stop-Process -Id $p.ProcessId -Force -ErrorAction Stop + Write-Host "Stopped NetX tray PID=$($p.ProcessId)" + } catch {} + } +} + +function ConvertTo-NetxSqlLiteral([string]$Value) { + # Single-quote escaping for PostgreSQL string literals. + return ($Value -replace "'", "''") +} diff --git a/packaging/assets/README.md b/packaging/assets/README.md new file mode 100644 index 0000000..27b599c --- /dev/null +++ b/packaging/assets/README.md @@ -0,0 +1,6 @@ +# NetX packaging assets + +- `netx.ico` 鈥?installer / shortcuts / tray (regenerate with `python gen_icon.py`) +- `netx-256.png` / `netx-64.png` 鈥?previews + +Mark matches `web/public/favicon.svg` (network 鈥淣鈥?nodes on navy). diff --git a/packaging/assets/gen_icon.py b/packaging/assets/gen_icon.py new file mode 100644 index 0000000..e5b96d5 --- /dev/null +++ b/packaging/assets/gen_icon.py @@ -0,0 +1,115 @@ +"""Generate NetX Windows .ico/.png from the brand mark. + +Mirrors web/public/favicon.svg and web/src/layout/BrandMark.tsx (24x24 viewBox): +blue gradient tile, an "X" of links, four nodes around a hub, one node lit cyan. +""" +from __future__ import annotations + +import os +from PIL import Image, ImageDraw + +OUT_DIR = os.path.dirname(os.path.abspath(__file__)) +GRAD_A = (0x3B, 0x82, 0xF6) # #3b82f6 (top-left) +GRAD_B = (0x1D, 0x4E, 0xD8) # #1d4ed8 (bottom-right) +WHITE = (255, 255, 255, 255) +LINK = (255, 255, 255, int(255 * 0.55)) +LIVE = (0x67, 0xE8, 0xF9, 255) # #67e8f9 +SS = 4 # supersample factor + + +def _gradient(px: int) -> Image.Image: + # Diagonal gradient along (0,0)->(24,24), like the SVG userSpaceOnUse gradient. + img = Image.new("RGBA", (px, px)) + pix = img.load() + span = 2 * (px - 1) or 1 + for y in range(px): + for x in range(px): + t = (x + y) / span + pix[x, y] = tuple(round(a + (b - a) * t) for a, b in zip(GRAD_A, GRAD_B)) + (255,) + return img + + +def make(size: int) -> Image.Image: + px = size * SS + u = px / 24.0 + + tile = _gradient(px) + mask = Image.new("L", (px, px), 0) + ImageDraw.Draw(mask).rounded_rectangle([0, 0, px - 1, px - 1], radius=6.5 * u, fill=255) + img = Image.new("RGBA", (px, px), (0, 0, 0, 0)) + img.paste(tile, (0, 0), mask) + + links = Image.new("RGBA", (px, px), (0, 0, 0, 0)) + ld = ImageDraw.Draw(links) + w = max(1, round(1.6 * u)) + for (x1, y1), (x2, y2) in (((7, 7), (17, 17)), ((17, 7), (7, 17))): + ld.line([(x1 * u, y1 * u), (x2 * u, y2 * u)], fill=LINK, width=w) + for cx, cy in ((x1, y1), (x2, y2)): # round caps + r = w / 2 + ld.ellipse([cx * u - r, cy * u - r, cx * u + r, cy * u + r], fill=LINK) + img = Image.alpha_composite(img, links) + + d = ImageDraw.Draw(img) + + def node(cx: float, cy: float, r: float, fill: tuple[int, int, int, int]) -> None: + d.ellipse([(cx - r) * u, (cy - r) * u, (cx + r) * u, (cy + r) * u], fill=fill) + + node(12, 12, 2.6, WHITE) + node(7, 7, 1.9, WHITE) + node(7, 17, 1.9, WHITE) + node(17, 17, 1.9, WHITE) + node(17, 7, 2.1, LIVE) + return img.resize((size, size), Image.LANCZOS) + + +WIZ_TOP = (0x0A, 0x12, 0x22) # near-black navy +WIZ_BOTTOM = (0x10, 0x2A, 0x55) # deep blue + + +def make_wizard(w: int, h: int) -> Image.Image: + """Inno Setup side image: dark navy gradient, faint link mesh, brand mark.""" + img = Image.new("RGBA", (w, h)) + pix = img.load() + for y in range(h): + t = y / max(1, h - 1) + row = tuple(round(a + (b - a) * t) for a, b in zip(WIZ_TOP, WIZ_BOTTOM)) + (255,) + for x in range(w): + pix[x, y] = row + + mesh = Image.new("RGBA", (w * SS, h * SS), (0, 0, 0, 0)) + md = ImageDraw.Draw(mesh) + pts = [(0.12, 0.62), (0.42, 0.70), (0.80, 0.60), (0.28, 0.86), (0.66, 0.90), (0.95, 0.80), (0.05, 0.95)] + pts = [(x * w * SS, y * h * SS) for x, y in pts] + line = (0x60, 0xA5, 0xFA, 46) + for a, b in ((0, 1), (1, 2), (0, 3), (1, 3), (1, 4), (2, 4), (2, 5), (4, 5), (3, 6)): + md.line([pts[a], pts[b]], fill=line, width=max(1, SS)) + for i, (x, y) in enumerate(pts): + r = (2.4 if i != 2 else 3.0) * SS * w / 164 + fill = (0x67, 0xE8, 0xF9, 150) if i == 2 else (0xBF, 0xDB, 0xFE, 90) + md.ellipse([x - r, y - r, x + r, y + r], fill=fill) + img = Image.alpha_composite(img, mesh.resize((w, h), Image.LANCZOS)) + + side = round(w * 0.42) + mark = make(side) + img.alpha_composite(mark, ((w - side) // 2, round(h * 0.26))) + return img.convert("RGB") + + +def main() -> None: + sizes = [16, 20, 24, 32, 40, 48, 64, 128, 256] + images = {s: make(s) for s in sizes} + ico_path = os.path.join(OUT_DIR, "netx.ico") + images[256].save(ico_path, format="ICO", sizes=[(s, s) for s in sizes], bitmap_format="bmp", + append_images=[images[s] for s in sizes[:-1]]) + images[256].save(os.path.join(OUT_DIR, "netx-256.png")) + images[64].save(os.path.join(OUT_DIR, "netx-64.png")) + # Inno Setup wizard images at 100/125/150/200% DPI (modern style base sizes). + for pct in (100, 125, 150, 200): + k = pct / 100 + make_wizard(round(164 * k), round(314 * k)).save(os.path.join(OUT_DIR, f"wizard-{pct}.png")) + make(round(55 * k)).save(os.path.join(OUT_DIR, f"wizard-small-{pct}.png")) + print(f"wrote {ico_path} ({os.path.getsize(ico_path)} bytes)") + + +if __name__ == "__main__": + main() diff --git a/packaging/assets/netx-256.png b/packaging/assets/netx-256.png new file mode 100644 index 0000000..073acea Binary files /dev/null and b/packaging/assets/netx-256.png differ diff --git a/packaging/assets/netx-64.png b/packaging/assets/netx-64.png new file mode 100644 index 0000000..0bbcfd1 Binary files /dev/null and b/packaging/assets/netx-64.png differ diff --git a/packaging/assets/netx.ico b/packaging/assets/netx.ico new file mode 100644 index 0000000..295a96d Binary files /dev/null and b/packaging/assets/netx.ico differ diff --git a/packaging/assets/wizard-100.png b/packaging/assets/wizard-100.png new file mode 100644 index 0000000..040de08 Binary files /dev/null and b/packaging/assets/wizard-100.png differ diff --git a/packaging/assets/wizard-125.png b/packaging/assets/wizard-125.png new file mode 100644 index 0000000..f4e211f Binary files /dev/null and b/packaging/assets/wizard-125.png differ diff --git a/packaging/assets/wizard-150.png b/packaging/assets/wizard-150.png new file mode 100644 index 0000000..94cc820 Binary files /dev/null and b/packaging/assets/wizard-150.png differ diff --git a/packaging/assets/wizard-200.png b/packaging/assets/wizard-200.png new file mode 100644 index 0000000..a2da6b3 Binary files /dev/null and b/packaging/assets/wizard-200.png differ diff --git a/packaging/assets/wizard-small-100.png b/packaging/assets/wizard-small-100.png new file mode 100644 index 0000000..65ba4ea Binary files /dev/null and b/packaging/assets/wizard-small-100.png differ diff --git a/packaging/assets/wizard-small-125.png b/packaging/assets/wizard-small-125.png new file mode 100644 index 0000000..a67a6a2 Binary files /dev/null and b/packaging/assets/wizard-small-125.png differ diff --git a/packaging/assets/wizard-small-150.png b/packaging/assets/wizard-small-150.png new file mode 100644 index 0000000..531f9a1 Binary files /dev/null and b/packaging/assets/wizard-small-150.png differ diff --git a/packaging/assets/wizard-small-200.png b/packaging/assets/wizard-small-200.png new file mode 100644 index 0000000..eb82868 Binary files /dev/null and b/packaging/assets/wizard-small-200.png differ diff --git a/packaging/build_release.ps1 b/packaging/build_release.ps1 index a347f03..722f9d7 100644 --- a/packaging/build_release.ps1 +++ b/packaging/build_release.ps1 @@ -1,10 +1,14 @@ -param( +param( [string]$Version = "", [string]$OutDir = "", [switch]$SkipWebBuild = $false, [switch]$SkipPostgresDownload = $false, + # Zip is no longer published; stage + Setup.exe only. Opt-in for local/dev testing. + [switch]$CreateZip = $false, + # Deprecated no-op (zip is off by default). Kept so older scripts that pass -SkipZip still run. [switch]$SkipZip = $false, - [switch]$CreateVenv = $false + # Default on: offline Setup must ship .venv so target PCs never pip-install. + [switch]$CreateVenv = $true ) $ErrorActionPreference = "Stop" @@ -32,13 +36,25 @@ if (-not $SkipWebBuild) { throw "npm_not_found" } Write-Host "==> npm build" - if (-not (Test-Path (Join-Path $webRoot "node_modules"))) { - & npm.cmd install --prefix $webRoot - if ($LASTEXITCODE -ne 0) { throw "npm_install_failed" } + Push-Location $webRoot + try { + # npm ci wipes node_modules first; on a dev box a running Vite locks native + # .node files and leaves it half-deleted. Skip when installed deps match the lockfile. + $installedLock = Join-Path $webRoot "node_modules\.package-lock.json" + $lock = Join-Path $webRoot "package-lock.json" + if ((Test-Path $installedLock) -and + (Get-Item $installedLock).LastWriteTimeUtc -ge (Get-Item $lock).LastWriteTimeUtc) { + Write-Host "==> node_modules up to date with package-lock.json - skip npm ci" + } else { + & npm.cmd ci + if ($LASTEXITCODE -ne 0) { throw "npm_install_failed" } + } + # Release packaging skips tsc (dev CI still uses `npm run build`). + & npm.cmd run build:release + if ($LASTEXITCODE -ne 0) { throw "web_build_failed" } + } finally { + Pop-Location } - # Release packaging skips tsc (dev CI still uses `npm run build`). - & npm.cmd run build:release --prefix $webRoot - if ($LASTEXITCODE -ne 0) { throw "web_build_failed" } } if (-not (Test-Path (Join-Path $dist "index.html"))) { throw "web_dist_missing: build web/ or drop -SkipWebBuild" @@ -51,6 +67,22 @@ if (-not $SkipPostgresDownload) { } } +# Bundle WinSW so offline installs can register a Windows Service without GitHub. +# .NET Framework 4.6.1 build (~0.6MB; in-box on Win10 1607+ / Server 2016+) instead of +# the self-contained x64 build (~18MB). +$winswCache = Join-Path $PSScriptRoot "cache\WinSW.NET461.exe" +$winswShip = Join-Path $PSScriptRoot "winsw\WinSW.exe" +if (-not (Test-Path $winswShip)) { + if (-not (Test-Path $winswCache)) { + $winswUrl = "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW.NET461.exe" + Write-Host "==> Downloading WinSW for offline bundle: $winswUrl" + New-Item -ItemType Directory -Path (Split-Path $winswCache -Parent) -Force | Out-Null + Invoke-WebRequest -Uri $winswUrl -OutFile $winswCache -UseBasicParsing + } + New-Item -ItemType Directory -Path (Split-Path $winswShip -Parent) -Force | Out-Null + Copy-Item -Path $winswCache -Destination $winswShip -Force +} + # Flat layout = same as repo so scripts\start_netx.ps1 works unchanged. foreach ($d in @("netx_api", "alembic", "scripts")) { Copy-Item -Path (Join-Path $repo $d) -Destination (Join-Path $stage $d) -Recurse -Force @@ -66,26 +98,90 @@ $webOut = Join-Path $stage "web\dist" New-Item -ItemType Directory -Path (Split-Path $webOut -Parent) -Force | Out-Null Copy-Item -Path $dist -Destination $webOut -Recurse -Force +function Write-Utf8BomFile { + param([string]$Path) + # Windows PowerShell 4/5 (Server 2012+) mis-parses UTF-8 without BOM when scripts contain CJK. + $utf8Bom = New-Object System.Text.UTF8Encoding $true + $bytes = [IO.File]::ReadAllBytes($Path) + $hasBom = ($bytes.Length -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) + $text = if ($hasBom) { + [Text.Encoding]::UTF8.GetString($bytes, 3, $bytes.Length - 3) + } else { + [Text.Encoding]::UTF8.GetString($bytes) + } + $text = $text -replace "`r`n", "`n" -replace "`n", "`r`n" + if (-not $text.EndsWith("`r`n")) { $text += "`r`n" } + [IO.File]::WriteAllText($Path, $text, $utf8Bom) +} + $packOut = Join-Path $stage "packaging" New-Item -ItemType Directory -Path $packOut -Force | Out-Null Copy-Item -Path (Join-Path $PSScriptRoot "_common.ps1") -Destination $packOut -Force foreach ($name in @( "download_postgres.ps1", "setup_first_run.ps1", "start_netx_app.ps1", - "stop_netx_app.ps1", "update_netx.ps1", "build_release.ps1", - "README.md", "manifest.example.json" + "stop_netx_app.ps1", "update_netx.ps1", "check_update.ps1", + "launch_shortcut.ps1", "netx_tray.ps1", "install_autostart.ps1", "install_service.ps1", + "service_run.ps1", "service_lifecycle.ps1", "install_update_task.ps1", "repair_venv.ps1", "runtime_ctl.ps1", "uninstall_prepare.ps1", "uninstall_delete_data.ps1", "sign_release.ps1", + "build_release.ps1", "publish_release.ps1", "publish_forgejo_release.ps1", "README.md", "manifest.example.json" )) { $src = Join-Path $PSScriptRoot $name if (Test-Path $src) { Copy-Item $src (Join-Path $packOut $name) -Force } } +Get-ChildItem -Path $packOut -Filter *.ps1 -File | ForEach-Object { Write-Utf8BomFile -Path $_.FullName } +Get-ChildItem -Path (Join-Path $stage "scripts") -Filter *.ps1 -File -ErrorAction SilentlyContinue | + ForEach-Object { Write-Utf8BomFile -Path $_.FullName } Copy-Item -Path (Join-Path $PSScriptRoot "config") -Destination (Join-Path $packOut "config") -Recurse -Force Copy-Item -Path (Join-Path $PSScriptRoot "installer") -Destination (Join-Path $packOut "installer") -Recurse -Force +$assetsSrc = Join-Path $PSScriptRoot "assets" +if (Test-Path $assetsSrc) { + Copy-Item -Path $assetsSrc -Destination (Join-Path $packOut "assets") -Recurse -Force +} +$winswOut = Join-Path $packOut "winsw" +New-Item -ItemType Directory -Path $winswOut -Force | Out-Null +if (Test-Path $winswShip) { + Copy-Item -Path $winswShip -Destination (Join-Path $winswOut "WinSW.exe") -Force + Write-Host "==> Bundled WinSW for offline service install" +} else { + Write-Host "[WARN] WinSW missing - offline service install will fail" -ForegroundColor Yellow +} New-Item -ItemType Directory -Path (Join-Path $packOut "postgres") -Force | Out-Null Copy-Item -Path (Join-Path $PSScriptRoot "postgres\README.md") -Destination (Join-Path $packOut "postgres\README.md") -Force if (Test-Path (Join-Path $pgsql "bin\pg_ctl.exe")) { - Write-Host "==> Copying bundled PostgreSQL" - New-Item -ItemType Directory -Path (Join-Path $stage "postgres") -Force | Out-Null - Copy-Item -Path $pgsql -Destination (Join-Path $stage "postgres\pgsql") -Recurse -Force + Write-Host '==> Copying bundled PostgreSQL (bin/lib/share; skip doc/)' + $pgStage = Join-Path $stage "postgres\pgsql" + New-Item -ItemType Directory -Path $pgStage -Force | Out-Null + foreach ($sub in @("bin", "lib", "share")) { + $srcSub = Join-Path $pgsql $sub + if (Test-Path $srcSub) { + Copy-Item -Path $srcSub -Destination (Join-Path $pgStage $sub) -Recurse -Force + } + } + # Runtime-only subset: NetX uses postgres/pg_ctl/initdb/psql/pg_dump (+ restore/isready + # for manual ops). Drop StackBuilder (wx GUI), dev/test tools, static libs, pgxs and + # non-Chinese message catalogs. + $pgBinKeep = @("postgres", "pg_ctl", "initdb", "psql", "pg_dump", "pg_dumpall", "pg_restore", "pg_isready") + Get-ChildItem (Join-Path $pgStage "bin") -File | Where-Object { + ($_.Extension -eq ".exe" -and $pgBinKeep -notcontains $_.BaseName) -or $_.Name -like "wx*.dll" + } | Remove-Item -Force + Get-ChildItem (Join-Path $pgStage "lib") -File -Filter *.lib | Remove-Item -Force + Remove-Item -Recurse -Force (Join-Path $pgStage "lib\pgxs") -ErrorAction SilentlyContinue + Get-ChildItem (Join-Path $pgStage "share\locale") -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notlike "zh_CN*" } | Remove-Item -Recurse -Force + # The EDB binaries link the VC++ 2015-2022 runtime (vcruntime140, msvcp140 via ICU), which a + # clean Windows install does not have. Deploy it app-local (allowed by the VC++ redist terms) + # so the bundled PostgreSQL starts on fresh / offline machines without the redistributable. + $sys32 = Join-Path $env:SystemRoot "System32" + foreach ($dll in @("vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll", "msvcp140_1.dll", "msvcp140_2.dll")) { + $src = Join-Path $sys32 $dll + if (-not (Test-Path -LiteralPath $src)) { throw "vc_runtime_missing_on_build_machine: $src (install the VC++ 2015-2022 x64 redistributable)" } + Copy-Item -LiteralPath $src -Destination (Join-Path $pgStage "bin\$dll") -Force + } + Get-ChildItem -Path $pgsql -File -ErrorAction SilentlyContinue | ForEach-Object { + Copy-Item -Path $_.FullName -Destination (Join-Path $pgStage $_.Name) -Force + } +} else { + throw "bundled_postgres_missing: Setup.exe must ship postgres for offline install. Run download_postgres.ps1 (build machine only)." } $builtAt = (Get-Date).ToUniversalTime().ToString("o") @@ -99,26 +195,122 @@ $builtAt = (Get-Date).ToUniversalTime().ToString("o") Set-Content -Path (Join-Path $stage ".portable") -Value "1" -Encoding ascii +# Root .cmd launchers (Explorer / Start Menu friendly; ASCII-only). +$cmdSrc = Join-Path $PSScriptRoot "cmd" +foreach ($cmdName in @("NetX-FirstRun.cmd", "NetX-Start.cmd", "NetX-Tray.cmd", "NetX-Stop.cmd")) { + $c = Join-Path $cmdSrc $cmdName + if (Test-Path $c) { + Copy-Item -Path $c -Destination (Join-Path $stage $cmdName) -Force + } +} + if ($CreateVenv) { - Write-Host "==> Creating .venv in stage (requires Python 3.11+ on PATH)" - $py = (Get-Command python -ErrorAction SilentlyContinue) - if (-not $py) { throw "python_not_found_for_venv" } - & $py.Source -m venv (Join-Path $stage ".venv") - & (Join-Path $stage ".venv\Scripts\python.exe") -m pip install --upgrade pip - & (Join-Path $stage ".venv\Scripts\python.exe") -m pip install -r (Join-Path $stage "requirements.txt") + Write-Host "==> Creating portable python/runtime + .venv (must not reference build-machine paths)" + # Releases pin 3.12 (same as CI); prefer it even when another version is first on PATH. + $pyPath = $null + $pyLauncher = Get-Command "py" -ErrorAction SilentlyContinue + if ($pyLauncher -and $pyLauncher.Source -notmatch '\\WindowsApps\\') { + try { + $resolved = (& $pyLauncher.Source -3.12 -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() + if ($resolved -and (Test-Path -LiteralPath $resolved)) { $pyPath = $resolved } + } catch {} + } + if (-not $pyPath) { + foreach ($p in @("$env:LOCALAPPDATA\Programs\Python\Python312\python.exe", "$env:ProgramFiles\Python312\python.exe")) { + if (Test-Path -LiteralPath $p) { $pyPath = $p; break } + } + } + if (-not $pyPath) { $pyPath = Get-NetxSystemPython } + if (-not $pyPath) { throw "python_not_found_for_venv: need Python 3.12.x (not WindowsApps stub)" } + $pyVersion = (& $pyPath -c "import sys; print('%d.%d' % sys.version_info[:2])" 2>$null | Out-String).Trim() + if ($pyVersion -ne '3.12') { throw "python_version_mismatch: Windows releases require Python 3.12.x for reproducible runtimes; found $pyVersion at $pyPath" } + Write-Host " Build Python: $pyPath" + + $prefix = (& $pyPath -c "import sys; print(sys.base_prefix)" 2>$null | Out-String).Trim() + if (-not $prefix -or -not (Test-Path -LiteralPath $prefix)) { + throw "python_base_prefix_not_found: $prefix" + } + $rtDir = Join-Path $stage "python\runtime" + if (Test-Path $rtDir) { Remove-Item -Recurse -Force $rtDir } + New-Item -ItemType Directory -Path $rtDir -Force | Out-Null + + Write-Host "==> Copying Python stdlib/runtime to $rtDir (exclude base site-packages)" + $spEx = Join-Path $prefix "Lib\site-packages" + $robocopy = Join-Path $env:SystemRoot "System32\robocopy.exe" + if (-not (Test-Path -LiteralPath $robocopy)) { throw "robocopy_not_found" } + # Headless server: drop docs, the stdlib test suite and Tk/IDLE (~100MB unpacked). + # Keep ensurepip - venv creation needs it. Same result on CI and local builds. + $xdRuntime = @("Doc", "tcl", "Tools", "Lib\test", "Lib\idlelib", "Lib\tkinter", "Lib\turtledemo") | + ForEach-Object { Join-Path $prefix $_ } + # python-*.exe: the GitHub runner's toolcache keeps the full Python installer (~25MB) + # in the prefix; _test*.pyd: stdlib test extensions. + $xfRuntime = @("_tkinter.pyd", "tcl86t.dll", "tk86t.dll", "*.pyc", "python-*.exe", "_test*.pyd", "_ctypes_test.pyd", "xxlimited*.pyd") + & $robocopy $prefix $rtDir /E /XD $spEx __pycache__ @xdRuntime /XF @xfRuntime /NFL /NDL /NJH /NJS /nc /ns /np | Out-Null + if ($LASTEXITCODE -ge 8) { throw "robocopy_python_runtime_failed: exit $LASTEXITCODE" } + + $rtPy = Join-Path $rtDir "python.exe" + if (-not (Test-Path -LiteralPath $rtPy)) { throw "python_runtime_missing: $rtPy" } + + $venvDir = Join-Path $stage ".venv" + if (Test-Path $venvDir) { Remove-Item -Recurse -Force $venvDir } + Write-Host '==> Creating .venv from shipped runtime (--copies)' + & $rtPy -m venv $venvDir --copies + if ($LASTEXITCODE -ne 0) { throw "venv_create_failed" } + + $venvPy = Join-Path $venvDir "Scripts\python.exe" + $null = Repair-NetxShippedVenv -ProgramRoot $stage + if (-not (Test-NetxVenvRunnable -VenvPython $venvPy)) { + throw "venv_not_runnable_after_build: check python/runtime copy" + } + & $venvPy -c "from pathlib import Path; import glob; print(Path.cwd()); print(glob.__file__)" + if ($LASTEXITCODE -ne 0) { throw "python_stdlib_smoke_failed: pathlib/glob import mismatch" } + + # --no-warn-script-location: the PATH check realpath()s every entry and aborts on a + # broken reparse point (e.g. a stale app folder on a detached volume). Irrelevant here. + & $venvPy -m pip install --upgrade pip --no-warn-script-location + & $venvPy -m pip install -r (Join-Path $stage "requirements.txt") --no-warn-script-location + if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } + + $null = Repair-NetxShippedVenv -ProgramRoot $stage + if (-not (Test-NetxVenvRunnable -VenvPython $venvPy)) { + throw "venv_not_runnable_after_pip" + } + & $venvPy -c "from pathlib import Path; import glob; print(Path.cwd()); print(glob.__file__)" + if ($LASTEXITCODE -ne 0) { throw "python_stdlib_smoke_failed_after_pip: pathlib/glob import mismatch" } + + Write-Host "==> Slimming .venv (drop tests / __pycache__ / *.pyc)" + $site = Join-Path $stage ".venv\Lib\site-packages" + if (Test-Path $site) { + Get-ChildItem -Path $site -Recurse -Directory -Force -ErrorAction SilentlyContinue | + Where-Object { + $_.Name -in @('__pycache__', 'tests', 'test', 'testing', 'Tests') -or + $_.FullName -match '\\pandas\\tests(\\|$)' -or + $_.FullName -match '\\numpy\\(_*tests|tests)(\\|$)' -or + $_.FullName -match '\\scipy\\(_*tests|tests)(\\|$)' + } | + Sort-Object { $_.FullName.Length } -Descending | + ForEach-Object { + Remove-Item -LiteralPath $_.FullName -Recurse -Force -ErrorAction SilentlyContinue + } + Get-ChildItem -Path $site -Recurse -Include *.pyc,*.pyo -Force -ErrorAction SilentlyContinue | + Remove-Item -Force -ErrorAction SilentlyContinue + } } Write-Host "==> Stage ready: $stage" -ForegroundColor Green -if (-not $SkipZip) { +if ($SkipZip -and $CreateZip) { + Write-Host "[WARN] -SkipZip ignored because -CreateZip was set" -ForegroundColor Yellow +} +if ($CreateZip) { $zip = Join-Path (Split-Path -Parent $stage) "NetX-$Version-win64.zip" if (Test-Path $zip) { Remove-Item -Force $zip } Compress-Archive -Path $stage -DestinationPath $zip -Force - Write-Host "==> Zip: $zip" -ForegroundColor Green + Write-Host "==> Zip (optional/dev): $zip" -ForegroundColor Yellow } Write-Host "" -Write-Host "Ship options:" -Write-Host " Zip: user unpacks, runs packaging\setup_first_run.ps1 then start_netx_app.ps1" +Write-Host "Ship:" Write-Host " Exe: compile packaging\installer\netx.iss with Inno Setup (needs stage above)" -Write-Host "Python: install 3.11+ on target, or rebuild with -CreateVenv and ship .venv" +Write-Host " (Zip packages are not published; use -CreateZip only for local testing.)" +Write-Host "Python: bundled python/runtime + .venv; target PCs do not need Python installed" diff --git a/packaging/check_update.ps1 b/packaging/check_update.ps1 new file mode 100644 index 0000000..c8ecae8 --- /dev/null +++ b/packaging/check_update.ps1 @@ -0,0 +1,501 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [string]$UpdateUrl = "", + [string]$FallbackUrl = "", + [string]$ForgejoUrl = "", + [string]$GithubRepo = "", + [string]$Channel = "", + [string]$Sources = "", + [switch]$Apply = $false, + [switch]$Quiet = $false, + # Only apply when NETX_UPDATE_AUTO=true (scheduled silent updates). + [switch]$AutoOnly = $false +) + +# Check for a newer NetX Windows package. +# +# Default (no config needed): +# GitHub primary + Forgejo mirror fallback (git.avelo.top). +# Probe every reachable source and pick the highest version; +# on equal versions prefer GitHub (listed first). +# +# Optional overrides: +# NETX_UPDATE_SOURCES=github,forgejo +# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest +# NETX_UPDATE_GITHUB_REPO=hansjone/netx +# NETX_UPDATE_URL=... manifest JSON +# NETX_UPDATE_TOKEN=... private API token + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters ` + -WindowStyle $(if ($Quiet) { "Hidden" } else { "Normal" }) + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$envPath = Join-Path $data ".env" +$map = @{} +if (Test-Path $envPath) { $map = Read-DotEnv -Path $envPath } + +function Get-Cfg([string]$Key, [string]$ParamVal = "") { + if ($ParamVal) { return $ParamVal } + if ($map.ContainsKey($Key) -and $map[$Key]) { return [string]$map[$Key] } + $envVal = [Environment]::GetEnvironmentVariable($Key) + if ($envVal) { return $envVal } + return "" +} + +$UpdateUrl = Get-Cfg "NETX_UPDATE_URL" $UpdateUrl +$FallbackUrl = Get-Cfg "NETX_UPDATE_FALLBACK_URL" $FallbackUrl +$ForgejoUrl = Get-Cfg "NETX_UPDATE_FORGEJO_URL" $ForgejoUrl +if (-not $ForgejoUrl) { + $ForgejoUrl = "https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest" +} +$GithubRepo = Get-Cfg "NETX_UPDATE_GITHUB_REPO" $GithubRepo +if (-not $GithubRepo) { $GithubRepo = "hansjone/netx" } +$Channel = Get-Cfg "NETX_UPDATE_CHANNEL" $Channel +if (-not $Channel) { $Channel = "stable" } +$Sources = Get-Cfg "NETX_UPDATE_SOURCES" $Sources +$UpdateToken = Get-Cfg "NETX_UPDATE_TOKEN" "" + +$current = Get-NetxVersion -ProgramRoot $prog + +function Compare-SemVer { + param([string]$A, [string]$B) + $pa = @($A.TrimStart('v', 'V').Split('.') | ForEach-Object { try { [int]$_ } catch { 0 } }) + $pb = @($B.TrimStart('v', 'V').Split('.') | ForEach-Object { try { [int]$_ } catch { 0 } }) + while ($pa.Count -lt 3) { $pa += 0 } + while ($pb.Count -lt 3) { $pb += 0 } + for ($i = 0; $i -lt 3; $i++) { + if ($pa[$i] -lt $pb[$i]) { return -1 } + if ($pa[$i] -gt $pb[$i]) { return 1 } + } + return 0 +} + +function Get-AuthHeaders { + param([string]$Style = "bearer") + $h = @{ "User-Agent" = "NetX-UpdateCheck" } + if (-not $UpdateToken) { return $h } + if ($Style -eq "token") { + $h["Authorization"] = "token $UpdateToken" + } else { + $h["Authorization"] = "Bearer $UpdateToken" + } + return $h +} + +function Resolve-AssetUrl { + param($Asset, $Rel, [string]$SourceName, [string]$ApiLatestUrl = "") + $url = [string]$Asset.browser_download_url + if ($url) { return $url } + # Forgejo/Gitea sometimes omit browser_download_url; synthesize release download URL. + if ($SourceName -eq "forgejo" -and $ApiLatestUrl -and $Rel.tag_name -and $Asset.name) { + if ($ApiLatestUrl -match '^(https?://[^/]+)/api/v1/repos/([^/]+)/([^/]+)/releases/latest') { + $base = $Matches[1] + $owner = $Matches[2] + $repo = $Matches[3] + $tag = [string]$Rel.tag_name + $name = [uri]::EscapeDataString([string]$Asset.name).Replace('%2F', '/') + # EscapeDataString encodes too much; use raw name (assets shouldn't need query encoding). + $name = [string]$Asset.name + return "$base/$owner/$repo/releases/download/$tag/$name" + } + } + return "" +} + +function Convert-ReleaseToManifest { + param($Rel, [string]$SourceName, [string]$ApiLatestUrl = "") + $tag = [string]$Rel.tag_name + $ver = $tag.TrimStart('v', 'V') + $assets = @($Rel.assets) + # Prefer Setup.exe (current ship format). Legacy win64.zip still accepted. + $setupAsset = $assets | Where-Object { $_.name -match 'Setup-.*\.exe$' } | Select-Object -First 1 + $zipAsset = $assets | Where-Object { $_.name -match 'win64\.zip$' } | Select-Object -First 1 + if (-not $setupAsset -and -not $zipAsset) { + throw "${SourceName}_release_missing_setup_or_zip" + } + + $setupUrl = "" + if ($setupAsset) { + $setupUrl = Resolve-AssetUrl -Asset $setupAsset -Rel $Rel -SourceName $SourceName -ApiLatestUrl $ApiLatestUrl + if (-not $setupUrl) { throw "${SourceName}_setup_url_missing" } + } + $zipUrl = "" + if ($zipAsset) { + $zipUrl = Resolve-AssetUrl -Asset $zipAsset -Rel $Rel -SourceName $SourceName -ApiLatestUrl $ApiLatestUrl + } + + if ($setupUrl) { + $primaryUrl = $setupUrl + $packageKind = "setup" + $primarySize = [int64]$(if ($setupAsset.size) { $setupAsset.size } else { 0 }) + } else { + if (-not $zipUrl) { throw "${SourceName}_zip_url_missing" } + $primaryUrl = $zipUrl + $packageKind = "zip" + $primarySize = [int64]$(if ($zipAsset.size) { $zipAsset.size } else { 0 }) + } + + return [pscustomobject]@{ + channel = "stable" + latest = $ver + min_compatible = $ver + notes_url = [string]$Rel.html_url + source = $SourceName + windows = [pscustomobject]@{ + url = $primaryUrl + sha256 = "" + size = $primarySize + setup_url = $setupUrl + package = $packageKind + } + } +} + +function Get-ManifestFromUrl { + param([string]$Url) + $headers = Get-AuthHeaders -Style "bearer" + $resp = Invoke-WebRequest -Uri $Url -Headers $headers -UseBasicParsing -TimeoutSec 30 + $m = $resp.Content | ConvertFrom-Json + if (-not $m.source) { + $m | Add-Member -NotePropertyName source -NotePropertyValue "manifest" -Force + } + return $m +} + +function Get-FromGitHubReleases { + $api = "https://api.github.com/repos/$GithubRepo/releases/latest" + $headers = Get-AuthHeaders -Style "bearer" + $headers["Accept"] = "application/vnd.github+json" + $rel = Invoke-RestMethod -Uri $api -Headers $headers -TimeoutSec 30 + return Convert-ReleaseToManifest -Rel $rel -SourceName "github" +} + +function Get-FromForgejoReleases { + param([string]$ApiUrl) + if (-not $ApiUrl) { throw "forgejo_url_empty" } + $headers = Get-AuthHeaders -Style "token" + $headers["Accept"] = "application/json" + $rel = Invoke-RestMethod -Uri $ApiUrl -Headers $headers -TimeoutSec 30 + return Convert-ReleaseToManifest -Rel $rel -SourceName "forgejo" -ApiLatestUrl $ApiUrl +} + +function Get-UpdateSourceList { + if ($Sources) { + return @($Sources.Split(',') | ForEach-Object { $_.Trim().ToLowerInvariant() } | Where-Object { $_ }) + } + # Default: GitHub primary, Forgejo mirror backup. Optional manifests prepended if configured. + $list = [System.Collections.Generic.List[string]]::new() + if ($UpdateUrl) { [void]$list.Add("manifest") } + [void]$list.Add("github") + [void]$list.Add("forgejo") + if ($FallbackUrl) { [void]$list.Add("manifest_fallback") } + return @($list) +} + +function Get-ManifestFromSource([string]$src) { + switch ($src) { + "manifest" { + if (-not $UpdateUrl) { throw "NETX_UPDATE_URL empty" } + Write-Host "==> Probing manifest: $UpdateUrl" + return Get-ManifestFromUrl -Url $UpdateUrl + } + "manifest_fallback" { + if (-not $FallbackUrl) { throw "NETX_UPDATE_FALLBACK_URL empty" } + Write-Host "==> Probing fallback manifest: $FallbackUrl" + $m = Get-ManifestFromUrl -Url $FallbackUrl + if ($m -and -not $m.source) { + $m | Add-Member -NotePropertyName source -NotePropertyValue "manifest_fallback" -Force + } + return $m + } + "forgejo" { + Write-Host "==> Probing Forgejo: $ForgejoUrl" + return Get-FromForgejoReleases -ApiUrl $ForgejoUrl + } + "github" { + Write-Host "==> Probing GitHub: $GithubRepo" + return Get-FromGitHubReleases + } + default { throw "unknown_source: $src" } + } +} + +Write-Host "==> Current version: $current" +$sourceList = Get-UpdateSourceList +Write-Host "==> Probe sources (pick newest reachable; tie → earlier in list): $($sourceList -join ', ')" + +$candidates = @() +$errors = @() +foreach ($src in $sourceList) { + try { + $m = Get-ManifestFromSource -src $src + if ($m.channel -and $Channel -and ($m.channel -ne $Channel)) { + Write-Host "[WARN] $src channel=$($m.channel) requested=$Channel" + } + if (-not $m.windows -or -not $m.windows.url) { + throw "missing_windows_download_url" + } + Write-Host " OK $($m.source) latest=$($m.latest)" -ForegroundColor DarkGreen + $candidates += $m + } catch { + $msg = $_.Exception.Message + $errors += "${src}: $msg" + Write-Host "[WARN] source '$src' unreachable/failed: $msg" -ForegroundColor Yellow + } +} + +if ($candidates.Count -eq 0) { + $joined = ($errors -join "; ") + if ($Quiet) { exit 2 } + throw "update_check_failed: all sources failed ($joined)" +} + +# Prefer highest semver; on tie keep earlier source in $sourceList (GitHub before Forgejo by default). +$manifest = $candidates[0] +foreach ($c in $candidates) { + $cmpCand = Compare-SemVer -A ([string]$manifest.latest) -B ([string]$c.latest) + if ($cmpCand -lt 0) { + $manifest = $c + } +} + +Write-Host "==> Selected source=$($manifest.source) latest=$($manifest.latest) (from $($candidates.Count) reachable)" -ForegroundColor Green + +# Prefer Setup.exe when a manifest still lists a legacy zip as windows.url but also has setup_url. +if ($manifest.windows -and $manifest.windows.setup_url) { + $setupCandidate = [string]$manifest.windows.setup_url + $urlNow = [string]$manifest.windows.url + $pkgNow = if ($manifest.windows.package) { [string]$manifest.windows.package } else { "" } + if ($setupCandidate -and ($pkgNow -eq "zip" -or $urlNow -match '\.zip(\?|$)' -or -not $urlNow)) { + $manifest.windows | Add-Member -NotePropertyName url -NotePropertyValue $setupCandidate -Force + $manifest.windows | Add-Member -NotePropertyName package -NotePropertyValue "setup" -Force + } +} + +$latest = [string]$manifest.latest +$cmp = Compare-SemVer -A $current -B $latest +$packageKind = "setup" +if ($manifest.windows.package) { + $packageKind = [string]$manifest.windows.package +} elseif ([string]$manifest.windows.url -match '\.zip(\?|$)') { + $packageKind = "zip" +} + +$result = [pscustomobject]@{ + current = $current + latest = $latest + update_available = ($cmp -lt 0) + source = $(if ($manifest.source) { [string]$manifest.source } else { "unknown" }) + download_url = [string]$manifest.windows.url + setup_url = $(if ($manifest.windows.setup_url) { [string]$manifest.windows.setup_url } else { "" }) + package = $packageKind + notes_url = $(if ($manifest.notes_url) { [string]$manifest.notes_url } else { "" }) + sha256 = $(if ($manifest.windows.sha256) { [string]$manifest.windows.sha256 } else { "" }) + reachable = @($candidates | ForEach-Object { "$($_.source)=$($_.latest)" }) +} + +if (-not $result.update_available) { + Write-Host "==> Up to date (latest=$latest, source=$($result.source))" -ForegroundColor Green + if (-not $Quiet) { + $result | ConvertTo-Json -Compress | Write-Output + } + exit 0 +} + +Write-Host "==> Update available: $current -> $latest (source=$($result.source))" -ForegroundColor Cyan +if ($result.notes_url) { Write-Host " Notes: $($result.notes_url)" } + +$autoEnabled = $false +$autoVal = Get-Cfg "NETX_UPDATE_AUTO" "" +if ($autoVal -match '^(1|true|yes|on)$') { $autoEnabled = $true } + +if (-not $Apply) { + Write-Host " Download ($($result.package)): $($result.download_url)" + Write-Host " To apply: .\packaging\check_update.ps1 -Apply" + if (-not $Quiet) { + $result | ConvertTo-Json -Compress | Write-Output + } + exit 10 +} + +if ($AutoOnly -and -not $autoEnabled) { + Write-Host "==> Update available but NETX_UPDATE_AUTO is not enabled; skip apply" + if (-not $Quiet) { + $result | ConvertTo-Json -Compress | Write-Output + } + exit 10 +} + +$lockFile = Join-Path $data "data\runtime\update.lock" +$lockDir = Split-Path -Parent $lockFile +if (-not (Test-Path $lockDir)) { + New-Item -ItemType Directory -Path $lockDir -Force | Out-Null +} +if (Test-Path $lockFile) { + $ageHrs = ((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalHours + if ($ageHrs -lt 2) { + Write-Host "==> Another update appears in progress ($lockFile); abort" + exit 3 + } + Remove-Item -Force $lockFile -ErrorAction SilentlyContinue +} +Set-Content -Path $lockFile -Value (Get-Date).ToString("o") -Encoding ascii + +try { + $dlDir = Join-Path $data "backups\downloads" + if (-not (Test-Path $dlDir)) { + New-Item -ItemType Directory -Path $dlDir -Force | Out-Null + } + + $isSetup = ($result.package -eq "setup") -or ($result.download_url -match '\.exe(\?|$)') + if ($isSetup) { + $pkgName = "NetX-Setup-$latest.exe" + } else { + $pkgName = "NetX-$latest-win64.zip" + } + $pkgPath = Join-Path $dlDir $pkgName + + $needDownload = $true + if ((Test-Path -LiteralPath $pkgPath) -and ((Get-Item -LiteralPath $pkgPath).Length -gt 1MB)) { + Write-Host "==> Using existing download: $pkgPath ($([math]::Round((Get-Item $pkgPath).Length/1MB,1)) MB)" + $needDownload = $false + } + if ($needDownload) { + Write-Host "==> Downloading $pkgName from $($result.source) ..." + # Only attach token for non-GitHub hosts (Forgejo/private). Public GitHub assets need no auth; + # a Forgejo token would break anonymous GitHub downloads. + $dlHeaders = @{ "User-Agent" = "NetX-UpdateCheck" } + $dlUri = [uri]$result.download_url + $isGithub = ($dlUri.Host -match '(^|\.)github\.com$' -or $dlUri.Host -match '(^|\.)githubusercontent\.com$') + if ($UpdateToken -and -not $isGithub) { + $dlHeaders["Authorization"] = "token $UpdateToken" + } + Invoke-WebRequest -Uri $result.download_url -OutFile $pkgPath -Headers $dlHeaders -UseBasicParsing + } + $expectedHash = [string]$result.sha256 + if (-not $expectedHash -or $expectedHash -match 'REPLACE') { + $checksumUri = [uri]$result.download_url + $checksumBuilder = New-Object -TypeName System.UriBuilder -ArgumentList $checksumUri + $checksumBuilder.Path = $checksumBuilder.Path + '.sha256' + $checksumBuilder.Query = '' + $checksumUrl = $checksumBuilder.Uri.AbsoluteUri + $checksumHeaders = @{ 'User-Agent' = 'NetX-UpdateCheck' } + $isChecksumGithub = ($checksumUri.Host -match '(^|\.)github\.com$' -or $checksumUri.Host -match '(^|\.)githubusercontent\.com$') + if ($UpdateToken -and -not $isChecksumGithub) { $checksumHeaders['Authorization'] = "token $UpdateToken" } + $checksumFile = "$pkgPath.sha256" + try { + Invoke-WebRequest -Uri $checksumUrl -OutFile $checksumFile -Headers $checksumHeaders -UseBasicParsing + $expectedHash = (Get-Content -LiteralPath $checksumFile -Raw).Trim() -split '\s+' | Select-Object -First 1 + } catch { + $statusCode = 0 + if ($_.Exception.Response -and $_.Exception.Response.StatusCode) { + $statusCode = [int]$_.Exception.Response.StatusCode + } + if ($statusCode -eq 404) { + Write-Host "[WARN] Release has no SHA-256 sidecar; accepting legacy asset without checksum verification." -ForegroundColor Yellow + } else { + throw + } + } finally { + Remove-Item -LiteralPath $checksumFile -Force -ErrorAction SilentlyContinue + } + } + if ($expectedHash) { + $expectedHash = $expectedHash.Trim().ToLowerInvariant() + if ($expectedHash -notmatch '^[0-9a-f]{64}$') { throw 'invalid_sha256_metadata' } + $actualHash = (Get-FileHash -LiteralPath $pkgPath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actualHash -ne $expectedHash) { + Remove-Item -LiteralPath $pkgPath -Force -ErrorAction SilentlyContinue + throw "sha256_mismatch: got $actualHash expected $expectedHash" + } + Write-Host '==> SHA256 OK' + } + + # Program Files installs need admin; elevate once (avoid loop via NETX_UPDATE_ELEVATED). + $progWritable = $false + try { + $probe = Join-Path $prog (".netx_w_" + [guid]::NewGuid().ToString("n")) + [IO.File]::WriteAllText($probe, "x") + Remove-Item -LiteralPath $probe -Force -ErrorAction SilentlyContinue + $progWritable = $true + } catch { + $progWritable = $false + } + if (-not $progWritable -and -not $env:NETX_UPDATE_ELEVATED) { + Write-Host "==> Program directory is not writable; relaunching update as Administrator (UAC)..." -ForegroundColor Yellow + $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply" + $ep = Start-Process -FilePath "powershell.exe" -ArgumentList $arg -WorkingDirectory $prog ` + -Verb RunAs -PassThru + Wait-NetxProcessExit -Process $ep + if ($null -eq $ep.ExitCode) { exit 1 } + exit $ep.ExitCode + } + + $env:NETX_UPDATE_ELEVATED = "1" + if ($isSetup) { + # Silent Setup update mode: skip DB wizard, keep ProgramData + # (see installer/netx.iss /InstallMode=update). + Write-Host "==> Applying update via silent Setup.exe" + $setupArgs = "/VERYSILENT /NORESTART /SUPPRESSMSGBOXES /DIR=`"$prog`" /InstallMode=update" + # Setup stops the tray (StopForUpdate -KillTray); remember to bring it back. + $trayWasRunning = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { + $_.CommandLine -and $_.CommandLine -match 'netx_tray\.ps1' + }).Count -gt 0 + $sp = Start-Process -FilePath $pkgPath -ArgumentList $setupArgs -PassThru + Wait-NetxProcessExit -Process $sp + if ($null -eq $sp.ExitCode -or $sp.ExitCode -ne 0) { + $code = if ($null -eq $sp.ExitCode) { "null" } else { $sp.ExitCode } + throw "setup_update_failed: exit $code" + } + # Setup post-install also repairs; belt-and-suspenders when running elevated apply. + $repairPs1 = Join-Path $prog "packaging\repair_venv.ps1" + if (Test-Path -LiteralPath $repairPs1) { + Write-Host "==> Relinking .venv to bundled python/runtime" + & powershell -NoProfile -ExecutionPolicy Bypass -File $repairPs1 ` + -ProgramRoot $prog -DataRoot $data + } + $managedRunnerActive = $false + $netxService = Get-Service -Name "NetX" -ErrorAction SilentlyContinue + if ($netxService -and $netxService.Status -eq "Running") { $managedRunnerActive = $true } + if (-not $managedRunnerActive) { + try { + $netxTask = Get-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -ErrorAction Stop + $managedRunnerActive = ([string]$netxTask.State -eq "Running") + } catch {} + } + if ($managedRunnerActive) { + Write-Host "==> Waiting for the previously running NetX service/task" + $netxPort = 8890 + $netxEnv = Read-DotEnv -Path (Join-Path $data '.env') + if ($netxEnv['NETX_PORT']) { try { $netxPort = [int]$netxEnv['NETX_PORT'] } catch {} } + if (-not (Wait-NetxApiHealthy -HostName "127.0.0.1" -Port $netxPort -TimeoutSec 180)) { + throw "updated_service_unhealthy: NetX service/task was resumed but /health did not become ready" + } + } else { + Write-Host "==> Restarting NetX after Setup" + & (Join-Path $PSScriptRoot "start_netx_app.ps1") ` + -ProgramRoot $prog -DataRoot $data -SkipBrowser + if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + if ($trayWasRunning) { + $trayPs1 = Join-Path $prog "packaging\netx_tray.ps1" + if (Test-Path -LiteralPath $trayPs1) { + Write-Host "==> Restarting NetX tray" + Start-NetxPowerShell -File $trayPs1 -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-QuietStart") ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } + } + } else { + Write-Host "==> Applying legacy zip update via update_netx.ps1" + & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "update_netx.ps1") ` + -PackagePath $pkgPath -ProgramRoot $prog -DataRoot $data + } + Write-Host "==> Update applied to $latest" -ForegroundColor Green +} finally { + Remove-Item -Force $lockFile -ErrorAction SilentlyContinue +} +exit 0 diff --git a/packaging/cmd/NetX-FirstRun.cmd b/packaging/cmd/NetX-FirstRun.cmd new file mode 100644 index 0000000..e5f9887 --- /dev/null +++ b/packaging/cmd/NetX-FirstRun.cmd @@ -0,0 +1,33 @@ +@echo off +setlocal EnableDelayedExpansion +net session >nul 2>&1 +if not "%errorLevel%"=="0" ( + powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" + exit /b !ERRORLEVEL! +) +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" +cd /d "%ROOT%" +title NetX - Reconfigure database +echo. +echo NetX database reconfigure / 重新配置数据库 +echo Prefer the installer wizard for first-time setup. +echo 首次安装请用安装向导选择内置或外置数据库。 +echo This window is for repair / reconfigure only. +echo 本窗口仅用于修复或重新配置。 +echo. +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\setup_first_run.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" +set "EC=!ERRORLEVEL!" +if not "!EC!"=="0" ( + echo. + echo Setup failed / 配置失败 exit=!EC! + pause + exit /b !EC! +) +echo. +echo Starting NetX tray... / 正在启动托盘... +start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\netx_tray.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -StartOnLaunch +echo. +echo Done. You can close this window. / 完成，可关闭本窗口。 +pause +exit /b 0 diff --git a/packaging/cmd/NetX-Start.cmd b/packaging/cmd/NetX-Start.cmd new file mode 100644 index 0000000..8342f92 --- /dev/null +++ b/packaging/cmd/NetX-Start.cmd @@ -0,0 +1,13 @@ +@echo off +setlocal EnableDelayedExpansion +REM Always elevate: Program Files install needs admin to start API / repair venv. +net session >nul 2>&1 +if not "%errorLevel%"=="0" ( + powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" + exit /b !ERRORLEVEL! +) +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" +cd /d "%ROOT%" +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action start -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" +exit /b !ERRORLEVEL! diff --git a/packaging/cmd/NetX-Stop.cmd b/packaging/cmd/NetX-Stop.cmd new file mode 100644 index 0000000..8a225f2 --- /dev/null +++ b/packaging/cmd/NetX-Stop.cmd @@ -0,0 +1,12 @@ +@echo off +setlocal EnableDelayedExpansion +net session >nul 2>&1 +if not "%errorLevel%"=="0" ( + powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" + exit /b !ERRORLEVEL! +) +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" +cd /d "%ROOT%" +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action stop -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" +exit /b !ERRORLEVEL! diff --git a/packaging/cmd/NetX-Tray.cmd b/packaging/cmd/NetX-Tray.cmd new file mode 100644 index 0000000..ac1f46c --- /dev/null +++ b/packaging/cmd/NetX-Tray.cmd @@ -0,0 +1,13 @@ +@echo off +setlocal EnableDelayedExpansion +REM Elevate once, then start tray (inherits admin for Start/Stop/Update). +net session >nul 2>&1 +if not "%errorLevel%"=="0" ( + powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" + exit /b !ERRORLEVEL! +) +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" +cd /d "%ROOT%" +start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\netx_tray.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -StartOnLaunch +exit /b 0 diff --git a/packaging/config/database.example.env b/packaging/config/database.example.env index a9d072a..f5e5555 100644 --- a/packaging/config/database.example.env +++ b/packaging/config/database.example.env @@ -5,5 +5,11 @@ # NETX_HOST=127.0.0.1 # NETX_PORT=8890 # NETX_UI_DIST_DIR=web/dist +# Defaults already: github + forgejo mirror git.avelo.top (newest reachable wins) +# NETX_UPDATE_SOURCES=github,forgejo +# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest +# NETX_UPDATE_GITHUB_REPO=hansjone/netx # NETX_UPDATE_URL= +# NETX_UPDATE_TOKEN= # NETX_UPDATE_CHANNEL=stable +# NETX_UPDATE_AUTO=false diff --git a/packaging/download_postgres.ps1 b/packaging/download_postgres.ps1 index 38c7b41..0a0fe29 100644 --- a/packaging/download_postgres.ps1 +++ b/packaging/download_postgres.ps1 @@ -1,4 +1,4 @@ -param( +param( [string]$Version = "16.4-1", [string]$OutDir = "" ) @@ -12,6 +12,7 @@ $url = "https://get.enterprisedb.com/postgresql/postgresql-$ver-windows-x64-bina $cache = if ($OutDir) { $OutDir } else { Join-Path $PSScriptRoot "cache" } $zipPath = Join-Path $cache "postgresql-$ver-windows-x64-binaries.zip" +$partialPath = "$zipPath.partial" $extractRoot = Join-Path $cache "pgsql-$ver" $destPgsql = Join-Path $PSScriptRoot "postgres\pgsql" @@ -22,7 +23,31 @@ if (-not (Test-Path $cache)) { Write-Host "==> PostgreSQL binaries URL: $url" if (-not (Test-Path $zipPath)) { Write-Host "==> Downloading (large; may take several minutes)..." - Invoke-WebRequest -Uri $url -OutFile $zipPath -UseBasicParsing + $curl = Get-Command curl.exe -ErrorAction SilentlyContinue + if ($curl) { + # PostgreSQL's archive is several hundred MB. Resume interrupted transfers + # instead of restarting the entire download when the CDN drops a connection. + & $curl.Source --fail --location --retry 8 --retry-all-errors --retry-delay 3 ` + --connect-timeout 30 --continue-at - --output $partialPath $url + if ($LASTEXITCODE -ne 0) { + throw "postgres_download_failed: curl exit $LASTEXITCODE (partial file kept at $partialPath)" + } + Move-Item -LiteralPath $partialPath -Destination $zipPath -Force + } else { + # Fallback for environments without curl; retry from scratch on transient errors. + $downloaded = $false + for ($attempt = 1; $attempt -le 3 -and -not $downloaded; $attempt++) { + try { + Invoke-WebRequest -Uri $url -OutFile $partialPath -UseBasicParsing + Move-Item -LiteralPath $partialPath -Destination $zipPath -Force + $downloaded = $true + } catch { + if ($attempt -eq 3) { throw } + Remove-Item -LiteralPath $partialPath -Force -ErrorAction SilentlyContinue + Start-Sleep -Seconds (5 * $attempt) + } + } + } } else { Write-Host "==> Using cached zip: $zipPath" } diff --git a/packaging/install_autostart.ps1 b/packaging/install_autostart.ps1 new file mode 100644 index 0000000..b0ac9ef --- /dev/null +++ b/packaging/install_autostart.ps1 @@ -0,0 +1,48 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [switch]$Remove = $false +) + +# Register / unregister NetX tray at logon with highest privileges (UAC admin). +# HKCU\Run cannot elevate reliably on fresh PCs; use a Scheduled Task instead. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$tray = Join-Path $PSScriptRoot "netx_tray.ps1" +$runKey = "HKCU:\Software\Microsoft\Windows\CurrentVersion\Run" +$legacyName = "NetX" +$taskName = "NetXTray" +$taskPath = "\NetX\" + +# Always clear legacy per-user Run key (non-elevated). +Remove-ItemProperty -Path $runKey -Name $legacyName -ErrorAction SilentlyContinue + +if ($Remove) { + Unregister-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Confirm:$false -ErrorAction SilentlyContinue + Write-Host "==> Removed NetX tray autostart task" + exit 0 +} + +# -QuietStart: bring NetX up at logon without popping a browser window. +$arg = "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$tray`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -StartOnLaunch -QuietStart" +$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog +# Any user logon, run for the local Administrators group: Setup runs elevated +# (often as a different admin account), so $env:USERNAME is not the desktop user. +$trig = New-ScheduledTaskTrigger -AtLogOn +$trig.Delay = "PT10S" +$admins = (New-Object System.Security.Principal.SecurityIdentifier("S-1-5-32-544")).Translate( + [System.Security.Principal.NTAccount]).Value +$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable ` + -ExecutionTimeLimit ([TimeSpan]::Zero) -MultipleInstances Parallel +$principal = New-ScheduledTaskPrincipal -GroupId $admins -RunLevel Highest +Register-ScheduledTask -TaskName $taskName -TaskPath $taskPath ` + -Action $action -Trigger $trig -Settings $settings -Principal $principal -Force | Out-Null + +Write-Host "==> NetX tray will start at logon (Scheduled Task, RunLevel Highest)" -ForegroundColor Green +Write-Host " $taskPath$taskName" +Write-Host " $arg" diff --git a/packaging/install_service.ps1 b/packaging/install_service.ps1 new file mode 100644 index 0000000..6a46763 --- /dev/null +++ b/packaging/install_service.ps1 @@ -0,0 +1,154 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [ValidateSet("winsw", "task")] + [string]$Mode = "winsw", + [switch]$Uninstall = $false, + [switch]$Start = $false, + # Offline-safe by default: use WinSW shipped in the package. Opt-in to download from GitHub. + [switch]$AllowDownload = $false +) + +# Install NetX as a Windows Service (WinSW) or as a SYSTEM startup Scheduled Task. +# Requires elevation for winsw / task modes that run as SYSTEM. +# WinSW binary is bundled at packaging\winsw\WinSW.exe by build_release.ps1 — no network needed. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$svcName = "NetX" +$winswDir = Join-Path $prog "packaging\winsw" +$winswExe = Join-Path $winswDir "NetX.exe" +$winswXml = Join-Path $winswDir "NetX.xml" +$cacheDir = Join-Path $PSScriptRoot "cache" + +function Test-IsAdmin { + return Test-NetxIsAdmin +} + +function ConvertTo-WinSWPath([string]$Path) { + return ($Path -replace '\\', '/') +} + +function Ensure-WinSW { + if (-not (Test-Path $winswDir)) { + New-Item -ItemType Directory -Path $winswDir -Force | Out-Null + } + if (-not (Test-Path $cacheDir)) { + New-Item -ItemType Directory -Path $cacheDir -Force | Out-Null + } + # WinSW.exe = .NET Framework build (current); WinSW-x64.exe = older self-contained bundles. + $bundled = Join-Path $PSScriptRoot "winsw\WinSW.exe" + if (-not (Test-Path $bundled)) { $bundled = Join-Path $PSScriptRoot "winsw\WinSW-x64.exe" } + $dl = Join-Path $cacheDir "WinSW.NET461.exe" + $src = $null + if (Test-Path $bundled) { + $src = $bundled + Write-Host "==> Using bundled WinSW: $bundled" + } elseif (Test-Path $dl) { + $src = $dl + Write-Host "==> Using cached WinSW: $dl" + } elseif ($AllowDownload) { + $url = "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW.NET461.exe" + Write-Host "==> Downloading WinSW: $url" + Invoke-WebRequest -Uri $url -OutFile $dl -UseBasicParsing + $src = $dl + } else { + throw "winsw_missing_offline: expected $bundled (rebuild with build_release.ps1). Or pass -AllowDownload when online." + } + Copy-Item -Path $src -Destination $winswExe -Force + + $runPs1 = ConvertTo-WinSWPath (Join-Path $PSScriptRoot "service_run.ps1") + $stopPs1 = ConvertTo-WinSWPath (Join-Path $PSScriptRoot "stop_netx_app.ps1") + $progFs = ConvertTo-WinSWPath $prog + $dataFs = ConvertTo-WinSWPath $data + # Own folder: WinSW writes .out.log / .err.log = NetX.out.log, which on Windows is + # the same file as start_netx.ps1's netx.out.log. WinSW keeps it open, so NetX could not + # truncate its log and every service start failed. + $svcLogDir = Join-Path $data "data\runtime\service" + $logPath = ConvertTo-WinSWPath $svcLogDir + if (-not (Test-Path $svcLogDir)) { + New-Item -ItemType Directory -Path $svcLogDir -Force | Out-Null + } + + # Stop hook via cmd: NUL stdin + output to a file. Launched by WinSW directly, + # powershell.exe never reached the script's first line (stuck on the inherited + # std handles), so the supervisor was never told to exit and stops hung. + $stopLog = Join-Path $svcLogDir "stopexec.log" + $stopCmd = "/d /s /c `"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$stopPs1`" -ProgramRoot `"$progFs`" -DataRoot `"$dataFs`" <NUL >>`"$stopLog`" 2>&1`"" + + $xml = @" + + $svcName + NetX Ops + NetX API, workers, and optional bundled PostgreSQL + powershell.exe + -NoProfile -ExecutionPolicy Bypass -File "$runPs1" -ProgramRoot "$progFs" -DataRoot "$dataFs" + $logPath + + 10240 + 4 + + + + 1 hour + 60sec + cmd.exe + $stopCmd + $progFs + +"@ + # WinSW expects UTF-8 without BOM issues; ASCII-compatible paths preferred. + [System.IO.File]::WriteAllText($winswXml, $xml) +} + +if ($Uninstall) { + if (-not (Test-IsAdmin)) { throw "admin_required_for_uninstall" } + if (Test-Path $winswExe) { + Write-Host "==> Stopping/uninstalling WinSW service" + & $winswExe stop 2>$null + & $winswExe uninstall 2>$null + } + Unregister-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -Confirm:$false -ErrorAction SilentlyContinue + Write-Host "==> Service/task removed" + exit 0 +} + +if (-not (Test-IsAdmin)) { + throw "admin_required: run elevated PowerShell to install the NetX service" +} + +if ($Mode -eq "winsw") { + Ensure-WinSW + Write-Host "==> Installing Windows Service via WinSW" + & $winswExe stop 2>$null + & $winswExe uninstall 2>$null + & $winswExe install + if ($LASTEXITCODE -ne 0) { throw "winsw_install_failed" } + if ($Start) { + & $winswExe start + Write-Host "==> Service started" -ForegroundColor Green + } else { + Write-Host "==> Installed. Start with: Start-Service NetX or `"$winswExe`" start" + } +} else { + Write-Host "==> Registering Scheduled Task NetX\NetXService (At startup, SYSTEM)" + $runPs1 = Join-Path $PSScriptRoot "service_run.ps1" + $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$runPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`"" + $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog + $trigger = New-ScheduledTaskTrigger -AtStartup + $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest + # ExecutionTimeLimit 0: the default (72h) would kill the long-running supervisor. + $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 1) ` + -ExecutionTimeLimit ([TimeSpan]::Zero) + Register-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Force | Out-Null + if ($Start) { + Start-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" + Write-Host "==> Task started" -ForegroundColor Green + } else { + Write-Host "==> Task registered. Start with: Start-ScheduledTask -TaskPath '\NetX\' -TaskName NetXService" + } +} diff --git a/packaging/install_update_task.ps1 b/packaging/install_update_task.ps1 new file mode 100644 index 0000000..abdecce --- /dev/null +++ b/packaging/install_update_task.ps1 @@ -0,0 +1,64 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [ValidateSet("Daily", "AtLogOn", "Hourly")] + [string]$Trigger = "Daily", + [string]$At = "03:30", + [switch]$Remove = $false, + [switch]$EnableAutoEnv = $true +) + +# Schedule silent update checks. With NETX_UPDATE_AUTO=true, applies updates when available. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$taskName = "NetXUpdate" +$taskPath = "\NetX\" +$checkPs1 = Join-Path $PSScriptRoot "check_update.ps1" + +if ($Remove) { + Unregister-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Confirm:$false -ErrorAction SilentlyContinue + # Tray also applies updates on launch when NETX_UPDATE_AUTO=true; turn that off too. + $envFile = Join-Path $data ".env" + if (Test-Path $envFile) { + Write-DotEnvValue -Path $envFile -Values @{ "NETX_UPDATE_AUTO" = "false" } + } + Write-Host "==> Update task removed" + exit 0 +} + +if ($EnableAutoEnv) { + $envFile = Join-Path $data ".env" + if (-not (Test-Path $data)) { + New-Item -ItemType Directory -Path $data -Force | Out-Null + } + Write-DotEnvValue -Path $envFile -Values @{ "NETX_UPDATE_AUTO" = "true" } + Write-Host "==> Set NETX_UPDATE_AUTO=true in $envFile" +} + +$arg = "-NoProfile -ExecutionPolicy Bypass -File `"$checkPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply -Quiet -AutoOnly" +$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog + +switch ($Trigger) { + "Hourly" { + $trig = New-ScheduledTaskTrigger -Once -At (Get-Date).Date.AddHours((Get-Date).Hour + 1) ` + -RepetitionInterval (New-TimeSpan -Hours 1) -RepetitionDuration ([TimeSpan]::MaxValue) + } + "AtLogOn" { + $trig = New-ScheduledTaskTrigger -AtLogOn + } + default { + $trig = New-ScheduledTaskTrigger -Daily -At $At + } +} + +$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -StartWhenAvailable +# Highest: silent apply must write Program Files / run Setup. +$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive -RunLevel Highest +Register-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Action $action -Trigger $trig -Settings $settings -Principal $principal -Force | Out-Null +Write-Host "==> Scheduled update task: $taskPath$taskName ($Trigger, RunLevel Highest)" -ForegroundColor Green +Write-Host " Manual run: .\packaging\check_update.ps1 -Apply -Quiet -AutoOnly" diff --git a/packaging/installer/Languages/ChineseSimplified.isl b/packaging/installer/Languages/ChineseSimplified.isl new file mode 100644 index 0000000..30d9973 --- /dev/null +++ b/packaging/installer/Languages/ChineseSimplified.isl @@ -0,0 +1,417 @@ +; *** Inno Setup version 6.5.0+ Chinese Simplified messages *** +; +; To download user-contributed translations of this file, go to: +; https://jrsoftware.org/files/istrans/ +; +; Note: When translating this text, do not add periods (.) to the end of +; messages that didn't have them already, because on those messages Inno +; Setup adds the periods automatically (appending a period would result in +; two periods being displayed). +; +; Maintainer: Zhenghan Yang (Kira) +; Email: 847320916@QQ.com +; Github: https://github.com/kira-96/Inno-Setup-Chinese-Simplified-Translation +; Encoding: UTF-8 +; Translation based on network resource +; + +[LangOptions] +; The following three entries are very important. Be sure to read and +; understand the '[LangOptions] section' topic in the help file. +LanguageName=简体中文 +; About LanguageID, to reference link: +; https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c +LanguageID=$0804 +; LanguageCodePage should always be set if possible, even if this file is Unicode +; For English it's set to zero anyway because English only uses ASCII characters +LanguageCodePage=936 +; If the language you are translating to requires special font faces or +; sizes, uncomment any of the following entries and change them accordingly. +;DialogFontName= +;DialogFontSize=9 +;DialogFontBaseScaleWidth=7 +;DialogFontBaseScaleHeight=15 +;WelcomeFontName=Segoe UI +;WelcomeFontSize=14 + +[Messages] + +; *** Application titles +SetupAppTitle=安装 +SetupWindowTitle=安装 - %1 +UninstallAppTitle=卸载 +UninstallAppFullTitle=%1 卸载 + +; *** Misc. common +InformationTitle=信息 +ConfirmTitle=确认 +ErrorTitle=错误 + +; *** SetupLdr messages +SetupLdrStartupMessage=现在将安装 %1。您想要继续吗？ +LdrCannotCreateTemp=无法创建临时文件。安装程序已中止 +LdrCannotExecTemp=无法执行临时目录中的文件。安装程序已中止 +HelpTextNote= + +; *** Startup error messages +LastErrorMessage=%1。%n%n错误 %2: %3 +SetupFileMissing=安装目录中缺少文件 %1。请修正这个问题或者获取程序的新副本。 +SetupFileCorrupt=安装文件已损坏。请获取程序的新副本。 +SetupFileCorruptOrWrongVer=安装文件已损坏，或是与这个安装程序的版本不兼容。请修正这个问题或获取新的程序副本。 +InvalidParameter=无效的命令行参数：%n%n%1 +SetupAlreadyRunning=安装程序已在运行。 +WindowsVersionNotSupported=此程序不支持当前计算机运行的 Windows 版本。 +WindowsServicePackRequired=此程序需要 %1 服务包 %2 或更高版本。 +NotOnThisPlatform=此程序不能在 %1 上运行。 +OnlyOnThisPlatform=此程序只能在 %1 上运行。 +OnlyOnTheseArchitectures=此程序只能安装到为下列处理器架构设计的 Windows 版本中：%n%n%1 +WinVersionTooLowError=此程序需要 %1 版本 %2 或更高。 +WinVersionTooHighError=此程序不能安装于 %1 版本 %2 或更高。 +AdminPrivilegesRequired=在安装此程序时您必须以管理员身份登录。 +PowerUserPrivilegesRequired=在安装此程序时您必须以管理员身份或高级用户组身份登录。 +SetupAppRunningError=安装程序检测到 %1 当前正在运行。%n%n请先关闭正在运行的程序，然后点击“确定”继续，或点击“取消”退出。 +UninstallAppRunningError=卸载程序检测到 %1 当前正在运行。%n%n请先关闭正在运行的程序，然后点击“确定”继续，或点击“取消”退出。 + +; *** Startup questions +PrivilegesRequiredOverrideTitle=选择安装程序安装模式 +PrivilegesRequiredOverrideInstruction=选择安装模式 +PrivilegesRequiredOverrideText1=%1 可以为所有用户安装（需要管理员权限），或仅为您安装。 +PrivilegesRequiredOverrideText2=%1 可以仅为您安装，或为所有用户安装（需要管理员权限）。 +PrivilegesRequiredOverrideAllUsers=为所有用户安装(&A) +PrivilegesRequiredOverrideAllUsersRecommended=为所有用户安装(&A)（推荐） +PrivilegesRequiredOverrideCurrentUser=仅为我安装(&M) +PrivilegesRequiredOverrideCurrentUserRecommended=仅为我安装(&M)（推荐） + +; *** Misc. errors +ErrorCreatingDir=安装程序无法创建目录“%1” +ErrorTooManyFilesInDir=无法在目录“%1”中创建文件，因为里面包含太多文件。 + +; *** Setup common messages +ExitSetupTitle=退出安装程序 +ExitSetupMessage=安装程序尚未完成。如果现在退出，将不会安装该程序。%n%n您之后可以再次运行安装程序完成安装。%n%n现在退出安装程序吗？ +AboutSetupMenuItem=关于安装程序(&A)... +AboutSetupTitle=关于安装程序 +AboutSetupMessage=%1 版本 %2%n%3%n%n%1 主页：%n%4 +AboutSetupNote= +TranslatorNote=简体中文翻译由 Kira（847320916@qq.com）维护。项目地址：https://github.com/kira-96/Inno-Setup-Chinese-Simplified-Translation + +; *** Buttons +ButtonBack=< 上一步(&B) +ButtonNext=下一步(&N) > +ButtonInstall=安装(&I) +ButtonOK=确定 +ButtonCancel=取消 +ButtonYes=是(&Y) +ButtonYesToAll=全是(&A) +ButtonNo=否(&N) +ButtonNoToAll=全否(&O) +ButtonFinish=完成(&F) +ButtonBrowse=浏览(&B)... +ButtonWizardBrowse=浏览(&R)... +ButtonNewFolder=新建文件夹(&M) + +; *** "Select Language" dialog messages +SelectLanguageTitle=选择安装语言 +SelectLanguageLabel=选择安装时使用的语言。 + +; *** Common wizard text +ClickNext=点击“下一步”继续，或点击“取消”退出安装程序。 +BeveledLabel= +BrowseDialogTitle=浏览文件夹 +BrowseDialogLabel=在下面的列表中选择一个文件夹，然后点击“确定”。 +NewFolderName=新建文件夹 + +; *** "Welcome" wizard page +WelcomeLabel1=欢迎使用 [name] 安装向导 +WelcomeLabel2=即将在您的计算机上安装 [name/ver]。%n%n建议您在继续安装前关闭所有其他应用程序。 + +; *** "Password" wizard page +WizardPassword=密码 +PasswordLabel1=此安装程序需要密码验证。 +PasswordLabel3=请输入密码，然后点击“下一步”继续。密码区分大小写。 +PasswordEditLabel=密码(&P)： +IncorrectPassword=您输入的密码不正确，请重新输入。 + +; *** "License Agreement" wizard page +WizardLicense=许可协议 +LicenseLabel=请在继续安装前阅读以下重要信息。 +LicenseLabel3=请阅读下列许可协议。在继续安装前您必须同意这些协议条款。 +LicenseAccepted=我同意此协议(&A) +LicenseNotAccepted=我不同意此协议(&D) + +; *** "Information" wizard pages +WizardInfoBefore=信息 +InfoBeforeLabel=请在继续安装前阅读以下重要信息。 +InfoBeforeClickLabel=准备好继续安装后，点击“下一步”。 +WizardInfoAfter=信息 +InfoAfterLabel=请在继续安装前阅读以下重要信息。 +InfoAfterClickLabel=准备好继续安装后，点击“下一步”。 + +; *** "User Information" wizard page +WizardUserInfo=用户信息 +UserInfoDesc=请输入您的信息。 +UserInfoName=用户名(&U)： +UserInfoOrg=组织(&O)： +UserInfoSerial=序列号(&S)： +UserInfoNameRequired=请输入用户名。 + +; *** "Select Destination Location" wizard page +WizardSelectDir=选择目标位置 +SelectDirDesc=您想将 [name] 安装在哪里？ +SelectDirLabel3=安装程序将安装 [name] 到下面的文件夹中。 +SelectDirBrowseLabel=点击“下一步”继续。如果您想选择其他文件夹，点击“浏览”。 +DiskSpaceGBLabel=至少需要有 [gb] GB 的可用磁盘空间。 +DiskSpaceMBLabel=至少需要有 [mb] MB 的可用磁盘空间。 +CannotInstallToNetworkDrive=安装程序无法安装到一个网络驱动器。 +CannotInstallToUNCPath=安装程序无法安装到一个 UNC 路径。 +InvalidPath=您必须输入一个带驱动器盘符的完整路径，例如：%n%nC:\App%n%n或UNC路径：%n%n\\server\share +InvalidDrive=您选定的驱动器或 UNC 共享不存在或不能访问。请选择其他位置。 +DiskSpaceWarningTitle=磁盘空间不足 +DiskSpaceWarning=安装程序至少需要 %1 KB 的可用空间才能安装，但选定驱动器只有 %2 KB 的可用空间。%n%n您确定要继续吗？ +DirNameTooLong=文件夹名称或路径太长。 +InvalidDirName=文件夹名称无效。 +BadDirName32=文件夹名称不能包含下列任何字符：%n%n%1 +DirExistsTitle=文件夹已存在 +DirExists=文件夹：%n%n%1%n%n已经存在。您确定安装到这个文件夹中吗？ +DirDoesntExistTitle=文件夹不存在 +DirDoesntExist=文件夹：%n%n%1%n%n不存在。您想要创建此文件夹吗？ + +; *** "Select Components" wizard page +WizardSelectComponents=选择组件 +SelectComponentsDesc=您想安装哪些程序组件？ +SelectComponentsLabel2=选中您想安装的组件；取消您不想安装的组件。然后点击“下一步”继续。 +FullInstallation=完全安装 +; if possible don't translate 'Compact' as 'Minimal' (I mean 'Minimal' in your language) +CompactInstallation=简洁安装 +CustomInstallation=自定义安装 +NoUninstallWarningTitle=组件已存在 +NoUninstallWarning=安装程序检测到下列组件已安装在您的计算机中：%n%n%1%n%n取消选中这些组件不会卸载它们。%n%n您确定要继续吗？ +ComponentSize1=%1 KB +ComponentSize2=%1 MB +ComponentsDiskSpaceGBLabel=当前选择的组件需要至少 [gb] GB 的磁盘空间。 +ComponentsDiskSpaceMBLabel=当前选择的组件需要至少 [mb] MB 的磁盘空间。 + +; *** "Select Additional Tasks" wizard page +WizardSelectTasks=选择附加任务 +SelectTasksDesc=您想要安装程序执行哪些附加任务？ +SelectTasksLabel2=选择您想要安装程序在安装 [name] 时执行的附加任务，然后点击“下一步”。 + +; *** "Select Start Menu Folder" wizard page +WizardSelectProgramGroup=选择开始菜单文件夹 +SelectStartMenuFolderDesc=安装程序应该在哪里放置程序的快捷方式？ +SelectStartMenuFolderLabel3=安装程序将在下列“开始”菜单文件夹中创建程序的快捷方式。 +SelectStartMenuFolderBrowseLabel=点击“下一步”继续。如果您想选择其他文件夹，点击“浏览”。 +MustEnterGroupName=您必须输入一个文件夹名称。 +GroupNameTooLong=文件夹名称或路径太长。 +InvalidGroupName=文件夹名称无效。 +BadGroupName=文件夹名称不能包含下列任何字符：%n%n%1 +NoProgramGroupCheck2=不创建开始菜单文件夹(&D) + +; *** "Ready to Install" wizard page +WizardReady=准备安装 +ReadyLabel1=安装程序准备就绪，现在可以开始安装 [name] 到您的计算机。 +ReadyLabel2a=点击“安装”继续此安装程序。如果您想重新查看或修改任何设置，点击“上一步”。 +ReadyLabel2b=点击“安装”继续此安装程序。 +ReadyMemoUserInfo=用户信息： +ReadyMemoDir=目标位置： +ReadyMemoType=安装类型： +ReadyMemoComponents=已选择组件： +ReadyMemoGroup=开始菜单文件夹： +ReadyMemoTasks=附加任务： + +; *** TDownloadWizardPage wizard page and DownloadTemporaryFile +DownloadingLabel2=正在下载文件... +ButtonStopDownload=停止下载(&S) +StopDownload=您确定要停止下载吗？ +ErrorDownloadAborted=下载已中止。 +ErrorDownloadFailed=下载失败：%1 %2。 +ErrorDownloadSizeFailed=获取大小失败：%1 %2。 +ErrorProgress=无效的进度：%1 / %2。 +ErrorFileSize=文件大小错误：预期 %1，实际 %2。 + +; *** TExtractionWizardPage wizard page and ExtractArchive +ExtractingLabel=正在提取文件... +ButtonStopExtraction=停止提取(&S) +StopExtraction=您确定要停止提取吗？ +ErrorExtractionAborted=提取已中止。 +ErrorExtractionFailed=提取失败：%1 + +; *** Archive extraction failure details +ArchiveIncorrectPassword=密码不正确。 +ArchiveIsCorrupted=压缩包已损坏。 +ArchiveUnsupportedFormat=不支持的压缩包格式。 + +; *** "Preparing to Install" wizard page +WizardPreparing=正在准备安装 +PreparingDesc=安装程序正在准备安装 [name] 到您的计算机。 +PreviousInstallNotCompleted=先前的程序安装或卸载未完成，需要您重启计算机以完成该安装。%n%n在重启计算机后，再次运行安装程序以完成 [name] 的安装。 +CannotContinue=安装程序不能继续。请点击“取消”退出。 +ApplicationsFound=以下应用程序正在使用将由安装程序更新的文件。建议您允许安装程序自动关闭这些应用程序。 +ApplicationsFound2=以下应用程序正在使用将由安装程序更新的文件。建议您允许安装程序自动关闭这些应用程序。安装完成后，安装程序将尝试重新启动这些应用程序。 +CloseApplications=自动关闭应用程序(&A) +DontCloseApplications=不要关闭应用程序(&D) +ErrorCloseApplications=安装程序无法自动关闭所有应用程序。建议您在继续之前，关闭所有在使用需要由安装程序更新的文件的应用程序。 +PrepareToInstallNeedsRestart=安装程序必须重启您的计算机。计算机重启后，请再次运行安装程序以完成 [name] 的安装。%n%n要立即重启吗？ + +; *** "Installing" wizard page +WizardInstalling=正在安装 +InstallingLabel=安装程序正在安装 [name] 到您的计算机，请稍候。 + +; *** "Setup Completed" wizard page +FinishedHeadingLabel=完成 [name] 安装向导 +FinishedLabelNoIcons=安装程序已在您的计算机中安装了 [name]。 +FinishedLabel=安装程序已在您的计算机中安装了 [name]。您可以通过已安装的快捷方式运行此应用程序。 +ClickFinish=点击“完成”退出安装程序。 +FinishedRestartLabel=为完成 [name] 的安装，安装程序必须重新启动您的计算机。要立即重启吗？ +FinishedRestartMessage=为完成 [name] 的安装，安装程序必须重新启动您的计算机。%n%n要立即重启吗？ +ShowReadmeCheck=是，我想查阅自述文件 +YesRadio=是，立即重启计算机(&Y) +NoRadio=否，稍后重启计算机(&N) +; used for example as 'Run MyProg.exe' +RunEntryExec=运行 %1 +; used for example as 'View Readme.txt' +RunEntryShellExec=查阅 %1 + +; *** "Setup Needs the Next Disk" stuff +ChangeDiskTitle=安装程序需要下一张磁盘 +SelectDiskLabel2=请插入磁盘 %1 并点击“确定”。%n%n如果这个磁盘中的文件可以在下列文件夹之外的文件夹中找到，请输入正确的路径或点击“浏览”。 +PathLabel=路径(&P)： +FileNotInDir2=“%2”中找不到文件“%1”。请插入正确的磁盘或选择其他文件夹。 +SelectDirectoryLabel=请指定下一张磁盘的位置。 + +; *** Installation phase messages +SetupAborted=安装程序未完成安装。%n%n请修正这个问题并重新运行安装程序。 +AbortRetryIgnoreSelectAction=选择操作 +AbortRetryIgnoreRetry=重试(&T) +AbortRetryIgnoreIgnore=忽略错误并继续(&I) +AbortRetryIgnoreCancel=取消安装 +RetryCancelSelectAction=选择操作 +RetryCancelRetry=重试(&T) +RetryCancelCancel=取消 + +; *** Installation status messages +StatusClosingApplications=正在关闭应用程序... +StatusCreateDirs=正在创建目录... +StatusExtractFiles=正在提取文件... +StatusDownloadFiles=正在下载文件... +StatusCreateIcons=正在创建快捷方式... +StatusCreateIniEntries=正在创建 INI 条目... +StatusCreateRegistryEntries=正在创建注册表条目... +StatusRegisterFiles=正在注册文件... +StatusSavingUninstall=正在保存卸载信息... +StatusRunProgram=正在完成安装... +StatusRestartingApplications=正在重启应用程序... +StatusRollback=正在撤销更改... + +; *** Misc. errors +ErrorInternal2=内部错误：%1。 +ErrorFunctionFailedNoCode=%1 失败。 +ErrorFunctionFailed=%1 失败；错误代码 %2。 +ErrorFunctionFailedWithMessage=%1 失败；错误代码 %2。%n%3 +ErrorExecutingProgram=无法执行文件：%n%1 + +; *** Registry errors +ErrorRegOpenKey=打开注册表项时出错：%n%1\%2 +ErrorRegCreateKey=创建注册表项时出错：%n%1\%2 +ErrorRegWriteKey=写入注册表项时出错：%n%1\%2 + +; *** INI errors +ErrorIniEntry=在文件“%1”中创建 INI 条目时出错。 + +; *** File copying errors +FileAbortRetryIgnoreSkipNotRecommended=跳过此文件(&S)（不推荐） +FileAbortRetryIgnoreIgnoreNotRecommended=忽略错误并继续(&I)（不推荐） +SourceIsCorrupted=源文件已损坏。 +SourceDoesntExist=源文件“%1”不存在。 +SourceVerificationFailed=源文件验证失败：%1 +VerificationSignatureDoesntExist=签名文件“%1”不存在。 +VerificationSignatureInvalid=签名文件“%1”无效。 +VerificationKeyNotFound=签名文件“%1”使用了未知的密钥。 +VerificationFileNameIncorrect=文件名不正确。 +VerificationFileTagIncorrect=文件标签不正确。 +VerificationFileSizeIncorrect=文件大小不正确。 +VerificationFileHashIncorrect=文件哈希值不正确。 +ExistingFileReadOnly2=无法替换已存在的文件，它是只读的。 +ExistingFileReadOnlyRetry=移除只读属性并重试(&R) +ExistingFileReadOnlyKeepExisting=保留已存在的文件(&K) +ErrorReadingExistingDest=尝试读取已存在的文件时出错： +FileExistsSelectAction=选择操作 +FileExists2=文件已经存在。 +FileExistsOverwriteExisting=覆盖已存在的文件(&O) +FileExistsKeepExisting=保留已存在的文件(&K) +FileExistsOverwriteOrKeepAll=为接下来的冲突文件执行此操作(&D) +ExistingFileNewerSelectAction=选择操作 +ExistingFileNewer2=已存在的文件比安装程序将要安装的文件还要新。 +ExistingFileNewerOverwriteExisting=覆盖已存在的文件(&O) +ExistingFileNewerKeepExisting=保留已存在的文件(&K)（推荐） +ExistingFileNewerOverwriteOrKeepAll=为接下来的冲突文件执行此操作(&D) +ErrorChangingAttr=尝试更改下列已存在的文件属性时出错： +ErrorCreatingTemp=尝试在目标目录创建文件时出错： +ErrorReadingSource=尝试读取下列源文件时出错： +ErrorCopying=尝试复制下列文件时出错： +ErrorDownloading=尝试下载文件时出错： +ErrorExtracting=尝试提取压缩包时出错： +ErrorReplacingExistingFile=尝试替换已存在的文件时出错： +ErrorRestartReplace=重启并替换失败： +ErrorRenamingTemp=尝试重命名下列目标目录中的一个文件时出错： +ErrorRegisterServer=无法注册 DLL/OCX：%1 +ErrorRegSvr32Failed=RegSvr32 失败；退出代码 %1。 +ErrorRegisterTypeLib=无法注册类型库：%1 + +; *** Uninstall display name markings +; used for example as 'My Program (32-bit)' +UninstallDisplayNameMark=%1 (%2) +; used for example as 'My Program (32-bit, All users)' +UninstallDisplayNameMarks=%1 (%2, %3) +UninstallDisplayNameMark32Bit=32 位 +UninstallDisplayNameMark64Bit=64 位 +UninstallDisplayNameMarkAllUsers=所有用户 +UninstallDisplayNameMarkCurrentUser=当前用户 + +; *** Post-installation errors +ErrorOpeningReadme=尝试打开自述文件时出错。 +ErrorRestartingComputer=安装程序无法重启计算机，请手动重启。 + +; *** Uninstaller messages +UninstallNotFound=文件“%1”不存在。无法卸载。 +UninstallOpenError=文件“%1”不能被打开。无法卸载 +UninstallUnsupportedVer=此版本的卸载程序无法识别卸载日志文件“%1”的格式。无法卸载。 +UninstallUnknownEntry=卸载日志中遇到一个未知条目（%1）。 +ConfirmUninstall=您确认要完全移除 %1 及其所有组件吗？ +UninstallOnlyOnWin64=仅允许在 64 位 Windows 中卸载此程序。 +OnlyAdminCanUninstall=仅使用管理员权限的用户能完成此卸载。 +UninstallStatusLabel=正在从您的计算机中移除 %1，请稍候。 +UninstalledAll=已顺利从您的计算机中移除 %1。 +UninstalledMost=%1 卸载完成。%n%n有部分内容未能被删除，但您可以手动删除它们。 +UninstalledAndNeedsRestart=为完成 %1 的卸载，需要重启您的计算机。%n%n要立即重启吗？ +UninstallDataCorrupted=文件“%1”已损坏。无法卸载。 + +; *** Uninstallation phase messages +ConfirmDeleteSharedFileTitle=删除共享文件？ +ConfirmDeleteSharedFile2=系统表示下列共享文件已不再有任何程序使用。您希望卸载程序删除此共享文件吗？%n%n如果仍有程序正在使用此文件，删除后这些程序可能无法正常运行。如果您不能确定，请选择“否”，保留此文件在系统中不会造成任何损害。 +SharedFileNameLabel=文件名： +SharedFileLocationLabel=位置： +WizardUninstalling=卸载状态 +StatusUninstalling=正在卸载 %1... + +; *** Shutdown block reasons +ShutdownBlockReasonInstallingApp=正在安装 %1。 +ShutdownBlockReasonUninstallingApp=正在卸载 %1。 + +; The custom messages below aren't used by Setup itself, but if you make +; use of them in your scripts, you'll want to translate them. + +[CustomMessages] + +NameAndVersion=%1 版本 %2 +AdditionalIcons=附加快捷方式： +CreateDesktopIcon=创建桌面快捷方式(&D) +CreateQuickLaunchIcon=创建快速启动栏快捷方式(&Q) +ProgramOnTheWeb=%1 网站 +UninstallProgram=卸载 %1 +LaunchProgram=运行 %1 +AssocFileExtension=将 %2 文件扩展名与 %1 建立关联(&A) +AssocingFileExtension=正在将 %2 文件扩展名与 %1 建立关联... +AutoStartProgramGroupDescription=启动： +AutoStartProgram=自动启动 %1 +AddonHostProgramNotFound=您选择的文件夹中无法找到 %1。%n%n您确定要继续吗？ diff --git a/packaging/installer/netx.iss b/packaging/installer/netx.iss index 55090ea..d630e0e 100644 --- a/packaging/installer/netx.iss +++ b/packaging/installer/netx.iss @@ -1,10 +1,11 @@ -; NetX Windows installer (Inno Setup 6+) +; NetX Windows installer (Inno Setup 6+) ; Compile after: packaging\build_release.ps1 ; ISCC.exe packaging\installer\netx.iss +; Encoding: UTF-8 with BOM (required for Chinese CustomMessages) #define MyAppName "NetX" #ifndef MyAppVersion - #define MyAppVersion "0.3.0" + #define MyAppVersion "0.4.18" #endif #define MyAppPublisher "NetX" #define MyAppURL "https://github.com/hansjone/netx" @@ -12,11 +13,17 @@ ; Stage directory produced by build_release.ps1 (relative to this .iss) #define StageDir "..\release\netx-win64" +#define IconFile "..\assets\netx.ico" +#define AssetsDir "..\assets" [Setup] AppId={{A7E3C2D1-9F40-4B8E-9C1A-NETXWIN64001} AppName={#MyAppName} AppVersion={#MyAppVersion} +AppVerName={#MyAppName} {#MyAppVersion} +VersionInfoVersion={#MyAppVersion} +VersionInfoProductName={#MyAppName} +UninstallDisplayName={#MyAppName} {#MyAppVersion} AppPublisher={#MyAppPublisher} AppPublisherURL={#MyAppURL} DefaultDirName={autopf}\NetX @@ -24,46 +31,1253 @@ DefaultGroupName=NetX DisableProgramGroupPage=yes LicenseFile={#StageDir}\LICENSE OutputDir=..\release -OutputBaseFilename=NetX-Setup-{#MyAppVersion} +#ifndef OutputBaseFilename + #define OutputBaseFilename "NetX-Setup-" + MyAppVersion +#endif +OutputBaseFilename={#OutputBaseFilename} +SetupIconFile={#IconFile} +UninstallDisplayIcon={app}\packaging\assets\netx.ico Compression=lzma2 SolidCompression=yes -WizardStyle=modern +; Follows the Windows light/dark setting; windows11 = flat Fluent-style controls. +WizardStyle=modern dynamic windows11 hidebevels +WizardImageFile={#AssetsDir}\wizard-100.png,{#AssetsDir}\wizard-125.png,{#AssetsDir}\wizard-150.png,{#AssetsDir}\wizard-200.png +WizardSmallImageFile={#AssetsDir}\wizard-small-100.png,{#AssetsDir}\wizard-small-125.png,{#AssetsDir}\wizard-small-150.png,{#AssetsDir}\wizard-small-200.png +WizardImageBackColor=#0a1222 ArchitecturesAllowed=x64compatible ArchitecturesInstallIn64BitMode=x64compatible PrivilegesRequired=admin -; Data lives under {commonappdata}\NetX — not overwritten by upgrades -CloseApplications=yes +; Data lives under {commonappdata}\NetX - not overwritten by upgrades. +; Do NOT use CloseApplications=yes — it can freeze/beep on the Tasks/Ready +; pages while scanning locked NetX tray/service processes with no visible dialog. +CloseApplications=no +; Pick from the Windows UI language; no extra dialog. +ShowLanguageDialog=auto [Languages] Name: "english"; MessagesFile: "compiler:Default.isl" +Name: "chinesesimplified"; MessagesFile: "Languages\ChineseSimplified.isl" + +[CustomMessages] +english.CreateDesktopIcon=Create a desktop shortcut +english.SetupOptions=After install: +english.StartTrayNow=Start NetX now +english.RunModeGroup=How NetX runs: +english.RunModeTray=Desktop tray (recommended) - control NetX from the notification area +english.EnableAutostart=Start automatically when I sign in to Windows +english.RunModeService=Windows service - runs at boot without anyone signed in (servers) +english.MaintenanceGroup=Maintenance: +english.EnableAutoUpdate=Install updates automatically every day (needs network) +english.ServiceInstallFailed=NetX was installed, but the Windows service could not be registered (exit %1).%n%nYou can still start NetX from the Start Menu. Log: %2 +english.IconStart=Start NetX +english.IconStop=Stop NetX +english.IconUpdate=Check for updates +english.UninstallDeleteData=Also delete NetX data (%ProgramData%\NetX)?%n%nYes = remove database, settings, and secrets%nNo = keep data for a later reinstall +english.DbPageCaption=Database +english.DbPageDescription=Choose built-in or external PostgreSQL. Optional credential key: paste from an existing install, or leave empty to auto-generate. +english.DbBundled=Built-in PostgreSQL (portable, offline) +english.DbExternal=External PostgreSQL (existing server) +english.DbHost=Host +english.DbPort=Port +english.DbUser=User +english.DbPassword=Password +english.DbName=Database +english.CredKey=Credential key +english.CredKeyHint=NETX_CREDENTIAL_SECRET_KEY (optional — empty = auto-generate) +english.DbFieldsRequired=Please fill in host, port, user, password, and database name. +english.DbTesting=Testing PostgreSQL connection, please wait… +english.DbTestFailed=Cannot connect to PostgreSQL with these settings.%n%n%1%n%nFix the settings and try again. +english.DbTestExtractFailed=Could not extract PostgreSQL client tools for connection test. +english.DbSilentExternalMissing=Silent install with external DB requires /DbHost /DbUser /DbPassword /DbName (optional /DbPort). +english.DbApplyFailed=Database configuration failed after file install (exit %1).%n%nLog: %2%n%nFix the problem, then use Start Menu → Reconfigure database — or reinstall. +english.DbApplyExecFailed=Could not start database configuration script. +english.DbBundledMissing=Built-in PostgreSQL files are missing from the install folder:%n%1%n%nThe Setup package is incomplete. Re-download NetX-Setup and install again. +english.DbEnvMissing=Database configuration did not write %ProgramData%\NetX\.env. +english.ReconfigureDb=Reconfigure database +english.UpgradeKeepDb=Existing NetX data found - database settings will be kept (no wizard). +english.ModePageCaption=Install mode +english.ModePageDescription=An existing NetX installation was found. Choose update or reinstall. +english.ModeUpdate=Update existing installation (keep database settings) +english.ModeReinstall=Reinstall and reconfigure database (same as first install) +english.ModeUpdateHint=Replaces program files only. ProgramData\.env and database data are kept. +english.ModeReinstallHint=Shows the Database page again. Does not delete ProgramData unless you uninstall and choose to. +chinesesimplified.CreateDesktopIcon=创建桌面快捷方式 +chinesesimplified.SetupOptions=安装完成后: +chinesesimplified.StartTrayNow=立即启动 NetX +chinesesimplified.RunModeGroup=运行方式： +chinesesimplified.RunModeTray=桌面托盘（推荐）— 在任务栏通知区域控制 NetX +chinesesimplified.EnableAutostart=登录 Windows 时自动启动 +chinesesimplified.RunModeService=Windows 服务 — 开机即运行，无需登录（适合服务器） +chinesesimplified.MaintenanceGroup=维护： +chinesesimplified.EnableAutoUpdate=每天自动安装更新（需要联网） +chinesesimplified.ServiceInstallFailed=NetX 已安装，但 Windows 服务注册失败（退出码 %1）。%n%n仍可从开始菜单启动 NetX。日志：%2 +chinesesimplified.IconStart=启动 NetX +chinesesimplified.IconStop=停止 NetX +chinesesimplified.IconUpdate=检查更新 +chinesesimplified.UninstallDeleteData=是否同时删除 NetX 数据（%ProgramData%\NetX）？%n%n是 = 删除数据库、配置和密钥%n否 = 保留数据以便以后重装 +chinesesimplified.DbPageCaption=数据库 +chinesesimplified.DbPageDescription=选择内置或外置 PostgreSQL。可选粘贴已有 NETX_CREDENTIAL_SECRET_KEY；留空则自动生成。 +chinesesimplified.DbBundled=内置 PostgreSQL（便携，可离线） +chinesesimplified.DbExternal=外置 PostgreSQL（已有服务器） +chinesesimplified.DbHost=主机 +chinesesimplified.DbPort=端口 +chinesesimplified.DbUser=用户 +chinesesimplified.DbPassword=密码 +chinesesimplified.DbName=数据库名 +chinesesimplified.CredKey=凭据密钥 +chinesesimplified.CredKeyHint=NETX_CREDENTIAL_SECRET_KEY（可选，留空=自动生成） +chinesesimplified.DbFieldsRequired=请填写主机、端口、用户、密码和数据库名。 +chinesesimplified.DbTesting=正在测试 PostgreSQL 连接，请稍候… +chinesesimplified.DbTestFailed=无法用当前设置连接 PostgreSQL。%n%n%1%n%n请改正后重试。 +chinesesimplified.DbTestExtractFailed=无法解压 PostgreSQL 客户端以测试连接。 +chinesesimplified.DbSilentExternalMissing=静默安装外置库需要参数 /DbHost /DbUser /DbPassword /DbName（可选 /DbPort）。 +chinesesimplified.DbApplyFailed=文件安装后数据库自动配置失败（退出码 %1）。%n%n日志: %2%n%n请处理后使用开始菜单 → 重新配置数据库，或重新安装。 +chinesesimplified.DbApplyExecFailed=无法启动数据库配置脚本。 +chinesesimplified.DbBundledMissing=安装目录中缺少内置 PostgreSQL 文件：%n%1%n%n安装包不完整，请重新下载 NetX-Setup 后再装。 +chinesesimplified.DbEnvMissing=数据库配置未写入 %ProgramData%\NetX\.env。 +chinesesimplified.ReconfigureDb=重新配置数据库 +chinesesimplified.UpgradeKeepDb=检测到已有 NetX 数据 - 将保留数据库配置（跳过向导）。 +chinesesimplified.ModePageCaption=安装模式 +chinesesimplified.ModePageDescription=检测到已有 NetX 安装。请选择更新或重装。 +chinesesimplified.ModeUpdate=更新现有安装（保留数据库配置） +chinesesimplified.ModeReinstall=重装并重新配置数据库（与首次安装相同） +chinesesimplified.ModeUpdateHint=仅替换程序文件，保留 ProgramData\.env 与数据库数据。 +chinesesimplified.ModeReinstallHint=再次显示数据库页。不会删除 ProgramData（除非卸载时选择删除）。 [Tasks] -Name: "desktopicon"; Description: "Create a desktop shortcut"; GroupDescription: "Additional icons:"; Flags: unchecked -Name: "firstrun"; Description: "Run first-time setup (database mode) after install"; GroupDescription: "Setup:"; Flags: checkedonce +; Run mode is exclusive: tray + service together would race for the same port. +; On upgrades CurPageChanged pre-selects whatever is actually installed. +Name: "runtray"; Description: "{cm:RunModeTray}"; GroupDescription: "{cm:RunModeGroup}"; Flags: exclusive +Name: "runtray\autostart"; Description: "{cm:EnableAutostart}" +Name: "runservice"; Description: "{cm:RunModeService}"; GroupDescription: "{cm:RunModeGroup}"; Flags: exclusive unchecked +Name: "autoupdate"; Description: "{cm:EnableAutoUpdate}"; GroupDescription: "{cm:MaintenanceGroup}"; Flags: unchecked +Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: checkedonce [Files] -; Entire release stage → {app}. Exclude .portable so installed builds use ProgramData. +; Entire release stage -> {app}. Exclude .portable so installed builds use ProgramData. Source: "{#StageDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs; Excludes: ".portable" +; Connection-test tools (wizard Next / silent PrepareToInstall only) +Source: "test_pg_conn.ps1"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\psql.exe"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libpq.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libssl-3-x64.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libcrypto-3-x64.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libintl-9.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libiconv-2.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libwinpthread-1.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\zlib1.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\liblz4.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libzstd.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libxml2.dll"; Flags: dontcopy +Source: "{#StageDir}\postgres\pgsql\bin\libxslt.dll"; Flags: dontcopy +; VC++ runtime deployed app-local by build_release.ps1 (clean PCs lack the redistributable). +Source: "{#StageDir}\postgres\pgsql\bin\vcruntime140.dll"; Flags: dontcopy +; These helpers are extracted for PrepareToInstall, before the new program tree is copied. +Source: "{#StageDir}\packaging\_common.ps1"; Flags: dontcopy +Source: "{#StageDir}\packaging\service_lifecycle.ps1"; Flags: dontcopy [Dirs] -Name: "{commonappdata}\NetX" -Name: "{commonappdata}\NetX\data" -Name: "{commonappdata}\NetX\data\auth" -Name: "{commonappdata}\NetX\data\runtime" -Name: "{commonappdata}\NetX\pgdata" -Name: "{commonappdata}\NetX\backups" +; users-modify so tray / non-admin reconfigure can update .env and runtime files. +Name: "{commonappdata}\NetX"; Permissions: users-modify +Name: "{commonappdata}\NetX\data"; Permissions: users-modify +Name: "{commonappdata}\NetX\data\auth"; Permissions: users-modify +Name: "{commonappdata}\NetX\data\runtime"; Permissions: users-modify +Name: "{commonappdata}\NetX\pgdata"; Permissions: users-modify +Name: "{commonappdata}\NetX\backups"; Permissions: users-modify + +[InstallDelete] +; Python minor releases can change stdlib layout (for example pathlib.py -> pathlib\\). +; Remove the previous runtime and venv before copying the new ones so an upgrade +; cannot leave orphaned modules that shadow files from the shipped Python version. +Type: filesandordirs; Name: "{app}\python\runtime" +Type: filesandordirs; Name: "{app}\.venv" +; Remove legacy desktop shortcuts from older Setup builds (keep a single NetX icon). +Type: files; Name: "{commondesktop}\NetX Start.lnk" +Type: files; Name: "{commondesktop}\NetX First-time setup.lnk" +Type: files; Name: "{commondesktop}\首次配置（内置或外置数据库）.lnk" +; Start Menu entries from <= 0.4.16 (English-only names; toggles moved into the tray / Tasks page). +Type: files; Name: "{group}\NetX Tray.lnk" +Type: files; Name: "{group}\Start NetX.lnk" +Type: files; Name: "{group}\Stop NetX.lnk" +Type: files; Name: "{group}\Check for updates.lnk" +Type: files; Name: "{group}\Open NetX UI.url" +Type: files; Name: "{group}\Install Windows Service (admin).lnk" +Type: files; Name: "{group}\Enable silent auto-update (daily).lnk" +Type: files; Name: "{group}\Enable start at logon.lnk" [Icons] -Name: "{group}\Start NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" -Name: "{group}\Stop NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\stop_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" -Name: "{group}\Open NetX UI"; Filename: "http://127.0.0.1:8890/" -Name: "{group}\First-time setup"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\setup_first_run.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" -Name: "{autodesktop}\Start NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Tasks: desktopicon +; Prefer .cmd targets so Server 2012 / classic Start Menu shows normal program entries. +; Autostart / auto-update / service toggles live in the tray Settings menu and the Tasks page. +Name: "{group}\NetX"; Filename: "{app}\NetX-Tray.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\{cm:IconStart}"; Filename: "{app}\NetX-Start.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\{cm:IconStop}"; Filename: "{app}\NetX-Stop.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\{cm:IconUpdate}"; Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\launch_shortcut.ps1"" -Action update -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +Name: "{group}\{cm:ReconfigureDb}"; Filename: "{app}\NetX-FirstRun.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" +; One desktop shortcut only (tray = start + tray UI). +Name: "{commondesktop}\NetX"; Filename: "{app}\NetX-Tray.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico"; Tasks: desktopicon [Run] -Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\setup_first_run.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Flags: postinstall skipifsilent; Tasks: firstrun -Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Description: "Start NetX now"; Flags: postinstall nowait skipifsilent unchecked +; Autostart / service / auto-update are applied from the Tasks page in CurStepChanged. +Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\netx_tray.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"" -StartOnLaunch"; WorkingDir: "{app}"; Description: "{cm:StartTrayNow}"; Flags: postinstall runhidden nowait skipifsilent; Check: NetxShouldStartTray + +; Stop runtime BEFORE files are deleted. Do not also Exec prepare from +; InitializeUninstall (that previously raced/killed unins000 or hung the UI). +[UninstallRun] +Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\uninstall_prepare.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; RunOnceId: "StopNetX"; Flags: runhidden waituntilterminated [UninstallDelete] -; Do NOT delete {commonappdata}\NetX — preserves DB and secrets across reinstall +; Program files always removed. Data dir only when user chose Yes in InitializeUninstall. Type: filesandordirs; Name: "{app}" + +[Code] +var + GDeleteNetxData: Boolean; + GTempDeleteDataScript: String; + DbPage: TWizardPage; + RbBundled: TNewRadioButton; + RbExternal: TNewRadioButton; + LblHost: TNewStaticText; + LblPort: TNewStaticText; + LblUser: TNewStaticText; + LblPassword: TNewStaticText; + LblDbName: TNewStaticText; + EdHost: TNewEdit; + EdPort: TNewEdit; + EdUser: TNewEdit; + EdPassword: TNewEdit; + EdDbName: TNewEdit; + LblCredKey: TNewStaticText; + LblCredHint: TNewStaticText; + EdCredKey: TNewEdit; + GDbMode: String; + GDbUrl: String; + GDbHost: String; + GDbPort: String; + GDbUser: String; + GDbPassword: String; + GDbName: String; + GCredKey: String; + GSkipDbPage: Boolean; + GIsUpgrade: Boolean; + GInstallMode: String; + ModePage: TWizardPage; + RbModeUpdate: TNewRadioButton; + RbModeReinstall: TNewRadioButton; + LblModeHint: TNewStaticText; + GDbConfigured: Boolean; + GPgToolsReady: Boolean; + GDbConnOk: Boolean; + GNextBtnCaption: String; + GTasksSynced: Boolean; + +function EnvHasDbMode(): Boolean; +var + Lines: TArrayOfString; + i: Integer; + Line: String; +begin + Result := False; + if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then + Exit; + for i := 0 to GetArrayLength(Lines) - 1 do + begin + Line := Trim(Lines[i]); + if Pos('NETX_DB_MODE=', Line) = 1 then + begin + if Length(Line) > Length('NETX_DB_MODE=') then + begin + Result := True; + Exit; + end; + end; + end; +end; + +function EnvModeIsExternal(): Boolean; +var + Lines: TArrayOfString; + i: Integer; + Line: String; +begin + Result := False; + if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then + Exit; + for i := 0 to GetArrayLength(Lines) - 1 do + begin + Line := Trim(Lines[i]); + if CompareText(Line, 'NETX_DB_MODE=external') = 0 then + begin + Result := True; + Exit; + end; + end; +end; + +function ParamIsTruthy(const ParamName: String): Boolean; +var + V: String; +begin + V := LowerCase(Trim(ExpandConstant('{param:' + ParamName + '|}'))); + Result := (V = '1') or (V = 'true') or (V = 'yes') or (V = 'on'); +end; + +function DetectExistingConfiguredInstall(): Boolean; +var + Lines: TArrayOfString; + i: Integer; + Line: String; +begin + { NETX_DATABASE_URL-only installs predate NETX_DB_MODE and use an external DB. } + Result := EnvHasDbMode(); + if Result then + Exit; + if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then + Exit; + for i := 0 to GetArrayLength(Lines) - 1 do + begin + Line := Trim(Lines[i]); + if (Pos('NETX_DATABASE_URL=', Line) = 1) and + (Length(Line) > Length('NETX_DATABASE_URL=')) then + begin + Result := True; + Exit; + end; + end; +end; + +procedure SyncSkipDbFromInstallMode(); +begin + GSkipDbPage := (GInstallMode = 'update'); +end; + +procedure ApplyInstallModeFromChoice(); +begin + if (RbModeReinstall <> nil) and RbModeReinstall.Checked then + GInstallMode := 'reinstall' + else if GIsUpgrade then + GInstallMode := 'update' + else + GInstallMode := 'fresh'; + SyncSkipDbFromInstallMode(); +end; + +procedure ModeChoiceClick(Sender: TObject); +begin + ApplyInstallModeFromChoice(); + if LblModeHint = nil then + Exit; + if GInstallMode = 'reinstall' then + LblModeHint.Caption := ExpandConstant('{cm:ModeReinstallHint}') + else + LblModeHint.Caption := ExpandConstant('{cm:ModeUpdateHint}'); +end; + +function ResolveInstallModeFromParams(): Boolean; +{ Returns True if a CLI param forced the mode. } +var + Mode: String; +begin + Result := False; + Mode := LowerCase(Trim(ExpandConstant('{param:InstallMode|}'))); + if (Mode = 'update') or (Mode = 'upgrade') then + begin + GInstallMode := 'update'; + Result := True; + end + else if (Mode = 'reinstall') or (Mode = 'fresh') then + begin + GInstallMode := 'reinstall'; + Result := True; + end + else if ParamIsTruthy('SkipDbPage') then + begin + GInstallMode := 'update'; + Result := True; + end + else if ParamIsTruthy('ForceDbPage') then + begin + GInstallMode := 'reinstall'; + Result := True; + end; + if Result then + SyncSkipDbFromInstallMode(); +end; + +procedure StopNetxBeforeFileReplace(); +var + ResultCode: Integer; + LifecycleScript: String; + Params: String; +begin + ExtractTemporaryFile('_common.ps1'); + ExtractTemporaryFile('service_lifecycle.ps1'); + LifecycleScript := ExpandConstant('{tmp}\service_lifecycle.ps1'); + Params := + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + LifecycleScript + '"' + + ' -Action StopForUpdate' + + ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + + ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; + if not Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode) then + RaiseException('Could not stop NetX before replacing program files.'); + if ResultCode <> 0 then + RaiseException('Could not stop NetX services/tasks before replacing program files. See ProgramData\\NetX\\data\\runtime\\service_lifecycle.log.'); +end; + +procedure ResumeNetxServiceAfterInstall(); +var + ResultCode: Integer; + LifecycleScript: String; + Params: String; +begin + LifecycleScript := ExpandConstant('{app}\packaging\service_lifecycle.ps1'); + if not FileExists(LifecycleScript) then + Exit; + Params := + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + LifecycleScript + '"' + + ' -Action ResumeAfterUpdate' + + ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + + ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; + if not Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode) then + RaiseException('NetX was updated, but its previously running service could not be restarted.'); + if ResultCode <> 0 then + RaiseException('NetX was updated, but its previously running service could not be restarted. See ProgramData\\NetX\\data\\runtime\\service_lifecycle.log.'); +end; + +function NetxShouldStartTray(): Boolean; +begin + Result := GDbConfigured or + FileExists(ExpandConstant('{commonappdata}\NetX\.env')); +end; + +procedure UpdateDbFieldState(); +var + Ext: Boolean; +begin + Ext := RbExternal.Checked; + LblHost.Enabled := Ext; + LblPort.Enabled := Ext; + LblUser.Enabled := Ext; + LblPassword.Enabled := Ext; + LblDbName.Enabled := Ext; + EdHost.Enabled := Ext; + EdPort.Enabled := Ext; + EdUser.Enabled := Ext; + EdPassword.Enabled := Ext; + EdDbName.Enabled := Ext; +end; + +procedure DbModeClick(Sender: TObject); +begin + UpdateDbFieldState(); +end; + +function ExtractPgClientTools(): Boolean; +begin + if GPgToolsReady and FileExists(ExpandConstant('{tmp}\psql.exe')) and + FileExists(ExpandConstant('{tmp}\test_pg_conn.ps1')) then + begin + Result := True; + Exit; + end; + Result := False; + try + ExtractTemporaryFile('test_pg_conn.ps1'); + ExtractTemporaryFile('psql.exe'); + ExtractTemporaryFile('libpq.dll'); + ExtractTemporaryFile('libssl-3-x64.dll'); + ExtractTemporaryFile('libcrypto-3-x64.dll'); + ExtractTemporaryFile('libintl-9.dll'); + ExtractTemporaryFile('libiconv-2.dll'); + ExtractTemporaryFile('libwinpthread-1.dll'); + ExtractTemporaryFile('zlib1.dll'); + ExtractTemporaryFile('liblz4.dll'); + ExtractTemporaryFile('libzstd.dll'); + ExtractTemporaryFile('libxml2.dll'); + ExtractTemporaryFile('libxslt.dll'); + ExtractTemporaryFile('vcruntime140.dll'); + Result := FileExists(ExpandConstant('{tmp}\psql.exe')) and + FileExists(ExpandConstant('{tmp}\test_pg_conn.ps1')); + GPgToolsReady := Result; + except + Result := False; + GPgToolsReady := False; + end; +end; + +procedure SetWizardBusy(const Busy: Boolean; const StatusText: String); +begin + if Busy then + begin + if GNextBtnCaption = '' then + GNextBtnCaption := WizardForm.NextButton.Caption; + WizardForm.NextButton.Caption := StatusText; + WizardForm.NextButton.Enabled := False; + WizardForm.BackButton.Enabled := False; + WizardForm.CancelButton.Enabled := False; + end + else + begin + if GNextBtnCaption <> '' then + WizardForm.NextButton.Caption := GNextBtnCaption; + WizardForm.NextButton.Enabled := True; + WizardForm.BackButton.Enabled := True; + WizardForm.CancelButton.Enabled := True; + end; + WizardForm.Refresh; +end; + +function TestExternalDb(const Host, Port, User, Password, DbName: String; var ErrMsg: String): Boolean; +var + ResultCode: Integer; + UrlFile: String; + ErrFile: String; + PwFile: String; + Params: String; + Lines: TArrayOfString; + i: Integer; +begin + Result := False; + ErrMsg := ''; + if not ExtractPgClientTools() then + begin + ErrMsg := ExpandConstant('{cm:DbTestExtractFailed}'); + Exit; + end; + + UrlFile := ExpandConstant('{tmp}\netx_db_url.txt'); + ErrFile := ExpandConstant('{tmp}\netx_db_test_err.txt'); + PwFile := ExpandConstant('{tmp}\netx_db_pw.txt'); + DeleteFile(UrlFile); + DeleteFile(ErrFile); + SaveStringToFile(PwFile, Password, False); + + Params := + '-NoProfile -ExecutionPolicy Bypass -File "' + ExpandConstant('{tmp}\test_pg_conn.ps1') + '"' + + ' -PgHost "' + Host + '"' + + ' -Port ' + Port + + ' -User "' + User + '"' + + ' -PasswordFile "' + PwFile + '"' + + ' -Database "' + DbName + '"' + + ' -PsqlPath "' + ExpandConstant('{tmp}\psql.exe') + '"' + + ' -OutUrlFile "' + UrlFile + '"' + + ' -OutErrFile "' + ErrFile + '"'; + + if not Exec('powershell.exe', Params, ExpandConstant('{tmp}'), SW_HIDE, ewWaitUntilTerminated, ResultCode) then + begin + ErrMsg := 'powershell_exec_failed'; + DeleteFile(PwFile); + Exit; + end; + DeleteFile(PwFile); + if ResultCode <> 0 then + begin + ErrMsg := 'psql/test exit ' + IntToStr(ResultCode); + if LoadStringsFromFile(ErrFile, Lines) then + begin + ErrMsg := ''; + for i := 0 to GetArrayLength(Lines) - 1 do + begin + if ErrMsg <> '' then + ErrMsg := ErrMsg + #13#10; + ErrMsg := ErrMsg + Lines[i]; + end; + if ErrMsg = '' then + ErrMsg := 'psql/test exit ' + IntToStr(ResultCode); + end; + Exit; + end; + if not LoadStringsFromFile(UrlFile, Lines) or (GetArrayLength(Lines) < 1) or (Trim(Lines[0]) = '') then + begin + ErrMsg := 'url_file_missing'; + Exit; + end; + GDbUrl := Trim(Lines[0]); + Result := True; +end; + +function SaveExternalFieldsFromControls(var ErrMsg: String): Boolean; +begin + Result := False; + GDbHost := Trim(EdHost.Text); + GDbPort := Trim(EdPort.Text); + GDbUser := Trim(EdUser.Text); + GDbPassword := EdPassword.Text; + GDbName := Trim(EdDbName.Text); + GDbConnOk := False; + GDbUrl := ''; + if (GDbHost = '') or (GDbPort = '') or (GDbUser = '') or + (GDbPassword = '') or (GDbName = '') then + begin + ErrMsg := ExpandConstant('{cm:DbFieldsRequired}'); + Exit; + end; + GDbMode := 'external'; + Result := True; +end; + +function ValidateExternalDbSaved(var ErrMsg: String): Boolean; +begin + Result := False; + if GDbConnOk and (GDbUrl <> '') then + begin + Result := True; + Exit; + end; + if not TestExternalDb(GDbHost, GDbPort, GDbUser, GDbPassword, GDbName, ErrMsg) then + begin + if ErrMsg = '' then + ErrMsg := 'connection_failed'; + ErrMsg := FmtMessage(ExpandConstant('{cm:DbTestFailed}'), [ErrMsg]); + GDbConnOk := False; + Exit; + end; + GDbConnOk := True; + Result := True; +end; + +function ApplySilentDbParams(var ErrMsg: String): Boolean; +var + Mode, Host, Port, User, Password, DbName: String; +begin + Result := False; + GCredKey := Trim(ExpandConstant('{param:CredentialSecretKey|}')); + Mode := LowerCase(ExpandConstant('{param:DbMode|bundled}')); + if Mode = '' then + Mode := 'bundled'; + if Mode = 'bundled' then + begin + GDbMode := 'bundled'; + GDbUrl := ''; + Result := True; + Exit; + end; + if Mode <> 'external' then + begin + ErrMsg := 'invalid DbMode (use bundled or external)'; + Exit; + end; + Host := ExpandConstant('{param:DbHost|}'); + Port := ExpandConstant('{param:DbPort|5432}'); + User := ExpandConstant('{param:DbUser|}'); + Password := ExpandConstant('{param:DbPassword|}'); + DbName := ExpandConstant('{param:DbName|}'); + if (Host = '') or (User = '') or (Password = '') or (DbName = '') then + begin + ErrMsg := ExpandConstant('{cm:DbSilentExternalMissing}'); + Exit; + end; + if not TestExternalDb(Host, Port, User, Password, DbName, ErrMsg) then + begin + if ErrMsg = '' then + ErrMsg := 'connection_failed'; + ErrMsg := FmtMessage(ExpandConstant('{cm:DbTestFailed}'), [ErrMsg]); + Exit; + end; + GDbMode := 'external'; + Result := True; +end; + +procedure CaptureCredKeyFromControls(); +begin + if EdCredKey <> nil then + GCredKey := Trim(EdCredKey.Text) + else + GCredKey := ''; +end; + +procedure InitializeWizard(); +var + TopPos: Integer; +begin + GDbMode := 'bundled'; + GDbUrl := ''; + GDbHost := ''; + GDbPort := '5432'; + GDbUser := ''; + GDbPassword := ''; + GDbName := ''; + GCredKey := ''; + GDbConfigured := False; + GPgToolsReady := False; + GDbConnOk := False; + { Existing install: interactive Mode page (Update vs Reinstall). + Silent: /InstallMode=update|reinstall (or /SkipDbPage=1 /ForceDbPage=1). } + GIsUpgrade := DetectExistingConfiguredInstall(); + ModePage := nil; + RbModeUpdate := nil; + RbModeReinstall := nil; + LblModeHint := nil; + if not ResolveInstallModeFromParams() then + begin + if GIsUpgrade then + GInstallMode := 'update' + else + GInstallMode := 'fresh'; + SyncSkipDbFromInstallMode(); + end; + + if GIsUpgrade and (not WizardSilent) then + begin + ModePage := CreateCustomPage(wpSelectDir, + ExpandConstant('{cm:ModePageCaption}'), + ExpandConstant('{cm:ModePageDescription}')); + + RbModeUpdate := TNewRadioButton.Create(ModePage); + RbModeUpdate.Parent := ModePage.Surface; + RbModeUpdate.Caption := ExpandConstant('{cm:ModeUpdate}'); + RbModeUpdate.Checked := (GInstallMode <> 'reinstall'); + RbModeUpdate.Top := ScaleY(8); + RbModeUpdate.Left := ScaleX(0); + RbModeUpdate.Width := ModePage.SurfaceWidth; + RbModeUpdate.OnClick := @ModeChoiceClick; + + RbModeReinstall := TNewRadioButton.Create(ModePage); + RbModeReinstall.Parent := ModePage.Surface; + RbModeReinstall.Caption := ExpandConstant('{cm:ModeReinstall}'); + RbModeReinstall.Checked := (GInstallMode = 'reinstall'); + RbModeReinstall.Top := RbModeUpdate.Top + ScaleY(28); + RbModeReinstall.Left := ScaleX(0); + RbModeReinstall.Width := ModePage.SurfaceWidth; + RbModeReinstall.OnClick := @ModeChoiceClick; + + LblModeHint := TNewStaticText.Create(ModePage); + LblModeHint.Parent := ModePage.Surface; + LblModeHint.Top := RbModeReinstall.Top + ScaleY(36); + LblModeHint.Left := ScaleX(0); + LblModeHint.Width := ModePage.SurfaceWidth; + LblModeHint.Height := ScaleY(60); + LblModeHint.AutoSize := False; + LblModeHint.WordWrap := True; + ModeChoiceClick(nil); + + DbPage := CreateCustomPage(ModePage.ID, + ExpandConstant('{cm:DbPageCaption}'), + ExpandConstant('{cm:DbPageDescription}')); + end + else + DbPage := CreateCustomPage(wpSelectDir, + ExpandConstant('{cm:DbPageCaption}'), + ExpandConstant('{cm:DbPageDescription}')); + + RbBundled := TNewRadioButton.Create(DbPage); + RbBundled.Parent := DbPage.Surface; + RbBundled.Caption := ExpandConstant('{cm:DbBundled}'); + RbBundled.Checked := True; + RbBundled.Top := ScaleY(8); + RbBundled.Left := ScaleX(0); + RbBundled.Width := DbPage.SurfaceWidth; + RbBundled.OnClick := @DbModeClick; + + RbExternal := TNewRadioButton.Create(DbPage); + RbExternal.Parent := DbPage.Surface; + RbExternal.Caption := ExpandConstant('{cm:DbExternal}'); + RbExternal.Top := RbBundled.Top + ScaleY(28); + RbExternal.Left := ScaleX(0); + RbExternal.Width := DbPage.SurfaceWidth; + RbExternal.OnClick := @DbModeClick; + + TopPos := RbExternal.Top + ScaleY(36); + + LblHost := TNewStaticText.Create(DbPage); + LblHost.Parent := DbPage.Surface; + LblHost.Caption := ExpandConstant('{cm:DbHost}'); + LblHost.Top := TopPos; + LblHost.Left := ScaleX(16); + + EdHost := TNewEdit.Create(DbPage); + EdHost.Parent := DbPage.Surface; + EdHost.Top := TopPos; + EdHost.Left := ScaleX(120); + EdHost.Width := ScaleX(220); + EdHost.Text := '127.0.0.1'; + + TopPos := TopPos + ScaleY(28); + LblPort := TNewStaticText.Create(DbPage); + LblPort.Parent := DbPage.Surface; + LblPort.Caption := ExpandConstant('{cm:DbPort}'); + LblPort.Top := TopPos; + LblPort.Left := ScaleX(16); + + EdPort := TNewEdit.Create(DbPage); + EdPort.Parent := DbPage.Surface; + EdPort.Top := TopPos; + EdPort.Left := ScaleX(120); + EdPort.Width := ScaleX(80); + EdPort.Text := '5432'; + + TopPos := TopPos + ScaleY(28); + LblUser := TNewStaticText.Create(DbPage); + LblUser.Parent := DbPage.Surface; + LblUser.Caption := ExpandConstant('{cm:DbUser}'); + LblUser.Top := TopPos; + LblUser.Left := ScaleX(16); + + EdUser := TNewEdit.Create(DbPage); + EdUser.Parent := DbPage.Surface; + EdUser.Top := TopPos; + EdUser.Left := ScaleX(120); + EdUser.Width := ScaleX(220); + EdUser.Text := 'netx'; + + TopPos := TopPos + ScaleY(28); + LblPassword := TNewStaticText.Create(DbPage); + LblPassword.Parent := DbPage.Surface; + LblPassword.Caption := ExpandConstant('{cm:DbPassword}'); + LblPassword.Top := TopPos; + LblPassword.Left := ScaleX(16); + + EdPassword := TNewEdit.Create(DbPage); + EdPassword.Parent := DbPage.Surface; + EdPassword.Top := TopPos; + EdPassword.Left := ScaleX(120); + EdPassword.Width := ScaleX(220); + EdPassword.PasswordChar := '*'; + + TopPos := TopPos + ScaleY(28); + LblDbName := TNewStaticText.Create(DbPage); + LblDbName.Parent := DbPage.Surface; + LblDbName.Caption := ExpandConstant('{cm:DbName}'); + LblDbName.Top := TopPos; + LblDbName.Left := ScaleX(16); + + EdDbName := TNewEdit.Create(DbPage); + EdDbName.Parent := DbPage.Surface; + EdDbName.Top := TopPos; + EdDbName.Left := ScaleX(120); + EdDbName.Width := ScaleX(220); + EdDbName.Text := 'netx'; + + TopPos := TopPos + ScaleY(36); + LblCredKey := TNewStaticText.Create(DbPage); + LblCredKey.Parent := DbPage.Surface; + LblCredKey.Caption := ExpandConstant('{cm:CredKey}'); + LblCredKey.Top := TopPos; + LblCredKey.Left := ScaleX(0); + + EdCredKey := TNewEdit.Create(DbPage); + EdCredKey.Parent := DbPage.Surface; + EdCredKey.Top := TopPos; + EdCredKey.Left := ScaleX(120); + EdCredKey.Width := DbPage.SurfaceWidth - ScaleX(120); + EdCredKey.Text := ''; + EdCredKey.PasswordChar := '*'; + + TopPos := TopPos + ScaleY(24); + LblCredHint := TNewStaticText.Create(DbPage); + LblCredHint.Parent := DbPage.Surface; + LblCredHint.Caption := ExpandConstant('{cm:CredKeyHint}'); + LblCredHint.Top := TopPos; + LblCredHint.Left := ScaleX(120); + LblCredHint.Width := DbPage.SurfaceWidth - ScaleX(120); + LblCredHint.AutoSize := False; + + UpdateDbFieldState(); + + { Prefill from existing ProgramData\.env when present. } + if EnvModeIsExternal() then + begin + RbExternal.Checked := True; + UpdateDbFieldState(); + end + else if EnvHasDbMode() then + begin + RbBundled.Checked := True; + UpdateDbFieldState(); + end; + + if WizardSilent then + begin + { Params applied in PrepareToInstall } + end; +end; + +function ShouldSkipPage(PageID: Integer): Boolean; +begin + Result := False; + if (ModePage <> nil) and (PageID = ModePage.ID) then + begin + { Only interactive existing installs show the mode page. } + Result := (not GIsUpgrade) or WizardSilent; + Exit; + end; + if (DbPage <> nil) and (PageID = DbPage.ID) then + Result := GSkipDbPage; +end; + +function NextButtonClick(CurPageID: Integer): Boolean; +var + ErrMsg: String; +begin + Result := True; + if (ModePage <> nil) and (CurPageID = ModePage.ID) then + begin + ApplyInstallModeFromChoice(); + Exit; + end; + { Only the DB page runs validation. Other pages must always proceed + (Tasks/Ready used to appear "dead" when CloseApplications hung). } + if (DbPage = nil) or (CurPageID <> DbPage.ID) or GSkipDbPage then + Exit; + + CaptureCredKeyFromControls(); + if RbBundled.Checked then + begin + GDbMode := 'bundled'; + GDbUrl := ''; + GDbConnOk := True; + Result := True; + Exit; + end; + + if not SaveExternalFieldsFromControls(ErrMsg) then + begin + MsgBox(ErrMsg, mbError, MB_OK); + Result := False; + Exit; + end; + + { Connection test here so bad settings never reach Tasks/Ready. + Show busy state — ExtractTemporaryFile + psql can take a few seconds. } + SetWizardBusy(True, ExpandConstant('{cm:DbTesting}')); + try + Result := ValidateExternalDbSaved(ErrMsg); + finally + SetWizardBusy(False, ''); + end; + if not Result then + MsgBox(ErrMsg, mbError, MB_OK); +end; + +function PrepareToInstall(var NeedsRestart: Boolean): String; +var + ErrMsg: String; +begin + Result := ''; + NeedsRestart := False; + + if (ModePage <> nil) and (not WizardSilent) then + ApplyInstallModeFromChoice(); + + { Stop tray / API / bundled PG so Program Files can be replaced safely. } + if GSkipDbPage or GIsUpgrade or DirExists(ExpandConstant('{app}\packaging')) then + StopNetxBeforeFileReplace(); + + if GSkipDbPage then + Exit; + + if WizardSilent then + begin + if not ApplySilentDbParams(ErrMsg) then + Result := ErrMsg; + Exit; + end; + + if GDbMode = '' then + GDbMode := 'bundled'; + + { Re-check only if user never got a successful test (should be rare). } + if (GDbMode = 'external') and (not GDbConnOk) then + begin + if not ValidateExternalDbSaved(ErrMsg) then + Result := ErrMsg; + end; +end; + +function ApplyDatabaseConfig(): Boolean; +var + ResultCode: Integer; + Params: String; + UrlFile: String; + KeyFile: String; + LogFile: String; + InitDb: String; + EnvFile: String; + Wrapper: String; + WrapBody: String; +begin + Result := False; + GDbConfigured := False; + if GDbMode = '' then + GDbMode := 'bundled'; + + InitDb := ExpandConstant('{app}\postgres\pgsql\bin\initdb.exe'); + if (GDbMode = 'bundled') and (not FileExists(InitDb)) then + begin + MsgBox(FmtMessage(ExpandConstant('{cm:DbBundledMissing}'), [InitDb]), mbError, MB_OK); + Exit; + end; + + ForceDirectories(ExpandConstant('{commonappdata}\NetX\data\runtime')); + LogFile := ExpandConstant('{commonappdata}\NetX\data\runtime\setup_first_run.log'); + Wrapper := ExpandConstant('{tmp}\netx_apply_db.ps1'); + + WrapBody := + '$ErrorActionPreference = ''Stop''' + #13#10 + + '$log = ''' + LogFile + '''' + #13#10 + + 'Start-Transcript -Path $log -Force | Out-Null' + #13#10 + + 'try {' + #13#10 + + ' & ''' + ExpandConstant('{app}\packaging\setup_first_run.ps1') + '''' + + ' -ProgramRoot ''' + ExpandConstant('{app}') + '''' + + ' -DataRoot ''' + ExpandConstant('{commonappdata}\NetX') + '''' + + ' -NonInteractive -DbMode ' + GDbMode; + + if GDbMode = 'external' then + begin + UrlFile := ExpandConstant('{tmp}\netx_db_url.txt'); + if GDbUrl <> '' then + SaveStringToFile(UrlFile, GDbUrl, False); + WrapBody := WrapBody + + ' -ExternalDatabaseUrlFile ''' + UrlFile + ''' -SkipExternalProbe'; + end; + + if GCredKey <> '' then + begin + KeyFile := ExpandConstant('{tmp}\netx_cred_key.txt'); + SaveStringToFile(KeyFile, GCredKey, False); + WrapBody := WrapBody + ' -CredentialSecretKeyFile ''' + KeyFile + ''''; + end; + + WrapBody := WrapBody + #13#10 + + ' if ($null -ne $LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE }' + #13#10 + + '} catch {' + #13#10 + + ' Write-Error $_' + #13#10 + + ' exit 1' + #13#10 + + '} finally {' + #13#10 + + ' Stop-Transcript | Out-Null' + #13#10 + + '}' + #13#10; + + SaveStringToFile(Wrapper, WrapBody, False); + Params := '-NoProfile -ExecutionPolicy Bypass -File "' + Wrapper + '"'; + + if not Exec( + 'powershell.exe', Params, ExpandConstant('{app}'), + SW_HIDE, ewWaitUntilTerminated, ResultCode) then + begin + MsgBox(ExpandConstant('{cm:DbApplyExecFailed}'), mbError, MB_OK); + Exit; + end; + if ResultCode <> 0 then + begin + MsgBox(FmtMessage(ExpandConstant('{cm:DbApplyFailed}'), [IntToStr(ResultCode), LogFile]), mbError, MB_OK); + Exit; + end; + + EnvFile := ExpandConstant('{commonappdata}\NetX\.env'); + if not FileExists(EnvFile) then + begin + MsgBox(ExpandConstant('{cm:DbEnvMissing}'), mbError, MB_OK); + Exit; + end; + + GDbConfigured := True; + Result := True; +end; + +function InitializeUninstall(): Boolean; +var + DeleteSrc: String; + LifecycleScript: String; + Params: String; + ResultCode: Integer; +begin + Result := True; + GDeleteNetxData := False; + GTempDeleteDataScript := ExpandConstant('{tmp}\netx_uninstall_delete_data.ps1'); + + // Copy data-wipe helper to TEMP before program files are removed. + DeleteSrc := ExpandConstant('{app}\packaging\uninstall_delete_data.ps1'); + if FileExists(DeleteSrc) then + CopyFile(DeleteSrc, GTempDeleteDataScript, False); + + if UninstallSilent then + begin + // Settings often uses QuietUninstallString; keep data by default. + GDeleteNetxData := False; + end + else + begin + GDeleteNetxData := + MsgBox(ExpandConstant('{cm:UninstallDeleteData}'), mbConfirmation, MB_YESNO) = IDYES; + end; + + LifecycleScript := ExpandConstant('{app}\packaging\service_lifecycle.ps1'); + if FileExists(LifecycleScript) then + begin + Params := + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + LifecycleScript + '"' + + ' -Action RemoveForUninstall' + + ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + + ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; + if (not Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode)) or + (ResultCode <> 0) then + begin + MsgBox('Could not stop/remove the NetX Windows service or startup task. Uninstall has been cancelled to protect the installation. See ProgramData\\NetX\\data\\runtime\\service_lifecycle.log.', mbError, MB_OK); + Result := False; + Exit; + end; + end; +end; + +procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep); +var + ResultCode: Integer; + DataRoot: String; +begin + if (CurUninstallStep = usPostUninstall) and GDeleteNetxData then + begin + DataRoot := ExpandConstant('{commonappdata}\NetX'); + if FileExists(GTempDeleteDataScript) then + begin + Exec( + 'powershell.exe', + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + + GTempDeleteDataScript + '" -DataRoot "' + DataRoot + '"', + ExpandConstant('{tmp}'), + SW_HIDE, + ewWaitUntilTerminated, + ResultCode + ); + end; + if DirExists(DataRoot) then + DelTree(DataRoot, True, True, True); + end; +end; + +procedure RepairNetxVenvAfterInstall(); +var + ResultCode: Integer; + RepairScript: String; + Params: String; +begin + RepairScript := ExpandConstant('{app}\packaging\repair_venv.ps1'); + if not FileExists(RepairScript) then + Exit; + Params := + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + RepairScript + '"' + + ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + + ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; + Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode); +end; + +function NetxTaskExists(const Name: String): Boolean; +var + RC: Integer; +begin + Result := Exec(ExpandConstant('{sys}\schtasks.exe'), '/Query /TN "\NetX\' + Name + '"', '', + SW_HIDE, ewWaitUntilTerminated, RC) and (RC = 0); +end; + +function NetxServiceExists(): Boolean; +var + RC: Integer; +begin + Result := Exec(ExpandConstant('{sys}\sc.exe'), 'query NetX', '', SW_HIDE, ewWaitUntilTerminated, RC) and (RC = 0); +end; + +procedure CurPageChanged(CurPageID: Integer); +begin + { Existing install: show the run mode / toggles that are really in place + (older versions set them from Finish-page checkboxes Inno never recorded). } + if (CurPageID = wpSelectTasks) and (not GTasksSynced) then + begin + GTasksSynced := True; + if FileExists(ExpandConstant('{app}\packaging\netx_tray.ps1')) then + begin + if NetxServiceExists() then + WizardSelectTasks('runservice') + else + WizardSelectTasks('runtray'); + if NetxTaskExists('NetXTray') then + WizardSelectTasks('runtray\autostart') + else + WizardSelectTasks('!runtray\autostart'); + if NetxTaskExists('NetXUpdate') then + WizardSelectTasks('autoupdate') + else + WizardSelectTasks('!autoupdate'); + end; + end; +end; + +function RunNetxScript(const Script, Extra: String): Integer; +var + Params: String; +begin + Params := + '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + ExpandConstant('{app}\packaging\' + Script) + '"' + + ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + + ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"' + Extra; + if not Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, Result) then + Result := -1; +end; + +function TasksGivenOnCommandLine(): Boolean; +var + i: Integer; + P: String; +begin + Result := False; + for i := 1 to ParamCount do + begin + P := Uppercase(ParamStr(i)); + if (Pos('/TASKS=', P) = 1) or (Pos('/MERGETASKS=', P) = 1) then + begin + Result := True; + Exit; + end; + end; +end; + +procedure ApplyRunModeTasks(); +var + Code: Integer; +begin + { Silent auto-updates keep whatever is installed unless /TASKS is passed. } + if WizardSilent and (not TasksGivenOnCommandLine()) then + Exit; + if not FileExists(ExpandConstant('{commonappdata}\NetX\.env')) then + Exit; + + if WizardIsTaskSelected('runservice') then + begin + Code := RunNetxScript('install_service.ps1', ' -Start'); + if Code <> 0 then + SuppressibleMsgBox(FmtMessage(ExpandConstant('{cm:ServiceInstallFailed}'), [IntToStr(Code), + ExpandConstant('{commonappdata}\NetX\data\runtime')]), mbError, MB_OK, IDOK); + end + else if NetxServiceExists() then + RunNetxScript('install_service.ps1', ' -Uninstall'); + + if WizardIsTaskSelected('runtray\autostart') then + RunNetxScript('install_autostart.ps1', '') + else + RunNetxScript('install_autostart.ps1', ' -Remove'); + + if WizardIsTaskSelected('autoupdate') then + RunNetxScript('install_update_task.ps1', '') + else if NetxTaskExists('NetXUpdate') then + RunNetxScript('install_update_task.ps1', ' -Remove'); +end; + +procedure CurStepChanged(CurStep: TSetupStep); +var + UninstallKey: String; +begin + if CurStep = ssPostInstall then + begin + { Always relink .venv -> local python/runtime (Setup is elevated). } + RepairNetxVenvAfterInstall(); + + { Upgrade keeps existing .env / pgdata - do not re-run DB wizard. } + if not GSkipDbPage then + ApplyDatabaseConfig() + else + GDbConfigured := FileExists(ExpandConstant('{commonappdata}\NetX\.env')); + + UninstallKey := + 'Software\Microsoft\Windows\CurrentVersion\Uninstall\{A7E3C2D1-9F40-4B8E-9C1A-NETXWIN64001}_is1'; + RegDeleteValue(HKLM, UninstallKey, 'QuietUninstallString'); + ApplyRunModeTasks(); + ResumeNetxServiceAfterInstall(); + end; +end; diff --git a/packaging/installer/test_pg_conn.ps1 b/packaging/installer/test_pg_conn.ps1 new file mode 100644 index 0000000..efc8323 --- /dev/null +++ b/packaging/installer/test_pg_conn.ps1 @@ -0,0 +1,104 @@ +param( + [Parameter(Mandatory = $true)][string]$PgHost, + [int]$Port = 5432, + [Parameter(Mandatory = $true)][string]$User, + [string]$Password = "", + [string]$PasswordFile = "", + [Parameter(Mandatory = $true)][string]$Database, + [Parameter(Mandatory = $true)][string]$PsqlPath, + [string]$OutUrlFile = "", + [string]$OutErrFile = "" +) + +$ErrorActionPreference = "Stop" + +function Write-ErrFile([string]$Message) { + if ($OutErrFile) { + Set-Content -LiteralPath $OutErrFile -Value $Message -Encoding utf8 + } + [Console]::Error.WriteLine($Message) +} + +if (-not (Test-Path -LiteralPath $PsqlPath)) { + Write-ErrFile "psql_not_found: $PsqlPath" + exit 2 +} +if ($PasswordFile) { + if (-not (Test-Path -LiteralPath $PasswordFile)) { + Write-ErrFile "password_file_missing" + exit 3 + } + $Password = [IO.File]::ReadAllText($PasswordFile).TrimEnd("`r", "`n") +} +if ([string]::IsNullOrWhiteSpace($PgHost)) { + Write-ErrFile "host_required" + exit 3 +} +if ([string]::IsNullOrWhiteSpace($User) -or [string]::IsNullOrWhiteSpace($Database)) { + Write-ErrFile "user_and_database_required" + exit 3 +} +if ([string]::IsNullOrWhiteSpace($Password)) { + Write-ErrFile "password_required" + exit 3 +} +if ($Port -lt 1 -or $Port -gt 65535) { + Write-ErrFile "invalid_port: $Port" + exit 3 +} + +$encUser = [uri]::EscapeDataString($User) +$encPw = [uri]::EscapeDataString($Password) +$encDb = [uri]::EscapeDataString($Database) +$url = "postgresql+psycopg://${encUser}:${encPw}@${PgHost}:${Port}/${encDb}" + +$psqlDir = Split-Path -Parent $PsqlPath +$prevPath = $env:PATH +$prevPw = $env:PGPASSWORD +try { + $env:PATH = "$psqlDir;$env:PATH" + $env:PGPASSWORD = $Password + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $PsqlPath + $psi.Arguments = "-h `"$PgHost`" -p $Port -U `"$User`" -d `"$Database`" -v ON_ERROR_STOP=1 -t -A -c `"SELECT 1`"" + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + $psi.WorkingDirectory = $psqlDir + $p = [Diagnostics.Process]::Start($psi) + $stdout = $p.StandardOutput.ReadToEnd() + $stderr = $p.StandardError.ReadToEnd() + $p.WaitForExit() + if ($p.ExitCode -ne 0) { + $msg = (($stderr + "`n" + $stdout).Trim()) + if (-not $msg) { $msg = "psql_exit_$($p.ExitCode)" } + Write-ErrFile "connection_failed: $msg" + exit 1 + } + if (($stdout.Trim()) -notmatch '1') { + Write-ErrFile "unexpected_result: $($stdout.Trim())" + exit 1 + } +} catch { + Write-ErrFile ("connection_failed: " + $_.Exception.Message) + exit 1 +} finally { + $env:PATH = $prevPath + if ($null -eq $prevPw) { + Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue + } else { + $env:PGPASSWORD = $prevPw + } +} + +if ($OutUrlFile) { + $dir = Split-Path -Parent $OutUrlFile + if ($dir -and -not (Test-Path -LiteralPath $dir)) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + } + Set-Content -LiteralPath $OutUrlFile -Value $url -Encoding ascii -NoNewline +} + +Write-Output "ok" +exit 0 diff --git a/packaging/launch_shortcut.ps1 b/packaging/launch_shortcut.ps1 new file mode 100644 index 0000000..fcc5fa2 --- /dev/null +++ b/packaging/launch_shortcut.ps1 @@ -0,0 +1,129 @@ +param( + [ValidateSet("tray", "start", "stop", "setup", "update")] + [string]$Action = "tray", + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [switch]$StartOnLaunch = $true +) + +# Visible entrypoint for Start Menu / desktop shortcuts. +# Nested PowerShell runs are hidden; only failures show a message box. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden + +Add-Type -AssemblyName System.Windows.Forms + +$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') +function T([string]$En, [string]$Zh) { + if ($zh) { return $Zh } else { return $En } +} + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$logDir = Join-Path $data "data\runtime" +if (-not (Test-Path $logDir)) { + New-Item -ItemType Directory -Path $logDir -Force | Out-Null +} +$log = Join-Path $logDir "launch.log" + +function Write-LaunchLog([string]$Msg) { + $line = "[{0}] {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Msg + Add-Content -Path $log -Value $line -Encoding utf8 +} + +function Show-NetxError([string]$Msg) { + Write-LaunchLog "ERROR $Msg" + [void][System.Windows.Forms.MessageBox]::Show( + $Msg, + "NetX", + [System.Windows.Forms.MessageBoxButtons]::OK, + [System.Windows.Forms.MessageBoxIcon]::Error + ) +} + +function Invoke-NetxHiddenPs1 { + param( + [string]$File, + [string[]]$ExtraArgs = @(), + [switch]$NoWait = $false + ) + return Start-NetxPowerShell -File $File ` + -Arguments (@("-ProgramRoot", $prog, "-DataRoot", $data) + $ExtraArgs) ` + -WorkingDirectory $prog -WindowStyle Hidden -PassThru -Wait:(-not $NoWait) +} + +function Ensure-NetxEnv { + $envPath = Join-Path $data ".env" + if (Test-Path $envPath) { return } + $cmd = Join-Path $prog "NetX-FirstRun.cmd" + throw (T ` + "NetX is not configured yet (missing $envPath).`nRe-run Setup and choose built-in or external DB,`nor Start Menu → Reconfigure database:`n$cmd" ` + "尚未完成数据库配置（缺少 $envPath）。`n请重新运行安装向导选择内置/外置库，`n或开始菜单 → 重新配置数据库：`n$cmd") +} + +try { + Write-LaunchLog "action=$Action prog=$prog data=$data" + switch ($Action) { + "setup" { + # Visible console so user can pick bundled vs external DB. + $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot "setup_first_run.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` + -WorkingDirectory $prog -WindowStyle Normal -Wait -PassThru + if ($p.ExitCode -ne 0) { throw "setup_exit_$($p.ExitCode)" } + if (Test-Path (Join-Path $data ".env")) { + Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-StartOnLaunch") ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } + } + "stop" { + $p = Invoke-NetxHiddenPs1 -File (Join-Path $PSScriptRoot "runtime_ctl.ps1") -ExtraArgs @("-Action", "stop") + if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0) { throw "stop_exit_$($p.ExitCode)" } + } + "update" { + Ensure-NetxEnv + # Update UI may need a visible window for prompts. + $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot "check_update.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` + -WorkingDirectory $prog -WindowStyle Normal -Wait -PassThru + if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0 -and $p.ExitCode -ne 10) { + throw "update_exit_$($p.ExitCode)" + } + } + "start" { + Ensure-NetxEnv + $hostBind = "127.0.0.1" + $port = 8890 + $envPath = Join-Path $data ".env" + if (Test-Path $envPath) { + $map = Read-DotEnv -Path $envPath + if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } + if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } + } + $hostBind = Get-NetxClientHost -BindHost $hostBind + $p = Invoke-NetxHiddenPs1 -File (Join-Path $PSScriptRoot "runtime_ctl.ps1") -ExtraArgs @("-Action", "start") + if ($p.ExitCode -ne 0) { throw "start_exit_$($p.ExitCode)" } + if (Wait-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 180) { + Open-NetxUrl -Url "http://${hostBind}:${port}/" + } else { + throw (T ` + "NetX started but /health not ready within 180s.`nSee: $logDir\netx.err.log" ` + "NetX 已启动但 /health 在 180 秒内未就绪。`n请查看: $logDir\netx.err.log") + } + } + "tray" { + Ensure-NetxEnv + $extra = @("-ProgramRoot", $prog, "-DataRoot", $data) + if ($StartOnLaunch) { $extra += "-StartOnLaunch" } + # Detach tray; do not leave a success MessageBox (keeps a console open). + Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` + -Arguments $extra -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } + } + Write-LaunchLog "action=$Action ok" +} catch { + Show-NetxError ((T "NetX failed to start:`n$($_.Exception.Message)" "NetX 启动失败：`n$($_.Exception.Message)") + "`n`n$log") + exit 1 +} diff --git a/packaging/manifest.example.json b/packaging/manifest.example.json index 9f51c14..ff24d0d 100644 --- a/packaging/manifest.example.json +++ b/packaging/manifest.example.json @@ -1,11 +1,13 @@ { "channel": "stable", - "latest": "0.3.0", + "latest": "0.4.18", "min_compatible": "0.3.0", - "notes_url": "", + "notes_url": "https://github.com/hansjone/netx/releases/tag/v0.4.18", "windows": { - "url": "https://example.com/netx/NetX-0.2.0-win64.zip", - "sha256": "REPLACE_WITH_SHA256", + "url": "https://github.com/hansjone/netx/releases/download/v0.4.18/NetX-Setup-0.4.18.exe", + "setup_url": "https://github.com/hansjone/netx/releases/download/v0.4.18/NetX-Setup-0.4.18.exe", + "package": "setup", + "sha256": "", "size": 0 } } diff --git a/packaging/netx_tray.ps1 b/packaging/netx_tray.ps1 new file mode 100644 index 0000000..c74b364 --- /dev/null +++ b/packaging/netx_tray.ps1 @@ -0,0 +1,985 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [switch]$StartOnLaunch = $true, + # Logon autostart / post-update relaunch: bring NetX up but never pop a browser. + [switch]$QuietStart = $false, + [switch]$CheckUpdateOnLaunch = $false, + [switch]$AutoUpdate = $false +) + +# System-tray controller for NetX (Windows packaged installs). +# Always elevated: Start/Stop/Update touch Program Files + services. +# The UI thread never blocks: /health and scheduled-task state are polled in a +# background runspace, child scripts are watched by a timer. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden + +Add-Type -AssemblyName System.Windows.Forms +Add-Type -AssemblyName System.Drawing + +if (-not ("NetxMenuRenderer" -as [type])) { + Add-Type -ReferencedAssemblies System.Windows.Forms, System.Drawing -TypeDefinition @" +using System; +using System.Drawing; +using System.Drawing.Drawing2D; +using System.Runtime.InteropServices; +using System.Windows.Forms; + +public static class NetxNative { + [DllImport("user32.dll")] public static extern bool SetProcessDPIAware(); + [DllImport("user32.dll")] public static extern bool DestroyIcon(IntPtr handle); + [DllImport("dwmapi.dll")] static extern int DwmSetWindowAttribute(IntPtr hwnd, int attr, ref int value, int size); + + // Windows 11: rounded popup corners + themed 1px border drawn by DWM. + public static bool RoundCorners(IntPtr hwnd, Color border) { + try { + int pref = 2; // DWMWCP_ROUND + if (DwmSetWindowAttribute(hwnd, 33, ref pref, 4) != 0) return false; + int c = ColorTranslator.ToWin32(border); + DwmSetWindowAttribute(hwnd, 34, ref c, 4); + return true; + } catch { return false; } + } +} + +public sealed class NetxTheme { + public Color Back, Hover, Text, Muted, Sep, Border; + public bool DwmRounded; +} + +public sealed class NetxMenuColorTable : ProfessionalColorTable { + readonly NetxTheme t; + public NetxMenuColorTable(NetxTheme theme) { t = theme; UseSystemColors = false; } + public override Color ToolStripDropDownBackground { get { return t.Back; } } + public override Color ImageMarginGradientBegin { get { return t.Back; } } + public override Color ImageMarginGradientMiddle { get { return t.Back; } } + public override Color ImageMarginGradientEnd { get { return t.Back; } } + public override Color MenuBorder { get { return t.Border; } } + public override Color MenuItemBorder { get { return t.Hover; } } + public override Color MenuItemSelected { get { return t.Hover; } } + public override Color MenuItemSelectedGradientBegin { get { return t.Hover; } } + public override Color MenuItemSelectedGradientEnd { get { return t.Hover; } } + public override Color MenuItemPressedGradientBegin { get { return t.Hover; } } + public override Color MenuItemPressedGradientMiddle { get { return t.Hover; } } + public override Color MenuItemPressedGradientEnd { get { return t.Hover; } } + public override Color SeparatorDark { get { return t.Sep; } } + public override Color SeparatorLight { get { return t.Sep; } } +} + +public sealed class NetxMenuRenderer : ToolStripProfessionalRenderer { + readonly NetxTheme t; + readonly float scale; + + public NetxMenuRenderer(NetxTheme theme, float dpiScale) : base(new NetxMenuColorTable(theme)) { + t = theme; + scale = dpiScale; + RoundedEdges = false; + } + + int S(int px) { return (int)Math.Round(px * scale); } + + static GraphicsPath Round(Rectangle r, int radius) { + int d = radius * 2; + GraphicsPath p = new GraphicsPath(); + p.AddArc(r.X, r.Y, d, d, 180, 90); + p.AddArc(r.Right - d, r.Y, d, d, 270, 90); + p.AddArc(r.Right - d, r.Bottom - d, d, d, 0, 90); + p.AddArc(r.X, r.Bottom - d, d, d, 90, 90); + p.CloseFigure(); + return p; + } + + protected override void OnRenderToolStripBackground(ToolStripRenderEventArgs e) { + using (SolidBrush b = new SolidBrush(t.Back)) e.Graphics.FillRectangle(b, e.AffectedBounds); + } + + protected override void OnRenderImageMargin(ToolStripRenderEventArgs e) { } + + protected override void OnRenderToolStripBorder(ToolStripRenderEventArgs e) { + if (t.DwmRounded) return; // DWM draws the rounded border on Windows 11 + Rectangle r = new Rectangle(0, 0, e.AffectedBounds.Width - 1, e.AffectedBounds.Height - 1); + using (Pen pen = new Pen(t.Border)) e.Graphics.DrawRectangle(pen, r); + } + + protected override void OnRenderMenuItemBackground(ToolStripItemRenderEventArgs e) { + ToolStripItem item = e.Item; + if (!item.Selected || !item.Enabled) return; + Rectangle r = new Rectangle(S(4), S(1), item.Width - S(8), item.Height - S(2)); + e.Graphics.SmoothingMode = SmoothingMode.AntiAlias; + using (GraphicsPath path = Round(r, S(4))) + using (SolidBrush b = new SolidBrush(t.Hover)) e.Graphics.FillPath(b, path); + } + + protected override void OnRenderItemText(ToolStripItemTextRenderEventArgs e) { + // Drawn directly: the base renderer forces grey on disabled (header) items. + string role = e.Item.Tag as string; + Color c; + if (role == "header") c = t.Text; + else if (role == "status") c = t.Muted; + else c = e.Item.Enabled ? t.Text : t.Muted; + TextRenderer.DrawText(e.Graphics, e.Text, e.TextFont, e.TextRectangle, c, e.TextFormat); + } + + protected override void OnRenderItemImage(ToolStripItemImageRenderEventArgs e) { + string role = e.Item.Tag as string; + if (e.Image != null && (e.Item.Enabled || role == "status" || role == "header")) { + e.Graphics.DrawImage(e.Image, e.ImageRectangle); + return; + } + base.OnRenderItemImage(e); + } + + protected override void OnRenderArrow(ToolStripArrowRenderEventArgs e) { + e.ArrowColor = t.Muted; + base.OnRenderArrow(e); + } + + protected override void OnRenderSeparator(ToolStripSeparatorRenderEventArgs e) { + int y = e.Item.Height / 2; + using (Pen pen = new Pen(t.Sep)) e.Graphics.DrawLine(pen, S(12), y, e.Item.Width - S(12), y); + } +} +"@ +} + +# Crisp text on high-DPI screens (powershell.exe is DPI-unaware by default). +try { [void][NetxNative]::SetProcessDPIAware() } catch {} +[System.Windows.Forms.Application]::EnableVisualStyles() + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$envPath = Join-Path $data ".env" +$logDir = Join-Path $data "data\runtime" + +$zh = ([cultureinfo]::CurrentUICulture.Name -match '^zh') +function T([string]$En, [string]$Zh) { + if ($zh) { return $Zh } else { return $En } +} + +if (-not (Test-Path $envPath)) { + [void][System.Windows.Forms.MessageBox]::Show( + (T ` + "NetX is not configured yet (missing $envPath).`nRe-run the installer and choose built-in or external DB,`nor use Start Menu → NetX → Reconfigure database." ` + "尚未完成数据库配置（缺少 $envPath）。`n请重新运行安装向导并选择内置或外置数据库，`n或使用「开始菜单 → NetX → 重新配置数据库」。"), + "NetX", + [System.Windows.Forms.MessageBoxButtons]::OK, + [System.Windows.Forms.MessageBoxIcon]::Warning + ) + exit 1 +} + +function Read-NetxTrayConfig { + $map = Read-DotEnv -Path $envPath + $bind = if ($map["NETX_HOST"]) { $map["NETX_HOST"] } else { "127.0.0.1" } + $p = 8890 + if ($map["NETX_PORT"]) { try { $p = [int]$map["NETX_PORT"] } catch {} } + $script:clientHost = Get-NetxClientHost -BindHost $bind + $script:port = $p + $script:uiUrl = "http://$($script:clientHost):$p/" + $script:dbModeLabel = if ((Get-DbMode -EnvMap $map) -eq "bundled") { (T "Built-in DB" "内置库") } else { (T "External DB" "外置库") } + $script:autoUpdateEnv = ([string]$map["NETX_UPDATE_AUTO"] -match '^(1|true|yes|on)$') -or + ([string]$env:NETX_UPDATE_AUTO -match '^(1|true|yes|on)$') +} +Read-NetxTrayConfig +if ($script:autoUpdateEnv) { + $AutoUpdate = $true + $CheckUpdateOnLaunch = $true +} +$script:ver = Get-NetxVersion -ProgramRoot $prog + +# Only one tray icon per logon session (desktop + postinstall + autostart can race). +$script:netxTrayMutex = $null +try { + $createdNew = $false + $script:netxTrayMutex = New-Object System.Threading.Mutex($true, "Local\NetX.WinTray.SingleInstance", [ref]$createdNew) + if (-not $createdNew) { + if (-not $QuietStart) { + if (Test-NetxApiHealthy -HostName $script:clientHost -Port $script:port -TimeoutSec 2) { + Open-NetxUrl -Url $script:uiUrl + } elseif ($StartOnLaunch) { + Start-NetxPowerShell -File (Join-Path $PSScriptRoot "runtime_ctl.ps1") ` + -Arguments @("-Action", "start", "-ProgramRoot", $prog, "-DataRoot", $data) ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } + } + exit 0 + } +} catch { + # If mutex fails, continue with a tray (better than no UI control). +} + +$runtimePs1 = Join-Path $PSScriptRoot "runtime_ctl.ps1" +$checkPs1 = Join-Path $PSScriptRoot "check_update.ps1" +$setupPs1 = Join-Path $PSScriptRoot "setup_first_run.ps1" +$autostartPs1 = Join-Path $PSScriptRoot "install_autostart.ps1" +$updateTaskPs1 = Join-Path $PSScriptRoot "install_update_task.ps1" + +# ---------------------------------------------------------------- background probe +# /health + runner/task state off the UI thread; the UI timer only reads $sync. +$script:sync = [hashtable]::Synchronized(@{ + HealthUrl = "$($script:uiUrl)health" + Healthy = $false + Seen = $false + Runner = "" + Autostart = $false + AutoUpdate = $false + Stop = $false + }) +$script:probeRunspace = [runspacefactory]::CreateRunspace() +$script:probeRunspace.Open() +$script:probeRunspace.SessionStateProxy.SetVariable("sync", $script:sync) +$script:probePs = [powershell]::Create() +$script:probePs.Runspace = $script:probeRunspace +[void]$script:probePs.AddScript({ + $i = 0 + while (-not $sync.Stop) { + $ok = $false + try { + $r = Invoke-WebRequest -Uri $sync.HealthUrl -UseBasicParsing -TimeoutSec 2 -MaximumRedirection 0 + if ($r.StatusCode -eq 200) { $ok = (($r.Content | ConvertFrom-Json).status -eq "ok") } + } catch {} + $sync.Healthy = $ok + $sync.Seen = $true + if (($i % 6) -eq 0) { + $runner = "" + $svc = Get-Service -Name "NetX" -ErrorAction SilentlyContinue + if ($svc -and [string]$svc.StartType -ne "Disabled") { $runner = "service" } + $tasks = @{} + try { + Get-ScheduledTask -TaskPath "\NetX\" -ErrorAction Stop | ForEach-Object { $tasks[$_.TaskName] = [string]$_.State } + } catch {} + if (-not $runner -and $tasks["NetXService"] -and $tasks["NetXService"] -ne "Disabled") { $runner = "task" } + $sync.Runner = $runner + $sync.Autostart = [bool]($tasks["NetXTray"] -and $tasks["NetXTray"] -ne "Disabled") + $sync.AutoUpdate = [bool]($tasks["NetXUpdate"] -and $tasks["NetXUpdate"] -ne "Disabled") + } + $i++ + for ($k = 0; $k -lt 15 -and -not $sync.Stop -and -not $sync.Poke; $k++) { Start-Sleep -Milliseconds 100 } + $sync.Poke = $false + } + }) +$script:probeHandle = $script:probePs.BeginInvoke() + +function Request-NetxProbe { $script:sync.Poke = $true } + +# ---------------------------------------------------------------- theme + drawing +$script:dpiScale = 1.0 +try { + $g = [System.Drawing.Graphics]::FromHwnd([IntPtr]::Zero) + $script:dpiScale = $g.DpiX / 96.0 + $g.Dispose() +} catch {} +function Get-NetxPx([int]$Px) { return [int][Math]::Round($Px * $script:dpiScale) } + +$script:glyphFont = "Segoe MDL2 Assets" +try { + $installed = (New-Object System.Drawing.Text.InstalledFontCollection).Families | ForEach-Object { $_.Name } + if ($installed -contains "Segoe Fluent Icons") { $script:glyphFont = "Segoe Fluent Icons" } +} catch {} +$script:uiFontName = if ($zh) { "Microsoft YaHei UI" } else { "Segoe UI" } + +function Test-NetxDarkMode { + try { + $v = Get-ItemPropertyValue -Path "HKCU:\Software\Microsoft\Windows\CurrentVersion\Themes\Personalize" -Name "AppsUseLightTheme" -ErrorAction Stop + return ([int]$v -eq 0) + } catch { return $false } +} + +function New-NetxTheme([bool]$Dark) { + $t = New-Object NetxTheme + $c = { param($r, $g, $b) [System.Drawing.Color]::FromArgb($r, $g, $b) } + if ($Dark) { + $t.Back = & $c 28 32 40 + $t.Hover = & $c 44 54 72 + $t.Text = & $c 232 237 244 + $t.Muted = & $c 140 150 166 + $t.Sep = & $c 50 56 68 + $t.Border = & $c 58 66 82 + } else { + $t.Back = & $c 251 252 253 + $t.Hover = & $c 228 237 251 + $t.Text = & $c 26 31 38 + $t.Muted = & $c 108 117 130 + $t.Sep = & $c 229 232 237 + $t.Border = & $c 212 218 226 + } + return $t +} + +$script:stateColors = @{ + running = [System.Drawing.Color]::FromArgb(34, 197, 94) + stopped = [System.Drawing.Color]::FromArgb(148, 160, 178) + starting = [System.Drawing.Color]::FromArgb(245, 166, 35) + stopping = [System.Drawing.Color]::FromArgb(245, 166, 35) + checking = [System.Drawing.Color]::FromArgb(148, 160, 178) +} + +function New-NetxGlyph([char]$Glyph, [System.Drawing.Color]$Color) { + $sz = Get-NetxPx 16 + $bmp = New-Object System.Drawing.Bitmap($sz, $sz) + $g = [System.Drawing.Graphics]::FromImage($bmp) + $g.TextRenderingHint = [System.Drawing.Text.TextRenderingHint]::AntiAliasGridFit + $font = New-Object System.Drawing.Font($script:glyphFont, [float]($sz * 0.78), [System.Drawing.GraphicsUnit]::Pixel) + $brush = New-Object System.Drawing.SolidBrush($Color) + $fmt = New-Object System.Drawing.StringFormat + $fmt.Alignment = [System.Drawing.StringAlignment]::Center + $fmt.LineAlignment = [System.Drawing.StringAlignment]::Center + $g.DrawString([string]$Glyph, $font, $brush, (New-Object System.Drawing.RectangleF(0, 1, $sz, $sz)), $fmt) + $fmt.Dispose(); $brush.Dispose(); $font.Dispose(); $g.Dispose() + return $bmp +} + +function New-NetxDot([System.Drawing.Color]$Color) { + $sz = Get-NetxPx 16 + $bmp = New-Object System.Drawing.Bitmap($sz, $sz) + $g = [System.Drawing.Graphics]::FromImage($bmp) + $g.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias + $d = [Math]::Max(6, [int]($sz * 0.5)) + $o = [int](($sz - $d) / 2) + $brush = New-Object System.Drawing.SolidBrush($Color) + $g.FillEllipse($brush, $o, $o, $d, $d) + $brush.Dispose(); $g.Dispose() + return $bmp +} + +$script:glyphs = @{ + open = [char]0xE8A7 + start = [char]0xE768 + stop = [char]0xE71A + restart = [char]0xE72C + settings = [char]0xE713 + update = [char]0xE895 + repair = [char]0xE90F + folder = [char]0xE8B7 + log = [char]0xE8A5 + check = [char]0xE73E + exit = [char]0xE8BB + power = [char]0xE7E8 +} + +# Brand mark from the 256px PNG (small .ico frames do not convert cleanly). +$script:brandPng = $null +try { $script:brandPng = [System.Drawing.Image]::FromFile((Join-Path $PSScriptRoot "assets\netx-256.png")) } catch {} +function New-NetxBrandBitmap([int]$Size) { + $bmp = New-Object System.Drawing.Bitmap($Size, $Size) + $g = [System.Drawing.Graphics]::FromImage($bmp) + $g.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic + $g.PixelOffsetMode = [System.Drawing.Drawing2D.PixelOffsetMode]::HighQuality + $rect = New-Object System.Drawing.Rectangle(0, 0, $Size, $Size) + if ($script:brandPng) { $g.DrawImage($script:brandPng, $rect) } + else { $g.DrawIcon([System.Drawing.SystemIcons]::Application, $rect) } + $g.Dispose() + return $bmp +} + +# Tray icons with a status dot, built once per state. +$script:iconHandles = @() +function New-NetxStateIcon([System.Drawing.Color]$Dot) { + $sz = [System.Windows.Forms.SystemInformation]::SmallIconSize.Width + $bmp = New-NetxBrandBitmap $sz + $g = [System.Drawing.Graphics]::FromImage($bmp) + $g.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias + $d = [Math]::Max(5, [int][Math]::Round($sz * 0.42)) + $x = $sz - $d + $y = $sz - $d + $ring = New-Object System.Drawing.SolidBrush([System.Drawing.Color]::FromArgb(15, 23, 36)) + $g.FillEllipse($ring, $x - 1, $y - 1, $d + 1, $d + 1) + $brush = New-Object System.Drawing.SolidBrush($Dot) + $g.FillEllipse($brush, $x + 1, $y + 1, $d - 3, $d - 3) + $ring.Dispose(); $brush.Dispose(); $g.Dispose() + $h = $bmp.GetHicon() + $bmp.Dispose() + $script:iconHandles += $h + return [System.Drawing.Icon]::FromHandle($h) +} +$script:trayIcons = @{} +foreach ($k in $script:stateColors.Keys) { + try { $script:trayIcons[$k] = New-NetxStateIcon $script:stateColors[$k] } catch {} +} + +# ---------------------------------------------------------------- window + icon +$form = New-Object System.Windows.Forms.Form +$form.Text = "NetX" +$form.ShowInTaskbar = $false +$form.WindowState = "Minimized" +$form.Visible = $false +$form.Size = New-Object System.Drawing.Size(0, 0) + +$notify = New-Object System.Windows.Forms.NotifyIcon +$notify.Icon = if ($script:trayIcons["checking"]) { $script:trayIcons["checking"] } else { [System.Drawing.SystemIcons]::Application } +$notify.Text = "NetX" +$notify.Visible = $true + +function Show-NetxToast([string]$Text, [string]$Kind = "Info", [int]$Ms = 4000) { + try { $notify.ShowBalloonTip($Ms, "NetX", $Text, [System.Windows.Forms.ToolTipIcon]::$Kind) } catch {} +} + +function Show-NetxDialog([string]$Text, [string]$Kind = "Information", [string]$Buttons = "OK") { + return [System.Windows.Forms.MessageBox]::Show( + $Text, "NetX", + [System.Windows.Forms.MessageBoxButtons]::$Buttons, + [System.Windows.Forms.MessageBoxIcon]::$Kind + ) +} + +# ---------------------------------------------------------------- child processes +# Never Start-Process -Wait on the WinForms UI thread — it freezes the tray menu. +$script:watches = New-Object System.Collections.ArrayList +$script:watchTimer = New-Object System.Windows.Forms.Timer +$script:watchTimer.Interval = 300 +$script:watchTimer.add_Tick({ + foreach ($w in @($script:watches)) { + $done = $true + try { $done = [bool]$w.Proc.HasExited } catch {} + if (-not $done) { continue } + [void]$script:watches.Remove($w) + if ($w.OnExit) { + try { & $w.OnExit $w.Proc $w.Context } catch { Show-NetxToast $_.Exception.Message "Warning" } + } + } + if ($script:watches.Count -eq 0) { $script:watchTimer.Stop() } + }) + +function Watch-NetxProcess($Process, [scriptblock]$OnExit, [hashtable]$Context = @{}) { + # OnExit runs later from the timer: pass anything it needs via -Context + # (function locals are gone by then). + if ($null -eq $Process) { + if ($OnExit) { & $OnExit $null $Context } + return + } + [void]$script:watches.Add(@{ Proc = $Process; OnExit = $OnExit; Context = $Context }) + $script:watchTimer.Start() +} + +function Start-NetxChild([string]$File, [string[]]$ExtraArgs = @(), [string]$WindowStyle = "Hidden") { + return Start-NetxPowerShell -File $File -Arguments (@("-ProgramRoot", $prog, "-DataRoot", $data) + $ExtraArgs) ` + -WorkingDirectory $prog -WindowStyle $WindowStyle -PassThru +} + +function Get-NetxExitCode($Process) { + if ($null -eq $Process) { return $null } + try { return $Process.ExitCode } catch { return $null } +} + +function Test-NetxSetupCancelled($ExitCode) { + # STATUS_CONTROL_C_EXIT (0xC000013A): console closed / Ctrl+C — not a failure. + return ($null -ne $ExitCode -and [int]$ExitCode -eq -1073741510) +} + +# ---------------------------------------------------------------- runtime state +# pending = an in-flight start/stop/restart; it clears once the child exits and +# /health matches the target (or on timeout). +$script:pending = $null +$script:openWhenReady = $null +$script:lastState = "" + +function Invoke-NetxRuntime([ValidateSet("start", "stop", "restart")][string]$Action, [switch]$OpenUi, [scriptblock]$Then) { + if ($script:pending) { return } + $kind = if ($Action -eq "stop") { "stopping" } else { "starting" } + $timeout = if ($Action -eq "stop") { 90 } else { 200 } + $proc = $null + try { + $proc = Start-NetxChild -File $runtimePs1 -ExtraArgs @("-Action", $Action) + } catch { + Show-NetxToast ((T "Could not run NetX control script: " "无法执行 NetX 控制脚本：") + $_.Exception.Message) "Error" + return + } + $script:pending = @{ + Kind = $kind + Target = ($Action -ne "stop") + Done = $false + Until = (Get-Date).AddSeconds($timeout) + Then = $Then + } + if ($OpenUi) { $script:openWhenReady = (Get-Date).AddSeconds($timeout) } + Update-NetxTrayState + Watch-NetxProcess $proc -Context @{ Action = $Action } -OnExit { + param($p, $ctx) + $code = Get-NetxExitCode $p + $pend = $script:pending + if ($null -ne $code -and $code -ne 0) { + $script:pending = $null + $script:openWhenReady = $null + Show-NetxToast ((T "NetX $($ctx.Action) failed (exit $code). Logs: " "NetX 操作失败（退出码 $code）。日志：") + $logDir) "Error" 8000 + } elseif ($pend) { + $pend.Done = $true + } + Request-NetxProbe + Update-NetxTrayState + if ($pend -and $pend.Then) { & $pend.Then $code } + } +} + +function Get-NetxTrayState { + $healthy = [bool]$script:sync.Healthy + $p = $script:pending + if ($p) { + if ((Get-Date) -gt $p.Until) { + $script:pending = $null + } elseif ($p.Done -and ($healthy -eq $p.Target)) { + $script:pending = $null + } else { + return $p.Kind + } + } + if (-not $script:sync.Seen) { return "checking" } + if ($healthy) { return "running" } + return "stopped" +} + +$script:stateText = @{ + running = (T "Running" "运行中") + stopped = (T "Stopped" "已停止") + starting = (T "Starting…" "正在启动…") + stopping = (T "Stopping…" "正在停止…") + checking = (T "Checking…" "检测中…") +} + +function Update-NetxTrayState { + $state = Get-NetxTrayState + if ($script:openWhenReady) { + if ($state -eq "running") { + $script:openWhenReady = $null + Open-NetxUrl -Url $script:uiUrl + } elseif ((Get-Date) -gt $script:openWhenReady -or ($state -eq "stopped" -and -not $script:pending)) { + $script:openWhenReady = $null + Show-NetxToast (T "NetX did not become ready. Check data\runtime\netx.err.log." "NetX 未能就绪，请查看 data\runtime\netx.err.log。") "Warning" 8000 + } + } + $runner = [string]$script:sync.Runner + $key = "$state|$runner|$($script:ver)" + if ($key -eq $script:lastState) { return } + $prevState = ($script:lastState -split '\|')[0] + $script:lastState = $key + + $text = $script:stateText[$state] + $miStatus.Text = $text + $miStatus.Image = $script:dots[$state] + $mode = if ($runner -eq "service") { (T "Service" "服务") } elseif ($runner -eq "task") { (T "Startup task" "启动任务") } else { $script:dbModeLabel } + $miStatus.ShortcutKeyDisplayString = "$mode · $($script:clientHost):$($script:port)" + $miHeader.ShortcutKeyDisplayString = "v$($script:ver)" + + $busy = ($state -in @("starting", "stopping", "checking")) + $miStart.Enabled = (-not $busy) -and ($state -eq "stopped") + $miStop.Enabled = (-not $busy) -and ($state -eq "running") + $miRestart.Enabled = (-not $busy) -and ($state -eq "running") + $miOpen.Enabled = ($state -ne "stopping") + + if ($script:trayIcons[$state]) { $notify.Icon = $script:trayIcons[$state] } + $tip = "NetX $($script:ver) · $text" + if ($tip.Length -gt 63) { $tip = $tip.Substring(0, 63) } + $notify.Text = $tip + + # Requested stops pass through "stopping"; a direct running→stopped is external. + if ($prevState -eq "running" -and $state -eq "stopped") { + Show-NetxToast (T "NetX has stopped." "NetX 已停止运行。") "Warning" + } +} + +# ---------------------------------------------------------------- menu +$menu = New-Object System.Windows.Forms.ContextMenuStrip +$menu.ShowImageMargin = $true +$menu.ShowCheckMargin = $false +$menu.ImageScalingSize = New-Object System.Drawing.Size((Get-NetxPx 16), (Get-NetxPx 16)) +$menu.Font = New-Object System.Drawing.Font($script:uiFontName, 9.0) +$menu.Padding = New-Object System.Windows.Forms.Padding((Get-NetxPx 4), (Get-NetxPx 6), (Get-NetxPx 4), (Get-NetxPx 6)) +$menu.DropShadowEnabled = $true + +$script:itemGlyph = @{} +function New-NetxMenuItem([string]$Text, [string]$Glyph = "", [string]$Role = "", [switch]$Bold) { + $item = New-Object System.Windows.Forms.ToolStripMenuItem + $item.Text = $Text + $item.Tag = $Role + $item.AutoSize = $true + $item.Padding = New-Object System.Windows.Forms.Padding((Get-NetxPx 6), (Get-NetxPx 6), (Get-NetxPx 18), (Get-NetxPx 6)) + $item.Margin = New-Object System.Windows.Forms.Padding(0) + if ($Glyph) { $script:itemGlyph[$item] = $Glyph } + $style = if ($Bold) { [System.Drawing.FontStyle]::Bold } else { [System.Drawing.FontStyle]::Regular } + $size = if ($Role -eq "status") { 8.5 } else { 9.0 } + $item.Font = New-Object System.Drawing.Font($script:uiFontName, [float]$size, $style) + if ($Role -in @("header", "status")) { $item.Enabled = $false } + return $item +} + +function New-NetxMenuSeparator { + $sep = New-Object System.Windows.Forms.ToolStripSeparator + $sep.Margin = New-Object System.Windows.Forms.Padding(0, (Get-NetxPx 3), 0, (Get-NetxPx 3)) + return $sep +} + +function Set-NetxDropDownChrome($DropDown) { + $DropDown.Renderer = $script:renderer + $DropDown.add_HandleCreated({ + param($s, $e) + [void][NetxNative]::RoundCorners($s.Handle, $script:theme.Border) + }) +} + +$miHeader = New-NetxMenuItem -Text "NetX" -Role "header" -Bold +$miStatus = New-NetxMenuItem -Text $script:stateText["checking"] -Role "status" +$miOpen = New-NetxMenuItem -Text (T "Open NetX" "打开界面") -Glyph "open" -Bold +$miStart = New-NetxMenuItem -Text (T "Start" "启动") -Glyph "start" +$miStop = New-NetxMenuItem -Text (T "Stop" "停止") -Glyph "stop" +$miRestart = New-NetxMenuItem -Text (T "Restart" "重启") -Glyph "restart" +$miSettings = New-NetxMenuItem -Text (T "Settings" "设置") -Glyph "settings" +$miAutostart = New-NetxMenuItem -Text (T "Start with Windows" "登录 Windows 时自动启动") -Glyph "check" +$miAutoUpdate = New-NetxMenuItem -Text (T "Daily automatic updates" "每日自动更新") -Glyph "check" +$miSource = New-NetxMenuItem -Text (T "Update source" "更新源") -Glyph "update" +$miSourceGithub = New-NetxMenuItem -Text (T "GitHub (default)" "GitHub（默认）") -Glyph "check" +$miSourceForgejo = New-NetxMenuItem -Text "git.avelo.top" -Glyph "check" +$miReconfig = New-NetxMenuItem -Text (T "Reconfigure database…" "重新配置数据库…") -Glyph "repair" +$miLogs = New-NetxMenuItem -Text (T "Open log folder" "打开日志目录") -Glyph "log" +$miData = New-NetxMenuItem -Text (T "Open data folder" "打开数据目录") -Glyph "folder" +$miUpdate = New-NetxMenuItem -Text (T "Check for updates…" "检查更新…") -Glyph "update" +$miExit = New-NetxMenuItem -Text (T "Exit tray (keep NetX running)" "退出托盘（NetX 保持运行）") -Glyph "exit" +$miStopExit = New-NetxMenuItem -Text (T "Stop NetX and exit" "停止 NetX 并退出") -Glyph "power" + +[void]$miSource.DropDownItems.AddRange(@($miSourceGithub, $miSourceForgejo)) +[void]$miSettings.DropDownItems.AddRange(@( + $miAutostart, $miAutoUpdate, $miSource, (New-NetxMenuSeparator), + $miReconfig, $miLogs, $miData + )) +[void]$menu.Items.AddRange(@( + $miHeader, $miStatus, (New-NetxMenuSeparator), + $miOpen, (New-NetxMenuSeparator), + $miStart, $miStop, $miRestart, (New-NetxMenuSeparator), + $miSettings, $miUpdate, (New-NetxMenuSeparator), + $miExit, $miStopExit + )) + +$script:themeDark = $null +function Set-NetxTrayTheme { + $dark = Test-NetxDarkMode + if ($dark -eq $script:themeDark) { return } + $script:themeDark = $dark + $script:theme = New-NetxTheme $dark + $script:theme.DwmRounded = ([Environment]::OSVersion.Version.Build -ge 22000) + $script:renderer = New-Object NetxMenuRenderer($script:theme, [float]$script:dpiScale) + $menu.Renderer = $script:renderer + $miSettings.DropDown.Renderer = $script:renderer + $miSource.DropDown.Renderer = $script:renderer + $script:glyphImages = @{} + foreach ($k in $script:glyphs.Keys) { + $script:glyphImages[$k] = New-NetxGlyph $script:glyphs[$k] $script:theme.Text + } + $script:glyphImages["open"] = New-NetxGlyph $script:glyphs["open"] ([System.Drawing.Color]::FromArgb(64, 140, 255)) + foreach ($it in $script:itemGlyph.Keys) { + $gk = $script:itemGlyph[$it] + if ($gk -ne "check") { $it.Image = $script:glyphImages[$gk] } + } + $script:dots = @{} + foreach ($k in $script:stateColors.Keys) { $script:dots[$k] = New-NetxDot $script:stateColors[$k] } + if (-not $miHeader.Image) { $miHeader.Image = New-NetxBrandBitmap (Get-NetxPx 16) } + $script:lastState = "" +} +Set-NetxTrayTheme +Set-NetxDropDownChrome $menu +Set-NetxDropDownChrome $miSettings.DropDown +Set-NetxDropDownChrome $miSource.DropDown + +# Update source = NETX_UPDATE_SOURCES in .env (read by check_update.ps1). +# Empty = default (GitHub first, git.avelo.top mirror as fallback); "forgejo" = mirror only. +function Get-NetxUpdateSource { + $v = [string](Read-DotEnv -Path $envPath)["NETX_UPDATE_SOURCES"] + if ($v.Trim().ToLowerInvariant() -eq "forgejo") { return "forgejo" } + return "github" +} + +$menu.add_Opening({ + Set-NetxTrayTheme + if ($menu.IsHandleCreated) { [void][NetxNative]::RoundCorners($menu.Handle, $script:theme.Border) } + Update-NetxTrayState + }) +$miSettings.DropDown.add_Opening({ + $dd = $miSettings.DropDown + if ($dd.IsHandleCreated) { [void][NetxNative]::RoundCorners($dd.Handle, $script:theme.Border) } + $miAutostart.Image = if ($script:sync.Autostart) { $script:glyphImages["check"] } else { $null } + $miAutoUpdate.Image = if ($script:sync.AutoUpdate) { $script:glyphImages["check"] } else { $null } + }) +$miSource.DropDown.add_Opening({ + $dd = $miSource.DropDown + if ($dd.IsHandleCreated) { [void][NetxNative]::RoundCorners($dd.Handle, $script:theme.Border) } + $src = Get-NetxUpdateSource + $miSourceGithub.Image = if ($src -eq "github") { $script:glyphImages["check"] } else { $null } + $miSourceForgejo.Image = if ($src -eq "forgejo") { $script:glyphImages["check"] } else { $null } + }) + +function Invoke-NetxTrayDeferred([scriptblock]$Work) { + # Close the menu first: ContextMenuStrip stays modal while a Click handler runs. + try { $menu.Close() } catch {} + $script:deferredWork = $Work + $t = New-Object System.Windows.Forms.Timer + $t.Interval = 30 + $t.add_Tick({ + param($s, $e) + $s.Stop(); $s.Dispose() + $w = $script:deferredWork + $script:deferredWork = $null + if ($w) { & $w } + }) + $t.Start() +} + +function Open-NetxUi { + if ($script:sync.Healthy) { + Open-NetxUrl -Url $script:uiUrl + } elseif ($script:pending -and $script:pending.Kind -eq "starting") { + $script:openWhenReady = $script:pending.Until + } else { + Show-NetxToast (T "NetX is not running. Starting…" "NetX 未运行，正在启动…") + Invoke-NetxRuntime -Action start -OpenUi + } +} + +$miOpen.add_Click({ Invoke-NetxTrayDeferred { Open-NetxUi } }) +$miStart.add_Click({ Invoke-NetxTrayDeferred { Invoke-NetxRuntime -Action start -OpenUi:(-not $QuietStart) } }) +$miStop.add_Click({ Invoke-NetxTrayDeferred { Invoke-NetxRuntime -Action stop } }) +$miRestart.add_Click({ Invoke-NetxTrayDeferred { Invoke-NetxRuntime -Action restart } }) +$miLogs.add_Click({ Invoke-NetxTrayDeferred { Open-NetxUrl -Url $logDir } }) +$miData.add_Click({ Invoke-NetxTrayDeferred { Open-NetxUrl -Url $data } }) + +function Switch-NetxTaskToggle([string]$Script, [string]$SyncKey, [string]$OnText, [string]$OffText) { + $enable = -not [bool]$script:sync[$SyncKey] + $extra = if ($enable) { @() } else { @("-Remove") } + try { + $p = Start-NetxChild -File $Script -ExtraArgs $extra + } catch { + Show-NetxToast $_.Exception.Message "Error" + return + } + $ctx = @{ Key = $SyncKey; Enable = $enable; Text = $(if ($enable) { $OnText } else { $OffText }) } + Watch-NetxProcess $p -Context $ctx -OnExit { + param($proc, $c) + $code = Get-NetxExitCode $proc + if ($null -eq $code -or $code -eq 0) { + $script:sync[$c.Key] = $c.Enable + Show-NetxToast $c.Text + } else { + Show-NetxToast (T "Setting change failed (exit $code)." "设置修改失败（退出码 $code）。") "Error" + } + } +} + +$miAutostart.add_Click({ + Invoke-NetxTrayDeferred { + Switch-NetxTaskToggle $autostartPs1 "Autostart" ` + (T "NetX will start when you sign in." "已开启：登录 Windows 时自动启动 NetX。") ` + (T "Start with Windows turned off." "已关闭登录自动启动。") + } + }) +function Set-NetxUpdateSource([string]$Source) { + $value = if ($Source -eq "forgejo") { "forgejo" } else { "" } + try { + Write-DotEnvValue -Path $envPath -Values @{ "NETX_UPDATE_SOURCES" = $value } + } catch { + Show-NetxToast ((T "Could not save the update source: " "无法保存更新源：") + $_.Exception.Message) "Error" + return + } + Show-NetxToast $(if ($value) { + (T "Updates now come from git.avelo.top." "更新源已切换为 git.avelo.top。") + } else { + (T "Updates now come from GitHub (git.avelo.top as fallback)." "更新源已切换为 GitHub（git.avelo.top 作为备用）。") + }) +} +$miSourceGithub.add_Click({ Invoke-NetxTrayDeferred { Set-NetxUpdateSource "github" } }) +$miSourceForgejo.add_Click({ Invoke-NetxTrayDeferred { Set-NetxUpdateSource "forgejo" } }) + +$miAutoUpdate.add_Click({ + Invoke-NetxTrayDeferred { + Switch-NetxTaskToggle $updateTaskPs1 "AutoUpdate" ` + (T "Daily automatic updates turned on." "已开启每日自动更新。") ` + (T "Daily automatic updates turned off." "已关闭每日自动更新。") + } + }) + +# --- reconfigure database (console wizard) --- +function Complete-NetxTrayReconfig { + Read-NetxTrayConfig + $script:sync.HealthUrl = "$($script:uiUrl)health" + $script:lastState = "" + $ans = Show-NetxDialog (T "Database configuration saved.`nStart NetX now?" "数据库配置已保存。`n是否立即启动 NetX？") "Information" "YesNo" + if ($ans -eq [System.Windows.Forms.DialogResult]::Yes) { + Invoke-NetxRuntime -Action start -OpenUi + } + Update-NetxTrayState +} + +function Start-NetxReconfigWizard { + $null = Ensure-NetxDataAcl -DataRoot $data -Quiet + try { + $p = Start-NetxChild -File $setupPs1 -WindowStyle Normal + } catch { + Show-NetxDialog ((T "Database setup failed: " "数据库配置失败：") + $_.Exception.Message) "Error" | Out-Null + return + } + Watch-NetxProcess $p { + param($sp) + $code = Get-NetxExitCode $sp + if (Test-NetxSetupCancelled $code) { + Show-NetxToast (T "Database setup cancelled." "已取消数据库配置。") + return + } + if ($null -ne $code -and $code -ne 0) { + Show-NetxDialog ((T "Database setup failed (exit $code).`nLog: " "数据库配置失败（退出码 $code）。`n日志：") + $logDir) "Error" | Out-Null + return + } + Complete-NetxTrayReconfig + } +} + +$miReconfig.add_Click({ + Invoke-NetxTrayDeferred { + $ans = Show-NetxDialog (T ` + "NetX will stop while you reconfigure the database.`n`nChoose built-in or external PostgreSQL in the console window that opens.`nContinue?" ` + "重新配置数据库时会先停止 NetX。`n`n请在随后打开的控制台窗口中选择内置或外置 PostgreSQL。`n是否继续？") "Question" "YesNo" + if ($ans -ne [System.Windows.Forms.DialogResult]::Yes) { return } + if ($script:sync.Healthy -or $script:sync.Runner) { + Invoke-NetxRuntime -Action stop -Then { param($c) Start-NetxReconfigWizard } + } else { + Start-NetxReconfigWizard + } + } + }) + +# --- updates --- +function Restart-NetxTraySelf { + $trayFile = Join-Path $PSScriptRoot "netx_tray.ps1" + try { + Start-NetxPowerShell -File $trayFile -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-QuietStart") ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + } catch {} + Exit-NetxTray +} + +function Start-NetxUpdateApply { + Show-NetxToast (T "Downloading and applying the update… keep the console open." "正在下载并应用更新…请勿关闭控制台窗口。") "Info" 5000 + try { + $ap = Start-NetxChild -File $checkPs1 -ExtraArgs @("-Apply") -WindowStyle Normal + } catch { + Show-NetxToast (T "Update cancelled." "已取消更新。") + return + } + Watch-NetxProcess $ap { + param($p) + $code = Get-NetxExitCode $p + if (Test-NetxSetupCancelled $code) { + Show-NetxToast (T "Update cancelled." "已取消更新。") + return + } + if ($code -eq 0) { + $newVer = Get-NetxVersion -ProgramRoot $prog + Show-NetxDialog (T "Updated to $newVer.`nThe tray will restart." "已更新到 $newVer。`n托盘即将重启。") | Out-Null + Restart-NetxTraySelf + return + } + Show-NetxDialog (T "Update failed (exit $code). Retry Check for updates, or run NetX-Setup manually." "更新失败（退出码 $code）。请重试「检查更新」，或手动运行 Setup 安装包。") "Warning" | Out-Null + } +} + +$miUpdate.add_Click({ + Invoke-NetxTrayDeferred { + Show-NetxToast (T "Checking for updates…" "正在检查更新…") "Info" 3000 + try { + $cp = Start-NetxChild -File $checkPs1 -WindowStyle Normal + } catch { + Show-NetxDialog ((T "Update check failed: " "检查更新失败：") + $_.Exception.Message) "Warning" | Out-Null + return + } + Watch-NetxProcess $cp { + param($p) + $code = Get-NetxExitCode $p + if (Test-NetxSetupCancelled $code) { return } + if ($code -eq 10) { + $ans = Show-NetxDialog (T "A newer NetX is available. Download and apply now?" "发现新版本，是否立即下载并更新？") "Question" "YesNo" + if ($ans -eq [System.Windows.Forms.DialogResult]::Yes) { Start-NetxUpdateApply } + } elseif ($code -eq 0) { + Show-NetxDialog (T "NetX is up to date ($($script:ver))." "已是最新版本（$($script:ver)）。") | Out-Null + } else { + Show-NetxDialog (T "Update check failed (exit $code)." "检查更新失败（退出码 $code）。") "Warning" | Out-Null + } + } + } + }) + +# --- exit --- +function Exit-NetxTray { + try { $menu.Close() } catch {} + $notify.Visible = $false + $form.Close() + [System.Windows.Forms.Application]::Exit() +} +$miExit.add_Click({ Invoke-NetxTrayDeferred { Exit-NetxTray } }) +$miStopExit.add_Click({ + Invoke-NetxTrayDeferred { + if (-not $script:sync.Healthy -and -not $script:sync.Runner) { Exit-NetxTray; return } + Show-NetxToast (T "Stopping NetX…" "正在停止 NetX…") "Info" 3000 + $script:pending = $null + Invoke-NetxRuntime -Action stop -Then { param($c) Exit-NetxTray } + } + }) + +$notify.ContextMenuStrip = $menu +# Single left click opens the menu like modern tray apps; double click opens the UI. +$notify.add_MouseUp({ + param($s, $e) + if ($e.Button -eq [System.Windows.Forms.MouseButtons]::Left) { + $mi = [System.Windows.Forms.NotifyIcon].GetMethod("ShowContextMenu", [System.Reflection.BindingFlags]"Instance,NonPublic") + if ($mi) { $mi.Invoke($notify, $null) } + } + }) +$notify.add_DoubleClick({ Invoke-NetxTrayDeferred { Open-NetxUi } }) + +# ---------------------------------------------------------------- lifecycle +$script:uiTimer = New-Object System.Windows.Forms.Timer +$script:uiTimer.Interval = 400 +$script:uiTimer.add_Tick({ + # Pick up in-place updates of version.json. + if (((Get-Date) - $script:verCheckedAt).TotalSeconds -gt 30) { + $script:verCheckedAt = Get-Date + try { $script:ver = Get-NetxVersion -ProgramRoot $prog } catch {} + } + Update-NetxTrayState + }) +$script:verCheckedAt = Get-Date + +function Start-NetxOnLaunch { + if (-not $StartOnLaunch) { return } + Invoke-NetxRuntime -Action start -OpenUi:(-not $QuietStart) + if (-not $QuietStart) { + Show-NetxToast (T "Starting NetX… the first run may take a minute." "正在启动 NetX…首次运行可能需要一分钟。") + } +} + +$form.add_Shown({ + $script:uiTimer.Start() + if ($AutoUpdate) { + # Apply silently first, then start; watched so the tray stays responsive. + try { + $up = Start-NetxChild -File $checkPs1 -ExtraArgs @("-Apply", "-Quiet", "-AutoOnly") + Watch-NetxProcess $up { param($p) Start-NetxOnLaunch } + } catch { + Start-NetxOnLaunch + } + } else { + if ($CheckUpdateOnLaunch) { + try { Start-NetxChild -File $checkPs1 -ExtraArgs @("-Quiet") | Out-Null } catch {} + } + Start-NetxOnLaunch + } + }) + +$form.add_FormClosing({ + try { $script:uiTimer.Stop() } catch {} + try { $script:watchTimer.Stop() } catch {} + $script:sync.Stop = $true + $notify.Visible = $false + $notify.Dispose() + foreach ($h in $script:iconHandles) { try { [void][NetxNative]::DestroyIcon($h) } catch {} } + }) + +[System.Windows.Forms.Application]::Run($form) +try { $script:probePs.EndInvoke($script:probeHandle) } catch {} +try { $script:probePs.Dispose(); $script:probeRunspace.Close() } catch {} diff --git a/packaging/publish_forgejo_release.ps1 b/packaging/publish_forgejo_release.ps1 new file mode 100644 index 0000000..c51c3e4 --- /dev/null +++ b/packaging/publish_forgejo_release.ps1 @@ -0,0 +1,134 @@ +param( + [string]$Version = "", + [string]$ForgejoBase = "https://git.avelo.top", + [string]$Owner = "hansjone", + [string]$Repo = "netx", + [string]$Token = "", + [string]$ReleaseDir = "" +) + +# Publish Windows Setup.exe to Forgejo/Gitea (mirror sync does NOT copy GitHub Release files). +# Requires a Forgejo token with repo write (Settings → Applications → Generate New Token). +# Default: https://git.avelo.top (public domain). LAN IP only when explicitly reachable. +# +# Example: +# $env:NETX_FORGEJO_TOKEN = "..." # or User env NETX_FORGEJO_TOKEN +# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 +# +# Optional LAN (when 10.0.0.131 is reachable): +# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 -ForgejoBase "http://10.0.0.131:3000" + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$repoRoot = Get-NetxRepoRoot +if (-not $Version) { + $Version = Get-NetxVersion -ProgramRoot $repoRoot +} +$tag = "v$Version" +$relDir = if ($ReleaseDir) { $ReleaseDir } else { Join-Path $PSScriptRoot "release" } +$setup = Join-Path $relDir "NetX-Setup-$Version.exe" +$checksum = "$setup.sha256" + +if (-not $Token) { + $Token = $env:NETX_FORGEJO_TOKEN + if (-not $Token) { $Token = $env:FORGEJO_TOKEN } +} +if (-not $Token) { + throw "forgejo_token_required: set NETX_FORGEJO_TOKEN or pass -Token" +} +if (-not (Test-Path $setup)) { + throw "missing_setup: $setup (run build_release.ps1 + ISCC first)" +} +$hash = (Get-FileHash -LiteralPath $setup -Algorithm SHA256).Hash.ToLowerInvariant() +Set-Content -LiteralPath $checksum -Value $hash -Encoding ascii -NoNewline +$apiBase = "$ForgejoBase/api/v1/repos/$Owner/$Repo" +$headers = @{ + "Authorization" = "token $Token" + "Accept" = "application/json" +} + +function Invoke-ForgejoJson { + param( + [string]$Method, + [string]$Url, + [object]$Body = $null + ) + $params = @{ + Method = $Method + Uri = $Url + Headers = $headers + ContentType = "application/json" + } + if ($null -ne $Body) { + $params["Body"] = ($Body | ConvertTo-Json -Depth 5) + } + return Invoke-RestMethod @params +} + +Write-Host "==> Forgejo publish $tag -> $ForgejoBase/$Owner/$Repo" + +# Ensure git tag exists on Forgejo (mirror usually already has it). +try { + $null = Invoke-ForgejoJson -Method GET -Url "$apiBase/git/refs/tags/$tag" + Write-Host " tag $tag present on Forgejo" +} catch { + Write-Host "[WARN] tag $tag not found on Forgejo yet — run mirror-sync or push tag first" -ForegroundColor Yellow +} + +$existing = $null +try { + $existing = Invoke-ForgejoJson -Method GET -Url "$apiBase/releases/tags/$tag" +} catch { + $existing = $null +} + +$notes = @" +NetX $Version Windows installer (published from GitHub release build). + +- NetX-Setup-$Version.exe +"@ + +if ($existing -and $existing.id) { + Write-Host "==> Release $tag already exists (id=$($existing.id)); deleting old release to re-upload assets" + Invoke-ForgejoJson -Method DELETE -Url "$apiBase/releases/$($existing.id)" | Out-Null +} + +Write-Host "==> Creating release $tag" +$rel = Invoke-ForgejoJson -Method POST -Url "$apiBase/releases" -Body @{ + tag_name = $tag + target = "main" + name = "NetX $Version" + body = $notes + draft = $false + prerelease = $false +} +$relId = $rel.id +if (-not $relId) { throw "forgejo_create_release_failed" } + +function Upload-ForgejoAsset { + param([string]$Path) + if (-not (Test-Path $Path)) { return } + $name = Split-Path -Leaf $Path + Write-Host "==> Uploading $name ..." + $url = "$apiBase/releases/$relId/assets" + $curl = Get-Command curl.exe -ErrorAction SilentlyContinue + if (-not $curl) { throw "curl.exe required for asset upload" } + # --ssl-no-revoke: schannel CRYPT_E_REVOCATION_OFFLINE often fails via proxy/offline CA. + # --noproxy: a desktop proxy in HTTP(S)_PROXY cannot reach a LAN Forgejo (502). + $forgejoHost = ([Uri]$ForgejoBase).Host + & $curl.Source -f -sS --ssl-no-revoke --noproxy $forgejoHost -X POST ` + -H "Authorization: token $Token" ` + -F "attachment=@$Path" ` + $url + if ($LASTEXITCODE -ne 0) { throw "upload_failed: $name" } + Write-Host " OK $name" -ForegroundColor Green +} + +Upload-ForgejoAsset -Path $setup +Upload-ForgejoAsset -Path $checksum + +$releaseUrl = "$ForgejoBase/$Owner/$Repo/releases/tag/$tag" +Write-Host "" +Write-Host "==> Forgejo release ready: $releaseUrl" -ForegroundColor Green +Write-Host " Update API: $ForgejoBase/api/v1/repos/$Owner/$Repo/releases/latest" diff --git a/packaging/publish_release.ps1 b/packaging/publish_release.ps1 index 193c36c..28181a2 100644 --- a/packaging/publish_release.ps1 +++ b/packaging/publish_release.ps1 @@ -1,9 +1,10 @@ -param( +param( [string]$Version = "", [switch]$SkipBuild = $false, [switch]$SkipInstaller = $false, [switch]$SkipGitHub = $false, - [switch]$Draft = $false + [switch]$Draft = $false, + [switch]$Sign = $false ) $ErrorActionPreference = "Stop" @@ -15,41 +16,69 @@ if (-not $Version) { } $tag = "v$Version" $releaseDir = Join-Path $PSScriptRoot "release" -$zip = Join-Path $releaseDir "NetX-$Version-win64.zip" $setup = Join-Path $releaseDir "NetX-Setup-$Version.exe" +$checksum = "$setup.sha256" +$stage = Join-Path $releaseDir "netx-win64" + +function Assert-NativeSuccess([string]$Operation) { + if ($LASTEXITCODE -ne 0) { throw "$Operation failed with exit code $LASTEXITCODE" } +} Write-Host "==> NetX publish $tag" if (-not $SkipBuild) { + if (Test-Path -LiteralPath $setup) { Remove-Item -LiteralPath $setup -Force } & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "build_release.ps1") ` -Version $Version -CreateVenv -} - -if (-not (Test-Path $zip)) { - throw "missing_zip: $zip" + Assert-NativeSuccess 'Build stage' } if (-not $SkipInstaller) { $isccCmd = Get-Command ISCC.exe -ErrorAction SilentlyContinue $isccCandidates = @( $(if ($isccCmd) { $isccCmd.Source }), + "$env:LOCALAPPDATA\Programs\Inno Setup 6\ISCC.exe", "${env:ProgramFiles(x86)}\Inno Setup 6\ISCC.exe", "$env:ProgramFiles\Inno Setup 6\ISCC.exe" ) | Where-Object { $_ -and (Test-Path $_) } $iscc = $isccCandidates | Select-Object -First 1 if (-not $iscc) { - Write-Host "[WARN] Inno Setup (ISCC) not found. Install: winget install JRSoftware.InnoSetup" -ForegroundColor Yellow - Write-Host " Skipping Setup.exe; zip-only release." - } else { - Write-Host "==> Compiling installer: $iscc" - & $iscc "/DMyAppVersion=$Version" (Join-Path $PSScriptRoot "installer\netx.iss") - if (-not (Test-Path $setup)) { - throw "installer_build_failed: expected $setup" - } - Write-Host "==> Setup.exe: $setup" -ForegroundColor Green + throw "innosetup_not_found: install with winget install JRSoftware.InnoSetup" } + Write-Host "==> Compiling installer: $iscc" + if (Test-Path -LiteralPath $setup) { Remove-Item -LiteralPath $setup -Force } + & $iscc "/DMyAppVersion=$Version" (Join-Path $PSScriptRoot "installer\netx.iss") + Assert-NativeSuccess 'Inno Setup compilation' + if (-not (Test-Path -LiteralPath $setup) -or (Get-Item -LiteralPath $setup).Length -lt 1MB) { + throw "installer_build_failed: expected $setup" + } + Write-Host "==> Setup.exe: $setup" -ForegroundColor Green } +if (-not (Test-Path $setup)) { + throw "missing_setup: $setup" +} + +$stageVersionPath = Join-Path $stage 'version.json' +if (-not (Test-Path -LiteralPath $stageVersionPath)) { throw "missing_release_stage: $stageVersionPath" } +$stageVersion = (Get-Content -LiteralPath $stageVersionPath -Raw | ConvertFrom-Json).version +if ([string]$stageVersion -ne $Version) { + throw "release_version_mismatch: requested=$Version stage=$stageVersion" +} +if ((Get-Item -LiteralPath $setup).LastWriteTimeUtc -lt (Get-Item -LiteralPath $stageVersionPath).LastWriteTimeUtc) { + throw "stale_installer: $setup predates the release stage; rebuild or recompile before publishing" +} + +if ($Sign) { + Write-Host "==> Signing release artifacts" + & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "sign_release.ps1") -Files @($setup) + Assert-NativeSuccess 'Code signing' +} + +$hash = (Get-FileHash -LiteralPath $setup -Algorithm SHA256).Hash.ToLowerInvariant() +Set-Content -LiteralPath $checksum -Value $hash -Encoding ascii -NoNewline +Write-Host "==> SHA-256 $hash" + if ($SkipGitHub) { Write-Host "==> Skip GitHub release (-SkipGitHub)" exit 0 @@ -60,30 +89,33 @@ if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { } Set-Location $repo +# gh writes "release not found" to stderr; keep Stop for the rest of the script. +$prevEap = $ErrorActionPreference +$ErrorActionPreference = "Continue" $existing = gh release view $tag 2>$null -if ($LASTEXITCODE -eq 0) { +$viewCode = $LASTEXITCODE +$ErrorActionPreference = $prevEap +if ($viewCode -eq 0) { Write-Host "==> Release $tag exists; uploading assets" - gh release upload $tag $zip --clobber - if (Test-Path $setup) { - gh release upload $tag $setup --clobber - } + gh release upload $tag $setup $checksum --clobber + Assert-NativeSuccess 'GitHub asset upload' } else { $notes = @" -## NetX $Version — first Windows installable release +## NetX $Version ### Downloads -- **NetX-Setup-$Version.exe** — recommended installer (Program Files + ProgramData) -- **NetX-$Version-win64.zip** — portable directory package +- **NetX-Setup-$Version.exe** — Windows installer (Program Files + ProgramData) ### Requirements -- Windows 10/11 x64 -- No separate Python or PostgreSQL install required (bundled in package) +- Windows 10/11 x64 (or Windows Server 2016+) +- No separate Python or PostgreSQL install required (bundled) -### Quick start (installer) +### Quick start 1. Run ``NetX-Setup-$Version.exe`` as administrator. -2. Complete first-time setup (choose **bundled** or **external** PostgreSQL). -3. Start menu → **Start NetX** → browser opens ``http://127.0.0.1:8890/`` -4. Default login: ``admin`` / ``admin123`` (change after first login) +2. **First install:** choose built-in or external PostgreSQL. +3. **Already installed:** Setup detects existing data and updates in place (keeps DB settings). +4. Start menu → **Start NetX** → ``http://127.0.0.1:8890/`` +5. Default login: ``admin`` / ``admin123`` (change after first login) ### Linux Unchanged — use your own PostgreSQL and ``scripts/start_netx.sh``. @@ -92,10 +124,12 @@ See [packaging/README.md](https://github.com/hansjone/netx/blob/main/packaging/R "@ $args = @("release", "create", $tag, "--title", "NetX $Version", "--notes", $notes) if ($Draft) { $args += "--draft" } - $args += $zip - if (Test-Path $setup) { $args += $setup } + $args += $setup + $args += $checksum & gh @args + Assert-NativeSuccess 'GitHub release creation' } Write-Host "==> Published $tag" -ForegroundColor Green gh release view $tag --web 2>$null +Assert-NativeSuccess 'GitHub release verification' diff --git a/packaging/repair_venv.ps1 b/packaging/repair_venv.ps1 new file mode 100644 index 0000000..3af0e60 --- /dev/null +++ b/packaging/repair_venv.ps1 @@ -0,0 +1,26 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "" +) + +# Relink shipped .venv to local python/runtime (must be elevated under Program Files). +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot + +Write-Host "==> Repairing .venv under $prog" +if (Repair-NetxShippedVenv -ProgramRoot $prog) { + $venvPy = Join-Path $prog ".venv\Scripts\python.exe" + if (Test-NetxVenvRunnable -VenvPython $venvPy) { + Write-Host "==> .venv OK -> bundled python/runtime" -ForegroundColor Green + Get-Content (Join-Path $prog ".venv\pyvenv.cfg") + exit 0 + } + Write-Host "[ERR] pyvenv.cfg written but venv still not runnable" -ForegroundColor Red + exit 2 +} +Write-Host "[ERR] repair failed (missing runtime/.venv or access denied)" -ForegroundColor Red +exit 1 diff --git a/packaging/runtime_ctl.ps1 b/packaging/runtime_ctl.ps1 new file mode 100644 index 0000000..a7e4a10 --- /dev/null +++ b/packaging/runtime_ctl.ps1 @@ -0,0 +1,80 @@ +param( + [ValidateSet("start", "stop", "restart")] + [string]$Action = "start", + [string]$ProgramRoot = "", + [string]$DataRoot = "" +) + +# Single start/stop/restart entry for tray + shortcuts. Routes through the +# Windows service / SYSTEM task when one is installed, else the plain scripts. + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot + +function Invoke-NetxChildScript([string]$Name, [string[]]$Extra = @()) { + # Never read the child's stdout: postgres / NetX processes it starts inherit the + # pipe, so `& powershell.exe` with captured output blocks until they exit (forever). + # Start-Process -Wait is no good either (it waits for the whole process tree). + $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot $Name) ` + -Arguments (@("-ProgramRoot", $prog, "-DataRoot", $data) + $Extra) ` + -WorkingDirectory $prog -WindowStyle Hidden -PassThru + $null = $p.Handle # keep a handle so ExitCode is available after exit + $p.WaitForExit() + return $p.ExitCode +} + +function Stop-NetxService { + # Stop-Service waits forever on a service that will not stop; poll with a deadline + # and kill the WinSW process tree as a last resort. + $svc = Get-Service -Name "NetX" -ErrorAction SilentlyContinue + if (-not $svc -or $svc.Status -eq "Stopped") { return } + # Ask service_run.ps1 to exit ourselves rather than relying only on WinSW's stop hook. + Write-NetxServiceStopFlag -DataRoot $data + & sc.exe stop NetX | Out-Null + $deadline = (Get-Date).AddSeconds(120) + while ((Get-Date) -lt $deadline) { + Start-Sleep -Seconds 1 + $svc.Refresh() + if ($svc.Status -eq "Stopped") { return } + } + $wmi = Get-CimInstance Win32_Service -Filter "Name='NetX'" -ErrorAction SilentlyContinue + if ($wmi -and $wmi.ProcessId) { & taskkill.exe /PID $wmi.ProcessId /T /F | Out-Null } + $null = Invoke-NetxChildScript "stop_netx_app.ps1" +} + +function Start-NetxService { + # Don't block on SERVICE_RUNNING: callers wait for /health instead. + $svc = Get-Service -Name "NetX" -ErrorAction SilentlyContinue + if ($svc -and $svc.Status -eq "Stopped") { & sc.exe start NetX | Out-Null } +} + +$runner = Get-NetxManagedRunner +switch ($runner) { + "service" { + if ($Action -ne "start") { Stop-NetxService } + if ($Action -ne "stop") { Start-NetxService } + } + "task" { + if ($Action -ne "start") { + Stop-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" + $null = Invoke-NetxChildScript "stop_netx_app.ps1" + } + if ($Action -ne "stop") { + Start-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" + } + } + default { + if ($Action -ne "start") { + $null = Invoke-NetxChildScript "stop_netx_app.ps1" + } + if ($Action -ne "stop") { + $code = Invoke-NetxChildScript "start_netx_app.ps1" @("-SkipBrowser") + if ($code) { exit $code } + } + } +} +exit 0 diff --git a/packaging/service_lifecycle.ps1 b/packaging/service_lifecycle.ps1 new file mode 100644 index 0000000..4431991 --- /dev/null +++ b/packaging/service_lifecycle.ps1 @@ -0,0 +1,137 @@ +param( + [Parameter(Mandatory = $true)] + [ValidateSet('StopForUpdate', 'ResumeAfterUpdate', 'RemoveForUninstall')] + [string]$Action, + [string]$ProgramRoot = '', + [string]$DataRoot = '' +) + +$ErrorActionPreference = 'Stop' +. "$PSScriptRoot\_common.ps1" +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$runtime = Join-Path $data 'data\runtime' +$null = New-Item -ItemType Directory -Path $runtime -Force +$stateFile = Join-Path $runtime 'service-update-state.json' +$logFile = Join-Path $runtime 'service_lifecycle.log' + +function Write-LifecycleLog([string]$Message) { + Add-Content -LiteralPath $logFile -Value "$(Get-Date -Format o) $Message" -Encoding utf8 +} + +function Get-NetxTask { + try { return Get-ScheduledTask -TaskName 'NetXService' -TaskPath '\NetX\' -ErrorAction Stop } + catch { return $null } +} + +function Stop-NetxRunners([switch]$Remove) { + $state = @{ serviceWasRunning = $false; taskWasRunning = $false } + $svc = Get-Service -Name 'NetX' -ErrorAction SilentlyContinue + if ($svc) { + $state.serviceWasRunning = ($svc.Status -ne 'Stopped') + $state | ConvertTo-Json | Set-Content -LiteralPath $stateFile -Encoding utf8 + if ($svc.Status -ne 'Stopped') { + Write-LifecycleLog 'Stopping Windows service NetX' + Write-NetxServiceStopFlag -DataRoot $data + Stop-Service -Name 'NetX' -Force -ErrorAction Stop + (Get-Service -Name 'NetX').WaitForStatus('Stopped', [TimeSpan]::FromSeconds(60)) + } + } + $task = Get-NetxTask + if ($task) { + $state.taskWasRunning = ([string]$task.State -eq 'Running') + $state | ConvertTo-Json | Set-Content -LiteralPath $stateFile -Encoding utf8 + if ($state.taskWasRunning) { + Write-LifecycleLog 'Stopping scheduled task NetXService' + Stop-ScheduledTask -TaskName 'NetXService' -TaskPath '\NetX\' -ErrorAction Stop + $deadline = (Get-Date).AddSeconds(60) + do { + Start-Sleep -Milliseconds 500 + $task = Get-NetxTask + } while ($task -and [string]$task.State -eq 'Running' -and (Get-Date) -lt $deadline) + if ($task -and [string]$task.State -eq 'Running') { throw 'NetXService scheduled task did not stop within 60 seconds.' } + } + } + if ($Remove) { + if ($svc) { + $winsw = Join-Path $prog 'packaging\winsw\NetX.exe' + if (Test-Path -LiteralPath $winsw) { + Write-LifecycleLog 'Uninstalling WinSW service' + & $winsw uninstall + if ($LASTEXITCODE -ne 0) { throw "WinSW uninstall failed: $LASTEXITCODE" } + } else { + throw "NetX service exists but WinSW executable is missing: $winsw" + } + } + if ($task) { + Unregister-ScheduledTask -TaskName 'NetXService' -TaskPath '\NetX\' -Confirm:$false -ErrorAction Stop + } + } + return $state +} + +try { + switch ($Action) { + 'StopForUpdate' { + $state = Stop-NetxRunners + $state | ConvertTo-Json | Set-Content -LiteralPath $stateFile -Encoding utf8 + $stopScript = Join-Path $prog 'packaging\stop_netx_app.ps1' + if (Test-Path -LiteralPath $stopScript) { + & powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File $stopScript ` + -ProgramRoot $prog -DataRoot $data -KillTray + if ($LASTEXITCODE -ne 0) { throw "NetX application stop failed: $LASTEXITCODE" } + } + } + 'ResumeAfterUpdate' { + if (-not (Test-Path -LiteralPath $stateFile)) { break } + $state = Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json + # Setup may have switched run mode (service -> tray) in between: skip what is gone. + if ($state.serviceWasRunning -and (Get-Service -Name 'NetX' -ErrorAction SilentlyContinue)) { + Write-LifecycleLog 'Restarting Windows service NetX after update' + Start-Service -Name 'NetX' -ErrorAction Stop + } + if ($state.taskWasRunning -and (Get-NetxTask)) { + Write-LifecycleLog 'Restarting scheduled task NetXService after update' + Start-ScheduledTask -TaskName 'NetXService' -TaskPath '\NetX\' -ErrorAction Stop + } + Remove-Item -LiteralPath $stateFile -Force + } + 'RemoveForUninstall' { + $null = Stop-NetxRunners -Remove + # Logon tray + daily update tasks would otherwise outlive the program files. + foreach ($t in @('NetXTray', 'NetXUpdate')) { + Unregister-ScheduledTask -TaskName $t -TaskPath '\NetX\' -Confirm:$false -ErrorAction SilentlyContinue + } + try { + $sched = New-Object -ComObject Schedule.Service + $sched.Connect() + $root = $sched.GetFolder('\') + $nx = $root.GetFolder('\NetX') + if ($nx.GetTasks(1).Count -eq 0 -and $nx.GetFolders(0).Count -eq 0) { $root.DeleteFolder('NetX', 0) } + } catch {} + $stopScript = Join-Path $prog 'packaging\stop_netx_app.ps1' + if (Test-Path -LiteralPath $stopScript) { + & powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File $stopScript ` + -ProgramRoot $prog -DataRoot $data -KillTray + if ($LASTEXITCODE -ne 0) { throw "NetX application stop failed: $LASTEXITCODE" } + } + } + } + Write-LifecycleLog "Action completed: $Action" + exit 0 +} catch { + Write-LifecycleLog "Action failed: $Action; $($_.Exception.Message)" + if ($Action -eq 'StopForUpdate' -and (Test-Path -LiteralPath $stateFile)) { + try { + $prior = Get-Content -LiteralPath $stateFile -Raw | ConvertFrom-Json + if ($prior.serviceWasRunning) { Start-Service -Name 'NetX' -ErrorAction Stop } + if ($prior.taskWasRunning) { Start-ScheduledTask -TaskName 'NetXService' -TaskPath '\NetX\' -ErrorAction Stop } + Remove-Item -LiteralPath $stateFile -Force + Write-LifecycleLog 'Restored service/task after update stop failure' + } catch { + Write-LifecycleLog "Could not restore service/task after update stop failure: $($_.Exception.Message)" + } + } + Write-Error $_ + exit 1 +} diff --git a/packaging/service_run.ps1 b/packaging/service_run.ps1 new file mode 100644 index 0000000..3504485 --- /dev/null +++ b/packaging/service_run.ps1 @@ -0,0 +1,109 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "", + [int]$HealthFailLimit = 6, + [int]$HealthIntervalSec = 10 +) + +# Long-running supervisor for Windows Service / Task Scheduler. +# Starts NetX, probes /health; repeated failures trigger restart (or exit for WinSW). + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\_common.ps1" + +$prog = Get-NetxProgramRoot -Override $ProgramRoot +$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot +$logDir = Join-Path $data "data\runtime" +if (-not (Test-Path $logDir)) { + New-Item -ItemType Directory -Path $logDir -Force | Out-Null +} +$logFile = Join-Path $logDir "service_run.log" +$stopFlag = Join-Path $logDir "service.stop" + +function Write-SvcLog([string]$Msg) { + $line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Msg + Add-Content -Path $logFile -Value $line -Encoding utf8 + Write-Host $line +} + +function Get-BindInfo { + $envPath = Join-Path $data ".env" + $hostBind = "127.0.0.1" + $port = 8890 + if (Test-Path $envPath) { + $map = Read-DotEnv -Path $envPath + if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } + if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } + } + return @{ Host = $hostBind; Port = $port } +} + +Remove-Item -Force $stopFlag -ErrorAction SilentlyContinue +Write-SvcLog "service_run starting program=$prog data=$data" + +$startPs1 = Join-Path $PSScriptRoot "start_netx_app.ps1" +$stopPs1 = Join-Path $PSScriptRoot "stop_netx_app.ps1" +$bind = Get-BindInfo +$failStreak = 0 +$restartCount = 0 + +function Start-NetxChildren { + & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startPs1 ` + -ProgramRoot $prog -DataRoot $data -SkipBrowser -Force + if ($LASTEXITCODE -ne 0) { + throw "start_netx_app_failed exit=$LASTEXITCODE" + } +} + +function Stop-NetxChildren { + & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $stopPs1 ` + -ProgramRoot $prog -DataRoot $data -KeepSupervisor +} + +function Wait-NetxStopFlag([int]$Seconds) { + # Sleep in 1s steps so a service stop (stop flag) is noticed quickly. + for ($i = 0; $i -lt $Seconds; $i++) { + if (Test-Path $stopFlag) { return $true } + Start-Sleep -Seconds 1 + } + return (Test-Path $stopFlag) +} + +try { + Start-NetxChildren + Write-SvcLog "NetX started; health watch host=$($bind.Host) port=$($bind.Port)" + + while ($true) { + if (Test-Path $stopFlag) { + Write-SvcLog "stop flag detected" + break + } + + if (Test-NetxApiHealthy -HostName $bind.Host -Port $bind.Port -TimeoutSec 3) { + $failStreak = 0 + } else { + $failStreak++ + Write-SvcLog "health fail streak=$failStreak/$HealthFailLimit" + if ($failStreak -ge $HealthFailLimit) { + $restartCount++ + Write-SvcLog "restarting NetX (attempt=$restartCount)" + try { Stop-NetxChildren } catch { Write-SvcLog "stop warning: $($_.Exception.Message)" } + Start-Sleep -Seconds 3 + Start-NetxChildren + $failStreak = 0 + # Give API time to come up before counting failures again. + $null = Wait-NetxStopFlag ([Math]::Max(15, $HealthIntervalSec)) + continue + } + } + $null = Wait-NetxStopFlag $HealthIntervalSec + } +} catch { + Write-SvcLog "ERROR: $($_.Exception.Message)" + throw +} finally { + Write-SvcLog "stopping NetX" + try { Stop-NetxChildren } catch { Write-SvcLog "stop warning: $($_.Exception.Message)" } + Remove-Item -Force $stopFlag -ErrorAction SilentlyContinue + Write-SvcLog "service_run exited" +} diff --git a/packaging/setup_first_run.ps1 b/packaging/setup_first_run.ps1 index fa8e0f5..0806bb8 100644 --- a/packaging/setup_first_run.ps1 +++ b/packaging/setup_first_run.ps1 @@ -4,31 +4,35 @@ param( [string]$ProgramRoot = "", [string]$DataRoot = "", [string]$ExternalDatabaseUrl = "", + [string]$ExternalDatabaseUrlFile = "", + [string]$CredentialSecretKey = "", + [string]$CredentialSecretKeyFile = "", [string]$BundledPassword = "", [int]$BundledPort = 15432, - [switch]$NonInteractive = $false + [switch]$NonInteractive = $false, + [switch]$SkipExternalProbe = $false ) $ErrorActionPreference = "Stop" . "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot $envPath = Join-Path $data ".env" +$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') -Write-Host "==> Program root: $prog" -Write-Host "==> Data root: $data" -Write-Host "==> Env file: $envPath" +function T([string]$En, [string]$Zh) { + if ($zh) { return $Zh } else { return $En } +} -if (-not (Test-Path $data)) { - New-Item -ItemType Directory -Path $data -Force | Out-Null -} -foreach ($sub in @("data", "data\auth", "data\runtime", "backups", "pgdata")) { - $p = Join-Path $data $sub - if (-not (Test-Path $p)) { - New-Item -ItemType Directory -Path $p -Force | Out-Null - } -} +try { + +Write-Host ("==> " + (T "Program root:" "程序目录:") + " $prog") +Write-Host ("==> " + (T "Data root:" "数据目录:") + " $data") +Write-Host ("==> " + (T "Env file:" "环境文件:") + " $envPath") + +Ensure-NetxDataDirectories -DataRoot $data $existing = Read-DotEnv -Path $envPath if (-not $DbMode) { @@ -42,10 +46,11 @@ if (-not $DbMode) { } } else { Write-Host "" - Write-Host "Choose database mode:" - Write-Host " 1) bundled — use NetX portable PostgreSQL (default for new installs)" - Write-Host " 2) external — connect to an existing PostgreSQL (Linux / already deployed)" - $choice = Read-Host "Enter 1 or 2" + Write-Host (T "Choose database mode:" "选择数据库模式:") -ForegroundColor Cyan + Write-Host (T " 1) bundled — NetX portable PostgreSQL (offline / default)" " 1) bundled — NetX 内置便携 PostgreSQL（可离线，默认）") + Write-Host (T " 2) external — existing PostgreSQL (you provide connection URL)" " 2) external — 已有外置 PostgreSQL（需填写连接串）") + Write-Host "" + $choice = Read-Host (T "Enter 1 or 2" "请输入 1 或 2") if ($choice -eq "2") { $DbMode = "external" } else { $DbMode = "bundled" } } } @@ -58,28 +63,44 @@ $values = @{} $values["NETX_DB_MODE"] = $DbMode $values["NETX_HOST"] = "127.0.0.1" $values["NETX_PORT"] = "8890" -$values["NETX_UI_DIST_DIR"] = "web/dist" +# Absolute UI path when present; else relative for source trees. +$distAbs = Join-Path $prog "web\dist" +if (Test-Path (Join-Path $distAbs "index.html")) { + $values["NETX_UI_DIST_DIR"] = (ConvertTo-NetxFsPath $distAbs) +} else { + $values["NETX_UI_DIST_DIR"] = "web/dist" +} -# Point runtime data dirs into the data root (absolute). -$values["NETX_AUTH_MCP_TOKEN_FILE"] = ((Join-Path $data "data\auth\mcp_token") -replace '\\', '/') -$values["NETX_SCHEDULER_HEARTBEAT_PATH"] = ((Join-Path $data "data\runtime\scheduler_heartbeat.json") -replace '\\', '/') +# All runtime data under data root (never under Program Files). +$dataEnv = Get-NetxDataEnvMap -DataRoot $data +foreach ($k in $dataEnv.Keys) { $values[$k] = $dataEnv[$k] } + +# Preload credential key from file/CLI only; interactive prompt comes AFTER DB settings +# so users are not left thinking the key alone finished setup. +if ($CredentialSecretKeyFile) { + if (-not (Test-Path -LiteralPath $CredentialSecretKeyFile)) { + throw "credential_secret_key_file_missing: $CredentialSecretKeyFile" + } + $CredentialSecretKey = [IO.File]::ReadAllText($CredentialSecretKeyFile).Trim() +} if ($DbMode -eq "bundled") { $pgsql = Join-Path $prog "postgres\pgsql" if (-not (Test-Path (Join-Path $pgsql "bin\initdb.exe"))) { - # In-repo layout $alt = Join-Path $PSScriptRoot "postgres\pgsql" if (Test-Path (Join-Path $alt "bin\initdb.exe")) { $pgsql = $alt } else { - throw "bundled_postgres_missing: run packaging\download_postgres.ps1 first (expected $pgsql)" + throw (T ` + "bundled_postgres_missing: incomplete package (expected $pgsql). Offline Setup must include postgres; rebuild with build_release.ps1 on a machine that already ran download_postgres.ps1." ` + "缺少内置 PostgreSQL（期望路径 $pgsql）。离线安装包必须自带数据库；请在已运行过 download_postgres.ps1 的机器上重新 build_release。") } } if (-not $BundledPassword) { if ($NonInteractive) { $BundledPassword = -join ((48..57) + (65..90) + (97..122) | Get-Random -Count 24 | ForEach-Object { [char]$_ }) } else { - $sec = Read-Host -Prompt "Password for bundled role 'netx' (empty = auto-generate)" -AsSecureString + $sec = Read-Host -Prompt (T "Password for bundled role 'netx' (empty = auto-generate)" "内置数据库用户 netx 的密码（回车=自动生成）") -AsSecureString $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($sec) try { $BundledPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto($bstr) @@ -88,24 +109,22 @@ if ($DbMode -eq "bundled") { } if (-not $BundledPassword) { $BundledPassword = -join ((48..57) + (65..90) + (97..122) | Get-Random -Count 24 | ForEach-Object { [char]$_ }) - Write-Host "Generated password (saved in .env only)." + Write-Host (T "Generated password (saved in .env only)." "已自动生成密码（仅保存在 .env）。") } } } $pgData = Join-Path $data "pgdata" $values["NETX_BUNDLED_PG_PORT"] = "$BundledPort" - $values["NETX_BUNDLED_PG_DATA_DIR"] = ($pgData -replace '\\', '/') + $values["NETX_BUNDLED_PG_DATA_DIR"] = (ConvertTo-NetxFsPath $pgData) $pwFile = Join-Path $data "pg_netx.pw" Set-Content -Path $pwFile -Value $BundledPassword -Encoding ascii -NoNewline $encPw = [uri]::EscapeDataString($BundledPassword) $values["NETX_DATABASE_URL"] = "postgresql+psycopg://netx:${encPw}@127.0.0.1:${BundledPort}/netx" - # Initialize cluster if needed $initdb = Join-Path $pgsql "bin\initdb.exe" $pgCtl = Join-Path $pgsql "bin\pg_ctl.exe" $psql = Join-Path $pgsql "bin\psql.exe" - $createdb = Join-Path $pgsql "bin\createdb.exe" - $createuser = Join-Path $pgsql "bin\createuser.exe" + $sqlPw = ConvertTo-NetxSqlLiteral $BundledPassword if (-not (Test-Path (Join-Path $pgData "PG_VERSION"))) { Write-Host "==> initdb $pgData" @@ -115,7 +134,6 @@ if ($DbMode -eq "bundled") { if ($LASTEXITCODE -ne 0) { throw "initdb_failed" } } - # Ensure listen / port in postgresql.conf $conf = Join-Path $pgData "postgresql.conf" $hba = Join-Path $pgData "pg_hba.conf" if (Test-Path $conf) { @@ -126,6 +144,8 @@ if ($DbMode -eq "bundled") { $confText = $confText -replace '(?m)^\s*port\s*=\s*\d+', "port = $BundledPort" if ($confText -notmatch "(?m)^\s*listen_addresses\s*=") { $confText += "`nlisten_addresses = '127.0.0.1'`n" + } else { + $confText = $confText -replace "(?m)^\s*listen_addresses\s*=\s*'[^']*'", "listen_addresses = '127.0.0.1'" } Set-Content -Path $conf -Value $confText -Encoding utf8 } @@ -137,49 +157,216 @@ if ($DbMode -eq "bundled") { } } - Write-Host "==> Starting bundled PostgreSQL for bootstrap" - & $pgCtl -D $pgData -l (Join-Path $data "pgdata\pg.log") start - Start-Sleep -Seconds 2 + $status = & $pgCtl -D $pgData status 2>&1 | Out-String + if ($status -notmatch "server is running") { + if (-not (Test-NetxTcpPortFree -HostName "127.0.0.1" -Port $BundledPort)) { + throw "port_in_use: 127.0.0.1:$BundledPort already occupied (bundled Postgres)" + } + Write-Host "==> Starting bundled PostgreSQL for bootstrap" + & $pgCtl -D $pgData -l (Join-Path $pgData "pg.log") start + if ($LASTEXITCODE -ne 0) { throw "pg_start_failed" } + Start-Sleep -Seconds 2 + } $env:PGPASSWORD = $BundledPassword try { - $role = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='netx'" - if ($role -notmatch "1") { - & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` - -c "CREATE ROLE netx LOGIN PASSWORD '$BundledPassword';" + function Invoke-NetxPsql { + param([string]$Sql, [string]$Database = "postgres") + $out = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d $Database -v ON_ERROR_STOP=1 -tAc $Sql 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "psql_failed ($LASTEXITCODE): $Sql`n$out" + } + return (($out | Out-String).Trim()) + } + + $role = Invoke-NetxPsql -Sql "SELECT 1 FROM pg_roles WHERE rolname='netx'" + if ($role -eq "1") { + Invoke-NetxPsql -Sql "ALTER ROLE netx WITH LOGIN PASSWORD '$sqlPw'" | Out-Null } else { - & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` - -c "ALTER ROLE netx WITH LOGIN PASSWORD '$BundledPassword';" + Invoke-NetxPsql -Sql "CREATE ROLE netx LOGIN PASSWORD '$sqlPw'" | Out-Null } - $db = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='netx'" - if ($db -notmatch "1") { - & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` - -c "CREATE DATABASE netx OWNER netx;" + $db = Invoke-NetxPsql -Sql "SELECT 1 FROM pg_database WHERE datname='netx'" + if ($db -ne "1") { + Invoke-NetxPsql -Sql "CREATE DATABASE netx OWNER netx" | Out-Null + } + Invoke-NetxPsql -Sql "GRANT ALL PRIVILEGES ON DATABASE netx TO netx" | Out-Null + # Verify login as app role (catches auth/hba mismatches early). + $prev = $env:PGPASSWORD + $env:PGPASSWORD = $BundledPassword + try { + $ping = & $psql -h 127.0.0.1 -p $BundledPort -U netx -d netx -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>&1 + if ($LASTEXITCODE -ne 0 -or (($ping | Out-String).Trim()) -ne "1") { + throw "netx_role_login_failed: $ping" + } + } finally { + $env:PGPASSWORD = $prev } - & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` - -c "GRANT ALL PRIVILEGES ON DATABASE netx TO netx;" } finally { Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue } Write-Host "==> Bundled Postgres ready on 127.0.0.1:$BundledPort" -ForegroundColor Green } else { + if ($ExternalDatabaseUrlFile) { + if (-not (Test-Path -LiteralPath $ExternalDatabaseUrlFile)) { + throw "external_url_file_missing: $ExternalDatabaseUrlFile" + } + $ExternalDatabaseUrl = [IO.File]::ReadAllText($ExternalDatabaseUrlFile).Trim() + } if (-not $ExternalDatabaseUrl) { - if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { - $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] - } elseif ($NonInteractive) { - throw "external_url_required" + if ($NonInteractive) { + if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { + $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] + } else { + throw "external_url_required" + } } else { - $ExternalDatabaseUrl = Read-Host "NETX_DATABASE_URL (postgresql+psycopg://user:pass@host:5432/netx)" + # Interactive: always ask. Empty = keep existing URL (never silent). + Write-Host "" + Write-Host (T ` + "Enter PostgreSQL URL (database/role must already exist):" ` + "请输入 PostgreSQL 连接串（库和用户需事先建好）:") -ForegroundColor Cyan + Write-Host " postgresql+psycopg://USER:PASSWORD@HOST:5432/DBNAME" + Write-Host (T ` + "Example: postgresql+psycopg://netx:secret@10.0.0.8:5432/netx" ` + "示例: postgresql+psycopg://netx:secret@10.0.0.8:5432/netx") + if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { + Write-Host (T ` + "Current: $($existing['NETX_DATABASE_URL'])" ` + "当前: $($existing['NETX_DATABASE_URL'])") -ForegroundColor DarkGray + Write-Host (T ` + "Press Enter to keep the current URL, or paste a new one." ` + "直接回车 = 保留当前连接串；或粘贴新的连接串。") + } + $entered = (Read-Host "NETX_DATABASE_URL").Trim() + if ($entered) { + $ExternalDatabaseUrl = $entered + } elseif ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { + $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] + Write-Host (T "==> Keeping existing NETX_DATABASE_URL" "==> 保留已有 NETX_DATABASE_URL") -ForegroundColor Green + } } } if (-not $ExternalDatabaseUrl) { throw "external_url_required" } + $ExternalDatabaseUrl = $ExternalDatabaseUrl.Trim() $values["NETX_DATABASE_URL"] = $ExternalDatabaseUrl - Write-Host "==> External Postgres configured (ensure role/db exist; see scripts\init_pg.ps1)" -ForegroundColor Green + Write-Host "==> External Postgres configured" -ForegroundColor Green + + if (-not $SkipExternalProbe) { + # Probe with bundled psql when available (wizard already tested; this covers CLI reconfigure). + $psqlProbe = Join-Path $prog "postgres\pgsql\bin\psql.exe" + $testHelper = Join-Path $PSScriptRoot "installer\test_pg_conn.ps1" + if (-not (Test-Path $testHelper)) { + $testHelper = Join-Path $PSScriptRoot "test_pg_conn.ps1" + } + if ((Test-Path $psqlProbe) -and ($ExternalDatabaseUrl -match '^(?:postgresql(?:\+psycopg)?|postgres)://([^:]+):([^@]*)@([^:/]+):?(\d+)?/([^?\s]+)')) { + $u = [uri]::UnescapeDataString($Matches[1]) + $pw = [uri]::UnescapeDataString($Matches[2]) + $h = $Matches[3] + $po = if ($Matches[4]) { [int]$Matches[4] } else { 5432 } + $dbn = [uri]::UnescapeDataString($Matches[5]) + Write-Host "==> Probing external PostgreSQL ${h}:${po}/${dbn} ..." + if (Test-Path $testHelper) { + $probeErr = Join-Path $env:TEMP ("netx-pg-probe-" + [Guid]::NewGuid().ToString("n") + ".txt") + & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $testHelper ` + -PgHost $h -Port $po -User $u -Password $pw -Database $dbn ` + -PsqlPath $psqlProbe -OutErrFile $probeErr + if ($LASTEXITCODE -ne 0) { + $detail = if (Test-Path $probeErr) { (Get-Content -LiteralPath $probeErr -Raw) } else { "exit $LASTEXITCODE" } + Remove-Item -LiteralPath $probeErr -Force -ErrorAction SilentlyContinue + throw (T "external_db_probe_failed: $detail" "外置数据库连接失败: $detail") + } + Remove-Item -LiteralPath $probeErr -Force -ErrorAction SilentlyContinue + Write-Host "==> External PostgreSQL OK" -ForegroundColor Green + } else { + $env:PGPASSWORD = $pw + try { + $ping = & $psqlProbe -h $h -p $po -U $u -d $dbn -t -A -c "SELECT 1" 2>&1 + if ($LASTEXITCODE -ne 0 -or (($ping | Out-String).Trim()) -notmatch '1') { + throw (T "external_db_probe_failed: $ping" "外置数据库连接失败: $ping") + } + } finally { + Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue + } + } + } else { + Write-Host (T ` + "[WARN] Skipped connection probe (no bundled psql or URL parse failed)." ` + "[WARN] 已跳过连接探测（缺少捆绑 psql 或连接串无法解析）。") -ForegroundColor Yellow + } + } +} + +# Fernet key AFTER database prompts (interactive UX). +# Priority: CLI/file > interactive prompt > existing .env > generate. +if (-not $CredentialSecretKey -and -not $NonInteractive) { + Write-Host "" + Write-Host (T ` + "Optional: paste NETX_CREDENTIAL_SECRET_KEY from an existing install" ` + "可选: 粘贴已有安装的 NETX_CREDENTIAL_SECRET_KEY（便于沿用已加密凭据）") -ForegroundColor Cyan + Write-Host (T ` + "Leave empty to keep existing .env key, or auto-generate if none." ` + "留空则保留已有 .env 中的密钥；若没有则自动生成。") + $CredentialSecretKey = (Read-Host "NETX_CREDENTIAL_SECRET_KEY").Trim() +} +if ($CredentialSecretKey) { + $values["NETX_CREDENTIAL_SECRET_KEY"] = $CredentialSecretKey.Trim() + Write-Host "==> Using provided NETX_CREDENTIAL_SECRET_KEY" -ForegroundColor Green +} elseif ($existing.ContainsKey("NETX_CREDENTIAL_SECRET_KEY") -and $existing["NETX_CREDENTIAL_SECRET_KEY"]) { + $values["NETX_CREDENTIAL_SECRET_KEY"] = $existing["NETX_CREDENTIAL_SECRET_KEY"] + Write-Host "==> Keeping existing NETX_CREDENTIAL_SECRET_KEY from .env" -ForegroundColor Green +} else { + $values["NETX_CREDENTIAL_SECRET_KEY"] = New-NetxFernetKey + Write-Host "==> Generated NETX_CREDENTIAL_SECRET_KEY (saved in .env)" -ForegroundColor Green } Write-DotEnvValue -Path $envPath -Values $values Write-Host "" Write-Host "Wrote $envPath" -ForegroundColor Green -Write-Host "Next: .\packaging\start_netx_app.ps1" + +# Official Setup ships python/runtime + .venv (build_release -CreateVenv). +$venvPy = Join-Path $prog ".venv\Scripts\python.exe" +try { + $null = Ensure-NetxVenv -ProgramRoot $prog + Write-Host "==> Bundled Python venv OK: $venvPy" -ForegroundColor Green +} catch { + if (Test-Path $venvPy) { + throw + } + Write-Host "==> Bundled .venv missing — creating one (Setup should normally ship it)." -ForegroundColor Yellow + Write-Host "[WARN] $($_.Exception.Message)" -ForegroundColor Yellow + Write-Host " Install a Setup built with: packaging\build_release.ps1 -CreateVenv" -ForegroundColor Yellow +} + +Write-Host "" +Write-Host (T "Setup complete." "首次配置完成。") -ForegroundColor Green +if (-not $NonInteractive) { + Write-Host (T ` + "Next: Start Menu → NetX Tray (or press Y below)." ` + "下一步: 开始菜单 → NetX 托盘 （或在下方按 Y 立即启动）。") + $ans = Read-Host (T "Start NetX tray now? [Y/n]" "现在启动 NetX 托盘？[Y/n]") + if ($ans -notmatch '^[Nn]') { + try { + Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` + -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-StartOnLaunch") ` + -WorkingDirectory $prog -WindowStyle Hidden | Out-Null + Write-Host (T "Tray started." "托盘已启动。") -ForegroundColor Green + } catch { + Write-Host "[WARN] $($_.Exception.Message)" -ForegroundColor Yellow + } + } +} else { + Write-Host "Next: .\packaging\start_netx_app.ps1 or NetX-Tray.cmd" +} + +} catch { + Write-Host "" + Write-Host ("[ERR] " + $_.Exception.Message) -ForegroundColor Red + if (-not $NonInteractive) { + try { + [void](Read-Host (T "Press Enter to close" "按回车关闭窗口")) + } catch {} + } + exit 1 +} diff --git a/packaging/sign_release.ps1 b/packaging/sign_release.ps1 new file mode 100644 index 0000000..a859447 --- /dev/null +++ b/packaging/sign_release.ps1 @@ -0,0 +1,78 @@ +param( + [Parameter(Mandatory = $true)] + [string[]]$Files, + [string]$Thumbprint = "", + [string]$PfxPath = "", + [string]$PfxPassword = "", + [string]$TimestampUrl = "http://timestamp.digicert.com", + [string]$Description = "NetX" +) + +# Sign release artifacts with signtool (Authenticode). +# Requires Windows SDK / signtool.exe and a code-signing certificate. +# +# Examples: +# .\sign_release.ps1 -Files .\packaging\release\NetX-Setup-0.4.0.exe -Thumbprint ABCDEF... +# .\sign_release.ps1 -Files .\packaging\release\*.exe -PfxPath .\certs\netx.pfx -PfxPassword *** + +$ErrorActionPreference = "Stop" + +function Find-SignTool { + $cmd = Get-Command signtool.exe -ErrorAction SilentlyContinue + if ($cmd) { return $cmd.Source } + $roots = @( + "${env:ProgramFiles(x86)}\Windows Kits\10\bin", + "${env:ProgramFiles}\Windows Kits\10\bin" + ) + foreach ($root in $roots) { + if (-not (Test-Path $root)) { continue } + $hit = Get-ChildItem -Path $root -Recurse -Filter signtool.exe -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -match '\\x64\\signtool\.exe$' } | + Select-Object -First 1 + if ($hit) { return $hit.FullName } + } + return $null +} + +$signtool = Find-SignTool +if (-not $signtool) { + throw "signtool_not_found: install Windows SDK or add signtool.exe to PATH" +} + +if (-not $Thumbprint -and -not $PfxPath) { + if ($env:NETX_SIGN_THUMBPRINT) { $Thumbprint = $env:NETX_SIGN_THUMBPRINT } + if ($env:NETX_SIGN_PFX) { $PfxPath = $env:NETX_SIGN_PFX } + if ($env:NETX_SIGN_PFX_PASSWORD) { $PfxPassword = $env:NETX_SIGN_PFX_PASSWORD } +} +if (-not $Thumbprint -and -not $PfxPath) { + throw "provide -Thumbprint or -PfxPath (or NETX_SIGN_THUMBPRINT / NETX_SIGN_PFX)" +} + +$resolved = @() +foreach ($pattern in $Files) { + $resolved += @(Resolve-Path -Path $pattern -ErrorAction Stop) +} +if ($resolved.Count -eq 0) { throw "no_files_to_sign" } + +foreach ($f in $resolved) { + $path = $f.Path + Write-Host "==> Signing $path" + $args = @( + "sign", "/fd", "SHA256", "/td", "SHA256", "/tr", $TimestampUrl, + "/d", $Description + ) + if ($PfxPath) { + $args += @("/f", $PfxPath) + if ($PfxPassword) { $args += @("/p", $PfxPassword) } + } else { + $args += @("/sha1", $Thumbprint) + } + $args += $path + & $signtool @args + if ($LASTEXITCODE -ne 0) { throw "sign_failed: $path" } + & $signtool verify /pa $path + if ($LASTEXITCODE -ne 0) { throw "verify_failed: $path" } + Write-Host " OK" -ForegroundColor Green +} + +Write-Host "==> Done. Without a trusted CA certificate, Windows SmartScreen may still warn." diff --git a/packaging/start_netx_app.ps1 b/packaging/start_netx_app.ps1 index 3fe2313..d25a0d4 100644 --- a/packaging/start_netx_app.ps1 +++ b/packaging/start_netx_app.ps1 @@ -2,11 +2,13 @@ param( [string]$ProgramRoot = "", [string]$DataRoot = "", [switch]$SkipBrowser = $false, - [switch]$InlineSchedulers = $false + [switch]$InlineSchedulers = $false, + [switch]$Force = $false ) $ErrorActionPreference = "Stop" . "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot @@ -21,25 +23,40 @@ if (-not (Test-Path (Join-Path $prog "netx_api"))) { throw "netx_api not found under program root: $prog" } +Ensure-NetxDataDirectories -DataRoot $data + $map = Import-NetxEnvFile -Path $envPath +# Force data-root paths even if an older .env missed them. +$dataEnv = Get-NetxDataEnvMap -DataRoot $data +foreach ($k in $dataEnv.Keys) { + Set-Item -Path "Env:$k" -Value $dataEnv[$k] +} + Set-Location $prog $env:PYTHONPATH = $prog -# Keep runtime artifacts in the data root (not under Program Files). -$env:NETX_AUTH_MCP_TOKEN_FILE = Join-Path $data "data\auth\mcp_token" -$env:NETX_SCHEDULER_HEARTBEAT_PATH = Join-Path $data "data\runtime\scheduler_heartbeat.json" -$env:NETX_AUTH_SECRET_FILE = Join-Path $data "data\auth\jwt_secret" - $dist = Join-Path $prog "web\dist" if (Test-Path (Join-Path $dist "index.html")) { $env:NETX_UI_DIST_DIR = $dist } $mode = Get-DbMode -EnvMap $map +$hostBind = if ($env:NETX_HOST) { $env:NETX_HOST } else { "127.0.0.1" } +$port = if ($env:NETX_PORT) { [int]$env:NETX_PORT } else { 8890 } +$clientHost = Get-NetxClientHost -BindHost $hostBind + Write-Host "==> Program: $prog" Write-Host "==> Data: $data" Write-Host "==> DB mode: $mode" +if (-not $Force -and (Test-NetxApiHealthy -HostName $clientHost -Port $port)) { + Write-Host "==> NetX already healthy at http://${clientHost}:${port}/ — skip start (use -Force to restart)" -ForegroundColor Green + if (-not $SkipBrowser) { + Open-NetxUrl -Url "http://${clientHost}:${port}/" + } + exit 0 +} + function Get-PgsqlBin { foreach ($b in @( (Join-Path $prog "postgres\pgsql\bin"), @@ -54,13 +71,20 @@ if ($mode -eq "bundled") { $pgBin = Get-PgsqlBin if (-not $pgBin) { throw "bundled_postgres_missing" } $pgData = if ($map["NETX_BUNDLED_PG_DATA_DIR"]) { - $map["NETX_BUNDLED_PG_DATA_DIR"] + $map["NETX_BUNDLED_PG_DATA_DIR"] -replace '/', '\' } else { Join-Path $data "pgdata" } + $pgPort = 15432 + if ($map["NETX_BUNDLED_PG_PORT"]) { + try { $pgPort = [int]$map["NETX_BUNDLED_PG_PORT"] } catch {} + } $pgCtl = Join-Path $pgBin "pg_ctl.exe" $status = & $pgCtl -D $pgData status 2>&1 | Out-String if ($status -notmatch "server is running") { + if (-not (Test-NetxTcpPortFree -HostName "127.0.0.1" -Port $pgPort)) { + throw "port_in_use: 127.0.0.1:$pgPort (bundled Postgres)" + } Write-Host "==> Starting bundled PostgreSQL" if (-not (Test-Path $pgData)) { New-Item -ItemType Directory -Path $pgData -Force | Out-Null @@ -74,39 +98,49 @@ if ($mode -eq "bundled") { } } -# Ensure venv exists for start_netx.ps1 -$venvPy = Join-Path $prog ".venv\Scripts\python.exe" -if (-not (Test-Path $venvPy)) { - Write-Host "==> Creating .venv (one-time)" - $pyCmd = Get-Command python -ErrorAction SilentlyContinue - if (-not $pyCmd) { throw "python_not_found: install Python 3.11+ and re-run" } - & $pyCmd.Source -m venv (Join-Path $prog ".venv") - & $venvPy -m pip install --upgrade pip - & $venvPy -m pip install -r (Join-Path $prog "requirements.txt") - if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } +try { + $null = Ensure-NetxVenv -ProgramRoot $prog +} catch { + Write-Host "[ERR] $($_.Exception.Message)" -ForegroundColor Red + Write-Host " Tip: run Start Menu → NetX → First-time setup once as Administrator," -ForegroundColor Yellow + Write-Host " or install a Setup built with packaging\\build_release.ps1 -CreateVenv." -ForegroundColor Yellow + exit 1 } -$hostBind = if ($env:NETX_HOST) { $env:NETX_HOST } else { "127.0.0.1" } -$port = if ($env:NETX_PORT) { [int]$env:NETX_PORT } else { 8890 } +if (-not (Test-NetxTcpPortFree -HostName $clientHost -Port $port)) { + if (Test-NetxApiHealthy -HostName $clientHost -Port $port) { + Write-Host "==> Port $port already serves NetX — skip start" -ForegroundColor Green + exit 0 + } + throw "port_in_use: ${hostBind}:${port} occupied by non-NetX process" +} $startScript = Join-Path $prog "scripts\start_netx.ps1" if (-not (Test-Path $startScript)) { throw "start_netx.ps1 not found at $startScript" } -$psArgs = @( - "-ExecutionPolicy", "Bypass", "-File", $startScript, - "-SkipInstall", "-Background", - "-BindHost", $hostBind, "-Port", "$port" -) -if ($InlineSchedulers) { $psArgs += "-InlineSchedulers" } - Write-Host "==> Starting NetX (API + workers; UI via API on :$port)" -& powershell @psArgs -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +# Run in-process with -Background so this console exits after /health (nested +# Start-Process -Wait on a Hidden child often never returns under UAC update). +$startArgs = @{ + SkipInstall = $true + Background = $true + BindHost = $hostBind + Port = $port +} +if ($InlineSchedulers) { $startArgs["InlineSchedulers"] = $true } +& $startScript @startArgs +if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -$url = "http://${hostBind}:${port}/" +$url = "http://${clientHost}:${port}/" +if (-not (Test-NetxApiHealthy -HostName $clientHost -Port $port -TimeoutSec 2)) { + if (-not (Wait-NetxApiHealthy -HostName $clientHost -Port $port -TimeoutSec 30)) { + Write-Host "[ERR] NetX started but /health not ready: $url" -ForegroundColor Red + exit 1 + } +} Write-Host "==> Open: $url" -ForegroundColor Green if (-not $SkipBrowser) { - try { Start-Process $url } catch {} + Open-NetxUrl -Url $url } diff --git a/packaging/stop_netx_app.ps1 b/packaging/stop_netx_app.ps1 index 174c631..5f091cb 100644 --- a/packaging/stop_netx_app.ps1 +++ b/packaging/stop_netx_app.ps1 @@ -1,11 +1,26 @@ param( [string]$ProgramRoot = "", [string]$DataRoot = "", - [switch]$KeepPostgres = $false + [switch]$KeepPostgres = $false, + [switch]$KillTray = $false, + # Used by service_run.ps1 when it restarts NetX itself: do not ask it to exit. + [switch]$KeepSupervisor = $false ) $ErrorActionPreference = "Continue" +# Timestamped trace (service stop path is hard to observe otherwise). +$script:stopTrace = $null +function Write-StopTrace([string]$Msg) { + if (-not $script:stopTrace) { return } + try { Add-Content -Path $script:stopTrace -Value ("{0} [{1}] {2}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss.fff"), $PID, $Msg) -Encoding utf8 } catch {} +} +if ($DataRoot) { + $traceDir = Join-Path $DataRoot "data\runtime\service" + if (Test-Path $traceDir) { $script:stopTrace = Join-Path $traceDir "stop_netx_app.log" } +} +Write-StopTrace "begin user=$([Environment]::UserName) keepSupervisor=$KeepSupervisor" . "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot @@ -15,17 +30,33 @@ if (Test-Path $envPath) { $map = Read-DotEnv -Path $envPath } +# Tell the service supervisor (service_run.ps1) to exit first. WinSW runs this script as +# and then waits for the supervisor to exit; without the flag it kept +# looping, restarted NetX, and the service never stopped. +if (-not $KeepSupervisor) { + Write-NetxServiceStopFlag -DataRoot $data + Write-StopTrace "stop flag written" +} + $stopScript = Join-Path $prog "scripts\stop_netx.ps1" if (-not (Test-Path $stopScript)) { $stopScript = Join-Path (Get-NetxRepoRoot) "scripts\stop_netx.ps1" } if (Test-Path $stopScript) { Write-Host "==> Stopping NetX processes" - & powershell -ExecutionPolicy Bypass -File $stopScript -Force + Write-StopTrace "stop_netx.ps1 begin" + Start-NetxPowerShell -File $stopScript -Arguments @("-Force") ` + -WorkingDirectory $prog -WindowStyle Hidden -Wait | Out-Null + Write-StopTrace "stop_netx.ps1 end" } else { Write-Host "[WARN] stop_netx.ps1 not found" } +if ($KillTray) { + Write-Host "==> Stopping NetX tray icons" + Stop-NetxTrayProcesses -ProgramRoot $prog +} + $mode = Get-DbMode -EnvMap $map if ($mode -eq "bundled" -and -not $KeepPostgres) { $pgBinCandidates = @( @@ -43,4 +74,5 @@ if ($mode -eq "bundled" -and -not $KeepPostgres) { } } +Write-StopTrace "done" Write-Host "==> Stop done" diff --git a/packaging/uninstall_delete_data.ps1 b/packaging/uninstall_delete_data.ps1 new file mode 100644 index 0000000..707d0cd --- /dev/null +++ b/packaging/uninstall_delete_data.ps1 @@ -0,0 +1,53 @@ +param( + [string]$DataRoot = "" +) + +# Post-uninstall optional data wipe with retries (handles briefly locked runtime logs). + +$ErrorActionPreference = "Continue" +if (-not $DataRoot) { + $DataRoot = Join-Path ([Environment]::GetFolderPath("CommonApplicationData")) "NetX" +} + +if (-not (Test-Path -LiteralPath $DataRoot)) { + Write-Host "==> Data root already gone: $DataRoot" + exit 0 +} + +Write-Host "==> Deleting data root: $DataRoot" + +# Kill anything still holding files under data root. +Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { + $_.CommandLine -and ($_.CommandLine.IndexOf($DataRoot, [StringComparison]::OrdinalIgnoreCase) -ge 0) +} | ForEach-Object { + try { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } catch {} + try { & taskkill.exe /F /PID $_.ProcessId 2>$null | Out-Null } catch {} +} + +$ok = $false +for ($i = 1; $i -le 5; $i++) { + try { + cmd /c "rmdir /s /q `"$DataRoot`"" + } catch {} + if (-not (Test-Path -LiteralPath $DataRoot)) { + $ok = $true + break + } + Start-Sleep -Seconds 1 +} + +if (-not $ok -and (Test-Path -LiteralPath $DataRoot)) { + try { + takeown /F $DataRoot /R /D Y | Out-Null + icacls $DataRoot /grant Administrators:F /T | Out-Null + cmd /c "rmdir /s /q `"$DataRoot`"" + } catch {} +} + +if (Test-Path -LiteralPath $DataRoot) { + Write-Host "[WARN] Could not fully delete $DataRoot" + exit 1 +} + +Write-Host "==> Data root deleted" +exit 0 diff --git a/packaging/uninstall_prepare.ps1 b/packaging/uninstall_prepare.ps1 new file mode 100644 index 0000000..05e581b --- /dev/null +++ b/packaging/uninstall_prepare.ps1 @@ -0,0 +1,71 @@ +param( + [string]$ProgramRoot = "", + [string]$DataRoot = "" +) + +# Fast, non-blocking uninstall prep for Inno [UninstallRun]. +# Must return within a few seconds and never kill unins000.exe / _unins.tmp. + +$ErrorActionPreference = "Continue" + +$prog = if ($ProgramRoot) { $ProgramRoot } else { "C:\Program Files\NetX" } +$data = if ($DataRoot) { $DataRoot } else { Join-Path $env:ProgramData "NetX" } +$prog = ($prog -replace '/', '\').TrimEnd('\') +$data = ($data -replace '/', '\').TrimEnd('\') + +function Test-Protected([string]$Name, [string]$Cmd) { + if ($Name -match '^(unins\d*|_unins\.tmp|setup)\.exe$') { return $true } + if ($Cmd -match 'unins\d*\.exe|_unins\.tmp|uninstall_prepare\.ps1|uninstall_delete_data\.ps1') { return $true } + return $false +} + +function Stop-Pid([int]$Id, [string]$Label) { + if ($Id -le 4 -or $Id -eq $PID) { return } + Write-Host "stop $Id $Label" + try { Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue } catch {} +} + +Write-Host "==> uninstall_prepare fast-stop prog=$prog" + +$patterns = @( + 'netx_tray\.ps1', + 'netx_api\.(main|worker|biz_state_worker)', + 'start_netx_app\.ps1', + 'stop_netx_app\.ps1', + 'launch_shortcut\.ps1', + [regex]::Escape((Join-Path $prog '.venv\Scripts\python.exe')), + [regex]::Escape((Join-Path $prog 'postgres\pgsql\bin')) +) + +Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | ForEach-Object { + $cl = [string]$_.CommandLine + $name = [string]$_.Name + if (-not $cl) { return } + if (Test-Protected -Name $name -Cmd $cl) { return } + foreach ($pat in $patterns) { + if ($cl -match $pat) { + Stop-Pid -Id ([int]$_.ProcessId) -Label $name + break + } + } +} + +# Best-effort postgres stop (no hang: immediate + ignore) +$pgCtl = Join-Path $prog "postgres\pgsql\bin\pg_ctl.exe" +$pgData = Join-Path $data "pgdata" +if ((Test-Path $pgCtl) -and (Test-Path $pgData)) { + try { + $p = Start-Process -FilePath $pgCtl -ArgumentList @('-D', $pgData, 'stop', '-m', 'immediate') ` + -WindowStyle Hidden -PassThru + if (-not $p.WaitForExit(5000)) { + try { Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue } catch {} + } + } catch {} +} + +try { + Remove-ItemProperty -Path "HKCU:\Software\Microsoft\Windows\CurrentVersion\Run" -Name "NetX" -ErrorAction SilentlyContinue +} catch {} + +Write-Host "==> uninstall_prepare done" +exit 0 diff --git a/packaging/update_netx.ps1 b/packaging/update_netx.ps1 index 1ddd0ec..02c128b 100644 --- a/packaging/update_netx.ps1 +++ b/packaging/update_netx.ps1 @@ -9,6 +9,7 @@ param( $ErrorActionPreference = "Stop" . "$PSScriptRoot\_common.ps1" +Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot @@ -22,10 +23,14 @@ New-Item -ItemType Directory -Path $work -Force | Out-Null try { Write-Host "==> Extracting update package" + # Preferred end-user path is NetX-Setup-*.exe (check_update / offline installer). + # This script remains for legacy/dev .zip packages (build_release.ps1 -CreateZip). if ($PackagePath.ToLowerInvariant().EndsWith(".zip")) { Expand-Archive -Path $PackagePath -DestinationPath $work -Force + } elseif ($PackagePath.ToLowerInvariant().EndsWith(".exe")) { + throw "unsupported_package: run NetX-Setup-*.exe (upgrade keeps DB), or use check_update.ps1 -Apply" } else { - throw "unsupported_package: use a .zip built by build_release.ps1" + throw "unsupported_package: use a .zip from build_release.ps1 -CreateZip, or NetX-Setup-*.exe" } $src = $work @@ -54,7 +59,35 @@ try { if (Test-Path $envFile) { Copy-Item $envFile (Join-Path $bak ".env") } - Write-Host "==> Backup marker: $bak (data/pgdata left in place; optional pg_dump not run)" + # Best-effort logical backup when psql is available (bundled or PATH). + $map = Read-DotEnv -Path $envFile + $pgDump = $null + foreach ($c in @( + (Join-Path $prog "postgres\pgsql\bin\pg_dump.exe"), + (Join-Path $PSScriptRoot "postgres\pgsql\bin\pg_dump.exe") + )) { + if (Test-Path $c) { $pgDump = $c; break } + } + if (-not $pgDump) { + $cmd = Get-Command pg_dump -ErrorAction SilentlyContinue + if ($cmd) { $pgDump = $cmd.Source } + } + if ($pgDump -and $map["NETX_DATABASE_URL"] -match 'postgresql\+?[^:]*://([^:]+):([^@]+)@([^:/]+):?(\d+)?/([^?\s]+)') { + $u = $Matches[1]; $p = [uri]::UnescapeDataString($Matches[2]); $h = $Matches[3] + $pt = if ($Matches[4]) { $Matches[4] } else { "5432" } + $dbn = $Matches[5] + $dumpOut = Join-Path $bak "netx.dump" + Write-Host "==> pg_dump -> $dumpOut" + $env:PGPASSWORD = $p + try { + & $pgDump -h $h -p $pt -U $u -d $dbn -Fc -f $dumpOut + } catch { + Write-Host "[WARN] pg_dump failed: $($_.Exception.Message)" -ForegroundColor Yellow + } finally { + Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue + } + } + Write-Host "==> Backup: $bak" } Write-Host "==> Stopping services" @@ -62,7 +95,8 @@ try { -ProgramRoot $prog -DataRoot $data # Replace program layers only — never wipe data root. - $replaceDirs = @("netx_api", "alembic", "web", "packaging", "scripts", "postgres", "packages") + # Include python/ (portable runtime) — required with shipped .venv since 0.4.6. + $replaceDirs = @("netx_api", "alembic", "web", "packaging", "scripts", "postgres", "packages", "python") foreach ($name in $replaceDirs) { $from = Join-Path $src $name $to = Join-Path $prog $name @@ -79,7 +113,7 @@ try { Copy-Item -Path $from -Destination (Join-Path $prog $f) -Force } } - # Optional runtime / .venv shipped in package + # Legacy top-level runtime/ (older packages) + shipped .venv if (Test-Path (Join-Path $src "runtime")) { $rt = Join-Path $prog "runtime" if (Test-Path $rt) { Remove-Item -Recurse -Force $rt } @@ -92,11 +126,21 @@ try { Copy-Item -Path (Join-Path $src ".venv") -Destination $venvTo -Recurse -Force } + # Builder-path pyvenv.cfg → local python/runtime (same as first install). + if (Get-Command Repair-NetxShippedVenv -ErrorAction SilentlyContinue) { + if (Repair-NetxShippedVenv -ProgramRoot $prog) { + Write-Host "==> Relinked .venv to bundled python/runtime" -ForegroundColor Green + } + } + Write-Host "==> Update files applied" -ForegroundColor Green if (-not $NoStart) { - & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "start_netx_app.ps1") ` + # Same process (no nested powershell) so the update console can exit cleanly. + & (Join-Path $PSScriptRoot "start_netx_app.ps1") ` -ProgramRoot $prog -DataRoot $data -SkipBrowser + if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } + Write-Host "==> Update complete. You can close this window." -ForegroundColor Green } finally { if (Test-Path $work) { Remove-Item -Recurse -Force $work -ErrorAction SilentlyContinue diff --git a/pyproject.toml b/pyproject.toml index a2836f0..1dd9858 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netx-ops" -version = "0.3.0" +version = "0.4.18" description = "netx operations tool: alarm-centric workflows with REST API and stdio MCP" readme = "README.md" requires-python = ">=3.11" @@ -16,7 +16,6 @@ dependencies = [ "httpx>=0.27.0", "sqlalchemy>=2.0.0", "psycopg[binary]>=3.2.0", - "pandas>=2.0.0", "openpyxl>=3.1.0", "PyYAML>=6.0.0", "pydantic>=2.8.0", @@ -43,3 +42,4 @@ netx-topology-mcp = "netx_topology_mcp.server:main" [tool.setuptools.packages.find] where = ["."] include = ["netx_api*"] + diff --git a/requirements.txt b/requirements.txt index d870d81..8bd50ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ websockets>=12.0 httpx>=0.27.0 sqlalchemy>=2.0.0 psycopg[binary]>=3.2.0 -pandas>=2.0.0 openpyxl>=3.1.0 PyYAML>=6.0.0 pydantic>=2.8.0 diff --git a/scripts/bench_webcrt.py b/scripts/bench_webcrt.py new file mode 100644 index 0000000..06f7c32 --- /dev/null +++ b/scripts/bench_webcrt.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""WebCRT latency / throughput / audit benchmark against a live netx + lab device. + + python scripts/bench_webcrt.py --ne-id [--prompt-re "|\\[~?r1\\]"] [--audit] + +Measures (client-side, real WebSocket): + * connect_ms create session -> first prompt visible + * echo latency single keystroke -> echoed byte (p50/p95/max) + * burst echo latency 20 keys sent 3ms apart -> last echo byte + * command RTT Enter -> next prompt for a short command + * bulk throughput long output command, bytes / seconds +With --audit it also types a set of tricky command lines (backspace, tab, history +recall, bare Enter) and prints what landed in audit.jsonl for the session. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import statistics +import sys +import time + +import httpx +import websockets + +BASE = os.environ.get("NETX_BASE", "http://127.0.0.1:8890").rstrip("/") +USER = os.environ.get("NETX_USER", "admin") +PWD = os.environ.get("NETX_PASSWORD", "Admin@1234") + + +def pct(vals: list[float], p: float) -> float: + if not vals: + return float("nan") + s = sorted(vals) + return s[min(len(s) - 1, int(len(s) * p))] + + +async def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--ne-id", default="") + ap.add_argument("--quick", default="", help="ip:port:user:pass -> quick-connect instead of --ne-id") + ap.add_argument("--prompt-re", default=r"(?:<[^>\r\n]+>|\[[^\]\r\n]+\]|[\w.\-]+(?:\([^)]+\))*[#>])\s*$") + ap.add_argument("--bulk-cmd", default="display current-configuration") + ap.add_argument("--probe-cmd", default="display clock") + ap.add_argument("--audit", action="store_true") + ap.add_argument("--echo-only", action="store_true", help="only connect + echo latency") + ap.add_argument("--audit-cmds", default="") + ap.add_argument("--audit-file", default=os.path.join("data", "webcrt", "audit.jsonl")) + args = ap.parse_args() + prompt_re = re.compile(args.prompt_re) + + c = httpx.Client(base_url=BASE, timeout=60) + tok = c.post("/v1/auth/login", json={"username": USER, "password": PWD}).json()["access_token"] + h = {"Authorization": f"Bearer {tok}"} + t_create = time.perf_counter() + if args.quick: + ip, port, user, pw = args.quick.split(":", 3) + r = c.post("/v1/webcrt/sessions/quick-connect", headers=h, json={ + "name": "bench", "ip_address": ip, "port": int(port), "protocol": "ssh", "username": user, + "password": pw, "save_password": False, "cols": 160, "rows": 50}) + else: + r = c.post("/v1/webcrt/sessions", headers=h, json={"ne_id": args.ne_id, "cols": 160, "rows": 50}) + r.raise_for_status() + sid = r.json()["session_id"] + ticket = c.post("/v1/webcrt/ws-ticket", headers=h).json()["ticket"] + url = f"{BASE.replace('http', 'ws', 1)}/v1/webcrt/sessions/{sid}/ws?ws_ticket={ticket}" + print(f"session={sid}") + + async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws: + buf = "" + ev = asyncio.Event() + last_rx = 0.0 + rx_bytes = 0 + + async def reader() -> None: + nonlocal buf, last_rx, rx_bytes + dec = __import__("codecs").getincrementaldecoder("utf-8")(errors="replace") + async for raw in ws: + if isinstance(raw, bytes): + buf += dec.decode(raw) + rx_bytes += len(raw) + last_rx = time.perf_counter() + ev.set() + else: + m = json.loads(raw) + if m.get("type") == "status" and m.get("state") in ("error", "closed", "warning"): + print("STATUS", m) + if m.get("type") == "stdout": + buf += m.get("data", "") + last_rx = time.perf_counter() + ev.set() + + rt = asyncio.create_task(reader()) + + def at_prompt() -> bool: + tail = re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", buf).replace("\r", "\n").rstrip(" ").split("\n")[-1] + return bool(prompt_re.search(tail)) and len(tail) < 80 + + async def wait_prompt(timeout: float = 60.0, quiet: float = 0.05) -> bool: + end = time.perf_counter() + timeout + while time.perf_counter() < end: + if at_prompt() and time.perf_counter() - last_rx >= quiet: + return True + await asyncio.sleep(0.005) + return False + + async def send(data: str, audit_line: str | None = None) -> None: + p = {"type": "stdin", "data": data} + if audit_line: + p["audit_line"] = audit_line + await ws.send(json.dumps(p)) + + ok = await wait_prompt(90, 0.3) + connect_ms = (time.perf_counter() - t_create) * 1000 + if not ok: + await send("\r") + ok = await wait_prompt(15, 0.3) + print(f"connect_ms={connect_ms:.0f} prompt={'yes' if ok else 'NO'} tail={buf[-80:]!r}") + if not ok: + return 2 + + # --- single-key echo latency --- + lat: list[float] = [] + for ch in "abcdefghijklmnopqrst": + n0 = rx_bytes + t0 = time.perf_counter() + await send(ch) + while rx_bytes == n0 and time.perf_counter() - t0 < 2: + await asyncio.sleep(0.0005) + lat.append((time.perf_counter() - t0) * 1000) + await asyncio.sleep(0.05) + await send("\x08" * 20) + await asyncio.sleep(0.4) + print(f"echo_ms p50={statistics.median(lat):.1f} p95={pct(lat, .95):.1f} max={max(lat):.1f}") + + if args.echo_only: + await ws.send(json.dumps({"type": "close"})) + await asyncio.sleep(0.3) + rt.cancel() + return 0 + + # --- burst typing: last echo arrival --- + await wait_prompt(5, 0.1) + n0 = len(buf) + t0 = time.perf_counter() + for ch in "abcdefghijklmnopqrst": + await send(ch) + await asyncio.sleep(0.003) + t_sent = time.perf_counter() + while time.perf_counter() - t_sent < 3 and not buf.endswith("abcdefghijklmnopqrst"): + await asyncio.sleep(0.002) + print(f"burst_echo_ms={(time.perf_counter() - t_sent) * 1000:.1f} (after last key sent)") + await send("\x08" * 20) + await asyncio.sleep(0.4) + + # --- command RTT --- + rtts: list[float] = [] + for _ in range(5): + await wait_prompt(10, 0.05) + n0 = len(buf) + t0 = time.perf_counter() + await send(args.probe_cmd + "\r") + await asyncio.sleep(0.02) + await wait_prompt(15, 0.0) + # prompt re-appeared after output + while len(buf) == n0 and time.perf_counter() - t0 < 10: + await asyncio.sleep(0.005) + rtts.append((time.perf_counter() - t0) * 1000) + print(f"cmd_rtt_ms p50={statistics.median(rtts):.0f} max={max(rtts):.0f} ({args.probe_cmd})") + + # --- bulk --- + await wait_prompt(10, 0.2) + rx0 = rx_bytes + t0 = time.perf_counter() + await send(args.bulk_cmd + "\r") + await asyncio.sleep(0.2) + # page through "More" prompts if any + deadline = time.perf_counter() + 90 + while time.perf_counter() < deadline: + tail = buf[-200:] + if "More" in tail[-40:]: + await send(" ") + await asyncio.sleep(0.01) + continue + if at_prompt() and time.perf_counter() - last_rx > 0.4 and rx_bytes - rx0 > 100: + break + await asyncio.sleep(0.01) + dt = time.perf_counter() - t0 - 0.4 + print(f"bulk bytes={rx_bytes - rx0} secs={dt:.2f} KB/s={(rx_bytes - rx0) / 1024 / max(dt, 1e-3):.0f}") + + if args.audit: + await wait_prompt(10, 0.2) + cmds = [__import__("codecs").decode(x, "unicode_escape") for x in args.audit_cmds.split("|") if x] or [ + "display clock\r", + "disX\x08play ver\t\r", + "dis ip int br\t\r", + "\r", + "display clock\r", + "\x1b[A\r", + "\x1b[A\x1b[A\x1b[B\r", + "system-view\r", + "sysname r1\r", + "quit\r", + ] + for cmd in cmds: + await send(cmd) + await asyncio.sleep(0.2) + await wait_prompt(10, 0.25) + if "More" in buf[-40:]: + await send(" ") + await asyncio.sleep(0.5) + + await ws.send(json.dumps({"type": "close"})) + await asyncio.sleep(0.3) + rt.cancel() + + time.sleep(0.8) + if args.audit: + print("--- audit.jsonl commands for session ---") + try: + with open(args.audit_file, encoding="utf-8") as fh: + for line in fh: + try: + o = json.loads(line) + except Exception: + continue + if o.get("event") == "command" and o.get("session_id") == sid: + print(repr(o.get("command")), o.get("source")) + except OSError as exc: + print("cannot read audit:", exc) + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/e2e_webcrt_input_scenarios.py b/scripts/e2e_webcrt_input_scenarios.py new file mode 100644 index 0000000..157e198 --- /dev/null +++ b/scripts/e2e_webcrt_input_scenarios.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Keystroke-level WebCRT audit scenarios against a live device. + +Every key is sent as its own stdin frame (~40ms apart, like a person typing) and the +command that landed in audit.jsonl for the session is compared with what the device +actually executed. + + python scripts/e2e_webcrt_input_scenarios.py --ne-id --vendor huawei + python scripts/e2e_webcrt_input_scenarios.py --ne-id --vendor cisco +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +import time + +import httpx +import websockets + +BASE = os.environ.get("NETX_BASE", "http://127.0.0.1:8890").rstrip("/") +USER = os.environ.get("NETX_USER", "admin") +PWD = os.environ.get("NETX_PASSWORD", "Admin@1234") + +LEFT, RIGHT, UP, DOWN = "\x1b[D", "\x1b[C", "\x1b[A", "\x1b[B" +HOME, END, DEL, BS, TAB, CR, CTRL_C = "\x1b[H", "\x1b[F", "\x1b[3~", "\x08", "\t", "\r", "\x03" + + +def keys(*parts: str | list[str]) -> list[str]: + """Flatten: a plain str is typed char by char, escape sequences stay whole.""" + out: list[str] = [] + for p in parts: + if isinstance(p, list): + out.extend(p) + elif p.startswith("\x1b") or p in (BS, TAB, CR, CTRL_C): + out.append(p) + else: + out.extend(list(p)) + return out + + +def scenarios(vendor: str) -> list[tuple[str, list[str], "str | list[str]"]]: + """(name, keystrokes, expected command text after the prompt).""" + if vendor == "cisco": + full, partial, tabbed, other = "show clock", "show cl", "sh clo", "show version" + wrong = "show clxock" + else: + full, partial, tabbed, other = "display clock", "display cl", "display clo", "display version" + wrong = "display clxock" + word = full.split()[-1] # clock + return [ + ("plain", keys(full, CR), full), + ("backspace fix", keys(full[:-3] + "x", BS, full[-3:], CR), full), + ("tab completion", keys(tabbed, TAB, CR), "sh clock" if vendor == "cisco" else full), + ("left-arrow insert mid-line", keys(full.replace("clock", "cock"), [LEFT] * 3, "l", CR), full), + ("left-arrow + backspace delete", keys(wrong, [LEFT] * 3, BS, CR), full), + ("left-arrow + Delete key", keys(wrong, [LEFT] * 4, DEL, CR), full), + ("left-arrow replace char", keys(full.replace("clock", "clxck"), [LEFT] * 2, BS, "o", CR), full), + ("home + insert first char", keys(full[1:], HOME, full[0], END, CR), full), + ("right-arrow after left", keys(full, LEFT, LEFT, RIGHT, RIGHT, CR), full), + ("up-arrow recall", keys(UP, CR), full), + ("up-arrow recall then edit tail", keys(UP, [BS] * len(word), other.split()[-1], CR), other), + ("up up down recall", keys(UP, UP, DOWN, CR), other), + ("recall + left-arrow delete+insert", keys(UP, [LEFT] * 2, BS, "c", CR), other[:-3] + "con"), + ("ctrl-c (device decides if line clears)", keys("garbage", CTRL_C, full, CR), ("garbage" + full) if vendor == "huawei" else full), + ("empty enter x2 (not audited)", keys(CR, CR), ""), + ("run then recall and run again", keys(full, CR, UP, CR), [full, full]), + ] + + +async def run(args: argparse.Namespace) -> int: + c = httpx.Client(base_url=BASE, timeout=60) + tok = c.post("/v1/auth/login", json={"username": USER, "password": PWD}).json()["access_token"] + h = {"Authorization": f"Bearer {tok}"} + sid = c.post("/v1/webcrt/sessions", headers=h, json={"ne_id": args.ne_id, "cols": 160, "rows": 50}).json()["session_id"] + ticket = c.post("/v1/webcrt/ws-ticket", headers=h).json()["ticket"] + url = f"{BASE.replace('http', 'ws', 1)}/v1/webcrt/sessions/{sid}/ws?ws_ticket={ticket}" + prompt_re = re.compile(r"(?:<[^>\r\n]+>|\[[^\]\r\n]+\]|[\w.\-]+(?:\([^)]+\))*[#>])\s*$") + results: list[tuple[str, str | None, list[str]]] = [] + + async with websockets.connect(url, max_size=None) as ws: + buf = "" + last_rx = 0.0 + + async def reader() -> None: + nonlocal buf, last_rx + try: + async for raw in ws: + if isinstance(raw, bytes): + buf += raw.decode("utf-8", "replace") + last_rx = time.perf_counter() + except Exception: + pass + + rt = asyncio.create_task(reader()) + + def at_prompt() -> bool: + plain = re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", buf).replace("\r", "\n") + tail = plain.rstrip(" ").split("\n")[-1] + return bool(prompt_re.search(tail)) and len(tail) < 80 + + async def settle(timeout: float = 20.0, quiet: float = 0.6) -> None: + end = time.perf_counter() + timeout + while time.perf_counter() < end: + if at_prompt() and time.perf_counter() - last_rx >= quiet: + return + await asyncio.sleep(0.02) + + async def send(data: str) -> None: + await ws.send(json.dumps({"type": "stdin", "data": data})) + + await settle(120, 1.0) + if not at_prompt(): + await send("\r") + await settle(15, 1.0) + sel = [x.strip() for x in args.only.split(",") if x.strip()] + for name, ks, expected in scenarios(args.vendor): + if sel and not name.startswith(tuple(sel)): + continue + before = len(read_cmds(sid, args.audit_file)) + for k in ks: + await send(k) + await asyncio.sleep(args.key_gap) + # Wait for the device (slow VMs echo seconds late), not a fixed delay: first for + # output after the last key, then until the expected audit rows exist. + sent_at = time.perf_counter() + while last_rx < sent_at and time.perf_counter() - sent_at < 15: + await asyncio.sleep(0.02) + await settle() + want = len(expected) if isinstance(expected, list) else (0 if expected == "" else 1) + waited = time.perf_counter() + while time.perf_counter() - waited < (2.0 if want == 0 else 15.0): + if len(read_cmds(sid, args.audit_file)) - before >= want: + break + await asyncio.sleep(0.1) + await asyncio.sleep(1.5) # Tab lines settle ~1.2s; also catches stray extra rows + got = read_cmds(sid, args.audit_file)[before:] + results.append((name, expected, got)) + await ws.send(json.dumps({"type": "close"})) + await asyncio.sleep(0.5) + rt.cancel() + + bad = 0 + print(f"{'scenario':38} {'result':6} recorded") + for name, expected, got in results: + tails = [re.sub(r"^(?:<[^>]*>|\[[^\]]*\]|[\w.\-]+(?:\([^)]*\))*[#>])\s*", "", g) for g in got] + if isinstance(expected, list): + ok = tails == expected + elif expected == "": + ok = not got + else: + ok = tails == [expected] + bad += 0 if ok else 1 + print(f"{name:38} {'OK' if ok else 'FAIL':6} {got!r}" + ("" if ok else f" expected {expected!r}")) + print(f"\n{len(results) - bad}/{len(results)} scenarios correct") + return 1 if bad else 0 + + +def read_cmds(sid: str, path: str) -> list[str]: + out: list[str] = [] + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + if '"event": "command"' not in line: + continue + o = json.loads(line) + if o.get("session_id") == sid: + out.append(str(o.get("command") or "")) + except OSError: + pass + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--ne-id", required=True) + ap.add_argument("--vendor", choices=["huawei", "cisco"], required=True) + ap.add_argument("--only", default="", help="comma-separated scenario name prefixes") + ap.add_argument("--key-gap", type=float, default=0.04, help="seconds between keystrokes") + ap.add_argument("--audit-file", default=os.path.join("data", "webcrt", "audit.jsonl")) + return asyncio.run(run(ap.parse_args())) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fake_ssh_device.py b/scripts/fake_ssh_device.py new file mode 100644 index 0000000..6b1cdbc --- /dev/null +++ b/scripts/fake_ssh_device.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Local fake network-device SSH server for WebCRT pipeline benchmarks. + +Zero device latency, so a benchmark against it measures netx overhead only. + + python scripts/fake_ssh_device.py [--port 2222] [--user lab] [--password lab] + +Behaviour: Cisco-like ``test#`` prompt, server-side echo with Tab completion +(``sh`` -> ``show ``), history on Up arrow, ``show clock`` (short output), +``bulk `` (large output) and ``quit``. +""" + +from __future__ import annotations + +import argparse +import socket +import threading +import time + +import paramiko + +HOST_KEY = paramiko.RSAKey.generate(2048) +PROMPT = b"test#" + + +class _Server(paramiko.ServerInterface): + def __init__(self, user: str, password: str) -> None: + self.user, self.password = user, password + self.event = threading.Event() + + def check_auth_password(self, username, password): # noqa: ANN001 + ok = username == self.user and password == self.password + return paramiko.AUTH_SUCCESSFUL if ok else paramiko.AUTH_FAILED + + def get_allowed_auths(self, username): # noqa: ANN001 + return "password" + + def check_channel_request(self, kind, chanid): # noqa: ANN001 + return paramiko.OPEN_SUCCEEDED if kind == "session" else paramiko.OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def check_channel_pty_request(self, *a): # noqa: ANN002 + return True + + def check_channel_shell_request(self, channel): # noqa: ANN001 + self.event.set() + return True + + def check_channel_window_change_request(self, *a): # noqa: ANN002 + return True + + +def _shell(chan: paramiko.Channel) -> None: + chan.send(b"\r\nFake device banner\r\n" + PROMPT) + line = b"" + history: list[bytes] = [] + hidx = 0 + try: + while True: + data = chan.recv(1024) + if not data: + return + i = 0 + while i < len(data): + b = data[i : i + 1] + if data[i : i + 3] == b"\x1b[A": # Up + if history: + hidx = max(0, hidx - 1) if hidx else len(history) - 1 + chan.send(b"\x08" * len(line) + b" " * len(line) + b"\x08" * len(line)) + line = history[hidx] + chan.send(line) + i += 3 + continue + if b == b"\t": + if line.startswith(b"sh") and b" " not in line: + chan.send(b"ow "[len(line) - 2 :] if len(line) <= 4 else b"") + line = b"show " + i += 1 + continue + if b in (b"\r", b"\n"): + chan.send(b"\r\n") + cmd = line.strip() + line = b"" + if cmd: + history.append(cmd) + hidx = len(history) + if cmd == b"quit": + chan.close() + return + if cmd == b"show clock": + chan.send(b"2026-10-12 12:00:00\r\nMonday\r\n") + elif cmd.startswith(b"bulk"): + mb = float(cmd.split()[1]) if len(cmd.split()) > 1 else 1.0 + row = b"Ethernet1/0/%d unassigned up down --\r\n" + sent = 0 + n = 0 + buf = [] + while sent < mb * 1024 * 1024: + r = row % (n % 999) + buf.append(r) + sent += len(r) + n += 1 + if len(buf) >= 400: + chan.sendall(b"".join(buf)) + buf = [] + if buf: + chan.sendall(b"".join(buf)) + elif cmd: + chan.send(b"%% Unknown command\r\n") + chan.send(PROMPT) + i += 1 + continue + if b in (b"\x08", b"\x7f"): + if line: + line = line[:-1] + chan.send(b"\x08 \x08") + i += 1 + continue + line += b + chan.send(b) + i += 1 + except Exception: # noqa: BLE001 + pass + finally: + try: + chan.close() + except Exception: # noqa: BLE001 + pass + + +def _serve(sock: socket.socket, user: str, password: str) -> None: + t = paramiko.Transport(sock) + t.add_server_key(HOST_KEY) + srv = _Server(user, password) + try: + t.start_server(server=srv) + chan = t.accept(30) + if chan is None: + return + srv.event.wait(10) + _shell(chan) + except Exception: # noqa: BLE001 + pass + finally: + try: + t.close() + except Exception: # noqa: BLE001 + pass + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--port", type=int, default=2222) + ap.add_argument("--user", default="lab") + ap.add_argument("--password", default="lab") + a = ap.parse_args() + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("127.0.0.1", a.port)) + s.listen(20) + print(f"fake device on 127.0.0.1:{a.port} user={a.user}", flush=True) + while True: + c, _ = s.accept() + threading.Thread(target=_serve, args=(c, a.user, a.password), daemon=True).start() + + +if __name__ == "__main__": + main() diff --git a/scripts/init_pg.ps1 b/scripts/init_pg.ps1 index d9c02de..cfcb909 100644 --- a/scripts/init_pg.ps1 +++ b/scripts/init_pg.ps1 @@ -1,4 +1,4 @@ -param( +param( [string]$PgHost = "127.0.0.1", [int]$PgPort = 5432, [string]$SuperUser = "postgres", diff --git a/scripts/start_netx.ps1 b/scripts/start_netx.ps1 index ed2caec..50cb267 100644 --- a/scripts/start_netx.ps1 +++ b/scripts/start_netx.ps1 @@ -23,9 +23,24 @@ if ($Backgtound -and -not $Background) { $projectRoot = Split-Path -Parent $PSScriptRoot Set-Location $projectRoot -$runDir = Join-Path $PSScriptRoot ".run" +$packCommon = Join-Path $projectRoot "packaging\_common.ps1" +if (Test-Path -LiteralPath $packCommon) { + . $packCommon + if (Get-Command Repair-NetxShippedVenv -ErrorAction SilentlyContinue) { + $null = Repair-NetxShippedVenv -ProgramRoot $projectRoot + } +} + +# Prefer writable data-root runtime dir (Program Files is not writable after Setup install). +if ($env:NETX_RUN_DIR) { + $runDir = $env:NETX_RUN_DIR +} elseif ($env:NETX_SCHEDULER_HEARTBEAT_PATH) { + $runDir = Split-Path -Parent ($env:NETX_SCHEDULER_HEARTBEAT_PATH -replace '/', '\') +} else { + $runDir = Join-Path $PSScriptRoot ".run" +} if (-not (Test-Path $runDir)) { - New-Item -ItemType Directory -Path $runDir | Out-Null + New-Item -ItemType Directory -Path $runDir -Force | Out-Null } $pidFile = Join-Path $runDir "netx.pid" @@ -38,14 +53,22 @@ $bizStateWorkerPidDir = Join-Path $runDir "biz_state_workers" $webPidFile = Join-Path $runDir "web.pid" $webLogFile = Join-Path $runDir "web.out.log" $webErrFile = Join-Path $runDir "web.err.log" +# Children share this console; give them an empty stdin so Vite (raw-mode CLI shortcuts) +# and python cannot steal keystrokes from the shell that launched them. +$nullStdin = Join-Path $runDir "stdin.empty" +if (-not (Test-Path -LiteralPath $nullStdin)) { New-Item -ItemType File -Path $nullStdin -Force | Out-Null } -$venvPython = Join-Path $projectRoot ".venv\\Scripts\\python.exe" -if (-not (Test-Path $venvPython)) { +$venvPython = Join-Path $projectRoot ".venv\Scripts\python.exe" +if (Get-Command Test-NetxVenvRunnable -ErrorAction SilentlyContinue) { + if (-not (Test-NetxVenvRunnable -VenvPython $venvPython)) { + throw "venv_not_runnable: reinstall NetX or run setup as administrator" + } +} elseif (-not (Test-Path -LiteralPath $venvPython)) { Write-Host "==> .venv not found, creating virtual environment" python -m venv (Join-Path $projectRoot ".venv") -} -if (-not (Test-Path $venvPython)) { - throw "failed_to_create_venv" + if (-not (Test-Path -LiteralPath $venvPython)) { + throw "failed_to_create_venv" + } } $pythonExe = $venvPython @@ -124,6 +147,8 @@ function Show-LogTail { function Test-NetxApiListening { param([string]$HostName, [int]$LocalPort, [int]$WaitSec = 45) $deadline = (Get-Date).AddSeconds($WaitSec) + # Wildcard binds are not connectable on Windows; probe via loopback. + if ($HostName -in @("0.0.0.0", "::", "[::]", "*")) { $HostName = "127.0.0.1" } $healthUrl = "http://${HostName}:${LocalPort}/health" while ((Get-Date) -lt $deadline) { try { @@ -165,6 +190,7 @@ function Start-NetxWorker { -ArgumentList @("-m", "netx_api.worker") ` -WorkingDirectory $projectRoot ` -WindowStyle Hidden ` + -RedirectStandardInput $nullStdin ` -RedirectStandardOutput $outLog ` -RedirectStandardError $errLog ` -PassThru @@ -209,6 +235,7 @@ function Start-BizStateWorkers { -ArgumentList @("-m", "netx_api.biz_state_worker") ` -WorkingDirectory $projectRoot ` -WindowStyle Hidden ` + -RedirectStandardInput $nullStdin ` -RedirectStandardOutput $outLog ` -RedirectStandardError $errLog ` -PassThru @@ -232,6 +259,7 @@ if ($Background) { -ArgumentList @("-m", "netx_api.main") ` -WorkingDirectory $projectRoot ` -WindowStyle Hidden ` + -RedirectStandardInput $nullStdin ` -RedirectStandardOutput $logFile ` -RedirectStandardError $errFile ` -PassThru @@ -253,7 +281,11 @@ if ($Background) { Show-LogTail -Path $logFile exit 1 } - $healthWaitSec = 45 + # Fresh installs run Alembic upgrades on first boot; 45s is often too short. + $healthWaitSec = 180 + if ($env:NETX_START_HEALTH_WAIT_SEC) { + try { $healthWaitSec = [int]$env:NETX_START_HEALTH_WAIT_SEC } catch {} + } if (-not (Test-NetxApiListening -HostName $BindHost -LocalPort $Port -WaitSec $healthWaitSec)) { Write-Host "[ERR] Port $Port not listening /health not OK within ${healthWaitSec}s (process may be stuck on DB or schema migration)." -ForegroundColor Red Show-LogTail -Path $errFile @@ -270,6 +302,7 @@ if ($Background) { -ArgumentList @("run", "dev", "--", "--host", $BindHost, "--port", "$WebPort") ` -WorkingDirectory $webRoot ` -WindowStyle Hidden ` + -RedirectStandardInput $nullStdin ` -RedirectStandardOutput $webLogFile ` -RedirectStandardError $webErrFile ` -PassThru @@ -309,6 +342,7 @@ if ($WithWeb) { -ArgumentList @("run", "dev", "--", "--host", $BindHost, "--port", "$WebPort") ` -WorkingDirectory $webRoot ` -WindowStyle Hidden ` + -RedirectStandardInput $nullStdin ` -RedirectStandardOutput $webLogFile ` -RedirectStandardError $webErrFile ` -PassThru diff --git a/scripts/stop_netx.ps1 b/scripts/stop_netx.ps1 index 2d91466..1106bb3 100644 --- a/scripts/stop_netx.ps1 +++ b/scripts/stop_netx.ps1 @@ -1,4 +1,4 @@ -param( +param( [int]$Port = 8890, [int]$WebPort = 5173, # Windows often ignores graceful Stop-Process on python; default hard-kill. @@ -9,7 +9,13 @@ param( $ErrorActionPreference = "Continue" $useForce = if ($NoForce) { $false } else { [bool]$Force } -$runDir = Join-Path $PSScriptRoot ".run" +if ($env:NETX_RUN_DIR) { + $runDir = $env:NETX_RUN_DIR +} elseif ($env:NETX_SCHEDULER_HEARTBEAT_PATH) { + $runDir = Split-Path -Parent ($env:NETX_SCHEDULER_HEARTBEAT_PATH -replace '/', '\') +} else { + $runDir = Join-Path $PSScriptRoot ".run" +} $pidFile = Join-Path $runDir "netx.pid" $workerPidFile = Join-Path $runDir "worker.pid" $webPidFile = Join-Path $runDir "web.pid" diff --git a/scripts/topology_clear_frozen_labels.py b/scripts/topology_clear_frozen_labels.py new file mode 100644 index 0000000..dedf6ad --- /dev/null +++ b/scripts/topology_clear_frozen_labels.py @@ -0,0 +1,79 @@ +"""Clear view-node labels that merely freeze the current fabric name/ip. + +Default is dry-run. Pass --apply to write. + +A placement label is considered frozen when it equals the fabric node's current +name or IP (case-insensitive, stripped). Clearing it restores follow-device-name +behavior on the next graph read. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from sqlalchemy import or_ + +from netx_api.db import SessionLocal +from netx_api.models import TopoFabricNode, TopoViewNode + + +def _norm(s: str | None) -> str: + return str(s or "").strip().casefold() + + +def clear_frozen_labels(db, *, apply: bool) -> tuple[int, int]: + """Return (scanned_nonempty_labels, frozen_cleared). Caller commits/rolls back.""" + rows = ( + db.query(TopoViewNode, TopoFabricNode) + .outerjoin(TopoFabricNode, TopoFabricNode.id == TopoViewNode.fabric_node_id) + .filter(TopoViewNode.label != "") + .filter(or_(TopoViewNode.label.isnot(None))) + .all() + ) + scanned = 0 + cleared = 0 + for vn, fn in rows: + scanned += 1 + label = _norm(vn.label) + if not label or fn is None: + continue + if label not in {_norm(fn.name), _norm(fn.ip)}: + continue + cleared += 1 + if apply: + vn.label = "" + return scanned, cleared + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apply", + action="store_true", + help="Write changes (default: dry-run only)", + ) + args = parser.parse_args(argv) + + db = SessionLocal() + try: + scanned, cleared = clear_frozen_labels(db, apply=args.apply) + if args.apply: + db.commit() + else: + db.rollback() + mode = "apply" if args.apply else "dry-run" + print(f"mode={mode} scanned={scanned} frozen_labels={cleared}") + return 0 + except Exception: + db.rollback() + raise + finally: + db.close() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c29b7da --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,41 @@ +"""Shared test hygiene: never connect the test suite to the application database.""" +import os +from pathlib import Path +from tempfile import TemporaryDirectory + +import pytest + + +# Set this before any test imports netx_api.config/db. Several older unittest +# cases use the global SessionLocal and delete entire tables in setUp. +_TEST_STATE = TemporaryDirectory(prefix="netx-tests-", ignore_cleanup_errors=True) +_TEST_DATABASE_URL = f"sqlite:///{(Path(_TEST_STATE.name) / 'netx.sqlite').as_posix()}" +os.environ["NETX_DATABASE_URL"] = _TEST_DATABASE_URL +os.environ["NETX_CREDENTIAL_SECRET_FILE"] = str(Path(_TEST_STATE.name) / "credential_secret") + + +def pytest_sessionstart(session): + from sqlalchemy.engine import make_url + + from netx_api.db import engine + + # Compare parsed URLs: newer SQLAlchemy renders "C:" as "C%3A" in str(url). + if engine.url != make_url(_TEST_DATABASE_URL): + raise pytest.UsageError("NetX tests require the isolated temporary database") + + +def pytest_sessionfinish(session, exitstatus): + from netx_api.db import engine + + engine.dispose() + _TEST_STATE.cleanup() + + +@pytest.fixture(autouse=True) +def _fresh_baseline_row_cache(): + """Test databases reuse batch ids; the process-wide baseline cache must not leak between tests.""" + from netx_api.biz_migration.service import BASELINE_ROWS + + BASELINE_ROWS.invalidate() + yield + BASELINE_ROWS.invalidate() diff --git a/tests/test_auth.py b/tests/test_auth.py index 0b00176..ddf807c 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -440,6 +440,32 @@ class AuthApiTests(unittest.TestCase): ) self.assertEqual(empty.status_code, 400) + def test_api_token_purge_removes_row(self) -> None: + token = self._login() + created = self.client.post( + "/v1/api-tokens", + headers={"Authorization": f"Bearer {token}"}, + json={"name": "to-purge", "expires_in_days": 7, "scopes": ["ne:read"]}, + ) + self.assertEqual(created.status_code, 200, created.text) + tid = created.json()["token"]["id"] + revoked = self.client.delete( + f"/v1/api-tokens/{tid}", + headers={"Authorization": f"Bearer {token}"}, + ) + self.assertEqual(revoked.status_code, 200, revoked.text) + listed = self.client.get("/v1/api-tokens", headers={"Authorization": f"Bearer {token}"}) + self.assertEqual(listed.status_code, 200, listed.text) + self.assertTrue(any(r["id"] == tid and r.get("revoked") for r in listed.json().get("items") or [])) + purged = self.client.delete( + f"/v1/api-tokens/{tid}/purge", + headers={"Authorization": f"Bearer {token}"}, + ) + self.assertEqual(purged.status_code, 200, purged.text) + listed2 = self.client.get("/v1/api-tokens", headers={"Authorization": f"Bearer {token}"}) + self.assertEqual(listed2.status_code, 200, listed2.text) + self.assertFalse(any(r["id"] == tid for r in listed2.json().get("items") or [])) + def test_api_token_max_count(self) -> None: token = self._login() with patch("netx_api.auth_service.settings.auth_api_token_max_count", 2): diff --git a/tests/test_biz_compare_templates.py b/tests/test_biz_compare_templates.py new file mode 100644 index 0000000..83dd0f1 --- /dev/null +++ b/tests/test_biz_compare_templates.py @@ -0,0 +1,280 @@ +"""Template validation, rule semantics, protected references and bounded queries.""" +from copy import deepcopy + +import pytest +from fastapi import HTTPException +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker + +from netx_api.db import Base +from netx_api.models import BizCompareJob, BizCompareRun, BizCompareTemplate, BizStateBatchCommand, BizStateBatch +from netx_api.biz_state import compare_service as svc +from netx_api.biz_state.compare_engine import compare_rows, row_key +from netx_api.biz_state.compare_rules import eval_leaf_filter, values_equal +from netx_api.biz_state.compare_sql import compile_row_filters_sql, _rk_sql, _field_differs_sql +from netx_api.biz_state_router import TemplateIn, TemplatePatchIn + + +@pytest.fixture +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine, expire_on_commit=False)() + yield session + session.close() + engine.dispose() + + +def body(): + return {"name": "Test", "metrics": [{"sheet_id": "s", "metric_id": "custom", "key_fields": ["id"], + "compare_fields": ["count"], "iface_fields": ["port"], "ignore_port_changes": False, + "row_filters": [], "field_rules": []}]} + + +@pytest.mark.parametrize("bad", [ + {"key_fields": []}, {"key_fields": ["id", "id"]}, {"metric_id": ""}, + {"row_filters": [{"field": "state", "op": "typo", "value": "up"}]}, + {"row_filters": [{"field": "state", "op": "regex", "value": "["}]}, + {"row_filters": [{"any": []}]}, {"row_filters": [{"field": "id", "op": "in", "value": "a,b"}]}, + {"field_rules": [{"field": "count", "compare": "typo"}]}, + {"field_rules": [{"field": "count", "normalize": "typo"}]}, + {"field_rules": [{"field": "count", "tolerance": -1}]}, + {"field_rules": [{"field": "count", "tolerance": float("inf")}]}, + {"field_rules": [{"field": "count", "tolerance": float("nan")}]}, + {"field_rules": [{"field": "count"}, {"field": "count"}]}, +]) +def test_invalid_sheet_never_silently_drops_or_saves(db, bad): + data = body() + data["metrics"].append({**deepcopy(data["metrics"][0]), "sheet_id": "second", **bad}) + with pytest.raises(HTTPException) as exc: + svc.create_template(db, data) + assert exc.value.status_code == 400 + assert "metrics[1]" in exc.value.detail["path"] + assert db.query(BizCompareTemplate).count() == 0 + + +def test_template_validation_precedes_mutation_and_rejects_duplicate_alias(db): + tpl = svc.create_template(db, body()) + data = body() + data["name"] = "Wrong name" + data["metrics"][0]["key_fields"] = [] + with pytest.raises(HTTPException): + svc.update_template(db, tpl["id"], data) + assert db.get(BizCompareTemplate, tpl["id"]).name == "Test" + with pytest.raises(HTTPException): + svc.create_template(db, {**body(), "iface_normalize_rules": [{"from": "GE", "to": "a"}, {"from": "ge", "to": "b"}]}) + + +def test_api_port_policy_roundtrip_and_legacy_patch(db): + saved = svc.create_template(db, TemplateIn.model_validate(body()).model_dump()) + assert saved["metrics"][0]["ignore_port_changes"] is False + updated = svc.update_template(db, saved["id"], TemplatePatchIn(ignore_port_changes=True).model_dump(exclude_unset=True)) + assert updated["metrics"][0]["ignore_port_changes"] is True + legacy = svc.create_template(db, {"name": "legacy", "metric_id": "custom", "key_fields": ["id"], "ignore_port_changes": False}) + assert legacy["metrics"][0]["ignore_port_changes"] is False + + +@pytest.mark.parametrize("value", [0, False]) +def test_filter_and_identity_preserve_false_and_zero(value): + assert not eval_leaf_filter({"x": value}, {"field": "x", "op": "empty"}) + assert eval_leaf_filter({"x": value}, {"field": "x", "op": "eq", "value": value}) + assert row_key({"id": value}, ["id"]) != row_key({"id": None}, ["id"]) + result = compare_rows(before_rows=[{"id": value}], after_rows=[{"id": None}], key_fields=["id"], iface_fields=[], compare_fields=[], port_map={}) + assert result["summary"]["removed"] == 1 and result["summary"]["added"] == 1 + _, params = compile_row_filters_sql([{"field": "x", "op": "eq", "value": value}]) + assert list(params.values()) == ["0" if value is not False else "false"] + + +@pytest.mark.parametrize("mode", ["numeric", "percent"]) +def test_missing_numeric_value_is_not_zero(mode): + assert not values_equal(None, 0, rule={"compare": mode, "tolerance": 5}) + assert values_equal(None, "", rule={"compare": mode}) + assert values_equal("+.5", "0.5", rule={"compare": mode}) + assert values_equal("1.", "1", rule={"compare": mode}) + assert values_equal("NaN", "NaN", rule={"compare": mode}) # textual fallback + + +def test_sql_filter_literal_contains_and_collision_free_composite_keys(): + sql, params = compile_row_filters_sql([{"field": "x", "op": "contains", "value": "a%_\\b"}]) + assert "ESCAPE" in sql + assert list(params.values()) == ["%a\\%\\_\\\\b%"] + sql, _ = compile_row_filters_sql([{"field": "x", "op": "contains", "value": ""}]) + assert "FALSE" in sql + assert "jsonb_build_array" in _rk_sql(["a", "b"]) + assert "concat_ws" not in _rk_sql(["a", "b"]) + assert "[+-]?" in _field_differs_sql("b", "a", {"compare": "numeric"}) + + +def test_sheet_order_uses_one_aggregate_query_without_raw_text(db): + for bid, metric, count in (("b", "large", 20), ("a", "large", 30), ("b", "small", 3)): + db.add(BizStateBatchCommand(id=f"{bid}-{metric}", batch_id=bid, metric_id=metric, row_count=count, raw_text="x" * 100000)) + db.commit() + sql = [] + event.listen(db.get_bind(), "before_cursor_execute", lambda c, cur, statement, p, ctx, many: sql.append(statement)) + sheets = [{"metric_id": "large", "sheet_id": "l1"}, {"metric_id": "small", "sheet_id": "s"}, {"metric_id": "large", "sheet_id": "l2"}] + ordered = svc._order_sheets_small_first(db, sheets, before_batch_id="b", after_batch_id="a") + assert [s["sheet_id"] for s in ordered] == ["s", "l1", "l2"] + assert len(sql) == 1 and "raw_text" not in sql[0] + + +def test_referenced_template_missing_sheets_and_running_delete_protected(db): + tpl = svc.create_template(db, body()) + db.add(BizCompareJob(id="j", template_id=tpl["id"], enabled_sheet_ids=["s"])) + db.add(BizCompareRun(id="r", job_id="j", status="running")) + db.commit() + for action in (lambda: svc.delete_template(db, tpl["id"]), lambda: svc.delete_job(db, "j"), lambda: svc.delete_run(db, "r")): + with pytest.raises(HTTPException) as exc: + action() + assert exc.value.status_code == 409 + with pytest.raises(HTTPException) as exc: + svc.update_job(db, "j", {"name": "invalid", "enabled_sheet_ids": ["missing"]}) + assert exc.value.detail["error"] == "unknown_enabled_sheets" + assert db.get(BizCompareJob, "j").name != "invalid" + + +def test_template_usage_counts_are_batched(db, monkeypatch): + monkeypatch.setattr(svc, "ensure_default_templates", lambda db: None) + monkeypatch.setattr(svc, "upgrade_builtin_split_sheets", lambda db: None) + used = svc.create_template(db, body()) + unused = svc.create_template(db, {**body(), "name": "Unused"}) + db.add_all([BizCompareJob(id="j1", template_id=used["id"]), BizCompareJob(id="j2", template_id=used["id"])]) + db.commit() + sql = [] + event.listen(db.get_bind(), "before_cursor_execute", lambda c, cur, statement, p, ctx, many: sql.append(statement)) + counts = {t["id"]: t["job_count"] for t in svc.list_templates(db)} + assert counts == {used["id"]: 2, unused["id"]: 0} + assert len(sql) == 3 # template query + one grouped query each for jobs and monitor templates + + +@pytest.mark.parametrize("status", ["queued", "running", "failed", "cancelled", "partial"]) +def test_compare_rejects_incomplete_sources_before_enqueue(db, status): + tpl = svc.create_template(db, body()) + db.add(BizStateBatch(id="before", status="success")) + db.add(BizStateBatch(id="after", status=status)) + db.add(BizCompareJob(id="j", template_id=tpl["id"], before_batch_id="before", after_batch_id="after")) + db.commit() + with pytest.raises(HTTPException) as exc: + svc._validate_compare_job(db, "j") + assert exc.value.detail == "source_batch_not_complete" + assert db.query(BizCompareRun).count() == 0 + + +def test_compare_rejects_partially_stale_sheet_scope_after_template_edit(db): + tpl = svc.create_template(db, body()) + db.add_all([BizStateBatch(id="before", status="success"), BizStateBatch(id="after", status="success"), + BizCompareJob(id="j", template_id=tpl["id"], before_batch_id="before", after_batch_id="after", enabled_sheet_ids=["s", "removed"])]) + db.commit() + with pytest.raises(HTTPException) as exc: + svc._validate_compare_job(db, "j") + assert exc.value.detail == {"error": "unknown_enabled_sheets", "sheet_ids": ["removed"]} + assert db.query(BizCompareRun).count() == 0 + + +@pytest.mark.parametrize("bad", [ + {"key_fields": []}, + {"row_filters": [{"field": "id", "op": "typo", "value": "a"}]}, + {"row_filters": [{"field": "id", "op": "regex", "value": "["}]}, + {"field_rules": [{"field": "count", "compare": "typo"}]}, +]) +def test_legacy_invalid_template_cannot_execute_as_partial_or_fail_open(db, bad): + tpl = svc.create_template(db, body()) + row = db.get(BizCompareTemplate, tpl["id"]) + row.metrics_json = [*row.metrics_json, {**body()["metrics"][0], "sheet_id": "invalid", **bad}] + db.add_all([BizStateBatch(id="before", status="success"), BizStateBatch(id="after", status="success"), + BizCompareJob(id="j", template_id=tpl["id"], before_batch_id="before", after_batch_id="after")]) + db.commit() + with pytest.raises(HTTPException) as exc: + svc._validate_compare_job(db, "j") + assert exc.value.detail["error"] == "invalid_template" + assert "metrics[1]" in exc.value.detail["path"] + assert db.query(BizCompareRun).count() == 0 + + +def _builtin_sheet(row, metric_id): + return next(s for s in svc.template_metrics(row) if s["metric_id"] == metric_id) + + +def _rule(sheet, field): + return next(r for r in sheet["field_rules"] if r["field"] == field) + + +def test_packaged_status_template_uses_corrected_defaults(db): + row = svc.ensure_default_zte_status_template(db) + detail = _builtin_sheet(row, "interface_detail") + assert _rule(detail, "output_bps")["compare"] == "percent" + for f in ("input_bps", "output_bps", "in_util", "out_util"): + assert f not in detail["compare_fields"] and f in detail["display_fields"] + rx = _rule(_builtin_sheet(row, "optical_brief"), "rx_power") + assert (rx["compare"], rx["tolerance"]) == ("numeric", 2) + assert _builtin_sheet(row, "l2vpn_mac")["iface_fields"] == ["ac_port"] + + +def _store_old_packaged_values(db, row): + sheets = svc.template_metrics(row) + for s in sheets: + if s["metric_id"] == "interface_detail": + _rule(s, "output_bps").update(compare="numeric", tolerance=5) + s["compare_fields"] = list(svc._INTERFACE_DETAIL_OLD_COMPARE) + elif s["metric_id"] == "optical_brief": + _rule(s, "rx_power").update(compare="percent", tolerance=5) + elif s["metric_id"] == "l2vpn_mac": + s["iface_fields"] = [] + svc._apply_sheets_to_row(row, sheets) + db.commit() + + +def test_old_packaged_values_are_upgraded_in_place(db): + row = svc.ensure_default_zte_status_template(db) + _store_old_packaged_values(db, row) + row = svc.ensure_default_zte_status_template(db) + detail = _builtin_sheet(row, "interface_detail") + assert _rule(detail, "output_bps")["compare"] == "percent" + assert "output_bps" not in detail["compare_fields"] + assert _rule(_builtin_sheet(row, "optical_brief"), "rx_power")["compare"] == "numeric" + assert _builtin_sheet(row, "l2vpn_mac")["iface_fields"] == ["ac_port"] + + +def test_operator_customised_values_survive_upgrade(db): + row = svc.ensure_default_zte_status_template(db) + _store_old_packaged_values(db, row) + sheets = svc.template_metrics(row) + for s in sheets: + if s["metric_id"] == "optical_brief": + _rule(s, "rx_power").update(compare="percent", tolerance=1) # operator tightened it + elif s["metric_id"] == "interface_detail": + s["compare_fields"] = ["admin", "output_bps"] # operator trimmed the list + elif s["metric_id"] == "l2vpn_mac": + s["iface_fields"] = ["neighbor"] + svc._apply_sheets_to_row(row, sheets) + db.commit() + row = svc.ensure_default_zte_status_template(db) + rx = _rule(_builtin_sheet(row, "optical_brief"), "rx_power") + assert (rx["compare"], rx["tolerance"]) == ("percent", 1) + assert _builtin_sheet(row, "interface_detail")["compare_fields"] == ["admin", "output_bps"] + assert _builtin_sheet(row, "l2vpn_mac")["iface_fields"] == ["neighbor"] + + +def test_packaged_arp_filter_uses_canonical_entry_type_and_old_filter_is_upgraded(db): + from netx_api.biz_state.compare_rules import arp_dynamic_row_filters + + row = svc.ensure_default_zte_status_template(db) + assert _builtin_sheet(row, "arp")["row_filters"] == arp_dynamic_row_filters() + sheets = svc.template_metrics(row) + for s in sheets: + if s["metric_id"] == "arp": + s["row_filters"] = [{"op": "age_timer", "field": "age", "value": ""}] + svc._apply_sheets_to_row(row, sheets) + db.commit() + assert _builtin_sheet(svc.ensure_default_zte_status_template(db), "arp")["row_filters"] == arp_dynamic_row_filters() + + +def test_customised_arp_filter_is_left_alone(db): + row = svc.ensure_default_zte_status_template(db) + sheets = svc.template_metrics(row) + mine = [{"op": "eq", "field": "entry_type", "value": "static"}] + for s in sheets: + if s["metric_id"] == "arp": + s["row_filters"] = mine + svc._apply_sheets_to_row(row, sheets) + db.commit() + assert _builtin_sheet(svc.ensure_default_zte_status_template(db), "arp")["row_filters"] == mine diff --git a/tests/test_biz_migration_async.py b/tests/test_biz_migration_async.py new file mode 100644 index 0000000..7c60cf1 --- /dev/null +++ b/tests/test_biz_migration_async.py @@ -0,0 +1,284 @@ +"""H2/H3/H5: background evaluation, manual route tables, duplicate-key scope.""" +from __future__ import annotations + +import unittest +from unittest import mock + +from fastapi import HTTPException +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_migration import service as mig +from netx_api.models import BizMigrationBatch, BizMigrationRun, BizStateBatchCommand, BizStateMetricRow +import tests.test_biz_migration_evidence as evidence + + +class _Inline: + """threading.Thread stand-in: remember the target, run it on demand.""" + + started: list = [] + + def __init__(self, target=None, **_): + self.target = target + + def start(self): + _Inline.started.append(self.target) + + +class _Fixture(unittest.TestCase): + def setUp(self) -> None: + self.ctx = evidence.EvidencePersistTests("test_evaluate_persists_display_ab_and_evidence_with_command") + self.ctx.setUp() + self.db = self.ctx.db + self.factory = sessionmaker(bind=self.db.get_bind(), expire_on_commit=False) + self.session_patch = mock.patch("netx_api.db.SessionLocal", self.factory) + self.session_patch.start() + _Inline.started = [] + proj = self.ctx._create_project() + c = self.ctx + for bid, tid in (("bl_old", c.old_p.id), ("bl_new", c.new_p.id), ("cur_old", proj["old_hf_task_id"]), + ("cur_new", proj["new_hf_task_id"])): + c._mk_batch(bid, tid) + c._seed_if_row(batch_id="bl_old", task_id=c.old_p.id, ne_id="ne-old", command_id="c1", + raw_command="show interface brief", interface="GE-old") + c._seed_if_row(batch_id="cur_new", task_id=proj["new_hf_task_id"], ne_id="ne-new", command_id="c2", + raw_command="show interface brief", interface="xgei-new") + mig.patch_project(self.db, proj["id"], {"old_baseline_batch_id": "bl_old", "new_baseline_batch_id": "bl_new"}) + self.batch = mig.create_batch(self.db, proj["id"], {"batch_label": "w", "expect_set": {"ports": ["gei-old"]}}) + mig.patch_batch(self.db, self.batch["id"], {"status": "active"}) + self.project_id = proj["id"] + + def tearDown(self) -> None: + self.session_patch.stop() + self.ctx.tearDown() + + def run_background(self): + for target in list(_Inline.started): + target() + _Inline.started.clear() + + +class AsyncEvaluateTests(_Fixture): + def test_evaluate_returns_a_running_run_immediately_then_completes(self): + with mock.patch.object(mig.threading, "Thread", _Inline): + started = mig.enqueue_evaluate(self.db, self.batch["id"]) + self.assertEqual(started["status"], "running") + self.assertEqual(self.db.get(BizMigrationRun, started["id"]).status, "running") + self.assertEqual(len(_Inline.started), 1) + self.run_background() + self.db.expire_all() + run = self.db.get(BizMigrationRun, started["id"]) + self.assertEqual(run.status, "success") + self.assertEqual(run.purpose, "manual") + self.assertIn("sheet_cards", run.summary_json) + self.assertEqual(self.db.query(BizMigrationRun).count(), 1) # the claim row was filled, not duplicated + + def test_second_request_while_one_runs_is_409(self): + with mock.patch.object(mig.threading, "Thread", _Inline): + mig.enqueue_evaluate(self.db, self.batch["id"]) + with self.assertRaises(HTTPException) as exc: + mig.enqueue_evaluate(self.db, self.batch["id"]) + with self.assertRaises(HTTPException) as exc_finish: + mig.enqueue_finish(self.db, self.batch["id"]) + self.assertEqual(exc.exception.status_code, 409) + self.assertEqual(exc.exception.detail, "evaluation_in_progress") + self.assertEqual(exc_finish.exception.status_code, 409) + + def test_failure_is_recorded_on_the_run_and_releases_the_project(self): + with mock.patch.object(mig.threading, "Thread", _Inline): + started = mig.enqueue_evaluate(self.db, self.batch["id"]) + with mock.patch.object(mig, "run_evaluate", side_effect=HTTPException(status_code=400, detail="boom")): + self.run_background() + self.db.expire_all() + run = self.db.get(BizMigrationRun, started["id"]) + self.assertEqual((run.status, run.message), ("failed", "boom")) + with mock.patch.object(mig.threading, "Thread", _Inline): + mig.enqueue_evaluate(self.db, self.batch["id"]) # no longer blocked + + def test_restart_cancels_orphaned_runs(self): + with mock.patch.object(mig.threading, "Thread", _Inline): + started = mig.enqueue_evaluate(self.db, self.batch["id"]) + self.assertEqual(mig.recover_interrupted_evaluations_on_startup(self.db), 1) + self.db.expire_all() + run = self.db.get(BizMigrationRun, started["id"]) + self.assertEqual((run.status, run.message), ("cancelled", "interrupted_by_restart")) + # a cancelled claim must not be resurrected by a late worker + with self.assertRaises(HTTPException) as exc: + mig.run_evaluate(self.db, batch_id=self.batch["id"], _run_id=started["id"]) + self.assertEqual(exc.exception.status_code, 409) + + def test_background_evaluation_locks_the_project_only_for_the_final_write(self): + from sqlalchemy.orm import Query + + calls: list[str] = [] + original = Query.with_for_update + + def spy(self_, *a, **k): + calls.append("lock") + return original(self_, *a, **k) + + with mock.patch.object(mig.threading, "Thread", _Inline): + mig.enqueue_evaluate(self.db, self.batch["id"]) # claim = one short lock + with mock.patch.object(Query, "with_for_update", spy): + self.run_background() + self.assertEqual(len(calls), 1, "computation must be unlocked; only the write takes the project lock") + calls.clear() + with mock.patch.object(Query, "with_for_update", spy): + self.db.query(BizMigrationRun).delete() + self.db.commit() + mig.run_evaluate(self.db, batch_id=self.batch["id"]) # synchronous callers keep the old behaviour + self.assertGreaterEqual(len(calls), 1) + + def test_finish_runs_in_the_background_and_closes_the_wave_only_on_success(self): + with mock.patch.object(mig.threading, "Thread", _Inline): + started = mig.enqueue_finish(self.db, self.batch["id"]) + self.assertEqual((started["status"], started["run"]["purpose"]), ("running", "acceptance")) + self.assertEqual(self.db.get(BizMigrationBatch, self.batch["id"]).status, "active") + self.run_background() + self.db.expire_all() + batch = self.db.get(BizMigrationBatch, self.batch["id"]) + self.assertEqual(batch.status, "review") + self.assertEqual(batch.accept_run_id, started["run_id"]) + self.assertEqual(self.db.get(BizMigrationRun, started["run_id"]).status, "success") + + def test_finish_failure_keeps_the_wave_operable(self): + with mock.patch.object(mig.threading, "Thread", _Inline): + started = mig.enqueue_finish(self.db, self.batch["id"]) + with mock.patch.object(mig, "run_evaluate", side_effect=HTTPException(status_code=400, detail="x")): + self.run_background() + self.db.expire_all() + self.assertEqual(self.db.get(BizMigrationBatch, self.batch["id"]).status, "active") + self.assertEqual(self.db.get(BizMigrationRun, started["run_id"]).status, "failed") + + def test_auto_monitor_stays_out_of_a_running_evaluation(self): + from netx_api.biz_migration import auto_monitor + + with mock.patch.object(mig.threading, "Thread", _Inline): + mig.enqueue_evaluate(self.db, self.batch["id"]) + with mock.patch.object(mig, "run_evaluate") as run_evaluate: + auto_monitor.try_auto_monitor_for_task(self.db, self.ctx.old_p.id) + run_evaluate.assert_not_called() + + +class ManualRouteAndDuplicateScopeTests(_Fixture): + """Share the fixture, add route sheets / duplicate rows to the compare template.""" + + def _add_sheet(self, sheet): + self.ctx.ct.metrics_json = [*self.ctx.ct.metrics_json, sheet] + self.ctx.mt.collect_metric_ids_json = [] + self.db.commit() + + def test_uncollected_route_table_is_manual_pending_and_does_not_fail_acceptance(self): + self._add_sheet({"sheet_id": "bgp_route", "title": "routes", "metric_id": "bgp_route", + "key_fields": ["network"], "iface_fields": [], "compare_fields": [], "row_filters": [], + "field_rules": []}) + out = mig.run_evaluate(self.db, batch_id=self.batch["id"], acceptance=True, purpose="acceptance") + card = next(c for c in out["summary"]["sheet_cards"] if c["metric_id"] == "bgp_route") + self.assertTrue(card["manual_pending"]) + self.assertFalse(card.get("current_missing")) + self.assertIn("routes", out["summary"]["manual_pending_metrics"]) + self.assertNotIn("routes", out["summary"]["current_missing_metrics"]) + + def test_duplicate_keys_outside_the_wave_are_warnings_not_failures(self): + c = self.ctx + for i, cmd in enumerate(("d1", "d2")): + self.db.add(BizStateMetricRow(id=f"dup-{i}", batch_id="bl_old", metric_id="interface_brief", seq=10 + i, + data_json={"interface": "GE-other", "admin": "up", "phy": "up", "prot": "up"})) + self.db.commit() + out = mig.run_evaluate(self.db, batch_id=self.batch["id"], acceptance=True, purpose="acceptance") + summary = out["summary"] + self.assertEqual(summary["duplicate_keys"], 0) + self.assertEqual(summary["duplicate_warnings"][0]["count"] >= 1, True) + + def test_duplicate_keys_inside_the_wave_still_fail(self): + for i in range(2): + self.db.add(BizStateMetricRow(id=f"dup-{i}", batch_id="bl_old", metric_id="interface_brief", seq=10 + i, + data_json={"interface": "GE-old", "admin": "up", "phy": "up", "prot": "up"})) + self.db.commit() + out = mig.run_evaluate(self.db, batch_id=self.batch["id"], acceptance=True, purpose="acceptance") + self.assertGreaterEqual(out["summary"]["duplicate_keys"], 1) + self.assertFalse(out["summary"]["coverage_complete"]) + + +class BaselineCacheTests(_Fixture): + def test_second_evaluation_does_not_read_baseline_rows_again(self): + mig.BASELINE_ROWS.invalidate() + real = mig._load_metric_rows + loaded: list[str] = [] + + def spy(db, *, batch_id, metric_id, **kw): + loaded.append(batch_id) + return real(db, batch_id=batch_id, metric_id=metric_id, **kw) + + with mock.patch.object(mig, "_load_metric_rows", spy): + mig.run_evaluate(self.db, batch_id=self.batch["id"]) + first = list(loaded) + loaded.clear() + mig.run_evaluate(self.db, batch_id=self.batch["id"]) + self.assertIn("bl_old", first) + self.assertNotIn("bl_old", loaded) + self.assertNotIn("bl_new", loaded) + self.assertIn("cur_new", loaded) # current samples are always re-read + + def test_lru_is_bounded_by_total_rows_and_can_be_invalidated(self): + cache = mig.BaselineRowCache(max_rows=5) + rows = lambda n: [{"i": i} for i in range(n)] # noqa: E731 + cache.get_or_load("a", "m", lambda: rows(3)) + cache.get_or_load("b", "m", lambda: rows(3)) # evicts a (3 + 3 > 5) + calls: list[str] = [] + cache.get_or_load("a", "m", lambda: calls.append("a") or rows(3)) + self.assertEqual(calls, ["a"]) + cache.get_or_load("big", "m", lambda: rows(9)) # larger than the cap: never cached + cache.get_or_load("big", "m", lambda: calls.append("big") or rows(9)) + self.assertEqual(calls, ["a", "big"]) + cache.invalidate("a") + cache.get_or_load("a", "m", lambda: calls.append("a2") or rows(1)) + self.assertEqual(calls[-1], "a2") + + +class RedTicketResolvableTests(_Fixture): + def _diff(self, run_id, key, color, verdict): + from netx_api.models import BizMigrationDiff + + self.db.add(BizMigrationDiff(id=f"d-{run_id}-{key}", run_id=run_id, metric_id="interface_brief", color=color, + verdict=verdict, key_json={"match_old_key": key, "sheet_id": "interface_brief", + "key_str": key})) + + def test_ticket_for_a_key_that_turned_green_is_flagged_not_closed(self): + from netx_api.models import BizMigrationRedTicket + + pid = self.project_id + for key in ("gei-1", "gei-2"): + self.db.add(BizMigrationRedTicket( + id=f"t-{key}", project_id=pid, batch_id="old", run_id="r0", metric_id="interface_brief", + match_key_str=key, key_str=key, status="open", detail_json={"sheet_id": "interface_brief", "match_old_key": key})) + self.db.add(BizMigrationRun(id="r1", project_id=pid, batch_id=self.batch["id"], purpose="acceptance")) + self._diff("r1", "gei-1", "green", "migrated") + self._diff("r1", "gei-2", "red", "anomaly") + self.db.commit() + mig._persist_red_tickets_from_run(self.db, project_id=pid, batch_id=self.batch["id"], run_id="r1") + self.db.commit() + from netx_api.models import BizMigrationRedTicket as T + + flagged = {t.id: mig._red_ticket_to_dict(t) for t in self.db.query(T)} + self.assertTrue(flagged["t-gei-1"]["resolvable"]) + self.assertFalse(flagged["t-gei-2"]["resolvable"]) + self.assertEqual(flagged["t-gei-1"]["status"], "open") + + out = mig.resolve_resolvable_red_tickets(self.db, pid) + self.assertEqual(out["ids"], ["t-gei-1"]) + self.assertEqual(self.db.get(T, "t-gei-1").status, "resolved") + self.assertEqual(self.db.get(T, "t-gei-2").status, "open") + + +class UncoveredSliceTests(_Fixture): + def test_a_slice_one_side_never_collected_is_not_collected_not_lost(self): + sheet = {"sheet_id": "bgp_peer.ipv4", "metric_id": "bgp_peer", "key_fields": ["neighbor"], + "iface_fields": [], "compare_fields": ["state"], + "row_filters": [{"op": "eq", "field": "afi", "value": "ipv4"}], "field_rules": []} + for batch, profile in (("cur_old", "zte.bgp_ipv4_summary"), ("cur_new", "zte.bgp_vpnv4_summary")): + self.db.add(BizStateBatchCommand(id=f"slice-{batch}", batch_id=batch, metric_id="bgp_peer", + profile_id=profile, parse_status="ok", row_count=0, raw_line_count=0)) + self.db.commit() + self.assertEqual(mig._uncovered_slice_sides(self.db, sheet, "bgp_peer", "cur_old", "cur_new"), ["new"]) + self.assertEqual(mig._uncovered_slice_sides(self.db, {**sheet, "row_filters": []}, "bgp_peer", + "cur_old", "cur_new"), []) diff --git a/tests/test_biz_migration_evaluate.py b/tests/test_biz_migration_evaluate.py index 0ac413c..99d650f 100644 --- a/tests/test_biz_migration_evaluate.py +++ b/tests/test_biz_migration_evaluate.py @@ -733,7 +733,7 @@ class EvaluateMetricDualTests(unittest.TestCase): migrated = [r for r in out["rows"] if r["verdict"] == "migrated"] self.assertTrue(migrated, out["rows"]) - def test_out_of_expect_ignore_no_red(self): + def test_out_of_expect_ignore_cannot_hide_business_drift(self): old_base = [{"interface": "gei-keep", "admin": "up", "phy": "up", "prot": "up"}] old_cur: list[dict] = [] new_cur = [{"interface": "gei-keep", "admin": "up", "phy": "up", "prot": "up"}] @@ -753,9 +753,9 @@ class EvaluateMetricDualTests(unittest.TestCase): out_of_expect="ignore", ) reds = [r for r in out["rows"] if r["color"] == "red"] - self.assertFalse(reds, out["rows"]) + self.assertTrue(reds, out["rows"]) involved = [r for r in out["rows"] if r["verdict"] != "not_involved"] - self.assertFalse(involved, out["rows"]) + self.assertTrue(involved, out["rows"]) def test_unexpected_loss_is_anomaly(self): old_base = [{"interface": "gei-keep", "admin": "up", "phy": "up", "prot": "up"}] @@ -823,7 +823,7 @@ class EvaluateMetricDualTests(unittest.TestCase): self.assertFalse(out["new_baseline_missing"]) self.assertEqual(out["new_baseline_mode"], "port_mapped") - def test_empty_provided_new_baseline_falls_back_or_missing(self): + def test_empty_collected_new_baseline_is_valid(self): old_base = [{"peer": "1.1.1.1", "state": "Established"}] new_cur = [{"peer": "1.1.1.1", "state": "Established"}] expect = parse_expect_set({"items": [{"metric_id": "bgp_peer", "key": "1.1.1.1"}]}) @@ -841,8 +841,8 @@ class EvaluateMetricDualTests(unittest.TestCase): window_active=True, sheet_override={}, ) - self.assertTrue(out["new_baseline_missing"]) - self.assertEqual(out["new_baseline_mode"], "empty") + self.assertFalse(out["new_baseline_missing"]) + self.assertEqual(out["new_baseline_mode"], "provided") # Still reports presence as added vs empty before — callers should heed the flag. kinds = {r["new_kind"] for r in out["rows"] if r.get("in_expect")} self.assertIn("added", kinds) @@ -1153,7 +1153,10 @@ class PortMapScopeTests(unittest.TestCase): ) keys = {r["key_str"] for r in out["rows"]} self.assertIn("10.0.0.1|gei-old", keys) - self.assertNotIn("10.0.0.2|gei-keep", keys) + self.assertIn("10.0.0.2|gei-keep", keys) + outside = next(r for r in out["rows"] if r["key_str"] == "10.0.0.2|gei-keep") + self.assertEqual(outside["color"], "red") + self.assertFalse(outside["in_expect"]) migrated = [r for r in out["rows"] if r["verdict"] == "migrated"] self.assertTrue(migrated, out["rows"]) self.assertEqual(migrated[0]["new_key_str"], "10.0.0.1|gei-new") diff --git a/tests/test_biz_migration_evidence.py b/tests/test_biz_migration_evidence.py index e22a259..8038f4e 100644 --- a/tests/test_biz_migration_evidence.py +++ b/tests/test_biz_migration_evidence.py @@ -138,10 +138,13 @@ class EvidencePersistTests(unittest.TestCase): return t def _mk_batch(self, bid: str, task_id: str) -> BizStateBatch: + task = self.db.get(BizStateTask, task_id) now = _utcnow() b = BizStateBatch( id=bid, task_id=task_id, + collect_group_id=task.collect_group_id or "", + collect_round_id="test-" + task.collect_group_id if task.collect_group_id else "", status="success", started_at=now, ended_at=now, @@ -149,6 +152,8 @@ class EvidencePersistTests(unittest.TestCase): command_count=1, ) self.db.add(b) + self.db.add(BizStateBatchCommand(id=f"receipt-{bid}", batch_id=bid, + metric_id="interface_brief", parse_status="ok", row_count=0)) self.db.commit() return b @@ -233,7 +238,7 @@ class EvidencePersistTests(unittest.TestCase): raw_command="show interface brief", interface="GE-old", ) - # New baseline empty → port_mapped mode uses mapped old + # Successfully collected empty new baseline is valid # Old current: gone (no rows) — migration off old # New current: xgei-new up self._seed_if_row( diff --git a/tests/test_biz_migration_hf_decouple.py b/tests/test_biz_migration_hf_decouple.py index 4fc3c53..8e34c77 100644 --- a/tests/test_biz_migration_hf_decouple.py +++ b/tests/test_biz_migration_hf_decouple.py @@ -14,6 +14,7 @@ from netx_api.models import ( BizMigrationProject, BizMonitorTemplate, BizStateBatch, + BizStateBatchCommand, BizStateTask, ) from netx_api.biz_migration import service as mig @@ -46,6 +47,7 @@ class HfDecoupleFlowTests(unittest.TestCase): ) Base.metadata.create_all(bind=engine) self.db = TestingSession() + self.Session = TestingSession self.mt = BizMonitorTemplate( id="mt1", @@ -93,16 +95,22 @@ class HfDecoupleFlowTests(unittest.TestCase): return t def _mk_batch(self, bid: str, task_id: str) -> BizStateBatch: + task = self.db.get(BizStateTask, task_id) now = datetime.utcnow() b = BizStateBatch( id=bid, task_id=task_id, + collect_group_id=task.collect_group_id or "", + collect_round_id="test-" + task.collect_group_id if task.collect_group_id else "", status="success", started_at=now, ended_at=now, row_count=1, ) self.db.add(b) + for mid in ("interface_brief", "arp"): + self.db.add(BizStateBatchCommand(id=f"receipt-{bid}-{mid}", batch_id=bid, + metric_id=mid, parse_status="ok", row_count=0)) self.db.commit() return b @@ -136,6 +144,82 @@ class HfDecoupleFlowTests(unittest.TestCase): self.assertNotEqual(out["old_hf_task_id"], "old_p") self.assertNotEqual(out["new_hf_task_id"], "new_p") + def test_same_interval_light_metrics_share_one_paired_task(self) -> None: + out = self._create_project(collect_metric_ids=["interface_brief", "arp"]) + old, new = out["old_hf_bindings"], out["new_hf_bindings"] + self.assertEqual((len(old), len(new)), (1, 1)) + self.assertEqual(old[0]["metric_ids"], ["interface_brief", "arp"]) + self.assertEqual(old[0]["interval_sec"], 60) + self.assertTrue(old[0]["collect_group_id"]) + self.assertEqual(old[0]["collect_group_id"], new[0]["collect_group_id"]) + from netx_api.models import BizStateTaskItem + + items = self.db.query(BizStateTaskItem).filter(BizStateTaskItem.task_id == old[0]["task_id"]).count() + self.assertEqual(items, 2) # one SSH login runs both commands + + def test_different_interval_or_lane_or_manual_route_stay_apart(self) -> None: + out = self._create_project(collect_metric_ids=["interface_brief", "arp", "interface_detail", "ip_route"], + metric_interval_sec={"arp": 300}) + by_metrics = {tuple(b["metric_ids"]): b for b in out["old_hf_bindings"]} + self.assertEqual(set(by_metrics), {("interface_brief",), ("arp",), ("interface_detail",), ("ip_route",)}) + self.assertEqual(by_metrics[("arp",)]["interval_sec"], 300) + self.assertTrue(by_metrics[("ip_route",)]["manual_only"]) + self.assertEqual(len({b["collect_group_id"] for b in out["old_hf_bindings"]}), 4) + + def test_pair_lookup_works_for_a_metric_inside_a_merged_task(self) -> None: + out = self._create_project(collect_metric_ids=["interface_brief", "arp"]) + proj = self.db.get(BizMigrationProject, out["id"]) + self.assertEqual(mig._hf_task_id_for_metric(proj, "old", "arp"), out["old_hf_bindings"][0]["task_id"]) + self.assertEqual(mig._hf_task_id_for_metric(proj, "new", "interface_brief"), + out["new_hf_bindings"][0]["task_id"]) + + def test_dynamic_routes_manual_only_and_removal_cancels_queued_pairs(self) -> None: + from netx_api.models import BizStateTaskItem, BizStateTaskItemBinding + + for tid in ("old_p", "new_p"): + self.db.add(BizStateTaskItem(id="route-item-" + tid, task_id=tid, source_profile_id="zte.ip_route_vrf", kind="catalog", enabled=True)) + self.db.add(BizStateTaskItemBinding(id="route-bind-" + tid, item_id="route-item-" + tid, placeholder="vrf", value="customer-A")) + self.db.commit() + out = self._create_project() + with mock.patch.object(biz_svc, "_ne_meta", side_effect=_stub_ne_meta): + updated = mig.patch_project(self.db, out["id"], {"collect_metric_ids": ["interface_brief", "ip_route"]}) + routes = [b for side in ("old", "new") for b in updated[f"{side}_hf_bindings"] if "ip_route" in b["metric_ids"]] + self.assertEqual(len(routes), 2) + for idx, binding in enumerate(routes): + task = self.db.get(BizStateTask, binding["task_id"]) + self.assertTrue(task.collect_manual_only) + self.assertEqual(task.status, "paused") + task.collect_running = True + self.db.add(BizStateBatch(id=f"queued-route-{idx}", task_id=task.id, status="queued", collect_round_id="route-round")) + self.db.commit() + with mock.patch("netx_api.biz_state.collect_stop.SessionLocal", self.Session), mock.patch.object(biz_svc, "_ne_meta", side_effect=_stub_ne_meta): + shrunk = mig.patch_project(self.db, out["id"], {"collect_metric_ids": ["interface_brief"]}) + self.db.expire_all() + self.assertEqual(len(shrunk["old_hf_bindings"]), 1) + for idx in range(2): + self.assertEqual(self.db.get(BizStateBatch, f"queued-route-{idx}").status, "cancelled") + + def test_route_profile_preserves_portrait_cli_variant_and_bindings(self) -> None: + from netx_api.models import BizStateTaskItem, BizStateTaskItemBinding + + self.db.add(BizStateTaskItem(id="vrf-item", task_id="old_p", source_profile_id="zte.ip_route_vrf", kind="catalog", enabled=True)) + self.db.add(BizStateTaskItemBinding(id="vrf-bind", item_id="vrf-item", placeholder="vrf", value="customer-A")) + self.db.commit() + out = self._create_project(collect_metric_ids=["interface_brief", "ip_route"]) + binding = next(b for b in out["old_hf_bindings"] if b["metric_ids"] == ["ip_route"]) + item = self.db.query(BizStateTaskItem).filter(BizStateTaskItem.task_id == binding["task_id"]).one() + self.assertEqual(item.source_profile_id, "zte.ip_route_vrf") + copied = self.db.query(BizStateTaskItemBinding).filter(BizStateTaskItemBinding.item_id == item.id).one() + self.assertEqual((copied.placeholder, copied.value), ("vrf", "customer-A")) + + def test_invalid_new_metric_does_not_save_broken_selection(self) -> None: + out = self._create_project() + with self.assertRaises(Exception): + mig.patch_project(self.db, out["id"], {"collect_metric_ids": ["interface_brief", "not-a-metric"]}) + self.db.rollback() + current = self.db.get(BizMigrationProject, out["id"]) + self.assertEqual(mig.resolve_collect_metric_ids(self.db, current), ["interface_brief"]) + def test_ensure_highfreq_does_not_overwrite_portrait(self) -> None: proj = self._create_project() res = self._ensure(proj["id"], interval_sec=60) @@ -170,7 +254,7 @@ class HfDecoupleFlowTests(unittest.TestCase): batch = mig.create_batch( self.db, pid, {"batch_label": "n1", "expect_set": {"ports": ["gei-0/1"]}} ) - with mock.patch.object(mig, "resolve_evaluate_sheets", return_value=([], [], {}, [])): + with mock.patch.object(mig, "resolve_evaluate_sheets", return_value=([mig.port_sheet_def()], [], {}, [])): out = mig.run_evaluate(self.db, batch_id=batch["id"]) self.assertEqual(out["old_batch_id"], "cur_old") self.assertEqual(out["new_batch_id"], "cur_new") @@ -209,7 +293,7 @@ class HfDecoupleFlowTests(unittest.TestCase): called = {c.args[0] for c in disp.call_args_list} self.assertEqual( called, - {proj["old_hf_task_id"], proj["new_hf_task_id"]}, + {proj["old_hf_task_id"]}, # one dispatch atomically enqueues the pair ) self.assertNotIn("old_p", called) @@ -497,3 +581,77 @@ class HfDecoupleFlowTests(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +class SliceUnionAndSameDeviceTests(HfDecoupleFlowTests): + """V2 (both devices collect the same slices) and H7 (same device on both sides).""" + + def _portrait_items(self, task_id, *profile_ids): + from netx_api.models import BizStateTaskItem + + for i, pid in enumerate(profile_ids): + self.db.add(BizStateTaskItem(id=f"{task_id}-{i}", task_id=task_id, source_profile_id=pid, + kind="catalog", enabled=True, sort_order=i)) + self.db.commit() + + def _hf_profiles(self, task_id): + from netx_api.models import BizStateTaskItem + + return sorted(r.source_profile_id for r in + self.db.query(BizStateTaskItem).filter(BizStateTaskItem.task_id == task_id)) + + def test_union_gives_the_device_that_lacks_a_slice_this_vendors_profile(self) -> None: + old = [{"source_profile_id": "zte.bgp_ipv4_summary", "bindings": []}, + {"source_profile_id": "zte.bgp_vpnv4_summary", "bindings": []}] + new = [{"source_profile_id": "zte.bgp_vpnv4_summary", "bindings": []}] + old2, new2 = mig._union_slice_items("bgp_peer", old, new, "zte", "zte") + self.assertEqual({i["source_profile_id"] for i in new2}, {"zte.bgp_ipv4_summary", "zte.bgp_vpnv4_summary"}) + self.assertEqual(old2, old) + + def test_missing_vendor_profile_is_a_clear_error_not_a_silent_gap(self) -> None: + old = [{"source_profile_id": "zte.bgp_ipv4_summary", "bindings": []}] + new = [{"source_profile_id": "zte.bgp_vpnv4_summary", "bindings": []}] + with self.assertRaises(Exception) as ctx: + mig._union_slice_items("bgp_peer", old, new, "zte", "huawei") + self.assertEqual(ctx.exception.detail, "no_profile_for_slice:bgp_peer:afi=ipv4:huawei") + + def test_wholemetric_profiles_need_no_union(self) -> None: + item = [{"source_profile_id": "zte.interface_brief", "bindings": []}] + self.assertEqual(mig._union_slice_items("interface_brief", item, [], "zte", "zte"), (item, [])) + + def test_project_hf_tasks_collect_the_union_of_both_portraits(self) -> None: + self._portrait_items("old_p", "zte.bgp_ipv4_summary", "zte.bgp_vpnv4_summary") + self._portrait_items("new_p", "zte.bgp_vpnv4_summary") + out = self._create_project(collect_metric_ids=["bgp_peer"]) + want = ["zte.bgp_ipv4_summary", "zte.bgp_vpnv4_summary"] + self.assertEqual(self._hf_profiles(out["old_hf_bindings"][0]["task_id"]), want) + self.assertEqual(self._hf_profiles(out["new_hf_bindings"][0]["task_id"]), want) + + def test_same_device_project_uses_one_task_for_both_sides(self) -> None: + self.new_portrait.ne_id = self.old_portrait.ne_id + self.db.commit() + out = self._create_project(collect_metric_ids=["interface_brief", "arp"]) + old, new = out["old_hf_bindings"], out["new_hf_bindings"] + self.assertEqual(len(old), 1) + self.assertEqual(old[0]["task_id"], new[0]["task_id"]) + self.assertEqual(old[0]["collect_group_id"], new[0]["collect_group_id"]) + self.assertEqual(self.db.query(BizStateTask).filter(BizStateTask.purpose == mig.PURPOSE_CUTOVER_HF).count(), 1) + + def test_same_device_sample_is_that_devices_latest_finished_round(self) -> None: + self.new_portrait.ne_id = self.old_portrait.ne_id + self.db.commit() + out = self._create_project(collect_metric_ids=["interface_brief"]) + proj = self.db.get(BizMigrationProject, out["id"]) + tid = out["old_hf_bindings"][0]["task_id"] + self.assertEqual(mig._current_pair_for_metric(self.db, proj, "interface_brief")[2]["sample_issues"], + ["collect_pair_missing"]) + batch = self._mk_batch("solo-1", tid) + old, new, meta = mig._current_pair_for_metric(self.db, proj, "interface_brief") + self.assertIs(old, new) + self.assertEqual(old.id, batch.id) + self.assertEqual(meta["old_collect_status"], "success") + + def test_two_devices_are_still_rejected_when_they_are_in_fact_one(self) -> None: + # different NE ids but resolving to one task id must keep failing loudly + out = self._create_project(collect_metric_ids=["interface_brief"]) + self.assertNotEqual(out["old_hf_bindings"][0]["task_id"], out["new_hf_bindings"][0]["task_id"]) diff --git a/tests/test_biz_migration_integrity.py b/tests/test_biz_migration_integrity.py new file mode 100644 index 0000000..079b8e8 --- /dev/null +++ b/tests/test_biz_migration_integrity.py @@ -0,0 +1,426 @@ +"""Business continuity, wave scope, sample uncertainty and live-monitor regressions.""" +from datetime import timedelta + +import pytest +from fastapi import HTTPException +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker + +from netx_api.db import Base +from netx_api.models import (BizCompareTemplate, BizMonitorTemplate, BizMigrationProject, + BizMigrationBatch, BizMigrationRun, BizMigrationDiff, BizStateTask, BizStateBatch, + BizStateBatchCommand, BizStateMetricRow) +from netx_api.timeutil import utcnow_naive +from netx_api.biz_migration import service as svc +from netx_api.biz_migration.auto_monitor import try_auto_monitor_for_task +from netx_api.biz_migration.evaluate import evaluate_metric_dual, parse_expect_set, _eval_field_op +from netx_api.biz_migration.validation import validate_monitor_rules + + +def evaluate(**changes): + args = dict(metric_id="arp", sheet_id="arp", key_fields=["ip"], iface_fields=["port"], + compare_fields=["mac", "port"], old_baseline_rows=[{"ip": "a", "mac": "good", "port": "old"}], + old_current_rows=[], new_baseline_rows=[], + new_current_rows=[{"ip": "a", "mac": "good", "port": "new"}], + port_map={"old": "new"}, expect=parse_expect_set({"items": [{"metric_id": "arp", "key": "a"}]}), + window_active=True, allow_mapping_scope=False) + return evaluate_metric_dual(**{**args, **changes}) + + +def test_target_content_checked_against_old_business(): + out = evaluate(new_current_rows=[{"ip": "a", "mac": "wrong", "port": "new"}]) + assert out["progress_ok"] == 0 + assert out["rows"][0]["rule_hit"] == "service_content_changed" + assert out["rows"][0]["integrity_changes"][0]["field"] == "mac" + + +def test_content_normalization_and_ignore_follow_compare_template(): + out = evaluate(new_current_rows=[{"ip": "a", "mac": "GOOD", "port": "new"}], + field_rules=[{"field": "mac", "normalize": "upper"}]) + assert out["progress_ok"] == 1 + + +def test_down_target_cannot_pass_permissive_success_rule(): + out = evaluate(new_current_rows=[{"ip": "a", "state": "down"}], + sheet_override={"status_fields": ["state"], "up_values": ["up"], "down_values": ["down"], + "success": [{"old": ["removed"], "new": ["added"]}]}) + assert out["rows"][0]["verdict"] == "lost" + assert out["rows"][0]["color"] == "red" + + +def test_mapping_does_not_expand_wave_selection(): + out = evaluate(old_baseline_rows=[{"ip": "a", "port": "old"}, {"ip": "b", "port": "old2"}], + new_current_rows=[{"ip": "a", "port": "new"}], compare_fields=[], + port_map={"old": "new", "old2": "new2"}) + assert out["progress_total"] == 1 + assert next(r for r in out["rows"] if r["old_key"] == "b")["color"] == "red" + + +def test_port_selection_covers_business_with_nonkey_interface(): + out = evaluate(expect=parse_expect_set({"ports": ["old"]})) + assert out["progress_total"] == out["progress_ok"] == 1 + + +@pytest.mark.parametrize("healthy", [True, False]) +def test_previous_wave_remains_protected_without_changing_current_progress(healthy): + out = evaluate(expect={}, previous_expect=parse_expect_set({"items": [{"metric_id": "arp", "key": "a"}]}), + new_current_rows=[{"ip": "a", "mac": "good", "port": "new"}] if healthy else []) + assert out["progress_total"] == 0 + assert out["rows"][0]["verdict"] == ("migrated_previous" if healthy else "lost") + assert out["anomaly"] == (0 if healthy else 1) + + +@pytest.mark.parametrize("value", [0, False]) +def test_zero_false_are_not_blank_in_keys_or_conditions(value): + assert _eval_field_op({"id": value}, "id", "eq", value) + out = evaluate(old_baseline_rows=[{"ip": value}], new_current_rows=[{"ip": value}], + compare_fields=[], expect=parse_expect_set({"items": [{"metric_id": "arp", "key": [str(value).lower()]}]})) + assert out["rows"][0]["old_key"] == str(value).lower() + + +@pytest.fixture +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + with sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)() as session: + now = utcnow_naive() - timedelta(seconds=1) + sheet = {"metric_id": "arp", "sheet_id": "s", "key_fields": ["ip"], "compare_fields": ["mac"]} + session.add(BizCompareTemplate(id="ct", name="cmp", metrics_json=[sheet])) + session.add(BizMonitorTemplate(id="mt", name="mon", compare_template_id="ct", + collect_metric_ids_json=["arp"], defaults_json={}, sheet_overrides_json=[])) + for tid in ("op", "np", "oh", "nh"): + session.add(BizStateTask(id=tid, ne_id=tid, ne_name=tid, purpose="cutover_hf" if tid.endswith("h") else "")) + session.add(BizMigrationProject(id="p", name="p", old_task_id="op", new_task_id="np", + old_hf_task_id="oh", new_hf_task_id="nh", old_baseline_batch_id="ob", new_baseline_batch_id="nb", + monitor_template_id="mt", hf_interval_sec=60, status="active")) + session.add(BizMigrationBatch(id="wave", project_id="p", batch_label="wave", status="active", + expect_set_json={"items": [{"metric_id": "arp", "sheet_id": "s", "key": "a"}]})) + for i, (bid, tid) in enumerate((("ob", "op"), ("nb", "np"), ("oc", "oh"), ("nc", "nh"))): + at = now - timedelta(milliseconds=100 * (4 - i)) + session.add(BizStateBatch(id=bid, task_id=tid, status="success", started_at=at, ended_at=at)) + session.add(BizStateBatchCommand(id="cmd-" + bid, batch_id=bid, metric_id="arp", parse_status="ok", raw_text="secret CLI payload")) + for bid in ("ob", "nc"): + session.add(BizStateMetricRow(id="row-" + bid, batch_id=bid, task_id="op" if bid == "ob" else "nh", + batch_command_id="cmd-" + bid, metric_id="arp", seq=0, data_json={"ip": "a", "mac": "good"})) + session.commit() + yield session + + +def test_collected_empty_baseline_can_pass_acceptance(db): + finished = svc.finish_batch(db, "wave") + assert finished["accept_summary"]["passed"] + assert finished["run"]["summary"]["config_snapshot"]["expect_set"] + + +def paired_samples(db): + for tid in ("oh", "nh"): + db.get(BizStateTask, tid).collect_group_id = "pair" + for bid in ("oc", "nc"): + sample = db.get(BizStateBatch, bid) + sample.collect_group_id = "pair" + sample.collect_round_id = "round" + sample.queued_at = sample.started_at + db.commit() + + +@pytest.mark.parametrize("problem", ["running", "queued", "failed", "round_mismatch"]) +def test_paired_samples_cannot_mix_rounds_or_use_old_success(db, problem): + paired_samples(db) + if problem == "round_mismatch": + db.get(BizStateBatch, "nc").collect_round_id = "another-round" + else: + db.get(BizStateBatch, "nc").status = problem + db.commit() + run = svc.run_evaluate(db, batch_id="wave") + assert not run["summary"]["coverage_complete"] + assert not run["summary"]["anomaly"] # incomplete pair is unknown, not service loss + + +def test_paired_samples_expose_duration_queue_and_real_start_skew(db): + paired_samples(db) + for bid in ("oc", "nc"): + sample = db.get(BizStateBatch, bid) + sample.queued_at = sample.started_at - timedelta(seconds=15) + sample.started_at -= timedelta(seconds=5) + db.commit() + card = svc.run_evaluate(db, batch_id="wave")["summary"]["sheet_cards"][0] + assert card["collect_round_id"] == "round" + assert card["old_queue_sec"] == 10 and card["old_duration_sec"] == 5 + assert card["start_skew_sec"] < 1 + + +def test_unselected_route_does_not_block_regular_acceptance(db): + ct = db.get(BizCompareTemplate, "ct") + ct.metrics_json = [*ct.metrics_json, {"metric_id": "ip_route", "sheet_id": "route", "key_fields": ["prefix"]}] + db.commit() + assert svc.finish_batch(db, "wave")["accept_summary"]["passed"] + + +def test_incremental_evaluation_reuses_route_rows_and_paged_evidence(db): + from unittest.mock import patch + ct = db.get(BizCompareTemplate, "ct") + ct.metrics_json = [*ct.metrics_json, {"metric_id": "ip_route", "sheet_id": "route", "key_fields": ["prefix"], "compare_fields": ["nexthop"]}] + db.get(BizMonitorTemplate, "mt").collect_metric_ids_json = ["arp", "ip_route"] + # Legacy combined samples also exercise compatibility with existing projects. + for bid in ("ob", "nb", "oc", "nc"): + db.add(BizStateBatchCommand(id="route-cmd-" + bid, batch_id=bid, metric_id="ip_route", parse_status="ok")) + for i in range(2000): + db.add(BizStateMetricRow(id=f"route-{i}", batch_id="ob", metric_id="ip_route", data_json={"prefix": str(i), "nexthop": "old"})) + db.commit() + first = svc.run_evaluate(db, batch_id="wave", purpose="auto") + with patch.object(svc, "_load_metric_rows", wraps=svc._load_metric_rows) as load: + second = svc.run_evaluate(db, batch_id="wave", purpose="auto", _metric_ids={"arp"}) + assert all(call.kwargs["metric_id"] == "arp" for call in load.call_args_list) + assert svc.list_run_diffs(db, second["id"], sheet_id="route", color="red", limit=100)["total"] == 2000 + assert db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id == second["id"]).count() == 1 + assert second["summary"]["anomaly"] == first["summary"]["anomaly"] + assert svc.list_run_diffs(db, second["id"], only_expect=True)["total"] == 1 + # Template correction invalidates cached verdicts and evidence. + ct.metrics_json = [ct.metrics_json[0], {**ct.metrics_json[1], "row_filters": [{"field": "prefix", "op": "eq", "value": "1"}]}] + db.commit() + third = svc.run_evaluate(db, batch_id="wave", purpose="auto", _metric_ids={"arp"}) + assert svc.list_run_diffs(db, third["id"], sheet_id="route")["total"] == 1 + + +def test_baseline_picker_never_loads_route_inventory(db): + from unittest.mock import patch + ct = db.get(BizCompareTemplate, "ct") + ct.metrics_json = [*ct.metrics_json, {"metric_id": "ip_route", "sheet_id": "route", "key_fields": ["prefix"]}] + db.get(BizMonitorTemplate, "mt").collect_metric_ids_json = ["arp", "ip_route"] + db.commit() + with patch.object(svc, "_load_metric_rows", wraps=svc._load_metric_rows) as load: + sheets = svc.list_baseline_expect_objects(db, "p")["sheets"] + assert {s["metric_id"] for s in sheets} == {"arp"} + assert all(c.kwargs["metric_id"] == "arp" for c in load.call_args_list) + + +def test_legacy_auto_monitor_refreshes_changed_samples_without_explicit_metric_bindings(db): + assert try_auto_monitor_for_task(db, "nh") == 1 + db.get(BizStateBatch, "nc").status = "failed" + db.commit() + assert try_auto_monitor_for_task(db, "nh") == 1 + last = db.query(BizMigrationRun).order_by(BizMigrationRun.created_at.desc()).first() + assert not last.summary_json["coverage_complete"] + + +@pytest.mark.parametrize("problem", ["missing_receipt", "failed_receipt", "stale", "skew", "duplicate", "skipped", "unknown_scope"]) +def test_uncertain_coverage_cannot_pass_acceptance(db, problem): + if problem == "missing_receipt": + db.delete(db.get(BizStateBatchCommand, "cmd-nb")) + elif problem == "failed_receipt": + db.get(BizStateBatchCommand, "cmd-nc").parse_status = "failed" + elif problem in ("stale", "skew"): + batch = db.get(BizStateBatch, "nc") + batch.started_at = batch.ended_at = utcnow_naive() - timedelta(minutes=10) + elif problem == "duplicate": + db.add(BizStateMetricRow(id="dup", batch_id="nc", metric_id="arp", data_json={"ip": "a", "mac": "good"})) + elif problem == "skipped": + db.get(BizMigrationProject, "p").collect_metric_ids_json = ["other"] + else: + db.get(BizMigrationBatch, "wave").expect_set_json = {"items": [{"sheet_id": "deleted", "metric_id": "arp", "key": "a"}]} + db.commit() + assert not svc.finish_batch(db, "wave")["accept_summary"]["passed"] + + +def test_invalid_baseline_and_pinned_batch_do_not_fallback(db): + with pytest.raises(HTTPException) as error: + svc.run_evaluate(db, batch_id="wave", old_batch_id="nb") + assert error.value.detail == "old_current_batch_task_mismatch" + db.get(BizMigrationProject, "p").old_baseline_batch_id = "nc" + db.commit() + with pytest.raises(HTTPException): + svc.finish_batch(db, "wave") + assert db.get(BizMigrationBatch, "wave").status == "active" + assert db.query(BizMigrationRun).count() == 0 + + +def test_latest_failed_collection_does_not_reuse_previous_success(db): + db.add(BizStateBatch(id="latest-failed", task_id="nh", status="failed", started_at=utcnow_naive(), ended_at=utcnow_naive())) + db.commit() + run = svc.run_evaluate(db, batch_id="wave") + assert run["summary"]["current_missing_metrics"] == ["s"] + assert not run["summary"]["coverage_complete"] + + +def test_target_older_than_source_is_uncertain_instead_of_false_loss(db): + db.delete(db.get(BizStateMetricRow, "row-nc")) + db.get(BizStateBatch, "oc").started_at = utcnow_naive() + db.get(BizStateBatch, "oc").ended_at = utcnow_naive() + db.commit() + run = svc.run_evaluate(db, batch_id="wave") + row = svc.list_run_diffs(db, run["id"])["items"][0] + assert row["verdict"] == "awaiting_peer" + assert run["summary"]["anomaly"] == 0 + assert not run["summary"]["coverage_complete"] + + +def test_auto_monitor_deduplicates_receipts_and_respects_closed_window(db): + assert try_auto_monitor_for_task(db, "oh") == 1 + assert try_auto_monitor_for_task(db, "nh") == 0 + assert db.query(BizMigrationRun).one().purpose == "auto" + db.get(BizMigrationBatch, "wave").status = "done" + db.commit() + assert try_auto_monitor_for_task(db, "oh") == 0 + + +def test_two_active_waves_are_rejected(db): + with pytest.raises(HTTPException) as error: + svc.create_batch(db, "p", {"status": "active", "batch_label": "overlap"}) + assert error.value.detail == "another_batch_active" + assert db.query(BizMigrationBatch).count() == 1 + + +def test_sheets_with_same_key_keep_separate_red_tickets(db): + run = svc.run_evaluate(db, batch_id="wave") + for sid in ("one", "two"): + db.add(BizMigrationDiff(id=sid, run_id=run["id"], metric_id="arp", color="red", + key_json={"sheet_id": sid, "match_old_key": "a"}, verdict="lost")) + db.flush() + tickets = svc._persist_red_tickets_from_run(db, project_id="p", batch_id="wave", run_id=run["id"]) + assert len(tickets) == 2 + assert {t.detail_json["sheet_id"] for t in tickets} == {"one", "two"} + + +def test_port_mapping_does_not_remap_business_key_with_same_text(): + out = evaluate(old_baseline_rows=[{"ip": "old", "port": "old"}], + new_current_rows=[{"ip": "old", "port": "new"}], compare_fields=["port"], + expect=parse_expect_set({"items": [{"metric_id": "arp", "key": "old"}]})) + assert out["progress_ok"] == 1 + + +def test_unmapped_same_name_target_is_not_used_as_mapped_target_evidence(): + out = evaluate(metric_id="interface_brief", key_fields=["port"], compare_fields=[], + old_baseline_rows=[{"port": "old"}], new_current_rows=[{"port": "old"}], + expect=parse_expect_set({"ports": ["old"]})) + row = next(r for r in out["rows"] if r["in_expect"]) + assert row["verdict"] == "lost" + assert row["new"] == {} + assert row["new_key"] == "" + assert row["match_new_key"] == "new" + + +def test_legacy_raw_interface_expect_uses_same_alias_normalization(): + out = evaluate(metric_id="interface_brief", key_fields=["port"], + old_baseline_rows=[{"port": "GE-1"}], new_current_rows=[{"port": "GE-2"}], + compare_fields=[], port_map={"gei-1": "gei-2"}, + expect=parse_expect_set({"ports": ["GE-1"]}), + iface_normalize_rules=[{"from": "GE", "to": "gei"}]) + assert out["progress_ok"] == 1 + + +def test_baseline_picker_matches_normalized_keys_but_displays_originals(db): + ct = db.get(BizCompareTemplate, "ct") + ct.metrics_json = [{"metric_id": "arp", "sheet_id": "s", "key_fields": ["ip", "port"], "iface_fields": ["port"]}] + ct.iface_normalize_json = [{"from": "GE", "to": "gei"}] + db.get(BizStateMetricRow, "row-ob").data_json = {"ip": 0, "port": "GE-1"} + db.commit() + item = svc.list_baseline_expect_objects(db, "p")["sheets"][0]["items"][0] + assert item["keys"] == ["0", "gei-1"] + assert item["label"] == "0|GE-1" + assert parse_expect_set({"items": [{"metric_id": "arp", "key": item["keys"]}]})["arp"] == {"0\x1fgei-1"} + + +def test_empty_composite_segments_are_preserved_and_false_is_not_blank(): + assert parse_expect_set({"items": [{"metric_id": "x", "key": [0, "", False]}]})["x"] == {"0\x1f\x1ffalse"} + + +def test_missing_status_field_cannot_match_negative_success_condition(): + assert not _eval_field_op({"ip": "a"}, "state", "ne", "down") + assert not _eval_field_op({"ip": "a"}, "state", "not_in", ["down"]) + assert _eval_field_op({"ip": "a"}, "state", "empty", None) + + +def test_stable_outside_inventory_is_counted_without_repeated_evidence_cards(): + row = {"ip": "a", "mac": "good", "port": "old"} + out = evaluate(expect={}, old_current_rows=[row], new_current_rows=[]) + assert out["rows"] == [] + assert out["steady_outside"] == 1 + assert out["anomaly"] == 0 + + +def test_removed_previous_wave_sheet_blocks_full_coverage_acceptance(db): + db.add(BizMigrationBatch(id="prior", project_id="p", status="done", + created_at=utcnow_naive() - timedelta(days=1), + expect_set_json={"items": [{"sheet_id": "removed-sheet", "metric_id": "arp", "key": "old-customer"}]})) + db.commit() + finished = svc.finish_batch(db, "wave") + assert not finished["accept_summary"]["passed"] + assert finished["run"]["summary"]["uncovered_scope"] == ["removed-sheet"] + + +def test_split_sheets_share_loaded_rows_and_command_metadata(db): + from unittest.mock import patch + + ct = db.get(BizCompareTemplate, "ct") + sheet = ct.metrics_json[0] + ct.metrics_json = [sheet, {**sheet, "sheet_id": "s2"}] + for i in range(500): + for bid in ("ob", "nc"): + db.add(BizStateMetricRow(id=f"bulk-{bid}-{i}", batch_id=bid, + batch_command_id="cmd-" + bid, metric_id="arp", data_json={"ip": f"bulk-{i}", "mac": "good"})) + db.commit() + with patch.object(svc, "_load_metric_rows", wraps=svc._load_metric_rows) as load, patch.object(svc, "_command_brief", wraps=svc._command_brief) as command: + run = svc.run_evaluate(db, batch_id="wave") + assert load.call_count == 4 # once for each (batch, metric), not once per sheet + assert command.call_count == 2 # baseline + current metadata, independent of row count + assert svc.list_run_diffs(db, run["id"], limit=1)["total"] == 1002 + + +def test_wave_acceptance_rolls_back_whole_transaction_on_ticket_failure(db): + from unittest.mock import patch + + with patch.object(svc, "_persist_red_tickets_from_run", side_effect=RuntimeError("write failed")): + with pytest.raises(RuntimeError): + svc.finish_batch(db, "wave") + db.rollback() + assert db.query(BizMigrationRun).count() == 0 + assert db.get(BizMigrationBatch, "wave").status == "active" + + +def test_command_metadata_is_cached_and_raw_cli_is_not_selected(db): + statements = [] + def capture(_conn, _cursor, statement, *_args): + statements.append(statement) + event.listen(db.bind, "before_cursor_execute", capture) + try: + run = svc.run_evaluate(db, batch_id="wave") + finally: + event.remove(db.bind, "before_cursor_execute", capture) + command_selects = [q for q in statements if q.startswith("SELECT") and "biz_state_batch_command" in q] + assert all("raw_text" not in q for q in command_selects) + assert run["summary"]["coverage_complete"] + + +def test_diff_filters_apply_before_pagination_and_escape_wildcards(db): + run = svc.run_evaluate(db, batch_id="wave") + db.add(BizMigrationDiff(id="outside", run_id=run["id"], metric_id="arp", seq=999, color="red", + key_json={"sheet_id": "outside"}, in_expect=False, search_text="a_100%")) + db.commit() + result = svc.list_run_diffs(db, run["id"], sheet_id="outside", color="red", limit=1) + assert result["total"] == 1 and result["items"][0]["id"] == "outside" + assert svc.list_run_diffs(db, run["id"], only_expect=True)["total"] == 1 + assert svc.list_run_diffs(db, run["id"], kw="_")["total"] == 1 + + +@pytest.mark.parametrize("condition", [{"type": "value", "field": "state", "op": "typo", "value": "up"}, {"type": "presence", "value": "typo"}]) +def test_invalid_correction_rules_fail_before_evaluation(condition): + with pytest.raises(HTTPException): + validate_monitor_rules({}, [{"metric_id": "arp", "success": [{"old_groups": [[condition]], "new": ["added"]}]}]) + + +@pytest.mark.parametrize("raw", [None, True, " v "]) +def test_odd_key_values_still_match_expected_key(raw): + """A6: diff-derived key strings must use the same form as the expect set.""" + from netx_api.biz_migration.monitor_templates import preset_override_for_metric + from netx_api.biz_state.compare_rules import scalar_text + + old = {"ip": "a", "vrf": raw, "mac": "m", "port": "old"} + new = {"ip": "a", "vrf": raw, "mac": "m", "port": "new"} + out = evaluate( + key_fields=["ip", "vrf"], compare_fields=["mac"], iface_fields=[], + old_baseline_rows=[old], old_current_rows=[], new_baseline_rows=[], new_current_rows=[new], + port_map={}, window_active=False, acceptance=True, + sheet_override=preset_override_for_metric("arp"), + expect=parse_expect_set({"items": [{"metric_id": "arp", "key": ["a", scalar_text(raw).strip()]}]}), + ) + assert [r["verdict"] for r in out["rows"]] == ["migrated"] diff --git a/tests/test_biz_migration_presets.py b/tests/test_biz_migration_presets.py new file mode 100644 index 0000000..15bfec3 --- /dev/null +++ b/tests/test_biz_migration_presets.py @@ -0,0 +1,156 @@ +"""Default monitor presets: port success condition and legacy upgrade (plan A4).""" +from copy import deepcopy + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_migration import monitor_templates as mt +from netx_api.biz_migration.evaluate import dual_verdict_ex +from netx_api.db import Base +from netx_api.models import BizCompareTemplate, BizMonitorTemplate + +NEW_UP = {"admin": "up", "phy": "up"} + +OLD_SIDES = { + "fibre_pulled_no_shutdown": ("changed", {"admin": "up", "phy": "down"}), + "shutdown_link_still_up": ("changed", {"admin": "down", "phy": "up"}), + "both_down": ("changed", {"admin": "down", "phy": "down"}), + "port_deleted": ("removed", {}), +} + + +def verdict(old_kind, old_row, new_row, *, acceptance, ov=None): + ov = ov or mt.preset_override_for_metric("interface_brief") + return dual_verdict_ex( + old_kind=old_kind, new_kind="changed", in_expect=True, + window_active=not acceptance, acceptance=acceptance, + old_status="none", new_status="up", + success_patterns=ov["success"], anomaly_patterns=ov["anomaly"], + old_row=old_row, new_row=new_row, + ) + + +@pytest.mark.parametrize("acceptance", [False, True], ids=["window", "acceptance"]) +@pytest.mark.parametrize("case", OLD_SIDES) +def test_old_port_out_of_service_with_new_up_is_migrated(case, acceptance): + kind, row = OLD_SIDES[case] + v, color, _hit = verdict(kind, row, NEW_UP, acceptance=acceptance) + assert (v, color) == ("migrated", "green") + + +@pytest.mark.parametrize("acceptance", [False, True], ids=["window", "acceptance"]) +def test_new_port_phy_down_is_never_green(acceptance): + v, color, _ = verdict("changed", {"admin": "down", "phy": "down"}, + {"admin": "up", "phy": "down"}, acceptance=acceptance) + assert color != "green" + + +def test_old_port_still_in_service_is_not_migrated(): + v, color, _ = verdict("unchanged", {"admin": "up", "phy": "up"}, NEW_UP, acceptance=True) + assert color != "green" + + +def test_legacy_form_still_requires_both_down(): + legacy = mt.preset_override_for_metric("interface_brief", legacy_port_and=True) + v, color, _ = verdict("changed", {"admin": "up", "phy": "down"}, NEW_UP, + acceptance=True, ov=legacy) + assert color != "green" # the pre-A4 bug this task fixes + + +@pytest.fixture() +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine, expire_on_commit=False)() + yield session + session.close() + engine.dispose() + + +def _template(db, overrides): + db.add(BizMonitorTemplate(id="m", name="m", compare_template_id="", sheet_overrides_json=overrides)) + db.commit() + + +def test_upgrade_replaces_exact_legacy_preset_only(db): + legacy = mt.preset_override_for_metric("interface_brief", legacy_port_and=True) + arp = mt.preset_override_for_metric("arp") + _template(db, [legacy, arp]) + assert mt.upgrade_legacy_port_overrides(db) == 1 + stored = db.get(BizMonitorTemplate, "m").sheet_overrides_json + assert stored[0] == mt.preset_override_for_metric("interface_brief") + assert stored[1] == arp + assert mt.upgrade_legacy_port_overrides(db) == 0 # idempotent + + +def test_customised_override_is_kept_and_warned(db): + custom = mt.preset_override_for_metric("interface_brief", legacy_port_and=True) + custom["down_values"] = ["down", "lowerlayerdown"] + _template(db, [deepcopy(custom)]) + assert mt.upgrade_legacy_port_overrides(db) == 0 + assert db.get(BizMonitorTemplate, "m").sheet_overrides_json == [custom] + db.add(BizCompareTemplate(id="ct", name="ct", metrics_json=[ + {"sheet_id": "interface_brief", "metric_id": "interface_brief", + "key_fields": ["interface"], "compare_fields": ["admin"]}])) + db.commit() + warnings = mt.validate_sheet_overrides(db, compare_template_id="ct", sheet_overrides=[custom]) + assert {"sheet_id": "", "metric_id": "interface_brief", + "reason": "port_success_legacy_and_semantics"} in warnings + assert not any(w["reason"] == "port_success_legacy_and_semantics" for w in + mt.validate_sheet_overrides(db, compare_template_id="ct", sheet_overrides=[ + mt.preset_override_for_metric("interface_brief")])) + + +def test_key_tuple_is_the_single_canonical_key_form(): + from netx_api.biz_state.compare_engine import key_tuple + + assert key_tuple({"a": None, "b": True, "c": " x "}, ["a", "b", "c", "missing"]) == ("", "true", "x", "") + + +# ---- H8: scope_by_port_map is declared, not guessed from the metric name ---------------------- + +def _scoped(override): + from netx_api.biz_migration.evaluate import evaluate_metric_dual, parse_expect_set + + return evaluate_metric_dual( + metric_id="custom_t", key_fields=["ifname"], iface_fields=["ifname"], compare_fields=[], + old_baseline_rows=[{"ifname": "a"}], old_current_rows=[], new_baseline_rows=[], + new_current_rows=[{"ifname": "b"}], port_map={"a": "b"}, expect=parse_expect_set({}), + window_active=True, sheet_override=override, allow_mapping_scope=True, + ) + + +def test_a_port_map_scopes_an_interface_sheet_unless_the_override_says_otherwise(): + assert _scoped({})["progress_total"] == 1 + assert _scoped({"scope_by_port_map": True})["progress_total"] == 1 + out = _scoped({"scope_by_port_map": False}) + assert out["progress_total"] == 0 and out["effective_scope_by_port_map"] is False + + +def test_presets_turn_scoping_off_for_bgp_and_pseudowire_only(): + assert mt.preset_override_for_metric("bgp_peer")["scope_by_port_map"] is False + assert mt.preset_override_for_metric("l2vpn_pw")["scope_by_port_map"] is False + assert "scope_by_port_map" not in mt.preset_override_for_metric("interface_brief") + assert "scope_by_port_map" not in mt.preset_override_for_metric("arp") + + +def test_stored_overrides_get_the_old_name_based_default_written_down(db): + _template(db, [{"metric_id": "bgp_peer", "sheet_id": "bgp_peer.vpnv4"}, {"metric_id": "arp"}, + {"metric_id": "bgp_peer", "sheet_id": "x", "scope_by_port_map": True}]) + assert mt.fill_scope_by_port_map(db) == 1 + stored = db.get(BizMonitorTemplate, "m").sheet_overrides_json + assert stored[0]["scope_by_port_map"] is False + assert "scope_by_port_map" not in stored[1] + assert stored[2]["scope_by_port_map"] is True # an explicit choice is kept + assert mt.fill_scope_by_port_map(db) == 0 + + +def test_scope_flag_must_be_boolean(): + from fastapi import HTTPException + + from netx_api.biz_migration.validation import validate_monitor_rules + + with pytest.raises(HTTPException): + validate_monitor_rules({}, [{"metric_id": "arp", "scope_by_port_map": "no"}]) + validate_monitor_rules({}, [{"metric_id": "arp", "scope_by_port_map": False}]) diff --git a/tests/test_biz_migration_retention.py b/tests/test_biz_migration_retention.py new file mode 100644 index 0000000..f021f6f --- /dev/null +++ b/tests/test_biz_migration_retention.py @@ -0,0 +1,96 @@ +"""H1: bounded auto-run history and run-aware batch protection.""" +from datetime import timedelta + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_state.retention import protected_batch_map, purge_auto_runs +from netx_api.db import Base +from netx_api.models import ( + BizCompareDiff, BizCompareJob, BizCompareRun, BizMigrationBatch, BizMigrationDiff, BizMigrationRedTicket, + BizMigrationRun, BizStateBatch, BizStateTask, +) +from netx_api.timeutil import utcnow_naive + + +@pytest.fixture() +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine, expire_on_commit=False)() + yield session + session.close() + engine.dispose() + + +def _runs(db, n, *, purpose="auto", batch="wave", prefix="r"): + base = utcnow_naive() - timedelta(hours=10) + for i in range(n): + db.add(BizMigrationRun(id=f"{prefix}{i}", project_id="p", batch_id=batch, purpose=purpose, + old_batch_id=f"o-{prefix}{i}", new_batch_id=f"n-{prefix}{i}", + created_at=base + timedelta(minutes=i), summary_json={})) + db.add(BizMigrationDiff(id=f"d-{prefix}{i}", run_id=f"{prefix}{i}", metric_id="m")) + db.commit() + + +def test_only_the_newest_auto_runs_survive_and_acceptance_runs_stay(db): + _runs(db, 400) + _runs(db, 1, purpose="acceptance", prefix="acc") + out = purge_auto_runs(db, migration_keep=300) + assert out["migration_runs"] == 100 + assert db.query(BizMigrationRun).count() == 301 + assert db.get(BizMigrationRun, "acc0") is not None + assert db.get(BizMigrationRun, "r399") is not None and db.get(BizMigrationRun, "r99") is None + assert db.query(BizMigrationDiff).count() == 301 # diffs follow their run + + +def test_runs_pinned_by_red_tickets_or_acceptance_are_never_dropped(db): + _runs(db, 12) + db.add(BizMigrationRedTicket(id="t", project_id="p", batch_id="wave", run_id="r0")) + db.add(BizMigrationBatch(id="wave", project_id="p", accept_run_id="r1")) + db.commit() + purge_auto_runs(db, migration_keep=5) + assert db.get(BizMigrationRun, "r0") is not None and db.get(BizMigrationRun, "r1") is not None + assert db.get(BizMigrationRun, "r2") is None and db.query(BizMigrationRun).count() == 7 + + +def test_keep_is_per_wave(db): + _runs(db, 6, batch="w1", prefix="a") + _runs(db, 6, batch="w2", prefix="b") + assert purge_auto_runs(db, migration_keep=4)["migration_runs"] == 4 + assert db.query(BizMigrationRun).filter_by(batch_id="w1").count() == 4 + + +def test_compare_keeps_manual_runs_and_trims_auto_runs_of_auto_jobs(db): + db.add(BizCompareJob(id="job", name="j", mode="auto")) + base = utcnow_naive() - timedelta(hours=5) + for i in range(8): + db.add(BizCompareRun(id=f"c{i}", job_id="job", status="success", created_at=base + timedelta(minutes=i), + summary_json={"purpose": "auto" if i != 0 else "manual"})) + db.add(BizCompareDiff(id=f"cd{i}", run_id=f"c{i}", metric_id="m")) + db.commit() + out = purge_auto_runs(db, compare_keep=3) + assert out["compare_runs"] == 4 # c1..c4; c0 is manual and survives, c5..c7 are the newest + assert {r.id for r in db.query(BizCompareRun)} == {"c0", "c5", "c6", "c7"} + assert db.query(BizCompareDiff).count() == 4 + + +def _batch(db, bid, task="t"): + db.add(BizStateBatch(id=bid, task_id=task, status="success")) + + +def test_protection_follows_surviving_runs_and_all_metric_batches(db): + db.add(BizStateTask(id="t", source="managed", ne_id="n")) + for bid in ("o-r0", "n-r0", "hf-a", "hf-b", "o-r1", "n-r1", "o-acc0", "n-acc0"): + _batch(db, bid) + db.commit() + _runs(db, 2) + _runs(db, 1, purpose="acceptance", prefix="acc") + acc = db.get(BizMigrationRun, "acc0") + acc.summary_json = {"metric_batches": {"arp": {"old_batch_id": "hf-a", "new_batch_id": "hf-b"}}} + db.commit() + reasons = protected_batch_map(db) + assert "hf-a" in reasons and "hf-b" in reasons # beyond the first binding + purge_auto_runs(db, migration_keep=1) # r0 goes away + assert "o-r0" not in protected_batch_map(db) and "o-r1" in protected_batch_map(db) diff --git a/tests/test_biz_state_claim.py b/tests/test_biz_state_claim.py index 12beb4f..8224ac5 100644 --- a/tests/test_biz_state_claim.py +++ b/tests/test_biz_state_claim.py @@ -95,7 +95,8 @@ class BizStateClaimTests(unittest.TestCase): self.assertEqual(r2.get("reason"), "already_collecting") def test_claim_ne_mutex_skips_same_ne(self) -> None: - with patch.object(claim_mod, "max_concurrent_tasks", return_value=10): + # Session cap 1 reproduces the classic "one collection per NE" mutex. + with patch.object(claim_mod, "max_concurrent_tasks", return_value=10), patch.object(claim_mod, "max_sessions_per_ne", return_value=1): ra = claim_mod.enqueue_collect("ta", manual=True) rb = claim_mod.enqueue_collect("tb", manual=True) rc = claim_mod.enqueue_collect("tc", manual=True) @@ -111,6 +112,47 @@ class BizStateClaimTests(unittest.TestCase): self.assertEqual(self.db.get(BizStateBatch, rb["batch_id"]).status, "queued") self.assertEqual(self.db.get(BizStateBatch, rc["batch_id"]).status, "running") + def test_a_ne_runs_up_to_its_session_cap_and_no_more(self) -> None: + self.db.add_all([BizStateTask(id=f"x{i}", source="managed", ne_id="ne-a", status="running") for i in range(3)]) + for i in range(3): + self.db.add(BizStateTaskItem(id=uuid4().hex, task_id=f"x{i}", source_profile_id="zte.arp", + kind="catalog", enabled=True)) + self.db.commit() + with patch.object(claim_mod, "max_concurrent_tasks", return_value=10), patch.object(claim_mod, "max_sessions_per_ne", return_value=2): + queued = [claim_mod.enqueue_collect(f"x{i}", manual=True)["batch_id"] for i in range(3)] + first = claim_mod.claim_queued_batches(10) + again = claim_mod.claim_queued_batches(10) + self.assertEqual(len(first), 2) # three batches queued for ne-a, only two sessions allowed + self.assertEqual(again, []) + self.db.expire_all() + states = sorted(self.db.get(BizStateBatch, b).status for b in queued) + self.assertEqual(states, ["queued", "running", "running"]) + + def test_default_session_cap_is_two(self) -> None: + self.assertEqual(claim_mod.max_sessions_per_ne(), 2) + + def test_single_task_group_is_a_valid_round_for_a_same_device_cutover(self) -> None: + task = self.db.get(BizStateTask, "ta") + task.collect_group_id = "solo" + self.db.commit() + with patch.object(claim_mod, "max_concurrent_tasks", return_value=10): + r = claim_mod.enqueue_collect("ta", manual=True) + self.assertTrue(r["queued"], r) + claimed = claim_mod.claim_queued_batches(10) + self.assertEqual([c["task_id"] for c in claimed], ["ta"]) + + def test_a_pair_needs_both_sides_before_it_is_claimed(self) -> None: + for tid, group in (("ta", "pair"), ("tc", "pair")): + self.db.get(BizStateTask, tid).collect_group_id = group + self.db.commit() + with patch.object(claim_mod, "max_concurrent_tasks", return_value=10): + r = claim_mod.enqueue_collect("ta", manual=True) + self.assertTrue(r["queued"], r) + self.db.expire_all() + self.db.query(BizStateBatch).filter(BizStateBatch.task_id == "tc").delete() + self.db.commit() + self.assertEqual(claim_mod.claim_queued_batches(10), []) + def test_claim_respects_slot_ceiling(self) -> None: with patch.object(claim_mod, "max_concurrent_tasks", return_value=1): claim_mod.enqueue_collect("ta", manual=True) diff --git a/tests/test_biz_state_compare.py b/tests/test_biz_state_compare.py index 8d610ad..dcb07e1 100644 --- a/tests/test_biz_state_compare.py +++ b/tests/test_biz_state_compare.py @@ -5,7 +5,8 @@ from __future__ import annotations import copy import unittest -from netx_api.biz_state.compare_engine import compare_rows, mapping_stats +from netx_api.biz_state.compare_engine import compare_rows, mapping_stats, stratify_take +from netx_api.biz_state.compare_service import _kind_allows from netx_api.biz_state.compare_rules import ( apply_row_filters, arp_dynamic_row_filters, @@ -96,7 +97,18 @@ class CompareEngineTests(unittest.TestCase): self.assertEqual(s["removed"], 1) self.assertEqual(s["added"], 1) kinds = {d["kind"] for d in out["diffs"]} - self.assertIn("unchanged", kinds) + # Default: unchanged counted but not listed (million-row safe) + self.assertNotIn("unchanged", kinds) + out_full = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["local_if", "remote_sys", "remote_if"], + iface_fields=["local_if"], + compare_fields=[], + port_map={}, + include_unchanged=True, + ) + self.assertIn("unchanged", {d["kind"] for d in out_full["diffs"]}) def test_empty_port_map_ignores_iface_in_key(self) -> None: """No port map → ignore local_if when matching (same neighbor, renamed port).""" @@ -149,10 +161,107 @@ class CompareEngineTests(unittest.TestCase): iface_fields=["local_if"], compare_fields=["remote_ip"], port_map={}, + include_unchanged=True, ) self.assertEqual(out["summary"]["unchanged"], 1) self.assertEqual(len(out["diffs"]), 1) self.assertEqual(out["diffs"][0]["kind"], "unchanged") + slim = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["local_if", "remote_sys", "remote_if"], + iface_fields=["local_if"], + compare_fields=["remote_ip"], + port_map={}, + ) + self.assertEqual(slim["summary"]["unchanged"], 1) + self.assertEqual(slim["diffs"], []) + + def test_unchanged_sample_limit_and_compact(self) -> None: + before = [ + {"k": str(i), "v": "1", "_netx": {"row_id": f"b{i}"}} for i in range(5) + ] + after = [ + {"k": str(i), "v": "1", "_netx": {"row_id": f"a{i}"}} for i in range(5) + ] + out = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["k"], + iface_fields=[], + compare_fields=["v"], + port_map={}, + include_unchanged=True, + unchanged_limit=2, + compact_unchanged=True, + ) + self.assertEqual(out["summary"]["unchanged"], 5) + self.assertEqual(out["summary"]["unchanged_listed"], 2) + self.assertTrue(out["summary"]["unchanged_truncated"]) + self.assertTrue(out["summary"]["unchanged_compact"]) + self.assertEqual(out["summary"]["unchanged_sample_mode"], "stratified") + self.assertEqual(len(out["diffs"]), 2) + self.assertEqual(out["diffs"][0]["before"], {}) + self.assertEqual(out["diffs"][0]["after"], {}) + self.assertEqual(out["diffs"][0]["before_row_id"], "b0") + self.assertEqual(out["diffs"][0]["after_row_id"], "a0") + + def test_stratify_take_round_robin(self) -> None: + items = [("a", i) for i in range(5)] + [("b", i) for i in range(5)] + got = stratify_take(items, 4, key_fn=lambda x: x[0]) + self.assertEqual([x[0] for x in got], ["a", "b", "a", "b"]) + + def test_kind_allows_tabs(self) -> None: + self.assertTrue(_kind_allows("all", "unchanged")) + self.assertTrue(_kind_allows("diff", "removed")) + self.assertTrue(_kind_allows("diff", "changed")) + self.assertFalse(_kind_allows("diff", "added")) + self.assertFalse(_kind_allows("diff", "unchanged")) + self.assertTrue(_kind_allows("unchanged", "unchanged")) + self.assertFalse(_kind_allows("unchanged", "removed")) + self.assertTrue(_kind_allows("removed", "removed")) + self.assertTrue(_kind_allows("changed", "changed")) + + def test_unchanged_sample_stratified_across_neighbors(self) -> None: + """Sample must cover multiple neighbors, not only the first command's rows.""" + before = [] + after = [] + for n_i, neigh in enumerate(("1.1.1.1", "2.2.2.2", "3.3.3.3")): + for j in range(10): + net = f"10.{n_i}.{j}.0/24" + before.append( + { + "neighbor": neigh, + "direction": "in", + "network": net, + "v": "1", + "_netx": {"row_id": f"b-{neigh}-{j}"}, + } + ) + after.append( + { + "neighbor": neigh, + "direction": "in", + "network": net, + "v": "1", + "_netx": {"row_id": f"a-{neigh}-{j}"}, + } + ) + out = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["neighbor", "direction", "network"], + iface_fields=[], + compare_fields=["v"], + port_map={}, + include_unchanged=True, + unchanged_limit=6, + compact_unchanged=True, + ) + self.assertEqual(out["summary"]["unchanged"], 30) + self.assertEqual(out["summary"]["unchanged_listed"], 6) + keys = [d["key"]["neighbor"] for d in out["diffs"]] + self.assertEqual(set(keys), {"1.1.1.1", "2.2.2.2", "3.3.3.3"}) def test_mac_normalize_via_field_rules(self) -> None: before = [{"ip": "1.1.1.1", "mac": "00:11:22:33:44:55", "iface": "gei-0/1"}] @@ -309,6 +418,57 @@ class CompareRulesTests(unittest.TestCase): class CompareDiffPagingTests(unittest.TestCase): + def test_resolve_unchanged_policy(self) -> None: + from netx_api.biz_state.compare_service import resolve_unchanged_policy + + small = resolve_unchanged_policy("auto", before_n=100, after_n=100) + self.assertTrue(small["include"]) + self.assertIsNone(small["limit"]) + self.assertFalse(small["compact"]) + large = resolve_unchanged_policy("auto", before_n=1_500_000, after_n=1_500_000) + self.assertTrue(large["include"]) + self.assertEqual(large["limit"], 5000) + self.assertTrue(large["compact"]) + self.assertFalse(resolve_unchanged_policy("never", before_n=10, after_n=10)["include"]) + keys = resolve_unchanged_policy("keys", before_n=1_500_000, after_n=1_500_000) + self.assertTrue(keys["include"]) + self.assertIsNone(keys["limit"]) + self.assertTrue(keys["compact"]) + + def test_hydrate_diff_rows_fills_sides(self) -> None: + from netx_api.biz_state.compare_service import _hydrate_diff_rows + + class _FakeDb: + pass + + items = [ + { + "kind": "unchanged", + "key": {"k": "1"}, + "before": {}, + "after": {}, + "mapped_before": {}, + "changes": {}, + "before_row_id": "b1", + "after_row_id": "a1", + } + ] + # Patch loader + import netx_api.biz_state.compare_service as cs + + orig = cs._metric_rows_by_ids + try: + cs._metric_rows_by_ids = lambda _db, ids: { # type: ignore[assignment] + "b1": {"k": "1", "v": "pre"}, + "a1": {"k": "1", "v": "post"}, + } + out = _hydrate_diff_rows(_FakeDb(), items) + finally: + cs._metric_rows_by_ids = orig + self.assertEqual(out[0]["before"]["v"], "pre") + self.assertEqual(out[0]["after"]["v"], "post") + self.assertEqual(out[0]["mapped_before"]["v"], "pre") + def test_filter_inline_diffs_kind_and_kw(self) -> None: from netx_api.biz_state.compare_service import _filter_inline_diffs @@ -354,25 +514,43 @@ class CompareSheetDefaultsTests(unittest.TestCase): self.assertIn("isis_adjacency.ipv6", ids) # Same source metric may appear multiple times (afi splits) self.assertEqual(sum(1 for s in sheets if s["metric_id"] == "bgp_peer"), 6) + self.assertGreaterEqual(sum(1 for s in sheets if s["metric_id"] == "bgp_route"), 4) self.assertIn("bgp_peer.evpn", ids) self.assertIn("bgp_peer.vpls", ids) self.assertIn("vrrp.ipv4", ids) self.assertIn("lldp_neighbor", ids) - detail = next(s for s in sheets if sheet_key(s) == "interface_detail") - self.assertEqual(detail["compare_fields"], ["port_status"]) - self.assertIn("input_bps", detail["display_fields"]) - optical = next(s for s in sheets if sheet_key(s) == "optical_brief") - self.assertEqual(optical["compare_fields"], ["status"]) - self.assertIn("rx_power", optical["display_fields"]) - bgp4 = next(s for s in sheets if sheet_key(s) == "bgp_peer.ipv4") - self.assertEqual(bgp4["compare_fields"], ["as_num", "state"]) - self.assertIn("pfx_rcd", bgp4["display_fields"]) + # Packaged IOH default: filtered bgp_route ipv4 + percent pfx_rcd + route4 = next(s for s in sheets if sheet_key(s) == "bgp_route") + self.assertEqual(route4["metric_id"], "bgp_route") + self.assertTrue( + any( + f.get("field") == "afi" and f.get("value") == "ipv4" + for f in (route4.get("row_filters") or []) + ) + ) + self.assertTrue( + any( + r.get("field") == "pfx_rcd" and r.get("compare") == "percent" + for r in (route4.get("field_rules") or []) + ) + ) + route_vpn = next(s for s in sheets if sheet_key(s) == "bgp_route.vpnv4") + self.assertTrue( + any( + f.get("field") == "afi" and f.get("value") == "vpnv4" + for f in (route_vpn.get("row_filters") or []) + ) + ) vpnv4 = next(s for s in sheets if sheet_key(s) == "bgp_peer.vpnv4") - self.assertEqual(vpnv4["row_filters"], [{"field": "afi", "op": "eq", "value": "vpnv4"}]) + self.assertEqual( + vpnv4["row_filters"], + [{"field": "afi", "op": "eq", "value": "vpnv4"}], + ) isis4 = next(s for s in sheets if sheet_key(s) == "isis_adjacency.ipv4") self.assertEqual(isis4["row_filters"][0]["op"], "contains") - def test_duplicate_match_keys_are_reported(self) -> None: + def test_ordered_same_key_pairing_2v2(self) -> None: + """Same key, two rows each: zip by order (not first-wins + duplicate).""" before = [ {"local_if": "a", "remote_sys": "X", "remote_if": "1", "remote_ip": "1"}, {"local_if": "a", "remote_sys": "X", "remote_if": "1", "remote_ip": "9"}, @@ -389,15 +567,47 @@ class CompareSheetDefaultsTests(unittest.TestCase): compare_fields=["remote_ip"], port_map={"a": "a"}, ) + self.assertEqual(out["summary"]["before_count"], 2) + self.assertEqual(out["summary"]["after_count"], 2) + self.assertEqual(out["summary"]["duplicate"], 0) self.assertEqual(out["summary"]["duplicate_keys_before"], 1) self.assertEqual(out["summary"]["duplicate_keys_after"], 1) - self.assertEqual(out["summary"]["duplicate"], 2) self.assertIn("a|X|1", out["summary"]["duplicate_key_list"]) kinds = [d["kind"] for d in out["diffs"]] - self.assertEqual(kinds.count("duplicate"), 2) - # First before wins → matches first after → unchanged (same remote_ip) + self.assertNotIn("duplicate", kinds) + # Pair 0: 1==1 unchanged; pair 1: 9!=2 changed self.assertEqual(out["summary"]["unchanged"], 1) + self.assertEqual(out["summary"]["changed"], 1) + self.assertEqual(out["summary"]["added"], 0) + self.assertEqual(out["summary"]["removed"], 0) + + def test_ordered_multipath_5_vs_2(self) -> None: + """5 before / 2 after same key → 2 compared + 3 removed failures.""" + key = {"local_if": "a", "remote_sys": "X", "remote_if": "1"} + before = [{**key, "remote_ip": str(i)} for i in range(5)] + after = [{**key, "remote_ip": "0"}, {**key, "remote_ip": "1"}] + out = compare_rows( + before_rows=before, + after_rows=after, + key_fields=["local_if", "remote_sys", "remote_if"], + iface_fields=["local_if"], + compare_fields=["remote_ip"], + port_map={"a": "a"}, + ) + self.assertEqual(out["summary"]["before_count"], 5) + self.assertEqual(out["summary"]["after_count"], 2) + self.assertEqual(out["summary"]["removed"], 3) + self.assertEqual(out["summary"]["added"], 0) + self.assertEqual( + out["summary"]["unchanged"] + out["summary"]["changed"], + 2, + ) + self.assertEqual(out["summary"]["unchanged"], 2) self.assertEqual(out["summary"]["changed"], 0) + self.assertEqual(out["summary"]["duplicate"], 0) + kinds = [d["kind"] for d in out["diffs"]] + self.assertEqual(kinds.count("removed"), 3) + self.assertNotIn("duplicate", kinds) def test_ignore_port_changes_false_keeps_iface(self) -> None: before = [ diff --git a/tests/test_biz_state_compare_export.py b/tests/test_biz_state_compare_export.py new file mode 100644 index 0000000..d176c40 --- /dev/null +++ b/tests/test_biz_state_compare_export.py @@ -0,0 +1,87 @@ +"""Export must page lazily, keep tied sequence rows, and produce valid UTF-8 CSV.""" + +from __future__ import annotations + +import csv +import io +import unittest +import zipfile +from unittest.mock import patch + +from sqlalchemy import create_engine, event +from sqlalchemy.orm import Session + +from netx_api.biz_state.compare_service import _iter_sheet_diffs, export_run_zip +from netx_api.db import Base +from netx_api.models import BizCompareDiff, BizCompareRun + + +class CompareExportTests(unittest.TestCase): + def setUp(self): + self.engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(self.engine) + self.db = Session(self.engine) + self.sheet = {"sheet_id": "sheet", "metric_id": "test", "key_fields": ["id"], + "compare_fields": ["note"], "display_fields": ["id", "note"], + "summary": {"unchanged": 7}, "status": "done"} + self.run = BizCompareRun(id="run", metric_id="test", status="success", + summary_json={"sheets": [self.sheet]}) + self.db.add(self.run) + for seq, ids in ((0, "cba"), (1, "ed"), (2, "gf")): + for rid in ids: + note = '中文, "quoted"\nsecond line' + self.db.add(BizCompareDiff(id=rid, run_id="run", metric_id="sheet", seq=seq, + kind="unchanged", key_json={"id": rid}, + before_json={"id": rid, "note": note}, + after_json={"id": rid, "note": note})) + self.db.commit() + + def tearDown(self): + self.db.close() + self.engine.dispose() + + def test_cursor_handles_tied_sequence_numbers_across_chunks(self): + statements = [] + def record(_conn, _cursor, statement, _parameters, _context, _executemany): + if "ORDER BY biz_compare_diff.seq" in statement: + statements.append(statement) + event.listen(self.engine, "before_cursor_execute", record) + with patch("netx_api.biz_state.compare_service._DIFF_CHUNK", 2): + iterator = _iter_sheet_diffs(self.db, "run", "sheet") + self.assertEqual(statements, []) + self.assertEqual(next(iterator)["key"]["id"], "a") + self.assertEqual(next(iterator)["key"]["id"], "b") + self.assertEqual(len(statements), 1, "do not load the next chunk early") + self.assertEqual([d["key"]["id"] for d in iterator], list("cdefg")) + self.assertEqual(len(statements), 4) + self.assertTrue(all("biz_compare_diff.seq >" in q and "biz_compare_diff.id >" in q + for q in statements[1:])) + + def test_export_csv_escapes_unicode_quotes_commas_and_newlines(self): + with patch("netx_api.biz_state.compare_service._DIFF_CHUNK", 2): + archive_bytes = export_run_zip(self.db, "run") + with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive: + payload = archive.read("tables/sheet.csv") + self.assertTrue(payload.startswith(b"\xef\xbb\xbf")) + rows = list(csv.DictReader(io.StringIO(payload.decode("utf-8-sig")))) + self.assertEqual([r["id"] for r in rows], list("abcdefg")) + self.assertTrue(all(r["note__pre"] == '中文, "quoted"\nsecond line' for r in rows)) + self.assertTrue(all(r["note__pre"] == r["note__post"] for r in rows)) + + def test_other_sheets_do_not_leak_into_export(self): + self.db.add(BizCompareDiff(id="other", run_id="run", metric_id="another", seq=0, + kind="added", key_json={"id": "other"}, after_json={"note": "other"})) + self.db.commit() + with patch("netx_api.biz_state.compare_service._DIFF_CHUNK", 2): + self.assertEqual([d["key"]["id"] for d in _iter_sheet_diffs(self.db, "run", "sheet")], list("abcdefg")) + + def test_legacy_inline_and_empty_exports(self): + self.db.query(BizCompareDiff).delete() + inline = {"kind": "added", "key": {"id": "legacy"}, "after": {"note": "value"}} + self.run.summary_json = {"sheets": [{**self.sheet, "diffs": [inline]}]} + self.db.commit() + self.assertEqual(list(_iter_sheet_diffs(self.db, "run", "sheet")), [inline]) + self.run.summary_json = {"sheets": [self.sheet]} + self.db.commit() + with zipfile.ZipFile(io.BytesIO(export_run_zip(self.db, "run"))) as archive: + self.assertEqual(archive.read("tables/sheet.csv").decode("utf-8-sig"), "kind,id,note__pre,note__post\n") diff --git a/tests/test_biz_state_compare_memory.py b/tests/test_biz_state_compare_memory.py new file mode 100644 index 0000000..7850798 --- /dev/null +++ b/tests/test_biz_state_compare_memory.py @@ -0,0 +1,103 @@ +"""Bounded sampling regressions and an opt-in comparison memory benchmark.""" + +from __future__ import annotations + +import gc +import hashlib +import json +import os +import random +import subprocess +import time +import tracemalloc +import types +from collections import defaultdict, deque +from copy import deepcopy + +import pytest + +from netx_api.biz_state.compare_engine import _StratifiedSample, compare_rows, stratify_take + + +def reference_sample(items, limit): + if len(items) <= limit: + return list(items) + buckets = defaultdict(deque) + for item in items: + buckets[item[0]].append(item) + out = [] + while buckets and len(out) < limit: + for key in list(buckets): + out.append(buckets[key].popleft()) + if not buckets[key]: + del buckets[key] + if len(out) >= limit: + break + return out + + +def test_streaming_sample_preserves_round_robin_and_small_input_order(): + rng = random.Random(42) + for _ in range(100): + items = [(str(rng.randrange(20)), i) for i in range(rng.randrange(100))] + for limit in (0, 1, 5, 20, 100): + expected = reference_sample(items, limit) + assert stratify_take(iter(items), limit, key_fn=lambda item: item[0]) == expected + + +def test_sample_storage_is_bounded_for_both_many_rows_and_many_strata(): + sample = _StratifiedSample(50) + for i in range(20_000): + sample.add(i, str(i) if i >= 10_000 else "one-large-peer") + assert len(sample.heap) == 50 + assert len(sample.strata) == 50 + assert sample.picked() == [0, *range(10_000, 10_049)] + + +@pytest.mark.parametrize("iface_fields,rules,pmap", [ + ([], [], {}), + (["interface"], [], {}), + (["interface"], [{"from": "GE", "to": "gei"}], {"gei-1": "gei-2"}), +]) +def test_compare_does_not_mutate_inputs_when_reusing_rows(iface_fields, rules, pmap): + before = [{"id": "1", "interface": "GE-1", "state": "up"}] + after = [{"id": "1", "interface": "GE-2", "state": "up"}] + original = deepcopy((before, after)) + compare_rows(before_rows=before, after_rows=after, key_fields=["id"], + iface_fields=iface_fields, compare_fields=["state"], port_map=pmap, + iface_normalize_rules=rules, include_unchanged=True, unchanged_limit=1) + assert (before, after) == original + + +def test_memory_benchmark_against_git_baseline(): + """Run explicitly with NETX_BENCH_ROWS=1000000; not part of routine CI.""" + n = int(os.environ.get("NETX_BENCH_ROWS", "0")) + if not n: + pytest.skip("opt-in large synthetic benchmark") + baseline_ref = os.environ.get("NETX_BENCH_REF", "HEAD") + source = subprocess.run(["git", "show", f"{baseline_ref}:netx_api/biz_state/compare_engine.py"], + check=True, capture_output=True, text=True, encoding="utf-8").stdout + baseline = types.ModuleType("netx_api.biz_state._benchmark_baseline") + baseline.__package__ = "netx_api.biz_state" + exec(compile(source, "baseline_compare_engine.py", "exec"), baseline.__dict__) + before = [{"id": str(i), "neighbor": str(i % 50), "state": "up", "afi": "vpnv4"} for i in range(n)] + after = [dict(row) for row in before] + records = [] + digests = [] + for name, compare in (("baseline", baseline.compare_rows), ("optimized", compare_rows)): + gc.collect() + tracemalloc.start() + started = time.perf_counter() + result = compare(before_rows=before, after_rows=after, key_fields=["id"], iface_fields=[], + compare_fields=["state"], include_unchanged=True, unchanged_limit=5000, + compact_unchanged=True) + elapsed = time.perf_counter() - started + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + digests.append(hashlib.sha256(json.dumps(result, sort_keys=True).encode()).hexdigest()) + records.append({"engine": name, "rows_per_side": n, "peak_engine_mib": round(peak / 2**20, 2), + "seconds_with_tracemalloc": round(elapsed, 3)}) + del result + assert digests[0] == digests[1], "large-sheet results and sample order must match" + print("\n" + json.dumps({"baseline_ref": baseline_ref, "input_allocations_included": False, + "results": records}, indent=2)) diff --git a/tests/test_biz_state_compare_recovery.py b/tests/test_biz_state_compare_recovery.py new file mode 100644 index 0000000..e24ea40 --- /dev/null +++ b/tests/test_biz_state_compare_recovery.py @@ -0,0 +1,98 @@ +"""Startup recovery for interrupted biz-state compare runs.""" + +from __future__ import annotations + +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_state.compare_service import ( + cancel_compare_run, + recover_interrupted_compares_on_startup, +) +from netx_api.db import Base +from netx_api.models import BizCompareJob, BizCompareRun + + +class BizStateCompareRecoveryTests(unittest.TestCase): + def setUp(self) -> None: + engine = create_engine("sqlite+pysqlite:///:memory:", future=True) + TestingSession = sessionmaker( + bind=engine, autoflush=False, autocommit=False, expire_on_commit=False + ) + Base.metadata.create_all(bind=engine) + self.db = TestingSession() + self.job = BizCompareJob( + id="j1", + name="cutover", + template_id="tpl1", + status="active", + ) + self.db.add(self.job) + self.running = BizCompareRun( + id="r_run", + job_id="j1", + status="running", + message="loading 1/2 · BGP", + summary_json={ + "progress": {"phase": "loading", "sheet_index": 1, "sheet_total": 2}, + "sheets": [ + {"sheet_id": "bgp", "status": "running"}, + {"sheet_id": "isis", "status": "pending"}, + ], + }, + ) + self.queued = BizCompareRun( + id="r_q", + job_id="j1", + status="queued", + message="queued", + summary_json={"sheets": [{"sheet_id": "bgp", "status": "pending"}]}, + ) + self.ok = BizCompareRun( + id="r_ok", + job_id="j1", + status="success", + message="done", + summary_json={"added": 0, "removed": 0, "changed": 0}, + ) + self.db.add_all([self.running, self.queued, self.ok]) + self.db.commit() + + def tearDown(self) -> None: + self.db.close() + + def test_startup_cancels_running_and_queued(self) -> None: + out = recover_interrupted_compares_on_startup(self.db) + self.assertEqual(out["runs"], 2) + + self.db.refresh(self.running) + self.db.refresh(self.queued) + self.db.refresh(self.ok) + self.assertEqual(self.running.status, "cancelled") + self.assertIn("interrupted_by_restart", self.running.message or "") + self.assertEqual( + (self.running.summary_json or {}).get("progress", {}).get("phase"), + "cancelled", + ) + sheets = list((self.running.summary_json or {}).get("sheets") or []) + self.assertEqual(sheets[0].get("status"), "cancelled") + self.assertEqual(sheets[1].get("status"), "cancelled") + self.assertEqual(self.queued.status, "cancelled") + self.assertEqual(self.ok.status, "success") + + def test_cancel_compare_run_user(self) -> None: + out = cancel_compare_run(self.db, "r_run") + self.assertEqual(out["status"], "cancelled") + self.db.refresh(self.running) + self.assertEqual(self.running.status, "cancelled") + self.assertIn("cancelled_by_user", self.running.message or "") + + def test_cancel_idempotent_on_success(self) -> None: + out = cancel_compare_run(self.db, "r_ok") + self.assertEqual(out["status"], "success") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_biz_state_compare_search.py b/tests/test_biz_state_compare_search.py new file mode 100644 index 0000000..0b30c8f --- /dev/null +++ b/tests/test_biz_state_compare_search.py @@ -0,0 +1,186 @@ +"""Regression tests: searching a pair must not change its comparison verdict.""" + +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from fastapi import HTTPException +from sqlalchemy import create_engine +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import Session + +from netx_api.biz_state.compare_engine import compare_rows +from netx_api.biz_state.compare_service import ( + _load_metric_rows_for_search, + _literal_search_pattern, + _persist_sheet_diffs, + _search_stored_diffs, + list_run_diffs, +) +from netx_api.db import Base +from netx_api.models import ( + BizCompareDiff, BizCompareRun, BizCompareTemplate, BizPortMappingRow, + BizStateLldpNeighbor, BizStateMetricRow, +) + + +class CompareSearchTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(self.engine, tables=[m.__table__ for m in ( + BizCompareDiff, BizCompareRun, BizCompareTemplate, BizPortMappingRow, + BizStateLldpNeighbor, BizStateMetricRow, + )]) + self.db = Session(self.engine) + + def tearDown(self) -> None: + self.db.close() + self.engine.dispose() + + def seed(self, before, after, *, metric="test", keys=None, fields=None, + iface=None, mapping=None, sample=False, compact=False, filters=None): + keys = keys or ["id"] + iface = iface or [] + fields = fields or ["state"] + for side, rows in (("before", before), ("after", after)): + for seq, row in enumerate(rows): + rid = f"{side}-{seq}" + if metric == "lldp_neighbor": + self.db.add(BizStateLldpNeighbor(id=rid, batch_id=side, **row)) + else: + self.db.add(BizStateMetricRow(id=rid, batch_id=side, metric_id=metric, + seq=seq, data_json=dict(row))) + row["_netx"] = {"row_id": rid} + if mapping: + for b, a in mapping.items(): + self.db.add(BizPortMappingRow(mapping_id="map", before_if=b, after_if=a)) + result = compare_rows(before_rows=before, after_rows=after, key_fields=keys, + compare_fields=fields, iface_fields=iface, port_map=mapping or {}, + include_unchanged=True, compact_unchanged=compact) + sheet = {"sheet_id": "sheet", "metric_id": metric, "key_fields": keys, + "compare_fields": fields, "iface_fields": iface, + "summary": {**result["summary"], "unchanged_listed": 0 if sample + else result["summary"]["unchanged"]}, + "row_filters": filters or []} + self.db.add(BizCompareRun(id="run", status="success", before_batch_id="before", + after_batch_id="after", mapping_id="map" if mapping else "", + summary_json={"sheets": [sheet]})) + self.db.commit() + _persist_sheet_diffs(self.db, run_id="run", metric_id="sheet", + diffs=[d for d in result["diffs"] + if not sample or d["kind"] != "unchanged"]) + return result + + def search(self, **kwargs): + return list_run_diffs(self.db, "run", metric_id="sheet", **kwargs) + + def test_search_before_value_preserves_changed_verdict(self): + self.seed([{"id": "1", "state": "up"}], [{"id": "1", "state": "down"}]) + with patch("netx_api.biz_state.compare_service.compare_rows") as compare: + result = self.search(kw="up", kind="changed") + compare.assert_not_called() + self.assertEqual(result["total"], 1) + self.assertEqual(result["items"][0]["after"]["state"], "down") + self.assertEqual(self.search(kw="up", kind="removed")["total"], 0) + + def test_field_filter_matches_after_value_on_compact_diff(self): + self.seed([{"id": "1", "state": "up"}], [{"id": "1", "state": "down"}], compact=True) + result = self.search(qf={"state": "down"}, kind="changed") + self.assertEqual(result["total"], 1) + self.assertEqual(result["items"][0]["before"]["state"], "up") + + def test_sample_lookup_completes_pair_matching_only_one_side(self): + self.seed([{"id": "1", "state": "up", "age": "old"}], + [{"id": "1", "state": "up", "age": "new"}], sample=True) + result = self.search(kw="old", kind="unchanged") + self.assertEqual(result["source"], "live") + self.assertEqual(result["total"], 1) + self.assertEqual(result["items"][0]["after"]["age"], "new") + + def test_sample_lookup_does_not_turn_changed_into_removed(self): + self.seed([{"id": "1", "state": "up"}, {"id": "2", "state": "up"}], + [{"id": "1", "state": "down"}, {"id": "2", "state": "up"}], sample=True) + result = self.search(qf={"id": "1", "state": "up"}, kind="all") + self.assertEqual([d["kind"] for d in result["items"]], ["changed"]) + + def test_sample_lookup_keeps_duplicate_group_pairing(self): + self.seed([{"id": "1", "state": "up", "note": "first"}, + {"id": "1", "state": "down", "note": "second"}], + [{"id": "1", "state": "up", "note": "other"}, + {"id": "1", "state": "down", "note": "other"}], sample=True) + result = self.search(kw="second", kind="all") + self.assertEqual([d["kind"] for d in result["items"]], ["unchanged"]) + self.assertEqual(result["items"][0]["after"]["state"], "down") + + def test_sample_search_is_not_limited_by_persisted_search_text_size(self): + self.seed([{"id": "1", "state": "up", "note": "x" * 5000 + "needle"}], + [{"id": "1", "state": "up", "note": "other"}], sample=True) + self.assertEqual(self.search(kw="needle", kind="unchanged")["total"], 1) + + def test_sample_lookup_applies_port_mapping_before_matching(self): + self.seed([{"id": "1", "interface": "old", "state": "up"}], + [{"id": "1", "interface": "new", "state": "up"}], + keys=["id", "interface"], iface=["interface"], mapping={"old": "new"}, sample=True) + result = self.search(qf={"interface": "old"}, kind="unchanged") + self.assertEqual(result["total"], 1) + self.assertEqual(result["items"][0]["after"]["interface"], "new") + + def test_lldp_source_search(self): + self.seed([{"local_if": "gei-1", "remote_sys": "peer", "remote_if": "port", "remote_ip": "1"}], + [{"local_if": "gei-1", "remote_sys": "peer", "remote_if": "port", "remote_ip": "1"}], + metric="lldp_neighbor", keys=["local_if", "remote_sys"], fields=["remote_ip"], + iface=["local_if"], sample=True) + self.assertEqual(self.search(kw="peer", kind="unchanged")["total"], 1) + + def test_lldp_compact_diff_field_search(self): + self.seed([{"local_if": "gei-1", "remote_sys": "peer", "remote_ip": "1"}], + [{"local_if": "gei-1", "remote_sys": "peer", "remote_ip": "2"}], + metric="lldp_neighbor", keys=["local_if", "remote_sys"], fields=["remote_ip"], + iface=["local_if"], compact=True) + self.assertEqual(self.search(qf={"remote_ip": "2"}, kind="changed")["total"], 1) + + def test_exact_load_cap_is_not_truncated(self): + self.seed([{"id": "1", "state": "up"}], [{"id": "1", "state": "up"}]) + rows, truncated = _load_metric_rows_for_search(self.db, batch_id="before", metric_id="test", + row_filters=[], key_fields=["id"], kw="up", cap=1) + self.assertEqual(len(rows), 1) + self.assertFalse(truncated) + + def test_incomplete_duplicate_groups_are_omitted(self): + self.seed([{"id": "1", "state": "up"}, {"id": "1", "state": "down"}], + [{"id": "1", "state": "up"}, {"id": "1", "state": "down"}], sample=True) + with patch("netx_api.biz_state.compare_service._LIVE_SEARCH_GROUP_LOAD_CAP", 1): + result = self.search(kw="up", kind="all") + self.assertEqual(result["items"], []) + self.assertTrue(result["truncated"]) + + def test_postgres_search_pattern_treats_wildcards_literally(self): + self.assertEqual(_literal_search_pattern("a_10%"), "%a\\_10\\%%") + + def test_postgres_unsupported_filter_uses_python(self): + self.seed([{"id": "1", "state": "up", "age": "H"}], + [{"id": "1", "state": "up", "age": "H"}]) + with patch("netx_api.biz_state.compare_sql._dialect_is_postgres", return_value=True): + rows, _ = _load_metric_rows_for_search(self.db, batch_id="before", metric_id="test", + row_filters=[{"field": "age", "op": "age_timer"}], key_fields=["id"], kw="up") + self.assertEqual(rows, []) + + def test_unknown_sheet_is_rejected(self): + self.seed([{"id": "1", "state": "up"}], [{"id": "1", "state": "up"}]) + with self.assertRaises(HTTPException) as error: + list_run_diffs(self.db, "run", metric_id="missing") + self.assertEqual(error.exception.detail, "sheet_not_found") + + def test_stored_search_escapes_like_wildcards(self): + self.seed([{"id": "x_1", "state": "up"}, {"id": "xy1", "state": "up"}], + [{"id": "x_1", "state": "down"}, {"id": "xy1", "state": "down"}]) + self.assertEqual(self.search(qf={"id": "x_1"}, kind="changed")["total"], 1) + query = _search_stored_diffs(self.db.query(BizCompareDiff), metric_id="test", + kw="up", field_q={"id": "x_1"}) + compiled = query.statement.compile(dialect=postgresql.dialect()) + self.assertIn("LEFT OUTER JOIN", str(compiled)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_biz_state_compare_snapshot.py b/tests/test_biz_state_compare_snapshot.py new file mode 100644 index 0000000..a531e0e --- /dev/null +++ b/tests/test_biz_state_compare_snapshot.py @@ -0,0 +1,206 @@ +"""Historical comparisons must use the settings captured when they were queued.""" + +from __future__ import annotations + +import csv +import io +import json +import unittest +import zipfile +from copy import deepcopy +from unittest.mock import patch + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_state.compare_service import ( + _create_running_run, _execute_compare_into_run, cancel_compare_run, + export_run_zip, get_run, list_run_diffs, template_metrics, +) +from netx_api.db import Base +from netx_api.models import ( + BizCompareDiff, BizCompareJob, BizCompareTemplate, BizPortMapping, + BizPortMappingRow, BizStateMetricRow, +) + + +class CompareSnapshotTests(unittest.TestCase): + def setUp(self): + self.engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(self.engine) + self.factory = sessionmaker(bind=self.engine, expire_on_commit=False) + self.db = self.factory() + # Progress uses a separate session in production; keep every test write local. + self.session_patch = patch("netx_api.db.SessionLocal", self.factory) + self.session_patch.start() + self.sheet = { + "sheet_id": "sheet", "metric_id": "test", "title": "Original sheet", + "key_fields": ["id", "interface"], "iface_fields": ["interface"], + "compare_fields": ["state"], "display_fields": ["id", "interface", "state"], + "row_filters": [], "ignore_port_changes": False, + } + self.tpl = BizCompareTemplate( + id="tpl", name="Original template", metrics_json=[deepcopy(self.sheet)], + iface_normalize_json=[{"from": "GE", "to": "gei"}], + ) + self.job = BizCompareJob(id="job", name="Original job", template_id="tpl", + mapping_id="map", store_unchanged="sample") + self.mapping = BizPortMapping(id="map", name="Original mapping") + self.pair = BizPortMappingRow(mapping_id="map", before_if="gei-1", after_if="gei-2") + self.db.add_all([self.tpl, self.job, self.mapping, self.pair]) + for side, iface, note in (("before", "GE-1.100", "old"), ("after", "GE-2.100", "new")): + self.db.add(BizStateMetricRow(id=side, batch_id=side, metric_id="test", seq=0, + data_json={"id": "1", "interface": iface, + "state": "up", "note": note})) + self.db.commit() + self.run = _create_running_run( + self.db, job=self.job, tpl=self.tpl, before_batch_id="before", + after_batch_id="after", sheets_cfg=template_metrics(self.tpl), + ) + + def tearDown(self): + self.session_patch.stop() + self.db.close() + self.engine.dispose() + + def edit_current_settings(self): + changed = {**self.sheet, "title": "Edited sheet", "compare_fields": ["note"], + "row_filters": [{"field": "state", "op": "eq", "value": "down"}]} + self.tpl.name = "Edited template" + self.tpl.metrics_json = [changed] + self.tpl.iface_normalize_json = [{"from": "GE", "to": "other"}] + self.pair.after_if = "gei-99" + self.job.name = "Edited job" + self.job.store_unchanged = "never" + self.job.enabled_sheet_ids = ["different-sheet"] + self.mapping.name = "Edited mapping" + self.db.commit() + + def execute(self): + return _execute_compare_into_run(self.db, self.run.id) + + def test_queued_run_uses_original_sheet_rules_mapping_and_storage(self): + original = deepcopy(self.run.summary_json["config_snapshot"]) + self.edit_current_settings() + detail = self.execute() + self.assertEqual(detail["status"], "success") + self.assertEqual(detail["summary"]["unchanged"], 1) + self.assertEqual(detail["summary"]["unchanged_listed"], 1) + self.assertEqual(detail["summary"]["store_unchanged"], "sample") + self.assertEqual(detail["sheets"][0]["compare_fields"], ["state"]) + self.assertEqual(self.run.summary_json["config_snapshot"], original) + + def _use_raw_port_map(self, version: int): + # Map written exactly as the device prints it (not the normalized form). + self.pair.before_if, self.pair.after_if = "GE-1", "GE-2" + self.db.commit() + self.run = _create_running_run( + self.db, job=self.job, tpl=self.tpl, before_batch_id="before", + after_batch_id="after", sheets_cfg=template_metrics(self.tpl), + ) + snap = deepcopy(self.run.summary_json) + snap["config_snapshot"]["version"] = version + self.run.summary_json = snap + self.db.commit() + + def test_v2_run_normalizes_raw_port_map_names(self): + self._use_raw_port_map(2) + detail = self.execute() + self.assertEqual(detail["summary"]["unchanged"], 1) + self.assertEqual(detail["summary"]["added"], 0) + self.assertEqual(detail["summary"]["removed"], 0) + + def test_v1_run_keeps_legacy_port_map_matching(self): + self._use_raw_port_map(1) + detail = self.execute() + self.assertEqual(detail["summary"]["unchanged"], 0) + self.assertEqual(detail["summary"]["added"], 1) + self.assertEqual(detail["summary"]["removed"], 1) + + def test_compact_hydration_restores_original_normalized_values(self): + self.execute() + self.edit_current_settings() + diff = list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged")["items"][0] + self.assertEqual(diff["before"]["interface"], "GE-1.100") + self.assertEqual(diff["mapped_before"]["interface"], "gei-2.100") + self.assertEqual(diff["after"]["interface"], "gei-2.100") + + def test_diff_rows_expose_the_before_side_name_as_printed(self): + self.execute() + diff = list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged")["items"][0] + self.assertEqual(diff["key"]["interface"], "gei-2.100") + self.assertEqual(diff["raw_key_before"], {"id": "1", "interface": "GE-1.100"}) + with zipfile.ZipFile(io.BytesIO(export_run_zip(self.db, self.run.id))) as archive: + rows = list(csv.DictReader(io.StringIO(archive.read("tables/sheet.csv").decode("utf-8-sig")))) + self.assertEqual(rows[0]["interface__raw_before"], "GE-1.100") + + def test_live_search_does_not_inherit_new_filters_or_transforms(self): + with patch("netx_api.biz_state.compare_service._UNCHANGED_SAMPLE_MAX", 0): + self.execute() + self.edit_current_settings() + result = list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged", kw="old") + self.assertEqual(result["source"], "live") + self.assertEqual(result["total"], 1) + self.assertEqual(result["items"][0]["mapped_before"]["interface"], "gei-2.100") + + def test_snapshot_can_be_read_after_mapping_and_template_are_deleted(self): + self.execute() + self.db.query(BizPortMappingRow).delete() + self.db.delete(self.mapping) + self.db.delete(self.tpl) + self.db.commit() + detail = get_run(self.db, self.run.id) + self.assertEqual(detail["template_name"], "Original template") + self.assertEqual(detail["mapping_name"], "Original mapping") + diff = list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged")["items"][0] + self.assertEqual(diff["mapped_before"]["interface"], "gei-2.100") + + def test_queued_run_can_execute_after_its_template_and_mapping_are_deleted(self): + self.db.query(BizPortMappingRow).delete() + self.db.delete(self.mapping) + self.db.delete(self.tpl) + self.db.commit() + detail = self.execute() + self.assertEqual(detail["status"], "success") + self.assertEqual(detail["summary"]["unchanged"], 1) + + def test_detail_returns_original_names_and_independent_template_copy(self): + self.edit_current_settings() + detail = get_run(self.db, self.run.id) + self.assertEqual(detail["job_name"], "Original job") + self.assertEqual(detail["config_snapshot_version"], 2) + self.assertEqual(detail["template"]["iface_normalize_rules"], [{"from": "GE", "to": "gei"}]) + self.assertNotIn("config_snapshot", detail["summary"]) + detail["template"]["iface_normalize_rules"].clear() + self.assertTrue(self.run.summary_json["config_snapshot"]["template"]["iface_normalize_rules"]) + + def test_export_contains_original_config_and_hydrated_csv(self): + self.execute() + self.edit_current_settings() + with zipfile.ZipFile(io.BytesIO(export_run_zip(self.db, self.run.id))) as archive: + config = json.loads(archive.read("config_snapshot.json")) + self.assertEqual(config["port_map"], {"gei-1": "gei-2"}) + self.assertEqual(config["template"]["name"], "Original template") + rows = list(csv.DictReader(io.StringIO(archive.read("tables/sheet.csv").decode("utf-8-sig")))) + self.assertEqual(rows[0]["interface"], "gei-2.100") + + def test_failed_and_cancelled_runs_keep_snapshot(self): + original = deepcopy(self.run.summary_json["config_snapshot"]) + with patch("netx_api.biz_state.compare_service._run_sheet", side_effect=RuntimeError("test-failure")): + with self.assertRaisesRegex(RuntimeError, "test-failure"): + self.execute() + self.assertEqual(self.run.summary_json["config_snapshot"], original) + self.run.status = "running" + self.db.commit() + cancel_compare_run(self.db, self.run.id) + self.assertEqual(self.run.summary_json["config_snapshot"], original) + + def test_legacy_runs_without_snapshot_remain_readable(self): + self.execute() + self.run.summary_json = {k: v for k, v in self.run.summary_json.items() if k != "config_snapshot"} + self.db.commit() + detail = get_run(self.db, self.run.id) + self.assertEqual(detail["status"], "success") + self.assertIsNone(detail["config_snapshot_version"]) + self.assertEqual(list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged")["total"], 1) + self.assertEqual(self.db.query(BizCompareDiff).count(), 1) diff --git a/tests/test_biz_state_compare_sql.py b/tests/test_biz_state_compare_sql.py new file mode 100644 index 0000000..44b2891 --- /dev/null +++ b/tests/test_biz_state_compare_sql.py @@ -0,0 +1,228 @@ +"""Unit tests for SQL compare eligibility and filter compilation.""" + +from __future__ import annotations + +import unittest +from unittest.mock import MagicMock + +from netx_api.biz_state.compare_sql import ( + can_sql_compare, + compile_row_filters_sql, + sql_compare_skip_reason, + _field_rules_sql_compatible, + _filters_sql_compatible, + _safe_field, +) + + +class _FakeDialect: + def __init__(self, name: str) -> None: + self.name = name + + +class _FakeBind: + def __init__(self, name: str) -> None: + self.dialect = _FakeDialect(name) + + +def _db(dialect: str = "postgresql") -> MagicMock: + db = MagicMock() + db.get_bind.return_value = _FakeBind(dialect) + return db + + +class CompareSqlGateTests(unittest.TestCase): + def test_safe_field_rejects_injection(self) -> None: + with self.assertRaises(ValueError): + _safe_field("a'; drop table x;--") + with self.assertRaises(ValueError): + _safe_field("x.y") + self.assertEqual(_safe_field("network"), "network") + + def test_requires_postgres(self) -> None: + sheet = { + "metric_id": "bgp_route", + "key_fields": ["network", "next_hop"], + "compare_fields": ["path"], + "iface_fields": [], + "field_rules": [], + "row_filters": [], + } + self.assertFalse(can_sql_compare(_db("sqlite"), sheet, port_map={})) + self.assertTrue(can_sql_compare(_db("postgresql"), sheet, port_map={})) + + def test_rejects_port_map(self) -> None: + sheet = { + "metric_id": "lldp_neighbor", + "key_fields": ["local_if", "remote_sys"], + "compare_fields": [], + "iface_fields": ["local_if"], + "ignore_port_changes": False, + "field_rules": [], + "row_filters": [], + } + self.assertFalse( + can_sql_compare(_db(), sheet, port_map={"gei-0/1": "gei-0/2"}) + ) + + def test_rejects_auto_ignore_ports_with_iface_key(self) -> None: + sheet = { + "metric_id": "lldp_neighbor", + "key_fields": ["local_if", "remote_sys"], + "compare_fields": [], + "iface_fields": ["local_if"], + "ignore_port_changes": None, + "field_rules": [], + "row_filters": [], + } + self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) + + def test_rejects_mac_normalize(self) -> None: + sheet = { + "metric_id": "arp", + "key_fields": ["ip", "vrf"], + "compare_fields": ["mac"], + "iface_fields": [], + "field_rules": [{"field": "mac", "normalize": "mac"}], + "row_filters": [], + } + self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) + + def test_rejects_age_timer_filter(self) -> None: + sheet = { + "metric_id": "arp", + "key_fields": ["ip"], + "compare_fields": [], + "iface_fields": [], + "field_rules": [], + "row_filters": [{"field": "age", "op": "age_timer"}], + } + self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) + + def test_accepts_bgp_route_style(self) -> None: + sheet = { + "metric_id": "bgp_route", + "key_fields": [ + "local_as", + "afi", + "vrf", + "neighbor", + "direction", + "rd", + "network", + "next_hop", + ], + "compare_fields": ["path", "as_num", "pfx_rcd"], + "iface_fields": [], + "field_rules": [ + {"field": "pfx_rcd", "compare": "percent", "tolerance": 5}, + ], + "row_filters": [ + {"field": "vrf", "op": "empty"}, + {"field": "afi", "op": "eq", "value": "ipv4"}, + ], + } + self.assertTrue(can_sql_compare(_db(), sheet, port_map={})) + # BGP afi/vrf sheet splits + percent rules must not force Python + self.assertEqual(sql_compare_skip_reason(_db(), sheet, port_map={}), "") + + def test_rejects_iface_normalize_when_key_uses_iface(self) -> None: + sheet = { + "metric_id": "interface_brief", + "key_fields": ["interface"], + "compare_fields": ["admin"], + "iface_fields": ["interface"], + "ignore_port_changes": False, + "field_rules": [], + "row_filters": [], + } + self.assertFalse( + can_sql_compare( + _db(), + sheet, + port_map={}, + iface_normalize_rules=[{"from": "GE", "to": "gei"}], + ) + ) + + def test_accepts_ignore_port_changes_false_without_normalize(self) -> None: + sheet = { + "metric_id": "interface_brief", + "key_fields": ["interface"], + "compare_fields": ["admin"], + "iface_fields": ["interface"], + "ignore_port_changes": False, + "field_rules": [], + "row_filters": [], + } + self.assertTrue(can_sql_compare(_db(), sheet, port_map={}, iface_normalize_rules=[])) + + +class CompareSqlFilterCompileTests(unittest.TestCase): + def test_empty_filters(self) -> None: + sql, params = compile_row_filters_sql([]) + self.assertEqual(sql, "TRUE") + self.assertEqual(params, {}) + + def test_eq_case_insensitive(self) -> None: + sql, params = compile_row_filters_sql( + [{"field": "afi", "op": "eq", "value": "IPv4"}] + ) + self.assertIn("lower(", sql) + self.assertIn("afi", sql) + self.assertEqual(list(params.values()), ["ipv4"]) + + def test_contains(self) -> None: + sql, params = compile_row_filters_sql( + [{"field": "af", "op": "contains", "value": "IPv4"}] + ) + self.assertIn("LIKE", sql) + self.assertEqual(list(params.values()), ["%ipv4%"]) + + def test_any_all_nesting(self) -> None: + sql, params = compile_row_filters_sql( + [ + { + "any": [ + {"field": "entry_type", "op": "eq", "value": "dynamic"}, + { + "all": [ + {"field": "entry_type", "op": "empty"}, + {"field": "ip", "op": "not_empty"}, + ] + }, + ] + } + ] + ) + self.assertIn(" OR ", sql) + self.assertIn(" AND ", sql) + self.assertTrue(_filters_sql_compatible([{"any": [{"field": "a", "op": "eq", "value": "1"}]}])) + + def test_in_list(self) -> None: + sql, params = compile_row_filters_sql( + [{"field": "afi", "op": "in", "value": ["ipv4", "ipv6"]}] + ) + self.assertIn(" IN (", sql) + self.assertEqual(sorted(params.values()), ["ipv4", "ipv6"]) + + def test_field_rules_matrix(self) -> None: + self.assertTrue(_field_rules_sql_compatible([{"field": "mac", "normalize": "lower"}])) + self.assertFalse(_field_rules_sql_compatible([{"field": "mac", "normalize": "mac"}])) + self.assertTrue( + _field_rules_sql_compatible( + [{"field": "rx", "compare": "numeric", "tolerance": 1}] + ) + ) + self.assertTrue( + _field_rules_sql_compatible( + [{"field": "pfx_rcd", "compare": "percent", "tolerance": 5}] + ) + ) + self.assertTrue( + _field_rules_sql_compatible([{"field": "x", "compare": "ignore"}]) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_biz_state_monitor_service.py b/tests/test_biz_state_monitor_service.py new file mode 100644 index 0000000..6f266cd --- /dev/null +++ b/tests/test_biz_state_monitor_service.py @@ -0,0 +1,144 @@ +"""Monitoring query correctness, bounded reads and active-collection protection.""" +import csv +import io +import zipfile +from datetime import datetime, timedelta + +import pytest +from fastapi import HTTPException +from sqlalchemy import create_engine, event +from sqlalchemy.orm import sessionmaker + +from netx_api.db import Base +from netx_api.models import ( + BizCompareRun, BizStateBatch, BizStateBatchCommand, BizStateLldpNeighbor, + BizStateMetricRow, BizStateTask, BizStateTaskItem, BizStateTaskItemBinding, +) +from netx_api.biz_state import service +from netx_api.biz_state.retention import protected_batch_map, purge_task_batches + + +@pytest.fixture +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine, expire_on_commit=False)() + session.add(BizStateTask(id="t", ne_id="ne", retention_days=1)) + session.add(BizStateBatch(id="b", task_id="t", status="success")) + session.commit() + yield session + session.close() + engine.dispose() + + +def test_task_detail_uses_one_binding_query_and_progress_omits_items(db): + for n in range(30): + db.add(BizStateTaskItem(id=f"i{n}", task_id="t", sort_order=n)) + for ph, val in (("vrf", f"v{n}"), ("neighbor", f"p{n}")): + db.add(BizStateTaskItemBinding(id=f"{n}-{ph}", item_id=f"i{n}", placeholder=ph, value=val)) + db.commit() + db.expunge_all() + statements = [] + event.listen(db.get_bind(), "before_cursor_execute", lambda c, cur, sql, p, ctx, many: statements.append(sql)) + result = service.get_task(db, "t") + assert len(statements) == 3 + assert len(result["items"]) == 30 + assert result["items"][9]["sort_order"] == 9 + assert {b["value"] for b in result["items"][9]["bindings"]} == {"v9", "p9"} + db.expunge_all() + statements.clear() + progress = service.get_task_progress(db, "t") + assert len(statements) == 1 + assert "items" not in progress + assert progress["id"] == "t" + with pytest.raises(HTTPException) as exc: + service.get_task_progress(db, "missing") + assert exc.value.status_code == 404 + + +@pytest.mark.parametrize("metric,column", [("custom", "name"), ("lldp_neighbor", "remote_sys")]) +@pytest.mark.parametrize("keyword", ["%", "_", "\\"]) +def test_metric_search_treats_wildcards_literally(db, metric, column, keyword): + for n, value in enumerate((f"exact{keyword}value", "different-value")): + if metric == "lldp_neighbor": + db.add(BizStateLldpNeighbor(id=f"r{n}", batch_id="b", local_if=f"ge{n}", remote_sys=value)) + else: + db.add(BizStateMetricRow(id=f"r{n}", batch_id="b", metric_id=metric, seq=n, data_json={column: value})) + db.commit() + result = service.list_batch_metric_rows(db, "b", metric, kw=keyword, column=column) + assert result["total"] == 1 + assert result["items"][0][column] == f"exact{keyword}value" + + +def test_summary_does_not_select_raw_text_and_keeps_early_aux_failure(db): + now = datetime(2026, 10, 10) + for cid, status, raw, stored in ( + ("a", "aux_failed", "first\nsecond\n", 0), + ("b", "ok", "X" * 1_000_000, 999), + ("c", "aux_cached", " \t\n", 0), + ): + db.add(BizStateBatchCommand(id=cid, batch_id="b", raw_command="show arp" if cid != "c" else "show config", + metric_id="arp", parse_status=status, raw_text=raw, raw_line_count=stored, created_at=now)) + db.commit() + db.expunge_all() + column_names = [] + event.listen(db.get_bind(), "after_cursor_execute", lambda c, cur, sql, p, ctx, many: + column_names.extend([d[0] for d in (cur.description or [])]) if "biz_state_batch_command" in sql else None) + summary = service.get_batch(db, "b") + cmds = {c["id"]: c for c in summary["commands"]} + assert "biz_state_batch_command_raw_text" not in column_names + assert cmds["a"]["raw_line_count"] == 2 + assert cmds["b"]["raw_line_count"] == 999 + assert cmds["b"]["has_raw"] is True + assert cmds["c"]["has_raw"] is False + assert summary["command_stats"]["aux_failed"] == 1 + assert service.get_batch_command(db, "b", "b")["raw_text"] == "X" * 1_000_000 + + +def test_active_batches_cannot_be_deleted_or_purged(db): + for status in ("queued", "running"): + db.add(BizStateBatch(id=status, task_id="t", status=status, started_at=datetime.utcnow() - timedelta(days=3))) + db.commit() + for bid in ("queued", "running"): + with pytest.raises(HTTPException) as exc: + service.delete_batch(db, bid) + assert exc.value.status_code == 409 + assert "active_collection" in exc.value.detail["reasons"] + result = service.delete_batches_bulk(db, ["queued", "running"]) + assert result["deleted"] == [] + purge_task_batches(db, db.get(BizStateTask, "t")) + assert db.get(BizStateBatch, "queued") is not None + assert db.get(BizStateBatch, "running") is not None + db.get(BizStateTask, "t").collect_running = True + db.commit() + with pytest.raises(HTTPException) as exc: + service.delete_task(db, "t") + assert exc.value.detail == "task_collecting" + + +def test_protection_reads_only_scoped_reference_columns(db): + db.add(BizStateBatch(id="other", task_id="other-task", status="success")) + db.add(BizCompareRun(id="r", before_batch_id="b", after_batch_id="other", summary_json={"large": "x" * 10000})) + db.commit() + db.expunge_all() + statements = [] + event.listen(db.get_bind(), "before_cursor_execute", lambda c, cur, sql, p, ctx, many: statements.append(sql)) + assert protected_batch_map(db, batch_ids=["b"]) == {"b": ["compare_run_before"]} + assert all("summary_json" not in sql for sql in statements) + assert protected_batch_map(db, task_id="t") == {"b": ["compare_run_before"]} + assert protected_batch_map(db, batch_ids=[]) == {} + + +def test_export_preserves_false_zero_csv_and_duplicate_sequences(db): + for n in range(1005): + rec = {"n": n, "enabled": False, "note": 'a,b"c\rd\ne' if n == 0 else None} + if n == 1004: + rec["late"] = "last-column" + db.add(BizStateMetricRow(id=f"r{n:04}", batch_id="b", metric_id="custom", seq=0, data_json=rec)) + db.commit() + with zipfile.ZipFile(io.BytesIO(service.export_batch_zip(db, "b"))) as archive: + rows = list(csv.DictReader(io.StringIO(archive.read("tables/custom.csv").decode("utf-8"), newline=""))) + assert len(rows) == 1005 + assert rows[0] == {"n": "0", "enabled": "False", "note": 'a,b"c\rd\ne', "late": ""} + assert rows[-1]["late"] == "last-column" + assert [row["n"] for row in rows] == [str(n) for n in range(1005)] diff --git a/tests/test_biz_state_paired_sampling.py b/tests/test_biz_state_paired_sampling.py new file mode 100644 index 0000000..22f6923 --- /dev/null +++ b/tests/test_biz_state_paired_sampling.py @@ -0,0 +1,186 @@ +"""Paired admission/dispatch, actual cadence, and per-metric isolation.""" +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from threading import Barrier, Event + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from netx_api.db import Base +from netx_api.models import BizStateTask, BizStateTaskItem, BizStateBatch +from netx_api.timeutil import utcnow_naive +from netx_api.biz_state import claim +from netx_api import biz_state_scheduler as scheduler + + +@pytest.fixture +def db(monkeypatch): + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False) + monkeypatch.setattr(claim, "SessionLocal", factory) + monkeypatch.setattr(scheduler, "SessionLocal", factory) + with factory() as session: + for tid in ("old", "new"): + session.add(BizStateTask(id=tid, source="managed", ne_id=tid, + status="running", interval_sec=60, collect_group_id="pair")) + session.add(BizStateTaskItem(id="item-" + tid, task_id=tid, source_profile_id="zte.arp", enabled=True)) + session.commit() + yield session + scheduler._in_flight.clear() + + +def test_individual_trigger_enqueues_both_with_one_identity(db): + result = claim.enqueue_collect("new", manual=True) + assert result["queued"] and result["task_id"] == "new" + rows = db.query(BizStateBatch).all() + assert len(rows) == 2 + assert {b.task_id for b in rows} == {"old", "new"} + assert len({b.collect_round_id for b in rows}) == 1 + assert len({b.queued_at for b in rows}) == 1 + assert not claim.enqueue_collect("old", manual=True)["queued"] + + +@pytest.mark.parametrize("problem", ["busy", "no_items", "paused", "same_device"]) +def test_pair_admission_never_creates_half_round(db, problem): + peer = db.get(BizStateTask, "new") + if problem == "busy": + peer.collect_running = True + elif problem == "no_items": + db.delete(db.get(BizStateTaskItem, "item-new")) + elif problem == "paused": + peer.status = "paused" + else: + peer.ne_id = "old" + db.commit() + assert not claim.enqueue_collect("old")["queued"] + assert db.query(BizStateBatch).count() == 0 + + +def test_pair_claim_requires_two_slots(db): + claim.enqueue_collect("old") + assert claim.claim_queued_batches(1) == [] + assert len(claim.claim_queued_batches(2)) == 2 + + +def test_busy_device_keeps_both_queued_and_no_ssh_overlap(db, monkeypatch): + # These cases are about one collection per device; the cap defaults to 2. + monkeypatch.setattr(claim, "max_sessions_per_ne", lambda: 1) + db.add(BizStateBatch(id="portrait", task_id="other", ne_id="old", status="running")) + db.commit() + claim.enqueue_collect("old") + assert claim.claim_queued_batches(8) == [] + assert db.query(BizStateBatch).filter(BizStateBatch.status == "queued").count() == 2 + + +def test_capacity_one_has_actionable_admission_error(db, monkeypatch): + monkeypatch.setattr(claim, "max_concurrent_tasks", lambda: 1) + assert claim.enqueue_collect("old")["reason"] == "collect_pair_capacity" + assert db.query(BizStateBatch).count() == 0 + + +def test_manual_routes_are_excluded_from_periodic_scheduler(db): + for t in db.query(BizStateTask).all(): + t.collect_manual_only = True + db.commit() + assert scheduler._enqueue_due_tasks() == 0 + assert claim.enqueue_collect("old", manual=True)["queued"] + + +def test_pair_cadence_uses_later_start_and_slow_peer_blocks_restart(db): + now = utcnow_naive() + db.get(BizStateTask, "old").last_collect_started_at = now - timedelta(seconds=100) + db.get(BizStateTask, "new").last_collect_started_at = now - timedelta(seconds=10) + db.commit() + assert scheduler._enqueue_due_tasks() == 0 + db.get(BizStateTask, "new").last_collect_started_at = now - timedelta(seconds=100) + db.get(BizStateTask, "new").collect_running = True + db.commit() + assert scheduler._enqueue_due_tasks() == 0 + db.get(BizStateTask, "new").collect_running = False + db.commit() + assert scheduler._enqueue_due_tasks() == 1 + assert db.query(BizStateBatch).count() == 2 + + +def test_inline_pair_uses_two_workers_without_waiting_for_one_side(db, monkeypatch): + from netx_api.biz_state import collect_runner + + both_started, release = Barrier(2), Event() + started, synchronized = [], [] + def fake_collect(**job): + started.append(job["task_id"]) + both_started.wait(timeout=3) + synchronized.append(job["task_id"]) + release.wait(timeout=3) + monkeypatch.setattr(scheduler, "execute_claimed_batch", fake_collect) + monkeypatch.setattr(collect_runner, "_should_execute_inline", lambda: True) + with ThreadPoolExecutor(max_workers=2) as pool: + monkeypatch.setattr(scheduler, "_dispatch_pool_get", lambda: pool) + result = collect_runner.dispatch_collect("old", manual=True) + assert result["queued"] + # Both worker entries must reach a barrier; serial old→new would fail it. + release.set() + assert set(started) == {"old", "new"} + assert set(synchronized) == {"old", "new"} + + +def test_faster_metrics_claim_before_manual_route_round(db, monkeypatch): + # These cases are about one collection per device; the cap defaults to 2. + monkeypatch.setattr(claim, "max_sessions_per_ne", lambda: 1) + for tid in ("ro", "rn"): + db.add(BizStateTask(id=tid, ne_id="old" if tid == "ro" else "new", + status="paused", collect_group_id="routes", collect_manual_only=True)) + db.add(BizStateTaskItem(id="i-" + tid, task_id=tid, source_profile_id="zte.ip_route", enabled=True)) + db.commit() + claim.enqueue_collect("ro", manual=True) + claim.enqueue_collect("old") + jobs = claim.claim_queued_batches(4) + assert {job["task_id"] for job in jobs} == {"old", "new"} + + +def test_manual_route_round_cannot_starve_behind_continuous_fast_metrics(db, monkeypatch): + # These cases are about one collection per device; the cap defaults to 2. + monkeypatch.setattr(claim, "max_sessions_per_ne", lambda: 1) + for tid in ("ro", "rn"): + db.add(BizStateTask(id=tid, ne_id="old" if tid == "ro" else "new", + status="paused", collect_group_id="routes", collect_manual_only=True)) + db.add(BizStateTaskItem(id="i-" + tid, task_id=tid, source_profile_id="zte.ip_route", enabled=True)) + db.commit() + claim.enqueue_collect("ro", manual=True) + claim.enqueue_collect("old") + for batch in db.query(BizStateBatch).filter(BizStateBatch.collect_group_id == "routes").all(): + batch.queued_at = utcnow_naive() - timedelta(seconds=301) + db.commit() + assert {job["task_id"] for job in claim.claim_queued_batches(4)} == {"ro", "rn"} + + +@pytest.mark.parametrize("running", [False, True]) +def test_stopping_one_side_stops_the_whole_round(db, monkeypatch, running): + from netx_api.biz_state import collect_stop + + monkeypatch.setattr(collect_stop, "SessionLocal", claim.SessionLocal) + claim.enqueue_collect("old") + if running: + claim.claim_queued_batches(2) + result = collect_stop.request_stop_collect("new") + assert result["ok"] and result["stopped"] + db.expire_all() + batches = db.query(BizStateBatch).all() + try: + if running: + assert len(result["signaled_batches"]) == 2 + assert all(b.message == collect_stop.STOP_REQUEST_TOKEN for b in batches) + assert all(t.collect_running for t in db.query(BizStateTask).all()) + else: + assert len(result["cancelled_batches"]) == 2 + assert all(b.status == "cancelled" for b in batches) + assert not any(t.collect_running for t in db.query(BizStateTask).all()) + assert claim.claim_queued_batches(2) == [] + assert claim.enqueue_collect("new")["queued"] + finally: + for batch in batches: + collect_stop.clear_stop_requested(batch.id) + diff --git a/tests/test_compare_preflight.py b/tests/test_compare_preflight.py new file mode 100644 index 0000000..408b380 --- /dev/null +++ b/tests/test_compare_preflight.py @@ -0,0 +1,167 @@ +"""Mapping pre-flight (B5), N:1 reporting (B7) and command-slice coverage (V2).""" +import pytest +from fastapi import HTTPException +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_migration import service as mig +from netx_api.biz_state import compare_service as svc +from netx_api.biz_state.compare_quality import classify_sheet, covers_slice, sheet_slice +from netx_api.db import Base +from netx_api.models import ( + BizCompareTemplate, BizMigrationProject, BizMonitorTemplate, BizPortMapping, BizPortMappingRow, + BizStateBatch, BizStateBatchCommand, BizStateMetricRow, BizStateTask, +) + + +@pytest.fixture() +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine, expire_on_commit=False)() + yield session + session.close() + engine.dispose() + + +SHEET = {"sheet_id": "ifb", "metric_id": "iface_t", "key_fields": ["interface"], "iface_fields": ["interface"], + "compare_fields": ["state"], "field_rules": [], "row_filters": []} + + +def _batches(db, before_ifaces, after_ifaces, *, receipts=True): + db.add_all([BizStateTask(id="t1", source="managed", ne_id="a", vendor="ZTE"), + BizStateTask(id="t2", source="managed", ne_id="b", vendor="ZTE"), + BizStateBatch(id="b1", task_id="t1", status="success"), + BizStateBatch(id="b2", task_id="t2", status="success"), + BizCompareTemplate(id="tpl", name="t", metrics_json=[SHEET], + iface_normalize_json=[{"from": "GE", "to": "gei"}])]) + for batch, names in (("b1", before_ifaces), ("b2", after_ifaces)): + for i, name in enumerate(names): + db.add(BizStateMetricRow(id=f"{batch}-{i}", batch_id=batch, metric_id="iface_t", seq=i, + data_json={"interface": name, "state": "up"})) + if receipts: + db.add(BizStateBatchCommand(id=f"c-{batch}", batch_id=batch, metric_id="iface_t", + profile_id="zte.x", parse_status="ok", row_count=len(names), + raw_line_count=len(names) + 3)) + db.commit() + + +def _preflight(db, **extra): + return svc.preflight_compare(db, {"template_id": "tpl", "before_batch_id": "b1", "after_batch_id": "b2", **extra}) + + +def test_exact_parent_same_name_and_suspect_are_classified(db): + _batches(db, ["GE0/1/0/1", "GE0/1/0/2.100", "GE0/1/0/3", "GE0/1/0/9"], + ["gei0/2/0/1", "gei0/2/0/2.100", "gei0/1/0/3"]) + report = _preflight(db, mapping_rows=[{"before_if": "GE0/1/0/1", "after_if": "GE0/2/0/1"}, + {"before_if": "GE0/1/0/2", "after_if": "GE0/2/0/2"}]) + (sheet,) = report["sheets"] + assert sheet["mapped_exact"] == ["gei0/1/0/1"] + assert sheet["mapped_by_parent"] == ["gei0/1/0/2.100"] + assert sheet["identity_same_name"] == ["gei0/1/0/3"] + assert sheet["suspect_unmapped"] == ["gei0/1/0/9"] + assert report["coverage"] == {"mapped": 2, "suspect_unmapped": 1, "identity_same_name": 1, "rate": 66.7} + assert report["verdict"] == "risk" and "suspect_unmapped" in report["risk_reasons"] + + +def test_unsaved_rows_are_enough_and_clean_map_is_ready(db): + _batches(db, ["GE0/1"], ["gei0/2"]) + report = _preflight(db, mapping_rows=[{"before_if": "GE0/1", "after_if": "GE0/2"}]) + assert report["verdict"] == "ready" and report["coverage"]["rate"] == 100.0 + assert report["mapping"]["unused_map_keys"] == [] + + +def test_unused_keys_missing_targets_and_n_to_1_are_listed(db): + _batches(db, ["GE0/1", "GE0/2"], ["gei0/9"]) + report = _preflight(db, mapping_rows=[ + {"before_if": "GE0/1", "after_if": "GE0/9"}, {"before_if": "GE0/2", "after_if": "GE0/9"}, + {"before_if": "GE0/77", "after_if": "GE0/88"}]) + m = report["mapping"] + assert m["n_to_1_targets"] == [{"target": "gei0/9", "sources": ["gei0/1", "gei0/2"]}] + assert m["unused_map_keys"] == ["gei0/77"] + assert m["map_targets_missing_on_after"] == ["gei0/88"] + assert report["verdict"] == "risk" + + +def test_duplicate_before_rows_are_reported(db): + _batches(db, ["GE0/1"], ["gei0/2"]) + report = _preflight(db, mapping_rows=[{"before_if": "GE0/1", "after_if": "GE0/2"}, + {"before_if": "GE0/1", "after_if": "GE0/3"}]) + assert report["mapping"]["duplicate_before"][0]["dropped_to"] == "GE0/3" + assert report["verdict"] == "risk" + + +def test_saved_mapping_id_is_used_when_no_rows_given(db): + _batches(db, ["GE0/1"], ["gei0/2"]) + db.add_all([BizPortMapping(id="m", name="m"), + BizPortMappingRow(mapping_id="m", before_if="GE0/1", after_if="GE0/2")]) + db.commit() + assert _preflight(db, mapping_id="m")["coverage"]["mapped"] == 1 + + +def test_missing_batch_is_404(db): + _batches(db, ["GE0/1"], ["gei0/2"]) + with pytest.raises(HTTPException) as exc: + svc.preflight_compare(db, {"template_id": "tpl", "before_batch_id": "nope", "after_batch_id": "b2"}) + assert exc.value.status_code == 404 + + +def test_n_to_1_is_rejected_by_monitoring_with_the_colliding_rows(): + with pytest.raises(HTTPException) as exc: + mig._reject_ambiguous_mapping({"a": "x", "b": "x", "c": "y"}) + assert exc.value.status_code == 400 + assert exc.value.detail["code"] == "mapping_target_ambiguous" + assert exc.value.detail["n_to_1"] == [{"target": "x", "sources": ["a", "b"]}] + assert "a, b -> x" in exc.value.detail["message"] + mig._reject_ambiguous_mapping({"a": "x", "b": "y"}) + + +def test_project_preflight_uses_the_portrait_baselines(db): + _batches(db, ["GE0/1", "GE0/5"], ["gei0/2"]) + db.add_all([BizPortMapping(id="m", name="m"), + BizPortMappingRow(mapping_id="m", before_if="GE0/1", after_if="GE0/2"), + BizMonitorTemplate(id="mt", name="mt", compare_template_id="tpl"), + BizMigrationProject(id="p", name="p", mapping_id="m", monitor_template_id="mt", old_task_id="t1", + new_task_id="t2", old_baseline_batch_id="b1", new_baseline_batch_id="b2")]) + db.commit() + report = mig.preflight_project(db, "p") + assert report["sheets"][0]["suspect_unmapped"] == ["gei0/5"] + assert report["verdict"] == "risk" + + +# ---- V2: slices --------------------------------------------------------------------------- + +def receipts(*profile_ids, status="ok"): + return [{"profile_id": p, "parse_status": status, "row_count": 1, "raw_line_count": 4} for p in profile_ids] + + +def test_sheet_slice_reads_only_top_level_eq_filters(): + s = {"row_filters": [{"op": "eq", "field": "afi", "value": "IPv4"}, + {"op": "contains", "field": "x", "value": "y"}, + {"any": [{"op": "eq", "field": "z", "value": "1"}]}]} + assert sheet_slice(s) == {"afi": "ipv4"} + + +def test_missing_afi_command_means_not_collected_not_lost(): + only_vpnv4 = receipts("zte.bgp_vpnv4_summary") + both = receipts("zte.bgp_vpnv4_summary", "zte.bgp_ipv4_summary") + state, reasons = classify_sheet(before=both, after=only_vpnv4, before_rows=3, after_rows=0, + wanted_slice={"afi": "ipv4"}) + assert state == "not_collected" and reasons == ["after:slice_not_collected"] + ok, _ = classify_sheet(before=both, after=both, before_rows=3, after_rows=3, wanted_slice={"afi": "ipv4"}) + assert ok == "ok" + + +def test_command_without_declared_slice_covers_everything(): + assert covers_slice(receipts("custom.unknown"), {"afi": "ipv4"}) + assert not covers_slice(receipts("zte.bgp_vpnv6_summary"), {"afi": "ipv4"}) + assert not covers_slice(receipts("zte.bgp_ipv4_summary", status="failed"), {"afi": "ipv4"}) + + +def test_command_coverage_compares_what_commands_produce(db): + _batches(db, ["GE0/1"], ["gei0/2"]) + db.add(BizStateBatchCommand(id="x", batch_id="b1", metric_id="bgp_peer", profile_id="zte.bgp_ipv4_summary", + parse_status="ok")) + db.commit() + cov = _preflight(db, mapping_rows=[])["command_coverage"] + assert cov["only_before"] == ["bgp_peer{afi=ipv4}"] diff --git a/tests/test_compare_quality.py b/tests/test_compare_quality.py new file mode 100644 index 0000000..c6079a4 --- /dev/null +++ b/tests/test_compare_quality.py @@ -0,0 +1,163 @@ +"""Sheet data quality (V1) and directional compare modes (V3).""" +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_state import compare_service as svc +from netx_api.biz_state.compare_rules import explain_diff, values_equal +from netx_api.biz_state.identity_pipeline import build_pipeline +from netx_api.db import Base +from netx_api.models import BizStateBatch, BizStateBatchCommand, BizStateMetricRow, BizStateTask + +SHEET = {"sheet_id": "peers", "metric_id": "bgp_peer", "key_fields": ["neighbor"], "iface_fields": [], + "compare_fields": ["state"], "field_rules": [], "row_filters": []} + + +@pytest.fixture() +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine, expire_on_commit=False)() + session.add_all([BizStateTask(id="t", source="managed", ne_id="n"), + BizStateBatch(id="b1", task_id="t", status="success"), + BizStateBatch(id="b2", task_id="t", status="success")]) + session.commit() + yield session + session.close() + engine.dispose() + + +def collect(db, batch, rows, *, lines=None, status="ok", command=True): + if command: + db.add(BizStateBatchCommand(id=f"c-{batch}", batch_id=batch, metric_id="bgp_peer", profile_id="zte.x", + parse_status=status, row_count=len(rows), + raw_line_count=len(rows) + 2 if lines is None else lines)) + for i, peer in enumerate(rows): + db.add(BizStateMetricRow(id=f"{batch}-{i}", batch_id=batch, metric_id="bgp_peer", seq=i, + data_json={"neighbor": peer, "state": "up"})) + db.commit() + + +def run(db): + return svc._run_sheet(db, sheet=SHEET, before_batch_id="b1", after_batch_id="b2", + pipeline=build_pipeline(SHEET, iface_rules=[], port_map={}))["summary"] + + +def enrich(summary): + sheet = {"sheet_id": "peers", "metric_id": "bgp_peer", "status": "done", "summary": summary} + return svc._enrich_summary({k: summary.get(k, 0) for k in + ("added", "removed", "changed", "unchanged", "before_count", "after_count")}, [sheet]) + + +def test_empty_against_empty_is_no_data_not_a_hundred_percent(db): + collect(db, "b1", [], lines=0) + collect(db, "b2", [], lines=0) + summary = run(db) + assert summary["data_state"] == "no_data" + out = enrich(summary) + assert out["pass_rate"] is None and out["sheet_cards"][0]["pass_rate"] is None + assert out["ok"] is None + + +def test_unparsed_output_is_suspect_parse_and_leaves_the_verdict_open(db): + collect(db, "b1", [f"p{i}" for i in range(10)]) + collect(db, "b2", [], lines=40) # 40 CLI lines, 0 rows parsed + summary = run(db) + assert summary["data_state"] == "suspect_parse" + assert summary["data_state_reasons"] == ["after:parsed_0_of_40_lines"] + out = enrich(summary) + assert out["sheet_cards"][0]["removed"] == 10 # still shown + assert out["ok"] is None and out["suspect_sheets"] == 1 + + +def test_side_without_any_command_is_not_collected_and_leaves_the_totals(db): + collect(db, "b1", ["a", "b"]) + db.add(BizStateBatchCommand(id="other", batch_id="b2", metric_id="other_metric", profile_id="zte.y", + parse_status="ok", row_count=1, raw_line_count=1)) + db.commit() + summary = run(db) + assert summary["data_state"] == "not_collected" + out = enrich(summary) + assert out["sheet_cards"][0]["pass_rate"] is None + assert out["fail_count"] == 0 and out["ok"] is None and out["unjudged_sheets"] == 1 + + +def test_batches_without_receipts_are_trusted(db): + collect(db, "b1", ["a"], command=False) + collect(db, "b2", ["a"], command=False) + summary = run(db) + assert summary["data_state"] == "ok" + assert enrich(summary)["pass_rate"] == 100.0 + + +def test_normal_comparison_keeps_the_verdict(db): + collect(db, "b1", ["a", "b"]) + collect(db, "b2", ["a"]) + out = enrich(run(db)) + assert out["ok"] is False and out["fail_count"] == 1 and out["pass_rate"] == 50.0 + + +# ---- V3 ----------------------------------------------------------------------------------- + +NOT_WORSE = {"compare": "not_worse", "rank": ["up", "down"]} + + +def test_not_worse_accepts_recovery_and_rejects_degradation(): + assert values_equal("down", "up", rule=NOT_WORSE) + assert values_equal("up", "up", rule=NOT_WORSE) + assert not values_equal("up", "down", rule=NOT_WORSE) + assert explain_diff("up", "down", rule=NOT_WORSE) == "worse: up→down" + + +def test_not_worse_falls_back_to_equality_for_unranked_values(): + assert not values_equal("up", "flapping", rule=NOT_WORSE) + assert values_equal("flapping", "flapping", rule=NOT_WORSE) + + +def test_numeric_min_allows_growth_and_bounded_loss(): + rule = {"compare": "numeric_min", "tolerance": 5} + assert values_equal("1000", "1200", rule=rule) + assert values_equal("1000", "960", rule=rule) + assert not values_equal("1000", "900", rule=rule) + assert explain_diff("1000", "900", rule=rule) == "drop 10% > 5%" + + +def test_numeric_max_is_the_mirror_image(): + rule = {"compare": "numeric_max", "tolerance": 10} + assert values_equal("100", "60", rule=rule) + assert values_equal("100", "110", rule=rule) + assert not values_equal("100", "150", rule=rule) + assert explain_diff("100", "150", rule=rule) == "rise 50% > 10%" + + +def test_directional_modes_pass_validation_and_keep_their_rank(): + from netx_api.biz_state.compare_validation import validate_template_body + + sheet = {"metric_id": "m", "key_fields": ["k"], "compare_fields": ["s"], + "field_rules": [{"field": "s", "compare": "not_worse", "rank": ["up", "down"]}]} + validate_template_body({"metrics": [sheet]}) + normalized = svc._normalize_sheet(sheet) + assert normalized["field_rules"][0]["rank"] == ["up", "down"] + with pytest.raises(Exception): + validate_template_body({"metrics": [{**sheet, "field_rules": [{"field": "s", "compare": "not_worse"}]}]}) + + +def test_directional_compare_runs_end_to_end_and_blocks_sql(db): + from tests.test_identity_pipeline import pg_db + from netx_api.biz_state.compare_engine import compare_rows + from netx_api.biz_state.compare_sql import sql_compare_skip_reason + + sheet = {**SHEET, "field_rules": [{"field": "state", **NOT_WORSE}]} + out = compare_rows(before_rows=[{"neighbor": "a", "state": "down"}, {"neighbor": "b", "state": "up"}], + after_rows=[{"neighbor": "a", "state": "up"}, {"neighbor": "b", "state": "down"}], + key_fields=["neighbor"], iface_fields=[], compare_fields=["state"], + field_rules=sheet["field_rules"]) + assert (out["summary"]["unchanged"], out["summary"]["changed"]) == (1, 1) + assert sql_compare_skip_reason(pg_db(), sheet) == "directional_compare" + + +def test_sorted_pairing_is_the_default_for_new_runs_only(): + assert build_pipeline(SHEET, iface_rules=[], port_map={}).duplicate_pairing == "sorted" + assert build_pipeline({**SHEET, "duplicate_pairing": "ordered"}, iface_rules=[], + port_map={}).duplicate_pairing == "ordered" + assert build_pipeline(SHEET, iface_rules=[], port_map={}, legacy=True).duplicate_pairing == "ordered" diff --git a/tests/test_cutover_references.py b/tests/test_cutover_references.py new file mode 100644 index 0000000..0f97c36 --- /dev/null +++ b/tests/test_cutover_references.py @@ -0,0 +1,204 @@ +"""Reference protection and input validation for compare/cutover (plan A2, A5).""" +import pytest +from fastapi import HTTPException +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_migration import service as mig +from netx_api.biz_state import compare_service as svc +from netx_api.db import Base +from netx_api.models import ( + BizCompareJob, BizCompareTemplate, BizMigrationProject, BizMonitorTemplate, + BizPortMapping, BizPortMappingRow, BizStateBatch, BizStateTask, +) + + +@pytest.fixture() +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine, expire_on_commit=False)() + yield session + session.close() + engine.dispose() + + +def _template(db, tid="tpl"): + db.add(BizCompareTemplate(id=tid, name=f"name-{tid}", metrics_json=[])) + db.commit() + + +def _mapping(db, mid="map"): + db.add(BizPortMapping(id=mid, name=f"map-{mid}")) + db.add(BizPortMappingRow(mapping_id=mid, before_if="a", after_if="b")) + db.commit() + + +def test_mapping_used_by_job_cannot_be_deleted(db): + _mapping(db) + db.add(BizCompareJob(id="j", name="job-1", template_id="tpl", mapping_id="map")) + db.commit() + with pytest.raises(HTTPException) as exc: + svc.delete_mapping(db, "map") + assert exc.value.status_code == 409 + assert exc.value.detail["code"] == "mapping_in_use" + assert exc.value.detail["jobs"] == ["job-1"] + assert db.get(BizPortMapping, "map") is not None + + +def test_mapping_used_by_project_cannot_be_deleted(db): + _mapping(db) + db.add(BizMigrationProject(id="p", name="proj-1", mapping_id="map")) + db.commit() + with pytest.raises(HTTPException) as exc: + svc.delete_mapping(db, "map") + assert exc.value.status_code == 409 + assert exc.value.detail["projects"] == ["proj-1"] + assert "proj-1" in exc.value.detail["message"] + + +def test_unreferenced_mapping_is_deleted_with_rows(db): + _mapping(db) + svc.delete_mapping(db, "map") + assert db.get(BizPortMapping, "map") is None + assert db.query(BizPortMappingRow).count() == 0 + + +def test_list_mappings_reports_reference_counts(db): + _mapping(db) + db.add_all([BizCompareJob(id="j1", template_id="tpl", mapping_id="map"), + BizCompareJob(id="j2", template_id="tpl", mapping_id="map"), + BizMigrationProject(id="p", name="p", mapping_id="map")]) + db.commit() + (item,) = svc.list_mappings(db) + assert (item["job_count"], item["project_count"]) == (2, 1) + + +def test_template_used_by_monitor_template_cannot_be_deleted(db): + _template(db) + db.add(BizMonitorTemplate(id="mt", name="monitor-1", compare_template_id="tpl")) + db.commit() + with pytest.raises(HTTPException) as exc: + svc.delete_template(db, "tpl") + assert exc.value.status_code == 409 + assert exc.value.detail["code"] == "template_in_use" + assert exc.value.detail["monitor_templates"] == ["monitor-1"] + assert exc.value.detail["jobs"] == [] + + +def test_template_used_by_job_reports_job_names(db): + _template(db) + db.add(BizCompareJob(id="j", name="job-1", template_id="tpl")) + db.commit() + with pytest.raises(HTTPException) as exc: + svc.delete_template(db, "tpl") + assert exc.value.detail["jobs"] == ["job-1"] + + +def test_list_templates_counts_monitor_templates(db, monkeypatch): + monkeypatch.setattr(svc, "ensure_default_templates", lambda db: None) + monkeypatch.setattr(svc, "upgrade_builtin_split_sheets", lambda db: None) + _template(db) + db.add_all([BizMonitorTemplate(id="m1", compare_template_id="tpl"), + BizMonitorTemplate(id="m2", compare_template_id="tpl")]) + db.commit() + (item,) = svc.list_templates(db) + assert item["monitor_template_count"] == 2 + + +def test_job_with_deleted_mapping_is_rejected_not_silently_unmapped(db): + _template(db) + db.add(BizCompareJob(id="j", template_id="tpl", mapping_id="gone", + before_batch_id="b", after_batch_id="a")) + db.commit() + with pytest.raises(HTTPException) as exc: + svc._validate_compare_job(db, "j") + assert exc.value.status_code == 400 + assert exc.value.detail == "mapping_missing" + + +def _project_tasks(db): + db.add_all([BizStateTask(id="old_p", source="managed", ne_id="ne-old", status="running"), + BizStateTask(id="new_p", source="managed", ne_id="ne-new", status="running"), + BizStateTask(id="other", source="managed", ne_id="ne-x", status="running")]) + db.add_all([BizStateBatch(id="old_ok", task_id="old_p", status="success"), + BizStateBatch(id="new_ok", task_id="new_p", status="partial"), + BizStateBatch(id="foreign", task_id="other", status="success"), + BizStateBatch(id="old_failed", task_id="old_p", status="failed")]) + db.commit() + + +def _body(**extra): + return {"name": "n", "old_task_id": "old_p", "new_task_id": "new_p", **extra} + + +@pytest.mark.parametrize("extra,code", [ + ({"old_baseline_batch_id": "foreign"}, "old_baseline_invalid"), + ({"new_baseline_batch_id": "foreign"}, "new_baseline_invalid"), + ({"old_baseline_batch_id": "old_failed"}, "old_baseline_invalid"), + ({"new_baseline_batch_id": "old_ok"}, "new_baseline_invalid"), +]) +def test_create_project_rejects_baseline_from_wrong_task(db, extra, code): + _project_tasks(db) + with pytest.raises(HTTPException) as exc: + mig.create_project(db, _body(**extra)) + assert exc.value.status_code == 400 + assert exc.value.detail == code + assert db.query(BizMigrationProject).count() == 0 + + +def test_baseline_ownership_accepts_success_and_partial(db): + _project_tasks(db) + mig._validate_baseline_ownership(db, "old", "old_ok", "old_p") + mig._validate_baseline_ownership(db, "new", "new_ok", "new_p") + mig._validate_baseline_ownership(db, "old", "", "old_p") + + +# ---- H9: reads do not write; project lists do not fan out -------------------------------------- + +def _project(db, n, **extra): + db.add(BizStateTask(id=f"to{n}", source="managed", ne_id=f"o{n}", status="running")) + db.add(BizStateTask(id=f"tn{n}", source="managed", ne_id=f"n{n}", status="running")) + db.add(BizStateBatch(id=f"bo{n}", task_id=f"to{n}", status="success")) + db.add(BizStateBatch(id=f"bn{n}", task_id=f"tn{n}", status="success")) + db.add(BizMigrationProject(id=f"p{n}", name=f"p{n}", old_task_id=f"to{n}", new_task_id=f"tn{n}", + old_baseline_batch_id=f"bo{n}", new_baseline_batch_id=f"bn{n}", **extra)) + + +def _statements(db, work): + from sqlalchemy import event + + seen: list[str] = [] + listener = lambda conn, cur, stmt, params, ctx, many: seen.append(stmt) # noqa: E731 + event.listen(db.get_bind(), "before_cursor_execute", listener) + try: + work() + finally: + event.remove(db.get_bind(), "before_cursor_execute", listener) + return seen + + +def test_get_project_with_unbound_monitor_template_does_not_write(db): + db.add_all([BizCompareTemplate(id="ct", name="ZTE status default", metrics_json=[]), + BizMonitorTemplate(id="mt", name="默认割接监控", compare_template_id="ct")]) + _project(db, 1) + db.commit() + seen = _statements(db, lambda: mig.get_project(db, "p1")) + assert not [s for s in seen if s.lstrip().upper().startswith(("INSERT", "UPDATE", "DELETE"))] + assert db.get(BizMigrationProject, "p1").monitor_template_id == "" + + +def test_project_list_query_count_does_not_grow_with_projects(db): + db.add_all([BizCompareTemplate(id="ct", name="c", metrics_json=[]), + BizMonitorTemplate(id="mt", name="默认割接监控", compare_template_id="ct")]) + for n in range(3): + _project(db, n, monitor_template_id="mt") + db.commit() + few = len(_statements(db, lambda: mig.list_projects(db))) + db.expire_all() + for n in range(3, 15): + _project(db, n, monitor_template_id="mt") + db.commit() + db.expire_all() + many = len(_statements(db, lambda: mig.list_projects(db))) + assert many <= few + 2, (few, many) diff --git a/tests/test_identity_pipeline.py b/tests/test_identity_pipeline.py new file mode 100644 index 0000000..a3ee5aa --- /dev/null +++ b/tests/test_identity_pipeline.py @@ -0,0 +1,341 @@ +"""Identity pipeline: one match key everywhere, per-side rules, value maps, post-normalize filters.""" +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from netx_api.biz_migration.evaluate import evaluate_metric_dual, parse_expect_set +from netx_api.biz_state import compare_service as svc +from netx_api.biz_state.compare_engine import compare_rows +from netx_api.biz_state.compare_sql import dup_pairing_order_sql, sql_compare_skip_reason +from netx_api.biz_state.identity_pipeline import build_pipeline, legacy_pipeline +from netx_api.db import Base +from netx_api.models import ( + BizCompareRun, BizCompareTemplate, BizStateBatch, BizStateBatchCommand, BizStateMetricRow, BizStateTask, +) + +ZTE_RULES = [{"from": "GE", "to": "gei"}, {"from": "XGE", "to": "xgei"}] + + +def sheet(**over): + base = { + "sheet_id": "s", "metric_id": "iface_t", "key_fields": ["interface"], "iface_fields": ["interface"], + "compare_fields": ["state"], "field_rules": [], "row_filters": [], + } + return {**base, **over} + + +def pg_db(): + db = MagicMock() + db.get_bind.return_value = MagicMock(dialect=MagicMock()) + db.get_bind.return_value.dialect.name = "postgresql" + return db + + +# ---- B2: per-side rules + vendor canonical ------------------------------------------------- + +def test_huawei_long_and_short_names_are_the_same_interface(): + p = build_pipeline(sheet(), iface_rules=ZTE_RULES, port_map={}, before_vendor="huawei", after_vendor="huawei") + a = p.match_key({"interface": "GigabitEthernet0/0/1"}, "before") + b = p.match_key({"interface": "GE0/0/1"}, "before") + assert a == b == ("gei0/0/1",) + + +def test_inner_spaces_are_removed(): + p = build_pipeline(sheet(), iface_rules=[], port_map={}, before_vendor="cisco") + assert p.match_key({"interface": "Gigabit Ethernet 0/0/1"}, "before") == p.match_key( + {"interface": "GigabitEthernet0/0/1"}, "before") + + +def test_cisco_and_h3c_canonical_forms(): + p = build_pipeline(sheet(), iface_rules=[], port_map={}, before_vendor="cisco", after_vendor="h3c") + assert p.normalize({"interface": "TenGigE0/0/0/1"}, "before")["interface"] == "Te0/0/0/1" + assert p.normalize({"interface": "Ten-GigabitEthernet1/0/1"}, "after")["interface"] == "XGE1/0/1" + + +def test_builtin_canonical_can_be_switched_off_per_sheet(): + p = build_pipeline(sheet(builtin_vendor_canonical=False), iface_rules=[], port_map={}, before_vendor="huawei") + assert p.normalize({"interface": "GigabitEthernet0/0/1"}, "before")["interface"] == "GigabitEthernet0/0/1" + + +def test_side_scoped_rule_only_touches_its_side(): + rules = [{"from": "GE", "to": "gei", "side": "before"}] + p = build_pipeline(sheet(), iface_rules=rules, port_map={}, before_vendor="huawei", after_vendor="zte") + assert p.normalize({"interface": "GE0/1"}, "before")["interface"] == "gei0/1" + assert p.normalize({"interface": "GE0/1"}, "after")["interface"] == "GE0/1" + + +def test_vendor_scoped_rule_follows_the_batch_vendor(): + rules = [{"from": "Eth-Trunk", "to": "ae", "vendor": "huawei"}] + p = build_pipeline(sheet(), iface_rules=rules, port_map={}, before_vendor="huawei", after_vendor="zte") + assert p.normalize({"interface": "Eth-Trunk1"}, "before")["interface"] == "ae1" + assert p.normalize({"interface": "Eth-Trunk1"}, "after")["interface"] == "Eth-Trunk1" + + +def test_port_map_keys_use_before_rules_and_values_use_after_rules(): + rules = [{"from": "GE", "to": "gei", "side": "after"}] + p = build_pipeline(sheet(), iface_rules=rules, port_map={"GE0/1": "GE0/2"}, + before_vendor="huawei", after_vendor="zte") + # before-side name stays "GE0/1" (rule is after-only); after-side value is normalized. + assert p.port_map == {"GigabitEthernet0/1".replace("GigabitEthernet", "GE"): "gei0/2"} + row = p.transform({"interface": "GigabitEthernet0/1"}, "before") + assert row["interface"] == "gei0/2" + + +def test_zte_to_huawei_numbering_change_with_alias_rules(): + p = build_pipeline( + sheet(), iface_rules=ZTE_RULES, port_map={"GE0/1/0/1": "GigabitEthernet0/2/0/1"}, + before_vendor="zte", after_vendor="huawei", + ) + out = compare_rows( + before_rows=[{"interface": "gei0/1/0/1", "state": "up"}], + after_rows=[{"interface": "GE0/2/0/1", "state": "up"}], + key_fields=["interface"], iface_fields=["interface"], compare_fields=["state"], pipeline=p, + ) + assert out["summary"]["unchanged"] == 1 + + +# ---- B3: value maps ---------------------------------------------------------------------- + +def test_value_map_aligns_vendor_state_names(): + rules = [{"field": "state", "value_map": {"estab": "established", "*down": "down"}}] + p = build_pipeline(sheet(iface_fields=[], key_fields=["peer"], field_rules=rules), iface_rules=[], + port_map={}) + out = compare_rows( + before_rows=[{"peer": "1.1.1.1", "state": "Estab"}, {"peer": "2.2.2.2", "state": "*down"}], + after_rows=[{"peer": "1.1.1.1", "state": "established"}, {"peer": "2.2.2.2", "state": "down"}], + key_fields=["peer"], iface_fields=[], compare_fields=["state"], pipeline=p, + ) + assert out["summary"]["unchanged"] == 2 and out["summary"]["changed"] == 0 + + +def test_value_map_can_be_scoped_to_one_side(): + rules = [{"field": "state", "side": "before", "value_map": {"estab": "established"}}] + p = build_pipeline(sheet(iface_fields=[], key_fields=["peer"], field_rules=rules), iface_rules=[], port_map={}) + assert p.normalize({"peer": "a", "state": "Estab"}, "before")["state"] == "established" + assert p.normalize({"peer": "a", "state": "Estab"}, "after")["state"] == "Estab" + + +def test_mac_key_with_different_formats_pairs(): + rules = [{"field": "mac", "normalize": "mac"}] + p = build_pipeline(sheet(metric_id="mac_t", iface_fields=[], key_fields=["mac"], compare_fields=["port"], + field_rules=rules), iface_rules=[], port_map={}) + out = compare_rows( + before_rows=[{"mac": "00e0-fc12-3456", "port": "1"}], after_rows=[{"mac": "00e0.fc12.3456", "port": "1"}], + key_fields=["mac"], iface_fields=[], compare_fields=["port"], pipeline=p, + ) + assert out["summary"]["unchanged"] == 1 and out["summary"]["added"] == out["summary"]["removed"] == 0 + + +def test_legacy_pipeline_never_normalizes_keys(): + rules = [{"field": "mac", "normalize": "mac"}] + p = legacy_pipeline(key_fields=["mac"], iface_fields=[], compare_fields=["port"], field_rules=rules) + out = compare_rows( + before_rows=[{"mac": "00e0-fc12-3456", "port": "1"}], after_rows=[{"mac": "00e0.fc12.3456", "port": "1"}], + key_fields=["mac"], iface_fields=[], compare_fields=["port"], field_rules=rules, pipeline=p, + ) + assert out["summary"]["added"] == out["summary"]["removed"] == 1 + + +def test_value_map_and_key_normalize_block_sql(): + s = sheet(metric_id="bgp_route", iface_fields=[], key_fields=["mac"], + field_rules=[{"field": "state", "value_map": {"a": "b"}}], compare_fields=["state"]) + p = build_pipeline(s, iface_rules=[], port_map={}) + assert sql_compare_skip_reason(pg_db(), s, pipeline=p) == "value_map" + s2 = sheet(metric_id="m", iface_fields=[], key_fields=["mac"], compare_fields=[], + field_rules=[{"field": "mac", "normalize": "lower"}]) + assert sql_compare_skip_reason(pg_db(), s2, pipeline=build_pipeline(s2, iface_rules=[], port_map={})) \ + == "key_normalize" + + +# ---- B4: filters run after normalization ----------------------------------------------------- + +CONTAINS_GEI = [{"field": "interface", "op": "contains", "value": "gei"}] + + +def test_v2_filter_sees_normalized_names_but_legacy_sees_raw(): + s = sheet(row_filters=CONTAINS_GEI) + v2 = build_pipeline(s, iface_rules=ZTE_RULES, port_map={}, before_vendor="huawei") + assert [raw["interface"] for raw, _ in v2.prepare([{"interface": "GE0/0/1"}], "before")] == ["GE0/0/1"] + old = legacy_pipeline(key_fields=["interface"], iface_fields=["interface"], compare_fields=[], + iface_normalize_rules=ZTE_RULES, row_filters=CONTAINS_GEI) + assert old.prepare([{"interface": "GE0/0/1"}], "before") == [] # the cross-vendor bug, kept for v1 + + +def test_filter_can_target_one_side(): + s = sheet(iface_fields=[], row_filters=[{"field": "type", "op": "eq", "value": "dynamic", "side": "before"}]) + p = build_pipeline(s, iface_rules=[], port_map={}) + rows = [{"interface": "a", "type": "static"}] + assert p.prepare(rows, "before") == [] + assert len(p.prepare(rows, "after")) == 1 + + +def test_pushdown_only_for_filters_that_read_untransformed_fields(): + s = sheet(row_filters=[CONTAINS_GEI[0], {"field": "afi", "op": "eq", "value": "ipv4"}]) + p = build_pipeline(s, iface_rules=ZTE_RULES, port_map={}) + assert p.pushdown_filters("before") == [{"field": "afi", "op": "eq", "value": "ipv4"}] + assert sql_compare_skip_reason(pg_db(), sheet(metric_id="m", iface_fields=["interface"], key_fields=["id"], + compare_fields=[], row_filters=CONTAINS_GEI, + ignore_port_changes=False), + pipeline=build_pipeline(sheet(row_filters=CONTAINS_GEI), iface_rules=[], + port_map={})) == "row_filters_post_normalize" + + +# ---- B6: a port map no longer pushes interface-free sheets to Python ---------------------------- + +def test_port_map_only_blocks_sql_for_sheets_that_use_interfaces(): + route = {"metric_id": "bgp_route", "key_fields": ["network"], "iface_fields": [], + "compare_fields": ["path"], "field_rules": [], "row_filters": []} + assert sql_compare_skip_reason(pg_db(), route, port_map={"a": "b"}) == "" + brief = {**route, "metric_id": "interface_brief", "key_fields": ["interface"], "iface_fields": ["interface"]} + assert sql_compare_skip_reason(pg_db(), brief, port_map={"a": "b"}) == "port_map" + # a map on a sheet whose interface column is only displayed does not matter either + shown = {**route, "iface_fields": ["interface"]} + assert sql_compare_skip_reason(pg_db(), shown, port_map={"a": "b"}) == "" + + +# ---- V4: duplicate keys pair by value ---------------------------------------------------------- + +def test_same_key_rows_pair_by_value_not_output_order(): + s = sheet(iface_fields=[], key_fields=["prefix"], compare_fields=["hop"]) + rows_b = [{"prefix": "p", "hop": "a"}, {"prefix": "p", "hop": "b"}] + rows_a = [{"prefix": "p", "hop": "b"}, {"prefix": "p", "hop": "a"}] + kw = dict(before_rows=rows_b, after_rows=rows_a, key_fields=["prefix"], iface_fields=[], compare_fields=["hop"]) + new = compare_rows(**kw, pipeline=build_pipeline(s, iface_rules=[], port_map={})) + assert new["summary"]["unchanged"] == 2 and new["summary"]["changed"] == 0 + old = compare_rows(**kw) # ordered, legacy + assert old["summary"]["changed"] == 2 + + +def test_sql_dup_order_mirrors_python_sort_key(): + sql = dup_pairing_order_sql(["hop", "metric"], {"hop": {"normalize": "lower"}}) + assert 'lower(trim(both from coalesce(data_json->>\'hop\', \'\'))) COLLATE "C" ASC' in sql.replace("(", "", 1) \ + or 'COLLATE "C"' in sql + assert sql.endswith(", ") + assert dup_pairing_order_sql([], {}) == "" + + +# ---- B7: partial maps --------------------------------------------------------------------------- + +def test_ignore_port_policy_keeps_unmapped_ports_matching_by_content(): + s = sheet(iface_fields=["interface"], key_fields=["interface", "ip"], compare_fields=["mac"], + unmapped_port_policy="ignore_port") + p = build_pipeline(s, iface_rules=[], port_map={"gei-1": "gei-9"}) + out = compare_rows( + before_rows=[{"interface": "gei-1", "ip": "10.0.0.1", "mac": "m1"}, + {"interface": "gei-2", "ip": "10.0.0.2", "mac": "m2"}], + after_rows=[{"interface": "gei-9", "ip": "10.0.0.1", "mac": "m1"}, + {"interface": "gei-7", "ip": "10.0.0.2", "mac": "m2"}], + key_fields=["interface", "ip"], iface_fields=["interface"], compare_fields=["mac"], pipeline=p, + ) + assert out["summary"]["unchanged"] == 2 + by_name = build_pipeline({**s, "unmapped_port_policy": "by_name"}, iface_rules=[], port_map={"gei-1": "gei-9"}) + out2 = compare_rows( + before_rows=[{"interface": "gei-2", "ip": "10.0.0.2", "mac": "m2"}], + after_rows=[{"interface": "gei-7", "ip": "10.0.0.2", "mac": "m2"}], + key_fields=["interface", "ip"], iface_fields=["interface"], compare_fields=["mac"], pipeline=by_name, + ) + assert out2["summary"]["added"] == out2["summary"]["removed"] == 1 + + +# ---- B1: every path derives the same match key -------------------------------------------------- + +@pytest.fixture() +def db(): + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all(engine) + session = sessionmaker(bind=engine, expire_on_commit=False)() + yield session + session.close() + engine.dispose() + + +def _seed_two_batches(db): + db.add_all([ + BizStateTask(id="t-old", source="managed", ne_id="o", vendor="Huawei", device_type="huawei"), + BizStateTask(id="t-new", source="managed", ne_id="n", vendor="ZTE", device_type="zte_zxros"), + BizStateBatch(id="b-old", task_id="t-old", status="success", vendor="Huawei"), + BizStateBatch(id="b-new", task_id="t-new", status="success", vendor="ZTE"), + ]) + db.add_all([ + BizStateMetricRow(id="r-old", batch_id="b-old", metric_id="iface_t", seq=0, + data_json={"interface": "GigabitEthernet0/1/0/1", "state": "up"}), + BizStateMetricRow(id="r-new", batch_id="b-new", metric_id="iface_t", seq=0, + data_json={"interface": "gei-0/2/0/1", "state": "up"}), + ]) + db.commit() + + +def test_all_paths_agree_on_the_match_key(db): + _seed_two_batches(db) + raw_map = {"GigabitEthernet0/1/0/1": "gei-0/2/0/1"} + rules = [{"from": "GE", "to": "gei"}] + s = sheet(sheet_id="s") + tpl = BizCompareTemplate(id="tpl", name="t", metrics_json=[s], iface_normalize_json=rules) + db.add(tpl) + db.commit() + from netx_api.biz_state.compare_service import _create_running_run, template_metrics + from netx_api.models import BizCompareJob, BizPortMapping, BizPortMappingRow + + db.add_all([BizPortMapping(id="m", name="m"), + BizPortMappingRow(mapping_id="m", before_if="GigabitEthernet0/1/0/1", after_if="gei-0/2/0/1"), + BizCompareJob(id="j", name="j", template_id="tpl", mapping_id="m")]) + db.commit() + job = db.get(BizCompareJob, "j") + run = _create_running_run(db, job=job, tpl=tpl, before_batch_id="b-old", after_batch_id="b-new", + sheets_cfg=template_metrics(tpl)) + assert run.summary_json["config_snapshot"]["vendors"] == {"before": "huawei", "after": "zte"} + pipe = svc._run_pipeline(db, run, s) + expected = ("gei-0/2/0/1",) + + # 1. execution + result = compare_rows(before_rows=[{"interface": "GigabitEthernet0/1/0/1", "state": "up"}], + after_rows=[{"interface": "gei-0/2/0/1", "state": "up"}], + key_fields=["interface"], iface_fields=["interface"], compare_fields=["state"], + pipeline=pipe) + assert result["summary"]["unchanged"] == 1 + # 2. pipeline directly + assert pipe.match_key({"interface": "GigabitEthernet0/1/0/1"}, "before") == expected + assert pipe.match_key({"interface": "gei-0/2/0/1"}, "after") == expected + # 3. search key groups + rows, complete = svc._load_search_key_groups( + db, batch_id="b-old", metric_id="iface_t", keys={expected}, match_keys=["interface"], + pipeline=pipe, side="before") + assert [r["interface"] for r in rows] == ["GigabitEthernet0/1/0/1"] and complete == {expected} + # 4. hydrate + item = {"before": {"interface": "GigabitEthernet0/1/0/1"}, "after": {}, "mapped_before": {}, + "after_row_id": "r-new", "before_row_id": ""} + hydrated = svc._hydrate_diff_rows(db, [item], run=run, sheet=s)[0] + assert hydrated["mapped_before"]["interface"] == hydrated["after"]["interface"] == "gei-0/2/0/1" + # 5. cutover evaluation (old = before, new = after) + dual = evaluate_metric_dual( + metric_id="iface_t", sheet_id="s", key_fields=["interface"], iface_fields=["interface"], + compare_fields=["state"], old_baseline_rows=[{"interface": "GigabitEthernet0/1/0/1", "state": "up"}], + old_current_rows=[], new_baseline_rows=[{"interface": "gei-0/2/0/1", "state": "down"}], + new_current_rows=[{"interface": "gei-0/2/0/1", "state": "up"}], port_map=pipe.port_map, + expect=parse_expect_set({"items": [{"metric_id": "iface_t", "key": "GE0/1/0/1"}]}), + window_active=False, acceptance=True, pipeline=pipe, + ) + (row,) = dual["rows"] + assert (row["match_old_key"], row["match_new_key"]) == ("gei0/1/0/1", "gei-0/2/0/1") + assert row["old_key"] == "GigabitEthernet0/1/0/1" and row["verdict"] == "migrated" + # 6. baseline expect picker + assert pipe.transform({"interface": "GigabitEthernet0/1/0/1"}, "before")["interface"] == "gei-0/2/0/1" + + +# ---- v1 replay must not change ----------------------------------------------------------------- + +def test_v1_snapshot_run_replays_with_legacy_semantics(db): + _seed_two_batches(db) + s = sheet(sheet_id="s", row_filters=CONTAINS_GEI) + run = BizCompareRun(id="r1", job_id="j", template_id="", before_batch_id="b-old", after_batch_id="b-new", + summary_json={"config_snapshot": { + "version": 1, "template": {"iface_normalize_rules": [{"from": "GE", "to": "gei"}]}, + "sheets": [s], "port_map": {"gei-0/1/0/1": "gei-0/2/0/1"}}}) + pipe = svc._run_pipeline(db, run, s) + assert pipe.legacy and pipe.duplicate_pairing == "ordered" + assert pipe.port_map == {"gei-0/1/0/1": "gei-0/2/0/1"} + assert pipe.prepare([{"interface": "GE0/1"}], "before") == [] diff --git a/tests/test_iface_normalize.py b/tests/test_iface_normalize.py index 94bbcbb..f266fa7 100644 --- a/tests/test_iface_normalize.py +++ b/tests/test_iface_normalize.py @@ -9,6 +9,7 @@ from netx_api.biz_state.iface_normalize import ( default_zte_iface_normalize_rules, normalize_iface_name, normalize_iface_rules, + normalize_port_map, resolve_mapped_iface, ) @@ -71,6 +72,32 @@ class PortMapSubifTests(unittest.TestCase): self.assertEqual(row["interface"], "xgei-0/1/0/1.55") +class NormalizePortMapTests(unittest.TestCase): + def test_keys_and_values_use_template_rules(self) -> None: + out, conflicts = normalize_port_map( + {"GE0/1/0/1": "GE0/2/0/1"}, + normalize_iface_rules([{"from": "GE", "to": "gei"}]), + ) + self.assertEqual(out, {"gei0/1/0/1": "gei0/2/0/1"}) + self.assertEqual(conflicts, []) + + def test_conflicting_keys_are_reported_first_wins(self) -> None: + out, conflicts = normalize_port_map( + {"GE0/1": "gei0/2", "gei0/1": "gei0/3"}, + normalize_iface_rules([{"from": "GE", "to": "gei"}]), + ) + self.assertEqual(out, {"gei0/1": "gei0/2"}) + self.assertEqual(len(conflicts), 1) + self.assertEqual(conflicts[0]["dropped_to"], "gei0/3") + + def test_same_target_duplicate_is_not_a_conflict(self) -> None: + _, conflicts = normalize_port_map( + {"GE0/1": "gei0/2", "gei0/1": "gei0/2"}, + normalize_iface_rules([{"from": "GE", "to": "gei"}]), + ) + self.assertEqual(conflicts, []) + + class ComparePipelineTests(unittest.TestCase): def test_normalize_then_map_matches(self) -> None: before = [{"interface": "GE-0/0/0/1.100", "admin": "up"}] diff --git a/tests/test_managed_ne_page.py b/tests/test_managed_ne_page.py new file mode 100644 index 0000000..3b82bc1 --- /dev/null +++ b/tests/test_managed_ne_page.py @@ -0,0 +1,205 @@ +"""NE management page: column filters, create/edit validation, batch and import fixes.""" +import json +from unittest.mock import patch + +import pytest +from cryptography.fernet import Fernet +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import StaticPool + +from netx_api.config import settings +from netx_api.db import Base, get_db +from netx_api.managed_ne_router import router +from netx_api.models import ManagedNE +from netx_api.ne_crypto import decrypt_secret, encrypt_secret +from netx_api.ne_service import import_managed_ne + + +@pytest.fixture +def engine(): + orig = settings.credential_secret_key + settings.credential_secret_key = Fernet.generate_key().decode() + eng = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) + Base.metadata.create_all(eng) + with Session(eng) as db: + db.add_all([ + ManagedNE(id="m1", name="PE-1", ip_address="10.0.0.1", vendor="ZTE", device_type="zte_zxros", + username="u1", tags="core edge", connect_status="pass", source=""), + ManagedNE(id="m2", name="", ip_address="10.0.0.2", vendor="Huawei", device_type="huawei", + username="u2", tags="edge", connect_status="fail", source="webcrt"), + ManagedNE(id="m3", name="SW-3", ip_address="10.0.0.3", vendor="Cisco", device_type="cisco_ios", + username="u3", tags="", connect_status="unknown", source="lldp"), + ManagedNE(id="m4", name="Lab_1", ip_address="10.0.0.4", vendor="Cisco", device_type="cisco_ios", + username="u4", tags="lab%x", connect_status="pass"), + ]) + db.commit() + yield eng + eng.dispose() + settings.credential_secret_key = orig + + +@pytest.fixture +def client(engine): + app = FastAPI() + app.include_router(router) + + def database(): + with Session(engine) as db: + yield db + + app.dependency_overrides[get_db] = database + with patch("netx_api.auth_deps.settings.auth_enabled", False), TestClient(app) as test_client: + yield test_client + + +def _list(client, filters=None, **params): + if filters: + params["column_filters"] = json.dumps(filters) + r = client.get("/v1/managed-ne", params=params) + assert r.status_code == 200, r.text + return r.json() + + +def _ids(body): + return sorted(x["id"] for x in body["items"]) + + +def test_column_filter_options_cover_full_dataset(client): + opts = _list(client, filter_field="vendor", page_size=1) + assert opts == {"field": "vendor", "options": ["Cisco", "Huawei", "ZTE"]} + # Blank names fall back to the IP shown in the table; blank source shows as manual. + assert "10.0.0.2" in _list(client, filter_field="name")["options"] + assert _list(client, filter_field="source")["options"] == ["lldp", "manual", "webcrt"] + + +def test_column_filters_apply_before_pagination(client): + body = _list(client, {"connectStatus": {"mode": "include", "values": ["pass"]}}, page_size=1) + assert body["total"] == 2 + body = _list(client, {"vendor": {"mode": "exclude", "values": ["Cisco"]}}) + assert _ids(body) == ["m1", "m2"] + body = _list(client, {"source": {"mode": "include", "values": ["manual"]}}) + assert _ids(body) == ["m1", "m4"] + + +def test_tags_filter_matches_individual_tags(client): + assert _list(client, filter_field="tags")["options"] == ["", "core", "edge", "lab%x"] + assert _ids(_list(client, {"tags": {"mode": "include", "values": ["edge"]}})) == ["m1", "m2"] + assert _ids(_list(client, {"tags": {"mode": "include", "values": [""]}})) == ["m3"] + assert _ids(_list(client, {"tags": {"mode": "exclude", "values": ["edge", ""]}})) == ["m4"] + # LIKE wildcards inside a tag are literal. + assert _ids(_list(client, {"tags": {"mode": "include", "values": ["lab%x"]}})) == ["m4"] + # Tag options respect the other columns' filters. + opts = _list(client, {"vendor": {"mode": "include", "values": ["Huawei"]}}, filter_field="tags") + assert opts["options"] == ["edge"] + + +def test_invalid_filter_field_rejected(client): + r = client.get("/v1/managed-ne", params={"filter_field": "password_enc"}) + assert r.status_code == 400 + + +@pytest.mark.parametrize("ip", ["10.0.0.300", "a b", "http://x", "10.0.0.9:22"]) +def test_create_rejects_bad_address(client, ip): + r = client.post("/v1/managed-ne", json={"vendor": "ZTE", "device_type": "zte_zxros", + "ip_address": ip, "username": "u"}) + assert r.status_code == 400 and r.json()["detail"] == "invalid_ip_address" + + +@pytest.mark.parametrize("ip", ["10.0.0.9", "2001:db8::1", "pe-01.example.net"]) +def test_create_accepts_ip_or_hostname(client, ip): + r = client.post("/v1/managed-ne", json={"vendor": "ZTE", "device_type": "zte_zxros", + "ip_address": ip, "username": "u"}) + assert r.status_code == 200, r.text + + +@pytest.mark.parametrize("device_type", ["linux", "generic"]) +def test_create_accepts_form_device_types(client, device_type): + r = client.post("/v1/managed-ne", json={"vendor": "Other", "device_type": device_type, + "ip_address": "10.0.0.9", "username": "u"}) + assert r.status_code == 200, r.text + + +@pytest.mark.parametrize("port", [0, 65536, -1]) +def test_create_rejects_bad_port(client, port): + r = client.post("/v1/managed-ne", json={"vendor": "ZTE", "device_type": "zte_zxros", + "ip_address": "10.0.0.9", "username": "u", "port": port}) + assert r.status_code == 422 + + +def test_update_resets_status_when_login_changes_and_keeps_name(client): + r = client.patch("/v1/managed-ne/m1", json={"remark": "note"}) + assert r.json()["connect_status"] == "pass" + r = client.patch("/v1/managed-ne/m1", json={"port": 2222}) + assert r.json()["connect_status"] == "unknown" + r = client.patch("/v1/managed-ne/m1", json={"name": " "}) + assert r.json()["name"] == "10.0.0.1" + + +def test_batch_hop_requires_password_for_ne_without_one(client, engine): + hop = {"hop_vendor": "zte", "hop_host": "1.1.1.1", "hop_username": "jump"} + r = client.post("/v1/managed-ne/batch-hop", json={"ids": ["m1"], "hop": hop}) + assert r.status_code == 400 and r.json()["detail"].startswith("hop_password_required") + with Session(engine) as db: + db.get(ManagedNE, "m1").hop_password_enc = encrypt_secret("old") + db.commit() + # Blank password keeps the stored one. + r = client.post("/v1/managed-ne/batch-hop", json={"ids": ["m1"], "hop": hop}) + assert r.status_code == 200, r.text + with Session(engine) as db: + row = db.get(ManagedNE, "m1") + assert decrypt_secret(row.hop_password_enc) == "old" + assert row.connect_status == "unknown" + # A bastion that manages target auth may run without a hop password. + bastion = {**hop, "hop_vendor": "bastion", "hop_target_auth_mode": "bastion_managed"} + r = client.post("/v1/managed-ne/batch-hop", json={"ids": ["m2"], "hop": bastion}) + assert r.status_code == 200, r.text + + +def test_batch_account_resets_status(client, engine): + r = client.post("/v1/managed-ne/batch-account", json={"ids": ["m1"], "account": {"username": "new"}}) + assert r.status_code == 200 + with Session(engine) as db: + assert db.get(ManagedNE, "m1").connect_status == "unknown" + + +def test_import_keeps_password_skips_bad_rows_and_promotes_placeholder(engine): + db = sessionmaker(bind=engine)() + try: + row = db.get(ManagedNE, "m1") + row.password_enc = encrypt_secret("secret") + db.commit() + csv = ( + "device_type,ip,username,password,port,protocol,name,vendor,tags\n" + "zte_zxros,10.0.0.1,u1,,22,ssh,PE-1,ZTE,core\n" # blank password keeps stored one + "zte_zxros,10.0.0.3,u3,,22,ssh,SW-3,ZTE,\n" # LLDP placeholder becomes manual + "zte_zxros,10.9.9.9,u,,70000,ssh,BAD,ZTE,\n" # bad port + "zte_zxros,999.1.1.1,u,,22,ssh,BAD,ZTE,\n" # bad ip + "zte_zxros,10.9.9.8,u,,22,ssh,NEW,ZTE,\n" + ).encode("utf-8") + result = import_managed_ne(db, csv, "devices.csv") + assert (result.inserted, result.updated) == (1, 2) + assert [(f.row, f.reason) for f in result.failed] == [(4, "invalid_port"), (5, "invalid_ip_address")] + db.expire_all() + assert decrypt_secret(db.get(ManagedNE, "m1").password_enc) == "secret" + assert db.get(ManagedNE, "m3").source == "" + assert db.query(ManagedNE).filter(ManagedNE.ip_address == "10.9.9.9").count() == 0 + finally: + db.close() + + +def test_import_failure_mid_row_leaves_no_partial_ne(engine): + db = sessionmaker(bind=engine)() + try: + csv = ( + "device_type,ip,username,password,port,protocol,name,vendor\n" + "zte_zxros,10.7.7.7,u,pw,22,ssh,X,ZTE\n" + ).encode("utf-8") + with patch("netx_api.ne_service_import.encrypt_secret", side_effect=RuntimeError("boom")): + result = import_managed_ne(db, csv, "devices.csv") + assert (result.inserted, result.updated, len(result.failed)) == (0, 0, 1) + assert db.query(ManagedNE).filter(ManagedNE.ip_address == "10.7.7.7").count() == 0 + finally: + db.close() diff --git a/tests/test_mcp_http.py b/tests/test_mcp_http.py index 88f5ff2..e0ac008 100644 --- a/tests/test_mcp_http.py +++ b/tests/test_mcp_http.py @@ -12,9 +12,9 @@ import pytest from netx_mcp.http_tools import HTTP_MCP_TOOLS, call_http_tool -def test_http_mcp_tool_list_has_fourteen_tools() -> None: +def test_http_mcp_tool_list_has_fifteen_tools() -> None: names = [str(t.get("name") or "") for t in HTTP_MCP_TOOLS] - assert len(names) == 14 + assert len(names) == 15 assert "queryNmsAlarms" in names assert "queryNmsAlarmsRaw" in names assert "execManagedNe" in names @@ -92,7 +92,7 @@ def test_stdio_initialize_and_tools_list() -> None: assert "error" not in list_resp, list_resp tools = list_resp["result"]["tools"] names = {t["name"] for t in tools} - assert len(tools) == 14 + assert len(tools) == 15 assert "queryNmsAlarms" in names assert "findTopologyPaths" in names diff --git a/tests/test_ne_netmiko_show.py b/tests/test_ne_netmiko_show.py index 2add1bd..5c57bb4 100644 --- a/tests/test_ne_netmiko_show.py +++ b/tests/test_ne_netmiko_show.py @@ -3,6 +3,7 @@ from __future__ import annotations import unittest +import re from unittest.mock import MagicMock, patch from netx_api.ne_netmiko import drain_read_channel, send_show_command @@ -23,6 +24,50 @@ class DrainReadChannelTests(unittest.TestCase): class SendShowCommandTests(unittest.TestCase): + @patch("netx_api.ne_netmiko.drain_read_channel") + def test_mikrotik_does_not_probe_prompt_after_drain(self, drain: MagicMock) -> None: + conn = MagicMock() + conn.device_type = "mikrotik_routeros_ssh" + + def delayed_routeros_output(**kwargs: object) -> str: + # A prompt probe queues an old prompt before the actual output. + if kwargs.get("auto_find_prompt", True): + return "" + return "name: MikroTik" + + conn.send_command.side_effect = delayed_routeros_output + self.assertEqual(send_show_command(conn, "/system identity print"), "name: MikroTik") + conn.send_command.assert_called_once() + conn.find_prompt.assert_not_called() + conn.send_command_timing.assert_not_called() + drain.assert_called_once_with(conn) + pattern = conn.send_command.call_args.kwargs["expect_string"] + for prompt in ("[admin@MikroTik] >", "[ops@edge-router] /ip/firewall>"): + self.assertIsNotNone(re.search(pattern, prompt)) + for echo in ("[admin@MikroTik]", "[admin@MikroTik] > /system resource print"): + self.assertIsNone(re.search(pattern, echo)) + + @patch("netx_api.ne_netmiko.drain_read_channel") + def test_mikrotik_wrapped_script_does_not_require_exact_echo(self, drain: MagicMock) -> None: + conn = MagicMock() + conn.device_type = "mikrotik_routeros" + conn.send_command.return_value = "finished" + command = ':put "' + "x" * 600 + '"' + self.assertEqual(send_show_command(conn, command, read_timeout=45), "finished") + conn.send_command.assert_called_once() + self.assertFalse(conn.send_command.call_args.kwargs["cmd_verify"]) + self.assertEqual(conn.send_command.call_args.kwargs["read_timeout"], 45) + + @patch("netx_api.ne_netmiko.drain_read_channel") + def test_mikrotik_timeout_never_resends_write(self, drain: MagicMock) -> None: + conn = MagicMock() + conn.device_type = "mikrotik_routeros" + conn.send_command.side_effect = TimeoutError("late prompt") + with self.assertRaises(TimeoutError): + send_show_command(conn, '/ip address add address=192.0.2.1/32 interface=lo') + conn.send_command.assert_called_once() + conn.send_command_timing.assert_not_called() + @patch("netx_api.ne_netmiko.drain_read_channel") def test_drains_then_sends_once(self, drain: MagicMock) -> None: conn = MagicMock() diff --git a/tests/test_paging_disable.py b/tests/test_paging_disable.py index d4030ec..74a6e02 100644 --- a/tests/test_paging_disable.py +++ b/tests/test_paging_disable.py @@ -9,6 +9,12 @@ from netx_api.ne_netmiko import disable_target_paging, paging_disable_commands class PagingDisableTests(unittest.TestCase): + def test_routeros_does_not_send_cisco_paging_command(self) -> None: + conn = MagicMock() + self.assertEqual(disable_target_paging(conn, device_type="mikrotik_routeros"), "") + conn.send_command_timing.assert_not_called() + conn.write_channel.assert_not_called() + def test_commands_by_vendor(self) -> None: self.assertEqual( paging_disable_commands(vendor="Huawei", device_type="huawei"), diff --git a/tests/test_table_column_filters.py b/tests/test_table_column_filters.py new file mode 100644 index 0000000..6f69ea5 --- /dev/null +++ b/tests/test_table_column_filters.py @@ -0,0 +1,112 @@ +"""Column filters must cover the full dataset and run before pagination.""" +import json +from unittest.mock import patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import Session +from sqlalchemy.pool import StaticPool + +from netx_api.cli_router import router as cli_router +from netx_api.config_sync_router import router as config_router +from netx_api.db import Base, get_db +from netx_api.lldp_collect_router import router as lldp_router +from netx_api.models import ( + ManagedNE, NeConfigSnapshot, TopoDiscoverJob, TopoDiscoverJobItem, + TopoFabricEdge, TopoFabricNode, UmeAlarmCurrent, UmeInventoryNE, +) +from netx_api.topology_router import router as topology_router +from netx_api.ume_alarms_router import router as alarm_router + + +@pytest.fixture +def client(): + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) + Base.metadata.create_all(engine) + with Session(engine) as db: + db.add_all([ + ManagedNE(id="m1", name="Device A", ip_address="10.0.0.1", vendor="Huawei", source="manual"), + ManagedNE(id="m2", name="Device B", ip_address="10.0.0.2", vendor="Cisco", source="manual"), + UmeInventoryNE(ne_id="u1", host_name="Fallback Host", ne_type="Router"), + NeConfigSnapshot(source="managed", target_id="m1", ne_name="Device A", vendor="Huawei", plain_size=100), + NeConfigSnapshot(source="ume", target_id="u1", ne_name="Device B", vendor="Cisco", plain_size=200), + UmeAlarmCurrent(alarm_key="a1", ne_id="u1", host_name="", perceived_severity="major", is_cleared="false"), + UmeAlarmCurrent(alarm_key="a2", host_name="Other Host", perceived_severity="minor", is_cleared="true"), + TopoFabricNode(id="n1", name="Device A", ip="10.0.0.1", managed_ne_id="m1", level=1), + TopoFabricNode(id="n2", name="Device B", ip="10.0.0.2", level=None), + TopoFabricEdge(id="e1", a_node_id="n1", b_node_id="n2", a_port="p1", status="stale", attrs={"miss_count": 2}), + TopoFabricEdge(id="e2", a_node_id="n1", b_node_id="n2", a_port="p2", status="active"), + TopoDiscoverJob(id="j1", status="done", total=2, done=2), + TopoDiscoverJob(id="j2", status="failed", total=1, done=1), + TopoDiscoverJobItem(id="i1", job_id="j1", ne_name="Device A", ok=True, neighbors=2), + TopoDiscoverJobItem(id="i2", job_id="j1", ne_name="Device B", ok=True, error="empty_cli_output"), + ]) + db.commit() + app = FastAPI() + for router in (cli_router, config_router, topology_router, lldp_router, alarm_router): + app.include_router(router) + + def database(): + with Session(engine) as db: + yield db + + app.dependency_overrides[get_db] = database + with patch("netx_api.auth_deps.settings.auth_enabled", False), TestClient(app) as test_client: + yield test_client + engine.dispose() + + +CASES = [ + ("/v1/cli/targets", {"source": "inventory"}, "name", "Device B", "items", "total"), + ("/v1/config-sync/snapshots", {}, "vendor", "Cisco", "items", "total"), + ("/v1/ume/alarms", {}, "hostName", "Fallback Host", "items", "total"), + ("/v1/topology/fabric/nodes", {}, "level", "", "items", "total"), + ("/v1/topology/fabric/edges", {}, "status", "missing", "items", "total"), + ("/v1/topology/lldp-collect/jobs", {}, "status", "failed", "items", "total"), + ("/v1/topology/lldp-collect/jobs/j1", {}, "status", "fail", "items", "items_total"), +] + + +@pytest.mark.parametrize("path,scope,field,value,items_key,total_key", CASES) +def test_options_and_selection_before_pagination(client, path, scope, field, value, items_key, total_key): + options = client.get(path, params={**scope, "filter_field": field, "page_size": 1}) + assert options.status_code == 200, options.text + assert value in options.json()["options"] + assert len(options.json()["options"]) >= 2 + for mode, selected, expected_total in [("include", [value], 1), ("include", [], 0), ("exclude", [], len(client.get(path, params=scope).json()[items_key]))]: + response = client.get(path, params={**scope, "page_size": 1, "column_filters": json.dumps({field: {"mode": mode, "values": selected}})}) + assert response.status_code == 200, response.text + assert response.json()[total_key] == expected_total + assert len(response.json()[items_key]) == min(expected_total, 1) + # Exclusion is the complement of inclusion, including blank values. + all_total = client.get(path, params=scope).json()[total_key] + excluded = client.get(path, params={**scope, "column_filters": json.dumps({field: {"mode": "exclude", "values": [value]}})}) + assert excluded.json()[total_key] == all_total - 1 + + +def test_multiple_columns_and_invalid_payload(client): + path = "/v1/config-sync/snapshots" + response = client.get(path, params={"column_filters": json.dumps({"source": {"mode": "include", "values": ["managed"]}, "vendor": {"mode": "include", "values": ["Cisco"]}})}) + assert response.status_code == 200 + assert response.json()["total"] == 0 + for payload in ("[]", "bad-json", '{"name":{"mode":"bad","values":[]}}', '{"name":{"mode":"include","values":[1]}}'): + assert client.get(path, params={"column_filters": payload}).status_code == 400 + assert client.get(path, params={"filter_field": "nonexistent"}).status_code == 400 + + +@pytest.mark.parametrize("path,field", [ + ("/v1/config-sync/snapshots", "size"), ("/v1/topology/fabric/nodes", "link"), + ("/v1/topology/fabric/edges", "aSide"), ("/v1/topology/fabric/edges", "missCount"), + ("/v1/topology/lldp-collect/jobs", "progress"), ("/v1/topology/lldp-collect/jobs/j1", "edges"), +]) +def test_computed_column_values_roundtrip(client, path, field): + response = client.get(path, params={"filter_field": field}) + assert response.status_code == 200, response.text + options = response.json()["options"] + assert options + for value in options: + filtered = client.get(path, params={"column_filters": json.dumps({field: {"mode": "include", "values": [value]}})}) + assert filtered.status_code == 200, filtered.text + assert filtered.json().get("total", filtered.json().get("items_total")) > 0 diff --git a/tests/test_tabular.py b/tests/test_tabular.py new file mode 100644 index 0000000..1826dee --- /dev/null +++ b/tests/test_tabular.py @@ -0,0 +1,83 @@ +"""xlsx/csv helpers that replaced pandas in alarm + managed-NE imports.""" +import io +from datetime import datetime, timezone + +from openpyxl import Workbook, load_workbook + +from netx_api.importer import _dt, _int, _str +from netx_api.tabular import parse_datetime, read_csv, read_table, read_xlsx, write_csv, write_xlsx + + +def _xlsx(rows): + wb = Workbook() + ws = wb.active + for r in rows: + ws.append(r) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def test_read_xlsx_types_blank_rows_and_sheet_row_numbers(): + content = _xlsx([ + ["Time", "Code", "Port", None, "Code"], + [datetime(2024, 1, 2, 3, 4, 5), 1001, 22.0, "x", "dup"], + [None, None, None, None, None], + [" ", "A", 2.5, None, None], + ]) + headers, rows = read_xlsx(content) + assert headers == ["Time", "Code", "Port", "Unnamed: 3", "Code.1"] + assert rows == [ + (2, {"Time": datetime(2024, 1, 2, 3, 4, 5), "Code": 1001, "Port": 22, "Unnamed: 3": "x", "Code.1": "dup"}), + (4, {"Time": None, "Code": "A", "Port": 2.5, "Unnamed: 3": None, "Code.1": None}), + ] + + +def test_read_csv_bom_quotes_short_rows_and_blank_lines(): + content = "a,b,c\n1,\"x, y\",\n\n2\n".encode("utf-8") + headers, rows = read_csv(content) + assert headers == ["a", "b", "c"] + assert rows == [(2, {"a": "1", "b": "x, y", "c": None}), (4, {"a": "2", "b": None, "c": None})] + + +def test_read_csv_falls_back_to_gbk(): + headers, rows = read_csv("名称,地址\n核心,10.0.0.1\n".encode("gbk")) + assert headers == ["名称", "地址"] + assert rows == [(2, {"名称": "核心", "地址": "10.0.0.1"})] + + +def test_read_table_dispatches_on_extension(): + assert read_table(b"h\nv\n", "X.CSV") == (["h"], [(2, {"h": "v"})]) + assert read_table(_xlsx([["h"], ["v"]]), "x.xlsx") == (["h"], [(2, {"h": "v"})]) + + +def test_write_csv_and_xlsx_round_trip(): + csv_bytes = write_csv(["a", "b"], [[1, None], ["中文", "x"]]) + assert csv_bytes.startswith("".encode("utf-8")) + assert read_csv(csv_bytes)[1] == [(2, {"a": "1", "b": None}), (3, {"a": "中文", "b": "x"})] + + xlsx = write_xlsx([("import", ["a"], [[1]]), ("types", ["t"], [["x"], ["y"]])]) + wb = load_workbook(io.BytesIO(xlsx)) + assert wb.sheetnames == ["import", "types"] + assert read_xlsx(xlsx) == (["a"], [(2, {"a": 1})]) + + +def test_parse_datetime_formats(): + want = datetime(2024, 1, 2, 3, 4, 5) + for text in ( + "2024-01-02 03:04:05", "2024/01/02 03:04:05", "2024/1/2 3:04:05", "2024-1-2 03:04:05", + "2024-01-02T03:04:05", "01/02/2024 03:04:05", "20240102030405", "02-Jan-2024 03:04:05", + "Jan 02, 2024 03:04:05", "2024年1月2日 03:04:05", + ): + assert parse_datetime(text) == want, text + assert parse_datetime("2024-01-02T03:04:05Z") == datetime(2024, 1, 2, 3, 4, 5, tzinfo=timezone.utc) + assert parse_datetime("2024-01-02") == datetime(2024, 1, 2) + for bad in (None, "", " ", "nan", "NaT", "abc"): + assert parse_datetime(bad) is None + + +def test_importer_cell_helpers(): + assert _dt("2024-01-02T03:04:05Z") == datetime(2024, 1, 2, 3, 4, 5) # stored as naive UTC + assert _dt(float("nan")) is None and _dt("garbage") is None + assert _str(1001) == "1001" and _str(None) == "" and _str(" nan ") == "" and _str(" a ") == "a" + assert _int("7") == 7 and _int(2.0) == 2 and _int("x") == 0 and _int(None) == 0 diff --git a/tests/test_topology.py b/tests/test_topology.py index fe86736..00de82b 100644 --- a/tests/test_topology.py +++ b/tests/test_topology.py @@ -27,12 +27,17 @@ from netx_api.models import ( TopoViewEdgeStyle, TopoViewNode, ) +from netx_api.ne_crypto import encrypt_secret from netx_api.topology_schemas import ( FabricDiscoverRequest, TopologyFolderCreate, TopologyPlaceholderCreate, TopologyViewCreate, TopologyViewUpdate, + ViewCommitEdgeCreate, + ViewCommitEdgePortEdit, + ViewCommitIn, + ViewEdgeStyleItem, ViewNodesAdd, ViewPopulateRequest, ViewPositionsPatch, @@ -384,6 +389,388 @@ class FabricTopologyTests(unittest.TestCase): self.db.refresh(view) return view.id + def test_patch_manual_edge_ports(self) -> None: + suffix = uuid4().hex[:8] + ne_a = ManagedNE( + id=f"ne-mp-a-{suffix}", + name=f"MPA-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.5.0.{(int(suffix[:2], 16) % 200) + 1}", + ) + ne_b = ManagedNE( + id=f"ne-mp-b-{suffix}", + name=f"MPB-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.5.1.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add_all([ne_a, ne_b]) + self.db.commit() + fa = svc.ensure_fabric_node_for_managed(self.db, ne_a) + fb = svc.ensure_fabric_node_for_managed(self.db, ne_b) + edge, _ = svc.upsert_fabric_edge( + self.db, + a_node_id=fa.id, + b_node_id=fb.id, + a_port="gi0/1", + b_port="gi0/2", + source="manual", + ) + self.db.commit() + assert edge is not None + updated = svc.patch_manual_edge_ports( + self.db, edge.id, a_port="Gi0/10", b_port="Gi0/20", commit=True + ) + self.assertEqual(updated.a_port.lower(), "gi0/10") + self.assertEqual(updated.b_port.lower(), "gi0/20") + for ne in (ne_a, ne_b): + self.db.delete(ne) + self.db.commit() + + def test_patch_non_manual_edge_ports_rejected(self) -> None: + from fastapi import HTTPException + + suffix = uuid4().hex[:8] + ne_a = ManagedNE( + id=f"ne-nm-a-{suffix}", + name=f"NMA-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.5.2.{(int(suffix[:2], 16) % 200) + 1}", + ) + ne_b = ManagedNE( + id=f"ne-nm-b-{suffix}", + name=f"NMB-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.5.3.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add_all([ne_a, ne_b]) + self.db.commit() + fa = svc.ensure_fabric_node_for_managed(self.db, ne_a) + fb = svc.ensure_fabric_node_for_managed(self.db, ne_b) + edge, _ = svc.upsert_fabric_edge( + self.db, + a_node_id=fa.id, + b_node_id=fb.id, + a_port="gi0/1", + b_port="gi0/2", + source="lldp", + ) + self.db.commit() + assert edge is not None + with self.assertRaises(HTTPException) as ctx: + svc.patch_manual_edge_ports( + self.db, edge.id, a_port="gi0/9", b_port="gi0/8", commit=True + ) + self.assertEqual(ctx.exception.status_code, 400) + self.assertEqual(ctx.exception.detail, "edge_not_manual") + for ne in (ne_a, ne_b): + self.db.delete(ne) + self.db.commit() + + def test_patch_manual_edge_ports_conflict_409(self) -> None: + from fastapi import HTTPException + + suffix = uuid4().hex[:8] + ne_a = ManagedNE( + id=f"ne-cf-a-{suffix}", + name=f"CFA-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.5.4.{(int(suffix[:2], 16) % 200) + 1}", + ) + ne_b = ManagedNE( + id=f"ne-cf-b-{suffix}", + name=f"CFB-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.5.5.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add_all([ne_a, ne_b]) + self.db.commit() + fa = svc.ensure_fabric_node_for_managed(self.db, ne_a) + fb = svc.ensure_fabric_node_for_managed(self.db, ne_b) + e1, _ = svc.upsert_fabric_edge( + self.db, + a_node_id=fa.id, + b_node_id=fb.id, + a_port="gi0/1", + b_port="gi0/2", + source="manual", + ) + e2, _ = svc.upsert_fabric_edge( + self.db, + a_node_id=fa.id, + b_node_id=fb.id, + a_port="gi0/3", + b_port="gi0/4", + source="manual", + ) + self.db.commit() + assert e1 is not None and e2 is not None + # Use the canonical A/B ports already stored on e1 (normalize may have swapped). + with self.assertRaises(HTTPException) as ctx: + svc.patch_manual_edge_ports( + self.db, + e2.id, + a_port=e1.a_port, + b_port=e1.b_port, + commit=True, + ) + self.assertEqual(ctx.exception.status_code, 409) + self.assertEqual(ctx.exception.detail, "edge_port_conflict") + for ne in (ne_a, ne_b): + self.db.delete(ne) + self.db.commit() + + def test_view_commit_atomic_rollback(self) -> None: + from fastapi import HTTPException + + suffix = uuid4().hex[:8] + ne = ManagedNE( + id=f"ne-cmt-fail-{suffix}", + name=f"CFail-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.4.0.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.commit() + view_id = self._canvas(f"CFail-{suffix}") + fa = svc.ensure_fabric_node_for_managed(self.db, ne) + self.db.commit() + before_nodes = ( + self.db.query(TopoViewNode).filter(TopoViewNode.view_id == view_id).count() + ) + before_edges = self.db.query(TopoFabricEdge).count() + with self.assertRaises(HTTPException) as ctx: + svc.commit_view_changes( + self.db, + view_id, + ViewCommitIn( + edge_creates=[ + ViewCommitEdgeCreate( + client_id="local-loop", + a_node_id=fa.id, + b_node_id=fa.id, + a_port="gi0/1", + b_port="gi0/2", + ) + ], + node_adds=[fa.id], + positions=[ViewNodeIn(fabric_node_id=fa.id, x=99, y=99)], + ), + ) + detail = ctx.exception.detail + self.assertIsInstance(detail, dict) + self.assertEqual(detail.get("failed_step"), "edge_creates") + self.assertEqual( + self.db.query(TopoViewNode).filter(TopoViewNode.view_id == view_id).count(), + before_nodes, + ) + self.assertEqual(self.db.query(TopoFabricEdge).count(), before_edges) + self.db.delete(ne) + self.db.commit() + + def test_view_commit_happy_path(self) -> None: + suffix = uuid4().hex[:8] + ne_a = ManagedNE( + id=f"ne-cmt-a-{suffix}", + name=f"CA-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.4.1.{(int(suffix[:2], 16) % 200) + 1}", + ) + ne_b = ManagedNE( + id=f"ne-cmt-b-{suffix}", + name=f"CB-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.4.2.{(int(suffix[:2], 16) % 200) + 1}", + ) + ne_c = ManagedNE( + id=f"ne-cmt-c-{suffix}", + name=f"CC-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.4.3.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add_all([ne_a, ne_b, ne_c]) + self.db.commit() + view_id = self._canvas(f"COk-{suffix}") + fa = svc.ensure_fabric_node_for_managed(self.db, ne_a) + fb = svc.ensure_fabric_node_for_managed(self.db, ne_b) + fc = svc.ensure_fabric_node_for_managed(self.db, ne_c) + self.db.commit() + svc.add_nodes_to_view( + self.db, view_id, ViewNodesAdd(fabric_node_ids=[fa.id, fb.id]) + ) + old_edge, _ = svc.upsert_fabric_edge( + self.db, + a_node_id=fa.id, + b_node_id=fb.id, + a_port="gi0/9", + b_port="gi0/9", + source="manual", + ) + self.db.commit() + assert old_edge is not None + + out = svc.commit_view_changes( + self.db, + view_id, + ViewCommitIn( + edge_creates=[ + ViewCommitEdgeCreate( + client_id="local-new", + a_node_id=fa.id, + b_node_id=fc.id, + a_port="gi0/1", + b_port="gi0/2", + ) + ], + edge_deletes=[old_edge.id], + node_adds=[fc.id], + node_removes=[fb.id], + positions=[ + ViewNodeIn(fabric_node_id=fa.id, x=11, y=22), + ViewNodeIn(fabric_node_id=fc.id, x=33, y=44), + ], + ), + ) + self.assertTrue(out.ok) + self.assertIn("local-new", out.edge_id_map) + self.assertIsNotNone(out.graph) + ids = {n.fabric_node_id for n in out.graph.nodes} + self.assertIn(fa.id, ids) + self.assertIn(fc.id, ids) + self.assertNotIn(fb.id, ids) + old_edge_id = str(old_edge.id) + self.db.expire_all() + self.assertIsNone(self.db.get(TopoFabricEdge, old_edge_id)) + new_eid = out.edge_id_map["local-new"] + self.assertIsNotNone(self.db.get(TopoFabricEdge, new_eid)) + pos_a = next(n for n in out.graph.nodes if n.fabric_node_id == fa.id) + self.assertEqual(float(pos_a.x), 11.0) + self.assertEqual(float(pos_a.y), 22.0) + # Node added in the same commit must also take its position (autoflush=False). + pos_c = next(n for n in out.graph.nodes if n.fabric_node_id == fc.id) + self.assertEqual((float(pos_c.x), float(pos_c.y)), (33.0, 44.0)) + for ne in (ne_a, ne_b, ne_c): + self.db.delete(ne) + self.db.commit() + + def _two_node_canvas(self, tag: str): + suffix = uuid4().hex[:8] + nes = [ + ManagedNE( + id=f"ne-{tag}-{i}-{suffix}", + name=f"{tag.upper()}{i}-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.7.{i}.{(int(suffix[:2], 16) % 200) + 1}", + ) + for i in (1, 2) + ] + self.db.add_all(nes) + self.db.commit() + view_id = self._canvas(f"{tag}-{suffix}") + fa, fb = (svc.ensure_fabric_node_for_managed(self.db, ne) for ne in nes) + self.db.commit() + svc.add_nodes_to_view(self.db, view_id, ViewNodesAdd(fabric_node_ids=[fa.id, fb.id])) + return view_id, fa, fb + + def test_view_commit_skips_edits_on_edges_deleted_in_same_commit(self) -> None: + view_id, fa, fb = self._two_node_canvas("cdel") + edge, _ = svc.upsert_fabric_edge( + self.db, a_node_id=fa.id, b_node_id=fb.id, a_port="gi0/1", b_port="gi0/1", source="manual" + ) + self.db.commit() + assert edge is not None + eid = str(edge.id) + out = svc.commit_view_changes( + self.db, + view_id, + ViewCommitIn( + edge_deletes=[eid], + edge_port_edits=[ViewCommitEdgePortEdit(id=eid, a_port="gi0/5", b_port="gi0/6")], + edge_styles=[ViewEdgeStyleItem(fabric_edge_id=eid, stroke_color="#ff0000")], + ), + ) + self.assertTrue(out.ok) + self.db.expire_all() + self.assertIsNone(self.db.get(TopoFabricEdge, eid)) + + def test_view_commit_styles_new_edge_by_client_id(self) -> None: + view_id, fa, fb = self._two_node_canvas("csty") + out = svc.commit_view_changes( + self.db, + view_id, + ViewCommitIn( + edge_creates=[ + ViewCommitEdgeCreate(client_id="local:s1", a_node_id=fa.id, b_node_id=fb.id) + ], + edge_styles=[ + ViewEdgeStyleItem(fabric_edge_id="local:s1", stroke_color="#00ff00", stroke_width=3) + ], + ), + ) + new_id = out.edge_id_map["local:s1"] + edge_out = next(e for e in out.graph.edges if e.id == new_id) + self.assertEqual(edge_out.stroke_color, "#00ff00") + self.assertEqual(edge_out.stroke_width, 3) + + def test_positions_persist_child_region_icon_without_placement(self) -> None: + suffix = uuid4().hex[:8] + parent_id = self._region(f"Rg-{suffix}") + view = self._physical_view(parent_id) + child = svc.create_folder( + self.db, TopologyFolderCreate(name=f"Sub-{suffix}", kind="region", parent_id=parent_id) + ) + nid = f"region:{child.id}" + # Legacy data: drop the icon's placement row so it is only synthesized. + self.db.query(TopoViewNode).filter( + TopoViewNode.view_id == view.id, TopoViewNode.fabric_node_id == nid + ).delete(synchronize_session=False) + self.db.commit() + g = svc.get_view_graph(self.db, view.id) + self.assertIn(nid, {n.fabric_node_id for n in g.nodes}) + + svc.patch_view_positions( + self.db, view.id, ViewPositionsPatch(positions=[ViewNodeIn(fabric_node_id=nid, x=321, y=123)]) + ) + g = svc.get_view_graph(self.db, view.id) + icon = next(n for n in g.nodes if n.fabric_node_id == nid) + self.assertEqual((icon.x, icon.y), (321.0, 123.0)) + + self.db.query(TopoViewNode).filter( + TopoViewNode.view_id == view.id, TopoViewNode.fabric_node_id == nid + ).delete(synchronize_session=False) + self.db.commit() + out = svc.commit_view_changes( + self.db, view.id, ViewCommitIn(positions=[ViewNodeIn(fabric_node_id=nid, x=55, y=66)]) + ) + icon = next(n for n in out.graph.nodes if n.fabric_node_id == nid) + self.assertEqual((icon.x, icon.y), (55.0, 66.0)) + # Placement follows the folder name (no frozen label). + self.assertEqual(icon.label_override, "") + + def test_positions_do_not_place_foreign_region_icon(self) -> None: + suffix = uuid4().hex[:8] + view_id = self._canvas(f"Rf-{suffix}") + other = svc.create_folder(self.db, TopologyFolderCreate(name=f"Other-{suffix}", kind="region")) + out = svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ViewNodeIn(fabric_node_id=f"region:{other.id}", x=1, y=1)], + return_graph=False, + ), + ) + self.assertEqual(out.skipped_missing, 1) + def test_view_crud_and_positions(self) -> None: suffix = uuid4().hex[:8] ne = ManagedNE( @@ -417,6 +804,420 @@ class FabricTopologyTests(unittest.TestCase): self.db.delete(ne) self.db.commit() + def test_positions_preserve_lock_when_omitted(self) -> None: + suffix = uuid4().hex[:8] + ne = ManagedNE( + id=f"ne-lock-{suffix}", + name=f"L-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.1.0.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.commit() + view_id = self._canvas(f"Lock-{suffix}") + graph = svc.add_nodes_to_view(self.db, view_id, ViewNodesAdd(managed_ne_ids=[ne.id])) + fid = graph.nodes[0].fabric_node_id + svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ViewNodeIn(fabric_node_id=fid, x=500, y=500, locked=True)], + return_graph=False, + ), + ) + # UI-style save: omit locked — must not move or unlock. + out = svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ViewNodeIn(fabric_node_id=fid, x=10, y=10)], + return_graph=False, + ), + ) + self.assertEqual(out.skipped_locked, 1) + self.assertEqual(out.updated, 0) + row = ( + self.db.query(TopoViewNode) + .filter(TopoViewNode.view_id == view_id, TopoViewNode.fabric_node_id == fid) + .one() + ) + self.assertTrue(row.locked) + self.assertEqual(float(row.x), 500.0) + self.assertEqual(float(row.y), 500.0) + svc.delete_view(self.db, view_id, force=True) + self.db.delete(ne) + self.db.commit() + + def test_positions_explicit_unlock_moves(self) -> None: + suffix = uuid4().hex[:8] + ne = ManagedNE( + id=f"ne-ulock-{suffix}", + name=f"U-{suffix}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.1.1.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.commit() + view_id = self._canvas(f"ULock-{suffix}") + graph = svc.add_nodes_to_view(self.db, view_id, ViewNodesAdd(managed_ne_ids=[ne.id])) + fid = graph.nodes[0].fabric_node_id + svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ViewNodeIn(fabric_node_id=fid, x=500, y=500, locked=True)], + return_graph=False, + ), + ) + out = svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ViewNodeIn(fabric_node_id=fid, x=10, y=20, locked=False)], + return_graph=False, + ), + ) + self.assertEqual(out.skipped_locked, 0) + self.assertEqual(out.updated, 1) + row = ( + self.db.query(TopoViewNode) + .filter(TopoViewNode.view_id == view_id, TopoViewNode.fabric_node_id == fid) + .one() + ) + self.assertFalse(row.locked) + self.assertEqual(float(row.x), 10.0) + self.assertEqual(float(row.y), 20.0) + svc.delete_view(self.db, view_id, force=True) + self.db.delete(ne) + self.db.commit() + + def test_positions_without_label_keeps_following_name(self) -> None: + suffix = uuid4().hex[:8] + ne = ManagedNE( + id=f"ne-lbl-{suffix}", + name="OLD-NAME", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.2.0.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.commit() + view_id = self._canvas(f"Lbl-{suffix}") + graph = svc.add_nodes_to_view(self.db, view_id, ViewNodesAdd(managed_ne_ids=[ne.id])) + fid = graph.nodes[0].fabric_node_id + self.assertEqual(graph.nodes[0].label, "OLD-NAME") + self.assertEqual(graph.nodes[0].label_override, "") + # Save positions without label — must not freeze the device name. + svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ViewNodeIn(fabric_node_id=fid, x=40, y=60)], + return_graph=False, + ), + ) + fn = self.db.get(TopoFabricNode, fid) + assert fn is not None + fn.name = "NEW-NAME" + self.db.commit() + refreshed = svc.get_view_graph(self.db, view_id) + self.assertEqual(refreshed.nodes[0].label, "NEW-NAME") + self.assertEqual(refreshed.nodes[0].label_override, "") + svc.delete_view(self.db, view_id, force=True) + self.db.delete(ne) + self.db.commit() + + def test_label_override_survives_rename(self) -> None: + suffix = uuid4().hex[:8] + ne = ManagedNE( + id=f"ne-ov-{suffix}", + name="DEVICE-A", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.2.1.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.commit() + view_id = self._canvas(f"Ov-{suffix}") + graph = svc.add_nodes_to_view(self.db, view_id, ViewNodesAdd(managed_ne_ids=[ne.id])) + fid = graph.nodes[0].fabric_node_id + svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ + ViewNodeIn(fabric_node_id=fid, x=1, y=2, label="CUSTOM"), + ], + return_graph=False, + ), + ) + fn = self.db.get(TopoFabricNode, fid) + assert fn is not None + fn.name = "DEVICE-B" + self.db.commit() + refreshed = svc.get_view_graph(self.db, view_id) + self.assertEqual(refreshed.nodes[0].label, "CUSTOM") + self.assertEqual(refreshed.nodes[0].label_override, "CUSTOM") + # Clear override → follow device name again. + svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ViewNodeIn(fabric_node_id=fid, x=1, y=2, label="")], + return_graph=False, + ), + ) + cleared = svc.get_view_graph(self.db, view_id) + self.assertEqual(cleared.nodes[0].label, "DEVICE-B") + self.assertEqual(cleared.nodes[0].label_override, "") + svc.delete_view(self.db, view_id, force=True) + self.db.delete(ne) + self.db.commit() + + def test_clear_frozen_labels_script_dry_run_and_apply(self) -> None: + from scripts.topology_clear_frozen_labels import clear_frozen_labels + + suffix = uuid4().hex[:8] + ne = ManagedNE( + id=f"ne-frz-{suffix}", + name="FROZEN-NAME", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.3.0.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.commit() + view_id = self._canvas(f"Frz-{suffix}") + graph = svc.add_nodes_to_view(self.db, view_id, ViewNodesAdd(managed_ne_ids=[ne.id])) + fid = graph.nodes[0].fabric_node_id + # Simulate a pre-T2 save that froze the device name into vn.label. + row = ( + self.db.query(TopoViewNode) + .filter(TopoViewNode.view_id == view_id, TopoViewNode.fabric_node_id == fid) + .one() + ) + row.label = "FROZEN-NAME" + self.db.commit() + + scanned, cleared = clear_frozen_labels(self.db, apply=False) + self.assertGreaterEqual(scanned, 1) + self.assertGreaterEqual(cleared, 1) + self.db.refresh(row) + self.assertEqual(row.label, "FROZEN-NAME") + + scanned2, cleared2 = clear_frozen_labels(self.db, apply=True) + self.db.commit() + self.assertEqual(cleared2, cleared) + self.db.refresh(row) + self.assertEqual(row.label, "") + + scanned3, cleared3 = clear_frozen_labels(self.db, apply=False) + self.assertEqual(cleared3, 0) + self.assertGreaterEqual(scanned3, 0) + + svc.delete_view(self.db, view_id, force=True) + self.db.delete(ne) + self.db.commit() + + def test_outside_peers_found_when_many_internal_edges(self) -> None: + """Internal edges must not consume the outside-peer scan budget.""" + from netx_api.topology_views_graph import _outside_peers_for_view + + suffix = uuid4().hex[:8] + nodes: list[TopoFabricNode] = [] + for i in range(6): + ne = ManagedNE( + id=f"ne-op-{suffix}-{i}", + name=f"OP-{suffix}-{i}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.8.{i}.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.flush() + nodes.append(svc.ensure_fabric_node_for_managed(self.db, ne)) + outside = TopoFabricNode( + id=f"out-{suffix}", + name=f"OUT-{suffix}", + ip="10.8.99.1", + ) + self.db.add(outside) + self.db.commit() + # Dense internal mesh among the first 5 nodes. + for i, a in enumerate(nodes[:5]): + for j, b in enumerate(nodes[:5]): + if i >= j: + continue + svc.upsert_fabric_edge( + self.db, + a_node_id=a.id, + b_node_id=b.id, + a_port=f"gi0/{i}", + b_port=f"gi0/{j}", + source="manual", + ) + # One external edge after the internal flood. + svc.upsert_fabric_edge( + self.db, + a_node_id=nodes[0].id, + b_node_id=outside.id, + a_port="gi0/99", + b_port="gi0/1", + source="manual", + ) + self.db.commit() + view_id = self._canvas(f"OP-{suffix}") + view = self.db.get(TopoView, view_id) + assert view is not None + svc.add_nodes_to_view( + self.db, + view_id, + ViewNodesAdd(fabric_node_ids=[n.id for n in nodes[:5]]), + ) + peers = _outside_peers_for_view( + self.db, + view, + member_ids={n.id for n in nodes[:5]}, + layer="physical", + limit=50, + edge_scan_limit=10, # would miss the peer if internals counted + ) + peer_ids = {p["fabric_node_id"] for p in peers} + self.assertIn(outside.id, peer_ids) + svc.delete_view(self.db, view_id, force=True) + self.db.commit() + + def test_view_truncation_is_deterministic(self) -> None: + from unittest.mock import patch + + from netx_api import topology_views_graph as vg + + suffix = uuid4().hex[:8] + view_id = self._canvas(f"Trunc-{suffix}") + fids: list[str] = [] + for i in range(5): + ne = ManagedNE( + id=f"ne-tr-{suffix}-{i}", + name=f"TR-{suffix}-{i}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.9.{i}.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.flush() + fn = svc.ensure_fabric_node_for_managed(self.db, ne) + fids.append(fn.id) + self.db.commit() + svc.add_nodes_to_view(self.db, view_id, ViewNodesAdd(fabric_node_ids=fids)) + with patch.object(vg, "VIEW_GRAPH_NODE_HARD_CAP", 3): + g1 = svc.get_view_graph(self.db, view_id) + g2 = svc.get_view_graph(self.db, view_id) + ids1 = [n.fabric_node_id for n in g1.nodes] + ids2 = [n.fabric_node_id for n in g2.nodes] + self.assertEqual(len(ids1), 3) + self.assertEqual(ids1, ids2) + self.assertTrue(g1.truncated) + svc.delete_view(self.db, view_id, force=True) + self.db.commit() + + def test_positions_batch_lookup_avoids_n_plus_one(self) -> None: + """200-node positions patch should issue a handful of SQL statements, not 200+.""" + from sqlalchemy import event + + suffix = uuid4().hex[:8] + view_id = self._canvas(f"N1-{suffix}") + fids: list[str] = [] + for i in range(40): + ne = ManagedNE( + id=f"ne-n1-{suffix}-{i}", + name=f"N1-{suffix}-{i}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.10.{i // 200}.{(i % 200) + 1}", + ) + self.db.add(ne) + self.db.flush() + fids.append(svc.ensure_fabric_node_for_managed(self.db, ne).id) + self.db.commit() + svc.add_nodes_to_view(self.db, view_id, ViewNodesAdd(fabric_node_ids=fids)) + statements: list[str] = [] + + def _count(conn, cursor, statement, parameters, context, executemany): # noqa: ANN001 + statements.append(str(statement)) + + bind = self.db.get_bind() + event.listen(bind, "before_cursor_execute", _count) + try: + svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ + ViewNodeIn(fabric_node_id=fid, x=float(i), y=float(i)) + for i, fid in enumerate(fids) + ], + return_graph=False, + ), + ) + finally: + event.remove(bind, "before_cursor_execute", _count) + # Cap well below one-query-per-node (40+). + self.assertLessEqual(len(statements), 15) + svc.delete_view(self.db, view_id, force=True) + self.db.commit() + + def test_positions_respects_max_nodes(self) -> None: + """Positions must skip unknown placements instead of inserting past the cap.""" + suffix = uuid4().hex[:8] + view_id = self._canvas( + f"Cap-{suffix}", + filter={"membership": {"max_nodes": 2, "mode": "manual"}}, + ) + nes = [] + fids = [] + for i in range(3): + ne = ManagedNE( + id=f"ne-cap-{suffix}-{i}", + name=f"CAP-{suffix}-{i}", + vendor="Cisco", + device_type="cisco_ios", + ip_address=f"10.11.{i}.{(int(suffix[:2], 16) % 200) + 1}", + ) + self.db.add(ne) + self.db.flush() + nes.append(ne) + fids.append(svc.ensure_fabric_node_for_managed(self.db, ne).id) + self.db.commit() + svc.add_nodes_to_view( + self.db, view_id, ViewNodesAdd(fabric_node_ids=fids[:2], return_graph=False) + ) + before = ( + self.db.query(TopoViewNode).filter(TopoViewNode.view_id == view_id).count() + ) + self.assertEqual(before, 2) + out = svc.patch_view_positions( + self.db, + view_id, + ViewPositionsPatch( + positions=[ + ViewNodeIn(fabric_node_id=fids[0], x=1, y=1), + ViewNodeIn(fabric_node_id=fids[2], x=9, y=9), # not on view + ], + return_graph=False, + ), + ) + self.assertGreaterEqual(out.skipped_missing, 1) + after = ( + self.db.query(TopoViewNode).filter(TopoViewNode.view_id == view_id).count() + ) + self.assertEqual(after, 2) + svc.delete_view(self.db, view_id, force=True) + for ne in nes: + self.db.delete(ne) + self.db.commit() + def test_create_topology_placeholder_on_view(self) -> None: suffix = uuid4().hex[:8] view_id = self._canvas(f"Rph-{suffix}") @@ -1249,6 +2050,8 @@ class FabricTopologyTests(unittest.TestCase): vendor="Cisco", device_type="cisco_ios", ip_address=f"203.0.113.{(int(suffix[:2], 16) % 100) + 1}", + username="u", + password_enc=encrypt_secret("p"), ) ne_b = ManagedNE( id=f"neb-{suffix}", @@ -1256,6 +2059,8 @@ class FabricTopologyTests(unittest.TestCase): vendor="Cisco", device_type="cisco_ios", ip_address=f"203.0.113.{(int(suffix[2:4], 16) % 100) + 101}", + username="u", + password_enc=encrypt_secret("p"), ) self.db.add(ne_a) self.db.add(ne_b) @@ -1270,7 +2075,12 @@ class FabricTopologyTests(unittest.TestCase): "output": CISCO_LLDP_DETAIL, "commands": ["show lldp neighbors detail"], } - with patch("netx_api.topology_discover_scan.execute_managed_ne_commands", return_value=fake_exec): + # Background workers open SessionLocal(); point them at this in-memory engine. + with ( + patch("netx_api.topology_discover_jobs.SessionLocal", self.Session), + patch("netx_api.topology_discover_scan.SessionLocal", self.Session), + patch("netx_api.topology_discover_scan.execute_managed_ne_commands", return_value=fake_exec), + ): job = svc.start_discover_job( self.db, FabricDiscoverRequest(scope="ne_ids", ne_ids=[ne_a.id], concurrency=1), @@ -1422,6 +2232,8 @@ class FabricTopologyTests(unittest.TestCase): device_type="cisco_ios", ip_address=f"203.0.113.{(int(suffix[:2], 16) % 80) + 40}", source="manual", + username="u", + password_enc=encrypt_secret("p"), ) self.db.add(ne_a) self.db.commit() @@ -1442,7 +2254,11 @@ Management Addresses: "output": lldp_out, "commands": ["show lldp neighbors detail"], } - with patch("netx_api.topology_discover_scan.execute_managed_ne_commands", return_value=fake_exec): + with ( + patch("netx_api.topology_discover_jobs.SessionLocal", self.Session), + patch("netx_api.topology_discover_scan.SessionLocal", self.Session), + patch("netx_api.topology_discover_scan.execute_managed_ne_commands", return_value=fake_exec), + ): job = svc.start_discover_job( self.db, FabricDiscoverRequest( @@ -2283,7 +3099,9 @@ Management Addresses: self.db.add(orphan) self.db.commit() - stats = run_fabric_reconcile_once() + # run_fabric_reconcile_once opens SessionLocal(); bind it to this engine. + with patch("netx_api.fabric_reconcile_scheduler.SessionLocal", self.Session): + stats = run_fabric_reconcile_once() self.assertGreaterEqual(int(stats.get("purged_orphans") or 0), 1) self.db.expire_all() self.assertIsNone(self.db.get(TopoFabricNode, orphan_id)) @@ -2586,6 +3404,27 @@ Management Addresses: self.assertEqual(full["detail"], "full") self.assertIn("attrs", full["paths"][0]["nodes"][0]) + # A parallel A—B link must not spawn a duplicate 1-hop path. + svc.upsert_fabric_edge( + self.db, + a_node_id=nodes[1].id, + b_node_id=nodes[0].id, + a_port="Gi0/9", + b_port="Gi0/9", + source="manual", + ) + self.db.commit() + dedup = svc.find_fabric_paths( + self.db, + from_managed_ne_id=nes[0].id, + to_managed_ne_id=nes[1].id, + max_paths=3, + max_hops=6, + ) + self.assertEqual([p["hops"] for p in dedup["paths"]], [1, 2]) + self.assertEqual(dedup["paths"][0]["edges"][0]["parallel_links"], 2) + self.assertNotIn("parallel_links", dedup["paths"][1]["edges"][0]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ume_topology_apply.py b/tests/test_ume_topology_apply.py index 3c6e819..73ecc6c 100644 --- a/tests/test_ume_topology_apply.py +++ b/tests/test_ume_topology_apply.py @@ -345,20 +345,12 @@ class UmeFabricApplyTests(unittest.TestCase): edge.source = "ume" edge.attrs = {"sources": ["ume"], "ume_link_id": "link-1"} self.db.commit() - a = ( - self.db.query(TopoFabricNode) - .filter(TopoFabricNode.ume_ne_id == "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa") - .one() - ) - b = ( - self.db.query(TopoFabricNode) - .filter(TopoFabricNode.ume_ne_id == "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb") - .one() - ) + # Ports are already assigned to the edge's canonical A/B order; pass the + # same node ids so _normalize_endpoints cannot swap ports onto the wrong ends. edge2, action = upsert_fabric_edge( self.db, - a_node_id=a.id, - b_node_id=b.id, + a_node_id=edge.a_node_id, + b_node_id=edge.b_node_id, a_port=rich_a, b_port=rich_b, source="lldp", diff --git a/tests/test_user_prefs.py b/tests/test_user_prefs.py new file mode 100644 index 0000000..9cd1254 --- /dev/null +++ b/tests/test_user_prefs.py @@ -0,0 +1,193 @@ +"""Per-user prefs API: workbench layout read/write, whitelist, validation, isolation.""" + +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool + +from netx_api.auth_middleware import AuthAuditMiddleware +from netx_api.auth_router import router as auth_router +from netx_api.auth_service import bootstrap_admin_if_needed, create_user +from netx_api.db import Base, get_db +from netx_api.models import AppUser + + +class UserPrefsApiTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine( + "sqlite+pysqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(bind=self.engine) + self.Session = sessionmaker(bind=self.engine, autoflush=False, autocommit=False) + + self.app = FastAPI() + self.app.add_middleware(AuthAuditMiddleware) + self.app.include_router(auth_router) + + def _override_db(): + db = self.Session() + try: + yield db + finally: + db.close() + + self.app.dependency_overrides[get_db] = _override_db + + self._sess_patch = patch("netx_api.auth_middleware.SessionLocal", self.Session) + self._sess_patch.start() + self._settings_patches = [ + patch("netx_api.auth_middleware.settings.auth_enabled", True), + patch("netx_api.audit_async.settings.audit_async", False), + patch("netx_api.auth_tokens.settings.auth_secret", "unit-test-auth-secret-32bytes!!"), + patch("netx_api.auth_tokens.settings.auth_token_ttl_sec", 3600), + patch("netx_api.auth_service.settings.bootstrap_admin_username", "admin"), + patch("netx_api.auth_service.settings.bootstrap_admin_password", "adminpass"), + patch("netx_api.auth_deps.settings.auth_enabled", True), + ] + for p in self._settings_patches: + p.start() + + db = self.Session() + try: + bootstrap_admin_if_needed(db) + admin = db.query(AppUser).filter(AppUser.username == "admin").one() + admin.must_change_password = False + create_user(db, username="alice", password="alice123", role="user", actor=admin) + create_user(db, username="bob", password="bob12345", role="user", actor=admin) + for name in ("alice", "bob"): + u = db.query(AppUser).filter(AppUser.username == name).one() + u.must_change_password = False + db.commit() + finally: + db.close() + + self.client = TestClient(self.app) + + def tearDown(self) -> None: + self._sess_patch.stop() + for p in self._settings_patches: + p.stop() + self.app.dependency_overrides.clear() + self.engine.dispose() + + def _login(self, username: str, password: str) -> str: + r = self.client.post("/v1/auth/login", json={"username": username, "password": password}) + self.assertEqual(r.status_code, 200, r.text) + return str(r.json()["access_token"]) + + def test_me_includes_empty_prefs(self) -> None: + token = self._login("alice", "alice123") + me = self.client.get("/v1/auth/me", headers={"Authorization": f"Bearer {token}"}) + self.assertEqual(me.status_code, 200, me.text) + self.assertEqual(me.json().get("prefs"), {}) + login = self.client.post("/v1/auth/login", json={"username": "alice", "password": "alice123"}) + self.assertEqual(login.json().get("prefs"), {}) + + def test_get_and_put_workbench(self) -> None: + token = self._login("alice", "alice123") + empty = self.client.get("/v1/auth/me/prefs", headers={"Authorization": f"Bearer {token}"}) + self.assertEqual(empty.status_code, 200, empty.text) + self.assertEqual(empty.json(), {"prefs": {}}) + + body = {"order": ["ne", "topology"], "hidden": ["webcrt"]} + put = self.client.put( + "/v1/auth/me/prefs/workbench", + headers={"Authorization": f"Bearer {token}"}, + json=body, + ) + self.assertEqual(put.status_code, 200, put.text) + self.assertEqual(put.json()["prefs"]["workbench"], body) + + got = self.client.get("/v1/auth/me/prefs", headers={"Authorization": f"Bearer {token}"}) + self.assertEqual(got.json()["prefs"]["workbench"], body) + me = self.client.get("/v1/auth/me", headers={"Authorization": f"Bearer {token}"}) + self.assertEqual(me.json()["prefs"]["workbench"], body) + + def test_put_strips_unknown_fields(self) -> None: + token = self._login("alice", "alice123") + put = self.client.put( + "/v1/auth/me/prefs/workbench", + headers={"Authorization": f"Bearer {token}"}, + json={"order": ["ne"], "hidden": [], "extra": True, "panels": {}}, + ) + self.assertEqual(put.status_code, 200, put.text) + self.assertEqual(put.json()["prefs"]["workbench"], {"order": ["ne"], "hidden": []}) + + def test_unknown_key_404(self) -> None: + token = self._login("alice", "alice123") + r = self.client.put( + "/v1/auth/me/prefs/panels", + headers={"Authorization": f"Bearer {token}"}, + json={"order": [], "hidden": []}, + ) + self.assertEqual(r.status_code, 404) + + def test_validation_422(self) -> None: + token = self._login("alice", "alice123") + headers = {"Authorization": f"Bearer {token}"} + + bad_char = self.client.put( + "/v1/auth/me/prefs/workbench", + headers=headers, + json={"order": ["NE_Admin"], "hidden": []}, + ) + self.assertEqual(bad_char.status_code, 422) + + too_long = self.client.put( + "/v1/auth/me/prefs/workbench", + headers=headers, + json={"order": [f"m{i}" for i in range(65)], "hidden": []}, + ) + self.assertEqual(too_long.status_code, 422) + + bad_type = self.client.put( + "/v1/auth/me/prefs/workbench", + headers=headers, + json={"order": "ne", "hidden": []}, + ) + self.assertEqual(bad_type.status_code, 422) + + def test_user_isolation(self) -> None: + alice = self._login("alice", "alice123") + bob = self._login("bob", "bob12345") + layout = {"order": ["topology", "ne"], "hidden": ["ume"]} + put = self.client.put( + "/v1/auth/me/prefs/workbench", + headers={"Authorization": f"Bearer {alice}"}, + json=layout, + ) + self.assertEqual(put.status_code, 200, put.text) + + bob_me = self.client.get("/v1/auth/me", headers={"Authorization": f"Bearer {bob}"}) + self.assertEqual(bob_me.status_code, 200, bob_me.text) + self.assertEqual(bob_me.json().get("prefs"), {}) + + alice_me = self.client.get("/v1/auth/me", headers={"Authorization": f"Bearer {alice}"}) + self.assertEqual(alice_me.json()["prefs"]["workbench"], layout) + + def test_auth_disabled_does_not_insert_system_user(self) -> None: + with patch("netx_api.auth_deps.settings.auth_enabled", False), patch( + "netx_api.auth_middleware.settings.auth_enabled", False + ): + r = self.client.put( + "/v1/auth/me/prefs/workbench", + json={"order": ["ne"], "hidden": []}, + ) + self.assertEqual(r.status_code, 409, r.text) + db = self.Session() + try: + self.assertIsNone(db.get(AppUser, "system")) + finally: + db.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_webcrt.py b/tests/test_webcrt.py index 4d93477..786ff23 100644 --- a/tests/test_webcrt.py +++ b/tests/test_webcrt.py @@ -94,6 +94,10 @@ class WebcrtServiceTests(unittest.TestCase): telnet = _FakeConn() telnet.RETURN = "\r\n" self.assertEqual(svc.map_network_cli_enter("\r", telnet), "\r\n") # type: ignore[arg-type] + self.assertEqual(svc.map_network_cli_enter("show clock\r\n\r\n", telnet), "show clock\r\n\r\n") + self.assertEqual(svc.map_network_cli_enter("a\nb\rc\r\n", telnet), "a\r\nb\r\nc\r\n") + telnet.RETURN = "\r" + self.assertEqual(svc.map_network_cli_enter("a\r\nb\n", telnet), "a\rb\r") self.assertEqual(svc.normalize_cli_transcript("R2#R2#\nR2#"), "R2#") self.assertEqual(svc.normalize_cli_transcript("banner\nR2#R2#"), "banner\nR2#") self.assertEqual(svc.prepare_bootstrap_output("login\nR2#\nR2#"), "login\nR2#") @@ -154,6 +158,47 @@ class WebcrtServiceTests(unittest.TestCase): self.assertIn("R2#", text) self.assertNotIn("MagicMock", text) + @patch.object(reg, "_audit") + @patch.object(reg, "get_cli_hop_guard", return_value=None) + @patch.object(reg, "_capture_raw_channel", return_value="") + @patch.object(reg, "_prime_interactive_channel", return_value="") + @patch.object(reg, "open_netmiko_connection") + def test_first_prompt_arriving_after_reader_start_is_preserved(self, mock_open, *_mocks) -> None: + sess = svc.WebcrtSession( + session_id="late-first-prompt", ne_id="lab", ne_name="lab", + ne_ip="192.0.2.1", protocol="telnet", cols=80, rows=24, + ) + mock_open.return_value = _FakeConn() + # Auth banner has been streamed, but the VM has not sent its first prompt yet. + sess.push_connect_echo("Login successful\r\n") + with patch.object(sess, "start_reader", side_effect=lambda: sess.out_queue.put(b"")), \ + patch.object(sess, "open_session_log"), patch.object(sess, "append_session_log"): + reg._finish_connect(sess, creds={}, device={}, connect_timeout=30, client="test") + self.assertEqual(sess.state, "ready") + self.assertEqual(sess.out_queue.get_nowait(), b"") + self.assertFalse(sess.needs_live_prompt) + + def test_quick_connect_honors_async_authentication(self) -> None: + from netx_api.webcrt_router import WebcrtQuickConnectBody, api_quick_connect + + ne = MagicMock() + ne.id = "lab" + ctx = MagicMock() + ctx.user.id = "user" + ctx.user.username = "operator" + with patch("netx_api.ne_service.upsert_webcrt_session_host", return_value=(ne, "reused")), \ + patch("netx_api.webcrt_router.create_session", return_value={"session_id": "pending"}) as create, \ + patch("netx_api.webcrt_router._client_label", return_value="test"): + for async_connect in (True, False): + body = WebcrtQuickConnectBody( + ip_address="192.0.2.1", protocol="ssh", username="operator", + password="one-shot", save_password=False, async_connect=async_connect, + ) + result = api_quick_connect(body, request=MagicMock(), db=MagicMock(), ctx=ctx) + self.assertEqual(result["session_id"], "pending") + self.assertEqual(create.call_args.kwargs["async_connect"], async_connect) + self.assertEqual(create.call_args.kwargs["password_override"], "one-shot") + @patch.object(reg, "_audit") @patch.object(reg, "open_netmiko_connection") @patch("netx_api.cli_resolve.resolve_cli_target") diff --git a/tests/test_webcrt_cmd_audit.py b/tests/test_webcrt_cmd_audit.py new file mode 100644 index 0000000..2e2a4e0 --- /dev/null +++ b/tests/test_webcrt_cmd_audit.py @@ -0,0 +1,295 @@ +"""Device-echo command audit: LineScreen / CommandTracker and the live session wiring.""" + +from __future__ import annotations + +import time +import unittest +from unittest.mock import MagicMock, patch + +from netx_api.webcrt_cmd_audit import CommandTracker, LineScreen, split_prompt +from netx_api.webcrt_session_model import WebcrtSession + + +def _run(steps, *, linux=False, settle=False): + t = CommandTracker(linux_like=linux) + res = [] + for kind, val in steps: + if kind == "out": + res += t.feed(val) + else: + lines, saw, enter = val + res += t.on_stdin(lines, saw_input=saw, had_enter=enter) + if settle: + res += t.expire(time.monotonic() + 5.0) + return [r.command for r in res] + + +class LineScreenTests(unittest.TestCase): + def _row(self, text: str) -> str: + s = LineScreen() + s.feed_row(text) + return s.text() + + def test_plain_and_cr(self) -> None: + self.assertEqual(self._row("abc\rX"), "Xbc") + + def test_backspace_overwrite(self) -> None: + self.assertEqual(self._row("disX\x08 \x08play"), "display") + + def test_csi_left_erase_and_delete(self) -> None: + self.assertEqual(self._row("display clock\x1b[13D\x1b[K"), "") + self.assertEqual(self._row("abcdef\x1b[3D\x1b[2P"), "abcf") + + def test_huawei_history_redraw(self) -> None: + raw = "display clock\x1b[13D \x1b[13Ddis ip int brief \x1b[17D \x1b[17Ddisplay clock" + self.assertEqual(self._row(raw), "display clock") + + def test_more_pager_wipe_leaves_blank_row(self) -> None: + self.assertEqual(self._row(" ---- More ----\x1b[16D \x1b[16D"), "") + + def test_partial_escape_is_carried_between_chunks(self) -> None: + t = CommandTracker() + t.feed("a\x1b[") + t.feed("1Db") + self.assertEqual(t.current_row(), "b") + + +class SplitPromptTests(unittest.TestCase): + def test_vendors(self) -> None: + self.assertEqual(split_prompt("display clock"), ("", "display clock")) + self.assertEqual(split_prompt("[~r1-GigabitEthernet0/0/0]ip address 1.1.1.1 24")[1], "ip address 1.1.1.1 24") + self.assertEqual(split_prompt("R2(config-if)#no shut"), ("R2(config-if)#", "no shut")) + self.assertEqual(split_prompt("[admin@MikroTik] /ip> print")[1], "print") + self.assertEqual(split_prompt("[user@host ~]$ ls -l")[1], "ls -l") + + def test_not_a_prompt(self) -> None: + self.assertIsNone(split_prompt("[Y/N]:")) + self.assertIsNone(split_prompt(" ---- More ----")) + self.assertIsNone(split_prompt("Username:admin")) + + +class TrackerScenarioTests(unittest.TestCase): + def test_echo_row_wins_over_lagging_typed_text(self) -> None: + got = _run([("out", ""), ("in", (["show clock"], True, True)), ("out", "sh"), ("out", "ow clock\r\nT\r\n")]) + self.assertEqual(got, ["show clock"]) + + def test_backspace_edit(self) -> None: + got = _run([("out", "R2#"), ("in", (["shoX"], True, False)), ("out", "shoX\b \bw"), + ("in", (["show clock"], True, True)), ("out", " clock\r\nT\r\nR2#")]) + self.assertEqual(got, ["R2#show clock"]) + + def test_bare_enter_not_audited(self) -> None: + self.assertEqual(_run([("out", ""), ("in", ([], False, True)), ("out", "\r\n")]), []) + + def test_history_recall_same_frame_and_separate(self) -> None: + a = _run([("out", ""), ("in", ([], True, True)), ("out", "display clock\r\nx\r\n")]) + b = _run([("out", ""), ("in", ([], True, False)), ("out", "display clock"), + ("in", ([], False, True)), ("out", "\r\nx\r\n")]) + self.assertEqual(a, ["display clock"]) + self.assertEqual(b, ["display clock"]) + + def test_tab_redraw_before_enter_echo(self) -> None: + # Huawei redraws the line after CRLF on Tab; the executed line is the later row. + got = _run([("out", ""), ("in", (["display ver\t"], True, True)), + ("out", "\tdisX\x1b[1D \x1b[1Dplay ver\r\ndisplay version \r\nVRP\r\n")]) + self.assertEqual(got, ["display version"]) + + def test_password_redacted(self) -> None: + self.assertEqual(_run([("out", "Password:"), ("in", (["secret"], True, True))]), ["***"]) + + def test_login_dialog_not_audited(self) -> None: + got = _run([("out", "Username:"), ("in", (["admin"], True, True)), ("out", "admin\r\nPassword:")], settle=True) + self.assertEqual(got, []) + + def test_multiline_paste_keeps_order_and_skips_empty_echo(self) -> None: + got = _run([("out", "R2#"), ("in", (["a", "b"], True, True)), ("out", "a\r\nR2#\r\nR2#b\r\nR2#")]) + self.assertEqual(got, ["R2#a", "R2#b"]) + + def test_pager_enter_is_not_a_command(self) -> None: + got = _run([("out", ""), ("in", (["display cur"], True, True)), + ("out", "display cur\r\nl1\r\n ---- More ----"), ("in", ([], False, True)), + ("out", "\x1b[16D \x1b[16D\r\nl2\r\n")]) + self.assertEqual(got, ["display cur"]) + + def test_no_echo_falls_back_to_snapshot(self) -> None: + self.assertEqual(_run([("out", "R2#show clock"), ("in", ([], False, True))], settle=True), ["R2#show clock"]) + + def test_linux_prompt_keeps_single_space(self) -> None: + got = _run([("out", "[u@h ~]$ "), ("in", (["ls"], True, True)), ("out", "ls\r\nf\r\n[u@h ~]$ ")], linux=True) + self.assertEqual(got, ["[u@h ~]$ ls"]) + + def test_enter_typed_ahead_during_output_then_recall(self) -> None: + # Up+Enter pressed while the previous command is still printing: output rows must + # not swallow the bare Enter. + got = _run([("out", ""), ("in", (["display clock"], True, True)), + ("in", ([], True, False)), ("in", ([], False, True)), + ("out", "display clock\r\n2026\r\nSunday\r\n"), + ("out", "display clock\r\n2026\r\nSunday\r\n")]) + self.assertEqual(got, ["display clock", "display clock"]) + + def test_pager_enter_then_typed_line_recorded_once(self) -> None: + got = _run([("out", ""), ("in", (["display cur"], True, True)), + ("out", "display cur\r\nl1\r\n ---- More ----"), ("in", ([], False, True)), + ("out", "\x1b[16D \x1b[16D\r\nl2\r\n"), + ("in", (["display clock"], True, True)), ("out", "display clock\r\nt\r\n")]) + self.assertEqual(got, ["display cur", "display clock"]) + + def test_ctrl_c_echo_does_not_steal_next_enter(self) -> None: + # IOS/bash echo the aborted row + CRLF; keys typed meanwhile must still be audited. + t = CommandTracker() + t.feed("R2#garbage") + res = t.on_stdin([], saw_input=True, had_enter=False, abort=True) + res += t.on_stdin(["show clock"], saw_input=True, had_enter=True) + res += t.feed("\r\nR2#show clock\r\n*22:57:17 UTC\r\nR2#") + self.assertEqual([r.command for r in res], ["R2#show clock"]) + + def test_ctrl_c_with_slow_echo_still_skips_abort_linefeed(self) -> None: + # Only "R2#garb" was echoed when Ctrl-C was typed; the device later prints the + # whole aborted row + CRLF, then echoes the line typed meanwhile. + t = CommandTracker() + t.feed("R2#") + t.on_stdin(["garbage"], saw_input=True, had_enter=False) + t.feed("garb") + res = t.on_stdin([], saw_input=True, had_enter=False, abort=True, abort_text="garbage") + res += t.on_stdin(["show clock"], saw_input=True, had_enter=True) + res += t.feed("age\r\nR2#show clock\r\n*22:57 UTC\r\nR2#") + self.assertEqual([r.command for r in res], ["R2#show clock"]) + + def test_ctrl_c_that_keeps_text_is_audited_as_displayed(self) -> None: + # VRP ignores Ctrl-C at the prompt: the executed line really is garbage+command. + t = CommandTracker() + t.feed("garbage") + res = t.on_stdin([], saw_input=True, had_enter=False, abort=True) + res += t.on_stdin(["display clock"], saw_input=True, had_enter=True) + res += t.feed("display clock\r\nError\r\n") + self.assertEqual([r.command for r in res], ["garbagedisplay clock"]) + + def test_network_prompt_classifier(self) -> None: + from netx_api.webcrt_channel import looks_like_network_prompt as f + + for ok in ("R2#", "", "[~r1]", "R2(config)#"): + self.assertTrue(f(ok), ok) + for bad in ("[u@h ~]$", "root@h:~#", "~ #", "user@host>"): + self.assertFalse(f(bad), bad) + + def test_bulk_output_skips_to_last_row_when_idle(self) -> None: + t = CommandTracker() + t.feed("".join(f"line {i}\r\n" for i in range(5000)) + "") + self.assertEqual(t.current_row(), "") + + +def _live_session(reader_alive: bool = True) -> tuple[WebcrtSession, MagicMock]: + conn = MagicMock() + conn.RETURN = "\n" + conn.remote_conn = MagicMock(spec=["recv_ready", "recv", "exit_status_ready", "resize_pty", "send"]) + conn.remote_conn.send.side_effect = lambda b: len(b) + sess = WebcrtSession( + session_id="s-live", ne_id="ne1", ne_name="lab", ne_ip="1.2.3.4", protocol="ssh", + cols=80, rows=24, cli_keymap=False, owner_user_id="u1", owner_username="bob", conn=conn, + ) + alive = MagicMock() + alive.is_alive.return_value = reader_alive + sess._reader = alive + return sess, conn + + +class LiveSessionAuditTests(unittest.TestCase): + @patch("netx_api.webcrt_session_model._audit") + def test_command_resolved_from_device_echo_not_client_hint(self, mock_audit: MagicMock) -> None: + sess, _ = _live_session() + sess._note_stdout_for_audit("") + sess.write_stdin("display clock\r", audit_line="dis") # stale xterm snapshot + mock_audit.assert_not_called() # nothing recorded until the device echoes + sess._note_stdout_for_audit("display clock\r\n2026-10-12\r\n") + kwargs = mock_audit.call_args.kwargs + self.assertEqual(kwargs["command"], "display clock") + self.assertEqual(kwargs["via"], "device_echo") + self.assertEqual(kwargs["owner_username"], "bob") + + @patch("netx_api.webcrt_session_model._audit") + def test_keystroke_sent_before_audit_work(self, mock_audit: MagicMock) -> None: + sess, conn = _live_session() + sess._note_stdout_for_audit("") + sess.write_stdin("display clock\r") + self.assertTrue(conn.remote_conn.send.called) + + @patch("netx_api.webcrt_session_model._audit") + def test_password_prompt_redacted_immediately(self, mock_audit: MagicMock) -> None: + sess, _ = _live_session() + sess._note_stdout_for_audit("Password:") + self.assertTrue(sess._password_mode) + sess.write_stdin("hunter2\r") + kwargs = mock_audit.call_args.kwargs + self.assertEqual(kwargs["command"], "***") + self.assertTrue(kwargs["redacted"]) + + @patch("netx_api.webcrt_session_model._audit") + def test_post_login_is_trusted(self, mock_audit: MagicMock) -> None: + sess, _ = _live_session() + sess.write_stdin("screen-length 0 temporary\r", audit_source="post_login") + kwargs = mock_audit.call_args.kwargs + self.assertEqual(kwargs["command"], "screen-length 0 temporary") + self.assertEqual(kwargs["source"], "post_login") + + @patch("netx_api.webcrt_session_model._audit") + def test_prompt_sync_enter_not_audited(self, mock_audit: MagicMock) -> None: + sess, _ = _live_session() + sess._note_stdout_for_audit("display clock") + sess.write_stdin("\r", audit_source="prompt_sync") + sess._note_stdout_for_audit("\r\n") + mock_audit.assert_not_called() + + +class StdoutHandOffTests(unittest.TestCase): + def test_drain_stdout_batches_and_reports_eof_and_stale(self) -> None: + sess, _ = _live_session() + sess.attach_gen = 3 + sess.out_queue.put(b"a") + sess.out_queue.put(b"b") + self.assertEqual(sess.drain_stdout(2), ("stale", b"")) + self.assertEqual(sess.drain_stdout(3), ("data", b"ab")) + self.assertEqual(sess.drain_stdout(3), ("empty", b"")) + sess.out_queue.put(b"z") + sess.out_queue.put(None) + self.assertEqual(sess.drain_stdout(3), ("eof", b"z")) + + def test_wake_is_coalesced(self) -> None: + sess, _ = _live_session() + loop = MagicMock() + sess.set_notifier(loop, MagicMock()) + sess._wake() + sess._wake() + sess._wake() + self.assertEqual(loop.call_soon_threadsafe.call_count, 1) + sess.drain_stdout(sess.attach_gen) # pump consumed -> next put wakes again + sess._wake() + self.assertEqual(loop.call_soon_threadsafe.call_count, 2) + + def test_reader_keeps_split_utf8_intact(self) -> None: + import threading + + sess, conn = _live_session(reader_alive=False) + chunks = [b"\xe4\xbd", b"\xa0\xe5\xa5\xbd\r\n", None] + ch = conn.remote_conn + ch.recv_ready.return_value = True + ch.exit_status_ready.return_value = False + seen: list[str] = [] + it = iter(chunks) + + def _recv(_n: int) -> bytes: + nxt = next(it) + if nxt is None: + sess.closed = True + return b"" + return nxt + + ch.recv.side_effect = _recv + sess._note_stdout_for_audit = lambda text: seen.append(text) # type: ignore[method-assign] + t = threading.Thread(target=sess._reader_loop, daemon=True) + t.start() + t.join(timeout=3) + self.assertEqual("".join(seen), "你好\r\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/web/WEB.md b/web/WEB.md index 691e194..5c9610e 100644 --- a/web/WEB.md +++ b/web/WEB.md @@ -7,26 +7,68 @@ - TanStack Query（服务端状态） - 自研 i18n（`src/i18n/`，无额外依赖） -## 目录结构 +## 代码结构 + +**按领域（feature）组织，依赖方向 `features → 共享层`，反向禁止**（ESLint `no-restricted-imports` 强制）。 ``` src/ - config/modules.ts # 模块注册表（工作台卡片、路由、标题 i18n） - constants/queryKeys.ts # React Query key - components/ # 可复用 UI（无业务 API 调用） - hooks/ # 通用 hooks（Toast、窗口注册） - i18n/ # 文案 zh / en - layout/ # AppLayout、ErrorBoundary - pages/ # 页面（按路由） - services/api.ts # HTTP 封装与 API 函数 - types.ts # 与后端契约类型 - utils/ - tabChannel.ts # 跨标签页聚焦（BroadcastChannel） - workbench.ts # 返回工作台 - moduleWindows.ts # 打开/聚焦模块页签 - display.ts # 纯展示辅助 + main.tsx App.tsx # 入口、路由表（lazy 引入各 feature 页面） + features/<领域>/ # 业务模块，kebab-case 目录名 + XxxPage.tsx # 路由入口（XxxLayout.tsx 为带子路由的壳） + components/ # 该领域私有组件（含 modals/） + hooks/ # 该领域私有 hooks（useXxx.ts） + lib/ # 纯逻辑（无 React），可单测 + layout/ # 应用壳：AppLayout、顶栏、ErrorBoundary + components/ (+ ui/) # 跨领域复用 UI（无业务 API 调用） + hooks/ utils/ # 跨领域 hooks / 纯工具 + services/api/ # 后端 API，按领域分文件，index.ts 汇总导出 + types/ # 后端契约类型，按领域分文件，index.ts 汇总导出 + i18n/locales/{zh,en}/.ts # 文案，一个命名空间一个文件 + config/ # 导航与模块注册表（modules / networkNav / auditNav） + constants/queryKeys.ts # React Query key + styles/ # 见「样式规范」 ``` +现有领域：`auth` `workbench` `managed-ne` `ume` `collect` `config-sync` `webcrt` `topology` +`network` `port-traffic` `cutover`（业务状态/对比/割接） `audit` `users` `api-tokens` `sessions`。 + +约定： + +- 只被**一个**领域用到的代码放进该领域的 `components/ hooks/ lib/`；被**两个以上**领域用到才上移到共享层。 +- 跨领域只引用对方的页面级导出或 `components/ lib/` 下的文件；现有合理依赖：`* → auth`、`topology/ume → managed-ne`。 +- 共享层（`components hooks utils services i18n types constants`）**不得** import `features/`。 +- 新增 API：写进 `services/api/<领域>.ts`；新增类型：写进 `types/<领域>.ts`；调用方仍 `from "…/services/api"` / `"…/types"`。 +- 单个组件函数超过 ~500 行视为需要拆分：状态/副作用抽成 `hooks/`，纯计算抽成 `lib/`，区块抽成 `components/`。 + （`BizComparePage` `WebcrtPage` `BizStatePage` `BizMigrationPage` `TopologyPage` 等仍是待拆分的单体组件。） + +## 样式规范 + +`src/index.css` 只是入口：按顺序 `@import` `styles/` 下的文件，**导入顺序即层叠顺序，不要调换**。 + +``` +styles/ + theme/tokens.css # 唯一的色板/圆角/阴影来源（--nx-*），并把旧 --nm-* 映射到它 + base/ # 全局基础（滚动条、body） + layout/ # 应用壳、网络/审计壳（左侧导航） + components/ # 共享原语（按钮、输入、表格、卡片、pill…） + pages/<领域>.css # 页面样式，类名带领域前缀（wb-* topo-* webcrt-* bs-* pt-*…） + legacy/ # 历史遗留的覆盖层，逐步并入 components/，不要往里加新规则 + theme/theme.css # 设计语言层，必须最后加载 +``` + +设计方向：**现代蓝**——沿用 NetX 原有深海军蓝色板、柔和渐变面板、8/12px 圆角、蓝色强调色（主按钮用渐变）， +胶囊状态标签；hover 有轻微抬升和蓝色光晕，focus 用柔和焦点环。 + +规则： + +- 颜色一律用 `var(--nx-*)`；半透明用 `rgb(var(--nx-ink-rgb) / .6)`，不要写 `#0f172a` / `rgba(15,23,42,.6)`。 +- 状态色用语义变量 `--nx-ok / --nx-warn / --nx-bad`。 +- 次要文字至少用 `--nx-text-faint`（在 `--nx-surface-1` 上对比度 ≥ 4.5:1）。 +- 顶栏高度固定 48px（`--nx-header-h`），多个页面用 `calc(100vh - 48px)` 布局。 +- 新样式放进所属 `pages/` 或 `components/` 文件；只有跨页面的视觉语言才改 `theme/theme.css`。 +- 避免 `!important` 和"覆盖全局 button"式规则；需要时先在 `components/` 里改原语。 + ## 路由与模块 | 路径 | 页面 | moduleId | @@ -61,7 +103,7 @@ src/ 1. 在 `MODULES` 增加一项（`moduleId`、`path`、`section`、i18n key、`iconTone`） 2. 在 `App.tsx` 增加 `` -3. 在 `i18n/zh.ts` 与 `en.ts` 补充文案 +3. 在 `i18n/locales/zh/.ts` 与 `en/.ts` 补充文案（新命名空间还需在各自 `index.ts` 注册） 4. 工作台卡片由 `modulesInSection` 自动生成，点击走 `openOrFocusModule` ## 跨标签页导航 @@ -79,7 +121,7 @@ src/ ## 数据请求 -- API 仅写在 `services/api.ts` +- API 仅写在 `services/api/<领域>.ts` - Query key 集中在 `constants/queryKeys.ts` - 失效缓存用 prefix key（如 `queryKeys.umeSyncStatusAll`） - 顶栏连接状态：`App` 轮询 `GET /v1/integrations/status`（5s），展示 **netx api** 与 **DSH alarm hub** @@ -184,7 +226,7 @@ src/ ## 样式 -- 全局样式：`index.css`（工作台浅色主题） +- 全局样式：见「样式规范」（`index.css` 为入口，令牌在 `styles/theme/tokens.css`） - 不使用 CSS Modules；类名 BEM 风格：`app-brand__logo`、`wb-card` ## 构建 diff --git a/web/eslint.config.js b/web/eslint.config.js index ef614d2..f872ad9 100644 --- a/web/eslint.config.js +++ b/web/eslint.config.js @@ -19,4 +19,21 @@ export default defineConfig([ globals: globals.browser, }, }, + { + // Dependency direction: features -> shared, never the reverse (see WEB.md "代码结构"). + files: ['src/{components,hooks,utils,services,i18n,types,constants}/**/*.{ts,tsx}'], + rules: { + 'no-restricted-imports': [ + 'error', + { + patterns: [ + { + group: ['**/features/**'], + message: '共享层不得依赖 features/（依赖方向：features → 共享层）。把共用部分下沉到共享层。', + }, + ], + }, + ], + }, + }, ]) diff --git a/web/package-lock.json b/web/package-lock.json index 8eb6317..a286dc3 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -12,6 +12,7 @@ "@heroui/styles": "^3.2.5", "@tanstack/react-query": "^5.100.8", "@xterm/addon-fit": "^0.11.0", + "@xterm/addon-webgl": "^0.19.0", "@xterm/xterm": "^6.0.0", "@xyflow/react": "^12.11.2", "dagre": "^0.8.5", @@ -2065,6 +2066,12 @@ "integrity": "sha512-jYcgT6xtVYhnhgxh3QgYDnnNMYTcf8ElbxxFzX0IZo+vabQqSPAjC3c1wJrKB5E19VwQei89QCiZZP86DCPF7g==", "license": "MIT" }, + "node_modules/@xterm/addon-webgl": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@xterm/addon-webgl/-/addon-webgl-0.19.0.tgz", + "integrity": "sha512-b3fMOsyLVuCeNJWxolACEUED0vm7qC0cy4wRvf3oURSzDTYVQiGPhTnhWZwIHdvC48Y+oLhvYXnY4XDXPoJo6A==", + "license": "MIT" + }, "node_modules/@xterm/xterm": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-6.0.0.tgz", diff --git a/web/package.json b/web/package.json index 9580af5..28bee6a 100644 --- a/web/package.json +++ b/web/package.json @@ -18,6 +18,7 @@ "@heroui/styles": "^3.2.5", "@tanstack/react-query": "^5.100.8", "@xterm/addon-fit": "^0.11.0", + "@xterm/addon-webgl": "^0.19.0", "@xterm/xterm": "^6.0.0", "@xyflow/react": "^12.11.2", "dagre": "^0.8.5", diff --git a/web/public/favicon.svg b/web/public/favicon.svg index 7097b3e..75b4ccc 100644 --- a/web/public/favicon.svg +++ b/web/public/favicon.svg @@ -1,16 +1,16 @@ - - - - - - - - - + + + + + + + + + + + + + + + diff --git a/web/public/login-space.webp b/web/public/login-space.webp deleted file mode 100644 index 54f3bc1..0000000 Binary files a/web/public/login-space.webp and /dev/null differ diff --git a/web/src/App.tsx b/web/src/App.tsx index 8c99625..159076a 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,75 +1,75 @@ import { lazy, Suspense } from "react"; import { Navigate, Route, Routes, useLocation } from "react-router-dom"; import { AppLayout } from "./layout/AppLayout"; -import { WorkbenchPage } from "./pages/WorkbenchPage"; -import { LoginPage } from "./pages/LoginPage"; -import { ForceChangePasswordPage } from "./pages/ForceChangePasswordPage"; -import { useAuth } from "./auth/AuthContext"; +import { WorkbenchPage } from "./features/workbench/WorkbenchPage"; +import { LoginPage } from "./features/auth/LoginPage"; +import { ForceChangePasswordPage } from "./features/auth/ForceChangePasswordPage"; +import { useAuth } from "./features/auth/AuthContext"; -const CollectPage = lazy(() => import("./pages/CollectPage").then((m) => ({ default: m.CollectPage }))); +const CollectPage = lazy(() => import("./features/collect/CollectPage").then((m) => ({ default: m.CollectPage }))); const ConfigSyncPage = lazy(() => - import("./pages/ConfigSyncPage").then((m) => ({ default: m.ConfigSyncPage })), + import("./features/config-sync/ConfigSyncPage").then((m) => ({ default: m.ConfigSyncPage })), ); -const NePage = lazy(() => import("./pages/NePage").then((m) => ({ default: m.NePage }))); -const UmePage = lazy(() => import("./pages/UmePage").then((m) => ({ default: m.UmePage }))); -const WebcrtPage = lazy(() => import("./pages/WebcrtPage").then((m) => ({ default: m.WebcrtPage }))); -const TopologyPage = lazy(() => import("./pages/TopologyPage").then((m) => ({ default: m.TopologyPage }))); +const NePage = lazy(() => import("./features/managed-ne/NePage").then((m) => ({ default: m.NePage }))); +const UmePage = lazy(() => import("./features/ume/UmePage").then((m) => ({ default: m.UmePage }))); +const WebcrtPage = lazy(() => import("./features/webcrt/WebcrtPage").then((m) => ({ default: m.WebcrtPage }))); +const TopologyPage = lazy(() => import("./features/topology/TopologyPage").then((m) => ({ default: m.TopologyPage }))); const NetworkLayout = lazy(() => - import("./pages/network/NetworkLayout").then((m) => ({ default: m.NetworkLayout })), + import("./features/network/NetworkLayout").then((m) => ({ default: m.NetworkLayout })), ); const NetworkDevicesPage = lazy(() => - import("./pages/network/NetworkDevicesPage").then((m) => ({ default: m.NetworkDevicesPage })), + import("./features/network/NetworkDevicesPage").then((m) => ({ default: m.NetworkDevicesPage })), ); const NetworkAlarmsPage = lazy(() => - import("./pages/network/NetworkAlarmsPage").then((m) => ({ default: m.NetworkAlarmsPage })), + import("./features/network/NetworkAlarmsPage").then((m) => ({ default: m.NetworkAlarmsPage })), ); const NetworkConfigsPage = lazy(() => - import("./pages/network/NetworkConfigsPage").then((m) => ({ default: m.NetworkConfigsPage })), + import("./features/network/NetworkConfigsPage").then((m) => ({ default: m.NetworkConfigsPage })), ); const LldpLinksPage = lazy(() => - import("./pages/network/LldpLinksPage").then((m) => ({ default: m.LldpLinksPage })), + import("./features/network/LldpLinksPage").then((m) => ({ default: m.LldpLinksPage })), ); const TopologyClassifyPage = lazy(() => - import("./pages/network/TopologyClassifyPage").then((m) => ({ default: m.TopologyClassifyPage })), + import("./features/network/TopologyClassifyPage").then((m) => ({ default: m.TopologyClassifyPage })), ); const PortTrafficPage = lazy(() => - import("./pages/network/PortTrafficPage").then((m) => ({ default: m.PortTrafficPage })), + import("./features/port-traffic/PortTrafficPage").then((m) => ({ default: m.PortTrafficPage })), ); const PortTrafficBoardListPage = lazy(() => - import("./pages/network/PortTrafficBoardListPage").then((m) => ({ + import("./features/port-traffic/PortTrafficBoardListPage").then((m) => ({ default: m.PortTrafficBoardListPage, })), ); const PortTrafficWallPage = lazy(() => - import("./pages/network/PortTrafficWallPage").then((m) => ({ default: m.PortTrafficWallPage })), + import("./features/port-traffic/PortTrafficWallPage").then((m) => ({ default: m.PortTrafficWallPage })), ); const BizStatePage = lazy(() => - import("./pages/network/BizStatePage").then((m) => ({ default: m.BizStatePage })), + import("./features/cutover/BizStatePage").then((m) => ({ default: m.BizStatePage })), ); const BizComparePage = lazy(() => - import("./pages/network/BizComparePage").then((m) => ({ default: m.BizComparePage })), + import("./features/cutover/BizComparePage").then((m) => ({ default: m.BizComparePage })), ); const BizMigrationPage = lazy(() => - import("./pages/network/BizMigrationPage").then((m) => ({ default: m.BizMigrationPage })), + import("./features/cutover/BizMigrationPage").then((m) => ({ default: m.BizMigrationPage })), ); const BizMonitorTemplatesPage = lazy(() => - import("./pages/network/BizMonitorTemplatesPage").then((m) => ({ + import("./features/cutover/BizMonitorTemplatesPage").then((m) => ({ default: m.BizMonitorTemplatesPage, })), ); -const UsersPage = lazy(() => import("./pages/UsersPage").then((m) => ({ default: m.UsersPage }))); +const UsersPage = lazy(() => import("./features/users/UsersPage").then((m) => ({ default: m.UsersPage }))); const AuditLayout = lazy(() => - import("./pages/audit/AuditLayout").then((m) => ({ default: m.AuditLayout })), + import("./features/audit/AuditLayout").then((m) => ({ default: m.AuditLayout })), ); const TaskOverviewPage = lazy(() => - import("./pages/audit/TaskOverviewPage").then((m) => ({ default: m.TaskOverviewPage })), + import("./features/audit/TaskOverviewPage").then((m) => ({ default: m.TaskOverviewPage })), ); -const AuditPage = lazy(() => import("./pages/AuditPage").then((m) => ({ default: m.AuditPage }))); +const AuditPage = lazy(() => import("./features/audit/AuditPage").then((m) => ({ default: m.AuditPage }))); const ApiTokensPage = lazy(() => - import("./pages/ApiTokensPage").then((m) => ({ default: m.ApiTokensPage })), + import("./features/api-tokens/ApiTokensPage").then((m) => ({ default: m.ApiTokensPage })), ); const SessionsPage = lazy(() => - import("./pages/SessionsPage").then((m) => ({ default: m.SessionsPage })), + import("./features/sessions/SessionsPage").then((m) => ({ default: m.SessionsPage })), ); /** Preserve query when redirecting legacy /network/webcrt → /webcrt. */ diff --git a/web/src/components/HelpHint.tsx b/web/src/components/HelpHint.tsx index 11c5c8c..e39287e 100644 --- a/web/src/components/HelpHint.tsx +++ b/web/src/components/HelpHint.tsx @@ -1,4 +1,4 @@ -import { useEffect, useLayoutEffect, useRef, useState } from "react"; +import { useEffect, useLayoutEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { createPortal } from "react-dom"; type Props = { @@ -9,20 +9,160 @@ type Props = { }; const VIEWPORT_MARGIN = 8; -const POPOVER_MAX_WIDTH = 360; +const POPOVER_MAX_WIDTH = 420; + +type HelpBlock = + | { kind: "title"; text: string } + | { kind: "list"; items: string[] } + | { kind: "para"; text: string }; + +const SECTION_RE = /^[【\[](.+?)[】\]]\s*$/; +const BULLET_RE = /^[·•\-\*]\s+(.+)$/; +/** Highlight field names / env keys / templates — skip short ALLCAPS acronyms (UME, DSH). */ +const CODEISH_RE = + /(`[^`]+`|\{[A-Za-z0-9_./:@-]+\}|[A-Z][A-Z0-9_]{5,}(?::[a-z]+)?|[a-z]+(?:_[a-z0-9]+)+|[a-z]+[A-Z][A-Za-z0-9]*)/g; + +function parseHelpBlocks(raw: string): HelpBlock[] { + const lines = String(raw || "") + .replace(/\r\n/g, "\n") + .split("\n") + .map((l) => l.trimEnd()); + const blocks: HelpBlock[] = []; + let listBuf: string[] = []; + let paraBuf: string[] = []; + + const flushList = () => { + if (!listBuf.length) return; + blocks.push({ kind: "list", items: listBuf }); + listBuf = []; + }; + const flushPara = () => { + if (!paraBuf.length) return; + blocks.push({ kind: "para", text: paraBuf.join(" ").trim() }); + paraBuf = []; + }; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed) { + flushList(); + flushPara(); + continue; + } + const section = trimmed.match(SECTION_RE); + if (section) { + flushList(); + flushPara(); + blocks.push({ kind: "title", text: section[1].trim() }); + continue; + } + const bullet = trimmed.match(BULLET_RE); + if (bullet) { + flushPara(); + listBuf.push(bullet[1].trim()); + continue; + } + flushList(); + paraBuf.push(trimmed); + } + flushList(); + flushPara(); + return blocks; +} + +function renderInline(text: string): ReactNode[] { + const nodes: ReactNode[] = []; + let last = 0; + let m: RegExpExecArray | null; + const re = new RegExp(CODEISH_RE.source, "g"); + while ((m = re.exec(text)) !== null) { + if (m.index > last) nodes.push(text.slice(last, m.index)); + const raw = m[0]; + const body = raw.startsWith("`") && raw.endsWith("`") ? raw.slice(1, -1) : raw; + nodes.push( + + {body} + , + ); + last = m.index + raw.length; + } + if (last < text.length) nodes.push(text.slice(last)); + return nodes; +} + +/** Long single paragraphs → clause list so plain help is still scannable. */ +function expandPlainPara(text: string): HelpBlock[] { + const parts = text + .split(/(?<=[。；;])\s*/) + .map((s) => s.replace(/[。；;]\s*$/, "").trim()) + .filter(Boolean); + if (parts.length >= 2 && text.length >= 48) { + return [{ kind: "list", items: parts }]; + } + return [{ kind: "para", text }]; +} + +function HelpBody({ text, nowrap }: { text: string; nowrap: boolean }) { + const blocks = useMemo(() => { + const parsed = parseHelpBlocks(text); + if (nowrap) return parsed; + if (parsed.length === 1 && parsed[0].kind === "para") { + return expandPlainPara(parsed[0].text); + } + return parsed.flatMap((b) => (b.kind === "para" && b.text.length >= 96 ? expandPlainPara(b.text) : [b])); + }, [text, nowrap]); + + if (nowrap) { + return
{renderInline(text)}
; + } + return ( +
+ {blocks.map((b, i) => { + if (b.kind === "title") { + return ( +
+ {b.text} +
+ ); + } + if (b.kind === "list") { + return ( +
    + {b.items.map((item, j) => ( +
  • {renderInline(item)}
    • + ))} +
+ ); + } + return ( +

+ {renderInline(b.text)} +

+ ); + })} +
+ ); +} export function HelpHint({ text, ariaLabel, align = "start", nowrap = false }: Props) { const [open, setOpen] = useState(false); const [popoverStyle, setPopoverStyle] = useState({}); const rootRef = useRef(null); const popoverRef = useRef(null); + const structured = useMemo(() => { + const parsed = parseHelpBlocks(text); + if (parsed.some((b) => b.kind === "title" || b.kind === "list")) return true; + // Long plain paragraphs are auto-split into a list in HelpBody. + return parsed.length === 1 && parsed[0].kind === "para" && parsed[0].text.length >= 48; + }, [text]); const updatePopoverPosition = () => { const el = rootRef.current; if (!el) return; const rect = el.getBoundingClientRect(); const popover = popoverRef.current; - const maxWidth = Math.min(POPOVER_MAX_WIDTH, window.innerWidth - VIEWPORT_MARGIN * 2); + const preferred = structured && !nowrap ? POPOVER_MAX_WIDTH : 360; + const maxWidth = Math.min(preferred, window.innerWidth - VIEWPORT_MARGIN * 2); const popWidth = popover?.offsetWidth || maxWidth; const popHeight = popover?.offsetHeight || 0; @@ -40,6 +180,7 @@ export function HelpHint({ text, ariaLabel, align = "start", nowrap = false }: P left, right: "auto", maxWidth, + maxHeight: Math.min(window.innerHeight - VIEWPORT_MARGIN * 2, 480), }); }; @@ -55,7 +196,7 @@ export function HelpHint({ text, ariaLabel, align = "start", nowrap = false }: P window.removeEventListener("resize", onReflow); window.removeEventListener("scroll", onReflow, true); }; - }, [open, align, text, nowrap]); + }, [open, align, text, nowrap, structured]); useEffect(() => { if (!open) return; @@ -98,11 +239,11 @@ export function HelpHint({ text, ariaLabel, align = "start", nowrap = false }: P ref={popoverRef} className={`help-hint__popover help-hint__popover--portal${ nowrap ? " help-hint__popover--nowrap" : "" - }`} + }${structured && !nowrap ? " help-hint__popover--structured" : ""}`} style={popoverStyle} role="tooltip" > - {text} + , document.body, ) diff --git a/web/src/components/ListPager.tsx b/web/src/components/ListPager.tsx index c68aed9..03574be 100644 --- a/web/src/components/ListPager.tsx +++ b/web/src/components/ListPager.tsx @@ -1,5 +1,5 @@ import { Button, Input } from "@heroui/react"; -import { useEffect, useState } from "react"; +import { useEffect, useState, type ReactNode } from "react"; import { useI18n } from "../i18n"; import { FieldSelect } from "./ui/FieldSelect"; @@ -13,6 +13,8 @@ export type ListPagerProps = { onPageSizeChange?: (size: number) => void; disabled?: boolean; className?: string; + /** Extra status shown after the totals (e.g. selection count), so pages need no separate row. */ + meta?: ReactNode; }; /** Shared denser pager: meta + prev/next + jump + optional page size. */ @@ -26,6 +28,7 @@ export function ListPager({ onPageSizeChange, disabled = false, className = "", + meta, }: ListPagerProps) { const { t } = useI18n(); const safePages = Math.max(1, pages || 1); @@ -54,6 +57,7 @@ export function ListPager({ page: String(page), pages: String(safePages), })} + {meta}
+ {open && portalContainer ? createPortal( +
{ + if (event.key === "Escape") { + event.preventDefault(); + event.stopPropagation(); + setOpen(false); + triggerRef.current?.focus(); + } + }}> + { + setOptionSearch(event.target.value); + setRenderLimit(100); + }} + /> + +
+ {optionsQuery.isLoading ? {loadingLabel} : null} + {!optionsQuery.isLoading && optionsQuery.isError ? ( +
+ {errorLabel} + +
+ ) : null} + {!optionsQuery.isLoading && !optionsQuery.isError && !visibleOptions.length ? ( + {noOptionsLabel} + ) : null} + {visibleOptions.slice(0, renderLimit).map((option) => ( + + ))} + {visibleOptions.length > renderLimit ? ( + + ) : null} +
+
+ + + +
+
, + portalContainer, + ) : null} + + ); +} diff --git a/web/src/components/ui/FieldSelect.tsx b/web/src/components/ui/FieldSelect.tsx index cc65495..2b12ebd 100644 --- a/web/src/components/ui/FieldSelect.tsx +++ b/web/src/components/ui/FieldSelect.tsx @@ -1,4 +1,4 @@ -import type { ReactNode, SelectHTMLAttributes } from "react"; +import { useId, type ReactNode, type SelectHTMLAttributes } from "react"; import { Label } from "@heroui/react"; /** Compact labeled native select — HeroUI Select is heavy for dense ops forms. */ @@ -9,6 +9,8 @@ export function FieldSelect({ fullWidth, className = "", children, + id, + "aria-describedby": describedBy, ...props }: { label?: ReactNode; @@ -18,17 +20,26 @@ export function FieldSelect({ className?: string; children: ReactNode; } & SelectHTMLAttributes) { + const generatedId = useId(); + const selectId = id || generatedId; + const hintId = `${selectId}-hint`; return ( -
diff --git a/web/src/pages/audit/TaskOverviewPage.tsx b/web/src/features/audit/TaskOverviewPage.tsx similarity index 99% rename from web/src/pages/audit/TaskOverviewPage.tsx rename to web/src/features/audit/TaskOverviewPage.tsx index 276801f..703fa6c 100644 --- a/web/src/pages/audit/TaskOverviewPage.tsx +++ b/web/src/features/audit/TaskOverviewPage.tsx @@ -97,7 +97,6 @@ export function TaskOverviewPage() { {query.isFetching ? t("common.refreshing") : t("common.refresh")} -

{t("audit.tasks.hint")}

diff --git a/web/src/auth/AuthContext.tsx b/web/src/features/auth/AuthContext.tsx similarity index 77% rename from web/src/auth/AuthContext.tsx rename to web/src/features/auth/AuthContext.tsx index 84af750..5b0e01f 100644 --- a/web/src/auth/AuthContext.tsx +++ b/web/src/features/auth/AuthContext.tsx @@ -7,7 +7,7 @@ import { useState, type ReactNode, } from "react"; -import { apiGet, apiPost, clearAuthToken } from "../services/api"; +import { apiGet, apiPost, clearAuthToken } from "../../services/api"; export type AuthUser = { id: string; @@ -21,11 +21,15 @@ export type AuthUser = { updated_at?: string | null; }; +export type UserPrefs = Record; + type AuthState = { ready: boolean; token: string | null; user: AuthUser | null; scopes: string[]; + prefs: UserPrefs; + setPrefs: (prefs: UserPrefs) => void; login: (username: string, password: string) => Promise; logout: () => Promise; refreshMe: () => Promise; @@ -35,24 +39,33 @@ type AuthState = { const AuthContext = createContext(null); +function asPrefs(value: unknown): UserPrefs { + return value && typeof value === "object" && !Array.isArray(value) ? (value as UserPrefs) : {}; +} + export function AuthProvider({ children }: { children: ReactNode }) { const [ready, setReady] = useState(false); // token is opaque for UI; cookie session means we only care about user presence. const [token, setToken] = useState(null); const [user, setUser] = useState(null); const [scopes, setScopes] = useState([]); + const [prefs, setPrefs] = useState({}); const refreshMe = useCallback(async () => { clearAuthToken(); // drop any legacy localStorage tokens try { - const data = await apiGet<{ user: AuthUser; scopes?: string[] }>("/v1/auth/me"); + const data = await apiGet<{ user: AuthUser; scopes?: string[]; prefs?: UserPrefs }>( + "/v1/auth/me", + ); setToken("cookie"); setUser(data.user); setScopes(data.scopes || data.user.scopes || []); + setPrefs(asPrefs(data.prefs)); } catch { setToken(null); setUser(null); setScopes([]); + setPrefs({}); } }, []); @@ -64,7 +77,7 @@ export function AuthProvider({ children }: { children: ReactNode }) { }, [refreshMe]); const login = useCallback(async (username: string, password: string) => { - const data = await apiPost<{ user: AuthUser }>("/v1/auth/login", { + const data = await apiPost<{ user: AuthUser; prefs?: UserPrefs }>("/v1/auth/login", { username, password, }); @@ -72,6 +85,7 @@ export function AuthProvider({ children }: { children: ReactNode }) { setToken("cookie"); setUser(data.user); setScopes(data.user.scopes || []); + setPrefs(asPrefs(data.prefs)); }, []); const logout = useCallback(async () => { @@ -84,6 +98,7 @@ export function AuthProvider({ children }: { children: ReactNode }) { setToken(null); setUser(null); setScopes([]); + setPrefs({}); }, []); const value = useMemo( @@ -92,13 +107,15 @@ export function AuthProvider({ children }: { children: ReactNode }) { token, user, scopes, + prefs, + setPrefs, login, logout, refreshMe, isAdmin: user?.role === "admin", hasScope: (scope: string) => scopes.includes(scope) || user?.role === "admin", }), - [ready, token, user, scopes, login, logout, refreshMe], + [ready, token, user, scopes, prefs, login, logout, refreshMe], ); return {children}; diff --git a/web/src/features/auth/DropletGlass.tsx b/web/src/features/auth/DropletGlass.tsx new file mode 100644 index 0000000..f48b34b --- /dev/null +++ b/web/src/features/auth/DropletGlass.tsx @@ -0,0 +1,115 @@ +import { useEffect, useState, type RefObject } from "react"; + +/** + * Displacement map for a rounded rect: neutral (128,128) in the middle, and + * within `bezel` px of the edge the sample point is pushed inward along the + * edge normal — the background bends at the rim like light through a droplet. + */ +function buildMap(w: number, h: number, radius: number, bezel: number): string { + const canvas = document.createElement("canvas"); + canvas.width = w; + canvas.height = h; + const ctx = canvas.getContext("2d"); + if (!ctx) return ""; + const img = ctx.createImageData(w, h); + const hx = w / 2; + const hy = h / 2; + const r = Math.min(radius, hx, hy); + + // Signed distance to the rounded-rect edge (negative inside). + const sdf = (x: number, y: number) => { + const qx = Math.abs(x - hx) - (hx - r); + const qy = Math.abs(y - hy) - (hy - r); + const ox = Math.max(qx, 0); + const oy = Math.max(qy, 0); + return Math.hypot(ox, oy) + Math.min(Math.max(qx, qy), 0) - r; + }; + + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + const i = (y * w + x) * 4; + const px = x + 0.5; + const py = y + 0.5; + const depth = -sdf(px, py); + let dx = 0; + let dy = 0; + if (depth < bezel) { + // Outward normal from the SDF gradient. + const gx = sdf(px + 1, py) - sdf(px - 1, py); + const gy = sdf(px, py + 1) - sdf(px, py - 1); + const gl = Math.hypot(gx, gy) || 1; + const t = 1 - Math.max(depth, 0) / bezel; + const f = t * t; // strongest right at the rim, fading inward + dx = -(gx / gl) * f; + dy = -(gy / gl) * f; + } + img.data[i] = Math.round(128 + dx * 127); + img.data[i + 1] = Math.round(128 + dy * 127); + img.data[i + 2] = 128; + img.data[i + 3] = 255; + } + } + ctx.putImageData(img, 0, 0); + return canvas.toDataURL(); +} + +/** + * SVG filter that refracts whatever sits behind `target`; reference it with + * `backdrop-filter: url(#id)` (Chromium only — other engines keep the plain + * blur declared before it). + */ +export function DropletGlassFilter({ + id, + target, + radius = 30, + bezel = 26, + strength = 46, +}: { + id: string; + target: RefObject; + radius?: number; + bezel?: number; + strength?: number; +}) { + const [map, setMap] = useState<{ url: string; w: number; h: number } | null>(null); + + useEffect(() => { + const el = target.current; + if (!el) return; + let timer = 0; + const rebuild = () => { + const w = Math.round(el.offsetWidth); + const h = Math.round(el.offsetHeight); + if (!w || !h) return; + setMap((prev) => (prev && prev.w === w && prev.h === h ? prev : { url: buildMap(w, h, radius, bezel), w, h })); + }; + const ro = new ResizeObserver(() => { + window.clearTimeout(timer); + timer = window.setTimeout(rebuild, 60); + }); + ro.observe(el); + rebuild(); + return () => { + ro.disconnect(); + window.clearTimeout(timer); + }; + }, [target, radius, bezel]); + + return ( + + ); +} diff --git a/web/src/pages/ForceChangePasswordPage.tsx b/web/src/features/auth/ForceChangePasswordPage.tsx similarity index 96% rename from web/src/pages/ForceChangePasswordPage.tsx rename to web/src/features/auth/ForceChangePasswordPage.tsx index 7470c08..7b3c914 100644 --- a/web/src/pages/ForceChangePasswordPage.tsx +++ b/web/src/features/auth/ForceChangePasswordPage.tsx @@ -1,8 +1,8 @@ import { Alert, Button, Card, Form, Input, Label, TextField } from "@heroui/react"; import { useState, type FormEvent } from "react"; -import { useAuth } from "../auth/AuthContext"; -import { useI18n } from "../i18n"; -import { apiPost } from "../services/api"; +import { useAuth } from "./AuthContext"; +import { useI18n } from "../../i18n"; +import { apiPost } from "../../services/api"; import { LoginShell } from "./LoginShell"; export function ForceChangePasswordPage() { diff --git a/web/src/pages/LoginPage.tsx b/web/src/features/auth/LoginPage.tsx similarity index 62% rename from web/src/pages/LoginPage.tsx rename to web/src/features/auth/LoginPage.tsx index acd79f9..51e3ec1 100644 --- a/web/src/pages/LoginPage.tsx +++ b/web/src/features/auth/LoginPage.tsx @@ -1,19 +1,46 @@ import { Alert, Button, Card, Form, Input, Label, TextField } from "@heroui/react"; -import { useState, type FormEvent } from "react"; +import { useRef, useState, type FormEvent, type KeyboardEvent } from "react"; import { Navigate, useSearchParams } from "react-router-dom"; -import { useAuth } from "../auth/AuthContext"; -import { useI18n } from "../i18n"; +import { useAuth } from "./AuthContext"; +import { useI18n } from "../../i18n"; import { LoginShell } from "./LoginShell"; +import { DropletGlassFilter } from "./DropletGlass"; +import { ApiRequestError } from "../../services/api/http"; + +const LAST_USER_KEY = "netx.login.lastUser"; + +function readLastUser(): string { + try { + return localStorage.getItem(LAST_USER_KEY) || ""; + } catch { + return ""; + } +} + +/** Map the login endpoint's error codes to readable text. */ +function loginErrorText(err: unknown, t: (key: string, vars?: Record) => string): string { + if (err instanceof ApiRequestError) { + if (err.detail === "invalid_credentials") return t("auth.badCredentials"); + const d = err.detail as { error?: string; retry_after_sec?: number } | null; + if (d && typeof d === "object" && d.error === "login_locked") { + return t("auth.loginLocked", { minutes: Math.max(1, Math.ceil((d.retry_after_sec || 60) / 60)) }); + } + } + return String(err instanceof Error ? err.message : err) || t("auth.loginFailed"); +} export function LoginPage() { const { t } = useI18n(); const { ready, user, login } = useAuth(); const [params] = useSearchParams(); - const [username, setUsername] = useState("admin"); + const [lastUser] = useState(readLastUser); + const [username, setUsername] = useState(lastUser || "admin"); const [password, setPassword] = useState(""); const [showPassword, setShowPassword] = useState(false); const [error, setError] = useState(""); const [busy, setBusy] = useState(false); + const [capsLock, setCapsLock] = useState(false); + const cardRef = useRef(null); if (ready && user) { const next = params.get("next") || "/"; @@ -24,23 +51,32 @@ export function LoginPage() { e.preventDefault(); setError(""); setBusy(true); + const name = username.trim(); try { - await login(username.trim(), password); + await login(name, password); + try { + localStorage.setItem(LAST_USER_KEY, name); + } catch { + /* storage blocked — nothing to remember */ + } } catch (err) { - setError(String(err instanceof Error ? err.message : err) || t("auth.loginFailed")); + setError(loginErrorText(err, t)); } finally { setBusy(false); } }; + const syncCaps = (e: KeyboardEvent) => setCapsLock(e.getModifierState("CapsLock")); + return ( - + + - {t("auth.loginTitle")} -
+ + {t("auth.loginTitle")}
void onSubmit(e)}> @@ -48,28 +84,36 @@ export function LoginPage() { fullWidth name="username" value={username} - onChange={setUsername} + onChange={(v) => { + setUsername(v); + setError(""); + }} isDisabled={busy || !ready} - autoFocus + autoFocus={!lastUser} autoComplete="username" > -
+
{ + setPassword(v); + setError(""); + }} isDisabled={busy || !ready} + autoFocus={!!lastUser} autoComplete="current-password" > + {capsLock ? {t("auth.capsLock")} : null}