diff --git a/.env.example b/.env.example index 23d7cfd..7ede38d 100644 --- a/.env.example +++ b/.env.example @@ -120,16 +120,8 @@ NETX_UME_NOTIFICATION_TOPIC=ALARM # bundled | external; unset = external (compatible with existing deploys) # NETX_BUNDLED_PG_PORT=15432 # NETX_BUNDLED_PG_DATA_DIR= -# Update: GitHub primary + Forgejo mirror (git.avelo.top); pick newest reachable -# NETX_UPDATE_SOURCES=github,forgejo -# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest -# NETX_UPDATE_GITHUB_REPO=hansjone/netx # NETX_UPDATE_URL= -# NETX_UPDATE_FALLBACK_URL= -# NETX_UPDATE_TOKEN= # NETX_UPDATE_CHANNEL=stable -# NETX_UPDATE_AUTO=false -# When true: tray/scheduled task may download+apply zip updates silently. # Heavier fleets: raise CLI/DB together; also ensure Postgres max_connections and bastion session limits. # One-click start (recommended): .\scripts\start_netx.ps1 -Background -WithWeb # -> API + netx_api.worker + N× netx_api.biz_state_worker + optional Vite diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index 3c8444b..12e2fa9 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -22,11 +22,10 @@ jobs: with: python-version: "3.12" - - name: Build release stage + - name: Build release zip shell: pwsh run: | - # Use pwsh -File (UTF-8). Nested Windows PowerShell 5.1 mis-parses UTF-8 scripts without BOM. - & ./packaging/build_release.ps1 -CreateVenv + powershell -ExecutionPolicy Bypass -File ./packaging/build_release.ps1 -CreateVenv - name: Install Inno Setup shell: pwsh @@ -44,5 +43,6 @@ jobs: uses: softprops/action-gh-release@v2 with: files: | + packaging/release/NetX-*-win64.zip packaging/release/NetX-Setup-*.exe generate_release_notes: true diff --git a/.gitignore b/.gitignore index 6720b41..92197b4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,18 +11,13 @@ scripts/.run/ scripts/_* scripts/archive/ packages/netx-topology-mcp/tests/_* -_tmp_* -.idea/ ume/ data/ne_collections/ data/webcrt/ data/auth/ data/runtime/ -data/ne_exec_jobs/ # Windows packaging artifacts (binaries / release trees) packaging/cache/ packaging/release/ packaging/postgres/pgsql/ -packaging/winsw/ *.exe -!packaging/installer/*.iss diff --git a/README.md b/README.md index 8a4d5e9..2ce7550 100644 --- a/README.md +++ b/README.md @@ -158,9 +158,9 @@ API base: `http://127.0.0.1:8890/` After `npm run build` in `web/`, the API can also serve the UI at `http://127.0.0.1:8890/` (same origin). See `NETX_UI_DIST_DIR` in `.env.example`. -### Windows installer (optional) +### Windows installer / portable package (optional) -Fool-proof Windows delivery via `NetX-Setup-*.exe` (bundled or external Postgres, program/data split, offline upgrade over existing installs) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`. +Fool-proof Windows delivery (bundled or external Postgres, program/data split, manual update) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`. ### 6) MCP（Cursor / oclaw / Claude） diff --git a/docs/reviews/assets/biz-compare-config-mobile.png b/docs/reviews/assets/biz-compare-config-mobile.png deleted file mode 100644 index 29539dc..0000000 Binary files a/docs/reviews/assets/biz-compare-config-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-config.png b/docs/reviews/assets/biz-compare-config.png deleted file mode 100644 index 98fe764..0000000 Binary files a/docs/reviews/assets/biz-compare-config.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-desktop.png b/docs/reviews/assets/biz-compare-desktop.png deleted file mode 100644 index 20904ac..0000000 Binary files a/docs/reviews/assets/biz-compare-desktop.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-error.png b/docs/reviews/assets/biz-compare-error.png deleted file mode 100644 index 4396388..0000000 Binary files a/docs/reviews/assets/biz-compare-error.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-jobs.png b/docs/reviews/assets/biz-compare-jobs.png deleted file mode 100644 index ace1467..0000000 Binary files a/docs/reviews/assets/biz-compare-jobs.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-legacy.png b/docs/reviews/assets/biz-compare-legacy.png deleted file mode 100644 index a6c03ba..0000000 Binary files a/docs/reviews/assets/biz-compare-legacy.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-memory-benchmark.json b/docs/reviews/assets/biz-compare-memory-benchmark.json deleted file mode 100644 index 70290b7..0000000 --- a/docs/reviews/assets/biz-compare-memory-benchmark.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "date": "2026-10-10", - "baseline_ref": "afa8e06b2de64100f963ae30b3babfcb5d7009f4", - "python": "3.14.3", - "dataset": { - "rows_per_side": 1000000, - "fields": ["id", "neighbor", "state", "afi"], - "strata": 50, - "unchanged_limit": 5000, - "compact_unchanged": true, - "interface_transforms": false, - "all_rows_unchanged": true - }, - "measurement": "tracemalloc peak allocated during compare_rows; excludes input allocations, DB loading and process overhead", - "results": [ - { - "engine": "baseline", - "peak_engine_mib": 1144.43, - "seconds_with_tracemalloc": 92.4 - }, - { - "engine": "optimized", - "peak_engine_mib": 420.49, - "seconds_with_tracemalloc": 63.681 - } - ], - "results_and_sample_order_identical": true, - "test_result": "1 passed in 157.88s" -} diff --git a/docs/reviews/assets/biz-compare-narrow.png b/docs/reviews/assets/biz-compare-narrow.png deleted file mode 100644 index bfec41c..0000000 Binary files a/docs/reviews/assets/biz-compare-narrow.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-runs-mobile.png b/docs/reviews/assets/biz-compare-runs-mobile.png deleted file mode 100644 index 9e53762..0000000 Binary files a/docs/reviews/assets/biz-compare-runs-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/biz-compare-runs.png b/docs/reviews/assets/biz-compare-runs.png deleted file mode 100644 index 9042eca..0000000 Binary files a/docs/reviews/assets/biz-compare-runs.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-batches.png b/docs/reviews/assets/biz-state-batches.png deleted file mode 100644 index e6b7248..0000000 Binary files a/docs/reviews/assets/biz-state-batches.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-config-mobile.png b/docs/reviews/assets/biz-state-config-mobile.png deleted file mode 100644 index 19244fc..0000000 Binary files a/docs/reviews/assets/biz-state-config-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-config.png b/docs/reviews/assets/biz-state-config.png deleted file mode 100644 index a4eb2e0..0000000 Binary files a/docs/reviews/assets/biz-state-config.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-error.png b/docs/reviews/assets/biz-state-error.png deleted file mode 100644 index 62cd975..0000000 Binary files a/docs/reviews/assets/biz-state-error.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-tasks.png b/docs/reviews/assets/biz-state-tasks.png deleted file mode 100644 index be971dc..0000000 Binary files a/docs/reviews/assets/biz-state-tasks.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-workbook-mobile.png b/docs/reviews/assets/biz-state-workbook-mobile.png deleted file mode 100644 index 5b2a506..0000000 Binary files a/docs/reviews/assets/biz-state-workbook-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/biz-state-workbook.png b/docs/reviews/assets/biz-state-workbook.png deleted file mode 100644 index cad4dee..0000000 Binary files a/docs/reviews/assets/biz-state-workbook.png and /dev/null differ diff --git a/docs/reviews/assets/compare-template-editor-mobile.png b/docs/reviews/assets/compare-template-editor-mobile.png deleted file mode 100644 index 3dc4212..0000000 Binary files a/docs/reviews/assets/compare-template-editor-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/compare-template-editor.png b/docs/reviews/assets/compare-template-editor.png deleted file mode 100644 index a6419a9..0000000 Binary files a/docs/reviews/assets/compare-template-editor.png and /dev/null differ diff --git a/docs/reviews/assets/cutover-monitor-board.png b/docs/reviews/assets/cutover-monitor-board.png deleted file mode 100644 index 9fe1dcf..0000000 Binary files a/docs/reviews/assets/cutover-monitor-board.png and /dev/null differ diff --git a/docs/reviews/assets/cutover-monitor-mobile.png b/docs/reviews/assets/cutover-monitor-mobile.png deleted file mode 100644 index e9e2c1b..0000000 Binary files a/docs/reviews/assets/cutover-monitor-mobile.png and /dev/null differ diff --git a/docs/reviews/assets/cutover-monitor-wall.png b/docs/reviews/assets/cutover-monitor-wall.png deleted file mode 100644 index f35d5db..0000000 Binary files a/docs/reviews/assets/cutover-monitor-wall.png and /dev/null differ diff --git a/docs/reviews/biz-compare-review-2026-10-10.md b/docs/reviews/biz-compare-review-2026-10-10.md deleted file mode 100644 index 65e342e..0000000 --- a/docs/reviews/biz-compare-review-2026-10-10.md +++ /dev/null @@ -1,131 +0,0 @@ -# 业务对比检查与优化记录 - -检查日期：2026-10-10。范围：网络管理 → 割接对比 → 业务对比，包括前端任务/批次/结果界面、Python 与 PostgreSQL 对比路径、结果分页与搜索、短期列表缓存。 - -本次直接修改项目代码并添加回归测试，没有连接设备执行采集，也没有修改生产数据库。浏览器截图与交互测试使用固定 API 响应。 - -## 已修复的问题 - -### 1. 搜索会改变业务判定（P1） - -原实现分别按搜索条件过滤割接前、后的原表，再重新对比。例如 `state=up → down`，搜索 `up` 后只剩前侧，原本的“变化”就可能变成“删除”。重复身份键的业务还可能在筛选后错位配对。 - -现在：已存储的判定直接在结果表分页筛选；需要回查未保存全量明细的成功行时，先找到候选身份键，再补齐两侧的完整分组，按原有配对规则对比，最后筛选。达到读取上限的分组整体跳过，并返回截断标记，避免产生虚假的新增或删除。 - -### 2. PostgreSQL 搜索漏用部分模板过滤条件（P1） - -原搜索只下推 SQL 支持的过滤规则，对 `age_timer` 等不支持的规则直接忽略，可能将静态 ARP 等不属于模板范围的数据纳入结果。现在不支持下推的规则走流式读取并在 Python 中完整过滤。 - -### 3. 快速切换时旧请求覆盖当前任务（P1） - -前后采集任务的批次列表、打开任务、加载历史结果以及轮询缺少完整的过期保护。现在对批次请求使用清理标记，对任务与结果使用请求序号；关闭弹窗、切换路由或打开其他任务后，旧响应不能覆盖当前界面。 - -### 4. 结果加载错误容易被误读（P2） - -原查询失败时保留上一组明细，界面已经显示新筛选条件；切到未完成的表后，加载状态也可能无法结束。现在加载和失败都有独立状态，错误时清除旧明细并提供重试入口；切换表/页签后清理加载状态。 - -### 5. LLDP 搜索漏查（P2） - -LLDP 数据存于 `biz_state_lldp_neighbor`，原搜索只读 `biz_state_metric_row`。现在原表回查与精简结果的字段搜索都支持 LLDP 表。 - -### 6. 其他已修复边界 - -- 恰好命中读取上限时，原非 PostgreSQL 路径错误地报告“已截断”；现在多读一条判断。 -- 无效或有歧义的表标识可能落到其他表；现在返回 `sheet_not_found`。 -- `%`、`_` 等搜索字符按字面值匹配，避免不同查询路径含义不一致。 -- 关闭最后一项检查后，空数组被解释为“启用全部”；现在阻止关闭最后一项并提示至少保留一项。 -- 两个 HeroUI Input 使用了 `isDisabled`，实际需要 `disabled`；过滤值与容差输入的禁用状态已修正。 -- 列表缓存缺少并发合并，慢请求可以覆盖强制刷新结果或在失效后重新填充缓存；现在共享请求、保护失效状态并抑制过期的后台通知。 -- 清理中英文文案中的重复键，解决相关类型检查错误。 -- 操作后任务选择“同操作前”时原来没有加载批次列表；现在按实际使用的任务读取并共享缓存。切换采集任务时清除原批次选择，避免把上一个任务的批次带入配置。 -- 原通用下拉框嵌套了两个 label，标签不能正确关联控件；现在使用明确的 `htmlFor/id`，并关联提示文本。 - -## 性能改动与实际收益范围 - -- 合并两套轮询，下一轮在本轮请求完成后启动。模拟浏览器测试中，运行任务在 5.2 秒内历史列表和详情各请求 2–3 次（包含可能尚未完成的初始加载），没有重复轮询，也不会因慢请求持续叠加。 -- 已保存的失败、新增、变化等结果在数据库中筛选并分页，避免每次翻页重新加载和对比原始批次。回归测试明确验证这一查询路径不调用 `compare_rows`。 -- 同一前后采集任务的批次请求共用短期缓存，列表后台刷新合并同一请求；6 项缓存测试覆盖并发、强制刷新、失败重试与失效竞态。 -- 模板查找使用 Map，避免列表搜索与绘制逐项线性查找模板。 -- 修改筛选后先回到第一页，避免先请求旧页码再重复请求第一页。 - -这些改动减少了重复工作。以下后续基准使用合成数据，没有测量生产环境延迟。 - -## 后续优化（2026-10-10） - -### 历史执行配置与精简行还原 - -新运行在创建时将模板、启用的检查项、字段规则/过滤、接口归一化、端口映射、成功行保存策略和名称存入 `summary_json.config_snapshot`。后台执行、成功行回查、精简行还原以及导出共用该配置；排队之后修改配置，甚至删除原模板/映射，也不会改变这一运行使用的配置。无需数据库结构迁移。 - -修复了精简存储明细还原时将 `mapped_before` 直接设为原始 `before` 的问题。现在前侧依次归一化和映射，后侧归一化，保持与完整结果的显示及 CSV 一致。ZIP 新增 `config_snapshot.json`，便于核对当时配置；详情接口只返回快照模板与版本，不将完整端口映射加入轮询响应。 - -页面显示“执行配置已保存”；旧记录显示“旧记录：未保存执行配置”。旧运行无法可靠补造当时的配置，仍保留兼容查询，成功行回查及精简行还原会使用当前配置。快照也不保存原始采集行的副本，若原始批次数据被清理，依赖行 ID 的还原仍会缺少内容。 - -### Python 大表内存 - -成功样本按“层内序号、层首次出现顺序”排序，只保留样本上限内的候选和分层状态，避免先积累全部成功行。保留原分层轮询顺序；未截断的小表仍保持采集顺序。不需要接口变换时复用输入行；有变换时创建新行，不修改输入数据。稳定身份键顺序直接使用分组字典，省去再次计算全表键、额外键列表与集合。 - -每侧 1,000,000 行、50 个邻居分层、全部成功、保存 5,000 条精简样本的测试：优化前引擎新增分配峰值 **1144.43 MiB**，优化后 **420.49 MiB**，下降约 **63.3%**。完整结果与样本顺序的 SHA-256 一致。开启 `tracemalloc` 时耗时为 92.400 秒和 63.681 秒；该计时含内存追踪开销，不代表生产耗时。输入行在追踪开始前创建，峰值不包含输入行、数据库加载或 Python 进程的总内存。 - -基线为 `afa8e06b2de64100f963ae30b3babfcb5d7009f4` 中的对比引擎。原始指标见 [百万行测试结果](assets/biz-compare-memory-benchmark.json)。身份键分组仍需要随原表行数增长的内存；这次不是完全流式对比。 - -### 大表导出 - -导出改为按 `(seq, id)` 游标分批查询、分批还原精简行、逐行将 UTF-8 CSV 写入 ZIP，避免将全部明细与完整 CSV 同时保存在内存。相同 `seq` 的多条结果按 `id` 连续读取，不漏掉跨批次的同行序号记录。保留 BOM、字段顺序及逗号/引号/换行转义。最终压缩包仍由现有接口在内存中返回，压缩文件本身的大小仍计入内存。 - -## UI 改动 - -任务列表、创建向导、任务配置、对比批次和结果区统一使用深色平面、细边框、小圆角、低饱和蓝色和等宽数字。任务配置按作业参数、前后数据源、检查范围、端口映射四个区块组织；数据源左右配对，显示设备地址和采集数量。保存配置作为配置页主操作，删除操作降低视觉权重。 - -批次页增加最近 50 条记录的统计、刷新、加载、错误重试；刷新失败保留已载入记录。已完成只表示执行结束，业务失败/成功/新增数量单独显示。排队与运行使用不同文案，未知进度使用不确定进度条；时间、批次标识和前后采集信息便于核查。页签支持方向键及 Home/End，窄屏配置改为单列，批次表在独立容器中横向滚动。 - -结果区分为当前检查项/全表通过率、结果类型、搜索与辅助操作三层。增加明确的分页范围、清空筛选、加载状态、错误重试和搜索无结果提示。筛选按钮增加按下状态，字段筛选增加展开状态与键盘焦点提示。后续增加了执行配置保存状态及旧记录限制提示。 - -窄屏检查项导航改为紧凑的横向排列，避免一个检查项占据整行的大块区域。浏览器检查覆盖 1440px、768px、390px；筛选栏没有横向溢出。结果表保留必要的横向滚动。 - -- [任务列表](assets/biz-compare-jobs.png) -- [桌面任务配置](assets/biz-compare-config.png) -- [390px 任务配置](assets/biz-compare-config-mobile.png) -- [桌面对比批次](assets/biz-compare-runs.png) -- [390px 对比批次](assets/biz-compare-runs-mobile.png) -- [桌面结果区](assets/biz-compare-desktop.png) -- [768px 结果区](assets/biz-compare-narrow.png) -- [加载失败与重试](assets/biz-compare-error.png) -- [旧记录执行配置提示](assets/biz-compare-legacy.png) - -## 验证 - -- Python：87 项通过，1 项百万行基准默认跳过；单独启用百万行基准后通过。新增测试覆盖搜索、历史配置快照、有界抽样与游标导出。 -- 缓存：6 项 Node 测试通过。 -- 浏览器：27 个场景通过，包括任务列表文案、配置保存数据、同任务批次加载与切换清理、自动模式、检查项开关、创建向导、键盘页签、批次统计/状态/刷新/错误重试、配置保存状态、旧记录提示、结果布局、筛选、窄屏布局、旧任务响应保护、单套轮询、无运行时异常。 -- Vite 生产打包成功。 -- 本次业务对比页面的类型错误已经修复。全项目 `tsc -b` 仍被 `BizStatePage.tsx`、`BizMigrationPage.tsx` 中原有的类型/未使用变量错误阻塞。 -- 页面 ESLint 原有 34 个错误，主要是 `any` 与现有 React Hook 写法；批次列表增加类型后减少了 `any`，本次未增加错误。结果请求依赖警告已修正。通用下拉框和缓存模块没有 lint 错误。 -- PostgreSQL 路径验证了 SQL 编译与规则降级分支，没有在真实 PostgreSQL 数据库上执行查询或测量执行计划。 - -可重复执行的检查： - -```powershell -# 项目根目录 -.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare_export.py tests/test_biz_state_compare_snapshot.py tests/test_biz_state_compare_memory.py tests/test_biz_state_compare_search.py tests/test_biz_state_compare.py tests/test_biz_state_compare_sql.py tests/test_biz_state_compare_recovery.py -q - -# 可选：百万行基准，耗时约数分钟，内存占用较大 -$env:NETX_BENCH_ROWS = '1000000' -$env:NETX_BENCH_REF = 'afa8e06b2de64100f963ae30b3babfcb5d7009f4' -.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare_memory.py::test_memory_benchmark_against_git_baseline -q -s -Remove-Item Env:NETX_BENCH_ROWS, Env:NETX_BENCH_REF - -# web 目录 -node --experimental-strip-types --test tests/cutoverDataCache.test.mjs -node node_modules/vite/bin/vite.js build -node node_modules/typescript/bin/tsc -b -``` - -浏览器脚本为 `web/tests/bizCompare.browser.mjs`。先启动本地 Vite 服务，再设置 `NETX_PLAYWRIGHT_MODULE` 为可用的 Playwright 包路径；`NETX_TEST_URL` 默认是 `http://127.0.0.1:5179`，浏览器默认使用已安装的 Chrome。所有 `/v1` 请求由脚本提供固定响应，不会访问实际设备或数据库。 - -## 后续值得优化的部分 - -1. **Python 大表分组。** 有界抽样和冗余复制优化已完成，百万行基准下降约 63% 的引擎新增分配峰值。分组与输入仍随原表规模增长；进一步降低内存可研究排序流式配对或分区执行，并保持重复键配对语义。 -2. **PostgreSQL 搜索与深分页。** JSON 字段包含搜索和精确总数统计仍可能扫描大量结果。需要用真实数据查看 `EXPLAIN (ANALYZE, BUFFERS)`，再决定常用字段索引、搜索文本索引或游标分页；本次没有盲目添加生产索引。 -3. **历史配置与原始数据保留。** 新运行的执行配置快照已完成。旧运行仍无法还原未记录的当时配置；精简结果还原依赖原批次行，后续可为批次清理增加引用保护或保留策略。 -4. **重复键的业务语义。** 引擎按采集顺序配对同键行。如果业务应按集合判断，CLI 行顺序改变可能产生多条“不一致”。当前保留原有语义；可优先完善身份键，或另行实现并测试 Python/SQL 一致的集合匹配。 -5. **原表回查的数量上限。** 每侧候选命中最多 2000 条，完整分组补齐每侧预算 10000 条，最终展示最多 200 对；这不是全表匹配总数。界面会提示截断，大范围检索需要收窄条件。 diff --git a/docs/reviews/biz-state-review-2026-10-11.md b/docs/reviews/biz-state-review-2026-10-11.md deleted file mode 100644 index 7ca0e8e..0000000 --- a/docs/reviews/biz-state-review-2026-10-11.md +++ /dev/null @@ -1,54 +0,0 @@ -# 业务监控分析与优化记录 - -检查日期：2026-10-11。范围：网络管理 → 割接对比 → 业务监控，包含任务配置、周期采集状态、批次保护、工作簿、原始日志、筛选与导出。 - -本次修改项目代码并使用内存 SQLite 和固定浏览器 API 响应验证，没有连接设备执行采集，也没有修改生产数据库。未新增解析模板或数据库字段。前后端应一起更新，前端轮询使用新增的 `/v1/biz-state/tasks/{id}/progress` 接口。 - -## 已修复的问题 - -1. **旧请求覆盖当前内容。** 任务、用途筛选、批次工作簿、指标表、采集详情、参数发现和原始日志增加请求序号或取消保护。关闭弹窗、切换任务/表后，慢响应不能重新打开弹窗或覆盖新内容。原始日志下载保留其实际批次 ID，避免根据当前另一层弹窗推断来源。 -2. **后台采集不能及时反映、慢轮询叠加。** 同时观察服务端 `collect_running` 与当前页面启动的任务；周期任务和打开的任务保留低频查询，能够发现后台启动的采集。活动采集每轮完成后等待 4 秒，空闲等待 15 秒，再发起下一轮。跨 effect 重建共享在途标记，避免慢请求叠加。用途筛选隐藏的本地采集任务单独读取进度，避免误报采集结束。 -3. **刷新覆盖未保存配置、批次数突然缩减。** 轮询只合并运行标志和时间，不重新写入周期、保留策略或监控项。进度刷新和首次加载统一读取最多 200 个最近批次，界面明确显示已载入数量与上限。 -4. **加载失败看起来像空表或上一份数据。** 任务列表、任务配置、批次和指标表增加加载/错误/重试状态。指标查询失败清除旧行，不显示“本表无数据”；配置未就绪时操作区域不可交互。列表刷新失败保留当前行并显示错误。 -5. **正在采集的批次可能被删除或清理。** `queued/running` 批次加入 `active_collection` 保护，单条删除、批量删除、保留清理均保留它们；采集中的任务返回 409。前端同步禁用选择与删除，保留基线、业务对比及割接引用保护。 -6. **查询与导出边界错误。** `%`、`_`、反斜杠按字面值搜索；JSON 单列提取和 LLDP 分页排序更稳定。早于主命令保存的辅助命令失败保持可见。CSV 保留 `0`、`False`，正确引用逗号、双引号和 CR/LF；重复 `seq` 的导出不遗漏跨分块行，完整保留后续行才出现的列。 - -## 性能改动与验证范围 - -- 任务详情将逐监控项读取绑定改为一次批量读取。测试中 30 个监控项、60 个绑定由原来的 32 次查询降为 **3 次**；轻量进度接口为 **1 次**，不返回监控项和绑定。 -- 批次保护只读取引用 ID 列，按本次批次范围限制查询，不再加载所有对比/割接运行的配置和汇总 JSON。 -- 工作簿摘要不将原始 CLI 日志传输到 Python；只读取 SQL 计算的行数与是否存在日志。已有完整行数优先使用，原始日志在查看或导出时加载。回归使用 1 MB 日志，确认摘要结果集没有原文列。 -- 通用指标 CSV 先分块扫描列名，再按 `(seq, id)` 游标每块 1000 行写入 ZIP；LLDP 分批写入。减少指标表的 ORM 对象、字典和完整 CSV 同时驻留。**最终 ZIP 仍驻留内存，命令原文导出仍需要原文；这不是端到端流式下载。** 两次扫描保留完整列名，正在写入的批次仍不保证导出为一致性快照。 -- 同一厂商/型号的采集模板缓存 30 秒；轮询不重新读取模板。任务列表默认每页渲染 50 行，可选 20/50/100/200；列名去重使用 Set，命令搜索只计算一次。 -- 指标搜索先回到第一页并等待防抖值一致，避免先请求旧页码、旧关键字。 - -以上是查询次数、请求并发及读取方式的验证，没有测量真实设备或生产 PostgreSQL 的延迟与内存。任务列表 API 仍返回当前用途下的完整任务集合；客户端分页减少 DOM，不减少列表响应大小。JSON 全文模糊搜索、旧日志的 SQL 行数计算、非常大的 ZIP 内存仍是后续基准重点。 - -## 界面调整 - -统一深色工业风：连续指标栏、细边框、小圆角、等宽任务标识、紧凑表头及明确的异常提示。任务配置使用响应式网格；采集周期数值与单位并排；工作簿横向页签、过滤工具栏和表格支持窄屏滚动；底部按钮可换行，避免被裁切。任务和工作簿页签支持方向键、Home/End，过滤控件增加可访问名称。 - -“采集成功 / 部分完成 / 采集失败”描述采集流程，不能据此判断业务验收通过；业务状态需要查看数据表或业务对比结果。 - -- [任务列表](assets/biz-state-tasks.png) -- [任务配置](assets/biz-state-config.png) · [390px 配置](assets/biz-state-config-mobile.png) -- [采集批次](assets/biz-state-batches.png) -- [工作簿](assets/biz-state-workbook.png) · [390px 工作簿](assets/biz-state-workbook-mobile.png) -- [查询失败与重试](assets/biz-state-error.png) - -## 验证 - -- 后端相关回归 **124 项通过**：新增监控服务、保留策略、日志导入、命令导出，以及停止/恢复/收尾/认领采集、双车道、spool、解析池与业务对比相关回归。 -- `web/tests/bizState.browser.mjs`：**30 项回归通过**。固定 120 个任务、54 个批次、120 行指标，覆盖分页、配置保存、模板缓存、过期响应、搜索、重试、日志下载、后台采集和窄屏布局。4500ms 列表延迟下最大并发轮询为 1；轮询阶段任务完整详情仅首次打开请求 1 次，未保存的周期值保持不变。全部 `/v1` 请求被固定响应拦截。 -- `node node_modules/vite/bin/vite.js build` 通过。项目原有主包超过 500 kB 提示仍在。 -- `node node_modules/typescript/bin/tsc -b`：本页已有类型问题已修复；全项目仍被 `BizMigrationPage.tsx` 的 4 个既有错误阻断（两处未使用变量、两处字符串传给数值控件）。 -- 本页 ESLint 从 16 个错误降为 15 个，保留既有 `any` 和 effect 状态同步问题及 1 个依赖警告；不宣称全项目 lint 通过。 - -浏览器回归运行方法（先启动 Vite，可通过环境变量指定 URL、截图目录和浏览器）： - -```powershell -cd web -node node_modules/vite/bin/vite.js --host 127.0.0.1 --port 5179 -# 另一个终端；Playwright 已安装时直接运行，否则设置 NETX_PLAYWRIGHT_MODULE 为其模块路径。 -node tests/bizState.browser.mjs -``` diff --git a/docs/reviews/compare-templates-review-2026-10-11.md b/docs/reviews/compare-templates-review-2026-10-11.md deleted file mode 100644 index 6c62f0d..0000000 --- a/docs/reviews/compare-templates-review-2026-10-11.md +++ /dev/null @@ -1,46 +0,0 @@ -# 对比模板与业务对比评估、优化记录 - -日期：2026-10-11。范围：网络管理 → 割接对比 → 对比模板、业务对比。延续此前的业务对比和业务监控优化。本轮使用内存 SQLite、SQL 编译断言和本地浏览器固定 API 响应验证，没有连接真实设备，也没有改动生产数据库。 - -## 修复的结果准确性问题 - -1. **模板导入可能只保存有效部分。** 前后端检查全部检查项，空 Key、重复 ID、无效规则不再被静默丢弃。服务器在修改模板之前校验，错误响应包括配置路径和原因。执行时也检查历史模板的原始检查项，防止旧的无效配置被归一化后跳过。 -2. **错误过滤规则可能放行数据。** 新写入与执行入口拒绝未知运算符、混合分组/叶节点、空分组、无效正则及超过 12 层的嵌套。集合过滤要求非空数组；绝对/相对容差必须为有限非负数。无效归一模式、重复字段规则和接口别名同样报错。 -3. **复杂嵌套过滤被可视化编辑器截断。** 编辑器只接管能够完整表达的 OR/AND 结构，其余结构完整保留并显示 JSON。保存、导入、导出不删除嵌套条件。修改复杂结构可导出 JSON 后编辑并重新导入。 -4. **端口变化策略在 API 和导入导出中丢失。** `ignore_port_changes` 贯通请求模型、兼容单表配置、编辑、复制、保存、导入与导出。界面明确提供自动判断、接口必须匹配、允许端口变化三种选项；显式 `false` 不再丢失。 -5. **零、布尔和空值混淆。** Python 行键与过滤保留 `0`、`False`，布尔文本与 JSON 提取一致；数值比对不再把空值当成 0。Python/SQL 使用同一数值格式，支持 `+.5`、`1.`、科学计数法；解析不了的值按文本比较。 -6. **SQL 过滤与行键误匹配。** `contains` 的 `%`、`_`、反斜杠按字面值处理，空包含条件与 Python 一致。复合键改为 JSON 数组文本，避免 `('a|b','c')` 和 `('a','b|c')` 被拼成相同键。 -7. **任务范围失效导致隐式扩大或缩小对比。** 编辑任务时保留已移除的检查项 ID，显示错误并禁用保存、执行；用户可明确重新选择或启用当前全部检查项。服务器在保存和执行时都检查 ID，部分有效、部分失效的范围也会被拒绝。 -8. **不完整采集被解释成业务变化。** 比对入口只接受两侧状态为 `success` 的源批次；排队、执行中、失败、取消和部分完成批次返回 `source_batch_not_complete`。界面禁用这些选项。成功批次没有某指标或没有行的原有处理保持不变；本轮没有新增逐指标采集完备性判定。 -9. **删除破坏引用或在途对比。** 已被任务引用的模板不能删除；存在排队/执行中记录的任务及活动记录不能删除。模板页显示任务引用数量。这些是 API 状态检查，未新增数据库外键或锁，不宣称消除跨请求的全部并发窗口。 -10. **重新启用忽略字段仍然无效。** 勾选参与比对时清除遗留的 ignore/skip/off 规则。字段表保留模板配置的自定义字段；过滤专用字段也能显示。兼容数值、相对偏差和归一规则别名的显示。 - -## 性能与界面 - -- 检查项排序从逐表、逐批次查询改为 **一次按批次/metric 聚合**。相同 metric 的拆分表共享统计，SQL 不投影原始 CLI 日志；回归验证查询次数为 1。 -- `batch_metric_collect_ok` 只读取解析状态和行数，不加载命令原文。 -- 模板引用数量由一次 grouped 查询计算，避免每行请求。默认模板初始化/升级仍执行原来的查询。 -- 模板页面只请求模板和字段目录，省去业务监控任务、端口映射、对比任务三个列表；字段目录缓存 60 秒。业务对比与模板列表使用独立缓存键。 -- 两个列表默认每页渲染 50 行，可选 20/50/100/200；模板 ID 查找使用 Map。**API 仍返回完整列表，客户端分页减少 DOM，不减少响应体。** -- 模板编辑器使用深色平面面板、细边框、小圆角、蓝色选中态、等宽字段和固定表头。接口归一化折叠，检查项导航与规则区并排；窄屏导航横向滚动，操作按钮独立一行，过滤控件可换行，底部保存始终可见。 -- 导航和规则页签支持方向键、Home/End；规则控件增加可访问名称。保存期间编辑区不可交互，并阻止关闭后打开新模板造成保存响应串扰。 - -没有测量生产 PostgreSQL 的耗时或内存；本轮 SQL 变更由生成 SQL/绑定参数断言验证，未执行真实 PostgreSQL 集成回归。极大数值指数的 SQL 浮点转换、Python 与 PostgreSQL 正则方言差异、全文 JSON 搜索和大型 ZIP 内存仍需单独基准。既有回退机制不等于所有 SQL/Python 语义已完全一致。 - -## 验证与预览 - -- 后端相关回归：**120 项通过，1 项跳过**。跳过的是需显式开启的百万行压力测试。覆盖模板校验、原子更新、旧记录执行保护、端口策略、过滤、数值、复合键、查询次数、删除保护、范围失效、源批次状态，以及既有对比、搜索、配置快照、导出、恢复与内存相关回归。 -- Node 测试：**14 项通过**（8 项模板模型、6 项缓存）。 -- 浏览器：**53 个场景通过**（原业务对比 27 项、模板新增 26 项）。全部 `/v1` 请求使用固定响应，120 个模板列表验证默认 50 行上限；覆盖导入导出、复杂过滤、端口策略、负容差、忽略字段恢复、保存期间交互锁定、范围修复、键盘和 390px 布局。 -- Vite 生产构建通过；原有主包超过 500 kB 提示仍在。 -- 全项目 TypeScript 检查仍有 `BizMigrationPage.tsx` 的 4 个既有错误：2 个未使用变量、2 处字符串传给数值控件；本轮没有新增错误。 -- 新模板模型 ESLint 通过；业务对比页保留原来的 **23 个错误**（17 个 any、6 个 effect 状态同步），与修改前数量一致。不宣称全项目 lint 通过。 - -前后端应一起更新；新增 API 字段为模板引用数量与端口策略，无数据库迁移。原来使用标量 `in/not_in`、未知规则、负容差的模板需要修正再保存/执行；部分完成的源批次需要重新完成采集后再对比。 - -- [桌面模板编辑器](assets/compare-template-editor.png) -- [390px 模板编辑器](assets/compare-template-editor-mobile.png) -- [业务对比列表](assets/biz-compare-jobs.png) -- [此前业务对比记录](biz-compare-review-2026-10-10.md) - -复现：先在 `web` 启动 `node node_modules/vite/bin/vite.js --host 127.0.0.1 --port 5179`，另一个终端运行 `node tests/bizCompareTemplates.browser.mjs` 和 `node tests/bizCompare.browser.mjs`。可用 `NETX_PLAYWRIGHT_MODULE` 指定已有 Playwright，`NETX_TEST_URL` 指定本地地址，`NETX_TEST_OUTPUT` 指定截图目录。 diff --git a/netx_api/app_startup.py b/netx_api/app_startup.py index 8acc5f5..ea16cd6 100644 --- a/netx_api/app_startup.py +++ b/netx_api/app_startup.py @@ -15,7 +15,6 @@ from .schema_patches import ( apply_topology_schema_safety_net, run_alembic_upgrade_to_head, ) -from .ne_crypto import ensure_credential_secret_key from .security_bootstrap import assert_secure_defaults_or_exit from .ume_runtime import start_api_sideband_threads, start_device_schedulers import netx_api.ume_support as ume_support @@ -43,8 +42,6 @@ def _configure_ume_diag_logging() -> None: def run_api_startup() -> None: """Full API boot sequence previously inlined in ``main.on_startup``.""" assert_secure_defaults_or_exit() - # Persist Fernet key for managed-NE passwords (same idea as JWT secret file). - ensure_credential_secret_key() _configure_ume_diag_logging() _log.info( "startup: ne_exec_policy_enabled=%s", @@ -159,17 +156,6 @@ def run_api_startup() -> None: ) except Exception: _log.exception("startup: biz_state collect recovery failed") - try: - from .biz_state.compare_service import recover_interrupted_compares_on_startup - - cmp_rec = recover_interrupted_compares_on_startup(db) - if cmp_rec.get("runs"): - _log.info( - "startup: cancelled %s interrupted biz compare run(s)", - cmp_rec.get("runs"), - ) - except Exception: - _log.exception("startup: biz compare recovery failed") try: from .port_traffic_migrate import backfill_port_traffic_series diff --git a/netx_api/biz_migration/auto_monitor.py b/netx_api/biz_migration/auto_monitor.py deleted file mode 100644 index 4cd8174..0000000 --- a/netx_api/biz_migration/auto_monitor.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Coalesced collect-completion events for live cutover evaluation. - -One worker per process bounds concurrency. Project row locks in run_evaluate -serialize persistence across processes. No device collection runs here. -""" -from __future__ import annotations - -import hashlib -import json -import logging -import threading - -from sqlalchemy.orm import Session - -from ..models import BizMigrationBatch, BizMigrationProject, BizMigrationRun, BizMonitorTemplate, BizStateBatch -from . import service - -_log = logging.getLogger(__name__) -_lock = threading.Lock() -_pending: set[str] = set() -_worker_running = False - - -def try_auto_monitor_for_task(db: Session, task_id: str) -> int: - count = 0 - # Bindings are JSON for multiple sampling intervals; match in Python after - # limiting to projects with an active operation batch. - projects = db.query(BizMigrationProject).filter( - BizMigrationProject.status != "done", - BizMigrationProject.id.in_(db.query(BizMigrationBatch.project_id).filter( - BizMigrationBatch.status == "active")), - ).all() - for candidate in projects: - tids = service._all_hf_task_ids(candidate, "old") + service._all_hf_task_ids(candidate, "new") - if task_id not in tids or not candidate.old_baseline_batch_id or not candidate.new_baseline_batch_id: - continue - # Lock before deduplication so two worker processes cannot save the same - # snapshot. run_evaluate holds the same lock through the transaction. - proj = db.query(BizMigrationProject).filter( - BizMigrationProject.id == candidate.id).with_for_update().populate_existing().first() - batch = db.query(BizMigrationBatch).filter( - BizMigrationBatch.project_id == proj.id, BizMigrationBatch.status == "active" - ).order_by(BizMigrationBatch.started_at.desc()).first() - if not batch or proj.status == "done": - db.rollback() - continue - mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None - if not mt: - db.rollback() - _log.error("cutover monitor template missing project=%s", proj.id) - continue - sheets, overrides, defaults, norms = service.resolve_evaluate_sheets(db, mt) - samples = [] - for tid in tids: - sample = db.query(BizStateBatch).filter( - BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued")) - ).order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() - samples.append([tid, sample.id if sample else "", sample.status if sample else ""]) - payload = [samples, proj.old_baseline_batch_id, proj.new_baseline_batch_id, - batch.expect_set_json, service._port_map_dict(db, proj.mapping_id), - sheets, overrides, defaults, norms, service.resolve_collect_metric_ids(db, proj)] - fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest() - last = db.query(BizMigrationRun).filter(BizMigrationRun.batch_id == batch.id).order_by( - BizMigrationRun.created_at.desc()).first() - if last and (last.summary_json or {}).get("auto_fingerprint") == fingerprint: - db.rollback() - continue - previous_samples = {s[0]: (s[1], s[2]) for s in (last.summary_json or {}).get("auto_samples", [])} if last else {} - if last and not previous_samples: - for sample in (last.summary_json or {}).get("metric_batches", {}).values(): - for side in ("old", "new"): - previous_samples[sample.get(f"{side}_task_id", "")] = ( - sample.get(f"{side}_batch_id", ""), sample.get(f"{side}_status", "success")) - changed_tasks = {tid for tid, bid, status in samples if previous_samples.get(tid) != (bid, status)} - changed_metrics = {mid for side in ("old", "new") for binding in service._hf_bindings(proj, side) - if binding["task_id"] in changed_tasks for mid in binding.get("metric_ids") or payload[-1]} - try: - result = service.run_evaluate(db, batch_id=batch.id, purpose="auto", _commit=False, - _metric_ids=changed_metrics) - run = db.get(BizMigrationRun, result["id"]) - run.summary_json = {**run.summary_json, "auto_fingerprint": fingerprint, "auto_samples": samples} - db.commit() - count += 1 - except Exception: - db.rollback() - _log.exception("cutover monitor evaluate failed project=%s", proj.id) - return count - - -def schedule_auto_monitor_for_task(task_id: str) -> None: - global _worker_running - with _lock: - _pending.add(str(task_id)) - if _worker_running: - return - _worker_running = True - - def work() -> None: - global _worker_running - from ..db import SessionLocal - - while True: - with _lock: - if not _pending: - _worker_running = False - return - tid = _pending.pop() - try: - with SessionLocal() as db: - try_auto_monitor_for_task(db, tid) - except Exception: - _log.exception("cutover monitor worker failed task=%s", tid) - - threading.Thread(target=work, name="biz-cutover-monitor", daemon=True).start() diff --git a/netx_api/biz_migration/evaluate.py b/netx_api/biz_migration/evaluate.py index f762a49..eb85841 100644 --- a/netx_api/biz_migration/evaluate.py +++ b/netx_api/biz_migration/evaluate.py @@ -5,9 +5,6 @@ from __future__ import annotations from typing import Any from ..biz_state.compare_engine import apply_port_map, compare_rows -from ..biz_state.compare_rules import ( - effective_compare_fields, field_rule_map, scalar_text, values_equal, -) PORT_METRIC_ID = "interface_brief" PORT_STATUS_FIELDS = ("admin", "phy", "prot") @@ -25,7 +22,7 @@ def normalize_expect_key(raw: str) -> str: Accepts legacy ``a|b`` multi-field keys and already-normalized ``\\x1f`` keys. """ - s = scalar_text(raw).strip() + s = str(raw or "").strip() if not s: return "" if KEY_SEP in s: @@ -68,13 +65,13 @@ def classify_status(row: dict[str, Any] | None, sheet_override: dict[str, Any] | return "none" down_vals = {str(x).lower() for x in (ov.get("down_values") or ["down"])} up_vals = {str(x).lower() for x in (ov.get("up_values") or ["up"])} - vals = [scalar_text(row.get(f)).strip().lower() for f in fields] + vals = [str(row.get(f) or "").strip().lower() for f in fields] vals = [v for v in vals if v] if not vals: return "none" if any(v in down_vals for v in vals): return "down" - if len(vals) == len(fields) and all(v in up_vals for v in vals): + if vals and all(v in up_vals for v in vals): return "up" return "other" @@ -108,11 +105,11 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]: if it.get("key") is not None: k = it.get("key") if isinstance(k, (list, tuple)): - joined = KEY_SEP.join(scalar_text(p).strip() for p in k) - if joined.strip(KEY_SEP): + joined = KEY_SEP.join(str(p).strip() for p in k if str(p).strip()) + if joined: bucket.add(joined) else: - s = normalize_expect_key(scalar_text(k)) + s = normalize_expect_key(str(k or "")) if s: bucket.add(s) keys = it.get("keys") @@ -120,14 +117,14 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]: # Flat list of segments → one composite key; else each entry is a key # (string or nested list/tuple of segments). if keys and all(not isinstance(x, (list, tuple, dict)) for x in keys): - joined = KEY_SEP.join(scalar_text(x).strip() for x in keys) - if joined.strip(KEY_SEP): + joined = KEY_SEP.join(str(x).strip() for x in keys if str(x).strip()) + if joined: bucket.add(joined) else: for x in keys: if isinstance(x, (list, tuple)): - joined = KEY_SEP.join(scalar_text(p).strip() for p in x) - if joined.strip(KEY_SEP): + joined = KEY_SEP.join(str(p).strip() for p in x if str(p).strip()) + if joined: bucket.add(joined) else: s = normalize_expect_key(str(x or "")) @@ -222,7 +219,7 @@ def _field_tokens_from_row(row: dict[str, Any] | None, fields: set[str]) -> set[ return set() out: set[str] = set() for f in fields: - v = scalar_text(row.get(f)).strip() + v = str(row.get(f) or "").strip() if v: out.add(field_token(f, v)) return out @@ -248,8 +245,8 @@ def _side_tokens( def _norm_list_values(raw: Any) -> list[str]: if isinstance(raw, list): - return [scalar_text(x).strip().lower() for x in raw if scalar_text(x).strip()] - s = scalar_text(raw).strip() + return [str(x).strip().lower() for x in raw if str(x).strip()] + s = str(raw or "").strip() if not s: return [] if "," in s: @@ -269,22 +266,18 @@ def _eval_field_op( f = str(field or "").strip() if not f: return False - raw = "" if not row else scalar_text(row.get(f)).strip() + raw = "" if not row else str(row.get(f) or "").strip() val = raw.lower() o = str(op or "eq").strip().lower() - if row and f not in row: - # A missing parser field is not evidence for a negative condition such as - # state != down. Only an explicit empty check may match that absence. - return o == "empty" if o in ("changed", "diff"): - base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower() + base = "" if not base_row else str(base_row.get(f) or "").strip().lower() if not row: return False return base != val if o in ("unchanged", "same"): if not row: return False - base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower() + base = "" if not base_row else str(base_row.get(f) or "").strip().lower() return base == val if o == "empty": return not val @@ -575,7 +568,7 @@ def _match_field_token_rules( if not field or not allowed: continue row = old_row if side == "old" else new_row if side == "new" else None - val = scalar_text((row or {}).get(field)).strip().lower() + val = str((row or {}).get(field) or "").strip().lower() if val not in allowed: return False return True @@ -865,14 +858,6 @@ def _port_identity(key_fields: list[str], iface_fields: list[str]) -> bool: return all(str(f) in iface_set for f in key_fields) -def _iface_selected(value: str, ports: set[str]) -> bool: - """Set lookup plus parent interface prefixes; independent of scope size.""" - if value in ports: - return True - parts = value.split(".") - return any(".".join(parts[:i]) in ports for i in range(len(parts) - 1, 0, -1)) - - def _unmapped_same_iface_anomaly( *, old_kind: str, @@ -907,8 +892,6 @@ def _remap_key_str( return key_str if not port_map: return key_str - if key_fields and not set(key_fields).intersection(iface_fields): - return key_str if (not key_fields or len(key_fields) == 1) and ( key_str in port_map or any( @@ -945,8 +928,6 @@ def _reverse_remap_key_str( return key_str if not rev_map: return key_str - if key_fields and not set(key_fields).intersection(iface_fields): - return key_str if (not key_fields or len(key_fields) == 1) and ( key_str in rev_map or any( @@ -1028,7 +1009,7 @@ def row_match_key( data = strip_netx(row) if iface_fields and rules: data = apply_iface_normalize(dict(data), iface_fields=iface_fields, rules=rules) - return KEY_SEP.join(scalar_text(data.get(k)).strip() for k in key_fields) + return KEY_SEP.join(str(data.get(k) or "").strip() for k in key_fields) def index_raw_by_match_key( @@ -1052,7 +1033,7 @@ def display_key_from_raw(raw: dict[str, Any] | None, key_fields: list[str]) -> s if not raw or not key_fields: return "" data = strip_netx(raw) - return "|".join(scalar_text(data.get(k)).strip() for k in key_fields) + return "|".join(str(data.get(k) or "").strip() for k in key_fields) def iface_lineage( @@ -1072,7 +1053,7 @@ def iface_lineage( pmap = port_map or {} out: list[dict[str, Any]] = [] for f in iface_fields: - raw_v = scalar_text(data.get(f)).strip() + raw_v = str(data.get(f) or "").strip() norm_v = normalize_iface_name(raw_v, rules) if rules else raw_v mapped = False map_to = "" @@ -1111,8 +1092,6 @@ def evaluate_metric_dual( out_of_expect: str = "strict", sheet_id: str = "", iface_normalize_rules: list[dict[str, str]] | None = None, - previous_expect: dict[str, set[str]] | None = None, - allow_mapping_scope: bool = True, ) -> dict[str, Any]: """Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge. @@ -1123,16 +1102,12 @@ def evaluate_metric_dual( from ..biz_state.iface_normalize import ( apply_iface_normalize_rows, normalize_iface_rules, - normalize_iface_name, ) sid = str(sheet_id or "").strip() or str(metric_id or "").strip() expect_keys = expect_keys_for_metric( expect, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid ) - previous_keys = expect_keys_for_metric( - previous_expect or {}, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid - ) ov = sheet_override or {} if ov.get("skip_dual"): return { @@ -1160,15 +1135,6 @@ def evaluate_metric_dual( ) norm_rules = normalize_iface_rules(iface_normalize_rules) - def normalized_key(key: str) -> str: - parts = key.split(KEY_SEP) if KEY_SEP in key else ([key] if len(key_fields) == 1 else key.split("|")) - if len(parts) != len(key_fields): - return key - return KEY_SEP.join(normalize_iface_name(p, norm_rules) if f in iface_fields else p - for p, f in zip(parts, key_fields)) - - expect_keys = {normalized_key(key) for key in expect_keys} - previous_keys = {normalized_key(key) for key in previous_keys} old_baseline_rows = apply_iface_normalize_rows( [strip_netx(r) for r in raw_old_base], iface_fields=iface_fields, rules=norm_rules ) @@ -1212,7 +1178,6 @@ def evaluate_metric_dual( port_map=None, field_rules=field_rules, iface_normalize_rules=None, # already applied - include_unchanged=True, ) old_idx = build_diff_index_from_compare(old_cmp) @@ -1221,8 +1186,18 @@ def evaluate_metric_dual( if new_baseline_rows is not None: new_before = list(new_baseline_rows) new_baseline_mode = "provided" - # A successfully collected empty baseline is valid on a replacement NE. - # Collection completeness is checked by the service, not inferred from rows. + # Explicit empty batch is as unusable as "no baseline" for presence semantics. + if not new_before and (old_baseline_rows or new_current_rows): + if port_map and iface_fields: + new_before = [ + apply_port_map(r, iface_fields=iface_fields, port_map=port_map) + for r in old_baseline_rows + ] + new_baseline_mode = "port_mapped" + new_baseline_missing = False + else: + new_baseline_missing = True + new_baseline_mode = "empty" elif port_map and iface_fields: new_before = [ apply_port_map(r, iface_fields=iface_fields, port_map=port_map) @@ -1243,22 +1218,12 @@ def evaluate_metric_dual( compare_fields=compare_fields, port_map=None, field_rules=field_rules, - include_unchanged=True, ) new_idx = build_diff_index_from_compare(new_cmp) rev_map = {v: k for k, v in port_map.items()} map_scoped = _map_defines_iface_expect(metric_id, iface_fields, port_map) - # Mapping describes identity; an explicit batch selection defines scope. - # Port selection also covers business rows attached to those interfaces. - for scope, keys in ((expect, expect_keys), (previous_expect or {}, previous_keys)): - ports = {normalize_iface_name(p, norm_rules) for p in scope.get("_ports") or set()} - if ports and not keys and iface_fields: - for ks, diff in old_idx.items(): - row = diff.get("before") or diff.get("after") or {} - if any(_iface_selected(scalar_text(row.get(f)), ports) for f in iface_fields): - keys.add(ks) - if allow_mapping_scope and map_scoped and not any(expect.values()): + if map_scoped: scoped: set[str] = set() for ks in old_idx: if _key_in_port_map( @@ -1288,8 +1253,6 @@ def evaluate_metric_dual( for ok in sorted(expect_keys): _add(ok) - for ok in sorted(previous_keys): - _add(ok) for ks in sorted(old_idx): _add(ks) for ks in sorted(new_idx): @@ -1307,11 +1270,6 @@ def evaluate_metric_dual( progress_total = 0 anomaly = 0 anomaly_in_expect = 0 - steady_outside = 0 - rules_by_field = field_rule_map(field_rules) - integrity_fields = effective_compare_fields(compare_fields, field_rules) - # Status transitions are interpreted by the monitor's correction rules. - integrity_fields = [f for f in integrity_fields if f not in (ov.get("status_fields") or [])] for old_ks in canon_keys: new_ks = _remap_key_str( @@ -1321,12 +1279,11 @@ def evaluate_metric_dual( port_map=port_map, ) in_exp = old_ks in expect_keys - previous = old_ks in previous_keys and not in_exp if in_exp: progress_total += 1 od = old_idx.get(old_ks) - nd = new_idx.get(new_ks) + nd = new_idx.get(new_ks) or new_idx.get(old_ks) old_kind = str((od or {}).get("kind") or "") new_kind = str((nd or {}).get("kind") or "") @@ -1344,7 +1301,11 @@ def evaluate_metric_dual( old_st = classify_status(old_cur, ov) new_st = classify_status(new_cur, ov) - if map_scoped and not (in_exp or previous) and ( + # Mapped iface rows are the expect set. Unmapped rows stay out of the + # migration compare, except same-name ports where old went down and the + # new device's same port is up — that is anomaly, not a successful cutover. + if map_scoped and not in_exp: + if not ( _port_identity(key_fields, iface_fields) and _unmapped_same_iface_anomaly( old_kind=old_kind, @@ -1353,63 +1314,26 @@ def evaluate_metric_dual( new_status=new_st, ) ): + continue verdict, color, rule_hit = "anomaly", "red", "unmapped_same_iface" else: verdict, color, rule_hit = dual_verdict_ex( old_kind=old_kind or "", new_kind=new_kind or "", - in_expect=in_exp or previous, + in_expect=in_exp, window_active=window_active, acceptance=acceptance, old_status=old_st, new_status=new_st, success_patterns=success_patterns or None, anomaly_patterns=anomaly_patterns or None, - # Business drift outside this batch must remain visible. - out_of_expect="strict", + out_of_expect=out_of_expect, old_row=old_cur or None, new_row=new_cur or None, old_base=old_base or None, new_base=new_base or None, sheet_override=ov, ) - integrity_changes: list[dict[str, Any]] = [] - if in_exp or previous: - old_departed = old_kind == "removed" or old_st == "down" - new_healthy = bool(new_cur) and ( - new_st == "up" if ov.get("status_fields") else True - ) - if old_departed and not new_healthy: - verdict, color, rule_hit = "lost", "red", "service_not_received" - elif verdict == "migrated": - if not new_healthy: - verdict, color, rule_hit = "anomaly", "red", "target_unhealthy" - elif not old_base: - verdict, color, rule_hit = "anomaly", "red", "old_service_not_in_baseline" - else: - reference = apply_port_map(old_base, iface_fields=iface_fields, port_map=port_map) - integrity_changes = [ - {"field": f, "before": reference.get(f), "after": new_cur.get(f)} - for f in integrity_fields - if not values_equal(reference.get(f), new_cur.get(f), rule=rules_by_field.get(f)) - ] - if integrity_changes: - verdict, color, rule_hit = "anomaly", "red", "service_content_changed" - elif not old_departed and not success_patterns: - verdict, color, rule_hit = ( - ("unfinished", "red", "old_service_still_present") if acceptance - else ("migrating", "yellow", "old_service_still_present") - ) - if previous: - if verdict == "migrated": - verdict, rule_hit = "migrated_previous", "previous_batch_received" - elif color != "red": - verdict, color, rule_hit = "unfinished_previous", "red", "previous_batch_not_received" - # Stable inventory is covered by the comparison and counted, but does - # not need a full persisted evidence card on every high-frequency run. - if not in_exp and verdict == "not_involved": - steady_outside += 1 - continue if in_exp and verdict == "migrated" and color == "green": progress_ok += 1 if color == "red": @@ -1420,9 +1344,9 @@ def evaluate_metric_dual( # Prefer live raw rows; fall back to baseline raw. Never use port-mapped # synthetic "before" as the new-side display row. raw_old = old_raw_cur_idx.get(old_ks) or old_raw_base_idx.get(old_ks) - raw_new = new_raw_cur_idx.get(new_ks) + raw_new = new_raw_cur_idx.get(new_ks) or new_raw_cur_idx.get(old_ks) if not raw_new: - raw_new = new_raw_base_idx.get(new_ks) + raw_new = new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks) display_old = display_key_from_raw(raw_old, key_fields) or old_ks display_new = display_key_from_raw(raw_new, key_fields) or ( @@ -1436,8 +1360,8 @@ def evaluate_metric_dual( new_disp = strip_netx(raw_new) if raw_new else {} old_base_raw = strip_netx(old_raw_base_idx.get(old_ks)) if old_raw_base_idx.get(old_ks) else {} new_base_raw = ( - strip_netx(new_raw_base_idx.get(new_ks)) - if new_raw_base_idx.get(new_ks) + strip_netx(new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks)) + if (new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks)) else {} ) @@ -1487,9 +1411,7 @@ def evaluate_metric_dual( "verdict": verdict, "color": color, "rule_hit": rule_hit, - "integrity_changes": integrity_changes, "in_expect": in_exp, - "previous_batch": previous, "old_kind": old_kind, "new_kind": new_kind, "old": old_disp, @@ -1528,7 +1450,6 @@ def evaluate_metric_dual( "progress_total": progress_total, "anomaly": anomaly, "anomaly_in_expect": anomaly_in_expect, - "steady_outside": steady_outside, "rows": rows_out, "new_baseline_mode": new_baseline_mode, "new_baseline_missing": new_baseline_missing, diff --git a/netx_api/biz_migration/monitor_templates.py b/netx_api/biz_migration/monitor_templates.py index 9ebc420..b55f487 100644 --- a/netx_api/biz_migration/monitor_templates.py +++ b/netx_api/biz_migration/monitor_templates.py @@ -8,8 +8,7 @@ from uuid import uuid4 from fastapi import HTTPException from sqlalchemy.orm import Session -from ..models import BizCompareTemplate, BizMigrationProject, BizMonitorTemplate -from .validation import validate_monitor_rules +from ..models import BizCompareTemplate, BizMonitorTemplate from ..timeutil import utcnow_naive from ..biz_state import compare_service as cmp_svc from ..biz_state.iface_normalize import default_zte_iface_normalize_rules @@ -34,8 +33,6 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None effective = seen if not effective: effective = [PORT_METRIC_ID] - ct = db.get(BizCompareTemplate, row.compare_template_id) if db and row.compare_template_id else None - available = list(dict.fromkeys(str(s.get("metric_id") or "") for s in cmp_svc.template_metrics(ct))) if ct else effective out: dict[str, Any] = { "id": row.id, "name": row.name, @@ -43,7 +40,6 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None "compare_template_name": compare_name, "collect_metric_ids": collect, "collect_metric_ids_effective": effective, - "available_metric_ids": [m for m in available if m], "defaults": dict(row.defaults_json or {}), "sheet_overrides": list(row.sheet_overrides_json or []), "note": row.note or "", @@ -460,7 +456,6 @@ def create_monitor_template(db: Session, body: dict[str, Any]) -> dict[str, Any] overrides = body.get("sheet_overrides") if not isinstance(overrides, list): overrides = [] - validate_monitor_rules(defaults, overrides) row = BizMonitorTemplate( id=uuid4().hex, name=name[:256], @@ -484,10 +479,6 @@ def update_monitor_template(db: Session, template_id: str, body: dict[str, Any]) row = db.get(BizMonitorTemplate, template_id) if not row: raise HTTPException(status_code=404, detail="monitor_template_not_found") - validate_monitor_rules(body.get("defaults", row.defaults_json or {}), - body.get("sheet_overrides", row.sheet_overrides_json or [])) - if body.get("compare_template_id") and not db.get(BizCompareTemplate, body["compare_template_id"]): - raise HTTPException(status_code=404, detail="compare_template_not_found") if "name" in body and body["name"] is not None: name = str(body["name"] or "").strip() if not name: @@ -524,7 +515,5 @@ def delete_monitor_template(db: Session, template_id: str) -> None: row = db.get(BizMonitorTemplate, template_id) if not row: raise HTTPException(status_code=404, detail="monitor_template_not_found") - if db.query(BizMigrationProject.id).filter(BizMigrationProject.monitor_template_id == template_id).first(): - raise HTTPException(status_code=409, detail="monitor_template_in_use") db.delete(row) db.commit() diff --git a/netx_api/biz_migration/service.py b/netx_api/biz_migration/service.py index d895809..760a6c0 100644 --- a/netx_api/biz_migration/service.py +++ b/netx_api/biz_migration/service.py @@ -2,20 +2,18 @@ from __future__ import annotations -import hashlib -from collections import Counter from datetime import datetime, timezone from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import and_, insert, or_ from sqlalchemy.orm import Session -from ..biz_state.compare_rules import apply_row_filters, scalar_text +from ..biz_state.compare_rules import apply_row_filters from ..biz_state.compare_service import ( _load_metric_rows, _port_map_dict, + batch_metric_collect_ok, sheet_key, sheet_title, template_metrics, @@ -30,7 +28,6 @@ from ..models import ( BizMonitorTemplate, BizPortMapping, BizStateBatch, - BizStateBatchCommand, BizStateTask, ) from ..timeutil import utcnow_naive @@ -44,7 +41,6 @@ from .evaluate import ( ) PURPOSE_CUTOVER_HF = "cutover_hf" -MANUAL_ROUTE_METRICS = frozenset({"ip_route", "ipv6_route", "bgp_route"}) def _parse_dt(raw: Any) -> datetime | None: @@ -153,7 +149,7 @@ def hf_task_id(proj: BizMigrationProject, side: str) -> str: def current_task_id(proj: BizMigrationProject, side: str) -> str: """Evaluate current batches: prefer HF, fall back to portrait.""" - return hf_task_id(proj, side) + return hf_task_id(proj, side) or portrait_task_id(proj, side) def _metric_interval_map(proj: BizMigrationProject) -> dict[str, int]: @@ -220,9 +216,7 @@ def _hf_bindings(proj: BizMigrationProject, side: str) -> list[dict[str, Any]]: iv = max(60, int(row.get("interval_sec") or 60)) except (TypeError, ValueError): iv = 60 - out.append({"task_id": tid, "metric_ids": mids, "interval_sec": iv, - "collect_group_id": str(row.get("collect_group_id") or ""), - "manual_only": bool(row.get("manual_only"))}) + out.append({"task_id": tid, "metric_ids": mids, "interval_sec": iv}) if out: return out # Legacy single slot @@ -274,94 +268,15 @@ def _current_batch_for_metric( # Only honor pin when it belongs to this metric's HF task (multi-interval safe) if pinned and (not tid or str(pinned.task_id or "") == tid): # Pinned partial / failed metric → treat as missing (avoid false red) - if not _metric_sample_ok(db, pinned, metric_id): + if not batch_metric_collect_ok(db, pinned.id, metric_id): return None return pinned - batch = ( - db.query(BizStateBatch) - .filter(BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued"))) - .order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() - ) if tid else None - if batch and not _metric_sample_ok(db, batch, metric_id): + batch = _latest_success_batch(db, tid) if tid else None + if batch and not batch_metric_collect_ok(db, batch.id, metric_id): return None return batch -def _metric_sample_ok(db: Session, batch: BizStateBatch, metric_id: str) -> bool: - """An empty result requires a successful command receipt, just like a full one. - - Batch success alone cannot prove this metric was collected. Unmatched and - skipped parsers are uncertainty, never evidence that a business disappeared. - """ - if batch.status not in ("success", "partial"): - return False - statuses = [str(s or "").lower() for (s,) in db.query(BizStateBatchCommand.parse_status) - .filter(BizStateBatchCommand.batch_id == batch.id, - BizStateBatchCommand.metric_id == metric_id).all()] - return bool(statuses) and all(s in ("ok", "ok_aux") for s in statuses) - - -def _current_pair_for_metric(db: Session, proj: BizMigrationProject, mid: str, - pin_old: str = "", pin_new: str = "") -> tuple[BizStateBatch | None, BizStateBatch | None, dict[str, Any]]: - old_tid, new_tid = (_hf_task_id_for_metric(proj, side, mid) for side in ("old", "new")) - old_task = db.get(BizStateTask, old_tid) if old_tid else None - new_task = db.get(BizStateTask, new_tid) if new_tid else None - gid = old_task.collect_group_id if old_task else "" - meta: dict[str, Any] = {"collect_group_id": gid, "manual_only": bool(old_task and old_task.collect_manual_only)} - if gid and not (pin_old or pin_new): - if not new_task or new_task.collect_group_id != gid: - return None, None, {**meta, "sample_issues": ["collect_pair_invalid"]} - latest = db.query(BizStateBatch).filter(BizStateBatch.collect_group_id == gid, - BizStateBatch.task_id.in_((old_tid, new_tid))).order_by( - BizStateBatch.queued_at.desc(), BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() - peers = db.query(BizStateBatch).filter(BizStateBatch.collect_round_id == latest.collect_round_id, - BizStateBatch.task_id.in_((old_tid, new_tid))).all() if latest and latest.collect_round_id else [] - meta.update({"collect_round_id": latest.collect_round_id if latest else "", - "old_collect_status": next((b.status for b in peers if b.task_id == old_tid), "missing"), - "new_collect_status": next((b.status for b in peers if b.task_id == new_tid), "missing")}) - if len(peers) != 2: - return None, None, {**meta, "sample_issues": ["collect_pair_missing"]} - if any(b.status in ("running", "queued") for b in peers): - return None, None, {**meta, "sample_issues": ["collect_pair_pending"]} - by_task = {b.task_id: b for b in peers} - if not all(_metric_sample_ok(db, b, mid) for b in peers): - return None, None, {**meta, "sample_issues": ["collect_pair_failed"]} - return by_task[old_tid], by_task[new_tid], meta - old = _current_batch_for_metric(db, proj, "old", mid, pinned_batch_id=pin_old) - new = _current_batch_for_metric(db, proj, "new", mid, pinned_batch_id=pin_new) - if old and new and (old.collect_round_id or new.collect_round_id) and old.collect_round_id != new.collect_round_id: - return None, None, {**meta, "sample_issues": ["collect_round_mismatch"]} - return old, new, meta - - -def _sample_timing(proj: BizMigrationProject, mid: str, old: BizStateBatch, - new: BizStateBatch, *, check_age: bool) -> dict[str, Any]: - interval = max(60, _metric_interval_map(proj).get(mid) or proj.hf_interval_sec or 60) - old_at, new_at = old.ended_at or old.started_at, new.ended_at or new.started_at - durations = [max(0, ((b.ended_at or b.started_at) - b.started_at).total_seconds()) for b in (old, new)] - age_limit, skew_limit = interval * 3 + max(durations), interval * 2 - reasons: list[str] = [] - if not old_at or not new_at: - reasons.append("sample_time_missing") - skew = None - else: - skew = abs((old_at - new_at).total_seconds()) - if skew > skew_limit: - reasons.append("sample_skew") - if check_age and max((utcnow_naive() - old_at).total_seconds(), - (utcnow_naive() - new_at).total_seconds()) > age_limit: - reasons.append("sample_stale") - start_skew = abs((old.started_at - new.started_at).total_seconds()) - if old.collect_round_id and start_skew > min(30, interval / 2): - reasons.append("sample_start_skew") - return {"old_collected_at": _dt_iso(old_at), "new_collected_at": _dt_iso(new_at), - "age_limit_sec": age_limit, "skew_limit_sec": skew_limit, - "skew_sec": skew, "start_skew_sec": start_skew, "sample_issues": reasons, - "old_duration_sec": durations[0], "new_duration_sec": durations[1], - "old_queue_sec": max(0, (old.started_at - (old.queued_at or old.started_at)).total_seconds()), - "new_queue_sec": max(0, (new.started_at - (new.queued_at or new.started_at)).total_seconds())} - - def find_portrait_task_for_ne(db: Session, *, source: str, ne_id: str) -> BizStateTask | None: """Best portrait (non-HF) biz_state task for an NE — for baseline binding.""" src = str(source or "managed").strip().lower() or "managed" @@ -740,14 +655,6 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st p = db.get(BizMigrationProject, project_id) if not p: raise HTTPException(status_code=404, detail="project_not_found") - # Validate incoming baseline ownership before mutating any project field. - for side in ("old", "new"): - bid = str(body.get(f"{side}_baseline_batch_id", getattr(p, f"{side}_baseline_batch_id")) or "").strip() - tid = str(body.get(f"{side}_task_id", getattr(p, f"{side}_task_id")) or "").strip() - if bid: - sample = db.get(BizStateBatch, bid) - if not sample or sample.task_id != tid or sample.status not in ("success", "partial"): - raise HTTPException(status_code=400, detail=f"{side}_baseline_invalid") if "name" in body and body["name"] is not None: p.name = str(body["name"]).strip() or p.name if "note" in body and body["note"] is not None: @@ -807,19 +714,6 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st p.hf_start_at = _parse_dt(body.get("hf_start_at")) if "hf_end_at" in body: p.hf_end_at = _parse_dt(body.get("hf_end_at")) - sampling_changed = any(k in body for k in ("collect_metric_ids", "metric_interval_sec", "hf_interval_sec", "monitor_template_id")) - has_hf = bool(_all_hf_task_ids(p, "old") and _all_hf_task_ids(p, "new")) - if sampling_changed and has_hf: - # Validate newly selected commands before saving the project selection. - for side in ("old", "new"): - portrait = db.get(BizStateTask, portrait_task_id(p, side)) - existing = db.get(BizStateTask, hf_task_id(p, side)) - fields = _template_from_task_or_ne(db, portrait=portrait, existing_hf=existing, ne=None) - for mid in resolve_collect_metric_ids(db, p): - task = db.get(BizStateTask, _hf_task_id_for_metric(p, side, mid)) - if (task and mid in _enabled_metric_ids(db, task.id)) or _portrait_metric_items(db, portrait, mid): - continue - _catalog_item_for_metric(vendor=fields["vendor"], device_type=fields["device_type"], metric_id=mid) p.updated_at = utcnow_naive() db.commit() db.refresh(p) @@ -827,9 +721,6 @@ def patch_project(db: Session, project_id: str, body: dict[str, Any]) -> dict[st if any(k in body for k in ("status", "hf_start_at", "hf_end_at")): _apply_hf_window_to_tasks(db, p) db.refresh(p) - if sampling_changed and has_hf: - ensure_highfreq(db, p.id, collect_now=False) - db.refresh(p) return project_to_dict(db, p) @@ -877,13 +768,11 @@ def create_batch(db: Session, project_id: str, body: dict[str, Any]) -> dict[str raise HTTPException(status_code=404, detail="project_not_found") label = str(body.get("batch_label") or "").strip() or "batch" expect = body.get("expect_set") if isinstance(body.get("expect_set"), dict) else {} - status = str(body.get("status") or "pending").strip() - _validate_batch_status(db, project_id, "", status) b = BizMigrationBatch( id=uuid4().hex, project_id=project_id, batch_label=label, - status=status, + status=str(body.get("status") or "pending").strip() or "pending", expect_set_json=dict(expect or {}), note=str(body.get("note") or "")[:500], ) @@ -930,50 +819,29 @@ def run_evaluate( new_batch_id: str = "", acceptance: bool = False, purpose: str = "manual", - _commit: bool = True, - _metric_ids: set[str] | None = None, ) -> dict[str, Any]: mb = db.get(BizMigrationBatch, batch_id) if not mb: raise HTTPException(status_code=404, detail="batch_not_found") - # PostgreSQL serializes evaluate/finish for this project across API workers. - proj = db.query(BizMigrationProject).filter(BizMigrationProject.id == mb.project_id).with_for_update().populate_existing().first() + proj = db.get(BizMigrationProject, mb.project_id) if not proj: raise HTTPException(status_code=404, detail="project_not_found") - db.refresh(mb) - if acceptance and mb.status not in ("active", "review"): - raise HTTPException(status_code=409, detail="monitor_window_closed") - if purpose == "auto" and (mb.status != "active" or proj.status == "done"): - raise HTTPException(status_code=409, detail="monitor_window_closed") if not proj.old_baseline_batch_id: raise HTTPException(status_code=400, detail="old_baseline_required") if not proj.new_baseline_batch_id: raise HTTPException(status_code=400, detail="new_baseline_required") - for side in ("old", "new"): - baseline = db.get(BizStateBatch, getattr(proj, f"{side}_baseline_batch_id")) - if not baseline or baseline.task_id != portrait_task_id(proj, side): - raise HTTPException(status_code=400, detail=f"{side}_baseline_task_mismatch") - if baseline.status not in ("success", "partial"): - raise HTTPException(status_code=400, detail=f"{side}_baseline_not_complete") - pin = (old_batch_id if side == "old" else new_batch_id).strip() - if pin: - pinned = db.get(BizStateBatch, pin) - if not pinned or pinned.task_id not in _all_hf_task_ids(proj, side): - raise HTTPException(status_code=400, detail=f"{side}_current_batch_task_mismatch") old_batch = db.get(BizStateBatch, old_batch_id.strip()) if old_batch_id.strip() else None new_batch = db.get(BizStateBatch, new_batch_id.strip()) if new_batch_id.strip() else None # Primary current batch ids (for run record / pin UX); per-metric may differ when multi-HF if not old_batch: for tid in _all_hf_task_ids(proj, "old") or [current_task_id(proj, "old")]: - old_batch = db.query(BizStateBatch).filter(BizStateBatch.task_id == tid).order_by( - BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() + old_batch = _latest_success_batch(db, tid) if old_batch: break if not new_batch: for tid in _all_hf_task_ids(proj, "new") or [current_task_id(proj, "new")]: - new_batch = db.query(BizStateBatch).filter(BizStateBatch.task_id == tid).order_by( - BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first() + new_batch = _latest_success_batch(db, tid) if new_batch: break if not old_batch: @@ -986,48 +854,11 @@ def run_evaluate( pin_new = new_batch_id.strip() port_map = _port_map_dict(db, proj.mapping_id) - if proj.mapping_id and not db.get(BizPortMapping, proj.mapping_id): - raise HTTPException(status_code=400, detail="mapping_missing") - if len(set(port_map.values())) != len(port_map): - raise HTTPException(status_code=400, detail="mapping_target_ambiguous") expect = parse_expect_set(mb.expect_set_json if isinstance(mb.expect_set_json, dict) else {}) - previous_expect: dict[str, set[str]] = {} - for prior in db.query(BizMigrationBatch).filter( - BizMigrationBatch.project_id == proj.id, BizMigrationBatch.id != mb.id, - BizMigrationBatch.status.in_(("review", "done")), - BizMigrationBatch.created_at <= mb.created_at, - ).all(): - for mid, keys in parse_expect_set(prior.expect_set_json).items(): - previous_expect.setdefault(mid, set()).update(keys) # Final acceptance: window closed → unfinished expect = red window_active = (mb.status == "active") and (not acceptance) - # Evaluations never seed or commit default configuration inside the lock. - mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None - if not mt: - raise HTTPException(status_code=400, detail="monitor_template_missing") - if mt.compare_template_id and not db.get(BizCompareTemplate, mt.compare_template_id): - raise HTTPException(status_code=400, detail="compare_template_missing") + mt = resolve_project_monitor_template(db, proj) sheets, sheet_overrides, defaults, iface_norm = resolve_evaluate_sheets(db, mt) - from .validation import validate_monitor_rules - - validate_monitor_rules(defaults, sheet_overrides) - from ..biz_state.compare_validation import validate_template_body - - validate_template_body({"metrics": sheets, "iface_normalize_rules": iface_norm}) - from ..biz_state.iface_normalize import normalize_iface_name - - normalized_map = {normalize_iface_name(k, iface_norm): normalize_iface_name(v, iface_norm) - for k, v in port_map.items()} - if len(normalized_map) != len(port_map) or len(set(normalized_map.values())) != len(normalized_map): - raise HTTPException(status_code=400, detail="mapping_target_ambiguous") - port_map = normalized_map - known_sheets = {sheet_key(s) for s in sheets} - known_metrics = {str(s.get("metric_id") or "") for s in sheets} - for override in sheet_overrides: - if (override.get("sheet_id") and override["sheet_id"] not in known_sheets) or ( - not override.get("sheet_id") and override.get("metric_id") not in known_metrics - ): - raise HTTPException(status_code=400, detail="monitor_override_sheet_missing") out_of_expect = str((defaults or {}).get("out_of_expect") or "strict").strip().lower() collect_ids = set(resolve_collect_metric_ids(db, proj)) @@ -1035,32 +866,7 @@ def run_evaluate( all_rows: list[dict[str, Any]] = [] metric_batches: dict[str, dict[str, str]] = {} seq = 0 - rows_cache: dict[tuple[str, str], list[dict[str, Any]]] = {} - command_cache: dict[str, dict[str, Any]] = {} - enabled_sheets = [s for s in sheets if str(s.get("metric_id") or "") in collect_ids and not - override_for_sheet(sheet_overrides, sheet_id=sheet_key(s), metric_id=s.get("metric_id", "")).get("skip_dual")] - scope_ids = {str(s.get("metric_id") or "") for s in enabled_sheets} | {sheet_key(s) for s in enabled_sheets} - if any(s.get("iface_fields") for s in enabled_sheets): - scope_ids.add("_ports") - uncovered_scope = sorted({k for scope in (expect, previous_expect) for k, values in scope.items() - if values and k not in scope_ids}) - config_snapshot = {"sheets": sheets, "sheet_overrides": sheet_overrides, - "defaults": defaults, "iface_normalize": iface_norm, "port_map": port_map, - "expect_set": mb.expect_set_json, "previous_expect": {k: sorted(v) for k, v in previous_expect.items()}, - "baseline_ids": [proj.old_baseline_batch_id, proj.new_baseline_batch_id], - "collect_metric_ids": sorted(collect_ids), "window_active": window_active, - "bindings": [_hf_bindings(proj, "old"), _hf_bindings(proj, "new")]} - prior_run = db.query(BizMigrationRun).filter(BizMigrationRun.batch_id == batch_id).order_by( - BizMigrationRun.created_at.desc()).first() if _metric_ids is not None else None - reusable = (prior_run.summary_json or {}).get("config_snapshot") == config_snapshot if prior_run else False - prior_cards = {sheet_key(c): c for c in (prior_run.summary_json or {}).get("sheet_cards", [])} if reusable else {} - pair_cache: dict[str, tuple[BizStateBatch | None, BizStateBatch | None, dict[str, Any]]] = {} - - def metric_rows(bid: str, mid: str) -> list[dict[str, Any]]: - key = (bid, mid) - if key not in rows_cache: - rows_cache[key] = _load_metric_rows(db, batch_id=bid, metric_id=mid) - return rows_cache[key] + verdict_counts: dict[str, int] = {} for sheet in sheets: mid = str(sheet.get("metric_id") or "").strip() @@ -1074,9 +880,6 @@ def run_evaluate( field_rules = list(sheet.get("field_rules") or []) sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid) if sheet_ov.get("skip_dual"): - sheet_cards.append({"metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), - "progress_ok": 0, "progress_total": 0, "anomaly": 0, - "collect_skipped": True}) continue # Metric not in HF collect set → skip dual eval (avoid false red) @@ -1099,17 +902,8 @@ def run_evaluate( ) continue - if _metric_ids is not None and mid not in _metric_ids and sid in prior_cards: - cached = {**prior_cards[sid], "source_run_id": prior_cards[sid].get("source_run_id") or prior_run.id} - times = [_parse_dt(cached.get(k)) for k in ("old_collected_at", "new_collected_at")] - if any(at and (utcnow_naive() - at).total_seconds() > cached.get("age_limit_sec", 180) for at in times): - cached.update({"current_missing": True, "sample_issues": ["sample_stale"]}) - sheet_cards.append(cached) - metric_batches[mid] = (prior_run.summary_json or {}).get("metric_batches", {}).get(mid, {}) - continue - if mid not in pair_cache: - pair_cache[mid] = _current_pair_for_metric(db, proj, mid, pin_old, pin_new) - old_cur_batch, new_cur_batch, pair_meta = pair_cache[mid] + old_cur_batch = _current_batch_for_metric(db, proj, "old", mid, pinned_batch_id=pin_old) + new_cur_batch = _current_batch_for_metric(db, proj, "new", mid, pinned_batch_id=pin_new) # Do NOT fall back to another HF task's batch — empty/wrong metric → false red if not old_cur_batch or not new_cur_batch: sheet_cards.append( @@ -1128,37 +922,26 @@ def run_evaluate( "current_missing": True, "collect_incomplete": True, "collect_skipped": False, - **pair_meta, } ) continue old_cur_mid = old_cur_batch.id new_cur_mid = new_cur_batch.id - timing = {**pair_meta, **_sample_timing(proj, mid, old_cur_batch, new_cur_batch, - check_age=acceptance or not (pin_old and pin_new))} - baseline_missing = [side for side in ("old", "new") if not _metric_sample_ok( - db, db.get(BizStateBatch, getattr(proj, f"{side}_baseline_batch_id")), mid)] - if timing["sample_issues"] or baseline_missing: - sheet_cards.append({"metric_id": mid, "sheet_id": sid, "title": sheet_title(sheet), - "progress_ok": 0, "progress_total": 0, "anomaly": 0, - "current_missing": True, "collect_incomplete": True, - "baseline_missing_sides": baseline_missing, **timing}) - continue old_base = apply_row_filters( - metric_rows(proj.old_baseline_batch_id, mid), + _load_metric_rows(db, batch_id=proj.old_baseline_batch_id, metric_id=mid), row_filters, ) old_now = apply_row_filters( - metric_rows(old_cur_mid, mid), + _load_metric_rows(db, batch_id=old_cur_mid, metric_id=mid), row_filters, ) new_base_rows = apply_row_filters( - metric_rows(proj.new_baseline_batch_id, mid), + _load_metric_rows(db, batch_id=proj.new_baseline_batch_id, metric_id=mid), row_filters, ) new_now = apply_row_filters( - metric_rows(new_cur_mid, mid), + _load_metric_rows(db, batch_id=new_cur_mid, metric_id=mid), row_filters, ) @@ -1180,19 +963,7 @@ def run_evaluate( sheet_override=sheet_ov, out_of_expect=out_of_expect, iface_normalize_rules=iface_norm, - previous_expect=previous_expect, - allow_mapping_scope=False, ) - # A target snapshot taken before the source's disappearance cannot prove - # loss. Wait for the next target receipt; final acceptance stays blocked. - awaiting = 0 - if (new_cur_batch.ended_at or new_cur_batch.started_at) < old_cur_batch.started_at: - for row in one["rows"]: - if row["in_expect"] and row["rule_hit"] == "service_not_received": - row.update(verdict="awaiting_peer", color="gray", rule_hit="target_sample_predates_source") - one["anomaly"] -= 1 - one["anomaly_in_expect"] -= 1 - awaiting += 1 old_tid = _hf_task_id_for_metric(proj, "old", mid) or current_task_id(proj, "old") new_tid = _hf_task_id_for_metric(proj, "new", mid) or current_task_id(proj, "new") old_task_brief = _task_brief(db, old_tid) @@ -1202,9 +973,6 @@ def run_evaluate( "new_batch_id": new_cur_mid, "old_task_id": old_tid, "new_task_id": new_tid, - "old_status": old_cur_batch.status, - "new_status": new_cur_batch.status, - **timing, } sheet_cards.append( { @@ -1215,7 +983,6 @@ def run_evaluate( "progress_total": one["progress_total"], "anomaly": one["anomaly"], "anomaly_in_expect": int(one.get("anomaly_in_expect") or 0), - "steady_outside": int(one.get("steady_outside") or 0), "old_summary": one["old_summary"], "new_summary": one["new_summary"], "new_baseline_mode": one.get("new_baseline_mode") or "provided", @@ -1226,10 +993,6 @@ def run_evaluate( "duplicate_key_list": list(one.get("duplicate_key_list") or []), "duplicate_keys_before": int(one.get("duplicate_keys_before") or 0), "duplicate_keys_after": int(one.get("duplicate_keys_after") or 0), - **timing, - "awaiting_peer": awaiting, - "verdict_counts": dict(Counter(r["verdict"] for r in one["rows"] if r.get("verdict"))), - "expected_count": sum(bool(r.get("in_expect")) for r in one["rows"]), } ) for r in one["rows"]: @@ -1240,18 +1003,15 @@ def run_evaluate( new_batch_id=new_cur_mid, old_task=old_task_brief, new_task=new_task_brief, - command_cache=command_cache, ) r["seq"] = seq r["sheet_id"] = sid seq += 1 all_rows.append(r) + v = str(r.get("verdict") or "") + if v: + verdict_counts[v] = verdict_counts.get(v, 0) + 1 - verdict_counts = {} - for card in sheet_cards: - if not card.get("current_missing"): - for verdict, count in card.get("verdict_counts", {}).items(): - verdict_counts[verdict] = verdict_counts.get(verdict, 0) + count active_cards = [ c for c in sheet_cards if not c.get("collect_skipped") and not c.get("current_missing") ] @@ -1292,12 +1052,6 @@ def run_evaluate( if c.get("collect_skipped") ], "verdict_counts": verdict_counts, - "anomaly_outside_expect": sum(int(c.get("anomaly") or 0) - int(c.get("anomaly_in_expect") or 0) for c in active_cards), - "steady_outside": sum(int(c.get("steady_outside") or 0) for c in active_cards), - "duplicate_keys": sum(int(c.get("duplicate_keys_before") or 0) + int(c.get("duplicate_keys_after") or 0) for c in active_cards), - "uncovered_scope": uncovered_scope, - "coverage_complete": bool(enabled_sheets) and not uncovered_scope and not any(c.get("current_missing") or c.get("awaiting_peer") or c.get("duplicate_keys_before") or c.get("duplicate_keys_after") for c in sheet_cards if not c.get("collect_skipped")), - "config_snapshot": config_snapshot, "window_active": window_active, "expect_ports": sorted(expect.get("_ports") or ()), "old_current_task_id": current_task_id(proj, "old"), @@ -1307,10 +1061,6 @@ def run_evaluate( ) db.add(run) db.flush() - for card in sheet_cards: - card.setdefault("source_run_id", run.id) - run.summary_json = {**run.summary_json, "sheet_cards": sheet_cards} - pending_diffs: list[dict[str, Any]] = [] for r in all_rows: key_list = r.get("key") or [] ev = r.get("evidence") if isinstance(r.get("evidence"), dict) else {} @@ -1329,8 +1079,8 @@ def run_evaluate( str(r.get("metric_id") or ""), ] ) - pending_diffs.append( - dict( + db.add( + BizMigrationDiff( id=uuid4().hex, run_id=run.id, metric_id=str(r.get("metric_id") or ""), @@ -1348,8 +1098,6 @@ def run_evaluate( "old_status": r.get("old_status"), "new_status": r.get("new_status"), "rule_hit": r.get("rule_hit") or "", - "integrity_changes": r.get("integrity_changes") or [], - "previous_batch": bool(r.get("previous_batch")), "sheet_id": r.get("sheet_id") or "", "evidence": r.get("evidence") or {}, }, @@ -1361,15 +1109,8 @@ def run_evaluate( search_text=search[:2000], ) ) - if len(pending_diffs) >= 500: - db.execute(insert(BizMigrationDiff), pending_diffs) - pending_diffs.clear() - if pending_diffs: - db.execute(insert(BizMigrationDiff), pending_diffs) - db.flush() - if _commit: - db.commit() - db.refresh(run) + db.commit() + db.refresh(run) return run_to_dict(db, run) @@ -1406,12 +1147,7 @@ def _command_brief(db: Session, command_id: str) -> dict[str, Any]: cid = str(command_id or "").strip() if not cid: return {} - # Select metadata only: raw CLI output can be several MB per command. - c = db.query(BizStateBatchCommand.id, BizStateBatchCommand.raw_command, - BizStateBatchCommand.parse_status, BizStateBatchCommand.row_count, - BizStateBatchCommand.profile_id, BizStateBatchCommand.parser_id, - BizStateBatchCommand.metric_id, BizStateBatchCommand.message).filter( - BizStateBatchCommand.id == cid).first() + c = db.get(BizStateBatchCommand, cid) if not c: return {"command_id": cid} return { @@ -1434,7 +1170,6 @@ def _enrich_row_evidence( new_batch_id: str, old_task: dict[str, Any] | None, new_task: dict[str, Any] | None, - command_cache: dict[str, dict[str, Any]] | None = None, ) -> None: """Attach device / collect / show-command onto evaluate evidence (in-place).""" ev = dict(row.get("evidence") or {}) @@ -1444,11 +1179,7 @@ def _enrich_row_evidence( ): side_ev = dict(ev.get(side) or {}) netx = dict(side_ev.get("netx") or {}) - cid = str(netx.get("batch_command_id") or "") - cache = command_cache if command_cache is not None else {} - if cid not in cache: - cache[cid] = _command_brief(db, cid) - cmd = cache[cid] + cmd = _command_brief(db, str(netx.get("batch_command_id") or "")) side_ev["device"] = { "ne_id": str(brief.get("ne_id") or netx.get("ne_id") or ""), "ne_name": str(brief.get("ne_name") or ""), @@ -1490,8 +1221,6 @@ def diff_to_dict(d: BizMigrationDiff) -> dict[str, Any]: "old_status": kj.get("old_status") or "", "new_status": kj.get("new_status") or "", "rule_hit": kj.get("rule_hit") or "", - "integrity_changes": kj.get("integrity_changes") or [], - "previous_batch": bool(kj.get("previous_batch")), "evidence": dict(kj.get("evidence") or {}), "old_kind": d.old_kind, "new_kind": d.new_kind, @@ -1548,11 +1277,6 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] collect_ids = set(resolve_collect_metric_ids(db, p)) port_map = _port_map_dict(db, p.mapping_id) out_sheets: list[dict[str, Any]] = [] - from ..biz_state.iface_normalize import apply_iface_normalize, resolve_mapped_iface, normalize_iface_name - from .evaluate import KEY_SEP - - rows_cache: dict[str, list[dict[str, Any]]] = {} - port_map = {normalize_iface_name(k, _norm): normalize_iface_name(v, _norm) for k, v in port_map.items()} for sheet in sheets: mid = str(sheet.get("metric_id") or "").strip() sid = sheet_key(sheet) @@ -1562,39 +1286,29 @@ def list_baseline_expect_objects(db: Session, project_id: str) -> dict[str, Any] # Only offer expect keys for metrics that are actually HF-collected if collect_ids and mid not in collect_ids: continue - if mid in MANUAL_ROUTE_METRICS: - # Never enumerate million-row route tables into batch scope pickers. - continue sheet_ov = override_for_sheet(sheet_overrides, sheet_id=sid, metric_id=mid) if sheet_ov.get("skip_dual"): continue iface_fields = [str(k) for k in (sheet.get("iface_fields") or []) if str(k).strip()] - if mid not in rows_cache: - rows_cache[mid] = _load_metric_rows(db, batch_id=p.old_baseline_batch_id, metric_id=mid) rows_raw = apply_row_filters( - rows_cache[mid], + _load_metric_rows(db, batch_id=p.old_baseline_batch_id, metric_id=mid), list(sheet.get("row_filters") or []), ) items: list[dict[str, Any]] = [] - seen_keys: set[str] = set() for r in rows_raw: - normalized = apply_iface_normalize(dict(r), iface_fields=iface_fields, rules=_norm) - keys = [scalar_text(normalized.get(f)).strip() for f in key_fields] + keys = [str(r.get(f) or "").strip() for f in key_fields] if not any(keys): continue - key_str = KEY_SEP.join(keys) - if key_str in seen_keys: - continue - seen_keys.add(key_str) + key_str = "|".join(keys) mapped = "" if iface_fields and key_fields and key_fields[0] in iface_fields: - mapped = resolve_mapped_iface(keys[0], port_map) if port_map else "" + mapped = port_map.get(keys[0]) or "" items.append( { "key": key_str, "keys": keys, "mapped_to": mapped, - "label": "|".join(scalar_text(r.get(f)).strip() for f in key_fields), + "label": key_str, "row": {f: r.get(f) for f in list(dict.fromkeys([*key_fields, *iface_fields, "description", "admin", "phy", "prot"])) if f in r}, } ) @@ -1623,18 +1337,20 @@ def _catalog_item_for_metric(*, vendor: str, device_type: str, metric_id: str) - mid = str(metric_id or "").strip() vkey = resolve_vendor_key(vendor, device_type) - candidates = [p for p in profiles_for_vendor(vkey) if p.metric_id == mid and p.kind == "collect"] - for p in sorted(candidates, key=lambda p: bool(p.placeholders)): - if not any(getattr(ph, "required", True) for ph in (p.placeholders or [])): + for p in profiles_for_vendor(vkey): + if p.metric_id == mid and p.kind == "collect": + if getattr(p, "placeholders", None): + ph = ",".join(str(x.name) for x in (p.placeholders or []) if getattr(x, "name", None)) + raise HTTPException( + status_code=400, + detail=f"metric_needs_bindings:{mid}:{ph or 'required'}", + ) return { "source_profile_id": p.profile_id, "kind": "catalog", "enabled": True, "title": p.title or mid, } - if candidates: - ph = ",".join(x.name for x in candidates[0].placeholders if getattr(x, "required", True)) - raise HTTPException(status_code=400, detail=f"metric_needs_bindings:{mid}:{ph}") raise HTTPException( status_code=400, detail=f"no_profile_for_metric:{mid}:{vkey or vendor or 'unknown'}", @@ -1738,26 +1454,6 @@ def _template_from_task_or_ne( raise HTTPException(status_code=400, detail="old_new_ne_or_task_required") -def _portrait_metric_items(db: Session, portrait: BizStateTask | None, mid: str) -> list[dict[str, Any]]: - """Preserve CLI variants and bindings already used by the device baseline.""" - from ..biz_state.profiles import get_profile - from ..models import BizStateTaskItem, BizStateTaskItemBinding - - if not portrait: - return [] - out = [] - for item in db.query(BizStateTaskItem).filter(BizStateTaskItem.task_id == portrait.id, - BizStateTaskItem.enabled.is_(True)).order_by(BizStateTaskItem.sort_order).all(): - profile = get_profile(item.source_profile_id) if item.kind == "catalog" else None - if not profile or profile.metric_id != mid: - continue - bindings = db.query(BizStateTaskItemBinding).filter(BizStateTaskItemBinding.item_id == item.id).all() - out.append({"source_profile_id": item.source_profile_id, "kind": item.kind, - "enabled": True, "title": item.title, "command_override": item.command_override, - "bindings": [{"placeholder": b.placeholder, "value": b.value} for b in bindings]}) - return out - - def _ensure_side_highfreq( db: Session, *, @@ -1768,7 +1464,6 @@ def _ensure_side_highfreq( retention_days: int, metric_ids: list[str], status: str = "running", - configured_items: list[dict[str, Any]] | None = None, ) -> tuple[BizStateTask, bool]: """Return (hf_task, created). Reuse existing HF if metrics match; never mutate portrait.""" from ..biz_state import service as biz_svc @@ -1776,9 +1471,6 @@ def _ensure_side_highfreq( want = {str(m).strip() for m in metric_ids if str(m).strip()} if not want: want = {PORT_METRIC_ID} - if existing_hf and existing_hf.collect_running and _enabled_metric_ids(db, existing_hf.id) != want: - # A running collection owns its item configuration until completion. - existing_hf = None if existing_hf and _is_highfreq_task(db, existing_hf, want): patch: dict[str, Any] = { @@ -1795,7 +1487,7 @@ def _ensure_side_highfreq( # Existing HF (purpose already cutover_hf) with wrong metrics → update items in place. # Never rewrite a task that only "looks like" HF by note — create a sibling instead. if existing_hf and str(getattr(existing_hf, "purpose", None) or "").strip() == PURPOSE_CUTOVER_HF: - items = configured_items or [ + items = [ _catalog_item_for_metric( vendor=str(template_fields.get("vendor") or existing_hf.vendor), device_type=str(template_fields.get("device_type") or existing_hf.device_type), @@ -1819,7 +1511,7 @@ def _ensure_side_highfreq( refreshed = db.get(BizStateTask, existing_hf.id) return refreshed or existing_hf, False - items = configured_items or [ + items = [ _catalog_item_for_metric( vendor=str(template_fields.get("vendor") or ""), device_type=str(template_fields.get("device_type") or ""), @@ -1873,12 +1565,11 @@ def _apply_hf_window_to_tasks(db: Session, proj: BizMigrationProject) -> None: from ..biz_state import service as biz_svc log = logging.getLogger("netx.biz_migration.hf_window") - window_status = _hf_window_status(proj) + want = _hf_window_status(proj) for tid in _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new"): task = db.get(BizStateTask, tid) if not task: continue - want = "paused" if task.collect_manual_only else window_status purpose = str(getattr(task, "purpose", None) or "").strip() if purpose and purpose != PURPOSE_CUTOVER_HF: continue @@ -1926,7 +1617,9 @@ def ensure_highfreq( old_ne: dict[str, Any] | None = None, new_ne: dict[str, Any] | None = None, ) -> dict[str, Any]: - """One task per metric/device, paired by project; route tables are manual-only.""" + """Create/bind HF tasks (one per interval group) — never overwrites portrait ids.""" + from ..biz_state.collect_runner import dispatch_collect + proj = db.get(BizMigrationProject, project_id) if not proj: raise HTTPException(status_code=404, detail="project_not_found") @@ -1940,28 +1633,33 @@ def ensure_highfreq( if not metric_ids: metric_ids = [PORT_METRIC_ID] by_metric = _metric_interval_map(proj) - groups = [(max(60, by_metric.get(mid) or iv_default), [mid]) for mid in metric_ids] + groups = _group_metrics_by_interval(metric_ids, default_interval=iv_default, by_metric=by_metric) want_status = _hf_window_status(proj) old_portrait = db.get(BizStateTask, proj.old_task_id) if proj.old_task_id else None new_portrait = db.get(BizStateTask, proj.new_task_id) if proj.new_task_id else None - # Reuse a single-metric binding; a legacy combined task can be assigned - # once, while the remaining metrics receive independent sibling tasks. - def _existing_by_metric(side: str) -> dict[str, BizStateTask]: - out: dict[str, BizStateTask] = {} - for binding in _hf_bindings(proj, side): - task = db.get(BizStateTask, binding["task_id"]) - if task: - mids = binding.get("metric_ids") or metric_ids - for mid in mids: - if mid in metric_ids: - out.setdefault(mid, task) - break + # Reuse existing bindings by interval when possible + def _existing_by_interval(side: str) -> dict[int, BizStateTask]: + out: dict[int, BizStateTask] = {} + for b in _hf_bindings(proj, side): + tid = str(b.get("task_id") or "").strip() + t = db.get(BizStateTask, tid) if tid else None + if not t: + continue + try: + iv = max(60, int(b.get("interval_sec") or t.interval_sec or 60)) + except (TypeError, ValueError): + iv = 60 + out[iv] = t + # legacy primary slot + primary = db.get(BizStateTask, hf_task_id(proj, side)) if hf_task_id(proj, side) else None + if primary and iv_default not in out: + out[iv_default] = primary return out - old_existing = _existing_by_metric("old") - new_existing = _existing_by_metric("new") + old_existing = _existing_by_interval("old") + new_existing = _existing_by_interval("new") old_prev_ids = set(_all_hf_task_ids(proj, "old")) new_prev_ids = set(_all_hf_task_ids(proj, "new")) @@ -1984,51 +1682,33 @@ def ensure_highfreq( new_any_created = False for g_iv, g_metrics in groups: - mid = g_metrics[0] - gid = "cutover:" + proj.id + ":" + hashlib.sha256(mid.encode()).hexdigest()[:16] - old_candidate, new_candidate = old_existing.get(mid), new_existing.get(mid) - if old_candidate and old_candidate.collect_group_id not in ("", gid): - old_candidate = None - if new_candidate and new_candidate.collect_group_id not in ("", gid): - new_candidate = None old_task, old_created = _ensure_side_highfreq( db, - existing_hf=old_candidate, + existing_hf=old_existing.get(g_iv), template_fields=old_fields, project_name=proj.name, interval_sec=g_iv, retention_days=ret, metric_ids=g_metrics, - status="paused", - configured_items=_portrait_metric_items(db, old_portrait, mid), + status=want_status, ) new_task, new_created = _ensure_side_highfreq( db, - existing_hf=new_candidate, + existing_hf=new_existing.get(g_iv), template_fields=new_fields, project_name=proj.name, interval_sec=g_iv, retention_days=ret, metric_ids=g_metrics, - status="paused", - configured_items=_portrait_metric_items(db, new_portrait, mid), + status=want_status, ) - if old_task.ne_id == new_task.ne_id: - raise HTTPException(status_code=400, detail="collect_pair_same_device") - for task in (old_task, new_task): - task.collect_group_id = gid - task.collect_manual_only = mid in MANUAL_ROUTE_METRICS - task.status = "paused" if task.collect_manual_only else want_status - db.commit() old_any_created = old_any_created or old_created new_any_created = new_any_created or new_created old_bindings.append( - {"task_id": old_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv, - "collect_group_id": gid, "manual_only": mid in MANUAL_ROUTE_METRICS} + {"task_id": old_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv} ) new_bindings.append( - {"task_id": new_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv, - "collect_group_id": gid, "manual_only": mid in MANUAL_ROUTE_METRICS} + {"task_id": new_task.id, "metric_ids": list(g_metrics), "interval_sec": g_iv} ) proj = db.get(BizMigrationProject, project_id) @@ -2052,24 +1732,27 @@ def ensure_highfreq( continue if str(getattr(t, "purpose", None) or "").strip() not in ("", PURPOSE_CUTOVER_HF): continue - if t.collect_group_id and not t.collect_group_id.startswith("cutover:" + proj.id + ":"): - continue - t.collect_group_id = "" - t.collect_manual_only = False - db.commit() if t.status == "running": try: biz_svc.update_task(db, tid, {"status": "paused"}) except Exception: # noqa: BLE001 pass - if t.collect_running: - from ..biz_state.collect_stop import request_stop_collect - - request_stop_collect(tid) collect: dict[str, Any] = {"old": [], "new": []} - if collect_now: - collect = collect_project_now(db, project_id) + if collect_now and want_status == "running": + for side, binds in (("old", old_bindings), ("new", new_bindings)): + for b in binds: + tid = b["task_id"] + try: + dispatch_collect(tid) + collect[side].append({"ok": True, "task_id": tid}) + except Exception as exc: # noqa: BLE001 + collect[side].append({"ok": False, "task_id": tid, "error": str(exc)[:200]}) + elif collect_now and want_status != "running": + collect = { + "old": [{"ok": False, "error": "hf_window_inactive"}], + "new": [{"ok": False, "error": "hf_window_inactive"}], + } return { "project": project_to_dict(db, proj), @@ -2112,33 +1795,26 @@ def collect_project_now(db: Session, project_id: str) -> dict[str, Any]: } out: dict[str, Any] = {"old": [], "new": [], "hf_status": want_status} - old_bindings, new_bindings = _hf_bindings(proj, "old"), _hf_bindings(proj, "new") - handled: set[str] = set() - for old_binding in old_bindings: - gid = old_binding.get("collect_group_id") - if not gid or gid in handled: + for side in ("old", "new"): + tids = _all_hf_task_ids(proj, side) + if not tids: + out[side] = [{"ok": False, "error": "hf_task_missing"}] continue - handled.add(gid) - new_binding = next((b for b in new_bindings if b.get("collect_group_id") == gid), None) - if not new_binding: - result = {"ok": False, "error": "collect_pair_invalid"} - else: - result = dispatch_collect(old_binding["task_id"], manual=True) - result = {**result, "ok": bool(result.get("queued")), "error": result.get("reason", "")} - for side, binding in (("old", old_binding), ("new", new_binding)): - out[side].append({**result, "task_id": binding["task_id"] if binding else "", - "metric_ids": old_binding.get("metric_ids", [])}) - # Legacy projects remain operable until their bindings are upgraded. - if not handled: - for side in ("old", "new"): - tids = _all_hf_task_ids(proj, side) - if not tids: - out[side] = [{"ok": False, "error": "hf_task_missing"}] - for tid in tids: - result = dispatch_collect(tid, manual=True) - out[side].append({**result, "ok": bool(result.get("queued")), "task_id": tid, - "error": result.get("reason", "")}) - + for tid in tids: + task = db.get(BizStateTask, tid) + if not task: + out[side].append({"ok": False, "error": "task_not_found", "task_id": tid}) + continue + if bool(task.collect_running): + out[side].append( + {"ok": False, "error": "collect_already_running", "task_id": tid} + ) + continue + try: + dispatch_collect(tid) + out[side].append({"ok": True, "task_id": tid}) + except Exception as exc: # noqa: BLE001 + out[side].append({"ok": False, "task_id": tid, "error": str(exc)[:200]}) return out @@ -2246,30 +1922,26 @@ def _persist_red_tickets_from_run( ) .all() ) - by_id: dict[tuple[str, str, str], BizMigrationRedTicket] = {} + by_id: dict[tuple[str, str], BizMigrationRedTicket] = {} for t in existing: - detail = t.detail_json if isinstance(t.detail_json, dict) else {} mk = str(getattr(t, "match_key_str", "") or "").strip() if not mk: detail = t.detail_json if isinstance(t.detail_json, dict) else {} mk = str(detail.get("match_old_key") or t.key_str or "").strip() if mk: - by_id[(str(t.metric_id or ""), str(detail.get("sheet_id") or t.metric_id or ""), - str(detail.get("match_old_key") or mk))] = t + by_id[(str(t.metric_id or ""), mk)] = t created_or_updated: list[BizMigrationRedTicket] = [] - seen: set[tuple[str, str, str]] = set() + seen: set[tuple[str, str]] = set() for d in diffs: kj = d.key_json if isinstance(d.key_json, dict) else {} mid = str(d.metric_id or "") match_key = str( kj.get("match_old_key") or kj.get("key_str") or kj.get("old_key") or "" - ).strip() + ).strip()[:256] if not match_key: match_key = str(kj.get("key_str") or "")[:256] - ident = (mid, str(kj.get("sheet_id") or mid), match_key) - from hashlib import sha256 - stored_key = match_key if len(match_key) <= 256 else "sha256:" + sha256(match_key.encode()).hexdigest() + ident = (mid, match_key) detail = { "old_kind": d.old_kind, "new_kind": d.new_kind, @@ -2280,8 +1952,6 @@ def _persist_red_tickets_from_run( "new_key": kj.get("new_key") or kj.get("new_key_str") or "", "match_old_key": kj.get("match_old_key") or match_key, "match_new_key": kj.get("match_new_key") or "", - "sheet_id": kj.get("sheet_id") or mid, - "integrity_changes": kj.get("integrity_changes") or [], "evidence": dict(kj.get("evidence") or {}), } prev = by_id.get(ident) @@ -2290,7 +1960,7 @@ def _persist_red_tickets_from_run( prev.run_id = run_id prev.key_str = str(kj.get("key_str") or "")[:256] prev.new_key_str = str(kj.get("new_key_str") or "")[:256] - prev.match_key_str = stored_key + prev.match_key_str = match_key prev.verdict = d.verdict prev.color = d.color or "red" prev.old_status = str(kj.get("old_status") or "")[:64] @@ -2310,7 +1980,7 @@ def _persist_red_tickets_from_run( metric_id=mid, key_str=str(kj.get("key_str") or "")[:256], new_key_str=str(kj.get("new_key_str") or "")[:256], - match_key_str=stored_key, + match_key_str=match_key, verdict=d.verdict, color=d.color or "red", old_status=str(kj.get("old_status") or "")[:64], @@ -2344,9 +2014,14 @@ def finish_batch( detail=f"batch_not_finishable:{mb.status}", ) - # acceptance=True closes the logical window. Persist batch closure only after - # evaluation succeeds, so an API/collection failure keeps the batch operable. - run = run_evaluate(db, batch_id=batch_id, acceptance=True, purpose="acceptance", _commit=False) + # Close window first so acceptance uses non-active rules + if mb.status == "active": + mb.ended_at = utcnow_naive() + mb.status = "done" if mark_done else "review" + mb.updated_at = utcnow_naive() + db.commit() + + run = run_evaluate(db, batch_id=batch_id, acceptance=True, purpose="acceptance") summary = dict(run.get("summary") or {}) progress = dict(summary.get("progress") or {}) anomaly = int(summary.get("anomaly") or 0) @@ -2361,10 +2036,6 @@ def finish_batch( fail_reasons.append("new_baseline_missing") if current_missing: fail_reasons.append("current_missing") - if not summary.get("coverage_complete", False): - fail_reasons.append("coverage_incomplete") - if summary.get("duplicate_keys"): - fail_reasons.append("duplicate_keys") if anomaly: fail_reasons.append("anomaly") if total > 0 and ok < total: @@ -2390,9 +2061,6 @@ def finish_batch( mb = db.get(BizMigrationBatch, batch_id) if not mb: raise HTTPException(status_code=404, detail="batch_not_found") - if mb.status == "active": - mb.ended_at = utcnow_naive() - mb.status = "done" if mark_done else "review" mb.accept_run_id = str(run.get("id") or "") mb.accept_status = "passed" if passed else "failed" mb.accept_summary_json = accept_summary @@ -2418,22 +2086,10 @@ def finish_batch( } -def _validate_batch_status(db: Session, project_id: str, batch_id: str, status: str) -> None: - if status not in ("pending", "active", "review", "done"): - raise HTTPException(status_code=400, detail="batch_status_invalid") - if status == "active": - db.query(BizMigrationProject).filter(BizMigrationProject.id == project_id).with_for_update().first() - if db.query(BizMigrationBatch.id).filter(BizMigrationBatch.project_id == project_id, - BizMigrationBatch.id != batch_id, BizMigrationBatch.status == "active").first(): - raise HTTPException(status_code=409, detail="another_batch_active") - - def patch_batch(db: Session, batch_id: str, body: dict[str, Any]) -> dict[str, Any]: b = db.get(BizMigrationBatch, batch_id) if not b: raise HTTPException(status_code=404, detail="batch_not_found") - if body.get("status") is not None: - _validate_batch_status(db, b.project_id, b.id, str(body["status"]).strip()) if "batch_label" in body and body["batch_label"] is not None: b.batch_label = str(body["batch_label"]).strip() or b.batch_label if "note" in body and body["note"] is not None: @@ -2479,44 +2135,30 @@ def list_run_diffs( run_id: str, *, metric_id: str = "", - sheet_id: str = "", - only_expect: bool = False, verdict: str = "", color: str = "", kw: str = "", offset: int = 0, limit: int = 100, ) -> dict[str, Any]: - run = db.get(BizMigrationRun, run_id) - if not run: + if not db.get(BizMigrationRun, run_id): raise HTTPException(status_code=404, detail="run_not_found") - sources = [and_(BizMigrationDiff.run_id == c["source_run_id"], - BizMigrationDiff.key_json["sheet_id"].as_string() == sheet_key(c)) - for c in (run.summary_json or {}).get("sheet_cards", []) - if c.get("source_run_id") and c["source_run_id"] != run_id - and not (only_expect and c.get("expected_count") == 0) - and (not metric_id or c.get("metric_id") == metric_id) - and (not sheet_id or sheet_key(c) == sheet_id)] - q = db.query(BizMigrationDiff).filter(or_(BizMigrationDiff.run_id == run_id, *sources)) + q = db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id == run_id) if metric_id: q = q.filter(BizMigrationDiff.metric_id == metric_id) - if sheet_id: - q = q.filter(BizMigrationDiff.key_json["sheet_id"].as_string() == sheet_id) - if only_expect: - q = q.filter(BizMigrationDiff.in_expect.is_(True)) if verdict: q = q.filter(BizMigrationDiff.verdict == verdict) if color: q = q.filter(BizMigrationDiff.color == color) if kw.strip(): - literal = kw.strip().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - q = q.filter(BizMigrationDiff.search_text.ilike(f"%{literal}%", escape="\\")) + like = f"%{kw.strip()}%" + q = q.filter(BizMigrationDiff.search_text.ilike(like)) total = q.count() - rows = q.order_by(BizMigrationDiff.metric_id.asc(), BizMigrationDiff.seq.asc(), BizMigrationDiff.id.asc()).offset(max(0, offset)).limit(min(500, max(1, limit))).all() + rows = q.order_by(BizMigrationDiff.seq.asc()).offset(max(0, offset)).limit(min(500, max(1, limit))).all() return { "total": total, - "offset": max(0, offset), - "limit": min(500, max(1, limit)), + "offset": offset, + "limit": limit, "items": [diff_to_dict(d) for d in rows], } @@ -2554,35 +2196,9 @@ def board(db: Session, batch_id: str, run_id: str = "") -> dict[str, Any]: "project": proj, "batch": batch_to_dict(mb), "run": run_to_dict(db, run) if run else None, - "sampling": sampling_state(db, db.get(BizMigrationProject, mb.project_id)) if not run_id else [], } -def sampling_state(db: Session, proj: BizMigrationProject) -> list[dict[str, Any]]: - """Small live read independent of evaluation and route row counts.""" - out = [] - for binding in _hf_bindings(proj, "old"): - gid = binding.get("collect_group_id") - if not gid: - continue - latest = db.query(BizStateBatch).filter(BizStateBatch.collect_group_id == gid).order_by( - BizStateBatch.queued_at.desc(), BizStateBatch.started_at.desc()).first() - peers = db.query(BizStateBatch).filter(BizStateBatch.collect_round_id == latest.collect_round_id).all() if latest else [] - old = next((b for b in peers if b.task_id == binding["task_id"]), None) - new = next((b for b in peers if b.task_id != binding["task_id"]), None) - card: dict[str, Any] = {"metric_id": (binding.get("metric_ids") or [""])[0], - "collect_group_id": gid, "collect_round_id": latest.collect_round_id if latest else "", - "manual_only": binding.get("manual_only", False), "interval_sec": binding["interval_sec"]} - for side, batch in (("old", old), ("new", new)): - card[f"{side}_collect_status"] = batch.status if batch else "idle" - card[f"{side}_duration_sec"] = max(0, ((batch.ended_at or utcnow_naive()) - batch.started_at).total_seconds()) if batch and batch.status != "queued" else 0 - card[f"{side}_queue_sec"] = max(0, ((utcnow_naive() if batch.status == "queued" else batch.started_at) - (batch.queued_at or batch.started_at)).total_seconds()) if batch else 0 - if old and new and all(b.status != "queued" for b in (old, new)): - card["start_skew_sec"] = abs((old.started_at - new.started_at).total_seconds()) - out.append(card) - return out - - def get_monitor_context( db: Session, *, diff --git a/netx_api/biz_migration/validation.py b/netx_api/biz_migration/validation.py deleted file mode 100644 index 881a3b3..0000000 --- a/netx_api/biz_migration/validation.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Validate monitor correction rules before they can affect live verdicts.""" -from typing import Any - -from fastapi import HTTPException - -from .evaluate import _groups_for_side, _normalize_cond_type - - -def validate_monitor_rules(defaults: Any, overrides: Any) -> None: - def fail(path: str) -> None: - raise HTTPException(status_code=400, detail=f"monitor_rule_invalid:{path}") - - if not isinstance(defaults, dict) or not isinstance(overrides, list): - fail("defaults/sheet_overrides") - if defaults.get("out_of_expect", "strict") not in ("strict", "warn", "ignore"): - fail("defaults.out_of_expect") - seen = set() - for i, ov in enumerate(overrides): - path = f"sheet_overrides[{i}]" - if not isinstance(ov, dict) or not (ov.get("sheet_id") or ov.get("metric_id")): - fail(path) - ident = ov.get("sheet_id") or ov.get("metric_id") - if ident in seen: - fail(path + ".duplicate") - seen.add(ident) - for key in ("status_fields", "up_values", "down_values"): - if key in ov and (not isinstance(ov[key], list) or any(not str(v).strip() for v in ov[key])): - fail(path + "." + key) - for key in ("success", "anomaly"): - patterns = ov.get(key, []) - if not isinstance(patterns, list): - fail(path + "." + key) - for j, pat in enumerate(patterns): - pp = f"{path}.{key}[{j}]" - if not isinstance(pat, dict): - fail(pp) - for side in ("old", "new"): - for attr in (side, side + "_conds", side + "_groups"): - if attr in pat and not isinstance(pat[attr], list): - fail(pp + "." + attr) - if any(isinstance(g, list) and not g for g in pat.get(side + "_groups", [])): - fail(pp + "." + side + "_groups.empty_group") - groups = _groups_for_side(pat, side) - if key == "success" and not groups: - fail(pp + "." + side + ".required") - for group in groups: - for cond in group: - cp = pp + "." + side - if isinstance(cond, str): - if cond not in ("added", "removed", "changed", "unchanged", "up", "down", "other", "none") and not (cond.startswith("field:") and cond.count(":") >= 2): - fail(cp + ".token") - elif isinstance(cond, dict): - typ = _normalize_cond_type(cond) - if typ not in ("presence", "status", "value"): - fail(cp + ".type") - val = cond.get("value", cond.get("kind", cond.get("status"))) - if typ == "presence" and val not in ("added", "removed", "changed", "unchanged"): - fail(cp + ".presence") - if typ == "status" and val not in ("up", "down", "other", "none"): - fail(cp + ".status") - if typ == "value": - op = cond.get("op", "eq") - if op not in ("eq", "ne", "in", "not_in", "empty", "not_empty", "changed", "unchanged", "same", "diff"): - fail(cp + ".op") - if op in ("eq", "ne", "in", "not_in") and (val is None or val == "" or val == []): - fail(cp + ".value") - else: - fail(cp + ".condition") - if key == "anomaly" and not any(_groups_for_side(pat, s) for s in ("old", "new")): - fail(pp + ".empty") diff --git a/netx_api/biz_migration_router.py b/netx_api/biz_migration_router.py index 52508f8..29b4377 100644 --- a/netx_api/biz_migration_router.py +++ b/netx_api/biz_migration_router.py @@ -285,8 +285,6 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)): def api_list_diffs( run_id: str, metric_id: str = "", - sheet_id: str = "", - only_expect: bool = False, verdict: str = "", color: str = "", kw: str = "", @@ -298,8 +296,6 @@ def api_list_diffs( db, run_id, metric_id=metric_id, - sheet_id=sheet_id, - only_expect=only_expect, verdict=verdict, color=color, kw=kw, diff --git a/netx_api/biz_state/claim.py b/netx_api/biz_state/claim.py index 2a592c0..a86674e 100644 --- a/netx_api/biz_state/claim.py +++ b/netx_api/biz_state/claim.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -from datetime import datetime, timedelta +from datetime import datetime from typing import Any from uuid import uuid4 @@ -39,10 +39,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: task = db.get(BizStateTask, tid) if not task: return {"ok": False, "queued": False, "reason": "task_not_found", "task_id": tid} - if task.collect_group_id: - group_id = task.collect_group_id - db.close() - return enqueue_collect_group(group_id, manual=manual, requested_task_id=tid) if str(task.source or "").strip().lower() == "import": return { "ok": False, @@ -103,7 +99,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: vendor=task.vendor, status="queued", started_at=now, - queued_at=now, message="queued" if not manual else "queued_manual", ) db.add(batch) @@ -126,61 +121,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: db.close() -def enqueue_collect_group(group_id: str, *, manual: bool = False, - requested_task_id: str = "") -> dict[str, Any]: - """Atomically enqueue both devices with one round identity, or neither.""" - db = SessionLocal() - try: - tasks = db.query(BizStateTask).filter(BizStateTask.collect_group_id == group_id).order_by( - BizStateTask.id).with_for_update().populate_existing().all() - result: dict[str, Any] = {"ok": False, "queued": False, "task_id": requested_task_id, - "collect_group_id": group_id} - if len(tasks) != 2 or len({t.ne_id for t in tasks}) != 2: - return {**result, "reason": "collect_pair_invalid"} - if max_concurrent_tasks() < 2: - return {**result, "reason": "collect_pair_capacity"} - from ..cli_budget import clamp_cli_workers - - worker_cap = clamp_cli_workers(int(getattr(settings, "biz_state_worker_collect_threads", None) - or getattr(settings, "biz_state_dispatch_workers", 8) or 8)) - if worker_cap < 2: - return {**result, "reason": "collect_pair_capacity"} - if any(t.collect_running for t in tasks): - return {**result, "reason": "collect_pair_busy"} - for task in tasks: - if task.source == "import" or (not manual and (task.status != "running" or task.collect_manual_only)): - return {**result, "reason": "not_scheduled"} - if task.status in ("", "deleted"): - return {**result, "reason": "bad_status"} - if not db.query(BizStateTaskItem.id).filter(BizStateTaskItem.task_id == task.id, - BizStateTaskItem.enabled.is_(True)).first(): - return {**result, "reason": "no_enabled_items"} - now, round_id = _utcnow(), uuid4().hex - batches = [] - for task in tasks: - task.collect_running = True - task.collect_queued_at = now - task.last_error = "" - task.updated_at = now - batch = BizStateBatch(id=uuid4().hex, task_id=task.id, source=task.source, - ne_id=task.ne_id, ne_name=task.ne_name, vendor=task.vendor, status="queued", - started_at=now, queued_at=now, collect_group_id=group_id, - collect_round_id=round_id, collect_priority=10 if task.collect_manual_only else 20, - message="queued_manual" if manual else "queued") - db.add(batch) - batches.append({"batch_id": batch.id, "task_id": task.id}) - db.commit() - selected = next((b for b in batches if b["task_id"] == requested_task_id), batches[0]) - return {**result, **selected, "ok": True, "queued": True, "collect_round_id": round_id, - "batches": batches, "manual": manual} - except Exception: - db.rollback() - _log.exception("enqueue collect pair failed group=%s", group_id) - return {"ok": False, "queued": False, "reason": "enqueue_error", "task_id": requested_task_id} - finally: - db.close() - - def _dialect_supports_skip_locked(db: Session) -> bool: try: bind = db.get_bind() @@ -211,16 +151,11 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: Global ceiling is always ``max_concurrent_tasks()``; ``limit`` only caps how many this caller wants in one call. """ - from sqlalchemy import and_, case, text + from sqlalchemy import text global_cap = max_concurrent_tasks() db = SessionLocal() try: - pg = _dialect_supports_skip_locked(db) - if pg: - # A short transaction lock makes capacity reservation and pair claims - # atomic across worker processes. Never held during device I/O. - db.execute(text("SELECT pg_advisory_xact_lock(hashtext('biz-state-claim-capacity'))")) running_n = ( db.query(BizStateBatch).filter(BizStateBatch.status == "running").count() ) @@ -238,54 +173,50 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: if str(r[0] or "").strip() } - # Manual route rounds yield to fresh monitoring, but must eventually - # run even when a busy device always has regular metrics queued. - priority = case((and_(BizStateBatch.collect_priority == 10, - BizStateBatch.queued_at <= _utcnow() - timedelta(seconds=300)), 30), - else_=BizStateBatch.collect_priority) q = ( db.query(BizStateBatch) .filter(BizStateBatch.status == "queued") - .order_by(priority.desc(), BizStateBatch.started_at.asc(), BizStateBatch.id.asc()) + .order_by(BizStateBatch.started_at.asc()) ) + pg = _dialect_supports_skip_locked(db) if pg: q = q.with_for_update(skip_locked=True) else: q = q.with_for_update() - candidates = q.limit(max(slots * 8, 64)).all() + candidates = q.limit(max(slots * 4, slots)).all() claimed_rows: list[BizStateBatch] = [] - visited: set[str] = set() for batch in candidates: - if batch.id in visited: - continue if len(claimed_rows) >= slots: break - group = [batch] - if batch.collect_round_id: - peers = db.query(BizStateBatch).filter( - BizStateBatch.collect_round_id == batch.collect_round_id, - ).with_for_update(skip_locked=pg).all() - if len(peers) != 2 or any(p.status != "queued" for p in peers): + ne = str(batch.ne_id or "").strip() + if ne and ne in busy_nes: + continue + # Serialize claims per NE across workers (Postgres). + if ne and pg: + try: + db.execute( + text("SELECT pg_advisory_xact_lock(hashtext(:ne))"), + {"ne": ne}, + ) + except Exception: + _log.exception("advisory lock failed ne=%s", ne) + conflict = ( + db.query(BizStateBatch.id) + .filter( + BizStateBatch.status == "running", + BizStateBatch.ne_id == ne, + BizStateBatch.id != batch.id, + ) + .first() + ) + if conflict: continue - group = peers - visited.update(p.id for p in group) - if len(group) > slots - len(claimed_rows): - continue - nes = {str(p.ne_id or "").strip() for p in group} - if nes & busy_nes: - continue - now = _utcnow() - for member in group: - member.status = "running" - member.message = "" - if member.collect_round_id: - member.started_at = now - task = db.get(BizStateTask, member.task_id) - if task: - task.last_collect_started_at = now - claimed_rows.append(member) - busy_nes.update(nes - {""}) + batch.status = "running" + batch.message = "" + claimed_rows.append(batch) + if ne: + busy_nes.add(ne) if not claimed_rows: db.rollback() @@ -300,13 +231,10 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]: db.query(BizStateBatch).filter(BizStateBatch.status == "running").count() ) while n_running > global_cap and claimed_rows: - last = claimed_rows[-1] - demoted = [b for b in claimed_rows if b.collect_round_id == last.collect_round_id] if last.collect_round_id else [last] - for demote in demoted: - claimed_rows.remove(demote) - demote.status = "queued" - demote.message = "queued" - n_running -= 1 + demote = claimed_rows.pop() + demote.status = "queued" + demote.message = "queued" + n_running -= 1 if not claimed_rows: db.rollback() diff --git a/netx_api/biz_state/collect_runner.py b/netx_api/biz_state/collect_runner.py index 91797ce..ad04d77 100644 --- a/netx_api/biz_state/collect_runner.py +++ b/netx_api/biz_state/collect_runner.py @@ -784,12 +784,6 @@ def dispatch_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]: result = enqueue_collect(task_id, manual=manual) if not result.get("queued"): return result - if result.get("collect_group_id"): - if _should_execute_inline(): - from ..biz_state_scheduler import _claim_and_dispatch - - _claim_and_dispatch() - return result batch_id = str(result.get("batch_id") or "") if not batch_id: return result @@ -876,11 +870,6 @@ def execute_claimed_batch( task = db.get(BizStateTask, task_id) if task_id else None if not batch: return - if batch.collect_round_id: - batch.started_at = _utcnow() - if task: - task.last_collect_started_at = batch.started_at - db.commit() if not task_id: task_id = str(batch.task_id or "") task = db.get(BizStateTask, task_id) if task_id else None @@ -1731,14 +1720,6 @@ def _finalize_batch_status( batch.message = "" status = str(batch.status or "") db.commit() - # Failed/partial receipts also update cutover coverage; never keep an old - # success looking healthy after the latest collection has failed. - try: - from ..biz_migration.auto_monitor import schedule_auto_monitor_for_task - - schedule_auto_monitor_for_task(task_id) - except Exception: - _log.exception("cutover monitor schedule failed task=%s", task_id) # Only full success triggers auto compare; never block the collect thread. if status == "success" and not stopped: try: diff --git a/netx_api/biz_state/collect_stop.py b/netx_api/biz_state/collect_stop.py index 1f27e31..8c92db0 100644 --- a/netx_api/biz_state/collect_stop.py +++ b/netx_api/biz_state/collect_stop.py @@ -131,7 +131,7 @@ def _force_close_holders(batch_id: str) -> int: def request_stop_collect(task_id: str) -> dict[str, Any]: - """Stop a task, including its paired collector, without stranding a half round.""" + """Cancel queued batches and signal running collect for this task to abort.""" tid = str(task_id or "").strip() if not tid: return {"ok": False, "reason": "missing_task_id"} @@ -141,37 +141,26 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: signaled_ids: list[str] = [] closed = 0 try: - from sqlalchemy import text - from .claim import _dialect_supports_skip_locked - - if _dialect_supports_skip_locked(db): - db.execute(text("SELECT pg_advisory_xact_lock(hashtext('biz-state-claim-capacity'))")) task = db.get(BizStateTask, tid) if not task: return {"ok": False, "reason": "task_not_found", "task_id": tid} - task_ids = [tid] - if task.collect_group_id: - task_ids = [r[0] for r in db.query(BizStateTask.id).filter( - BizStateTask.collect_group_id == task.collect_group_id).all()] active = ( db.query(BizStateBatch) .filter( - BizStateBatch.task_id.in_(task_ids), + BizStateBatch.task_id == tid, BizStateBatch.status.in_(("queued", "running")), ) - .order_by(BizStateBatch.id).with_for_update().all() + .all() ) - tasks = db.query(BizStateTask).filter(BizStateTask.id.in_(task_ids)).order_by( - BizStateTask.id).with_for_update().populate_existing().all() if not active: # Nothing to stop — clear sticky collect_running if orphaned. - for member in tasks: - if member.collect_running: - member.collect_running = False - member.collect_queued_at = None - member.updated_at = utcnow_naive() - db.commit() + if bool(task.collect_running): + task.collect_running = False + if hasattr(task, "collect_queued_at"): + task.collect_queued_at = None + task.updated_at = utcnow_naive() + db.commit() return { "ok": True, "task_id": tid, @@ -182,7 +171,7 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: } now = utcnow_naive() - running_tasks: set[str] = set() + still_running = False for batch in active: bid = str(batch.id) mark_stop_requested(bid) @@ -194,16 +183,16 @@ def request_stop_collect(task_id: str) -> dict[str, Any]: else: batch.message = STOP_REQUEST_TOKEN signaled_ids.append(bid) - running_tasks.add(batch.task_id) + still_running = True closed += _force_close_holders(bid) - for member in tasks: - if member.id not in running_tasks: - member.collect_running = False - member.collect_queued_at = None - member.last_collect_ended_at = now - member.last_error = STOP_USER_MESSAGE - member.updated_at = now + if not still_running: + task.collect_running = False + if hasattr(task, "collect_queued_at"): + task.collect_queued_at = None + task.last_collect_ended_at = now + task.last_error = STOP_USER_MESSAGE + task.updated_at = now db.commit() _log.info( diff --git a/netx_api/biz_state/compare_engine.py b/netx_api/biz_state/compare_engine.py index 87af097..b11cae8 100644 --- a/netx_api/biz_state/compare_engine.py +++ b/netx_api/biz_state/compare_engine.py @@ -2,81 +2,15 @@ from __future__ import annotations -from collections import defaultdict -from heapq import heappush, heapreplace -from itertools import chain -from typing import Any, Iterable, Mapping, Sequence +from typing import Any, Mapping, Sequence -from .compare_rules import field_rule_map, values_equal, explain_diff, scalar_text +from .compare_rules import field_rule_map, values_equal, explain_diff from .iface_normalize import ( apply_iface_normalize_rows, normalize_iface_rules, resolve_mapped_iface, ) -# Prefer these fields when stratifying success samples (BGP multipath / multi-cmd). -_STRATUM_FIELDS = ("neighbor", "direction", "afi", "vrf") - - -def stratum_key(row: Mapping[str, Any] | None) -> str: - """Bucket key for stratified unchanged sampling.""" - if not isinstance(row, Mapping): - return "_" - parts: list[str] = [] - for f in _STRATUM_FIELDS: - v = str(row.get(f) or "").strip() - if v: - parts.append(f"{f}={v}") - return "|".join(parts) if parts else "_" - - -class _StratifiedSample: - """Keep the first N round-robin ranks in O(N) space, even with many strata.""" - - def __init__(self, limit: int) -> None: - self.limit = max(0, int(limit)) - self.count = 0 - self.strata: dict[str, tuple[int, int]] = {} - self.heap: list[tuple[int, int, int, Any]] = [] - - def add(self, item: Any, key: str) -> None: - serial = self.count - self.count += 1 - if not self.limit: - return - state = self.strata.get(key) - if state is None: - # Later strata cannot beat the first item of N earlier strata. - if len(self.strata) >= self.limit: - return - index, round_n = len(self.strata), 0 - else: - index, round_n = state - self.strata[key] = (index, round_n + 1) - entry = (-round_n, -index, serial, item) - if len(self.heap) < self.limit: - heappush(self.heap, entry) - elif entry[:2] > self.heap[0][:2]: - heapreplace(self.heap, entry) - - def picked(self) -> list[Any]: - # The legacy helper preserves encounter order when no truncation occurs. - if self.count <= self.limit: - entries = sorted(self.heap, key=lambda e: e[2]) - else: - entries = sorted(self.heap, key=lambda e: (-e[0], -e[1])) - return [entry[3] for entry in entries] - - -def stratify_take(items: Iterable[Any], limit: int, *, key_fn) -> list[Any]: - """Round-robin across strata without retaining every candidate.""" - sample = _StratifiedSample(limit) - if not sample.limit: - return [] - for item in items: - sample.add(item, str(key_fn(item) or "_")) - return sample.picked() - def apply_port_map( row: dict[str, Any], @@ -94,7 +28,7 @@ def apply_port_map( def row_key(row: dict[str, Any], key_fields: list[str]) -> tuple[str, ...]: - return tuple(scalar_text(row.get(k)).strip() for k in key_fields) + return tuple(str(row.get(k) or "").strip() for k in key_fields) def mapping_stats( @@ -171,28 +105,12 @@ def compare_rows( field_rules: Sequence[Mapping[str, Any]] | None = None, iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, ignore_port_changes: bool | None = None, - include_unchanged: bool = False, - unchanged_limit: int | None = None, - compact_unchanged: bool = False, ) -> dict[str, Any]: """Return summary + diffs list. - Diff kinds: added | removed | changed | unchanged + Diff kinds: added | removed | changed | unchanged | duplicate - Pipeline: iface normalize (both sides) → port map (before) → ordered - same-key pairing (load / list order within each match key). - - Same match key with N before and M after rows: zip by index - ``0..min(N,M)-1`` for field compare; extras become ``removed`` (before) - or ``added`` (after). Example: 5 vs 2 → 2 compared + 3 removed. - - ``include_unchanged``: when False, matching rows still increment - ``summary.unchanged`` but are omitted from ``diffs``. - - ``unchanged_limit``: max unchanged diffs to emit (None = no cap). Use with - large sheets so the UI can browse a sample without writing millions of rows. - - ``compact_unchanged``: emit key only (empty before/after) to cut storage. + Pipeline: iface normalize (both sides) → port map (before) → match. ``ignore_port_changes``: - ``None`` (default): auto — drop iface from match key only when remaining @@ -200,8 +118,8 @@ def compare_rows( - ``True``: force drop iface from match key when a non-empty candidate exists. - ``False``: never drop iface from match key. - ``summary.duplicate`` is always 0. ``duplicate_keys_*`` count match keys - that appear more than once on a side (diagnostic only). + Duplicate match keys are not silently discarded: extras become ``duplicate`` + diffs and ``summary.duplicate_key_list`` lists the colliding keys. ``field_rules`` drives normalize / numeric tolerance / per-field compare mode (template-driven; no metric-specific branches here). @@ -216,10 +134,10 @@ def compare_rows( before_norm = apply_iface_normalize_rows( before_rows, iface_fields=iface_list, rules=norm_rules - ) if norm_rules and iface_list else before_rows + ) after_norm = apply_iface_normalize_rows( after_rows, iface_fields=iface_list, rules=norm_rules - ) if norm_rules and iface_list else after_rows + ) ignore_ports = False # No map → optionally ignore port renames by dropping iface from match key. @@ -234,16 +152,9 @@ def compare_rows( match_keys = list(key_fields) else: # Auto heuristic (legacy default) - def unique_keys(rows: list[dict[str, Any]]) -> bool: - seen: set[tuple[str, ...]] = set() - for row in rows: - key = row_key(row, candidate) - if key in seen: - return False - seen.add(key) - return True - - if unique_keys(before_norm) and unique_keys(after_norm): + before_c = [row_key(r, candidate) for r in before_norm] + after_c = [row_key(r, candidate) for r in after_norm] + if len(before_c) == len(set(before_c)) and len(after_c) == len(set(after_c)): match_keys = candidate ignore_ports = True else: @@ -251,154 +162,131 @@ def compare_rows( else: match_keys = list(key_fields) - before_groups: dict[tuple[str, ...], list[tuple[dict[str, Any], dict[str, Any]]]] = ( - defaultdict(list) - ) - after_groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) - for orig, norm in zip(before_rows, before_norm): - mapped = apply_port_map(norm, iface_fields=iface_list, port_map=pmap) if iface_list else norm - before_groups[row_key(mapped, match_keys)].append((orig, mapped)) - for r in after_norm: - after_groups[row_key(r, match_keys)].append(r) + before_mapped: list[dict[str, Any]] = [ + apply_port_map(r, iface_fields=iface_list, port_map=pmap) for r in before_norm + ] + after_index: dict[tuple[str, ...], dict[str, Any]] = {} + after_dup = 0 + after_dup_keys: list[tuple[str, ...]] = [] + after_dup_rows: list[tuple[tuple[str, ...], dict[str, Any]]] = [] + for r in after_norm: + k = row_key(r, match_keys) + if k in after_index: + after_dup += 1 + after_dup_keys.append(k) + after_dup_rows.append((k, r)) + continue # first wins — do not overwrite + after_index[k] = r + + before_keys: set[tuple[str, ...]] = set() + before_dup = 0 + before_dup_keys: list[tuple[str, ...]] = [] diffs: list[dict[str, Any]] = [] - added = removed = changed = unchanged = 0 - unchanged_listed = 0 - limit_n = None if unchanged_limit is None else max(0, int(unchanged_limit)) - multi_before_keys: list[tuple[str, ...]] = [] - multi_after_keys: list[tuple[str, ...]] = [] - unchanged_candidates: list[tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = [] - sample = _StratifiedSample(limit_n) if include_unchanged and limit_n is not None else None + added = removed = changed = unchanged = duplicate = 0 def _key_obj(row: dict[str, Any]) -> dict[str, Any]: return {f: row.get(f, "") for f in key_fields} - def _row_id(row: dict[str, Any] | None) -> str: - if not isinstance(row, dict): - return "" - netx = row.get("_netx") - if isinstance(netx, dict): - return str(netx.get("row_id") or "") - return "" - - def _append_unchanged_diff( - orig: dict[str, Any], mapped: dict[str, Any], after_row: dict[str, Any] - ) -> None: - nonlocal unchanged_listed - unchanged_listed += 1 - before_rid = _row_id(orig) - after_rid = _row_id(after_row) - if compact_unchanged: + for orig, mapped in zip(before_rows, before_mapped): + k = row_key(mapped, match_keys) + if k in before_keys: + before_dup += 1 + before_dup_keys.append(k) + duplicate += 1 diffs.append( { - "kind": "unchanged", + "kind": "duplicate", + "side": "before", "key": _key_obj(mapped), - "before": {}, - "after": {}, - "mapped_before": {}, + "before": orig, + "after": after_index.get(k), + "mapped_before": mapped, "changes": {}, - "compact": True, - "before_row_id": before_rid, - "after_row_id": after_rid, + } + ) + continue # only first before row participates in match + before_keys.add(k) + after = after_index.get(k) + if after is None: + removed += 1 + diffs.append( + { + "kind": "removed", + "key": _key_obj(mapped), + "before": orig, + "after": None, + "mapped_before": mapped, + "changes": {}, + } + ) + continue + # Empty compare_fields = presence-only: keyed rows that exist on both + # sides are unchanged (no value checks). + field_changes: dict[str, dict[str, Any]] = {} + for f in compare_fields: + bv = mapped.get(f, "") + av = after.get(f, "") + rule = rules.get(f) + if not values_equal(bv, av, rule=rule): + entry: dict[str, Any] = {"before": bv, "after": av} + reason = explain_diff(bv, av, rule=rule) + if reason: + entry["reason"] = reason + field_changes[f] = entry + if field_changes: + changed += 1 + diffs.append( + { + "kind": "changed", + "key": _key_obj(mapped), + "before": orig, + "after": after, + "mapped_before": mapped, + "changes": field_changes, } ) else: + unchanged += 1 diffs.append( { "kind": "unchanged", "key": _key_obj(mapped), "before": orig, - "after": after_row, + "after": after, "mapped_before": mapped, "changes": {}, - "before_row_id": before_rid, - "after_row_id": after_rid, } ) - # Stable key order: before encounter order, then after-only keys - ordered_keys = chain(before_groups, (k for k in after_groups if k not in before_groups)) - for k in ordered_keys: - b_list = before_groups.get(k) or [] - a_list = after_groups.get(k) or [] - if len(b_list) > 1: - multi_before_keys.append(k) - if len(a_list) > 1: - multi_after_keys.append(k) - n = max(len(b_list), len(a_list)) - for i in range(n): - if i >= len(b_list): - after = a_list[i] - added += 1 - diffs.append( - { - "kind": "added", - "key": _key_obj(after), - "before": None, - "after": after, - "mapped_before": None, - "changes": {}, - "before_row_id": "", - "after_row_id": _row_id(after), - } - ) - continue - if i >= len(a_list): - orig, mapped = b_list[i] - removed += 1 - diffs.append( - { - "kind": "removed", - "key": _key_obj(mapped), - "before": orig, - "after": None, - "mapped_before": mapped, - "changes": {}, - "before_row_id": _row_id(orig), - "after_row_id": "", - } - ) - continue - orig, mapped = b_list[i] - after = a_list[i] - field_changes: dict[str, dict[str, Any]] = {} - for f in compare_fields: - bv = mapped.get(f, "") - av = after.get(f, "") - rule = rules.get(f) - if not values_equal(bv, av, rule=rule): - entry: dict[str, Any] = {"before": bv, "after": av} - reason = explain_diff(bv, av, rule=rule) - if reason: - entry["reason"] = reason - field_changes[f] = entry - if field_changes: - changed += 1 - diffs.append( - { - "kind": "changed", - "key": _key_obj(mapped), - "before": orig, - "after": after, - "mapped_before": mapped, - "changes": field_changes, - "before_row_id": _row_id(orig), - "after_row_id": _row_id(after), - } - ) - else: - unchanged += 1 - if include_unchanged: - if sample is not None: - if sample.limit: - sample.add((orig, mapped, after), stratum_key(mapped)) - else: - unchanged_candidates.append((orig, mapped, after)) + for k, after in after_index.items(): + if k in before_keys: + continue + added += 1 + diffs.append( + { + "kind": "added", + "key": _key_obj(after), + "before": None, + "after": after, + "mapped_before": None, + "changes": {}, + } + ) - if include_unchanged: - picked = sample.picked() if sample is not None else unchanged_candidates - for orig, mapped, after_row in picked: - _append_unchanged_diff(orig, mapped, after_row) + for k, after in after_dup_rows: + duplicate += 1 + diffs.append( + { + "kind": "duplicate", + "side": "after", + "key": _key_obj(after), + "before": None, + "after": after, + "mapped_before": None, + "changes": {}, + } + ) def _fmt_keys(keys: list[tuple[str, ...]]) -> list[str]: seen: set[str] = set() @@ -408,7 +296,7 @@ def compare_rows( if s not in seen: seen.add(s) out.append(s) - return out[:64] + return out stats = mapping_stats( before_rows=before_norm, @@ -426,21 +314,11 @@ def compare_rows( "removed": removed, "changed": changed, "unchanged": unchanged, - "duplicate": 0, + "duplicate": duplicate, "match_key_fields": match_keys, - "duplicate_keys_before": len(multi_before_keys), - "duplicate_keys_after": len(multi_after_keys), - "duplicate_key_list": _fmt_keys(multi_before_keys + multi_after_keys), - "unchanged_listed": unchanged_listed, - "unchanged_truncated": bool( - include_unchanged and limit_n is not None and unchanged > unchanged_listed - ), - "unchanged_compact": bool(compact_unchanged and unchanged_listed > 0), - "unchanged_sample_mode": ( - "stratified" - if include_unchanged and limit_n is not None and unchanged > unchanged_listed - else ("full" if include_unchanged else "none") - ), + "duplicate_keys_before": before_dup, + "duplicate_keys_after": after_dup, + "duplicate_key_list": _fmt_keys(before_dup_keys + after_dup_keys), }, "diffs": diffs, "mapping_stats": stats, diff --git a/netx_api/biz_state/compare_rules.py b/netx_api/biz_state/compare_rules.py index 269a32a..97e454b 100644 --- a/netx_api/biz_state/compare_rules.py +++ b/netx_api/biz_state/compare_rules.py @@ -7,21 +7,10 @@ All metric-specific compare behavior belongs in the sheet template from __future__ import annotations import re -import math from typing import Any, Mapping, Sequence _AGE_TIMER_RE = re.compile(r"^\d{1,2}:\d{2}:\d{2}$") -NUMBER_PATTERN = r"^[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?$" - - -def scalar_text(value: Any) -> str: - """Match JSON text extraction: only null is blank; keep zero and booleans.""" - if value is None: - return "" - if isinstance(value, bool): - return "true" if value else "false" - return str(value) def _as_list(raw: Any) -> list[Any]: @@ -33,7 +22,7 @@ def _as_list(raw: Any) -> list[Any]: def _field_val(row: Mapping[str, Any], field: str) -> str: - return scalar_text((row or {}).get(field)).strip() + return str((row or {}).get(field) or "").strip() def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: @@ -46,17 +35,17 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: expect = filt.get("value") if op in ("eq", "=="): - return raw.lower() == scalar_text(expect).strip().lower() + return raw.lower() == str(expect or "").strip().lower() if op in ("ne", "!="): - return raw.lower() != scalar_text(expect).strip().lower() + return raw.lower() != str(expect or "").strip().lower() if op == "in": - opts = {scalar_text(x).strip().lower() for x in _as_list(expect) if scalar_text(x).strip()} + opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()} return raw.lower() in opts if op in ("not_in", "nin"): - opts = {scalar_text(x).strip().lower() for x in _as_list(expect) if scalar_text(x).strip()} + opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()} return raw.lower() not in opts if op == "contains": - needle = scalar_text(expect).strip().lower() + needle = str(expect or "").strip().lower() return bool(needle) and needle in raw.lower() if op == "empty": return not raw @@ -74,7 +63,7 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool: # HH:MM:SS dynamic ARP age return bool(_AGE_TIMER_RE.match(raw)) if op == "ci_eq": - return raw.lower() == scalar_text(expect).strip().lower() + return raw.lower() == str(expect or "").strip().lower() return True @@ -110,7 +99,7 @@ def apply_row_filters( def normalize_value(value: Any, how: str) -> str: - text = scalar_text(value).strip() + text = str(value if value is not None else "").strip() mode = str(how or "").strip().lower() if not mode or mode in ("none", "strip"): return text @@ -201,11 +190,8 @@ def effective_display_fields( def _parse_float(text: str) -> float | None: - if not re.fullmatch(NUMBER_PATTERN, text): - return None try: - value = float(text) - return value if math.isfinite(value) else None + return float(text) if text else 0.0 except ValueError: return None diff --git a/netx_api/biz_state/compare_service.py b/netx_api/biz_state/compare_service.py index 286c29b..195e752 100644 --- a/netx_api/biz_state/compare_service.py +++ b/netx_api/biz_state/compare_service.py @@ -6,16 +6,13 @@ import io import json import logging import threading -import time import zipfile -from copy import deepcopy from datetime import datetime -from typing import Any, Callable, Iterable, Iterator +from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import String, and_, cast, func, or_, tuple_ -from sqlalchemy.orm import Session, aliased +from sqlalchemy.orm import Session from ..models import ( BizCompareDiff, @@ -30,14 +27,13 @@ from ..models import ( BizStateTask, ) from ..timeutil import utcnow_naive -from .compare_engine import apply_port_map, compare_rows, mapping_stats, row_key +from .compare_engine import compare_rows, mapping_stats from .compare_rules import ( ROW_FILTER_PRESETS, apply_row_filters, arp_dynamic_row_filters, effective_compare_fields, effective_display_fields, - row_matches_filter, ) from .iface_normalize import ( apply_iface_normalize_rows, @@ -45,7 +41,6 @@ from .iface_normalize import ( normalize_iface_rules, ) from .profiles import metric_field_map -from .compare_validation import validate_template_body _log = logging.getLogger("netx.biz_state.compare") @@ -93,7 +88,7 @@ def batch_metric_collect_ok(db: Session, batch_id: str, metric_id: str) -> bool: if status == "success": return True cmds = ( - db.query(BizStateBatchCommand.parse_status, BizStateBatchCommand.row_count) + db.query(BizStateBatchCommand) .filter( BizStateBatchCommand.batch_id == bid, BizStateBatchCommand.metric_id == mid, @@ -186,93 +181,10 @@ def _compare_side( } _DIFF_CHUNK = 2000 -_LOAD_YIELD_PER = 5000 _SEARCH_TEXT_MAX = 4000 -# Heartbeat while bulk-inserting large fail/ok diff sets (vpnv4-scale). -_PERSIST_PROGRESS_EVERY = 10_000 -# Above this, store fail diffs as key + row_id + changes (hydrate sides on read). -_FAIL_COMPACT_MIN = 50_000 -# Live search (kw): load at most this many matching rows per side, return ≤ this many pairs. -_LIVE_SEARCH_LOAD_CAP = 2_000 -_LIVE_SEARCH_RESULT_CAP = 200 -_LIVE_SEARCH_GROUP_LOAD_CAP = 10_000 -# Success-row persist policy (see resolve_unchanged_policy) -_STORE_UNCHANGED_MODES = frozenset({"auto", "always", "never", "sample", "keys"}) -_UNCHANGED_FULL_MAX = 20_000 -_UNCHANGED_SAMPLE_MAX = 5_000 -def normalize_store_unchanged(raw: Any) -> str: - m = str(raw or "auto").strip().lower() - return m if m in _STORE_UNCHANGED_MODES else "auto" - - -def compute_pass_rate(*, success: int, fail: int, empty_as: float = 100.0) -> float: - """Pass rate for cutover UI. - - With million-row sheets, 3 fails / 50k can round to 100.0% at 1 decimal — - never report 100 when any fail remains; add decimals until it stays < 100. - """ - s = max(0, int(success or 0)) - f = max(0, int(fail or 0)) - judged = s + f - if judged <= 0: - return float(empty_as) - rate = (s / judged) * 100.0 - if f <= 0: - return round(rate, 1) - for nd in (1, 2, 3, 4): - r = round(rate, nd) - if r < 100.0: - return r - return 99.9999 - - -def resolve_unchanged_policy( - mode: str, *, before_n: int, after_n: int -) -> dict[str, Any]: - """Decide whether / how many success rows to emit for one sheet. - - - always: full before/after for every match (slow on million-row sheets) - - never: count only - - sample: key + row_id sample (cap) — cutover spot-check default for large sheets - - keys: all success as key + row_id (full browse; write still heavy) - - auto: full when sheet is small; otherwise sample compact - """ - m = normalize_store_unchanged(mode) - n = max(int(before_n or 0), int(after_n or 0)) - if m == "never": - return {"mode": m, "include": False, "limit": None, "compact": False} - if m == "always": - return {"mode": m, "include": True, "limit": None, "compact": False} - if m == "keys": - return {"mode": m, "include": True, "limit": None, "compact": True} - if m == "sample": - return { - "mode": m, - "include": True, - "limit": _UNCHANGED_SAMPLE_MAX, - "compact": True, - } - # auto — cutover-oriented: large sheets sample, not full key dump - if n <= _UNCHANGED_FULL_MAX: - return {"mode": m, "include": True, "limit": None, "compact": False} - return { - "mode": m, - "include": True, - "limit": _UNCHANGED_SAMPLE_MAX, - "compact": True, - } - - -def _strip_netx(row: Any) -> dict[str, Any]: - """Drop collector provenance before persisting compare payloads.""" - if not isinstance(row, dict): - return {} - return {k: v for k, v in row.items() if k != "_netx"} - - -def _diff_search_text(d: dict[str, Any], *, limit: int | None = _SEARCH_TEXT_MAX) -> str: +def _diff_search_text(d: dict[str, Any]) -> str: parts = [str(d.get("kind") or "")] for key in ("key", "before", "after", "mapped_before", "changes"): val = d.get(key) @@ -281,8 +193,7 @@ def _diff_search_text(d: dict[str, Any], *, limit: int | None = _SEARCH_TEXT_MAX parts.append(json.dumps(val, ensure_ascii=False, default=str, separators=(",", ":"))) except Exception: parts.append(str(val)) - value = " ".join(parts) - return value if limit is None else value[:limit] + return " ".join(parts)[:_SEARCH_TEXT_MAX] def _top_changed_fields(diffs: list[dict[str, Any]], *, limit: int = 8) -> list[dict[str, Any]]: @@ -304,161 +215,30 @@ def _persist_sheet_diffs( run_id: str, metric_id: str, diffs: list[dict[str, Any]], - seq_start: int = 0, - on_progress: Callable[[int, int], None] | None = None, -) -> int: - """Bulk-insert diff rows already selected by the engine policy. - - Compact success rows carry key + before/after_row_id; JSON sides stay empty - and are hydrated from metric tables on read. - - ``on_progress(written, total)`` fires periodically so UI elapsed time moves - during multi-minute inserts (e.g. large vpnv4 fail sets). - """ +) -> None: + """Bulk-insert diff rows; avoids embedding million-row arrays in summary_json.""" buf: list[dict[str, Any]] = [] - seq = int(seq_start or 0) - written = 0 - total = len(diffs) - last_prog = 0 - last_prog_t = time.monotonic() - - def _maybe_prog(force: bool = False) -> None: - nonlocal last_prog, last_prog_t - if not on_progress: - return - now = time.monotonic() - if ( - not force - and written - last_prog < _PERSIST_PROGRESS_EVERY - and now - last_prog_t < 2.0 - ): - return - last_prog = written - last_prog_t = now - try: - on_progress(written, total) - except Exception: - _log.exception("persist progress callback failed run=%s metric=%s", run_id, metric_id) - - for d in diffs: - kind = str(d.get("kind") or "") - before = _strip_netx(d.get("before")) - after = _strip_netx(d.get("after")) - mapped = _strip_netx(d.get("mapped_before")) - payload = { - "kind": kind, - "key": dict(d.get("key") or {}), - "before": before, - "after": after, - "mapped_before": mapped, - "changes": dict(d.get("changes") or {}), - } - # Compact rows: search_text = kind + key (+ changes) only — no fat sides - search_src = ( - { - "kind": kind, - "key": payload["key"], - **({"changes": payload["changes"]} if payload["changes"] else {}), - } - if bool(d.get("compact")) - else payload - ) + for i, d in enumerate(diffs): buf.append( { "id": uuid4().hex, "run_id": run_id, "metric_id": metric_id, - "seq": seq, - "kind": kind, - "key_json": payload["key"], - "before_json": before, - "after_json": after, - "mapped_before_json": mapped, - "changes_json": payload["changes"], - "before_row_id": str(d.get("before_row_id") or "")[:64], - "after_row_id": str(d.get("after_row_id") or "")[:64], - "search_text": _diff_search_text(search_src), + "seq": i, + "kind": str(d.get("kind") or ""), + "key_json": dict(d.get("key") or {}), + "before_json": dict(d.get("before") or {}), + "after_json": dict(d.get("after") or {}), + "mapped_before_json": dict(d.get("mapped_before") or {}), + "changes_json": dict(d.get("changes") or {}), + "search_text": _diff_search_text(d), } ) - seq += 1 - written += 1 if len(buf) >= _DIFF_CHUNK: db.bulk_insert_mappings(BizCompareDiff, buf) buf.clear() - # Commit chunks so progress/UI can see mid-write fail rows and - # elapsed_ms advances (otherwise persisting_* looks frozen). - db.commit() - _maybe_prog() if buf: db.bulk_insert_mappings(BizCompareDiff, buf) - db.commit() - _maybe_prog(force=True) - return written - - -def _metric_rows_by_ids(db: Session, ids: list[str]) -> dict[str, dict[str, Any]]: - """Load BizStateMetricRow / LLDP rows by primary key for hydrate.""" - clean = [str(i).strip() for i in ids if str(i or "").strip()] - if not clean: - return {} - from ..models import BizStateMetricRow - - out: dict[str, dict[str, Any]] = {} - # Chunk IN lists for large pages - for i in range(0, len(clean), 500): - chunk = clean[i : i + 500] - for r in db.query(BizStateMetricRow).filter(BizStateMetricRow.id.in_(chunk)).all(): - out[str(r.id)] = dict(r.data_json or {}) - missing = [x for x in chunk if x not in out] - if missing: - for n in ( - db.query(BizStateLldpNeighbor) - .filter(BizStateLldpNeighbor.id.in_(missing)) - .all() - ): - out[str(n.id)] = { - "local_if": n.local_if, - "remote_sys": n.remote_sys, - "remote_if": n.remote_if, - "remote_ip": n.remote_ip, - "protocol": n.protocol, - } - return out - - -def _hydrate_diff_rows( - db: Session, items: list[dict[str, Any]], *, - run: BizCompareRun | None = None, sheet: dict[str, Any] | None = None, -) -> list[dict[str, Any]]: - """Fill empty before/after from metric tables when row_ids are present.""" - need: list[str] = [] - for it in items: - if not it.get("before") and it.get("before_row_id"): - need.append(str(it["before_row_id"])) - if not it.get("after") and it.get("after_row_id"): - need.append(str(it["after_row_id"])) - if not need: - return items - by_id = _metric_rows_by_ids(db, need) - pmap, norm_rules = _run_transform_config(db, run) if run else ({}, []) - iface_fields = list((sheet or {}).get("iface_fields") or []) - for it in items: - brid = str(it.get("before_row_id") or "") - arid = str(it.get("after_row_id") or "") - if not it.get("before") and brid and brid in by_id: - it["before"] = by_id[brid] - if not it.get("after") and arid and arid in by_id: - it["after"] = apply_iface_normalize_rows( - [by_id[arid]], iface_fields=iface_fields, rules=norm_rules, - )[0] - if not it.get("mapped_before") and it.get("before"): - normalized = apply_iface_normalize_rows( - [it["before"]], iface_fields=iface_fields, rules=norm_rules, - )[0] - it["mapped_before"] = apply_port_map( - normalized, iface_fields=iface_fields, port_map=pmap, - ) - return items def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]: @@ -469,8 +249,6 @@ def _diff_row_out(r: BizCompareDiff) -> dict[str, Any]: "after": r.after_json or {}, "mapped_before": r.mapped_before_json or {}, "changes": r.changes_json or {}, - "before_row_id": getattr(r, "before_row_id", "") or "", - "after_row_id": getattr(r, "after_row_id", "") or "", } @@ -486,7 +264,6 @@ def _filter_inline_diffs( *, kind: str, kw: str, - field_q: dict[str, str] | None = None, ) -> list[dict[str, Any]]: kind_n = (kind or "diff").strip().lower() kw_n = (kw or "").strip().lower() @@ -500,70 +277,17 @@ def _filter_inline_diffs( elif kind_n != "all" and dk != kind_n: continue if kw_n: - blob = _diff_search_text(d, limit=None).lower() + blob = _diff_search_text(d).lower() if kw_n not in blob: continue - if field_q and not _diff_matches_search(d, kw="", field_q=field_q): - continue out.append(d) return out -def _diff_matches_search(d: dict[str, Any], *, kw: str, field_q: dict[str, str]) -> bool: - """Search a completed pair, never filter its two sides before pairing.""" - if kw and kw.lower() not in _diff_search_text(d, limit=None).lower(): - return False - return all( - any( - val.lower() in str((d.get(side) or {}).get(name, "")).lower() - for side in ("key", "before", "mapped_before", "after") - ) - for name, val in field_q.items() - ) - - -def _search_stored_diffs(q: Any, *, metric_id: str, kw: str, field_q: dict[str, str]) -> Any: - """Filter persisted verdicts in SQL; join source rows for compact payloads.""" - from ..models import BizStateMetricRow - - sides = [BizCompareDiff.key_json, BizCompareDiff.before_json, - BizCompareDiff.mapped_before_json, BizCompareDiff.after_json] - sources = [] - for row_id in (BizCompareDiff.before_row_id, BizCompareDiff.after_row_id): - model = BizStateLldpNeighbor if metric_id == "lldp_neighbor" else BizStateMetricRow - src = aliased(model) - q = q.outerjoin(src, src.id == row_id) - sources.append(src) - if kw: - expressions = [BizCompareDiff.search_text, *[cast(s, String) for s in sides]] - for src in sources: - if metric_id == "lldp_neighbor": - expressions.extend(getattr(src, f) for f in - ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")) - else: - expressions.append(cast(src.data_json, String)) - q = q.filter(or_(*[func.lower(e).contains(kw.lower(), autoescape=True) for e in expressions])) - for name, val in field_q.items(): - expressions = [s[name].as_string() for s in sides] - for src in sources: - if metric_id == "lldp_neighbor": - if name in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol"): - expressions.append(getattr(src, name)) - else: - expressions.append(src.data_json[name].as_string()) - q = q.filter(or_(*[func.lower(cast(e, String)).contains(val.lower(), autoescape=True) - for e in expressions])) - return q - - def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: Any) -> list[dict[str, Any]]: sheets = list(summary.get("sheets") or []) if sheets: return sheets - # Running / empty: do NOT invent a fake first-metric sheet (was showing - # isis 100% with 0→0 while BGP was still loading). - if str(run.status or "") in ("running", "queued"): - return [] return [ { "metric_id": run.metric_id, @@ -573,7 +297,6 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A "iface_fields": list((tpl.iface_fields if tpl else None) or []), "compare_fields": list((tpl.compare_fields if tpl else None) or []), "mode": "fields", - "status": "done", "summary": { k: summary.get(k, 0) for k in ("added", "removed", "changed", "unchanged", "before_count", "after_count") @@ -583,62 +306,6 @@ def _sheet_meta_from_summary(summary: dict[str, Any], run: BizCompareRun, tpl: A ] -def _order_sheets_small_first( - db: Session, - sheets: list[dict[str, Any]], - *, - before_batch_id: str, - after_batch_id: str, -) -> list[dict[str, Any]]: - """Run smaller metrics first so field engineers can review early results.""" - if len(sheets) <= 1: - return list(sheets) - # Split sheets share metrics. Aggregate all counters once, without raw CLI blobs. - counts: dict[str, int] = {} - for bid, mid, total in db.query( - BizStateBatchCommand.batch_id, BizStateBatchCommand.metric_id, - func.sum(BizStateBatchCommand.row_count), - ).filter( - BizStateBatchCommand.batch_id.in_([before_batch_id, after_batch_id]), - BizStateBatchCommand.metric_id.in_({str(s.get("metric_id") or "") for s in sheets}), - ).group_by(BizStateBatchCommand.batch_id, BizStateBatchCommand.metric_id): - counts[mid] = max(counts.get(mid, 0), int(total or 0)) - scored: list[tuple[int, int, dict[str, Any]]] = [] - for i, sheet in enumerate(sheets): - n = counts.get(str(sheet.get("metric_id") or ""), 0) - scored.append((n, i, sheet)) - scored.sort(key=lambda x: (x[0], x[1])) - return [s for _, _, s in scored] - - -def _pending_sheet_meta(sheet: dict[str, Any]) -> dict[str, Any]: - """Placeholder meta so the UI lists all check items while a run is in flight.""" - key_fields = list(sheet.get("key_fields") or []) - compare_fields = list(sheet.get("compare_fields") or []) - return { - "sheet_id": sheet_key(sheet), - "title": sheet_title(sheet), - "metric_id": sheet.get("metric_id") or "", - "key_fields": key_fields, - "iface_fields": list(sheet.get("iface_fields") or []), - "compare_fields": compare_fields, - "display_fields": list(sheet.get("display_fields") or []), - "field_rules": list(sheet.get("field_rules") or []), - "row_filters": list(sheet.get("row_filters") or []), - "ignore_port_changes": sheet.get("ignore_port_changes"), - "mode": "presence" if not compare_fields else "fields", - "status": "pending", - "summary": { - "added": 0, - "removed": 0, - "changed": 0, - "unchanged": 0, - "before_count": 0, - "after_count": 0, - }, - } - - def _str_list(raw: Any) -> list[str]: return [str(x).strip() for x in (raw or []) if str(x).strip()] @@ -884,35 +551,7 @@ def _builtin_source_splits() -> dict[str, list[dict[str, Any]]]: } -def _packaged_zte_status_template_path(): - from pathlib import Path - - return Path(__file__).resolve().parent / "data" / "default_zte_status_template.json" - - -def _load_packaged_zte_status_template() -> dict[str, Any]: - """IOH CN migration sheet set shipped as the built-in status default.""" - path = _packaged_zte_status_template_path() - if not path.is_file(): - return {} - try: - return dict(json.loads(path.read_text(encoding="utf-8")) or {}) - except Exception: - _log.exception("failed to load packaged ZTE status template %s", path) - return {} - - def _default_zte_status_sheets() -> list[dict[str, Any]]: - """Built-in status sheets — prefer packaged IOH CN migration rules.""" - raw = _load_packaged_zte_status_template() - out: list[dict[str, Any]] = [] - for item in list(raw.get("metrics") or []): - sheet = _normalize_sheet(item) - if sheet: - out.append(sheet) - if out: - return out - # Fallback if package missing (tests / incomplete install) return [ *_isis_af_sheets(), _default_sheet_for_metric("interface_brief", compare_roles=("state",)), @@ -929,23 +568,6 @@ def _default_zte_status_sheets() -> list[dict[str, Any]]: ] -def _builtin_status_needs_packaged_upgrade(existing: list[dict[str, Any]]) -> bool: - """True when built-in template still lacks filtered BGP route sheets.""" - mids = {str(s.get("metric_id") or "") for s in existing} - if "bgp_route" not in mids and "l2vpn_mac" not in mids: - return True - has_filtered_route = any( - str(s.get("metric_id") or "") == "bgp_route" and list(s.get("row_filters") or []) - for s in existing - ) - if not has_filtered_route: - return True - packaged_keys = {sheet_key(s) for s in _default_zte_status_sheets()} - have_keys = {sheet_key(s) for s in existing} - # Missing several packaged sheet ids → sync to packaged default - return len(packaged_keys - have_keys) >= 3 - - def _default_zte_config_sheets() -> list[dict[str, Any]]: """Config-intent metrics for cutover / intent-vs-intent compare.""" return [ @@ -1114,7 +736,6 @@ def _parse_metrics_body(body: dict[str, Any]) -> list[dict[str, Any]]: display_fields=disp_arg, row_filters=_normalize_row_filters(body.get("row_filters")), field_rules=rules, - ignore_port_changes=body.get("ignore_port_changes"), ) ] @@ -1213,63 +834,52 @@ def ensure_default_lldp_template(db: Session) -> BizCompareTemplate: def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: name = "ZTE status default" row = db.query(BizCompareTemplate).filter(BizCompareTemplate.name == name).one_or_none() - packaged = _load_packaged_zte_status_template() sheets = _default_zte_status_sheets() - note = str( - packaged.get("note") - or "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP route AF sheets)" - )[:512] - iface_rules = list(packaged.get("iface_normalize_rules") or []) + # Volatile counters must not stay in compare_fields on upgraded installs. + _STRIP_COMPARE: dict[str, frozenset[str]] = { + "interface_detail": frozenset({"input_bps", "output_bps", "in_util", "out_util"}), + "optical_brief": frozenset({"rx_power", "tx_power"}), + "bgp_peer": frozenset({"pfx_rcd"}), + } if row: existing = template_metrics(row) - if _builtin_status_needs_packaged_upgrade(existing) and sheets: - _apply_sheets_to_row(row, sheets) - row.note = note - row.updated_at = _utcnow() - if iface_rules: - _set_template_iface_normalize(row, iface_rules) - elif not template_iface_normalize(row): - # Packaged IOH rules use empty normalize; leave empty when explicit - _set_template_iface_normalize(row, []) - db.commit() - db.refresh(row) - return row - # Incremental patches for already-upgraded installs - changed = False + want = {s["metric_id"] for s in sheets} + have = {s["metric_id"] for s in existing} + changed = bool(want - have) upgraded: list[dict[str, Any]] = [] - by_sid = {sheet_key(s): s for s in sheets} + by_want = {s["metric_id"]: s for s in sheets} for s in existing: cur = dict(s) mid = str(cur.get("metric_id") or "") - sid = sheet_key(cur) if mid == "arp" and not cur.get("row_filters"): - src = by_sid.get(sid) or next( - (x for x in sheets if x.get("metric_id") == "arp"), None - ) - cur["row_filters"] = list( - (src or {}).get("row_filters") or arp_dynamic_row_filters() - ) - if not cur.get("field_rules") and src and src.get("field_rules"): - cur["field_rules"] = list(src["field_rules"]) + cur["row_filters"] = list(by_want.get("arp", {}).get("row_filters") or arp_dynamic_row_filters()) + if not cur.get("field_rules") and by_want.get("arp", {}).get("field_rules"): + cur["field_rules"] = list(by_want["arp"]["field_rules"]) changed = True + strip = _STRIP_COMPARE.get(mid) + if strip: + old_cmp = list(cur.get("compare_fields") or []) + new_cmp = [f for f in old_cmp if f not in strip] + if new_cmp != old_cmp: + cur["compare_fields"] = new_cmp + want_disp = list((by_want.get(mid) or {}).get("display_fields") or []) + if want_disp: + cur["display_fields"] = want_disp + changed = True upgraded.append(_normalize_sheet(cur) or cur) - have_mids = {str(s.get("metric_id") or "") for s in upgraded} - for s in sheets: - if str(s.get("metric_id") or "") not in have_mids: - # Only append wholly missing metrics (e.g. bgp_route family) - if str(s.get("metric_id") or "") == "bgp_route" and "bgp_route" not in have_mids: - upgraded.extend( - [x for x in sheets if x.get("metric_id") == "bgp_route"] - ) - have_mids.add("bgp_route") - changed = True - elif str(s.get("metric_id") or "") not in have_mids: + if want - have: + for s in sheets: + if s["metric_id"] not in have: upgraded.append(s) - have_mids.add(str(s.get("metric_id") or "")) - changed = True + changed = True if changed: _apply_sheets_to_row(row, upgraded if upgraded else sheets) - row.note = note + row.note = "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)" + row.updated_at = _utcnow() + db.commit() + db.refresh(row) + if not template_iface_normalize(row): + _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) row.updated_at = _utcnow() db.commit() db.refresh(row) @@ -1277,12 +887,12 @@ def ensure_default_zte_status_template(db: Session) -> BizCompareTemplate: row = BizCompareTemplate( id=uuid4().hex, name=name, - note=note, + note="Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP/LLDP)", created_at=_utcnow(), updated_at=_utcnow(), ) _apply_sheets_to_row(row, sheets) - _set_template_iface_normalize(row, iface_rules) + _set_template_iface_normalize(row, default_zte_iface_normalize_rules()) db.add(row) db.commit() db.refresh(row) @@ -1338,10 +948,10 @@ def ensure_default_templates(db: Session) -> None: def upgrade_builtin_split_sheets(db: Session) -> None: - """Upgrade built-in ZTE status template to packaged AF / BGP route sheets. + """Split unfiltered whole-table sheets on the built-in ZTE template only. - Custom templates are left alone. Built-in is replaced wholesale when it - still lacks filtered ``bgp_route`` sheets (IOH CN migration default). + A sheet is replaced when its id is still the source metric and it has no + row filters. Custom templates and already-split sheets are left alone. """ row = ( db.query(BizCompareTemplate) @@ -1351,18 +961,6 @@ def upgrade_builtin_split_sheets(db: Session) -> None: if not row: return existing = template_metrics(row) - packaged_sheets = _default_zte_status_sheets() - if _builtin_status_needs_packaged_upgrade(existing) and packaged_sheets: - packaged = _load_packaged_zte_status_template() - _apply_sheets_to_row(row, packaged_sheets) - row.note = str( - packaged.get("note") - or "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP route AF sheets)" - )[:512] - _set_template_iface_normalize(row, list(packaged.get("iface_normalize_rules") or [])) - row.updated_at = _utcnow() - db.commit() - return splits = _builtin_source_splits() out: list[dict[str, Any]] = [] changed = False @@ -1388,8 +986,7 @@ def list_templates(db: Session) -> list[dict[str, Any]]: ensure_default_templates(db) upgrade_builtin_split_sheets(db) rows = db.query(BizCompareTemplate).order_by(BizCompareTemplate.name.asc()).all() - usage = dict(db.query(BizCompareJob.template_id, func.count(BizCompareJob.id)).group_by(BizCompareJob.template_id).all()) - return [{**_template_out(t), "job_count": usage.get(t.id, 0)} for t in rows] + return [_template_out(t) for t in rows] def list_metric_schemas() -> list[dict[str, Any]]: @@ -1426,7 +1023,6 @@ def list_row_filter_presets() -> list[dict[str, Any]]: def create_template(db: Session, body: dict[str, Any]) -> dict[str, Any]: - validate_template_body(body) sheets = _parse_metrics_body(body) t = BizCompareTemplate( id=uuid4().hex, @@ -1449,7 +1045,6 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict t = db.get(BizCompareTemplate, template_id) if not t: raise HTTPException(status_code=404, detail="template_not_found") - validate_template_body(body, partial=True) if "name" in body: t.name = str(body.get("name") or "")[:256] if "note" in body: @@ -1471,7 +1066,6 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict "display_fields", "row_filters", "field_rules", - "ignore_port_changes", ) ): # Prefer explicit metrics; otherwise merge into current sheets from legacy keys @@ -1509,8 +1103,6 @@ def update_template(db: Session, template_id: str, body: dict[str, Any]) -> dict first["row_filters"] = _normalize_row_filters(body.get("row_filters")) if "field_rules" in body: first["field_rules"] = _normalize_field_rules(body.get("field_rules")) - if "ignore_port_changes" in body: - first["ignore_port_changes"] = body["ignore_port_changes"] sheets[0] = _normalize_sheet(first) or first _apply_sheets_to_row(t, sheets) t.updated_at = _utcnow() @@ -1522,8 +1114,6 @@ def delete_template(db: Session, template_id: str) -> None: t = db.get(BizCompareTemplate, template_id) if not t: raise HTTPException(status_code=404, detail="template_not_found") - if db.query(BizCompareJob.id).filter(BizCompareJob.template_id == template_id).first(): - raise HTTPException(status_code=409, detail="template_in_use") db.delete(t) db.commit() @@ -1616,191 +1206,61 @@ def _port_map_dict(db: Session, mapping_id: str) -> dict[str, str]: return {str(r.before_if): str(r.after_if) for r in rows if r.before_if and r.after_if} -def _load_metric_rows( - db: Session, - *, - batch_id: str, - metric_id: str, - on_chunk: Callable[[int], None] | None = None, - row_filters: list[dict[str, Any]] | None = None, -) -> list[dict[str, Any]]: - """Load metric rows in keyset chunks (stable on million-row sheets). - - When ``row_filters`` are SQL-pushdown-safe on PostgreSQL, they are applied in - the SELECT (critical for BGP afi/vrf sheet splits — avoids loading 1M+ then - discarding). Otherwise filters are applied in Python after each chunk. - """ - from .compare_sql import ( - _dialect_is_postgres, - _filters_sql_compatible, - compile_row_filters_sql, - ) - - filters = [f for f in (row_filters or []) if isinstance(f, dict)] - pushdown = bool( - filters and _dialect_is_postgres(db) and _filters_sql_compatible(filters) - ) - filter_sql, filter_params = ("TRUE", {}) - if pushdown: - filter_sql, filter_params = compile_row_filters_sql(filters) - +def _load_metric_rows(db: Session, *, batch_id: str, metric_id: str) -> list[dict[str, Any]]: if metric_id == "lldp_neighbor": - out: list[dict[str, Any]] = [] - last_id = "" - while True: - q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id) - if last_id: - q = q.filter(BizStateLldpNeighbor.id > last_id) - chunk = q.order_by(BizStateLldpNeighbor.id.asc()).limit(_LOAD_YIELD_PER).all() - if not chunk: - break - for n in chunk: - row = { - "local_if": n.local_if, - "remote_sys": n.remote_sys, - "remote_if": n.remote_if, - "remote_ip": n.remote_ip, - "protocol": n.protocol, - "_netx": { - "batch_id": batch_id, - "batch_command_id": n.batch_command_id or "", - "task_id": n.task_id or "", - "ne_id": n.ne_id or "", - "collected_at": n.collected_at.isoformat() + "Z" - if n.collected_at - else None, - "row_id": n.id, - }, - } - if filters and not pushdown and not all( - row_matches_filter(row, f) for f in filters - ): - db.expunge(n) - continue - out.append(row) - db.expunge(n) - last_id = str(chunk[-1].id) - if on_chunk: - on_chunk(len(out)) - if len(chunk) < _LOAD_YIELD_PER: - break - if filters and not pushdown: - return apply_row_filters(out, filters) - return out - - # Generic tabular metrics — PG + pushdown uses SQL keyset with JSON filters - from ..models import BizStateMetricRow - from sqlalchemy import text as sql_text - - out: list[dict[str, Any]] = [] - last_seq = -1 - last_id = "" - while True: - if pushdown: - params = { - "bid": batch_id, - "mid": metric_id, - "last_seq": last_seq, - "last_id": last_id, - "lim": int(_LOAD_YIELD_PER), - **filter_params, + rows = ( + db.query(BizStateLldpNeighbor) + .filter(BizStateLldpNeighbor.batch_id == batch_id) + .all() + ) + return [ + { + "local_if": n.local_if, + "remote_sys": n.remote_sys, + "remote_if": n.remote_if, + "remote_ip": n.remote_ip, + "protocol": n.protocol, + "_netx": { + "batch_id": batch_id, + "batch_command_id": n.batch_command_id or "", + "task_id": n.task_id or "", + "ne_id": n.ne_id or "", + "collected_at": n.collected_at.isoformat() + "Z" if n.collected_at else None, + "row_id": n.id, + }, } - keyset = ( - "(seq > :last_seq OR (seq = :last_seq AND id > :last_id))" - if last_id - else "TRUE" - ) - rows = db.execute( - sql_text( - f""" - SELECT id, batch_command_id, task_id, ne_id, seq, data_json, collected_at - FROM biz_state_metric_row - WHERE batch_id = :bid - AND metric_id = :mid - AND ({filter_sql}) - AND ({keyset}) - ORDER BY seq ASC, id ASC - LIMIT :lim - """ - ), - params, - ).mappings().all() - if not rows: - break - for r in rows: - data = dict(r["data_json"] or {}) - collected = r["collected_at"] - out.append( - { - **data, - "_netx": { - "batch_id": batch_id, - "batch_command_id": str(r["batch_command_id"] or ""), - "task_id": str(r["task_id"] or ""), - "ne_id": str(r["ne_id"] or ""), - "collected_at": collected.isoformat() + "Z" - if collected is not None - else None, - "row_id": str(r["id"]), - }, - } - ) - last_seq = int(rows[-1]["seq"] or 0) - last_id = str(rows[-1]["id"]) - if on_chunk: - on_chunk(len(out)) - if len(rows) < _LOAD_YIELD_PER: - break - continue + for n in rows + ] + # Generic tabular metrics (ISIS / interface / ARP / ND6 / BGP …) + from ..models import BizStateMetricRow - q = db.query(BizStateMetricRow).filter( + rows = ( + db.query(BizStateMetricRow) + .filter( BizStateMetricRow.batch_id == batch_id, BizStateMetricRow.metric_id == metric_id, ) - if last_id: - q = q.filter( - or_( - BizStateMetricRow.seq > last_seq, - and_( - BizStateMetricRow.seq == last_seq, - BizStateMetricRow.id > last_id, - ), - ) - ) - chunk = ( - q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) - .limit(_LOAD_YIELD_PER) - .all() - ) - if not chunk: - break - for r in chunk: - row = { + .order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) + .all() + ) + if rows: + # Raw rows only — filtering belongs to the compare sheet template + # (``row_filters``), not metric-specific branches here. + # ``_netx`` is collector provenance (stripped before field compare). + return [ + { **dict(r.data_json or {}), "_netx": { "batch_id": batch_id, "batch_command_id": r.batch_command_id or "", "task_id": r.task_id or "", "ne_id": r.ne_id or "", - "collected_at": r.collected_at.isoformat() + "Z" - if r.collected_at - else None, + "collected_at": r.collected_at.isoformat() + "Z" if r.collected_at else None, "row_id": r.id, }, } - if filters and not all(row_matches_filter(row, f) for f in filters): - db.expunge(r) - continue - out.append(row) - db.expunge(r) - last_seq = int(chunk[-1].seq or 0) - last_id = str(chunk[-1].id) - if on_chunk: - on_chunk(len(out)) - if len(chunk) < _LOAD_YIELD_PER: - break - if out: - return out + for r in rows + ] # Known metric with zero rows is OK; unknown metric still errors if metric_id in metric_field_map(): return [] @@ -1858,7 +1318,6 @@ def _job_out(j: BizCompareJob) -> dict[str, Any]: "mode": j.mode, "status": j.status, "enabled_sheet_ids": _str_list(getattr(j, "enabled_sheet_ids", None)), - "store_unchanged": normalize_store_unchanged(getattr(j, "store_unchanged", None)), "note": j.note, "updated_at": j.updated_at.isoformat() + "Z" if j.updated_at else None, } @@ -1879,16 +1338,6 @@ def list_jobs(db: Session) -> list[dict[str, Any]]: return [_job_out(j) for j in rows] -def _validate_job_sheets(db: Session, template_id: str, enabled: list[str]) -> None: - tpl = db.get(BizCompareTemplate, template_id) - if not tpl: - raise HTTPException(status_code=404, detail="template_not_found") - sheets = {sheet_key(s) for s in template_metrics(tpl)} - unknown = sorted(set(enabled) - sheets) - if unknown: - raise HTTPException(status_code=400, detail={"error": "unknown_enabled_sheets", "sheet_ids": unknown}) - - def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: ensure_default_templates(db) template_id = str(body.get("template_id") or "").strip() @@ -1899,7 +1348,6 @@ def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: if not db.get(BizCompareTemplate, template_id): raise HTTPException(status_code=404, detail="template_not_found") enabled = _str_list(body.get("enabled_sheet_ids")) - _validate_job_sheets(db, template_id, enabled) j = BizCompareJob( id=uuid4().hex, name=str(body.get("name") or "compare")[:256], @@ -1912,7 +1360,6 @@ def create_job(db: Session, body: dict[str, Any]) -> dict[str, Any]: mode=str(body.get("mode") or "manual")[:16], status="ready", enabled_sheet_ids=enabled, - store_unchanged=normalize_store_unchanged(body.get("store_unchanged")), note=str(body.get("note") or "")[:512], created_at=_utcnow(), updated_at=_utcnow(), @@ -1928,8 +1375,6 @@ def update_job(db: Session, job_id: str, body: dict[str, Any]) -> dict[str, Any] j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") - _validate_job_sheets(db, str(body.get("template_id", j.template_id) or ""), - _str_list(body.get("enabled_sheet_ids", j.enabled_sheet_ids))) for key in ( "name", "template_id", @@ -1946,8 +1391,6 @@ def update_job(db: Session, job_id: str, body: dict[str, Any]) -> dict[str, Any] setattr(j, key, str(body.get(key) or "")) if "enabled_sheet_ids" in body: j.enabled_sheet_ids = _str_list(body.get("enabled_sheet_ids")) - if "store_unchanged" in body: - j.store_unchanged = normalize_store_unchanged(body.get("store_unchanged")) j.updated_at = _utcnow() db.commit() return _job_out(j) @@ -1957,9 +1400,6 @@ def delete_job(db: Session, job_id: str) -> None: j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") - if db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id, - BizCompareRun.status.in_(("queued", "running"))).first(): - raise HTTPException(status_code=409, detail="compare_running") run_ids = [ rid for (rid,) in db.query(BizCompareRun.id).filter(BizCompareRun.job_id == job_id).all() ] @@ -1977,8 +1417,6 @@ def delete_run(db: Session, run_id: str) -> dict[str, Any]: r = db.get(BizCompareRun, run_id) if not r: raise HTTPException(status_code=404, detail="run_not_found") - if r.status in ("queued", "running"): - raise HTTPException(status_code=409, detail="compare_running") job_id = str(r.job_id or "") db.query(BizCompareDiff).filter(BizCompareDiff.run_id == run_id).delete( synchronize_session=False @@ -2006,40 +1444,6 @@ def _resolve_after_batch(db: Session, job: BizCompareJob) -> str: return str(latest.id) if latest else "" -def _sheet_result_envelope( - sheet: dict[str, Any], - *, - key_fields: list[str], - iface_fields: list[str], - compare_fields: list[str], - display_fields: list[str], - row_filters: list[Any], - field_rules: list[Any], - ignore_ports: bool | None, - mode: str, - summary: dict[str, Any], - diffs: list[Any], - mapping_stats: dict[str, Any], -) -> dict[str, Any]: - return { - "sheet_id": sheet_key(sheet), - "title": sheet_title(sheet), - "metric_id": sheet["metric_id"], - "key_fields": key_fields, - "iface_fields": iface_fields, - "compare_fields": compare_fields, - "display_fields": display_fields, - "row_filters": row_filters, - "field_rules": field_rules, - "ignore_port_changes": ignore_ports, - "mode": mode, - "status": "done", - "summary": summary, - "diffs": diffs, - "mapping_stats": mapping_stats, - } - - def _run_sheet( db: Session, *, @@ -2048,8 +1452,6 @@ def _run_sheet( after_batch_id: str, port_map: dict[str, str], iface_normalize_rules: list[dict[str, str]] | None = None, - store_unchanged: str = "auto", - on_load_progress: Callable[..., None] | None = None, ) -> dict[str, Any]: key_fields = list(sheet.get("key_fields") or []) iface_fields = list(sheet.get("iface_fields") or []) @@ -2070,117 +1472,10 @@ def _run_sheet( ignore_ports = sheet.get("ignore_port_changes") if ignore_ports is not None: ignore_ports = bool(ignore_ports) - mid = sheet["metric_id"] - - def _emit_load(side: str, n: int, **meta: Any) -> None: - if not on_load_progress: - return - try: - on_load_progress(side, n, **meta) - except TypeError: - on_load_progress(side, n) - - # PostgreSQL path: pushdown-safe sheets join in-DB (BGP-scale). - from .compare_sql import SqlCompareSkip, run_sql_sheet_compare, sql_compare_skip_reason - - skip_reason = sql_compare_skip_reason( - db, - sheet, - port_map=port_map, - iface_normalize_rules=iface_normalize_rules, - ) - if not skip_reason: - try: - result = run_sql_sheet_compare( - db, - sheet=sheet, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - store_unchanged=store_unchanged, - on_progress=_emit_load, - ) - summary = dict(result["summary"]) - # Engine already sets raw counts / policy; keep keys stable - if "unchanged_policy" not in summary: - summary["unchanged_policy"] = resolve_unchanged_policy( - store_unchanged, - before_n=int(summary.get("before_count") or 0), - after_n=int(summary.get("after_count") or 0), - ) - return _sheet_result_envelope( - sheet, - key_fields=key_fields, - iface_fields=iface_fields, - compare_fields=compare_fields, - display_fields=display_fields, - row_filters=row_filters, - field_rules=field_rules, - ignore_ports=ignore_ports, - mode=mode, - summary=summary, - diffs=list(result.get("diffs") or []), - mapping_stats=dict(result.get("mapping_stats") or {}), - ) - except SqlCompareSkip as skip: - skip_reason = skip.reason or "skip" - _log.info( - "python compare sheet=%s metric=%s skip_sql=%s", - sheet_key(sheet), - mid, - skip_reason, - ) - try: - db.rollback() - except Exception: - pass - except Exception: - _log.exception( - "sql compare fallback sheet=%s metric=%s — using Python engine", - sheet_key(sheet), - mid, - ) - skip_reason = "sql_error_fallback" - # Roll back aborted SQL transaction so Python path can use the session - try: - db.rollback() - except Exception: - pass - else: - _log.info( - "python compare sheet=%s metric=%s skip_sql=%s", - sheet_key(sheet), - mid, - skip_reason, - ) - - _emit_load("before", 0, engine="python", note=skip_reason or "python", phase="loading") - - def _before_chunk(n: int) -> None: - _emit_load("before", n, engine="python", note=skip_reason or "python", phase="loading") - - def _after_chunk(n: int) -> None: - _emit_load("after", n, engine="python", note=skip_reason or "python", phase="loading") - - before_raw = _load_metric_rows( - db, - batch_id=before_batch_id, - metric_id=mid, - on_chunk=_before_chunk, - row_filters=row_filters, - ) - after_raw = _load_metric_rows( - db, - batch_id=after_batch_id, - metric_id=mid, - on_chunk=_after_chunk, - row_filters=row_filters, - ) - # Filters already applied in load when pushdown-safe; keep apply for safety - before_rows = apply_row_filters(before_raw, row_filters) if row_filters else before_raw - after_rows = apply_row_filters(after_raw, row_filters) if row_filters else after_raw - policy = resolve_unchanged_policy( - store_unchanged, before_n=len(before_rows), after_n=len(after_rows) - ) + before_raw = _load_metric_rows(db, batch_id=before_batch_id, metric_id=sheet["metric_id"]) + after_raw = _load_metric_rows(db, batch_id=after_batch_id, metric_id=sheet["metric_id"]) + before_rows = apply_row_filters(before_raw, row_filters) + after_rows = apply_row_filters(after_raw, row_filters) result = compare_rows( before_rows=before_rows, after_rows=after_rows, @@ -2191,36 +1486,42 @@ def _run_sheet( field_rules=field_rules, iface_normalize_rules=iface_normalize_rules, ignore_port_changes=ignore_ports, - include_unchanged=bool(policy["include"]), - unchanged_limit=policy.get("limit"), - compact_unchanged=bool(policy.get("compact")), ) summary = dict(result["summary"]) summary["before_raw_count"] = len(before_raw) summary["after_raw_count"] = len(after_raw) summary["row_filters"] = len(row_filters) - summary["unchanged_policy"] = policy - summary.setdefault("engine", "python") - return _sheet_result_envelope( - sheet, - key_fields=key_fields, - iface_fields=iface_fields, - compare_fields=compare_fields, - display_fields=display_fields, - row_filters=row_filters, - field_rules=field_rules, - ignore_ports=ignore_ports, - mode=mode, - summary=summary, - diffs=list(result.get("diffs") or []), - mapping_stats=dict(result.get("mapping_stats") or {}), - ) + return { + "sheet_id": sheet_key(sheet), + "title": sheet_title(sheet), + "metric_id": sheet["metric_id"], + "key_fields": key_fields, + "iface_fields": iface_fields, + "compare_fields": compare_fields, + "display_fields": display_fields, + "row_filters": row_filters, + "field_rules": field_rules, + "ignore_port_changes": ignore_ports, + "mode": mode, + "summary": summary, + "diffs": result["diffs"], + "mapping_stats": result["mapping_stats"], + } -def _validate_compare_job( +def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]: + lock = _job_compare_lock(job_id) + if not lock.acquire(blocking=False): + raise HTTPException(status_code=409, detail="compare_already_running") + try: + return _run_compare_unlocked(db, job_id, force_after_batch_id=force_after_batch_id) + finally: + lock.release() + + +def _run_compare_unlocked( db: Session, job_id: str, *, force_after_batch_id: str = "" ) -> dict[str, Any]: - """Resolve job/template/batches/sheets; raises HTTPException on bad input.""" j = db.get(BizCompareJob, job_id) if not j: raise HTTPException(status_code=404, detail="job_not_found") @@ -2231,237 +1532,19 @@ def _validate_compare_job( after_batch_id = str(force_after_batch_id or "").strip() or _resolve_after_batch(db, j) if not before_batch_id or not after_batch_id: raise HTTPException(status_code=400, detail="before_and_after_batch_required") - source_batches = [db.get(BizStateBatch, bid) for bid in (before_batch_id, after_batch_id)] - if not all(source_batches): + if not db.get(BizStateBatch, before_batch_id) or not db.get(BizStateBatch, after_batch_id): raise HTTPException(status_code=404, detail="batch_not_found") - if any(b.status != "success" for b in source_batches): - raise HTTPException(status_code=409, detail="source_batch_not_complete") - # Existing records may predate write validation. Do not normalize invalid - # sheets away or run unknown rules as successful comparisons. - if tpl.metrics_json: - validate_template_body({"metrics": tpl.metrics_json, - "iface_normalize_rules": tpl.iface_normalize_json}) sheets_cfg = template_metrics(tpl) if not sheets_cfg: raise HTTPException(status_code=400, detail="template_has_no_metrics") - _validate_job_sheets(db, tpl.id, _str_list(j.enabled_sheet_ids)) sheets_cfg = _filter_enabled_sheets(sheets_cfg, getattr(j, "enabled_sheet_ids", None)) if not sheets_cfg: raise HTTPException(status_code=400, detail="no_enabled_sheets") - sheets_cfg = _order_sheets_small_first( - db, - sheets_cfg, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - ) - return { - "job": j, - "template": tpl, - "before_batch_id": before_batch_id, - "after_batch_id": after_batch_id, - "sheets_cfg": sheets_cfg, - } - -def _create_running_run( - db: Session, - *, - job: BizCompareJob, - tpl: BizCompareTemplate, - before_batch_id: str, - after_batch_id: str, - sheets_cfg: list[dict[str, Any]], -) -> BizCompareRun: - first_metric = str(sheets_cfg[0].get("metric_id") or "") - pending_sheets = [_pending_sheet_meta(s) for s in sheets_cfg] - mapping = db.get(BizPortMapping, job.mapping_id) if job.mapping_id else None - config_snapshot = deepcopy({ - "version": 1, - "template": _template_out(tpl), - "sheets": sheets_cfg, - "port_map": _port_map_dict(db, str(job.mapping_id or "")), - "store_unchanged": normalize_store_unchanged(getattr(job, "store_unchanged", None)), - "job_name": job.name or "", - "before_task_id": job.before_task_id or "", - "after_task_id": job.after_task_id or "", - "mapping_name": (mapping.name if mapping else "") or "", - }) - run = BizCompareRun( - id=uuid4().hex, - job_id=job.id, - template_id=tpl.id, - mapping_id=job.mapping_id, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - metric_id=first_metric, - status="running", - summary_json={ - "config_snapshot": config_snapshot, - "progress": { - "phase": "queued", - "sheet_index": 0, - "sheet_total": len(sheets_cfg), - "sheet_id": "", - "sheet_title": "", - "elapsed_ms": 0, - }, - "sheet_count": len(sheets_cfg), - "added": 0, - "removed": 0, - "changed": 0, - "unchanged": 0, - "duplicate": 0, - "before_count": 0, - "after_count": 0, - "sheets": pending_sheets, - }, - diffs_json=[], - mapping_stats_json={}, - message="queued", - created_at=_utcnow(), - ) - db.add(run) - db.commit() - db.refresh(run) - return run - - -def _run_config_snapshot(run: BizCompareRun) -> dict[str, Any] | None: - snapshot = (run.summary_json or {}).get("config_snapshot") - if isinstance(snapshot, dict) and snapshot.get("version") == 1: - return snapshot - return None - - -def _run_transform_config( - db: Session, run: BizCompareRun, -) -> tuple[dict[str, str], list[dict[str, str]]]: - snapshot = _run_config_snapshot(run) - if snapshot is not None: - return dict(snapshot.get("port_map") or {}), normalize_iface_rules( - (snapshot.get("template") or {}).get("iface_normalize_rules") or [], - ) - # Legacy runs did not record these settings; retain their fallback behavior. - tpl = db.get(BizCompareTemplate, run.template_id) if run.template_id else None - return _port_map_dict(db, str(run.mapping_id or "")), template_iface_normalize(tpl) - - -def _set_run_progress( - db: Session, - run: BizCompareRun, - *, - phase: str, - sheet_index: int, - sheet_total: int, - sheet: dict[str, Any] | None, - started_mono: float, - extra: dict[str, Any] | None = None, - detach: bool = False, -) -> None: - """Update run progress. - - ``detach=True`` writes via a fresh session so SQL compare can keep an open - transaction (TEMP CTAS) without mid-flight commits on the worker ``db``. - """ - elapsed_ms = int((time.monotonic() - started_mono) * 1000) - progress = { - "phase": phase, - "sheet_index": sheet_index, - "sheet_total": sheet_total, - "sheet_id": sheet_key(sheet) if sheet else "", - "sheet_title": sheet_title(sheet) if sheet else "", - "elapsed_ms": elapsed_ms, - } - if extra: - progress.update(extra) - title = progress["sheet_title"] or progress["sheet_id"] or "" - message = ( - f"{phase} {sheet_index}/{sheet_total}" - + (f" · {title}" if title else "") - + f" · {elapsed_ms // 1000}s" - )[:1024] - - if detach: - from ..db import SessionLocal - - s = SessionLocal() - try: - r = s.get(BizCompareRun, str(run.id)) - if not r: - return - prev = dict(r.summary_json or {}) - prev["progress"] = progress - r.summary_json = prev - if str(r.status or "") != "cancelled": - r.status = "running" - r.message = message - s.commit() - # Mirror into worker instance for later in-memory reads (do not commit db) - prev_w = dict(run.summary_json or {}) - prev_w["progress"] = progress - run.summary_json = prev_w - if str(run.status or "") != "cancelled": - run.message = message - finally: - s.close() - return - - prev = dict(run.summary_json or {}) - prev["progress"] = progress - run.summary_json = prev - # Re-read status from DB — cancel may have been committed by another session - # (UI cancel / startup recovery) while this worker still holds a stale "running". - db.expire(run, ["status", "message"]) - if str(run.status or "") != "cancelled": - run.status = "running" - run.message = message - db.commit() - - -def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]: - """Run compare into an existing ``running`` BizCompareRun; persist sheet-by-sheet.""" - run = db.get(BizCompareRun, run_id) - if not run: - raise HTTPException(status_code=404, detail="run_not_found") - snapshot = _run_config_snapshot(run) - j = db.get(BizCompareJob, run.job_id) - if not j: - run.status = "failed" - run.message = "job_not_found" - db.commit() - raise HTTPException(status_code=404, detail="job_not_found") - tpl = db.get(BizCompareTemplate, run.template_id) if snapshot is None else None - if not tpl and snapshot is None: - run.status = "failed" - run.message = "template_not_found" - db.commit() - raise HTTPException(status_code=404, detail="template_not_found") - - before_batch_id = str(run.before_batch_id or "") - after_batch_id = str(run.after_batch_id or "") - if snapshot is not None: - sheets_cfg = deepcopy(snapshot.get("sheets") or []) - else: - sheets_cfg = _filter_enabled_sheets(template_metrics(tpl), getattr(j, "enabled_sheet_ids", None)) - if not sheets_cfg: - run.status = "failed" - run.message = "no_enabled_sheets" - db.commit() - raise HTTPException(status_code=400, detail="no_enabled_sheets") - if snapshot is None: - sheets_cfg = _order_sheets_small_first( - db, sheets_cfg, before_batch_id=before_batch_id, after_batch_id=after_batch_id, - ) - - started_mono = time.monotonic() - store_mode = normalize_store_unchanged( - snapshot.get("store_unchanged") if snapshot is not None else getattr(j, "store_unchanged", None), - ) - pmap, norm_rules = _run_transform_config(db, run) - unchanged_listed_total = 0 - unchanged_truncated_any = False - unchanged_compact_any = False + pmap = _port_map_dict(db, j.mapping_id) + norm_rules = template_iface_normalize(tpl) + sheet_results: list[dict[str, Any]] = [] agg = { "before_count": 0, "after_count": 0, @@ -2472,485 +1555,87 @@ def _execute_compare_into_run(db: Session, run_id: str) -> dict[str, Any]: "duplicate": 0, } mapping_by_metric: dict[str, Any] = {} + for sheet in sheets_cfg: + one = _run_sheet( + db, + sheet=sheet, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + port_map=pmap, + iface_normalize_rules=norm_rules, + ) + sheet_results.append(one) + s = one["summary"] + for k in agg: + agg[k] += int(s.get(k) or 0) + mapping_by_metric[sheet_key(one)] = one["mapping_stats"] + + first = sheet_results[0] field_counts: dict[str, int] = {} - total = len(sheets_cfg) + for s in sheet_results: + for d in list(s.get("diffs") or []): + if str(d.get("kind") or "") != "changed": + continue + for fname in d.get("changes") or {}: + field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1 + top_fields = sorted( + [{"field": k, "count": v} for k, v in field_counts.items()], + key=lambda x: (-int(x["count"]), str(x["field"])), + )[:8] - # Prefer seeded pending sheets from create; realign to small-first order - prev_metas = list((run.summary_json or {}).get("sheets") or []) - by_key = {sheet_key(m): m for m in prev_metas} - sheet_metas = [ - by_key.get(sheet_key(s)) or _pending_sheet_meta(s) for s in sheets_cfg - ] - - def _publish_sheets() -> None: - prev = dict(run.summary_json or {}) - prev["sheets"] = list(sheet_metas) - prev.update({k: agg[k] for k in agg}) - run.summary_json = prev - db.expire(run, ["status"]) - # Never resurrect cancelled while publishing incremental sheet metas - if str(run.status or "") == "cancelled": - db.commit() - return - db.commit() - - try: - for idx, sheet in enumerate(sheets_cfg, start=1): - if _run_is_cancelled(db, run_id): - run = db.get(BizCompareRun, run_id) or run - return get_run(db, run.id) - sid = sheet_key(sheet) - # Mark current sheet running in the sidebar list - for meta in sheet_metas: - if sheet_key(meta) == sid: - meta["status"] = "running" - break - _publish_sheets() - - _load_pub = {"t": 0.0, "n": -1, "engine": ""} - - def _on_load( - side: str, - n: int, - *, - engine: str = "python", - note: str = "", - phase: str | None = None, - _idx: int = idx, - _sheet: dict = sheet, - ) -> None: - now = time.monotonic() - eng = str(engine or "python") - # SQL emits sparse updates; Python still throttle chunk spam - if eng != "sql": - if n - _load_pub["n"] < 25_000 and now - _load_pub["t"] < 2.0: - return - _load_pub["t"] = now - _load_pub["n"] = n - _load_pub["engine"] = eng - extra: dict[str, Any] = { - "load_side": side, - "rows_loaded": n, - "engine": eng, - } - if note: - extra["engine_note"] = str(note)[:128] - # SQL path: detach progress commits so TEMP CTAS stays in one txn - _set_run_progress( - db, - run, - phase=str(phase or ("loading" if eng != "sql" else "sql_count")), - sheet_index=_idx, - sheet_total=total, - sheet=_sheet, - started_mono=started_mono, - extra=extra, - detach=(eng == "sql"), - ) - - _set_run_progress( - db, - run, - phase="loading", - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={"engine": "", "engine_note": ""}, - ) - one = _run_sheet( - db, - sheet=sheet, - before_batch_id=before_batch_id, - after_batch_id=after_batch_id, - port_map=pmap, - iface_normalize_rules=norm_rules, - store_unchanged=store_mode, - on_load_progress=_on_load, - ) - s = one["summary"] - listed = int(s.get("unchanged_listed") or 0) - unchanged_listed_total += listed - if s.get("unchanged_truncated"): - unchanged_truncated_any = True - if s.get("unchanged_compact"): - unchanged_compact_any = True - _set_run_progress( - db, - run, - phase="comparing", - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={ - "before_count": int(s.get("before_count") or 0), - "after_count": int(s.get("after_count") or 0), - "diff_rows": int(s.get("added") or 0) - + int(s.get("removed") or 0) - + int(s.get("changed") or 0) - + int(s.get("duplicate") or 0) - + listed, - }, - ) - diffs = list(one.get("diffs") or []) - for d in diffs: - if str(d.get("kind") or "") != "changed": - continue - for fname in d.get("changes") or {}: - field_counts[str(fname)] = field_counts.get(str(fname), 0) + 1 - # Cutover-first: persist fails so the UI can open the fail tab ASAP, - # then write success slim keys (sample / keys mode). - fail_diffs = [d for d in diffs if str(d.get("kind") or "") != "unchanged"] - ok_diffs = [d for d in diffs if str(d.get("kind") or "") == "unchanged"] - mid = sheet_key(one) - # Million-row vpnv4 with many diffs: keep key/row_id/changes only. - if len(fail_diffs) >= _FAIL_COMPACT_MIN: - for d in fail_diffs: - d["before"] = {} - d["after"] = {} - d["mapped_before"] = {} - d["compact"] = True - - def _on_persist( - written: int, - total_n: int, - *, - phase: str, - kind_key: str, - ) -> None: - _set_run_progress( - db, - run, - phase=phase, - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={ - kind_key: total_n, - "persisted": written, - "persist_total": total_n, - }, - # Separate session so chunk commits in persist do not race - # with progress JSON writes on the worker session. - detach=True, - ) - - _set_run_progress( - db, - run, - phase="persisting_fail", - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={ - "fail_rows": len(fail_diffs), - "persisted": 0, - "persist_total": len(fail_diffs), - }, - ) - n_fail = _persist_sheet_diffs( - db, - run_id=run.id, - metric_id=mid, - diffs=fail_diffs, - seq_start=0, - on_progress=lambda w, n: _on_persist( - w, n, phase="persisting_fail", kind_key="fail_rows" - ), - ) - if ok_diffs: - _set_run_progress( - db, - run, - phase="persisting_ok", - sheet_index=idx, - sheet_total=total, - sheet=sheet, - started_mono=started_mono, - extra={ - "ok_rows": len(ok_diffs), - "persisted": 0, - "persist_total": len(ok_diffs), - }, - ) - _persist_sheet_diffs( - db, - run_id=run.id, - metric_id=mid, - diffs=ok_diffs, - seq_start=n_fail, - on_progress=lambda w, n: _on_persist( - w, n, phase="persisting_ok", kind_key="ok_rows" - ), - ) - for k in agg: - agg[k] += int(s.get(k) or 0) - mapping_by_metric[mid] = one["mapping_stats"] - done_meta = { - "sheet_id": one.get("sheet_id") or one["metric_id"], - "title": one.get("title") or one.get("sheet_id") or one["metric_id"], - "metric_id": one["metric_id"], - "key_fields": one["key_fields"], - "iface_fields": one["iface_fields"], - "compare_fields": one["compare_fields"], - "display_fields": one.get("display_fields") or [], - "field_rules": one.get("field_rules") or [], - "row_filters": one.get("row_filters") or [], - "ignore_port_changes": one.get("ignore_port_changes"), - "mode": one["mode"], - "status": "done", - "summary": one["summary"], + run_id = uuid4().hex + # Persist counts/meta only — diffs go to biz_compare_diff rows + summary_payload = { + **agg, + "sheet_count": len(sheet_results), + "top_changed_fields": top_fields, + "sheets": [ + { + "sheet_id": s.get("sheet_id") or s["metric_id"], + "title": s.get("title") or s.get("sheet_id") or s["metric_id"], + "metric_id": s["metric_id"], + "key_fields": s["key_fields"], + "iface_fields": s["iface_fields"], + "compare_fields": s["compare_fields"], + "display_fields": s.get("display_fields") or [], + "field_rules": s.get("field_rules") or [], + "ignore_port_changes": s.get("ignore_port_changes"), + "mode": s["mode"], + "summary": s["summary"], } - replaced = False - for i, meta in enumerate(sheet_metas): - if sheet_key(meta) == mid: - sheet_metas[i] = done_meta - replaced = True - break - if not replaced: - sheet_metas.append(done_meta) - _publish_sheets() - # Drop heavy diffs before next sheet - one.clear() - diffs.clear() - fail_diffs.clear() - ok_diffs.clear() - db.commit() - if _run_is_cancelled(db, run_id): - run = db.get(BizCompareRun, run_id) or run - return get_run(db, run.id) + for s in sheet_results + ], + } - if _run_is_cancelled(db, run_id): - run = db.get(BizCompareRun, run_id) or run - return get_run(db, run.id) - - duration_ms = int((time.monotonic() - started_mono) * 1000) - top_fields = sorted( - [{"field": k, "count": v} for k, v in field_counts.items()], - key=lambda x: (-int(x["count"]), str(x["field"])), - )[:8] - summary_payload = { - **agg, - "sheet_count": len(sheet_metas), - "top_changed_fields": top_fields, - "duration_ms": duration_ms, - "store_unchanged": store_mode, - "unchanged_stored": unchanged_listed_total > 0, - "unchanged_listed": unchanged_listed_total, - "unchanged_truncated": unchanged_truncated_any, - "unchanged_compact": unchanged_compact_any, - "progress": { - "phase": "done", - "sheet_index": total, - "sheet_total": total, - "sheet_id": "", - "sheet_title": "", - "elapsed_ms": duration_ms, - }, - "sheets": sheet_metas, - } - if snapshot is not None: - summary_payload["config_snapshot"] = snapshot - run = db.get(BizCompareRun, run_id) or run - if str(run.status or "") == "cancelled": - return get_run(db, run.id) - run.status = "success" - run.summary_json = summary_payload - run.mapping_stats_json = mapping_by_metric - run.metric_id = str(sheet_metas[0]["metric_id"]) if sheet_metas else run.metric_id - run.message = f"done · {duration_ms // 1000}s" - run.diffs_json = [] - j.updated_at = _utcnow() - if j.mode == "manual": - j.after_batch_id = after_batch_id - db.commit() - return get_run(db, run.id) - except HTTPException as exc: - run = db.get(BizCompareRun, run_id) or run - if str(run.status or "") != "cancelled": - run.status = "failed" - run.message = str(getattr(exc, "detail", "") or exc)[:1024] - prev = dict(run.summary_json or {}) - prog = dict(prev.get("progress") or {}) - prog["phase"] = "failed" - prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000) - prev["progress"] = prog - prev["duration_ms"] = prog["elapsed_ms"] - run.summary_json = prev - db.commit() - raise - except Exception as exc: - _log.exception("compare run failed run=%s job=%s", run_id, j.id) - run = db.get(BizCompareRun, run_id) or run - if str(run.status or "") != "cancelled": - run.status = "failed" - run.message = str(exc)[:1024] - prev = dict(run.summary_json or {}) - prog = dict(prev.get("progress") or {}) - prog["phase"] = "failed" - prog["elapsed_ms"] = int((time.monotonic() - started_mono) * 1000) - prev["progress"] = prog - prev["duration_ms"] = prog["elapsed_ms"] - run.summary_json = prev - db.commit() - raise - - -_INTERRUPT_MARK = "interrupted_by_restart" - - -def _job_has_active_run(db: Session, job_id: str) -> BizCompareRun | None: - return ( - db.query(BizCompareRun) - .filter( - BizCompareRun.job_id == str(job_id or ""), - BizCompareRun.status.in_(("running", "queued")), + run = BizCompareRun( + id=run_id, + job_id=j.id, + template_id=tpl.id, + mapping_id=j.mapping_id, + before_batch_id=before_batch_id, + after_batch_id=after_batch_id, + metric_id=first["metric_id"], + status="success", + summary_json=summary_payload, + diffs_json=[], + mapping_stats_json=mapping_by_metric, + message="", + created_at=_utcnow(), + ) + db.add(run) + db.flush() + for s in sheet_results: + _persist_sheet_diffs( + db, + run_id=run_id, + metric_id=sheet_key(s), + diffs=list(s.get("diffs") or []), ) - .order_by(BizCompareRun.created_at.desc()) - .first() - ) - - -def _run_is_cancelled(db: Session, run_id: str) -> bool: - """Re-read status so user/startup cancel is visible to the worker thread.""" - db.expire_all() - r = db.get(BizCompareRun, run_id) - return bool(r and str(r.status or "") == "cancelled") - - -def recover_interrupted_compares_on_startup(db: Session) -> dict[str, Any]: - """Mark orphaned running/queued compare runs as cancelled after process restart. - - In-memory job locks die with the process; without this, the UI stays on - 「比对中」and blocks a new run. - """ - now = _utcnow() - rows = ( - db.query(BizCompareRun) - .filter(BizCompareRun.status.in_(("running", "queued"))) - .all() - ) - n = 0 - for r in rows: - r.status = "cancelled" - msg = str(r.message or "").strip() - if _INTERRUPT_MARK not in msg: - r.message = f"{msg} | {_INTERRUPT_MARK}".strip(" |")[:1024] - prev = dict(r.summary_json or {}) - prog = dict(prev.get("progress") or {}) - prog["phase"] = "cancelled" - prog["elapsed_ms"] = int(prog.get("elapsed_ms") or 0) - prev["progress"] = prog - # Mark in-flight sheet placeholders so UI does not show fake pass - sheets = list(prev.get("sheets") or []) - for sh in sheets: - st = str(sh.get("status") or "") - if st in ("pending", "running", "queued"): - sh["status"] = "cancelled" - prev["sheets"] = sheets - r.summary_json = prev - n += 1 - if n: - db.commit() - _log.info("startup: cancelled %s interrupted compare run(s)", n) - return {"runs": n, "at": now.isoformat() + "Z"} - - -def cancel_compare_run(db: Session, run_id: str) -> dict[str, Any]: - """Cancel a running/queued compare so a new run can start.""" - r = db.get(BizCompareRun, run_id) - if not r: - raise HTTPException(status_code=404, detail="run_not_found") - st = str(r.status or "") - if st not in ("running", "queued"): - return get_run(db, run_id) - r.status = "cancelled" - msg = str(r.message or "").strip() - r.message = f"{msg} | cancelled_by_user".strip(" |")[:1024] - prev = dict(r.summary_json or {}) - prog = dict(prev.get("progress") or {}) - prog["phase"] = "cancelled" - prev["progress"] = prog - for sh in list(prev.get("sheets") or []): - if str(sh.get("status") or "") in ("pending", "running", "queued"): - sh["status"] = "cancelled" - r.summary_json = prev + j.updated_at = _utcnow() + if j.mode == "manual": + j.after_batch_id = after_batch_id db.commit() - return get_run(db, run_id) - - -def run_compare(db: Session, job_id: str, *, force_after_batch_id: str = "") -> dict[str, Any]: - """Synchronous compare (auto-compare / tests). Blocks the caller until done.""" - if _job_has_active_run(db, job_id): - raise HTTPException(status_code=409, detail="compare_already_running") - lock = _job_compare_lock(job_id) - if not lock.acquire(blocking=False): - raise HTTPException(status_code=409, detail="compare_already_running") - try: - ctx = _validate_compare_job(db, job_id, force_after_batch_id=force_after_batch_id) - run = _create_running_run( - db, - job=ctx["job"], - tpl=ctx["template"], - before_batch_id=ctx["before_batch_id"], - after_batch_id=ctx["after_batch_id"], - sheets_cfg=ctx["sheets_cfg"], - ) - return _execute_compare_into_run(db, run.id) - finally: - lock.release() - - -def enqueue_compare( - db: Session, job_id: str, *, force_after_batch_id: str = "" -) -> dict[str, Any]: - """Create a ``running`` run and execute compare on a daemon thread. - - Returns immediately so the HTTP worker / UI stay responsive. Poll - ``GET /compare/runs/{id}`` for progress (``summary.progress``). - """ - if _job_has_active_run(db, job_id): - raise HTTPException(status_code=409, detail="compare_already_running") - lock = _job_compare_lock(job_id) - if not lock.acquire(blocking=False): - raise HTTPException(status_code=409, detail="compare_already_running") - run_id = "" - try: - ctx = _validate_compare_job(db, job_id, force_after_batch_id=force_after_batch_id) - run = _create_running_run( - db, - job=ctx["job"], - tpl=ctx["template"], - before_batch_id=ctx["before_batch_id"], - after_batch_id=ctx["after_batch_id"], - sheets_cfg=ctx["sheets_cfg"], - ) - run_id = run.id - except Exception: - lock.release() - raise - - def _bg() -> None: - from ..db import SessionLocal - - s = SessionLocal() - try: - _execute_compare_into_run(s, run_id) - except Exception: - _log.exception("bg compare failed job=%s run=%s", job_id, run_id) - finally: - s.close() - lock.release() - - threading.Thread( - target=_bg, - name=f"biz-cmp-{run_id[:8]}", - daemon=True, - ).start() - return get_run(db, run_id) + return get_run(db, run.id) def _csv_cell(v: Any) -> str: @@ -2960,7 +1645,7 @@ def _csv_cell(v: Any) -> str: return s -def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) -> Iterator[str]: +def _sheet_csv(sheet: dict[str, Any]) -> str: keys = list(sheet.get("key_fields") or []) key_set = set(keys) compare = [f for f in list(sheet.get("compare_fields") or []) if f not in key_set] @@ -2979,8 +1664,8 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) -> headers.append(f"{f}__post") else: headers.append(f) - yield ",".join(_csv_cell(h) for h in headers) - for d in diffs: + lines = [",".join(_csv_cell(h) for h in headers)] + for d in list(sheet.get("diffs") or []): kind = str(d.get("kind") or "") pre = dict(d.get("mapped_before") or d.get("before") or {}) post = dict(d.get("after") or {}) @@ -3005,11 +1690,8 @@ def _sheet_csv_lines(sheet: dict[str, Any], diffs: Iterable[dict[str, Any]]) -> row.append(pre.get(f, "")) else: row.append(post.get(f, pre.get(f, ""))) - yield ",".join(_csv_cell(x) for x in row) - - -def _sheet_csv(sheet: dict[str, Any]) -> str: - return "\ufeff" + "\n".join(_sheet_csv_lines(sheet, sheet.get("diffs") or [])) + "\n" + lines.append(",".join(_csv_cell(x) for x in row)) + return "\ufeff" + "\n".join(lines) + "\n" def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> dict[str, Any]: @@ -3027,9 +1709,7 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di matched = changed + unchanged diff_count = fail_count pass_rate = ( - compute_pass_rate(success=success_count, fail=fail_count, empty_as=100.0 if total == 0 else 0.0) - if judged - else (100.0 if total == 0 else 0.0) + round((success_count / judged) * 100, 1) if judged else (100.0 if total == 0 else 0.0) ) diff_rate = round((fail_count / judged) * 100, 1) if judged else 0.0 @@ -3043,15 +1723,12 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di st = sa + sr + sc + su sf = sr + sc sj = sf + su - status = str(sh.get("status") or "done") - pending = status in ("pending", "running", "queued") sheet_cards.append( { "sheet_id": sheet_key(sh), "title": sheet_title(sh), "metric_id": sh.get("metric_id") or "", "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"), - "status": status, "added": sa, "removed": sr, "changed": sc, @@ -3061,14 +1738,7 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di "fail_count": sf, "success_count": su, "diff_count": sf, - # Pending sheets must not look like "100% pass" - "pass_rate": None - if pending - else ( - compute_pass_rate(success=su, fail=sf, empty_as=100.0 if st == 0 else 0.0) - if sj - else (100.0 if st == 0 else 0.0) - ), + "pass_rate": round((su / sj) * 100, 1) if sj else (100.0 if st == 0 else 0.0), } ) @@ -3087,9 +1757,6 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di key=lambda x: (-int(x["count"]), str(x["field"])), )[:8] - any_pending = any( - str(sh.get("status") or "") in ("pending", "running", "queued") for sh in sheets - ) return { "added": added, "removed": removed, @@ -3103,18 +1770,11 @@ def _enrich_summary(summary: dict[str, Any], sheets: list[dict[str, Any]]) -> di "fail_count": fail_count, "success_count": success_count, "diff_count": diff_count, - "pass_rate": None if any_pending else pass_rate, + "pass_rate": pass_rate, "diff_rate": diff_rate, - "ok": False if any_pending else fail_count == 0, + "ok": fail_count == 0, "sheet_cards": sheet_cards, "top_changed_fields": top_fields, - "duration_ms": int(summary.get("duration_ms") or 0), - "store_unchanged": str(summary.get("store_unchanged") or ""), - "unchanged_stored": bool(summary.get("unchanged_stored", True)), - "unchanged_listed": int(summary.get("unchanged_listed") or 0), - "unchanged_truncated": bool(summary.get("unchanged_truncated")), - "unchanged_compact": bool(summary.get("unchanged_compact")), - "progress": dict(summary.get("progress") or {}), } @@ -3122,13 +1782,8 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: r = db.get(BizCompareRun, run_id) if not r: raise HTTPException(status_code=404, detail="run_not_found") + tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None summary = dict(r.summary_json or {}) - snapshot = _run_config_snapshot(r) - tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id and snapshot is None else None - template_out = ( - deepcopy(snapshot.get("template")) if snapshot is not None - else (_template_out(tpl) if tpl else None) - ) raw_sheets = _sheet_meta_from_summary(summary, r, tpl) # Never return full diffs in run detail (million-row safe) sheets = [ @@ -3142,38 +1797,28 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: "display_fields": list(sh.get("display_fields") or []), "field_rules": list(sh.get("field_rules") or []), "mode": sh.get("mode") or ("presence" if not sh.get("compare_fields") else "fields"), - "status": str(sh.get("status") or "done"), "summary": dict(sh.get("summary") or {}), } for sh in raw_sheets ] enriched = _enrich_summary(summary, raw_sheets) stored = "rows" if _run_has_diff_rows(db, run_id) else "inline" - job = db.get(BizCompareJob, r.job_id) if r.job_id and snapshot is None else None - mapping = db.get(BizPortMapping, r.mapping_id) if r.mapping_id and snapshot is None else None + job = db.get(BizCompareJob, r.job_id) if r.job_id else None + mapping = db.get(BizPortMapping, r.mapping_id) if r.mapping_id else None before_side = _compare_side( - db, r.before_batch_id, fallback_task_id=( - snapshot.get("before_task_id", "") if snapshot is not None else (job.before_task_id if job else "") - ), + db, r.before_batch_id, fallback_task_id=(job.before_task_id if job else "") ) after_side = _compare_side( - db, r.after_batch_id, fallback_task_id=( - snapshot.get("after_task_id", "") if snapshot is not None else (job.after_task_id if job else "") - ), + db, r.after_batch_id, fallback_task_id=(job.after_task_id if job else "") ) return { "id": r.id, "job_id": r.job_id, - "job_name": ( - snapshot.get("job_name", "") if snapshot is not None else ((job.name if job else "") or "") - ), + "job_name": (job.name if job else "") or "", "template_id": r.template_id, - "template_name": (template_out or {}).get("name") or "", + "template_name": (tpl.name if tpl else "") or "", "mapping_id": r.mapping_id, - "mapping_name": ( - snapshot.get("mapping_name", "") if snapshot is not None else ((mapping.name if mapping else "") or "") - ), - "config_snapshot_version": snapshot["version"] if snapshot is not None else None, + "mapping_name": (mapping.name if mapping else "") or "", "before_batch_id": r.before_batch_id, "after_batch_id": r.after_batch_id, "before": before_side, @@ -3187,7 +1832,7 @@ def get_run(db: Session, run_id: str) -> dict[str, Any]: "mapping_stats": r.mapping_stats_json or {}, "message": r.message, "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, - "template": template_out, + "template": _template_out(tpl) if tpl else None, } @@ -3205,451 +1850,6 @@ def _lookup_sheet(sheets: list[dict[str, Any]], key: str) -> dict[str, Any] | No return None -def _kind_allows(kind_n: str, diff_kind: str) -> bool: - dk = str(diff_kind or "") - if kind_n == "all": - return True - if kind_n == "diff": - return dk in ("removed", "changed") - return dk == kind_n - - -def _parse_qf(raw: Any) -> dict[str, str]: - """Normalize field query map: {field: value} with safe names only.""" - from .compare_sql import _FIELD_RE - - if raw is None or raw == "": - return {} - obj: Any = raw - if isinstance(raw, str): - s = raw.strip() - if not s: - return {} - try: - obj = json.loads(s) - except Exception: - return {} - if not isinstance(obj, dict): - return {} - out: dict[str, str] = {} - for k, v in obj.items(): - name = str(k or "").strip() - val = str(v or "").strip() - if not name or not val or not _FIELD_RE.match(name): - continue - out[name] = val - return out - - -def _split_kw_and_field_tokens(kw: str) -> tuple[str, dict[str, str]]: - """Parse ``direction:out network:1.1.1.1 foo`` → free kw + field map.""" - from .compare_sql import _FIELD_RE - - free: list[str] = [] - fields: dict[str, str] = {} - for tok in str(kw or "").split(): - if ":" in tok: - name, _, val = tok.partition(":") - name = name.strip() - val = val.strip() - if name and val and _FIELD_RE.match(name): - fields[name] = val - continue - if tok.strip(): - free.append(tok.strip()) - return " ".join(free), fields - - -def _kw_match_sql(key_fields: list[str], *, param: str = "kw") -> str: - """OR of ILIKE on key fields (and data_json::text fallback).""" - from .compare_sql import _FIELD_RE, _safe_field - - parts: list[str] = [] - for f in key_fields: - name = str(f or "").strip() - if not name or not _FIELD_RE.match(name): - continue - sf = _safe_field(name) - parts.append(f"lower(trim(both from coalesce(data_json->>'{sf}', ''))) LIKE :{param}") - # Broad fallback so free-text still hits non-key columns (path, next_hop, …) - parts.append(f"lower(data_json::text) LIKE :{param}") - return "(" + " OR ".join(parts) + ")" if parts else f"(lower(data_json::text) LIKE :{param})" - - -def _field_qf_sql(field_q: dict[str, str], *, prefix: str = "qf") -> tuple[str, dict[str, Any]]: - """AND of ILIKE contains on each field.""" - from .compare_sql import _safe_field - - if not field_q: - return "TRUE", {} - parts: list[str] = [] - params: dict[str, Any] = {} - for i, (name, val) in enumerate(field_q.items()): - sf = _safe_field(name) - key = f"{prefix}_{i}" - params[key] = _literal_search_pattern(val) - parts.append( - f"lower(trim(both from coalesce(data_json->>'{sf}', ''))) LIKE :{key}" - ) - return "(" + " AND ".join(parts) + ")", params - - -def _literal_search_pattern(value: str) -> str: - return "%" + value.lower().replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" - - -def _load_metric_rows_for_search( - db: Session, - *, - batch_id: str, - metric_id: str, - row_filters: list[dict[str, Any]] | None, - key_fields: list[str], - kw: str = "", - field_q: dict[str, str] | None = None, - cap: int = _LIVE_SEARCH_LOAD_CAP, -) -> tuple[list[dict[str, Any]], bool]: - """Load rows matching sheet filters + optional free kw / field_q. Returns (rows, truncated).""" - from .compare_sql import ( - _dialect_is_postgres, - _filters_sql_compatible, - compile_row_filters_sql, - ) - from ..models import BizStateMetricRow - from sqlalchemy import text as sql_text - - bid = str(batch_id or "").strip() - mid = str(metric_id or "").strip() - needle = str(kw or "").strip() - fq = {k: v for k, v in (field_q or {}).items() if str(v or "").strip()} - if not bid or not mid or (not needle and not fq): - return [], False - lim = max(1, min(int(cap), _LIVE_SEARCH_LOAD_CAP)) - filters = [f for f in (row_filters or []) if isinstance(f, dict)] - - if mid != "lldp_neighbor" and _dialect_is_postgres(db) and ( - not filters or _filters_sql_compatible(filters) - ): - filter_sql, filter_params = ("TRUE", {}) - if filters and _filters_sql_compatible(filters): - filter_sql, filter_params = compile_row_filters_sql(filters) - qf_sql, qf_params = _field_qf_sql(fq) - search_parts = [qf_sql] - params: dict[str, Any] = { - "bid": bid, - "mid": mid, - "lim": lim + 1, - **filter_params, - **qf_params, - } - if needle: - params["kw"] = _literal_search_pattern(needle) - search_parts.append(_kw_match_sql(key_fields)) - search_sql = " AND ".join(f"({p})" for p in search_parts if p and p != "TRUE") - if not search_sql: - search_sql = "TRUE" - rows = db.execute( - sql_text( - f""" - SELECT id, batch_command_id, task_id, ne_id, seq, data_json, collected_at - FROM biz_state_metric_row - WHERE batch_id = :bid - AND metric_id = :mid - AND ({filter_sql}) - AND ({search_sql}) - ORDER BY seq ASC, id ASC - LIMIT :lim - """ - ), - params, - ).mappings().all() - truncated = len(rows) > lim - rows = rows[:lim] - out: list[dict[str, Any]] = [] - for r in rows: - data = dict(r["data_json"] or {}) - collected = r["collected_at"] - out.append( - { - **data, - "_netx": { - "batch_id": bid, - "batch_command_id": str(r["batch_command_id"] or ""), - "task_id": str(r["task_id"] or ""), - "ne_id": str(r["ne_id"] or ""), - "collected_at": collected.isoformat() + "Z" - if collected is not None - else None, - "row_id": str(r["id"]), - }, - } - ) - return out, truncated - - # Non-PG / fallback: scan with early stop (OK for tests / small sheets) - model = BizStateLldpNeighbor if mid == "lldp_neighbor" else BizStateMetricRow - q = db.query(model).filter(model.batch_id == bid) - if mid == "lldp_neighbor": - q = q.order_by(model.id.asc()) - else: - q = q.filter(model.metric_id == mid).order_by(model.seq.asc(), model.id.asc()) - out = [] - truncated = False - needle_l = needle.lower() - key_set = [str(k).strip() for k in key_fields if str(k).strip()] - for r in q.yield_per(500): - data = ( - {f: getattr(r, f) for f in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")} - if mid == "lldp_neighbor" else dict(r.data_json or {}) - ) - row = { - **data, - "_netx": { - "batch_id": bid, - "batch_command_id": r.batch_command_id or "", - "task_id": r.task_id or "", - "ne_id": r.ne_id or "", - "collected_at": r.collected_at.isoformat() + "Z" - if r.collected_at - else None, - "row_id": r.id, - }, - } - if filters and not all(row_matches_filter(row, f) for f in filters): - db.expunge(r) - continue - if fq: - ok_f = True - for fname, fval in fq.items(): - if fval.lower() not in str(row.get(fname) or "").lower(): - ok_f = False - break - if not ok_f: - db.expunge(r) - continue - if needle_l: - hit = False - for kf in key_set: - if needle_l in str(row.get(kf) or "").lower(): - hit = True - break - if not hit: - blob = json.dumps(data, ensure_ascii=False, default=str).lower() - hit = needle_l in blob - if not hit: - db.expunge(r) - continue - db.expunge(r) - if len(out) >= lim: - truncated = True - break - out.append(row) - return out, truncated - - -def _load_search_key_groups( - db: Session, *, batch_id: str, metric_id: str, keys: set[tuple[str, ...]], - match_keys: list[str], iface_fields: list[str], row_filters: list[dict[str, Any]], - norm_rules: list[dict[str, str]], port_map: dict[str, str], -) -> tuple[list[dict[str, Any]], set[tuple[str, ...]]]: - """Complete candidate key groups so a one-sided search cannot change verdicts. - - Interface transforms run in Python. Other identity fields narrow the SQL - query. If the load budget is exceeded, omit entire groups rather than compare - partial groups and manufacture additions/deletions. - """ - from ..models import BizStateMetricRow - - model = BizStateLldpNeighbor if metric_id == "lldp_neighbor" else BizStateMetricRow - invariant = [f for f in match_keys if f not in iface_fields] - positions = [match_keys.index(f) for f in invariant] - coarse = sorted({tuple(k[i] for i in positions) for k in keys}) - out: list[dict[str, Any]] = [] - complete: set[tuple[str, ...]] = set() - # Bound memory even when a candidate identifies a very large duplicate group. - budget = _LIVE_SEARCH_GROUP_LOAD_CAP - for start in range(0, len(coarse), 100): - group = coarse[start:start + 100] - group_set = set(group) - target = {k for k in keys if tuple(k[i] for i in positions) in group_set} - q = db.query(model).filter(model.batch_id == batch_id) - if metric_id == "lldp_neighbor": - q = q.order_by(model.id.asc()) - else: - q = q.filter(model.metric_id == metric_id).order_by(model.seq.asc(), model.id.asc()) - if invariant: - exprs = [func.trim(func.coalesce( - cast(getattr(model, f) if metric_id == "lldp_neighbor" - else model.data_json[f].as_string(), String), "")) for f in invariant] - q = q.filter(tuple_(*exprs).in_(group)) - rows = q.limit(budget + 1).all() - if len(rows) > budget: - for r in rows: - db.expunge(r) - # No partial verdicts. Later groups may still fit the remaining budget. - continue - budget -= len(rows) - complete.update(target) - for r in rows: - data = ( - {f: getattr(r, f) for f in ("local_if", "remote_sys", "remote_if", "remote_ip", "protocol")} - if metric_id == "lldp_neighbor" else dict(r.data_json or {}) - ) - row = {**data, "_netx": {"row_id": r.id}} - db.expunge(r) - if row_filters and not all(row_matches_filter(row, f) for f in row_filters): - continue - normalized = apply_iface_normalize_rows( - [row], iface_fields=iface_fields, rules=norm_rules, - )[0] - mapped = apply_port_map(normalized, iface_fields=iface_fields, port_map=port_map) - if row_key(mapped, match_keys) in target: - out.append(row) - return out, complete - - -def _live_search_sheet_diffs( - db: Session, - run: BizCompareRun, - sheet: dict[str, Any], - *, - kind: str, - kw: str = "", - field_q: dict[str, str] | None = None, - page: int, - page_size: int, -) -> dict[str, Any]: - """Find candidate keys, complete both sides, then compare and filter pairs.""" - mid_src = str(sheet.get("metric_id") or "").strip() - sid = sheet_key(sheet) - key_fields = list(sheet.get("key_fields") or []) - iface_fields = list(sheet.get("iface_fields") or []) - field_rules = list(sheet.get("field_rules") or []) - compare_fields = effective_compare_fields( - list(sheet.get("compare_fields") or []), - field_rules, - ) - row_filters = list(sheet.get("row_filters") or []) - # Older runs may lack row_filters on sheet meta — fall back to template - if "row_filters" not in sheet and _run_config_snapshot(run) is None and run.template_id: - tpl = db.get(BizCompareTemplate, run.template_id) - if tpl: - for s in template_metrics(tpl): - if sheet_key(s) == sid: - row_filters = list(s.get("row_filters") or []) - if not key_fields: - key_fields = list(s.get("key_fields") or []) - if not field_rules: - field_rules = list(s.get("field_rules") or []) - compare_fields = effective_compare_fields( - list(s.get("compare_fields") or compare_fields), - field_rules, - ) - break - - free_kw, tok_fields = _split_kw_and_field_tokens(kw) - merged_q = {**tok_fields, **(field_q or {})} - # Drop empty - merged_q = {k: v for k, v in merged_q.items() if str(v or "").strip()} - - if not key_fields or not mid_src: - return { - "total": 0, - "page": page, - "page_size": page_size, - "metric_id": sid, - "items": [], - "source": "live", - "truncated": False, - } - - ignore_ports = sheet.get("ignore_port_changes") - if ignore_ports is not None: - ignore_ports = bool(ignore_ports) - pmap, norm_rules = _run_transform_config(db, run) - - before_rows, trunc_b = _load_metric_rows_for_search( - db, - batch_id=str(run.before_batch_id or ""), - metric_id=mid_src, - row_filters=row_filters, - key_fields=key_fields, - kw=free_kw, - field_q=merged_q, - ) - after_rows, trunc_a = _load_metric_rows_for_search( - db, - batch_id=str(run.after_batch_id or ""), - metric_id=mid_src, - row_filters=row_filters, - key_fields=key_fields, - kw=free_kw, - field_q=merged_q, - ) - match_keys = list((sheet.get("summary") or {}).get("match_key_fields") or key_fields) - if ignore_ports is True and not pmap: - match_keys = [f for f in key_fields if f not in iface_fields] or key_fields - keys: set[tuple[str, ...]] = set() - for rows, mapping in ((before_rows, pmap), (after_rows, {})): - for row in apply_iface_normalize_rows(rows, iface_fields=iface_fields, rules=norm_rules): - keys.add(row_key(apply_port_map(row, iface_fields=iface_fields, port_map=mapping), match_keys)) - before_rows, complete_b = _load_search_key_groups( - db, batch_id=str(run.before_batch_id or ""), metric_id=mid_src, keys=keys, - match_keys=match_keys, iface_fields=iface_fields, row_filters=row_filters, - norm_rules=norm_rules, port_map=pmap, - ) - after_rows, complete_a = _load_search_key_groups( - db, batch_id=str(run.after_batch_id or ""), metric_id=mid_src, keys=keys, - match_keys=match_keys, iface_fields=iface_fields, row_filters=row_filters, - norm_rules=norm_rules, port_map={}, - ) - complete = complete_b & complete_a - def keep_complete(rows: list[dict[str, Any]], mapping: dict[str, str]) -> list[dict[str, Any]]: - normalized = apply_iface_normalize_rows(rows, iface_fields=iface_fields, rules=norm_rules) - return [orig for orig, norm in zip(rows, normalized) if row_key( - apply_port_map(norm, iface_fields=iface_fields, port_map=mapping), match_keys, - ) in complete] - before_rows = keep_complete(before_rows, pmap) - after_rows = keep_complete(after_rows, {}) - result = compare_rows( - before_rows=before_rows, - after_rows=after_rows, - key_fields=key_fields, - iface_fields=iface_fields, - compare_fields=compare_fields, - port_map=pmap, - field_rules=field_rules, - iface_normalize_rules=norm_rules, - ignore_port_changes=(match_keys != key_fields) if not pmap else False, - include_unchanged=True, - unchanged_limit=None, - compact_unchanged=False, - ) - kind_n = (kind or "diff").strip().lower() - filtered = [ - d for d in list(result.get("diffs") or []) - if _kind_allows(kind_n, str(d.get("kind") or "")) - and _diff_matches_search(d, kw=free_kw, field_q=merged_q) - ] - # Cap pairs returned to keep UI snappy - truncated = bool(trunc_b or trunc_a or complete != keys or len(filtered) > _LIVE_SEARCH_RESULT_CAP) - filtered = filtered[:_LIVE_SEARCH_RESULT_CAP] - total = len(filtered) - start = (page - 1) * page_size - page_items = filtered[start : start + page_size] - return { - "total": total, - "page": page, - "page_size": page_size, - "metric_id": sid, - "items": page_items, - "source": "live", - "truncated": truncated, - "live_before_matched": len(before_rows), - "live_after_matched": len(after_rows), - } - - def list_run_diffs( db: Session, run_id: str, @@ -3657,7 +1857,6 @@ def list_run_diffs( metric_id: str = "", kind: str = "diff", kw: str = "", - qf: Any = None, page: int = 1, page_size: int = 100, ) -> dict[str, Any]: @@ -3668,40 +1867,14 @@ def list_run_diffs( size_n = max(1, min(500, int(page_size or 100))) kind_n = (kind or "diff").strip().lower() kw_n = (kw or "").strip() - field_q = _parse_qf(qf) - # Also accept field:value tokens inside kw - free_from_kw, tok_fields = _split_kw_and_field_tokens(kw_n) - if tok_fields: - field_q = {**tok_fields, **field_q} - kw_n = free_from_kw summary = dict(r.summary_json or {}) tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None sheets = _sheet_meta_from_summary(summary, r, tpl) asked = (metric_id or "").strip() sheet = _lookup_sheet(sheets, asked) if asked else (sheets[0] if sheets else None) - if asked and sheet is None: - raise HTTPException(status_code=404, detail="sheet_not_found") mid = sheet_key(sheet) if sheet else (asked or str(r.metric_id or "")) - # Only reconstruct missing success rows. Persisted verdicts must stay stable - # while searching and avoid loading/recomparing source tables on every page. - sheet_summary = (sheet or {}).get("summary") or {} - success_missing = int(sheet_summary.get("unchanged") or 0) > int( - sheet_summary.get("unchanged_listed") or 0 - ) - if (kw_n or field_q) and sheet and kind_n in ("all", "unchanged") and success_missing: - return _live_search_sheet_diffs( - db, - r, - sheet, - kind=kind_n, - kw=kw_n, - field_q=field_q, - page=page_n, - page_size=size_n, - ) - if _run_has_diff_rows(db, run_id): q = db.query(BizCompareDiff).filter( BizCompareDiff.run_id == run_id, @@ -3709,15 +1882,10 @@ def list_run_diffs( ) if kind_n == "diff": q = q.filter(BizCompareDiff.kind.in_(("removed", "changed"))) - elif kind_n == "removed": - q = q.filter(BizCompareDiff.kind == "removed") - elif kind_n == "changed": - q = q.filter(BizCompareDiff.kind == "changed") elif kind_n != "all": q = q.filter(BizCompareDiff.kind == kind_n) - if kw_n or field_q: - q = _search_stored_diffs(q, metric_id=str((sheet or {}).get("metric_id") or mid), - kw=kw_n, field_q=field_q) + if kw_n: + q = q.filter(BizCompareDiff.search_text.ilike(f"%{kw_n}%")) total = q.count() rows = ( q.order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc()) @@ -3725,15 +1893,12 @@ def list_run_diffs( .limit(size_n) .all() ) - items = _hydrate_diff_rows(db, [_diff_row_out(x) for x in rows], run=r, sheet=sheet) return { "total": total, "page": page_n, "page_size": size_n, "metric_id": mid, - "items": items, - "source": "stored", - "truncated": False, + "items": [_diff_row_out(x) for x in rows], } # Legacy: diffs embedded in summary_json / diffs_json @@ -3743,7 +1908,7 @@ def list_run_diffs( inline = list((sheet or {}).get("diffs") or []) if not inline and mid == r.metric_id: inline = list(r.diffs_json or []) - filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n, field_q=field_q) + filtered = _filter_inline_diffs(inline, kind=kind_n, kw=kw_n) total = len(filtered) start = (page_n - 1) * size_n page_items = filtered[start : start + size_n] @@ -3753,58 +1918,43 @@ def list_run_diffs( "page_size": size_n, "metric_id": mid, "items": page_items, - "source": "stored", - "truncated": False, } -def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> Iterator[dict[str, Any]]: - """Stream one sheet using a stable (seq, id) cursor; fall back to inline. - - Compact success rows are hydrated in chunks from metric tables. - """ - r = db.get(BizCompareRun, run_id) - if not r: - return - tpl = db.get(BizCompareTemplate, r.template_id) if r.template_id else None - sheets = _sheet_meta_from_summary(dict(r.summary_json or {}), r, tpl) - sheet = next((s for s in sheets if sheet_key(s) == metric_id), None) +def _iter_sheet_diffs(db: Session, run_id: str, metric_id: str) -> list[dict[str, Any]]: + """Load all diffs for one sheet (export). Prefer row table; fall back to inline.""" if _run_has_diff_rows(db, run_id): - cursor: tuple[int, str] | None = None + out: list[dict[str, Any]] = [] + offset = 0 while True: - q = db.query(BizCompareDiff).filter( - BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id, - ) - if cursor is not None: - seq, diff_id = cursor - q = q.filter(or_(BizCompareDiff.seq > seq, and_( - BizCompareDiff.seq == seq, BizCompareDiff.id > diff_id, - ))) - rows = (q + rows = ( + db.query(BizCompareDiff) + .filter(BizCompareDiff.run_id == run_id, BizCompareDiff.metric_id == metric_id) .order_by(BizCompareDiff.seq.asc(), BizCompareDiff.id.asc()) + .offset(offset) .limit(_DIFF_CHUNK) .all() ) if not rows: break - chunk = _hydrate_diff_rows(db, [_diff_row_out(x) for x in rows], run=r, sheet=sheet) - cursor = (rows[-1].seq, rows[-1].id) - yield from chunk + out.extend(_diff_row_out(x) for x in rows) + offset += len(rows) if len(rows) < _DIFF_CHUNK: break - # Release the previous chunk before loading the next one. - del rows, chunk - return + return out + r = db.get(BizCompareRun, run_id) + if not r: + return [] summary = dict(r.summary_json or {}) sheets = list(summary.get("sheets") or []) for sh in sheets: if sheet_key(sh) == metric_id or ( str(sh.get("metric_id") or "") == metric_id and sheet_key(sh) == metric_id ): - yield from sh.get("diffs") or [] - return + return list(sh.get("diffs") or []) if metric_id == r.metric_id: - yield from r.diffs_json or [] + return list(r.diffs_json or []) + return [] def export_run_zip(db: Session, run_id: str) -> bytes: @@ -3833,17 +1983,14 @@ def export_run_zip(db: Session, run_id: str) -> bytes: f"+{card.get('added')}/-{card.get('removed')}/~{card.get('changed')}/= {card.get('unchanged')}" ) zf.writestr("manifest.txt", "\n".join(manifest) + "\n") - run = db.get(BizCompareRun, run_id) - snapshot = _run_config_snapshot(run) if run else None - if snapshot is not None: - zf.writestr("config_snapshot.json", json.dumps(snapshot, ensure_ascii=False, indent=2)) for sheet in list(detail.get("sheets") or []): sid = sheet_key(sheet) or "sheet" safe = "".join(ch if ch.isalnum() or ch in "-_." else "_" for ch in sid)[:80] or "sheet" - with zf.open(f"tables/{safe}.csv", "w", force_zip64=True) as csv_file: - csv_file.write(b"\xef\xbb\xbf") - for line in _sheet_csv_lines(sheet, _iter_sheet_diffs(db, run_id, sid)): - csv_file.write((line + "\n").encode("utf-8")) + sheet_full = { + **sheet, + "diffs": _iter_sheet_diffs(db, run_id, sid), + } + zf.writestr(f"tables/{safe}.csv", _sheet_csv(sheet_full)) sum_lines = ["metric_id,mode,before,after,added,removed,changed,unchanged,diff_count,pass_rate"] for card in list(s.get("sheet_cards") or []): sum_lines.append( @@ -3878,34 +2025,22 @@ def list_runs(db: Session, job_id: str, *, limit: int = 20) -> list[dict[str, An job = db.get(BizCompareJob, job_id) before_tid = str(job.before_task_id or "") if job else "" after_tid = str(job.after_task_id or "") if job else "" - out: list[dict[str, Any]] = [] - for r in rows: - summary = dict(r.summary_json or {}) - out.append( - { - "id": r.id, - "before_batch_id": r.before_batch_id, - "after_batch_id": r.after_batch_id, - "before": _compare_side(db, r.before_batch_id, fallback_task_id=before_tid), - "after": _compare_side(db, r.after_batch_id, fallback_task_id=after_tid), - "status": r.status, - "message": r.message or "", - "summary": { - k: summary.get(k, 0) - for k in ( - "added", - "removed", - "changed", - "unchanged", - "sheet_count", - "duration_ms", - ) - }, - "progress": dict(summary.get("progress") or {}), - "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, - } - ) - return out + return [ + { + "id": r.id, + "before_batch_id": r.before_batch_id, + "after_batch_id": r.after_batch_id, + "before": _compare_side(db, r.before_batch_id, fallback_task_id=before_tid), + "after": _compare_side(db, r.after_batch_id, fallback_task_id=after_tid), + "status": r.status, + "summary": { + k: (r.summary_json or {}).get(k, 0) + for k in ("added", "removed", "changed", "unchanged", "sheet_count") + }, + "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, + } + for r in rows + ] def try_auto_compare_for_task(db: Session, task_id: str, batch_id: str) -> int: diff --git a/netx_api/biz_state/compare_sql.py b/netx_api/biz_state/compare_sql.py deleted file mode 100644 index 672a15a..0000000 --- a/netx_api/biz_state/compare_sql.py +++ /dev/null @@ -1,757 +0,0 @@ -"""PostgreSQL-backed sheet compare (FULL OUTER JOIN) for pushdown-safe sheets. - -Falls back to the Python engine when port-map / complex rules cannot be expressed -in SQL. See ``can_sql_compare``. -""" - -from __future__ import annotations - -import logging -import re -from typing import Any, Callable, Mapping, Sequence -from uuid import uuid4 - -from sqlalchemy import text -from sqlalchemy.orm import Session - -from .compare_rules import effective_compare_fields, field_rule_map, NUMBER_PATTERN, scalar_text - -_log = logging.getLogger("netx.biz_state.compare_sql") - -# Below this, Python hash-join is faster and avoids TEMP CTAS / progress races. -_SQL_MIN_ROWS = 20_000 -# Cap a single SQL statement so a stuck planner/lock surfaces as fallback. -_SQL_STATEMENT_TIMEOUT_MS = 180_000 - -_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") - - -class SqlCompareSkip(Exception): - """Soft skip — caller should use the Python engine (not an error).""" - - def __init__(self, reason: str) -> None: - super().__init__(reason) - self.reason = str(reason or "skip") -_SQL_FILTER_OPS = frozenset( - {"eq", "==", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "ci_eq"} -) -_SQL_NORMALIZE = frozenset({"", "none", "strip", "lower", "upper", "empty_as_blank"}) -_SQL_COMPARE_MODES = frozenset( - { - "", - "eq", - "ignore", - "skip", - "off", - "numeric", - "number", - "int", - "float", - "percent", - "pct", - "rel", - } -) -_EMPTY_AS_BLANK = ("n/a", "na", "-", "--", "none", "null") -_NUM_RE_SQL = NUMBER_PATTERN - - -def _dialect_is_postgres(db: Session) -> bool: - bind = db.get_bind() - if bind is None: - return False - return str(getattr(bind.dialect, "name", "") or "").lower() in ("postgresql", "postgres") - - -def _safe_field(name: str) -> str: - f = str(name or "").strip() - if not f or not _FIELD_RE.match(f): - raise ValueError(f"unsafe_json_field:{name!r}") - return f - - -def _json_text_expr(alias: str, field: str) -> str: - """SQL expression: trimmed text from JSONB column ``alias.data`` / ``data_json``.""" - f = _safe_field(field) - # alias is caller-controlled (_b / data_json) — not user input - return f"trim(both from coalesce({alias}->>'{f}', ''))" - - -def _norm_expr(alias: str, field: str, normalize: str) -> str: - base = _json_text_expr(alias, field) - mode = str(normalize or "strip").strip().lower() or "strip" - if mode in ("", "none", "strip"): - return base - if mode == "lower": - return f"lower({base})" - if mode == "upper": - return f"upper({base})" - if mode == "empty_as_blank": - opts = ", ".join(f"'{x}'" for x in _EMPTY_AS_BLANK) - return ( - f"CASE WHEN lower({base}) IN ({opts}) THEN '' ELSE {base} END" - ) - raise ValueError(f"unsupported_normalize:{mode}") - - -def _filters_sql_compatible(filters: Sequence[Mapping[str, Any]] | None) -> bool: - fl = [f for f in (filters or []) if isinstance(f, dict)] - return all(_filter_node_compatible(f) for f in fl) - - -def _filter_node_compatible(filt: Mapping[str, Any]) -> bool: - if "any" in filt: - kids = filt.get("any") or [] - return isinstance(kids, list) and all( - isinstance(k, dict) and _filter_node_compatible(k) for k in kids - ) - if "all" in filt: - kids = filt.get("all") or [] - return isinstance(kids, list) and all( - isinstance(k, dict) and _filter_node_compatible(k) for k in kids - ) - field = str(filt.get("field") or "").strip() - op = str(filt.get("op") or "eq").strip().lower() - if op not in _SQL_FILTER_OPS: - return False - if op in ("empty", "not_empty", "nonempty"): - return bool(field) and bool(_FIELD_RE.match(field)) - if not field or not _FIELD_RE.match(field): - return False - if op in ("in", "not_in", "nin"): - vals = filt.get("value") - if vals is None: - return True - if not isinstance(vals, (list, tuple)): - vals = [vals] - return all(isinstance(x, (str, int, float, bool)) or x is None for x in vals) - return True - - -def _field_rules_sql_compatible(rules: Sequence[Mapping[str, Any]] | None) -> bool: - for raw in rules or []: - if not isinstance(raw, dict): - return False - name = str(raw.get("field") or "").strip() - if name and not _FIELD_RE.match(name): - return False - mode = str(raw.get("compare") or "eq").strip().lower() or "eq" - if mode not in _SQL_COMPARE_MODES: - return False - if raw.get("ignore") is True: - continue - if mode in ("ignore", "skip", "off"): - continue - norm = str(raw.get("normalize") or "strip").strip().lower() or "strip" - if norm not in _SQL_NORMALIZE: - return False - return True - - -def sql_compare_skip_reason( - db: Session, - sheet: Mapping[str, Any], - *, - port_map: Mapping[str, str] | None = None, - iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, -) -> str: - """Empty string when SQL is allowed; otherwise a short reason for progress/logs. - - Simple ``row_filters`` (eq/in/contains/…) used for BGP sheet splits are fine — - they do **not** force Python by themselves. - """ - if not _dialect_is_postgres(db): - return "not_postgres" - if port_map: - return "port_map" - key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] - if not key_fields: - return "no_key_fields" - if any(not _FIELD_RE.match(k) for k in key_fields): - return "unsafe_key_field" - iface_fields = [str(f).strip() for f in (sheet.get("iface_fields") or []) if str(f).strip()] - iface_set = set(iface_fields) - ignore_ports = sheet.get("ignore_port_changes") - if ignore_ports is True: - return "ignore_port_changes" - if ignore_ports is None and iface_set and any(k in iface_set for k in key_fields): - return "auto_ignore_ports" - field_rules = list(sheet.get("field_rules") or []) - if not _field_rules_sql_compatible(field_rules): - return "field_rules" - compare_fields = effective_compare_fields( - list(sheet.get("compare_fields") or []), - field_rules, - ) - if any(not _FIELD_RE.match(str(f).strip()) for f in compare_fields if str(f).strip()): - return "unsafe_compare_field" - used = set(key_fields) | {str(f).strip() for f in compare_fields if str(f).strip()} - if iface_normalize_rules and (used & iface_set): - return "iface_normalize" - if not _filters_sql_compatible(list(sheet.get("row_filters") or [])): - return "row_filters" - if not str(sheet.get("metric_id") or "").strip(): - return "no_metric_id" - return "" - - -def can_sql_compare( - db: Session, - sheet: Mapping[str, Any], - *, - port_map: Mapping[str, str] | None = None, - iface_normalize_rules: Sequence[Mapping[str, str]] | None = None, -) -> bool: - """True when this sheet can run entirely as a PostgreSQL JOIN.""" - return not bool( - sql_compare_skip_reason( - db, - sheet, - port_map=port_map, - iface_normalize_rules=iface_normalize_rules, - ) - ) - - -def compile_row_filters_sql( - filters: Sequence[Mapping[str, Any]] | None, - *, - json_col: str = "data_json", - param_prefix: str = "f", -) -> tuple[str, dict[str, Any]]: - """Compile template row_filters to SQL AND-clause + bind params. - - Returns ``(sql_fragment, params)``. Empty filters → ``(\"TRUE\", {})``. - ``json_col`` is the JSONB column/expression name (trusted). - """ - fl = [f for f in (filters or []) if isinstance(f, dict)] - if not fl: - return "TRUE", {} - params: dict[str, Any] = {} - counter = {"n": 0} - - def _next(name: str) -> str: - counter["n"] += 1 - return f"{param_prefix}_{counter['n']}_{name}" - - def _node(filt: Mapping[str, Any]) -> str: - if "any" in filt: - kids = [k for k in (filt.get("any") or []) if isinstance(k, dict)] - if not kids: - return "TRUE" - return "(" + " OR ".join(_node(k) for k in kids) + ")" - if "all" in filt: - kids = [k for k in (filt.get("all") or []) if isinstance(k, dict)] - if not kids: - return "TRUE" - return "(" + " AND ".join(_node(k) for k in kids) + ")" - field = _safe_field(str(filt.get("field") or "")) - op = str(filt.get("op") or "eq").strip().lower() - expr = _json_text_expr(json_col, field) - # _json_text_expr uses alias->> ; for bare column use data_json directly - if json_col == "data_json": - expr = f"trim(both from coalesce(data_json->>'{field}', ''))" - if op in ("empty",): - return f"({expr} = '')" - if op in ("not_empty", "nonempty"): - return f"({expr} <> '')" - expect = filt.get("value") - if op in ("eq", "==", "ci_eq"): - key = _next("v") - params[key] = scalar_text(expect).strip() - if op == "ci_eq" or op in ("eq", "=="): - # Python eq is case-insensitive via .lower() - params[key] = scalar_text(expect).strip().lower() - return f"(lower({expr}) = :{key})" - if op in ("ne", "!="): - key = _next("v") - params[key] = scalar_text(expect).strip().lower() - return f"(lower({expr}) <> :{key})" - if op == "contains": - key = _next("v") - needle = scalar_text(expect).strip().lower() - if not needle: - return "FALSE" - escaped = needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - params[key] = f"%{escaped}%" - return f"(lower({expr}) LIKE :{key} ESCAPE E'\\\\')" - if op in ("in", "not_in", "nin"): - vals = expect - if vals is None: - vals = [] - if not isinstance(vals, (list, tuple)): - vals = [vals] - clean = [scalar_text(x).strip().lower() for x in vals if scalar_text(x).strip()] - if not clean: - return "FALSE" if op == "in" else "TRUE" - keys = [] - for i, v in enumerate(clean): - k = _next(f"in{i}") - params[k] = v - keys.append(f":{k}") - inside = f"lower({expr}) IN ({', '.join(keys)})" - return f"({inside})" if op == "in" else f"(NOT {inside})" - raise ValueError(f"unsupported_filter_op:{op}") - - parts = [_node(f) for f in fl] - return "(" + " AND ".join(parts) + ")", params - - -def _rk_sql(key_fields: list[str], *, json_col: str = "data_json") -> str: - parts = [] - for k in key_fields: - f = _safe_field(k) - if json_col == "data_json": - parts.append(f"trim(both from coalesce(data_json->>'{f}', ''))") - else: - parts.append(f"trim(both from coalesce({json_col}->>'{f}', ''))") - if len(parts) == 1: - return parts[0] - # Delimiter concatenation conflates ('a|b', 'c') with ('a', 'b|c'). - return "jsonb_build_array(" + ", ".join(parts) + ")::text" - - -def _field_differs_sql(bv: str, av: str, rule: Mapping[str, Any]) -> str: - """SQL boolean: True when before/after values differ under the field rule.""" - mode = str(rule.get("compare") or "eq").strip().lower() or "eq" - if mode in ("ignore", "skip", "off") or rule.get("ignore") is True: - return "FALSE" - try: - tol = float(rule.get("tolerance") or 0) - except (TypeError, ValueError): - tol = 0.0 - if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"): - # Match Python values_equal: parseable → numeric compare; else string eq - num_b = f"(({bv}) ~ '{_NUM_RE_SQL}')" - num_a = f"(({av}) ~ '{_NUM_RE_SQL}')" - bn = f"({bv})::double precision" - an = f"({av})::double precision" - if mode in ("percent", "pct", "rel"): - # differ when relative % > tol (before==0 → after must be 0) - num_diff = ( - f"(CASE WHEN {bn} = 0 THEN {an} IS DISTINCT FROM 0 " - f"ELSE (abs({an} - {bn}) / abs({bn}) * 100.0) > {tol} END)" - ) - else: - num_diff = f"(abs({an} - {bn}) > {tol})" - return ( - f"(CASE WHEN {num_b} AND {num_a} THEN {num_diff} " - f"ELSE ({bv} IS DISTINCT FROM {av}) END)" - ) - return f"({bv} IS DISTINCT FROM {av})" - - -def _changed_predicate( - compare_fields: list[str], - rules: dict[str, dict[str, Any]], - *, - before_alias: str = "b", - after_alias: str = "a", -) -> str: - """SQL boolean: True when any compare field differs (IS DISTINCT FROM).""" - if not compare_fields: - return "FALSE" - clauses: list[str] = [] - for f in compare_fields: - name = str(f).strip() - if not name: - continue - rule = rules.get(name) or {} - mode = str(rule.get("compare") or "eq").strip().lower() or "eq" - if mode in ("ignore", "skip", "off") or rule.get("ignore") is True: - continue - norm = str(rule.get("normalize") or "strip").strip().lower() or "strip" - bv = _norm_expr(f"{before_alias}.data", name, norm) - av = _norm_expr(f"{after_alias}.data", name, norm) - clauses.append(_field_differs_sql(bv, av, rule)) - if not clauses: - return "FALSE" - return "(" + " OR ".join(clauses) + ")" - - -def _key_obj_from_data(data: dict[str, Any] | None, key_fields: list[str]) -> dict[str, Any]: - row = data if isinstance(data, dict) else {} - return {f: row.get(f, "") for f in key_fields} - - -def _changes_from_rows( - before: dict[str, Any] | None, - after: dict[str, Any] | None, - compare_fields: list[str], - rules: dict[str, dict[str, Any]], -) -> dict[str, dict[str, Any]]: - from .compare_rules import explain_diff, values_equal - - out: dict[str, dict[str, Any]] = {} - b = before or {} - a = after or {} - for f in compare_fields: - rule = rules.get(f) - bv = b.get(f, "") - av = a.get(f, "") - if values_equal(bv, av, rule=rule): - continue - entry: dict[str, Any] = {"before": bv, "after": av} - reason = explain_diff(bv, av, rule=rule) - if reason: - entry["reason"] = reason - out[f] = entry - return out - - -def run_sql_sheet_compare( - db: Session, - *, - sheet: Mapping[str, Any], - before_batch_id: str, - after_batch_id: str, - store_unchanged: str = "auto", - on_progress: Callable[..., None] | None = None, -) -> dict[str, Any]: - """Compare one sheet via PostgreSQL TEMP tables + FULL OUTER JOIN. - - Same-key multipath: rows get ``dup_rn`` by ``seq,id`` and join on - ``(rk, dup_rn)`` (ordered zip). Extras are added/removed, not duplicate. - - Returns the same ``{summary, diffs, mapping_stats}`` shape as ``compare_rows``. - ``on_progress(side, n, *, engine=\"sql\", note=..., phase=...)``. - """ - if not _dialect_is_postgres(db): - raise RuntimeError("sql_compare_requires_postgres") - - def _prog(side: str, n: int, *, phase: str = "sql_count", note: str = "") -> None: - if not on_progress: - return - try: - on_progress(side, n, engine="sql", note=note, phase=phase) - except TypeError: - on_progress(side, n) - - key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()] - field_rules = list(sheet.get("field_rules") or []) - rules = field_rule_map(field_rules) - compare_fields = effective_compare_fields( - list(sheet.get("compare_fields") or []), - field_rules, - ) - row_filters = list(sheet.get("row_filters") or []) - mid = str(sheet.get("metric_id") or "").strip() - bid_b = str(before_batch_id or "").strip() - bid_a = str(after_batch_id or "").strip() - if not mid or not bid_b or not bid_a or not key_fields: - raise ValueError("sql_compare_missing_args") - - filter_sql, filter_params = compile_row_filters_sql(row_filters) - rk = _rk_sql(key_fields) - tag = uuid4().hex[:8] - tb = f"_netx_cmp_b_{tag}" - ta = f"_netx_cmp_a_{tag}" - - # Keep worker transaction open across CTAS — progress must NOT commit this session. - db.execute(text(f"SET LOCAL statement_timeout = '{int(_SQL_STATEMENT_TIMEOUT_MS)}'")) - - _prog("before", 0, phase="sql_count", note="count") - - # Raw counts (no row_filters) - raw_b = int( - db.execute( - text( - "SELECT count(*) FROM biz_state_metric_row " - "WHERE batch_id = :bid AND metric_id = :mid" - ), - {"bid": bid_b, "mid": mid}, - ).scalar() - or 0 - ) - _prog("before", raw_b, phase="sql_count") - raw_a = int( - db.execute( - text( - "SELECT count(*) FROM biz_state_metric_row " - "WHERE batch_id = :bid AND metric_id = :mid" - ), - {"bid": bid_a, "mid": mid}, - ).scalar() - or 0 - ) - _prog("after", raw_a, phase="sql_count") - - if max(raw_b, raw_a) < int(_SQL_MIN_ROWS): - raise SqlCompareSkip("small_sheet") - - base_params = {"bid": bid_b, "mid": mid, **filter_params} - # Build TEMP sides (one transaction — no mid-flight commits on ``db``) - _prog("before", raw_b, phase="sql_project", note="temp_before") - for tname, batch_id, side, note in ( - (tb, bid_b, "before", "temp_before"), - (ta, bid_a, "after", "temp_after"), - ): - db.execute(text(f"DROP TABLE IF EXISTS {tname}")) - params = {**base_params, "bid": batch_id} - if side == "after": - _prog(side, raw_a, phase="sql_project", note=note) - db.execute( - text( - f""" - CREATE TEMP TABLE {tname} ON COMMIT PRESERVE ROWS AS - SELECT - id, - ({rk}) AS rk, - data_json AS data, - row_number() OVER ( - PARTITION BY ({rk}) - ORDER BY seq ASC, id ASC - ) AS dup_rn - FROM biz_state_metric_row - WHERE batch_id = :bid - AND metric_id = :mid - AND ({filter_sql}) - """ - ), - params, - ) - db.execute(text(f"CREATE INDEX IF NOT EXISTS {tname}_rk ON {tname} (rk, dup_rn)")) - - before_n = int( - db.execute(text(f"SELECT count(*) FROM {tb}")).scalar() or 0 - ) - after_n = int( - db.execute(text(f"SELECT count(*) FROM {ta}")).scalar() or 0 - ) - _prog("after", after_n, phase="sql_join", note="join") - - changed_pred = _changed_predicate(compare_fields, rules) - kind_expr = f""" - CASE - WHEN b.id IS NULL THEN 'added' - WHEN a.id IS NULL THEN 'removed' - WHEN {changed_pred} THEN 'changed' - ELSE 'unchanged' - END - """ - - # Ordered same-key pairing: join on (rk, dup_rn) so 5 vs 2 → 2 compared + 3 removed - agg_rows = db.execute( - text( - f""" - SELECT {kind_expr} AS kind, count(*)::bigint AS n - FROM {tb} b - FULL OUTER JOIN {ta} a - ON b.rk = a.rk AND b.dup_rn = a.dup_rn - GROUP BY 1 - """ - ) - ).mappings().all() - counts = {str(r["kind"]): int(r["n"] or 0) for r in agg_rows} - added = counts.get("added", 0) - removed = counts.get("removed", 0) - changed = counts.get("changed", 0) - unchanged = counts.get("unchanged", 0) - - # Diagnostic: count of match keys with >1 row (not fail rows) - multi_b = int( - db.execute( - text(f"SELECT count(*) FROM (SELECT rk FROM {tb} GROUP BY rk HAVING count(*) > 1) t") - ).scalar() - or 0 - ) - multi_a = int( - db.execute( - text(f"SELECT count(*) FROM (SELECT rk FROM {ta} GROUP BY rk HAVING count(*) > 1) t") - ).scalar() - or 0 - ) - - # Lazy import — avoid circular import with compare_service - from .compare_service import resolve_unchanged_policy - - policy = resolve_unchanged_policy( - store_unchanged, before_n=before_n, after_n=after_n - ) - diffs: list[dict[str, Any]] = [] - - # Fail rows (stream into Python — should be << million) - fail_sql = text( - f""" - SELECT - {kind_expr} AS kind, - b.id AS before_row_id, - a.id AS after_row_id, - b.data AS before_data, - a.data AS after_data, - COALESCE(b.rk, a.rk) AS rk - FROM {tb} b - FULL OUTER JOIN {ta} a - ON b.rk = a.rk AND b.dup_rn = a.dup_rn - WHERE {kind_expr} IN ('added', 'removed', 'changed') - """ - ) - fail_n = 0 - _prog("after", after_n, phase="sql_fail_fetch", note="fetch_fails") - for row in db.execute(fail_sql).mappings(): - kind = str(row["kind"] or "") - before = dict(row["before_data"] or {}) if row["before_data"] is not None else None - after = dict(row["after_data"] or {}) if row["after_data"] is not None else None - key_src = after if kind == "added" else (before or after or {}) - item: dict[str, Any] = { - "kind": kind, - "key": _key_obj_from_data(key_src, key_fields), - "before": before, - "after": after, - "mapped_before": before, - "changes": {}, - "before_row_id": str(row["before_row_id"] or ""), - "after_row_id": str(row["after_row_id"] or ""), - } - if kind == "changed": - item["changes"] = _changes_from_rows(before, after, compare_fields, rules) - diffs.append(item) - fail_n += 1 - if fail_n == 1 or fail_n % 25_000 == 0: - _prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails") - if fail_n: - _prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails") - - unchanged_listed = 0 - include_u = bool(policy.get("include")) - compact = bool(policy.get("compact")) - limit_n = policy.get("limit") - if include_u and unchanged > 0: - params_u: dict[str, Any] = {} - # Stratified sample: round-robin by neighbor|direction|afi|vrf so one - # BGP command cannot consume the whole success sample. - stratum_expr = """ - concat_ws('|', - coalesce(nullif(trim(both from coalesce(b.data->>'neighbor','')), ''), '_'), - coalesce(nullif(trim(both from coalesce(b.data->>'direction','')), ''), '_'), - coalesce(nullif(trim(both from coalesce(b.data->>'afi','')), ''), '_'), - coalesce(nullif(trim(both from coalesce(b.data->>'vrf','')), ''), '_') - ) - """ - if limit_n is not None: - params_u["lim"] = max(0, int(limit_n)) - u_sql = text( - f""" - WITH u AS ( - SELECT - b.id AS before_row_id, - a.id AS after_row_id, - b.data AS before_data, - a.data AS after_data, - {stratum_expr} AS stratum - FROM {tb} b - INNER JOIN {ta} a - ON b.rk = a.rk AND b.dup_rn = a.dup_rn - WHERE NOT ({changed_pred}) - ), - ranked AS ( - SELECT *, - row_number() OVER ( - PARTITION BY stratum ORDER BY before_row_id ASC - ) AS rn_in - FROM u - ), - picked AS ( - SELECT *, - row_number() OVER ( - ORDER BY rn_in ASC, stratum ASC, before_row_id ASC - ) AS pick_ord - FROM ranked - ) - SELECT before_row_id, after_row_id, before_data, after_data - FROM picked - WHERE pick_ord <= :lim - """ - ) - else: - u_sql = text( - f""" - SELECT - b.id AS before_row_id, - a.id AS after_row_id, - b.data AS before_data, - a.data AS after_data - FROM {tb} b - INNER JOIN {ta} a - ON b.rk = a.rk AND b.dup_rn = a.dup_rn - WHERE NOT ({changed_pred}) - """ - ) - for row in db.execute(u_sql, params_u).mappings(): - before = dict(row["before_data"] or {}) - after = dict(row["after_data"] or {}) - unchanged_listed += 1 - if compact: - diffs.append( - { - "kind": "unchanged", - "key": _key_obj_from_data(before, key_fields), - "before": {}, - "after": {}, - "mapped_before": {}, - "changes": {}, - "compact": True, - "before_row_id": str(row["before_row_id"] or ""), - "after_row_id": str(row["after_row_id"] or ""), - } - ) - else: - diffs.append( - { - "kind": "unchanged", - "key": _key_obj_from_data(before, key_fields), - "before": before, - "after": after, - "mapped_before": before, - "changes": {}, - "before_row_id": str(row["before_row_id"] or ""), - "after_row_id": str(row["after_row_id"] or ""), - } - ) - - # Cleanup (also ON COMMIT DROP) - db.execute(text(f"DROP TABLE IF EXISTS {tb}")) - db.execute(text(f"DROP TABLE IF EXISTS {ta}")) - - summary = { - "before_count": before_n, - "after_count": after_n, - "added": added, - "removed": removed, - "changed": changed, - "unchanged": unchanged, - "duplicate": 0, - "match_key_fields": list(key_fields), - "duplicate_keys_before": multi_b, - "duplicate_keys_after": multi_a, - "duplicate_key_list": [], - "unchanged_listed": unchanged_listed, - "unchanged_truncated": bool( - include_u and limit_n is not None and unchanged > unchanged_listed - ), - "unchanged_compact": bool(compact and unchanged_listed > 0), - "unchanged_sample_mode": ( - "stratified" - if include_u and limit_n is not None and unchanged > unchanged_listed - else ("full" if include_u else "none") - ), - "engine": "sql", - "before_raw_count": raw_b, - "after_raw_count": raw_a, - "row_filters": len(row_filters), - "unchanged_policy": policy, - } - mapping_stats = { - "before_iface_count": 0, - "after_iface_count": 0, - "map_pairs": 0, - "hit_before": [], - "miss_before": [], - "hit_after": [], - "miss_after": [], - "unused_before_keys": [], - "ok": True, - "ignore_port_changes": False, - "engine": "sql", - } - return {"summary": summary, "diffs": diffs, "mapping_stats": mapping_stats} diff --git a/netx_api/biz_state/compare_validation.py b/netx_api/biz_state/compare_validation.py deleted file mode 100644 index 442b10f..0000000 --- a/netx_api/biz_state/compare_validation.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Validate template writes without silently dropping parts of the comparison.""" -from __future__ import annotations - -import math -import re -from typing import Any - -from fastapi import HTTPException - -FILTER_OPS = {"eq", "==", "ci_eq", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "regex", "age_timer"} -COMPARE_MODES = {"", "eq", "ignore", "skip", "off", "numeric", "number", "int", "float", "percent", "pct", "rel"} -NORMALIZE_MODES = {"", "none", "strip", "lower", "upper", "mac", "empty_as_blank"} - - -def invalid(path: str, reason: str) -> None: - raise HTTPException(status_code=400, detail={"error": "invalid_template", "path": path, "reason": reason}) - - -def _strings(value: Any, path: str, *, required: bool = False) -> None: - if not isinstance(value, list) or any(not isinstance(x, str) or not x.strip() for x in value): - invalid(path, "nonempty_strings_required") - if required and not value: - invalid(path, "key_fields_required") - if len({x.strip() for x in value}) != len(value): - invalid(path, "duplicate_field") - - -def validate_filters(value: Any, path: str, depth: int = 0) -> None: - if not isinstance(value, list): - invalid(path, "list_required") - if depth > 12: - invalid(path, "filter_nesting_too_deep") - for i, node in enumerate(value): - here = f"{path}[{i}]" - if not isinstance(node, dict) or not node: - invalid(here, "filter_required") - groups = [key for key in ("any", "all") if key in node] - if groups: - if len(groups) != 1 or any(node.get(key) is not None for key in ("field", "op", "value")): - invalid(here, "group_or_leaf_required") - kids = node[groups[0]] - if not isinstance(kids, list) or not kids: - invalid(here, "nonempty_group_required") - validate_filters(kids, f"{here}.{groups[0]}", depth + 1) - continue - if not isinstance(node.get("field"), str) or not node["field"].strip(): - invalid(here, "filter_field_required") - op = str(node.get("op") or "eq").strip().lower() - if op not in FILTER_OPS: - invalid(here, "unknown_filter_operator") - expect = node.get("value") - if op in ("in", "not_in", "nin"): - if not isinstance(expect, list) or not expect or any(not isinstance(x, (str, int, float, bool)) for x in expect): - invalid(here, "nonempty_value_list_required") - elif op not in ("empty", "not_empty", "nonempty", "age_timer"): - if not isinstance(expect, (str, int, float, bool)): - invalid(here, "scalar_value_required") - if op in ("contains", "regex") and not str(expect).strip(): - invalid(here, "value_required") - if op == "regex": - try: - re.compile(str(expect), re.I) - except re.error: - invalid(here, "invalid_regex") - - -def validate_sheet(sheet: Any, path: str, *, partial: bool = False) -> None: - if not isinstance(sheet, dict): - invalid(path, "sheet_required") - if not partial or "metric_id" in sheet: - if not isinstance(sheet.get("metric_id"), str) or not sheet["metric_id"].strip(): - invalid(path, "metric_id_required") - for key in ("key_fields", "iface_fields", "compare_fields", "display_fields", "ignore_fields"): - if key in sheet or (key == "key_fields" and not partial): - _strings(sheet.get(key), f"{path}.{key}", required=key == "key_fields") - if "ignore_port_changes" in sheet and sheet["ignore_port_changes"] is not None and not isinstance(sheet["ignore_port_changes"], bool): - invalid(path, "boolean_port_policy_required") - if "row_filters" in sheet: - validate_filters(sheet["row_filters"], f"{path}.row_filters") - if "field_rules" in sheet: - rules = sheet["field_rules"] - if not isinstance(rules, list): - invalid(path, "field_rules_list_required") - seen = set() - for i, rule in enumerate(rules): - here = f"{path}.field_rules[{i}]" - if not isinstance(rule, dict) or not isinstance(rule.get("field"), str) or not rule["field"].strip(): - invalid(here, "rule_field_required") - field = rule["field"].strip() - if field in seen: - invalid(here, "duplicate_field_rule") - seen.add(field) - if str(rule.get("compare") or "").strip().lower() not in COMPARE_MODES: - invalid(here, "unknown_compare_mode") - if str(rule.get("normalize") or "").strip().lower() not in NORMALIZE_MODES: - invalid(here, "unknown_normalize_mode") - tol = rule.get("tolerance") - if tol is not None: - if isinstance(tol, bool) or not isinstance(tol, (int, float)) or not math.isfinite(tol) or tol < 0: - invalid(here, "finite_nonnegative_tolerance_required") - - -def validate_template_body(body: dict[str, Any], *, partial: bool = False) -> None: - metrics = body.get("metrics") - if metrics is not None: - if not isinstance(metrics, list) or not metrics: - invalid("metrics", "metrics_required") - seen = set() - for i, sheet in enumerate(metrics): - path = f"metrics[{i}]" - validate_sheet(sheet, path) - sid = str(sheet.get("sheet_id") or sheet["metric_id"]).strip() - if sid in seen: - invalid(path, "duplicate_sheet_id") - seen.add(sid) - else: - validate_sheet(body, "template", partial=partial) - if "iface_normalize_rules" in body and body["iface_normalize_rules"] is not None: - rules = body["iface_normalize_rules"] - if not isinstance(rules, list): - invalid("iface_normalize_rules", "list_required") - seen = set() - for i, rule in enumerate(rules): - path = f"iface_normalize_rules[{i}]" - if not isinstance(rule, dict) or any(not isinstance(rule.get(k), str) or not rule[k].strip() for k in ("from", "to")): - invalid(path, "alias_pair_required") - key = rule["from"].strip().lower() - if key in seen: - invalid(path, "duplicate_alias") - seen.add(key) diff --git a/netx_api/biz_state/data/default_zte_status_template.json b/netx_api/biz_state/data/default_zte_status_template.json deleted file mode 100644 index d643cbc..0000000 --- a/netx_api/biz_state/data/default_zte_status_template.json +++ /dev/null @@ -1,1051 +0,0 @@ -{ - "format": "netx.biz_compare_template", - "version": 1, - "name": "ZTE status default", - "note": "Built-in ZTE status cutover (ISIS/IF/ARP/ND6/BGP, address-family sheets)", - "iface_normalize_rules": [], - "metrics": [ - { - "sheet_id": "isis_adjacency.ipv4", - "title": "ISIS IPv4", - "metric_id": "isis_adjacency", - "key_fields": [ - "process_id", - "interface", - "system_id" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state", - "lev", - "af" - ], - "display_fields": [ - "process_id", - "interface", - "system_id", - "state", - "lev", - "af" - ], - "row_filters": [ - { - "op": "contains", - "field": "af", - "value": "IPv4" - } - ], - "field_rules": [] - }, - { - "sheet_id": "isis_adjacency.ipv6", - "title": "ISIS IPv6", - "metric_id": "isis_adjacency", - "key_fields": [ - "process_id", - "interface", - "system_id" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state", - "lev", - "af" - ], - "display_fields": [ - "process_id", - "interface", - "system_id", - "state", - "lev", - "af" - ], - "row_filters": [ - { - "op": "contains", - "field": "af", - "value": "IPv6" - } - ], - "field_rules": [] - }, - { - "sheet_id": "interface_brief", - "title": "interface_brief", - "metric_id": "interface_brief", - "key_fields": [ - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "prot" - ], - "display_fields": [ - "interface", - "prot", - "admin", - "phy" - ], - "row_filters": [ - { - "op": "regex", - "field": "interface", - "value": "^[^.]+$" - }, - { - "op": "contains", - "field": "interface", - "value": "gei" - } - ], - "field_rules": [] - }, - { - "sheet_id": "arp", - "title": "arp", - "metric_id": "arp", - "key_fields": [ - "ip", - "interface", - "vrf" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "mac" - ], - "display_fields": [ - "ip", - "interface", - "vrf", - "mac", - "age", - "entry_type" - ], - "row_filters": [ - { - "op": "age_timer", - "field": "age", - "value": "" - } - ], - "field_rules": [ - { - "field": "mac", - "normalize": "mac" - } - ] - }, - { - "sheet_id": "nd6_cache", - "title": "nd6_cache", - "metric_id": "nd6_cache", - "key_fields": [ - "address", - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "link_address", - "status" - ], - "display_fields": [ - "address", - "interface", - "link_address", - "status", - "type", - "age" - ], - "row_filters": [ - { - "op": "ne", - "field": "status", - "value": " Incomplete" - } - ], - "field_rules": [] - }, - { - "sheet_id": "bgp_peer.ipv4", - "title": "BGP IPv4", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "ipv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.ipv6", - "title": "BGP IPv6", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "ipv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.vpnv4", - "title": "BGP VPNv4", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpnv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.vpnv6", - "title": "BGP VPNv6", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpnv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.evpn", - "title": "BGP EVPN", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "evpn" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_peer.vpls", - "title": "BGP VPLS", - "metric_id": "bgp_peer", - "key_fields": [ - "afi", - "vrf", - "neighbor", - "as_num" - ], - "iface_fields": [], - "compare_fields": [ - "state", - "pfx_rcd" - ], - "display_fields": [ - "afi", - "vrf", - "neighbor", - "as_num", - "state", - "pfx_rcd" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpls" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "interface_detail", - "title": "interface_detail", - "metric_id": "interface_detail", - "key_fields": [ - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "admin", - "input_bps", - "output_bps", - "port_status", - "in_util", - "out_util", - "ip_mtu", - "mtu", - "mpls_mtu", - "ipv6_mtu", - "port_media", - "negotiation" - ], - "display_fields": [ - "interface", - "admin", - "input_bps", - "output_bps", - "port_status", - "in_util", - "out_util", - "ip_mtu", - "mtu", - "mpls_mtu", - "ipv6_mtu", - "port_media", - "negotiation", - "bw", - "description", - "rate_period" - ], - "row_filters": [], - "field_rules": [ - { - "field": "input_bps", - "compare": "percent", - "tolerance": 5 - }, - { - "field": "output_bps", - "compare": "numeric", - "tolerance": 5 - }, - { - "field": "in_util", - "compare": "numeric", - "tolerance": 5 - }, - { - "field": "out_util", - "compare": "numeric", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "ospf_neighbor", - "title": "ospf_neighbor", - "metric_id": "ospf_neighbor", - "key_fields": [ - "neighbor_id", - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state" - ], - "display_fields": [ - "neighbor_id", - "interface", - "state", - "process_id", - "address" - ], - "row_filters": [], - "field_rules": [] - }, - { - "sheet_id": "vrrp.ipv4", - "title": "VRRP IPv4", - "metric_id": "vrrp", - "key_fields": [ - "af", - "interface", - "vr_id" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state", - "priority", - "master_addr", - "vrouter_addr" - ], - "display_fields": [ - "af", - "interface", - "vr_id", - "state", - "priority", - "master_addr", - "vrouter_addr" - ], - "row_filters": [ - { - "op": "eq", - "field": "af", - "value": "ipv4" - } - ], - "field_rules": [] - }, - { - "sheet_id": "vrrp.ipv6", - "title": "VRRP IPv6", - "metric_id": "vrrp", - "key_fields": [ - "af", - "interface", - "vr_id" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "state", - "priority", - "master_addr", - "vrouter_addr" - ], - "display_fields": [ - "af", - "interface", - "vr_id", - "state", - "priority", - "master_addr", - "vrouter_addr" - ], - "row_filters": [ - { - "op": "eq", - "field": "af", - "value": "ipv6" - } - ], - "field_rules": [] - }, - { - "sheet_id": "optical_brief", - "title": "optical_brief", - "metric_id": "optical_brief", - "key_fields": [ - "interface" - ], - "iface_fields": [ - "interface" - ], - "compare_fields": [ - "status", - "rx_power" - ], - "display_fields": [ - "interface", - "status", - "rx_power", - "tx_power", - "optic_type", - "wavelength", - "rx_threshold", - "tx_threshold" - ], - "row_filters": [], - "field_rules": [ - { - "field": "rx_power", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "lldp_neighbor", - "title": "lldp_neighbor", - "metric_id": "lldp_neighbor", - "key_fields": [ - "local_if", - "remote_sys", - "remote_if" - ], - "iface_fields": [ - "local_if" - ], - "compare_fields": [], - "display_fields": [ - "local_if", - "remote_sys", - "remote_if", - "remote_ip", - "protocol" - ], - "row_filters": [], - "field_rules": [] - }, - { - "sheet_id": "l2vpn_pw_detail", - "title": "l2vpn_pw_detail", - "metric_id": "l2vpn_pw_detail", - "key_fields": [ - "peer", - "vcid", - "service_instance" - ], - "iface_fields": [], - "compare_fields": [ - "vc_status", - "remote_status", - "vc_type", - "control_word" - ], - "display_fields": [ - "peer", - "vcid", - "service_instance", - "vc_status", - "remote_status", - "vc_type", - "control_word", - "pw_name", - "activation_status", - "service_instance_type", - "conn_mode", - "signaling", - "tunnel_dest" - ], - "row_filters": [], - "field_rules": [] - }, - { - "sheet_id": "l2vpn_mac", - "title": "l2vpn_mac", - "metric_id": "l2vpn_mac", - "key_fields": [ - "mac", - "vpn", - "neighbor", - "ac_port", - "exter_vlan", - "vpn_sid", - "neighbor_sid" - ], - "iface_fields": [], - "compare_fields": [], - "display_fields": [ - "mac", - "vpn", - "neighbor", - "ac_port", - "exter_vlan", - "vpn_sid", - "neighbor_sid", - "vlan", - "pw", - "attribute" - ], - "row_filters": [], - "field_rules": [] - }, - { - "sheet_id": "bgp_route", - "title": "bgp_route_ipv4", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "empty", - "field": "vrf", - "value": "" - }, - { - "op": "eq", - "field": "afi", - "value": "ipv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.vpnv4", - "title": "bgp_route_vpnv4", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpnv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.ipv6", - "title": "bgp_route_ipv6", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "empty", - "field": "vrf", - "value": "" - }, - { - "op": "eq", - "field": "afi", - "value": "ipv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.vpnv6", - "title": "bgp_route_vpnv6", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "eq", - "field": "afi", - "value": "vpnv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.vpnv6_vrf", - "title": "bgp_route_vpnv6_vrf", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "not_empty", - "field": "vrf", - "value": "" - }, - { - "op": "eq", - "field": "afi", - "value": "ipv6" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - }, - { - "sheet_id": "bgp_route.vpnv4_vrf", - "title": "bgp_route_vpnv4_vrf", - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf" - ], - "iface_fields": [], - "compare_fields": [ - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as" - ], - "display_fields": [ - "local_as", - "afi", - "neighbor", - "direction", - "rd", - "network", - "vrf", - "metric", - "loc_prf", - "path", - "as_num", - "state", - "pfx_rcd", - "remote_as", - "next_hop", - "tag", - "status_codes" - ], - "row_filters": [ - { - "op": "not_empty", - "field": "vrf", - "value": "" - }, - { - "op": "eq", - "field": "afi", - "value": "ipv4" - } - ], - "field_rules": [ - { - "field": "pfx_rcd", - "compare": "percent", - "tolerance": 5 - } - ] - } - ] -} diff --git a/netx_api/biz_state/retention.py b/netx_api/biz_state/retention.py index fde4107..d14c554 100644 --- a/netx_api/biz_state/retention.py +++ b/netx_api/biz_state/retention.py @@ -6,7 +6,6 @@ from collections import defaultdict from datetime import datetime, timedelta from typing import Any -from sqlalchemy import or_, select from sqlalchemy.orm import Session from ..models import ( @@ -27,54 +26,38 @@ def _utcnow() -> datetime: return datetime.utcnow() -def protected_batch_map( - db: Session, *, task_id: str = "", batch_ids: list[str] | None = None, -) -> dict[str, list[str]]: - """Only read reference columns, scoped to the requested task/batches when given.""" +def protected_batch_map(db: Session, *, task_id: str = "") -> dict[str, list[str]]: + """batch_id → reason codes. Empty task_id = all tasks.""" out: dict[str, list[str]] = defaultdict(list) - wanted = set(batch_ids) if batch_ids is not None else None - if wanted == set(): - return {} - scope = select(BizStateBatch.id) - q = db.query(BizStateBatch.id, BizStateBatch.is_baseline, BizStateBatch.status) - if task_id: - scope = scope.where(BizStateBatch.task_id == task_id) - q = q.filter(BizStateBatch.task_id == task_id) - if wanted is not None: - scope = scope.where(BizStateBatch.id.in_(wanted)) - q = q.filter(BizStateBatch.id.in_(wanted)) - batch_rows = q.all() if task_id or wanted is not None else q.filter(or_( - BizStateBatch.is_baseline.is_(True), BizStateBatch.status.in_(("queued", "running")), - )).all() - if task_id: - wanted = {row.id for row in batch_rows} def add(bid: str, reason: str) -> None: b = str(bid or "").strip() - if not b or (wanted is not None and b not in wanted): + if not b: return if reason not in out[b]: out[b].append(reason) - for b in batch_rows: - if b.is_baseline: - add(b.id, "manual_baseline") - if b.status in ("queued", "running"): - add(b.id, "active_collection") + q = db.query(BizStateBatch) + if task_id: + q = q.filter(BizStateBatch.task_id == task_id) + for b in q.filter(BizStateBatch.is_baseline.is_(True)).all(): + add(b.id, "manual_baseline") - references = ( - (BizCompareJob.before_batch_id, BizCompareJob.after_batch_id, "compare_job_before", "compare_job_after"), - (BizCompareRun.before_batch_id, BizCompareRun.after_batch_id, "compare_run_before", "compare_run_after"), - (BizMigrationProject.old_baseline_batch_id, BizMigrationProject.new_baseline_batch_id, "migration_old_baseline", "migration_new_baseline"), - (BizMigrationRun.old_batch_id, BizMigrationRun.new_batch_id, "migration_run_old", "migration_run_new"), - ) - for before, after, before_reason, after_reason in references: - refs = db.query(before, after) - if task_id or batch_ids is not None: - refs = refs.filter(or_(before.in_(scope), after.in_(scope))) - for before_id, after_id in refs: - add(before_id, before_reason) - add(after_id, after_reason) + for j in db.query(BizCompareJob).all(): + add(j.before_batch_id, "compare_job_before") + add(j.after_batch_id, "compare_job_after") + + for r in db.query(BizCompareRun).all(): + add(r.before_batch_id, "compare_run_before") + add(r.after_batch_id, "compare_run_after") + + for p in db.query(BizMigrationProject).all(): + add(p.old_baseline_batch_id, "migration_old_baseline") + add(p.new_baseline_batch_id, "migration_new_baseline") + + for r in db.query(BizMigrationRun).all(): + add(r.old_batch_id, "migration_run_old") + add(r.new_batch_id, "migration_run_new") return dict(out) @@ -84,7 +67,7 @@ def protected_batch_ids(db: Session, *, task_id: str = "") -> set[str]: def batch_protect_info(db: Session, batch_id: str) -> dict[str, Any]: - reasons = protected_batch_map(db, batch_ids=[batch_id]).get(batch_id, []) + reasons = protected_batch_map(db).get(batch_id, []) b = db.get(BizStateBatch, batch_id) if b and bool(getattr(b, "is_baseline", False)) and "manual_baseline" not in reasons: reasons = ["manual_baseline", *reasons] diff --git a/netx_api/biz_state/schema_ensure.py b/netx_api/biz_state/schema_ensure.py index 1c11933..6b1013f 100644 --- a/netx_api/biz_state/schema_ensure.py +++ b/netx_api/biz_state/schema_ensure.py @@ -23,7 +23,6 @@ def apply_biz_state_schema(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_command_batch_id ON biz_state_batch_command (batch_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_lldp_neighbor_batch_id ON biz_state_lldp_neighbor (batch_id)", "ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS enabled_sheet_ids JSON DEFAULT '[]'", - "ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS store_unchanged VARCHAR(16) DEFAULT 'auto'", "CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)", "CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)", @@ -32,8 +31,6 @@ def apply_biz_state_schema(conn: Connection) -> None: "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_id ON biz_compare_diff (run_id)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_kind ON biz_compare_diff (run_id, metric_id, kind)", "CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_seq ON biz_compare_diff (run_id, metric_id, seq)", - "ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS before_row_id VARCHAR(64) DEFAULT ''", - "ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS after_row_id VARCHAR(64) DEFAULT ''", "CREATE INDEX IF NOT EXISTS ix_biz_migration_project_status ON biz_migration_project (status)", "CREATE INDEX IF NOT EXISTS ix_biz_migration_batch_project_id ON biz_migration_batch (project_id)", "CREATE INDEX IF NOT EXISTS ix_biz_migration_run_batch_id ON biz_migration_run (batch_id)", @@ -82,15 +79,6 @@ def apply_biz_state_schema(conn: Connection) -> None: "ALTER TABLE biz_migration_project ADD COLUMN IF NOT EXISTS new_hf_bindings_json JSON DEFAULT '[]'", "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS purpose VARCHAR(32) DEFAULT ''", "CREATE INDEX IF NOT EXISTS ix_biz_state_task_purpose ON biz_state_task (purpose)", - "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS collect_group_id VARCHAR(128) DEFAULT ''", - "ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS collect_manual_only BOOLEAN DEFAULT FALSE", - "CREATE INDEX IF NOT EXISTS ix_biz_state_task_collect_group_id ON biz_state_task (collect_group_id)", - "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_group_id VARCHAR(128) DEFAULT ''", - "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_round_id VARCHAR(64) DEFAULT ''", - "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_priority INTEGER DEFAULT 0", - "ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS queued_at TIMESTAMP", - "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_collect_group_id ON biz_state_batch (collect_group_id)", - "CREATE INDEX IF NOT EXISTS ix_biz_state_batch_collect_round_id ON biz_state_batch (collect_round_id)", "ALTER TABLE biz_state_batch_command ADD COLUMN IF NOT EXISTS raw_line_count INTEGER DEFAULT 0", "ALTER TABLE biz_state_batch_command ADD COLUMN IF NOT EXISTS declared_total INTEGER DEFAULT 0", "ALTER TABLE biz_migration_red_ticket ADD COLUMN IF NOT EXISTS match_key_str VARCHAR(256) DEFAULT ''", diff --git a/netx_api/biz_state/service.py b/netx_api/biz_state/service.py index 95e191f..f3ad60a 100644 --- a/netx_api/biz_state/service.py +++ b/netx_api/biz_state/service.py @@ -9,8 +9,8 @@ from typing import Any from uuid import uuid4 from fastapi import HTTPException -from sqlalchemy import String, case, cast, func, or_ -from sqlalchemy.orm import Session, defer +from sqlalchemy import String, cast, func, or_ +from sqlalchemy.orm import Session from ..lldp_shared import resolve_vendor_key from ..models import ( @@ -354,15 +354,13 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]: .order_by(BizStateTaskItem.sort_order.asc()) .all() ) - bindings_by_item: dict[str, list[Any]] = {} - if items: - for binding in db.query(BizStateTaskItemBinding).filter( - BizStateTaskItemBinding.item_id.in_([it.id for it in items]) - ).all(): - bindings_by_item.setdefault(binding.item_id, []).append(binding) item_out = [] for it in items: - binds = bindings_by_item.get(it.id, []) + binds = ( + db.query(BizStateTaskItemBinding) + .filter(BizStateTaskItemBinding.item_id == it.id) + .all() + ) item_out.append( { "id": it.id, @@ -375,11 +373,6 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]: "bindings": [{"placeholder": b.placeholder, "value": b.value} for b in binds], } ) - return {**_task_summary(task), "items": item_out} - - -def _task_summary(task: BizStateTask) -> dict[str, Any]: - """Task metadata without command items or bindings (safe for frequent polling).""" return { "id": task.id, "source": task.source, @@ -396,8 +389,6 @@ def _task_summary(task: BizStateTask) -> dict[str, Any]: "daily_keep_enabled": bool(getattr(task, "daily_keep_enabled", False)), "daily_keep_count": int(getattr(task, "daily_keep_count", None) or 10), "collect_running": bool(task.collect_running), - "collect_group_id": task.collect_group_id or "", - "collect_manual_only": bool(task.collect_manual_only), "last_collect_started_at": task.last_collect_started_at.isoformat() + "Z" if task.last_collect_started_at else None, @@ -405,16 +396,10 @@ def _task_summary(task: BizStateTask) -> dict[str, Any]: if task.last_collect_ended_at else None, "last_error": task.last_error, + "items": item_out, } -def get_task_progress(db: Session, task_id: str) -> dict[str, Any]: - task = db.get(BizStateTask, task_id) - if not task: - raise HTTPException(status_code=404, detail="task_not_found") - return _task_summary(task) - - def list_tasks(db: Session, *, purpose: str | None = None) -> list[dict[str, Any]]: from sqlalchemy import or_ @@ -446,8 +431,6 @@ def list_tasks(db: Session, *, purpose: str | None = None) -> list[dict[str, Any "status": t.status, "interval_sec": t.interval_sec, "collect_running": bool(t.collect_running), - "collect_group_id": t.collect_group_id or "", - "collect_manual_only": bool(t.collect_manual_only), "last_error": t.last_error, "last_collect_started_at": t.last_collect_started_at.isoformat() + "Z" if t.last_collect_started_at @@ -464,8 +447,6 @@ def delete_task(db: Session, task_id: str) -> None: task = db.get(BizStateTask, task_id) if not task: raise HTTPException(status_code=404, detail="task_not_found") - if task.collect_running: - raise HTTPException(status_code=409, detail="task_collecting") batches = db.query(BizStateBatch).filter(BizStateBatch.task_id == task_id).all() # Refuse if any batch is still referenced by compare/migration (manual baseline is OK to drop with task) pmap = protected_batch_map(db, task_id=task_id) @@ -519,7 +500,7 @@ def list_batches(db: Session, task_id: str, *, limit: int = 50) -> list[dict[str .limit(max(1, min(500, int(limit)))) .all() ) - pmap = protected_batch_map(db, batch_ids=[b.id for b in rows]) + pmap = protected_batch_map(db, task_id=task_id) out: list[dict[str, Any]] = [] for b in rows: reasons = list(pmap.get(b.id, [])) @@ -641,29 +622,12 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: b = db.get(BizStateBatch, batch_id) if not b: raise HTTPException(status_code=404, detail="batch_not_found") - raw = func.coalesce(BizStateBatchCommand.raw_text, "") - raw_len = func.length(raw) - # Compute legacy counts in SQL; workbook summaries never transfer raw CLI text. - line_count = case( - (BizStateBatchCommand.raw_line_count > 0, BizStateBatchCommand.raw_line_count), - (raw_len == 0, 0), - else_=raw_len - func.length(func.replace(raw, "\n", "")) - + case((func.substr(raw, raw_len, 1) == "\n", 0), else_=1), - ) - whitespace = " \t\r\n\v\f" - trimmed = (func.btrim(raw, whitespace) if db.get_bind().dialect.name == "postgresql" - else func.trim(raw, whitespace)) - cmd_raw_stats: dict[str, tuple[int, bool]] = {} - cmds = [] - for c, lines, has_raw in ( - db.query(BizStateBatchCommand, line_count, func.length(trimmed) > 0) - .options(defer(BizStateBatchCommand.raw_text)) + cmds = ( + db.query(BizStateBatchCommand) .filter(BizStateBatchCommand.batch_id == batch_id) - .order_by(BizStateBatchCommand.created_at.asc(), BizStateBatchCommand.id.asc()) + .order_by(BizStateBatchCommand.created_at.asc()) .all() - ): - cmds.append(c) - cmd_raw_stats[c.id] = (int(lines or 0), bool(has_raw)) + ) protect = batch_protect_info(db, batch_id) # Per-metric row counts (generic table) @@ -704,14 +668,11 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: # Prefer primary collect rows over aux / aux_cached for the same CLI sheet_cmds[id_].append(cmd_info) - # Aux failures must remain visible even if their primary command was stored later. - primary_cmds_by_cli = { - normalize_command(str(c.raw_command or "")) for c in cmds - if not str(c.parse_status or "").strip().lower().startswith("aux") - } + primary_cmds_by_cli: dict[str, dict[str, Any]] = {} for c in cmds: status = str(c.parse_status or "").strip().lower() is_aux = status.startswith("aux") + raw = c.raw_text or "" info = { "id": c.id, "profile_id": c.profile_id, @@ -721,13 +682,15 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: "params": c.params_json or {}, "parse_status": c.parse_status, "row_count": c.row_count, - "raw_line_count": cmd_raw_stats[c.id][0], + "raw_line_count": _cmd_raw_line_count(c), "declared_total": _cmd_declared_total(c), "message": c.message, - "has_raw": cmd_raw_stats[c.id][1], + "has_raw": bool(str(raw).strip()), "is_aux": is_aux, } cmd_n = normalize_command(str(c.raw_command or "")) + if not is_aux and cmd_n: + primary_cmds_by_cli.setdefault(cmd_n, info) # Commands sheet: hide successful aux when the same CLI already has a primary row; # keep failed/skipped aux visible so partial reasons are not hidden. if is_aux and cmd_n and cmd_n in primary_cmds_by_cli: @@ -783,10 +746,10 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]: "params": c.params_json or {}, "parse_status": c.parse_status, "row_count": c.row_count, - "raw_line_count": cmd_raw_stats[c.id][0], + "raw_line_count": _cmd_raw_line_count(c), "declared_total": _cmd_declared_total(c), "message": c.message, - "has_raw": cmd_raw_stats[c.id][1], + "has_raw": bool(str(c.raw_text or "").strip()), "is_aux": True, } ) @@ -881,7 +844,6 @@ def list_batch_metric_rows( size_n = max(1, min(200, int(page_size or 50))) kw_n = str(kw or "").strip() col_n = str(column or "").strip() - like = "%" + kw_n.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%" fields = metric_field_map().get(mid) or [] columns = [ @@ -897,30 +859,30 @@ def list_batch_metric_rows( if mid == "lldp_neighbor": q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id) if kw_n: + like = f"%{kw_n}%" if col_n == "local_if": - q = q.filter(BizStateLldpNeighbor.local_if.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.local_if.ilike(like)) elif col_n == "remote_sys": - q = q.filter(BizStateLldpNeighbor.remote_sys.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.remote_sys.ilike(like)) elif col_n == "remote_if": - q = q.filter(BizStateLldpNeighbor.remote_if.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.remote_if.ilike(like)) elif col_n == "remote_ip": - q = q.filter(BizStateLldpNeighbor.remote_ip.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.remote_ip.ilike(like)) elif col_n == "protocol": - q = q.filter(BizStateLldpNeighbor.protocol.ilike(like, escape="\\")) + q = q.filter(BizStateLldpNeighbor.protocol.ilike(like)) else: q = q.filter( or_( - BizStateLldpNeighbor.local_if.ilike(like, escape="\\"), - BizStateLldpNeighbor.remote_sys.ilike(like, escape="\\"), - BizStateLldpNeighbor.remote_if.ilike(like, escape="\\"), - BizStateLldpNeighbor.remote_ip.ilike(like, escape="\\"), - BizStateLldpNeighbor.protocol.ilike(like, escape="\\"), + BizStateLldpNeighbor.local_if.ilike(like), + BizStateLldpNeighbor.remote_sys.ilike(like), + BizStateLldpNeighbor.remote_if.ilike(like), + BizStateLldpNeighbor.remote_ip.ilike(like), + BizStateLldpNeighbor.protocol.ilike(like), ) ) total = int(q.count() or 0) rows_db = ( - q.order_by(BizStateLldpNeighbor.local_if.asc(), BizStateLldpNeighbor.remote_sys.asc(), - BizStateLldpNeighbor.remote_if.asc(), BizStateLldpNeighbor.id.asc()) + q.order_by(BizStateLldpNeighbor.local_if.asc()) .offset((page_n - 1) * size_n) .limit(size_n) .all() @@ -949,11 +911,12 @@ def list_batch_metric_rows( BizStateMetricRow.metric_id == mid, ) if kw_n: + like = f"%{kw_n}%" if col_n: # JSON path as text — works on Postgres JSONB and SQLite JSON - q = q.filter(cast(BizStateMetricRow.data_json[col_n].as_string(), String).ilike(like, escape="\\")) + q = q.filter(cast(BizStateMetricRow.data_json[col_n], String).ilike(like)) else: - q = q.filter(cast(BizStateMetricRow.data_json, String).ilike(like, escape="\\")) + q = q.filter(cast(BizStateMetricRow.data_json, String).ilike(like)) total = int(q.count() or 0) rows_db = ( q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) @@ -1056,64 +1019,56 @@ def export_batch_zip(db: Session, batch_id: str) -> bytes: db.query(BizStateLldpNeighbor) .filter(BizStateLldpNeighbor.batch_id == batch_id) .order_by(BizStateLldpNeighbor.local_if.asc()) - .yield_per(1000) + .all() ) - with zf.open("tables/lldp_neighbor.csv", "w") as dest: - dest.write(b"local_if,remote_sys,remote_if,remote_ip,protocol\n") - for n in neighbors: - line = ",".join(_csv(v) for v in ( - n.local_if, n.remote_sys, n.remote_if, n.remote_ip, n.protocol, - )) + "\n" - dest.write(line.encode("utf-8")) + csv_lines = ["local_if,remote_sys,remote_if,remote_ip,protocol"] + for n in neighbors: + csv_lines.append( + ",".join( + [ + _csv(n.local_if), + _csv(n.remote_sys), + _csv(n.remote_if), + _csv(n.remote_ip), + _csv(n.protocol), + ] + ) + ) + zf.writestr("tables/lldp_neighbor.csv", "\n".join(csv_lines) + "\n") # Generic metrics CSV (stream by metric_id) for sheet in detail.get("sheets") or []: mid = str(sheet.get("metric_id") or "").strip() if not mid or mid == "lldp_neighbor": continue - # First scan discovers the same complete, ordered header as the legacy export. - # Second scan writes rows without materializing the table or CSV. - col_keys: dict[str, None] = {} - has_rows = False - for rec in _iter_metric_export_rows(db, batch_id, mid): - has_rows = True - for k in rec.keys(): - col_keys.setdefault(str(k), None) - if not has_rows: + rows = ( + db.query(BizStateMetricRow) + .filter( + BizStateMetricRow.batch_id == batch_id, + BizStateMetricRow.metric_id == mid, + ) + .order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()) + .all() + ) + if not rows: continue - cols = list(col_keys) + recs = [dict(r.data_json or {}) for r in rows] + cols: list[str] = [] + for rec in recs: + for k in rec.keys(): + if k not in cols: + cols.append(str(k)) + out_lines = [",".join(_csv(c) for c in cols)] + for rec in recs: + out_lines.append(",".join(_csv(str(rec.get(c, "") or "")) for c in cols)) safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "metric" - with zf.open(f"tables/{safe}.csv", "w") as dest: - dest.write((",".join(_csv(c) for c in cols) + "\n").encode("utf-8")) - for rec in _iter_metric_export_rows(db, batch_id, mid): - line = ",".join(_csv(rec.get(c)) for c in cols) + "\n" - dest.write(line.encode("utf-8")) + zf.writestr(f"tables/{safe}.csv", "\n".join(out_lines) + "\n") return buf.getvalue() -def _iter_metric_export_rows(db: Session, batch_id: str, metric_id: str, chunk_size: int = 1000): - cursor: tuple[int, str] | None = None - while True: - query = db.query(BizStateMetricRow.seq, BizStateMetricRow.id, BizStateMetricRow.data_json).filter( - BizStateMetricRow.batch_id == batch_id, BizStateMetricRow.metric_id == metric_id, - ) - if cursor is not None: - seq, row_id = cursor - query = query.filter(or_( - BizStateMetricRow.seq > seq, - (BizStateMetricRow.seq == seq) & (BizStateMetricRow.id > row_id), - )) - chunk = query.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()).limit(chunk_size).all() - if not chunk: - return - for row in chunk: - yield row.data_json or {} - cursor = (chunk[-1].seq, chunk[-1].id) - - -def _csv(v: Any) -> str: - s = "" if v is None else str(v) - if any(ch in s for ch in ",\"\r\n"): +def _csv(v: str) -> str: + s = str(v or "") + if any(ch in s for ch in ",\"\n"): return '"' + s.replace('"', '""') + '"' return s diff --git a/netx_api/biz_state_router.py b/netx_api/biz_state_router.py index fa1e936..4a4f3ed 100644 --- a/netx_api/biz_state_router.py +++ b/netx_api/biz_state_router.py @@ -173,11 +173,6 @@ def api_get_task(task_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return svc.get_task(db, task_id) -@router.get("/tasks/{task_id}/progress") -def api_task_progress(task_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: - return svc.get_task_progress(db, task_id) - - @router.get("/tasks/{task_id}/commands") def api_plan_task_commands( task_id: str, @@ -574,7 +569,6 @@ class TemplateMetricIn(BaseModel): display_fields: list[str] = Field(default_factory=list) row_filters: list[dict[str, Any]] = Field(default_factory=list) field_rules: list[FieldRuleIn] = Field(default_factory=list) - ignore_port_changes: bool | None = None class TemplateIn(BaseModel): @@ -593,7 +587,6 @@ class TemplateIn(BaseModel): display_fields: list[str] = Field(default_factory=list) row_filters: list[dict[str, Any]] = Field(default_factory=list) field_rules: list[FieldRuleIn] = Field(default_factory=list) - ignore_port_changes: bool | None = None class TemplatePatchIn(BaseModel): @@ -609,7 +602,6 @@ class TemplatePatchIn(BaseModel): display_fields: list[str] | None = None row_filters: list[dict[str, Any]] | None = None field_rules: list[FieldRuleIn] | None = None - ignore_port_changes: bool | None = None class MappingRowIn(BaseModel): @@ -641,8 +633,6 @@ class CompareJobIn(BaseModel): mode: str = "manual" # Empty = all template sheets; non-empty = only these sheet_id values enabled_sheet_ids: list[str] = Field(default_factory=list) - # auto|always|never|sample|keys — how to persist matching (success) rows - store_unchanged: str = "auto" note: str = "" @@ -733,15 +723,8 @@ def api_delete_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any] @router.post("/compare/jobs/{job_id}/run") -def api_run_job( - job_id: str, - sync: bool = Query(False, description="If true, block until compare finishes (tests/debug)"), - db: Session = Depends(get_db), -) -> dict[str, Any]: - """Start a compare run. Default is async (returns status=running immediately).""" - if sync: - return cmp_svc.run_compare(db, job_id) - return cmp_svc.enqueue_compare(db, job_id) +def api_run_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: + return cmp_svc.run_compare(db, job_id) @router.get("/compare/jobs/{job_id}/runs") @@ -754,12 +737,6 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return cmp_svc.get_run(db, run_id) -@router.post("/compare/runs/{run_id}/cancel") -def api_cancel_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: - """Cancel a stuck/running compare so a new run can start.""" - return cmp_svc.cancel_compare_run(db, run_id) - - @router.delete("/compare/runs/{run_id}") def api_delete_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]: return cmp_svc.delete_run(db, run_id) @@ -771,7 +748,6 @@ def api_list_run_diffs( metric_id: str = "", kind: str = "diff", kw: str = "", - qf: str = "", page: int = 1, page_size: int = 100, db: Session = Depends(get_db), @@ -782,7 +758,6 @@ def api_list_run_diffs( metric_id=metric_id, kind=kind, kw=kw, - qf=qf, page=page, page_size=page_size, ) diff --git a/netx_api/biz_state_scheduler.py b/netx_api/biz_state_scheduler.py index 49dcd07..d44250f 100644 --- a/netx_api/biz_state_scheduler.py +++ b/netx_api/biz_state_scheduler.py @@ -110,21 +110,12 @@ def _sync_cutover_hf_windows() -> None: projects = db.query(BizMigrationProject).all() for proj in projects: - tids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new") - if tids and any((t := db.get(BizStateTask, tid)) and not t.collect_group_id for tid in tids): - try: - from .biz_migration.service import ensure_highfreq - - ensure_highfreq(db, proj.id, collect_now=False) - except Exception: - _log.exception("upgrade cutover collection pairs failed project=%s", proj.id) - window_status = _hf_window_status(proj) + want = _hf_window_status(proj) tids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new") for tid in tids: task = db.get(BizStateTask, tid) if not task: continue - want = "paused" if task.collect_manual_only else window_status purpose = str(getattr(task, "purpose", None) or "").strip() if purpose and purpose != PURPOSE_CUTOVER_HF: continue @@ -158,26 +149,12 @@ def _enqueue_due_tasks() -> int: .filter( BizStateTask.status == "running", BizStateTask.collect_running.is_(False), - BizStateTask.collect_manual_only.is_(False), ) .all() ) due_ids: list[str] = [] - seen_groups: set[str] = set() now = _utcnow() for task in tasks: - if task.collect_group_id: - if task.collect_group_id in seen_groups: - continue - seen_groups.add(task.collect_group_id) - peers = db.query(BizStateTask).filter(BizStateTask.collect_group_id == task.collect_group_id).all() - if len(peers) != 2 or any(p.collect_running or p.status != "running" or p.collect_manual_only for p in peers): - continue - anchors = [p.last_collect_started_at or p.last_collect_ended_at for p in peers] - anchor = max((a for a in anchors if a), default=None) - if anchor is None or (now - anchor).total_seconds() >= max(p.interval_sec or 60 for p in peers): - due_ids.append(task.id) - continue interval = max(60, int(task.interval_sec or 300)) # Prefer start-based interval to avoid drift when collect duration # approaches the interval (end-based: 40s collect + 60s → 100s cycle). diff --git a/netx_api/config.py b/netx_api/config.py index e12b5fd..1126fc5 100644 --- a/netx_api/config.py +++ b/netx_api/config.py @@ -80,9 +80,8 @@ class Settings(BaseSettings): ume_topo_nodes_path: str = "/restconf/data/zte-resources-module:TopoNodes" ume_topological_links_path: str = "/restconf/data/zte-resources-module:TopologicalLinks" ume_sync_alarms_history_every_hours: int = 24 - # Managed NE credentials (Fernet). Empty = auto-generate & persist to credential_secret_file. + # Managed NE credentials (Fernet key; generate with cryptography.fernet.Fernet.generate_key()) credential_secret_key: str = "" - credential_secret_file: str = "data/auth/credential_secret" # Shared-server worker caps (sized for multi-operator use; raise if bastion/DB allow). ne_connect_max_workers: int = 8 ne_connect_timeout_sec: int = 30 diff --git a/netx_api/models/biz_state.py b/netx_api/models/biz_state.py index 864f85d..bd8c311 100644 --- a/netx_api/models/biz_state.py +++ b/netx_api/models/biz_state.py @@ -32,9 +32,6 @@ class BizStateTask(Base): note: Mapped[str] = mapped_column(String(256), default="") # "" | portrait | cutover_hf — filter cutover HF in biz-state list purpose: Mapped[str] = mapped_column(String(32), default="", index=True) - # Optional paired sampling; ordinary per-device tasks remain independent. - collect_group_id: Mapped[str] = mapped_column(String(128), default="", index=True) - collect_manual_only: Mapped[bool] = mapped_column(Boolean, default=False) status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|running|paused|stopped interval_sec: Mapped[int] = mapped_column(Integer, default=300) # Keep snapshots for N calendar days (protected baselines/refs never auto-deleted) @@ -89,10 +86,6 @@ class BizStateBatch(Base): id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex) task_id: Mapped[str] = mapped_column(String(64), default="", index=True) - collect_group_id: Mapped[str] = mapped_column(String(128), default="", index=True) - collect_round_id: Mapped[str] = mapped_column(String(64), default="", index=True) - collect_priority: Mapped[int] = mapped_column(Integer, default=0) - queued_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) source: Mapped[str] = mapped_column(String(32), default="managed", index=True) ne_id: Mapped[str] = mapped_column(String(128), default="", index=True) ne_name: Mapped[str] = mapped_column(String(256), default="") @@ -310,8 +303,6 @@ class BizCompareJob(Base): status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|ready|auto # Empty = all template sheets; non-empty = only these sheet_id values enabled_sheet_ids: Mapped[list] = mapped_column(_JsonType, default=list) - # auto|always|never|sample — how to persist matching (success) rows - store_unchanged: Mapped[str] = mapped_column(String(16), default="auto") note: Mapped[str] = mapped_column(String(512), default="") created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive) @@ -356,7 +347,4 @@ class BizCompareDiff(Base): after_json: Mapped[dict] = mapped_column(_JsonType, default=dict) mapped_before_json: Mapped[dict] = mapped_column(_JsonType, default=dict) changes_json: Mapped[dict] = mapped_column(_JsonType, default=dict) - # Pointers into BizStateMetricRow / LLDP tables for compact success hydrate - before_row_id: Mapped[str] = mapped_column(String(64), default="") - after_row_id: Mapped[str] = mapped_column(String(64), default="") search_text: Mapped[str] = mapped_column(Text, default="") diff --git a/netx_api/ne_crypto.py b/netx_api/ne_crypto.py index 7624c2a..4ba7617 100644 --- a/netx_api/ne_crypto.py +++ b/netx_api/ne_crypto.py @@ -1,73 +1,18 @@ from __future__ import annotations -import logging -from pathlib import Path - from cryptography.fernet import Fernet, InvalidToken from .config import settings -_log = logging.getLogger("netx.crypto") -_cached_credential_key: str | None = None - class CredentialCryptoError(RuntimeError): pass -def credential_secret_file_path() -> Path: - raw = str(getattr(settings, "credential_secret_file", None) or "data/auth/credential_secret").strip() - path = Path(raw) - if not path.is_absolute(): - path = Path.cwd() / path - return path - - -def ensure_credential_secret_key() -> str: - """Resolve Fernet key: env ``NETX_CREDENTIAL_SECRET_KEY`` > file > generate once. - - Packaged installs should also write the key into ``.env`` via setup_first_run; - this startup/path fallback covers upgrades and missed first-run keys. - """ - global _cached_credential_key - configured = str(settings.credential_secret_key or "").strip() - if configured: - return configured - if _cached_credential_key: - return _cached_credential_key - - path = credential_secret_file_path() - try: - if path.is_file(): - existing = path.read_text(encoding="utf-8").strip() - if existing: - _cached_credential_key = existing - settings.credential_secret_key = existing - return existing - except Exception: - _log.exception("read credential secret file failed path=%s", path) - - generated = Fernet.generate_key().decode("ascii") - try: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(generated + "\n", encoding="utf-8") - try: - path.chmod(0o600) - except Exception: - pass - _log.info("wrote per-install credential Fernet key to %s", path) - except Exception: - _log.exception( - "write credential secret file failed path=%s; using in-memory key only", - path, - ) - _cached_credential_key = generated - settings.credential_secret_key = generated - return generated - - def _fernet() -> Fernet: - key = ensure_credential_secret_key() + key = str(settings.credential_secret_key or "").strip() + if not key: + raise CredentialCryptoError("credential_secret_key_not_configured") try: return Fernet(key.encode("ascii")) except Exception as exc: @@ -92,7 +37,4 @@ def decrypt_secret(value: str) -> str: def credentials_configured() -> bool: - try: - return bool(ensure_credential_secret_key()) - except Exception: - return False + return bool(str(settings.credential_secret_key or "").strip()) diff --git a/packaging/README.md b/packaging/README.md index 847943d..4f22d6f 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -7,22 +7,14 @@ This directory builds a Windows deliverable with: - Optional **bundled** portable PostgreSQL **or** **external** existing Postgres - API-hosted UI (`web/dist`) — no separate Vite process for end users - Program / data split so upgrades do not wipe the database -- Manual update script + check/apply updates (GitHub Releases or custom manifest) -- Optional system tray + start-at-logon +- Manual update script + auto-update **manifest contract** (fetch not implemented yet) ## What users get | Artifact | How | |----------|-----| -| `NetX-Setup-x.y.z.exe` | Stage with `build_release.ps1`, then compile `installer/netx.iss` with [Inno Setup](https://jrsoftware.org/isinfo.php) | - -Zip packages are **not** published. Releases ship **Setup.exe only**. - -**Installer:** English + 简体中文 (language dialog). Icons use `packaging/assets/netx.ico`. - -**Offline:** Setup ships portable PostgreSQL, **`python/runtime` + `.venv`** (portable; not tied to the build PC’s user profile), and WinSW. **First install** shows the Database page (built-in or external PostgreSQL; external credentials validated with `psql SELECT 1`). **Re-running Setup when `%ProgramData%\NetX\.env` already has `NETX_DB_MODE`** skips the DB wizard and updates program files in place — **no GitHub/EDB download on the target PC**. Service install uses bundled WinSW (pass `-AllowDownload` only on a build/dev machine if the binary is missing). Auto-update still needs network later, and is unchecked by default. - -**OS:** Windows 10/11 or Windows Server **2016+** recommended. Packaging scripts are UTF-8 **with BOM** so Chinese UI works on Windows PowerShell 5.x. Bundled Python in current releases is **3.13+**, which does **not** support Windows Server 2012 R2 — use Server 2016+ or a newer desktop OS. +| `NetX-x.y.z-win64.zip` | `build_release.ps1` | +| `NetX-Setup-x.y.z.exe` | Compile `installer/netx.iss` with [Inno Setup](https://jrsoftware.org/isinfo.php) after staging | ## Build (developer machine) @@ -33,13 +25,14 @@ cd netx # Optional: download portable Postgres into packaging\postgres\pgsql powershell -ExecutionPolicy Bypass -File .\packaging\download_postgres.ps1 -# Build web + stage (-CreateVenv ships a ready .venv; large). No zip by default. +# Build web + stage + zip (-CreateVenv ships a ready .venv; large) powershell -ExecutionPolicy Bypass -File .\packaging\build_release.ps1 -CreateVenv ``` Output: -- `packaging/release/netx-win64/` — stage tree (input to Inno Setup) +- `packaging/release/netx-win64/` — stage tree +- `packaging/release/NetX--win64.zip` Inno Setup: @@ -47,29 +40,29 @@ Inno Setup: ISCC.exe packaging\installer\netx.iss ``` -Override version in the `.iss` or edit `#define MyAppVersion`. -Optional local zip only: `build_release.ps1 -CreateZip` (not for release upload). +Override version in the `.iss` or edit `#define MyAppVersion`. ## End-user install ### Setup.exe 1. Run `NetX-Setup-x.y.z.exe` (admin). -2. **First install:** Database page — choose built-in (offline) or external PostgreSQL (host/port/user/password/db; connection must succeed to continue). -3. **Already installed:** if `%ProgramData%\NetX\.env` already has `NETX_DB_MODE`, Setup shows an **Install mode** page: - - **Update** (default) — skip Database page; stop NetX; replace program files; **keep** ProgramData / DB settings. - - **Reinstall** — same Database wizard as first install (`ApplyDatabaseConfig`); does **not** delete ProgramData (use Uninstall → delete data for a wipe). -4. Files → `%ProgramFiles%\NetX\` (program root). -5. Data → `%ProgramData%\NetX\` (`.env`, `pgdata`, spool, secrets). -6. Start menu: **Start NetX** / **Stop NetX** / **Open NetX UI** / **Reconfigure database**. +2. Files → `%ProgramFiles%\NetX\` (program root). +3. Data → `%ProgramData%\NetX\` (`.env`, `pgdata`, spool, secrets). +4. Optional post-install task runs `setup_first_run.ps1` (choose bundled vs external DB). +5. Start menu: **Start NetX** / **Stop NetX** / **Open NetX UI**. -**Elevation (required on fresh PCs):** all end-user entry points (`NetX-Start.cmd`, `NetX-Tray.cmd`, `NetX-Stop.cmd`, `NetX-FirstRun.cmd`, tray, start/stop/setup/update) **self-elevate via UAC**. Program Files installs cannot start reliably as a normal user (`.venv` repair / writes). Approve the UAC prompt once per launch (tray stays elevated afterward). +### Zip (portable) -Silent first install (bundled default): `/SILENT /DbMode=bundled` -Silent external: `/SILENT /DbMode=external /DbHost=... /DbPort=5432 /DbUser=... /DbPassword=... /DbName=...` -Silent update over existing data: `/VERYSILENT /NORESTART /InstallMode=update` (also `/SkipDbPage=1`) -Silent reinstall (DB wizard via params): `/VERYSILENT /InstallMode=reinstall /DbMode=bundled` (or external `/DbHost`…) — also `/ForceDbPage=1` -Optional credential key (reuse encrypted NE passwords from another install): `/CredentialSecretKey=...` — omit to auto-generate. +1. Unpack anywhere. +2. Marker `.portable` → data root is sibling `NetXData\`. +3. Target needs **Python 3.11+** on PATH unless the zip was built with `-CreateVenv`. +4. Run: + +```powershell +.\packaging\setup_first_run.ps1 +.\packaging\start_netx_app.ps1 +``` ## Database modes @@ -83,123 +76,15 @@ Existing Windows deploys that only set `NETX_DATABASE_URL` keep working: never s ## Manual update -Preferred: run a newer `NetX-Setup-*.exe` and choose **Update** (or silent `/InstallMode=update`) so ProgramData is kept. Setup (elevated) always relinks `.venv` → `python/runtime` after file copy; runtime scripts also self-elevate so Start/Tray work on locked-down fresh PCs. - -Legacy/dev zip (only if you built with `-CreateZip`): - ```powershell -.\packaging\update_netx.ps1 -PackagePath .\NetX-0.4.0-win64.zip +.\packaging\update_netx.ps1 -PackagePath .\NetX-0.3.0-win64.zip ``` Stops services, replaces program folders (`netx_api`, `web`, `packaging`, `postgres`, …), **keeps** the data root, restarts. Schema migrations still run via Alembic on API start. -## Check / apply updates +## Auto-update (reserved) -**Defaults (no `.env` needed):** probe **GitHub** (`hansjone/netx`) and Forgejo mirror (`https://git.avelo.top/hansjone/netx`), then use the **newest reachable** version. Same version → prefer GitHub. - -```env -# Optional overrides in ProgramData\NetX\.env -# NETX_UPDATE_SOURCES=github,forgejo -# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest -# NETX_UPDATE_GITHUB_REPO=hansjone/netx -# NETX_UPDATE_URL=https://cdn.example.com/netx/manifest.json -# NETX_UPDATE_TOKEN=****** -``` - -| Variable | Role | -|----------|------| -| `NETX_UPDATE_FORGEJO_URL` | Forgejo/Gitea `releases/latest` API (default: git.avelo.top mirror) | -| `NETX_UPDATE_GITHUB_REPO` | GitHub `owner/repo` (default `hansjone/netx`) | -| `NETX_UPDATE_SOURCES` | Probe order / tie-break order, e.g. `github,forgejo` | -| `NETX_UPDATE_URL` | Optional custom JSON manifest | - -```powershell -.\packaging\check_update.ps1 -.\packaging\check_update.ps1 -Apply -``` - -Both sides should publish the same **Setup.exe**. `check_update.ps1` prefers `NetX-Setup-*.exe` (legacy `win64.zip` still works if present). **Code mirror alone is not enough** — Forgejo pull-mirror syncs git/tags only; Release assets must be uploaded separately (or clients can only fall back to GitHub). - -### Maintainer release checklist (Windows) - -Do this on the **dev PC on the home LAN** after bumping version: - -1. **Build** — `.\packaging\build_release.ps1 -CreateVenv` + Inno Setup → `NetX-Setup-*.exe` -2. **GitHub** — `.\packaging\publish_release.ps1 -Version x.y.z` (uploads Setup.exe only) -3. **Forgejo** — upload the same Setup.exe (default: public domain `https://git.avelo.top`): - -```powershell -# One-time: User env var (never commit the token) -# Forgejo → Settings → Applications → Generate New Token (repo write) -[Environment]::SetEnvironmentVariable("NETX_FORGEJO_TOKEN", "your_token", "User") -# Restart Cursor / open a new terminal so the agent/scripts see it - -# Default: https://git.avelo.top -.\packaging\publish_forgejo_release.ps1 -Version 0.4.11 - -# Optional when LAN IP is reachable: -# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 -ForgejoBase "http://10.0.0.131:3000" -``` - -| Role | URL | -|------|-----| -| Publish default | `https://git.avelo.top` (`-ForgejoBase` default) | -| Optional LAN | `http://10.0.0.131:3000` | -| Update probe (default) | GitHub + `https://git.avelo.top/.../releases/latest` | - -Verify: - -- https://git.avelo.top/hansjone/netx/releases -- Probe: `.\packaging\check_update.ps1` → both `github` and `forgejo` reachable with the new version - -Token: `NETX_FORGEJO_TOKEN` (or `FORGEJO_TOKEN`). - -## Tray & autostart - -```powershell -# System tray: Start / Stop / Open UI / Check updates (self-elevates) -.\packaging\netx_tray.ps1 -StartOnLaunch - -# Start tray at Windows logon — Scheduled Task with RunLevel Highest -# (replaces legacy HKCU\Run which cannot elevate on fresh installs) -.\packaging\install_autostart.ps1 -# Remove: .\packaging\install_autostart.ps1 -Remove -``` - -## Windows Service (admin) - -Uses [WinSW](https://github.com/winsw/winsw) (downloaded on first install into `packaging/cache`). -`service_run.ps1` probes `/health` and restarts children after consecutive failures. - -```powershell -# Elevated PowerShell -.\packaging\install_service.ps1 -Start -# Uninstall: .\packaging\install_service.ps1 -Uninstall - -# Fallback without WinSW binary management: SYSTEM scheduled task at startup -.\packaging\install_service.ps1 -Mode task -Start -``` - -## Silent auto-update - -```powershell -# Writes NETX_UPDATE_AUTO=true and registers a daily task (default 03:30, RunLevel Highest) -.\packaging\install_update_task.ps1 - -# Manual silent path (only applies when NETX_UPDATE_AUTO=true; self-elevates) -.\packaging\check_update.ps1 -Apply -Quiet -AutoOnly -``` - -Tray also auto-applies on launch when `NETX_UPDATE_AUTO=true`. - -## Code signing (optional, publisher machine) - -```powershell -.\packaging\sign_release.ps1 -Files .\packaging\release\NetX-Setup-0.4.0.exe -Thumbprint -# or: -PfxPath .\certs\code.pfx -PfxPassword *** -``` - -Needs `signtool.exe` (Windows SDK) and a real code-signing certificate. Without a trusted cert, SmartScreen may still warn. +See `manifest.example.json`. Future: set `NETX_UPDATE_URL` + `NETX_UPDATE_CHANNEL`; a checker will download the zip and call `update_netx.ps1`. Not implemented in this phase. ## Layout reminder diff --git a/packaging/_common.ps1 b/packaging/_common.ps1 index 194e8f7..4a1ef66 100644 --- a/packaging/_common.ps1 +++ b/packaging/_common.ps1 @@ -1,4 +1,4 @@ -# Shared path helpers for Windows packaging scripts. +# Shared path helpers for Windows packaging scripts. # Dot-source: . "$PSScriptRoot\_common.ps1" $ErrorActionPreference = "Stop" @@ -114,33 +114,7 @@ function Write-DotEnvValue { if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } - $text = ($lines -join "`r`n") - try { - Set-Content -LiteralPath $Path -Value $text -Encoding utf8 - } catch { - # Admin-created .env is often Users:RX only — repair ACL then retry once. - $root = $dir - if ((Split-Path -Leaf $dir) -eq "NetX" -or $dir -match '\\NetX$') { - $root = $dir - } else { - $parent = Split-Path -Parent $dir - if ($parent -and ((Split-Path -Leaf $parent) -eq "NetX")) { $root = $parent } - } - $null = Ensure-NetxDataAcl -DataRoot $root -Quiet - try { - # Reset .env ACL explicitly if still blocked. - $icacls = Join-Path $env:SystemRoot "System32\icacls.exe" - if (Test-Path -LiteralPath $icacls) { - & $icacls $Path /grant "*S-1-5-32-545:M" /C /Q 2>$null | Out-Null - } - Set-Content -LiteralPath $Path -Value $text -Encoding utf8 - } catch { - throw (New-Object System.UnauthorizedAccessException( - ("access_denied: cannot write {0}. Run Start Menu → NetX → Reconfigure database as Administrator, or: icacls `"{1}`" /grant Users:(OI)(CI)M /T" -f $Path, $root), - $_.Exception - )) - } - } + Set-Content -Path $Path -Value ($lines -join "`r`n") -Encoding utf8 } function Get-DbMode { @@ -161,473 +135,3 @@ function Import-NetxEnvFile { } return $map } - -function ConvertTo-NetxFsPath([string]$Path) { - # Forward slashes work in .env and most Windows APIs via Python pathlib. - return ($Path -replace '\\', '/') -} - -function Get-NetxDataEnvMap { - param([string]$DataRoot) - $d = $DataRoot - return @{ - "NETX_AUTH_MCP_TOKEN_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\mcp_token")) - "NETX_AUTH_SECRET_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\jwt_secret")) - "NETX_CREDENTIAL_SECRET_FILE" = (ConvertTo-NetxFsPath (Join-Path $d "data\auth\credential_secret")) - "NETX_SCHEDULER_HEARTBEAT_PATH" = (ConvertTo-NetxFsPath (Join-Path $d "data\runtime\scheduler_heartbeat.json")) - "NETX_RUN_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\runtime")) - "NETX_BIZ_STATE_SPOOL_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\biz_state_spool")) - "NETX_NE_COLLECTION_DATA_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\ne_collections")) - "NETX_NE_EXEC_JOB_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\ne_exec_jobs")) - "NETX_WEBCRT_DATA_DIR" = (ConvertTo-NetxFsPath (Join-Path $d "data\webcrt")) - } -} - -function New-NetxFernetKey { - # cryptography.fernet.Fernet.generate_key() == urlsafe_b64encode(os.urandom(32)) - $bytes = New-Object byte[] 32 - $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() - try { - $rng.GetBytes($bytes) - } finally { - $rng.Dispose() - } - return [Convert]::ToBase64String($bytes).Replace('+', '-').Replace('/', '_') -} - -function Ensure-NetxDataAcl { - param( - [Parameter(Mandatory = $true)][string]$DataRoot, - [switch]$Quiet = $false - ) - # Installer creates %ProgramData%\NetX as Administrators. Tray / normal users must - # be able to update .env and runtime files when reconfiguring DB. - if (-not (Test-Path -LiteralPath $DataRoot)) { return $false } - try { - $acl = Get-Acl -LiteralPath $DataRoot - $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( - "BUILTIN\Users", - "Modify", - "ContainerInherit,ObjectInherit", - "None", - "Allow" - ) - $acl.SetAccessRule($rule) - Set-Acl -LiteralPath $DataRoot -AclObject $acl - - # Also fix already-created files that only inherited RX for Users. - $icacls = Join-Path $env:SystemRoot "System32\icacls.exe" - if (Test-Path -LiteralPath $icacls) { - & $icacls $DataRoot /grant "*S-1-5-32-545:(OI)(CI)M" /T /C /Q 2>$null | Out-Null - } - if (-not $Quiet) { - Write-Host "==> Data ACL: BUILTIN\Users Modify on $DataRoot" -ForegroundColor DarkGray - } - return $true - } catch { - if (-not $Quiet) { - Write-Host "[WARN] Ensure-NetxDataAcl: $($_.Exception.Message)" -ForegroundColor Yellow - } - return $false - } -} - -function Ensure-NetxDataDirectories { - param([string]$DataRoot) - if (-not (Test-Path -LiteralPath $DataRoot)) { - New-Item -ItemType Directory -Path $DataRoot -Force | Out-Null - } - $subs = @( - "data", "data\auth", "data\runtime", "data\biz_state_spool", - "data\ne_collections", "data\ne_exec_jobs", "data\webcrt", - "backups", "pgdata" - ) - foreach ($sub in $subs) { - $p = Join-Path $DataRoot $sub - if (-not (Test-Path $p)) { - New-Item -ItemType Directory -Path $p -Force | Out-Null - } - } - # Best-effort; succeeds when running elevated (installer / first-run as admin). - $null = Ensure-NetxDataAcl -DataRoot $DataRoot -Quiet -} - -function Test-NetxTcpPortFree { - param([string]$HostName = "127.0.0.1", [int]$Port) - try { - $client = New-Object System.Net.Sockets.TcpClient - $iar = $client.BeginConnect($HostName, $Port, $null, $null) - $ok = $iar.AsyncWaitHandle.WaitOne(400) - if ($ok -and $client.Connected) { - $client.Close() - return $false - } - $client.Close() - return $true - } catch { - return $true - } -} - -function Test-NetxApiHealthy { - param([string]$HostName = "127.0.0.1", [int]$Port = 8890, [int]$TimeoutSec = 2) - try { - $url = "http://${HostName}:${Port}/health" - $r = Invoke-WebRequest -Uri $url -UseBasicParsing -TimeoutSec $TimeoutSec -MaximumRedirection 0 - if ($r.StatusCode -ne 200) { return $false } - $body = $r.Content | ConvertFrom-Json -ErrorAction Stop - return ($body.status -eq "ok") - } catch { - return $false - } -} - -function Wait-NetxApiHealthy { - param( - [string]$HostName = "127.0.0.1", - [int]$Port = 8890, - [int]$TimeoutSec = 180, - [int]$PollMs = 500 - ) - $deadline = (Get-Date).AddSeconds($TimeoutSec) - while ((Get-Date) -lt $deadline) { - if (Test-NetxApiHealthy -HostName $HostName -Port $Port -TimeoutSec 2) { - return $true - } - Start-Sleep -Milliseconds $PollMs - } - return $false -} - -function ConvertTo-NetxProcessArgument { - param([Parameter(Mandatory = $true)][AllowEmptyString()][string]$Value) - # Start-Process joins string[] args with spaces and does NOT quote values that - # contain spaces (e.g. C:\Program Files\NetX). Always emit one argv token. - if ($Value -match '[\s"]') { - return '"' + ($Value -replace '"', '\"') + '"' - } - return $Value -} - -function ConvertTo-NetxProcessArgumentString { - param([Parameter(Mandatory = $true)][string[]]$Arguments) - return (($Arguments | ForEach-Object { ConvertTo-NetxProcessArgument -Value "$_" }) -join " ") -} - -function Start-NetxPowerShell { - param( - [Parameter(Mandatory = $true)][string]$File, - [string[]]$Arguments = @(), - [string]$WorkingDirectory = "", - [ValidateSet("Hidden", "Normal", "Minimized")] - [string]$WindowStyle = "Hidden", - [switch]$Wait = $false, - [switch]$PassThru = $false, - [switch]$RunAs = $false - ) - $parts = @( - "-NoProfile", - "-WindowStyle", $WindowStyle, - "-ExecutionPolicy", "Bypass", - "-File", $File - ) + $Arguments - $sp = @{ - FilePath = "powershell.exe" - ArgumentList = (ConvertTo-NetxProcessArgumentString -Arguments $parts) - } - # -Verb RunAs implies UseShellExecute; cannot combine with -WindowStyle. - if (-not $RunAs) { - $sp.WindowStyle = $WindowStyle - } - if ($WorkingDirectory) { $sp.WorkingDirectory = $WorkingDirectory } - if ($Wait) { $sp.Wait = $true } - if ($PassThru -or $Wait) { $sp.PassThru = $true } - if ($RunAs) { $sp.Verb = "RunAs" } - return Start-Process @sp -} - -function Test-NetxIsAdmin { - try { - $id = [Security.Principal.WindowsIdentity]::GetCurrent() - $p = New-Object Security.Principal.WindowsPrincipal($id) - return $p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) - } catch { - return $false - } -} - -function ConvertTo-NetxRelaunchArgumentList { - param([System.Collections.IDictionary]$BoundParameters) - $parts = [System.Collections.Generic.List[string]]::new() - if (-not $BoundParameters) { return @() } - foreach ($key in $BoundParameters.Keys) { - $val = $BoundParameters[$key] - if ($val -is [System.Management.Automation.SwitchParameter]) { - if ($val.IsPresent) { [void]$parts.Add("-$key") } - continue - } - if ($val -is [bool]) { - if ($val) { [void]$parts.Add("-$key") } - continue - } - if ($null -eq $val) { continue } - $s = [string]$val - if ($s -eq "") { continue } - [void]$parts.Add("-$key") - [void]$parts.Add($s) - } - return @($parts) -} - -function Assert-NetxAdminOrRelaunch { - <# - .SYNOPSIS - End-user packaged scripts must run elevated (Program Files + ProgramData). - If not admin, re-launch the same script via UAC and exit with child exit code. - #> - param( - [Parameter(Mandatory = $true)][string]$ScriptPath, - [System.Collections.IDictionary]$BoundParameters = @{}, - [string]$WorkingDirectory = "", - [ValidateSet("Hidden", "Normal", "Minimized")] - [string]$WindowStyle = "Normal" - ) - if (Test-NetxIsAdmin) { return } - - if ($env:NETX_ELEVATION_RELAUNCH -eq "1") { - throw "admin_required: still not elevated after UAC. Run NetX-Start / NetX-Tray as Administrator." - } - - if (-not (Test-Path -LiteralPath $ScriptPath)) { - throw "elevation_script_missing: $ScriptPath" - } - $wd = $WorkingDirectory - if (-not $wd) { $wd = Split-Path -Parent $ScriptPath } - - # Window style belongs on powershell.exe argv, not Start-Process (incompatible with -Verb RunAs). - $parts = @( - "-NoProfile", - "-WindowStyle", $WindowStyle, - "-ExecutionPolicy", "Bypass", - "-File", $ScriptPath - ) + (ConvertTo-NetxRelaunchArgumentList -BoundParameters $BoundParameters) - - $prevRel = $env:NETX_ELEVATION_RELAUNCH - $env:NETX_ELEVATION_RELAUNCH = "1" - try { - $p = Start-Process -FilePath "powershell.exe" ` - -ArgumentList (ConvertTo-NetxProcessArgumentString -Arguments $parts) ` - -WorkingDirectory $wd ` - -Verb RunAs -Wait -PassThru - } catch { - if ($null -ne $prevRel) { $env:NETX_ELEVATION_RELAUNCH = $prevRel } else { Remove-Item Env:NETX_ELEVATION_RELAUNCH -ErrorAction SilentlyContinue } - throw ("admin_required: UAC cancelled or elevation failed. Run NetX as Administrator. {0}" -f $_.Exception.Message) - } - if ($null -ne $prevRel) { $env:NETX_ELEVATION_RELAUNCH = $prevRel } else { Remove-Item Env:NETX_ELEVATION_RELAUNCH -ErrorAction SilentlyContinue } - if ($null -eq $p) { - throw "admin_required: elevation failed (no process)" - } - $code = 0 - if ($null -ne $p.ExitCode) { $code = [int]$p.ExitCode } - exit $code -} - -function Get-NetxSystemPython { - # Prefer a real interpreter; skip the WindowsApps Store stub. - $candidates = @() - foreach ($cmd in @("python", "python3")) { - $c = Get-Command $cmd -ErrorAction SilentlyContinue - if ($c -and $c.Source -and ($c.Source -notmatch '\\WindowsApps\\')) { - $candidates += $c.Source - } - } - $pyLauncher = Get-Command "py" -ErrorAction SilentlyContinue - if ($pyLauncher -and $pyLauncher.Source -and ($pyLauncher.Source -notmatch '\\WindowsApps\\')) { - try { - $resolved = (& $pyLauncher.Source -3 -c "import sys; print(sys.executable)" 2>$null | Out-String).Trim() - if ($resolved -and (Test-Path -LiteralPath $resolved)) { - $candidates += $resolved - } - } catch {} - } - foreach ($p in @( - "$env:LOCALAPPDATA\Python\pythoncore-3.14-64\python.exe", - "$env:LOCALAPPDATA\Programs\Python\Python312\python.exe", - "$env:LOCALAPPDATA\Programs\Python\Python311\python.exe", - "$env:ProgramFiles\Python312\python.exe", - "$env:ProgramFiles\Python311\python.exe" - )) { - if ($p -and (Test-Path $p)) { $candidates += $p } - } - foreach ($p in ($candidates | Select-Object -Unique)) { - try { - $v = & $p -c "import sys; print('%d.%d'%sys.version_info[:2])" 2>$null - if ($v -match '^(3\.(1[1-9]|[2-9]\d))$') { return $p } - } catch {} - } - return $null -} - -function Get-NetxBundledPythonRoot { - param([Parameter(Mandatory = $true)][string]$ProgramRoot) - return Join-Path $ProgramRoot "python\runtime" -} - -function Repair-NetxShippedVenv { - param([Parameter(Mandatory = $true)][string]$ProgramRoot) - $cfgPath = Join-Path $ProgramRoot ".venv\pyvenv.cfg" - $rtRoot = Get-NetxBundledPythonRoot -ProgramRoot $ProgramRoot - $rtPy = Join-Path $rtRoot "python.exe" - if (-not (Test-Path -LiteralPath $rtPy)) { return $false } - if (-not (Test-Path -LiteralPath $cfgPath)) { return $false } - - $rtRootAbs = (Resolve-Path -LiteralPath $rtRoot).Path - $rtPyAbs = (Resolve-Path -LiteralPath $rtPy).Path - $ver = "3.14.0" - try { - $verOut = & $rtPyAbs -c "import sys; print('%d.%d.%d' % sys.version_info[:3])" 2>$null - if ($verOut) { $ver = ($verOut | Out-String).Trim() } - } catch {} - - $lines = @( - "home = $rtRootAbs", - "include-system-site-packages = false", - "version = $ver", - "executable = $rtPyAbs" - ) - $desired = ($lines -join "`r`n") - # Skip write when already correct — Users cannot modify Program Files after Setup. - try { - $cur = ([IO.File]::ReadAllText($cfgPath) -replace "`r`n", "`n" -replace "`r", "`n").Trim() - $want = ($desired -replace "`r`n", "`n" -replace "`r", "`n").Trim() - if ($cur -eq $want) { - return $true - } - } catch {} - - try { - Set-Content -LiteralPath $cfgPath -Value $desired -Encoding ascii -ErrorAction Stop - return $true - } catch { - # Non-elevated tray/start: leave cfg as-is; caller may still run if paths happen to work. - return $false - } -} - -function Test-NetxVenvRunnable { - param([Parameter(Mandatory = $true)][string]$VenvPython) - if (-not (Test-Path -LiteralPath $VenvPython)) { return $false } - - $outFile = Join-Path $env:TEMP ("netx-venv-out-" + [Guid]::NewGuid().ToString("n") + ".txt") - $errFile = Join-Path $env:TEMP ("netx-venv-err-" + [Guid]::NewGuid().ToString("n") + ".txt") - try { - $psi = New-Object System.Diagnostics.ProcessStartInfo - $psi.FileName = $VenvPython - $psi.Arguments = '-c "import encodings, sys; sys.exit(0 if sys.prefix else 1)"' - $psi.UseShellExecute = $false - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - $psi.CreateNoWindow = $true - $p = [Diagnostics.Process]::Start($psi) - $stderr = $p.StandardError.ReadToEnd() - $null = $p.StandardOutput.ReadToEnd() - $p.WaitForExit() - if ($stderr -match 'did not find executable|No Python at|Fatal Python error') { - return $false - } - if ($p.ExitCode -ne 0) { return $false } - return $true - } finally { - Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue - } -} - -function Ensure-NetxVenv { - param( - [Parameter(Mandatory = $true)][string]$ProgramRoot, - [switch]$ForcePip = $false - ) - $venvPy = Join-Path $ProgramRoot ".venv\Scripts\python.exe" - $req = Join-Path $ProgramRoot "requirements.txt" - $rtPy = Join-Path (Get-NetxBundledPythonRoot -ProgramRoot $ProgramRoot) "python.exe" - - if (Test-Path -LiteralPath $rtPy) { - try { - $null = Repair-NetxShippedVenv -ProgramRoot $ProgramRoot - } catch { - # Access denied under Program Files when not elevated — ignore if venv already runs. - } - } - - if ((Test-Path -LiteralPath $venvPy) -and -not $ForcePip) { - if (Test-NetxVenvRunnable -VenvPython $venvPy) { - return $venvPy - } - Write-Host "[WARN] Shipped .venv exists but Python cannot start (broken pyvenv.cfg or missing runtime)." -ForegroundColor Yellow - if (Test-Path -LiteralPath $rtPy) { - $repaired = $false - try { - $repaired = [bool](Repair-NetxShippedVenv -ProgramRoot $ProgramRoot) - } catch { - $repaired = $false - } - if ($repaired -and (Test-NetxVenvRunnable -VenvPython $venvPy)) { - Write-Host "==> Repaired .venv to use bundled python/runtime" -ForegroundColor Green - return $venvPy - } - if (-not $repaired) { - throw "venv_needs_admin_repair: pyvenv.cfg still points at the build PC. Re-run Setup (Update) as Administrator, or Start Menu → Reconfigure database once elevated." - } - } - if (-not $ForcePip) { - throw "venv_not_runnable: reinstall NetX Setup (ships python/runtime + .venv) or run repair as admin" - } - } - if (-not (Test-Path $venvPy)) { - $py = Get-NetxSystemPython - if (-not $py) { - throw "python_not_found: install Python 3.11+ (or ship Setup built with -CreateVenv)" - } - Write-Host "==> Creating .venv with $py" - $venvDir = Join-Path $ProgramRoot ".venv" - try { - & $py -m venv $venvDir - } catch { - throw "venv_create_failed: cannot write $venvDir (need admin / ship -CreateVenv). $($_.Exception.Message)" - } - if (-not (Test-Path $venvPy)) { - throw "venv_create_failed: missing $venvPy (Program Files may be read-only without elevation)" - } - $ForcePip = $true - } - if ($ForcePip) { - if (-not (Test-Path $req)) { throw "requirements_missing: $req" } - Write-Host "==> pip install -r requirements.txt" - & $venvPy -m pip install --upgrade pip - & $venvPy -m pip install -r $req - if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } - } - return $venvPy -} - -function Stop-NetxTrayProcesses { - param([string]$ProgramRoot = "") - $hits = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { - $_.CommandLine -and $_.CommandLine -match 'netx_tray\.ps1' - }) - if ($ProgramRoot) { - $esc = [regex]::Escape($ProgramRoot) - $hits = @($hits | Where-Object { $_.CommandLine -match $esc -or $_.CommandLine -match 'netx_tray\.ps1' }) - } - foreach ($p in $hits) { - try { - Stop-Process -Id $p.ProcessId -Force -ErrorAction Stop - Write-Host "Stopped NetX tray PID=$($p.ProcessId)" - } catch {} - } -} - -function ConvertTo-NetxSqlLiteral([string]$Value) { - # Single-quote escaping for PostgreSQL string literals. - return ($Value -replace "'", "''") -} diff --git a/packaging/assets/README.md b/packaging/assets/README.md deleted file mode 100644 index 2310893..0000000 --- a/packaging/assets/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# NetX packaging assets - -- `netx.ico` — installer / shortcuts / tray (regenerate with `python gen_icon.py`) -- `netx-256.png` / `netx-64.png` — previews - -Mark matches `web/public/favicon.svg` (network “N” nodes on navy). diff --git a/packaging/assets/gen_icon.py b/packaging/assets/gen_icon.py deleted file mode 100644 index f6e556c..0000000 --- a/packaging/assets/gen_icon.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Generate NetX Windows .ico from brand mark (matches web/public/favicon.svg).""" -from __future__ import annotations - -import os -from PIL import Image, ImageDraw - -OUT_DIR = os.path.dirname(os.path.abspath(__file__)) -BG = (15, 39, 68, 255) -LINE = (126, 182, 232, 255) -NODE = (232, 242, 252, 255) -ACCENT = (61, 130, 247, 255) - - -def make(size: int) -> Image.Image: - img = Image.new("RGBA", (size, size), (0, 0, 0, 0)) - d = ImageDraw.Draw(img) - radius = max(2, int(size * 0.22)) - d.rounded_rectangle([0, 0, size - 1, size - 1], radius=radius, fill=BG) - s = size / 32.0 - - def xy(x: float, y: float) -> tuple[float, float]: - return (x * s, y * s) - - stroke = max(2, int(round(2.2 * s))) - for a, b in ( - (xy(9.5, 8.5), xy(9.5, 23.5)), - (xy(9.5, 8.5), xy(22.5, 23.5)), - (xy(22.5, 8.5), xy(22.5, 23.5)), - ): - d.line([a, b], fill=LINE, width=stroke) - - def circle(cx: float, cy: float, rad: float, fill: tuple[int, int, int, int]) -> None: - x, y = xy(cx, cy) - rr = rad * s - d.ellipse([x - rr, y - rr, x + rr, y + rr], fill=fill) - - circle(9.5, 8.5, 2.35, NODE) - circle(9.5, 23.5, 2.35, NODE) - circle(22.5, 8.5, 2.35, NODE) - circle(22.5, 23.5, 2.35, NODE) - circle(16, 16, 1.7, ACCENT) - return img - - -def main() -> None: - sizes = [16, 24, 32, 48, 64, 128, 256] - images = [make(s) for s in sizes] - ico_path = os.path.join(OUT_DIR, "netx.ico") - images[-1].save(ico_path, format="ICO", sizes=[(s, s) for s in sizes]) - images[-1].save(os.path.join(OUT_DIR, "netx-256.png")) - make(64).save(os.path.join(OUT_DIR, "netx-64.png")) - print(f"wrote {ico_path} ({os.path.getsize(ico_path)} bytes)") - - -if __name__ == "__main__": - main() diff --git a/packaging/assets/netx-256.png b/packaging/assets/netx-256.png deleted file mode 100644 index 167327b..0000000 Binary files a/packaging/assets/netx-256.png and /dev/null differ diff --git a/packaging/assets/netx-64.png b/packaging/assets/netx-64.png deleted file mode 100644 index afeb76f..0000000 Binary files a/packaging/assets/netx-64.png and /dev/null differ diff --git a/packaging/assets/netx.ico b/packaging/assets/netx.ico deleted file mode 100644 index 179e51b..0000000 Binary files a/packaging/assets/netx.ico and /dev/null differ diff --git a/packaging/build_release.ps1 b/packaging/build_release.ps1 index 3f35475..a347f03 100644 --- a/packaging/build_release.ps1 +++ b/packaging/build_release.ps1 @@ -1,14 +1,10 @@ -param( +param( [string]$Version = "", [string]$OutDir = "", [switch]$SkipWebBuild = $false, [switch]$SkipPostgresDownload = $false, - # Zip is no longer published; stage + Setup.exe only. Opt-in for local/dev testing. - [switch]$CreateZip = $false, - # Deprecated no-op (zip is off by default). Kept so older scripts that pass -SkipZip still run. [switch]$SkipZip = $false, - # Default on: offline Setup must ship .venv so target PCs never pip-install. - [switch]$CreateVenv = $true + [switch]$CreateVenv = $false ) $ErrorActionPreference = "Stop" @@ -55,20 +51,6 @@ if (-not $SkipPostgresDownload) { } } -# Bundle WinSW so offline installs can register a Windows Service without GitHub. -$winswCache = Join-Path $PSScriptRoot "cache\WinSW-x64.exe" -$winswShip = Join-Path $PSScriptRoot "winsw\WinSW-x64.exe" -if (-not (Test-Path $winswShip)) { - if (-not (Test-Path $winswCache)) { - $winswUrl = "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW-x64.exe" - Write-Host "==> Downloading WinSW for offline bundle: $winswUrl" - New-Item -ItemType Directory -Path (Split-Path $winswCache -Parent) -Force | Out-Null - Invoke-WebRequest -Uri $winswUrl -OutFile $winswCache -UseBasicParsing - } - New-Item -ItemType Directory -Path (Split-Path $winswShip -Parent) -Force | Out-Null - Copy-Item -Path $winswCache -Destination $winswShip -Force -} - # Flat layout = same as repo so scripts\start_netx.ps1 works unchanged. foreach ($d in @("netx_api", "alembic", "scripts")) { Copy-Item -Path (Join-Path $repo $d) -Destination (Join-Path $stage $d) -Recurse -Force @@ -84,70 +66,26 @@ $webOut = Join-Path $stage "web\dist" New-Item -ItemType Directory -Path (Split-Path $webOut -Parent) -Force | Out-Null Copy-Item -Path $dist -Destination $webOut -Recurse -Force -function Write-Utf8BomFile { - param([string]$Path) - # Windows PowerShell 4/5 (Server 2012+) mis-parses UTF-8 without BOM when scripts contain CJK. - $utf8Bom = New-Object System.Text.UTF8Encoding $true - $bytes = [IO.File]::ReadAllBytes($Path) - $hasBom = ($bytes.Length -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) - $text = if ($hasBom) { - [Text.Encoding]::UTF8.GetString($bytes, 3, $bytes.Length - 3) - } else { - [Text.Encoding]::UTF8.GetString($bytes) - } - $text = $text -replace "`r`n", "`n" -replace "`n", "`r`n" - if (-not $text.EndsWith("`r`n")) { $text += "`r`n" } - [IO.File]::WriteAllText($Path, $text, $utf8Bom) -} - $packOut = Join-Path $stage "packaging" New-Item -ItemType Directory -Path $packOut -Force | Out-Null Copy-Item -Path (Join-Path $PSScriptRoot "_common.ps1") -Destination $packOut -Force foreach ($name in @( "download_postgres.ps1", "setup_first_run.ps1", "start_netx_app.ps1", - "stop_netx_app.ps1", "update_netx.ps1", "check_update.ps1", - "launch_shortcut.ps1", "netx_tray.ps1", "install_autostart.ps1", "install_service.ps1", - "service_run.ps1", "install_update_task.ps1", "uninstall_prepare.ps1", "uninstall_delete_data.ps1", "sign_release.ps1", - "build_release.ps1", "publish_release.ps1", "publish_forgejo_release.ps1", "README.md", "manifest.example.json" + "stop_netx_app.ps1", "update_netx.ps1", "build_release.ps1", + "README.md", "manifest.example.json" )) { $src = Join-Path $PSScriptRoot $name if (Test-Path $src) { Copy-Item $src (Join-Path $packOut $name) -Force } } -Get-ChildItem -Path $packOut -Filter *.ps1 -File | ForEach-Object { Write-Utf8BomFile -Path $_.FullName } -Get-ChildItem -Path (Join-Path $stage "scripts") -Filter *.ps1 -File -ErrorAction SilentlyContinue | - ForEach-Object { Write-Utf8BomFile -Path $_.FullName } Copy-Item -Path (Join-Path $PSScriptRoot "config") -Destination (Join-Path $packOut "config") -Recurse -Force Copy-Item -Path (Join-Path $PSScriptRoot "installer") -Destination (Join-Path $packOut "installer") -Recurse -Force -$assetsSrc = Join-Path $PSScriptRoot "assets" -if (Test-Path $assetsSrc) { - Copy-Item -Path $assetsSrc -Destination (Join-Path $packOut "assets") -Recurse -Force -} -$winswOut = Join-Path $packOut "winsw" -New-Item -ItemType Directory -Path $winswOut -Force | Out-Null -if (Test-Path $winswShip) { - Copy-Item -Path $winswShip -Destination (Join-Path $winswOut "WinSW-x64.exe") -Force - Write-Host "==> Bundled WinSW for offline service install" -} else { - Write-Host "[WARN] WinSW missing - offline service install will fail" -ForegroundColor Yellow -} New-Item -ItemType Directory -Path (Join-Path $packOut "postgres") -Force | Out-Null Copy-Item -Path (Join-Path $PSScriptRoot "postgres\README.md") -Destination (Join-Path $packOut "postgres\README.md") -Force if (Test-Path (Join-Path $pgsql "bin\pg_ctl.exe")) { - Write-Host '==> Copying bundled PostgreSQL (bin/lib/share; skip doc/)' - $pgStage = Join-Path $stage "postgres\pgsql" - New-Item -ItemType Directory -Path $pgStage -Force | Out-Null - foreach ($sub in @("bin", "lib", "share")) { - $srcSub = Join-Path $pgsql $sub - if (Test-Path $srcSub) { - Copy-Item -Path $srcSub -Destination (Join-Path $pgStage $sub) -Recurse -Force - } - } - Get-ChildItem -Path $pgsql -File -ErrorAction SilentlyContinue | ForEach-Object { - Copy-Item -Path $_.FullName -Destination (Join-Path $pgStage $_.Name) -Force - } -} else { - throw "bundled_postgres_missing: Setup.exe must ship postgres for offline install. Run download_postgres.ps1 (build machine only)." + Write-Host "==> Copying bundled PostgreSQL" + New-Item -ItemType Directory -Path (Join-Path $stage "postgres") -Force | Out-Null + Copy-Item -Path $pgsql -Destination (Join-Path $stage "postgres\pgsql") -Recurse -Force } $builtAt = (Get-Date).ToUniversalTime().ToString("o") @@ -161,93 +99,26 @@ $builtAt = (Get-Date).ToUniversalTime().ToString("o") Set-Content -Path (Join-Path $stage ".portable") -Value "1" -Encoding ascii -# Root .cmd launchers (Explorer / Start Menu friendly; ASCII-only). -$cmdSrc = Join-Path $PSScriptRoot "cmd" -foreach ($cmdName in @("NetX-FirstRun.cmd", "NetX-Start.cmd", "NetX-Tray.cmd", "NetX-Stop.cmd")) { - $c = Join-Path $cmdSrc $cmdName - if (Test-Path $c) { - Copy-Item -Path $c -Destination (Join-Path $stage $cmdName) -Force - } -} - if ($CreateVenv) { - Write-Host "==> Creating portable python/runtime + .venv (must not reference build-machine paths)" - $pyPath = Get-NetxSystemPython - if (-not $pyPath) { throw "python_not_found_for_venv: need Python 3.11+ (not WindowsApps stub)" } - Write-Host " Build Python: $pyPath" - - $prefix = (& $pyPath -c "import sys; print(sys.base_prefix)" 2>$null | Out-String).Trim() - if (-not $prefix -or -not (Test-Path -LiteralPath $prefix)) { - throw "python_base_prefix_not_found: $prefix" - } - $rtDir = Join-Path $stage "python\runtime" - if (Test-Path $rtDir) { Remove-Item -Recurse -Force $rtDir } - New-Item -ItemType Directory -Path $rtDir -Force | Out-Null - - Write-Host "==> Copying Python stdlib/runtime to $rtDir (exclude base site-packages)" - $spEx = Join-Path $prefix "Lib\site-packages" - $robocopy = Join-Path $env:SystemRoot "System32\robocopy.exe" - if (-not (Test-Path -LiteralPath $robocopy)) { throw "robocopy_not_found" } - & $robocopy $prefix $rtDir /E /XD $spEx __pycache__ /XF *.pyc /NFL /NDL /NJH /NJS /nc /ns /np | Out-Null - if ($LASTEXITCODE -ge 8) { throw "robocopy_python_runtime_failed: exit $LASTEXITCODE" } - - $rtPy = Join-Path $rtDir "python.exe" - if (-not (Test-Path -LiteralPath $rtPy)) { throw "python_runtime_missing: $rtPy" } - - $venvDir = Join-Path $stage ".venv" - if (Test-Path $venvDir) { Remove-Item -Recurse -Force $venvDir } - Write-Host '==> Creating .venv from shipped runtime (--copies)' - & $rtPy -m venv $venvDir --copies - if ($LASTEXITCODE -ne 0) { throw "venv_create_failed" } - - $venvPy = Join-Path $venvDir "Scripts\python.exe" - $null = Repair-NetxShippedVenv -ProgramRoot $stage - if (-not (Test-NetxVenvRunnable -VenvPython $venvPy)) { - throw "venv_not_runnable_after_build: check python/runtime copy" - } - - & $venvPy -m pip install --upgrade pip - & $venvPy -m pip install -r (Join-Path $stage "requirements.txt") - if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } - - $null = Repair-NetxShippedVenv -ProgramRoot $stage - if (-not (Test-NetxVenvRunnable -VenvPython $venvPy)) { - throw "venv_not_runnable_after_pip" - } - - Write-Host "==> Slimming .venv (drop tests / __pycache__ / *.pyc)" - $site = Join-Path $stage ".venv\Lib\site-packages" - if (Test-Path $site) { - Get-ChildItem -Path $site -Recurse -Directory -Force -ErrorAction SilentlyContinue | - Where-Object { - $_.Name -in @('__pycache__', 'tests', 'test', 'testing', 'Tests') -or - $_.FullName -match '\\pandas\\tests(\\|$)' -or - $_.FullName -match '\\numpy\\(_*tests|tests)(\\|$)' -or - $_.FullName -match '\\scipy\\(_*tests|tests)(\\|$)' - } | - Sort-Object { $_.FullName.Length } -Descending | - ForEach-Object { - Remove-Item -LiteralPath $_.FullName -Recurse -Force -ErrorAction SilentlyContinue - } - Get-ChildItem -Path $site -Recurse -Include *.pyc,*.pyo -Force -ErrorAction SilentlyContinue | - Remove-Item -Force -ErrorAction SilentlyContinue - } + Write-Host "==> Creating .venv in stage (requires Python 3.11+ on PATH)" + $py = (Get-Command python -ErrorAction SilentlyContinue) + if (-not $py) { throw "python_not_found_for_venv" } + & $py.Source -m venv (Join-Path $stage ".venv") + & (Join-Path $stage ".venv\Scripts\python.exe") -m pip install --upgrade pip + & (Join-Path $stage ".venv\Scripts\python.exe") -m pip install -r (Join-Path $stage "requirements.txt") } Write-Host "==> Stage ready: $stage" -ForegroundColor Green -if ($SkipZip -and $CreateZip) { - Write-Host "[WARN] -SkipZip ignored because -CreateZip was set" -ForegroundColor Yellow -} -if ($CreateZip) { +if (-not $SkipZip) { $zip = Join-Path (Split-Path -Parent $stage) "NetX-$Version-win64.zip" if (Test-Path $zip) { Remove-Item -Force $zip } Compress-Archive -Path $stage -DestinationPath $zip -Force - Write-Host "==> Zip (optional/dev): $zip" -ForegroundColor Yellow + Write-Host "==> Zip: $zip" -ForegroundColor Green } Write-Host "" -Write-Host "Ship:" +Write-Host "Ship options:" +Write-Host " Zip: user unpacks, runs packaging\setup_first_run.ps1 then start_netx_app.ps1" Write-Host " Exe: compile packaging\installer\netx.iss with Inno Setup (needs stage above)" -Write-Host " (Zip packages are not published; use -CreateZip only for local testing.)" Write-Host "Python: install 3.11+ on target, or rebuild with -CreateVenv and ship .venv" diff --git a/packaging/check_update.ps1 b/packaging/check_update.ps1 deleted file mode 100644 index 7c4f39f..0000000 --- a/packaging/check_update.ps1 +++ /dev/null @@ -1,438 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [string]$UpdateUrl = "", - [string]$FallbackUrl = "", - [string]$ForgejoUrl = "", - [string]$GithubRepo = "", - [string]$Channel = "", - [string]$Sources = "", - [switch]$Apply = $false, - [switch]$Quiet = $false, - # Only apply when NETX_UPDATE_AUTO=true (scheduled silent updates). - [switch]$AutoOnly = $false -) - -# Check for a newer NetX Windows package. -# -# Default (no config needed): -# GitHub primary + Forgejo mirror fallback (git.avelo.top). -# Probe every reachable source and pick the highest version; -# on equal versions prefer GitHub (listed first). -# -# Optional overrides: -# NETX_UPDATE_SOURCES=github,forgejo -# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest -# NETX_UPDATE_GITHUB_REPO=hansjone/netx -# NETX_UPDATE_URL=... manifest JSON -# NETX_UPDATE_TOKEN=... private API token - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters ` - -WindowStyle $(if ($Quiet) { "Hidden" } else { "Normal" }) - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$envPath = Join-Path $data ".env" -$map = @{} -if (Test-Path $envPath) { $map = Read-DotEnv -Path $envPath } - -function Get-Cfg([string]$Key, [string]$ParamVal = "") { - if ($ParamVal) { return $ParamVal } - if ($map.ContainsKey($Key) -and $map[$Key]) { return [string]$map[$Key] } - $envVal = [Environment]::GetEnvironmentVariable($Key) - if ($envVal) { return $envVal } - return "" -} - -$UpdateUrl = Get-Cfg "NETX_UPDATE_URL" $UpdateUrl -$FallbackUrl = Get-Cfg "NETX_UPDATE_FALLBACK_URL" $FallbackUrl -$ForgejoUrl = Get-Cfg "NETX_UPDATE_FORGEJO_URL" $ForgejoUrl -if (-not $ForgejoUrl) { - $ForgejoUrl = "https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest" -} -$GithubRepo = Get-Cfg "NETX_UPDATE_GITHUB_REPO" $GithubRepo -if (-not $GithubRepo) { $GithubRepo = "hansjone/netx" } -$Channel = Get-Cfg "NETX_UPDATE_CHANNEL" $Channel -if (-not $Channel) { $Channel = "stable" } -$Sources = Get-Cfg "NETX_UPDATE_SOURCES" $Sources -$UpdateToken = Get-Cfg "NETX_UPDATE_TOKEN" "" - -$current = Get-NetxVersion -ProgramRoot $prog - -function Compare-SemVer { - param([string]$A, [string]$B) - $pa = @($A.TrimStart('v', 'V').Split('.') | ForEach-Object { try { [int]$_ } catch { 0 } }) - $pb = @($B.TrimStart('v', 'V').Split('.') | ForEach-Object { try { [int]$_ } catch { 0 } }) - while ($pa.Count -lt 3) { $pa += 0 } - while ($pb.Count -lt 3) { $pb += 0 } - for ($i = 0; $i -lt 3; $i++) { - if ($pa[$i] -lt $pb[$i]) { return -1 } - if ($pa[$i] -gt $pb[$i]) { return 1 } - } - return 0 -} - -function Get-AuthHeaders { - param([string]$Style = "bearer") - $h = @{ "User-Agent" = "NetX-UpdateCheck" } - if (-not $UpdateToken) { return $h } - if ($Style -eq "token") { - $h["Authorization"] = "token $UpdateToken" - } else { - $h["Authorization"] = "Bearer $UpdateToken" - } - return $h -} - -function Resolve-AssetUrl { - param($Asset, $Rel, [string]$SourceName, [string]$ApiLatestUrl = "") - $url = [string]$Asset.browser_download_url - if ($url) { return $url } - # Forgejo/Gitea sometimes omit browser_download_url; synthesize release download URL. - if ($SourceName -eq "forgejo" -and $ApiLatestUrl -and $Rel.tag_name -and $Asset.name) { - if ($ApiLatestUrl -match '^(https?://[^/]+)/api/v1/repos/([^/]+)/([^/]+)/releases/latest') { - $base = $Matches[1] - $owner = $Matches[2] - $repo = $Matches[3] - $tag = [string]$Rel.tag_name - $name = [uri]::EscapeDataString([string]$Asset.name).Replace('%2F', '/') - # EscapeDataString encodes too much; use raw name (assets shouldn't need query encoding). - $name = [string]$Asset.name - return "$base/$owner/$repo/releases/download/$tag/$name" - } - } - return "" -} - -function Convert-ReleaseToManifest { - param($Rel, [string]$SourceName, [string]$ApiLatestUrl = "") - $tag = [string]$Rel.tag_name - $ver = $tag.TrimStart('v', 'V') - $assets = @($Rel.assets) - # Prefer Setup.exe (current ship format). Legacy win64.zip still accepted. - $setupAsset = $assets | Where-Object { $_.name -match 'Setup-.*\.exe$' } | Select-Object -First 1 - $zipAsset = $assets | Where-Object { $_.name -match 'win64\.zip$' } | Select-Object -First 1 - if (-not $setupAsset -and -not $zipAsset) { - throw "${SourceName}_release_missing_setup_or_zip" - } - - $setupUrl = "" - if ($setupAsset) { - $setupUrl = Resolve-AssetUrl -Asset $setupAsset -Rel $Rel -SourceName $SourceName -ApiLatestUrl $ApiLatestUrl - if (-not $setupUrl) { throw "${SourceName}_setup_url_missing" } - } - $zipUrl = "" - if ($zipAsset) { - $zipUrl = Resolve-AssetUrl -Asset $zipAsset -Rel $Rel -SourceName $SourceName -ApiLatestUrl $ApiLatestUrl - } - - if ($setupUrl) { - $primaryUrl = $setupUrl - $packageKind = "setup" - $primarySize = [int64]$(if ($setupAsset.size) { $setupAsset.size } else { 0 }) - } else { - if (-not $zipUrl) { throw "${SourceName}_zip_url_missing" } - $primaryUrl = $zipUrl - $packageKind = "zip" - $primarySize = [int64]$(if ($zipAsset.size) { $zipAsset.size } else { 0 }) - } - - return [pscustomobject]@{ - channel = "stable" - latest = $ver - min_compatible = $ver - notes_url = [string]$Rel.html_url - source = $SourceName - windows = [pscustomobject]@{ - url = $primaryUrl - sha256 = "" - size = $primarySize - setup_url = $setupUrl - package = $packageKind - } - } -} - -function Get-ManifestFromUrl { - param([string]$Url) - $headers = Get-AuthHeaders -Style "bearer" - $resp = Invoke-WebRequest -Uri $Url -Headers $headers -UseBasicParsing -TimeoutSec 30 - $m = $resp.Content | ConvertFrom-Json - if (-not $m.source) { - $m | Add-Member -NotePropertyName source -NotePropertyValue "manifest" -Force - } - return $m -} - -function Get-FromGitHubReleases { - $api = "https://api.github.com/repos/$GithubRepo/releases/latest" - $headers = Get-AuthHeaders -Style "bearer" - $headers["Accept"] = "application/vnd.github+json" - $rel = Invoke-RestMethod -Uri $api -Headers $headers -TimeoutSec 30 - return Convert-ReleaseToManifest -Rel $rel -SourceName "github" -} - -function Get-FromForgejoReleases { - param([string]$ApiUrl) - if (-not $ApiUrl) { throw "forgejo_url_empty" } - $headers = Get-AuthHeaders -Style "token" - $headers["Accept"] = "application/json" - $rel = Invoke-RestMethod -Uri $ApiUrl -Headers $headers -TimeoutSec 30 - return Convert-ReleaseToManifest -Rel $rel -SourceName "forgejo" -ApiLatestUrl $ApiUrl -} - -function Get-UpdateSourceList { - if ($Sources) { - return @($Sources.Split(',') | ForEach-Object { $_.Trim().ToLowerInvariant() } | Where-Object { $_ }) - } - # Default: GitHub primary, Forgejo mirror backup. Optional manifests prepended if configured. - $list = [System.Collections.Generic.List[string]]::new() - if ($UpdateUrl) { [void]$list.Add("manifest") } - [void]$list.Add("github") - [void]$list.Add("forgejo") - if ($FallbackUrl) { [void]$list.Add("manifest_fallback") } - return @($list) -} - -function Get-ManifestFromSource([string]$src) { - switch ($src) { - "manifest" { - if (-not $UpdateUrl) { throw "NETX_UPDATE_URL empty" } - Write-Host "==> Probing manifest: $UpdateUrl" - return Get-ManifestFromUrl -Url $UpdateUrl - } - "manifest_fallback" { - if (-not $FallbackUrl) { throw "NETX_UPDATE_FALLBACK_URL empty" } - Write-Host "==> Probing fallback manifest: $FallbackUrl" - $m = Get-ManifestFromUrl -Url $FallbackUrl - if ($m -and -not $m.source) { - $m | Add-Member -NotePropertyName source -NotePropertyValue "manifest_fallback" -Force - } - return $m - } - "forgejo" { - Write-Host "==> Probing Forgejo: $ForgejoUrl" - return Get-FromForgejoReleases -ApiUrl $ForgejoUrl - } - "github" { - Write-Host "==> Probing GitHub: $GithubRepo" - return Get-FromGitHubReleases - } - default { throw "unknown_source: $src" } - } -} - -Write-Host "==> Current version: $current" -$sourceList = Get-UpdateSourceList -Write-Host "==> Probe sources (pick newest reachable; tie → earlier in list): $($sourceList -join ', ')" - -$candidates = @() -$errors = @() -foreach ($src in $sourceList) { - try { - $m = Get-ManifestFromSource -src $src - if ($m.channel -and $Channel -and ($m.channel -ne $Channel)) { - Write-Host "[WARN] $src channel=$($m.channel) requested=$Channel" - } - if (-not $m.windows -or -not $m.windows.url) { - throw "missing_windows_download_url" - } - Write-Host " OK $($m.source) latest=$($m.latest)" -ForegroundColor DarkGreen - $candidates += $m - } catch { - $msg = $_.Exception.Message - $errors += "${src}: $msg" - Write-Host "[WARN] source '$src' unreachable/failed: $msg" -ForegroundColor Yellow - } -} - -if ($candidates.Count -eq 0) { - $joined = ($errors -join "; ") - if ($Quiet) { exit 2 } - throw "update_check_failed: all sources failed ($joined)" -} - -# Prefer highest semver; on tie keep earlier source in $sourceList (GitHub before Forgejo by default). -$manifest = $candidates[0] -foreach ($c in $candidates) { - $cmpCand = Compare-SemVer -A ([string]$manifest.latest) -B ([string]$c.latest) - if ($cmpCand -lt 0) { - $manifest = $c - } -} - -Write-Host "==> Selected source=$($manifest.source) latest=$($manifest.latest) (from $($candidates.Count) reachable)" -ForegroundColor Green - -# Prefer Setup.exe when a manifest still lists a legacy zip as windows.url but also has setup_url. -if ($manifest.windows -and $manifest.windows.setup_url) { - $setupCandidate = [string]$manifest.windows.setup_url - $urlNow = [string]$manifest.windows.url - $pkgNow = if ($manifest.windows.package) { [string]$manifest.windows.package } else { "" } - if ($setupCandidate -and ($pkgNow -eq "zip" -or $urlNow -match '\.zip(\?|$)' -or -not $urlNow)) { - $manifest.windows | Add-Member -NotePropertyName url -NotePropertyValue $setupCandidate -Force - $manifest.windows | Add-Member -NotePropertyName package -NotePropertyValue "setup" -Force - } -} - -$latest = [string]$manifest.latest -$cmp = Compare-SemVer -A $current -B $latest -$packageKind = "setup" -if ($manifest.windows.package) { - $packageKind = [string]$manifest.windows.package -} elseif ([string]$manifest.windows.url -match '\.zip(\?|$)') { - $packageKind = "zip" -} - -$result = [pscustomobject]@{ - current = $current - latest = $latest - update_available = ($cmp -lt 0) - source = $(if ($manifest.source) { [string]$manifest.source } else { "unknown" }) - download_url = [string]$manifest.windows.url - setup_url = $(if ($manifest.windows.setup_url) { [string]$manifest.windows.setup_url } else { "" }) - package = $packageKind - notes_url = $(if ($manifest.notes_url) { [string]$manifest.notes_url } else { "" }) - sha256 = $(if ($manifest.windows.sha256) { [string]$manifest.windows.sha256 } else { "" }) - reachable = @($candidates | ForEach-Object { "$($_.source)=$($_.latest)" }) -} - -if (-not $result.update_available) { - Write-Host "==> Up to date (latest=$latest, source=$($result.source))" -ForegroundColor Green - if (-not $Quiet) { - $result | ConvertTo-Json -Compress | Write-Output - } - exit 0 -} - -Write-Host "==> Update available: $current -> $latest (source=$($result.source))" -ForegroundColor Cyan -if ($result.notes_url) { Write-Host " Notes: $($result.notes_url)" } - -$autoEnabled = $false -$autoVal = Get-Cfg "NETX_UPDATE_AUTO" "" -if ($autoVal -match '^(1|true|yes|on)$') { $autoEnabled = $true } - -if (-not $Apply) { - Write-Host " Download ($($result.package)): $($result.download_url)" - Write-Host " To apply: .\packaging\check_update.ps1 -Apply" - if (-not $Quiet) { - $result | ConvertTo-Json -Compress | Write-Output - } - exit 10 -} - -if ($AutoOnly -and -not $autoEnabled) { - Write-Host "==> Update available but NETX_UPDATE_AUTO is not enabled; skip apply" - if (-not $Quiet) { - $result | ConvertTo-Json -Compress | Write-Output - } - exit 10 -} - -$lockFile = Join-Path $data "data\runtime\update.lock" -$lockDir = Split-Path -Parent $lockFile -if (-not (Test-Path $lockDir)) { - New-Item -ItemType Directory -Path $lockDir -Force | Out-Null -} -if (Test-Path $lockFile) { - $ageHrs = ((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalHours - if ($ageHrs -lt 2) { - Write-Host "==> Another update appears in progress ($lockFile); abort" - exit 3 - } - Remove-Item -Force $lockFile -ErrorAction SilentlyContinue -} -Set-Content -Path $lockFile -Value (Get-Date).ToString("o") -Encoding ascii - -try { - $dlDir = Join-Path $data "backups\downloads" - if (-not (Test-Path $dlDir)) { - New-Item -ItemType Directory -Path $dlDir -Force | Out-Null - } - - $isSetup = ($result.package -eq "setup") -or ($result.download_url -match '\.exe(\?|$)') - if ($isSetup) { - $pkgName = "NetX-Setup-$latest.exe" - } else { - $pkgName = "NetX-$latest-win64.zip" - } - $pkgPath = Join-Path $dlDir $pkgName - - $needDownload = $true - if ((Test-Path -LiteralPath $pkgPath) -and ((Get-Item -LiteralPath $pkgPath).Length -gt 1MB)) { - Write-Host "==> Using existing download: $pkgPath ($([math]::Round((Get-Item $pkgPath).Length/1MB,1)) MB)" - $needDownload = $false - } - if ($needDownload) { - Write-Host "==> Downloading $pkgName from $($result.source) ..." - # Only attach token for non-GitHub hosts (Forgejo/private). Public GitHub assets need no auth; - # a Forgejo token would break anonymous GitHub downloads. - $dlHeaders = @{ "User-Agent" = "NetX-UpdateCheck" } - $dlUri = [uri]$result.download_url - $isGithub = ($dlUri.Host -match '(^|\.)github\.com$' -or $dlUri.Host -match '(^|\.)githubusercontent\.com$') - if ($UpdateToken -and -not $isGithub) { - $dlHeaders["Authorization"] = "token $UpdateToken" - } - Invoke-WebRequest -Uri $result.download_url -OutFile $pkgPath -Headers $dlHeaders -UseBasicParsing - } - if ($result.sha256 -and $result.sha256 -notmatch 'REPLACE' -and $result.sha256.Trim()) { - $hash = (Get-FileHash -Path $pkgPath -Algorithm SHA256).Hash.ToLowerInvariant() - $expect = $result.sha256.ToLowerInvariant() - if ($hash -ne $expect) { - throw "sha256_mismatch: got $hash expected $expect" - } - Write-Host "==> SHA256 OK" - } - - # Program Files installs need admin; elevate once (avoid loop via NETX_UPDATE_ELEVATED). - $progWritable = $false - try { - $probe = Join-Path $prog (".netx_w_" + [guid]::NewGuid().ToString("n")) - [IO.File]::WriteAllText($probe, "x") - Remove-Item -LiteralPath $probe -Force -ErrorAction SilentlyContinue - $progWritable = $true - } catch { - $progWritable = $false - } - if (-not $progWritable -and -not $env:NETX_UPDATE_ELEVATED) { - Write-Host "==> Program directory is not writable; relaunching update as Administrator (UAC)..." -ForegroundColor Yellow - $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$PSCommandPath`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply" - $ep = Start-Process -FilePath "powershell.exe" -ArgumentList $arg -WorkingDirectory $prog ` - -Verb RunAs -Wait -PassThru - if ($null -eq $ep.ExitCode) { exit 1 } - exit $ep.ExitCode - } - - $env:NETX_UPDATE_ELEVATED = "1" - if ($isSetup) { - # Silent Setup update mode: skip DB wizard, keep ProgramData - # (see installer/netx.iss /InstallMode=update). - Write-Host "==> Applying update via silent Setup.exe" - $setupArgs = "/VERYSILENT /NORESTART /SUPPRESSMSGBOXES /DIR=`"$prog`" /InstallMode=update" - $sp = Start-Process -FilePath $pkgPath -ArgumentList $setupArgs -Wait -PassThru - if ($null -eq $sp.ExitCode -or $sp.ExitCode -ne 0) { - $code = if ($null -eq $sp.ExitCode) { "null" } else { $sp.ExitCode } - throw "setup_update_failed: exit $code" - } - # Setup post-install also repairs; belt-and-suspenders when running elevated apply. - $repairPs1 = Join-Path $prog "packaging\repair_venv.ps1" - if (Test-Path -LiteralPath $repairPs1) { - Write-Host "==> Relinking .venv to bundled python/runtime" - & powershell -NoProfile -ExecutionPolicy Bypass -File $repairPs1 ` - -ProgramRoot $prog -DataRoot $data - } - Write-Host "==> Restarting NetX after Setup" - & (Join-Path $PSScriptRoot "start_netx_app.ps1") ` - -ProgramRoot $prog -DataRoot $data -SkipBrowser - if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - } else { - Write-Host "==> Applying legacy zip update via update_netx.ps1" - & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "update_netx.ps1") ` - -PackagePath $pkgPath -ProgramRoot $prog -DataRoot $data - } - Write-Host "==> Update applied to $latest" -ForegroundColor Green -} finally { - Remove-Item -Force $lockFile -ErrorAction SilentlyContinue -} -exit 0 diff --git a/packaging/cmd/NetX-FirstRun.cmd b/packaging/cmd/NetX-FirstRun.cmd deleted file mode 100644 index e5f9887..0000000 --- a/packaging/cmd/NetX-FirstRun.cmd +++ /dev/null @@ -1,33 +0,0 @@ -@echo off -setlocal EnableDelayedExpansion -net session >nul 2>&1 -if not "%errorLevel%"=="0" ( - powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" - exit /b !ERRORLEVEL! -) -set "ROOT=%~dp0" -if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" -cd /d "%ROOT%" -title NetX - Reconfigure database -echo. -echo NetX database reconfigure / 重新配置数据库 -echo Prefer the installer wizard for first-time setup. -echo 首次安装请用安装向导选择内置或外置数据库。 -echo This window is for repair / reconfigure only. -echo 本窗口仅用于修复或重新配置。 -echo. -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\setup_first_run.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -set "EC=!ERRORLEVEL!" -if not "!EC!"=="0" ( - echo. - echo Setup failed / 配置失败 exit=!EC! - pause - exit /b !EC! -) -echo. -echo Starting NetX tray... / 正在启动托盘... -start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\netx_tray.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -StartOnLaunch -echo. -echo Done. You can close this window. / 完成，可关闭本窗口。 -pause -exit /b 0 diff --git a/packaging/cmd/NetX-Start.cmd b/packaging/cmd/NetX-Start.cmd deleted file mode 100644 index 8342f92..0000000 --- a/packaging/cmd/NetX-Start.cmd +++ /dev/null @@ -1,13 +0,0 @@ -@echo off -setlocal EnableDelayedExpansion -REM Always elevate: Program Files install needs admin to start API / repair venv. -net session >nul 2>&1 -if not "%errorLevel%"=="0" ( - powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" - exit /b !ERRORLEVEL! -) -set "ROOT=%~dp0" -if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" -cd /d "%ROOT%" -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action start -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -exit /b !ERRORLEVEL! diff --git a/packaging/cmd/NetX-Stop.cmd b/packaging/cmd/NetX-Stop.cmd deleted file mode 100644 index 8a225f2..0000000 --- a/packaging/cmd/NetX-Stop.cmd +++ /dev/null @@ -1,12 +0,0 @@ -@echo off -setlocal EnableDelayedExpansion -net session >nul 2>&1 -if not "%errorLevel%"=="0" ( - powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" - exit /b !ERRORLEVEL! -) -set "ROOT=%~dp0" -if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" -cd /d "%ROOT%" -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%ROOT%\packaging\launch_shortcut.ps1" -Action stop -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -exit /b !ERRORLEVEL! diff --git a/packaging/cmd/NetX-Tray.cmd b/packaging/cmd/NetX-Tray.cmd deleted file mode 100644 index ac1f46c..0000000 --- a/packaging/cmd/NetX-Tray.cmd +++ /dev/null @@ -1,13 +0,0 @@ -@echo off -setlocal EnableDelayedExpansion -REM Elevate once, then start tray (inherits admin for Start/Stop/Update). -net session >nul 2>&1 -if not "%errorLevel%"=="0" ( - powershell.exe -NoProfile -ExecutionPolicy Bypass -Command "try { $p = Start-Process -FilePath '%~f0' -Verb RunAs -Wait -PassThru; if ($null -eq $p -or $null -eq $p.ExitCode) { exit 1 }; exit $p.ExitCode } catch { exit 1 }" - exit /b !ERRORLEVEL! -) -set "ROOT=%~dp0" -if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" -cd /d "%ROOT%" -start "" powershell.exe -NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "%ROOT%\packaging\netx_tray.ps1" -ProgramRoot "%ROOT%" -DataRoot "%ProgramData%\NetX" -StartOnLaunch -exit /b 0 diff --git a/packaging/config/database.example.env b/packaging/config/database.example.env index f5e5555..a9d072a 100644 --- a/packaging/config/database.example.env +++ b/packaging/config/database.example.env @@ -5,11 +5,5 @@ # NETX_HOST=127.0.0.1 # NETX_PORT=8890 # NETX_UI_DIST_DIR=web/dist -# Defaults already: github + forgejo mirror git.avelo.top (newest reachable wins) -# NETX_UPDATE_SOURCES=github,forgejo -# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest -# NETX_UPDATE_GITHUB_REPO=hansjone/netx # NETX_UPDATE_URL= -# NETX_UPDATE_TOKEN= # NETX_UPDATE_CHANNEL=stable -# NETX_UPDATE_AUTO=false diff --git a/packaging/download_postgres.ps1 b/packaging/download_postgres.ps1 index 6398fd1..38c7b41 100644 --- a/packaging/download_postgres.ps1 +++ b/packaging/download_postgres.ps1 @@ -1,4 +1,4 @@ -param( +param( [string]$Version = "16.4-1", [string]$OutDir = "" ) diff --git a/packaging/install_autostart.ps1 b/packaging/install_autostart.ps1 deleted file mode 100644 index 4cba106..0000000 --- a/packaging/install_autostart.ps1 +++ /dev/null @@ -1,41 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [switch]$Remove = $false -) - -# Register / unregister NetX tray at logon with highest privileges (UAC admin). -# HKCU\Run cannot elevate reliably on fresh PCs; use a Scheduled Task instead. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$tray = Join-Path $PSScriptRoot "netx_tray.ps1" -$runKey = "HKCU:\Software\Microsoft\Windows\CurrentVersion\Run" -$legacyName = "NetX" -$taskName = "NetXTray" -$taskPath = "\NetX\" - -# Always clear legacy per-user Run key (non-elevated). -Remove-ItemProperty -Path $runKey -Name $legacyName -ErrorAction SilentlyContinue - -if ($Remove) { - Unregister-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Confirm:$false -ErrorAction SilentlyContinue - Write-Host "==> Removed NetX tray autostart task" - exit 0 -} - -$arg = "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File `"$tray`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -StartOnLaunch" -$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog -$trig = New-ScheduledTaskTrigger -AtLogOn -User $env:USERNAME -$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable -$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive -RunLevel Highest -Register-ScheduledTask -TaskName $taskName -TaskPath $taskPath ` - -Action $action -Trigger $trig -Settings $settings -Principal $principal -Force | Out-Null - -Write-Host "==> NetX tray will start at logon (Scheduled Task, RunLevel Highest)" -ForegroundColor Green -Write-Host " $taskPath$taskName" -Write-Host " $arg" diff --git a/packaging/install_service.ps1 b/packaging/install_service.ps1 deleted file mode 100644 index 02ea09c..0000000 --- a/packaging/install_service.ps1 +++ /dev/null @@ -1,140 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [ValidateSet("winsw", "task")] - [string]$Mode = "winsw", - [switch]$Uninstall = $false, - [switch]$Start = $false, - # Offline-safe by default: use WinSW shipped in the package. Opt-in to download from GitHub. - [switch]$AllowDownload = $false -) - -# Install NetX as a Windows Service (WinSW) or as a SYSTEM startup Scheduled Task. -# Requires elevation for winsw / task modes that run as SYSTEM. -# WinSW binary is bundled at packaging\winsw\WinSW-x64.exe by build_release.ps1 — no network needed. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$svcName = "NetX" -$winswDir = Join-Path $prog "packaging\winsw" -$winswExe = Join-Path $winswDir "NetX.exe" -$winswXml = Join-Path $winswDir "NetX.xml" -$cacheDir = Join-Path $PSScriptRoot "cache" - -function Test-IsAdmin { - return Test-NetxIsAdmin -} - -function ConvertTo-WinSWPath([string]$Path) { - return ($Path -replace '\\', '/') -} - -function Ensure-WinSW { - if (-not (Test-Path $winswDir)) { - New-Item -ItemType Directory -Path $winswDir -Force | Out-Null - } - if (-not (Test-Path $cacheDir)) { - New-Item -ItemType Directory -Path $cacheDir -Force | Out-Null - } - $bundled = Join-Path $PSScriptRoot "winsw\WinSW-x64.exe" - $dl = Join-Path $cacheDir "WinSW-x64.exe" - $src = $null - if (Test-Path $bundled) { - $src = $bundled - Write-Host "==> Using bundled WinSW: $bundled" - } elseif (Test-Path $dl) { - $src = $dl - Write-Host "==> Using cached WinSW: $dl" - } elseif ($AllowDownload) { - $url = "https://github.com/winsw/winsw/releases/download/v2.12.0/WinSW-x64.exe" - Write-Host "==> Downloading WinSW: $url" - Invoke-WebRequest -Uri $url -OutFile $dl -UseBasicParsing - $src = $dl - } else { - throw "winsw_missing_offline: expected $bundled (rebuild with build_release.ps1). Or pass -AllowDownload when online." - } - Copy-Item -Path $src -Destination $winswExe -Force - - $runPs1 = ConvertTo-WinSWPath (Join-Path $PSScriptRoot "service_run.ps1") - $stopPs1 = ConvertTo-WinSWPath (Join-Path $PSScriptRoot "stop_netx_app.ps1") - $progFs = ConvertTo-WinSWPath $prog - $dataFs = ConvertTo-WinSWPath $data - $logPath = ConvertTo-WinSWPath (Join-Path $data "data\runtime") - if (-not (Test-Path (Join-Path $data "data\runtime"))) { - New-Item -ItemType Directory -Path (Join-Path $data "data\runtime") -Force | Out-Null - } - - $xml = @" - - $svcName - NetX Ops - NetX API, workers, and optional bundled PostgreSQL - powershell.exe - -NoProfile -ExecutionPolicy Bypass -File "$runPs1" -ProgramRoot "$progFs" -DataRoot "$dataFs" - $logPath - - 10240 - 4 - - - - 1 hour - 60sec - powershell.exe - -NoProfile -ExecutionPolicy Bypass -File "$stopPs1" -ProgramRoot "$progFs" -DataRoot "$dataFs" - $progFs - -"@ - # WinSW expects UTF-8 without BOM issues; ASCII-compatible paths preferred. - [System.IO.File]::WriteAllText($winswXml, $xml) -} - -if ($Uninstall) { - if (-not (Test-IsAdmin)) { throw "admin_required_for_uninstall" } - if (Test-Path $winswExe) { - Write-Host "==> Stopping/uninstalling WinSW service" - & $winswExe stop 2>$null - & $winswExe uninstall 2>$null - } - Unregister-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -Confirm:$false -ErrorAction SilentlyContinue - Write-Host "==> Service/task removed" - exit 0 -} - -if (-not (Test-IsAdmin)) { - throw "admin_required: run elevated PowerShell to install the NetX service" -} - -if ($Mode -eq "winsw") { - Ensure-WinSW - Write-Host "==> Installing Windows Service via WinSW" - & $winswExe stop 2>$null - & $winswExe uninstall 2>$null - & $winswExe install - if ($LASTEXITCODE -ne 0) { throw "winsw_install_failed" } - if ($Start) { - & $winswExe start - Write-Host "==> Service started" -ForegroundColor Green - } else { - Write-Host "==> Installed. Start with: Start-Service NetX or `"$winswExe`" start" - } -} else { - Write-Host "==> Registering Scheduled Task NetX\NetXService (At startup, SYSTEM)" - $runPs1 = Join-Path $PSScriptRoot "service_run.ps1" - $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$runPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`"" - $action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog - $trigger = New-ScheduledTaskTrigger -AtStartup - $principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest - $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -RestartCount 3 -RestartInterval (New-TimeSpan -Minutes 1) - Register-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Force | Out-Null - if ($Start) { - Start-ScheduledTask -TaskName "NetXService" -TaskPath "\NetX\" - Write-Host "==> Task started" -ForegroundColor Green - } else { - Write-Host "==> Task registered. Start with: Start-ScheduledTask -TaskPath '\NetX\' -TaskName NetXService" - } -} diff --git a/packaging/install_update_task.ps1 b/packaging/install_update_task.ps1 deleted file mode 100644 index afcf712..0000000 --- a/packaging/install_update_task.ps1 +++ /dev/null @@ -1,59 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [ValidateSet("Daily", "AtLogOn", "Hourly")] - [string]$Trigger = "Daily", - [string]$At = "03:30", - [switch]$Remove = $false, - [switch]$EnableAutoEnv = $true -) - -# Schedule silent update checks. With NETX_UPDATE_AUTO=true, applies updates when available. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$taskName = "NetXUpdate" -$taskPath = "\NetX\" -$checkPs1 = Join-Path $PSScriptRoot "check_update.ps1" - -if ($Remove) { - Unregister-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Confirm:$false -ErrorAction SilentlyContinue - Write-Host "==> Update task removed" - exit 0 -} - -if ($EnableAutoEnv) { - $envFile = Join-Path $data ".env" - if (-not (Test-Path $data)) { - New-Item -ItemType Directory -Path $data -Force | Out-Null - } - Write-DotEnvValue -Path $envFile -Values @{ "NETX_UPDATE_AUTO" = "true" } - Write-Host "==> Set NETX_UPDATE_AUTO=true in $envFile" -} - -$arg = "-NoProfile -ExecutionPolicy Bypass -File `"$checkPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply -Quiet -AutoOnly" -$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument $arg -WorkingDirectory $prog - -switch ($Trigger) { - "Hourly" { - $trig = New-ScheduledTaskTrigger -Once -At (Get-Date).Date.AddHours((Get-Date).Hour + 1) ` - -RepetitionInterval (New-TimeSpan -Hours 1) -RepetitionDuration ([TimeSpan]::MaxValue) - } - "AtLogOn" { - $trig = New-ScheduledTaskTrigger -AtLogOn - } - default { - $trig = New-ScheduledTaskTrigger -Daily -At $At - } -} - -$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -StartWhenAvailable -# Highest: silent apply must write Program Files / run Setup. -$principal = New-ScheduledTaskPrincipal -UserId $env:USERNAME -LogonType Interactive -RunLevel Highest -Register-ScheduledTask -TaskName $taskName -TaskPath $taskPath -Action $action -Trigger $trig -Settings $settings -Principal $principal -Force | Out-Null -Write-Host "==> Scheduled update task: $taskPath$taskName ($Trigger, RunLevel Highest)" -ForegroundColor Green -Write-Host " Manual run: .\packaging\check_update.ps1 -Apply -Quiet -AutoOnly" diff --git a/packaging/installer/Languages/ChineseSimplified.isl b/packaging/installer/Languages/ChineseSimplified.isl deleted file mode 100644 index 30d9973..0000000 --- a/packaging/installer/Languages/ChineseSimplified.isl +++ /dev/null @@ -1,417 +0,0 @@ -; *** Inno Setup version 6.5.0+ Chinese Simplified messages *** -; -; To download user-contributed translations of this file, go to: -; https://jrsoftware.org/files/istrans/ -; -; Note: When translating this text, do not add periods (.) to the end of -; messages that didn't have them already, because on those messages Inno -; Setup adds the periods automatically (appending a period would result in -; two periods being displayed). -; -; Maintainer: Zhenghan Yang (Kira) -; Email: 847320916@QQ.com -; Github: https://github.com/kira-96/Inno-Setup-Chinese-Simplified-Translation -; Encoding: UTF-8 -; Translation based on network resource -; - -[LangOptions] -; The following three entries are very important. Be sure to read and -; understand the '[LangOptions] section' topic in the help file. -LanguageName=简体中文 -; About LanguageID, to reference link: -; https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/a9eac961-e77d-41a6-90a5-ce1a8b0cdb9c -LanguageID=$0804 -; LanguageCodePage should always be set if possible, even if this file is Unicode -; For English it's set to zero anyway because English only uses ASCII characters -LanguageCodePage=936 -; If the language you are translating to requires special font faces or -; sizes, uncomment any of the following entries and change them accordingly. -;DialogFontName= -;DialogFontSize=9 -;DialogFontBaseScaleWidth=7 -;DialogFontBaseScaleHeight=15 -;WelcomeFontName=Segoe UI -;WelcomeFontSize=14 - -[Messages] - -; *** Application titles -SetupAppTitle=安装 -SetupWindowTitle=安装 - %1 -UninstallAppTitle=卸载 -UninstallAppFullTitle=%1 卸载 - -; *** Misc. common -InformationTitle=信息 -ConfirmTitle=确认 -ErrorTitle=错误 - -; *** SetupLdr messages -SetupLdrStartupMessage=现在将安装 %1。您想要继续吗？ -LdrCannotCreateTemp=无法创建临时文件。安装程序已中止 -LdrCannotExecTemp=无法执行临时目录中的文件。安装程序已中止 -HelpTextNote= - -; *** Startup error messages -LastErrorMessage=%1。%n%n错误 %2: %3 -SetupFileMissing=安装目录中缺少文件 %1。请修正这个问题或者获取程序的新副本。 -SetupFileCorrupt=安装文件已损坏。请获取程序的新副本。 -SetupFileCorruptOrWrongVer=安装文件已损坏，或是与这个安装程序的版本不兼容。请修正这个问题或获取新的程序副本。 -InvalidParameter=无效的命令行参数：%n%n%1 -SetupAlreadyRunning=安装程序已在运行。 -WindowsVersionNotSupported=此程序不支持当前计算机运行的 Windows 版本。 -WindowsServicePackRequired=此程序需要 %1 服务包 %2 或更高版本。 -NotOnThisPlatform=此程序不能在 %1 上运行。 -OnlyOnThisPlatform=此程序只能在 %1 上运行。 -OnlyOnTheseArchitectures=此程序只能安装到为下列处理器架构设计的 Windows 版本中：%n%n%1 -WinVersionTooLowError=此程序需要 %1 版本 %2 或更高。 -WinVersionTooHighError=此程序不能安装于 %1 版本 %2 或更高。 -AdminPrivilegesRequired=在安装此程序时您必须以管理员身份登录。 -PowerUserPrivilegesRequired=在安装此程序时您必须以管理员身份或高级用户组身份登录。 -SetupAppRunningError=安装程序检测到 %1 当前正在运行。%n%n请先关闭正在运行的程序，然后点击“确定”继续，或点击“取消”退出。 -UninstallAppRunningError=卸载程序检测到 %1 当前正在运行。%n%n请先关闭正在运行的程序，然后点击“确定”继续，或点击“取消”退出。 - -; *** Startup questions -PrivilegesRequiredOverrideTitle=选择安装程序安装模式 -PrivilegesRequiredOverrideInstruction=选择安装模式 -PrivilegesRequiredOverrideText1=%1 可以为所有用户安装（需要管理员权限），或仅为您安装。 -PrivilegesRequiredOverrideText2=%1 可以仅为您安装，或为所有用户安装（需要管理员权限）。 -PrivilegesRequiredOverrideAllUsers=为所有用户安装(&A) -PrivilegesRequiredOverrideAllUsersRecommended=为所有用户安装(&A)（推荐） -PrivilegesRequiredOverrideCurrentUser=仅为我安装(&M) -PrivilegesRequiredOverrideCurrentUserRecommended=仅为我安装(&M)（推荐） - -; *** Misc. errors -ErrorCreatingDir=安装程序无法创建目录“%1” -ErrorTooManyFilesInDir=无法在目录“%1”中创建文件，因为里面包含太多文件。 - -; *** Setup common messages -ExitSetupTitle=退出安装程序 -ExitSetupMessage=安装程序尚未完成。如果现在退出，将不会安装该程序。%n%n您之后可以再次运行安装程序完成安装。%n%n现在退出安装程序吗？ -AboutSetupMenuItem=关于安装程序(&A)... -AboutSetupTitle=关于安装程序 -AboutSetupMessage=%1 版本 %2%n%3%n%n%1 主页：%n%4 -AboutSetupNote= -TranslatorNote=简体中文翻译由 Kira（847320916@qq.com）维护。项目地址：https://github.com/kira-96/Inno-Setup-Chinese-Simplified-Translation - -; *** Buttons -ButtonBack=< 上一步(&B) -ButtonNext=下一步(&N) > -ButtonInstall=安装(&I) -ButtonOK=确定 -ButtonCancel=取消 -ButtonYes=是(&Y) -ButtonYesToAll=全是(&A) -ButtonNo=否(&N) -ButtonNoToAll=全否(&O) -ButtonFinish=完成(&F) -ButtonBrowse=浏览(&B)... -ButtonWizardBrowse=浏览(&R)... -ButtonNewFolder=新建文件夹(&M) - -; *** "Select Language" dialog messages -SelectLanguageTitle=选择安装语言 -SelectLanguageLabel=选择安装时使用的语言。 - -; *** Common wizard text -ClickNext=点击“下一步”继续，或点击“取消”退出安装程序。 -BeveledLabel= -BrowseDialogTitle=浏览文件夹 -BrowseDialogLabel=在下面的列表中选择一个文件夹，然后点击“确定”。 -NewFolderName=新建文件夹 - -; *** "Welcome" wizard page -WelcomeLabel1=欢迎使用 [name] 安装向导 -WelcomeLabel2=即将在您的计算机上安装 [name/ver]。%n%n建议您在继续安装前关闭所有其他应用程序。 - -; *** "Password" wizard page -WizardPassword=密码 -PasswordLabel1=此安装程序需要密码验证。 -PasswordLabel3=请输入密码，然后点击“下一步”继续。密码区分大小写。 -PasswordEditLabel=密码(&P)： -IncorrectPassword=您输入的密码不正确，请重新输入。 - -; *** "License Agreement" wizard page -WizardLicense=许可协议 -LicenseLabel=请在继续安装前阅读以下重要信息。 -LicenseLabel3=请阅读下列许可协议。在继续安装前您必须同意这些协议条款。 -LicenseAccepted=我同意此协议(&A) -LicenseNotAccepted=我不同意此协议(&D) - -; *** "Information" wizard pages -WizardInfoBefore=信息 -InfoBeforeLabel=请在继续安装前阅读以下重要信息。 -InfoBeforeClickLabel=准备好继续安装后，点击“下一步”。 -WizardInfoAfter=信息 -InfoAfterLabel=请在继续安装前阅读以下重要信息。 -InfoAfterClickLabel=准备好继续安装后，点击“下一步”。 - -; *** "User Information" wizard page -WizardUserInfo=用户信息 -UserInfoDesc=请输入您的信息。 -UserInfoName=用户名(&U)： -UserInfoOrg=组织(&O)： -UserInfoSerial=序列号(&S)： -UserInfoNameRequired=请输入用户名。 - -; *** "Select Destination Location" wizard page -WizardSelectDir=选择目标位置 -SelectDirDesc=您想将 [name] 安装在哪里？ -SelectDirLabel3=安装程序将安装 [name] 到下面的文件夹中。 -SelectDirBrowseLabel=点击“下一步”继续。如果您想选择其他文件夹，点击“浏览”。 -DiskSpaceGBLabel=至少需要有 [gb] GB 的可用磁盘空间。 -DiskSpaceMBLabel=至少需要有 [mb] MB 的可用磁盘空间。 -CannotInstallToNetworkDrive=安装程序无法安装到一个网络驱动器。 -CannotInstallToUNCPath=安装程序无法安装到一个 UNC 路径。 -InvalidPath=您必须输入一个带驱动器盘符的完整路径，例如：%n%nC:\App%n%n或UNC路径：%n%n\\server\share -InvalidDrive=您选定的驱动器或 UNC 共享不存在或不能访问。请选择其他位置。 -DiskSpaceWarningTitle=磁盘空间不足 -DiskSpaceWarning=安装程序至少需要 %1 KB 的可用空间才能安装，但选定驱动器只有 %2 KB 的可用空间。%n%n您确定要继续吗？ -DirNameTooLong=文件夹名称或路径太长。 -InvalidDirName=文件夹名称无效。 -BadDirName32=文件夹名称不能包含下列任何字符：%n%n%1 -DirExistsTitle=文件夹已存在 -DirExists=文件夹：%n%n%1%n%n已经存在。您确定安装到这个文件夹中吗？ -DirDoesntExistTitle=文件夹不存在 -DirDoesntExist=文件夹：%n%n%1%n%n不存在。您想要创建此文件夹吗？ - -; *** "Select Components" wizard page -WizardSelectComponents=选择组件 -SelectComponentsDesc=您想安装哪些程序组件？ -SelectComponentsLabel2=选中您想安装的组件；取消您不想安装的组件。然后点击“下一步”继续。 -FullInstallation=完全安装 -; if possible don't translate 'Compact' as 'Minimal' (I mean 'Minimal' in your language) -CompactInstallation=简洁安装 -CustomInstallation=自定义安装 -NoUninstallWarningTitle=组件已存在 -NoUninstallWarning=安装程序检测到下列组件已安装在您的计算机中：%n%n%1%n%n取消选中这些组件不会卸载它们。%n%n您确定要继续吗？ -ComponentSize1=%1 KB -ComponentSize2=%1 MB -ComponentsDiskSpaceGBLabel=当前选择的组件需要至少 [gb] GB 的磁盘空间。 -ComponentsDiskSpaceMBLabel=当前选择的组件需要至少 [mb] MB 的磁盘空间。 - -; *** "Select Additional Tasks" wizard page -WizardSelectTasks=选择附加任务 -SelectTasksDesc=您想要安装程序执行哪些附加任务？ -SelectTasksLabel2=选择您想要安装程序在安装 [name] 时执行的附加任务，然后点击“下一步”。 - -; *** "Select Start Menu Folder" wizard page -WizardSelectProgramGroup=选择开始菜单文件夹 -SelectStartMenuFolderDesc=安装程序应该在哪里放置程序的快捷方式？ -SelectStartMenuFolderLabel3=安装程序将在下列“开始”菜单文件夹中创建程序的快捷方式。 -SelectStartMenuFolderBrowseLabel=点击“下一步”继续。如果您想选择其他文件夹，点击“浏览”。 -MustEnterGroupName=您必须输入一个文件夹名称。 -GroupNameTooLong=文件夹名称或路径太长。 -InvalidGroupName=文件夹名称无效。 -BadGroupName=文件夹名称不能包含下列任何字符：%n%n%1 -NoProgramGroupCheck2=不创建开始菜单文件夹(&D) - -; *** "Ready to Install" wizard page -WizardReady=准备安装 -ReadyLabel1=安装程序准备就绪，现在可以开始安装 [name] 到您的计算机。 -ReadyLabel2a=点击“安装”继续此安装程序。如果您想重新查看或修改任何设置，点击“上一步”。 -ReadyLabel2b=点击“安装”继续此安装程序。 -ReadyMemoUserInfo=用户信息： -ReadyMemoDir=目标位置： -ReadyMemoType=安装类型： -ReadyMemoComponents=已选择组件： -ReadyMemoGroup=开始菜单文件夹： -ReadyMemoTasks=附加任务： - -; *** TDownloadWizardPage wizard page and DownloadTemporaryFile -DownloadingLabel2=正在下载文件... -ButtonStopDownload=停止下载(&S) -StopDownload=您确定要停止下载吗？ -ErrorDownloadAborted=下载已中止。 -ErrorDownloadFailed=下载失败：%1 %2。 -ErrorDownloadSizeFailed=获取大小失败：%1 %2。 -ErrorProgress=无效的进度：%1 / %2。 -ErrorFileSize=文件大小错误：预期 %1，实际 %2。 - -; *** TExtractionWizardPage wizard page and ExtractArchive -ExtractingLabel=正在提取文件... -ButtonStopExtraction=停止提取(&S) -StopExtraction=您确定要停止提取吗？ -ErrorExtractionAborted=提取已中止。 -ErrorExtractionFailed=提取失败：%1 - -; *** Archive extraction failure details -ArchiveIncorrectPassword=密码不正确。 -ArchiveIsCorrupted=压缩包已损坏。 -ArchiveUnsupportedFormat=不支持的压缩包格式。 - -; *** "Preparing to Install" wizard page -WizardPreparing=正在准备安装 -PreparingDesc=安装程序正在准备安装 [name] 到您的计算机。 -PreviousInstallNotCompleted=先前的程序安装或卸载未完成，需要您重启计算机以完成该安装。%n%n在重启计算机后，再次运行安装程序以完成 [name] 的安装。 -CannotContinue=安装程序不能继续。请点击“取消”退出。 -ApplicationsFound=以下应用程序正在使用将由安装程序更新的文件。建议您允许安装程序自动关闭这些应用程序。 -ApplicationsFound2=以下应用程序正在使用将由安装程序更新的文件。建议您允许安装程序自动关闭这些应用程序。安装完成后，安装程序将尝试重新启动这些应用程序。 -CloseApplications=自动关闭应用程序(&A) -DontCloseApplications=不要关闭应用程序(&D) -ErrorCloseApplications=安装程序无法自动关闭所有应用程序。建议您在继续之前，关闭所有在使用需要由安装程序更新的文件的应用程序。 -PrepareToInstallNeedsRestart=安装程序必须重启您的计算机。计算机重启后，请再次运行安装程序以完成 [name] 的安装。%n%n要立即重启吗？ - -; *** "Installing" wizard page -WizardInstalling=正在安装 -InstallingLabel=安装程序正在安装 [name] 到您的计算机，请稍候。 - -; *** "Setup Completed" wizard page -FinishedHeadingLabel=完成 [name] 安装向导 -FinishedLabelNoIcons=安装程序已在您的计算机中安装了 [name]。 -FinishedLabel=安装程序已在您的计算机中安装了 [name]。您可以通过已安装的快捷方式运行此应用程序。 -ClickFinish=点击“完成”退出安装程序。 -FinishedRestartLabel=为完成 [name] 的安装，安装程序必须重新启动您的计算机。要立即重启吗？ -FinishedRestartMessage=为完成 [name] 的安装，安装程序必须重新启动您的计算机。%n%n要立即重启吗？ -ShowReadmeCheck=是，我想查阅自述文件 -YesRadio=是，立即重启计算机(&Y) -NoRadio=否，稍后重启计算机(&N) -; used for example as 'Run MyProg.exe' -RunEntryExec=运行 %1 -; used for example as 'View Readme.txt' -RunEntryShellExec=查阅 %1 - -; *** "Setup Needs the Next Disk" stuff -ChangeDiskTitle=安装程序需要下一张磁盘 -SelectDiskLabel2=请插入磁盘 %1 并点击“确定”。%n%n如果这个磁盘中的文件可以在下列文件夹之外的文件夹中找到，请输入正确的路径或点击“浏览”。 -PathLabel=路径(&P)： -FileNotInDir2=“%2”中找不到文件“%1”。请插入正确的磁盘或选择其他文件夹。 -SelectDirectoryLabel=请指定下一张磁盘的位置。 - -; *** Installation phase messages -SetupAborted=安装程序未完成安装。%n%n请修正这个问题并重新运行安装程序。 -AbortRetryIgnoreSelectAction=选择操作 -AbortRetryIgnoreRetry=重试(&T) -AbortRetryIgnoreIgnore=忽略错误并继续(&I) -AbortRetryIgnoreCancel=取消安装 -RetryCancelSelectAction=选择操作 -RetryCancelRetry=重试(&T) -RetryCancelCancel=取消 - -; *** Installation status messages -StatusClosingApplications=正在关闭应用程序... -StatusCreateDirs=正在创建目录... -StatusExtractFiles=正在提取文件... -StatusDownloadFiles=正在下载文件... -StatusCreateIcons=正在创建快捷方式... -StatusCreateIniEntries=正在创建 INI 条目... -StatusCreateRegistryEntries=正在创建注册表条目... -StatusRegisterFiles=正在注册文件... -StatusSavingUninstall=正在保存卸载信息... -StatusRunProgram=正在完成安装... -StatusRestartingApplications=正在重启应用程序... -StatusRollback=正在撤销更改... - -; *** Misc. errors -ErrorInternal2=内部错误：%1。 -ErrorFunctionFailedNoCode=%1 失败。 -ErrorFunctionFailed=%1 失败；错误代码 %2。 -ErrorFunctionFailedWithMessage=%1 失败；错误代码 %2。%n%3 -ErrorExecutingProgram=无法执行文件：%n%1 - -; *** Registry errors -ErrorRegOpenKey=打开注册表项时出错：%n%1\%2 -ErrorRegCreateKey=创建注册表项时出错：%n%1\%2 -ErrorRegWriteKey=写入注册表项时出错：%n%1\%2 - -; *** INI errors -ErrorIniEntry=在文件“%1”中创建 INI 条目时出错。 - -; *** File copying errors -FileAbortRetryIgnoreSkipNotRecommended=跳过此文件(&S)（不推荐） -FileAbortRetryIgnoreIgnoreNotRecommended=忽略错误并继续(&I)（不推荐） -SourceIsCorrupted=源文件已损坏。 -SourceDoesntExist=源文件“%1”不存在。 -SourceVerificationFailed=源文件验证失败：%1 -VerificationSignatureDoesntExist=签名文件“%1”不存在。 -VerificationSignatureInvalid=签名文件“%1”无效。 -VerificationKeyNotFound=签名文件“%1”使用了未知的密钥。 -VerificationFileNameIncorrect=文件名不正确。 -VerificationFileTagIncorrect=文件标签不正确。 -VerificationFileSizeIncorrect=文件大小不正确。 -VerificationFileHashIncorrect=文件哈希值不正确。 -ExistingFileReadOnly2=无法替换已存在的文件，它是只读的。 -ExistingFileReadOnlyRetry=移除只读属性并重试(&R) -ExistingFileReadOnlyKeepExisting=保留已存在的文件(&K) -ErrorReadingExistingDest=尝试读取已存在的文件时出错： -FileExistsSelectAction=选择操作 -FileExists2=文件已经存在。 -FileExistsOverwriteExisting=覆盖已存在的文件(&O) -FileExistsKeepExisting=保留已存在的文件(&K) -FileExistsOverwriteOrKeepAll=为接下来的冲突文件执行此操作(&D) -ExistingFileNewerSelectAction=选择操作 -ExistingFileNewer2=已存在的文件比安装程序将要安装的文件还要新。 -ExistingFileNewerOverwriteExisting=覆盖已存在的文件(&O) -ExistingFileNewerKeepExisting=保留已存在的文件(&K)（推荐） -ExistingFileNewerOverwriteOrKeepAll=为接下来的冲突文件执行此操作(&D) -ErrorChangingAttr=尝试更改下列已存在的文件属性时出错： -ErrorCreatingTemp=尝试在目标目录创建文件时出错： -ErrorReadingSource=尝试读取下列源文件时出错： -ErrorCopying=尝试复制下列文件时出错： -ErrorDownloading=尝试下载文件时出错： -ErrorExtracting=尝试提取压缩包时出错： -ErrorReplacingExistingFile=尝试替换已存在的文件时出错： -ErrorRestartReplace=重启并替换失败： -ErrorRenamingTemp=尝试重命名下列目标目录中的一个文件时出错： -ErrorRegisterServer=无法注册 DLL/OCX：%1 -ErrorRegSvr32Failed=RegSvr32 失败；退出代码 %1。 -ErrorRegisterTypeLib=无法注册类型库：%1 - -; *** Uninstall display name markings -; used for example as 'My Program (32-bit)' -UninstallDisplayNameMark=%1 (%2) -; used for example as 'My Program (32-bit, All users)' -UninstallDisplayNameMarks=%1 (%2, %3) -UninstallDisplayNameMark32Bit=32 位 -UninstallDisplayNameMark64Bit=64 位 -UninstallDisplayNameMarkAllUsers=所有用户 -UninstallDisplayNameMarkCurrentUser=当前用户 - -; *** Post-installation errors -ErrorOpeningReadme=尝试打开自述文件时出错。 -ErrorRestartingComputer=安装程序无法重启计算机，请手动重启。 - -; *** Uninstaller messages -UninstallNotFound=文件“%1”不存在。无法卸载。 -UninstallOpenError=文件“%1”不能被打开。无法卸载 -UninstallUnsupportedVer=此版本的卸载程序无法识别卸载日志文件“%1”的格式。无法卸载。 -UninstallUnknownEntry=卸载日志中遇到一个未知条目（%1）。 -ConfirmUninstall=您确认要完全移除 %1 及其所有组件吗？ -UninstallOnlyOnWin64=仅允许在 64 位 Windows 中卸载此程序。 -OnlyAdminCanUninstall=仅使用管理员权限的用户能完成此卸载。 -UninstallStatusLabel=正在从您的计算机中移除 %1，请稍候。 -UninstalledAll=已顺利从您的计算机中移除 %1。 -UninstalledMost=%1 卸载完成。%n%n有部分内容未能被删除，但您可以手动删除它们。 -UninstalledAndNeedsRestart=为完成 %1 的卸载，需要重启您的计算机。%n%n要立即重启吗？ -UninstallDataCorrupted=文件“%1”已损坏。无法卸载。 - -; *** Uninstallation phase messages -ConfirmDeleteSharedFileTitle=删除共享文件？ -ConfirmDeleteSharedFile2=系统表示下列共享文件已不再有任何程序使用。您希望卸载程序删除此共享文件吗？%n%n如果仍有程序正在使用此文件，删除后这些程序可能无法正常运行。如果您不能确定，请选择“否”，保留此文件在系统中不会造成任何损害。 -SharedFileNameLabel=文件名： -SharedFileLocationLabel=位置： -WizardUninstalling=卸载状态 -StatusUninstalling=正在卸载 %1... - -; *** Shutdown block reasons -ShutdownBlockReasonInstallingApp=正在安装 %1。 -ShutdownBlockReasonUninstallingApp=正在卸载 %1。 - -; The custom messages below aren't used by Setup itself, but if you make -; use of them in your scripts, you'll want to translate them. - -[CustomMessages] - -NameAndVersion=%1 版本 %2 -AdditionalIcons=附加快捷方式： -CreateDesktopIcon=创建桌面快捷方式(&D) -CreateQuickLaunchIcon=创建快速启动栏快捷方式(&Q) -ProgramOnTheWeb=%1 网站 -UninstallProgram=卸载 %1 -LaunchProgram=运行 %1 -AssocFileExtension=将 %2 文件扩展名与 %1 建立关联(&A) -AssocingFileExtension=正在将 %2 文件扩展名与 %1 建立关联... -AutoStartProgramGroupDescription=启动： -AutoStartProgram=自动启动 %1 -AddonHostProgramNotFound=您选择的文件夹中无法找到 %1。%n%n您确定要继续吗？ diff --git a/packaging/installer/netx.iss b/packaging/installer/netx.iss index fc2feaa..55090ea 100644 --- a/packaging/installer/netx.iss +++ b/packaging/installer/netx.iss @@ -1,11 +1,10 @@ -; NetX Windows installer (Inno Setup 6+) +; NetX Windows installer (Inno Setup 6+) ; Compile after: packaging\build_release.ps1 ; ISCC.exe packaging\installer\netx.iss -; Encoding: UTF-8 with BOM (required for Chinese CustomMessages) #define MyAppName "NetX" #ifndef MyAppVersion - #define MyAppVersion "0.4.13" + #define MyAppVersion "0.3.0" #endif #define MyAppPublisher "NetX" #define MyAppURL "https://github.com/hansjone/netx" @@ -13,7 +12,6 @@ ; Stage directory produced by build_release.ps1 (relative to this .iss) #define StageDir "..\release\netx-win64" -#define IconFile "..\assets\netx.ico" [Setup] AppId={{A7E3C2D1-9F40-4B8E-9C1A-NETXWIN64001} @@ -26,1048 +24,46 @@ DefaultGroupName=NetX DisableProgramGroupPage=yes LicenseFile={#StageDir}\LICENSE OutputDir=..\release -#ifndef OutputBaseFilename - #define OutputBaseFilename "NetX-Setup-" + MyAppVersion -#endif -OutputBaseFilename={#OutputBaseFilename} -SetupIconFile={#IconFile} -UninstallDisplayIcon={app}\packaging\assets\netx.ico +OutputBaseFilename=NetX-Setup-{#MyAppVersion} Compression=lzma2 SolidCompression=yes WizardStyle=modern ArchitecturesAllowed=x64compatible ArchitecturesInstallIn64BitMode=x64compatible PrivilegesRequired=admin -; Data lives under {commonappdata}\NetX - not overwritten by upgrades. -; Do NOT use CloseApplications=yes — it can freeze/beep on the Tasks/Ready -; pages while scanning locked NetX tray/service processes with no visible dialog. -CloseApplications=no -ShowLanguageDialog=yes +; Data lives under {commonappdata}\NetX — not overwritten by upgrades +CloseApplications=yes [Languages] Name: "english"; MessagesFile: "compiler:Default.isl" -Name: "chinesesimplified"; MessagesFile: "Languages\ChineseSimplified.isl" - -[CustomMessages] -english.CreateDesktopIcon=Create a desktop shortcut -english.SetupOptions=After install: -english.StartTrayNow=Start NetX tray now -english.EnableAutostart=Start NetX tray at Windows logon -english.EnableAutoUpdate=Enable daily silent auto-update (needs network later) -english.InstallService=Install as Windows Service (uses bundled WinSW, offline) -english.UninstallDeleteData=Also delete NetX data (%ProgramData%\NetX)?%n%nYes = remove database, settings, and secrets%nNo = keep data for a later reinstall -english.DbPageCaption=Database -english.DbPageDescription=Choose built-in or external PostgreSQL. Optional credential key: paste from an existing install, or leave empty to auto-generate. -english.DbBundled=Built-in PostgreSQL (portable, offline) -english.DbExternal=External PostgreSQL (existing server) -english.DbHost=Host -english.DbPort=Port -english.DbUser=User -english.DbPassword=Password -english.DbName=Database -english.CredKey=Credential key -english.CredKeyHint=NETX_CREDENTIAL_SECRET_KEY (optional — empty = auto-generate) -english.DbFieldsRequired=Please fill in host, port, user, password, and database name. -english.DbTesting=Testing PostgreSQL connection, please wait… -english.DbTestFailed=Cannot connect to PostgreSQL with these settings.%n%n%1%n%nFix the settings and try again. -english.DbTestExtractFailed=Could not extract PostgreSQL client tools for connection test. -english.DbSilentExternalMissing=Silent install with external DB requires /DbHost /DbUser /DbPassword /DbName (optional /DbPort). -english.DbApplyFailed=Database configuration failed after file install (exit %1).%n%nLog: %2%n%nFix the problem, then use Start Menu → Reconfigure database — or reinstall. -english.DbApplyExecFailed=Could not start database configuration script. -english.DbBundledMissing=Built-in PostgreSQL files are missing from the install folder:%n%1%n%nThe Setup package is incomplete. Re-download NetX-Setup and install again. -english.DbEnvMissing=Database configuration did not write %ProgramData%\NetX\.env. -english.ReconfigureDb=Reconfigure database -english.UpgradeKeepDb=Existing NetX data found - database settings will be kept (no wizard). -english.ModePageCaption=Install mode -english.ModePageDescription=An existing NetX installation was found. Choose update or reinstall. -english.ModeUpdate=Update existing installation (keep database settings) -english.ModeReinstall=Reinstall and reconfigure database (same as first install) -english.ModeUpdateHint=Replaces program files only. ProgramData\.env and database data are kept. -english.ModeReinstallHint=Shows the Database page again. Does not delete ProgramData unless you uninstall and choose to. -chinesesimplified.CreateDesktopIcon=创建桌面快捷方式 -chinesesimplified.SetupOptions=安装完成后: -chinesesimplified.StartTrayNow=立即启动 NetX 托盘 -chinesesimplified.EnableAutostart=开机时自动启动 NetX 托盘 -chinesesimplified.EnableAutoUpdate=启用每日静默自动更新（以后需要联网） -chinesesimplified.InstallService=安装为 Windows 服务（使用已捆绑的 WinSW，可离线） -chinesesimplified.UninstallDeleteData=是否同时删除 NetX 数据（%ProgramData%\NetX）？%n%n是 = 删除数据库、配置和密钥%n否 = 保留数据以便以后重装 -chinesesimplified.DbPageCaption=数据库 -chinesesimplified.DbPageDescription=选择内置或外置 PostgreSQL。可选粘贴已有 NETX_CREDENTIAL_SECRET_KEY；留空则自动生成。 -chinesesimplified.DbBundled=内置 PostgreSQL（便携，可离线） -chinesesimplified.DbExternal=外置 PostgreSQL（已有服务器） -chinesesimplified.DbHost=主机 -chinesesimplified.DbPort=端口 -chinesesimplified.DbUser=用户 -chinesesimplified.DbPassword=密码 -chinesesimplified.DbName=数据库名 -chinesesimplified.CredKey=凭据密钥 -chinesesimplified.CredKeyHint=NETX_CREDENTIAL_SECRET_KEY（可选，留空=自动生成） -chinesesimplified.DbFieldsRequired=请填写主机、端口、用户、密码和数据库名。 -chinesesimplified.DbTesting=正在测试 PostgreSQL 连接，请稍候… -chinesesimplified.DbTestFailed=无法用当前设置连接 PostgreSQL。%n%n%1%n%n请改正后重试。 -chinesesimplified.DbTestExtractFailed=无法解压 PostgreSQL 客户端以测试连接。 -chinesesimplified.DbSilentExternalMissing=静默安装外置库需要参数 /DbHost /DbUser /DbPassword /DbName（可选 /DbPort）。 -chinesesimplified.DbApplyFailed=文件安装后数据库自动配置失败（退出码 %1）。%n%n日志: %2%n%n请处理后使用开始菜单 → 重新配置数据库，或重新安装。 -chinesesimplified.DbApplyExecFailed=无法启动数据库配置脚本。 -chinesesimplified.DbBundledMissing=安装目录中缺少内置 PostgreSQL 文件：%n%1%n%n安装包不完整，请重新下载 NetX-Setup 后再装。 -chinesesimplified.DbEnvMissing=数据库配置未写入 %ProgramData%\NetX\.env。 -chinesesimplified.ReconfigureDb=重新配置数据库 -chinesesimplified.UpgradeKeepDb=检测到已有 NetX 数据 - 将保留数据库配置（跳过向导）。 -chinesesimplified.ModePageCaption=安装模式 -chinesesimplified.ModePageDescription=检测到已有 NetX 安装。请选择更新或重装。 -chinesesimplified.ModeUpdate=更新现有安装（保留数据库配置） -chinesesimplified.ModeReinstall=重装并重新配置数据库（与首次安装相同） -chinesesimplified.ModeUpdateHint=仅替换程序文件，保留 ProgramData\.env 与数据库数据。 -chinesesimplified.ModeReinstallHint=再次显示数据库页。不会删除 ProgramData（除非卸载时选择删除）。 [Tasks] -Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: checkedonce +Name: "desktopicon"; Description: "Create a desktop shortcut"; GroupDescription: "Additional icons:"; Flags: unchecked +Name: "firstrun"; Description: "Run first-time setup (database mode) after install"; GroupDescription: "Setup:"; Flags: checkedonce [Files] -; Entire release stage -> {app}. Exclude .portable so installed builds use ProgramData. +; Entire release stage → {app}. Exclude .portable so installed builds use ProgramData. Source: "{#StageDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs; Excludes: ".portable" -; Connection-test tools (wizard Next / silent PrepareToInstall only) -Source: "test_pg_conn.ps1"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\psql.exe"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libpq.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libssl-3-x64.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libcrypto-3-x64.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libintl-9.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libiconv-2.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libwinpthread-1.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\zlib1.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\liblz4.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libzstd.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libxml2.dll"; Flags: dontcopy -Source: "{#StageDir}\postgres\pgsql\bin\libxslt.dll"; Flags: dontcopy [Dirs] -; users-modify so tray / non-admin reconfigure can update .env and runtime files. -Name: "{commonappdata}\NetX"; Permissions: users-modify -Name: "{commonappdata}\NetX\data"; Permissions: users-modify -Name: "{commonappdata}\NetX\data\auth"; Permissions: users-modify -Name: "{commonappdata}\NetX\data\runtime"; Permissions: users-modify -Name: "{commonappdata}\NetX\pgdata"; Permissions: users-modify -Name: "{commonappdata}\NetX\backups"; Permissions: users-modify - -[InstallDelete] -; Remove legacy desktop shortcuts from older Setup builds (keep a single NetX icon). -Type: files; Name: "{commondesktop}\NetX Start.lnk" -Type: files; Name: "{commondesktop}\NetX First-time setup.lnk" -Type: files; Name: "{commondesktop}\首次配置（内置或外置数据库）.lnk" +Name: "{commonappdata}\NetX" +Name: "{commonappdata}\NetX\data" +Name: "{commonappdata}\NetX\data\auth" +Name: "{commonappdata}\NetX\data\runtime" +Name: "{commonappdata}\NetX\pgdata" +Name: "{commonappdata}\NetX\backups" [Icons] -; Prefer .cmd targets so Server 2012 / classic Start Menu shows normal program entries. -Name: "{group}\NetX Tray"; Filename: "{app}\NetX-Tray.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\Start NetX"; Filename: "{app}\NetX-Start.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\Stop NetX"; Filename: "{app}\NetX-Stop.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\Check for updates"; Filename: "powershell.exe"; Parameters: "-NoProfile -ExecutionPolicy Bypass -File ""{app}\packaging\launch_shortcut.ps1"" -Action update -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\Open NetX UI"; Filename: "http://127.0.0.1:8890/"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\{cm:ReconfigureDb}"; Filename: "{app}\NetX-FirstRun.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\Install Windows Service (admin)"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\install_service.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"" -Start"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\Enable silent auto-update (daily)"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\install_update_task.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -Name: "{group}\Enable start at logon"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\install_autostart.ps1"" -ProgramRoot ""{app}"""; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico" -; One desktop shortcut only (tray = start + tray UI). Start Menu keeps Start/Stop entries. -Name: "{commondesktop}\NetX"; Filename: "{app}\NetX-Tray.cmd"; WorkingDir: "{app}"; IconFilename: "{app}\packaging\assets\netx.ico"; Tasks: desktopicon +Name: "{group}\Start NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" +Name: "{group}\Stop NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\stop_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" +Name: "{group}\Open NetX UI"; Filename: "http://127.0.0.1:8890/" +Name: "{group}\First-time setup"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\setup_first_run.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}" +Name: "{autodesktop}\Start NetX"; Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Tasks: desktopicon [Run] -Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\netx_tray.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"" -StartOnLaunch"; WorkingDir: "{app}"; Description: "{cm:StartTrayNow}"; Flags: postinstall runhidden nowait skipifsilent; Check: NetxShouldStartTray -Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\install_autostart.ps1"" -ProgramRoot ""{app}"""; WorkingDir: "{app}"; Description: "{cm:EnableAutostart}"; Flags: postinstall runhidden skipifsilent unchecked -Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\install_update_task.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Description: "{cm:EnableAutoUpdate}"; Flags: postinstall runhidden skipifsilent unchecked -Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\install_service.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"" -Start"; WorkingDir: "{app}"; Description: "{cm:InstallService}"; Flags: postinstall runhidden skipifsilent unchecked - -; Stop runtime BEFORE files are deleted. Do not also Exec prepare from -; InitializeUninstall (that previously raced/killed unins000 or hung the UI). -[UninstallRun] -Filename: "powershell.exe"; Parameters: "-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File ""{app}\packaging\uninstall_prepare.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; RunOnceId: "StopNetX"; Flags: runhidden waituntilterminated +Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\setup_first_run.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Flags: postinstall skipifsilent; Tasks: firstrun +Filename: "powershell.exe"; Parameters: "-ExecutionPolicy Bypass -File ""{app}\packaging\start_netx_app.ps1"" -ProgramRoot ""{app}"" -DataRoot ""{commonappdata}\NetX"""; WorkingDir: "{app}"; Description: "Start NetX now"; Flags: postinstall nowait skipifsilent unchecked [UninstallDelete] -; Program files always removed. Data dir only when user chose Yes in InitializeUninstall. +; Do NOT delete {commonappdata}\NetX — preserves DB and secrets across reinstall Type: filesandordirs; Name: "{app}" - -[Code] -var - GDeleteNetxData: Boolean; - GTempDeleteDataScript: String; - DbPage: TWizardPage; - RbBundled: TNewRadioButton; - RbExternal: TNewRadioButton; - LblHost: TNewStaticText; - LblPort: TNewStaticText; - LblUser: TNewStaticText; - LblPassword: TNewStaticText; - LblDbName: TNewStaticText; - EdHost: TNewEdit; - EdPort: TNewEdit; - EdUser: TNewEdit; - EdPassword: TNewEdit; - EdDbName: TNewEdit; - LblCredKey: TNewStaticText; - LblCredHint: TNewStaticText; - EdCredKey: TNewEdit; - GDbMode: String; - GDbUrl: String; - GDbHost: String; - GDbPort: String; - GDbUser: String; - GDbPassword: String; - GDbName: String; - GCredKey: String; - GSkipDbPage: Boolean; - GIsUpgrade: Boolean; - GInstallMode: String; - ModePage: TWizardPage; - RbModeUpdate: TNewRadioButton; - RbModeReinstall: TNewRadioButton; - LblModeHint: TNewStaticText; - GDbConfigured: Boolean; - GPgToolsReady: Boolean; - GDbConnOk: Boolean; - GNextBtnCaption: String; - -function EnvHasDbMode(): Boolean; -var - Lines: TArrayOfString; - i: Integer; - Line: String; -begin - Result := False; - if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then - Exit; - for i := 0 to GetArrayLength(Lines) - 1 do - begin - Line := Trim(Lines[i]); - if Pos('NETX_DB_MODE=', Line) = 1 then - begin - if Length(Line) > Length('NETX_DB_MODE=') then - begin - Result := True; - Exit; - end; - end; - end; -end; - -function EnvModeIsExternal(): Boolean; -var - Lines: TArrayOfString; - i: Integer; - Line: String; -begin - Result := False; - if not LoadStringsFromFile(ExpandConstant('{commonappdata}\NetX\.env'), Lines) then - Exit; - for i := 0 to GetArrayLength(Lines) - 1 do - begin - Line := Trim(Lines[i]); - if CompareText(Line, 'NETX_DB_MODE=external') = 0 then - begin - Result := True; - Exit; - end; - end; -end; - -function ParamIsTruthy(const ParamName: String): Boolean; -var - V: String; -begin - V := LowerCase(Trim(ExpandConstant('{param:' + ParamName + '|}'))); - Result := (V = '1') or (V = 'true') or (V = 'yes') or (V = 'on'); -end; - -function DetectExistingConfiguredInstall(): Boolean; -begin - { Configured install = ProgramData\.env already has NETX_DB_MODE. } - Result := EnvHasDbMode(); -end; - -procedure SyncSkipDbFromInstallMode(); -begin - GSkipDbPage := (GInstallMode = 'update'); -end; - -procedure ApplyInstallModeFromChoice(); -begin - if (RbModeReinstall <> nil) and RbModeReinstall.Checked then - GInstallMode := 'reinstall' - else if GIsUpgrade then - GInstallMode := 'update' - else - GInstallMode := 'fresh'; - SyncSkipDbFromInstallMode(); -end; - -procedure ModeChoiceClick(Sender: TObject); -begin - ApplyInstallModeFromChoice(); - if LblModeHint = nil then - Exit; - if GInstallMode = 'reinstall' then - LblModeHint.Caption := ExpandConstant('{cm:ModeReinstallHint}') - else - LblModeHint.Caption := ExpandConstant('{cm:ModeUpdateHint}'); -end; - -function ResolveInstallModeFromParams(): Boolean; -{ Returns True if a CLI param forced the mode. } -var - Mode: String; -begin - Result := False; - Mode := LowerCase(Trim(ExpandConstant('{param:InstallMode|}'))); - if (Mode = 'update') or (Mode = 'upgrade') then - begin - GInstallMode := 'update'; - Result := True; - end - else if (Mode = 'reinstall') or (Mode = 'fresh') then - begin - GInstallMode := 'reinstall'; - Result := True; - end - else if ParamIsTruthy('SkipDbPage') then - begin - GInstallMode := 'update'; - Result := True; - end - else if ParamIsTruthy('ForceDbPage') then - begin - GInstallMode := 'reinstall'; - Result := True; - end; - if Result then - SyncSkipDbFromInstallMode(); -end; - -procedure StopNetxBeforeFileReplace(); -var - ResultCode: Integer; - StopScript: String; - Params: String; -begin - StopScript := ExpandConstant('{app}\packaging\stop_netx_app.ps1'); - if not FileExists(StopScript) then - Exit; - Params := - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + StopScript + '"' + - ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + - ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"' + - ' -KillTray'; - Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode); -end; - -function NetxShouldStartTray(): Boolean; -begin - Result := GDbConfigured or - FileExists(ExpandConstant('{commonappdata}\NetX\.env')); -end; - -procedure UpdateDbFieldState(); -var - Ext: Boolean; -begin - Ext := RbExternal.Checked; - LblHost.Enabled := Ext; - LblPort.Enabled := Ext; - LblUser.Enabled := Ext; - LblPassword.Enabled := Ext; - LblDbName.Enabled := Ext; - EdHost.Enabled := Ext; - EdPort.Enabled := Ext; - EdUser.Enabled := Ext; - EdPassword.Enabled := Ext; - EdDbName.Enabled := Ext; -end; - -procedure DbModeClick(Sender: TObject); -begin - UpdateDbFieldState(); -end; - -function ExtractPgClientTools(): Boolean; -begin - if GPgToolsReady and FileExists(ExpandConstant('{tmp}\psql.exe')) and - FileExists(ExpandConstant('{tmp}\test_pg_conn.ps1')) then - begin - Result := True; - Exit; - end; - Result := False; - try - ExtractTemporaryFile('test_pg_conn.ps1'); - ExtractTemporaryFile('psql.exe'); - ExtractTemporaryFile('libpq.dll'); - ExtractTemporaryFile('libssl-3-x64.dll'); - ExtractTemporaryFile('libcrypto-3-x64.dll'); - ExtractTemporaryFile('libintl-9.dll'); - ExtractTemporaryFile('libiconv-2.dll'); - ExtractTemporaryFile('libwinpthread-1.dll'); - ExtractTemporaryFile('zlib1.dll'); - ExtractTemporaryFile('liblz4.dll'); - ExtractTemporaryFile('libzstd.dll'); - ExtractTemporaryFile('libxml2.dll'); - ExtractTemporaryFile('libxslt.dll'); - Result := FileExists(ExpandConstant('{tmp}\psql.exe')) and - FileExists(ExpandConstant('{tmp}\test_pg_conn.ps1')); - GPgToolsReady := Result; - except - Result := False; - GPgToolsReady := False; - end; -end; - -procedure SetWizardBusy(const Busy: Boolean; const StatusText: String); -begin - if Busy then - begin - if GNextBtnCaption = '' then - GNextBtnCaption := WizardForm.NextButton.Caption; - WizardForm.NextButton.Caption := StatusText; - WizardForm.NextButton.Enabled := False; - WizardForm.BackButton.Enabled := False; - WizardForm.CancelButton.Enabled := False; - end - else - begin - if GNextBtnCaption <> '' then - WizardForm.NextButton.Caption := GNextBtnCaption; - WizardForm.NextButton.Enabled := True; - WizardForm.BackButton.Enabled := True; - WizardForm.CancelButton.Enabled := True; - end; - WizardForm.Refresh; -end; - -function TestExternalDb(const Host, Port, User, Password, DbName: String; var ErrMsg: String): Boolean; -var - ResultCode: Integer; - UrlFile: String; - ErrFile: String; - PwFile: String; - Params: String; - Lines: TArrayOfString; - i: Integer; -begin - Result := False; - ErrMsg := ''; - if not ExtractPgClientTools() then - begin - ErrMsg := ExpandConstant('{cm:DbTestExtractFailed}'); - Exit; - end; - - UrlFile := ExpandConstant('{tmp}\netx_db_url.txt'); - ErrFile := ExpandConstant('{tmp}\netx_db_test_err.txt'); - PwFile := ExpandConstant('{tmp}\netx_db_pw.txt'); - DeleteFile(UrlFile); - DeleteFile(ErrFile); - SaveStringToFile(PwFile, Password, False); - - Params := - '-NoProfile -ExecutionPolicy Bypass -File "' + ExpandConstant('{tmp}\test_pg_conn.ps1') + '"' + - ' -PgHost "' + Host + '"' + - ' -Port ' + Port + - ' -User "' + User + '"' + - ' -PasswordFile "' + PwFile + '"' + - ' -Database "' + DbName + '"' + - ' -PsqlPath "' + ExpandConstant('{tmp}\psql.exe') + '"' + - ' -OutUrlFile "' + UrlFile + '"' + - ' -OutErrFile "' + ErrFile + '"'; - - if not Exec('powershell.exe', Params, ExpandConstant('{tmp}'), SW_HIDE, ewWaitUntilTerminated, ResultCode) then - begin - ErrMsg := 'powershell_exec_failed'; - DeleteFile(PwFile); - Exit; - end; - DeleteFile(PwFile); - if ResultCode <> 0 then - begin - ErrMsg := 'psql/test exit ' + IntToStr(ResultCode); - if LoadStringsFromFile(ErrFile, Lines) then - begin - ErrMsg := ''; - for i := 0 to GetArrayLength(Lines) - 1 do - begin - if ErrMsg <> '' then - ErrMsg := ErrMsg + #13#10; - ErrMsg := ErrMsg + Lines[i]; - end; - if ErrMsg = '' then - ErrMsg := 'psql/test exit ' + IntToStr(ResultCode); - end; - Exit; - end; - if not LoadStringsFromFile(UrlFile, Lines) or (GetArrayLength(Lines) < 1) or (Trim(Lines[0]) = '') then - begin - ErrMsg := 'url_file_missing'; - Exit; - end; - GDbUrl := Trim(Lines[0]); - Result := True; -end; - -function SaveExternalFieldsFromControls(var ErrMsg: String): Boolean; -begin - Result := False; - GDbHost := Trim(EdHost.Text); - GDbPort := Trim(EdPort.Text); - GDbUser := Trim(EdUser.Text); - GDbPassword := EdPassword.Text; - GDbName := Trim(EdDbName.Text); - GDbConnOk := False; - GDbUrl := ''; - if (GDbHost = '') or (GDbPort = '') or (GDbUser = '') or - (GDbPassword = '') or (GDbName = '') then - begin - ErrMsg := ExpandConstant('{cm:DbFieldsRequired}'); - Exit; - end; - GDbMode := 'external'; - Result := True; -end; - -function ValidateExternalDbSaved(var ErrMsg: String): Boolean; -begin - Result := False; - if GDbConnOk and (GDbUrl <> '') then - begin - Result := True; - Exit; - end; - if not TestExternalDb(GDbHost, GDbPort, GDbUser, GDbPassword, GDbName, ErrMsg) then - begin - if ErrMsg = '' then - ErrMsg := 'connection_failed'; - ErrMsg := FmtMessage(ExpandConstant('{cm:DbTestFailed}'), [ErrMsg]); - GDbConnOk := False; - Exit; - end; - GDbConnOk := True; - Result := True; -end; - -function ApplySilentDbParams(var ErrMsg: String): Boolean; -var - Mode, Host, Port, User, Password, DbName: String; -begin - Result := False; - GCredKey := Trim(ExpandConstant('{param:CredentialSecretKey|}')); - Mode := LowerCase(ExpandConstant('{param:DbMode|bundled}')); - if Mode = '' then - Mode := 'bundled'; - if Mode = 'bundled' then - begin - GDbMode := 'bundled'; - GDbUrl := ''; - Result := True; - Exit; - end; - if Mode <> 'external' then - begin - ErrMsg := 'invalid DbMode (use bundled or external)'; - Exit; - end; - Host := ExpandConstant('{param:DbHost|}'); - Port := ExpandConstant('{param:DbPort|5432}'); - User := ExpandConstant('{param:DbUser|}'); - Password := ExpandConstant('{param:DbPassword|}'); - DbName := ExpandConstant('{param:DbName|}'); - if (Host = '') or (User = '') or (Password = '') or (DbName = '') then - begin - ErrMsg := ExpandConstant('{cm:DbSilentExternalMissing}'); - Exit; - end; - if not TestExternalDb(Host, Port, User, Password, DbName, ErrMsg) then - begin - if ErrMsg = '' then - ErrMsg := 'connection_failed'; - ErrMsg := FmtMessage(ExpandConstant('{cm:DbTestFailed}'), [ErrMsg]); - Exit; - end; - GDbMode := 'external'; - Result := True; -end; - -procedure CaptureCredKeyFromControls(); -begin - if EdCredKey <> nil then - GCredKey := Trim(EdCredKey.Text) - else - GCredKey := ''; -end; - -procedure InitializeWizard(); -var - TopPos: Integer; -begin - GDbMode := 'bundled'; - GDbUrl := ''; - GDbHost := ''; - GDbPort := '5432'; - GDbUser := ''; - GDbPassword := ''; - GDbName := ''; - GCredKey := ''; - GDbConfigured := False; - GPgToolsReady := False; - GDbConnOk := False; - { Existing install: interactive Mode page (Update vs Reinstall). - Silent: /InstallMode=update|reinstall (or /SkipDbPage=1 /ForceDbPage=1). } - GIsUpgrade := DetectExistingConfiguredInstall(); - ModePage := nil; - RbModeUpdate := nil; - RbModeReinstall := nil; - LblModeHint := nil; - if not ResolveInstallModeFromParams() then - begin - if GIsUpgrade then - GInstallMode := 'update' - else - GInstallMode := 'fresh'; - SyncSkipDbFromInstallMode(); - end; - - if GIsUpgrade and (not WizardSilent) then - begin - ModePage := CreateCustomPage(wpSelectDir, - ExpandConstant('{cm:ModePageCaption}'), - ExpandConstant('{cm:ModePageDescription}')); - - RbModeUpdate := TNewRadioButton.Create(ModePage); - RbModeUpdate.Parent := ModePage.Surface; - RbModeUpdate.Caption := ExpandConstant('{cm:ModeUpdate}'); - RbModeUpdate.Checked := (GInstallMode <> 'reinstall'); - RbModeUpdate.Top := ScaleY(8); - RbModeUpdate.Left := ScaleX(0); - RbModeUpdate.Width := ModePage.SurfaceWidth; - RbModeUpdate.OnClick := @ModeChoiceClick; - - RbModeReinstall := TNewRadioButton.Create(ModePage); - RbModeReinstall.Parent := ModePage.Surface; - RbModeReinstall.Caption := ExpandConstant('{cm:ModeReinstall}'); - RbModeReinstall.Checked := (GInstallMode = 'reinstall'); - RbModeReinstall.Top := RbModeUpdate.Top + ScaleY(28); - RbModeReinstall.Left := ScaleX(0); - RbModeReinstall.Width := ModePage.SurfaceWidth; - RbModeReinstall.OnClick := @ModeChoiceClick; - - LblModeHint := TNewStaticText.Create(ModePage); - LblModeHint.Parent := ModePage.Surface; - LblModeHint.Top := RbModeReinstall.Top + ScaleY(36); - LblModeHint.Left := ScaleX(0); - LblModeHint.Width := ModePage.SurfaceWidth; - LblModeHint.Height := ScaleY(60); - LblModeHint.AutoSize := False; - LblModeHint.WordWrap := True; - ModeChoiceClick(nil); - - DbPage := CreateCustomPage(ModePage.ID, - ExpandConstant('{cm:DbPageCaption}'), - ExpandConstant('{cm:DbPageDescription}')); - end - else - DbPage := CreateCustomPage(wpSelectDir, - ExpandConstant('{cm:DbPageCaption}'), - ExpandConstant('{cm:DbPageDescription}')); - - RbBundled := TNewRadioButton.Create(DbPage); - RbBundled.Parent := DbPage.Surface; - RbBundled.Caption := ExpandConstant('{cm:DbBundled}'); - RbBundled.Checked := True; - RbBundled.Top := ScaleY(8); - RbBundled.Left := ScaleX(0); - RbBundled.Width := DbPage.SurfaceWidth; - RbBundled.OnClick := @DbModeClick; - - RbExternal := TNewRadioButton.Create(DbPage); - RbExternal.Parent := DbPage.Surface; - RbExternal.Caption := ExpandConstant('{cm:DbExternal}'); - RbExternal.Top := RbBundled.Top + ScaleY(28); - RbExternal.Left := ScaleX(0); - RbExternal.Width := DbPage.SurfaceWidth; - RbExternal.OnClick := @DbModeClick; - - TopPos := RbExternal.Top + ScaleY(36); - - LblHost := TNewStaticText.Create(DbPage); - LblHost.Parent := DbPage.Surface; - LblHost.Caption := ExpandConstant('{cm:DbHost}'); - LblHost.Top := TopPos; - LblHost.Left := ScaleX(16); - - EdHost := TNewEdit.Create(DbPage); - EdHost.Parent := DbPage.Surface; - EdHost.Top := TopPos; - EdHost.Left := ScaleX(120); - EdHost.Width := ScaleX(220); - EdHost.Text := '127.0.0.1'; - - TopPos := TopPos + ScaleY(28); - LblPort := TNewStaticText.Create(DbPage); - LblPort.Parent := DbPage.Surface; - LblPort.Caption := ExpandConstant('{cm:DbPort}'); - LblPort.Top := TopPos; - LblPort.Left := ScaleX(16); - - EdPort := TNewEdit.Create(DbPage); - EdPort.Parent := DbPage.Surface; - EdPort.Top := TopPos; - EdPort.Left := ScaleX(120); - EdPort.Width := ScaleX(80); - EdPort.Text := '5432'; - - TopPos := TopPos + ScaleY(28); - LblUser := TNewStaticText.Create(DbPage); - LblUser.Parent := DbPage.Surface; - LblUser.Caption := ExpandConstant('{cm:DbUser}'); - LblUser.Top := TopPos; - LblUser.Left := ScaleX(16); - - EdUser := TNewEdit.Create(DbPage); - EdUser.Parent := DbPage.Surface; - EdUser.Top := TopPos; - EdUser.Left := ScaleX(120); - EdUser.Width := ScaleX(220); - EdUser.Text := 'netx'; - - TopPos := TopPos + ScaleY(28); - LblPassword := TNewStaticText.Create(DbPage); - LblPassword.Parent := DbPage.Surface; - LblPassword.Caption := ExpandConstant('{cm:DbPassword}'); - LblPassword.Top := TopPos; - LblPassword.Left := ScaleX(16); - - EdPassword := TNewEdit.Create(DbPage); - EdPassword.Parent := DbPage.Surface; - EdPassword.Top := TopPos; - EdPassword.Left := ScaleX(120); - EdPassword.Width := ScaleX(220); - EdPassword.PasswordChar := '*'; - - TopPos := TopPos + ScaleY(28); - LblDbName := TNewStaticText.Create(DbPage); - LblDbName.Parent := DbPage.Surface; - LblDbName.Caption := ExpandConstant('{cm:DbName}'); - LblDbName.Top := TopPos; - LblDbName.Left := ScaleX(16); - - EdDbName := TNewEdit.Create(DbPage); - EdDbName.Parent := DbPage.Surface; - EdDbName.Top := TopPos; - EdDbName.Left := ScaleX(120); - EdDbName.Width := ScaleX(220); - EdDbName.Text := 'netx'; - - TopPos := TopPos + ScaleY(36); - LblCredKey := TNewStaticText.Create(DbPage); - LblCredKey.Parent := DbPage.Surface; - LblCredKey.Caption := ExpandConstant('{cm:CredKey}'); - LblCredKey.Top := TopPos; - LblCredKey.Left := ScaleX(0); - - EdCredKey := TNewEdit.Create(DbPage); - EdCredKey.Parent := DbPage.Surface; - EdCredKey.Top := TopPos; - EdCredKey.Left := ScaleX(120); - EdCredKey.Width := DbPage.SurfaceWidth - ScaleX(120); - EdCredKey.Text := ''; - - TopPos := TopPos + ScaleY(24); - LblCredHint := TNewStaticText.Create(DbPage); - LblCredHint.Parent := DbPage.Surface; - LblCredHint.Caption := ExpandConstant('{cm:CredKeyHint}'); - LblCredHint.Top := TopPos; - LblCredHint.Left := ScaleX(120); - LblCredHint.Width := DbPage.SurfaceWidth - ScaleX(120); - LblCredHint.AutoSize := False; - - UpdateDbFieldState(); - - { Prefill from existing ProgramData\.env when present. } - if EnvModeIsExternal() then - begin - RbExternal.Checked := True; - UpdateDbFieldState(); - end - else if EnvHasDbMode() then - begin - RbBundled.Checked := True; - UpdateDbFieldState(); - end; - - if WizardSilent then - begin - { Params applied in PrepareToInstall } - end; -end; - -function ShouldSkipPage(PageID: Integer): Boolean; -begin - Result := False; - if (ModePage <> nil) and (PageID = ModePage.ID) then - begin - { Only interactive existing installs show the mode page. } - Result := (not GIsUpgrade) or WizardSilent; - Exit; - end; - if (DbPage <> nil) and (PageID = DbPage.ID) then - Result := GSkipDbPage; -end; - -function NextButtonClick(CurPageID: Integer): Boolean; -var - ErrMsg: String; -begin - Result := True; - if (ModePage <> nil) and (CurPageID = ModePage.ID) then - begin - ApplyInstallModeFromChoice(); - Exit; - end; - { Only the DB page runs validation. Other pages must always proceed - (Tasks/Ready used to appear "dead" when CloseApplications hung). } - if (DbPage = nil) or (CurPageID <> DbPage.ID) or GSkipDbPage then - Exit; - - CaptureCredKeyFromControls(); - if RbBundled.Checked then - begin - GDbMode := 'bundled'; - GDbUrl := ''; - GDbConnOk := True; - Result := True; - Exit; - end; - - if not SaveExternalFieldsFromControls(ErrMsg) then - begin - MsgBox(ErrMsg, mbError, MB_OK); - Result := False; - Exit; - end; - - { Connection test here so bad settings never reach Tasks/Ready. - Show busy state — ExtractTemporaryFile + psql can take a few seconds. } - SetWizardBusy(True, ExpandConstant('{cm:DbTesting}')); - try - Result := ValidateExternalDbSaved(ErrMsg); - finally - SetWizardBusy(False, ''); - end; - if not Result then - MsgBox(ErrMsg, mbError, MB_OK); -end; - -function PrepareToInstall(var NeedsRestart: Boolean): String; -var - ErrMsg: String; -begin - Result := ''; - NeedsRestart := False; - - if (ModePage <> nil) and (not WizardSilent) then - ApplyInstallModeFromChoice(); - - { Stop tray / API / bundled PG so Program Files can be replaced safely. } - if GSkipDbPage or GIsUpgrade or DirExists(ExpandConstant('{app}\packaging')) then - StopNetxBeforeFileReplace(); - - if GSkipDbPage then - Exit; - - if WizardSilent then - begin - if not ApplySilentDbParams(ErrMsg) then - Result := ErrMsg; - Exit; - end; - - if GDbMode = '' then - GDbMode := 'bundled'; - - { Re-check only if user never got a successful test (should be rare). } - if (GDbMode = 'external') and (not GDbConnOk) then - begin - if not ValidateExternalDbSaved(ErrMsg) then - Result := ErrMsg; - end; -end; - -function ApplyDatabaseConfig(): Boolean; -var - ResultCode: Integer; - Params: String; - UrlFile: String; - KeyFile: String; - LogFile: String; - InitDb: String; - EnvFile: String; - Wrapper: String; - WrapBody: String; -begin - Result := False; - GDbConfigured := False; - if GDbMode = '' then - GDbMode := 'bundled'; - - InitDb := ExpandConstant('{app}\postgres\pgsql\bin\initdb.exe'); - if (GDbMode = 'bundled') and (not FileExists(InitDb)) then - begin - MsgBox(FmtMessage(ExpandConstant('{cm:DbBundledMissing}'), [InitDb]), mbError, MB_OK); - Exit; - end; - - ForceDirectories(ExpandConstant('{commonappdata}\NetX\data\runtime')); - LogFile := ExpandConstant('{commonappdata}\NetX\data\runtime\setup_first_run.log'); - Wrapper := ExpandConstant('{tmp}\netx_apply_db.ps1'); - - WrapBody := - '$ErrorActionPreference = ''Stop''' + #13#10 + - '$log = ''' + LogFile + '''' + #13#10 + - 'Start-Transcript -Path $log -Force | Out-Null' + #13#10 + - 'try {' + #13#10 + - ' & ''' + ExpandConstant('{app}\packaging\setup_first_run.ps1') + '''' + - ' -ProgramRoot ''' + ExpandConstant('{app}') + '''' + - ' -DataRoot ''' + ExpandConstant('{commonappdata}\NetX') + '''' + - ' -NonInteractive -DbMode ' + GDbMode; - - if GDbMode = 'external' then - begin - UrlFile := ExpandConstant('{tmp}\netx_db_url.txt'); - if GDbUrl <> '' then - SaveStringToFile(UrlFile, GDbUrl, False); - WrapBody := WrapBody + - ' -ExternalDatabaseUrlFile ''' + UrlFile + ''' -SkipExternalProbe'; - end; - - if GCredKey <> '' then - begin - KeyFile := ExpandConstant('{tmp}\netx_cred_key.txt'); - SaveStringToFile(KeyFile, GCredKey, False); - WrapBody := WrapBody + ' -CredentialSecretKeyFile ''' + KeyFile + ''''; - end; - - WrapBody := WrapBody + #13#10 + - ' if ($null -ne $LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE }' + #13#10 + - '} catch {' + #13#10 + - ' Write-Error $_' + #13#10 + - ' exit 1' + #13#10 + - '} finally {' + #13#10 + - ' Stop-Transcript | Out-Null' + #13#10 + - '}' + #13#10; - - SaveStringToFile(Wrapper, WrapBody, False); - Params := '-NoProfile -ExecutionPolicy Bypass -File "' + Wrapper + '"'; - - if not Exec( - 'powershell.exe', Params, ExpandConstant('{app}'), - SW_HIDE, ewWaitUntilTerminated, ResultCode) then - begin - MsgBox(ExpandConstant('{cm:DbApplyExecFailed}'), mbError, MB_OK); - Exit; - end; - if ResultCode <> 0 then - begin - MsgBox(FmtMessage(ExpandConstant('{cm:DbApplyFailed}'), [IntToStr(ResultCode), LogFile]), mbError, MB_OK); - Exit; - end; - - EnvFile := ExpandConstant('{commonappdata}\NetX\.env'); - if not FileExists(EnvFile) then - begin - MsgBox(ExpandConstant('{cm:DbEnvMissing}'), mbError, MB_OK); - Exit; - end; - - GDbConfigured := True; - Result := True; -end; - -function InitializeUninstall(): Boolean; -var - DeleteSrc: String; -begin - Result := True; - GDeleteNetxData := False; - GTempDeleteDataScript := ExpandConstant('{tmp}\netx_uninstall_delete_data.ps1'); - - // Copy data-wipe helper to TEMP before program files are removed. - DeleteSrc := ExpandConstant('{app}\packaging\uninstall_delete_data.ps1'); - if FileExists(DeleteSrc) then - CopyFile(DeleteSrc, GTempDeleteDataScript, False); - - if UninstallSilent then - begin - // Settings often uses QuietUninstallString; keep data by default. - GDeleteNetxData := False; - end - else - begin - GDeleteNetxData := - MsgBox(ExpandConstant('{cm:UninstallDeleteData}'), mbConfirmation, MB_YESNO) = IDYES; - end; -end; - -procedure CurUninstallStepChanged(CurUninstallStep: TUninstallStep); -var - ResultCode: Integer; - DataRoot: String; -begin - if (CurUninstallStep = usPostUninstall) and GDeleteNetxData then - begin - DataRoot := ExpandConstant('{commonappdata}\NetX'); - if FileExists(GTempDeleteDataScript) then - begin - Exec( - 'powershell.exe', - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + - GTempDeleteDataScript + '" -DataRoot "' + DataRoot + '"', - ExpandConstant('{tmp}'), - SW_HIDE, - ewWaitUntilTerminated, - ResultCode - ); - end; - if DirExists(DataRoot) then - DelTree(DataRoot, True, True, True); - end; -end; - -procedure RepairNetxVenvAfterInstall(); -var - ResultCode: Integer; - RepairScript: String; - Params: String; -begin - RepairScript := ExpandConstant('{app}\packaging\repair_venv.ps1'); - if not FileExists(RepairScript) then - Exit; - Params := - '-NoProfile -WindowStyle Hidden -ExecutionPolicy Bypass -File "' + RepairScript + '"' + - ' -ProgramRoot "' + ExpandConstant('{app}') + '"' + - ' -DataRoot "' + ExpandConstant('{commonappdata}\NetX') + '"'; - Exec('powershell.exe', Params, ExpandConstant('{app}'), SW_HIDE, ewWaitUntilTerminated, ResultCode); -end; - -procedure CurStepChanged(CurStep: TSetupStep); -var - UninstallKey: String; -begin - if CurStep = ssPostInstall then - begin - { Always relink .venv -> local python/runtime (Setup is elevated). } - RepairNetxVenvAfterInstall(); - - { Upgrade keeps existing .env / pgdata - do not re-run DB wizard. } - if not GSkipDbPage then - ApplyDatabaseConfig() - else - GDbConfigured := FileExists(ExpandConstant('{commonappdata}\NetX\.env')); - - UninstallKey := - 'Software\Microsoft\Windows\CurrentVersion\Uninstall\{A7E3C2D1-9F40-4B8E-9C1A-NETXWIN64001}_is1'; - RegDeleteValue(HKLM, UninstallKey, 'QuietUninstallString'); - end; -end; diff --git a/packaging/installer/test_pg_conn.ps1 b/packaging/installer/test_pg_conn.ps1 deleted file mode 100644 index efc8323..0000000 --- a/packaging/installer/test_pg_conn.ps1 +++ /dev/null @@ -1,104 +0,0 @@ -param( - [Parameter(Mandatory = $true)][string]$PgHost, - [int]$Port = 5432, - [Parameter(Mandatory = $true)][string]$User, - [string]$Password = "", - [string]$PasswordFile = "", - [Parameter(Mandatory = $true)][string]$Database, - [Parameter(Mandatory = $true)][string]$PsqlPath, - [string]$OutUrlFile = "", - [string]$OutErrFile = "" -) - -$ErrorActionPreference = "Stop" - -function Write-ErrFile([string]$Message) { - if ($OutErrFile) { - Set-Content -LiteralPath $OutErrFile -Value $Message -Encoding utf8 - } - [Console]::Error.WriteLine($Message) -} - -if (-not (Test-Path -LiteralPath $PsqlPath)) { - Write-ErrFile "psql_not_found: $PsqlPath" - exit 2 -} -if ($PasswordFile) { - if (-not (Test-Path -LiteralPath $PasswordFile)) { - Write-ErrFile "password_file_missing" - exit 3 - } - $Password = [IO.File]::ReadAllText($PasswordFile).TrimEnd("`r", "`n") -} -if ([string]::IsNullOrWhiteSpace($PgHost)) { - Write-ErrFile "host_required" - exit 3 -} -if ([string]::IsNullOrWhiteSpace($User) -or [string]::IsNullOrWhiteSpace($Database)) { - Write-ErrFile "user_and_database_required" - exit 3 -} -if ([string]::IsNullOrWhiteSpace($Password)) { - Write-ErrFile "password_required" - exit 3 -} -if ($Port -lt 1 -or $Port -gt 65535) { - Write-ErrFile "invalid_port: $Port" - exit 3 -} - -$encUser = [uri]::EscapeDataString($User) -$encPw = [uri]::EscapeDataString($Password) -$encDb = [uri]::EscapeDataString($Database) -$url = "postgresql+psycopg://${encUser}:${encPw}@${PgHost}:${Port}/${encDb}" - -$psqlDir = Split-Path -Parent $PsqlPath -$prevPath = $env:PATH -$prevPw = $env:PGPASSWORD -try { - $env:PATH = "$psqlDir;$env:PATH" - $env:PGPASSWORD = $Password - $psi = New-Object System.Diagnostics.ProcessStartInfo - $psi.FileName = $PsqlPath - $psi.Arguments = "-h `"$PgHost`" -p $Port -U `"$User`" -d `"$Database`" -v ON_ERROR_STOP=1 -t -A -c `"SELECT 1`"" - $psi.UseShellExecute = $false - $psi.RedirectStandardOutput = $true - $psi.RedirectStandardError = $true - $psi.CreateNoWindow = $true - $psi.WorkingDirectory = $psqlDir - $p = [Diagnostics.Process]::Start($psi) - $stdout = $p.StandardOutput.ReadToEnd() - $stderr = $p.StandardError.ReadToEnd() - $p.WaitForExit() - if ($p.ExitCode -ne 0) { - $msg = (($stderr + "`n" + $stdout).Trim()) - if (-not $msg) { $msg = "psql_exit_$($p.ExitCode)" } - Write-ErrFile "connection_failed: $msg" - exit 1 - } - if (($stdout.Trim()) -notmatch '1') { - Write-ErrFile "unexpected_result: $($stdout.Trim())" - exit 1 - } -} catch { - Write-ErrFile ("connection_failed: " + $_.Exception.Message) - exit 1 -} finally { - $env:PATH = $prevPath - if ($null -eq $prevPw) { - Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue - } else { - $env:PGPASSWORD = $prevPw - } -} - -if ($OutUrlFile) { - $dir = Split-Path -Parent $OutUrlFile - if ($dir -and -not (Test-Path -LiteralPath $dir)) { - New-Item -ItemType Directory -Path $dir -Force | Out-Null - } - Set-Content -LiteralPath $OutUrlFile -Value $url -Encoding ascii -NoNewline -} - -Write-Output "ok" -exit 0 diff --git a/packaging/launch_shortcut.ps1 b/packaging/launch_shortcut.ps1 deleted file mode 100644 index 7f13188..0000000 --- a/packaging/launch_shortcut.ps1 +++ /dev/null @@ -1,128 +0,0 @@ -param( - [ValidateSet("tray", "start", "stop", "setup", "update")] - [string]$Action = "tray", - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [switch]$StartOnLaunch = $true -) - -# Visible entrypoint for Start Menu / desktop shortcuts. -# Nested PowerShell runs are hidden; only failures show a message box. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden - -Add-Type -AssemblyName System.Windows.Forms - -$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') -function T([string]$En, [string]$Zh) { - if ($zh) { return $Zh } else { return $En } -} - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$logDir = Join-Path $data "data\runtime" -if (-not (Test-Path $logDir)) { - New-Item -ItemType Directory -Path $logDir -Force | Out-Null -} -$log = Join-Path $logDir "launch.log" - -function Write-LaunchLog([string]$Msg) { - $line = "[{0}] {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Msg - Add-Content -Path $log -Value $line -Encoding utf8 -} - -function Show-NetxError([string]$Msg) { - Write-LaunchLog "ERROR $Msg" - [void][System.Windows.Forms.MessageBox]::Show( - $Msg, - "NetX", - [System.Windows.Forms.MessageBoxButtons]::OK, - [System.Windows.Forms.MessageBoxIcon]::Error - ) -} - -function Invoke-NetxHiddenPs1 { - param( - [string]$File, - [string[]]$ExtraArgs = @(), - [switch]$NoWait = $false - ) - return Start-NetxPowerShell -File $File ` - -Arguments (@("-ProgramRoot", $prog, "-DataRoot", $data) + $ExtraArgs) ` - -WorkingDirectory $prog -WindowStyle Hidden -PassThru -Wait:(-not $NoWait) -} - -function Ensure-NetxEnv { - $envPath = Join-Path $data ".env" - if (Test-Path $envPath) { return } - $cmd = Join-Path $prog "NetX-FirstRun.cmd" - throw (T ` - "NetX is not configured yet (missing $envPath).`nRe-run Setup and choose built-in or external DB,`nor Start Menu → Reconfigure database:`n$cmd" ` - "尚未完成数据库配置（缺少 $envPath）。`n请重新运行安装向导选择内置/外置库，`n或开始菜单 → 重新配置数据库：`n$cmd") -} - -try { - Write-LaunchLog "action=$Action prog=$prog data=$data" - switch ($Action) { - "setup" { - # Visible console so user can pick bundled vs external DB. - $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot "setup_first_run.ps1") ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` - -WorkingDirectory $prog -WindowStyle Normal -Wait -PassThru - if ($p.ExitCode -ne 0) { throw "setup_exit_$($p.ExitCode)" } - if (Test-Path (Join-Path $data ".env")) { - Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-StartOnLaunch") ` - -WorkingDirectory $prog -WindowStyle Hidden | Out-Null - } - } - "stop" { - $p = Invoke-NetxHiddenPs1 -File (Join-Path $PSScriptRoot "stop_netx_app.ps1") - if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0) { throw "stop_exit_$($p.ExitCode)" } - } - "update" { - Ensure-NetxEnv - # Update UI may need a visible window for prompts. - $p = Start-NetxPowerShell -File (Join-Path $PSScriptRoot "check_update.ps1") ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` - -WorkingDirectory $prog -WindowStyle Normal -Wait -PassThru - if ($null -ne $p.ExitCode -and $p.ExitCode -ne 0 -and $p.ExitCode -ne 10) { - throw "update_exit_$($p.ExitCode)" - } - } - "start" { - Ensure-NetxEnv - $hostBind = "127.0.0.1" - $port = 8890 - $envPath = Join-Path $data ".env" - if (Test-Path $envPath) { - $map = Read-DotEnv -Path $envPath - if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } - if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } - } - $p = Invoke-NetxHiddenPs1 -File (Join-Path $PSScriptRoot "start_netx_app.ps1") -ExtraArgs @("-SkipBrowser") - if ($p.ExitCode -ne 0) { throw "start_exit_$($p.ExitCode)" } - if (Wait-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 180) { - try { Start-Process "http://${hostBind}:${port}/" } catch {} - } else { - throw (T ` - "NetX started but /health not ready within 180s.`nSee: $logDir\netx.err.log" ` - "NetX 已启动但 /health 在 180 秒内未就绪。`n请查看: $logDir\netx.err.log") - } - } - "tray" { - Ensure-NetxEnv - $extra = @("-ProgramRoot", $prog, "-DataRoot", $data) - if ($StartOnLaunch) { $extra += "-StartOnLaunch" } - # Detach tray; do not leave a success MessageBox (keeps a console open). - Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` - -Arguments $extra -WorkingDirectory $prog -WindowStyle Hidden | Out-Null - } - } - Write-LaunchLog "action=$Action ok" -} catch { - Show-NetxError ((T "NetX failed to start:`n$($_.Exception.Message)" "NetX 启动失败：`n$($_.Exception.Message)") + "`n`n$log") - exit 1 -} diff --git a/packaging/manifest.example.json b/packaging/manifest.example.json index e8748ef..9f51c14 100644 --- a/packaging/manifest.example.json +++ b/packaging/manifest.example.json @@ -1,13 +1,11 @@ { "channel": "stable", - "latest": "0.4.13", + "latest": "0.3.0", "min_compatible": "0.3.0", - "notes_url": "https://github.com/hansjone/netx/releases/tag/v0.4.13", + "notes_url": "", "windows": { - "url": "https://github.com/hansjone/netx/releases/download/v0.4.13/NetX-Setup-0.4.13.exe", - "setup_url": "https://github.com/hansjone/netx/releases/download/v0.4.13/NetX-Setup-0.4.13.exe", - "package": "setup", - "sha256": "", + "url": "https://example.com/netx/NetX-0.2.0-win64.zip", + "sha256": "REPLACE_WITH_SHA256", "size": 0 } } diff --git a/packaging/netx_tray.ps1 b/packaging/netx_tray.ps1 deleted file mode 100644 index bffcc9c..0000000 --- a/packaging/netx_tray.ps1 +++ /dev/null @@ -1,749 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [switch]$StartOnLaunch = $true, - [switch]$CheckUpdateOnLaunch = $false, - [switch]$AutoUpdate = $false -) - -# System-tray controller for NetX (Windows packaged installs). -# Always elevated: Start/Stop/Update touch Program Files + services. - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden - -Add-Type -AssemblyName System.Windows.Forms -Add-Type -AssemblyName System.Drawing - -# Custom tray menu: hover/selected highlight + flat light chrome (default OS menu looks dead). -if (-not ("NetxMenuRenderer" -as [type])) { - Add-Type -ReferencedAssemblies System.Windows.Forms, System.Drawing -TypeDefinition @" -using System.Drawing; -using System.Drawing.Drawing2D; -using System.Windows.Forms; - -public sealed class NetxMenuColorTable : ProfessionalColorTable { - static readonly Color Hover = Color.FromArgb(232, 240, 252); - static readonly Color Press = Color.FromArgb(214, 228, 248); - static readonly Color Border = Color.FromArgb(170, 198, 240); - static readonly Color Edge = Color.FromArgb(210, 214, 220); - static readonly Color Sep = Color.FromArgb(228, 230, 235); - static readonly Color Bg = Color.FromArgb(252, 252, 253); - - public override Color MenuItemSelected { get { return Hover; } } - public override Color MenuItemSelectedGradientBegin { get { return Hover; } } - public override Color MenuItemSelectedGradientEnd { get { return Hover; } } - public override Color MenuItemBorder { get { return Border; } } - public override Color MenuItemPressedGradientBegin { get { return Press; } } - public override Color MenuItemPressedGradientMiddle { get { return Press; } } - public override Color MenuItemPressedGradientEnd { get { return Press; } } - public override Color MenuBorder { get { return Edge; } } - public override Color ToolStripDropDownBackground { get { return Bg; } } - public override Color ImageMarginGradientBegin { get { return Bg; } } - public override Color ImageMarginGradientMiddle { get { return Bg; } } - public override Color ImageMarginGradientEnd { get { return Bg; } } - public override Color SeparatorDark { get { return Sep; } } - public override Color SeparatorLight { get { return Sep; } } -} - -public sealed class NetxMenuRenderer : ToolStripProfessionalRenderer { - public NetxMenuRenderer() : base(new NetxMenuColorTable()) { - RoundedEdges = false; - } - - protected override void OnRenderMenuItemBackground(ToolStripItemRenderEventArgs e) { - ToolStripMenuItem item = e.Item as ToolStripMenuItem; - if (item == null || !item.Selected || !item.Enabled) { - base.OnRenderMenuItemBackground(e); - return; - } - Rectangle r = new Rectangle(2, 0, e.Item.Width - 4, e.Item.Height); - using (SolidBrush brush = new SolidBrush(Color.FromArgb(232, 240, 252))) - using (Pen pen = new Pen(Color.FromArgb(170, 198, 240))) { - e.Graphics.SmoothingMode = SmoothingMode.AntiAlias; - e.Graphics.FillRectangle(brush, r); - e.Graphics.DrawRectangle(pen, r.X, r.Y, r.Width - 1, r.Height - 1); - } - } - - protected override void OnRenderToolStripBorder(ToolStripRenderEventArgs e) { - Rectangle r = new Rectangle(0, 0, e.AffectedBounds.Width - 1, e.AffectedBounds.Height - 1); - using (Pen pen = new Pen(Color.FromArgb(210, 214, 220))) { - e.Graphics.DrawRectangle(pen, r); - } - } - - protected override void OnRenderSeparator(ToolStripSeparatorRenderEventArgs e) { - int y = e.Item.Height / 2; - using (Pen pen = new Pen(Color.FromArgb(228, 230, 235))) { - e.Graphics.DrawLine(pen, 10, y, e.Item.Width - 10, y); - } - } -} -"@ -} - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$hostBind = "127.0.0.1" -$port = 8890 -$envPath = Join-Path $data ".env" -$map = @{} -if (Test-Path $envPath) { - $map = Read-DotEnv -Path $envPath - if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } - if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } -} -$autoVal = "" -if ($map["NETX_UPDATE_AUTO"]) { $autoVal = $map["NETX_UPDATE_AUTO"] } -elseif ($env:NETX_UPDATE_AUTO) { $autoVal = $env:NETX_UPDATE_AUTO } -if ($autoVal -match '^(1|true|yes|on)$') { - $AutoUpdate = $true - $CheckUpdateOnLaunch = $true -} -$uiUrl = "http://${hostBind}:${port}/" -$script:ver = Get-NetxVersion -ProgramRoot $prog -$ver = $script:ver -$dbMode = Get-DbMode -EnvMap $map -$dbModeLabel = if ($dbMode -eq "bundled") { - if (([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)')) { "内置库" } else { "built-in DB" } -} else { - if (([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)')) { "外置库" } else { "external DB" } -} -$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') -function T([string]$En, [string]$Zh) { - if ($zh) { return $Zh } else { return $En } -} - -if (-not (Test-Path $envPath)) { - [void][System.Windows.Forms.MessageBox]::Show( - (T ` - "NetX is not configured yet (missing $envPath).`nRe-run the installer and choose built-in or external DB,`nor use Start Menu → NetX → Reconfigure database." ` - "尚未完成数据库配置（缺少 $envPath）。`n请重新运行安装向导并选择内置或外置数据库，`n或使用「开始菜单 → NetX → 重新配置数据库」。"), - "NetX", - [System.Windows.Forms.MessageBoxButtons]::OK, - [System.Windows.Forms.MessageBoxIcon]::Warning - ) - exit 1 -} - -# Only one tray icon per machine session (desktop + postinstall + autostart can race). -$script:netxTrayMutex = $null -try { - $createdNew = $false - $script:netxTrayMutex = New-Object System.Threading.Mutex($true, "Local\NetX.WinTray.SingleInstance", [ref]$createdNew) - if (-not $createdNew) { - if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { - try { Start-Process $uiUrl } catch {} - } elseif ($StartOnLaunch) { - Start-NetxPowerShell -File (Join-Path $PSScriptRoot "start_netx_app.ps1") ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-SkipBrowser") ` - -WorkingDirectory $prog -WindowStyle Hidden | Out-Null - } - exit 0 - } -} catch { - # If mutex fails, continue with a tray (better than no UI control). -} - -$startPs1 = Join-Path $PSScriptRoot "start_netx_app.ps1" -$stopPs1 = Join-Path $PSScriptRoot "stop_netx_app.ps1" -$checkPs1 = Join-Path $PSScriptRoot "check_update.ps1" -$setupPs1 = Join-Path $PSScriptRoot "setup_first_run.ps1" - -function Invoke-NetxScript { - param( - [string]$File, - [string[]]$ExtraArgs = @(), - [switch]$Wait = $false - ) - return Start-NetxPowerShell -File $File -Arguments (@("-ProgramRoot", $prog, "-DataRoot", $data) + $ExtraArgs) ` - -WorkingDirectory $prog -WindowStyle Hidden -PassThru -Wait:$Wait -} - -function Update-NetxTrayTip { - # Refresh version after in-place updates (tray process may outlive version.json). - try { - $nowVer = Get-NetxVersion -ProgramRoot $prog - if ($nowVer -and $nowVer -ne $script:ver) { - $script:ver = $nowVer - if ($null -ne $script:miHeader) { $script:miHeader.Text = "NetX $script:ver" } - } - } catch {} - $running = Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 1 - $state = if ($running) { - (T "Running" "运行中") - } else { - (T "Stopped" "已停止") - } - $v = $script:ver - $tip = "NetX $v · $state · $dbModeLabel`n$uiUrl" - if ($tip.Length -gt 63) { - # NotifyIcon.Text max ~63 chars on older Windows. - $tip = "NetX $v · $state · $dbModeLabel" - } - $notify.Text = $tip -} - -function Close-NetxTrayMenu { - # ContextMenuStrip stays modal if Click handlers block; close before dialogs/waits. - try { $menu.Close() } catch {} - try { [System.Windows.Forms.Application]::DoEvents() } catch {} -} - -function Start-NetxTrayDeferred { - param([scriptblock]$Work) - Close-NetxTrayMenu - $timer = New-Object System.Windows.Forms.Timer - $timer.Interval = 50 - $script:netxTrayDeferredWork = $Work - $timer.add_Tick({ - $t = $script:netxTrayDeferredTimer - try { if ($t) { $t.Stop(); $t.Dispose() } } catch {} - $script:netxTrayDeferredTimer = $null - $w = $script:netxTrayDeferredWork - $script:netxTrayDeferredWork = $null - if ($w) { & $w } - }) - $script:netxTrayDeferredTimer = $timer - $timer.Start() -} - -function Start-NetxTrayWatchProcess { - # Never Start-Process -Wait on the WinForms UI thread — it freezes the tray menu. - param( - [Parameter(Mandatory = $true)]$Process, - [scriptblock]$OnExit - ) - if ($null -eq $Process) { - if ($OnExit) { & $OnExit $null } - return - } - $script:netxWatchProc = $Process - $script:netxWatchOnExit = $OnExit - if ($script:netxWatchTimer) { - try { $script:netxWatchTimer.Stop(); $script:netxWatchTimer.Dispose() } catch {} - } - $timer = New-Object System.Windows.Forms.Timer - $timer.Interval = 400 - $timer.add_Tick({ - $p = $script:netxWatchProc - if ($null -eq $p) { - try { $script:netxWatchTimer.Stop(); $script:netxWatchTimer.Dispose() } catch {} - $script:netxWatchTimer = $null - return - } - $done = $false - try { $done = [bool]$p.HasExited } catch { $done = $true } - if (-not $done) { return } - try { $script:netxWatchTimer.Stop(); $script:netxWatchTimer.Dispose() } catch {} - $script:netxWatchTimer = $null - $cb = $script:netxWatchOnExit - $script:netxWatchOnExit = $null - $script:netxWatchProc = $null - if ($cb) { & $cb $p } - }) - $script:netxWatchTimer = $timer - $timer.Start() -} - -function Restart-NetxTraySelf { - # Fresh process picks up new version.json / scripts after in-place update. - try { - $notify.ShowBalloonTip( - 2500, "NetX", - (T "Restarting tray…" "正在重启托盘…"), - [System.Windows.Forms.ToolTipIcon]::Info - ) - } catch {} - $trayFile = Join-Path $PSScriptRoot "netx_tray.ps1" - $arg = "-NoProfile -ExecutionPolicy Bypass -WindowStyle Hidden -File `"$trayFile`" -ProgramRoot `"$prog`" -DataRoot `"$data`"" - try { - Start-Process -FilePath "powershell.exe" -ArgumentList $arg -WindowStyle Hidden | Out-Null - } catch {} - try { $notify.Visible = $false } catch {} - try { $form.Close() } catch {} - [System.Windows.Forms.Application]::Exit() -} - -function Test-NetxSetupCancelled { - param([AllowNull()][Nullable[int]]$ExitCode) - if ($null -eq $ExitCode) { return $false } - # STATUS_CONTROL_C_EXIT (0xC000013A): console closed / Ctrl+C — not a setup failure. - return ([int]$ExitCode -eq -1073741510) -} - -function Complete-NetxTrayReconfig { - # Reload .env after reconfigure (host/port/mode may change). - if (Test-Path $envPath) { - $map = Read-DotEnv -Path $envPath - if ($map["NETX_HOST"]) { $script:hostBind = $map["NETX_HOST"]; $hostBind = $script:hostBind } - if ($map["NETX_PORT"]) { - try { - $script:port = [int]$map["NETX_PORT"] - $port = $script:port - } catch {} - } - $script:uiUrl = "http://${hostBind}:${port}/" - $uiUrl = $script:uiUrl - $dbMode = Get-DbMode -EnvMap $map - $script:dbModeLabel = if ($dbMode -eq "bundled") { - (T "built-in DB" "内置库") - } else { - (T "external DB" "外置库") - } - $dbModeLabel = $script:dbModeLabel - if ($miSub) { $miSub.Text = "$dbModeLabel · ${hostBind}:$port" } - } - - $restart = [System.Windows.Forms.MessageBox]::Show( - (T "Database configuration saved.`nStart NetX now?" "数据库配置已保存。`n是否立即启动 NetX？"), - "NetX", - [System.Windows.Forms.MessageBoxButtons]::YesNo, - [System.Windows.Forms.MessageBoxIcon]::Information - ) - if ($restart -eq [System.Windows.Forms.DialogResult]::Yes) { - Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null - Open-NetxUiWhenReady - } else { - Update-NetxTrayTip - } -} - -function Open-NetxUiWhenReady { - param([int]$TimeoutSec = 180) - $notify.ShowBalloonTip( - 5000, - "NetX", - (T "Starting… first run may take a minute." "正在启动…首次迁移可能需要一分钟。"), - [System.Windows.Forms.ToolTipIcon]::Info - ) - # Poll on a timer — do not block the tray UI thread with Wait-NetxApiHealthy. - if ($script:netxUiReadyTimer) { - try { $script:netxUiReadyTimer.Stop(); $script:netxUiReadyTimer.Dispose() } catch {} - } - $script:netxUiReadyTicks = 0 - $script:netxUiReadyMax = [Math]::Max(1, [int]($TimeoutSec * 2)) - $timer = New-Object System.Windows.Forms.Timer - $timer.Interval = 500 - $timer.add_Tick({ - $script:netxUiReadyTicks++ - if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { - try { $script:netxUiReadyTimer.Stop(); $script:netxUiReadyTimer.Dispose() } catch {} - $script:netxUiReadyTimer = $null - try { Start-Process $uiUrl } catch {} - Update-NetxTrayTip - return - } - if ($script:netxUiReadyTicks -ge $script:netxUiReadyMax) { - try { $script:netxUiReadyTimer.Stop(); $script:netxUiReadyTimer.Dispose() } catch {} - $script:netxUiReadyTimer = $null - $notify.ShowBalloonTip( - 8000, - "NetX", - (T "UI not ready yet. Use tray → Open UI when ready, or check data\runtime\netx.err.log." "界面尚未就绪。就绪后请用托盘「打开界面」，或查看 data\runtime\netx.err.log。"), - [System.Windows.Forms.ToolTipIcon]::Warning - ) - Update-NetxTrayTip - } - }) - $script:netxUiReadyTimer = $timer - $timer.Start() -} - -$form = New-Object System.Windows.Forms.Form -$form.Text = "NetX" -$form.ShowInTaskbar = $false -$form.WindowState = "Minimized" -$form.Visible = $false -$form.Size = New-Object System.Drawing.Size(0, 0) - -$notify = New-Object System.Windows.Forms.NotifyIcon -$notify.Visible = $true -$iconPath = Join-Path $PSScriptRoot "assets\netx.ico" -try { - if (Test-Path $iconPath) { - $notify.Icon = New-Object System.Drawing.Icon $iconPath - } else { - $notify.Icon = [System.Drawing.SystemIcons]::Application - } -} catch { - try { $notify.Icon = [System.Drawing.SystemIcons]::Application } catch {} -} -Update-NetxTrayTip - -function New-NetxMenuItem { - param( - [string]$Text, - [switch]$Header, - [switch]$Primary, - [switch]$Muted - ) - $item = New-Object System.Windows.Forms.ToolStripMenuItem - $item.Text = $Text - $item.AutoSize = $true - $item.Padding = New-Object System.Windows.Forms.Padding(10, 5, 28, 5) - $item.Margin = New-Object System.Windows.Forms.Padding(0) - if ($Header) { - $item.Enabled = $false - $item.Font = New-Object System.Drawing.Font("Segoe UI", 9.0, [System.Drawing.FontStyle]::Bold) - $item.ForeColor = [System.Drawing.Color]::FromArgb(55, 65, 80) - $item.Padding = New-Object System.Windows.Forms.Padding(10, 6, 28, 2) - } elseif ($Muted) { - $item.Enabled = $false - $item.Font = New-Object System.Drawing.Font("Segoe UI", 8.25) - $item.ForeColor = [System.Drawing.Color]::FromArgb(120, 128, 140) - $item.Padding = New-Object System.Windows.Forms.Padding(10, 1, 28, 6) - } elseif ($Primary) { - $item.Font = New-Object System.Drawing.Font("Segoe UI", 9.0, [System.Drawing.FontStyle]::Bold) - $item.ForeColor = [System.Drawing.Color]::FromArgb(20, 40, 70) - } else { - $item.Font = New-Object System.Drawing.Font("Segoe UI", 9.0) - $item.ForeColor = [System.Drawing.Color]::FromArgb(35, 40, 48) - } - return $item -} - -function New-NetxMenuSeparator { - $sep = New-Object System.Windows.Forms.ToolStripSeparator - $sep.Margin = New-Object System.Windows.Forms.Padding(0, 3, 0, 3) - $sep.Padding = New-Object System.Windows.Forms.Padding(0) - return $sep -} - -$menu = New-Object System.Windows.Forms.ContextMenuStrip -$menu.ShowImageMargin = $false -$menu.ShowCheckMargin = $false -$menu.Font = New-Object System.Drawing.Font("Segoe UI", 9.0) -$menu.BackColor = [System.Drawing.Color]::FromArgb(252, 252, 253) -$menu.ForeColor = [System.Drawing.Color]::FromArgb(35, 40, 48) -$menu.Padding = New-Object System.Windows.Forms.Padding(4, 4, 4, 4) -$menu.Renderer = New-Object NetxMenuRenderer - -# --- header (info only) --- -$script:miHeader = New-NetxMenuItem -Text "NetX $script:ver" -Header -$miHeader = $script:miHeader -[void]$menu.Items.Add($miHeader) - -$miSub = New-NetxMenuItem -Text "$dbModeLabel · ${hostBind}:$port" -Muted -[void]$menu.Items.Add($miSub) - -[void]$menu.Items.Add((New-NetxMenuSeparator)) - -# --- primary --- -$miOpen = New-NetxMenuItem -Text (T "Open UI" "打开界面") -Primary -$miOpen.add_Click({ - Start-NetxTrayDeferred { - if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { - Start-Process $uiUrl - } else { - $notify.ShowBalloonTip(4000, "NetX", (T "NetX is not running. Starting…" "NetX 未运行，正在启动…"), [System.Windows.Forms.ToolTipIcon]::Info) - Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null - Open-NetxUiWhenReady - } - } -}) -[void]$menu.Items.Add($miOpen) - -[void]$menu.Items.Add((New-NetxMenuSeparator)) - -# --- service control --- -$miStart = New-NetxMenuItem -Text (T "Start" "启动") -$miStart.add_Click({ - Start-NetxTrayDeferred { - Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null - Open-NetxUiWhenReady - } -}) -[void]$menu.Items.Add($miStart) - -$miStop = New-NetxMenuItem -Text (T "Stop" "停止") -$miStop.add_Click({ - Start-NetxTrayDeferred { - $notify.ShowBalloonTip(3000, "NetX", (T "Stopping…" "正在停止…"), [System.Windows.Forms.ToolTipIcon]::Info) - Invoke-NetxScript -File $stopPs1 | Out-Null - Update-NetxTrayTip - } -}) -[void]$menu.Items.Add($miStop) - -[void]$menu.Items.Add((New-NetxMenuSeparator)) - -# --- setup / update --- -$miReconfig = New-NetxMenuItem -Text (T "Reconfigure database…" "重新配置数据库…") -$miReconfig.add_Click({ - Start-NetxTrayDeferred { - $ans = [System.Windows.Forms.MessageBox]::Show( - (T ` - "NetX will stop while you reconfigure the database.`n`nChoose built-in or external PostgreSQL in the console window.`nContinue?" ` - "重新配置数据库时会先停止 NetX。`n`n请在随后的控制台窗口中选择内置或外置 PostgreSQL。`n是否继续？"), - (T "Reconfigure database" "重新配置数据库"), - [System.Windows.Forms.MessageBoxButtons]::YesNo, - [System.Windows.Forms.MessageBoxIcon]::Question - ) - if ($ans -ne [System.Windows.Forms.DialogResult]::Yes) { return } - - $notify.ShowBalloonTip(3000, "NetX", (T "Stopping…" "正在停止…"), [System.Windows.Forms.ToolTipIcon]::Info) - Invoke-NetxScript -File $stopPs1 | Out-Null - - # ProgramData\.env is often admin-owned after Setup; repair ACL before reconfigure. - $null = Ensure-NetxDataAcl -DataRoot $data -Quiet - - try { - $p = Start-NetxPowerShell -File $setupPs1 ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` - -WorkingDirectory $prog -WindowStyle Normal -PassThru - Start-NetxTrayWatchProcess -Process $p -OnExit { - param($sp) - if ($null -eq $sp) { Update-NetxTrayTip; return } - if (Test-NetxSetupCancelled -ExitCode $sp.ExitCode) { - $notify.ShowBalloonTip( - 4000, "NetX", - (T "Database setup cancelled." "已取消数据库配置。"), - [System.Windows.Forms.ToolTipIcon]::Info - ) - Update-NetxTrayTip - return - } - if ($null -ne $sp.ExitCode -and $sp.ExitCode -ne 0) { - $elev = [System.Windows.Forms.MessageBox]::Show( - (T ` - "Database setup failed (exit $($sp.ExitCode)).`n`nIf this was a permission error on %ProgramData%\NetX\.env, click Yes to retry as Administrator." ` - "数据库配置失败（退出码 $($sp.ExitCode)）。`n`n若是 %ProgramData%\NetX\.env 权限问题，点「是」将以管理员身份重试。"), - "NetX", - [System.Windows.Forms.MessageBoxButtons]::YesNo, - [System.Windows.Forms.MessageBoxIcon]::Warning - ) - if ($elev -ne [System.Windows.Forms.DialogResult]::Yes) { - Update-NetxTrayTip - return - } - $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$setupPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`"" - try { - $ep = Start-Process -FilePath "powershell.exe" -ArgumentList $arg ` - -WorkingDirectory $prog -Verb RunAs -PassThru - } catch { - Update-NetxTrayTip - return - } - Start-NetxTrayWatchProcess -Process $ep -OnExit { - param($ep2) - if ($null -eq $ep2 -or (Test-NetxSetupCancelled -ExitCode $ep2.ExitCode)) { - Update-NetxTrayTip - return - } - if ($null -ne $ep2.ExitCode -and $ep2.ExitCode -ne 0) { - [System.Windows.Forms.MessageBox]::Show( - (T "Elevated database setup still failed (exit $($ep2.ExitCode))." "管理员配置仍失败（退出码 $($ep2.ExitCode)）。"), - "NetX", - [System.Windows.Forms.MessageBoxButtons]::OK, - [System.Windows.Forms.MessageBoxIcon]::Error - ) - Update-NetxTrayTip - return - } - Complete-NetxTrayReconfig - } - return - } - Complete-NetxTrayReconfig - } - } catch { - [System.Windows.Forms.MessageBox]::Show( - (T "Database setup failed: $($_.Exception.Message)" "数据库配置失败：$($_.Exception.Message)"), - "NetX", - [System.Windows.Forms.MessageBoxButtons]::OK, - [System.Windows.Forms.MessageBoxIcon]::Error - ) - Update-NetxTrayTip - } - } -}) -[void]$menu.Items.Add($miReconfig) - -$miUpdate = New-NetxMenuItem -Text (T "Check for updates…" "检查更新…") -$miUpdate.add_Click({ - Start-NetxTrayDeferred { - try { - $notify.ShowBalloonTip( - 3000, "NetX", - (T "Checking for updates…" "正在检查更新…"), - [System.Windows.Forms.ToolTipIcon]::Info - ) - # No -Wait on UI thread (freezes ContextMenuStrip / tray). - $p = Start-NetxPowerShell -File $checkPs1 -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data) ` - -WorkingDirectory $prog -WindowStyle Normal -PassThru - Start-NetxTrayWatchProcess -Process $p -OnExit { - param($cp) - try { - if ($null -eq $cp) { return } - if (Test-NetxSetupCancelled -ExitCode $cp.ExitCode) { return } - if ($cp.ExitCode -eq 10) { - $ans = [System.Windows.Forms.MessageBox]::Show( - (T "A newer NetX is available. Download and apply now?" "发现新版本，是否立即下载并更新？"), - (T "NetX update" "NetX 更新"), - [System.Windows.Forms.MessageBoxButtons]::YesNo, - [System.Windows.Forms.MessageBoxIcon]::Question - ) - if ($ans -ne [System.Windows.Forms.DialogResult]::Yes) { return } - $arg = "-NoProfile -ExecutionPolicy Bypass -File `"$checkPs1`" -ProgramRoot `"$prog`" -DataRoot `"$data`" -Apply" - $notify.ShowBalloonTip( - 5000, "NetX", - (T "Downloading / applying update… keep the console open." "正在下载/应用更新…请勿关闭控制台窗口。"), - [System.Windows.Forms.ToolTipIcon]::Info - ) - try { - $ap = Start-Process -FilePath "powershell.exe" -ArgumentList $arg ` - -WorkingDirectory $prog -Verb RunAs -PassThru - } catch { - $notify.ShowBalloonTip(4000, "NetX", (T "Update cancelled (UAC)." "已取消更新（UAC）。"), [System.Windows.Forms.ToolTipIcon]::Info) - return - } - Start-NetxTrayWatchProcess -Process $ap -OnExit { - param($ap2) - if ($null -eq $ap2) { return } - if (Test-NetxSetupCancelled -ExitCode $ap2.ExitCode) { - $notify.ShowBalloonTip(4000, "NetX", (T "Update cancelled." "已取消更新。"), [System.Windows.Forms.ToolTipIcon]::Info) - return - } - if ($null -ne $ap2.ExitCode -and $ap2.ExitCode -eq 0) { - $newVer = Get-NetxVersion -ProgramRoot $prog - [System.Windows.Forms.MessageBox]::Show( - (T "Updated to $newVer.`nTray will restart." "已更新到 $newVer。`n即将重启托盘。"), - (T "NetX update" "NetX 更新"), - [System.Windows.Forms.MessageBoxButtons]::OK, - [System.Windows.Forms.MessageBoxIcon]::Information - ) - Restart-NetxTraySelf - return - } - [System.Windows.Forms.MessageBox]::Show( - (T "Update failed (exit $($ap2.ExitCode)). Re-run Check for updates, or install NetX-Setup manually." "更新失败（退出码 $($ap2.ExitCode)）。请重试「检查更新」，或手动运行 Setup 安装包。"), - (T "NetX update" "NetX 更新"), - [System.Windows.Forms.MessageBoxButtons]::OK, - [System.Windows.Forms.MessageBoxIcon]::Warning - ) - Update-NetxTrayTip - } - } elseif ($cp.ExitCode -eq 0) { - [System.Windows.Forms.MessageBox]::Show( - (T "NetX is up to date ($script:ver)." "已是最新版本 ($script:ver)。"), - (T "NetX update" "NetX 更新") - ) - } - } catch { - [System.Windows.Forms.MessageBox]::Show( - (T "Update check failed: $($_.Exception.Message)" "检查更新失败：$($_.Exception.Message)"), - "NetX" - ) - } - } - } catch { - [System.Windows.Forms.MessageBox]::Show( - (T "Update check failed: $($_.Exception.Message)" "检查更新失败：$($_.Exception.Message)"), - "NetX" - ) - } - } -}) -[void]$menu.Items.Add($miUpdate) - -[void]$menu.Items.Add((New-NetxMenuSeparator)) - -# --- exit --- -$miExit = New-NetxMenuItem -Text (T "Exit tray" "退出托盘") -$miExit.ToolTipText = (T "Leave NetX services running" "NetX 服务继续运行") -$miExit.add_Click({ - Close-NetxTrayMenu - $notify.Visible = $false - $form.Close() - [System.Windows.Forms.Application]::Exit() -}) -[void]$menu.Items.Add($miExit) - -$miStopExit = New-NetxMenuItem -Text (T "Stop and exit" "停止并退出") -$miStopExit.add_Click({ - Start-NetxTrayDeferred { - $notify.ShowBalloonTip(3000, "NetX", (T "Stopping…" "正在停止…"), [System.Windows.Forms.ToolTipIcon]::Info) - Invoke-NetxScript -File $stopPs1 -Wait | Out-Null - $notify.Visible = $false - $form.Close() - [System.Windows.Forms.Application]::Exit() - } -}) -[void]$menu.Items.Add($miStopExit) - -$notify.ContextMenuStrip = $menu -$notify.add_DoubleClick({ - if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { - Start-Process $uiUrl - } else { - Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null - Open-NetxUiWhenReady - } -}) - -# Refresh tooltip periodically (running / stopped). -$script:tipTimer = New-Object System.Windows.Forms.Timer -$script:tipTimer.Interval = 5000 -$script:tipTimer.add_Tick({ Update-NetxTrayTip }) -$script:tipTimer.Start() - -$form.add_Shown({ - if ($AutoUpdate) { - $notify.ShowBalloonTip(4000, "NetX", (T "Checking for updates…" "正在检查更新…"), [System.Windows.Forms.ToolTipIcon]::Info) - Start-NetxPowerShell -File $checkPs1 ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-Apply", "-Quiet", "-AutoOnly") ` - -WorkingDirectory $prog -WindowStyle Hidden -Wait | Out-Null - } elseif ($CheckUpdateOnLaunch) { - Start-NetxPowerShell -File $checkPs1 ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-Quiet") ` - -WorkingDirectory $prog -WindowStyle Hidden | Out-Null - } - if ($StartOnLaunch) { - Invoke-NetxScript -File $startPs1 -ExtraArgs @("-SkipBrowser") | Out-Null - $script:netxUiWaitTicks = 0 - $script:netxUiWaitMax = 360 # 360 * 500ms = 180s - $script:netxUiTimer = New-Object System.Windows.Forms.Timer - $script:netxUiTimer.Interval = 500 - $script:netxUiTimer.add_Tick({ - $script:netxUiWaitTicks++ - if (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2) { - $script:netxUiTimer.Stop() - $script:netxUiTimer.Dispose() - try { Start-Process $uiUrl } catch {} - Update-NetxTrayTip - return - } - if ($script:netxUiWaitTicks -eq 1) { - $notify.ShowBalloonTip( - 5000, - "NetX", - (T "Starting… first run may take a minute." "正在启动…首次迁移可能需要一分钟。"), - [System.Windows.Forms.ToolTipIcon]::Info - ) - } - if ($script:netxUiWaitTicks -ge $script:netxUiWaitMax) { - $script:netxUiTimer.Stop() - $script:netxUiTimer.Dispose() - $notify.ShowBalloonTip( - 8000, - "NetX", - (T "UI not ready yet. Use tray → Open UI later, or check data\runtime\netx.err.log." "界面尚未就绪。请稍后用托盘「打开界面」，或查看 data\runtime\netx.err.log。"), - [System.Windows.Forms.ToolTipIcon]::Warning - ) - Update-NetxTrayTip - } - }) - $script:netxUiTimer.Start() - } -}) - -$form.add_FormClosing({ - try { $script:tipTimer.Stop(); $script:tipTimer.Dispose() } catch {} - $notify.Visible = $false - $notify.Dispose() -}) - -[System.Windows.Forms.Application]::Run($form) diff --git a/packaging/publish_forgejo_release.ps1 b/packaging/publish_forgejo_release.ps1 deleted file mode 100644 index 0d633af..0000000 --- a/packaging/publish_forgejo_release.ps1 +++ /dev/null @@ -1,129 +0,0 @@ -param( - [string]$Version = "", - [string]$ForgejoBase = "https://git.avelo.top", - [string]$Owner = "hansjone", - [string]$Repo = "netx", - [string]$Token = "", - [string]$ReleaseDir = "" -) - -# Publish Windows Setup.exe to Forgejo/Gitea (mirror sync does NOT copy GitHub Release files). -# Requires a Forgejo token with repo write (Settings → Applications → Generate New Token). -# Default: https://git.avelo.top (public domain). LAN IP only when explicitly reachable. -# -# Example: -# $env:NETX_FORGEJO_TOKEN = "..." # or User env NETX_FORGEJO_TOKEN -# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 -# -# Optional LAN (when 10.0.0.131 is reachable): -# .\packaging\publish_forgejo_release.ps1 -Version 0.4.11 -ForgejoBase "http://10.0.0.131:3000" - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" - -$repoRoot = Get-NetxRepoRoot -if (-not $Version) { - $Version = Get-NetxVersion -ProgramRoot $repoRoot -} -$tag = "v$Version" -$relDir = if ($ReleaseDir) { $ReleaseDir } else { Join-Path $PSScriptRoot "release" } -$setup = Join-Path $relDir "NetX-Setup-$Version.exe" - -if (-not $Token) { - $Token = $env:NETX_FORGEJO_TOKEN - if (-not $Token) { $Token = $env:FORGEJO_TOKEN } -} -if (-not $Token) { - throw "forgejo_token_required: set NETX_FORGEJO_TOKEN or pass -Token" -} -if (-not (Test-Path $setup)) { - throw "missing_setup: $setup (run build_release.ps1 + ISCC first)" -} - -$apiBase = "$ForgejoBase/api/v1/repos/$Owner/$Repo" -$headers = @{ - "Authorization" = "token $Token" - "Accept" = "application/json" -} - -function Invoke-ForgejoJson { - param( - [string]$Method, - [string]$Url, - [object]$Body = $null - ) - $params = @{ - Method = $Method - Uri = $Url - Headers = $headers - ContentType = "application/json" - } - if ($null -ne $Body) { - $params["Body"] = ($Body | ConvertTo-Json -Depth 5) - } - return Invoke-RestMethod @params -} - -Write-Host "==> Forgejo publish $tag -> $ForgejoBase/$Owner/$Repo" - -# Ensure git tag exists on Forgejo (mirror usually already has it). -try { - $null = Invoke-ForgejoJson -Method GET -Url "$apiBase/git/refs/tags/$tag" - Write-Host " tag $tag present on Forgejo" -} catch { - Write-Host "[WARN] tag $tag not found on Forgejo yet — run mirror-sync or push tag first" -ForegroundColor Yellow -} - -$existing = $null -try { - $existing = Invoke-ForgejoJson -Method GET -Url "$apiBase/releases/tags/$tag" -} catch { - $existing = $null -} - -$notes = @" -NetX $Version Windows installer (published from GitHub release build). - -- NetX-Setup-$Version.exe -"@ - -if ($existing -and $existing.id) { - Write-Host "==> Release $tag already exists (id=$($existing.id)); deleting old release to re-upload assets" - Invoke-ForgejoJson -Method DELETE -Url "$apiBase/releases/$($existing.id)" | Out-Null -} - -Write-Host "==> Creating release $tag" -$rel = Invoke-ForgejoJson -Method POST -Url "$apiBase/releases" -Body @{ - tag_name = $tag - target = "main" - name = "NetX $Version" - body = $notes - draft = $false - prerelease = $false -} -$relId = $rel.id -if (-not $relId) { throw "forgejo_create_release_failed" } - -function Upload-ForgejoAsset { - param([string]$Path) - if (-not (Test-Path $Path)) { return } - $name = Split-Path -Leaf $Path - Write-Host "==> Uploading $name ..." - $url = "$apiBase/releases/$relId/assets" - $curl = Get-Command curl.exe -ErrorAction SilentlyContinue - if (-not $curl) { throw "curl.exe required for asset upload" } - # --ssl-no-revoke: schannel CRYPT_E_REVOCATION_OFFLINE often fails via proxy/offline CA. - & $curl.Source -f -sS --ssl-no-revoke -X POST ` - -H "Authorization: token $Token" ` - -F "attachment=@$Path" ` - $url - if ($LASTEXITCODE -ne 0) { throw "upload_failed: $name" } - Write-Host " OK $name" -ForegroundColor Green -} - -Upload-ForgejoAsset -Path $setup - -$releaseUrl = "$ForgejoBase/$Owner/$Repo/releases/tag/$tag" -Write-Host "" -Write-Host "==> Forgejo release ready: $releaseUrl" -ForegroundColor Green -Write-Host " Update API: $ForgejoBase/api/v1/repos/$Owner/$Repo/releases/latest" diff --git a/packaging/publish_release.ps1 b/packaging/publish_release.ps1 index 122df57..193c36c 100644 --- a/packaging/publish_release.ps1 +++ b/packaging/publish_release.ps1 @@ -1,10 +1,9 @@ -param( +param( [string]$Version = "", [switch]$SkipBuild = $false, [switch]$SkipInstaller = $false, [switch]$SkipGitHub = $false, - [switch]$Draft = $false, - [switch]$Sign = $false + [switch]$Draft = $false ) $ErrorActionPreference = "Stop" @@ -16,6 +15,7 @@ if (-not $Version) { } $tag = "v$Version" $releaseDir = Join-Path $PSScriptRoot "release" +$zip = Join-Path $releaseDir "NetX-$Version-win64.zip" $setup = Join-Path $releaseDir "NetX-Setup-$Version.exe" Write-Host "==> NetX publish $tag" @@ -25,33 +25,29 @@ if (-not $SkipBuild) { -Version $Version -CreateVenv } +if (-not (Test-Path $zip)) { + throw "missing_zip: $zip" +} + if (-not $SkipInstaller) { $isccCmd = Get-Command ISCC.exe -ErrorAction SilentlyContinue $isccCandidates = @( $(if ($isccCmd) { $isccCmd.Source }), - "$env:LOCALAPPDATA\Programs\Inno Setup 6\ISCC.exe", "${env:ProgramFiles(x86)}\Inno Setup 6\ISCC.exe", "$env:ProgramFiles\Inno Setup 6\ISCC.exe" ) | Where-Object { $_ -and (Test-Path $_) } $iscc = $isccCandidates | Select-Object -First 1 if (-not $iscc) { - throw "innosetup_not_found: install with winget install JRSoftware.InnoSetup" + Write-Host "[WARN] Inno Setup (ISCC) not found. Install: winget install JRSoftware.InnoSetup" -ForegroundColor Yellow + Write-Host " Skipping Setup.exe; zip-only release." + } else { + Write-Host "==> Compiling installer: $iscc" + & $iscc "/DMyAppVersion=$Version" (Join-Path $PSScriptRoot "installer\netx.iss") + if (-not (Test-Path $setup)) { + throw "installer_build_failed: expected $setup" + } + Write-Host "==> Setup.exe: $setup" -ForegroundColor Green } - Write-Host "==> Compiling installer: $iscc" - & $iscc "/DMyAppVersion=$Version" (Join-Path $PSScriptRoot "installer\netx.iss") - if (-not (Test-Path $setup)) { - throw "installer_build_failed: expected $setup" - } - Write-Host "==> Setup.exe: $setup" -ForegroundColor Green -} - -if (-not (Test-Path $setup)) { - throw "missing_setup: $setup" -} - -if ($Sign) { - Write-Host "==> Signing release artifacts" - & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "sign_release.ps1") -Files @($setup) } if ($SkipGitHub) { @@ -64,32 +60,30 @@ if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { } Set-Location $repo -# gh writes "release not found" to stderr; keep Stop for the rest of the script. -$prevEap = $ErrorActionPreference -$ErrorActionPreference = "Continue" $existing = gh release view $tag 2>$null -$viewCode = $LASTEXITCODE -$ErrorActionPreference = $prevEap -if ($viewCode -eq 0) { +if ($LASTEXITCODE -eq 0) { Write-Host "==> Release $tag exists; uploading assets" - gh release upload $tag $setup --clobber + gh release upload $tag $zip --clobber + if (Test-Path $setup) { + gh release upload $tag $setup --clobber + } } else { $notes = @" -## NetX $Version +## NetX $Version — first Windows installable release ### Downloads -- **NetX-Setup-$Version.exe** — Windows installer (Program Files + ProgramData) +- **NetX-Setup-$Version.exe** — recommended installer (Program Files + ProgramData) +- **NetX-$Version-win64.zip** — portable directory package ### Requirements -- Windows 10/11 x64 (or Windows Server 2016+) -- No separate Python or PostgreSQL install required (bundled) +- Windows 10/11 x64 +- No separate Python or PostgreSQL install required (bundled in package) -### Quick start +### Quick start (installer) 1. Run ``NetX-Setup-$Version.exe`` as administrator. -2. **First install:** choose built-in or external PostgreSQL. -3. **Already installed:** Setup detects existing data and updates in place (keeps DB settings). -4. Start menu → **Start NetX** → ``http://127.0.0.1:8890/`` -5. Default login: ``admin`` / ``admin123`` (change after first login) +2. Complete first-time setup (choose **bundled** or **external** PostgreSQL). +3. Start menu → **Start NetX** → browser opens ``http://127.0.0.1:8890/`` +4. Default login: ``admin`` / ``admin123`` (change after first login) ### Linux Unchanged — use your own PostgreSQL and ``scripts/start_netx.sh``. @@ -98,7 +92,8 @@ See [packaging/README.md](https://github.com/hansjone/netx/blob/main/packaging/R "@ $args = @("release", "create", $tag, "--title", "NetX $Version", "--notes", $notes) if ($Draft) { $args += "--draft" } - $args += $setup + $args += $zip + if (Test-Path $setup) { $args += $setup } & gh @args } diff --git a/packaging/repair_venv.ps1 b/packaging/repair_venv.ps1 deleted file mode 100644 index 3af0e60..0000000 --- a/packaging/repair_venv.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "" -) - -# Relink shipped .venv to local python/runtime (must be elevated under Program Files). -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot - -Write-Host "==> Repairing .venv under $prog" -if (Repair-NetxShippedVenv -ProgramRoot $prog) { - $venvPy = Join-Path $prog ".venv\Scripts\python.exe" - if (Test-NetxVenvRunnable -VenvPython $venvPy) { - Write-Host "==> .venv OK -> bundled python/runtime" -ForegroundColor Green - Get-Content (Join-Path $prog ".venv\pyvenv.cfg") - exit 0 - } - Write-Host "[ERR] pyvenv.cfg written but venv still not runnable" -ForegroundColor Red - exit 2 -} -Write-Host "[ERR] repair failed (missing runtime/.venv or access denied)" -ForegroundColor Red -exit 1 diff --git a/packaging/service_run.ps1 b/packaging/service_run.ps1 deleted file mode 100644 index e514352..0000000 --- a/packaging/service_run.ps1 +++ /dev/null @@ -1,100 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "", - [int]$HealthFailLimit = 6, - [int]$HealthIntervalSec = 10 -) - -# Long-running supervisor for Windows Service / Task Scheduler. -# Starts NetX, probes /health; repeated failures trigger restart (or exit for WinSW). - -$ErrorActionPreference = "Stop" -. "$PSScriptRoot\_common.ps1" - -$prog = Get-NetxProgramRoot -Override $ProgramRoot -$data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot -$logDir = Join-Path $data "data\runtime" -if (-not (Test-Path $logDir)) { - New-Item -ItemType Directory -Path $logDir -Force | Out-Null -} -$logFile = Join-Path $logDir "service_run.log" -$stopFlag = Join-Path $logDir "service.stop" - -function Write-SvcLog([string]$Msg) { - $line = "{0} {1}" -f (Get-Date -Format "yyyy-MM-dd HH:mm:ss"), $Msg - Add-Content -Path $logFile -Value $line -Encoding utf8 - Write-Host $line -} - -function Get-BindInfo { - $envPath = Join-Path $data ".env" - $hostBind = "127.0.0.1" - $port = 8890 - if (Test-Path $envPath) { - $map = Read-DotEnv -Path $envPath - if ($map["NETX_HOST"]) { $hostBind = $map["NETX_HOST"] } - if ($map["NETX_PORT"]) { try { $port = [int]$map["NETX_PORT"] } catch {} } - } - return @{ Host = $hostBind; Port = $port } -} - -Remove-Item -Force $stopFlag -ErrorAction SilentlyContinue -Write-SvcLog "service_run starting program=$prog data=$data" - -$startPs1 = Join-Path $PSScriptRoot "start_netx_app.ps1" -$stopPs1 = Join-Path $PSScriptRoot "stop_netx_app.ps1" -$bind = Get-BindInfo -$failStreak = 0 -$restartCount = 0 - -function Start-NetxChildren { - & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $startPs1 ` - -ProgramRoot $prog -DataRoot $data -SkipBrowser -Force - if ($LASTEXITCODE -ne 0) { - throw "start_netx_app_failed exit=$LASTEXITCODE" - } -} - -function Stop-NetxChildren { - & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $stopPs1 ` - -ProgramRoot $prog -DataRoot $data -} - -try { - Start-NetxChildren - Write-SvcLog "NetX started; health watch host=$($bind.Host) port=$($bind.Port)" - - while ($true) { - if (Test-Path $stopFlag) { - Write-SvcLog "stop flag detected" - break - } - - if (Test-NetxApiHealthy -HostName $bind.Host -Port $bind.Port -TimeoutSec 3) { - $failStreak = 0 - } else { - $failStreak++ - Write-SvcLog "health fail streak=$failStreak/$HealthFailLimit" - if ($failStreak -ge $HealthFailLimit) { - $restartCount++ - Write-SvcLog "restarting NetX (attempt=$restartCount)" - try { Stop-NetxChildren } catch { Write-SvcLog "stop warning: $($_.Exception.Message)" } - Start-Sleep -Seconds 3 - Start-NetxChildren - $failStreak = 0 - # Give API time to come up before counting failures again. - Start-Sleep -Seconds ([Math]::Max(15, $HealthIntervalSec)) - continue - } - } - Start-Sleep -Seconds $HealthIntervalSec - } -} catch { - Write-SvcLog "ERROR: $($_.Exception.Message)" - throw -} finally { - Write-SvcLog "stopping NetX" - try { Stop-NetxChildren } catch { Write-SvcLog "stop warning: $($_.Exception.Message)" } - Remove-Item -Force $stopFlag -ErrorAction SilentlyContinue - Write-SvcLog "service_run exited" -} diff --git a/packaging/setup_first_run.ps1 b/packaging/setup_first_run.ps1 index 0806bb8..fa8e0f5 100644 --- a/packaging/setup_first_run.ps1 +++ b/packaging/setup_first_run.ps1 @@ -4,35 +4,31 @@ param( [string]$ProgramRoot = "", [string]$DataRoot = "", [string]$ExternalDatabaseUrl = "", - [string]$ExternalDatabaseUrlFile = "", - [string]$CredentialSecretKey = "", - [string]$CredentialSecretKeyFile = "", [string]$BundledPassword = "", [int]$BundledPort = 15432, - [switch]$NonInteractive = $false, - [switch]$SkipExternalProbe = $false + [switch]$NonInteractive = $false ) $ErrorActionPreference = "Stop" . "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot $envPath = Join-Path $data ".env" -$zh = ([cultureinfo]::CurrentUICulture.Name -match '^(zh|zh-)') -function T([string]$En, [string]$Zh) { - if ($zh) { return $Zh } else { return $En } +Write-Host "==> Program root: $prog" +Write-Host "==> Data root: $data" +Write-Host "==> Env file: $envPath" + +if (-not (Test-Path $data)) { + New-Item -ItemType Directory -Path $data -Force | Out-Null +} +foreach ($sub in @("data", "data\auth", "data\runtime", "backups", "pgdata")) { + $p = Join-Path $data $sub + if (-not (Test-Path $p)) { + New-Item -ItemType Directory -Path $p -Force | Out-Null + } } - -try { - -Write-Host ("==> " + (T "Program root:" "程序目录:") + " $prog") -Write-Host ("==> " + (T "Data root:" "数据目录:") + " $data") -Write-Host ("==> " + (T "Env file:" "环境文件:") + " $envPath") - -Ensure-NetxDataDirectories -DataRoot $data $existing = Read-DotEnv -Path $envPath if (-not $DbMode) { @@ -46,11 +42,10 @@ if (-not $DbMode) { } } else { Write-Host "" - Write-Host (T "Choose database mode:" "选择数据库模式:") -ForegroundColor Cyan - Write-Host (T " 1) bundled — NetX portable PostgreSQL (offline / default)" " 1) bundled — NetX 内置便携 PostgreSQL（可离线，默认）") - Write-Host (T " 2) external — existing PostgreSQL (you provide connection URL)" " 2) external — 已有外置 PostgreSQL（需填写连接串）") - Write-Host "" - $choice = Read-Host (T "Enter 1 or 2" "请输入 1 或 2") + Write-Host "Choose database mode:" + Write-Host " 1) bundled — use NetX portable PostgreSQL (default for new installs)" + Write-Host " 2) external — connect to an existing PostgreSQL (Linux / already deployed)" + $choice = Read-Host "Enter 1 or 2" if ($choice -eq "2") { $DbMode = "external" } else { $DbMode = "bundled" } } } @@ -63,44 +58,28 @@ $values = @{} $values["NETX_DB_MODE"] = $DbMode $values["NETX_HOST"] = "127.0.0.1" $values["NETX_PORT"] = "8890" -# Absolute UI path when present; else relative for source trees. -$distAbs = Join-Path $prog "web\dist" -if (Test-Path (Join-Path $distAbs "index.html")) { - $values["NETX_UI_DIST_DIR"] = (ConvertTo-NetxFsPath $distAbs) -} else { - $values["NETX_UI_DIST_DIR"] = "web/dist" -} +$values["NETX_UI_DIST_DIR"] = "web/dist" -# All runtime data under data root (never under Program Files). -$dataEnv = Get-NetxDataEnvMap -DataRoot $data -foreach ($k in $dataEnv.Keys) { $values[$k] = $dataEnv[$k] } - -# Preload credential key from file/CLI only; interactive prompt comes AFTER DB settings -# so users are not left thinking the key alone finished setup. -if ($CredentialSecretKeyFile) { - if (-not (Test-Path -LiteralPath $CredentialSecretKeyFile)) { - throw "credential_secret_key_file_missing: $CredentialSecretKeyFile" - } - $CredentialSecretKey = [IO.File]::ReadAllText($CredentialSecretKeyFile).Trim() -} +# Point runtime data dirs into the data root (absolute). +$values["NETX_AUTH_MCP_TOKEN_FILE"] = ((Join-Path $data "data\auth\mcp_token") -replace '\\', '/') +$values["NETX_SCHEDULER_HEARTBEAT_PATH"] = ((Join-Path $data "data\runtime\scheduler_heartbeat.json") -replace '\\', '/') if ($DbMode -eq "bundled") { $pgsql = Join-Path $prog "postgres\pgsql" if (-not (Test-Path (Join-Path $pgsql "bin\initdb.exe"))) { + # In-repo layout $alt = Join-Path $PSScriptRoot "postgres\pgsql" if (Test-Path (Join-Path $alt "bin\initdb.exe")) { $pgsql = $alt } else { - throw (T ` - "bundled_postgres_missing: incomplete package (expected $pgsql). Offline Setup must include postgres; rebuild with build_release.ps1 on a machine that already ran download_postgres.ps1." ` - "缺少内置 PostgreSQL（期望路径 $pgsql）。离线安装包必须自带数据库；请在已运行过 download_postgres.ps1 的机器上重新 build_release。") + throw "bundled_postgres_missing: run packaging\download_postgres.ps1 first (expected $pgsql)" } } if (-not $BundledPassword) { if ($NonInteractive) { $BundledPassword = -join ((48..57) + (65..90) + (97..122) | Get-Random -Count 24 | ForEach-Object { [char]$_ }) } else { - $sec = Read-Host -Prompt (T "Password for bundled role 'netx' (empty = auto-generate)" "内置数据库用户 netx 的密码（回车=自动生成）") -AsSecureString + $sec = Read-Host -Prompt "Password for bundled role 'netx' (empty = auto-generate)" -AsSecureString $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($sec) try { $BundledPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto($bstr) @@ -109,22 +88,24 @@ if ($DbMode -eq "bundled") { } if (-not $BundledPassword) { $BundledPassword = -join ((48..57) + (65..90) + (97..122) | Get-Random -Count 24 | ForEach-Object { [char]$_ }) - Write-Host (T "Generated password (saved in .env only)." "已自动生成密码（仅保存在 .env）。") + Write-Host "Generated password (saved in .env only)." } } } $pgData = Join-Path $data "pgdata" $values["NETX_BUNDLED_PG_PORT"] = "$BundledPort" - $values["NETX_BUNDLED_PG_DATA_DIR"] = (ConvertTo-NetxFsPath $pgData) + $values["NETX_BUNDLED_PG_DATA_DIR"] = ($pgData -replace '\\', '/') $pwFile = Join-Path $data "pg_netx.pw" Set-Content -Path $pwFile -Value $BundledPassword -Encoding ascii -NoNewline $encPw = [uri]::EscapeDataString($BundledPassword) $values["NETX_DATABASE_URL"] = "postgresql+psycopg://netx:${encPw}@127.0.0.1:${BundledPort}/netx" + # Initialize cluster if needed $initdb = Join-Path $pgsql "bin\initdb.exe" $pgCtl = Join-Path $pgsql "bin\pg_ctl.exe" $psql = Join-Path $pgsql "bin\psql.exe" - $sqlPw = ConvertTo-NetxSqlLiteral $BundledPassword + $createdb = Join-Path $pgsql "bin\createdb.exe" + $createuser = Join-Path $pgsql "bin\createuser.exe" if (-not (Test-Path (Join-Path $pgData "PG_VERSION"))) { Write-Host "==> initdb $pgData" @@ -134,6 +115,7 @@ if ($DbMode -eq "bundled") { if ($LASTEXITCODE -ne 0) { throw "initdb_failed" } } + # Ensure listen / port in postgresql.conf $conf = Join-Path $pgData "postgresql.conf" $hba = Join-Path $pgData "pg_hba.conf" if (Test-Path $conf) { @@ -144,8 +126,6 @@ if ($DbMode -eq "bundled") { $confText = $confText -replace '(?m)^\s*port\s*=\s*\d+', "port = $BundledPort" if ($confText -notmatch "(?m)^\s*listen_addresses\s*=") { $confText += "`nlisten_addresses = '127.0.0.1'`n" - } else { - $confText = $confText -replace "(?m)^\s*listen_addresses\s*=\s*'[^']*'", "listen_addresses = '127.0.0.1'" } Set-Content -Path $conf -Value $confText -Encoding utf8 } @@ -157,216 +137,49 @@ if ($DbMode -eq "bundled") { } } - $status = & $pgCtl -D $pgData status 2>&1 | Out-String - if ($status -notmatch "server is running") { - if (-not (Test-NetxTcpPortFree -HostName "127.0.0.1" -Port $BundledPort)) { - throw "port_in_use: 127.0.0.1:$BundledPort already occupied (bundled Postgres)" - } - Write-Host "==> Starting bundled PostgreSQL for bootstrap" - & $pgCtl -D $pgData -l (Join-Path $pgData "pg.log") start - if ($LASTEXITCODE -ne 0) { throw "pg_start_failed" } - Start-Sleep -Seconds 2 - } + Write-Host "==> Starting bundled PostgreSQL for bootstrap" + & $pgCtl -D $pgData -l (Join-Path $data "pgdata\pg.log") start + Start-Sleep -Seconds 2 $env:PGPASSWORD = $BundledPassword try { - function Invoke-NetxPsql { - param([string]$Sql, [string]$Database = "postgres") - $out = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d $Database -v ON_ERROR_STOP=1 -tAc $Sql 2>&1 - if ($LASTEXITCODE -ne 0) { - throw "psql_failed ($LASTEXITCODE): $Sql`n$out" - } - return (($out | Out-String).Trim()) - } - - $role = Invoke-NetxPsql -Sql "SELECT 1 FROM pg_roles WHERE rolname='netx'" - if ($role -eq "1") { - Invoke-NetxPsql -Sql "ALTER ROLE netx WITH LOGIN PASSWORD '$sqlPw'" | Out-Null + $role = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='netx'" + if ($role -notmatch "1") { + & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` + -c "CREATE ROLE netx LOGIN PASSWORD '$BundledPassword';" } else { - Invoke-NetxPsql -Sql "CREATE ROLE netx LOGIN PASSWORD '$sqlPw'" | Out-Null + & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` + -c "ALTER ROLE netx WITH LOGIN PASSWORD '$BundledPassword';" } - $db = Invoke-NetxPsql -Sql "SELECT 1 FROM pg_database WHERE datname='netx'" - if ($db -ne "1") { - Invoke-NetxPsql -Sql "CREATE DATABASE netx OWNER netx" | Out-Null - } - Invoke-NetxPsql -Sql "GRANT ALL PRIVILEGES ON DATABASE netx TO netx" | Out-Null - # Verify login as app role (catches auth/hba mismatches early). - $prev = $env:PGPASSWORD - $env:PGPASSWORD = $BundledPassword - try { - $ping = & $psql -h 127.0.0.1 -p $BundledPort -U netx -d netx -v ON_ERROR_STOP=1 -tAc "SELECT 1" 2>&1 - if ($LASTEXITCODE -ne 0 -or (($ping | Out-String).Trim()) -ne "1") { - throw "netx_role_login_failed: $ping" - } - } finally { - $env:PGPASSWORD = $prev + $db = & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='netx'" + if ($db -notmatch "1") { + & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` + -c "CREATE DATABASE netx OWNER netx;" } + & $psql -h 127.0.0.1 -p $BundledPort -U postgres -d postgres -v ON_ERROR_STOP=1 ` + -c "GRANT ALL PRIVILEGES ON DATABASE netx TO netx;" } finally { Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue } Write-Host "==> Bundled Postgres ready on 127.0.0.1:$BundledPort" -ForegroundColor Green } else { - if ($ExternalDatabaseUrlFile) { - if (-not (Test-Path -LiteralPath $ExternalDatabaseUrlFile)) { - throw "external_url_file_missing: $ExternalDatabaseUrlFile" - } - $ExternalDatabaseUrl = [IO.File]::ReadAllText($ExternalDatabaseUrlFile).Trim() - } if (-not $ExternalDatabaseUrl) { - if ($NonInteractive) { - if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { - $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] - } else { - throw "external_url_required" - } + if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { + $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] + } elseif ($NonInteractive) { + throw "external_url_required" } else { - # Interactive: always ask. Empty = keep existing URL (never silent). - Write-Host "" - Write-Host (T ` - "Enter PostgreSQL URL (database/role must already exist):" ` - "请输入 PostgreSQL 连接串（库和用户需事先建好）:") -ForegroundColor Cyan - Write-Host " postgresql+psycopg://USER:PASSWORD@HOST:5432/DBNAME" - Write-Host (T ` - "Example: postgresql+psycopg://netx:secret@10.0.0.8:5432/netx" ` - "示例: postgresql+psycopg://netx:secret@10.0.0.8:5432/netx") - if ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { - Write-Host (T ` - "Current: $($existing['NETX_DATABASE_URL'])" ` - "当前: $($existing['NETX_DATABASE_URL'])") -ForegroundColor DarkGray - Write-Host (T ` - "Press Enter to keep the current URL, or paste a new one." ` - "直接回车 = 保留当前连接串；或粘贴新的连接串。") - } - $entered = (Read-Host "NETX_DATABASE_URL").Trim() - if ($entered) { - $ExternalDatabaseUrl = $entered - } elseif ($existing.ContainsKey("NETX_DATABASE_URL") -and $existing["NETX_DATABASE_URL"]) { - $ExternalDatabaseUrl = $existing["NETX_DATABASE_URL"] - Write-Host (T "==> Keeping existing NETX_DATABASE_URL" "==> 保留已有 NETX_DATABASE_URL") -ForegroundColor Green - } + $ExternalDatabaseUrl = Read-Host "NETX_DATABASE_URL (postgresql+psycopg://user:pass@host:5432/netx)" } } if (-not $ExternalDatabaseUrl) { throw "external_url_required" } - $ExternalDatabaseUrl = $ExternalDatabaseUrl.Trim() $values["NETX_DATABASE_URL"] = $ExternalDatabaseUrl - Write-Host "==> External Postgres configured" -ForegroundColor Green - - if (-not $SkipExternalProbe) { - # Probe with bundled psql when available (wizard already tested; this covers CLI reconfigure). - $psqlProbe = Join-Path $prog "postgres\pgsql\bin\psql.exe" - $testHelper = Join-Path $PSScriptRoot "installer\test_pg_conn.ps1" - if (-not (Test-Path $testHelper)) { - $testHelper = Join-Path $PSScriptRoot "test_pg_conn.ps1" - } - if ((Test-Path $psqlProbe) -and ($ExternalDatabaseUrl -match '^(?:postgresql(?:\+psycopg)?|postgres)://([^:]+):([^@]*)@([^:/]+):?(\d+)?/([^?\s]+)')) { - $u = [uri]::UnescapeDataString($Matches[1]) - $pw = [uri]::UnescapeDataString($Matches[2]) - $h = $Matches[3] - $po = if ($Matches[4]) { [int]$Matches[4] } else { 5432 } - $dbn = [uri]::UnescapeDataString($Matches[5]) - Write-Host "==> Probing external PostgreSQL ${h}:${po}/${dbn} ..." - if (Test-Path $testHelper) { - $probeErr = Join-Path $env:TEMP ("netx-pg-probe-" + [Guid]::NewGuid().ToString("n") + ".txt") - & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $testHelper ` - -PgHost $h -Port $po -User $u -Password $pw -Database $dbn ` - -PsqlPath $psqlProbe -OutErrFile $probeErr - if ($LASTEXITCODE -ne 0) { - $detail = if (Test-Path $probeErr) { (Get-Content -LiteralPath $probeErr -Raw) } else { "exit $LASTEXITCODE" } - Remove-Item -LiteralPath $probeErr -Force -ErrorAction SilentlyContinue - throw (T "external_db_probe_failed: $detail" "外置数据库连接失败: $detail") - } - Remove-Item -LiteralPath $probeErr -Force -ErrorAction SilentlyContinue - Write-Host "==> External PostgreSQL OK" -ForegroundColor Green - } else { - $env:PGPASSWORD = $pw - try { - $ping = & $psqlProbe -h $h -p $po -U $u -d $dbn -t -A -c "SELECT 1" 2>&1 - if ($LASTEXITCODE -ne 0 -or (($ping | Out-String).Trim()) -notmatch '1') { - throw (T "external_db_probe_failed: $ping" "外置数据库连接失败: $ping") - } - } finally { - Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue - } - } - } else { - Write-Host (T ` - "[WARN] Skipped connection probe (no bundled psql or URL parse failed)." ` - "[WARN] 已跳过连接探测（缺少捆绑 psql 或连接串无法解析）。") -ForegroundColor Yellow - } - } -} - -# Fernet key AFTER database prompts (interactive UX). -# Priority: CLI/file > interactive prompt > existing .env > generate. -if (-not $CredentialSecretKey -and -not $NonInteractive) { - Write-Host "" - Write-Host (T ` - "Optional: paste NETX_CREDENTIAL_SECRET_KEY from an existing install" ` - "可选: 粘贴已有安装的 NETX_CREDENTIAL_SECRET_KEY（便于沿用已加密凭据）") -ForegroundColor Cyan - Write-Host (T ` - "Leave empty to keep existing .env key, or auto-generate if none." ` - "留空则保留已有 .env 中的密钥；若没有则自动生成。") - $CredentialSecretKey = (Read-Host "NETX_CREDENTIAL_SECRET_KEY").Trim() -} -if ($CredentialSecretKey) { - $values["NETX_CREDENTIAL_SECRET_KEY"] = $CredentialSecretKey.Trim() - Write-Host "==> Using provided NETX_CREDENTIAL_SECRET_KEY" -ForegroundColor Green -} elseif ($existing.ContainsKey("NETX_CREDENTIAL_SECRET_KEY") -and $existing["NETX_CREDENTIAL_SECRET_KEY"]) { - $values["NETX_CREDENTIAL_SECRET_KEY"] = $existing["NETX_CREDENTIAL_SECRET_KEY"] - Write-Host "==> Keeping existing NETX_CREDENTIAL_SECRET_KEY from .env" -ForegroundColor Green -} else { - $values["NETX_CREDENTIAL_SECRET_KEY"] = New-NetxFernetKey - Write-Host "==> Generated NETX_CREDENTIAL_SECRET_KEY (saved in .env)" -ForegroundColor Green + Write-Host "==> External Postgres configured (ensure role/db exist; see scripts\init_pg.ps1)" -ForegroundColor Green } Write-DotEnvValue -Path $envPath -Values $values Write-Host "" Write-Host "Wrote $envPath" -ForegroundColor Green - -# Official Setup ships python/runtime + .venv (build_release -CreateVenv). -$venvPy = Join-Path $prog ".venv\Scripts\python.exe" -try { - $null = Ensure-NetxVenv -ProgramRoot $prog - Write-Host "==> Bundled Python venv OK: $venvPy" -ForegroundColor Green -} catch { - if (Test-Path $venvPy) { - throw - } - Write-Host "==> Bundled .venv missing — creating one (Setup should normally ship it)." -ForegroundColor Yellow - Write-Host "[WARN] $($_.Exception.Message)" -ForegroundColor Yellow - Write-Host " Install a Setup built with: packaging\build_release.ps1 -CreateVenv" -ForegroundColor Yellow -} - -Write-Host "" -Write-Host (T "Setup complete." "首次配置完成。") -ForegroundColor Green -if (-not $NonInteractive) { - Write-Host (T ` - "Next: Start Menu → NetX Tray (or press Y below)." ` - "下一步: 开始菜单 → NetX 托盘 （或在下方按 Y 立即启动）。") - $ans = Read-Host (T "Start NetX tray now? [Y/n]" "现在启动 NetX 托盘？[Y/n]") - if ($ans -notmatch '^[Nn]') { - try { - Start-NetxPowerShell -File (Join-Path $PSScriptRoot "netx_tray.ps1") ` - -Arguments @("-ProgramRoot", $prog, "-DataRoot", $data, "-StartOnLaunch") ` - -WorkingDirectory $prog -WindowStyle Hidden | Out-Null - Write-Host (T "Tray started." "托盘已启动。") -ForegroundColor Green - } catch { - Write-Host "[WARN] $($_.Exception.Message)" -ForegroundColor Yellow - } - } -} else { - Write-Host "Next: .\packaging\start_netx_app.ps1 or NetX-Tray.cmd" -} - -} catch { - Write-Host "" - Write-Host ("[ERR] " + $_.Exception.Message) -ForegroundColor Red - if (-not $NonInteractive) { - try { - [void](Read-Host (T "Press Enter to close" "按回车关闭窗口")) - } catch {} - } - exit 1 -} +Write-Host "Next: .\packaging\start_netx_app.ps1" diff --git a/packaging/sign_release.ps1 b/packaging/sign_release.ps1 deleted file mode 100644 index a859447..0000000 --- a/packaging/sign_release.ps1 +++ /dev/null @@ -1,78 +0,0 @@ -param( - [Parameter(Mandatory = $true)] - [string[]]$Files, - [string]$Thumbprint = "", - [string]$PfxPath = "", - [string]$PfxPassword = "", - [string]$TimestampUrl = "http://timestamp.digicert.com", - [string]$Description = "NetX" -) - -# Sign release artifacts with signtool (Authenticode). -# Requires Windows SDK / signtool.exe and a code-signing certificate. -# -# Examples: -# .\sign_release.ps1 -Files .\packaging\release\NetX-Setup-0.4.0.exe -Thumbprint ABCDEF... -# .\sign_release.ps1 -Files .\packaging\release\*.exe -PfxPath .\certs\netx.pfx -PfxPassword *** - -$ErrorActionPreference = "Stop" - -function Find-SignTool { - $cmd = Get-Command signtool.exe -ErrorAction SilentlyContinue - if ($cmd) { return $cmd.Source } - $roots = @( - "${env:ProgramFiles(x86)}\Windows Kits\10\bin", - "${env:ProgramFiles}\Windows Kits\10\bin" - ) - foreach ($root in $roots) { - if (-not (Test-Path $root)) { continue } - $hit = Get-ChildItem -Path $root -Recurse -Filter signtool.exe -ErrorAction SilentlyContinue | - Where-Object { $_.FullName -match '\\x64\\signtool\.exe$' } | - Select-Object -First 1 - if ($hit) { return $hit.FullName } - } - return $null -} - -$signtool = Find-SignTool -if (-not $signtool) { - throw "signtool_not_found: install Windows SDK or add signtool.exe to PATH" -} - -if (-not $Thumbprint -and -not $PfxPath) { - if ($env:NETX_SIGN_THUMBPRINT) { $Thumbprint = $env:NETX_SIGN_THUMBPRINT } - if ($env:NETX_SIGN_PFX) { $PfxPath = $env:NETX_SIGN_PFX } - if ($env:NETX_SIGN_PFX_PASSWORD) { $PfxPassword = $env:NETX_SIGN_PFX_PASSWORD } -} -if (-not $Thumbprint -and -not $PfxPath) { - throw "provide -Thumbprint or -PfxPath (or NETX_SIGN_THUMBPRINT / NETX_SIGN_PFX)" -} - -$resolved = @() -foreach ($pattern in $Files) { - $resolved += @(Resolve-Path -Path $pattern -ErrorAction Stop) -} -if ($resolved.Count -eq 0) { throw "no_files_to_sign" } - -foreach ($f in $resolved) { - $path = $f.Path - Write-Host "==> Signing $path" - $args = @( - "sign", "/fd", "SHA256", "/td", "SHA256", "/tr", $TimestampUrl, - "/d", $Description - ) - if ($PfxPath) { - $args += @("/f", $PfxPath) - if ($PfxPassword) { $args += @("/p", $PfxPassword) } - } else { - $args += @("/sha1", $Thumbprint) - } - $args += $path - & $signtool @args - if ($LASTEXITCODE -ne 0) { throw "sign_failed: $path" } - & $signtool verify /pa $path - if ($LASTEXITCODE -ne 0) { throw "verify_failed: $path" } - Write-Host " OK" -ForegroundColor Green -} - -Write-Host "==> Done. Without a trusted CA certificate, Windows SmartScreen may still warn." diff --git a/packaging/start_netx_app.ps1 b/packaging/start_netx_app.ps1 index 55b8d6d..3fe2313 100644 --- a/packaging/start_netx_app.ps1 +++ b/packaging/start_netx_app.ps1 @@ -2,13 +2,11 @@ param( [string]$ProgramRoot = "", [string]$DataRoot = "", [switch]$SkipBrowser = $false, - [switch]$InlineSchedulers = $false, - [switch]$Force = $false + [switch]$InlineSchedulers = $false ) $ErrorActionPreference = "Stop" . "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot @@ -23,39 +21,25 @@ if (-not (Test-Path (Join-Path $prog "netx_api"))) { throw "netx_api not found under program root: $prog" } -Ensure-NetxDataDirectories -DataRoot $data - $map = Import-NetxEnvFile -Path $envPath -# Force data-root paths even if an older .env missed them. -$dataEnv = Get-NetxDataEnvMap -DataRoot $data -foreach ($k in $dataEnv.Keys) { - Set-Item -Path "Env:$k" -Value $dataEnv[$k] -} - Set-Location $prog $env:PYTHONPATH = $prog +# Keep runtime artifacts in the data root (not under Program Files). +$env:NETX_AUTH_MCP_TOKEN_FILE = Join-Path $data "data\auth\mcp_token" +$env:NETX_SCHEDULER_HEARTBEAT_PATH = Join-Path $data "data\runtime\scheduler_heartbeat.json" +$env:NETX_AUTH_SECRET_FILE = Join-Path $data "data\auth\jwt_secret" + $dist = Join-Path $prog "web\dist" if (Test-Path (Join-Path $dist "index.html")) { $env:NETX_UI_DIST_DIR = $dist } $mode = Get-DbMode -EnvMap $map -$hostBind = if ($env:NETX_HOST) { $env:NETX_HOST } else { "127.0.0.1" } -$port = if ($env:NETX_PORT) { [int]$env:NETX_PORT } else { 8890 } - Write-Host "==> Program: $prog" Write-Host "==> Data: $data" Write-Host "==> DB mode: $mode" -if (-not $Force -and (Test-NetxApiHealthy -HostName $hostBind -Port $port)) { - Write-Host "==> NetX already healthy at http://${hostBind}:${port}/ — skip start (use -Force to restart)" -ForegroundColor Green - if (-not $SkipBrowser) { - try { Start-Process "http://${hostBind}:${port}/" } catch {} - } - exit 0 -} - function Get-PgsqlBin { foreach ($b in @( (Join-Path $prog "postgres\pgsql\bin"), @@ -70,20 +54,13 @@ if ($mode -eq "bundled") { $pgBin = Get-PgsqlBin if (-not $pgBin) { throw "bundled_postgres_missing" } $pgData = if ($map["NETX_BUNDLED_PG_DATA_DIR"]) { - $map["NETX_BUNDLED_PG_DATA_DIR"] -replace '/', '\' + $map["NETX_BUNDLED_PG_DATA_DIR"] } else { Join-Path $data "pgdata" } - $pgPort = 15432 - if ($map["NETX_BUNDLED_PG_PORT"]) { - try { $pgPort = [int]$map["NETX_BUNDLED_PG_PORT"] } catch {} - } $pgCtl = Join-Path $pgBin "pg_ctl.exe" $status = & $pgCtl -D $pgData status 2>&1 | Out-String if ($status -notmatch "server is running") { - if (-not (Test-NetxTcpPortFree -HostName "127.0.0.1" -Port $pgPort)) { - throw "port_in_use: 127.0.0.1:$pgPort (bundled Postgres)" - } Write-Host "==> Starting bundled PostgreSQL" if (-not (Test-Path $pgData)) { New-Item -ItemType Directory -Path $pgData -Force | Out-Null @@ -97,48 +74,38 @@ if ($mode -eq "bundled") { } } -try { - $null = Ensure-NetxVenv -ProgramRoot $prog -} catch { - Write-Host "[ERR] $($_.Exception.Message)" -ForegroundColor Red - Write-Host " Tip: run Start Menu → NetX → First-time setup once as Administrator," -ForegroundColor Yellow - Write-Host " or install a Setup built with packaging\\build_release.ps1 -CreateVenv." -ForegroundColor Yellow - exit 1 +# Ensure venv exists for start_netx.ps1 +$venvPy = Join-Path $prog ".venv\Scripts\python.exe" +if (-not (Test-Path $venvPy)) { + Write-Host "==> Creating .venv (one-time)" + $pyCmd = Get-Command python -ErrorAction SilentlyContinue + if (-not $pyCmd) { throw "python_not_found: install Python 3.11+ and re-run" } + & $pyCmd.Source -m venv (Join-Path $prog ".venv") + & $venvPy -m pip install --upgrade pip + & $venvPy -m pip install -r (Join-Path $prog "requirements.txt") + if ($LASTEXITCODE -ne 0) { throw "pip_install_failed" } } -if (-not (Test-NetxTcpPortFree -HostName $hostBind -Port $port)) { - if (Test-NetxApiHealthy -HostName $hostBind -Port $port) { - Write-Host "==> Port $port already serves NetX — skip start" -ForegroundColor Green - exit 0 - } - throw "port_in_use: ${hostBind}:${port} occupied by non-NetX process" -} +$hostBind = if ($env:NETX_HOST) { $env:NETX_HOST } else { "127.0.0.1" } +$port = if ($env:NETX_PORT) { [int]$env:NETX_PORT } else { 8890 } $startScript = Join-Path $prog "scripts\start_netx.ps1" if (-not (Test-Path $startScript)) { throw "start_netx.ps1 not found at $startScript" } +$psArgs = @( + "-ExecutionPolicy", "Bypass", "-File", $startScript, + "-SkipInstall", "-Background", + "-BindHost", $hostBind, "-Port", "$port" +) +if ($InlineSchedulers) { $psArgs += "-InlineSchedulers" } + Write-Host "==> Starting NetX (API + workers; UI via API on :$port)" -# Run in-process with -Background so this console exits after /health (nested -# Start-Process -Wait on a Hidden child often never returns under UAC update). -$startArgs = @{ - SkipInstall = $true - Background = $true - BindHost = $hostBind - Port = $port -} -if ($InlineSchedulers) { $startArgs["InlineSchedulers"] = $true } -& $startScript @startArgs -if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +& powershell @psArgs +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $url = "http://${hostBind}:${port}/" -if (-not (Test-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 2)) { - if (-not (Wait-NetxApiHealthy -HostName $hostBind -Port $port -TimeoutSec 30)) { - Write-Host "[ERR] NetX started but /health not ready: $url" -ForegroundColor Red - exit 1 - } -} Write-Host "==> Open: $url" -ForegroundColor Green if (-not $SkipBrowser) { try { Start-Process $url } catch {} diff --git a/packaging/stop_netx_app.ps1 b/packaging/stop_netx_app.ps1 index 48d99c1..174c631 100644 --- a/packaging/stop_netx_app.ps1 +++ b/packaging/stop_netx_app.ps1 @@ -1,13 +1,11 @@ param( [string]$ProgramRoot = "", [string]$DataRoot = "", - [switch]$KeepPostgres = $false, - [switch]$KillTray = $false + [switch]$KeepPostgres = $false ) $ErrorActionPreference = "Continue" . "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Hidden $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot @@ -23,17 +21,11 @@ if (-not (Test-Path $stopScript)) { } if (Test-Path $stopScript) { Write-Host "==> Stopping NetX processes" - Start-NetxPowerShell -File $stopScript -Arguments @("-Force") ` - -WorkingDirectory $prog -WindowStyle Hidden -Wait | Out-Null + & powershell -ExecutionPolicy Bypass -File $stopScript -Force } else { Write-Host "[WARN] stop_netx.ps1 not found" } -if ($KillTray) { - Write-Host "==> Stopping NetX tray icons" - Stop-NetxTrayProcesses -ProgramRoot $prog -} - $mode = Get-DbMode -EnvMap $map if ($mode -eq "bundled" -and -not $KeepPostgres) { $pgBinCandidates = @( diff --git a/packaging/uninstall_delete_data.ps1 b/packaging/uninstall_delete_data.ps1 deleted file mode 100644 index 707d0cd..0000000 --- a/packaging/uninstall_delete_data.ps1 +++ /dev/null @@ -1,53 +0,0 @@ -param( - [string]$DataRoot = "" -) - -# Post-uninstall optional data wipe with retries (handles briefly locked runtime logs). - -$ErrorActionPreference = "Continue" -if (-not $DataRoot) { - $DataRoot = Join-Path ([Environment]::GetFolderPath("CommonApplicationData")) "NetX" -} - -if (-not (Test-Path -LiteralPath $DataRoot)) { - Write-Host "==> Data root already gone: $DataRoot" - exit 0 -} - -Write-Host "==> Deleting data root: $DataRoot" - -# Kill anything still holding files under data root. -Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | Where-Object { - $_.CommandLine -and ($_.CommandLine.IndexOf($DataRoot, [StringComparison]::OrdinalIgnoreCase) -ge 0) -} | ForEach-Object { - try { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue } catch {} - try { & taskkill.exe /F /PID $_.ProcessId 2>$null | Out-Null } catch {} -} - -$ok = $false -for ($i = 1; $i -le 5; $i++) { - try { - cmd /c "rmdir /s /q `"$DataRoot`"" - } catch {} - if (-not (Test-Path -LiteralPath $DataRoot)) { - $ok = $true - break - } - Start-Sleep -Seconds 1 -} - -if (-not $ok -and (Test-Path -LiteralPath $DataRoot)) { - try { - takeown /F $DataRoot /R /D Y | Out-Null - icacls $DataRoot /grant Administrators:F /T | Out-Null - cmd /c "rmdir /s /q `"$DataRoot`"" - } catch {} -} - -if (Test-Path -LiteralPath $DataRoot) { - Write-Host "[WARN] Could not fully delete $DataRoot" - exit 1 -} - -Write-Host "==> Data root deleted" -exit 0 diff --git a/packaging/uninstall_prepare.ps1 b/packaging/uninstall_prepare.ps1 deleted file mode 100644 index 05e581b..0000000 --- a/packaging/uninstall_prepare.ps1 +++ /dev/null @@ -1,71 +0,0 @@ -param( - [string]$ProgramRoot = "", - [string]$DataRoot = "" -) - -# Fast, non-blocking uninstall prep for Inno [UninstallRun]. -# Must return within a few seconds and never kill unins000.exe / _unins.tmp. - -$ErrorActionPreference = "Continue" - -$prog = if ($ProgramRoot) { $ProgramRoot } else { "C:\Program Files\NetX" } -$data = if ($DataRoot) { $DataRoot } else { Join-Path $env:ProgramData "NetX" } -$prog = ($prog -replace '/', '\').TrimEnd('\') -$data = ($data -replace '/', '\').TrimEnd('\') - -function Test-Protected([string]$Name, [string]$Cmd) { - if ($Name -match '^(unins\d*|_unins\.tmp|setup)\.exe$') { return $true } - if ($Cmd -match 'unins\d*\.exe|_unins\.tmp|uninstall_prepare\.ps1|uninstall_delete_data\.ps1') { return $true } - return $false -} - -function Stop-Pid([int]$Id, [string]$Label) { - if ($Id -le 4 -or $Id -eq $PID) { return } - Write-Host "stop $Id $Label" - try { Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue } catch {} -} - -Write-Host "==> uninstall_prepare fast-stop prog=$prog" - -$patterns = @( - 'netx_tray\.ps1', - 'netx_api\.(main|worker|biz_state_worker)', - 'start_netx_app\.ps1', - 'stop_netx_app\.ps1', - 'launch_shortcut\.ps1', - [regex]::Escape((Join-Path $prog '.venv\Scripts\python.exe')), - [regex]::Escape((Join-Path $prog 'postgres\pgsql\bin')) -) - -Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | ForEach-Object { - $cl = [string]$_.CommandLine - $name = [string]$_.Name - if (-not $cl) { return } - if (Test-Protected -Name $name -Cmd $cl) { return } - foreach ($pat in $patterns) { - if ($cl -match $pat) { - Stop-Pid -Id ([int]$_.ProcessId) -Label $name - break - } - } -} - -# Best-effort postgres stop (no hang: immediate + ignore) -$pgCtl = Join-Path $prog "postgres\pgsql\bin\pg_ctl.exe" -$pgData = Join-Path $data "pgdata" -if ((Test-Path $pgCtl) -and (Test-Path $pgData)) { - try { - $p = Start-Process -FilePath $pgCtl -ArgumentList @('-D', $pgData, 'stop', '-m', 'immediate') ` - -WindowStyle Hidden -PassThru - if (-not $p.WaitForExit(5000)) { - try { Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue } catch {} - } - } catch {} -} - -try { - Remove-ItemProperty -Path "HKCU:\Software\Microsoft\Windows\CurrentVersion\Run" -Name "NetX" -ErrorAction SilentlyContinue -} catch {} - -Write-Host "==> uninstall_prepare done" -exit 0 diff --git a/packaging/update_netx.ps1 b/packaging/update_netx.ps1 index 02c128b..1ddd0ec 100644 --- a/packaging/update_netx.ps1 +++ b/packaging/update_netx.ps1 @@ -9,7 +9,6 @@ param( $ErrorActionPreference = "Stop" . "$PSScriptRoot\_common.ps1" -Assert-NetxAdminOrRelaunch -ScriptPath $PSCommandPath -BoundParameters $PSBoundParameters -WindowStyle Normal $prog = Get-NetxProgramRoot -Override $ProgramRoot $data = Get-NetxDataRoot -ProgramRoot $prog -Override $DataRoot @@ -23,14 +22,10 @@ New-Item -ItemType Directory -Path $work -Force | Out-Null try { Write-Host "==> Extracting update package" - # Preferred end-user path is NetX-Setup-*.exe (check_update / offline installer). - # This script remains for legacy/dev .zip packages (build_release.ps1 -CreateZip). if ($PackagePath.ToLowerInvariant().EndsWith(".zip")) { Expand-Archive -Path $PackagePath -DestinationPath $work -Force - } elseif ($PackagePath.ToLowerInvariant().EndsWith(".exe")) { - throw "unsupported_package: run NetX-Setup-*.exe (upgrade keeps DB), or use check_update.ps1 -Apply" } else { - throw "unsupported_package: use a .zip from build_release.ps1 -CreateZip, or NetX-Setup-*.exe" + throw "unsupported_package: use a .zip built by build_release.ps1" } $src = $work @@ -59,35 +54,7 @@ try { if (Test-Path $envFile) { Copy-Item $envFile (Join-Path $bak ".env") } - # Best-effort logical backup when psql is available (bundled or PATH). - $map = Read-DotEnv -Path $envFile - $pgDump = $null - foreach ($c in @( - (Join-Path $prog "postgres\pgsql\bin\pg_dump.exe"), - (Join-Path $PSScriptRoot "postgres\pgsql\bin\pg_dump.exe") - )) { - if (Test-Path $c) { $pgDump = $c; break } - } - if (-not $pgDump) { - $cmd = Get-Command pg_dump -ErrorAction SilentlyContinue - if ($cmd) { $pgDump = $cmd.Source } - } - if ($pgDump -and $map["NETX_DATABASE_URL"] -match 'postgresql\+?[^:]*://([^:]+):([^@]+)@([^:/]+):?(\d+)?/([^?\s]+)') { - $u = $Matches[1]; $p = [uri]::UnescapeDataString($Matches[2]); $h = $Matches[3] - $pt = if ($Matches[4]) { $Matches[4] } else { "5432" } - $dbn = $Matches[5] - $dumpOut = Join-Path $bak "netx.dump" - Write-Host "==> pg_dump -> $dumpOut" - $env:PGPASSWORD = $p - try { - & $pgDump -h $h -p $pt -U $u -d $dbn -Fc -f $dumpOut - } catch { - Write-Host "[WARN] pg_dump failed: $($_.Exception.Message)" -ForegroundColor Yellow - } finally { - Remove-Item Env:PGPASSWORD -ErrorAction SilentlyContinue - } - } - Write-Host "==> Backup: $bak" + Write-Host "==> Backup marker: $bak (data/pgdata left in place; optional pg_dump not run)" } Write-Host "==> Stopping services" @@ -95,8 +62,7 @@ try { -ProgramRoot $prog -DataRoot $data # Replace program layers only — never wipe data root. - # Include python/ (portable runtime) — required with shipped .venv since 0.4.6. - $replaceDirs = @("netx_api", "alembic", "web", "packaging", "scripts", "postgres", "packages", "python") + $replaceDirs = @("netx_api", "alembic", "web", "packaging", "scripts", "postgres", "packages") foreach ($name in $replaceDirs) { $from = Join-Path $src $name $to = Join-Path $prog $name @@ -113,7 +79,7 @@ try { Copy-Item -Path $from -Destination (Join-Path $prog $f) -Force } } - # Legacy top-level runtime/ (older packages) + shipped .venv + # Optional runtime / .venv shipped in package if (Test-Path (Join-Path $src "runtime")) { $rt = Join-Path $prog "runtime" if (Test-Path $rt) { Remove-Item -Recurse -Force $rt } @@ -126,21 +92,11 @@ try { Copy-Item -Path (Join-Path $src ".venv") -Destination $venvTo -Recurse -Force } - # Builder-path pyvenv.cfg → local python/runtime (same as first install). - if (Get-Command Repair-NetxShippedVenv -ErrorAction SilentlyContinue) { - if (Repair-NetxShippedVenv -ProgramRoot $prog) { - Write-Host "==> Relinked .venv to bundled python/runtime" -ForegroundColor Green - } - } - Write-Host "==> Update files applied" -ForegroundColor Green if (-not $NoStart) { - # Same process (no nested powershell) so the update console can exit cleanly. - & (Join-Path $PSScriptRoot "start_netx_app.ps1") ` + & powershell -ExecutionPolicy Bypass -File (Join-Path $PSScriptRoot "start_netx_app.ps1") ` -ProgramRoot $prog -DataRoot $data -SkipBrowser - if ($LASTEXITCODE -and $LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } - Write-Host "==> Update complete. You can close this window." -ForegroundColor Green } finally { if (Test-Path $work) { Remove-Item -Recurse -Force $work -ErrorAction SilentlyContinue diff --git a/pyproject.toml b/pyproject.toml index 1a39136..a2836f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netx-ops" -version = "0.4.13" +version = "0.3.0" description = "netx operations tool: alarm-centric workflows with REST API and stdio MCP" readme = "README.md" requires-python = ">=3.11" @@ -43,4 +43,3 @@ netx-topology-mcp = "netx_topology_mcp.server:main" [tool.setuptools.packages.find] where = ["."] include = ["netx_api*"] - diff --git a/scripts/init_pg.ps1 b/scripts/init_pg.ps1 index cfcb909..d9c02de 100644 --- a/scripts/init_pg.ps1 +++ b/scripts/init_pg.ps1 @@ -1,4 +1,4 @@ -param( +param( [string]$PgHost = "127.0.0.1", [int]$PgPort = 5432, [string]$SuperUser = "postgres", diff --git a/scripts/start_netx.ps1 b/scripts/start_netx.ps1 index d1ba8ea..ed2caec 100644 --- a/scripts/start_netx.ps1 +++ b/scripts/start_netx.ps1 @@ -23,24 +23,9 @@ if ($Backgtound -and -not $Background) { $projectRoot = Split-Path -Parent $PSScriptRoot Set-Location $projectRoot -$packCommon = Join-Path $projectRoot "packaging\_common.ps1" -if (Test-Path -LiteralPath $packCommon) { - . $packCommon - if (Get-Command Repair-NetxShippedVenv -ErrorAction SilentlyContinue) { - $null = Repair-NetxShippedVenv -ProgramRoot $projectRoot - } -} - -# Prefer writable data-root runtime dir (Program Files is not writable after Setup install). -if ($env:NETX_RUN_DIR) { - $runDir = $env:NETX_RUN_DIR -} elseif ($env:NETX_SCHEDULER_HEARTBEAT_PATH) { - $runDir = Split-Path -Parent ($env:NETX_SCHEDULER_HEARTBEAT_PATH -replace '/', '\') -} else { - $runDir = Join-Path $PSScriptRoot ".run" -} +$runDir = Join-Path $PSScriptRoot ".run" if (-not (Test-Path $runDir)) { - New-Item -ItemType Directory -Path $runDir -Force | Out-Null + New-Item -ItemType Directory -Path $runDir | Out-Null } $pidFile = Join-Path $runDir "netx.pid" @@ -54,17 +39,13 @@ $webPidFile = Join-Path $runDir "web.pid" $webLogFile = Join-Path $runDir "web.out.log" $webErrFile = Join-Path $runDir "web.err.log" -$venvPython = Join-Path $projectRoot ".venv\Scripts\python.exe" -if (Get-Command Test-NetxVenvRunnable -ErrorAction SilentlyContinue) { - if (-not (Test-NetxVenvRunnable -VenvPython $venvPython)) { - throw "venv_not_runnable: reinstall NetX or run setup as administrator" - } -} elseif (-not (Test-Path -LiteralPath $venvPython)) { +$venvPython = Join-Path $projectRoot ".venv\\Scripts\\python.exe" +if (-not (Test-Path $venvPython)) { Write-Host "==> .venv not found, creating virtual environment" python -m venv (Join-Path $projectRoot ".venv") - if (-not (Test-Path -LiteralPath $venvPython)) { - throw "failed_to_create_venv" - } +} +if (-not (Test-Path $venvPython)) { + throw "failed_to_create_venv" } $pythonExe = $venvPython @@ -272,11 +253,7 @@ if ($Background) { Show-LogTail -Path $logFile exit 1 } - # Fresh installs run Alembic upgrades on first boot; 45s is often too short. - $healthWaitSec = 180 - if ($env:NETX_START_HEALTH_WAIT_SEC) { - try { $healthWaitSec = [int]$env:NETX_START_HEALTH_WAIT_SEC } catch {} - } + $healthWaitSec = 45 if (-not (Test-NetxApiListening -HostName $BindHost -LocalPort $Port -WaitSec $healthWaitSec)) { Write-Host "[ERR] Port $Port not listening /health not OK within ${healthWaitSec}s (process may be stuck on DB or schema migration)." -ForegroundColor Red Show-LogTail -Path $errFile diff --git a/scripts/stop_netx.ps1 b/scripts/stop_netx.ps1 index 1106bb3..2d91466 100644 --- a/scripts/stop_netx.ps1 +++ b/scripts/stop_netx.ps1 @@ -1,4 +1,4 @@ -param( +param( [int]$Port = 8890, [int]$WebPort = 5173, # Windows often ignores graceful Stop-Process on python; default hard-kill. @@ -9,13 +9,7 @@ $ErrorActionPreference = "Continue" $useForce = if ($NoForce) { $false } else { [bool]$Force } -if ($env:NETX_RUN_DIR) { - $runDir = $env:NETX_RUN_DIR -} elseif ($env:NETX_SCHEDULER_HEARTBEAT_PATH) { - $runDir = Split-Path -Parent ($env:NETX_SCHEDULER_HEARTBEAT_PATH -replace '/', '\') -} else { - $runDir = Join-Path $PSScriptRoot ".run" -} +$runDir = Join-Path $PSScriptRoot ".run" $pidFile = Join-Path $runDir "netx.pid" $workerPidFile = Join-Path $runDir "worker.pid" $webPidFile = Join-Path $runDir "web.pid" diff --git a/tests/test_biz_compare_templates.py b/tests/test_biz_compare_templates.py deleted file mode 100644 index a1f76f0..0000000 --- a/tests/test_biz_compare_templates.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Template validation, rule semantics, protected references and bounded queries.""" -from copy import deepcopy - -import pytest -from fastapi import HTTPException -from sqlalchemy import create_engine, event -from sqlalchemy.orm import sessionmaker - -from netx_api.db import Base -from netx_api.models import BizCompareJob, BizCompareRun, BizCompareTemplate, BizStateBatchCommand, BizStateBatch -from netx_api.biz_state import compare_service as svc -from netx_api.biz_state.compare_engine import compare_rows, row_key -from netx_api.biz_state.compare_rules import eval_leaf_filter, values_equal -from netx_api.biz_state.compare_sql import compile_row_filters_sql, _rk_sql, _field_differs_sql -from netx_api.biz_state_router import TemplateIn, TemplatePatchIn - - -@pytest.fixture -def db(): - engine = create_engine("sqlite+pysqlite:///:memory:") - Base.metadata.create_all(engine) - session = sessionmaker(bind=engine, expire_on_commit=False)() - yield session - session.close() - engine.dispose() - - -def body(): - return {"name": "Test", "metrics": [{"sheet_id": "s", "metric_id": "custom", "key_fields": ["id"], - "compare_fields": ["count"], "iface_fields": ["port"], "ignore_port_changes": False, - "row_filters": [], "field_rules": []}]} - - -@pytest.mark.parametrize("bad", [ - {"key_fields": []}, {"key_fields": ["id", "id"]}, {"metric_id": ""}, - {"row_filters": [{"field": "state", "op": "typo", "value": "up"}]}, - {"row_filters": [{"field": "state", "op": "regex", "value": "["}]}, - {"row_filters": [{"any": []}]}, {"row_filters": [{"field": "id", "op": "in", "value": "a,b"}]}, - {"field_rules": [{"field": "count", "compare": "typo"}]}, - {"field_rules": [{"field": "count", "normalize": "typo"}]}, - {"field_rules": [{"field": "count", "tolerance": -1}]}, - {"field_rules": [{"field": "count", "tolerance": float("inf")}]}, - {"field_rules": [{"field": "count", "tolerance": float("nan")}]}, - {"field_rules": [{"field": "count"}, {"field": "count"}]}, -]) -def test_invalid_sheet_never_silently_drops_or_saves(db, bad): - data = body() - data["metrics"].append({**deepcopy(data["metrics"][0]), "sheet_id": "second", **bad}) - with pytest.raises(HTTPException) as exc: - svc.create_template(db, data) - assert exc.value.status_code == 400 - assert "metrics[1]" in exc.value.detail["path"] - assert db.query(BizCompareTemplate).count() == 0 - - -def test_template_validation_precedes_mutation_and_rejects_duplicate_alias(db): - tpl = svc.create_template(db, body()) - data = body() - data["name"] = "Wrong name" - data["metrics"][0]["key_fields"] = [] - with pytest.raises(HTTPException): - svc.update_template(db, tpl["id"], data) - assert db.get(BizCompareTemplate, tpl["id"]).name == "Test" - with pytest.raises(HTTPException): - svc.create_template(db, {**body(), "iface_normalize_rules": [{"from": "GE", "to": "a"}, {"from": "ge", "to": "b"}]}) - - -def test_api_port_policy_roundtrip_and_legacy_patch(db): - saved = svc.create_template(db, TemplateIn.model_validate(body()).model_dump()) - assert saved["metrics"][0]["ignore_port_changes"] is False - updated = svc.update_template(db, saved["id"], TemplatePatchIn(ignore_port_changes=True).model_dump(exclude_unset=True)) - assert updated["metrics"][0]["ignore_port_changes"] is True - legacy = svc.create_template(db, {"name": "legacy", "metric_id": "custom", "key_fields": ["id"], "ignore_port_changes": False}) - assert legacy["metrics"][0]["ignore_port_changes"] is False - - -@pytest.mark.parametrize("value", [0, False]) -def test_filter_and_identity_preserve_false_and_zero(value): - assert not eval_leaf_filter({"x": value}, {"field": "x", "op": "empty"}) - assert eval_leaf_filter({"x": value}, {"field": "x", "op": "eq", "value": value}) - assert row_key({"id": value}, ["id"]) != row_key({"id": None}, ["id"]) - result = compare_rows(before_rows=[{"id": value}], after_rows=[{"id": None}], key_fields=["id"], iface_fields=[], compare_fields=[], port_map={}) - assert result["summary"]["removed"] == 1 and result["summary"]["added"] == 1 - _, params = compile_row_filters_sql([{"field": "x", "op": "eq", "value": value}]) - assert list(params.values()) == ["0" if value is not False else "false"] - - -@pytest.mark.parametrize("mode", ["numeric", "percent"]) -def test_missing_numeric_value_is_not_zero(mode): - assert not values_equal(None, 0, rule={"compare": mode, "tolerance": 5}) - assert values_equal(None, "", rule={"compare": mode}) - assert values_equal("+.5", "0.5", rule={"compare": mode}) - assert values_equal("1.", "1", rule={"compare": mode}) - assert values_equal("NaN", "NaN", rule={"compare": mode}) # textual fallback - - -def test_sql_filter_literal_contains_and_collision_free_composite_keys(): - sql, params = compile_row_filters_sql([{"field": "x", "op": "contains", "value": "a%_\\b"}]) - assert "ESCAPE" in sql - assert list(params.values()) == ["%a\\%\\_\\\\b%"] - sql, _ = compile_row_filters_sql([{"field": "x", "op": "contains", "value": ""}]) - assert "FALSE" in sql - assert "jsonb_build_array" in _rk_sql(["a", "b"]) - assert "concat_ws" not in _rk_sql(["a", "b"]) - assert "[+-]?" in _field_differs_sql("b", "a", {"compare": "numeric"}) - - -def test_sheet_order_uses_one_aggregate_query_without_raw_text(db): - for bid, metric, count in (("b", "large", 20), ("a", "large", 30), ("b", "small", 3)): - db.add(BizStateBatchCommand(id=f"{bid}-{metric}", batch_id=bid, metric_id=metric, row_count=count, raw_text="x" * 100000)) - db.commit() - sql = [] - event.listen(db.get_bind(), "before_cursor_execute", lambda c, cur, statement, p, ctx, many: sql.append(statement)) - sheets = [{"metric_id": "large", "sheet_id": "l1"}, {"metric_id": "small", "sheet_id": "s"}, {"metric_id": "large", "sheet_id": "l2"}] - ordered = svc._order_sheets_small_first(db, sheets, before_batch_id="b", after_batch_id="a") - assert [s["sheet_id"] for s in ordered] == ["s", "l1", "l2"] - assert len(sql) == 1 and "raw_text" not in sql[0] - - -def test_referenced_template_missing_sheets_and_running_delete_protected(db): - tpl = svc.create_template(db, body()) - db.add(BizCompareJob(id="j", template_id=tpl["id"], enabled_sheet_ids=["s"])) - db.add(BizCompareRun(id="r", job_id="j", status="running")) - db.commit() - for action in (lambda: svc.delete_template(db, tpl["id"]), lambda: svc.delete_job(db, "j"), lambda: svc.delete_run(db, "r")): - with pytest.raises(HTTPException) as exc: - action() - assert exc.value.status_code == 409 - with pytest.raises(HTTPException) as exc: - svc.update_job(db, "j", {"name": "invalid", "enabled_sheet_ids": ["missing"]}) - assert exc.value.detail["error"] == "unknown_enabled_sheets" - assert db.get(BizCompareJob, "j").name != "invalid" - - -def test_template_usage_counts_are_batched(db, monkeypatch): - monkeypatch.setattr(svc, "ensure_default_templates", lambda db: None) - monkeypatch.setattr(svc, "upgrade_builtin_split_sheets", lambda db: None) - used = svc.create_template(db, body()) - unused = svc.create_template(db, {**body(), "name": "Unused"}) - db.add_all([BizCompareJob(id="j1", template_id=used["id"]), BizCompareJob(id="j2", template_id=used["id"])]) - db.commit() - sql = [] - event.listen(db.get_bind(), "before_cursor_execute", lambda c, cur, statement, p, ctx, many: sql.append(statement)) - counts = {t["id"]: t["job_count"] for t in svc.list_templates(db)} - assert counts == {used["id"]: 2, unused["id"]: 0} - assert len(sql) == 2 # one template query and one grouped reference query - - -@pytest.mark.parametrize("status", ["queued", "running", "failed", "cancelled", "partial"]) -def test_compare_rejects_incomplete_sources_before_enqueue(db, status): - tpl = svc.create_template(db, body()) - db.add(BizStateBatch(id="before", status="success")) - db.add(BizStateBatch(id="after", status=status)) - db.add(BizCompareJob(id="j", template_id=tpl["id"], before_batch_id="before", after_batch_id="after")) - db.commit() - with pytest.raises(HTTPException) as exc: - svc._validate_compare_job(db, "j") - assert exc.value.detail == "source_batch_not_complete" - assert db.query(BizCompareRun).count() == 0 - - -def test_compare_rejects_partially_stale_sheet_scope_after_template_edit(db): - tpl = svc.create_template(db, body()) - db.add_all([BizStateBatch(id="before", status="success"), BizStateBatch(id="after", status="success"), - BizCompareJob(id="j", template_id=tpl["id"], before_batch_id="before", after_batch_id="after", enabled_sheet_ids=["s", "removed"])]) - db.commit() - with pytest.raises(HTTPException) as exc: - svc._validate_compare_job(db, "j") - assert exc.value.detail == {"error": "unknown_enabled_sheets", "sheet_ids": ["removed"]} - assert db.query(BizCompareRun).count() == 0 - - -@pytest.mark.parametrize("bad", [ - {"key_fields": []}, - {"row_filters": [{"field": "id", "op": "typo", "value": "a"}]}, - {"row_filters": [{"field": "id", "op": "regex", "value": "["}]}, - {"field_rules": [{"field": "count", "compare": "typo"}]}, -]) -def test_legacy_invalid_template_cannot_execute_as_partial_or_fail_open(db, bad): - tpl = svc.create_template(db, body()) - row = db.get(BizCompareTemplate, tpl["id"]) - row.metrics_json = [*row.metrics_json, {**body()["metrics"][0], "sheet_id": "invalid", **bad}] - db.add_all([BizStateBatch(id="before", status="success"), BizStateBatch(id="after", status="success"), - BizCompareJob(id="j", template_id=tpl["id"], before_batch_id="before", after_batch_id="after")]) - db.commit() - with pytest.raises(HTTPException) as exc: - svc._validate_compare_job(db, "j") - assert exc.value.detail["error"] == "invalid_template" - assert "metrics[1]" in exc.value.detail["path"] - assert db.query(BizCompareRun).count() == 0 diff --git a/tests/test_biz_migration_evaluate.py b/tests/test_biz_migration_evaluate.py index 99d650f..0ac413c 100644 --- a/tests/test_biz_migration_evaluate.py +++ b/tests/test_biz_migration_evaluate.py @@ -733,7 +733,7 @@ class EvaluateMetricDualTests(unittest.TestCase): migrated = [r for r in out["rows"] if r["verdict"] == "migrated"] self.assertTrue(migrated, out["rows"]) - def test_out_of_expect_ignore_cannot_hide_business_drift(self): + def test_out_of_expect_ignore_no_red(self): old_base = [{"interface": "gei-keep", "admin": "up", "phy": "up", "prot": "up"}] old_cur: list[dict] = [] new_cur = [{"interface": "gei-keep", "admin": "up", "phy": "up", "prot": "up"}] @@ -753,9 +753,9 @@ class EvaluateMetricDualTests(unittest.TestCase): out_of_expect="ignore", ) reds = [r for r in out["rows"] if r["color"] == "red"] - self.assertTrue(reds, out["rows"]) + self.assertFalse(reds, out["rows"]) involved = [r for r in out["rows"] if r["verdict"] != "not_involved"] - self.assertTrue(involved, out["rows"]) + self.assertFalse(involved, out["rows"]) def test_unexpected_loss_is_anomaly(self): old_base = [{"interface": "gei-keep", "admin": "up", "phy": "up", "prot": "up"}] @@ -823,7 +823,7 @@ class EvaluateMetricDualTests(unittest.TestCase): self.assertFalse(out["new_baseline_missing"]) self.assertEqual(out["new_baseline_mode"], "port_mapped") - def test_empty_collected_new_baseline_is_valid(self): + def test_empty_provided_new_baseline_falls_back_or_missing(self): old_base = [{"peer": "1.1.1.1", "state": "Established"}] new_cur = [{"peer": "1.1.1.1", "state": "Established"}] expect = parse_expect_set({"items": [{"metric_id": "bgp_peer", "key": "1.1.1.1"}]}) @@ -841,8 +841,8 @@ class EvaluateMetricDualTests(unittest.TestCase): window_active=True, sheet_override={}, ) - self.assertFalse(out["new_baseline_missing"]) - self.assertEqual(out["new_baseline_mode"], "provided") + self.assertTrue(out["new_baseline_missing"]) + self.assertEqual(out["new_baseline_mode"], "empty") # Still reports presence as added vs empty before — callers should heed the flag. kinds = {r["new_kind"] for r in out["rows"] if r.get("in_expect")} self.assertIn("added", kinds) @@ -1153,10 +1153,7 @@ class PortMapScopeTests(unittest.TestCase): ) keys = {r["key_str"] for r in out["rows"]} self.assertIn("10.0.0.1|gei-old", keys) - self.assertIn("10.0.0.2|gei-keep", keys) - outside = next(r for r in out["rows"] if r["key_str"] == "10.0.0.2|gei-keep") - self.assertEqual(outside["color"], "red") - self.assertFalse(outside["in_expect"]) + self.assertNotIn("10.0.0.2|gei-keep", keys) migrated = [r for r in out["rows"] if r["verdict"] == "migrated"] self.assertTrue(migrated, out["rows"]) self.assertEqual(migrated[0]["new_key_str"], "10.0.0.1|gei-new") diff --git a/tests/test_biz_migration_evidence.py b/tests/test_biz_migration_evidence.py index 8038f4e..e22a259 100644 --- a/tests/test_biz_migration_evidence.py +++ b/tests/test_biz_migration_evidence.py @@ -138,13 +138,10 @@ class EvidencePersistTests(unittest.TestCase): return t def _mk_batch(self, bid: str, task_id: str) -> BizStateBatch: - task = self.db.get(BizStateTask, task_id) now = _utcnow() b = BizStateBatch( id=bid, task_id=task_id, - collect_group_id=task.collect_group_id or "", - collect_round_id="test-" + task.collect_group_id if task.collect_group_id else "", status="success", started_at=now, ended_at=now, @@ -152,8 +149,6 @@ class EvidencePersistTests(unittest.TestCase): command_count=1, ) self.db.add(b) - self.db.add(BizStateBatchCommand(id=f"receipt-{bid}", batch_id=bid, - metric_id="interface_brief", parse_status="ok", row_count=0)) self.db.commit() return b @@ -238,7 +233,7 @@ class EvidencePersistTests(unittest.TestCase): raw_command="show interface brief", interface="GE-old", ) - # Successfully collected empty new baseline is valid + # New baseline empty → port_mapped mode uses mapped old # Old current: gone (no rows) — migration off old # New current: xgei-new up self._seed_if_row( diff --git a/tests/test_biz_migration_hf_decouple.py b/tests/test_biz_migration_hf_decouple.py index cf9a575..4fc3c53 100644 --- a/tests/test_biz_migration_hf_decouple.py +++ b/tests/test_biz_migration_hf_decouple.py @@ -14,7 +14,6 @@ from netx_api.models import ( BizMigrationProject, BizMonitorTemplate, BizStateBatch, - BizStateBatchCommand, BizStateTask, ) from netx_api.biz_migration import service as mig @@ -47,7 +46,6 @@ class HfDecoupleFlowTests(unittest.TestCase): ) Base.metadata.create_all(bind=engine) self.db = TestingSession() - self.Session = TestingSession self.mt = BizMonitorTemplate( id="mt1", @@ -95,22 +93,16 @@ class HfDecoupleFlowTests(unittest.TestCase): return t def _mk_batch(self, bid: str, task_id: str) -> BizStateBatch: - task = self.db.get(BizStateTask, task_id) now = datetime.utcnow() b = BizStateBatch( id=bid, task_id=task_id, - collect_group_id=task.collect_group_id or "", - collect_round_id="test-" + task.collect_group_id if task.collect_group_id else "", status="success", started_at=now, ended_at=now, row_count=1, ) self.db.add(b) - for mid in ("interface_brief", "arp"): - self.db.add(BizStateBatchCommand(id=f"receipt-{bid}-{mid}", batch_id=bid, - metric_id=mid, parse_status="ok", row_count=0)) self.db.commit() return b @@ -144,64 +136,6 @@ class HfDecoupleFlowTests(unittest.TestCase): self.assertNotEqual(out["old_hf_task_id"], "old_p") self.assertNotEqual(out["new_hf_task_id"], "new_p") - def test_same_interval_metrics_are_separate_paired_tasks(self) -> None: - out = self._create_project(collect_metric_ids=["interface_brief", "arp"]) - old, new = out["old_hf_bindings"], out["new_hf_bindings"] - self.assertEqual(len(old), 2) - self.assertEqual({b["interval_sec"] for b in old}, {60}) - for o, n in zip(old, new): - self.assertEqual(len(o["metric_ids"]), 1) - self.assertTrue(o["collect_group_id"]) - self.assertEqual(o["collect_group_id"], n["collect_group_id"]) - self.assertEqual(o["metric_ids"], n["metric_ids"]) - - def test_dynamic_routes_manual_only_and_removal_cancels_queued_pairs(self) -> None: - from netx_api.models import BizStateTaskItem, BizStateTaskItemBinding - - for tid in ("old_p", "new_p"): - self.db.add(BizStateTaskItem(id="route-item-" + tid, task_id=tid, source_profile_id="zte.ip_route_vrf", kind="catalog", enabled=True)) - self.db.add(BizStateTaskItemBinding(id="route-bind-" + tid, item_id="route-item-" + tid, placeholder="vrf", value="customer-A")) - self.db.commit() - out = self._create_project() - with mock.patch.object(biz_svc, "_ne_meta", side_effect=_stub_ne_meta): - updated = mig.patch_project(self.db, out["id"], {"collect_metric_ids": ["interface_brief", "ip_route"]}) - routes = [b for side in ("old", "new") for b in updated[f"{side}_hf_bindings"] if "ip_route" in b["metric_ids"]] - self.assertEqual(len(routes), 2) - for idx, binding in enumerate(routes): - task = self.db.get(BizStateTask, binding["task_id"]) - self.assertTrue(task.collect_manual_only) - self.assertEqual(task.status, "paused") - task.collect_running = True - self.db.add(BizStateBatch(id=f"queued-route-{idx}", task_id=task.id, status="queued", collect_round_id="route-round")) - self.db.commit() - with mock.patch("netx_api.biz_state.collect_stop.SessionLocal", self.Session), mock.patch.object(biz_svc, "_ne_meta", side_effect=_stub_ne_meta): - shrunk = mig.patch_project(self.db, out["id"], {"collect_metric_ids": ["interface_brief"]}) - self.db.expire_all() - self.assertEqual(len(shrunk["old_hf_bindings"]), 1) - for idx in range(2): - self.assertEqual(self.db.get(BizStateBatch, f"queued-route-{idx}").status, "cancelled") - - def test_route_profile_preserves_portrait_cli_variant_and_bindings(self) -> None: - from netx_api.models import BizStateTaskItem, BizStateTaskItemBinding - - self.db.add(BizStateTaskItem(id="vrf-item", task_id="old_p", source_profile_id="zte.ip_route_vrf", kind="catalog", enabled=True)) - self.db.add(BizStateTaskItemBinding(id="vrf-bind", item_id="vrf-item", placeholder="vrf", value="customer-A")) - self.db.commit() - out = self._create_project(collect_metric_ids=["interface_brief", "ip_route"]) - binding = next(b for b in out["old_hf_bindings"] if b["metric_ids"] == ["ip_route"]) - item = self.db.query(BizStateTaskItem).filter(BizStateTaskItem.task_id == binding["task_id"]).one() - self.assertEqual(item.source_profile_id, "zte.ip_route_vrf") - copied = self.db.query(BizStateTaskItemBinding).filter(BizStateTaskItemBinding.item_id == item.id).one() - self.assertEqual((copied.placeholder, copied.value), ("vrf", "customer-A")) - - def test_invalid_new_metric_does_not_save_broken_selection(self) -> None: - out = self._create_project() - with self.assertRaises(Exception): - mig.patch_project(self.db, out["id"], {"collect_metric_ids": ["interface_brief", "not-a-metric"]}) - self.db.rollback() - current = self.db.get(BizMigrationProject, out["id"]) - self.assertEqual(mig.resolve_collect_metric_ids(self.db, current), ["interface_brief"]) - def test_ensure_highfreq_does_not_overwrite_portrait(self) -> None: proj = self._create_project() res = self._ensure(proj["id"], interval_sec=60) @@ -236,7 +170,7 @@ class HfDecoupleFlowTests(unittest.TestCase): batch = mig.create_batch( self.db, pid, {"batch_label": "n1", "expect_set": {"ports": ["gei-0/1"]}} ) - with mock.patch.object(mig, "resolve_evaluate_sheets", return_value=([mig.port_sheet_def()], [], {}, [])): + with mock.patch.object(mig, "resolve_evaluate_sheets", return_value=([], [], {}, [])): out = mig.run_evaluate(self.db, batch_id=batch["id"]) self.assertEqual(out["old_batch_id"], "cur_old") self.assertEqual(out["new_batch_id"], "cur_new") @@ -275,7 +209,7 @@ class HfDecoupleFlowTests(unittest.TestCase): called = {c.args[0] for c in disp.call_args_list} self.assertEqual( called, - {proj["old_hf_task_id"]}, # one dispatch atomically enqueues the pair + {proj["old_hf_task_id"], proj["new_hf_task_id"]}, ) self.assertNotIn("old_p", called) diff --git a/tests/test_biz_migration_integrity.py b/tests/test_biz_migration_integrity.py deleted file mode 100644 index 6f2070d..0000000 --- a/tests/test_biz_migration_integrity.py +++ /dev/null @@ -1,408 +0,0 @@ -"""Business continuity, wave scope, sample uncertainty and live-monitor regressions.""" -from datetime import timedelta - -import pytest -from fastapi import HTTPException -from sqlalchemy import create_engine, event -from sqlalchemy.orm import sessionmaker - -from netx_api.db import Base -from netx_api.models import (BizCompareTemplate, BizMonitorTemplate, BizMigrationProject, - BizMigrationBatch, BizMigrationRun, BizMigrationDiff, BizStateTask, BizStateBatch, - BizStateBatchCommand, BizStateMetricRow) -from netx_api.timeutil import utcnow_naive -from netx_api.biz_migration import service as svc -from netx_api.biz_migration.auto_monitor import try_auto_monitor_for_task -from netx_api.biz_migration.evaluate import evaluate_metric_dual, parse_expect_set, _eval_field_op -from netx_api.biz_migration.validation import validate_monitor_rules - - -def evaluate(**changes): - args = dict(metric_id="arp", sheet_id="arp", key_fields=["ip"], iface_fields=["port"], - compare_fields=["mac", "port"], old_baseline_rows=[{"ip": "a", "mac": "good", "port": "old"}], - old_current_rows=[], new_baseline_rows=[], - new_current_rows=[{"ip": "a", "mac": "good", "port": "new"}], - port_map={"old": "new"}, expect=parse_expect_set({"items": [{"metric_id": "arp", "key": "a"}]}), - window_active=True, allow_mapping_scope=False) - return evaluate_metric_dual(**{**args, **changes}) - - -def test_target_content_checked_against_old_business(): - out = evaluate(new_current_rows=[{"ip": "a", "mac": "wrong", "port": "new"}]) - assert out["progress_ok"] == 0 - assert out["rows"][0]["rule_hit"] == "service_content_changed" - assert out["rows"][0]["integrity_changes"][0]["field"] == "mac" - - -def test_content_normalization_and_ignore_follow_compare_template(): - out = evaluate(new_current_rows=[{"ip": "a", "mac": "GOOD", "port": "new"}], - field_rules=[{"field": "mac", "normalize": "upper"}]) - assert out["progress_ok"] == 1 - - -def test_down_target_cannot_pass_permissive_success_rule(): - out = evaluate(new_current_rows=[{"ip": "a", "state": "down"}], - sheet_override={"status_fields": ["state"], "up_values": ["up"], "down_values": ["down"], - "success": [{"old": ["removed"], "new": ["added"]}]}) - assert out["rows"][0]["verdict"] == "lost" - assert out["rows"][0]["color"] == "red" - - -def test_mapping_does_not_expand_wave_selection(): - out = evaluate(old_baseline_rows=[{"ip": "a", "port": "old"}, {"ip": "b", "port": "old2"}], - new_current_rows=[{"ip": "a", "port": "new"}], compare_fields=[], - port_map={"old": "new", "old2": "new2"}) - assert out["progress_total"] == 1 - assert next(r for r in out["rows"] if r["old_key"] == "b")["color"] == "red" - - -def test_port_selection_covers_business_with_nonkey_interface(): - out = evaluate(expect=parse_expect_set({"ports": ["old"]})) - assert out["progress_total"] == out["progress_ok"] == 1 - - -@pytest.mark.parametrize("healthy", [True, False]) -def test_previous_wave_remains_protected_without_changing_current_progress(healthy): - out = evaluate(expect={}, previous_expect=parse_expect_set({"items": [{"metric_id": "arp", "key": "a"}]}), - new_current_rows=[{"ip": "a", "mac": "good", "port": "new"}] if healthy else []) - assert out["progress_total"] == 0 - assert out["rows"][0]["verdict"] == ("migrated_previous" if healthy else "lost") - assert out["anomaly"] == (0 if healthy else 1) - - -@pytest.mark.parametrize("value", [0, False]) -def test_zero_false_are_not_blank_in_keys_or_conditions(value): - assert _eval_field_op({"id": value}, "id", "eq", value) - out = evaluate(old_baseline_rows=[{"ip": value}], new_current_rows=[{"ip": value}], - compare_fields=[], expect=parse_expect_set({"items": [{"metric_id": "arp", "key": [str(value).lower()]}]})) - assert out["rows"][0]["old_key"] == str(value).lower() - - -@pytest.fixture -def db(): - engine = create_engine("sqlite+pysqlite:///:memory:") - Base.metadata.create_all(engine) - with sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)() as session: - now = utcnow_naive() - timedelta(seconds=1) - sheet = {"metric_id": "arp", "sheet_id": "s", "key_fields": ["ip"], "compare_fields": ["mac"]} - session.add(BizCompareTemplate(id="ct", name="cmp", metrics_json=[sheet])) - session.add(BizMonitorTemplate(id="mt", name="mon", compare_template_id="ct", - collect_metric_ids_json=["arp"], defaults_json={}, sheet_overrides_json=[])) - for tid in ("op", "np", "oh", "nh"): - session.add(BizStateTask(id=tid, ne_id=tid, ne_name=tid, purpose="cutover_hf" if tid.endswith("h") else "")) - session.add(BizMigrationProject(id="p", name="p", old_task_id="op", new_task_id="np", - old_hf_task_id="oh", new_hf_task_id="nh", old_baseline_batch_id="ob", new_baseline_batch_id="nb", - monitor_template_id="mt", hf_interval_sec=60, status="active")) - session.add(BizMigrationBatch(id="wave", project_id="p", batch_label="wave", status="active", - expect_set_json={"items": [{"metric_id": "arp", "sheet_id": "s", "key": "a"}]})) - for i, (bid, tid) in enumerate((("ob", "op"), ("nb", "np"), ("oc", "oh"), ("nc", "nh"))): - at = now - timedelta(milliseconds=100 * (4 - i)) - session.add(BizStateBatch(id=bid, task_id=tid, status="success", started_at=at, ended_at=at)) - session.add(BizStateBatchCommand(id="cmd-" + bid, batch_id=bid, metric_id="arp", parse_status="ok", raw_text="secret CLI payload")) - for bid in ("ob", "nc"): - session.add(BizStateMetricRow(id="row-" + bid, batch_id=bid, task_id="op" if bid == "ob" else "nh", - batch_command_id="cmd-" + bid, metric_id="arp", seq=0, data_json={"ip": "a", "mac": "good"})) - session.commit() - yield session - - -def test_collected_empty_baseline_can_pass_acceptance(db): - finished = svc.finish_batch(db, "wave") - assert finished["accept_summary"]["passed"] - assert finished["run"]["summary"]["config_snapshot"]["expect_set"] - - -def paired_samples(db): - for tid in ("oh", "nh"): - db.get(BizStateTask, tid).collect_group_id = "pair" - for bid in ("oc", "nc"): - sample = db.get(BizStateBatch, bid) - sample.collect_group_id = "pair" - sample.collect_round_id = "round" - sample.queued_at = sample.started_at - db.commit() - - -@pytest.mark.parametrize("problem", ["running", "queued", "failed", "round_mismatch"]) -def test_paired_samples_cannot_mix_rounds_or_use_old_success(db, problem): - paired_samples(db) - if problem == "round_mismatch": - db.get(BizStateBatch, "nc").collect_round_id = "another-round" - else: - db.get(BizStateBatch, "nc").status = problem - db.commit() - run = svc.run_evaluate(db, batch_id="wave") - assert not run["summary"]["coverage_complete"] - assert not run["summary"]["anomaly"] # incomplete pair is unknown, not service loss - - -def test_paired_samples_expose_duration_queue_and_real_start_skew(db): - paired_samples(db) - for bid in ("oc", "nc"): - sample = db.get(BizStateBatch, bid) - sample.queued_at = sample.started_at - timedelta(seconds=15) - sample.started_at -= timedelta(seconds=5) - db.commit() - card = svc.run_evaluate(db, batch_id="wave")["summary"]["sheet_cards"][0] - assert card["collect_round_id"] == "round" - assert card["old_queue_sec"] == 10 and card["old_duration_sec"] == 5 - assert card["start_skew_sec"] < 1 - - -def test_unselected_route_does_not_block_regular_acceptance(db): - ct = db.get(BizCompareTemplate, "ct") - ct.metrics_json = [*ct.metrics_json, {"metric_id": "ip_route", "sheet_id": "route", "key_fields": ["prefix"]}] - db.commit() - assert svc.finish_batch(db, "wave")["accept_summary"]["passed"] - - -def test_incremental_evaluation_reuses_route_rows_and_paged_evidence(db): - from unittest.mock import patch - ct = db.get(BizCompareTemplate, "ct") - ct.metrics_json = [*ct.metrics_json, {"metric_id": "ip_route", "sheet_id": "route", "key_fields": ["prefix"], "compare_fields": ["nexthop"]}] - db.get(BizMonitorTemplate, "mt").collect_metric_ids_json = ["arp", "ip_route"] - # Legacy combined samples also exercise compatibility with existing projects. - for bid in ("ob", "nb", "oc", "nc"): - db.add(BizStateBatchCommand(id="route-cmd-" + bid, batch_id=bid, metric_id="ip_route", parse_status="ok")) - for i in range(2000): - db.add(BizStateMetricRow(id=f"route-{i}", batch_id="ob", metric_id="ip_route", data_json={"prefix": str(i), "nexthop": "old"})) - db.commit() - first = svc.run_evaluate(db, batch_id="wave", purpose="auto") - with patch.object(svc, "_load_metric_rows", wraps=svc._load_metric_rows) as load: - second = svc.run_evaluate(db, batch_id="wave", purpose="auto", _metric_ids={"arp"}) - assert all(call.kwargs["metric_id"] == "arp" for call in load.call_args_list) - assert svc.list_run_diffs(db, second["id"], sheet_id="route", color="red", limit=100)["total"] == 2000 - assert db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id == second["id"]).count() == 1 - assert second["summary"]["anomaly"] == first["summary"]["anomaly"] - assert svc.list_run_diffs(db, second["id"], only_expect=True)["total"] == 1 - # Template correction invalidates cached verdicts and evidence. - ct.metrics_json = [ct.metrics_json[0], {**ct.metrics_json[1], "row_filters": [{"field": "prefix", "op": "eq", "value": "1"}]}] - db.commit() - third = svc.run_evaluate(db, batch_id="wave", purpose="auto", _metric_ids={"arp"}) - assert svc.list_run_diffs(db, third["id"], sheet_id="route")["total"] == 1 - - -def test_baseline_picker_never_loads_route_inventory(db): - from unittest.mock import patch - ct = db.get(BizCompareTemplate, "ct") - ct.metrics_json = [*ct.metrics_json, {"metric_id": "ip_route", "sheet_id": "route", "key_fields": ["prefix"]}] - db.get(BizMonitorTemplate, "mt").collect_metric_ids_json = ["arp", "ip_route"] - db.commit() - with patch.object(svc, "_load_metric_rows", wraps=svc._load_metric_rows) as load: - sheets = svc.list_baseline_expect_objects(db, "p")["sheets"] - assert {s["metric_id"] for s in sheets} == {"arp"} - assert all(c.kwargs["metric_id"] == "arp" for c in load.call_args_list) - - -def test_legacy_auto_monitor_refreshes_changed_samples_without_explicit_metric_bindings(db): - assert try_auto_monitor_for_task(db, "nh") == 1 - db.get(BizStateBatch, "nc").status = "failed" - db.commit() - assert try_auto_monitor_for_task(db, "nh") == 1 - last = db.query(BizMigrationRun).order_by(BizMigrationRun.created_at.desc()).first() - assert not last.summary_json["coverage_complete"] - - -@pytest.mark.parametrize("problem", ["missing_receipt", "failed_receipt", "stale", "skew", "duplicate", "skipped", "unknown_scope"]) -def test_uncertain_coverage_cannot_pass_acceptance(db, problem): - if problem == "missing_receipt": - db.delete(db.get(BizStateBatchCommand, "cmd-nb")) - elif problem == "failed_receipt": - db.get(BizStateBatchCommand, "cmd-nc").parse_status = "failed" - elif problem in ("stale", "skew"): - batch = db.get(BizStateBatch, "nc") - batch.started_at = batch.ended_at = utcnow_naive() - timedelta(minutes=10) - elif problem == "duplicate": - db.add(BizStateMetricRow(id="dup", batch_id="nc", metric_id="arp", data_json={"ip": "a", "mac": "good"})) - elif problem == "skipped": - db.get(BizMigrationProject, "p").collect_metric_ids_json = ["other"] - else: - db.get(BizMigrationBatch, "wave").expect_set_json = {"items": [{"sheet_id": "deleted", "metric_id": "arp", "key": "a"}]} - db.commit() - assert not svc.finish_batch(db, "wave")["accept_summary"]["passed"] - - -def test_invalid_baseline_and_pinned_batch_do_not_fallback(db): - with pytest.raises(HTTPException) as error: - svc.run_evaluate(db, batch_id="wave", old_batch_id="nb") - assert error.value.detail == "old_current_batch_task_mismatch" - db.get(BizMigrationProject, "p").old_baseline_batch_id = "nc" - db.commit() - with pytest.raises(HTTPException): - svc.finish_batch(db, "wave") - assert db.get(BizMigrationBatch, "wave").status == "active" - assert db.query(BizMigrationRun).count() == 0 - - -def test_latest_failed_collection_does_not_reuse_previous_success(db): - db.add(BizStateBatch(id="latest-failed", task_id="nh", status="failed", started_at=utcnow_naive(), ended_at=utcnow_naive())) - db.commit() - run = svc.run_evaluate(db, batch_id="wave") - assert run["summary"]["current_missing_metrics"] == ["s"] - assert not run["summary"]["coverage_complete"] - - -def test_target_older_than_source_is_uncertain_instead_of_false_loss(db): - db.delete(db.get(BizStateMetricRow, "row-nc")) - db.get(BizStateBatch, "oc").started_at = utcnow_naive() - db.get(BizStateBatch, "oc").ended_at = utcnow_naive() - db.commit() - run = svc.run_evaluate(db, batch_id="wave") - row = svc.list_run_diffs(db, run["id"])["items"][0] - assert row["verdict"] == "awaiting_peer" - assert run["summary"]["anomaly"] == 0 - assert not run["summary"]["coverage_complete"] - - -def test_auto_monitor_deduplicates_receipts_and_respects_closed_window(db): - assert try_auto_monitor_for_task(db, "oh") == 1 - assert try_auto_monitor_for_task(db, "nh") == 0 - assert db.query(BizMigrationRun).one().purpose == "auto" - db.get(BizMigrationBatch, "wave").status = "done" - db.commit() - assert try_auto_monitor_for_task(db, "oh") == 0 - - -def test_two_active_waves_are_rejected(db): - with pytest.raises(HTTPException) as error: - svc.create_batch(db, "p", {"status": "active", "batch_label": "overlap"}) - assert error.value.detail == "another_batch_active" - assert db.query(BizMigrationBatch).count() == 1 - - -def test_sheets_with_same_key_keep_separate_red_tickets(db): - run = svc.run_evaluate(db, batch_id="wave") - for sid in ("one", "two"): - db.add(BizMigrationDiff(id=sid, run_id=run["id"], metric_id="arp", color="red", - key_json={"sheet_id": sid, "match_old_key": "a"}, verdict="lost")) - db.flush() - tickets = svc._persist_red_tickets_from_run(db, project_id="p", batch_id="wave", run_id=run["id"]) - assert len(tickets) == 2 - assert {t.detail_json["sheet_id"] for t in tickets} == {"one", "two"} - - -def test_port_mapping_does_not_remap_business_key_with_same_text(): - out = evaluate(old_baseline_rows=[{"ip": "old", "port": "old"}], - new_current_rows=[{"ip": "old", "port": "new"}], compare_fields=["port"], - expect=parse_expect_set({"items": [{"metric_id": "arp", "key": "old"}]})) - assert out["progress_ok"] == 1 - - -def test_unmapped_same_name_target_is_not_used_as_mapped_target_evidence(): - out = evaluate(metric_id="interface_brief", key_fields=["port"], compare_fields=[], - old_baseline_rows=[{"port": "old"}], new_current_rows=[{"port": "old"}], - expect=parse_expect_set({"ports": ["old"]})) - row = next(r for r in out["rows"] if r["in_expect"]) - assert row["verdict"] == "lost" - assert row["new"] == {} - assert row["new_key"] == "" - assert row["match_new_key"] == "new" - - -def test_legacy_raw_interface_expect_uses_same_alias_normalization(): - out = evaluate(metric_id="interface_brief", key_fields=["port"], - old_baseline_rows=[{"port": "GE-1"}], new_current_rows=[{"port": "GE-2"}], - compare_fields=[], port_map={"gei-1": "gei-2"}, - expect=parse_expect_set({"ports": ["GE-1"]}), - iface_normalize_rules=[{"from": "GE", "to": "gei"}]) - assert out["progress_ok"] == 1 - - -def test_baseline_picker_matches_normalized_keys_but_displays_originals(db): - ct = db.get(BizCompareTemplate, "ct") - ct.metrics_json = [{"metric_id": "arp", "sheet_id": "s", "key_fields": ["ip", "port"], "iface_fields": ["port"]}] - ct.iface_normalize_json = [{"from": "GE", "to": "gei"}] - db.get(BizStateMetricRow, "row-ob").data_json = {"ip": 0, "port": "GE-1"} - db.commit() - item = svc.list_baseline_expect_objects(db, "p")["sheets"][0]["items"][0] - assert item["keys"] == ["0", "gei-1"] - assert item["label"] == "0|GE-1" - assert parse_expect_set({"items": [{"metric_id": "arp", "key": item["keys"]}]})["arp"] == {"0\x1fgei-1"} - - -def test_empty_composite_segments_are_preserved_and_false_is_not_blank(): - assert parse_expect_set({"items": [{"metric_id": "x", "key": [0, "", False]}]})["x"] == {"0\x1f\x1ffalse"} - - -def test_missing_status_field_cannot_match_negative_success_condition(): - assert not _eval_field_op({"ip": "a"}, "state", "ne", "down") - assert not _eval_field_op({"ip": "a"}, "state", "not_in", ["down"]) - assert _eval_field_op({"ip": "a"}, "state", "empty", None) - - -def test_stable_outside_inventory_is_counted_without_repeated_evidence_cards(): - row = {"ip": "a", "mac": "good", "port": "old"} - out = evaluate(expect={}, old_current_rows=[row], new_current_rows=[]) - assert out["rows"] == [] - assert out["steady_outside"] == 1 - assert out["anomaly"] == 0 - - -def test_removed_previous_wave_sheet_blocks_full_coverage_acceptance(db): - db.add(BizMigrationBatch(id="prior", project_id="p", status="done", - created_at=utcnow_naive() - timedelta(days=1), - expect_set_json={"items": [{"sheet_id": "removed-sheet", "metric_id": "arp", "key": "old-customer"}]})) - db.commit() - finished = svc.finish_batch(db, "wave") - assert not finished["accept_summary"]["passed"] - assert finished["run"]["summary"]["uncovered_scope"] == ["removed-sheet"] - - -def test_split_sheets_share_loaded_rows_and_command_metadata(db): - from unittest.mock import patch - - ct = db.get(BizCompareTemplate, "ct") - sheet = ct.metrics_json[0] - ct.metrics_json = [sheet, {**sheet, "sheet_id": "s2"}] - for i in range(500): - for bid in ("ob", "nc"): - db.add(BizStateMetricRow(id=f"bulk-{bid}-{i}", batch_id=bid, - batch_command_id="cmd-" + bid, metric_id="arp", data_json={"ip": f"bulk-{i}", "mac": "good"})) - db.commit() - with patch.object(svc, "_load_metric_rows", wraps=svc._load_metric_rows) as load, patch.object(svc, "_command_brief", wraps=svc._command_brief) as command: - run = svc.run_evaluate(db, batch_id="wave") - assert load.call_count == 4 # once for each (batch, metric), not once per sheet - assert command.call_count == 2 # baseline + current metadata, independent of row count - assert svc.list_run_diffs(db, run["id"], limit=1)["total"] == 1002 - - -def test_wave_acceptance_rolls_back_whole_transaction_on_ticket_failure(db): - from unittest.mock import patch - - with patch.object(svc, "_persist_red_tickets_from_run", side_effect=RuntimeError("write failed")): - with pytest.raises(RuntimeError): - svc.finish_batch(db, "wave") - db.rollback() - assert db.query(BizMigrationRun).count() == 0 - assert db.get(BizMigrationBatch, "wave").status == "active" - - -def test_command_metadata_is_cached_and_raw_cli_is_not_selected(db): - statements = [] - def capture(_conn, _cursor, statement, *_args): - statements.append(statement) - event.listen(db.bind, "before_cursor_execute", capture) - try: - run = svc.run_evaluate(db, batch_id="wave") - finally: - event.remove(db.bind, "before_cursor_execute", capture) - command_selects = [q for q in statements if q.startswith("SELECT") and "biz_state_batch_command" in q] - assert all("raw_text" not in q for q in command_selects) - assert run["summary"]["coverage_complete"] - - -def test_diff_filters_apply_before_pagination_and_escape_wildcards(db): - run = svc.run_evaluate(db, batch_id="wave") - db.add(BizMigrationDiff(id="outside", run_id=run["id"], metric_id="arp", seq=999, color="red", - key_json={"sheet_id": "outside"}, in_expect=False, search_text="a_100%")) - db.commit() - result = svc.list_run_diffs(db, run["id"], sheet_id="outside", color="red", limit=1) - assert result["total"] == 1 and result["items"][0]["id"] == "outside" - assert svc.list_run_diffs(db, run["id"], only_expect=True)["total"] == 1 - assert svc.list_run_diffs(db, run["id"], kw="_")["total"] == 1 - - -@pytest.mark.parametrize("condition", [{"type": "value", "field": "state", "op": "typo", "value": "up"}, {"type": "presence", "value": "typo"}]) -def test_invalid_correction_rules_fail_before_evaluation(condition): - with pytest.raises(HTTPException): - validate_monitor_rules({}, [{"metric_id": "arp", "success": [{"old_groups": [[condition]], "new": ["added"]}]}]) diff --git a/tests/test_biz_state_compare.py b/tests/test_biz_state_compare.py index dcb07e1..8d610ad 100644 --- a/tests/test_biz_state_compare.py +++ b/tests/test_biz_state_compare.py @@ -5,8 +5,7 @@ from __future__ import annotations import copy import unittest -from netx_api.biz_state.compare_engine import compare_rows, mapping_stats, stratify_take -from netx_api.biz_state.compare_service import _kind_allows +from netx_api.biz_state.compare_engine import compare_rows, mapping_stats from netx_api.biz_state.compare_rules import ( apply_row_filters, arp_dynamic_row_filters, @@ -97,18 +96,7 @@ class CompareEngineTests(unittest.TestCase): self.assertEqual(s["removed"], 1) self.assertEqual(s["added"], 1) kinds = {d["kind"] for d in out["diffs"]} - # Default: unchanged counted but not listed (million-row safe) - self.assertNotIn("unchanged", kinds) - out_full = compare_rows( - before_rows=before, - after_rows=after, - key_fields=["local_if", "remote_sys", "remote_if"], - iface_fields=["local_if"], - compare_fields=[], - port_map={}, - include_unchanged=True, - ) - self.assertIn("unchanged", {d["kind"] for d in out_full["diffs"]}) + self.assertIn("unchanged", kinds) def test_empty_port_map_ignores_iface_in_key(self) -> None: """No port map → ignore local_if when matching (same neighbor, renamed port).""" @@ -161,107 +149,10 @@ class CompareEngineTests(unittest.TestCase): iface_fields=["local_if"], compare_fields=["remote_ip"], port_map={}, - include_unchanged=True, ) self.assertEqual(out["summary"]["unchanged"], 1) self.assertEqual(len(out["diffs"]), 1) self.assertEqual(out["diffs"][0]["kind"], "unchanged") - slim = compare_rows( - before_rows=before, - after_rows=after, - key_fields=["local_if", "remote_sys", "remote_if"], - iface_fields=["local_if"], - compare_fields=["remote_ip"], - port_map={}, - ) - self.assertEqual(slim["summary"]["unchanged"], 1) - self.assertEqual(slim["diffs"], []) - - def test_unchanged_sample_limit_and_compact(self) -> None: - before = [ - {"k": str(i), "v": "1", "_netx": {"row_id": f"b{i}"}} for i in range(5) - ] - after = [ - {"k": str(i), "v": "1", "_netx": {"row_id": f"a{i}"}} for i in range(5) - ] - out = compare_rows( - before_rows=before, - after_rows=after, - key_fields=["k"], - iface_fields=[], - compare_fields=["v"], - port_map={}, - include_unchanged=True, - unchanged_limit=2, - compact_unchanged=True, - ) - self.assertEqual(out["summary"]["unchanged"], 5) - self.assertEqual(out["summary"]["unchanged_listed"], 2) - self.assertTrue(out["summary"]["unchanged_truncated"]) - self.assertTrue(out["summary"]["unchanged_compact"]) - self.assertEqual(out["summary"]["unchanged_sample_mode"], "stratified") - self.assertEqual(len(out["diffs"]), 2) - self.assertEqual(out["diffs"][0]["before"], {}) - self.assertEqual(out["diffs"][0]["after"], {}) - self.assertEqual(out["diffs"][0]["before_row_id"], "b0") - self.assertEqual(out["diffs"][0]["after_row_id"], "a0") - - def test_stratify_take_round_robin(self) -> None: - items = [("a", i) for i in range(5)] + [("b", i) for i in range(5)] - got = stratify_take(items, 4, key_fn=lambda x: x[0]) - self.assertEqual([x[0] for x in got], ["a", "b", "a", "b"]) - - def test_kind_allows_tabs(self) -> None: - self.assertTrue(_kind_allows("all", "unchanged")) - self.assertTrue(_kind_allows("diff", "removed")) - self.assertTrue(_kind_allows("diff", "changed")) - self.assertFalse(_kind_allows("diff", "added")) - self.assertFalse(_kind_allows("diff", "unchanged")) - self.assertTrue(_kind_allows("unchanged", "unchanged")) - self.assertFalse(_kind_allows("unchanged", "removed")) - self.assertTrue(_kind_allows("removed", "removed")) - self.assertTrue(_kind_allows("changed", "changed")) - - def test_unchanged_sample_stratified_across_neighbors(self) -> None: - """Sample must cover multiple neighbors, not only the first command's rows.""" - before = [] - after = [] - for n_i, neigh in enumerate(("1.1.1.1", "2.2.2.2", "3.3.3.3")): - for j in range(10): - net = f"10.{n_i}.{j}.0/24" - before.append( - { - "neighbor": neigh, - "direction": "in", - "network": net, - "v": "1", - "_netx": {"row_id": f"b-{neigh}-{j}"}, - } - ) - after.append( - { - "neighbor": neigh, - "direction": "in", - "network": net, - "v": "1", - "_netx": {"row_id": f"a-{neigh}-{j}"}, - } - ) - out = compare_rows( - before_rows=before, - after_rows=after, - key_fields=["neighbor", "direction", "network"], - iface_fields=[], - compare_fields=["v"], - port_map={}, - include_unchanged=True, - unchanged_limit=6, - compact_unchanged=True, - ) - self.assertEqual(out["summary"]["unchanged"], 30) - self.assertEqual(out["summary"]["unchanged_listed"], 6) - keys = [d["key"]["neighbor"] for d in out["diffs"]] - self.assertEqual(set(keys), {"1.1.1.1", "2.2.2.2", "3.3.3.3"}) def test_mac_normalize_via_field_rules(self) -> None: before = [{"ip": "1.1.1.1", "mac": "00:11:22:33:44:55", "iface": "gei-0/1"}] @@ -418,57 +309,6 @@ class CompareRulesTests(unittest.TestCase): class CompareDiffPagingTests(unittest.TestCase): - def test_resolve_unchanged_policy(self) -> None: - from netx_api.biz_state.compare_service import resolve_unchanged_policy - - small = resolve_unchanged_policy("auto", before_n=100, after_n=100) - self.assertTrue(small["include"]) - self.assertIsNone(small["limit"]) - self.assertFalse(small["compact"]) - large = resolve_unchanged_policy("auto", before_n=1_500_000, after_n=1_500_000) - self.assertTrue(large["include"]) - self.assertEqual(large["limit"], 5000) - self.assertTrue(large["compact"]) - self.assertFalse(resolve_unchanged_policy("never", before_n=10, after_n=10)["include"]) - keys = resolve_unchanged_policy("keys", before_n=1_500_000, after_n=1_500_000) - self.assertTrue(keys["include"]) - self.assertIsNone(keys["limit"]) - self.assertTrue(keys["compact"]) - - def test_hydrate_diff_rows_fills_sides(self) -> None: - from netx_api.biz_state.compare_service import _hydrate_diff_rows - - class _FakeDb: - pass - - items = [ - { - "kind": "unchanged", - "key": {"k": "1"}, - "before": {}, - "after": {}, - "mapped_before": {}, - "changes": {}, - "before_row_id": "b1", - "after_row_id": "a1", - } - ] - # Patch loader - import netx_api.biz_state.compare_service as cs - - orig = cs._metric_rows_by_ids - try: - cs._metric_rows_by_ids = lambda _db, ids: { # type: ignore[assignment] - "b1": {"k": "1", "v": "pre"}, - "a1": {"k": "1", "v": "post"}, - } - out = _hydrate_diff_rows(_FakeDb(), items) - finally: - cs._metric_rows_by_ids = orig - self.assertEqual(out[0]["before"]["v"], "pre") - self.assertEqual(out[0]["after"]["v"], "post") - self.assertEqual(out[0]["mapped_before"]["v"], "pre") - def test_filter_inline_diffs_kind_and_kw(self) -> None: from netx_api.biz_state.compare_service import _filter_inline_diffs @@ -514,43 +354,25 @@ class CompareSheetDefaultsTests(unittest.TestCase): self.assertIn("isis_adjacency.ipv6", ids) # Same source metric may appear multiple times (afi splits) self.assertEqual(sum(1 for s in sheets if s["metric_id"] == "bgp_peer"), 6) - self.assertGreaterEqual(sum(1 for s in sheets if s["metric_id"] == "bgp_route"), 4) self.assertIn("bgp_peer.evpn", ids) self.assertIn("bgp_peer.vpls", ids) self.assertIn("vrrp.ipv4", ids) self.assertIn("lldp_neighbor", ids) - # Packaged IOH default: filtered bgp_route ipv4 + percent pfx_rcd - route4 = next(s for s in sheets if sheet_key(s) == "bgp_route") - self.assertEqual(route4["metric_id"], "bgp_route") - self.assertTrue( - any( - f.get("field") == "afi" and f.get("value") == "ipv4" - for f in (route4.get("row_filters") or []) - ) - ) - self.assertTrue( - any( - r.get("field") == "pfx_rcd" and r.get("compare") == "percent" - for r in (route4.get("field_rules") or []) - ) - ) - route_vpn = next(s for s in sheets if sheet_key(s) == "bgp_route.vpnv4") - self.assertTrue( - any( - f.get("field") == "afi" and f.get("value") == "vpnv4" - for f in (route_vpn.get("row_filters") or []) - ) - ) + detail = next(s for s in sheets if sheet_key(s) == "interface_detail") + self.assertEqual(detail["compare_fields"], ["port_status"]) + self.assertIn("input_bps", detail["display_fields"]) + optical = next(s for s in sheets if sheet_key(s) == "optical_brief") + self.assertEqual(optical["compare_fields"], ["status"]) + self.assertIn("rx_power", optical["display_fields"]) + bgp4 = next(s for s in sheets if sheet_key(s) == "bgp_peer.ipv4") + self.assertEqual(bgp4["compare_fields"], ["as_num", "state"]) + self.assertIn("pfx_rcd", bgp4["display_fields"]) vpnv4 = next(s for s in sheets if sheet_key(s) == "bgp_peer.vpnv4") - self.assertEqual( - vpnv4["row_filters"], - [{"field": "afi", "op": "eq", "value": "vpnv4"}], - ) + self.assertEqual(vpnv4["row_filters"], [{"field": "afi", "op": "eq", "value": "vpnv4"}]) isis4 = next(s for s in sheets if sheet_key(s) == "isis_adjacency.ipv4") self.assertEqual(isis4["row_filters"][0]["op"], "contains") - def test_ordered_same_key_pairing_2v2(self) -> None: - """Same key, two rows each: zip by order (not first-wins + duplicate).""" + def test_duplicate_match_keys_are_reported(self) -> None: before = [ {"local_if": "a", "remote_sys": "X", "remote_if": "1", "remote_ip": "1"}, {"local_if": "a", "remote_sys": "X", "remote_if": "1", "remote_ip": "9"}, @@ -567,47 +389,15 @@ class CompareSheetDefaultsTests(unittest.TestCase): compare_fields=["remote_ip"], port_map={"a": "a"}, ) - self.assertEqual(out["summary"]["before_count"], 2) - self.assertEqual(out["summary"]["after_count"], 2) - self.assertEqual(out["summary"]["duplicate"], 0) self.assertEqual(out["summary"]["duplicate_keys_before"], 1) self.assertEqual(out["summary"]["duplicate_keys_after"], 1) + self.assertEqual(out["summary"]["duplicate"], 2) self.assertIn("a|X|1", out["summary"]["duplicate_key_list"]) kinds = [d["kind"] for d in out["diffs"]] - self.assertNotIn("duplicate", kinds) - # Pair 0: 1==1 unchanged; pair 1: 9!=2 changed + self.assertEqual(kinds.count("duplicate"), 2) + # First before wins → matches first after → unchanged (same remote_ip) self.assertEqual(out["summary"]["unchanged"], 1) - self.assertEqual(out["summary"]["changed"], 1) - self.assertEqual(out["summary"]["added"], 0) - self.assertEqual(out["summary"]["removed"], 0) - - def test_ordered_multipath_5_vs_2(self) -> None: - """5 before / 2 after same key → 2 compared + 3 removed failures.""" - key = {"local_if": "a", "remote_sys": "X", "remote_if": "1"} - before = [{**key, "remote_ip": str(i)} for i in range(5)] - after = [{**key, "remote_ip": "0"}, {**key, "remote_ip": "1"}] - out = compare_rows( - before_rows=before, - after_rows=after, - key_fields=["local_if", "remote_sys", "remote_if"], - iface_fields=["local_if"], - compare_fields=["remote_ip"], - port_map={"a": "a"}, - ) - self.assertEqual(out["summary"]["before_count"], 5) - self.assertEqual(out["summary"]["after_count"], 2) - self.assertEqual(out["summary"]["removed"], 3) - self.assertEqual(out["summary"]["added"], 0) - self.assertEqual( - out["summary"]["unchanged"] + out["summary"]["changed"], - 2, - ) - self.assertEqual(out["summary"]["unchanged"], 2) self.assertEqual(out["summary"]["changed"], 0) - self.assertEqual(out["summary"]["duplicate"], 0) - kinds = [d["kind"] for d in out["diffs"]] - self.assertEqual(kinds.count("removed"), 3) - self.assertNotIn("duplicate", kinds) def test_ignore_port_changes_false_keeps_iface(self) -> None: before = [ diff --git a/tests/test_biz_state_compare_export.py b/tests/test_biz_state_compare_export.py deleted file mode 100644 index d176c40..0000000 --- a/tests/test_biz_state_compare_export.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Export must page lazily, keep tied sequence rows, and produce valid UTF-8 CSV.""" - -from __future__ import annotations - -import csv -import io -import unittest -import zipfile -from unittest.mock import patch - -from sqlalchemy import create_engine, event -from sqlalchemy.orm import Session - -from netx_api.biz_state.compare_service import _iter_sheet_diffs, export_run_zip -from netx_api.db import Base -from netx_api.models import BizCompareDiff, BizCompareRun - - -class CompareExportTests(unittest.TestCase): - def setUp(self): - self.engine = create_engine("sqlite+pysqlite:///:memory:") - Base.metadata.create_all(self.engine) - self.db = Session(self.engine) - self.sheet = {"sheet_id": "sheet", "metric_id": "test", "key_fields": ["id"], - "compare_fields": ["note"], "display_fields": ["id", "note"], - "summary": {"unchanged": 7}, "status": "done"} - self.run = BizCompareRun(id="run", metric_id="test", status="success", - summary_json={"sheets": [self.sheet]}) - self.db.add(self.run) - for seq, ids in ((0, "cba"), (1, "ed"), (2, "gf")): - for rid in ids: - note = '中文, "quoted"\nsecond line' - self.db.add(BizCompareDiff(id=rid, run_id="run", metric_id="sheet", seq=seq, - kind="unchanged", key_json={"id": rid}, - before_json={"id": rid, "note": note}, - after_json={"id": rid, "note": note})) - self.db.commit() - - def tearDown(self): - self.db.close() - self.engine.dispose() - - def test_cursor_handles_tied_sequence_numbers_across_chunks(self): - statements = [] - def record(_conn, _cursor, statement, _parameters, _context, _executemany): - if "ORDER BY biz_compare_diff.seq" in statement: - statements.append(statement) - event.listen(self.engine, "before_cursor_execute", record) - with patch("netx_api.biz_state.compare_service._DIFF_CHUNK", 2): - iterator = _iter_sheet_diffs(self.db, "run", "sheet") - self.assertEqual(statements, []) - self.assertEqual(next(iterator)["key"]["id"], "a") - self.assertEqual(next(iterator)["key"]["id"], "b") - self.assertEqual(len(statements), 1, "do not load the next chunk early") - self.assertEqual([d["key"]["id"] for d in iterator], list("cdefg")) - self.assertEqual(len(statements), 4) - self.assertTrue(all("biz_compare_diff.seq >" in q and "biz_compare_diff.id >" in q - for q in statements[1:])) - - def test_export_csv_escapes_unicode_quotes_commas_and_newlines(self): - with patch("netx_api.biz_state.compare_service._DIFF_CHUNK", 2): - archive_bytes = export_run_zip(self.db, "run") - with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive: - payload = archive.read("tables/sheet.csv") - self.assertTrue(payload.startswith(b"\xef\xbb\xbf")) - rows = list(csv.DictReader(io.StringIO(payload.decode("utf-8-sig")))) - self.assertEqual([r["id"] for r in rows], list("abcdefg")) - self.assertTrue(all(r["note__pre"] == '中文, "quoted"\nsecond line' for r in rows)) - self.assertTrue(all(r["note__pre"] == r["note__post"] for r in rows)) - - def test_other_sheets_do_not_leak_into_export(self): - self.db.add(BizCompareDiff(id="other", run_id="run", metric_id="another", seq=0, - kind="added", key_json={"id": "other"}, after_json={"note": "other"})) - self.db.commit() - with patch("netx_api.biz_state.compare_service._DIFF_CHUNK", 2): - self.assertEqual([d["key"]["id"] for d in _iter_sheet_diffs(self.db, "run", "sheet")], list("abcdefg")) - - def test_legacy_inline_and_empty_exports(self): - self.db.query(BizCompareDiff).delete() - inline = {"kind": "added", "key": {"id": "legacy"}, "after": {"note": "value"}} - self.run.summary_json = {"sheets": [{**self.sheet, "diffs": [inline]}]} - self.db.commit() - self.assertEqual(list(_iter_sheet_diffs(self.db, "run", "sheet")), [inline]) - self.run.summary_json = {"sheets": [self.sheet]} - self.db.commit() - with zipfile.ZipFile(io.BytesIO(export_run_zip(self.db, "run"))) as archive: - self.assertEqual(archive.read("tables/sheet.csv").decode("utf-8-sig"), "kind,id,note__pre,note__post\n") diff --git a/tests/test_biz_state_compare_memory.py b/tests/test_biz_state_compare_memory.py deleted file mode 100644 index 7850798..0000000 --- a/tests/test_biz_state_compare_memory.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Bounded sampling regressions and an opt-in comparison memory benchmark.""" - -from __future__ import annotations - -import gc -import hashlib -import json -import os -import random -import subprocess -import time -import tracemalloc -import types -from collections import defaultdict, deque -from copy import deepcopy - -import pytest - -from netx_api.biz_state.compare_engine import _StratifiedSample, compare_rows, stratify_take - - -def reference_sample(items, limit): - if len(items) <= limit: - return list(items) - buckets = defaultdict(deque) - for item in items: - buckets[item[0]].append(item) - out = [] - while buckets and len(out) < limit: - for key in list(buckets): - out.append(buckets[key].popleft()) - if not buckets[key]: - del buckets[key] - if len(out) >= limit: - break - return out - - -def test_streaming_sample_preserves_round_robin_and_small_input_order(): - rng = random.Random(42) - for _ in range(100): - items = [(str(rng.randrange(20)), i) for i in range(rng.randrange(100))] - for limit in (0, 1, 5, 20, 100): - expected = reference_sample(items, limit) - assert stratify_take(iter(items), limit, key_fn=lambda item: item[0]) == expected - - -def test_sample_storage_is_bounded_for_both_many_rows_and_many_strata(): - sample = _StratifiedSample(50) - for i in range(20_000): - sample.add(i, str(i) if i >= 10_000 else "one-large-peer") - assert len(sample.heap) == 50 - assert len(sample.strata) == 50 - assert sample.picked() == [0, *range(10_000, 10_049)] - - -@pytest.mark.parametrize("iface_fields,rules,pmap", [ - ([], [], {}), - (["interface"], [], {}), - (["interface"], [{"from": "GE", "to": "gei"}], {"gei-1": "gei-2"}), -]) -def test_compare_does_not_mutate_inputs_when_reusing_rows(iface_fields, rules, pmap): - before = [{"id": "1", "interface": "GE-1", "state": "up"}] - after = [{"id": "1", "interface": "GE-2", "state": "up"}] - original = deepcopy((before, after)) - compare_rows(before_rows=before, after_rows=after, key_fields=["id"], - iface_fields=iface_fields, compare_fields=["state"], port_map=pmap, - iface_normalize_rules=rules, include_unchanged=True, unchanged_limit=1) - assert (before, after) == original - - -def test_memory_benchmark_against_git_baseline(): - """Run explicitly with NETX_BENCH_ROWS=1000000; not part of routine CI.""" - n = int(os.environ.get("NETX_BENCH_ROWS", "0")) - if not n: - pytest.skip("opt-in large synthetic benchmark") - baseline_ref = os.environ.get("NETX_BENCH_REF", "HEAD") - source = subprocess.run(["git", "show", f"{baseline_ref}:netx_api/biz_state/compare_engine.py"], - check=True, capture_output=True, text=True, encoding="utf-8").stdout - baseline = types.ModuleType("netx_api.biz_state._benchmark_baseline") - baseline.__package__ = "netx_api.biz_state" - exec(compile(source, "baseline_compare_engine.py", "exec"), baseline.__dict__) - before = [{"id": str(i), "neighbor": str(i % 50), "state": "up", "afi": "vpnv4"} for i in range(n)] - after = [dict(row) for row in before] - records = [] - digests = [] - for name, compare in (("baseline", baseline.compare_rows), ("optimized", compare_rows)): - gc.collect() - tracemalloc.start() - started = time.perf_counter() - result = compare(before_rows=before, after_rows=after, key_fields=["id"], iface_fields=[], - compare_fields=["state"], include_unchanged=True, unchanged_limit=5000, - compact_unchanged=True) - elapsed = time.perf_counter() - started - _, peak = tracemalloc.get_traced_memory() - tracemalloc.stop() - digests.append(hashlib.sha256(json.dumps(result, sort_keys=True).encode()).hexdigest()) - records.append({"engine": name, "rows_per_side": n, "peak_engine_mib": round(peak / 2**20, 2), - "seconds_with_tracemalloc": round(elapsed, 3)}) - del result - assert digests[0] == digests[1], "large-sheet results and sample order must match" - print("\n" + json.dumps({"baseline_ref": baseline_ref, "input_allocations_included": False, - "results": records}, indent=2)) diff --git a/tests/test_biz_state_compare_recovery.py b/tests/test_biz_state_compare_recovery.py deleted file mode 100644 index e24ea40..0000000 --- a/tests/test_biz_state_compare_recovery.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Startup recovery for interrupted biz-state compare runs.""" - -from __future__ import annotations - -import unittest - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from netx_api.biz_state.compare_service import ( - cancel_compare_run, - recover_interrupted_compares_on_startup, -) -from netx_api.db import Base -from netx_api.models import BizCompareJob, BizCompareRun - - -class BizStateCompareRecoveryTests(unittest.TestCase): - def setUp(self) -> None: - engine = create_engine("sqlite+pysqlite:///:memory:", future=True) - TestingSession = sessionmaker( - bind=engine, autoflush=False, autocommit=False, expire_on_commit=False - ) - Base.metadata.create_all(bind=engine) - self.db = TestingSession() - self.job = BizCompareJob( - id="j1", - name="cutover", - template_id="tpl1", - status="active", - ) - self.db.add(self.job) - self.running = BizCompareRun( - id="r_run", - job_id="j1", - status="running", - message="loading 1/2 · BGP", - summary_json={ - "progress": {"phase": "loading", "sheet_index": 1, "sheet_total": 2}, - "sheets": [ - {"sheet_id": "bgp", "status": "running"}, - {"sheet_id": "isis", "status": "pending"}, - ], - }, - ) - self.queued = BizCompareRun( - id="r_q", - job_id="j1", - status="queued", - message="queued", - summary_json={"sheets": [{"sheet_id": "bgp", "status": "pending"}]}, - ) - self.ok = BizCompareRun( - id="r_ok", - job_id="j1", - status="success", - message="done", - summary_json={"added": 0, "removed": 0, "changed": 0}, - ) - self.db.add_all([self.running, self.queued, self.ok]) - self.db.commit() - - def tearDown(self) -> None: - self.db.close() - - def test_startup_cancels_running_and_queued(self) -> None: - out = recover_interrupted_compares_on_startup(self.db) - self.assertEqual(out["runs"], 2) - - self.db.refresh(self.running) - self.db.refresh(self.queued) - self.db.refresh(self.ok) - self.assertEqual(self.running.status, "cancelled") - self.assertIn("interrupted_by_restart", self.running.message or "") - self.assertEqual( - (self.running.summary_json or {}).get("progress", {}).get("phase"), - "cancelled", - ) - sheets = list((self.running.summary_json or {}).get("sheets") or []) - self.assertEqual(sheets[0].get("status"), "cancelled") - self.assertEqual(sheets[1].get("status"), "cancelled") - self.assertEqual(self.queued.status, "cancelled") - self.assertEqual(self.ok.status, "success") - - def test_cancel_compare_run_user(self) -> None: - out = cancel_compare_run(self.db, "r_run") - self.assertEqual(out["status"], "cancelled") - self.db.refresh(self.running) - self.assertEqual(self.running.status, "cancelled") - self.assertIn("cancelled_by_user", self.running.message or "") - - def test_cancel_idempotent_on_success(self) -> None: - out = cancel_compare_run(self.db, "r_ok") - self.assertEqual(out["status"], "success") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_biz_state_compare_search.py b/tests/test_biz_state_compare_search.py deleted file mode 100644 index 0b30c8f..0000000 --- a/tests/test_biz_state_compare_search.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Regression tests: searching a pair must not change its comparison verdict.""" - -from __future__ import annotations - -import unittest -from unittest.mock import patch - -from fastapi import HTTPException -from sqlalchemy import create_engine -from sqlalchemy.dialects import postgresql -from sqlalchemy.orm import Session - -from netx_api.biz_state.compare_engine import compare_rows -from netx_api.biz_state.compare_service import ( - _load_metric_rows_for_search, - _literal_search_pattern, - _persist_sheet_diffs, - _search_stored_diffs, - list_run_diffs, -) -from netx_api.db import Base -from netx_api.models import ( - BizCompareDiff, BizCompareRun, BizCompareTemplate, BizPortMappingRow, - BizStateLldpNeighbor, BizStateMetricRow, -) - - -class CompareSearchTests(unittest.TestCase): - def setUp(self) -> None: - self.engine = create_engine("sqlite+pysqlite:///:memory:") - Base.metadata.create_all(self.engine, tables=[m.__table__ for m in ( - BizCompareDiff, BizCompareRun, BizCompareTemplate, BizPortMappingRow, - BizStateLldpNeighbor, BizStateMetricRow, - )]) - self.db = Session(self.engine) - - def tearDown(self) -> None: - self.db.close() - self.engine.dispose() - - def seed(self, before, after, *, metric="test", keys=None, fields=None, - iface=None, mapping=None, sample=False, compact=False, filters=None): - keys = keys or ["id"] - iface = iface or [] - fields = fields or ["state"] - for side, rows in (("before", before), ("after", after)): - for seq, row in enumerate(rows): - rid = f"{side}-{seq}" - if metric == "lldp_neighbor": - self.db.add(BizStateLldpNeighbor(id=rid, batch_id=side, **row)) - else: - self.db.add(BizStateMetricRow(id=rid, batch_id=side, metric_id=metric, - seq=seq, data_json=dict(row))) - row["_netx"] = {"row_id": rid} - if mapping: - for b, a in mapping.items(): - self.db.add(BizPortMappingRow(mapping_id="map", before_if=b, after_if=a)) - result = compare_rows(before_rows=before, after_rows=after, key_fields=keys, - compare_fields=fields, iface_fields=iface, port_map=mapping or {}, - include_unchanged=True, compact_unchanged=compact) - sheet = {"sheet_id": "sheet", "metric_id": metric, "key_fields": keys, - "compare_fields": fields, "iface_fields": iface, - "summary": {**result["summary"], "unchanged_listed": 0 if sample - else result["summary"]["unchanged"]}, - "row_filters": filters or []} - self.db.add(BizCompareRun(id="run", status="success", before_batch_id="before", - after_batch_id="after", mapping_id="map" if mapping else "", - summary_json={"sheets": [sheet]})) - self.db.commit() - _persist_sheet_diffs(self.db, run_id="run", metric_id="sheet", - diffs=[d for d in result["diffs"] - if not sample or d["kind"] != "unchanged"]) - return result - - def search(self, **kwargs): - return list_run_diffs(self.db, "run", metric_id="sheet", **kwargs) - - def test_search_before_value_preserves_changed_verdict(self): - self.seed([{"id": "1", "state": "up"}], [{"id": "1", "state": "down"}]) - with patch("netx_api.biz_state.compare_service.compare_rows") as compare: - result = self.search(kw="up", kind="changed") - compare.assert_not_called() - self.assertEqual(result["total"], 1) - self.assertEqual(result["items"][0]["after"]["state"], "down") - self.assertEqual(self.search(kw="up", kind="removed")["total"], 0) - - def test_field_filter_matches_after_value_on_compact_diff(self): - self.seed([{"id": "1", "state": "up"}], [{"id": "1", "state": "down"}], compact=True) - result = self.search(qf={"state": "down"}, kind="changed") - self.assertEqual(result["total"], 1) - self.assertEqual(result["items"][0]["before"]["state"], "up") - - def test_sample_lookup_completes_pair_matching_only_one_side(self): - self.seed([{"id": "1", "state": "up", "age": "old"}], - [{"id": "1", "state": "up", "age": "new"}], sample=True) - result = self.search(kw="old", kind="unchanged") - self.assertEqual(result["source"], "live") - self.assertEqual(result["total"], 1) - self.assertEqual(result["items"][0]["after"]["age"], "new") - - def test_sample_lookup_does_not_turn_changed_into_removed(self): - self.seed([{"id": "1", "state": "up"}, {"id": "2", "state": "up"}], - [{"id": "1", "state": "down"}, {"id": "2", "state": "up"}], sample=True) - result = self.search(qf={"id": "1", "state": "up"}, kind="all") - self.assertEqual([d["kind"] for d in result["items"]], ["changed"]) - - def test_sample_lookup_keeps_duplicate_group_pairing(self): - self.seed([{"id": "1", "state": "up", "note": "first"}, - {"id": "1", "state": "down", "note": "second"}], - [{"id": "1", "state": "up", "note": "other"}, - {"id": "1", "state": "down", "note": "other"}], sample=True) - result = self.search(kw="second", kind="all") - self.assertEqual([d["kind"] for d in result["items"]], ["unchanged"]) - self.assertEqual(result["items"][0]["after"]["state"], "down") - - def test_sample_search_is_not_limited_by_persisted_search_text_size(self): - self.seed([{"id": "1", "state": "up", "note": "x" * 5000 + "needle"}], - [{"id": "1", "state": "up", "note": "other"}], sample=True) - self.assertEqual(self.search(kw="needle", kind="unchanged")["total"], 1) - - def test_sample_lookup_applies_port_mapping_before_matching(self): - self.seed([{"id": "1", "interface": "old", "state": "up"}], - [{"id": "1", "interface": "new", "state": "up"}], - keys=["id", "interface"], iface=["interface"], mapping={"old": "new"}, sample=True) - result = self.search(qf={"interface": "old"}, kind="unchanged") - self.assertEqual(result["total"], 1) - self.assertEqual(result["items"][0]["after"]["interface"], "new") - - def test_lldp_source_search(self): - self.seed([{"local_if": "gei-1", "remote_sys": "peer", "remote_if": "port", "remote_ip": "1"}], - [{"local_if": "gei-1", "remote_sys": "peer", "remote_if": "port", "remote_ip": "1"}], - metric="lldp_neighbor", keys=["local_if", "remote_sys"], fields=["remote_ip"], - iface=["local_if"], sample=True) - self.assertEqual(self.search(kw="peer", kind="unchanged")["total"], 1) - - def test_lldp_compact_diff_field_search(self): - self.seed([{"local_if": "gei-1", "remote_sys": "peer", "remote_ip": "1"}], - [{"local_if": "gei-1", "remote_sys": "peer", "remote_ip": "2"}], - metric="lldp_neighbor", keys=["local_if", "remote_sys"], fields=["remote_ip"], - iface=["local_if"], compact=True) - self.assertEqual(self.search(qf={"remote_ip": "2"}, kind="changed")["total"], 1) - - def test_exact_load_cap_is_not_truncated(self): - self.seed([{"id": "1", "state": "up"}], [{"id": "1", "state": "up"}]) - rows, truncated = _load_metric_rows_for_search(self.db, batch_id="before", metric_id="test", - row_filters=[], key_fields=["id"], kw="up", cap=1) - self.assertEqual(len(rows), 1) - self.assertFalse(truncated) - - def test_incomplete_duplicate_groups_are_omitted(self): - self.seed([{"id": "1", "state": "up"}, {"id": "1", "state": "down"}], - [{"id": "1", "state": "up"}, {"id": "1", "state": "down"}], sample=True) - with patch("netx_api.biz_state.compare_service._LIVE_SEARCH_GROUP_LOAD_CAP", 1): - result = self.search(kw="up", kind="all") - self.assertEqual(result["items"], []) - self.assertTrue(result["truncated"]) - - def test_postgres_search_pattern_treats_wildcards_literally(self): - self.assertEqual(_literal_search_pattern("a_10%"), "%a\\_10\\%%") - - def test_postgres_unsupported_filter_uses_python(self): - self.seed([{"id": "1", "state": "up", "age": "H"}], - [{"id": "1", "state": "up", "age": "H"}]) - with patch("netx_api.biz_state.compare_sql._dialect_is_postgres", return_value=True): - rows, _ = _load_metric_rows_for_search(self.db, batch_id="before", metric_id="test", - row_filters=[{"field": "age", "op": "age_timer"}], key_fields=["id"], kw="up") - self.assertEqual(rows, []) - - def test_unknown_sheet_is_rejected(self): - self.seed([{"id": "1", "state": "up"}], [{"id": "1", "state": "up"}]) - with self.assertRaises(HTTPException) as error: - list_run_diffs(self.db, "run", metric_id="missing") - self.assertEqual(error.exception.detail, "sheet_not_found") - - def test_stored_search_escapes_like_wildcards(self): - self.seed([{"id": "x_1", "state": "up"}, {"id": "xy1", "state": "up"}], - [{"id": "x_1", "state": "down"}, {"id": "xy1", "state": "down"}]) - self.assertEqual(self.search(qf={"id": "x_1"}, kind="changed")["total"], 1) - query = _search_stored_diffs(self.db.query(BizCompareDiff), metric_id="test", - kw="up", field_q={"id": "x_1"}) - compiled = query.statement.compile(dialect=postgresql.dialect()) - self.assertIn("LEFT OUTER JOIN", str(compiled)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_biz_state_compare_snapshot.py b/tests/test_biz_state_compare_snapshot.py deleted file mode 100644 index 3736fa8..0000000 --- a/tests/test_biz_state_compare_snapshot.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Historical comparisons must use the settings captured when they were queued.""" - -from __future__ import annotations - -import csv -import io -import json -import unittest -import zipfile -from copy import deepcopy -from unittest.mock import patch - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from netx_api.biz_state.compare_service import ( - _create_running_run, _execute_compare_into_run, cancel_compare_run, - export_run_zip, get_run, list_run_diffs, template_metrics, -) -from netx_api.db import Base -from netx_api.models import ( - BizCompareDiff, BizCompareJob, BizCompareTemplate, BizPortMapping, - BizPortMappingRow, BizStateMetricRow, -) - - -class CompareSnapshotTests(unittest.TestCase): - def setUp(self): - self.engine = create_engine("sqlite+pysqlite:///:memory:") - Base.metadata.create_all(self.engine) - self.factory = sessionmaker(bind=self.engine, expire_on_commit=False) - self.db = self.factory() - # Progress uses a separate session in production; keep every test write local. - self.session_patch = patch("netx_api.db.SessionLocal", self.factory) - self.session_patch.start() - self.sheet = { - "sheet_id": "sheet", "metric_id": "test", "title": "Original sheet", - "key_fields": ["id", "interface"], "iface_fields": ["interface"], - "compare_fields": ["state"], "display_fields": ["id", "interface", "state"], - "row_filters": [], "ignore_port_changes": False, - } - self.tpl = BizCompareTemplate( - id="tpl", name="Original template", metrics_json=[deepcopy(self.sheet)], - iface_normalize_json=[{"from": "GE", "to": "gei"}], - ) - self.job = BizCompareJob(id="job", name="Original job", template_id="tpl", - mapping_id="map", store_unchanged="sample") - self.mapping = BizPortMapping(id="map", name="Original mapping") - self.pair = BizPortMappingRow(mapping_id="map", before_if="gei-1", after_if="gei-2") - self.db.add_all([self.tpl, self.job, self.mapping, self.pair]) - for side, iface, note in (("before", "GE-1.100", "old"), ("after", "GE-2.100", "new")): - self.db.add(BizStateMetricRow(id=side, batch_id=side, metric_id="test", seq=0, - data_json={"id": "1", "interface": iface, - "state": "up", "note": note})) - self.db.commit() - self.run = _create_running_run( - self.db, job=self.job, tpl=self.tpl, before_batch_id="before", - after_batch_id="after", sheets_cfg=template_metrics(self.tpl), - ) - - def tearDown(self): - self.session_patch.stop() - self.db.close() - self.engine.dispose() - - def edit_current_settings(self): - changed = {**self.sheet, "title": "Edited sheet", "compare_fields": ["note"], - "row_filters": [{"field": "state", "op": "eq", "value": "down"}]} - self.tpl.name = "Edited template" - self.tpl.metrics_json = [changed] - self.tpl.iface_normalize_json = [{"from": "GE", "to": "other"}] - self.pair.after_if = "gei-99" - self.job.name = "Edited job" - self.job.store_unchanged = "never" - self.job.enabled_sheet_ids = ["different-sheet"] - self.mapping.name = "Edited mapping" - self.db.commit() - - def execute(self): - return _execute_compare_into_run(self.db, self.run.id) - - def test_queued_run_uses_original_sheet_rules_mapping_and_storage(self): - original = deepcopy(self.run.summary_json["config_snapshot"]) - self.edit_current_settings() - detail = self.execute() - self.assertEqual(detail["status"], "success") - self.assertEqual(detail["summary"]["unchanged"], 1) - self.assertEqual(detail["summary"]["unchanged_listed"], 1) - self.assertEqual(detail["summary"]["store_unchanged"], "sample") - self.assertEqual(detail["sheets"][0]["compare_fields"], ["state"]) - self.assertEqual(self.run.summary_json["config_snapshot"], original) - - def test_compact_hydration_restores_original_normalized_values(self): - self.execute() - self.edit_current_settings() - diff = list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged")["items"][0] - self.assertEqual(diff["before"]["interface"], "GE-1.100") - self.assertEqual(diff["mapped_before"]["interface"], "gei-2.100") - self.assertEqual(diff["after"]["interface"], "gei-2.100") - - def test_live_search_does_not_inherit_new_filters_or_transforms(self): - with patch("netx_api.biz_state.compare_service._UNCHANGED_SAMPLE_MAX", 0): - self.execute() - self.edit_current_settings() - result = list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged", kw="old") - self.assertEqual(result["source"], "live") - self.assertEqual(result["total"], 1) - self.assertEqual(result["items"][0]["mapped_before"]["interface"], "gei-2.100") - - def test_snapshot_can_be_read_after_mapping_and_template_are_deleted(self): - self.execute() - self.db.query(BizPortMappingRow).delete() - self.db.delete(self.mapping) - self.db.delete(self.tpl) - self.db.commit() - detail = get_run(self.db, self.run.id) - self.assertEqual(detail["template_name"], "Original template") - self.assertEqual(detail["mapping_name"], "Original mapping") - diff = list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged")["items"][0] - self.assertEqual(diff["mapped_before"]["interface"], "gei-2.100") - - def test_queued_run_can_execute_after_its_template_and_mapping_are_deleted(self): - self.db.query(BizPortMappingRow).delete() - self.db.delete(self.mapping) - self.db.delete(self.tpl) - self.db.commit() - detail = self.execute() - self.assertEqual(detail["status"], "success") - self.assertEqual(detail["summary"]["unchanged"], 1) - - def test_detail_returns_original_names_and_independent_template_copy(self): - self.edit_current_settings() - detail = get_run(self.db, self.run.id) - self.assertEqual(detail["job_name"], "Original job") - self.assertEqual(detail["config_snapshot_version"], 1) - self.assertEqual(detail["template"]["iface_normalize_rules"], [{"from": "GE", "to": "gei"}]) - self.assertNotIn("config_snapshot", detail["summary"]) - detail["template"]["iface_normalize_rules"].clear() - self.assertTrue(self.run.summary_json["config_snapshot"]["template"]["iface_normalize_rules"]) - - def test_export_contains_original_config_and_hydrated_csv(self): - self.execute() - self.edit_current_settings() - with zipfile.ZipFile(io.BytesIO(export_run_zip(self.db, self.run.id))) as archive: - config = json.loads(archive.read("config_snapshot.json")) - self.assertEqual(config["port_map"], {"gei-1": "gei-2"}) - self.assertEqual(config["template"]["name"], "Original template") - rows = list(csv.DictReader(io.StringIO(archive.read("tables/sheet.csv").decode("utf-8-sig")))) - self.assertEqual(rows[0]["interface"], "gei-2.100") - - def test_failed_and_cancelled_runs_keep_snapshot(self): - original = deepcopy(self.run.summary_json["config_snapshot"]) - with patch("netx_api.biz_state.compare_service._run_sheet", side_effect=RuntimeError("test-failure")): - with self.assertRaisesRegex(RuntimeError, "test-failure"): - self.execute() - self.assertEqual(self.run.summary_json["config_snapshot"], original) - self.run.status = "running" - self.db.commit() - cancel_compare_run(self.db, self.run.id) - self.assertEqual(self.run.summary_json["config_snapshot"], original) - - def test_legacy_runs_without_snapshot_remain_readable(self): - self.execute() - self.run.summary_json = {k: v for k, v in self.run.summary_json.items() if k != "config_snapshot"} - self.db.commit() - detail = get_run(self.db, self.run.id) - self.assertEqual(detail["status"], "success") - self.assertIsNone(detail["config_snapshot_version"]) - self.assertEqual(list_run_diffs(self.db, self.run.id, metric_id="sheet", kind="unchanged")["total"], 1) - self.assertEqual(self.db.query(BizCompareDiff).count(), 1) diff --git a/tests/test_biz_state_compare_sql.py b/tests/test_biz_state_compare_sql.py deleted file mode 100644 index 44b2891..0000000 --- a/tests/test_biz_state_compare_sql.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Unit tests for SQL compare eligibility and filter compilation.""" - -from __future__ import annotations - -import unittest -from unittest.mock import MagicMock - -from netx_api.biz_state.compare_sql import ( - can_sql_compare, - compile_row_filters_sql, - sql_compare_skip_reason, - _field_rules_sql_compatible, - _filters_sql_compatible, - _safe_field, -) - - -class _FakeDialect: - def __init__(self, name: str) -> None: - self.name = name - - -class _FakeBind: - def __init__(self, name: str) -> None: - self.dialect = _FakeDialect(name) - - -def _db(dialect: str = "postgresql") -> MagicMock: - db = MagicMock() - db.get_bind.return_value = _FakeBind(dialect) - return db - - -class CompareSqlGateTests(unittest.TestCase): - def test_safe_field_rejects_injection(self) -> None: - with self.assertRaises(ValueError): - _safe_field("a'; drop table x;--") - with self.assertRaises(ValueError): - _safe_field("x.y") - self.assertEqual(_safe_field("network"), "network") - - def test_requires_postgres(self) -> None: - sheet = { - "metric_id": "bgp_route", - "key_fields": ["network", "next_hop"], - "compare_fields": ["path"], - "iface_fields": [], - "field_rules": [], - "row_filters": [], - } - self.assertFalse(can_sql_compare(_db("sqlite"), sheet, port_map={})) - self.assertTrue(can_sql_compare(_db("postgresql"), sheet, port_map={})) - - def test_rejects_port_map(self) -> None: - sheet = { - "metric_id": "lldp_neighbor", - "key_fields": ["local_if", "remote_sys"], - "compare_fields": [], - "iface_fields": ["local_if"], - "ignore_port_changes": False, - "field_rules": [], - "row_filters": [], - } - self.assertFalse( - can_sql_compare(_db(), sheet, port_map={"gei-0/1": "gei-0/2"}) - ) - - def test_rejects_auto_ignore_ports_with_iface_key(self) -> None: - sheet = { - "metric_id": "lldp_neighbor", - "key_fields": ["local_if", "remote_sys"], - "compare_fields": [], - "iface_fields": ["local_if"], - "ignore_port_changes": None, - "field_rules": [], - "row_filters": [], - } - self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) - - def test_rejects_mac_normalize(self) -> None: - sheet = { - "metric_id": "arp", - "key_fields": ["ip", "vrf"], - "compare_fields": ["mac"], - "iface_fields": [], - "field_rules": [{"field": "mac", "normalize": "mac"}], - "row_filters": [], - } - self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) - - def test_rejects_age_timer_filter(self) -> None: - sheet = { - "metric_id": "arp", - "key_fields": ["ip"], - "compare_fields": [], - "iface_fields": [], - "field_rules": [], - "row_filters": [{"field": "age", "op": "age_timer"}], - } - self.assertFalse(can_sql_compare(_db(), sheet, port_map={})) - - def test_accepts_bgp_route_style(self) -> None: - sheet = { - "metric_id": "bgp_route", - "key_fields": [ - "local_as", - "afi", - "vrf", - "neighbor", - "direction", - "rd", - "network", - "next_hop", - ], - "compare_fields": ["path", "as_num", "pfx_rcd"], - "iface_fields": [], - "field_rules": [ - {"field": "pfx_rcd", "compare": "percent", "tolerance": 5}, - ], - "row_filters": [ - {"field": "vrf", "op": "empty"}, - {"field": "afi", "op": "eq", "value": "ipv4"}, - ], - } - self.assertTrue(can_sql_compare(_db(), sheet, port_map={})) - # BGP afi/vrf sheet splits + percent rules must not force Python - self.assertEqual(sql_compare_skip_reason(_db(), sheet, port_map={}), "") - - def test_rejects_iface_normalize_when_key_uses_iface(self) -> None: - sheet = { - "metric_id": "interface_brief", - "key_fields": ["interface"], - "compare_fields": ["admin"], - "iface_fields": ["interface"], - "ignore_port_changes": False, - "field_rules": [], - "row_filters": [], - } - self.assertFalse( - can_sql_compare( - _db(), - sheet, - port_map={}, - iface_normalize_rules=[{"from": "GE", "to": "gei"}], - ) - ) - - def test_accepts_ignore_port_changes_false_without_normalize(self) -> None: - sheet = { - "metric_id": "interface_brief", - "key_fields": ["interface"], - "compare_fields": ["admin"], - "iface_fields": ["interface"], - "ignore_port_changes": False, - "field_rules": [], - "row_filters": [], - } - self.assertTrue(can_sql_compare(_db(), sheet, port_map={}, iface_normalize_rules=[])) - - -class CompareSqlFilterCompileTests(unittest.TestCase): - def test_empty_filters(self) -> None: - sql, params = compile_row_filters_sql([]) - self.assertEqual(sql, "TRUE") - self.assertEqual(params, {}) - - def test_eq_case_insensitive(self) -> None: - sql, params = compile_row_filters_sql( - [{"field": "afi", "op": "eq", "value": "IPv4"}] - ) - self.assertIn("lower(", sql) - self.assertIn("afi", sql) - self.assertEqual(list(params.values()), ["ipv4"]) - - def test_contains(self) -> None: - sql, params = compile_row_filters_sql( - [{"field": "af", "op": "contains", "value": "IPv4"}] - ) - self.assertIn("LIKE", sql) - self.assertEqual(list(params.values()), ["%ipv4%"]) - - def test_any_all_nesting(self) -> None: - sql, params = compile_row_filters_sql( - [ - { - "any": [ - {"field": "entry_type", "op": "eq", "value": "dynamic"}, - { - "all": [ - {"field": "entry_type", "op": "empty"}, - {"field": "ip", "op": "not_empty"}, - ] - }, - ] - } - ] - ) - self.assertIn(" OR ", sql) - self.assertIn(" AND ", sql) - self.assertTrue(_filters_sql_compatible([{"any": [{"field": "a", "op": "eq", "value": "1"}]}])) - - def test_in_list(self) -> None: - sql, params = compile_row_filters_sql( - [{"field": "afi", "op": "in", "value": ["ipv4", "ipv6"]}] - ) - self.assertIn(" IN (", sql) - self.assertEqual(sorted(params.values()), ["ipv4", "ipv6"]) - - def test_field_rules_matrix(self) -> None: - self.assertTrue(_field_rules_sql_compatible([{"field": "mac", "normalize": "lower"}])) - self.assertFalse(_field_rules_sql_compatible([{"field": "mac", "normalize": "mac"}])) - self.assertTrue( - _field_rules_sql_compatible( - [{"field": "rx", "compare": "numeric", "tolerance": 1}] - ) - ) - self.assertTrue( - _field_rules_sql_compatible( - [{"field": "pfx_rcd", "compare": "percent", "tolerance": 5}] - ) - ) - self.assertTrue( - _field_rules_sql_compatible([{"field": "x", "compare": "ignore"}]) - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_biz_state_monitor_service.py b/tests/test_biz_state_monitor_service.py deleted file mode 100644 index 6f266cd..0000000 --- a/tests/test_biz_state_monitor_service.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Monitoring query correctness, bounded reads and active-collection protection.""" -import csv -import io -import zipfile -from datetime import datetime, timedelta - -import pytest -from fastapi import HTTPException -from sqlalchemy import create_engine, event -from sqlalchemy.orm import sessionmaker - -from netx_api.db import Base -from netx_api.models import ( - BizCompareRun, BizStateBatch, BizStateBatchCommand, BizStateLldpNeighbor, - BizStateMetricRow, BizStateTask, BizStateTaskItem, BizStateTaskItemBinding, -) -from netx_api.biz_state import service -from netx_api.biz_state.retention import protected_batch_map, purge_task_batches - - -@pytest.fixture -def db(): - engine = create_engine("sqlite+pysqlite:///:memory:") - Base.metadata.create_all(engine) - session = sessionmaker(bind=engine, expire_on_commit=False)() - session.add(BizStateTask(id="t", ne_id="ne", retention_days=1)) - session.add(BizStateBatch(id="b", task_id="t", status="success")) - session.commit() - yield session - session.close() - engine.dispose() - - -def test_task_detail_uses_one_binding_query_and_progress_omits_items(db): - for n in range(30): - db.add(BizStateTaskItem(id=f"i{n}", task_id="t", sort_order=n)) - for ph, val in (("vrf", f"v{n}"), ("neighbor", f"p{n}")): - db.add(BizStateTaskItemBinding(id=f"{n}-{ph}", item_id=f"i{n}", placeholder=ph, value=val)) - db.commit() - db.expunge_all() - statements = [] - event.listen(db.get_bind(), "before_cursor_execute", lambda c, cur, sql, p, ctx, many: statements.append(sql)) - result = service.get_task(db, "t") - assert len(statements) == 3 - assert len(result["items"]) == 30 - assert result["items"][9]["sort_order"] == 9 - assert {b["value"] for b in result["items"][9]["bindings"]} == {"v9", "p9"} - db.expunge_all() - statements.clear() - progress = service.get_task_progress(db, "t") - assert len(statements) == 1 - assert "items" not in progress - assert progress["id"] == "t" - with pytest.raises(HTTPException) as exc: - service.get_task_progress(db, "missing") - assert exc.value.status_code == 404 - - -@pytest.mark.parametrize("metric,column", [("custom", "name"), ("lldp_neighbor", "remote_sys")]) -@pytest.mark.parametrize("keyword", ["%", "_", "\\"]) -def test_metric_search_treats_wildcards_literally(db, metric, column, keyword): - for n, value in enumerate((f"exact{keyword}value", "different-value")): - if metric == "lldp_neighbor": - db.add(BizStateLldpNeighbor(id=f"r{n}", batch_id="b", local_if=f"ge{n}", remote_sys=value)) - else: - db.add(BizStateMetricRow(id=f"r{n}", batch_id="b", metric_id=metric, seq=n, data_json={column: value})) - db.commit() - result = service.list_batch_metric_rows(db, "b", metric, kw=keyword, column=column) - assert result["total"] == 1 - assert result["items"][0][column] == f"exact{keyword}value" - - -def test_summary_does_not_select_raw_text_and_keeps_early_aux_failure(db): - now = datetime(2026, 10, 10) - for cid, status, raw, stored in ( - ("a", "aux_failed", "first\nsecond\n", 0), - ("b", "ok", "X" * 1_000_000, 999), - ("c", "aux_cached", " \t\n", 0), - ): - db.add(BizStateBatchCommand(id=cid, batch_id="b", raw_command="show arp" if cid != "c" else "show config", - metric_id="arp", parse_status=status, raw_text=raw, raw_line_count=stored, created_at=now)) - db.commit() - db.expunge_all() - column_names = [] - event.listen(db.get_bind(), "after_cursor_execute", lambda c, cur, sql, p, ctx, many: - column_names.extend([d[0] for d in (cur.description or [])]) if "biz_state_batch_command" in sql else None) - summary = service.get_batch(db, "b") - cmds = {c["id"]: c for c in summary["commands"]} - assert "biz_state_batch_command_raw_text" not in column_names - assert cmds["a"]["raw_line_count"] == 2 - assert cmds["b"]["raw_line_count"] == 999 - assert cmds["b"]["has_raw"] is True - assert cmds["c"]["has_raw"] is False - assert summary["command_stats"]["aux_failed"] == 1 - assert service.get_batch_command(db, "b", "b")["raw_text"] == "X" * 1_000_000 - - -def test_active_batches_cannot_be_deleted_or_purged(db): - for status in ("queued", "running"): - db.add(BizStateBatch(id=status, task_id="t", status=status, started_at=datetime.utcnow() - timedelta(days=3))) - db.commit() - for bid in ("queued", "running"): - with pytest.raises(HTTPException) as exc: - service.delete_batch(db, bid) - assert exc.value.status_code == 409 - assert "active_collection" in exc.value.detail["reasons"] - result = service.delete_batches_bulk(db, ["queued", "running"]) - assert result["deleted"] == [] - purge_task_batches(db, db.get(BizStateTask, "t")) - assert db.get(BizStateBatch, "queued") is not None - assert db.get(BizStateBatch, "running") is not None - db.get(BizStateTask, "t").collect_running = True - db.commit() - with pytest.raises(HTTPException) as exc: - service.delete_task(db, "t") - assert exc.value.detail == "task_collecting" - - -def test_protection_reads_only_scoped_reference_columns(db): - db.add(BizStateBatch(id="other", task_id="other-task", status="success")) - db.add(BizCompareRun(id="r", before_batch_id="b", after_batch_id="other", summary_json={"large": "x" * 10000})) - db.commit() - db.expunge_all() - statements = [] - event.listen(db.get_bind(), "before_cursor_execute", lambda c, cur, sql, p, ctx, many: statements.append(sql)) - assert protected_batch_map(db, batch_ids=["b"]) == {"b": ["compare_run_before"]} - assert all("summary_json" not in sql for sql in statements) - assert protected_batch_map(db, task_id="t") == {"b": ["compare_run_before"]} - assert protected_batch_map(db, batch_ids=[]) == {} - - -def test_export_preserves_false_zero_csv_and_duplicate_sequences(db): - for n in range(1005): - rec = {"n": n, "enabled": False, "note": 'a,b"c\rd\ne' if n == 0 else None} - if n == 1004: - rec["late"] = "last-column" - db.add(BizStateMetricRow(id=f"r{n:04}", batch_id="b", metric_id="custom", seq=0, data_json=rec)) - db.commit() - with zipfile.ZipFile(io.BytesIO(service.export_batch_zip(db, "b"))) as archive: - rows = list(csv.DictReader(io.StringIO(archive.read("tables/custom.csv").decode("utf-8"), newline=""))) - assert len(rows) == 1005 - assert rows[0] == {"n": "0", "enabled": "False", "note": 'a,b"c\rd\ne', "late": ""} - assert rows[-1]["late"] == "last-column" - assert [row["n"] for row in rows] == [str(n) for n in range(1005)] diff --git a/tests/test_biz_state_paired_sampling.py b/tests/test_biz_state_paired_sampling.py deleted file mode 100644 index 1594824..0000000 --- a/tests/test_biz_state_paired_sampling.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Paired admission/dispatch, actual cadence, and per-metric isolation.""" -from concurrent.futures import ThreadPoolExecutor -from datetime import timedelta -from threading import Barrier, Event - -import pytest -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import StaticPool - -from netx_api.db import Base -from netx_api.models import BizStateTask, BizStateTaskItem, BizStateBatch -from netx_api.timeutil import utcnow_naive -from netx_api.biz_state import claim -from netx_api import biz_state_scheduler as scheduler - - -@pytest.fixture -def db(monkeypatch): - engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) - Base.metadata.create_all(engine) - factory = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False) - monkeypatch.setattr(claim, "SessionLocal", factory) - monkeypatch.setattr(scheduler, "SessionLocal", factory) - with factory() as session: - for tid in ("old", "new"): - session.add(BizStateTask(id=tid, source="managed", ne_id=tid, - status="running", interval_sec=60, collect_group_id="pair")) - session.add(BizStateTaskItem(id="item-" + tid, task_id=tid, source_profile_id="zte.arp", enabled=True)) - session.commit() - yield session - scheduler._in_flight.clear() - - -def test_individual_trigger_enqueues_both_with_one_identity(db): - result = claim.enqueue_collect("new", manual=True) - assert result["queued"] and result["task_id"] == "new" - rows = db.query(BizStateBatch).all() - assert len(rows) == 2 - assert {b.task_id for b in rows} == {"old", "new"} - assert len({b.collect_round_id for b in rows}) == 1 - assert len({b.queued_at for b in rows}) == 1 - assert not claim.enqueue_collect("old", manual=True)["queued"] - - -@pytest.mark.parametrize("problem", ["busy", "no_items", "paused", "same_device"]) -def test_pair_admission_never_creates_half_round(db, problem): - peer = db.get(BizStateTask, "new") - if problem == "busy": - peer.collect_running = True - elif problem == "no_items": - db.delete(db.get(BizStateTaskItem, "item-new")) - elif problem == "paused": - peer.status = "paused" - else: - peer.ne_id = "old" - db.commit() - assert not claim.enqueue_collect("old")["queued"] - assert db.query(BizStateBatch).count() == 0 - - -def test_pair_claim_requires_two_slots(db): - claim.enqueue_collect("old") - assert claim.claim_queued_batches(1) == [] - assert len(claim.claim_queued_batches(2)) == 2 - - -def test_busy_device_keeps_both_queued_and_no_ssh_overlap(db): - db.add(BizStateBatch(id="portrait", task_id="other", ne_id="old", status="running")) - db.commit() - claim.enqueue_collect("old") - assert claim.claim_queued_batches(8) == [] - assert db.query(BizStateBatch).filter(BizStateBatch.status == "queued").count() == 2 - - -def test_capacity_one_has_actionable_admission_error(db, monkeypatch): - monkeypatch.setattr(claim, "max_concurrent_tasks", lambda: 1) - assert claim.enqueue_collect("old")["reason"] == "collect_pair_capacity" - assert db.query(BizStateBatch).count() == 0 - - -def test_manual_routes_are_excluded_from_periodic_scheduler(db): - for t in db.query(BizStateTask).all(): - t.collect_manual_only = True - db.commit() - assert scheduler._enqueue_due_tasks() == 0 - assert claim.enqueue_collect("old", manual=True)["queued"] - - -def test_pair_cadence_uses_later_start_and_slow_peer_blocks_restart(db): - now = utcnow_naive() - db.get(BizStateTask, "old").last_collect_started_at = now - timedelta(seconds=100) - db.get(BizStateTask, "new").last_collect_started_at = now - timedelta(seconds=10) - db.commit() - assert scheduler._enqueue_due_tasks() == 0 - db.get(BizStateTask, "new").last_collect_started_at = now - timedelta(seconds=100) - db.get(BizStateTask, "new").collect_running = True - db.commit() - assert scheduler._enqueue_due_tasks() == 0 - db.get(BizStateTask, "new").collect_running = False - db.commit() - assert scheduler._enqueue_due_tasks() == 1 - assert db.query(BizStateBatch).count() == 2 - - -def test_inline_pair_uses_two_workers_without_waiting_for_one_side(db, monkeypatch): - from netx_api.biz_state import collect_runner - - both_started, release = Barrier(2), Event() - started, synchronized = [], [] - def fake_collect(**job): - started.append(job["task_id"]) - both_started.wait(timeout=3) - synchronized.append(job["task_id"]) - release.wait(timeout=3) - monkeypatch.setattr(scheduler, "execute_claimed_batch", fake_collect) - monkeypatch.setattr(collect_runner, "_should_execute_inline", lambda: True) - with ThreadPoolExecutor(max_workers=2) as pool: - monkeypatch.setattr(scheduler, "_dispatch_pool_get", lambda: pool) - result = collect_runner.dispatch_collect("old", manual=True) - assert result["queued"] - # Both worker entries must reach a barrier; serial old→new would fail it. - release.set() - assert set(started) == {"old", "new"} - assert set(synchronized) == {"old", "new"} - - -def test_faster_metrics_claim_before_manual_route_round(db): - for tid in ("ro", "rn"): - db.add(BizStateTask(id=tid, ne_id="old" if tid == "ro" else "new", - status="paused", collect_group_id="routes", collect_manual_only=True)) - db.add(BizStateTaskItem(id="i-" + tid, task_id=tid, source_profile_id="zte.ip_route", enabled=True)) - db.commit() - claim.enqueue_collect("ro", manual=True) - claim.enqueue_collect("old") - jobs = claim.claim_queued_batches(4) - assert {job["task_id"] for job in jobs} == {"old", "new"} - - -def test_manual_route_round_cannot_starve_behind_continuous_fast_metrics(db): - for tid in ("ro", "rn"): - db.add(BizStateTask(id=tid, ne_id="old" if tid == "ro" else "new", - status="paused", collect_group_id="routes", collect_manual_only=True)) - db.add(BizStateTaskItem(id="i-" + tid, task_id=tid, source_profile_id="zte.ip_route", enabled=True)) - db.commit() - claim.enqueue_collect("ro", manual=True) - claim.enqueue_collect("old") - for batch in db.query(BizStateBatch).filter(BizStateBatch.collect_group_id == "routes").all(): - batch.queued_at = utcnow_naive() - timedelta(seconds=301) - db.commit() - assert {job["task_id"] for job in claim.claim_queued_batches(4)} == {"ro", "rn"} - - -@pytest.mark.parametrize("running", [False, True]) -def test_stopping_one_side_stops_the_whole_round(db, monkeypatch, running): - from netx_api.biz_state import collect_stop - - monkeypatch.setattr(collect_stop, "SessionLocal", claim.SessionLocal) - claim.enqueue_collect("old") - if running: - claim.claim_queued_batches(2) - result = collect_stop.request_stop_collect("new") - assert result["ok"] and result["stopped"] - db.expire_all() - batches = db.query(BizStateBatch).all() - try: - if running: - assert len(result["signaled_batches"]) == 2 - assert all(b.message == collect_stop.STOP_REQUEST_TOKEN for b in batches) - assert all(t.collect_running for t in db.query(BizStateTask).all()) - else: - assert len(result["cancelled_batches"]) == 2 - assert all(b.status == "cancelled" for b in batches) - assert not any(t.collect_running for t in db.query(BizStateTask).all()) - assert claim.claim_queued_batches(2) == [] - assert claim.enqueue_collect("new")["queued"] - finally: - for batch in batches: - collect_stop.clear_stop_requested(batch.id) - diff --git a/web/src/components/ui/FieldSelect.tsx b/web/src/components/ui/FieldSelect.tsx index 2b12ebd..cc65495 100644 --- a/web/src/components/ui/FieldSelect.tsx +++ b/web/src/components/ui/FieldSelect.tsx @@ -1,4 +1,4 @@ -import { useId, type ReactNode, type SelectHTMLAttributes } from "react"; +import type { ReactNode, SelectHTMLAttributes } from "react"; import { Label } from "@heroui/react"; /** Compact labeled native select — HeroUI Select is heavy for dense ops forms. */ @@ -9,8 +9,6 @@ export function FieldSelect({ fullWidth, className = "", children, - id, - "aria-describedby": describedBy, ...props }: { label?: ReactNode; @@ -20,26 +18,17 @@ export function FieldSelect({ className?: string; children: ReactNode; } & SelectHTMLAttributes) { - const generatedId = useId(); - const selectId = id || generatedId; - const hintId = `${selectId}-hint`; return ( -
+
+ {hint ? {hint} : null} + ); } diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 7681f11..8e14743 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -174,20 +174,6 @@ const en = { }, bizState: { title: "Business monitor", - loadedTasks: "Loaded tasks", - loadedBatches: "Recent batches · {{n}} loaded (up to 200)", - collectErrors: "Recent collection errors", - retry: "Retry", - taskLoading: "Loading task configuration…", - tasksLoadError: "Could not load tasks", - taskLoadError: "Could not load task configuration", - batchLoadError: "Could not load batch", - sheetLoadError: "Could not load structured data", - statusQueued: "Queued", - batchSuccess: "Collected", - batchPartial: "Partially completed", - batchFailed: "Collection failed", - batchCancelled: "Cancelled", create: "Create task", createHint: "Pick any valid NE (managed / UME); search supported", createModeDevice: "Pick device", @@ -239,7 +225,6 @@ const en = { statusCollecting: "Collecting", detail: "Details", interval: "Interval", - intervalUnit: "Interval unit", intervalUnitDays: "days", intervalUnitHours: "hours", intervalUnitSeconds: "seconds", @@ -386,17 +371,6 @@ const en = { templateExported: "Template exported", templateImported: "Template imported", templateImportInvalid: "Invalid file: need name and metrics", - templateValidationFailed: "Template validation failed; correct the indicated settings", - filterOperator: "Filter operator", - complexFiltersPreserved: "These nested filters are preserved in full. Export, edit the JSON and import it to update them.", - portPolicy: "Port change policy", - portPolicyHint: "Port mappings take priority. This policy controls whether interface identity fields participate in matching without a mapping.", - portPolicyAuto: "Automatic", - portPolicyStrict: "Interfaces must match", - portPolicyIgnore: "Allow port changes", - missingSheets: "Configured checks were removed or renamed; select the scope again", - enableCurrentSheets: "Enable all current checks", - templateUsage: "Used by {{n}} jobs", templateImportHint: "Import/export JSON to share templates privately (no job/batch data)", templateHint: "Pick metrics, then set Key / interface / compare / display. Compare modes: exact, abs delta, or relative %. Display can include context-only columns. Empty compare = presence-only.", @@ -500,10 +474,6 @@ const en = { sideAfter: "After", sidesVs: "vs", sidesTitle: "Compare sides", - configSnapshot: "Execution settings saved", - configSnapshotHint: "Template rules and port mapping are fixed for this run. Later edits do not affect historical lookup. The downloaded report includes the execution settings.", - configLegacy: "Legacy run: settings not saved", - configLegacyHint: "This run has no configuration snapshot. Success lookup and compact row reconstruction use the current settings, which may differ from the original execution.", pick: "Select…", sameAsBefore: "Same as before / select…", autoHint: "auto: new after-task batches trigger compare", @@ -524,60 +494,6 @@ const en = { needBeforeBatch: "Select before task and batch", needAfterBatch: "Select after batch", runNow: "Run now", - runStarted: "Compare started in background — you can close this page; check run history for progress", - runFailed: "Compare failed", - runStatusRunning: "Running", - runStatusFailed: "Failed", - runStatusDoneSec: "Done {{s}}s", - runProgress: "{{phase}} · sheet {{i}}/{{n}} · {{sheet}} · {{s}}s elapsed", - runSheetProgress: "sheet {{i}}/{{n}}", - runElapsed: "{{s}}s elapsed", - runRowsLoaded: "Loaded {{side}} {{n}} rows", - runRowsSqlCount: "DB count {{side}} {{n}} rows", - runPersisting: "Wrote {{done}}/{{total}} rows", - runEngineSql: "engine SQL", - runEnginePython: "engine Python", - runEngineNote: "reason {{note}}", - phaseQueued: "Queued", - phaseLoading: "Loading", - phaseSqlCount: "Counting", - phaseSqlProject: "Preparing tables", - phaseSqlJoin: "Joining", - phaseSqlFailFetch: "Fetching fails", - phaseComparing: "Comparing", - phasePersisting: "Writing", - phasePersistingFail: "Writing fails", - phasePersistingOk: "Writing passes", - phaseDone: "Done", - phaseFailed: "Failed", - phaseCancelled: "Cancelled", - sheetPending: "Pending", - colProgress: "Progress / result", - passRateScope: "all rows", - keyFiltersToggle: "Field filters", - hideDisplayCols: "Hide display cols", - showDisplayCols: "Show display cols", - displayColsToggle: "Display cols", - hideDisplayColsHint: "Compact: display-only columns are hidden by default; highlight means they are shown", - ranWithDuration: "Compare finished ({{s}}s)", - unchangedNotStored: "Success rows were counted but not stored. Set “Store success rows” to sample and re-run for spot-check.", - unchangedSampleHint: "{{total}} success rows total; browsing a stratified sample of {{listed}}. Search any route for live source lookup. Pass rate uses all {{total}}.", - liveSearchHint: "Pairs are completed from source batches before filtering. Pass rate still uses the full sheet.", - liveSearchTruncatedHint: "Search limit reached; some entries are omitted. Narrow the filters. Incomplete key groups were skipped.", - clearSearch: "Clear filters", - resultLoading: "Loading comparison details…", - resultRange: "{{from}}–{{to}} of {{total}}", - resultLoadFailed: "Could not load comparison details", - retry: "Retry", - searchEmpty: "No results match these filters. Clear filters or select another result type.", - clearKeyFilters: "Clear fields", - storeUnchanged: "Store success rows", - storeUnchangedAuto: "Auto (full if small / sample 5k + hydrate if large)", - storeUnchangedSample: "Sample only (up to 5k keys + hydrate)", - storeUnchangedKeys: "All success keys (full browse; large write still slow)", - storeUnchangedAlways: "Always full JSON (slow on huge sheets)", - storeUnchangedNever: "Never (counts only)", - storeUnchangedHint: "Cutover focuses on fails and pass rate; success defaults to a stratified sample. Type a filter to look up any route from source tables. Pass rate uses the full success count.", saveJob: "Save config", jobSaved: "Job config saved", jobDeleted: "Compare job deleted", @@ -586,46 +502,15 @@ const en = { edit: "Edit", delete: "Delete", tabConfig: "Job config", - tabRuns: "Compare batches", tabResult: "Result", - detailTabs: "Business comparison workspace", - jobReady: "Ready", - configBasics: "Job parameters", - configSources: "Before and after sources", - configScope: "Check scope", - selectMapping: "Select mapping", - mappingPairs: "Interface pairs", - sourceBaseline: "Fixed baseline", - sourceTarget: "Acceptance target", - sourceLatest: "Latest batch", - sourceRows: "{{n}} collected rows", - autoSourceHint: "Keep the before batch fixed. Compare against the latest completed batch from the after task.", - runHistoryTitle: "Execution history", - runHistoryHint: "Showing the latest 50 runs. Execution status and business verdicts are displayed separately. Open results to inspect.", - refreshRuns: "Refresh history", - loadedRuns: "Loaded runs", - activeRuns: "In progress", - completedRuns: "Completed", - stoppedRuns: "Failed / cancelled", - runFinished: "Completed", - runStatusQueued: "Queued", - runsLoading: "Loading execution history…", - runsLoadError: "Could not load execution history", runs: "Run history", - runsHint: "Each “Run now” creates a batch. Stuck “Running” after restart is auto-cancelled; you can also cancel manually and re-run.", noRuns: "No runs yet — run a compare first", result: "Result", - viewResult: "View result", - cancelRun: "Cancel", - confirmCancelRun: "Cancel this compare batch? You can start a new run afterward.", - runCancelled: "Compare cancelled", - runStatusCancelled: "Cancelled", - compareAlreadyRunning: "A compare is already running — cancel it under Compare batches or wait", - runBatchSummary: "Fail {{fail}} · Pass {{ok}} · Added {{added}}", - colTime: "Time", pickBatchRun: "Select compare run", pickRun: "Select run…", runCount: "{{n}} runs", + resultEmpty: "No matching diff rows", + resultFilterPh: "Filter key / values…", kindAll: "All", kindDiff: "Fail", kindAdded: "Added", @@ -657,7 +542,7 @@ const en = { filterFailField: "Failed compare field", filterFailFieldAll: "Any failed field", resultEmpty: "No matching rows", - resultFilterPh: "Free text, or direction:out network:1.1.1.1", + resultFilterPh: "Filter identity / values…", diffCount: "{{n}} failed", passRate: "Pass rate", passOk: "All passed", @@ -696,88 +581,6 @@ const en = { colChange: "Changes", }, bizMigration: { - fullscreen: "Wall mode", - exitFullscreen: "Exit wall mode", - fullscreenHint: "Display cutover status fullscreen. Press Esc to return; use a full-window layout when fullscreen is unavailable.", - wallTitle: "Cutover monitoring · Service migration", - wallProgress: "Wave migration progress · Current snapshot", - wallFilters: "Show filters and evaluation controls", - wallState_waiting: "Awaiting evaluation", - wallState_unknown: "Data needs confirmation", - wallState_anomaly: "Service anomalies detected", - wallState_normal: "No service anomalies in current samples", - pairCollectionHint: "Each metric has a paired collection round. Saving settings updates task bindings. Route tables are manual-only for final checks; use a separate business comparison for millions of routes. Actual cadence depends on command duration and device availability.", - pairedSampling: "Paired sampling · Timeliness", - manualRoute: "Manual final check", - pairedPeriodic: "Paired periodic", - samplingMode: "Sampling mode", - samplingDuration: "Duration old / new", - samplingQueue: "Queue old / new", - startSkew: "Start skew", - sameRound: "Round", - metricBaselineMissing: "This metric needs a matching baseline receipt", - samplePhase_idle: "Not collected", - samplePhase_queued: "Queued", - samplePhase_running: "Collecting", - samplePhase_success: "Complete", - samplePhase_partial: "Partial", - samplePhase_failed: "Failed", - samplePhase_cancelled: "Stopped", - collect_pair_invalid: "Incomplete collection pair", - collect_pair_missing: "No complete paired sample", - collect_pair_pending: "Waiting for both sides of this round", - collect_pair_failed: "Current round failed; older samples cannot replace it", - collect_round_mismatch: "Samples belong to different rounds", - sample_start_skew: "Actual collection starts are too far apart", - latestSnapshot: "Latest snapshot", - steadyOutside: "{{n}} stable items outside scope", - "followLive": "Follow live", - "historySnapshot": "Historical snapshot", - "evaluatedAt": "Evaluated", - "refreshBoard": "Refresh board", - "oldestSample": "Oldest sample", - "liveOverview": "Business migration overview", - "remaining": "Remaining items", - "anomalyInBatch": "Current wave anomalies", - "anomalyOutsideBatch": "Outside wave anomalies", - "coverageUnknown": "Uncertain coverage; acceptance cannot pass", - "coverageUnknownHint": "Resolve missing, stale or misaligned samples and duplicate keys. Healthy samples cover their own monitor items.", - "awaitingPeer": "Awaiting target sample", - "migratedPrevious": "Previous wave received", - "unfinishedPrevious": "Previous wave unfinished", - "sample_time_missing": "Sample time missing", - "sample_skew": "Samples too far apart", - "sample_stale": "Stale samples", - "duplicateKeys": "Duplicate business keys", - "uncoveredScope": "Missing scope sheets", - "refreshFailed": "Refresh failed; showing retained snapshot:", - "lastAcceptance": "Last acceptance", - "diffSearch": "Search business key / verdict / status", - "verdictFilter": "Verdict filter", - "allVerdicts": "All verdicts", - "warningVerdict": "In progress / warning", - "healthyVerdict": "Received / healthy", - "unknownVerdict": "Uncertain", - "loadingBoard": "Updating…", - "rowsMatched": "{{n}} matching items", - "showOutsideAnomalies": "Show outside anomalies", - "scopeCurrent": "Current wave", - "scopePrevious": "Previous wave", - "scopeOutside": "Outside scope", - "fieldChanges": "{{n}} changed fields", - "outsideProtection": "Protect businesses outside this wave", - "outsideProtectionHint": "Report drift outside this wave. Continue checking migrated businesses from previous waves without counting them in current progress.", - "reason_service_not_received": "Source departed; target not healthy", - "reason_target_unhealthy": "Target unhealthy", - "reason_old_service_not_in_baseline": "Business missing from source baseline", - "reason_service_content_changed": "Received business content differs", - "reason_old_service_still_present": "Business still on source", - "reason_target_sample_predates_source": "Target sample predates source departure", - "reason_previous_batch_received": "Previous wave remains healthy", - "reason_previous_batch_not_received": "Previous wave not received", - "reason_unmapped_same_iface": "Drift on unmapped interface", - "reason_out_of_expect:anomaly": "Business drift outside scope", - "reason_out_of_expect:lost": "Business lost outside scope", title: "Cutover monitor", hint: "Separate from business compare: baseline drift, expect set, dual-device verdict. Reuses collect tasks and port mappings.", hintPort: @@ -851,6 +654,7 @@ const en = { newBaseline: "New baseline batch", needProjectFields: "Enter a name and pick old/new portrait tasks", needExpectPorts: "Select at least one expected object for this batch", + needBaselineFirst: "Save an old baseline batch first to list expect objects", needBothBaselines: "Save both old and new baseline batches before evaluating", errOldBaselineRequired: "Old baseline batch is required", errNewBaselineRequired: "New baseline batch is required", @@ -881,10 +685,6 @@ const en = { tabBoard: "Board", saveBaseline: "Save", baselineSaved: "Baseline saved", - errPairCapacity: "Paired sampling needs at least two collection slots. Check task concurrency, collection threads and the CLI budget.", - errPairBusy: "This metric is still collecting or queued on either side. Wait for the round to finish.", - errPairInvalid: "The collector pair is incomplete. Save configuration again to restore it.", - errPairSameDevice: "Cutover monitoring requires two different devices.", pickExpectPorts: "Expect", portFilterPh: "Filter", emptyBaselinePorts: "No baseline data", @@ -948,8 +748,8 @@ const en = { acceptFailedShort: "Fail", colMapped: "Mapped new", colDesc: "Description", - colOldPort: "Old business key", - colNewPort: "New business key", + colOldPort: "Old port", + colNewPort: "New port", colOldStatus: "Old admin/phy/prot", colNewStatus: "New admin/phy/prot", colOld: "Old", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 07712d0..89e55aa 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -174,20 +174,6 @@ const zh = { }, bizState: { title: "业务监控", - loadedTasks: "已载入任务", - loadedBatches: "最近批次 · 已载入 {{n}} 条（最多 200）", - collectErrors: "最近采集异常", - retry: "重试", - taskLoading: "正在读取任务配置…", - tasksLoadError: "任务列表读取失败", - taskLoadError: "任务配置读取失败", - batchLoadError: "批次读取失败", - sheetLoadError: "结构化数据读取失败", - statusQueued: "排队中", - batchSuccess: "采集成功", - batchPartial: "部分完成", - batchFailed: "采集失败", - batchCancelled: "已取消", create: "创建任务", createHint: "从托管 / UME 等有效网元中选择（支持查找）", createModeDevice: "选择设备", @@ -239,7 +225,6 @@ const zh = { statusCollecting: "采集中", detail: "详情", interval: "采集周期", - intervalUnit: "周期单位", intervalUnitDays: "天", intervalUnitHours: "小时", intervalUnitSeconds: "秒", @@ -386,17 +371,6 @@ const zh = { templateExported: "模板已导出", templateImported: "模板已导入", templateImportInvalid: "导入文件无效：需要 name 与 metrics", - templateValidationFailed: "模板校验失败，请修正标明的配置项", - filterOperator: "过滤运算符", - complexFiltersPreserved: "该过滤规则包含复杂嵌套，已完整保留。可通过导出、修改 JSON 后重新导入来更新。", - portPolicy: "端口变化策略", - portPolicyHint: "存在端口映射时按映射匹配；此策略控制无映射时接口身份字段是否参与配对。", - portPolicyAuto: "自动判断", - portPolicyStrict: "接口必须匹配", - portPolicyIgnore: "允许端口变化", - missingSheets: "配置中的检查项已被移除或改名，请重新选择检查范围", - enableCurrentSheets: "启用当前全部检查项", - templateUsage: "已被 {{n}} 个任务引用", templateImportHint: "导入/导出 JSON，便于私下传递模板（不含任务/批次数据）", templateHint: "先勾选监控项，再配置 Key / 接口 / 比对 / 展示。比对可设完全相等、绝对容差或相对偏差%；展示列可含不参与比对的上下文。不勾比对字段则只校验条目存在性。", @@ -499,10 +473,6 @@ const zh = { sideAfter: "操作后", sidesVs: "比对", sidesTitle: "比对双方", - configSnapshot: "执行配置已保存", - configSnapshotHint: "此记录的模板规则和端口映射已固定；后续编辑不影响历史回查，下载报告包含执行配置。", - configLegacy: "旧记录：未保存执行配置", - configLegacyHint: "此记录创建时未保存配置快照；成功行回查和精简明细还原使用当前配置，修改配置后可能与当时结果不同。", pick: "选择…", sameAsBefore: "同操作前 / 选择…", autoHint: "auto：操作后任务新批次到达会自动跑比对", @@ -523,60 +493,6 @@ const zh = { needBeforeBatch: "请选择操作前任务与批次", needAfterBatch: "请选择操作后批次", runNow: "立即比对", - runStarted: "比对已在后台启动，可关闭本页；进度见历史记录", - runFailed: "比对失败", - runStatusRunning: "比对中", - runStatusFailed: "失败", - runStatusDoneSec: "完成 {{s}}s", - runProgress: "{{phase}} · 表 {{i}}/{{n}} · {{sheet}} · 已用 {{s}}s", - runSheetProgress: "表 {{i}}/{{n}}", - runElapsed: "已用 {{s}}s", - runRowsLoaded: "已加载 {{side}} {{n}} 行", - runRowsSqlCount: "库内统计 {{side}} {{n}} 行", - runPersisting: "已写入 {{done}}/{{total}} 行", - runEngineSql: "引擎 SQL", - runEnginePython: "引擎 Python", - runEngineNote: "原因 {{note}}", - phaseQueued: "排队中", - phaseLoading: "加载数据", - phaseSqlCount: "库内统计", - phaseSqlProject: "准备比对表", - phaseSqlJoin: "库内比对", - phaseSqlFailFetch: "拉取失败行", - phaseComparing: "比对中", - phasePersisting: "写入结果", - phasePersistingFail: "写入失败行", - phasePersistingOk: "写入成功行", - phaseDone: "完成", - phaseFailed: "失败", - phaseCancelled: "已取消", - sheetPending: "等待中", - colProgress: "进度 / 结果", - passRateScope: "全表", - keyFiltersToggle: "字段筛选", - hideDisplayCols: "隐藏展示列", - showDisplayCols: "显示展示列", - displayColsToggle: "展示列", - hideDisplayColsHint: "紧凑模式：默认隐藏不参与比对的展示列；高亮表示当前已显示", - ranWithDuration: "比对完成（耗时 {{s}} 秒）", - unchangedNotStored: "成功行仅统计数量未落库。可在任务配置将「成功行保存」改为抽样后重新比对（抽查用）。", - unchangedSampleHint: "成功共 {{total}} 条，明细抽样 {{listed}} 条（分层抽查）。要查任意路由请输入筛选条件——将按原表即时判定。通过率按全部 {{total}} 计。", - liveSearchHint: "从原始批次补齐同一业务的前后数据后筛选；通过率仍按全表统计。", - liveSearchTruncatedHint: "搜索已达上限，部分业务未展示。请收窄条件；不完整的业务分组已跳过。", - clearSearch: "清空筛选", - resultLoading: "正在加载对比明细…", - resultRange: "{{from}}–{{to}} / {{total}} 条", - resultLoadFailed: "对比明细加载失败", - retry: "重试", - searchEmpty: "当前筛选条件下没有匹配结果，可清空筛选或切换结果类型。", - clearKeyFilters: "清空字段", - storeUnchanged: "成功行保存", - storeUnchangedAuto: "自动（小表全量 / 大表抽样 5000+原表补全）", - storeUnchangedSample: "仅抽样（最多 5000，身份键+原表补全）", - storeUnchangedKeys: "全量身份键（可翻完全部成功，大表写入仍较久）", - storeUnchangedAlways: "始终全量 JSON（大表很慢，慎用）", - storeUnchangedNever: "不保存（只计数量）", - storeUnchangedHint: "割接优先看失败与通过率；成功默认分层抽样。输入筛选可回查原表（不依赖抽样）。通过率按全部成功计数。", saveJob: "保存配置", jobSaved: "任务配置已保存", jobDeleted: "比对任务已删除", @@ -585,48 +501,15 @@ const zh = { edit: "编辑", delete: "删除", tabConfig: "任务配置", - tabRuns: "对比批次", tabResult: "比对结果", - detailTabs: "业务对比工作区", - jobReady: "已就绪", - configBasics: "作业参数", - configSources: "前后数据源", - configScope: "检查范围", - selectMapping: "选择映射", - mappingPairs: "接口对应关系", - sourceBaseline: "固定基准", - sourceTarget: "验收目标", - sourceLatest: "最新批次", - sourceRows: "采集 {{n}} 条", - autoSourceHint: "保持操作前批次固定；操作后任务采集完成时，自动取最新批次进行对比。", - runHistoryTitle: "执行记录", - runHistoryHint: "展示最近 50 条记录。执行状态与业务判定分别显示，可打开结果核查。", - refreshRuns: "刷新记录", - loadedRuns: "已载入批次", - activeRuns: "正在执行", - completedRuns: "执行完成", - stoppedRuns: "失败 / 取消", - runFinished: "已完成", - runStatusQueued: "排队中", - runsLoading: "正在读取执行记录…", - runsLoadError: "执行记录读取失败", runs: "历史比对", - runsHint: "每次「立即比对」生成一条批次。重启后卡住的「比对中」会自动标为已取消；也可手动终止后重跑。", noRuns: "尚无比对记录，请先执行比对", result: "比对结果", - viewResult: "查看结果", - cancelRun: "终止", - confirmCancelRun: "确定终止该比对批次？终止后可重新发起比对。", - runCancelled: "比对已取消", - runStatusCancelled: "已取消", - compareAlreadyRunning: "已有比对在进行中，请先在「对比批次」中终止或等待完成", - runBatchSummary: "失败 {{fail}} · 成功 {{ok}} · 新增 {{added}}", - colTime: "时间", pickBatchRun: "选择比对记录", pickRun: "选择比对记录…", runCount: "{{n}} 次", resultEmpty: "无匹配失败/结果行", - resultFilterPh: "自由词，或 direction:out network:1.1.1.1", + resultFilterPh: "筛选身份 / 字段值…", kindAll: "全部", kindDiff: "失败", kindAdded: "新增", @@ -695,88 +578,6 @@ const zh = { colChange: "变更", }, bizMigration: { - fullscreen: "大屏模式", - exitFullscreen: "退出大屏", - fullscreenHint: "全屏展示割接状态，按 Esc 返回看板；浏览器不支持全屏时使用窗口大屏。", - wallTitle: "割接监控 · 业务迁移", - wallProgress: "本批迁移进度 · 当前快照", - wallFilters: "显示筛选与判定操作", - wallState_waiting: "等待判定", - wallState_unknown: "数据待确认", - wallState_anomaly: "存在业务异常", - wallState_normal: "当前样本无业务异常", - pairCollectionHint: "每个指标两侧同轮关联采集。保存配置会同步增删采集任务；路由项仅手动采集，收尾时再选择，百万路由建议使用独立业务对比任务。频率是目标间隔，实际周期受命令耗时与设备占用影响。", - pairedSampling: "双端关联采集 · 及时性", - manualRoute: "手动收尾", - pairedPeriodic: "关联周期采集", - samplingMode: "采集方式", - samplingDuration: "采集耗时 老 / 新", - samplingQueue: "排队耗时 老 / 新", - startSkew: "启动时差", - sameRound: "同轮", - metricBaselineMissing: "基线未覆盖此指标，需要补充对应基线", - samplePhase_idle: "尚未采集", - samplePhase_queued: "排队", - samplePhase_running: "采集中", - samplePhase_success: "完成", - samplePhase_partial: "部分完成", - samplePhase_failed: "采集失败", - samplePhase_cancelled: "已停止", - collect_pair_invalid: "双端关联配置不完整", - collect_pair_missing: "尚无完整同轮样本", - collect_pair_pending: "等待两侧同轮采集完成", - collect_pair_failed: "本轮采集失败，不能复用旧样本代替", - collect_round_mismatch: "两侧样本不属于同一轮次", - sample_start_skew: "两侧实际启动时差过大", - latestSnapshot: "最新快照", - steadyOutside: "范围外稳定 {{n}} 条", - "followLive": "实时跟随", - "historySnapshot": "历史快照", - "evaluatedAt": "最近判定", - "refreshBoard": "刷新看板", - "oldestSample": "最早采集", - "liveOverview": "业务迁移概览", - "remaining": "待迁移条目", - "anomalyInBatch": "本批异常", - "anomalyOutsideBatch": "范围外异常", - "coverageUnknown": "数据待确认，不能视为验收通过", - "coverageUnknownHint": "采集缺失、过期、不同步或键冲突需要处理；正常样本仅代表对应监控条目。", - "awaitingPeer": "等待新侧采集确认", - "migratedPrevious": "前批已接续", - "unfinishedPrevious": "前批未接续", - "sample_time_missing": "采集时间缺失", - "sample_skew": "两侧采集间隔过大", - "sample_stale": "采集已过期", - "duplicateKeys": "重复业务键", - "uncoveredScope": "范围对应表已失效", - "refreshFailed": "获取失败，当前显示保留快照：", - "lastAcceptance": "上次终验", - "diffSearch": "搜索业务键 / 判定 / 状态", - "verdictFilter": "判定筛选", - "allVerdicts": "全部判定", - "warningVerdict": "迁移中 / 提示", - "healthyVerdict": "已接续 / 正常", - "unknownVerdict": "待确认", - "loadingBoard": "正在更新…", - "rowsMatched": "匹配 {{n}} 条", - "showOutsideAnomalies": "查看范围外异常", - "scopeCurrent": "本批范围", - "scopePrevious": "前批范围", - "scopeOutside": "范围之外", - "fieldChanges": "字段变化 {{n}} 项", - "outsideProtection": "范围外业务持续保护", - "outsideProtectionHint": "本批之外的业务漂移仍报异常；前批已迁移业务持续检查新侧健康，不计入本批进度。", - "reason_service_not_received": "老侧退出，新侧未健康接续", - "reason_target_unhealthy": "新侧业务状态异常", - "reason_old_service_not_in_baseline": "老基线中未找到业务", - "reason_service_content_changed": "接续业务内容不一致", - "reason_old_service_still_present": "老侧业务尚未退出", - "reason_target_sample_predates_source": "新侧样本早于老侧退出", - "reason_previous_batch_received": "前批业务持续健康", - "reason_previous_batch_not_received": "前批业务未正常接续", - "reason_unmapped_same_iface": "未映射端口发生迁移变化", - "reason_out_of_expect:anomaly": "范围外业务漂移", - "reason_out_of_expect:lost": "范围外业务丢失", title: "割接监控", hint: "与业务对比独立：相对基线漂移 + 本批预期 + 老/新双端判定。复用采集任务与端口映射。", hintPort: "当前监控项：端口状态（interface_brief / admin·phy·prot）。选基线 → 勾选本批端口 → 判定看板。", @@ -877,10 +678,6 @@ const zh = { tabBoard: "判定看板", saveBaseline: "保存配置", baselineSaved: "基线已保存", - errPairCapacity: "双端关联采集至少需要两个采集并发槽位，请检查任务并发、采集线程和 CLI 预算配置。", - errPairBusy: "该指标的两侧仍在采集或排队，请等待本轮结束。", - errPairInvalid: "双端采集关联不完整，请重新保存配置建立关联。", - errPairSameDevice: "割接监控需要选择两台不同设备。", pickExpectPorts: "本批预期", portFilterPh: "筛选", emptyBaselinePorts: "基线无数据", @@ -923,6 +720,7 @@ const zh = { metaBaseline: "基线", baselineOk: "已设", baselineMissing: "缺", + needBaselineFirst: "先设基线", enableHighfreq: "开启高频", highfreqReady: "高频已就绪", collectNow: "立即采集", @@ -943,8 +741,8 @@ const zh = { acceptFailedShort: "未过", colMapped: "映射新口", colDesc: "描述", - colOldPort: "老侧业务标识", - colNewPort: "新侧业务标识", + colOldPort: "老端口", + colNewPort: "新端口", colOldStatus: "老状态 admin/phy/prot", colNewStatus: "新状态 admin/phy/prot", colOld: "老侧", @@ -974,6 +772,10 @@ const zh = { currentMissingBanner: "缺当前批次：{{metrics}}", highfreqReadyInactive: "高频已绑定（窗口未开，未立即采集）", collectPartial: "部分失败（成功 {{ok}} / 失败 {{fail}}）", + confirmProjectDone: "确认完成项目并暂停全部高频？", + projectMarkedDone: "已完成，高频已停", + confirmDeleteProject: "确认删除项目？将停高频并清批次记录。", + projectDeleted: "已删除", errMetricNeedsBindings: "监控项「{{metric}}」需要参数绑定（{{params}}），割接高频暂不支持，请去掉该项", errNoProfileForMetric: "监控项「{{metric}}」在当前设备厂商下无可用采集 profile", acceptTitle: "本批验收", diff --git a/web/src/index.css b/web/src/index.css index 52627d5..eb97361 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -11175,60 +11175,6 @@ html.login-page--paused .login-page__flare { overflow: auto; } -/* Cutover operations console: readable device samples, continuity and scope. */ -.bm-live-header, .bm-live-header > div, .bm-live-filters { - display: flex; - align-items: center; - flex-wrap: wrap; - gap: 10px; -} -.bm-live-header { justify-content: space-between; border-bottom: 1px solid rgba(148,163,184,.25); padding-bottom: 10px; } -.bm-live-state { font-size: 11px; padding: 3px 8px; border: 1px solid rgba(148,163,184,.35); border-radius: 3px; } -.bm-live-state.is-live { color: #34d399; border-color: rgba(52,211,153,.45); } -.bm-live-state.is-live::before { content: ""; display: inline-block; width: 6px; height: 6px; background: currentColor; margin-right: 6px; border-radius: 50%; } -.bm-live-sources { display: grid; grid-template-columns: repeat(2,minmax(0,1fr)); gap: 10px; } -.bm-live-sources > div { display: flex; flex-wrap: wrap; align-items: baseline; gap: 6px 12px; padding: 10px 12px; border: 1px solid rgba(148,163,184,.24); border-left: 3px solid #64748b; border-radius: 4px; min-width: 0; } -.bm-live-sources > div:last-child { border-left-color: #38bdf8; } -.bm-live-sources small { flex-basis: 100%; font-variant-numeric: tabular-nums; } -.bm-live-sources strong { overflow-wrap: anywhere; } -.bm-live-kpis { display: grid; grid-template-columns: repeat(4,minmax(0,1fr)); gap: 10px; } -.bm-live-kpis > div { display: flex; flex-direction: column; gap: 5px; padding: 12px 14px; border: 1px solid rgba(148,163,184,.25); border-radius: 4px; } -.bm-live-kpis span { font-size: 12px; opacity: .8; } -.bm-live-kpis strong { font: 600 26px/1.2 ui-monospace,monospace; font-variant-numeric: tabular-nums; } -.bm-live-kpis strong small { font-size: 13px; opacity: .7; } -.bm-live-kpis .is-ok strong { color: #34d399; } -.bm-live-kpis .is-bad strong { color: #f87171; } -.bm-live-warning { display: flex; flex-wrap: wrap; gap: 6px 14px; padding: 10px 12px; border: 1px solid rgba(251,191,36,.3); border-left: 3px solid #fbbf24; border-radius: 3px; background: rgba(251,191,36,.06); font-size: 12px; overflow-wrap: anywhere; } -.bm-live-warning strong { flex-basis: 100%; } -.bm-live-filters > input { flex: 1 1 240px; min-width: 0; } -.bm-live-filters > .ui-field-select { max-width: 230px; } -.bm-diff-row.is-red > td:first-child { box-shadow: inset 3px 0 #f87171; } -.bm-diff-row.is-yellow > td:first-child { box-shadow: inset 3px 0 #fbbf24; } -.bm-scope-label { display: block; font-size: 10px; opacity: .65; padding-top: 3px; } -.bm-field-changes { font-size: 11px; margin-top: 5px; } -.bm-field-changes summary { cursor: pointer; color: #fbbf24; } -.bm-field-changes div { padding-top: 5px; overflow-wrap: anywhere; } -.bm-field-changes ins { color: #f87171; text-decoration: none; } -.bm-board .bs-cmp-main { min-width: 0; } -.bm-sampling { border: 1px solid rgba(148,163,184,.25); border-radius: 4px; min-width: 0; } -.bm-sampling > summary { padding: 8px 12px; cursor: pointer; font-size: 12px; } -.app-heroui-modal .bm-sampling .pt-list-table-wrap { max-width: 100%; min-height: 0; } -.bm-live-header__actions { display: flex; gap: 8px; align-items: center; } -.bm-board-controls > summary { display: none; } -.bm-board-controls > .bm-live-filters { margin-top: 8px; } -@media (max-width: 720px) { - .bm-live-sources { grid-template-columns: 1fr; } - .bm-live-kpis { grid-template-columns: repeat(2,minmax(0,1fr)); gap: 6px; } - .bm-live-header > .muted { flex: 1 1 100%; font-size: 11px; } - .bm-board .bs-cmp-board__toolbar { display: grid; grid-template-columns: minmax(0,1fr); } - .bm-board .bs-id-row { min-width: 0; } - .bm-board .bs-cmp-board__body { display: flex; flex-direction: column; min-width: 0; } - .bm-board .bs-cmp-nav { width: 100%; max-width: none; } - .bm-board .bs-cmp-nav__list { max-height: 180px; } - .bm-live-filters > .ui-field-select { max-width: none; width: 100%; } - .bm-board__table { max-width: 100%; } -} - /* Cutover create wizard */ .bm-create { min-height: min(52vh, 520px); @@ -11752,939 +11698,6 @@ html.login-page--paused .login-page__flare { rgba(8, 13, 24, 0.4); } -.bs-cmp-progress { - display: flex; - flex-direction: column; - gap: 6px; - padding: 10px 12px; - border-radius: 8px; - border: 1px solid rgba(59, 130, 246, 0.35); - background: rgba(37, 99, 235, 0.12); - color: var(--bs-cmp-ink, #e2e8f0); - font-size: 0.875rem; -} - -.bs-cmp-progress.is-failed { - border-color: rgba(239, 68, 68, 0.4); - background: rgba(239, 68, 68, 0.12); -} - -.bs-cmp-progress.is-cancelled { - border-color: rgba(245, 158, 11, 0.4); - background: rgba(245, 158, 11, 0.1); -} - -.bs-cmp-progress__head { - display: flex; - flex-wrap: wrap; - align-items: center; - gap: 8px 12px; -} - -.bs-cmp-progress__actions { - margin-left: auto; - display: flex; - align-items: center; - gap: 6px; -} - -.bs-cmp-progress__engine, -.bs-cmp-progress__elapsed { - font-size: 0.8125rem; -} - -.bs-cmp-progress__track, -.bs-cmp-run-prog__track { - height: 6px; - border-radius: 999px; - background: rgba(148, 163, 184, 0.28); - overflow: hidden; -} - -.bs-cmp-progress__fill, -.bs-cmp-run-prog__fill { - height: 100%; - width: 0; - border-radius: inherit; - background: linear-gradient(90deg, #2563eb, #60a5fa); - transition: width 0.35s ease; -} - -.bs-cmp-progress__fill.is-indeterminate, -.bs-cmp-run-prog__fill.is-indeterminate { - width: 36%; - animation: bs-cmp-prog-slide 1.25s ease-in-out infinite; -} - -@keyframes bs-cmp-prog-slide { - 0% { - transform: translateX(-120%); - } - 100% { - transform: translateX(320%); - } -} - -.bs-cmp-progress__meta { - font-size: 0.8125rem; - color: var(--bs-cmp-muted, #94a3b8); - line-height: 1.4; - word-break: break-word; -} - -.bs-cmp-progress__msg { - flex: 1 1 100%; - font-size: 0.8125rem; -} - -/* Compare batch list (job detail → 对比批次) */ -.bs-cmp-runs { - display: flex; - flex-direction: column; - gap: 10px; - min-height: 260px; -} - -.bs-cmp-runs__toolbar { - display: flex; - flex-wrap: wrap; - align-items: flex-start; - justify-content: space-between; - gap: 10px 14px; -} - -.bs-cmp-runs__hint { - margin: 0; - flex: 1 1 220px; - line-height: 1.45; -} - -.bs-cmp-runs__table tbody tr.is-selected td { - background: rgba(37, 99, 235, 0.08); -} - -.bs-cmp-runs__elapsed { - margin-top: 4px; - font-size: 0.75rem; - color: var(--muted, #64748b); - font-variant-numeric: tabular-nums; -} - -.bs-cmp-runs__sides { - display: flex; - flex-wrap: wrap; - align-items: baseline; - gap: 2px 6px; - font-size: 0.8125rem; - line-height: 1.35; - max-width: 22rem; -} - -.bs-cmp-runs__arrow { - color: var(--muted, #94a3b8); - flex: 0 0 auto; -} - -.bs-cmp-runs__summary { - font-size: 0.8125rem; - line-height: 1.4; - color: var(--ink, #334155); - max-width: 18rem; -} - -.bs-cmp-run-prog { - display: flex; - flex-direction: column; - gap: 5px; - min-width: 160px; - max-width: 22rem; -} - -.bs-cmp-run-prog__meta { - font-size: 0.75rem; - line-height: 1.35; - color: var(--muted, #64748b); - display: -webkit-box; - -webkit-line-clamp: 2; - -webkit-box-orient: vertical; - overflow: hidden; -} - -/* Business compare workspace: compact, flat ops surfaces. */ -.bs-cmp-workspace, -.bs-cmp-job-modal, -.bs-cmp-create { - --cmp-line: #2b3748; - --cmp-muted: #9aa9bc; - --cmp-surface: #111c2a; - --cmp-accent: #6ba9ec; -} - -.panel.bs-cmp-workspace { - border-radius: 6px; - border-color: var(--cmp-line); - background: #0c1521; -} - -.bs-cmp-workspace > .pt-list > .pt-list-table-wrap { - border-radius: 4px; - border-color: var(--cmp-line); -} - -.bs-cmp-workspace > .pt-list > .pt-list-table-wrap .pt-list-table td { - border-bottom: 1px solid var(--cmp-line); -} - -.bs-cmp-workspace .button, -.bs-cmp-workspace .input { - border-radius: 4px; - box-shadow: none; -} - -.bs-cmp-workspace .bs-cmp-runs__delete { - color: #fca5a5; -} - -.bs-cmp-job-modal .modal__dialog, -.bs-cmp-create .modal__dialog { - border-radius: 6px; - background: #0c1521; - border-color: var(--cmp-line); - max-height: min(92dvh, 1000px); -} - -.bs-cmp-job-modal:not(.bs-cmp-board-modal) .modal__dialog { - width: min(1180px, 96vw); - max-width: min(1180px, 96vw); - display: flex; - flex-direction: column; -} - -.bs-cmp-job-modal:not(.bs-cmp-board-modal) .modal__body.bs-cmp-job-body { - min-height: 0; - max-height: none; - flex: 1 1 auto; - overflow: auto; -} - -.bs-cmp-job-modal .button, -.bs-cmp-create .button, -.bs-cmp-job-modal .ui-field__select, -.bs-cmp-create .ui-field__select, -.bs-cmp-job-modal .input, -.bs-cmp-create .input { - border-radius: 4px; - box-shadow: none; -} - -.bs-cmp-job-heading { - display: flex; - flex-direction: column; - gap: 5px; - min-width: 0; - overflow-wrap: anywhere; - font-size: 16px; -} - -.bs-cmp-job-heading__id { - color: var(--cmp-muted); - font: 10px/1.4 ui-monospace, SFMono-Regular, Consolas, monospace; - letter-spacing: 0.08em; -} - -.bs-cmp-job-modal .bs-cmp-job-tabs { - margin: 0; - gap: 0; - flex-wrap: nowrap; - border-bottom: 1px solid var(--cmp-line); -} - -.bs-cmp-job-tabs .button { - flex: 0 1 auto; - min-width: 0; - height: 38px; - padding: 0 15px; - border: 0; - border-bottom: 2px solid transparent; - border-radius: 0; - background: transparent; - color: var(--cmp-muted); -} - -.bs-cmp-job-modal .bs-cmp-job-tabs .button.is-active { - outline: none; - box-shadow: none; - border-bottom-color: var(--cmp-accent); - color: #edf5ff; - background: rgba(107, 169, 236, 0.08); -} - -.bs-cmp-job-tabs__index { - font: 10px ui-monospace, SFMono-Regular, Consolas, monospace; - opacity: 0.7; -} - -.bs-cmp-job-tabs__count { - padding: 1px 5px; - background: #263347; - border-radius: 3px; - font-size: 10px; -} - -.bs-cmp-config { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 12px; -} - -.bs-cmp-config__section { - min-width: 0; - padding: 14px; - border: 1px solid var(--cmp-line); - border-radius: 4px; - background: var(--cmp-surface); -} - -.bs-cmp-config .is-wide, -.bs-cmp-config__fields > .is-wide { - grid-column: 1 / -1; -} - -.bs-cmp-config__title { - display: flex; - align-items: center; - gap: 9px; - margin: 0 0 14px; - color: #e7eef7; - font-size: 12px; - font-weight: 650; -} - -.bs-cmp-config__title > span { - color: var(--cmp-accent); - font: 11px ui-monospace, SFMono-Regular, Consolas, monospace; -} - -.bs-cmp-config__fields, -.bs-cmp-mapping__fields, -.bs-cmp-sources { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 12px 16px; - min-width: 0; -} - -.bs-cmp-config__fields > .ui-field--full, -.bs-cmp-mapping__fields > .ui-field--full { - grid-column: auto; -} - -.bs-cmp-config__hint { - color: var(--cmp-muted); - font-size: 11px; - line-height: 1.65; - margin: 0; -} - -.bs-cmp-source { - display: flex; - flex-direction: column; - gap: 12px; - min-width: 0; - padding: 12px; - border: 1px solid var(--cmp-line); - border-radius: 3px; - background: #0c1521; -} - -.bs-cmp-source__head { - display: flex; - flex-wrap: wrap; - align-items: center; - gap: 8px; - font-size: 12px; -} - -.bs-cmp-source__head > span:last-child { - margin-left: auto; - color: var(--cmp-muted); - font-size: 10px; -} - -.bs-cmp-source__tag { - padding: 2px 5px; - border: 1px solid #465569; - color: #c6d3e3; - font: 10px ui-monospace, SFMono-Regular, Consolas, monospace; -} - -.bs-cmp-source.is-after .bs-cmp-source__tag { - border-color: #3f6a97; - color: #93c5fd; -} - -.bs-cmp-source__meta { - display: flex; - justify-content: space-between; - flex-wrap: wrap; - gap: 5px; - margin-top: auto; - padding-top: 9px; - border-top: 1px solid var(--cmp-line); - color: var(--cmp-muted); - font: 11px/1.4 ui-monospace, SFMono-Regular, Consolas, monospace; -} - -.bs-cmp-source__auto { - min-height: 58px; -} - -.bs-cmp-scope__heading { - display: flex; - justify-content: space-between; - align-items: center; - font-size: 12px; -} - -.bs-cmp-scope__count { - color: var(--cmp-accent); - font: 12px ui-monospace, SFMono-Regular, Consolas, monospace; -} - -.bs-cmp-scope .bm-hint { - margin: 8px 0 12px; - font-size: 11px; - line-height: 1.55; -} - -.bs-cmp-scope__items { - display: grid; - grid-template-columns: repeat(auto-fit, minmax(min(180px, 100%), 1fr)); - gap: 6px; -} - -.bs-cmp-scope__item { - display: flex; - align-items: center; - gap: 9px; - min-width: 0; - padding: 9px 10px; - border: 1px solid var(--cmp-line); - border-radius: 3px; - background: #0c1521; - color: var(--cmp-muted); - text-align: left; - cursor: pointer; -} - -.bs-cmp-scope__item.is-on { - border-color: #3c5f89; - background: rgba(61, 108, 160, 0.1); - color: #dbeafe; -} - -.bs-cmp-scope__item:disabled { - cursor: default; -} - -.bs-cmp-scope__check { - display: grid; - place-items: center; - flex: 0 0 15px; - height: 15px; - border: 1px solid #53677f; - border-radius: 2px; - color: var(--cmp-accent); - font-size: 11px; -} - -.bs-cmp-scope__content { - display: flex; - flex-direction: column; - gap: 3px; - min-width: 0; - overflow-wrap: anywhere; -} - -.bs-cmp-scope__content strong { - font-size: 12px; - font-weight: 550; -} - -.bs-cmp-scope__content > span { - color: var(--cmp-muted); - font-size: 10px; -} - -.bs-cmp-mapping { - display: flex; - flex-direction: column; - gap: 12px; - min-width: 0; -} - -.bs-cmp-mapping__editor { - width: 100%; - min-height: 100px; - padding: 8px 10px; - border: 1px solid var(--cmp-line); - border-radius: 3px; - background: #0a121d; - color: #dce8f6; - resize: vertical; - font: 11px/1.7 ui-monospace, SFMono-Regular, Consolas, monospace; -} - -.bs-cmp-mapping__actions { - display: flex; - justify-content: flex-end; - flex-wrap: wrap; - gap: 6px; -} - -.bs-cmp-create .bm-create__pane, -.bs-cmp-create .bm-steps__item { - border-radius: 4px; - box-shadow: none; -} - -.bs-cmp-job-modal .bs-cmp-runs { - gap: 14px; -} - -.bs-cmp-runs__intro { - flex: 1 1 300px; -} - -.bs-cmp-runs__intro h3 { - margin: 0 0 5px; - color: #e7eef7; - font-size: 14px; -} - -.bs-cmp-job-modal .bs-cmp-runs__hint { - color: var(--cmp-muted); - font-size: 11px; - line-height: 1.55; -} - -.bs-cmp-run-stats { - display: grid; - grid-template-columns: repeat(4, minmax(0, 1fr)); - margin: 0; - border: 1px solid var(--cmp-line); - border-radius: 4px; - background: var(--cmp-surface); -} - -.bs-cmp-run-stats > div { - padding: 12px 15px; - border-right: 1px solid var(--cmp-line); -} - -.bs-cmp-run-stats > div:last-child { - border-right: 0; -} - -.bs-cmp-run-stats dt { - color: var(--cmp-muted); - font-size: 11px; -} - -.bs-cmp-run-stats dd { - margin: 5px 0 0; - color: #e7eef7; - font: 24px/1.2 ui-monospace, SFMono-Regular, Consolas, monospace; - font-variant-numeric: tabular-nums; -} - -.bs-cmp-run-stats .is-active dd { - color: #93c5fd; -} - -.bs-cmp-run-stats .is-stopped dd { - color: #fbbf24; -} - -.bs-cmp-job-modal .bs-cmp-runs .pt-list-table-wrap { - max-height: min(54vh, 580px); - overflow: auto; - border: 1px solid var(--cmp-line); - border-radius: 4px; -} - -.bs-cmp-runs__table { - min-width: 1000px; - table-layout: fixed; -} - -.bs-cmp-runs__table th:nth-child(1) { width: 12%; } -.bs-cmp-runs__table th:nth-child(2) { width: 18%; } -.bs-cmp-runs__table th:nth-child(3) { width: 27%; } -.bs-cmp-runs__table th:nth-child(4) { width: 25%; } -.bs-cmp-runs__table th:nth-child(5) { width: 18%; } - -.bs-cmp-job-modal .bs-cmp-runs__table th { - position: sticky; - top: 0; - z-index: 1; - background: #162232; - color: #a7b6ca; - font-size: 11px; - font-weight: 550; -} - -.bs-cmp-job-modal .bs-cmp-runs__table td { - height: 80px; - border-bottom: 1px solid var(--cmp-line); - background: #0c1521; - vertical-align: middle; - font-size: 12px; -} - -.bs-cmp-job-modal .bs-cmp-runs__table tr.is-selected td { - background: #142237; -} - -.bs-cmp-runs__table tr td:first-child { - border-left: 3px solid transparent; -} - -.bs-cmp-runs__table tr.is-selected td:first-child, -.bs-cmp-runs__table tr[data-status="running"] td:first-child, -.bs-cmp-runs__table tr[data-status="queued"] td:first-child { - border-left-color: var(--cmp-accent); -} - -.bs-cmp-runs__table tr[data-status="failed"] td:first-child { - border-left-color: #f87171; -} - -.bs-cmp-runs__id { - display: block; - margin-top: 5px; - color: var(--cmp-muted); - font-size: 10px; -} - -.bs-cmp-job-modal .bs-cmp-runs__sides { - display: grid; - gap: 8px; - max-width: none; - font-size: 11px; -} - -.bs-cmp-runs__sides > div { - display: grid; - grid-template-columns: 42px minmax(0, 1fr) auto; - align-items: center; - gap: 6px; -} - -.bs-cmp-runs__side-tag, -.bs-cmp-runs__sides time { - color: var(--cmp-muted); - font-size: 10px; - white-space: nowrap; -} - -.bs-cmp-runs__sides strong { - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - font-weight: 550; -} - -.bs-cmp-run-verdict { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 8px; -} - -.bs-cmp-run-verdict > span { - display: flex; - flex-direction: column; - gap: 4px; - color: var(--cmp-muted); - font-size: 10px; -} - -.bs-cmp-run-verdict strong { - color: #dbe7f5; - font: 16px/1.2 ui-monospace, SFMono-Regular, Consolas, monospace; - font-variant-numeric: tabular-nums; -} - -.bs-cmp-run-verdict .is-fail strong { color: #fca5a5; } -.bs-cmp-run-verdict .is-pass strong { color: #86efac; } - -.bs-cmp-job-modal .bs-cmp-runs__summary, -.bs-cmp-job-modal .bs-cmp-run-prog__meta, -.bs-cmp-job-modal .bs-cmp-runs__elapsed { - color: var(--cmp-muted); - font-size: 11px; - overflow-wrap: anywhere; -} - -.bs-cmp-runs__summary.is-stopped { - color: #fbbf24; -} - -.bs-cmp-job-modal .bs-cmp-runs__delete { - color: #fca5a5; -} - -.bs-cmp-runs__error { - display: flex; - justify-content: space-between; - align-items: center; - flex-wrap: wrap; - gap: 8px; - padding: 10px; - border: 1px solid #7f4343; - border-radius: 4px; - color: #fecaca; - background: rgba(127, 67, 67, 0.1); - font-size: 12px; - overflow-wrap: anywhere; -} - -.bs-cmp-job-modal .bs-cmp-board, -.bs-cmp-job-modal .bs-cmp-strip, -.bs-cmp-job-modal .bs-cmp-sides, -.bs-cmp-job-modal .bs-cmp-nav, -.bs-cmp-job-modal .bs-cmp-progress { - border-radius: 4px; - background-image: none; -} - -.bs-cmp-job-modal .bs-cmp-progress__track, -.bs-cmp-job-modal .bs-cmp-run-prog__track { - border-radius: 2px; -} - -.bs-cmp-job-modal .bs-cmp-progress__fill, -.bs-cmp-job-modal .bs-cmp-run-prog__fill { - background: #5795d9; -} - -.bs-cmp-scope__item:focus-visible, -.bs-cmp-job-modal .bs-cmp-job-tabs .button:focus-visible, -.bs-cmp-mapping__editor:focus-visible { - outline: 2px solid var(--cmp-accent); - outline-offset: 2px; -} - -@media (max-width: 800px) { - .bs-cmp-config { grid-template-columns: minmax(0, 1fr); } - .bs-cmp-job-tabs .button { padding: 0 10px; } -} - -@media (max-width: 600px) { - .bs-cmp-sources, - .bs-cmp-config__fields, - .bs-cmp-mapping__fields { grid-template-columns: minmax(0, 1fr); } - .bs-cmp-config__section { padding: 11px; } - .bs-cmp-run-stats { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .bs-cmp-run-stats > div:nth-child(2) { border-right: 0; } - .bs-cmp-run-stats > div:nth-child(-n+2) { border-bottom: 1px solid var(--cmp-line); } - .bs-cmp-job-tabs__index { display: none; } - .bs-cmp-job-modal .bs-cmp-job-tabs .button { flex: 1 1 0; padding: 0 7px; font-size: 12px; } -} - -@media (prefers-reduced-motion: reduce) { - .bs-cmp-job-modal .bs-cmp-progress__fill, - .bs-cmp-job-modal .bs-cmp-run-prog__fill { transition: none; } - .bs-cmp-job-modal .is-indeterminate { animation: none; } -} - -/* Business monitoring uses the same compact operations surfaces as comparison. */ -.bs-monitor-workspace, -.bs-monitor-modal { - --mon-line: #2b3748; - --mon-muted: #9aa9bc; - --mon-accent: #6ba9ec; -} - -.panel.bs-monitor-workspace, -.bs-monitor-modal .modal__dialog { - border-color: var(--mon-line); - border-radius: 6px; - background: #0c1521; -} - -.bs-monitor-workspace .button, -.bs-monitor-modal .button, -.bs-monitor-workspace .input, -.bs-monitor-modal .input, -.bs-monitor-modal .ui-field__select { - border-radius: 4px; - box-shadow: none; -} - -.bs-monitor-workspace .pt-list-kpis { - gap: 0; - border: 1px solid var(--mon-line); - border-radius: 4px; - overflow: hidden; - background: #111c2a; -} - -.bs-monitor-workspace .pt-list-kpi { - border: 0; - border-right: 1px solid var(--mon-line); - border-radius: 0; - background: transparent; -} - -.bs-monitor-workspace .pt-list-kpi:last-child { border-right: 0; } -.bs-monitor-workspace .pt-list-kpi__value { font-family: ui-monospace, Consolas, monospace; } -.bs-monitor-workspace .pt-list-kpi__label { font-size: 11px; color: var(--mon-muted); } - -.bs-monitor-workspace .filter-inline, -.bs-monitor-modal .filter-inline { - flex-wrap: wrap; - border-radius: 4px; -} - -.bs-monitor-workspace > .pt-list > .pt-list-table-wrap, -.bs-monitor-modal .pt-list-table-wrap { - border-color: var(--mon-line); - border-radius: 4px; - min-width: 0; -} - -.bs-monitor-workspace > .pt-list > .pt-list-table-wrap .pt-list-table { min-width: 1120px; } -.bs-monitor-batches .pt-list-table { min-width: 1100px; } -.bs-monitor-workspace .pt-list-table th, -.bs-monitor-modal .pt-list-table th { background: #162232; font-size: 11px; } -.bs-monitor-workspace .pt-list-table td, -.bs-monitor-modal .pt-list-table td { border-bottom-color: var(--mon-line); } -.bs-monitor-workspace .pt-list-table td { font-size: 12px; } -.bs-monitor-workspace .pt-list-task-name + .muted { font-size: 11px; } - -.bs-monitor-heading { display: flex; flex-direction: column; gap: 5px; min-width: 0; } -.bs-monitor-id { color: var(--mon-muted); font: 10px/1.4 ui-monospace, Consolas, monospace; } -.bs-monitor-heading > span:last-child { font-size: 16px; overflow-wrap: anywhere; } -.bs-monitor-task-content { display: flex; flex-direction: column; gap: 14px; min-width: 0; } -.bs-monitor-task-content[hidden] { display: none; } -.bs-monitor-task-content[inert] { opacity: 0.5; } - -.bs-monitor-task .bs-schedule-row { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - align-items: start; - gap: 12px; - padding: 14px; - border: 1px solid var(--mon-line); - border-radius: 4px; - background: #111c2a; -} - -.bs-monitor-task .bs-schedule-row > .muted { grid-column: 1 / -1; font-size: 11px; line-height: 1.6; } -.bs-monitor-task .config-sync-policy-field { min-width: 0; flex-wrap: wrap; } -.bs-monitor-task .config-sync-policy-field .input { min-width: 0; width: 100%; } -.bs-monitor-task .config-sync-policy-field > select { min-width: 0; border-radius: 4px; } -.bs-monitor-task .bs-monitor-interval-field { display: grid; grid-template-columns: minmax(0, 1fr) auto; } -.bs-monitor-task .bs-monitor-interval-field > span { grid-column: 1 / -1; } -.bs-monitor-modal .modal__footer { min-width: 0; max-width: 100%; flex-wrap: wrap; } -.bs-monitor-task .bs-monitor-task-tabs { gap: 0; margin: 0; border-bottom: 1px solid var(--mon-line); } -.bs-monitor-task-tabs .button { border: 0; border-radius: 0; border-bottom: 2px solid transparent; background: transparent; color: var(--mon-muted); } -.bs-monitor-task .bs-monitor-task-tabs .button.is-active { border-bottom-color: var(--mon-accent); color: #edf5ff; background: rgba(107,169,236,0.08); outline: none; box-shadow: none; } -.bs-monitor-task-tabs .button:focus-visible { outline: 2px solid var(--mon-accent); outline-offset: -2px; } -.bs-monitor-history-count { flex: 1 1 240px; font-size: 11px; } -.bs-monitor-batches > .btn-row { padding: 10px; } -.bs-monitor-workspace .bs-monitor-delete, -.bs-monitor-modal .bs-monitor-delete { color: #fca5a5; } - -.bs-monitor-loading { padding: 16px; color: var(--mon-muted); font-size: 12px; } -.bs-monitor-error { - display: flex; - align-items: center; - justify-content: space-between; - flex-wrap: wrap; - gap: 8px; - padding: 10px; - border: 1px solid #7f4343; - border-radius: 4px; - color: #fecaca; - background: rgba(127,67,67,0.1); - font-size: 12px; - overflow-wrap: anywhere; -} - -.bs-monitor-modal .bs-workbook-meta, -.bs-monitor-modal .bs-sheet-cmd-bar, -.bs-monitor-modal .bs-sheet-cmd-row, -.bs-monitor-modal .bs-sheet-tab { border-radius: 4px; background-image: none; } -.bs-monitor-modal .bs-sheet-tabs { min-width: 0; flex-wrap: nowrap; overflow-x: auto; } -.bs-monitor-modal .bs-sheet-tab { flex: 0 0 auto; } -.bs-monitor-modal .bs-sheet-filter { gap: 8px; } -.bs-monitor-modal .bs-sheet-filter .input { flex: 1 1 200px; min-width: 0; } -.bs-monitor-modal .bs-sheet-filter .ui-field { min-width: 0; max-width: 100%; } -.bs-monitor-modal .bs-sheet-filter .ui-field__select { max-width: 100%; } -.bs-monitor-modal .bs-sheet-count { margin-left: auto; } - -@media (max-width: 800px) { - .bs-monitor-workspace .pt-list-kpis { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .bs-monitor-workspace .pt-list-kpi:nth-child(2) { border-right: 0; } - .bs-monitor-workspace .pt-list-kpi:nth-child(-n+2) { border-bottom: 1px solid var(--mon-line); } - .bs-monitor-task .bs-schedule-row { grid-template-columns: repeat(2, minmax(0, 1fr)); } -} - -@media (max-width: 600px) { - .bs-monitor-task .bs-schedule-row { grid-template-columns: minmax(0, 1fr); } - .bs-monitor-modal .bs-sheet-filter > .input { flex-basis: 100%; } -} - -/* Compare templates: compact operations editor and safe nested-rule readout. */ -.bs-template-editor .modal__dialog { border-radius: 6px; border-color: #2b3748; background: #0c1521; } -.bs-template-editor .button, -.bs-template-editor .input, -.bs-template-editor textarea, -.bs-template-editor .mt-select, -.bs-template-editor .ui-field__select { border-radius: 4px; box-shadow: none; } -.bs-template-editor .modal__footer { flex-wrap: wrap; } -.bs-template-editor .ct-editor__norm, -.bs-template-editor .ct-editor__pick, -.bs-template-editor .ct-editor__nav, -.bs-template-editor .ct-editor__pane { border: 1px solid #2b3748; border-radius: 4px; background: #111c2a; min-width: 0; } -.bs-template-editor .ct-editor__norm { padding: 10px; } -.bs-template-editor .ct-editor__norm summary { cursor: pointer; color: #c4ddfa; } -.bs-template-editor .ct-editor__norm-area { width: 100%; max-width: 100%; resize: vertical; font: 12px/1.6 ui-monospace, Consolas, monospace; } -.bs-template-editor .ct-editor__main { grid-template-columns: 190px minmax(0, 1fr); min-width: 0; } -.bs-template-editor .ct-editor__nav { max-height: 560px; } -.bs-template-editor .ct-editor__nav-item { border-radius: 3px; font-size: 12px; } -.bs-template-editor .ct-editor__nav-item.is-active { background: #1b2c43; border-color: #526f94; } -.bs-template-editor .ct-editor__meta { font: 11px/1.6 ui-monospace, Consolas, monospace; overflow-wrap: anywhere; } -.bs-template-editor .mt-rule-tab.is-active { color: #9cc7f7; border-color: #526f94; background: #1b2c43; } -.bs-template-editor .ct-field-table-wrap { min-width: 0; overflow: auto; border: 1px solid #2b3748; border-radius: 4px; } -.bs-template-editor .ct-field-table { min-width: 800px; } -.bs-template-editor .ct-field-table td { font-size: 12px; } -.bs-template-editor .ct-field-table th { font-size: 11px; } -.bs-template-editor .ct-field-table .input { max-width: 100px; } -.bs-template-editor .ct-filter-row { min-width: 0; } -.bs-template-editor .ct-filter-row .mt-select { min-width: 0; max-width: 100%; } -.bs-template-editor .ct-complex-filters pre { max-height: 360px; overflow: auto; background: #0c1521; border: 1px solid #2b3748; border-radius: 4px; padding: 10px; font: 11px/1.6 ui-monospace, Consolas, monospace; } -.bs-template-editor button:focus-visible { outline: 2px solid #6ba9ec; outline-offset: 2px; } -.bs-template-workspace .pt-list-table { min-width: 720px; } -.bs-template-workspace .bs-cmp-metric-chips { max-height: 84px; overflow: auto; } -@media (max-width: 900px) { - .bs-template-editor .ct-editor__main { grid-template-columns: minmax(0, 1fr); } - .bs-template-editor .ct-editor__nav-list { flex-direction: row; overflow-x: auto; } - .bs-template-editor .ct-editor__nav-item { flex: 0 0 160px; } - .bs-template-editor .ct-editor__nav { max-height: none; } -} -@media (max-width: 600px) { - .bs-template-editor .ct-editor__pane { padding: 10px; } - .bs-template-editor .ct-editor__pane-head { flex-direction: column; } - .bs-template-editor .ct-editor__pane-head > div:first-child { width: 100%; } - .bs-template-editor .ct-editor__pane-head > .btn-row { align-self: flex-end; } - .bs-template-editor .ct-filter-row { grid-template-columns: minmax(0, 1fr) minmax(0, 1fr) auto; } - .bs-template-editor .ct-filter-row > .input { grid-column: 1 / 3; grid-row: 2; } - .bs-template-editor .ct-filter-row > button { grid-column: 3; grid-row: 1 / 3; } -} - /* Native :fullscreen + CSS immersive fallback (class only when FS API blocked) */ .bs-cmp-board.is-fullscreen, .bs-cmp-board:fullscreen { @@ -12760,22 +11773,6 @@ body:has(.bs-cmp-board:fullscreen) { opacity: 0.75; } -.bs-cmp-board__config { - flex: 0 1 auto; - min-width: 0; - padding: 4px 7px; - border: 1px solid var(--bs-cmp-border, rgba(148, 163, 184, 0.22)); - border-radius: 6px; - color: var(--nm-muted, #94a3b8); - font-size: 11px; - line-height: 1.4; - overflow-wrap: anywhere; -} - -.bs-cmp-board__config.is-legacy { - color: #fbbf24; -} - .bs-cmp-board__actions { margin-left: auto; flex: 0 0 auto; @@ -13132,15 +12129,6 @@ body:has(.bs-cmp-board:fullscreen) { box-shadow: 0 0 0 2px rgba(245, 158, 11, 0.22); } -.bs-cmp-nav__item.is-pending .bs-cmp-nav__dot { - background: #64748b; - box-shadow: 0 0 0 2px rgba(100, 116, 139, 0.25); -} - -.bs-cmp-nav__item.is-pending .bs-cmp-nav__name { - opacity: 0.75; -} - .bs-cmp-nav__name { font-size: 12px; font-weight: 600; @@ -13342,7 +12330,7 @@ body:has(.bs-cmp-board:fullscreen) { flex-wrap: wrap; align-items: center; gap: 4px; - flex: 1 1 100%; + flex: 1 1 280px; min-width: 0; padding: 2px; border-radius: 8px; @@ -13460,13 +12448,13 @@ body:has(.bs-cmp-board:fullscreen) { .bs-cmp-filter-bar { display: flex; - flex-wrap: wrap; + flex-wrap: nowrap; gap: 8px; align-items: center; - flex: 1 1 100%; - min-width: 0; - max-width: none; - margin-left: 0; + flex: 1 1 200px; + min-width: 160px; + max-width: 320px; + margin-left: auto; min-height: 32px; } @@ -13475,8 +12463,6 @@ body:has(.bs-cmp-board:fullscreen) { .bs-cmp-filter-bar input { min-width: 0; width: 100%; - flex: 1 1 220px; - max-width: 440px; } .bs-cmp-filter-bar .bs-sheet-count { @@ -13486,36 +12472,6 @@ body:has(.bs-cmp-board:fullscreen) { white-space: nowrap; } -.bs-cmp-key-filters { - display: flex; - flex-wrap: wrap; - gap: 8px 10px; - align-items: flex-end; - width: 100%; -} - -.bs-cmp-key-filter { - display: flex; - flex-direction: column; - gap: 2px; - min-width: 110px; - max-width: 160px; - flex: 0 1 140px; -} - -.bs-cmp-key-filter > span { - font-size: 11px; - line-height: 1.2; -} - -.bs-cmp-key-filter .input, -.bs-cmp-key-filter [data-slot="input"], -.bs-cmp-key-filter input { - min-width: 0; - width: 100%; - font-size: 12px; -} - .bs-cmp-kind-pills { display: flex; flex-wrap: wrap; @@ -13564,57 +12520,11 @@ body:has(.bs-cmp-board:fullscreen) { } .bs-cmp-result-table.is-loading { - opacity: 1; + opacity: 0.55; /* Keep header/filters clickable; only mute row hit-testing via opacity cue */ pointer-events: auto; } -.bs-cmp-result-state { - min-height: 180px; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - gap: 12px; - padding: 24px; - text-align: center; - white-space: normal; - overflow-wrap: anywhere; - color: var(--bs-cmp-muted); -} - -.bs-cmp-result-state strong { - color: var(--bs-cmp-ink); - font-size: 14px; -} - -.bs-cmp-result-state.is-error strong { - color: var(--danger, #fca5a5); -} - -.bs-cmp-strip__kind:focus-visible, -.bs-cmp-strip__toggle:focus-visible, -.bs-cmp-nav__item:focus-visible { - outline: 2px solid var(--accent, #60a5fa); - outline-offset: 2px; -} - -@media (max-width: 600px) { - .bs-cmp-strip__sheet { - flex-wrap: wrap; - } - - .bs-cmp-filter-bar input { - flex-basis: 100%; - max-width: none; - } - - .bs-cmp-key-filter { - flex: 1 1 120px; - max-width: none; - } -} - .bs-cmp-board.is-fullscreen .bs-cmp-result-table { max-height: none; min-height: 0; @@ -13645,7 +12555,6 @@ body:has(.bs-cmp-board:fullscreen) { @media (max-width: 960px) { .bs-cmp-board__body { grid-template-columns: 1fr; - grid-template-rows: auto minmax(0, 1fr); } .bs-cmp-board__body.is-nav-collapsed { @@ -13654,7 +12563,6 @@ body:has(.bs-cmp-board:fullscreen) { .bs-cmp-nav { max-height: 180px; - align-self: start; } .bs-cmp-nav.is-collapsed { @@ -13664,8 +12572,6 @@ body:has(.bs-cmp-board:fullscreen) { .bs-cmp-nav__list { flex-direction: row; flex-wrap: nowrap; - flex: 0 0 auto; - align-items: flex-start; overflow-x: auto; overflow-y: hidden; } @@ -13676,9 +12582,8 @@ body:has(.bs-cmp-board:fullscreen) { .bs-cmp-nav__item { flex: 0 0 auto; - width: auto; - min-width: 240px; - grid-template-columns: 8px minmax(0, 1fr) auto; + min-width: 180px; + grid-template-columns: 8px minmax(0, 1fr); } } @@ -13868,6 +12773,10 @@ body:has(.bs-cmp-board:fullscreen) { vertical-align: middle; } +.bs-cmp-val-cell--diff { + background: rgba(245, 158, 11, 0.08); +} + .bs-cmp-pair { display: flex; flex-direction: column; @@ -13875,10 +12784,6 @@ body:has(.bs-cmp-board:fullscreen) { min-width: 4.5rem; } -.bs-cmp-pair--same { - min-width: 3rem; -} - .bs-cmp-pair__row { display: grid; grid-template-columns: 1.6em minmax(0, 1fr); @@ -13903,20 +12808,6 @@ body:has(.bs-cmp-board:fullscreen) { color: #86efac; } -.bs-cmp-val-cell--pair.is-same { - background: transparent; -} - -.bs-cmp-val--same { - color: #cbd5e1; - opacity: 0.92; -} - -.bs-cmp-val-cell--diff { - background: rgba(245, 158, 11, 0.1); - box-shadow: inset 0 0 0 1px rgba(245, 158, 11, 0.22); -} - .bs-cmp-val { font-family: ui-monospace, SFMono-Regular, Consolas, monospace; color: #e2e8f0; @@ -13924,63 +12815,6 @@ body:has(.bs-cmp-board:fullscreen) { line-height: 1.35; } -.bs-cmp-progress.is-compact { - padding: 6px 10px; - gap: 4px; -} - -.bs-cmp-progress.is-compact .bs-cmp-progress__track { - height: 4px; -} - -.bs-cmp-progress.is-compact .bs-cmp-progress__meta { - font-size: 0.75rem; -} - -.bs-cmp-strip__pass-scope { - margin-left: 4px; - font-size: 9px; - font-weight: 600; - color: #64748b; - text-transform: none; -} - -.bs-cmp-strip__toggle { - flex: 0 0 auto; - height: 28px; - padding: 0 8px; - border-radius: 6px; - border: 1px solid rgba(148, 163, 184, 0.28); - background: rgba(15, 23, 42, 0.45); - color: #cbd5e1; - font-size: 11px; - cursor: pointer; - white-space: nowrap; -} - -.bs-cmp-strip__toggle:hover { - border-color: rgba(96, 165, 250, 0.45); - color: #f1f5f9; -} - -.bs-cmp-strip__toggle.is-active { - border-color: rgba(59, 130, 246, 0.5); - background: rgba(37, 99, 235, 0.18); - color: #93c5fd; -} - -.bs-cmp-strip__hint { - flex: 1 1 100%; - margin: 0; - font-size: 0.75rem; - line-height: 1.35; -} - -.bs-cmp-strip .bs-cmp-key-filters { - flex: 1 1 100%; - margin-top: 0; -} - .bs-cmp-val.is-empty { color: #64748b; font-style: italic; @@ -15790,106 +14624,3 @@ option { .pager__jump-input { width: 4.25rem; } - -/* Cutover wall: keep one board/poller alive in native or window fullscreen. */ -.app-heroui-modal.bm-wall-modal { - width: 100vw !important; - max-width: 100vw !important; - height: 100dvh; - max-height: 100dvh; - padding: 0; - margin: 0; - transform: none; - box-sizing: border-box; - border: 0; -} -.app-heroui-modal.bm-wall-modal .app-heroui-modal__dialog { - width: 100% !important; - max-width: 100% !important; - height: 100%; - max-height: 100%; - margin: 0; - padding: 0; - border: 0; - border-radius: 0; - transform: none; - overflow: hidden; - box-sizing: border-box; -} -.bm-wall-modal .modal__header, -.bm-wall-modal .modal__footer, -.bm-wall-modal .bm-detail__chrome, -.bm-wall-modal .nm-config-modal__tabs { display: none; } -.app-heroui-modal.bm-wall-modal .bm-detail { - width: 100%; - min-width: 0; - height: 100%; - max-height: 100%; - margin: 0; - padding: 0; - gap: 0; - overflow: hidden; -} -.bm-board.is-wall, -.bm-board:fullscreen { - width: 100%; - min-width: 0; - height: 100dvh; - min-height: 0; - max-height: 100dvh; - margin: 0; - padding: clamp(12px, 1.25vw, 28px); - gap: 12px; - border-radius: 0; - background: #0b1220; - color: #e2e8f0; - box-sizing: border-box; - overflow: auto; -} -.bm-board.is-wall > * { flex-shrink: 0; } -.bm-wall-heading { display: flex; justify-content: space-between; align-items: center; gap: 20px; min-width: 0; } -.bm-wall-heading > div { min-width: 0; } -.bm-wall-heading__label { color: #94a3b8; font-size: 13px; letter-spacing: .08em; } -.bm-wall-heading h2 { margin: 4px 0 0; font-size: clamp(22px, 2vw, 36px); line-height: 1.2; overflow-wrap: anywhere; } -.bm-wall-heading__status { display: flex; flex-direction: column; align-items: flex-end; gap: 8px; } -.bm-wall-heading time { color: #94a3b8; font: 14px ui-monospace, monospace; white-space: nowrap; } -.bm-wall-state { padding: 6px 12px; font-size: 15px; border: 1px solid currentColor; border-radius: 3px; } -.bm-wall-state.is-normal { color: #34d399; } -.bm-wall-state.is-anomaly { color: #f87171; } -.bm-wall-state.is-waiting, .bm-wall-state.is-unknown { color: #fbbf24; } -.bm-board.is-wall .bm-live-header { position: sticky; top: 0; z-index: 5; background: #0b1220; padding: 8px 0; } -.bm-board.is-wall .bm-live-header strong { font-size: 17px; } -.bm-board.is-wall .bm-live-sources strong { font-size: 19px; } -.bm-board.is-wall .bm-live-kpis > div { padding: 16px 20px; } -.bm-board.is-wall .bm-live-kpis span { font-size: 16px; } -.bm-board.is-wall .bm-live-kpis strong { font-size: clamp(40px, 3.5vw, 68px); } -.bm-board.is-wall .bm-live-kpis strong small { font-size: 20px; } -.bm-board.is-wall .bm-live-warning { font-size: 14px; } -.bm-wall-progress { display: flex; align-items: center; gap: 20px; padding: 4px 0; color: #34d399; } -.bm-wall-progress > div:first-child { display: flex; align-items: baseline; gap: 16px; } -.bm-wall-progress span { font-size: 13px; } -.bm-wall-progress strong { font: 600 24px ui-monospace, monospace; } -.bm-wall-progress.is-unknown { color: #fbbf24; } -.bm-wall-progress__track { flex: 1; height: 8px; min-width: 60px; background: #243044; border-radius: 2px; overflow: hidden; } -.bm-wall-progress__track > span { display: block; height: 100%; background: currentColor; } -.bm-board.is-wall .bm-board-controls > summary { display: list-item; cursor: pointer; color: #94a3b8; font-size: 13px; padding: 6px 0; } -.bm-board.is-wall .bm-sampling { max-height: 240px; overflow: auto; } -.bm-board.is-wall .bm-sampling td, .bm-board.is-wall .bm-sampling th { font-size: 14px; } -.bm-board.is-wall .bs-cmp-board__body { flex: 1 0 260px; min-height: 260px; } -.bm-board.is-wall .bs-cmp-main { display: flex; min-height: 0; } -.bm-board.is-wall .bm-board__table { flex: 1; width: 100%; min-height: 0; max-height: none; } -.bm-board.is-wall .bm-board__table td { font-size: 15px; } -.bm-board.is-wall .bs-cmp-nav__list { max-height: none; } -@media (max-width: 720px) { - .bm-wall-heading { align-items: flex-start; gap: 12px; } - .bm-wall-heading__status { max-width: 48%; } - .bm-wall-heading time { font-size: 10px; white-space: normal; } - .bm-wall-state { font-size: 12px; padding: 4px 6px; } - .bm-board.is-wall .bm-live-kpis > div { padding: 12px; } - .bm-board.is-wall .bm-live-kpis strong { font-size: 36px; } - .bm-board.is-wall .bm-live-kpis span { font-size: 13px; } - .bm-wall-progress { flex-wrap: wrap; gap: 8px; } - .bm-wall-progress__track { flex-basis: 100%; } - .bm-board.is-wall .bs-cmp-board__body { flex-basis: auto; } - .bm-board.is-wall .bm-board__table { max-height: 50dvh; } -} diff --git a/web/src/pages/network/BizComparePage.tsx b/web/src/pages/network/BizComparePage.tsx index 8c215e1..8ffddee 100644 --- a/web/src/pages/network/BizComparePage.tsx +++ b/web/src/pages/network/BizComparePage.tsx @@ -1,8 +1,5 @@ import { Button, Input, Modal } from "@heroui/react"; import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; -import type { KeyboardEvent as ReactKeyboardEvent } from "react"; -import { parseTemplateImport, parseIfaceNormText, toOrGroups, fromOrGroups, validateTemplateDraft } from "./compareTemplateModel"; -import type { MetricSheet, FieldRule, RowFilter } from "./compareTemplateModel"; import { ListPager } from "../../components/ListPager"; import { AppModalShell } from "../../components/ui/AppModalShell"; import { FieldSelect } from "../../components/ui/FieldSelect"; @@ -10,7 +7,6 @@ import { useDebouncedValue } from "../../hooks/useDebouncedValue"; import { useToast } from "../../hooks/useToast"; import { useI18n } from "../../i18n"; import { - bizCompareCancelRun, bizCompareCreateJob, bizCompareCreateMapping, bizCompareCreateTemplate, @@ -39,9 +35,8 @@ import { cutoverCachedGet, cutoverCachedGetSWR, invalidateCutoverCache } from ". import { jobChipColor, NmStatusChip } from "./nmChips"; type PageTab = "templates" | "jobs"; -type JobDetailTab = "config" | "runs" | "result"; +type JobDetailTab = "config" | "result"; type KindFilter = "diff" | "all" | "added" | "removed" | "changed" | "unchanged"; -type DiffListSource = "stored" | "live" | ""; type CreateJobStep = 0 | 1 | 2 | 3; const CREATE_JOB_STEPS = 4; @@ -63,6 +58,34 @@ type MetricField = { }; type MetricSchema = { metric_id: string; fields: MetricField[] }; +type MetricSheet = { + sheet_id?: string; + title?: string; + metric_id: string; + key_fields: string[]; + iface_fields: string[]; + compare_fields: string[]; + display_fields?: string[]; + row_filters?: RowFilter[]; + field_rules?: FieldRule[]; +}; + +type FieldRule = { + field: string; + compare?: string; + normalize?: string; + ignore?: boolean; + tolerance?: number; +}; + +type RowFilter = { + field?: string; + op?: string; + value?: string | string[]; + any?: RowFilter[]; + all?: RowFilter[]; +}; + type Template = { id: string; name: string; @@ -74,12 +97,9 @@ type Template = { compare_fields: string[]; iface_normalize_rules?: { from: string; to: string }[]; note: string; - job_count?: number; }; type Mapping = { id: string; name: string; rows: { before_if: string; after_if: string }[] }; -type StoreUnchanged = "auto" | "always" | "never" | "sample" | "keys"; - type Job = { id: string; name: string; @@ -92,7 +112,6 @@ type Job = { mode: string; status: string; enabled_sheet_ids?: string[]; - store_unchanged?: StoreUnchanged | string; note?: string; }; @@ -115,27 +134,10 @@ type RunSheet = { display_fields?: string[]; field_rules?: FieldRule[]; mode?: string; - /** pending|running|queued|done|cancelled — set while overall run is active */ - status?: string; summary?: Record; diffs?: DiffRow[]; }; -type RunRecord = { - id: string; - status?: string; - created_at?: string; - before?: SideInfo; - after?: SideInfo; - summary?: { - added?: number; removed?: number; changed?: number; unchanged?: number; - before_count?: number; after_count?: number; duration_ms?: number; - progress?: RunProgressInfo; - }; - progress?: RunProgressInfo; - message?: string; -}; - function fmtTime(v?: string | null) { if (!v) return "—"; return formatSystemTime(v) || v; @@ -187,8 +189,6 @@ function PairCell(props: { const post = afterText || "—"; const isAdded = kind === "added"; const isRemoved = kind === "removed"; - const same = - !mismatch && !isAdded && !isRemoved && String(beforeText || "") === String(afterText || ""); // Whole-row missing/extra: emphasize the present side; do not strike it out. const preClass = [ "bs-cmp-val", @@ -210,31 +210,20 @@ function PairCell(props: { - {same ? ( -
- {pre || "—"} +
+
+ {beforeLabel} + {isAdded ? "—" : pre}
- ) : ( -
-
- {beforeLabel} - {isAdded ? "—" : pre} -
-
- {afterLabel} - {isRemoved ? "—" : post} -
+
+ {afterLabel} + {isRemoved ? "—" : post}
- )} +
{reason ?
{reason}
: null} ); @@ -340,109 +329,6 @@ function sheetLabel(s: { title?: string; sheet_id?: string; metric_id?: string } return String(s?.title || s?.sheet_id || s?.metric_id || "").trim() || "—"; } -type RunProgressInfo = { - phase?: string; - sheet_index?: number; - sheet_total?: number; - sheet_title?: string; - sheet_id?: string; - elapsed_ms?: number; - load_side?: string; - rows_loaded?: number; - engine?: string; - engine_note?: string; - persisted?: number; - persist_total?: number; -}; - -function localizeRunPhase(phase: string | undefined, t: (key: string) => string): string { - const p = String(phase || "").trim().toLowerCase(); - if (!p || p === "…") return "…"; - if (p.startsWith("persisting")) { - if (p.includes("fail")) return t("bizCompare.phasePersistingFail"); - if (p.includes("ok")) return t("bizCompare.phasePersistingOk"); - return t("bizCompare.phasePersisting"); - } - const map: Record = { - queued: "bizCompare.phaseQueued", - loading: "bizCompare.phaseLoading", - sql_count: "bizCompare.phaseSqlCount", - sql_project: "bizCompare.phaseSqlProject", - sql_join: "bizCompare.phaseSqlJoin", - sql_fail_fetch: "bizCompare.phaseSqlFailFetch", - comparing: "bizCompare.phaseComparing", - done: "bizCompare.phaseDone", - failed: "bizCompare.phaseFailed", - cancelled: "bizCompare.phaseCancelled", - }; - const key = map[p]; - return key ? t(key) : String(phase); -} - -function runProgressPercent(prog: RunProgressInfo): number { - const total = Number(prog.sheet_total || 0); - const idx = Number(prog.sheet_index || 0); - if (total <= 0) return 8; - const phase = String(prog.phase || "").toLowerCase(); - if (phase === "queued") return 4; - if (phase === "done") return 100; - let base = Math.max(0, Math.min(idx, total)) / total; - const persistTotal = Number(prog.persist_total || 0); - const persisted = Number(prog.persisted || 0); - if (persistTotal > 0 && phase.startsWith("persisting")) { - const within = Math.min(1, persisted / persistTotal) * (1 / total); - base = Math.max(0, (idx - 1) / total) + within; - } else if (phase === "loading" || phase === "sql_count" || phase === "comparing") { - base = Math.max(0, (idx - 1) / total) + 0.35 / total; - } - return Math.max(4, Math.min(99, Math.round(base * 100))); -} - -function runProgressMetaLine( - prog: RunProgressInfo, - t: (key: string, vars?: Record) => string, -): string { - const parts: string[] = []; - const phase = localizeRunPhase(prog.phase, t); - if (phase && phase !== "…") parts.push(phase); - const total = Number(prog.sheet_total || 0); - if (total > 0) { - parts.push( - t("bizCompare.runSheetProgress", { - i: String(prog.sheet_index || 0), - n: String(total), - }), - ); - } - const title = String(prog.sheet_title || prog.sheet_id || "").trim(); - if (title) parts.push(title); - const rows = Number(prog.rows_loaded || 0); - if (rows > 0) { - const eng = String(prog.engine || "").toLowerCase(); - parts.push( - eng === "sql" - ? t("bizCompare.runRowsSqlCount", { - side: String(prog.load_side || "—"), - n: String(rows), - }) - : t("bizCompare.runRowsLoaded", { - side: String(prog.load_side || "—"), - n: String(rows), - }), - ); - } - const persistTotal = Number(prog.persist_total || 0); - if (String(prog.phase || "").startsWith("persisting") && persistTotal > 0) { - parts.push( - t("bizCompare.runPersisting", { - done: String(prog.persisted || 0), - total: String(persistTotal), - }), - ); - } - return parts.join(" · "); -} - function templateSheets(tpl?: Template | null): MetricSheet[] { if (!tpl) return []; if (tpl.metrics?.length) return tpl.metrics; @@ -524,7 +410,6 @@ function cloneSheet(s: MetricSheet): MetricSheet { display_fields: display, row_filters: JSON.parse(JSON.stringify(s.row_filters || [])), field_rules: JSON.parse(JSON.stringify(s.field_rules || [])), - ignore_port_changes: s.ignore_port_changes, }; } @@ -545,7 +430,6 @@ function templateExportPayload(tpl: Template) { : [...(s.key_fields || []), ...(s.compare_fields || [])], row_filters: JSON.parse(JSON.stringify(s.row_filters || [])), field_rules: JSON.parse(JSON.stringify(s.field_rules || [])), - ignore_port_changes: s.ignore_port_changes, })); return { format: TEMPLATE_EXPORT_FORMAT, @@ -558,10 +442,105 @@ function templateExportPayload(tpl: Template) { }; } +function parseTemplateImport(raw: unknown): { + name: string; + note: string; + metrics: MetricSheet[]; + iface_normalize_rules?: { from: string; to: string }[]; +} | null { + if (!raw || typeof raw !== "object") return null; + const o = raw as Record; + // Accept our export shape or a bare {name, metrics} / TemplateIn body + const name = String(o.name || "").trim(); + const note = String(o.note || ""); + let metricsRaw = o.metrics; + if (!Array.isArray(metricsRaw) && o.metric_id) { + metricsRaw = [ + { + metric_id: o.metric_id, + key_fields: o.key_fields || [], + iface_fields: o.iface_fields || [], + compare_fields: o.compare_fields || [], + display_fields: o.display_fields || [], + row_filters: o.row_filters || [], + field_rules: o.field_rules || [], + }, + ]; + } + if (!name || !Array.isArray(metricsRaw) || !metricsRaw.length) return null; + const metrics: MetricSheet[] = []; + for (const item of metricsRaw) { + if (!item || typeof item !== "object") continue; + const m = item as Record; + const mid = String(m.metric_id || "").trim(); + const keys = Array.isArray(m.key_fields) + ? (m.key_fields as unknown[]).map((x) => String(x).trim()).filter(Boolean) + : []; + if (!mid || !keys.length) continue; + metrics.push({ + sheet_id: String(m.sheet_id || mid).trim() || mid, + title: String(m.title || m.sheet_id || mid).trim() || mid, + metric_id: mid, + key_fields: keys, + iface_fields: Array.isArray(m.iface_fields) + ? (m.iface_fields as unknown[]).map((x) => String(x).trim()).filter(Boolean) + : [], + compare_fields: Array.isArray(m.compare_fields) + ? (m.compare_fields as unknown[]).map((x) => String(x).trim()).filter(Boolean) + : [], + display_fields: Array.isArray(m.display_fields) + ? (m.display_fields as unknown[]).map((x) => String(x).trim()).filter(Boolean) + : undefined, + row_filters: Array.isArray(m.row_filters) ? (m.row_filters as RowFilter[]) : [], + field_rules: Array.isArray(m.field_rules) ? (m.field_rules as FieldRule[]) : [], + }); + } + if (!metrics.length) return null; + const normRaw = o.iface_normalize_rules; + const iface_normalize_rules: { from: string; to: string }[] = []; + if (Array.isArray(normRaw)) { + for (const item of normRaw) { + if (!item || typeof item !== "object") continue; + const r = item as Record; + const fr = String(r.from || "").trim(); + const to = String(r.to || "").trim(); + if (fr && to) iface_normalize_rules.push({ from: fr, to }); + } + } + return { name, note, metrics, iface_normalize_rules }; +} + function ifaceNormRulesToText(rules?: { from: string; to: string }[]): string { return (rules || []).map((r) => `${r.from},${r.to}`).join("\n"); } +function parseIfaceNormText(text: string): { from: string; to: string }[] { + const out: { from: string; to: string }[] = []; + for (const line of text.split(/\r?\n/)) { + const s = line.trim(); + if (!s || s.startsWith("#")) continue; + let fr = ""; + let to = ""; + if (s.includes(",")) { + const i = s.indexOf(","); + fr = s.slice(0, i).trim(); + to = s.slice(i + 1).trim(); + } else if (s.includes("\t")) { + const i = s.indexOf("\t"); + fr = s.slice(0, i).trim(); + to = s.slice(i + 1).trim(); + } else { + const parts = s.split(/\s+/); + if (parts.length >= 2) { + fr = parts[0]; + to = parts.slice(1).join(" "); + } + } + if (fr && to) out.push({ from: fr, to }); + } + return out; +} + function downloadJsonFile(filename: string, data: unknown) { const blob = new Blob([JSON.stringify(data, null, 2)], { type: "application/json" }); const url = URL.createObjectURL(blob); @@ -595,6 +574,51 @@ function upsertFieldRule(sheet: MetricSheet, field: string, patch: Partial { + if (g?.all && Array.isArray(g.all)) return g.all.filter(isLeafFilter); + if (isLeafFilter(g)) return [g]; + return [] as RowFilter[]; + }) + .filter((g) => g.length); + } + if (list.every(isLeafFilter)) return [list.map((f) => ({ ...f }))]; + // Mixed / opaque: keep editable leaves only + const leaves = list.filter(isLeafFilter); + return leaves.length ? [leaves] : []; +} + +/** Serialize OR-groups back to engine row_filters (AND of leaves, or single any-of). */ +function fromOrGroups(groups: RowFilter[][]): RowFilter[] { + const clean = groups + .map((g) => + g + .map((f) => ({ + field: f.field || "", + op: f.op || "eq", + value: f.value ?? "", + })) + .filter((f) => f.field), + ) + .filter((g) => g.length); + if (!clean.length) return []; + if (clean.length === 1) return clean[0]; + return [ + { + any: clean.map((g) => (g.length === 1 ? g[0] : { all: g })), + }, + ]; +} + function emptyLeaf(field = ""): RowFilter { return { field, op: "eq", value: "" }; } @@ -664,7 +688,6 @@ function TplRowFiltersEditor({
{ const next = groups.map((g) => g.map((x) => ({ ...x }))); const op = next[gi][fi]?.op || "eq"; let value: string | string[] = e.target.value; - if (["in", "not_in", "nin"].includes(op)) { + if (op === "in" || op === "not_in") { value = e.target.value .split(",") .map((x) => x.trim()) @@ -755,6 +770,10 @@ function TplRowFiltersEditor({ ); } +function metricLabel(id: string) { + return id; +} + export type BizComparePageMode = "jobs" | "templates" | "all"; export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePageMode }) { @@ -776,13 +795,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const [mappings, setMappings] = useState([]); const [jobs, setJobs] = useState([]); const [listKw, setListKw] = useState(""); - const [listPage, setListPage] = useState(1); - const [listPageSize, setListPageSize] = useState(50); const debouncedListKw = useDebouncedValue(listKw, 250); // template editor const [tplOpen, setTplOpen] = useState(false); - const [tplError, setTplError] = useState(""); const [tplEditId, setTplEditId] = useState(""); const [tplName, setTplName] = useState(""); const [tplNote, setTplNote] = useState(""); @@ -808,13 +824,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const [beforeBatchId, setBeforeBatchId] = useState(""); const [afterBatchId, setAfterBatchId] = useState(""); const [mode, setMode] = useState<"manual" | "auto">("manual"); - const [storeUnchanged, setStoreUnchanged] = useState("auto"); const [mapName, setMapName] = useState("端口映射"); const [mapText, setMapText] = useState(""); const [validateOut, setValidateOut] = useState(null); - const [runs, setRuns] = useState([]); - const [runsLoading, setRunsLoading] = useState(false); - const [runsError, setRunsError] = useState(""); + const [runs, setRuns] = useState([]); const [runDetail, setRunDetail] = useState(null); const [resultSheetId, setResultSheetId] = useState(""); @@ -822,41 +835,11 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const [kindFilter, setKindFilter] = useState("diff"); const [resultKw, setResultKw] = useState(""); const debouncedResultKw = useDebouncedValue(resultKw, 300); - const [resultKeyFilters, setResultKeyFilters] = useState>({}); - const [keyFiltersOpen, setKeyFiltersOpen] = useState(false); - const [hideDisplayCols, setHideDisplayCols] = useState(true); - const debouncedKeyFiltersJson = useDebouncedValue(JSON.stringify(resultKeyFilters), 300); - const debouncedKeyFilters = useMemo(() => { - try { - const o = JSON.parse(debouncedKeyFiltersJson || "{}") as Record; - const out: Record = {}; - for (const [k, v] of Object.entries(o || {})) { - if (String(k || "").trim() && String(v || "").trim()) out[String(k)] = String(v).trim(); - } - return out; - } catch { - return {} as Record; - } - }, [debouncedKeyFiltersJson]); - const hasResultSearch = - Boolean(debouncedResultKw.trim()) || Object.keys(debouncedKeyFilters).length > 0; - const activeKeyFilterCount = Object.keys(resultKeyFilters).filter((k) => - String(resultKeyFilters[k] || "").trim(), - ).length; - const keyFiltersVisible = keyFiltersOpen || activeKeyFilterCount > 0; const [resultPage, setResultPage] = useState(1); const [resultPageSize, setResultPageSize] = useState(100); const [resultTotal, setResultTotal] = useState(0); const [pagedDiffs, setPagedDiffs] = useState([]); const [diffsLoading, setDiffsLoading] = useState(false); - const [diffsSource, setDiffsSource] = useState(""); - const [diffsTruncated, setDiffsTruncated] = useState(false); - const [diffsError, setDiffsError] = useState(""); - const [diffsRetry, setDiffsRetry] = useState(0); - const jobRequestRef = useRef(0); - const runRequestRef = useRef(0); - const listsRequestRef = useRef(0); - const diffFiltersRef = useRef(""); const boardRef = useRef(null); const tableScrollRef = useRef(null); const tableScrollPosRef = useRef({ top: 0, left: 0 }); @@ -865,7 +848,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const tplImportRef = useRef(null); const refresh = useCallback(async (opts?: { force?: boolean }) => { - const request = ++listsRequestRef.current; type Bundle = { taskRes: Awaited>; tpl: Awaited>; @@ -875,16 +857,15 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; const fetchBundle = async (): Promise => { const [taskRes, tpl, maps, j, met] = await Promise.all([ - pageMode === "templates" ? Promise.resolve({items: []}) : bizStateListTasks(), + bizStateListTasks(), bizCompareListTemplates(), - pageMode === "templates" ? Promise.resolve({items: []}) : bizCompareListMappings(), - pageMode === "templates" ? Promise.resolve({items: []}) : bizCompareListJobs(), - cutoverCachedGet("bizCompare:metrics", () => bizCompareListMetrics(), {ttlMs: 60000}), + bizCompareListMappings(), + bizCompareListJobs(), + bizCompareListMetrics(), ]); return { taskRes, tpl, maps, j, met }; }; const apply = (b: Bundle) => { - if (request !== listsRequestRef.current) return; setTasks((b.taskRes.items || []) as TaskOpt[]); setTemplates((b.tpl.items || []) as Template[]); setMappings((b.maps.items || []) as Mapping[]); @@ -892,11 +873,11 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage setMetrics((b.met.items || []) as MetricSchema[]); }; if (opts?.force) { - apply(await cutoverCachedGet(`bizCompare:lists:${pageMode}`, fetchBundle, { force: true })); + apply(await cutoverCachedGet("bizCompare:lists", fetchBundle, { force: true })); return; } - apply(await cutoverCachedGetSWR(`bizCompare:lists:${pageMode}`, fetchBundle, apply)); - }, [pageMode]); + apply(await cutoverCachedGetSWR("bizCompare:lists", fetchBundle, apply)); + }, []); useEffect(() => { void (async () => { @@ -909,12 +890,8 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage // eslint-disable-next-line react-hooks/exhaustive-deps }, [refresh]); - useEffect(() => () => { listsRequestRef.current += 1; }, []); - // Keep local tab + dismiss overlays when route mode flips (component may be reused). useEffect(() => { - jobRequestRef.current += 1; - runRequestRef.current += 1; if (pageMode === "templates") setPageTab("templates"); else if (pageMode === "jobs") setPageTab("jobs"); setTplOpen(false); @@ -922,64 +899,45 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage setCreateStep(0); setJobId(""); setRuns([]); - setRunsLoading(false); - setRunsError(""); setRunDetail(null); setResultSheetId(""); if (document.fullscreenElement === boardRef.current) { void document.exitFullscreen().catch(() => undefined); } setBoardFs(false); - return () => { - jobRequestRef.current += 1; - runRequestRef.current += 1; - }; }, [pageMode]); useEffect(() => { - let cancelled = false; - setBeforeBatches([]); void (async () => { - if (!beforeTaskId) return; - try { - const b = await cutoverCachedGet(`bizCompare:batches:${beforeTaskId}`, - () => bizStateListBatches(beforeTaskId), { ttlMs: 5000 }); - if (!cancelled) setBeforeBatches((b.items || []) as BatchOpt[]); - } catch (e) { - if (!cancelled) showError(formatErr(e)); + if (!beforeTaskId) { + setBeforeBatches([]); + return; } + const b = await bizStateListBatches(beforeTaskId); + setBeforeBatches((b.items || []) as BatchOpt[]); })(); - return () => { cancelled = true; }; - }, [beforeTaskId, showError]); + }, [beforeTaskId]); - const effectiveAfterTaskId = afterTaskId || beforeTaskId; useEffect(() => { - let cancelled = false; - setAfterBatches([]); void (async () => { - if (!effectiveAfterTaskId) return; - try { - const b = await cutoverCachedGet(`bizCompare:batches:${effectiveAfterTaskId}`, - () => bizStateListBatches(effectiveAfterTaskId), { ttlMs: 5000 }); - if (!cancelled) setAfterBatches((b.items || []) as BatchOpt[]); - } catch (e) { - if (!cancelled) showError(formatErr(e)); + if (!afterTaskId) { + setAfterBatches([]); + return; } + const b = await bizStateListBatches(afterTaskId); + setAfterBatches((b.items || []) as BatchOpt[]); })(); - return () => { cancelled = true; }; - }, [effectiveAfterTaskId, showError]); - - const templatesById = useMemo(() => new Map(templates.map((x) => [x.id, x])), [templates]); + }, [afterTaskId]); const filteredJobs = useMemo(() => { const kw = debouncedListKw.trim().toLowerCase(); if (!kw) return jobs; return jobs.filter((j) => { - const tpl = templatesById.get(j.template_id); + const tpl = templates.find((x) => x.id === j.template_id); const mids = (tpl?.metric_ids || templateSheets(tpl).map((s) => s.metric_id)).join(" "); return `${j.name} ${j.mode} ${j.status} ${tpl?.name || ""} ${mids}`.toLowerCase().includes(kw); }); - }, [jobs, templatesById, debouncedListKw]); + }, [jobs, templates, debouncedListKw]); const filteredTemplates = useMemo(() => { const kw = debouncedListKw.trim().toLowerCase(); @@ -989,13 +947,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage return `${x.name} ${mids} ${x.note}`.toLowerCase().includes(kw); }); }, [templates, debouncedListKw]); - const listTotal = activeTab === "templates" ? filteredTemplates.length : filteredJobs.length; - const listPages = Math.max(1, Math.ceil(listTotal / listPageSize)); - const visibleListPage = Math.min(listPage, listPages); const selectedJobTemplate = useMemo( - () => templatesById.get(templateId) || null, - [templatesById, templateId], + () => templates.find((x) => x.id === templateId) || null, + [templates, templateId], ); const jobTemplateSheets = useMemo( @@ -1032,46 +987,32 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const next = currentlyOn.includes(sid) ? currentlyOn.filter((id) => id !== sid) : [...currentlyOn, sid]; - if (!next.length) { - showError(t("bizCompare.needSheets")); - return; - } // Empty list means "all on" (new template sheets auto-included) setEnabledSheetIds(next.length === all.length ? [] : next); }, - [jobSheetAllIds, enabledSheetIds, showError, t], + [jobSheetAllIds, enabledSheetIds], ); const setJobTemplateAndSheets = useCallback( (nextTplId: string, presetIds?: string[] | null) => { setTemplateId(nextTplId); + const tpl = templates.find((x) => x.id === nextTplId); + const all = tpl ? templateSheets(tpl).map((s) => sheetIdentity(s)).filter(Boolean) : []; if (presetIds && presetIds.length) { - // Keep stale IDs visible as a configuration error; [] would enable every sheet. - setEnabledSheetIds([...presetIds]); + const kept = presetIds.filter((id) => all.includes(id)); + setEnabledSheetIds(kept.length === all.length ? [] : kept); } else { setEnabledSheetIds([]); } }, - [], + [templates], ); const activeTplSheet = tplSheets[tplSheetIdx] || null; const activeTplFields = useMemo(() => { if (!activeTplSheet) return []; - const fields = [...(metrics.find((m) => m.metric_id === activeTplSheet.metric_id)?.fields || [])]; - const seen = new Set(fields.map((f) => f.name)); - for (const name of [...activeTplSheet.key_fields, ...activeTplSheet.iface_fields, - ...activeTplSheet.compare_fields, ...(activeTplSheet.display_fields || []), - ...(activeTplSheet.field_rules || []).map((r) => r.field)]) { - if (!seen.has(name)) { - fields.push({name, display_name: name, dtype: "string", is_key: false, is_interface: false, role: "meta"}); - seen.add(name); - } - } - return fields; + return metrics.find((m) => m.metric_id === activeTplSheet.metric_id)?.fields || []; }, [metrics, activeTplSheet]); - const activeTplFilterGroups = activeTplSheet ? toOrGroups(activeTplSheet.row_filters || []) : []; - const missingJobSheets = enabledSheetIds.filter((id) => !jobSheetAllIds.includes(id)); const runSheets: RunSheet[] = useMemo(() => { const sheets = (runDetail?.sheets || []) as RunSheet[]; @@ -1106,66 +1047,35 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage () => runSheets.find((s) => sheetIdentity(s) === resultSheetId) || runSheets[0] || null, [runSheets, resultSheetId], ); - const activeRunSheetId = sheetIdentity(activeRunSheet); - const activeRunSheetStatus = String(activeRunSheet?.status || ""); - // Clear field filters when switching sheet + // Reset page when sheet / filter / page size changes useEffect(() => { - setResultKeyFilters({}); - }, [resultSheetId]); + setResultPage(1); + }, [resultSheetId, kindFilter, debouncedResultKw, resultPageSize, runDetail?.id]); useEffect(() => { const runId = String(runDetail?.id || ""); - const mid = resultSheetId || activeRunSheetId || ""; - const sheetSt = activeRunSheetStatus; - // Block only while *this* sheet is still in flight — done sheets are readable mid-run - const sheetStillRunning = ["pending", "running", "queued"].includes(sheetSt); - if (!runId || !mid || jobDetailTab !== "result" || sheetStillRunning) { + const mid = resultSheetId || sheetIdentity(activeRunSheet) || ""; + if (!runId || !mid || jobDetailTab !== "result") { setPagedDiffs([]); setResultTotal(0); - setDiffsSource(""); - setDiffsTruncated(false); - setDiffsLoading(false); - setDiffsError(""); return; } - const filterKey = JSON.stringify([runId, mid, kindFilter, debouncedResultKw, - debouncedKeyFiltersJson, resultPageSize]); - if (diffFiltersRef.current !== filterKey) { - diffFiltersRef.current = filterKey; - if (resultPage !== 1) { - setPagedDiffs([]); - setResultTotal(0); - setDiffsLoading(true); - setResultPage(1); - return; - } - } let cancelled = false; void (async () => { setDiffsLoading(true); - setDiffsError(""); - setPagedDiffs([]); - setResultTotal(0); - setDiffsSource(""); - setDiffsTruncated(false); try { const res = await bizCompareListRunDiffs({ runId, metricId: mid, kind: kindFilter, kw: debouncedResultKw.trim(), - qf: debouncedKeyFilters, page: resultPage, pageSize: resultPageSize, }); if (cancelled) return; setPagedDiffs((res.items || []) as DiffRow[]); setResultTotal(Number(res.total || 0)); - setDiffsSource( - String((res as any).source || "").toLowerCase() === "live" ? "live" : "stored", - ); - setDiffsTruncated(Boolean((res as any).truncated)); const pages = Math.max( 1, Math.ceil(Number(res.total || 0) / Number(res.page_size || resultPageSize)), @@ -1173,7 +1083,8 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage if (resultPage > pages) setResultPage(pages); } catch (e) { if (!cancelled) { - setDiffsError(formatErr(e)); + // Keep previous rows to avoid strip/table jump; only clear on hard empty run + showError(formatErr(e)); } } finally { if (!cancelled) setDiffsLoading(false); @@ -1184,18 +1095,14 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; }, [ runDetail?.id, - runDetail?.status, resultSheetId, - activeRunSheetId, - activeRunSheetStatus, + activeRunSheet?.metric_id, kindFilter, debouncedResultKw, - debouncedKeyFiltersJson, - debouncedKeyFilters, resultPage, resultPageSize, jobDetailTab, - diffsRetry, + showError, ]); const resultColumns = useMemo(() => { @@ -1279,8 +1186,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage Number(c.success_count ?? c.unchanged ?? 0); const sheetPassRateOf = (c: { - pass_rate?: number | null; - status?: string; + pass_rate?: number; fail_count?: number; diff_count?: number; removed?: number; @@ -1288,30 +1194,12 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage success_count?: number; unchanged?: number; added?: number; - }): number | null => { - const st = String(c.status || ""); - if (st === "pending" || st === "running" || st === "queued") return null; - if (c.pass_rate === null) return null; + }) => { // Always recompute from fail/success so added never skews pass rate const fail = sheetFailOf(c); const ok = sheetSuccessOf(c); const judged = fail + ok; - if (!judged) return 100; - const rate = (ok / judged) * 100; - if (fail <= 0) return Math.round(rate * 10) / 10; - // 3 fails / 50k ≈ 99.994% — 1-decimal round becomes 100%; keep digits until < 100 - for (const nd of [1, 2, 3, 4]) { - const r = Math.round(rate * 10 ** nd) / 10 ** nd; - if (r < 100) return r; - } - return 99.9999; - }; - - const formatPassRate = (rate: number | null | undefined) => { - if (rate === null || rate === undefined || Number.isNaN(rate)) return "…"; - // Trim trailing zeros but keep enough precision for near-100 rates - const s = Number(rate).toFixed(4).replace(/\.?0+$/, ""); - return `${s}%`; + return judged ? Math.round((ok / judged) * 1000) / 10 : 100; }; const summary = runDetail?.summary || {}; @@ -1322,7 +1210,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage title?: string; metric_id: string; mode?: string; - status?: string; added?: number; removed?: number; changed?: number; @@ -1332,13 +1219,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage fail_count?: number; success_count?: number; diff_count?: number; - pass_rate?: number | null; + pass_rate?: number; }>; - // Failures first; keep pending/running at the end so completed fails stay visible + // Failures first so ops can scan quickly when many sheets return [...raw].sort((a, b) => { - const pa = ["pending", "running", "queued"].includes(String(a.status || "")) ? 1 : 0; - const pb = ["pending", "running", "queued"].includes(String(b.status || "")) ? 1 : 0; - if (pa !== pb) return pa - pb; const da = sheetFailOf(a); const db = sheetFailOf(b); if (da !== db) return db - da; @@ -1349,39 +1233,14 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const activeFail = sheetFailOf(activeSheetCard || {}); const activeSuccess = sheetSuccessOf(activeSheetCard || {}); const activePassRate = sheetPassRateOf(activeSheetCard || {}); - const activeSheetPending = ["pending", "running", "queued"].includes( - String(activeSheetCard?.status || ""), - ); const activeAdded = Number(activeSheetCard?.added || 0); - const activeRemoved = Number(activeSheetCard?.removed || 0); - const activeChanged = Number(activeSheetCard?.changed || 0); const showFailCol = - kindFilter === "diff" || - kindFilter === "all" || - kindFilter === "added" || - kindFilter === "removed" || - kindFilter === "changed"; - const isLiveSearch = diffsSource === "live" && hasResultSearch; - const resultSearchKeyFields = useMemo(() => { - const keys = (activeRunSheet?.key_fields || []).map((f) => String(f || "").trim()).filter(Boolean); - // Prefer BGP-ish fields first for compact UI - const prefer = ["direction", "neighbor", "network", "rd", "afi", "vrf", "local_as"]; - const ranked = [ - ...prefer.filter((p) => keys.includes(p)), - ...keys.filter((k) => !prefer.includes(k)), - ]; - return ranked.slice(0, 8); - }, [activeRunSheet?.key_fields]); + kindFilter === "diff" || kindFilter === "all" || kindFilter === "added"; const resultEmptyColSpan = 1 + (showFailCol ? 1 : 0) + resultColumns.keys.length + - Math.max( - resultColumns.extras.filter( - (f) => resultColumns.compareSet.has(f) || !hideDisplayCols, - ).length, - 0, - ); + Math.max(resultColumns.extras.length, 0); const rememberTableScroll = useCallback(() => { const wrap = tableScrollRef.current; @@ -1455,7 +1314,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; const updateActiveSheet = (patch: Partial) => { - setTplError(""); setTplSheets((prev) => prev.map((s, i) => { if (i !== tplSheetIdx) return s; @@ -1493,7 +1351,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; const openNewTemplate = () => { - setTplError(""); setTplEditId(""); setTplName(""); setTplNote(""); @@ -1506,7 +1363,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; const openEditTemplate = (tpl: Template) => { - setTplError(""); const sheets = templateSheets(tpl); setTplEditId(tpl.id); setTplName(tpl.name); @@ -1578,7 +1434,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; const saveTemplate = async () => { - setTplError(""); if (!tplSheets.length) { showError(t("bizCompare.metricsRequired")); return; @@ -1600,7 +1455,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage } setBusy(true); try { - validateTemplateDraft(tplSheets); const body = { name: tplName || tplSheets.map((s) => sheetLabel(s)).join("+"), note: tplNote, @@ -1615,7 +1469,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage display_fields: s.display_fields || [...s.key_fields, ...s.compare_fields], row_filters: s.row_filters || [], field_rules: s.field_rules || [], - ignore_port_changes: s.ignore_port_changes, })), }; if (tplEditId) await bizCompareUpdateTemplate(tplEditId, body); @@ -1626,7 +1479,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage invalidateCutoverCache("bizMigration:"); await refresh({ force: true }); } catch (e) { - setTplError(formatErr(e)); showError(formatErr(e)); } finally { setBusy(false); @@ -1686,7 +1538,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage display_fields: s.display_fields || [...s.key_fields, ...s.compare_fields], row_filters: s.row_filters || [], field_rules: s.field_rules || [], - ignore_port_changes: s.ignore_port_changes, })), }); showOk(t("bizCompare.templateImported")); @@ -1783,9 +1634,12 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const tplId = preset?.template_id || templates[0]?.id || ""; setName(preset?.name || t("bizCompare.defaultJobName")); setTemplateId(tplId); + const tpl = templates.find((x) => x.id === tplId); + const all = tpl ? templateSheets(tpl).map((s) => sheetIdentity(s)).filter(Boolean) : []; const presetIds = (preset?.enabled_sheet_ids || []).map(String).filter(Boolean); if (presetIds.length) { - setEnabledSheetIds(presetIds); + const kept = presetIds.filter((id) => all.includes(id)); + setEnabledSheetIds(kept.length === all.length ? [] : kept); } else { setEnabledSheetIds([]); } @@ -1795,14 +1649,6 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage setBeforeBatchId(preset?.before_batch_id || ""); setAfterBatchId(preset?.after_batch_id || ""); setMode(preset?.mode === "auto" ? "auto" : "manual"); - { - const su = String(preset?.store_unchanged || "auto"); - setStoreUnchanged( - su === "always" || su === "never" || su === "sample" || su === "keys" - ? su - : "auto", - ); - } setValidateOut(null); if (preset?.mapping_id) loadMappingText(preset.mapping_id); else { @@ -1821,24 +1667,8 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage after_batch_id: mode === "manual" ? afterBatchId : "", mode, enabled_sheet_ids: enabledSheetIds, - store_unchanged: storeUnchanged, }); - const renderStoreUnchangedField = () => ( - setStoreUnchanged(e.target.value as StoreUnchanged)} - fullWidth - > - - - - - - - ); - const closeCreateJob = () => { setJobCreateOpen(false); setCreateStep(0); @@ -1916,87 +1746,40 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; const openJob = async (id: string) => { - const request = ++jobRequestRef.current; - runRequestRef.current += 1; setJobId(id); setJobDetailTab("config"); setRunDetail(null); - setRuns([]); - setRunsLoading(true); - setRunsError(""); setResultSheetId(""); setKindFilter("diff"); setResultKw(""); const job = jobs.find((x) => x.id === id); if (job) resetJobForm(job); try { - const r = await bizCompareListRuns(id, 50); - if (request !== jobRequestRef.current) return; - const items = (r.items || []) as RunRecord[]; - setRuns(items); - if (items.length) { - const latest = await bizCompareGetRun(items[0].id); - if (request !== jobRequestRef.current) return; + const r = await bizCompareListRuns(id); + setRuns(r.items || []); + if ((r.items || []).length) { + const latest = await bizCompareGetRun(String((r.items as any[])[0].id)); setRunDetail(latest); - // Land on batch list so stuck/cancelled runs are visible and actionable - setJobDetailTab("runs"); + setJobDetailTab("result"); } } catch (e) { - if (request === jobRequestRef.current) { - setRunsError(formatErr(e)); - showError(formatErr(e)); - } - } finally { - if (request === jobRequestRef.current) setRunsLoading(false); + showError(formatErr(e)); } }; - const refreshRuns = async () => { - if (!jobId || runsLoading) return; - const request = jobRequestRef.current; - setRunsLoading(true); - setRunsError(""); - try { - const response = await bizCompareListRuns(jobId, 50); - if (request === jobRequestRef.current) setRuns((response.items || []) as RunRecord[]); - } catch (e) { - if (request === jobRequestRef.current) setRunsError(formatErr(e)); - } finally { - if (request === jobRequestRef.current) setRunsLoading(false); - } - }; - - const handleJobTabKeyDown = (event: ReactKeyboardEvent) => { - const tabs: JobDetailTab[] = ["config", "runs", "result"]; - const current = tabs.indexOf(jobDetailTab); - const next = event.key === "ArrowRight" ? (current + 1) % tabs.length - : event.key === "ArrowLeft" ? (current + tabs.length - 1) % tabs.length - : event.key === "Home" ? 0 : event.key === "End" ? tabs.length - 1 : -1; - if (next < 0) return; - event.preventDefault(); - setJobDetailTab(tabs[next]); - if (tabs[next] === "runs") void refreshRuns(); - event.currentTarget.querySelector(`#compare-${tabs[next]}-tab`)?.focus(); - }; - const closeJob = () => { - jobRequestRef.current += 1; - runRequestRef.current += 1; if (document.fullscreenElement === boardRef.current) { void document.exitFullscreen().catch(() => undefined); } setBoardFs(false); setJobId(""); setRuns([]); - setRunsLoading(false); - setRunsError(""); setRunDetail(null); setResultSheetId(""); }; const saveJobConfig = async () => { if (!jobId) return; - if (missingJobSheets.length) { showError(t("bizCompare.missingSheets")); return; } if (!enabledJobSheetCount) { showError(t("bizCompare.needSheets")); return; @@ -2013,134 +1796,22 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage } }; - const runStatus = String(runDetail?.status || ""); - const runIsActive = runStatus === "running" || runStatus === "queued"; - const jobHasActiveRun = runs.some((r) => { - const st = String(r.status || ""); - return st === "running" || st === "queued"; - }); - const runCounts = useMemo(() => ({ - total: runs.length, - active: runs.filter((r) => r.status === "running" || r.status === "queued").length, - completed: runs.filter((r) => r.status === "success").length, - stopped: runs.filter((r) => r.status === "failed" || r.status === "cancelled").length, - }), [runs]); - const runNow = async () => { if (!jobId) return; - if (missingJobSheets.length) { showError(t("bizCompare.missingSheets")); return; } if (!enabledJobSheetCount) { showError(t("bizCompare.needSheets")); return; } - if (jobHasActiveRun) { - showError(t("bizCompare.compareAlreadyRunning")); - setJobDetailTab("runs"); - return; - } - const request = jobRequestRef.current; setBusy(true); try { await bizCompareUpdateJob(jobId, jobConfigBody()); - // Async enqueue — returns immediately with status=running; poll below. const run = await bizCompareRunJob(jobId); - if (request !== jobRequestRef.current) return; - runRequestRef.current += 1; setRunDetail(run); - setJobDetailTab("runs"); - showOk(t("bizCompare.runStarted")); - const r = await bizCompareListRuns(jobId, 50); - if (request !== jobRequestRef.current) return; - setRuns((r.items || []) as RunRecord[]); - await refresh({ force: true }); - } catch (e) { - if (request === jobRequestRef.current) { - showError(formatErr(e)); - setJobDetailTab("runs"); - } - } finally { - setBusy(false); - } - }; - const runProgress = (runDetail?.summary?.progress || {}) as RunProgressInfo; - const runEngine = String(runProgress.engine || "").toLowerCase(); - - // One poller; schedule the next tick after requests settle to prevent overlap. - useEffect(() => { - if (!jobId || (!runIsActive && !(jobHasActiveRun && jobDetailTab === "runs"))) return; - let cancelled = false; - let notified = false; - let timer: number | undefined; - const jobRequest = jobRequestRef.current; - const runId = String(runDetail?.id || ""); - const tick = async () => { - const runRequest = runRequestRef.current; - try { - const [r, d] = await Promise.all([ - bizCompareListRuns(jobId, 50), - runIsActive && runId ? bizCompareGetRun(runId) : Promise.resolve(null), - ]); - if (cancelled || jobRequest !== jobRequestRef.current) return; - setRuns((r.items || []) as RunRecord[]); - if (!d || runRequest !== runRequestRef.current) return; - setRunDetail((prev: typeof runDetail) => String(prev?.id || "") === runId ? d : prev); - const st = String(d.status || ""); - if (!notified && st === "success") { - notified = true; - const ms = Number((d.summary as any)?.duration_ms || 0); - const sec = ms > 0 ? Math.round(ms / 1000) : 0; - showOk(sec > 0 ? t("bizCompare.ranWithDuration", { s: String(sec) }) : t("bizCompare.ran")); - } else if (!notified && st === "failed") { - notified = true; - showError(String(d.message || t("bizCompare.runFailed"))); - } else if (!notified && st === "cancelled") { - notified = true; - showError(String(d.message || t("bizCompare.runCancelled"))); - } - } catch (e) { - if (!cancelled) showError(formatErr(e)); - } finally { - if (!cancelled) timer = window.setTimeout(() => void tick(), 2000); - } - }; - void tick(); - return () => { - cancelled = true; - window.clearTimeout(timer); - }; - }, [runDetail?.id, runIsActive, jobId, jobHasActiveRun, jobDetailTab, showOk, showError, t]); - - const loadRun = async (runId: string) => { - const request = ++runRequestRef.current; - const jobRequest = jobRequestRef.current; - try { - const d = await bizCompareGetRun(runId); - if (request !== runRequestRef.current || jobRequest !== jobRequestRef.current) return; - setRunDetail(d); - setKindFilter("diff"); - setResultKw(""); - setResultKeyFilters({}); setJobDetailTab("result"); - } catch (e) { - if (request === runRequestRef.current && jobRequest === jobRequestRef.current) showError(formatErr(e)); - } - }; - - const cancelRun = async (runId: string) => { - if (!runId) return; - if (!window.confirm(t("bizCompare.confirmCancelRun"))) return; - const request = jobRequestRef.current; - setBusy(true); - try { - const d = await bizCompareCancelRun(runId); - if (request !== jobRequestRef.current) return; - setRunDetail((prev: typeof runDetail) => String(prev?.id || "") === runId ? d : prev); - if (jobId) { - const r = await bizCompareListRuns(jobId, 50); - if (request !== jobRequestRef.current) return; - setRuns((r.items || []) as RunRecord[]); - } - showOk(t("bizCompare.runCancelled")); + showOk(t("bizCompare.ran")); + const r = await bizCompareListRuns(jobId); + setRuns(r.items || []); + await refresh({ force: true }); } catch (e) { showError(formatErr(e)); } finally { @@ -2148,30 +1819,40 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage } }; + const loadRun = async (runId: string) => { + try { + const d = await bizCompareGetRun(runId); + setRunDetail(d); + setKindFilter("diff"); + setResultKw(""); + setJobDetailTab("result"); + } catch (e) { + showError(formatErr(e)); + } + }; + const removeRun = async (runId: string) => { if (!runId) return; if (!window.confirm(t("bizCompare.confirmDeleteRun"))) return; - const request = jobRequestRef.current; - const runRequest = runRequestRef.current; setBusy(true); try { const wasCurrent = String(runDetail?.id || "") === runId; await bizCompareDeleteRun(runId); - if (request !== jobRequestRef.current) return; let nextRuns: typeof runs = []; if (jobId) { - const r = await bizCompareListRuns(jobId, 50); - if (request !== jobRequestRef.current) return; - nextRuns = (r.items || []) as RunRecord[]; + const r = await bizCompareListRuns(jobId); + nextRuns = r.items || []; } else { nextRuns = (runs || []).filter((r) => String(r.id) !== runId); } setRuns(nextRuns); - if (wasCurrent && runRequest === runRequestRef.current) { - runRequestRef.current += 1; - setRunDetail(null); - setResultSheetId(""); - setJobDetailTab("runs"); + if (wasCurrent) { + if (nextRuns.length) { + await loadRun(String(nextRuns[0].id)); + } else { + setRunDetail(null); + setResultSheetId(""); + } } showOk(t("bizCompare.runDeleted")); } catch (e) { @@ -2182,16 +1863,15 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage }; const renderSheetChips = () => ( -
-
+
+
{t("bizCompare.enabledSheets")} - + {enabledJobSheetCount}/{jobSheetAllIds.length || 0}

{t("bizCompare.enabledSheetsHint")}

- {missingJobSheets.length ?
{t("bizCompare.missingSheets")} · {missingJobSheets.join(", ")}
: null} -
+
{jobTemplateSheets.map((s) => { const sid = sheetIdentity(s); const on = isJobSheetOn(sid); @@ -2199,17 +1879,11 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage ); })} @@ -2225,10 +1899,10 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage const renderMappingBlock = () => (
-

{t("bizCompare.mappingOptionalHint")}

-
+

{t("bizCompare.mapping")}

+

{t("bizCompare.mappingOptionalHint")}

+
{ const id = e.target.value; @@ -2246,23 +1920,7 @@ export function BizComparePage({ pageMode = "all" }: { pageMode?: BizComparePage ))} - -
-