mirror of
https://github.com/hansjone/netx.git
synced 2026-10-14 10:36:00 +08:00
Compare commits
No commits in common. "main" and "v0.3.0" have entirely different histories.
462 changed files with 31484 additions and 67586 deletions
|
|
@ -120,16 +120,8 @@ NETX_UME_NOTIFICATION_TOPIC=ALARM
|
|||
# bundled | external; unset = external (compatible with existing deploys)
|
||||
# NETX_BUNDLED_PG_PORT=15432
|
||||
# NETX_BUNDLED_PG_DATA_DIR=
|
||||
# Update: GitHub primary + Forgejo mirror (git.avelo.top); pick newest reachable
|
||||
# NETX_UPDATE_SOURCES=github,forgejo
|
||||
# NETX_UPDATE_FORGEJO_URL=https://git.avelo.top/api/v1/repos/hansjone/netx/releases/latest
|
||||
# NETX_UPDATE_GITHUB_REPO=hansjone/netx
|
||||
# NETX_UPDATE_URL=
|
||||
# NETX_UPDATE_FALLBACK_URL=
|
||||
# NETX_UPDATE_TOKEN=
|
||||
# NETX_UPDATE_CHANNEL=stable
|
||||
# NETX_UPDATE_AUTO=false
|
||||
# When true: tray/scheduled task may download+apply zip updates silently.
|
||||
# Heavier fleets: raise CLI/DB together; also ensure Postgres max_connections and bastion session limits.
|
||||
# One-click start (recommended): .\scripts\start_netx.ps1 -Background -WithWeb
|
||||
# -> API + netx_api.worker + N× netx_api.biz_state_worker + optional Vite
|
||||
|
|
|
|||
1
.gitattributes
vendored
1
.gitattributes
vendored
|
|
@ -1,2 +1 @@
|
|||
*.sh text eol=lf
|
||||
packaging/linux/** text eol=lf
|
||||
|
|
|
|||
92
.github/workflows/release-linux.yml
vendored
92
.github/workflows/release-linux.yml
vendored
|
|
@ -1,92 +0,0 @@
|
|||
name: Release Linux
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Check tag matches pyproject version
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
run: |
|
||||
ver="$(sed -n 's/^[[:space:]]*version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' pyproject.toml | head -n1)"
|
||||
[ "${GITHUB_REF_NAME}" = "v${ver}" ] || { echo "tag ${GITHUB_REF_NAME} != v${ver}"; exit 1; }
|
||||
|
||||
- name: Cache PostgreSQL build + downloads
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: packaging/cache/linux
|
||||
key: linux-x86_64-${{ hashFiles('packaging/linux/versions.env', 'packaging/linux/build_in_container.sh') }}-${{ hashFiles('requirements.txt') }}
|
||||
restore-keys: |
|
||||
linux-x86_64-${{ hashFiles('packaging/linux/versions.env', 'packaging/linux/build_in_container.sh') }}-
|
||||
|
||||
- name: Build package
|
||||
run: bash packaging/linux/build_linux_release.sh
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: netx-linux-x86_64
|
||||
path: |
|
||||
packaging/release/netx-*-linux-x86_64.tar.gz
|
||||
packaging/release/netx-*-linux-x86_64.tar.gz.sha256
|
||||
retention-days: 7
|
||||
|
||||
smoke:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
image:
|
||||
- centos:7
|
||||
- rockylinux:9
|
||||
- ubuntu:18.04
|
||||
- ubuntu:24.04
|
||||
- debian:12
|
||||
- opensuse/leap:15
|
||||
- openeuler/openeuler:22.03
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: netx-linux-x86_64
|
||||
path: pkg
|
||||
- name: Offline install smoke (${{ matrix.image }})
|
||||
run: |
|
||||
tarball="$(ls pkg/netx-*-linux-x86_64.tar.gz)"
|
||||
# Minimal images may not ship tar: unpack on the runner and mount both.
|
||||
mkdir -p pkgx && tar xzf "$tarball" -C pkgx
|
||||
docker run --rm --network none \
|
||||
-v "$PWD/pkg:/pkg:ro" -v "$PWD/pkgx:/pkgx:ro" \
|
||||
-v "$PWD/packaging/linux/smoke_test.sh:/smoke.sh:ro" \
|
||||
"${{ matrix.image }}" bash /smoke.sh "/pkg/$(basename "$tarball")" "/pkgx/$(ls pkgx)"
|
||||
|
||||
release:
|
||||
needs: smoke
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: netx-linux-x86_64
|
||||
path: pkg
|
||||
# Release notes come from the Windows workflow; only attach files here.
|
||||
- name: Upload to GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
pkg/netx-*-linux-x86_64.tar.gz
|
||||
pkg/netx-*-linux-x86_64.tar.gz.sha256
|
||||
13
.github/workflows/release-windows.yml
vendored
13
.github/workflows/release-windows.yml
vendored
|
|
@ -22,11 +22,10 @@ jobs:
|
|||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Build release stage
|
||||
- name: Build release zip
|
||||
shell: pwsh
|
||||
run: |
|
||||
# Use pwsh -File (UTF-8). Nested Windows PowerShell 5.1 mis-parses UTF-8 scripts without BOM.
|
||||
& ./packaging/build_release.ps1 -CreateVenv
|
||||
powershell -ExecutionPolicy Bypass -File ./packaging/build_release.ps1 -CreateVenv
|
||||
|
||||
- name: Install Inno Setup
|
||||
shell: pwsh
|
||||
|
|
@ -38,18 +37,12 @@ jobs:
|
|||
shell: pwsh
|
||||
run: |
|
||||
$ver = (Get-Content pyproject.toml | Select-String '^\s*version\s*=').Line -replace '.*"(.+)"','$1'
|
||||
if ($env:GITHUB_REF_NAME -ne "v$ver") { throw "Release tag $env:GITHUB_REF_NAME does not match pyproject version $ver" }
|
||||
ISCC.exe "/DMyAppVersion=$ver" packaging/installer/netx.iss
|
||||
if ($LASTEXITCODE -ne 0) { throw "Inno Setup failed with exit code $LASTEXITCODE" }
|
||||
$setup = "packaging/release/NetX-Setup-$ver.exe"
|
||||
if (-not (Test-Path $setup) -or (Get-Item $setup).Length -lt 1MB) { throw "Missing or invalid installer: $setup" }
|
||||
$hash = (Get-FileHash $setup -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
Set-Content "$setup.sha256" -Value $hash -Encoding ascii -NoNewline
|
||||
|
||||
- name: Upload to GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
packaging/release/NetX-*-win64.zip
|
||||
packaging/release/NetX-Setup-*.exe
|
||||
packaging/release/NetX-Setup-*.exe.sha256
|
||||
generate_release_notes: true
|
||||
|
|
|
|||
11
.gitignore
vendored
11
.gitignore
vendored
|
|
@ -8,25 +8,16 @@ build/
|
|||
dist/
|
||||
scripts/.run/
|
||||
# Local one-off probes / staging dumps (not for commit)
|
||||
artifacts/
|
||||
scripts/_*
|
||||
scripts/archive/
|
||||
packages/netx-topology-mcp/tests/_*
|
||||
_tmp_*
|
||||
.idea/
|
||||
/ume/
|
||||
ume/
|
||||
data/ne_collections/
|
||||
data/webcrt/
|
||||
data/auth/
|
||||
data/runtime/
|
||||
data/ne_exec_jobs/
|
||||
# Windows packaging artifacts (binaries / release trees)
|
||||
packaging/cache/
|
||||
packaging/release/
|
||||
packaging/postgres/pgsql/
|
||||
packaging/winsw/
|
||||
*.exe
|
||||
!packaging/installer/*.iss
|
||||
|
||||
# Claude Code personal overrides
|
||||
.claude/settings.local.json
|
||||
|
|
|
|||
|
|
@ -158,9 +158,9 @@ API base: `http://127.0.0.1:8890/`
|
|||
|
||||
After `npm run build` in `web/`, the API can also serve the UI at `http://127.0.0.1:8890/` (same origin). See `NETX_UI_DIST_DIR` in `.env.example`.
|
||||
|
||||
### Windows installer (optional)
|
||||
### Windows installer / portable package (optional)
|
||||
|
||||
Fool-proof Windows delivery via `NetX-Setup-*.exe` (bundled or external Postgres, program/data split, offline upgrade over existing installs) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`.
|
||||
Fool-proof Windows delivery (bundled or external Postgres, program/data split, manual update) lives under [`packaging/`](packaging/README.md). **Linux installs are unchanged** — keep using your own Postgres and `scripts/start_netx.sh`.
|
||||
|
||||
### 6) MCP（Cursor / oclaw / Claude）
|
||||
|
||||
|
|
|
|||
|
|
@ -1,52 +0,0 @@
|
|||
"""Add app_user.prefs JSON for per-user UI preferences.
|
||||
|
||||
Revision ID: 20261013_user_prefs
|
||||
Revises: 20260922_biz_state_queued
|
||||
Create Date: 2026-10-13
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "20261013_user_prefs"
|
||||
down_revision: Union[str, Sequence[str], None] = "20260922_biz_state_queued"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
dialect = str(getattr(bind.dialect, "name", "") or "").lower()
|
||||
if dialect.startswith("postgres"):
|
||||
op.execute(
|
||||
"ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'::json NOT NULL"
|
||||
)
|
||||
# Ensure NOT NULL + default for brownfield rows that already had a nullable column.
|
||||
op.execute("UPDATE app_user SET prefs = '{}'::json WHERE prefs IS NULL")
|
||||
op.execute("ALTER TABLE app_user ALTER COLUMN prefs SET DEFAULT '{}'::json")
|
||||
op.execute("ALTER TABLE app_user ALTER COLUMN prefs SET NOT NULL")
|
||||
else:
|
||||
insp = sa.inspect(bind)
|
||||
if "app_user" in insp.get_table_names():
|
||||
cols = {c["name"] for c in insp.get_columns("app_user")}
|
||||
if "prefs" not in cols:
|
||||
op.add_column(
|
||||
"app_user",
|
||||
sa.Column("prefs", sa.JSON(), nullable=False, server_default="{}"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
dialect = str(getattr(bind.dialect, "name", "") or "").lower()
|
||||
if dialect.startswith("postgres"):
|
||||
op.execute("ALTER TABLE app_user DROP COLUMN IF EXISTS prefs")
|
||||
else:
|
||||
try:
|
||||
op.drop_column("app_user", "prefs")
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
# 割接对比：规范值契约
|
||||
|
||||
适用范围：业务对比（`BizCompare*`）和割接监控（`BizMigration*`）。两种模式用同一条身份流水线（`netx_api/biz_state/identity_pipeline.py`），跨厂商对比靠它把两侧设备的原始输出对齐到同一套“规范值”。
|
||||
|
||||
## 1. 流水线
|
||||
|
||||
同一行在某一侧（`before` / `after`）依次经过：
|
||||
|
||||
1. **归一化**：接口字段去内部空格 → 厂商内规范名（`VENDOR_CANONICAL`，如华为 `GigabitEthernet`→`GE`）→ 模板别名规则（`iface_normalize_rules`，如 `GE`→`gei`）；键字段按 `field_rules[].normalize`（`lower`/`upper`/`mac`…）；再按 `field_rules[].value_map` 改写状态值。
|
||||
2. **行过滤**：`row_filters` 在归一化之后求值，所以按规范值书写；带 `side` 的过滤只作用于该侧。
|
||||
3. **端口映射**：只作用于 `before` 侧，键用 before 侧规则归一化、值用 after 侧规则归一化。
|
||||
4. **取键**：`scalar_text(value).strip()` 逐字段取值，得到 match key。
|
||||
|
||||
快照版本 `config_snapshot.version >= 2` 的 run 走完整流水线；`version == 1` 的历史 run 保持旧语义（过滤看原始行、键不归一化、同键按采集顺序配对），回填、检索、导出结果不变。
|
||||
|
||||
## 2. 规范值
|
||||
|
||||
比对层只认下面这些取值。设备原始值不同的，用 `value_map` 或 parser 输出规范值解决，**引擎里不写厂商特例**。
|
||||
|
||||
| metric | 字段 | 规范取值 |
|
||||
|---|---|---|
|
||||
| `interface_brief` | `admin` / `phy` / `prot` | `up` / `down`（华为 `*down` 记 `admin=down`） |
|
||||
| `bgp_peer` | `state` | `established` / `idle` / `active` / `connect` / `opensent` / `openconfirm` |
|
||||
| `bgp_peer` | `afi` | `ipv4` / `ipv6` / `vpnv4` / `vpnv6` / `evpn` / `vpls` |
|
||||
| `arp` | `entry_type` | `dynamic` / `static` / `interface` |
|
||||
| `isis_adjacency` / `ospf_neighbor` | `state` | `up` / `full` / `2way` / `down` / `init` / `idle` |
|
||||
| `vrrp` | `af` | `ipv4` / `ipv6` |
|
||||
|
||||
### 中兴（已实现）现状
|
||||
|
||||
中兴 parser 输出设备打印的原值，没有改写：
|
||||
|
||||
- `interface_brief.admin/phy/prot`：`up` / `down`，已是规范值。
|
||||
- `arp.entry_type`：由 `age` 推断，`HH:MM:SS` 为 `dynamic`，否则 `static`。
|
||||
- `bgp_peer.state`：设备原值，如 `Established`（首字母大写）。与中兴自身对比时一致；与其他厂商对比时在模板里加值映射：
|
||||
|
||||
```json
|
||||
{"field": "state", "value_map": {"established": "established", "estab": "established", "*down": "down"}}
|
||||
```
|
||||
|
||||
`value_map` 大小写不敏感匹配，输出为右侧的规范值；两侧都经过同一张表。
|
||||
|
||||
> 没有改动中兴 parser 的输出值：已采集批次里的值是原值，直接改成小写会让“旧批次 vs 新批次”出现假差异。需要统一时在模板里加 `value_map`。
|
||||
|
||||
### 其他厂商（待办）
|
||||
|
||||
华为 VRP 的状态 parser（`interface_brief`、`arp`、`bgp_peer`、`isis_adjacency`、`ospf_neighbor`）**尚未实现**：仓库里没有真实输出样本，按规则不凭空编写 TextFSM。收到样本后按 `skills/biz-state/netx-biz-state/SKILL.md` 第 2 节逐项完成，输出与中兴相同的字段名，状态值输出上表的规范值，并给 profile 补 `slice`。
|
||||
|
||||
目前只有 LLDP 在跨厂商场景下有真实 parser。
|
||||
|
||||
## 3. 命令切片（`ParseProfile.slice`）
|
||||
|
||||
一个采集命令产出 metric 的哪一部分，由 profile 的 `slice` 声明，例如 `zte.bgp_ipv4_summary` → `{"afi": "ipv4"}`、`zte.vrrp_ipv6` → `{"af": "ipv6"}`。比对表单（sheet）用顶层 `eq` 行过滤声明自己要的切片。
|
||||
|
||||
- 一侧没有任何命令覆盖该切片 → 该 sheet 状态 `not_collected`，不判业务缺失、不计入通过率。
|
||||
- 割接监控创建 HF 采集任务时，取两侧画像命令切片的并集；某侧找不到对应切片的 profile 时返回 `no_profile_for_slice:<metric>:<slice>:<vendor>`。
|
||||
- 没声明 `slice` 的命令视为覆盖整个 metric。
|
||||
|
||||
## 4. sheet 数据状态
|
||||
|
||||
| 状态 | 条件 | 通过率 |
|
||||
|---|---|---|
|
||||
| `ok` | 两侧采集正常 | 正常计算 |
|
||||
| `no_data` | 两侧过滤后都是 0 行，命令都成功 | `null`，不计入总通过率 |
|
||||
| `suspect_parse` | 某侧 `raw_line_count > 5` 但 `row_count == 0`，或全部命令失败 | 失败行照常显示，任务级 `ok = null` |
|
||||
| `not_collected` | 某侧没有任何命令（或切片）产出该 metric | `null`，不计入总通过率 |
|
||||
|
||||
没有任何命令回执的批次（导入或历史数据）无法判断，按正常处理。
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 46 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 84 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 88 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 83 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 57 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 74 KiB |
|
|
@ -1,29 +0,0 @@
|
|||
{
|
||||
"date": "2026-10-10",
|
||||
"baseline_ref": "afa8e06b2de64100f963ae30b3babfcb5d7009f4",
|
||||
"python": "3.14.3",
|
||||
"dataset": {
|
||||
"rows_per_side": 1000000,
|
||||
"fields": ["id", "neighbor", "state", "afi"],
|
||||
"strata": 50,
|
||||
"unchanged_limit": 5000,
|
||||
"compact_unchanged": true,
|
||||
"interface_transforms": false,
|
||||
"all_rows_unchanged": true
|
||||
},
|
||||
"measurement": "tracemalloc peak allocated during compare_rows; excludes input allocations, DB loading and process overhead",
|
||||
"results": [
|
||||
{
|
||||
"engine": "baseline",
|
||||
"peak_engine_mib": 1144.43,
|
||||
"seconds_with_tracemalloc": 92.4
|
||||
},
|
||||
{
|
||||
"engine": "optimized",
|
||||
"peak_engine_mib": 420.49,
|
||||
"seconds_with_tracemalloc": 63.681
|
||||
}
|
||||
],
|
||||
"results_and_sample_order_identical": true,
|
||||
"test_result": "1 passed in 157.88s"
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 74 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 42 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 95 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 120 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 40 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 102 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 71 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 148 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 43 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 77 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 37 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 69 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 112 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 47 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 96 KiB |
|
|
@ -1,131 +0,0 @@
|
|||
# 业务对比检查与优化记录
|
||||
|
||||
检查日期：2026-10-10。范围：网络管理 → 割接对比 → 业务对比，包括前端任务/批次/结果界面、Python 与 PostgreSQL 对比路径、结果分页与搜索、短期列表缓存。
|
||||
|
||||
本次直接修改项目代码并添加回归测试，没有连接设备执行采集，也没有修改生产数据库。浏览器截图与交互测试使用固定 API 响应。
|
||||
|
||||
## 已修复的问题
|
||||
|
||||
### 1. 搜索会改变业务判定（P1）
|
||||
|
||||
原实现分别按搜索条件过滤割接前、后的原表，再重新对比。例如 `state=up → down`，搜索 `up` 后只剩前侧，原本的“变化”就可能变成“删除”。重复身份键的业务还可能在筛选后错位配对。
|
||||
|
||||
现在：已存储的判定直接在结果表分页筛选；需要回查未保存全量明细的成功行时，先找到候选身份键，再补齐两侧的完整分组，按原有配对规则对比，最后筛选。达到读取上限的分组整体跳过，并返回截断标记，避免产生虚假的新增或删除。
|
||||
|
||||
### 2. PostgreSQL 搜索漏用部分模板过滤条件（P1）
|
||||
|
||||
原搜索只下推 SQL 支持的过滤规则，对 `age_timer` 等不支持的规则直接忽略，可能将静态 ARP 等不属于模板范围的数据纳入结果。现在不支持下推的规则走流式读取并在 Python 中完整过滤。
|
||||
|
||||
### 3. 快速切换时旧请求覆盖当前任务（P1）
|
||||
|
||||
前后采集任务的批次列表、打开任务、加载历史结果以及轮询缺少完整的过期保护。现在对批次请求使用清理标记，对任务与结果使用请求序号；关闭弹窗、切换路由或打开其他任务后，旧响应不能覆盖当前界面。
|
||||
|
||||
### 4. 结果加载错误容易被误读（P2）
|
||||
|
||||
原查询失败时保留上一组明细，界面已经显示新筛选条件；切到未完成的表后，加载状态也可能无法结束。现在加载和失败都有独立状态，错误时清除旧明细并提供重试入口；切换表/页签后清理加载状态。
|
||||
|
||||
### 5. LLDP 搜索漏查（P2）
|
||||
|
||||
LLDP 数据存于 `biz_state_lldp_neighbor`，原搜索只读 `biz_state_metric_row`。现在原表回查与精简结果的字段搜索都支持 LLDP 表。
|
||||
|
||||
### 6. 其他已修复边界
|
||||
|
||||
- 恰好命中读取上限时，原非 PostgreSQL 路径错误地报告“已截断”；现在多读一条判断。
|
||||
- 无效或有歧义的表标识可能落到其他表；现在返回 `sheet_not_found`。
|
||||
- `%`、`_` 等搜索字符按字面值匹配，避免不同查询路径含义不一致。
|
||||
- 关闭最后一项检查后，空数组被解释为“启用全部”；现在阻止关闭最后一项并提示至少保留一项。
|
||||
- 两个 HeroUI Input 使用了 `isDisabled`，实际需要 `disabled`；过滤值与容差输入的禁用状态已修正。
|
||||
- 列表缓存缺少并发合并，慢请求可以覆盖强制刷新结果或在失效后重新填充缓存；现在共享请求、保护失效状态并抑制过期的后台通知。
|
||||
- 清理中英文文案中的重复键，解决相关类型检查错误。
|
||||
- 操作后任务选择“同操作前”时原来没有加载批次列表；现在按实际使用的任务读取并共享缓存。切换采集任务时清除原批次选择，避免把上一个任务的批次带入配置。
|
||||
- 原通用下拉框嵌套了两个 label，标签不能正确关联控件；现在使用明确的 `htmlFor/id`，并关联提示文本。
|
||||
|
||||
## 性能改动与实际收益范围
|
||||
|
||||
- 合并两套轮询，下一轮在本轮请求完成后启动。模拟浏览器测试中，运行任务在 5.2 秒内历史列表和详情各请求 2–3 次（包含可能尚未完成的初始加载），没有重复轮询，也不会因慢请求持续叠加。
|
||||
- 已保存的失败、新增、变化等结果在数据库中筛选并分页，避免每次翻页重新加载和对比原始批次。回归测试明确验证这一查询路径不调用 `compare_rows`。
|
||||
- 同一前后采集任务的批次请求共用短期缓存，列表后台刷新合并同一请求；6 项缓存测试覆盖并发、强制刷新、失败重试与失效竞态。
|
||||
- 模板查找使用 Map，避免列表搜索与绘制逐项线性查找模板。
|
||||
- 修改筛选后先回到第一页，避免先请求旧页码再重复请求第一页。
|
||||
|
||||
这些改动减少了重复工作。以下后续基准使用合成数据，没有测量生产环境延迟。
|
||||
|
||||
## 后续优化（2026-10-10）
|
||||
|
||||
### 历史执行配置与精简行还原
|
||||
|
||||
新运行在创建时将模板、启用的检查项、字段规则/过滤、接口归一化、端口映射、成功行保存策略和名称存入 `summary_json.config_snapshot`。后台执行、成功行回查、精简行还原以及导出共用该配置；排队之后修改配置，甚至删除原模板/映射，也不会改变这一运行使用的配置。无需数据库结构迁移。
|
||||
|
||||
修复了精简存储明细还原时将 `mapped_before` 直接设为原始 `before` 的问题。现在前侧依次归一化和映射，后侧归一化，保持与完整结果的显示及 CSV 一致。ZIP 新增 `config_snapshot.json`，便于核对当时配置；详情接口只返回快照模板与版本，不将完整端口映射加入轮询响应。
|
||||
|
||||
页面显示“执行配置已保存”；旧记录显示“旧记录：未保存执行配置”。旧运行无法可靠补造当时的配置，仍保留兼容查询，成功行回查及精简行还原会使用当前配置。快照也不保存原始采集行的副本，若原始批次数据被清理，依赖行 ID 的还原仍会缺少内容。
|
||||
|
||||
### Python 大表内存
|
||||
|
||||
成功样本按“层内序号、层首次出现顺序”排序，只保留样本上限内的候选和分层状态，避免先积累全部成功行。保留原分层轮询顺序；未截断的小表仍保持采集顺序。不需要接口变换时复用输入行；有变换时创建新行，不修改输入数据。稳定身份键顺序直接使用分组字典，省去再次计算全表键、额外键列表与集合。
|
||||
|
||||
每侧 1,000,000 行、50 个邻居分层、全部成功、保存 5,000 条精简样本的测试：优化前引擎新增分配峰值 **1144.43 MiB**，优化后 **420.49 MiB**，下降约 **63.3%**。完整结果与样本顺序的 SHA-256 一致。开启 `tracemalloc` 时耗时为 92.400 秒和 63.681 秒；该计时含内存追踪开销，不代表生产耗时。输入行在追踪开始前创建，峰值不包含输入行、数据库加载或 Python 进程的总内存。
|
||||
|
||||
基线为 `afa8e06b2de64100f963ae30b3babfcb5d7009f4` 中的对比引擎。原始指标见 [百万行测试结果](assets/biz-compare-memory-benchmark.json)。身份键分组仍需要随原表行数增长的内存；这次不是完全流式对比。
|
||||
|
||||
### 大表导出
|
||||
|
||||
导出改为按 `(seq, id)` 游标分批查询、分批还原精简行、逐行将 UTF-8 CSV 写入 ZIP，避免将全部明细与完整 CSV 同时保存在内存。相同 `seq` 的多条结果按 `id` 连续读取，不漏掉跨批次的同行序号记录。保留 BOM、字段顺序及逗号/引号/换行转义。最终压缩包仍由现有接口在内存中返回，压缩文件本身的大小仍计入内存。
|
||||
|
||||
## UI 改动
|
||||
|
||||
任务列表、创建向导、任务配置、对比批次和结果区统一使用深色平面、细边框、小圆角、低饱和蓝色和等宽数字。任务配置按作业参数、前后数据源、检查范围、端口映射四个区块组织；数据源左右配对，显示设备地址和采集数量。保存配置作为配置页主操作，删除操作降低视觉权重。
|
||||
|
||||
批次页增加最近 50 条记录的统计、刷新、加载、错误重试；刷新失败保留已载入记录。已完成只表示执行结束，业务失败/成功/新增数量单独显示。排队与运行使用不同文案，未知进度使用不确定进度条；时间、批次标识和前后采集信息便于核查。页签支持方向键及 Home/End，窄屏配置改为单列，批次表在独立容器中横向滚动。
|
||||
|
||||
结果区分为当前检查项/全表通过率、结果类型、搜索与辅助操作三层。增加明确的分页范围、清空筛选、加载状态、错误重试和搜索无结果提示。筛选按钮增加按下状态，字段筛选增加展开状态与键盘焦点提示。后续增加了执行配置保存状态及旧记录限制提示。
|
||||
|
||||
窄屏检查项导航改为紧凑的横向排列，避免一个检查项占据整行的大块区域。浏览器检查覆盖 1440px、768px、390px；筛选栏没有横向溢出。结果表保留必要的横向滚动。
|
||||
|
||||
- [任务列表](assets/biz-compare-jobs.png)
|
||||
- [桌面任务配置](assets/biz-compare-config.png)
|
||||
- [390px 任务配置](assets/biz-compare-config-mobile.png)
|
||||
- [桌面对比批次](assets/biz-compare-runs.png)
|
||||
- [390px 对比批次](assets/biz-compare-runs-mobile.png)
|
||||
- [桌面结果区](assets/biz-compare-desktop.png)
|
||||
- [768px 结果区](assets/biz-compare-narrow.png)
|
||||
- [加载失败与重试](assets/biz-compare-error.png)
|
||||
- [旧记录执行配置提示](assets/biz-compare-legacy.png)
|
||||
|
||||
## 验证
|
||||
|
||||
- Python：87 项通过，1 项百万行基准默认跳过；单独启用百万行基准后通过。新增测试覆盖搜索、历史配置快照、有界抽样与游标导出。
|
||||
- 缓存：6 项 Node 测试通过。
|
||||
- 浏览器：27 个场景通过，包括任务列表文案、配置保存数据、同任务批次加载与切换清理、自动模式、检查项开关、创建向导、键盘页签、批次统计/状态/刷新/错误重试、配置保存状态、旧记录提示、结果布局、筛选、窄屏布局、旧任务响应保护、单套轮询、无运行时异常。
|
||||
- Vite 生产打包成功。
|
||||
- 本次业务对比页面的类型错误已经修复。全项目 `tsc -b` 仍被 `BizStatePage.tsx`、`BizMigrationPage.tsx` 中原有的类型/未使用变量错误阻塞。
|
||||
- 页面 ESLint 原有 34 个错误，主要是 `any` 与现有 React Hook 写法；批次列表增加类型后减少了 `any`，本次未增加错误。结果请求依赖警告已修正。通用下拉框和缓存模块没有 lint 错误。
|
||||
- PostgreSQL 路径验证了 SQL 编译与规则降级分支，没有在真实 PostgreSQL 数据库上执行查询或测量执行计划。
|
||||
|
||||
可重复执行的检查：
|
||||
|
||||
```powershell
|
||||
# 项目根目录
|
||||
.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare_export.py tests/test_biz_state_compare_snapshot.py tests/test_biz_state_compare_memory.py tests/test_biz_state_compare_search.py tests/test_biz_state_compare.py tests/test_biz_state_compare_sql.py tests/test_biz_state_compare_recovery.py -q
|
||||
|
||||
# 可选：百万行基准，耗时约数分钟，内存占用较大
|
||||
$env:NETX_BENCH_ROWS = '1000000'
|
||||
$env:NETX_BENCH_REF = 'afa8e06b2de64100f963ae30b3babfcb5d7009f4'
|
||||
.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare_memory.py::test_memory_benchmark_against_git_baseline -q -s
|
||||
Remove-Item Env:NETX_BENCH_ROWS, Env:NETX_BENCH_REF
|
||||
|
||||
# web 目录
|
||||
node --experimental-strip-types --test tests/cutoverDataCache.test.mjs
|
||||
node node_modules/vite/bin/vite.js build
|
||||
node node_modules/typescript/bin/tsc -b
|
||||
```
|
||||
|
||||
浏览器脚本为 `web/tests/bizCompare.browser.mjs`。先启动本地 Vite 服务，再设置 `NETX_PLAYWRIGHT_MODULE` 为可用的 Playwright 包路径；`NETX_TEST_URL` 默认是 `http://127.0.0.1:5179`，浏览器默认使用已安装的 Chrome。所有 `/v1` 请求由脚本提供固定响应，不会访问实际设备或数据库。
|
||||
|
||||
## 后续值得优化的部分
|
||||
|
||||
1. **Python 大表分组。** 有界抽样和冗余复制优化已完成，百万行基准下降约 63% 的引擎新增分配峰值。分组与输入仍随原表规模增长；进一步降低内存可研究排序流式配对或分区执行，并保持重复键配对语义。
|
||||
2. **PostgreSQL 搜索与深分页。** JSON 字段包含搜索和精确总数统计仍可能扫描大量结果。需要用真实数据查看 `EXPLAIN (ANALYZE, BUFFERS)`，再决定常用字段索引、搜索文本索引或游标分页；本次没有盲目添加生产索引。
|
||||
3. **历史配置与原始数据保留。** 新运行的执行配置快照已完成。旧运行仍无法还原未记录的当时配置；精简结果还原依赖原批次行，后续可为批次清理增加引用保护或保留策略。
|
||||
4. **重复键的业务语义。** 引擎按采集顺序配对同键行。如果业务应按集合判断，CLI 行顺序改变可能产生多条“不一致”。当前保留原有语义；可优先完善身份键，或另行实现并测试 Python/SQL 一致的集合匹配。
|
||||
5. **原表回查的数量上限。** 每侧候选命中最多 2000 条，完整分组补齐每侧预算 10000 条，最终展示最多 200 对；这不是全表匹配总数。界面会提示截断，大范围检索需要收窄条件。
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
# 业务监控分析与优化记录
|
||||
|
||||
检查日期：2026-10-11。范围：网络管理 → 割接对比 → 业务监控，包含任务配置、周期采集状态、批次保护、工作簿、原始日志、筛选与导出。
|
||||
|
||||
本次修改项目代码并使用内存 SQLite 和固定浏览器 API 响应验证，没有连接设备执行采集，也没有修改生产数据库。未新增解析模板或数据库字段。前后端应一起更新，前端轮询使用新增的 `/v1/biz-state/tasks/{id}/progress` 接口。
|
||||
|
||||
## 已修复的问题
|
||||
|
||||
1. **旧请求覆盖当前内容。** 任务、用途筛选、批次工作簿、指标表、采集详情、参数发现和原始日志增加请求序号或取消保护。关闭弹窗、切换任务/表后，慢响应不能重新打开弹窗或覆盖新内容。原始日志下载保留其实际批次 ID，避免根据当前另一层弹窗推断来源。
|
||||
2. **后台采集不能及时反映、慢轮询叠加。** 同时观察服务端 `collect_running` 与当前页面启动的任务；周期任务和打开的任务保留低频查询，能够发现后台启动的采集。活动采集每轮完成后等待 4 秒，空闲等待 15 秒，再发起下一轮。跨 effect 重建共享在途标记，避免慢请求叠加。用途筛选隐藏的本地采集任务单独读取进度，避免误报采集结束。
|
||||
3. **刷新覆盖未保存配置、批次数突然缩减。** 轮询只合并运行标志和时间，不重新写入周期、保留策略或监控项。进度刷新和首次加载统一读取最多 200 个最近批次，界面明确显示已载入数量与上限。
|
||||
4. **加载失败看起来像空表或上一份数据。** 任务列表、任务配置、批次和指标表增加加载/错误/重试状态。指标查询失败清除旧行，不显示“本表无数据”；配置未就绪时操作区域不可交互。列表刷新失败保留当前行并显示错误。
|
||||
5. **正在采集的批次可能被删除或清理。** `queued/running` 批次加入 `active_collection` 保护，单条删除、批量删除、保留清理均保留它们；采集中的任务返回 409。前端同步禁用选择与删除，保留基线、业务对比及割接引用保护。
|
||||
6. **查询与导出边界错误。** `%`、`_`、反斜杠按字面值搜索；JSON 单列提取和 LLDP 分页排序更稳定。早于主命令保存的辅助命令失败保持可见。CSV 保留 `0`、`False`，正确引用逗号、双引号和 CR/LF；重复 `seq` 的导出不遗漏跨分块行，完整保留后续行才出现的列。
|
||||
|
||||
## 性能改动与验证范围
|
||||
|
||||
- 任务详情将逐监控项读取绑定改为一次批量读取。测试中 30 个监控项、60 个绑定由原来的 32 次查询降为 **3 次**；轻量进度接口为 **1 次**，不返回监控项和绑定。
|
||||
- 批次保护只读取引用 ID 列，按本次批次范围限制查询，不再加载所有对比/割接运行的配置和汇总 JSON。
|
||||
- 工作簿摘要不将原始 CLI 日志传输到 Python；只读取 SQL 计算的行数与是否存在日志。已有完整行数优先使用，原始日志在查看或导出时加载。回归使用 1 MB 日志，确认摘要结果集没有原文列。
|
||||
- 通用指标 CSV 先分块扫描列名，再按 `(seq, id)` 游标每块 1000 行写入 ZIP；LLDP 分批写入。减少指标表的 ORM 对象、字典和完整 CSV 同时驻留。**最终 ZIP 仍驻留内存，命令原文导出仍需要原文；这不是端到端流式下载。** 两次扫描保留完整列名，正在写入的批次仍不保证导出为一致性快照。
|
||||
- 同一厂商/型号的采集模板缓存 30 秒；轮询不重新读取模板。任务列表默认每页渲染 50 行，可选 20/50/100/200；列名去重使用 Set，命令搜索只计算一次。
|
||||
- 指标搜索先回到第一页并等待防抖值一致，避免先请求旧页码、旧关键字。
|
||||
|
||||
以上是查询次数、请求并发及读取方式的验证，没有测量真实设备或生产 PostgreSQL 的延迟与内存。任务列表 API 仍返回当前用途下的完整任务集合；客户端分页减少 DOM，不减少列表响应大小。JSON 全文模糊搜索、旧日志的 SQL 行数计算、非常大的 ZIP 内存仍是后续基准重点。
|
||||
|
||||
## 界面调整
|
||||
|
||||
统一深色工业风：连续指标栏、细边框、小圆角、等宽任务标识、紧凑表头及明确的异常提示。任务配置使用响应式网格；采集周期数值与单位并排；工作簿横向页签、过滤工具栏和表格支持窄屏滚动；底部按钮可换行，避免被裁切。任务和工作簿页签支持方向键、Home/End，过滤控件增加可访问名称。
|
||||
|
||||
“采集成功 / 部分完成 / 采集失败”描述采集流程，不能据此判断业务验收通过；业务状态需要查看数据表或业务对比结果。
|
||||
|
||||
- [任务列表](assets/biz-state-tasks.png)
|
||||
- [任务配置](assets/biz-state-config.png) · [390px 配置](assets/biz-state-config-mobile.png)
|
||||
- [采集批次](assets/biz-state-batches.png)
|
||||
- [工作簿](assets/biz-state-workbook.png) · [390px 工作簿](assets/biz-state-workbook-mobile.png)
|
||||
- [查询失败与重试](assets/biz-state-error.png)
|
||||
|
||||
## 验证
|
||||
|
||||
- 后端相关回归 **124 项通过**：新增监控服务、保留策略、日志导入、命令导出，以及停止/恢复/收尾/认领采集、双车道、spool、解析池与业务对比相关回归。
|
||||
- `web/tests/bizState.browser.mjs`：**30 项回归通过**。固定 120 个任务、54 个批次、120 行指标，覆盖分页、配置保存、模板缓存、过期响应、搜索、重试、日志下载、后台采集和窄屏布局。4500ms 列表延迟下最大并发轮询为 1；轮询阶段任务完整详情仅首次打开请求 1 次，未保存的周期值保持不变。全部 `/v1` 请求被固定响应拦截。
|
||||
- `node node_modules/vite/bin/vite.js build` 通过。项目原有主包超过 500 kB 提示仍在。
|
||||
- `node node_modules/typescript/bin/tsc -b`：本页已有类型问题已修复；全项目仍被 `BizMigrationPage.tsx` 的 4 个既有错误阻断（两处未使用变量、两处字符串传给数值控件）。
|
||||
- 本页 ESLint 从 16 个错误降为 15 个，保留既有 `any` 和 effect 状态同步问题及 1 个依赖警告；不宣称全项目 lint 通过。
|
||||
|
||||
浏览器回归运行方法（先启动 Vite，可通过环境变量指定 URL、截图目录和浏览器）：
|
||||
|
||||
```powershell
|
||||
cd web
|
||||
node node_modules/vite/bin/vite.js --host 127.0.0.1 --port 5179
|
||||
# 另一个终端；Playwright 已安装时直接运行，否则设置 NETX_PLAYWRIGHT_MODULE 为其模块路径。
|
||||
node tests/bizState.browser.mjs
|
||||
```
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
# 对比模板与业务对比评估、优化记录
|
||||
|
||||
日期：2026-10-11。范围：网络管理 → 割接对比 → 对比模板、业务对比。延续此前的业务对比和业务监控优化。本轮使用内存 SQLite、SQL 编译断言和本地浏览器固定 API 响应验证，没有连接真实设备，也没有改动生产数据库。
|
||||
|
||||
## 修复的结果准确性问题
|
||||
|
||||
1. **模板导入可能只保存有效部分。** 前后端检查全部检查项，空 Key、重复 ID、无效规则不再被静默丢弃。服务器在修改模板之前校验，错误响应包括配置路径和原因。执行时也检查历史模板的原始检查项，防止旧的无效配置被归一化后跳过。
|
||||
2. **错误过滤规则可能放行数据。** 新写入与执行入口拒绝未知运算符、混合分组/叶节点、空分组、无效正则及超过 12 层的嵌套。集合过滤要求非空数组；绝对/相对容差必须为有限非负数。无效归一模式、重复字段规则和接口别名同样报错。
|
||||
3. **复杂嵌套过滤被可视化编辑器截断。** 编辑器只接管能够完整表达的 OR/AND 结构，其余结构完整保留并显示 JSON。保存、导入、导出不删除嵌套条件。修改复杂结构可导出 JSON 后编辑并重新导入。
|
||||
4. **端口变化策略在 API 和导入导出中丢失。** `ignore_port_changes` 贯通请求模型、兼容单表配置、编辑、复制、保存、导入与导出。界面明确提供自动判断、接口必须匹配、允许端口变化三种选项；显式 `false` 不再丢失。
|
||||
5. **零、布尔和空值混淆。** Python 行键与过滤保留 `0`、`False`，布尔文本与 JSON 提取一致；数值比对不再把空值当成 0。Python/SQL 使用同一数值格式，支持 `+.5`、`1.`、科学计数法；解析不了的值按文本比较。
|
||||
6. **SQL 过滤与行键误匹配。** `contains` 的 `%`、`_`、反斜杠按字面值处理，空包含条件与 Python 一致。复合键改为 JSON 数组文本，避免 `('a|b','c')` 和 `('a','b|c')` 被拼成相同键。
|
||||
7. **任务范围失效导致隐式扩大或缩小对比。** 编辑任务时保留已移除的检查项 ID，显示错误并禁用保存、执行；用户可明确重新选择或启用当前全部检查项。服务器在保存和执行时都检查 ID，部分有效、部分失效的范围也会被拒绝。
|
||||
8. **不完整采集被解释成业务变化。** 比对入口只接受两侧状态为 `success` 的源批次；排队、执行中、失败、取消和部分完成批次返回 `source_batch_not_complete`。界面禁用这些选项。成功批次没有某指标或没有行的原有处理保持不变；本轮没有新增逐指标采集完备性判定。
|
||||
9. **删除破坏引用或在途对比。** 已被任务引用的模板不能删除；存在排队/执行中记录的任务及活动记录不能删除。模板页显示任务引用数量。这些是 API 状态检查，未新增数据库外键或锁，不宣称消除跨请求的全部并发窗口。
|
||||
10. **重新启用忽略字段仍然无效。** 勾选参与比对时清除遗留的 ignore/skip/off 规则。字段表保留模板配置的自定义字段；过滤专用字段也能显示。兼容数值、相对偏差和归一规则别名的显示。
|
||||
|
||||
## 性能与界面
|
||||
|
||||
- 检查项排序从逐表、逐批次查询改为 **一次按批次/metric 聚合**。相同 metric 的拆分表共享统计，SQL 不投影原始 CLI 日志；回归验证查询次数为 1。
|
||||
- `batch_metric_collect_ok` 只读取解析状态和行数，不加载命令原文。
|
||||
- 模板引用数量由一次 grouped 查询计算，避免每行请求。默认模板初始化/升级仍执行原来的查询。
|
||||
- 模板页面只请求模板和字段目录，省去业务监控任务、端口映射、对比任务三个列表；字段目录缓存 60 秒。业务对比与模板列表使用独立缓存键。
|
||||
- 两个列表默认每页渲染 50 行，可选 20/50/100/200；模板 ID 查找使用 Map。**API 仍返回完整列表，客户端分页减少 DOM，不减少响应体。**
|
||||
- 模板编辑器使用深色平面面板、细边框、小圆角、蓝色选中态、等宽字段和固定表头。接口归一化折叠，检查项导航与规则区并排；窄屏导航横向滚动，操作按钮独立一行，过滤控件可换行，底部保存始终可见。
|
||||
- 导航和规则页签支持方向键、Home/End；规则控件增加可访问名称。保存期间编辑区不可交互，并阻止关闭后打开新模板造成保存响应串扰。
|
||||
|
||||
没有测量生产 PostgreSQL 的耗时或内存；本轮 SQL 变更由生成 SQL/绑定参数断言验证，未执行真实 PostgreSQL 集成回归。极大数值指数的 SQL 浮点转换、Python 与 PostgreSQL 正则方言差异、全文 JSON 搜索和大型 ZIP 内存仍需单独基准。既有回退机制不等于所有 SQL/Python 语义已完全一致。
|
||||
|
||||
## 验证与预览
|
||||
|
||||
- 后端相关回归：**120 项通过，1 项跳过**。跳过的是需显式开启的百万行压力测试。覆盖模板校验、原子更新、旧记录执行保护、端口策略、过滤、数值、复合键、查询次数、删除保护、范围失效、源批次状态，以及既有对比、搜索、配置快照、导出、恢复与内存相关回归。
|
||||
- Node 测试：**14 项通过**（8 项模板模型、6 项缓存）。
|
||||
- 浏览器：**53 个场景通过**（原业务对比 27 项、模板新增 26 项）。全部 `/v1` 请求使用固定响应，120 个模板列表验证默认 50 行上限；覆盖导入导出、复杂过滤、端口策略、负容差、忽略字段恢复、保存期间交互锁定、范围修复、键盘和 390px 布局。
|
||||
- Vite 生产构建通过；原有主包超过 500 kB 提示仍在。
|
||||
- 全项目 TypeScript 检查仍有 `BizMigrationPage.tsx` 的 4 个既有错误：2 个未使用变量、2 处字符串传给数值控件；本轮没有新增错误。
|
||||
- 新模板模型 ESLint 通过；业务对比页保留原来的 **23 个错误**（17 个 any、6 个 effect 状态同步），与修改前数量一致。不宣称全项目 lint 通过。
|
||||
|
||||
前后端应一起更新；新增 API 字段为模板引用数量与端口策略，无数据库迁移。原来使用标量 `in/not_in`、未知规则、负容差的模板需要修正再保存/执行；部分完成的源批次需要重新完成采集后再对比。
|
||||
|
||||
- [桌面模板编辑器](assets/compare-template-editor.png)
|
||||
- [390px 模板编辑器](assets/compare-template-editor-mobile.png)
|
||||
- [业务对比列表](assets/biz-compare-jobs.png)
|
||||
- [此前业务对比记录](biz-compare-review-2026-10-10.md)
|
||||
|
||||
复现：先在 `web` 启动 `node node_modules/vite/bin/vite.js --host 127.0.0.1 --port 5179`，另一个终端运行 `node tests/bizCompareTemplates.browser.mjs` 和 `node tests/bizCompare.browser.mjs`。可用 `NETX_PLAYWRIGHT_MODULE` 指定已有 Playwright，`NETX_TEST_URL` 指定本地地址，`NETX_TEST_OUTPUT` 指定截图目录。
|
||||
|
|
@ -1,477 +0,0 @@
|
|||
# 割接对比（业务对比 / 割接监控）问题分析与优化计划
|
||||
|
||||
日期：2026-10-12。范围：网络管理 → 割接对比下的 5 个子模块：业务监控（采集）、业务对比、割接监控、对比模板、监控模板。
|
||||
用途：交给 Cursor 按任务逐项实施；Claude 负责验收。
|
||||
|
||||
本文基于代码阅读和本地脚本验证，没有连接真实设备，也没有在真实 PostgreSQL 上执行。标了 **[已验证]** 的问题已经用脚本复现（脚本放在会话 scratchpad，结论见各条目）。基线：开始修改前，相关后端回归 **279 项通过**，`tsc -b` 通过，`cutoverDataCache` / `compareTemplateModel` Node 测试通过。
|
||||
|
||||
---
|
||||
|
||||
## 0. 结论摘要
|
||||
|
||||
1. **两种模式共用一套“归一化 + 映射”概念，但各个代码路径的实现不一致。** 割接监控会先对端口映射的键和值做接口归一化，业务对比不会（**[已验证]**：按设备原始名称填写映射，业务对比全部变成“删除 + 新增”）。行过滤用原始值，映射作用在归一化之后。键字段不受 `field_rules` 归一化影响。跨厂商割接最容易在这里产生误报。
|
||||
2. **目前只有两种映射：接口类型别名（模板级，两侧同一套规则）和端口映射（before→after，一对一）。** 没有分侧（按厂商）归一化，没有字段值映射（状态枚举、对端 IP、VRF 改名），也无法提前检查映射覆盖率。映射漏配会直接表现为业务“缺失”，用户很难判断是映射问题还是业务问题。
|
||||
3. **除 LLDP 外，状态采集解析只实现了中兴。** 华为、思科等的 parser 都是会抛出 `NotImplementedError` 的空壳，且没有注册。内置对比模板（`ZTE status default`）里的过滤和容差也都按中兴设定。所以现在的“跨厂商映射”只对 LLDP 真正有效，其余检查项在跨厂商场景下要么采集不到，要么全量误报。
|
||||
4. **判定语义有几处会把正常割接判成失败：**
|
||||
- 端口预设要求旧端口 `admin=down 且 phy=down`。现场常见的做法是拔纤但不 shutdown，这种情况验收时会判为“未完成/红”（**[已验证]**）。
|
||||
- 两侧都是 0 行时通过率是 100%（空对空误判为通过）。
|
||||
- 解析不到数据会被当成业务缺失。
|
||||
- 新旧两侧的采集命令集不一致时，会被当成业务丢失。
|
||||
- 内置模板对 `output_bps` 用绝对值容差 5 bps，对 `rx_power`（dBm）用百分比容差。
|
||||
5. **割接监控的运行数据无限增长。** 每个 `BizMigrationRun` 或 `BizCompareRun` 引用的采集批次都会被永久保护，run 和 diff 本身也没有保留策略。60 秒一轮的高频监控跑几天，数据库和磁盘会持续膨胀。
|
||||
6. **性能：** 业务对比只要配置了端口映射，所有检查项（包括不含接口字段的 BGP 路由百万行表）都会退回 Python 引擎。割接验收是同步 HTTP 请求，会把路由表全量载入内存。
|
||||
7. **引用保护有漏洞：** 被监控模板引用的对比模板、被任务或割接项目引用的端口映射都能被删除。删除后，业务对比会静默地按“无映射”执行；割接监控会直接报错停摆。
|
||||
|
||||
---
|
||||
|
||||
## 1. 现状框架梳理（给 Cursor 的背景）
|
||||
|
||||
### 1.1 数据流
|
||||
|
||||
```
|
||||
业务监控任务(biz_state_task, 每台设备)
|
||||
└─ 采集批次 batch ──> 每条命令 batch_command(parse_status/row_count/raw_line_count)
|
||||
└─ 解析(ParseProfile → parser → FieldDef) ──> biz_state_metric_row(data_json) / biz_state_lldp_neighbor
|
||||
│
|
||||
对比模板(biz_compare_template.metrics_json = 多个 sheet)
|
||||
sheet: metric_id, key_fields, iface_fields, compare_fields, display_fields,
|
||||
row_filters, field_rules(normalize/compare/tolerance), ignore_port_changes
|
||||
模板级: iface_normalize_json（接口类型别名，两侧同一套）
|
||||
│
|
||||
┌───────────── 业务对比（BizCompareJob/Run/Diff）─────────────┐
|
||||
│ before 批次 vs after 批次（两台设备或同一设备前后） │
|
||||
│ 行过滤(原始值) → 接口归一化(两侧) → 端口映射(仅 before) │
|
||||
│ → 按 match_key 分组 → 同键按采集顺序 zip 配对 │
|
||||
│ → added/removed/changed/unchanged → 通过率 │
|
||||
│ 引擎：PG + 可下推 → SQL FULL OUTER JOIN；否则 Python │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
┌───────────── 割接监控（BizMigrationProject/Batch/Run/Diff）──┐
|
||||
│ 旧设备: 画像基线 vs 高频当前 → old_kind │
|
||||
│ 新设备: 画像基线 vs 高频当前 → new_kind │
|
||||
│ 预期集(expect_set: 本批要迁移的端口/对象) + 监控模板(成功/异常规则)│
|
||||
│ → dual_verdict: migrated/migrating/anomaly/lost/unfinished… │
|
||||
│ → 本批完成(finish_batch)=终验 → 红单(带红继续) │
|
||||
│ 高频采集：每个 metric 一对(新/旧)任务，同轮次配对采样 │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 1.2 现有“映射/归一化”层与使用点（不一致的根源）
|
||||
|
||||
| 层 | 归属 | 作用对象 | 业务对比 | 业务对比·实时检索/回填/导出 | 割接监控 |
|
||||
|---|---|---|---|---|---|
|
||||
| 接口类型别名 `iface_normalize_json` | 对比模板 | `iface_fields`，两侧同规则，只替换前缀 | 有 | 有（`_run_transform_config`） | 有 |
|
||||
| 端口映射 `biz_port_mapping` | 任务 / 项目 | `iface_fields`，before→after，精确匹配 + 父接口继承 | **键/值不归一化** | 同左 | **键/值先归一化**（`service.py:1019`） |
|
||||
| 行过滤 `row_filters` | sheet | **原始值**（归一化之前） | 有（PG 可下推） | 有 | 有（Python） |
|
||||
| 字段规则 `field_rules.normalize` | sheet | **仅 compare 字段**，键字段不处理 | 有 | 有 | 仅完整性校验 |
|
||||
| 监控模板 `sheet_overrides` | 监控模板 | 状态语义（up/down 值、成功/异常模式） | — | — | 有 |
|
||||
|
||||
关键文件：
|
||||
- 引擎：`netx_api/biz_state/compare_engine.py`、`compare_rules.py`、`iface_normalize.py`、`compare_sql.py`
|
||||
- 业务对比服务：`netx_api/biz_state/compare_service.py`（约 4000 行）
|
||||
- 割接监控：`netx_api/biz_migration/evaluate.py`、`service.py`、`monitor_templates.py`、`auto_monitor.py`
|
||||
- 采集：`netx_api/biz_state/collect_runner.py`、`claim.py`、`profiles.py`、`parsers/*`
|
||||
- 保留：`netx_api/biz_state/retention.py`
|
||||
- 前端：`web/src/features/cutover/BizComparePage.tsx`、`BizMigrationPage.tsx`、`BizMonitorTemplatesPage.tsx`、`BizStatePage.tsx`
|
||||
- 内置模板：`netx_api/biz_state/data/default_zte_status_template.json`
|
||||
|
||||
---
|
||||
|
||||
## 2. 问题清单
|
||||
|
||||
严重度：**P0** 会产生错误结论或数据损坏；**P1** 高概率误报或明显性能/可用性问题；**P2** 边界问题或体验问题；**P3** 清理项。
|
||||
|
||||
### 2.1 业务对比（BizCompare）
|
||||
|
||||
| ID | 严重度 | 问题 | 位置 | 影响 |
|
||||
|---|---|---|---|---|
|
||||
| B-01 | P0 **[已验证]** | 端口映射的键/值不按模板接口归一化规则转换，必须由用户手工填写“归一化后”的名称；割接监控却会自动归一化。两种模式行为不一致。 | `compare_engine.py:224-260`（`apply_port_map` 用原始 `pmap`），`compare_service.py:2461`、`_run_transform_config:2337` | 模板有 `GE→gei` 规则时，映射 `GE0/1/0/1→gei0/2/0/1` 完全不命中，结果 added=1 / removed=1、`map_ok=False` |
|
||||
| B-02 | P1 | 任务里只要有端口映射，所有 sheet 都会跳过 SQL 引擎，包括没有接口字段的 BGP/路由表 | `compare_sql.py:165`（`if port_map: return "port_map"`） | 百万行 vpnv4 路由退回 Python，内存和耗时都大幅上升；割接任务几乎都会配映射 |
|
||||
| B-03 | P1 | 端口映射被删除后，业务对比静默地按“无映射”执行（还会触发自动忽略端口的启发式），删除时也不检查引用 | `delete_mapping:1603`，`_port_map_dict:1612`，`_validate_compare_job:2220` | 结果语义悄悄改变 |
|
||||
| B-04 | P1 | 对比模板删除只检查 `BizCompareJob` 的引用，不检查 `BizMonitorTemplate.compare_template_id` | `delete_template:1521`，`list_templates:1391` 的 `job_count` | 割接窗口中删掉模板后，`run_evaluate` 报 `compare_template_missing`，监控停摆 |
|
||||
| B-05 | P1 | 创建向导或任务配置里编辑了映射文本但没点“保存映射”，创建/保存/执行任务时映射文本被丢弃，没有任何提示 | `BizComparePage.tsx:1814 jobConfigBody`、`1895 createJob`、`1997 saveJobConfig`、`2029 runNow` | 用户以为配了映射，实际没生效，产生大量假“缺失/新增” |
|
||||
| B-06 | P2 | 映射文本解析只认 `, | Tab` 作分隔符；空格分隔或格式不对的行被静默丢弃，没有解析计数或错误提示 | `BizComparePage.tsx:1720 parseMapRows` | 映射少行而不自知 |
|
||||
| B-07 | P1 | 映射校验只针对“第一个含接口字段的 sheet”，结果以原始 JSON 展示；只报告“映射键未命中”，不报告“before 侧端口没有映射且在 after 侧不存在”（漏配） | `validate_mapping:1810`，`mapping_stats`（`compare_engine.py:100`），`BizComparePage.tsx:2272` | 漏配映射最终表现为业务缺失 |
|
||||
| B-08 | P1 | `row_filters` 作用在原始行（归一化、映射之前）；跨厂商时，同一条过滤在两侧含义不同 | `_load_metric_rows:1619`（SQL/Python 都在加载时过滤），`_run_sheet:2179` | 例如 `interface contains gei` 会把华为 `GE0/0/1` 整侧滤掉；ARP 的 `age_timer`（HH:MM:SS）只适用于中兴 |
|
||||
| B-09 | P2 | 键字段不应用 `field_rules.normalize`（如 MAC 格式），跨厂商的 MAC/大小写差异导致配对失败 | `compare_engine.py:96 row_key`，`compare_sql.py _rk_sql` | 键里含 MAC 的 sheet（如 `l2vpn_mac`）全量误报 |
|
||||
| B-10 | P1 | 两侧都是 0 行时通过率为 100%；解析不到数据（命令执行了但解析 0 行）被当成业务“缺失” | `_enrich_summary:3015`，`compute_pass_rate:210` | 空对空误判为通过；模板不匹配被当成业务丢失 |
|
||||
| B-11 | P1 | 新旧两侧采集命令集不一致（例如旧设备采了 bgp ipv4 summary，新设备没采），对比结果是“全部缺失” | 没有任何检查 | 采集配置问题被误判为业务问题 |
|
||||
| B-12 | P2 | 同键多行按采集顺序 zip 配对，CLI 输出顺序变化会产生假“不一致” | `compare_engine.py:331+`、`compare_sql.py` 中 `dup_rn ORDER BY seq` | ECMP/多路径场景误报 |
|
||||
| B-13 | P2 | 只要提供了映射，`ignore_port_changes` 就被忽略；部分映射时，未映射端口既不按名称匹配也不忽略端口 | `compare_engine.py:224` | 部分映射时策略不可控 |
|
||||
| B-14 | P2 | 结果表的键列只显示映射后的名称，看不到 before 侧原始端口名 | `BizComparePage.tsx:4161`（`d.key[k] ?? pre[k]`，`pre` 取 `mapped_before`） | 跨厂商时现场人员无法对照原端口 |
|
||||
| B-15 | P2 | 判定没有方向：状态 down→up、前缀数增加也算失败；流量计数器被当成严格比较项 | `compare_rules.values_equal` | 割接后业务恢复或流量迁入被判为失败 |
|
||||
| B-16 | P3 | `batch_metric_collect_ok` 是死代码；`_load_metric_rows` 中 LLDP 分支重复过滤；PG 下推时 `before_raw_count` 实际上是过滤后的数量 | `compare_service.py:78`、`1675-1688`、`2199` | 清理 |
|
||||
| B-17 | P3 | 防止同一任务并发对比的锁是进程内的；`_job_has_active_run` 检查和插入不是原子操作。biz_state worker 多副本时，自动对比有极小概率并发 | `compare_service.py:57`、`2887` | 低概率 |
|
||||
|
||||
### 2.2 割接监控（BizMigration）
|
||||
|
||||
| ID | 严重度 | 问题 | 位置 | 影响 |
|
||||
|---|---|---|---|---|
|
||||
| M-01 | P0 **[已验证]** | 端口预设的成功条件要求旧侧 `removed` 或 `admin=down 且 phy=down`。旧端口拔纤但没有 shutdown（admin up / phy down）、新端口已 up 时：窗口内是 `migrating/黄`，**验收时是 `unfinished/红`** | `monitor_templates.py:255-275`（`_success_stateful` 的 `old_down_groups` 用的是 AND） | 正常割接被判为验收失败 |
|
||||
| M-02 | P1 | 每个 run 引用的 HF 批次都会被永久保护，run 和 diff 本身也没有保留策略 | `retention.py:65-75` | 高频监控期间数据库无限增长 |
|
||||
| M-03 | P1 | 验收（`finish_batch`）和手动评估是同步 HTTP 请求；路由类 sheet（`bgp_route`/`ip_route`）会把 4 个批次的全量行载入 Python | `biz_migration_router.py` 的 evaluate/finish，`service.py:1148-1163` | 超时、内存暴涨；而且这期间一直持有项目行锁（`with_for_update`，`service.py:940`） |
|
||||
| M-04 | P1 | 默认监控模板会采集对比模板里的全部 metric，包括只能手动采集的路由表。没有手动采集时，该 sheet 是 `current_missing`，验收**必然**失败（`current_missing` / `coverage_incomplete`） | `resolve_collect_metric_ids:525`、`finish_batch:2358-2371`、`MANUAL_ROUTE_METRICS` | 默认配置跑不通验收 |
|
||||
| M-05 | P1 | 只要任意 sheet 有重复键，验收就判失败（`duplicate_keys`，同时 `coverage_complete=False`），不管这个 sheet 是否在本批范围内 | `service.py:1299`、`2366` | 例如 ISIS L1/L2 两行同键，就永远过不了验收 |
|
||||
| M-06 | P1 | 新旧两侧画像任务的命令集可能不同，HF 任务沿用各自画像的命令项，旧侧有、新侧没采的切片会被判为 `lost` | `_portrait_metric_items:1741`、`ensure_highfreq:1919` | 采集配置问题被当成业务丢失 |
|
||||
| M-07 | P1 | 创建项目时不校验基线批次是否属于该侧画像任务（patch 时会校验） | `create_project:686-691`（对比 `patch_project:744-750`） | 可能拿错设备的基线；直到评估时才报 `baseline_task_mismatch` |
|
||||
| M-08 | P2 | 不支持同设备割接（板卡更换、同框端口迁移）：`collect_pair_same_device` 直接报错 | `service.py:2016` | 常见场景无法使用 |
|
||||
| M-09 | P2 | 每个 metric、每台设备一个 HF 任务；15 个检查项就是每分钟每台设备约 15 次 SSH 登录；没有单设备并发上限 | `ensure_highfreq:1943`，`claim.py` | VTY 耗尽、AAA 日志刷屏，进而采集失败，看板大量 `current_missing` |
|
||||
| M-10 | P2 | 每轮自动评估都重新载入基线全量行（基线不会变） | `run_evaluate.metric_rows:1059` | 白白消耗 CPU/IO |
|
||||
| M-11 | P2 | 双侧评估里，诊断索引的键串用 `str(v)` 构造，而其他地方用 `scalar_text().strip()`：`None` 变成 `"None"`，`True` 变成 `"True"`，空白不裁剪，预期键匹配不上 | `evaluate.py:759`、`compare_engine.py:273 _key_obj` | **[已验证]**：键字段为 `None` 时，存在且未变化的对象被判为 `unfinished` |
|
||||
| M-12 | P2 | 用 metric 名称包含 `bgp` / `pw` 来决定是否用端口映射推导范围（字符串启发式） | `evaluate.py:804` | 新增 metric 时行为不可预期，也违反“引擎不写 metric 特例”的原则 |
|
||||
| M-13 | P2 | 映射允许 N:1 吗？业务对比允许（结果是重复键），割接监控直接报 `mapping_target_ambiguous`；1:N（拆分 breakout）两边都不支持（`before_if` 唯一） | `models/biz_state.py BizPortMappingRow`，`service.py:991/1021` | 端口汇聚、拆分割接无法表达，报错也没说明是哪几行 |
|
||||
| M-14 | P3 | `project_to_dict`（GET/列表）里会写库（`resolve_project_monitor_template` 持久化默认模板）；`list_projects` 存在 N+1 | `service.py:486-503`、`543` | 只读接口有副作用 |
|
||||
| M-15 | P3 | 一个键在后续验收中已经变绿，对应红单仍然保持 open，只能人工关闭 | `_persist_red_tickets_from_run:2326` | 红单堆积 |
|
||||
|
||||
### 2.3 内置模板与跨厂商能力
|
||||
|
||||
| ID | 严重度 | 问题 | 位置 |
|
||||
|---|---|---|---|
|
||||
| T-01 | P1 | `interface_detail.output_bps` 用 `numeric` 绝对容差 5（bps），而 `input_bps` 用 5%：相当于对出方向流量做严格比较 | `default_zte_status_template.json` |
|
||||
| T-02 | P1 | `optical_brief.rx_power` 用 `percent 5`：dBm 是对数单位，-0.5 和 -0.6 已经差 20%。应改为绝对值 dB 容差 | 同上 |
|
||||
| T-03 | P2 | 流量计数器（bps/util）默认参与比较；割接后流量迁移是预期行为，会产生大量噪声 | 同上 |
|
||||
| T-04 | P2 | `l2vpn_mac` 键里的 `ac_port` 是接口，但没放进 `iface_fields`，端口映射不生效 | 同上 |
|
||||
| T-05 | P1 | 华为、思科、华三、Juniper、诺基亚、爱立信的状态 parser 都是空壳（没有注册）；只有 LLDP 跨厂商可用 | `parsers/huawei/__init__.py` 等 |
|
||||
| T-06 | P2 | 各厂商的状态值不统一（`Established`/`ESTAB`、`UP`/`up`/`*down`、`Full/DR`），而对比层只有 `lower`/`upper`/`mac` 几种归一化，没有枚举别名映射 | `compare_rules.normalize_value` |
|
||||
|
||||
---
|
||||
|
||||
## 3. 优化方案（分阶段任务，按顺序执行）
|
||||
|
||||
### 通用执行约束（Cursor 每个任务都要遵守）
|
||||
|
||||
1. **引擎里不写 metric 特例**：所有 metric 相关的行为放进模板、sheet 或 profile 声明（遵循 `skills/biz-state/netx-biz-state/SKILL.md` 第 4、5 节）。
|
||||
2. **Python 和 SQL 必须一致**：凡是影响配对或判定的改动，要么两个引擎都实现，要么在 `sql_compare_skip_reason` 里让 SQL 自动降级到 Python，并补“同输入两引擎结果一致”的对照测试（SQL 侧至少断言生成的 SQL 和降级原因）。
|
||||
3. **历史 run 不变**：新增的执行配置写进 `config_snapshot`，并把 `version` 升到 2。`version==1` 和旧记录的回填、检索、导出保持原行为，不能因为新逻辑改变历史结果。
|
||||
4. **数据库结构**：尽量用 JSON 字段扩展，不加列。确实需要加列或表时，同时补 `alembic/versions/` 和 `netx_api/schema_patches.py`（参照现有做法），并写回归测试。
|
||||
5. **每个任务**：补单元测试；跑第 5 节的回归命令；前端改动跑 `tsc -b` 和 `vite build`，涉及交互的补 `web/tests/*.browser.mjs` 场景（全部 `/v1` 用固定响应）。
|
||||
6. 一个任务一个提交，提交信息写清楚任务 ID（例如 `fix(cutover): B-01 normalize port-map keys`）。
|
||||
|
||||
---
|
||||
|
||||
### 阶段 A：正确性快修（P0/P1，改动小，先做）
|
||||
|
||||
#### A1 端口映射键/值统一归一化（B-01）
|
||||
- 新增函数 `normalize_port_map(port_map, rules) -> (normalized_map, conflicts)`，放在 `iface_normalize.py`：键和值都用 `normalize_iface_name` 处理。归一化后键冲突（两个原始键归一后相同但目标不同）要作为 conflicts 返回。
|
||||
- 业务对比：在 `_run_transform_config`（`compare_service.py:2337`）统一处理，覆盖执行、实时检索、回填、导出所有路径。**只对 `config_snapshot.version>=2` 的 run 生效**；`version==1` 和旧记录保持原样。
|
||||
- 割接监控：`run_evaluate` 和 `list_baseline_expect_objects` 里的内联归一化改为调用这个函数，消除重复实现。
|
||||
- `mapping_stats` 的 `hint` 文案去掉“必须填写归一化后名称”。
|
||||
- **验收测试**：模板规则 `GE→gei`、映射写原始名 `GE0/1/0/1→gei0/2/0/1`，新 run 的结果是 `unchanged=1`；同样的数据在 `version=1` 的 run 上回填，结果不变。
|
||||
|
||||
#### A2 引用保护和缺失检测（B-03、B-04）
|
||||
- `delete_mapping`：被 `BizCompareJob.mapping_id` 或 `BizMigrationProject.mapping_id` 引用时返回 409 `mapping_in_use`，detail 里带引用列表（任务名/项目名）。
|
||||
- `delete_template`：同时检查 `BizMonitorTemplate.compare_template_id`，返回 409 `template_in_use`（detail 区分 job/monitor）。`list_templates` 增加 `monitor_template_count`。
|
||||
- `_validate_compare_job`：`job.mapping_id` 非空但映射不存在时返回 400 `mapping_missing`。
|
||||
- 前端：删除失败时显示引用对象；模板列表显示两类引用数。
|
||||
- **验收测试**：上述 3 种 409/400 的单测；被引用的映射无法删除。
|
||||
|
||||
#### A3 映射文本不会丢失（B-05、B-06）
|
||||
- `parseMapRows` 改为返回 `{rows, invalidLines[]}`：两段文本即使用空格分隔也接受（正好两个 token 时）；重复的 before 行要提示。
|
||||
- 映射文本相对于已保存版本“有修改”时：创建任务、保存配置、立即执行都要先拦截，提供“保存映射并继续 / 放弃修改”两个选项（或者自动先保存映射再保存任务，二选一，推荐自动保存并提示）。
|
||||
- 编辑区下方实时显示“已解析 N 对 / 无效 M 行（行号）”。
|
||||
- 保存一个被其他任务或项目共用的映射时，提示“该映射被 X 个任务 / Y 个割接项目使用”（后端 `list_mappings` 返回引用计数）。
|
||||
- **验收（浏览器脚本）**：在向导里输入映射后直接点“创建”，请求体里的 `mapping_id` 非空，或者被拦截并出现提示；空格分隔的行能被解析；无效行有提示。
|
||||
|
||||
#### A4 端口预设成功条件修正（M-01）
|
||||
- `preset_override_for_metric("interface_brief")` 中旧侧的条件改为三个 OR 组：`[removed]`、`[phy in down]`、`[admin in down]`；新侧保持 `admin up 且 phy up`。
|
||||
- 已存在的监控模板：在 `ensure_default_monitor_templates` 中增加一个幂等升级，**只替换结构上和旧预设完全一致的 override**（逐字段对比），用户自定义过的不动，并在 `override_warnings` 里提示“端口成功条件为旧版 AND 语义”。
|
||||
- **验收测试**：新增 4 个用例——旧侧 admin up/phy down、admin down/phy up、两者都 down、端口已删除；在窗口内和验收时，新侧 up 都应判为 `migrated/green`。另外，新侧 phy down 时仍判为红。
|
||||
|
||||
#### A5 创建项目时校验基线归属（M-07）
|
||||
- `create_project` 复用 `patch_project` 的校验：基线批次的 `task_id` 必须等于对应侧的画像任务，状态必须是 `success` 或 `partial`。抽成公共函数 `_validate_baseline_ownership`。
|
||||
- **验收测试**：拿其他设备的批次作为基线创建项目，返回 400 `old_baseline_invalid` / `new_baseline_invalid`。
|
||||
|
||||
#### A6 键串规范化统一（M-11）
|
||||
- 在 `compare_engine.py` 新增 `key_tuple(row, fields)`，统一用 `scalar_text(...).strip()`。`_key_obj` 的输出键值和 `build_diff_index_from_compare` 都改用它（`evaluate.py:759` 不再用 `str(v)`）。
|
||||
- **验收测试**：键字段为 `None`、`True`、带首尾空格时，预期键能命中，判定为 `ok`/`migrated` 而不是 `unfinished`。
|
||||
|
||||
#### A7 内置模板参数修正（T-01、T-02、T-04）
|
||||
- `default_zte_status_template.json`：
|
||||
- `output_bps` 改为 `percent 5`（和 `input_bps` 对称）。
|
||||
- `rx_power` 改为 `numeric`，`tolerance: 2`（单位 dB）。
|
||||
- `l2vpn_mac.iface_fields` 加上 `ac_port`。
|
||||
- `interface_detail` 的 `input_bps/output_bps/in_util/out_util` 移出 `compare_fields`，只保留在 `display_fields`（见 V3 的方向性规则；在 V3 完成之前先不比较）。
|
||||
- `ensure_default_zte_status_template` 增量升级：**只修改值仍等于旧打包值的字段**，用户改过的保留。
|
||||
- **验收测试**：旧打包值的模板升级后变成新值；用户自定义过 `rx_power` 规则的模板升级后保持不变。
|
||||
|
||||
---
|
||||
|
||||
### 阶段 B：统一的“身份归一化流水线”（跨厂商核心）
|
||||
|
||||
目标：业务对比执行、实时检索、回填、导出、映射校验、割接评估、预期集选择器这 7 条路径，**对同一行算出相同的 match key**，并支持按侧（按厂商）的规则和字段值映射。
|
||||
|
||||
#### B1 抽出 `IdentityPipeline`（解决“各路径各写一遍”）
|
||||
- 新文件 `netx_api/biz_state/identity_pipeline.py`：
|
||||
```python
|
||||
@dataclass(frozen=True)
|
||||
class SideSpec:
|
||||
iface_rules: list[dict] # 该侧接口归一化（已合并 both + side）
|
||||
value_maps: dict[str, dict] # field -> {from_lower: to}
|
||||
@dataclass(frozen=True)
|
||||
class Pipeline:
|
||||
before: SideSpec; after: SideSpec
|
||||
port_map: dict[str, str] # 已归一化（A1）
|
||||
key_fields, iface_fields, compare_fields, field_rules, row_filters, match_keys
|
||||
def transform(self, row, side) -> dict # 归一化 → 值映射 →（before 侧）端口映射
|
||||
def match_key(self, row, side) -> tuple
|
||||
def keep(self, row, side) -> bool # 行过滤（在 transform 之后求值，见 B4）
|
||||
```
|
||||
- 由 `build_pipeline(sheet, snapshot_or_template, port_map, before_vendor, after_vendor)` 统一构造；`compare_rows`、`_live_search_sheet_diffs`、`_load_search_key_groups`、`_hydrate_diff_rows`、`validate_mapping`、`evaluate_metric_dual`、`list_baseline_expect_objects` 全部改用它。
|
||||
- **验收测试**：参数化对照测试，用同一份 fixture 断言 7 条路径对同一行给出的 match key 相同（重点覆盖：映射、父接口继承、QinQ、值映射、归一化规则）。
|
||||
|
||||
#### B2 分侧接口归一化 + 厂商内置规范化
|
||||
- `iface_normalize_rules` 每条增加可选字段 `side: "both"|"before"|"after"`（默认 `both`）和 `vendor`（可选，按批次的厂商自动生效）。前端模板编辑器支持选择。
|
||||
- 内置“厂商内全称/缩写”规范化表（`iface_normalize.py` 中的 `VENDOR_CANONICAL`）：
|
||||
- 华为：`GigabitEthernet`↔`GE`、`XGigabitEthernet`↔`XGE`、`100GE`、`Eth-Trunk`
|
||||
- 思科：`GigabitEthernet`↔`Gi`、`TenGigE`↔`Te`、`HundredGigE`↔`Hu`、`Bundle-Ether`↔`BE`
|
||||
- 华三：`GigabitEthernet`↔`GE`、`Ten-GigabitEthernet`↔`XGE`、`Bridge-Aggregation`↔`BAGG`
|
||||
- 中兴：大小写
|
||||
- 统一去掉名称中的空格。
|
||||
|
||||
只做厂商内同义统一，**不做跨厂商编号转换**（那是端口映射的职责）。模板 sheet 可以用 `builtin_vendor_canonical: false` 关闭。
|
||||
- 批次的厂商取 `BizStateBatch.vendor`，取不到再用任务的 vendor，经 `resolve_vendor_key` 解析。
|
||||
- `config_snapshot` v2 记录两侧最终生效的规则。
|
||||
- **验收测试**：华为 `GigabitEthernet0/0/1` 与 `GE0/0/1` 互认；before 华为、after 中兴，只对 before 生效的规则不影响 after。
|
||||
|
||||
#### B3 字段值映射（T-06、B-09）
|
||||
- `field_rules[]` 新增可选字段：
|
||||
- `value_map: {"estab": "established", "*down": "down", ...}`（大小写不敏感匹配，输出规范值）
|
||||
- `side: both|before|after`
|
||||
- `apply_to_key: true|false`（默认：字段在 key_fields 中时为 true）
|
||||
- `normalize` 的处理顺序：`normalize`（strip/lower/mac…）→ `value_map`。**键字段也要应用**（解决 MAC 作为键时跨厂商对不上的问题）。
|
||||
- SQL：`value_map` 作用在键或比较字段上时，`sql_compare_skip_reason` 返回 `value_map`，降级到 Python（以后可以用 `CASE` 下推，本期不做）。
|
||||
- `compare_validation.py` 校验 `value_map` 必须是 `str→str` 映射，并且每个 sheet 最多 500 项。
|
||||
- 模板编辑器：字段规则行增加“值映射”编辑（每行一对，可以粘贴）。
|
||||
- **验收测试**：`state: Estab` 对 `established`，配置值映射后为 unchanged；MAC 作为键字段时，`00e0-fc12-3456` 与 `00e0.fc12.3456` 能够配对；Python 与 SQL 降级原因断言。
|
||||
|
||||
#### B4 行过滤在归一化之后求值（B-08）
|
||||
- 新 run（snapshot v2）：先 `transform` 再 `row_filters`。
|
||||
- PG 下推只在“过滤涉及的字段都不在 `iface_fields` 中、也没有值映射”时启用；否则 Python 过滤。
|
||||
- 支持过滤条件带 `side`（只作用于某一侧），满足“旧侧只看 dynamic ARP”这类需求。
|
||||
- 模板编辑器提示“过滤在归一化之后执行，请按规范值书写”。
|
||||
- **验收测试**：规则 `GE→gei` 加过滤 `interface contains gei`，before 侧华为 `GE0/0/1` 不再被滤掉；`version=1` 的 run 回放结果不变。
|
||||
|
||||
#### B5 端口映射覆盖率预检（B-07、B-11、M-06 的基础）
|
||||
- 新接口：`POST /v1/biz-state/compare/preflight`，请求体 `{template_id, enabled_sheet_ids, mapping_id | mapping_rows, before_batch_id, after_batch_id}`。割接监控复用同一个服务函数：`/v1/biz-migration/projects/{id}/preflight`，传入基线和最新 HF 批次。
|
||||
- 按 sheet 返回：
|
||||
- `before_count / after_count`（过滤后）、`parse` 状态（见 V1）
|
||||
- `mapped_exact[]`、`mapped_by_parent[]`、`identity_same_name[]`（未映射但两侧同名）
|
||||
- **`suspect_unmapped[]`**：before 侧接口没有映射，after 侧也不存在 → 疑似漏配
|
||||
- `unused_map_keys[]`、`map_targets_missing_on_after[]`、`n_to_1_targets[]`
|
||||
- `command_coverage`：两侧执行过的 profile_id 集合和差集（见 V2）
|
||||
- 汇总：映射覆盖率 = 已映射 / (已映射 + 疑似漏配)，以及“可执行 / 有风险”的结论。
|
||||
- 业务对比 run 的 summary 也写入同结构的 `mapping_coverage`（按 sheet），结果页可以查看。
|
||||
- 前端：映射块里的“校验”按钮改为显示预检表格（取代原始 JSON）；疑似漏配可以一键加入映射编辑区（目标留空，待填）。
|
||||
- **验收测试**：构造 3 个端口，分别是精确映射、父接口继承、漏配，预检分类正确；`mapping_rows` 未保存时也能预检。
|
||||
|
||||
#### B6 SQL 引擎条件放宽（B-02）
|
||||
- `sql_compare_skip_reason`：`port_map` 只在 `iface_fields ∩ (key_fields ∪ compare_fields)` 非空时才阻止 SQL。
|
||||
- **验收测试**：有端口映射的任务，`bgp_route` sheet（`iface_fields=[]`）走 SQL（断言 skip reason 为空），`interface_brief` 仍走 Python。
|
||||
|
||||
#### B7 映射语义：N:1 与部分映射（B-13、M-13）
|
||||
- 决策见第 6 节，默认方案：
|
||||
- **N:1**：允许，但预检和 summary 中必须列出；业务对比按重复键规则处理（V4）；割接监控继续拒绝，但错误信息里列出冲突行。
|
||||
- **1:N（breakout）**：本期不支持，在映射编辑器里检测到重复 before 时明确报错，不再只报 `duplicate_before_if`。
|
||||
- **部分映射**：新增 sheet 选项 `unmapped_port_policy: "by_name"|"ignore_port"`（默认 `by_name`，即当前行为），让 `ignore_port_changes` 与映射可以组合使用。
|
||||
- **验收测试**：N:1 在预检中出现在 `n_to_1_targets`；割接监控的报错包含冲突对。
|
||||
|
||||
---
|
||||
|
||||
### 阶段 C：判定语义与数据可信度
|
||||
|
||||
#### V1 sheet 级数据可信度（B-10）
|
||||
- `_run_sheet` 和割接评估为每侧补充 `collect_quality`：该 metric 的命令数、`parse_status` 计数、`row_count`、`raw_line_count`、`declared_total`（只读这些列，不读 `raw_text`）。
|
||||
- 新增 sheet 状态：
|
||||
- `no_data`：两侧过滤后都是 0 行，且命令都执行成功 → `pass_rate=null`，不计入总通过率，卡片显示“无数据”。
|
||||
- `suspect_parse`：某侧 `raw_line_count > 5` 但 `row_count == 0` → 卡片显示“疑似解析失败”，失败行数照常显示，但给出明确提示。
|
||||
- `not_collected`：某侧没有任何命令产生该 metric → 不做判定，`pass_rate=null`。
|
||||
- 任务级 `ok`：任一 sheet 处于 `not_collected` / `suspect_parse` 时为 `null`（未判定），不再是 `true`。
|
||||
- 前端卡片和 CSV `_summary.csv` 增加状态列。
|
||||
- **验收测试**：0 对 0 → `no_data`；before 1000 行、after 有原文但 0 行 → `suspect_parse`；after 侧无命令 → `not_collected`。
|
||||
|
||||
#### V2 采集命令集对齐（B-11、M-06）
|
||||
- `ParseProfile` 增加可选字段 `slice: dict[str, str]`（例如 `zte.bgp_ipv4_summary` 的 slice 是 `{"afi": "ipv4"}`），声明这个 profile 产出哪个切片。先给中兴的 `bgp_peer` / `bgp_route` / `vrrp` / `isis` profile 补齐。
|
||||
- 判断一个 sheet 是否“被某侧覆盖”：该侧存在 `metric_id` 相同的成功命令，且（sheet 的 row_filters 里有 `eq` 条件时）存在 slice 与之相符的 profile。
|
||||
- 业务对比的预检（B5）和执行时都做这个检查：未覆盖的 sheet 标记 `not_collected`（V1）。
|
||||
- 割接监控：`ensure_highfreq` 生成 HF 任务项时，取**两侧画像命令项的并集**（按 profile 的 slice 转换成该侧厂商可用的 profile；找不到时返回明确错误 `no_profile_for_slice:<metric>:<slice>:<vendor>`）。评估时，未覆盖的 sheet 标记为 `not_collected`，不判红。
|
||||
- **验收测试**：旧画像有 ipv4 + vpnv4，新画像只有 vpnv4 → HF 任务两侧都有 ipv4 + vpnv4；如果强制只有一侧有，评估结果是 `not_collected`，而不是 `lost`。
|
||||
|
||||
#### V3 方向性判定（B-15、T-03）
|
||||
- `field_rules.compare` 新增：
|
||||
- `not_worse`：配合 `rank: ["up", "down"]`（排在前面表示更好）；after 不差于 before 即通过。
|
||||
- `numeric_min`：配合 `tolerance`（百分比），`after >= before*(1-tol%)` 即通过（适用于前缀数、流量“不应明显下降”）。
|
||||
- `numeric_max`：对称方向。
|
||||
- `explain_diff` 输出方向原因，例如 `worse: up→down`、`drop 37% > 10%`。
|
||||
- SQL：先降级（skip reason 为 `directional_compare`），后续再下推。
|
||||
- 内置模板：`interface_brief.prot`、`bgp_peer.state` 默认仍为 `eq`（保守）；在模板编辑器里提供这些新模式供选择；流量计数器若要比较，推荐用 `numeric_min`。
|
||||
- **验收测试**：`down→up` 用 `not_worse` 判为通过，`up→down` 判为失败；前缀数 1000→1200 用 `numeric_min 5%` 判为通过，1000→900 判为失败。
|
||||
|
||||
#### V4 重复键配对：先排序再配对（B-12）
|
||||
- Python：同一个 match key 下，两侧各自按“归一化后的 compare 字段元组, 原始 seq”排序，然后 zip。
|
||||
- SQL：`dup_rn` 改为 `row_number() OVER (PARTITION BY rk ORDER BY <compare 字段归一化表达式...>, seq, id)`。
|
||||
- 用 sheet 选项 `duplicate_pairing: "sorted"|"ordered"` 控制，默认 `sorted`，只对 snapshot v2 生效；v1 回放保持 `ordered`。
|
||||
- **验收测试**：同键两行的值 `[a, b]` 对 `[b, a]` → 2 个 unchanged（原来是 2 个 changed）；Python 和 SQL 生成 SQL 的断言。
|
||||
|
||||
#### V5 原始身份展示（B-14）
|
||||
- diff 增加 `raw_key_before`（before 原始键值，未归一化/映射）。实现方式：在 Python 路径生成；对精简行，在回填时由 `IdentityPipeline` 反推；SQL 路径没有映射，原始键等于 key。
|
||||
- 结果表：当原始值与映射后的值不同时，键单元格显示 `cgei-0/2/0/1 ← GE0/0/1`；CSV 增加 `<field>__raw_before` 列（只针对接口键）。
|
||||
- **验收（浏览器脚本）**：映射行同时显示两侧端口名。
|
||||
|
||||
---
|
||||
|
||||
### 阶段 D：割接监控的健壮性与性能
|
||||
|
||||
#### H1 run、diff 保留策略 + 批次保护收敛（M-02）
|
||||
- 新配置项：`biz_migration_auto_run_keep`（默认每个 batch 保留 300 个 auto run）、`biz_compare_auto_run_keep`（默认每个 job 保留 50 个 auto run）。
|
||||
- 清理规则：
|
||||
- 永久保留：`purpose in (acceptance, manual)` 的 run，以及被红单 `run_id` 引用的 run。
|
||||
- 其余 auto run：超过数量上限的删除（连同其 diff）。
|
||||
- 保护范围：`retention.protected_batch_map` 只保护“仍然存在的 run”所引用的批次，并**加上 `summary.metric_batches` 里的所有批次**（目前只保护首个 binding 的批次）。
|
||||
- 清理挂在 `biz_state_scheduler._maybe_purge_retention` 上，每小时执行一次。
|
||||
- **验收测试**：插入 400 个 auto run 和 1 个验收 run，清理后剩 300 + 1；被删除 run 所引用的 HF 批次不再受保护，能按 `retention_days` 被清理；验收 run 引用的所有批次（包括 `metric_batches`）都受保护。
|
||||
|
||||
#### H2 评估和验收改为后台执行（M-03）
|
||||
- 仿照业务对比的 `enqueue_compare`：`POST /batches/{id}/evaluate` 和 `/finish` 返回 `{run_id, status: "running"}`，在后台线程里执行，`BizMigrationRun.status` 依次经过 `running/success/failed`，`summary.progress` 记录进度。
|
||||
- 项目行锁只在“写 run 和 diff”这个短事务里持有，不再覆盖整个计算过程。
|
||||
- 重启恢复：参照 `recover_interrupted_compares_on_startup`，把遗留的 `running` 标记为 `cancelled`。
|
||||
- 前端：轮询 run 的状态；验收按钮在执行期间不可重复点击。
|
||||
- **验收测试**：评估请求立即返回；并发两次评估，第二次返回 409；模拟重启后状态被正确恢复。
|
||||
|
||||
#### H3 路由类 sheet 的处理（M-04）
|
||||
- 默认：`MANUAL_ROUTE_METRICS` 对应的 sheet **不参与验收门槛**。没有在本批窗口内手动采集时，显示为 `manual_pending`（灰色），不计入 `current_missing` 或 `coverage_incomplete`；已采集时参与判定。
|
||||
- 路由 sheet 的双侧比对改为复用业务对比的 SQL 路径（旧基线 vs 旧当前、新基线 vs 新当前，各执行一次 `run_sql_sheet_compare`，只取计数和失败行），避免在 Python 里载入百万行。
|
||||
- **验收测试**：默认监控模板、没有采集路由时，验收不因路由失败；采集了路由时，判定会出现在结果里。
|
||||
|
||||
#### H4 基线行缓存（M-10）
|
||||
- 进程内 LRU：键为 `(batch_id, metric_id)`，只缓存基线批次（基线不可变），总行数上限可配置（默认 50 万行）。基线变更或项目删除时失效。
|
||||
- **验收测试**：连续两次 auto 评估，第二次不再查询基线行（mock 计数）。
|
||||
|
||||
#### H5 重复键的处理口径（M-05）
|
||||
- `duplicate_keys` 只在“本批范围内的键（expect）有重复”时作为验收失败原因；其他 sheet 的重复键作为警告，写进 `accept_summary.warnings`。
|
||||
- 卡片上展示具体是哪些重复键（已有 `duplicate_key_list`），并提示“完善 key_fields（例如 ISIS 加上 `lev`）或在 sheet 上启用 `duplicate_pairing: sorted`”。
|
||||
- **验收测试**：范围外的 sheet 有重复键时验收可以通过；范围内有重复时验收失败。
|
||||
|
||||
#### H6 HF 采集合并与单设备并发上限（M-09）
|
||||
- 决策见第 6 节，默认：**同间隔、同 lane（light/heavy）的 metric 合并为一对任务**（一次 SSH 采多条命令）；heavy 和 manual 的路由表保持独立。
|
||||
- 新配置项 `biz_state_max_sessions_per_ne`（默认 2）：`claim_queued_batches` 认领时，同一个 `ne_id` 正在运行的批次达到上限就跳过（成对认领时要两侧都满足）。
|
||||
- `_hf_bindings` 的结构保持兼容（`metric_ids` 本来就是列表）。
|
||||
- **验收测试**：15 个 light metric、同一间隔 → 每侧 1 个 HF 任务；单设备并发上限生效（3 个同 NE 批次只认领 2 个）。
|
||||
|
||||
#### H7 同设备割接模式（M-08）
|
||||
- 当 `old_ne == new_ne` 时允许创建项目：每个 metric 只建一个 HF 任务，新旧两侧共用同一批次（`collect_group_id` 只含一个任务，轮次天然对齐），映射表达同框端口迁移。
|
||||
- `evaluate_metric_dual` 不需要改（两侧是同一份行数据，靠映射区分新旧端口）。
|
||||
- `enqueue_collect_group` 允许单任务组（`mode=single_ne`）。
|
||||
- **验收测试**：同设备项目可以创建；端口 A 迁移到 B（A 变为 down、B 变为 up）判为 `migrated`。
|
||||
|
||||
#### H8 去掉 metric 名称启发式（M-12）
|
||||
- `sheet_overrides` 增加 `scope_by_port_map: true|false`；不填时按当前规则推导，并把推导结果写进 override 的 `effective_*` 字段供界面展示。`_map_defines_iface_expect` 不再读取 metric 名称。
|
||||
- **验收测试**：一个名称不含 bgp 的 sheet 设置 `scope_by_port_map=false` 后，不会被映射推导出范围。
|
||||
|
||||
#### H9 其他（M-14、M-15）
|
||||
- GET 接口不写库：`project_to_dict` 中的默认模板绑定改为只读解析，持久化只在 create/patch 时进行。
|
||||
- 红单：在后续验收中已转绿的键标记为 `resolvable=true`（不自动关闭），前端提供“一键关闭可关闭红单”。
|
||||
|
||||
---
|
||||
|
||||
### 阶段 E：跨厂商解析（需要你提供真实样本）
|
||||
|
||||
#### P1 华为 VRP 首批状态解析（T-05）
|
||||
- **前提**：每个命令提供至少 2 份真实输出样本，放到 `tests/fixtures/huawei/`，并做脱敏（不同版本、带子接口和 VRF 的各一份）。**没有样本时 Cursor 不得凭空编写 TextFSM。**
|
||||
- 顺序：`interface_brief`（`display interface brief`）、`arp`（`display arp all`）、`bgp_peer`（`display bgp peer` / `display bgp vpnv4 all peer`）、`isis_adjacency`（`display isis peer`）、`ospf_neighbor`（`display ospf peer brief`）。
|
||||
- 每个 metric 输出**与中兴相同的字段名**（FieldDef 共用），状态值输出**规范枚举**：
|
||||
- 接口 `admin/phy/prot ∈ {up, down}`；华为的 `*down` 输出为 `admin=down`
|
||||
- BGP `state ∈ {established, idle, active, connect, opensent, openconfirm}`
|
||||
- ARP `entry_type ∈ {dynamic, static, interface}`
|
||||
|
||||
这样对比层的过滤和预设不需要按厂商分叉。
|
||||
- 按 `skills/biz-state/netx-biz-state/SKILL.md` 第 2 节逐项完成：TextFSM、`prefer_fsm`、注册、profile（附 `slice`）、单测。
|
||||
- 新增“规范值契约”文档：`docs/biz-state-canonical-fields.md`，按 metric 列出字段、取值范围，以及各厂商原始值到规范值的对照。中兴 parser 也要对照这份契约补齐规范化（例如 `entry_type`）。
|
||||
- ARP 过滤：内置模板从 `age_timer` 改为 `entry_type eq dynamic`（依赖各厂商 parser 输出规范的 `entry_type`）；中兴 parser 在 `entry_type` 为空时，根据 age 推断出 `dynamic`。
|
||||
- **验收**：fixture 单测；中兴→华为的跨厂商对比用例（固定行数据）在配置映射和值映射后通过率为 100%。
|
||||
|
||||
---
|
||||
|
||||
### 阶段 F：前端体验（和上面各任务一起做）
|
||||
|
||||
1. 映射编辑器升级为表格模式（保留文本模式）：两列可编辑，支持粘贴 CSV/Excel，逐行显示校验状态（重复、无效、归一化后的值）。“从基线生成”：左列 before 端口、右列 after 端口，可按“相同槽位/端口号”自动推荐。
|
||||
2. 预检结果表（B5）：疑似漏配、未使用的键、N:1、命令覆盖差集，每项都可以展开查看明细。
|
||||
3. sheet 卡片显示 `no_data` / `suspect_parse` / `not_collected` / `manual_pending` 状态和原因（V1、H3）。
|
||||
4. 结果表的键列显示原始端口名（V5）。
|
||||
5. 模板编辑器：接口归一化规则增加 side/vendor 选择；字段规则增加值映射和方向性比较；过滤条件可选 side；行过滤处提示“按归一化后的值书写”。
|
||||
6. 割接看板：重复键警告、未覆盖切片、预设语义说明（端口成功条件）。
|
||||
|
||||
---
|
||||
|
||||
## 4. 建议执行顺序
|
||||
|
||||
| 顺序 | 任务 | 说明 |
|
||||
|---|---|---|
|
||||
| 1 | A1–A7 | 正确性快修，相互独立，可以并行 |
|
||||
| 2 | B1 → B2 → B3 → B4 | 流水线是 B2–B4 的基础，必须先做 |
|
||||
| 3 | B5、B6、B7 | 预检依赖流水线 |
|
||||
| 4 | V1 → V2 → V3 → V4 → V5 | V2 依赖 V1 的状态定义 |
|
||||
| 5 | H1 → H2 → H3 → H4 → H5 → H6 → H7 → H8 → H9 | H1/H2 优先（生产风险最大） |
|
||||
| 6 | 阶段 E | 等你提供样本 |
|
||||
| 7 | 阶段 F | 随各任务交付；最后做一次整体走查 |
|
||||
|
||||
---
|
||||
|
||||
## 5. 验收方式（Claude 验收时执行）
|
||||
|
||||
### 5.1 回归命令
|
||||
```powershell
|
||||
# 项目根目录：割接对比相关后端回归（基线 279 项通过）
|
||||
.venv\Scripts\python.exe -m pytest tests/test_biz_state_compare.py tests/test_biz_state_compare_sql.py tests/test_biz_state_compare_search.py tests/test_biz_state_compare_snapshot.py tests/test_biz_state_compare_export.py tests/test_biz_state_compare_recovery.py tests/test_biz_compare_templates.py tests/test_biz_migration_evaluate.py tests/test_biz_migration_evidence.py tests/test_biz_migration_hf_decouple.py tests/test_biz_migration_integrity.py tests/test_iface_normalize.py tests/test_biz_state_monitor_service.py tests/test_biz_state_paired_sampling.py tests/test_biz_state_retention.py -q
|
||||
# 全量
|
||||
.venv\Scripts\python.exe -m pytest -q
|
||||
|
||||
# web 目录
|
||||
node node_modules/typescript/bin/tsc -b
|
||||
node node_modules/vite/bin/vite.js build
|
||||
node --experimental-strip-types --test tests/cutoverDataCache.test.mjs tests/compareTemplateModel.test.mjs
|
||||
# 浏览器（先启动 vite :5179）
|
||||
node tests/bizCompare.browser.mjs
|
||||
node tests/bizCompareTemplates.browser.mjs
|
||||
node tests/bizMigration.browser.mjs
|
||||
node tests/bizState.browser.mjs
|
||||
```
|
||||
|
||||
### 5.2 必须新增的测试文件（便于验收定位）
|
||||
- `tests/test_identity_pipeline.py`：B1 的 7 路径对照、B2、B3、B4
|
||||
- `tests/test_compare_preflight.py`：B5、B7、V2
|
||||
- `tests/test_compare_quality.py`：V1、V3、V4
|
||||
- `tests/test_biz_migration_presets.py`：A4 的 4 个端口场景、H5、H8
|
||||
- `tests/test_biz_migration_retention.py`：H1
|
||||
- `tests/test_biz_migration_async.py`：H2、H3
|
||||
- `tests/test_cutover_references.py`：A2、A5
|
||||
- 浏览器：`bizCompare.browser.mjs` 增加 A3、B5、V5 场景；`bizMigration.browser.mjs` 增加 H2、V1 卡片状态场景
|
||||
|
||||
### 5.3 我会逐条核对的场景
|
||||
1. 原始名称映射（A1）：`GE0/1/0/1→gei0/2/0/1` 加规则 `GE→gei`，结果为 unchanged。
|
||||
2. 拔纤但未 shutdown（A4）：验收判 `migrated`。
|
||||
3. 删除被引用的映射或模板（A2）：返回 409。
|
||||
4. 向导中映射未保存（A3）：不会被静默丢弃。
|
||||
5. 跨厂商过滤（B4）：华为侧不会被 `contains gei` 整侧滤掉。
|
||||
6. MAC 作为键（B3）：不同格式能配对。
|
||||
7. 空对空（V1）：显示 `no_data`，不是 100%。
|
||||
8. 命令集不一致（V2）：显示 `not_collected`，不是 `lost`。
|
||||
9. 同键乱序（V4）：不再出现假“不一致”。
|
||||
10. 有映射时 BGP 路由走 SQL（B6）。
|
||||
11. auto run 被清理、被引用批次的保护随之释放（H1）。
|
||||
12. 验收异步执行、不阻塞请求（H2）；默认模板不因路由未采集而验收失败（H3）。
|
||||
13. 历史 run（snapshot v1）的检索、回填、导出结果与改动前一致。
|
||||
|
||||
---
|
||||
|
||||
## 6. 需要你确认的决策（Cursor 按“默认”执行，除非你另行指定）
|
||||
|
||||
| # | 决策 | 默认 |
|
||||
|---|---|---|
|
||||
| D1 | 业务对比是否允许 N:1 映射 | 允许，但在预检和结果中明确列出；割接监控继续拒绝并说明冲突行 |
|
||||
| D2 | 1:N（端口拆分 breakout）本期是否支持 | 不支持，明确报错 |
|
||||
| D3 | 空对空 / 疑似解析失败时，任务级“是否通过” | `null`（未判定），不计入通过率，不当作失败 |
|
||||
| D4 | 路由表是否参与割接验收门槛 | 未采集时不参与（`manual_pending`），采集后参与 |
|
||||
| D5 | HF 采集是否按“同间隔 + 同 lane”合并任务 | 合并；每台设备并发 SSH 上限默认 2 |
|
||||
| D6 | 是否支持同设备割接 | 支持（H7） |
|
||||
| D7 | auto run 保留数量 | 割接每批 300、业务对比每个任务 50；验收和手动 run 永久保留 |
|
||||
| D8 | 流量计数器默认是否参与比较 | 不参与；需要时用 `numeric_min` |
|
||||
| D9 | 华为样本由谁提供、首批覆盖哪些命令 | 由你提供；首批为接口、ARP、BGP、ISIS、OSPF |
|
||||
|
|
@ -1,288 +0,0 @@
|
|||
# NetX Linux 发行版实施计划（2026-10-14）
|
||||
|
||||
> 修订（同日）：用户确认**只出 x86_64**；**API/Web 端口也必须可自选**（服务器上服务多，避免冲突）。下文已按此调整；由 Claude 直接实现。
|
||||
|
||||
## 0. 目标与已确认的需求
|
||||
|
||||
| # | 需求（用户确认） | 落地方式 |
|
||||
|---|---|---|
|
||||
| 1 | 覆盖尽量多的 Linux 发行版 | 以 **glibc ≥ 2.17** 为基线（CentOS 7 级别），只出 **x86_64** 包 |
|
||||
| 2 | tar.gz + install.sh；离线安装，目标机上**什么都不用装** | 包内自带 Python 3.12、全部依赖、PostgreSQL 16 和 web/dist；目标机只需要 bash、tar、gzip、coreutils 和 useradd，这些是任何 glibc 发行版的最小系统都有的 |
|
||||
| 3 | 在线更新 | `netxctl update`，更新源与 Windows 版一致（GitHub / Forgejo / manifest），可选每日自动更新（systemd timer）；失败自动回滚程序 |
|
||||
| 4a | API/Web 端口可配置 | 安装时 `--port` 或交互输入（检查占用，被占用时重新输入）；装好后用 `netxctl set-port`，失败自动回滚 |
|
||||
| 4b | 内置 PG 端口可配置 | Linux：安装时 `--pg-port` 或交互输入，装好后用 `netxctl set-pg-port`；**Windows 同步补齐**：Setup 数据库页加端口框，静默参数 `/BundledPgPort=`，另提供改端口脚本 |
|
||||
|
||||
现状：后端是纯 Python，依赖都有 manylinux2014 wheel，[host_metrics.py](../../netx_api/host_metrics.py) 已有 Linux 分支。所以 **`netx_api` 业务代码不需要改**，工作量都在 `packaging/linux/`、CI 和 Windows 的端口配置上。`scripts/start_netx.sh` / `stop_netx.sh` 保留，继续作为源码/开发模式的启动方式，不动。
|
||||
|
||||
### 支持范围
|
||||
|
||||
- **支持**（glibc ≥ 2.17，x86_64）：CentOS 7/8/Stream 9、RHEL 7–9、Rocky/Alma 8–9、Oracle Linux、Fedora、Ubuntu 18.04+、Debian 9+、openSUSE Leap/SLES 12+、openEuler 20.03+、麒麟 V10、统信 UOS 20、Anolis 等。
|
||||
- **不支持（第一期）**：ARM（鲲鹏/飞腾）、Alpine 等 musl 系统、32 位系统、CentOS 6（glibc 2.12）、龙芯 LoongArch / MIPS（没有现成的 standalone Python，后续单独评估）。
|
||||
- 有没有 systemd 都能用：有 systemd 就注册服务；没有（容器、精简系统）就用 `netxctl start/stop` 后台运行。
|
||||
|
||||
---
|
||||
|
||||
## 1. 产物与目录布局
|
||||
|
||||
### 1.1 发布产物（与 Windows Setup 挂在同一个 GitHub/Forgejo Release 下）
|
||||
|
||||
```
|
||||
netx-<ver>-linux-x86_64.tar.gz + .sha256
|
||||
```
|
||||
|
||||
解压后：
|
||||
|
||||
```
|
||||
netx-<ver>-linux-x86_64/
|
||||
install.sh # 入口：安装 / 更新 / 重装
|
||||
version.json # {"version": "...", "arch": "x86_64", "pg_major": 16, "python": "3.12.x"}
|
||||
netx_api/ alembic/ alembic.ini pyproject.toml requirements.txt README.md LICENSE
|
||||
web/dist/ # 由 API 托管，不需要 Node
|
||||
python/ # python-build-standalone 3.12，依赖直接装在它的 site-packages 里（不用 venv）
|
||||
pgsql/{bin,lib,share} # PostgreSQL 16 只保留运行需要的部分；rpath=$ORIGIN/../lib
|
||||
packaging/linux/ # netxctl、netx_admin.py、systemd 单元模板、logrotate、README
|
||||
```
|
||||
|
||||
### 1.2 安装后的目录
|
||||
|
||||
| 路径 | 内容 | 属主 / 权限 |
|
||||
|---|---|---|
|
||||
| `/opt/netx/releases/<ver>/` | 某个版本的程序（只读） | root:root 0755 |
|
||||
| `/opt/netx/current` → `releases/<ver>` | 当前版本的软链，切换是原子的 | |
|
||||
| `/usr/bin/netxctl` → `/opt/netx/current/packaging/linux/netxctl` | 管理命令 | |
|
||||
| `/var/lib/netx/` | `.env`、`pgdata/`、`data/…`（与 Windows 的 `Get-NetxDataEnvMap` 子目录一致）、`backups/`、`logs/` | netx:netx 0750，`.env` 0600 |
|
||||
| `/etc/systemd/system/netx.service`、`netx-update.{service,timer}` | 服务单元 | |
|
||||
|
||||
`--prefix` 和 `--data` 都可以改。始终只保留最近 2 个 release 目录，用于回滚。
|
||||
|
||||
---
|
||||
|
||||
## 2. 任务清单
|
||||
|
||||
### L1 构建：PostgreSQL 16 可移植二进制（glibc 2.17）
|
||||
|
||||
- 新文件 `packaging/linux/build_pg.sh <arch>`：在 `quay.io/pypa/manylinux2014_<arch>` 容器里从源码编译 PostgreSQL 16.x，**和 Windows 用同一个小版本**（现在是 16.4，见 `packaging/cache/pgsql-16.4-1`）。
|
||||
- `./configure --prefix=/pgsql --without-readline --without-icu --with-zlib --without-openssl LDFLAGS='-Wl,-rpath,$ORIGIN/../lib'`
|
||||
- 把 `libz.so.1` 复制进 `pgsql/lib/`，避免最小系统里没有 zlib。
|
||||
- 只保留运行需要的二进制：`postgres pg_ctl initdb psql pg_dump pg_dumpall pg_restore pg_isready`（与 [build_release.ps1:163](../../packaging/build_release.ps1:163) 一致），外加 `lib/`、`share/`。
|
||||
- 打包成 `pg-16.x-linux-<arch>.tar.gz`；CI 按版本+架构缓存，本地放 `packaging/cache/`。
|
||||
- 版本号和下载校验值统一写在 `packaging/linux/versions.env`（PG 源码 URL+sha256，python-build-standalone URL+sha256）。
|
||||
|
||||
**验收**：在 `centos:7` 容器里，`pgsql/bin/initdb` / `pg_ctl start` / `psql -c 'select version()'` 都能跑；`L4` 的检查脚本通过。
|
||||
|
||||
### L2 构建：Python 运行时 + 依赖
|
||||
|
||||
- `packaging/linux/build_python.sh <arch>`：下载 python-build-standalone `cpython-3.12.x+<date>-<arch>-unknown-linux-gnu-install_only_stripped.tar.gz`（pin 住版本并校验 sha256）。
|
||||
- **在 manylinux2014 容器里**执行 `python/bin/python3 -m pip install --only-binary=:all: -r requirements.txt`。容器的 glibc 是 2.17，pip 只会选兼容的 wheel；任何依赖要编译或要求更高的 glibc，构建就会直接失败。
|
||||
- 删掉 `test/`、`idlelib/`、`tkinter/`、`ensurepip/`，再执行 `compileall -q`，这样 `/opt` 只读也不会在运行时写 pyc。
|
||||
- 不用 venv（standalone Python 本身可以搬动，venv 的软链和绝对路径反而碍事）。所有启动都用 `python/bin/python3 -m …`，不依赖 `bin/` 下脚本的 shebang。
|
||||
|
||||
**验收**：在 `centos:7` 和 `ubuntu:24.04` 里，`python/bin/python3 -c "import netx_api.main, psycopg, cryptography, netmiko, bcrypt, psutil"` 都成功。
|
||||
|
||||
### L3 构建：打包脚本 `packaging/linux/build_linux_release.sh`
|
||||
|
||||
- 参数：`--version`（默认读 pyproject）、`--web-dist <path>`、`--skip-web-build`。
|
||||
- 流程：先检查 web/dist（没有就执行 `npm ci && npm run build:release`），调用 L1/L2，拷贝程序文件（清单与 [build_release.ps1:88-93](../../packaging/build_release.ps1:88) 一致），写 `version.json`，执行 `tar czf` 并生成 `.sha256`，最后调用 L4 检查。
|
||||
- 依赖 docker（manylinux 容器）。本机是 Windows，所以本地构建要在 WSL2 里的 Ubuntu+docker 上做，或者用一台 Linux 构建机。**CI 是主要的构建途径**。
|
||||
- 文件：`packaging/linux/README.md`，写清楚构建、安装、运维、支持范围。同时把 [packaging/README.md](../../packaging/README.md) 第 3 行"Linux is unchanged"改成指向 Linux 发行版的说明。
|
||||
|
||||
### L4 兼容性闸门 `packaging/linux/check_bundle.sh`
|
||||
|
||||
- 用 `objdump -T` 扫描包里所有 ELF（`python/`、`site-packages/**/*.so`、`pgsql/`），引用的 `GLIBC_x.y` 最高不能超过 **2.17**。
|
||||
- 用 `readelf -d` 检查 NEEDED：除了 `libc/libm/libdl/libpthread/librt/libutil/libcrypt/ld-linux*/libgcc_s/libstdc++` 以外，都必须能在包内找到。
|
||||
- 不通过就让构建失败。
|
||||
|
||||
### L5 `install.sh`（安装 / 更新 / 重装）
|
||||
|
||||
只依赖 bash、coreutils、tar、gzip、useradd（或 `adduser`）。凡是复杂逻辑（改 .env、检查端口、URL 编码、生成密钥）都交给**包里自带的 Python**（`packaging/linux/netx_admin.py`，只用标准库），不依赖 curl、ss、openssl 这些。
|
||||
|
||||
参数（交互模式下会逐项提问；中文还是英文按 `$LANG` 判断；`--yes` 为非交互）：
|
||||
|
||||
```
|
||||
--mode install|update|reinstall # 已有安装时默认 update
|
||||
--prefix /opt/netx --data /var/lib/netx
|
||||
--db bundled|external
|
||||
--pg-port 15432 # 内置 PG 端口（需求 4）
|
||||
--pg-password <pw> # 不填则自动生成
|
||||
--db-url postgresql+psycopg://… # external 用；也可以拆成 --db-host/--db-port/--db-user/--db-password/--db-name
|
||||
--bind 0.0.0.0 --port 8890 # Linux 一般是无桌面的服务器，默认绑 0.0.0.0（Windows 是 127.0.0.1）
|
||||
--credential-key-file <path> # 复用别的安装的 NE 密码加密密钥（对应 Windows 的 /CredentialSecretKey）
|
||||
--no-service # 不注册 systemd 服务
|
||||
--auto-update # 打开每日自动更新（默认关闭，与 Windows 一致）
|
||||
--open-firewall # 有 firewall-cmd / ufw 时放行 --port；默认只打印提示
|
||||
--no-backup # 更新前不做 pg_dump
|
||||
```
|
||||
|
||||
预检（失败时给出中英文提示并退出，不留下半装的状态）：
|
||||
1. 必须是 root；
|
||||
2. 架构和包一致；`getconf GNU_LIBC_VERSION` ≥ 2.17；检测到 musl 就拒绝；
|
||||
3. 磁盘空间够（≥ 1 GB）；
|
||||
4. 用自带 Python 检查 `--port`、`--pg-port` 是否空闲（被占用时交互模式重新问，`--yes` 模式直接报 `port_in_use`）；
|
||||
5. external 模式用 `psql SELECT 1` 测试连接（对应 Windows 的行为）。
|
||||
|
||||
**首次安装**的步骤：
|
||||
1. 创建系统用户 `netx`（`useradd -r -s /sbin/nologin -d /var/lib/netx`；没有 useradd 就退到 `adduser -S`）；
|
||||
2. 解压到 `releases/<ver>`，切换 `current` 软链，建立 `netxctl` 链接；
|
||||
3. 创建数据目录、写 `.env`（键与 Windows [setup_first_run.ps1](../../packaging/setup_first_run.ps1) 一致：`NETX_DB_MODE`、`NETX_HOST`、`NETX_PORT`、`NETX_UI_DIST_DIR`、`NETX_BUNDLED_PG_PORT`、`NETX_DATABASE_URL`、`Get-NetxDataEnvMap` 里的路径、凭据密钥）；
|
||||
4. 内置 PG：用 `netx` 用户执行 `initdb -U postgres -A scram-sha-256 -E UTF8 --locale=C`，在 `postgresql.conf` 写入 `port = <pg-port>` 和 `listen_addresses = '127.0.0.1'`，`pg_hba` 只允许本机，然后创建 `netx` 角色和库（照搬 Windows 的 bootstrap SQL）；
|
||||
5. 如果系统有 SELinux 并且有 `restorecon`，执行 `restorecon -R <prefix>`；
|
||||
6. 有 systemd（`[ -d /run/systemd/system ]`）就安装 `netx.service` 并 `enable --now`；没有就执行 `netxctl start`；
|
||||
7. 等待 `/health` 返回正常（最多 90 秒），最后打印访问地址、首次登录说明和防火墙提示。
|
||||
|
||||
**更新**（`--mode update`，也是在线更新最后调用的入口）：
|
||||
1. 解压到 `releases/<newver>`（版本已存在就覆盖）；
|
||||
2. 内置 PG 时做 `pg_dump -Fc` 到 `backups/pre-<newver>-<ts>.dump`，只保留最近 3 份；
|
||||
3. 停服务，切换 `current` 软链，启动，等待 `/health`；
|
||||
4. 如果没有起来：软链切回旧版本再启动，返回非 0，并提示备份文件的位置（数据库可能已经被 alembic 迁移过，所以只回滚程序，数据库靠这个 dump 恢复）；
|
||||
5. 成功后清理旧 release，只保留 2 个；
|
||||
6. 不会改动 `.env` 和 PG 端口；systemd 单元按新版本的模板重新生成。
|
||||
|
||||
`--mode reinstall`：重新走数据库配置，但**不删除** `/var/lib/netx`（和 Windows 一样）。
|
||||
|
||||
### L6 `netxctl` 管理命令
|
||||
|
||||
bash 写的薄入口，复杂的子命令转交 `netx_admin.py`。以 root 调用时，需要降权的操作用 `runuser -u netx --`（没有 runuser 就用 `su -s /bin/sh netx -c`）。
|
||||
|
||||
```
|
||||
netxctl run # 前台监督进程（systemd 的 ExecStart 就是它）
|
||||
netxctl start|stop|restart|status
|
||||
netxctl logs [-f] [api|worker|pg]
|
||||
netxctl set-pg-port <port> # 需求 4
|
||||
netxctl set-port <port> / set-bind <host>
|
||||
netxctl reconfigure-db # 等于 install.sh --mode reinstall 里的数据库部分
|
||||
netxctl check-update / update [--yes] [--file <tar.gz>]
|
||||
netxctl auto-update on|off
|
||||
netxctl backup [<file>] # pg_dump + data/auth 下的密钥，打成一个 tar.gz
|
||||
netxctl uninstall [--purge] # --purge 删除数据，需要输入 "DELETE" 确认
|
||||
netxctl version
|
||||
```
|
||||
|
||||
`netxctl run`（前台监督进程，对应 Windows 的 [service_run.ps1](../../packaging/service_run.ps1)）：
|
||||
- 加载 `/var/lib/netx/.env`，并强制覆盖为数据目录下的路径；设置 `PYTHONPATH=current`、`NETX_UI_DIST_DIR`；
|
||||
- 内置模式先执行 `pg_ctl -D pgdata -w start`。启动前检查 `pgdata/PG_VERSION` 的主版本与 `version.json` 里的 `pg_major` 是否一致，不一致就报 `pg_major_mismatch`，不擅自升级数据；
|
||||
- 启动 API（`-m netx_api.main`）、worker（`-m netx_api.worker`）和 biz_state workers，副本数和参数照搬 [start_netx.sh:122-160](../../scripts/start_netx.sh:122)，`NETX_RUN_INLINE_SCHEDULERS=false`；
|
||||
- `trap TERM INT`：先停所有子进程，再 `pg_ctl stop -m fast`；
|
||||
- 用 `wait -n` 等待：任何一个子进程退出就整体退出并返回非 0，由 systemd `Restart=on-failure` 重启；
|
||||
- 日志写到 `/var/lib/netx/logs/*.log`，并安装 `/etc/logrotate.d/netx`（copytruncate，7 份）。
|
||||
|
||||
没有 systemd 时：`netxctl start` 执行 `nohup netxctl run &`，pid 写到 `data/runtime/netx.pid`；`stop` 发 TERM，30 秒后还没退出就 KILL。
|
||||
|
||||
`netx.service`：
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=NetX
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
[Service]
|
||||
Type=simple
|
||||
User=netx
|
||||
Group=netx
|
||||
ExecStart=/opt/netx/current/packaging/linux/netxctl run
|
||||
KillMode=mixed
|
||||
TimeoutStopSec=60
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
LimitNOFILE=65536
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
### L7a API/Web 端口可配置
|
||||
|
||||
- `install.sh`：支持 `--port` 和 `--bind`；交互模式下默认 8890 / 0.0.0.0，会检查端口是否被占用（自带 Python 的 socket 检测），被占用时重新问，`--yes` 模式直接报 `port_in_use`。
|
||||
- `netxctl set-port <port>` / `set-bind <host>`：校验 → 停服务 → 改 `.env` 里的 `NETX_PORT` / `NETX_HOST` → 启动 → 等 `/health`；失败就恢复 `.env` 并重新启动。改完后打印 firewalld/ufw 的放行命令。
|
||||
- 重装（`--mode reinstall`）时，默认值取当前配置，并允许沿用自己正在占用的端口。
|
||||
|
||||
### L7 内置 PG 端口可配置（Linux 部分）
|
||||
|
||||
- `install.sh`：支持 `--pg-port`；交互模式默认值 15432，会检查是否被占用，被占用时重新问。
|
||||
- `netxctl set-pg-port <port>`：校验范围在 1024–65535 且端口空闲 → 停服务 → 改 `postgresql.conf` 里的 `port` → 改 `.env` 里的 `NETX_BUNDLED_PG_PORT`，并把 `NETX_DATABASE_URL` 中的端口一并改掉（只换 host:port 部分，密码保持原样）→ 启动 → 等 `/health`；失败就把三处改回去。
|
||||
- external 模式下执行这个命令会报错，提示改用 `reconfigure-db`。
|
||||
|
||||
### L8 在线更新 `packaging/linux/netx_update.py`（只用标准库）
|
||||
|
||||
- 更新源和配置项与 Windows [check_update.ps1](../../packaging/check_update.ps1) **完全一致**：`NETX_UPDATE_SOURCES`（默认 `github,forgejo`）、`NETX_UPDATE_GITHUB_REPO`、`NETX_UPDATE_FORGEJO_URL`、`NETX_UPDATE_URL`（manifest）、`NETX_UPDATE_TOKEN`；会探测所有源，选最新的那个；版本相同时优先 GitHub；token 只发给非 GitHub 的主机。
|
||||
- 资产匹配规则：`^netx-(?P<ver>[\d.]+)-linux-<arch>\.tar\.gz$`；sha256 优先取 `<asset>.sha256` 资产，其次取 manifest 里的值，**两者都没有就拒绝更新**。
|
||||
- 代理：遵循 `HTTPS_PROXY/NO_PROXY`（urllib 默认就支持）。
|
||||
- 流程：下载到 `/var/lib/netx/updates/` → 校验 → 解压到临时目录 → 执行**新包里的** `install.sh --mode update --yes` → 清理临时文件。
|
||||
- `netxctl update --file <tar.gz>`：离线升级，跳过下载这一步。
|
||||
- 自动更新：`netx-update.timer`（`OnCalendar=*-*-* 03:30`，`RandomizedDelaySec=1h`，`Persistent=true`），执行 `netxctl update --yes --quiet`。没有 systemd 但有 crontab 时写到 `/etc/cron.d/netx-update`；两者都没有就提示不支持。
|
||||
- [manifest.example.json](../../packaging/manifest.example.json) 增加：
|
||||
```json
|
||||
"linux": {
|
||||
"x86_64": {"url": "...netx-0.4.19-linux-x86_64.tar.gz", "sha256": "", "size": 0},
|
||||
}
|
||||
```
|
||||
- Windows 这边 [check_update.ps1:115](../../packaging/check_update.ps1:115) 只匹配 `Setup-.*\.exe`，同一个 Release 里出现 Linux 资产不受影响。需要加一条回归验证。
|
||||
|
||||
### L9 CI：`.github/workflows/release-linux.yml`
|
||||
|
||||
- 触发：`v*` tag（和 Windows 一致）；同样校验 tag 必须等于 pyproject 里的版本。
|
||||
- job `web`：Node 20，执行 `npm ci && npm run build:release`，上传 `web/dist` 作为 artifact。
|
||||
- job `build`：`ubuntu-latest` + manylinux2014_x86_64 容器
|
||||
- 步骤：缓存/构建 PG（L1），执行 L2、L3、L4，上传 tar.gz 和 sha256 artifact。
|
||||
- job `smoke`（必须通过才发布），x86_64 在下面这些镜像里各跑一遍（`docker run --network none`，同时验证离线安装）：
|
||||
`centos:7`、`rockylinux:9`、`ubuntu:18.04`、`ubuntu:24.04`、`debian:12`、`opensuse/leap:15`、`openeuler/openeuler:22.03`。
|
||||
每个镜像执行 `packaging/linux/smoke_test.sh`：
|
||||
1. `install.sh --yes --no-service --db bundled --pg-port 25432`
|
||||
2. `/health` 返回 200；用默认管理员登录拿到 token
|
||||
3. `netxctl set-pg-port 25433`，之后 `/health` 仍然是 200
|
||||
4. `install.sh --mode update --yes`（用同一个包模拟升级）：`backups/` 里多出一份 dump，数据还在（登录前先建一条测试数据）
|
||||
5. `netxctl stop`，检查没有残留进程
|
||||
6. `netxctl uninstall --purge`（用 `--yes` 跳过确认）
|
||||
- job `release`：用 `softprops/action-gh-release@v2` 把 4 个文件上传到同一个 tag 的 Release（**不要**设 `generate_release_notes`，release notes 由 Windows 的 workflow 生成，避免互相覆盖）。
|
||||
|
||||
### L10 发布脚本
|
||||
|
||||
- [publish_forgejo_release.ps1](../../packaging/publish_forgejo_release.ps1)：`packaging/release/` 下如果有 `netx-<ver>-linux-*.tar.gz(.sha256)`，就一起上传；Release 说明里加上 Linux 安装方式。
|
||||
- [publish_release.ps1](../../packaging/publish_release.ps1)：GitHub Release 说明模板（第 107 行附近）加一段 Linux 的说明。
|
||||
- 本地发布流程（写进 `packaging/linux/README.md`）：CI 发布 tag 后，用 `gh release download vX -p 'netx-*-linux-*'` 把 Linux 包取回 `packaging/release/`，再执行 `publish_forgejo_release.ps1` 同步到 Forgejo。
|
||||
|
||||
### W1 Windows：Setup 数据库页增加内置 PG 端口
|
||||
|
||||
- [netx.iss](../../packaging/installer/netx.iss)：在 `RbBundled`（约第 762 行）下面加一个"端口"输入框 `EdBundledPort`，默认 15432，只有选内置时才可用（`DbModeClick`，第 469 行）。翻页时校验它是 1024–65535 的数字，并且端口没有被占用（可以复用 `Test-NetxTcpPortFree`，或者在 Pascal 里用 `netstat -ano` 检查）。
|
||||
- 静默安装参数 `/BundledPgPort=25432`（加在第 644 行附近解析 `DbMode` 的地方）。
|
||||
- 执行 `setup_first_run.ps1` 时传入 `-BundledPort`（这个参数已经存在，见 [setup_first_run.ps1:11](../../packaging/setup_first_run.ps1:11)）。
|
||||
- **Update 模式**不显示、也不修改端口。
|
||||
- 中英文文案（`english.` / `chinesesimplified.`）。
|
||||
|
||||
### W2 Windows：安装后修改端口
|
||||
|
||||
- 新文件 `packaging/set_pg_port.ps1 -Port <n>`（会自动提权）：做的事和 L7 一样：校验 → 通过 `runtime_ctl.ps1` 停服务 → 改 `postgresql.conf`、`.env` 里的 `NETX_BUNDLED_PG_PORT` 和 `NETX_DATABASE_URL` 的端口 → 启动 → 等 `/health`，失败就回滚。
|
||||
- 托盘的"设置"里加一项"内置数据库端口…"，调用这个脚本。
|
||||
- [build_release.ps1:121](../../packaging/build_release.ps1:121) 的拷贝清单里加上 `set_pg_port.ps1`。
|
||||
|
||||
---
|
||||
|
||||
## 3. 验收清单
|
||||
|
||||
| ID | 场景 | 通过标准 |
|
||||
|---|---|---|
|
||||
| A1 | CI 构建 | `release-linux.yml` 在测试 tag 上完整跑通；两个架构的 tar.gz 都 ≤ 150 MB；L4 检查通过 |
|
||||
| A2 | 离线安装矩阵 | L9 的 smoke 在全部镜像里通过，并且都是 `--network none` |
|
||||
| A3 | 真机 + systemd | 在一台真实的 Linux 服务器（例如 10.0.0.131 或一台 CentOS 7 / 麒麟虚拟机）上交互式安装：开机自启；`systemctl restart netx` 正常；`kill -9` API 进程后 5 秒内被拉起；浏览器能打开 `http://<ip>:8890/` 并登录 |
|
||||
| A4 | PG 端口 | 安装时 `--pg-port 25432` 生效（`ss -lnt` 只看到 `127.0.0.1:25432`）；`netxctl set-pg-port 25433` 后服务正常；改成一个已被占用的端口会被拒绝，原配置不变 |
|
||||
| A4b | API 端口 | 安装时 `--port 18890` 生效；`netxctl set-port 18891` 后新端口能访问、旧端口不再监听；设成一个已被占用的端口会被拒绝 |
|
||||
| A5 | 在线更新 | 先装 vN，在 Release 上挂 vN+1，执行 `netxctl check-update` 能看到新版本；`netxctl update --yes` 成功，`current` 指向新版本，数据还在，`backups/` 里有 dump |
|
||||
| A6 | 更新回滚 | 构造一个启动就失败的包（比如删掉 `netx_api/main.py`），`netxctl update --file` 之后 `current` 自动切回旧版本，服务正常，命令返回非 0 |
|
||||
| A7 | 校验失败 | 篡改 `.sha256` 后执行 update，结果是拒绝更新，并且不会解压 |
|
||||
| A8 | 自动更新 | `netxctl auto-update on` 后 `systemctl list-timers` 里能看到 `netx-update.timer`；`off` 后消失 |
|
||||
| A9 | external 模式 | `--db external --db-url …` 连接到外部 PG 能正常工作；`set-pg-port` 会给出明确的报错 |
|
||||
| A10 | 卸载 | `netxctl uninstall` 保留 `/var/lib/netx`；`--purge` 加确认后全部清掉；systemd 单元和 `netxctl` 链接都被删除 |
|
||||
| A11 | Windows 端口 | Setup 选内置并把端口填成 25432：装好后 `.env` 和 `postgresql.conf` 一致，服务正常；静默 `/BundledPgPort=25432` 一样生效；Update 模式下端口不变；托盘改端口成功 |
|
||||
| A12 | Windows 回归 | 现有的 Windows 沙箱离线验证（静默安装、启动、登录、停止、服务模式）全部通过；同一个 Release 里有 Linux 资产时，Windows 的 `check_update.ps1` 仍然只选 Setup.exe |
|
||||
|
||||
## 4. 建议的实现顺序
|
||||
|
||||
1. L1 → L2 → L4（先把"能在 CentOS 7 上运行的运行时"做出来，这是风险最大的部分）
|
||||
2. L6 `netxctl run`/start/stop → L5 install.sh → L7
|
||||
3. L3 + L9（CI + smoke 矩阵）
|
||||
4. L8 在线更新 + L10 发布脚本
|
||||
5. W1、W2（Windows 端口，相对独立，可以并行做）
|
||||
|
||||
## 5. 风险与说明
|
||||
|
||||
- **PG 大版本**：固定为 16。以后要升到 17 需要单独做 `pg_upgrade` 流程；第一期只检测、不处理（`pg_major_mismatch`）。
|
||||
- **API 默认绑 0.0.0.0**：Linux 服务器一般通过远程访问，所以默认对外。安装完成后会醒目提示修改默认管理员密码，并提醒放通防火墙端口。
|
||||
- **本地构建**：本机目前没有 docker，WSL 里也没有装发行版。要在本地构建 Linux 包，需要 `wsl --install -d Ubuntu` 再装 docker，或者在一台 Linux 构建机上执行。日常发布以 CI 为主。
|
||||
|
|
@ -1,295 +0,0 @@
|
|||
# 拓扑管理问题分析与优化计划
|
||||
|
||||
日期：2026-10-12。范围：拓扑管理画布（`web/src/features/topology/*`）及其视图接口（`netx_api/topology_views_graph.py`、`topology_router.py`、`topology_fabric_links.py`）。不含：UME World 平铺大图 LOD、LLDP 采集策略、分类规则、MCP 布局算法。
|
||||
用途：交给 Cursor 按任务逐项实施；Claude 负责验收。
|
||||
|
||||
本文结论来自代码阅读和本地脚本验证，没有在真实 PostgreSQL 或真实设备上执行。标了 **[已验证]** 的问题已经用脚本复现。
|
||||
|
||||
**基线（修改前）：**
|
||||
- `pytest tests/test_topology.py tests/test_ume_topology_apply.py`：**69 项通过，4 项失败**（见 T13，修改前就失败，与本计划无关）
|
||||
- `web`：`npx tsc -b` 通过
|
||||
- 拓扑前端目前没有任何单元测试
|
||||
|
||||
**实施进度（2026-10-12）：**
|
||||
| 任务 | 状态 |
|
||||
|---|---|
|
||||
| T11 基线失败测试 | ✅ 完成 |
|
||||
| T1 保存保留 lock | ✅ 完成 |
|
||||
| T2 不固化设备名为 label | ✅ 完成 |
|
||||
| T6 原子 commit + useTopologySave | ✅ 完成 |
|
||||
| T3 脏状态添加确认 | ✅ 完成 |
|
||||
| T4 手工边端口持久化 | ✅ 完成 |
|
||||
| T5 边样式走保存模型 | ✅ 完成 |
|
||||
| T7 N+1 / 截断确定性 | ✅ 完成 |
|
||||
| T8 positions 跳过缺失节点 | ✅ 完成（MCP 不依赖 positions 顺便加点） |
|
||||
| T9 点击不标脏 | ✅ 完成 |
|
||||
| T10 拆分 TopologyPage | ✅ 完成（`TopologyPage.tsx` ≤1200；抽出 palette/edge/keyboard/find/export/nodeOps/moduleLinks + `TopologyPageView`） |
|
||||
|
||||
回归：`pytest` 87 passed；`npx tsc -b` 通过；`topologySaveDiff.test.mjs` 4 passed。
|
||||
|
||||
---
|
||||
|
||||
## 0. 结论摘要
|
||||
|
||||
整体架构没有大问题（服务端图 + 本地编辑 + 一次保存）。主要问题集中在**“本地编辑状态”和“服务端状态”的同步边界**，有几处会静默丢数据或误删数据：
|
||||
|
||||
1. **UI 保存会清掉 MCP 设置的节点锁定，并把设备名固化成标签。** **[已验证]** 保存后 `locked` 从 True 变为 False，节点被移走；之后设备改名，画布上仍显示旧名称。
|
||||
2. **有未保存修改时“添加网元/投影邻居”会静默丢弃这些修改，并留下隐形删除。** 被删除的连线会重新出现在画布上，但仍留在待删除队列里，下次保存时会被真正删掉。
|
||||
3. **修改已有连线的端口不会被保存。** 界面上可以改，也会显示新值，但不会标记为未保存，保存时也不会提交，刷新后就恢复原值。
|
||||
4. **连线样式绕过了“编辑→保存”模型。** 样式修改会立即写库，但撤销只回退本地。撤销之后，画布显示和数据库就不一致了。
|
||||
5. **保存不是原子的。** 一次保存会依次调用 N 次建边、1 次删边、1 次加点、1 次删点和 1 次写坐标，其中 3 次会返回完整的图数据。中途失败会留下“部分已保存”的状态。另外，同一套保存逻辑在 `purgePlaceholderById` 里重复实现了一遍。
|
||||
6. **后端有 N+1 查询和不确定的截断。** 写坐标时每个点各查一次；外部邻居查询的 `limit(5000)` 把视图内部的边也算进去了，边密集的视图可能找不到外部邻居；节点超过上限时截断没有排序。
|
||||
7. **`TopologyPage.tsx` 有 2509 行。** 保存、面板添加、连线编辑、快捷键等逻辑都堆在这一个组件里，所以上面这些问题很难被发现，也很难测试。
|
||||
|
||||
---
|
||||
|
||||
## 1. 现状：画布编辑模型（给 Cursor 的背景）
|
||||
|
||||
```
|
||||
GET /views/{id} ──> graphQuery.data ──(graphToFlow)──> nodes / edges (本地 React state)
|
||||
│ 拖动/改名/连线/删除
|
||||
▼
|
||||
dirtyRef + historyRef(撤销栈, UNDO_MAX=40)
|
||||
pendingEdgeCreatesRef / pendingEdgeDeletesRef
|
||||
│ Ctrl+S / 保存
|
||||
▼
|
||||
saveMut (TopologyPage.tsx:516)
|
||||
1. 每条待建边 POST /fabric/edges (逐条串行)
|
||||
2. POST /fabric/edges/delete (待删边)
|
||||
3. POST /views/{id}/nodes (服务端没有的本地点；返回整图)
|
||||
4. POST /views/{id}/nodes/remove (本地没有的服务端点；返回整图)
|
||||
5. PATCH /views/{id}/positions (全部点的 x/y/label；返回整图)
|
||||
```
|
||||
|
||||
旁路（不走保存，直接写库）：
|
||||
- 添加网元/拖入面板 `addPaletteItems` (`TopologyPage.tsx:621`)、投影外部邻居 `addOutsidePeers` (`:1047`)：写库后**用服务端图整体覆盖本地并 `clearDirty()`**
|
||||
- 连线样式 `patchSelectedEdgeStyle` (`:765`)：立即 `PATCH /edge-style`
|
||||
- 删除占位网元 `purgePlaceholderById` (`:1238`)：先把保存逻辑复制执行一遍，再清理
|
||||
|
||||
脏状态下的轮询 (`useTopologyCanvas.ts:306-318`)：只合并在线状态、层级等叠加字段，不动几何信息，这部分是对的。
|
||||
|
||||
---
|
||||
|
||||
## 2. 问题清单与任务
|
||||
|
||||
优先级：**P0** = 静默丢数据/误删，**P1** = 可靠性/性能，**P2** = 体验/可维护性。
|
||||
|
||||
### T1 [P0] UI 保存会清除节点锁定 **[已验证]**
|
||||
|
||||
- `flowToPositions` (`web/src/features/topology/lib/graphFlow.ts:90`) 不发送 `locked` 字段。
|
||||
- `ViewNodeIn.locked` 的默认值是 `False` (`netx_api/topology_schemas.py:277`)。
|
||||
- `patch_view_positions` 只在“行已锁定且请求也带 locked=True”时才跳过 (`topology_views_graph.py:738`)，否则会覆盖坐标并写回 `locked=False`。
|
||||
- MCP 会用 `locked=true` 固定布局（`packages/netx-topology-mcp/.../http_tools.py:1985`）。用户在 UI 里随便拖一下再保存，这个锁就没了。
|
||||
|
||||
复现：MCP 把节点锁定在 (500,500)，UI 保存 (10,10)，结果节点变成 `locked=False x=10`。
|
||||
|
||||
**修改：**
|
||||
1. 后端：把 `ViewNodeIn.locked` 改为 `bool | None = None`，`None` 表示保持原值。对已锁定的行，只有请求里明确带 `locked=False` 才能移动（解锁）；否则跳过并计入 `skipped_locked`。MCP 显式传 `locked` 的行为保持不变。
|
||||
2. 前端：在 `NeNodeData` 里加 `locked`（由 `graphToFlow` 从 `n.locked` 填入）。锁定的节点设 `draggable: false`，并显示一个锁形角标；`flowToPositions` 不再发送 `locked`。
|
||||
3. 保存结果里如果 `skipped_locked > 0`，用 toast 提示。
|
||||
|
||||
**验收：**
|
||||
- 新增 `tests/test_topology.py::test_positions_preserve_lock_when_omitted`：锁定后用不带 locked 的 positions 请求，坐标不变，`locked` 仍为 True。
|
||||
- 新增 `test_positions_explicit_unlock_moves`：`locked=False` 可以移动并解锁。
|
||||
- 现有 MCP 相关测试全部通过。
|
||||
|
||||
### T2 [P0] 保存会把设备名固化成节点标签 **[已验证]**
|
||||
|
||||
- 接口返回的 `label` 已经做过回退：`vn.label` 为空时用 `fn.name / ip` (`topology_views_graph.py:195-197`)。
|
||||
- 前端把它放进 `data.label`，保存时 `flowToPositions` 又原样写回 `vn.label` (`graphFlow.ts:100`)。
|
||||
- 所以保存过一次之后，所有节点的 `vn.label` 都是当时的设备名。以后设备改名（例如 UME/网管同步），画布不会跟着变。
|
||||
|
||||
复现：保存后把 fabric 节点改名为 `NEW-NAME`，画布仍显示 `OLD-NAME`。
|
||||
|
||||
**修改：**
|
||||
1. 后端 `ViewNodeOut` 增加 `label_override: str`（原始 `vn.label`）；`label` 仍是回退后的显示值，保持兼容。`ViewNodeIn.label` 改为 `str | None = None`，`None` 表示不修改。
|
||||
2. 前端 `NeNodeData` 增加 `label_override`，只有用户“重命名节点”（`renameSelectedNode`，`TopologyPage.tsx:820`）时才写入。`flowToPositions` 只对 `label_override` 有变化的节点发送 `label`；重命名为空字符串表示清除覆盖，恢复跟随设备名。
|
||||
3. 数据修复脚本 `scripts/topology_clear_frozen_labels.py`，默认 dry-run：`vn.label` 等于当前 `fn.name` 或 `fn.ip` 的行，把 label 置空。输出受影响的行数，加 `--apply` 才真正写库。
|
||||
|
||||
**验收：**
|
||||
- 新增 `test_positions_without_label_keeps_following_name`：保存不带 label，改名后画布 label 是新名称。
|
||||
- 新增 `test_label_override_survives_rename`：显式覆盖后改名，仍显示覆盖值。
|
||||
- 脚本 dry-run 在测试库上输出正确的计数，`--apply` 之后再跑一次计数为 0。
|
||||
|
||||
### T3 [P0] 有未保存修改时“添加网元 / 投影邻居”会丢弃修改并留下隐形删除
|
||||
|
||||
- `addPaletteItems` (`TopologyPage.tsx:681-686`) 和 `addOutsidePeers` (`:1096-1101`) 都是直接写库，然后 `applyViewGraph(refreshed)` 再 `clearDirty()`：
|
||||
- 本地未保存的拖动、改名、删点**直接丢失**，也没有提示；
|
||||
- `pendingEdgeDeletesRef` **没有清空**。被删除的边按服务端数据重新画回画布，但它的 id 还在待删队列里。用户以后再改任何东西并保存时，这条**看得见的边会被删掉**；
|
||||
- 撤销栈也没有清空，撤销会回到和服务端不一致的快照。
|
||||
- 同类问题：`useTopologyCanvas.ts:320-355` 在非脏状态下整体刷新时，也不会清理撤销栈。
|
||||
|
||||
**修改（推荐“先保存再添加”，简单而且语义清楚）：**
|
||||
1. 抽出 `commitCanvas()`（见 T6）。`addPaletteItems` / `addOutsidePeers` 开始时如果 `dirtyRef.current` 为 true，就弹窗“当前有未保存修改，保存后继续添加？”：确认则先 `commitCanvas()` 再添加，取消则中止。
|
||||
2. 新增一个统一的辅助函数 `resetLocalFromServer(graph)`，用于所有“用服务端图整体覆盖本地”的地方：同时清空两个 pending 集合、撤销栈和重做栈，然后 `clearDirty()`。目前 `purgePlaceholderById` (`:1289-1297`) 是手写的这套逻辑，其他地方都漏了。
|
||||
|
||||
**验收（手工，Cursor 需要在 PR 描述里写明步骤和结果）：**
|
||||
1. 删除一条已有边 → 不保存 → 从面板拖入一个网元 → 应弹出确认框；选择保存后，该边被删除，新网元出现。
|
||||
2. 同上，但选择取消 → 不添加网元，删除操作仍处于未保存状态。
|
||||
3. 删除边 → 投影邻居（确认保存）→ 刷新页面，边确实被删除，没有“复活后又消失”。
|
||||
4. 用 grep 检查：所有 `applyViewGraph(` 调用点要么使用 `localPositions` 合并，要么经过 `resetLocalFromServer`。
|
||||
|
||||
### T4 [P0] 修改已有连线的端口不会被保存
|
||||
|
||||
- 右键菜单对任意单条边都显示端口输入框 (`TopologyCtxMenu.tsx:400-429`)。
|
||||
- `patchSelectedEdgeStyle` 修改本地端口后，只把样式字段发给 `PATCH /edge-style`（`ViewEdgeStylePatch` 本来就没有端口字段，`topology_schemas.py:450`），也没有 `markDirty()`。
|
||||
- 保存只处理 `pendingEdgeCreatesRef` 里的新边，已有边的端口修改永远不会提交。
|
||||
|
||||
**修改：**
|
||||
1. 只有 `source === "manual"` 的边可以编辑端口；LLDP/UME 边的端口改为只读（这两类以采集结果为准，手工修改会在下次发现时被覆盖）。
|
||||
2. 未保存的新手工边：端口修改继续只改本地，保存时随建边一起提交（现有逻辑已支持）。
|
||||
3. 已保存的手工边：新增 `PATCH /v1/topology/fabric/edges/{id}`，body `{a_port, b_port}`，只允许 `source=manual`，否则 400 `edge_not_manual`；端口要做归一化，与已有边冲突（违反唯一约束）时返回 409。前端把修改记入 `pendingEdgePortEditsRef: Map<id,{a,b}>` 并 `markDirty()`，在 `commitCanvas()` 里提交。
|
||||
|
||||
**验收：**
|
||||
- 新增 `test_patch_manual_edge_ports`、`test_patch_non_manual_edge_ports_rejected`、`test_patch_manual_edge_ports_conflict_409`。
|
||||
- 手工测试：修改已保存手工边的端口 → 出现未保存标记 → 保存 → 刷新后仍是新端口；LLDP 边的端口输入框为只读。
|
||||
|
||||
### T5 [P1] 连线样式与撤销/保存模型不一致
|
||||
|
||||
- `patchSelectedEdgeStyle` (`TopologyPage.tsx:807-815`) 每次修改都立即发请求（颜色选择器拖动时会连续发很多次）。同时调用了 `pushHistory()`，但撤销只回退本地状态，所以撤销后画布和数据库不一致。
|
||||
|
||||
**修改：** 样式也改为走保存：记入 `pendingEdgeStyleRef: Map<edgeId, style>`，`markDirty()`，在 `commitCanvas()` 里批量提交（后端 `PATCH /edge-style` 支持 `items: []` 批量，旧的单条 body 保留兼容）。撤销快照要包含这个 Map。
|
||||
|
||||
**验收：** 改颜色 → 撤销 → 保存 → 刷新，颜色是修改前的值；改颜色 → 保存 → 刷新，颜色是修改后的值；拖动颜色选择器期间，网络面板里没有 `/edge-style` 请求。
|
||||
|
||||
### T6 [P1] 保存原子化，并去掉重复的保存逻辑
|
||||
|
||||
现状见第 1 节。中途失败时，前面已执行的步骤（例如已建的边、已加的点）不会回滚，`pending*` 集合也只清了一部分。建边是幂等 upsert（`topology_fabric_links.py:65`），重试不会建出重复边，但删点和写坐标的顺序问题依然存在。
|
||||
|
||||
**修改：**
|
||||
1. 后端新增 `POST /v1/topology/views/{id}/commit`，在一个事务里依次处理：
|
||||
```json
|
||||
{
|
||||
"edge_creates": [{"client_id":"local-..","a_node_id":"","b_node_id":"","a_port":"","b_port":""}],
|
||||
"edge_deletes": ["edge-id"],
|
||||
"edge_port_edits": [{"id":"","a_port":"","b_port":""}],
|
||||
"edge_styles": [{"fabric_edge_id":"","stroke_color":"","stroke_width":0,"line_style":""}],
|
||||
"node_adds": ["fabric-id"],
|
||||
"node_removes": ["fabric-id"],
|
||||
"positions": [{"fabric_node_id":"","x":0,"y":0,"label":null}]
|
||||
}
|
||||
```
|
||||
任何一步失败都整体回滚，返回 4xx 和 `{"failed_step": "...", "detail": "..."}`。成功时只返回**一次**完整图数据，外加 `edge_id_map: {client_id: server_id}`。节点上限和冻结规则与 `add_nodes_to_view` 一致。
|
||||
2. 前端新增 `hooks/useTopologySave.ts`，导出 `commitCanvas()`：从本地状态计算差异（纯函数 `lib/saveDiff.ts`，可以单独测试），调用 `/commit`，成功后调用 `resetLocalFromServer`。`saveMut`、`purgePlaceholderById` (`:1248-1281`)、T3 的“先保存”都改用它。
|
||||
3. 旧的分步接口保留（MCP 在用）。
|
||||
|
||||
**验收：**
|
||||
- 新增 `test_view_commit_atomic_rollback`：构造一个 `edge_creates` 自环（第一步就失败），断言 `node_adds` 和 `positions` 都没有落库。
|
||||
- 新增 `test_view_commit_happy_path`：一次提交包含建边、删边、加点、删点、写坐标，结果图正确，`edge_id_map` 正确。
|
||||
- `web/tests/topologySaveDiff.test.mjs`：覆盖“本地新增/删除节点”“region 节点不进入增删列表”“只发送有变化的 label”“删除后又撤销的边不出现在 edge_deletes 中”等情况。
|
||||
- 手工测试：一次保存只产生 **1 个** 写请求（网络面板截图附在 PR 描述里）。
|
||||
|
||||
### T7 [P1] 后端视图接口的 N+1 查询与不确定截断
|
||||
|
||||
| 位置 | 问题 | 修改 |
|
||||
|---|---|---|
|
||||
| `topology_views_graph.py:719` | 写坐标时每个点各执行一次 `db.get(TopoFabricNode)`，最多 2000 次 | 先用一次 `IN` 查询拿到存在的 id 集合 |
|
||||
| `:160-166` | 节点超过 `VIEW_GRAPH_NODE_HARD_CAP` 时截断没有 `ORDER BY`，每次显示哪些节点不确定 | `order_by(TopoViewNode.created_at, TopoViewNode.id)` |
|
||||
| `:354-363` | 外部邻居的 `limit(5000)` 把**两端都在视图内**的边也算进去了，边密集的视图会漏掉外部邻居 | SQL 里排除两端都在视图内的边（`~and_(a.in_, b.in_)`），再加 `limit` |
|
||||
| `:375` | 每个外部邻居执行一次 `db.get` | 收集 id 后用一次 `IN` 查询 |
|
||||
| `:178-179` | region 节点每个执行一次 `db.get(TopoFolder)` | 一次 `IN` 查询 |
|
||||
|
||||
**验收：**
|
||||
- 新增 `test_outside_peers_found_when_many_internal_edges`：视图内有 >5000 条内部边（测试中可以把上限改成参数，例如用 10 条模拟），外部邻居仍然能返回。
|
||||
- 新增 `test_view_truncation_is_deterministic`：两次 GET 返回同一批节点。
|
||||
- 用 SQLAlchemy `before_cursor_execute` 统计 2000 个点的 positions 请求，语句数 ≤ 10（写成测试）。
|
||||
|
||||
### T8 [P1] positions 接口可以绕过节点上限加点
|
||||
|
||||
`patch_view_positions` 遇到视图中没有的 `fabric_node_id` 会直接插入 `TopoViewNode`（`:720-733`），不检查 `max_nodes` 和冻结状态，而 `add_nodes_to_view` 会检查（`:778`）。
|
||||
|
||||
**修改：** 先确认 MCP 是否依赖“positions 顺便加点”（用 grep 查 `packages/netx-topology-mcp` 里的 `positions` 调用）。如果不依赖，就改为跳过不存在的节点，并在返回里计入 `skipped_missing`；如果依赖，就加上与 `add_nodes_to_view` 相同的上限和冻结检查。**这一项的结论需要写进 PR 描述。**
|
||||
|
||||
**验收：** 新增 `test_positions_respects_max_nodes`。
|
||||
|
||||
### T9 [P2] 点击节点就会产生“未保存”和无效撤销记录
|
||||
|
||||
`onNodeDragStart` 会无条件 `pushHistory()`；`onNodesChange` 收到任何 `position` 变更都会 `markDirty()`，包括没有实际移动的点击 (`TopologyReactFlowView.tsx:442-449`)。后果：单击一个节点就出现“未保存”，并且触发脏锁，live sync 不再整体刷新。
|
||||
|
||||
**修改：** 在 `onNodeDragStart` 记录起点坐标；`onNodeDragStop` 时只有位移超过 1px 才把起始快照压入撤销栈并 `markDirty()`。`onNodesChange` 不再对 `position` 变更调用 `markDirty`（`remove`/`add` 保持不变）。
|
||||
|
||||
**验收：** 单击节点后不出现未保存标记，撤销按钮保持禁用；拖动 10px 后出现未保存标记，并且可以撤销。
|
||||
|
||||
### T10 [P2] 拆分 `TopologyPage.tsx`（在 T1–T6 完成后进行）
|
||||
|
||||
目标是把主组件压到 ≤ 1200 行，**只搬代码、不改行为**：
|
||||
- `hooks/useTopologySave.ts`（T6）
|
||||
- `hooks/useTopologyPalette.ts`：`addPaletteItems`、`addOutsidePeers`、拖放、`neQuery`、`umeQuery`
|
||||
- `hooks/useTopologyEdgeEditing.ts`：`patchSelectedEdgeStyle`、`queueDeleteEdges`、`removeSelected`、`onConnect`、`isValidConnection`
|
||||
- `hooks/useTopologyKeyboard.ts`：全局快捷键（`:1130-1182`）。依赖改为通过 ref 读取，避免每次 `nodes`/`edges` 变化都重新绑定监听
|
||||
- `lib/moduleLinks.ts`：`openNeInventory`、`openWebcrtFor`、`openAlarmsFor`、`openPortTrafficForEdge`
|
||||
|
||||
**验收：** `tsc -b` 通过；第 4 节的手工回归清单全部通过；`git diff --stat` 显示 TopologyPage.tsx 净减少 ≥ 1000 行。
|
||||
|
||||
### T11 [P2] 修复修改前就存在的失败测试
|
||||
|
||||
| 测试 | 原因 | 处理 |
|
||||
|---|---|---|
|
||||
| `test_ume_topology_apply.py::test_lldp_upgrades_bare_ume_ports_and_keeps_ume_primary` | **测试本身不稳定**：`rich_a/rich_b` 取自 `edge.a_port/b_port`（按节点 id 排序后的 A/B），但调用 upsert 时传的是 `a.id/b.id`（按 UME 名称取的节点）。uuid 排序不同时端口就配反了，结果约有一半概率失败。已用调试脚本确认这不是产品 bug。 | 改为传 `a_node_id=edge.a_node_id, b_node_id=edge.b_node_id`（`:358-365`）。`:297`、`:393` 有同样的写法，一起检查 |
|
||||
| `test_topology.py::test_lldp_discover_writes_fabric_edge`、`test_lldp_unmatched_creates_ssh_placeholder_ne` | 发现任务一直是 `pending`，没有变成 `done`（`:1285`、`:1458`） | 排查后台 worker 在 SQLite 内存库下是否被启动或被跳过，修好测试的启动方式，或者改为同步执行 job |
|
||||
| `test_topology.py::test_fabric_reconcile_scheduler_once_sweeps_orphans` | `0 not >= 1`（`:2287`） | 排查，可能和上一条同源 |
|
||||
|
||||
**验收：** 两个测试文件 **0 失败**，并且连续跑 5 次都通过（用于检验不稳定性）。
|
||||
|
||||
### 产品决定（2026-10-12 已确认）
|
||||
|
||||
- **Q1**：`isValidConnection` (`TopologyPage.tsx:568`) 不允许在同一对网元之间画第二条手工连线，但 LLDP/UME 支持并行链路，画布也支持聚合显示。是否应该允许手工建并行链路（端口不同即可）？
|
||||
- **Q2**：T3 选择了“先保存再添加”。如果希望“添加后保留本地未保存修改”，实现会复杂很多（需要把服务端新增部分合并进本地），请确认。
|
||||
|
||||
- **结论**：Q1 不允许手工画并行链路，保持 `isValidConnection`（现位于 `hooks/useTopologyEdgeEditing.ts`）现状；Q2 沿用“先保存再添加”。
|
||||
|
||||
---
|
||||
|
||||
## 3. 建议实施顺序
|
||||
|
||||
1. **T11**（先把基线修到全部通过，后面每一步的回归才有意义）
|
||||
2. **T1 + T2**（后端 schema 改动，两项一起做，再加数据修复脚本）
|
||||
3. **T6**（commit 接口 + `useTopologySave`），然后在它之上做 **T3、T4、T5**
|
||||
4. **T7、T8**
|
||||
5. **T9**，最后做 **T10**（纯重构，单独一个 PR）
|
||||
|
||||
每一步单独提交；T10 必须单独成 PR。
|
||||
|
||||
## 4. 回归命令与手工清单
|
||||
|
||||
```bash
|
||||
python -m pytest tests/test_topology.py tests/test_ume_topology_apply.py -q
|
||||
```
|
||||
|
||||
```bash
|
||||
cd web && npx tsc -b
|
||||
```
|
||||
|
||||
```bash
|
||||
cd web && node --experimental-strip-types --test tests/topologySaveDiff.test.mjs
|
||||
```
|
||||
|
||||
> `lib/saveDiff.ts` 只放纯函数，除 `import type` 外不要有运行时 import（Node strip-types 不会解析不带扩展名的相对路径）。写法参考现有的 `web/tests/cutoverDataCache.test.mjs`。
|
||||
|
||||
手工回归（每个 PR 都要过一遍，结果写进 PR 描述）：
|
||||
1. 拖动节点 → 保存 → 刷新，位置保持。
|
||||
2. 新建手工连线（带端口）→ 保存 → 刷新，连线和端口都在。
|
||||
3. 删除连线 → 撤销 → 保存，连线仍然存在。
|
||||
4. 删除节点 → 保存，节点从视图中移除，但 fabric 中仍存在（可以通过搜索找到）。
|
||||
5. 重命名节点 → 保存 → 刷新，显示新名称；再把名称清空 → 保存，恢复为设备名。
|
||||
6. 打开 live sync，另一个窗口用 MCP 移动节点，3 秒内本窗口同步；本窗口有未保存修改时不会被覆盖，只刷新在线状态。
|
||||
7. 只读用户（没有 `ne:write`）：不能拖动、连线、保存，右键菜单里没有写操作。
|
||||
8. 删除占位网元（有未保存修改时），修改先被保存，占位网元被清理。
|
||||
9. 导出 SVG/XML 正常。
|
||||
|
||||
---
|
||||
|
||||
## 5. 验收记录（2026-10-12）
|
||||
|
||||
Cursor 完成 T1–T11 后由 Claude 验收并接手修复，结论：主体实现正确，另修 6 个问题（均补了回归测试）：
|
||||
|
||||
1. `/commit` 同次 `node_adds` 的节点坐标被忽略（`autoflush=False` 下重查看不到新行）→ 先 `flush`；`_apply_positions` 的逐点 `db.get` 改为批量。
|
||||
2. T8 收紧后，无放置行的旧子区域图标拖动无法保存 → `add_missing_region_placement`（仅限本画布的手工子区域）。
|
||||
3. MCP `updateTopologyViewPositions` 总是发送 `label:""`、`locked:false`，会清掉 UI 重命名并解锁 → 未提供则不发送。
|
||||
4. 已改样式/端口的边被删除后保存 404 → 前端 `saveDiff` 与后端都跳过已删边；新建本地边的样式经 `edge_id_map` 映射。
|
||||
5. 框选拖动、方向键移动不标记未保存 → 补 `onSelectionDrag*` 和键盘移动判定。
|
||||
6. （改动前已存在）大视图打开空白、远处节点定位失效：xyflow `fitView` 等待节点测量，而 `onlyRenderVisibleElements` 下视口外节点永不测量 → `lib/fitNodes.ts` 按已知几何计算 `fitBounds`。
|
||||
|
||||
开发库数据修复：`topology_clear_frozen_labels.py --apply` 清理 516 条固化标签（执行前已备份原值）。
|
||||
|
||||
回归：拓扑后端 91 通过；Web Node 测试 30 通过；`tsc -b` 通过；MCP 包 238 通过 / 5 失败（与改动前同一批，既有问题）。
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
# 工作台功能卡片自定义 — 实现方案（2026-10-13）
|
||||
|
||||
## 目标
|
||||
|
||||
工作台功能区（UME 同步、网元管理、网络管理、拓扑、WebCRT…）以后会越来越多，每个人关心的不一样。
|
||||
让每个用户自己选择：**显示哪些卡片、按什么顺序**；设置跟着账号走（换电脑/换浏览器仍生效）。
|
||||
|
||||
## 现状
|
||||
|
||||
- 卡片注册表：`web/src/config/modules.ts:33` `MODULES`，按 `section` 分 `monitoring / operations / system`。
|
||||
- 过滤逻辑：`web/src/features/workbench/WorkbenchPage.tsx:323` `visibleBySection` —— 只按权限（`adminOnly` / `requiredScope`）和 `workbenchHidden` 过滤。
|
||||
- 渲染：`web/src/features/workbench/components/WorkbenchModules.tsx:23`，operations 是大磁贴网格，system 是底部一排小链接。
|
||||
- 后端无任何“用户偏好”存储；`AppUser`（`netx_api/models/auth.py:13`）只有账号字段。
|
||||
|
||||
## 设计决定
|
||||
|
||||
| 项 | 决定 | 理由 |
|
||||
|---|---|---|
|
||||
| 可自定义范围 | 只做 **operations 区的磁贴**（含 monitoring 合并进来的 UME 同步）。system 区链接行保持固定 | 系统管理入口少且是管理类，隐藏后反而找不到；需求说的是“功能卡片” |
|
||||
| 存储位置 | **服务端，按用户**：`app_user.prefs` JSON 列 | “每个人不一样”= 按账号；localStorage 只按浏览器，换机就丢 |
|
||||
| 本地缓存 | 同时写一份 localStorage（key 带 user id），首屏先用缓存渲染，避免卡片闪烁 | 首屏体验 |
|
||||
| 数据结构 | `{ order: string[], hidden: string[] }`，存的是“隐藏了哪些”而不是“显示哪些” | 以后新增模块默认对所有人出现，不需要每人手动开启 |
|
||||
| 新模块默认隐藏 | `ModuleDefinition` 增加可选 `defaultHidden?: boolean`；不在 `order` 里的这类模块视为隐藏 | 给以后“小众功能”留开关 |
|
||||
| 权限 | 没权限的模块永远不出现在卡片区，也不出现在“可添加”列表；偏好里残留的 id 静默忽略 | 偏好不能绕过权限 |
|
||||
| 交互 | 卡片区右上角一个小齿轮 → 进入**原地编辑模式**：已显示卡片可拖拽排序、右上角 × 移除；下方出现“可添加”托盘（灰显卡片，点击加入）；按钮只有「恢复默认」「完成」 | 和 iOS 主屏编辑一致，最少文案；不弹窗 |
|
||||
| 拖拽实现 | HTML5 原生 drag & drop，不引入新依赖 | 卡片数量小（<20），原生够用；项目里没有 dnd 库 |
|
||||
| 保存时机 | 点「完成」时一次 PUT；失败则 toast 并保留编辑状态 | 避免拖动过程中频繁请求 |
|
||||
| 全部隐藏 | 允许，但功能区显示一个空态（只有齿轮 + 一句“添加功能”） | 不阻止用户，但要能找回 |
|
||||
|
||||
## 任务
|
||||
|
||||
### 后端
|
||||
|
||||
**B1 — `app_user.prefs` 列**
|
||||
- `netx_api/models/auth.py:13` `AppUser` 加 `prefs: Mapped[dict] = mapped_column(_JsonType, default=dict)`。
|
||||
- 新迁移 `alembic/versions/20261013_user_prefs.py`，`down_revision = "20260922_biz_state_queued"`，`add_column("app_user", sa.Column("prefs", JSON, nullable=False, server_default="{}"))`。
|
||||
|
||||
**B2 — 读写接口**（`netx_api/auth_router.py`，紧跟 `api_me` 之后，约 :273）
|
||||
- `GET /v1/auth/me/prefs` → `{"prefs": {...}}`
|
||||
- `PUT /v1/auth/me/prefs/{key}`，body 为该 key 的值；`key` 白名单，目前只允许 `workbench`。
|
||||
- 校验 `workbench`：`{order: list[str], hidden: list[str]}`，每个列表 ≤ 64 项、每个 id ≤ 64 字符、只允许 `[a-z0-9-]`；其他字段丢弃。超限 422。
|
||||
- 只改自己的偏好（`require_user`），不需要额外 scope；不写审计（纯 UI 偏好，避免刷屏）。
|
||||
- `/v1/auth/me` 响应里顺带返回 `prefs`，前端登录后一次拿到，不多一次请求。
|
||||
|
||||
### 前端
|
||||
|
||||
**F1 — 注册表扩展** `web/src/config/modules.ts`
|
||||
- `ModuleDefinition` 加 `defaultHidden?: boolean`。
|
||||
- 新增 `export function customizableModules()`：operations + monitoring 中非 `workbenchHidden` 的模块（即当前 `displaySections` 里 operations 那组的来源）。
|
||||
|
||||
**F2 — 偏好 hook** 新建 `web/src/features/workbench/useWorkbenchLayout.ts`
|
||||
- 输入：权限过滤后的可用模块列表；输出：`{ shown: ModuleDefinition[], available: ModuleDefinition[], save(order, hidden), reset() }`。
|
||||
- 解析规则（纯函数 `resolveLayout(all, prefs)`，单独导出便于测试）：
|
||||
1. 先按 `prefs.order` 排已知 id，丢弃不存在/无权限的 id；
|
||||
2. 剩余模块按注册表顺序追加到末尾；
|
||||
3. 在 `hidden` 中的、或 `defaultHidden` 且不在 `order` 中的，归入 `available`。
|
||||
- 数据来源：`AuthContext` 的 `me.prefs.workbench` → 没有则 localStorage 缓存 `netx.wb.layout.<userId>` → 默认。
|
||||
- 保存：PUT 成功后更新 AuthContext 中的 prefs 和 localStorage。localStorage 读写全部 try/catch。
|
||||
|
||||
**F3 — AuthContext** `web/src/features/auth/AuthContext.tsx:48`
|
||||
- `refreshMe` / `login` 把响应里的 `prefs` 存进 state，并暴露 `prefs` 与 `setPrefs`。
|
||||
|
||||
**F4 — 编辑模式 UI** `web/src/features/workbench/components/WorkbenchModules.tsx`
|
||||
- `WorkbenchPage.tsx:334` `displaySections` 的 operations 组改为用 `useWorkbenchLayout().shown`。
|
||||
- 卡片区右上角齿轮按钮（`ConsoleIcon` 加一个 `settings` 图标），`aria-label` = “自定义功能区”。
|
||||
- 编辑态：
|
||||
- 磁贴轻微抖动或描边（遵循 glass 风格：**不发光**，只用 `--nx-line` 边框）、右上角 × 按钮；
|
||||
- `draggable`，`onDragStart/onDragOver/onDrop` 重排本地草稿数组；
|
||||
- 下方“可添加”托盘：灰显磁贴，点击加入末尾；托盘为空时不显示；
|
||||
- 底部两个按钮：「恢复默认」（清空草稿为默认）、「完成」（保存）；Esc = 放弃草稿退出。
|
||||
- 编辑态下点击磁贴不打开模块。
|
||||
- 非编辑态 shown 为空：显示空态 + “添加功能”按钮（直接进入编辑态）。
|
||||
- 样式写在 `web/src/styles/pages/workbench.css`，颜色只用现有 token（蓝黑 200–215°，无紫色）。
|
||||
|
||||
**F5 — 文案** `web/src/i18n/locales/{zh,en}/workbench.ts`
|
||||
- 新增：`customize`（自定义功能区）、`customizeDone`（完成）、`customizeReset`（恢复默认）、`customizeAvailable`（可添加）、`customizeEmpty`（添加功能）、`customizeSaveFailed`（保存失败）。只这些，不加说明文字。
|
||||
|
||||
## 验收
|
||||
|
||||
| ID | 场景 | 期望 |
|
||||
|---|---|---|
|
||||
| A1 | 新用户首次登录 | 卡片与改动前完全一致（顺序、数量） |
|
||||
| A2 | 隐藏「拓扑」、把「网元管理」拖到第一位，点完成 | 立即生效；刷新、换浏览器登录同一账号仍是该布局 |
|
||||
| A3 | 用户 A 定制后，用户 B 登录 | B 不受影响 |
|
||||
| A4 | 在 `MODULES` 新增一个模块 | 已定制过的用户自动在末尾看到它；若设了 `defaultHidden` 则出现在“可添加”里 |
|
||||
| A5 | 偏好里包含无权限模块 id（如去掉用户 `ne:read` 后） | 不显示、不出现在可添加；不报错 |
|
||||
| A6 | 全部移除后完成 | 显示空态，点“添加功能”可恢复 |
|
||||
| A7 | 编辑中按 Esc / 保存失败（断网） | Esc 恢复原布局；失败提示并停留在编辑态 |
|
||||
| A8 | `PUT /v1/auth/me/prefs/workbench` 传非法 key、超长列表、非法字符 | 404 / 422 |
|
||||
| A9 | 视觉 | 编辑态无发光、无紫色，与现有 glass 风格一致 |
|
||||
|
||||
## 回归命令
|
||||
|
||||
```bash
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
```bash
|
||||
python -m pytest tests/test_auth.py tests/test_user_prefs.py -q
|
||||
```
|
||||
|
||||
```bash
|
||||
cd web && npm run lint && npm run build
|
||||
```
|
||||
|
||||
- 新增 `tests/test_user_prefs.py`：覆盖 B2 的读写、白名单、校验、用户隔离（A3、A8）。
|
||||
- `resolveLayout` 纯函数若项目有前端单测则补上（A1、A4、A5）；没有则在验收时手测。
|
||||
|
||||
## 以后可扩展（本次不做）
|
||||
|
||||
- 同一机制用 `prefs.workbench.panels` 控制右侧任务栏、网元/拓扑/调度三张统计卡的显隐。
|
||||
- 管理员给角色设默认布局（`role_default_prefs`），用户未定制时用角色默认。
|
||||
|
|
@ -15,7 +15,6 @@ from .schema_patches import (
|
|||
apply_topology_schema_safety_net,
|
||||
run_alembic_upgrade_to_head,
|
||||
)
|
||||
from .ne_crypto import ensure_credential_secret_key
|
||||
from .security_bootstrap import assert_secure_defaults_or_exit
|
||||
from .ume_runtime import start_api_sideband_threads, start_device_schedulers
|
||||
import netx_api.ume_support as ume_support
|
||||
|
|
@ -43,8 +42,6 @@ def _configure_ume_diag_logging() -> None:
|
|||
def run_api_startup() -> None:
|
||||
"""Full API boot sequence previously inlined in ``main.on_startup``."""
|
||||
assert_secure_defaults_or_exit()
|
||||
# Persist Fernet key for managed-NE passwords (same idea as JWT secret file).
|
||||
ensure_credential_secret_key()
|
||||
_configure_ume_diag_logging()
|
||||
_log.info(
|
||||
"startup: ne_exec_policy_enabled=%s",
|
||||
|
|
@ -159,25 +156,6 @@ def run_api_startup() -> None:
|
|||
)
|
||||
except Exception:
|
||||
_log.exception("startup: biz_state collect recovery failed")
|
||||
try:
|
||||
from .biz_state.compare_service import recover_interrupted_compares_on_startup
|
||||
|
||||
cmp_rec = recover_interrupted_compares_on_startup(db)
|
||||
if cmp_rec.get("runs"):
|
||||
_log.info(
|
||||
"startup: cancelled %s interrupted biz compare run(s)",
|
||||
cmp_rec.get("runs"),
|
||||
)
|
||||
except Exception:
|
||||
_log.exception("startup: biz compare recovery failed")
|
||||
try:
|
||||
from .biz_migration.service import recover_interrupted_evaluations_on_startup
|
||||
|
||||
n_eval = recover_interrupted_evaluations_on_startup(db)
|
||||
if n_eval:
|
||||
_log.info("startup: cancelled %s interrupted cutover evaluation(s)", n_eval)
|
||||
except Exception:
|
||||
_log.exception("startup: cutover evaluation recovery failed")
|
||||
try:
|
||||
from .port_traffic_migrate import backfill_port_traffic_series
|
||||
|
||||
|
|
|
|||
|
|
@ -2,12 +2,10 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
|
||||
from fastapi import APIRouter, Depends, Query, Request, Response
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.attributes import flag_modified
|
||||
|
||||
from .auth_cookies import clear_auth_cookies, read_refresh_cookie, set_auth_cookies
|
||||
from .auth_deps import AuthContext, require_admin, require_user
|
||||
|
|
@ -32,13 +30,11 @@ from .auth_service import (
|
|||
list_users,
|
||||
login_issue_token,
|
||||
refresh_login_tokens,
|
||||
delete_api_token,
|
||||
revoke_api_token,
|
||||
revoke_auth_session_for_user,
|
||||
revoke_auth_sessions,
|
||||
update_api_token,
|
||||
update_user,
|
||||
user_prefs,
|
||||
user_public,
|
||||
write_audit,
|
||||
)
|
||||
|
|
@ -49,14 +45,9 @@ from .auth_rate_limit import (
|
|||
)
|
||||
from .client_ip import resolve_client_ip
|
||||
from .db import get_db
|
||||
from .models import AppUser
|
||||
|
||||
router = APIRouter(tags=["auth"])
|
||||
|
||||
_PREF_KEYS = frozenset({"workbench"})
|
||||
_PREF_ID_RE = re.compile(r"^[a-z0-9-]{1,64}$")
|
||||
_PREF_LIST_MAX = 64
|
||||
|
||||
|
||||
def _client_meta(request: Request) -> tuple[str, str]:
|
||||
ip = resolve_client_ip(request)
|
||||
|
|
@ -64,34 +55,6 @@ def _client_meta(request: Request) -> tuple[str, str]:
|
|||
return ip, ua
|
||||
|
||||
|
||||
def _validate_id_list(value: Any, *, field: str) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
raise HTTPException(status_code=422, detail=f"invalid_{field}")
|
||||
if len(value) > _PREF_LIST_MAX:
|
||||
raise HTTPException(status_code=422, detail=f"{field}_too_long")
|
||||
out: list[str] = []
|
||||
for item in value:
|
||||
if not isinstance(item, str) or not _PREF_ID_RE.fullmatch(item):
|
||||
raise HTTPException(status_code=422, detail=f"invalid_{field}_item")
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
def _validate_workbench_prefs(value: Any) -> dict[str, list[str]]:
|
||||
if not isinstance(value, dict):
|
||||
raise HTTPException(status_code=422, detail="invalid_workbench_prefs")
|
||||
return {
|
||||
"order": _validate_id_list(value.get("order", []), field="order"),
|
||||
"hidden": _validate_id_list(value.get("hidden", []), field="hidden"),
|
||||
}
|
||||
|
||||
|
||||
def _validate_pref_value(key: str, value: Any) -> Any:
|
||||
if key == "workbench":
|
||||
return _validate_workbench_prefs(value)
|
||||
raise HTTPException(status_code=404, detail="unknown_pref_key")
|
||||
|
||||
|
||||
def _token_response(response: Response, request: Request, out: dict[str, Any]) -> dict[str, Any]:
|
||||
access = str(out.get("access_token") or "")
|
||||
refresh = str(out.get("refresh_token") or "")
|
||||
|
|
@ -305,39 +268,9 @@ def api_me(ctx: Annotated[AuthContext, Depends(require_user)]) -> dict[str, Any]
|
|||
"auth_via": ctx.auth_via,
|
||||
"scopes": sorted(ctx.scopes),
|
||||
"all_scopes": sorted(ALL_SCOPES),
|
||||
"prefs": user_prefs(ctx.user),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/v1/auth/me/prefs")
|
||||
def api_get_prefs(ctx: Annotated[AuthContext, Depends(require_user)]) -> dict[str, Any]:
|
||||
return {"prefs": user_prefs(ctx.user)}
|
||||
|
||||
|
||||
@router.put("/v1/auth/me/prefs/{key}")
|
||||
def api_put_pref(
|
||||
key: str,
|
||||
body: dict[str, Any],
|
||||
ctx: Annotated[AuthContext, Depends(require_user)],
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
if key not in _PREF_KEYS:
|
||||
raise HTTPException(status_code=404, detail="unknown_pref_key")
|
||||
validated = _validate_pref_value(key, body)
|
||||
# Reload in this session: ctx.user may be the middleware's detached copy, or the
|
||||
# synthetic "system" user when auth is disabled (no row — never insert it).
|
||||
user = db.get(AppUser, ctx.user.id)
|
||||
if user is None:
|
||||
raise HTTPException(status_code=409, detail="prefs_unavailable")
|
||||
prefs = user_prefs(user)
|
||||
prefs[key] = validated
|
||||
user.prefs = prefs
|
||||
flag_modified(user, "prefs")
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
return {"prefs": user_prefs(user)}
|
||||
|
||||
|
||||
@router.post("/v1/auth/change-password")
|
||||
def api_change_password(
|
||||
body: ChangePasswordRequest,
|
||||
|
|
@ -604,27 +537,3 @@ def api_revoke_token(
|
|||
detail={"token_id": row.id, "name": row.name},
|
||||
)
|
||||
return {"ok": True, "id": row.id}
|
||||
|
||||
|
||||
@router.delete("/v1/api-tokens/{token_id}/purge")
|
||||
def api_purge_token(
|
||||
token_id: str,
|
||||
request: Request,
|
||||
ctx: Annotated[AuthContext, Depends(require_user)],
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
row = delete_api_token(db, token_id=token_id, actor=ctx.user)
|
||||
ip, ua = _client_meta(request)
|
||||
write_audit(
|
||||
db,
|
||||
action="api_tokens.purge",
|
||||
actor_user_id=ctx.user.id,
|
||||
actor_username=ctx.user.username,
|
||||
method="DELETE",
|
||||
path=f"/v1/api-tokens/{token_id}/purge",
|
||||
status_code=200,
|
||||
client_ip=ip,
|
||||
user_agent=ua,
|
||||
detail={"token_id": row.id, "name": row.name},
|
||||
)
|
||||
return {"ok": True, "id": row.id}
|
||||
|
|
|
|||
|
|
@ -76,11 +76,6 @@ def user_public(user: AppUser) -> dict[str, Any]:
|
|||
}
|
||||
|
||||
|
||||
def user_prefs(user: AppUser) -> dict[str, Any]:
|
||||
raw = getattr(user, "prefs", None)
|
||||
return dict(raw) if isinstance(raw, dict) else {}
|
||||
|
||||
|
||||
def sanitize_detail(detail: Any) -> Any:
|
||||
"""Recursively drop secret-looking keys from audit detail payloads."""
|
||||
if isinstance(detail, dict):
|
||||
|
|
@ -436,7 +431,6 @@ def login_issue_token(
|
|||
"expires_in": ttl,
|
||||
"refresh_expires_in": refresh_ttl,
|
||||
"user": user_public(user),
|
||||
"prefs": user_prefs(user),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -673,18 +667,6 @@ def revoke_api_token(db: Session, *, token_id: str, actor: AppUser) -> ApiToken:
|
|||
return row
|
||||
|
||||
|
||||
def delete_api_token(db: Session, *, token_id: str, actor: AppUser) -> ApiToken:
|
||||
"""Permanently remove an API key row (revoked or active)."""
|
||||
row = db.query(ApiToken).filter(ApiToken.id == str(token_id)).one_or_none()
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="api_token_not_found")
|
||||
if actor.role != "admin" and row.user_id != actor.id:
|
||||
raise HTTPException(status_code=403, detail="forbidden")
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return row
|
||||
|
||||
|
||||
def update_api_token(
|
||||
db: Session,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -1,120 +0,0 @@
|
|||
"""Coalesced collect-completion events for live cutover evaluation.
|
||||
|
||||
One worker per process bounds concurrency. Project row locks in run_evaluate
|
||||
serialize persistence across processes. No device collection runs here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import BizMigrationBatch, BizMigrationProject, BizMigrationRun, BizMonitorTemplate, BizStateBatch
|
||||
from . import service
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
_lock = threading.Lock()
|
||||
_pending: set[str] = set()
|
||||
_worker_running = False
|
||||
|
||||
|
||||
def try_auto_monitor_for_task(db: Session, task_id: str) -> int:
|
||||
count = 0
|
||||
# Bindings are JSON for multiple sampling intervals; match in Python after
|
||||
# limiting to projects with an active operation batch.
|
||||
projects = db.query(BizMigrationProject).filter(
|
||||
BizMigrationProject.status != "done",
|
||||
BizMigrationProject.id.in_(db.query(BizMigrationBatch.project_id).filter(
|
||||
BizMigrationBatch.status == "active")),
|
||||
).all()
|
||||
for candidate in projects:
|
||||
tids = service._all_hf_task_ids(candidate, "old") + service._all_hf_task_ids(candidate, "new")
|
||||
if task_id not in tids or not candidate.old_baseline_batch_id or not candidate.new_baseline_batch_id:
|
||||
continue
|
||||
# Lock before deduplication so two worker processes cannot save the same
|
||||
# snapshot. run_evaluate holds the same lock through the transaction.
|
||||
proj = db.query(BizMigrationProject).filter(
|
||||
BizMigrationProject.id == candidate.id).with_for_update().populate_existing().first()
|
||||
batch = db.query(BizMigrationBatch).filter(
|
||||
BizMigrationBatch.project_id == proj.id, BizMigrationBatch.status == "active"
|
||||
).order_by(BizMigrationBatch.started_at.desc()).first()
|
||||
if not batch or proj.status == "done":
|
||||
db.rollback()
|
||||
continue
|
||||
if db.query(BizMigrationRun.id).filter(
|
||||
BizMigrationRun.project_id == proj.id, BizMigrationRun.status == "running"
|
||||
).first():
|
||||
db.rollback() # a manual / acceptance evaluation owns the project right now
|
||||
continue
|
||||
mt = db.get(BizMonitorTemplate, proj.monitor_template_id) if proj.monitor_template_id else None
|
||||
if not mt:
|
||||
db.rollback()
|
||||
_log.error("cutover monitor template missing project=%s", proj.id)
|
||||
continue
|
||||
sheets, overrides, defaults, norms = service.resolve_evaluate_sheets(db, mt)
|
||||
samples = []
|
||||
for tid in tids:
|
||||
sample = db.query(BizStateBatch).filter(
|
||||
BizStateBatch.task_id == tid, BizStateBatch.status.notin_(("running", "queued"))
|
||||
).order_by(BizStateBatch.started_at.desc(), BizStateBatch.id.desc()).first()
|
||||
samples.append([tid, sample.id if sample else "", sample.status if sample else ""])
|
||||
payload = [samples, proj.old_baseline_batch_id, proj.new_baseline_batch_id,
|
||||
batch.expect_set_json, service._port_map_dict(db, proj.mapping_id),
|
||||
sheets, overrides, defaults, norms, service.resolve_collect_metric_ids(db, proj)]
|
||||
fingerprint = hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
last = db.query(BizMigrationRun).filter(
|
||||
BizMigrationRun.batch_id == batch.id, BizMigrationRun.status == "success").order_by(
|
||||
BizMigrationRun.created_at.desc()).first()
|
||||
if last and (last.summary_json or {}).get("auto_fingerprint") == fingerprint:
|
||||
db.rollback()
|
||||
continue
|
||||
previous_samples = {s[0]: (s[1], s[2]) for s in (last.summary_json or {}).get("auto_samples", [])} if last else {}
|
||||
if last and not previous_samples:
|
||||
for sample in (last.summary_json or {}).get("metric_batches", {}).values():
|
||||
for side in ("old", "new"):
|
||||
previous_samples[sample.get(f"{side}_task_id", "")] = (
|
||||
sample.get(f"{side}_batch_id", ""), sample.get(f"{side}_status", "success"))
|
||||
changed_tasks = {tid for tid, bid, status in samples if previous_samples.get(tid) != (bid, status)}
|
||||
changed_metrics = {mid for side in ("old", "new") for binding in service._hf_bindings(proj, side)
|
||||
if binding["task_id"] in changed_tasks for mid in binding.get("metric_ids") or payload[-1]}
|
||||
try:
|
||||
result = service.run_evaluate(db, batch_id=batch.id, purpose="auto", _commit=False,
|
||||
_metric_ids=changed_metrics)
|
||||
run = db.get(BizMigrationRun, result["id"])
|
||||
run.summary_json = {**run.summary_json, "auto_fingerprint": fingerprint, "auto_samples": samples}
|
||||
db.commit()
|
||||
count += 1
|
||||
except Exception:
|
||||
db.rollback()
|
||||
_log.exception("cutover monitor evaluate failed project=%s", proj.id)
|
||||
return count
|
||||
|
||||
|
||||
def schedule_auto_monitor_for_task(task_id: str) -> None:
|
||||
global _worker_running
|
||||
with _lock:
|
||||
_pending.add(str(task_id))
|
||||
if _worker_running:
|
||||
return
|
||||
_worker_running = True
|
||||
|
||||
def work() -> None:
|
||||
global _worker_running
|
||||
from ..db import SessionLocal
|
||||
|
||||
while True:
|
||||
with _lock:
|
||||
if not _pending:
|
||||
_worker_running = False
|
||||
return
|
||||
tid = _pending.pop()
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
try_auto_monitor_for_task(db, tid)
|
||||
except Exception:
|
||||
_log.exception("cutover monitor worker failed task=%s", tid)
|
||||
|
||||
threading.Thread(target=work, name="biz-cutover-monitor", daemon=True).start()
|
||||
|
|
@ -4,10 +4,7 @@ from __future__ import annotations
|
|||
|
||||
from typing import Any
|
||||
|
||||
from ..biz_state.compare_engine import apply_port_map, compare_rows, key_tuple
|
||||
from ..biz_state.compare_rules import (
|
||||
effective_compare_fields, field_rule_map, scalar_text, values_equal,
|
||||
)
|
||||
from ..biz_state.compare_engine import apply_port_map, compare_rows
|
||||
|
||||
PORT_METRIC_ID = "interface_brief"
|
||||
PORT_STATUS_FIELDS = ("admin", "phy", "prot")
|
||||
|
|
@ -25,7 +22,7 @@ def normalize_expect_key(raw: str) -> str:
|
|||
|
||||
Accepts legacy ``a|b`` multi-field keys and already-normalized ``\\x1f`` keys.
|
||||
"""
|
||||
s = scalar_text(raw).strip()
|
||||
s = str(raw or "").strip()
|
||||
if not s:
|
||||
return ""
|
||||
if KEY_SEP in s:
|
||||
|
|
@ -68,13 +65,13 @@ def classify_status(row: dict[str, Any] | None, sheet_override: dict[str, Any] |
|
|||
return "none"
|
||||
down_vals = {str(x).lower() for x in (ov.get("down_values") or ["down"])}
|
||||
up_vals = {str(x).lower() for x in (ov.get("up_values") or ["up"])}
|
||||
vals = [scalar_text(row.get(f)).strip().lower() for f in fields]
|
||||
vals = [str(row.get(f) or "").strip().lower() for f in fields]
|
||||
vals = [v for v in vals if v]
|
||||
if not vals:
|
||||
return "none"
|
||||
if any(v in down_vals for v in vals):
|
||||
return "down"
|
||||
if len(vals) == len(fields) and all(v in up_vals for v in vals):
|
||||
if vals and all(v in up_vals for v in vals):
|
||||
return "up"
|
||||
return "other"
|
||||
|
||||
|
|
@ -108,11 +105,11 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]:
|
|||
if it.get("key") is not None:
|
||||
k = it.get("key")
|
||||
if isinstance(k, (list, tuple)):
|
||||
joined = KEY_SEP.join(scalar_text(p).strip() for p in k)
|
||||
if joined.strip(KEY_SEP):
|
||||
joined = KEY_SEP.join(str(p).strip() for p in k if str(p).strip())
|
||||
if joined:
|
||||
bucket.add(joined)
|
||||
else:
|
||||
s = normalize_expect_key(scalar_text(k))
|
||||
s = normalize_expect_key(str(k or ""))
|
||||
if s:
|
||||
bucket.add(s)
|
||||
keys = it.get("keys")
|
||||
|
|
@ -120,14 +117,14 @@ def parse_expect_set(raw: dict[str, Any] | None) -> dict[str, set[str]]:
|
|||
# Flat list of segments → one composite key; else each entry is a key
|
||||
# (string or nested list/tuple of segments).
|
||||
if keys and all(not isinstance(x, (list, tuple, dict)) for x in keys):
|
||||
joined = KEY_SEP.join(scalar_text(x).strip() for x in keys)
|
||||
if joined.strip(KEY_SEP):
|
||||
joined = KEY_SEP.join(str(x).strip() for x in keys if str(x).strip())
|
||||
if joined:
|
||||
bucket.add(joined)
|
||||
else:
|
||||
for x in keys:
|
||||
if isinstance(x, (list, tuple)):
|
||||
joined = KEY_SEP.join(scalar_text(p).strip() for p in x)
|
||||
if joined.strip(KEY_SEP):
|
||||
joined = KEY_SEP.join(str(p).strip() for p in x if str(p).strip())
|
||||
if joined:
|
||||
bucket.add(joined)
|
||||
else:
|
||||
s = normalize_expect_key(str(x or ""))
|
||||
|
|
@ -222,7 +219,7 @@ def _field_tokens_from_row(row: dict[str, Any] | None, fields: set[str]) -> set[
|
|||
return set()
|
||||
out: set[str] = set()
|
||||
for f in fields:
|
||||
v = scalar_text(row.get(f)).strip()
|
||||
v = str(row.get(f) or "").strip()
|
||||
if v:
|
||||
out.add(field_token(f, v))
|
||||
return out
|
||||
|
|
@ -248,8 +245,8 @@ def _side_tokens(
|
|||
|
||||
def _norm_list_values(raw: Any) -> list[str]:
|
||||
if isinstance(raw, list):
|
||||
return [scalar_text(x).strip().lower() for x in raw if scalar_text(x).strip()]
|
||||
s = scalar_text(raw).strip()
|
||||
return [str(x).strip().lower() for x in raw if str(x).strip()]
|
||||
s = str(raw or "").strip()
|
||||
if not s:
|
||||
return []
|
||||
if "," in s:
|
||||
|
|
@ -269,22 +266,18 @@ def _eval_field_op(
|
|||
f = str(field or "").strip()
|
||||
if not f:
|
||||
return False
|
||||
raw = "" if not row else scalar_text(row.get(f)).strip()
|
||||
raw = "" if not row else str(row.get(f) or "").strip()
|
||||
val = raw.lower()
|
||||
o = str(op or "eq").strip().lower()
|
||||
if row and f not in row:
|
||||
# A missing parser field is not evidence for a negative condition such as
|
||||
# state != down. Only an explicit empty check may match that absence.
|
||||
return o == "empty"
|
||||
if o in ("changed", "diff"):
|
||||
base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower()
|
||||
base = "" if not base_row else str(base_row.get(f) or "").strip().lower()
|
||||
if not row:
|
||||
return False
|
||||
return base != val
|
||||
if o in ("unchanged", "same"):
|
||||
if not row:
|
||||
return False
|
||||
base = "" if not base_row else scalar_text(base_row.get(f)).strip().lower()
|
||||
base = "" if not base_row else str(base_row.get(f) or "").strip().lower()
|
||||
return base == val
|
||||
if o == "empty":
|
||||
return not val
|
||||
|
|
@ -575,7 +568,7 @@ def _match_field_token_rules(
|
|||
if not field or not allowed:
|
||||
continue
|
||||
row = old_row if side == "old" else new_row if side == "new" else None
|
||||
val = scalar_text((row or {}).get(field)).strip().lower()
|
||||
val = str((row or {}).get(field) or "").strip().lower()
|
||||
if val not in allowed:
|
||||
return False
|
||||
return True
|
||||
|
|
@ -756,10 +749,10 @@ def build_diff_index_from_compare(result: dict[str, Any]) -> dict[str, dict[str,
|
|||
continue
|
||||
key = d.get("key")
|
||||
if isinstance(key, dict):
|
||||
key_list = [scalar_text(v).strip() for v in key.values()]
|
||||
key_list = [str(v) for v in key.values()]
|
||||
ks = _key_str(key_list)
|
||||
elif isinstance(key, (list, tuple)):
|
||||
key_list = [scalar_text(x).strip() for x in key]
|
||||
key_list = [str(x) for x in key]
|
||||
ks = _key_str(key_list)
|
||||
else:
|
||||
ks = str(key or "")
|
||||
|
|
@ -801,18 +794,17 @@ def expect_keys_for_metric(
|
|||
return keys
|
||||
|
||||
|
||||
def _map_defines_iface_expect(
|
||||
iface_fields: list[str], port_map: dict[str, str], scope_by_port_map: bool | None = None,
|
||||
) -> bool:
|
||||
"""Whether a non-empty port map is this sheet's expect scope.
|
||||
def _map_defines_iface_expect(metric_id: str, iface_fields: list[str], port_map: dict[str, str]) -> bool:
|
||||
"""Iface sheets: a non-empty port map is the expect scope.
|
||||
|
||||
The sheet override decides (``scope_by_port_map``); the built-in presets set it to
|
||||
false for BGP / pseudowire sheets, which keep the batch expect picker. Unset means
|
||||
"an interface sheet with a map is scoped by the map".
|
||||
BGP / PW keep the batch expect picker even if a column is marked iface.
|
||||
"""
|
||||
if not port_map or not iface_fields:
|
||||
return False
|
||||
return True if scope_by_port_map is None else bool(scope_by_port_map)
|
||||
mid = str(metric_id or "").strip().lower()
|
||||
if "bgp" in mid or "pw" in mid or "pseudowire" in mid:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _iface_values(key_str: str, *, key_fields: list[str], iface_fields: list[str]) -> list[str]:
|
||||
|
|
@ -866,14 +858,6 @@ def _port_identity(key_fields: list[str], iface_fields: list[str]) -> bool:
|
|||
return all(str(f) in iface_set for f in key_fields)
|
||||
|
||||
|
||||
def _iface_selected(value: str, ports: set[str]) -> bool:
|
||||
"""Set lookup plus parent interface prefixes; independent of scope size."""
|
||||
if value in ports:
|
||||
return True
|
||||
parts = value.split(".")
|
||||
return any(".".join(parts[:i]) in ports for i in range(len(parts) - 1, 0, -1))
|
||||
|
||||
|
||||
def _unmapped_same_iface_anomaly(
|
||||
*,
|
||||
old_kind: str,
|
||||
|
|
@ -908,8 +892,6 @@ def _remap_key_str(
|
|||
return key_str
|
||||
if not port_map:
|
||||
return key_str
|
||||
if key_fields and not set(key_fields).intersection(iface_fields):
|
||||
return key_str
|
||||
if (not key_fields or len(key_fields) == 1) and (
|
||||
key_str in port_map
|
||||
or any(
|
||||
|
|
@ -946,8 +928,6 @@ def _reverse_remap_key_str(
|
|||
return key_str
|
||||
if not rev_map:
|
||||
return key_str
|
||||
if key_fields and not set(key_fields).intersection(iface_fields):
|
||||
return key_str
|
||||
if (not key_fields or len(key_fields) == 1) and (
|
||||
key_str in rev_map
|
||||
or any(
|
||||
|
|
@ -1023,16 +1003,13 @@ def row_match_key(
|
|||
key_fields: list[str],
|
||||
iface_fields: list[str],
|
||||
rules: list[dict[str, str]] | None,
|
||||
normalizer: Any = None,
|
||||
) -> str:
|
||||
from ..biz_state.iface_normalize import apply_iface_normalize
|
||||
|
||||
data = strip_netx(row)
|
||||
if normalizer is not None:
|
||||
data = normalizer(dict(data))
|
||||
elif iface_fields and rules:
|
||||
if iface_fields and rules:
|
||||
data = apply_iface_normalize(dict(data), iface_fields=iface_fields, rules=rules)
|
||||
return KEY_SEP.join(scalar_text(data.get(k)).strip() for k in key_fields)
|
||||
return KEY_SEP.join(str(data.get(k) or "").strip() for k in key_fields)
|
||||
|
||||
|
||||
def index_raw_by_match_key(
|
||||
|
|
@ -1041,14 +1018,11 @@ def index_raw_by_match_key(
|
|||
key_fields: list[str],
|
||||
iface_fields: list[str],
|
||||
rules: list[dict[str, str]] | None,
|
||||
normalizer: Any = None,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""First raw row per internal match key (normalized iface, raw elsewhere)."""
|
||||
idx: dict[str, dict[str, Any]] = {}
|
||||
for raw in raw_rows:
|
||||
ks = row_match_key(
|
||||
raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules, normalizer=normalizer,
|
||||
)
|
||||
ks = row_match_key(raw, key_fields=key_fields, iface_fields=iface_fields, rules=rules)
|
||||
if ks and ks not in idx:
|
||||
idx[ks] = raw
|
||||
return idx
|
||||
|
|
@ -1059,7 +1033,7 @@ def display_key_from_raw(raw: dict[str, Any] | None, key_fields: list[str]) -> s
|
|||
if not raw or not key_fields:
|
||||
return ""
|
||||
data = strip_netx(raw)
|
||||
return "|".join(scalar_text(data.get(k)).strip() for k in key_fields)
|
||||
return "|".join(str(data.get(k) or "").strip() for k in key_fields)
|
||||
|
||||
|
||||
def iface_lineage(
|
||||
|
|
@ -1069,7 +1043,6 @@ def iface_lineage(
|
|||
rules: list[dict[str, str]] | None,
|
||||
port_map: dict[str, str] | None,
|
||||
apply_map: bool,
|
||||
namer: Any = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Per iface field: raw → normalized → (optional) mapped."""
|
||||
from ..biz_state.iface_normalize import normalize_iface_name, resolve_mapped_iface
|
||||
|
|
@ -1080,8 +1053,8 @@ def iface_lineage(
|
|||
pmap = port_map or {}
|
||||
out: list[dict[str, Any]] = []
|
||||
for f in iface_fields:
|
||||
raw_v = scalar_text(data.get(f)).strip()
|
||||
norm_v = namer(raw_v) if namer is not None else (normalize_iface_name(raw_v, rules) if rules else raw_v)
|
||||
raw_v = str(data.get(f) or "").strip()
|
||||
norm_v = normalize_iface_name(raw_v, rules) if rules else raw_v
|
||||
mapped = False
|
||||
map_to = ""
|
||||
if apply_map and pmap and norm_v:
|
||||
|
|
@ -1119,9 +1092,6 @@ def evaluate_metric_dual(
|
|||
out_of_expect: str = "strict",
|
||||
sheet_id: str = "",
|
||||
iface_normalize_rules: list[dict[str, str]] | None = None,
|
||||
previous_expect: dict[str, set[str]] | None = None,
|
||||
allow_mapping_scope: bool = True,
|
||||
pipeline: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run old vs old-baseline, new vs new-baseline (or mapped old baseline), dual merge.
|
||||
|
||||
|
|
@ -1129,21 +1099,15 @@ def evaluate_metric_dual(
|
|||
Board / AI fields ``old_key`` / ``new_key`` and ``old`` / ``new`` stay **raw**
|
||||
device values (A/B), never the post-map BB form.
|
||||
"""
|
||||
from ..biz_state.identity_pipeline import legacy_pipeline
|
||||
|
||||
# Old device = "before" side, new device = "after" side of the identity pipeline.
|
||||
pipe = pipeline or legacy_pipeline(
|
||||
key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields,
|
||||
field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules,
|
||||
from ..biz_state.iface_normalize import (
|
||||
apply_iface_normalize_rows,
|
||||
normalize_iface_rules,
|
||||
)
|
||||
|
||||
sid = str(sheet_id or "").strip() or str(metric_id or "").strip()
|
||||
expect_keys = expect_keys_for_metric(
|
||||
expect, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid
|
||||
)
|
||||
previous_keys = expect_keys_for_metric(
|
||||
previous_expect or {}, metric_id=metric_id, iface_fields=iface_fields, sheet_id=sid
|
||||
)
|
||||
ov = sheet_override or {}
|
||||
if ov.get("skip_dual"):
|
||||
return {
|
||||
|
|
@ -1170,44 +1134,39 @@ def evaluate_metric_dual(
|
|||
[dict(r) for r in new_baseline_rows] if new_baseline_rows is not None else None
|
||||
)
|
||||
|
||||
def normalized_key(key: str) -> str:
|
||||
parts = key.split(KEY_SEP) if KEY_SEP in key else ([key] if len(key_fields) == 1 else key.split("|"))
|
||||
if len(parts) != len(key_fields):
|
||||
return key
|
||||
return KEY_SEP.join(pipe.normalize_name(p, "before") if f in iface_fields else p
|
||||
for p, f in zip(parts, key_fields))
|
||||
|
||||
def norm_before(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return pipe.normalize(row, "before")
|
||||
|
||||
def norm_after(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return pipe.normalize(row, "after")
|
||||
|
||||
expect_keys = {normalized_key(key) for key in expect_keys}
|
||||
previous_keys = {normalized_key(key) for key in previous_keys}
|
||||
old_baseline_rows = [norm_before(strip_netx(r)) for r in raw_old_base]
|
||||
old_current_rows = [norm_before(strip_netx(r)) for r in raw_old_cur]
|
||||
new_current_rows = [norm_after(strip_netx(r)) for r in raw_new_cur]
|
||||
norm_rules = normalize_iface_rules(iface_normalize_rules)
|
||||
old_baseline_rows = apply_iface_normalize_rows(
|
||||
[strip_netx(r) for r in raw_old_base], iface_fields=iface_fields, rules=norm_rules
|
||||
)
|
||||
old_current_rows = apply_iface_normalize_rows(
|
||||
[strip_netx(r) for r in raw_old_cur], iface_fields=iface_fields, rules=norm_rules
|
||||
)
|
||||
new_current_rows = apply_iface_normalize_rows(
|
||||
[strip_netx(r) for r in raw_new_cur], iface_fields=iface_fields, rules=norm_rules
|
||||
)
|
||||
if raw_new_base is not None:
|
||||
new_baseline_rows = [norm_after(strip_netx(r)) for r in raw_new_base]
|
||||
new_baseline_rows = apply_iface_normalize_rows(
|
||||
[strip_netx(r) for r in raw_new_base],
|
||||
iface_fields=iface_fields,
|
||||
rules=norm_rules,
|
||||
)
|
||||
else:
|
||||
new_baseline_rows = None
|
||||
|
||||
old_raw_cur_idx = index_raw_by_match_key(
|
||||
raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before
|
||||
raw_old_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
|
||||
)
|
||||
old_raw_base_idx = index_raw_by_match_key(
|
||||
raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_before
|
||||
raw_old_base, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
|
||||
)
|
||||
new_raw_cur_idx = index_raw_by_match_key(
|
||||
raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=None, normalizer=norm_after
|
||||
raw_new_cur, key_fields=key_fields, iface_fields=iface_fields, rules=norm_rules
|
||||
)
|
||||
new_raw_base_idx = index_raw_by_match_key(
|
||||
raw_new_base or [],
|
||||
key_fields=key_fields,
|
||||
iface_fields=iface_fields,
|
||||
rules=None,
|
||||
normalizer=norm_after,
|
||||
rules=norm_rules,
|
||||
)
|
||||
|
||||
old_cmp = compare_rows(
|
||||
|
|
@ -1219,8 +1178,6 @@ def evaluate_metric_dual(
|
|||
port_map=None,
|
||||
field_rules=field_rules,
|
||||
iface_normalize_rules=None, # already applied
|
||||
include_unchanged=True,
|
||||
duplicate_pairing=pipe.duplicate_pairing,
|
||||
)
|
||||
old_idx = build_diff_index_from_compare(old_cmp)
|
||||
|
||||
|
|
@ -1229,8 +1186,18 @@ def evaluate_metric_dual(
|
|||
if new_baseline_rows is not None:
|
||||
new_before = list(new_baseline_rows)
|
||||
new_baseline_mode = "provided"
|
||||
# A successfully collected empty baseline is valid on a replacement NE.
|
||||
# Collection completeness is checked by the service, not inferred from rows.
|
||||
# Explicit empty batch is as unusable as "no baseline" for presence semantics.
|
||||
if not new_before and (old_baseline_rows or new_current_rows):
|
||||
if port_map and iface_fields:
|
||||
new_before = [
|
||||
apply_port_map(r, iface_fields=iface_fields, port_map=port_map)
|
||||
for r in old_baseline_rows
|
||||
]
|
||||
new_baseline_mode = "port_mapped"
|
||||
new_baseline_missing = False
|
||||
else:
|
||||
new_baseline_missing = True
|
||||
new_baseline_mode = "empty"
|
||||
elif port_map and iface_fields:
|
||||
new_before = [
|
||||
apply_port_map(r, iface_fields=iface_fields, port_map=port_map)
|
||||
|
|
@ -1251,23 +1218,12 @@ def evaluate_metric_dual(
|
|||
compare_fields=compare_fields,
|
||||
port_map=None,
|
||||
field_rules=field_rules,
|
||||
include_unchanged=True,
|
||||
duplicate_pairing=pipe.duplicate_pairing,
|
||||
)
|
||||
new_idx = build_diff_index_from_compare(new_cmp)
|
||||
|
||||
rev_map = {v: k for k, v in port_map.items()}
|
||||
map_scoped = _map_defines_iface_expect(iface_fields, port_map, ov.get("scope_by_port_map"))
|
||||
# Mapping describes identity; an explicit batch selection defines scope.
|
||||
# Port selection also covers business rows attached to those interfaces.
|
||||
for scope, keys in ((expect, expect_keys), (previous_expect or {}, previous_keys)):
|
||||
ports = {pipe.normalize_name(p, "before") for p in scope.get("_ports") or set()}
|
||||
if ports and not keys and iface_fields:
|
||||
for ks, diff in old_idx.items():
|
||||
row = diff.get("before") or diff.get("after") or {}
|
||||
if any(_iface_selected(scalar_text(row.get(f)), ports) for f in iface_fields):
|
||||
keys.add(ks)
|
||||
if allow_mapping_scope and map_scoped and not any(expect.values()):
|
||||
map_scoped = _map_defines_iface_expect(metric_id, iface_fields, port_map)
|
||||
if map_scoped:
|
||||
scoped: set[str] = set()
|
||||
for ks in old_idx:
|
||||
if _key_in_port_map(
|
||||
|
|
@ -1287,21 +1243,6 @@ def evaluate_metric_dual(
|
|||
scoped.add(canon)
|
||||
expect_keys = scoped
|
||||
|
||||
# Duplicate keys only block when they sit inside this wave's scope.
|
||||
from collections import Counter
|
||||
|
||||
def _dup_keys(rows: list[dict[str, Any]]) -> set[str]:
|
||||
return {KEY_SEP.join(k) for k, n in Counter(key_tuple(r, key_fields) for r in rows).items() if n > 1}
|
||||
|
||||
in_scope = expect_keys | previous_keys
|
||||
new_dups = _dup_keys(new_current_rows) | _dup_keys(new_baseline_rows or [])
|
||||
flagged = (
|
||||
_dup_keys(old_baseline_rows) | _dup_keys(old_current_rows) | new_dups
|
||||
| {_reverse_remap_key_str(k, key_fields=key_fields, iface_fields=iface_fields, rev_map=rev_map)
|
||||
for k in new_dups}
|
||||
)
|
||||
duplicate_in_expect = sorted(flagged & in_scope)[:64]
|
||||
|
||||
canon_keys: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
|
|
@ -1312,8 +1253,6 @@ def evaluate_metric_dual(
|
|||
|
||||
for ok in sorted(expect_keys):
|
||||
_add(ok)
|
||||
for ok in sorted(previous_keys):
|
||||
_add(ok)
|
||||
for ks in sorted(old_idx):
|
||||
_add(ks)
|
||||
for ks in sorted(new_idx):
|
||||
|
|
@ -1331,11 +1270,6 @@ def evaluate_metric_dual(
|
|||
progress_total = 0
|
||||
anomaly = 0
|
||||
anomaly_in_expect = 0
|
||||
steady_outside = 0
|
||||
rules_by_field = field_rule_map(field_rules)
|
||||
integrity_fields = effective_compare_fields(compare_fields, field_rules)
|
||||
# Status transitions are interpreted by the monitor's correction rules.
|
||||
integrity_fields = [f for f in integrity_fields if f not in (ov.get("status_fields") or [])]
|
||||
|
||||
for old_ks in canon_keys:
|
||||
new_ks = _remap_key_str(
|
||||
|
|
@ -1345,12 +1279,11 @@ def evaluate_metric_dual(
|
|||
port_map=port_map,
|
||||
)
|
||||
in_exp = old_ks in expect_keys
|
||||
previous = old_ks in previous_keys and not in_exp
|
||||
if in_exp:
|
||||
progress_total += 1
|
||||
|
||||
od = old_idx.get(old_ks)
|
||||
nd = new_idx.get(new_ks)
|
||||
nd = new_idx.get(new_ks) or new_idx.get(old_ks)
|
||||
|
||||
old_kind = str((od or {}).get("kind") or "")
|
||||
new_kind = str((nd or {}).get("kind") or "")
|
||||
|
|
@ -1368,7 +1301,11 @@ def evaluate_metric_dual(
|
|||
old_st = classify_status(old_cur, ov)
|
||||
new_st = classify_status(new_cur, ov)
|
||||
|
||||
if map_scoped and not (in_exp or previous) and (
|
||||
# Mapped iface rows are the expect set. Unmapped rows stay out of the
|
||||
# migration compare, except same-name ports where old went down and the
|
||||
# new device's same port is up — that is anomaly, not a successful cutover.
|
||||
if map_scoped and not in_exp:
|
||||
if not (
|
||||
_port_identity(key_fields, iface_fields)
|
||||
and _unmapped_same_iface_anomaly(
|
||||
old_kind=old_kind,
|
||||
|
|
@ -1377,63 +1314,26 @@ def evaluate_metric_dual(
|
|||
new_status=new_st,
|
||||
)
|
||||
):
|
||||
continue
|
||||
verdict, color, rule_hit = "anomaly", "red", "unmapped_same_iface"
|
||||
else:
|
||||
verdict, color, rule_hit = dual_verdict_ex(
|
||||
old_kind=old_kind or "",
|
||||
new_kind=new_kind or "",
|
||||
in_expect=in_exp or previous,
|
||||
in_expect=in_exp,
|
||||
window_active=window_active,
|
||||
acceptance=acceptance,
|
||||
old_status=old_st,
|
||||
new_status=new_st,
|
||||
success_patterns=success_patterns or None,
|
||||
anomaly_patterns=anomaly_patterns or None,
|
||||
# Business drift outside this batch must remain visible.
|
||||
out_of_expect="strict",
|
||||
out_of_expect=out_of_expect,
|
||||
old_row=old_cur or None,
|
||||
new_row=new_cur or None,
|
||||
old_base=old_base or None,
|
||||
new_base=new_base or None,
|
||||
sheet_override=ov,
|
||||
)
|
||||
integrity_changes: list[dict[str, Any]] = []
|
||||
if in_exp or previous:
|
||||
old_departed = old_kind == "removed" or old_st == "down"
|
||||
new_healthy = bool(new_cur) and (
|
||||
new_st == "up" if ov.get("status_fields") else True
|
||||
)
|
||||
if old_departed and not new_healthy:
|
||||
verdict, color, rule_hit = "lost", "red", "service_not_received"
|
||||
elif verdict == "migrated":
|
||||
if not new_healthy:
|
||||
verdict, color, rule_hit = "anomaly", "red", "target_unhealthy"
|
||||
elif not old_base:
|
||||
verdict, color, rule_hit = "anomaly", "red", "old_service_not_in_baseline"
|
||||
else:
|
||||
reference = apply_port_map(old_base, iface_fields=iface_fields, port_map=port_map)
|
||||
integrity_changes = [
|
||||
{"field": f, "before": reference.get(f), "after": new_cur.get(f)}
|
||||
for f in integrity_fields
|
||||
if not values_equal(reference.get(f), new_cur.get(f), rule=rules_by_field.get(f))
|
||||
]
|
||||
if integrity_changes:
|
||||
verdict, color, rule_hit = "anomaly", "red", "service_content_changed"
|
||||
elif not old_departed and not success_patterns:
|
||||
verdict, color, rule_hit = (
|
||||
("unfinished", "red", "old_service_still_present") if acceptance
|
||||
else ("migrating", "yellow", "old_service_still_present")
|
||||
)
|
||||
if previous:
|
||||
if verdict == "migrated":
|
||||
verdict, rule_hit = "migrated_previous", "previous_batch_received"
|
||||
elif color != "red":
|
||||
verdict, color, rule_hit = "unfinished_previous", "red", "previous_batch_not_received"
|
||||
# Stable inventory is covered by the comparison and counted, but does
|
||||
# not need a full persisted evidence card on every high-frequency run.
|
||||
if not in_exp and verdict == "not_involved":
|
||||
steady_outside += 1
|
||||
continue
|
||||
if in_exp and verdict == "migrated" and color == "green":
|
||||
progress_ok += 1
|
||||
if color == "red":
|
||||
|
|
@ -1444,9 +1344,9 @@ def evaluate_metric_dual(
|
|||
# Prefer live raw rows; fall back to baseline raw. Never use port-mapped
|
||||
# synthetic "before" as the new-side display row.
|
||||
raw_old = old_raw_cur_idx.get(old_ks) or old_raw_base_idx.get(old_ks)
|
||||
raw_new = new_raw_cur_idx.get(new_ks)
|
||||
raw_new = new_raw_cur_idx.get(new_ks) or new_raw_cur_idx.get(old_ks)
|
||||
if not raw_new:
|
||||
raw_new = new_raw_base_idx.get(new_ks)
|
||||
raw_new = new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks)
|
||||
|
||||
display_old = display_key_from_raw(raw_old, key_fields) or old_ks
|
||||
display_new = display_key_from_raw(raw_new, key_fields) or (
|
||||
|
|
@ -1460,8 +1360,8 @@ def evaluate_metric_dual(
|
|||
new_disp = strip_netx(raw_new) if raw_new else {}
|
||||
old_base_raw = strip_netx(old_raw_base_idx.get(old_ks)) if old_raw_base_idx.get(old_ks) else {}
|
||||
new_base_raw = (
|
||||
strip_netx(new_raw_base_idx.get(new_ks))
|
||||
if new_raw_base_idx.get(new_ks)
|
||||
strip_netx(new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks))
|
||||
if (new_raw_base_idx.get(new_ks) or new_raw_base_idx.get(old_ks))
|
||||
else {}
|
||||
)
|
||||
|
||||
|
|
@ -1471,10 +1371,9 @@ def evaluate_metric_dual(
|
|||
"iface": iface_lineage(
|
||||
raw_old,
|
||||
iface_fields=iface_fields,
|
||||
rules=None,
|
||||
rules=norm_rules,
|
||||
port_map=port_map,
|
||||
apply_map=True,
|
||||
namer=lambda v: pipe.normalize_name(v, "before"),
|
||||
),
|
||||
},
|
||||
"new": {
|
||||
|
|
@ -1482,10 +1381,9 @@ def evaluate_metric_dual(
|
|||
"iface": iface_lineage(
|
||||
raw_new,
|
||||
iface_fields=iface_fields,
|
||||
rules=None,
|
||||
rules=norm_rules,
|
||||
port_map=None,
|
||||
apply_map=False,
|
||||
namer=lambda v: pipe.normalize_name(v, "after"),
|
||||
),
|
||||
},
|
||||
"port_map": {
|
||||
|
|
@ -1513,9 +1411,7 @@ def evaluate_metric_dual(
|
|||
"verdict": verdict,
|
||||
"color": color,
|
||||
"rule_hit": rule_hit,
|
||||
"integrity_changes": integrity_changes,
|
||||
"in_expect": in_exp,
|
||||
"previous_batch": previous,
|
||||
"old_kind": old_kind,
|
||||
"new_kind": new_kind,
|
||||
"old": old_disp,
|
||||
|
|
@ -1548,14 +1444,12 @@ def evaluate_metric_dual(
|
|||
|
||||
return {
|
||||
"metric_id": metric_id,
|
||||
"effective_scope_by_port_map": map_scoped,
|
||||
"old_summary": old_cmp.get("summary") or {},
|
||||
"new_summary": new_cmp.get("summary") or {},
|
||||
"progress_ok": progress_ok,
|
||||
"progress_total": progress_total,
|
||||
"anomaly": anomaly,
|
||||
"anomaly_in_expect": anomaly_in_expect,
|
||||
"steady_outside": steady_outside,
|
||||
"rows": rows_out,
|
||||
"new_baseline_mode": new_baseline_mode,
|
||||
"new_baseline_missing": new_baseline_missing,
|
||||
|
|
@ -1563,7 +1457,6 @@ def evaluate_metric_dual(
|
|||
+ int((new_cmp.get("summary") or {}).get("duplicate_keys_before") or 0),
|
||||
"duplicate_keys_after": int((old_cmp.get("summary") or {}).get("duplicate_keys_after") or 0)
|
||||
+ int((new_cmp.get("summary") or {}).get("duplicate_keys_after") or 0),
|
||||
"duplicate_in_expect": duplicate_in_expect,
|
||||
"duplicate_key_list": list(
|
||||
dict.fromkeys(
|
||||
list((old_cmp.get("summary") or {}).get("duplicate_key_list") or [])
|
||||
|
|
|
|||
|
|
@ -8,8 +8,7 @@ from uuid import uuid4
|
|||
from fastapi import HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import BizCompareTemplate, BizMigrationProject, BizMonitorTemplate
|
||||
from .validation import validate_monitor_rules
|
||||
from ..models import BizCompareTemplate, BizMonitorTemplate
|
||||
from ..timeutil import utcnow_naive
|
||||
from ..biz_state import compare_service as cmp_svc
|
||||
from ..biz_state.iface_normalize import default_zte_iface_normalize_rules
|
||||
|
|
@ -34,8 +33,6 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None
|
|||
effective = seen
|
||||
if not effective:
|
||||
effective = [PORT_METRIC_ID]
|
||||
ct = db.get(BizCompareTemplate, row.compare_template_id) if db and row.compare_template_id else None
|
||||
available = list(dict.fromkeys(str(s.get("metric_id") or "") for s in cmp_svc.template_metrics(ct))) if ct else effective
|
||||
out: dict[str, Any] = {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
|
|
@ -43,7 +40,6 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None
|
|||
"compare_template_name": compare_name,
|
||||
"collect_metric_ids": collect,
|
||||
"collect_metric_ids_effective": effective,
|
||||
"available_metric_ids": [m for m in available if m],
|
||||
"defaults": dict(row.defaults_json or {}),
|
||||
"sheet_overrides": list(row.sheet_overrides_json or []),
|
||||
"note": row.note or "",
|
||||
|
|
@ -61,18 +57,6 @@ def _out(row: BizMonitorTemplate, compare_name: str = "", *, db: Session | None
|
|||
return out
|
||||
|
||||
|
||||
def _has_legacy_port_and_group(override: dict[str, Any]) -> bool:
|
||||
"""True when a success rule's old side demands admin down AND phy down."""
|
||||
legacy = _port_old_down_groups(legacy_and=True)[0]
|
||||
for rule in override.get("success") or []:
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
for group in rule.get("old_groups") or []:
|
||||
if group == legacy:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def validate_sheet_overrides(
|
||||
db: Session,
|
||||
*,
|
||||
|
|
@ -106,9 +90,6 @@ def validate_sheet_overrides(
|
|||
for ov in overrides:
|
||||
sid = str(ov.get("sheet_id") or "").strip()
|
||||
mid = str(ov.get("metric_id") or "").strip()
|
||||
if _has_legacy_port_and_group(ov):
|
||||
warnings.append({"sheet_id": sid, "metric_id": mid,
|
||||
"reason": "port_success_legacy_and_semantics"})
|
||||
if sid:
|
||||
if sid not in sheet_ids:
|
||||
warnings.append(
|
||||
|
|
@ -243,29 +224,9 @@ def _success_stateful(
|
|||
}
|
||||
|
||||
|
||||
_PORT_ADMIN_DOWN = {"type": "value", "field": "admin", "op": "in", "value": ["down"]}
|
||||
_PORT_PHY_DOWN = {"type": "value", "field": "phy", "op": "in", "value": ["down"]}
|
||||
|
||||
|
||||
def _port_old_down_groups(*, legacy_and: bool = False) -> list[list[dict[str, Any]]]:
|
||||
"""Old-side "port is out of service" groups (OR of groups, AND within one).
|
||||
|
||||
A fibre pulled without ``shutdown`` leaves admin up / phy down, so either
|
||||
signal alone counts. ``legacy_and`` is the pre-A4 form requiring both.
|
||||
"""
|
||||
if legacy_and:
|
||||
return [[dict(_PORT_ADMIN_DOWN), dict(_PORT_PHY_DOWN)]]
|
||||
return [[dict(_PORT_PHY_DOWN)], [dict(_PORT_ADMIN_DOWN)]]
|
||||
|
||||
|
||||
def preset_override_for_metric(
|
||||
metric_id: str, *, sheet_id: str = "", legacy_port_and: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
def preset_override_for_metric(metric_id: str, *, sheet_id: str = "") -> dict[str, Any]:
|
||||
"""Default dual-verdict overlay for a compare-template sheet.
|
||||
|
||||
``legacy_port_and`` rebuilds the pre-A4 port preset (old side must be
|
||||
admin down AND phy down) so stored copies can be recognised and upgraded.
|
||||
|
||||
``metric_id`` is the collected source table; ``sheet_id`` identifies a
|
||||
filtered split (e.g. bgp_peer.vpnv4). Presets key off the source metric.
|
||||
"""
|
||||
|
|
@ -279,8 +240,6 @@ def preset_override_for_metric(
|
|||
look = mid.split(".", 1)[0] if "." in mid else mid
|
||||
|
||||
def _out(body: dict[str, Any]) -> dict[str, Any]:
|
||||
if _scope_by_map_default_off(look):
|
||||
body["scope_by_port_map"] = False
|
||||
body["metric_id"] = mid or look
|
||||
if sid and sid != (mid or look):
|
||||
body["sheet_id"] = sid
|
||||
|
|
@ -298,7 +257,12 @@ def preset_override_for_metric(
|
|||
"up_values": ["up"],
|
||||
"success": [
|
||||
_success_stateful(
|
||||
old_down_groups=_port_old_down_groups(legacy_and=legacy_port_and),
|
||||
old_down_groups=[
|
||||
[
|
||||
{"type": "value", "field": "admin", "op": "in", "value": ["down"]},
|
||||
{"type": "value", "field": "phy", "op": "in", "value": ["down"]},
|
||||
]
|
||||
],
|
||||
new_up_conds=up,
|
||||
)
|
||||
],
|
||||
|
|
@ -401,8 +365,6 @@ DEFAULT_MONITOR_TEMPLATE_NAME = "默认割接监控"
|
|||
def ensure_default_monitor_templates(db: Session) -> None:
|
||||
"""Seed once when monitor-template table is empty — one template on the full status compare."""
|
||||
if db.query(BizMonitorTemplate.id).limit(1).first():
|
||||
upgrade_legacy_port_overrides(db)
|
||||
fill_scope_by_port_map(db)
|
||||
return
|
||||
cmp_svc.ensure_default_templates(db)
|
||||
zte = (
|
||||
|
|
@ -437,67 +399,8 @@ def ensure_default_monitor_templates(db: Session) -> None:
|
|||
db.commit()
|
||||
|
||||
|
||||
def _scope_by_map_default_off(name: str) -> bool:
|
||||
"""BGP / pseudowire sheets keep the batch expect picker; a port map does not scope them."""
|
||||
low = str(name or "").lower()
|
||||
return "bgp" in low or "pw" in low or "pseudowire" in low
|
||||
|
||||
|
||||
def fill_scope_by_port_map(db: Session) -> int:
|
||||
"""Make the old name-based default explicit on stored overrides (one-off, idempotent)."""
|
||||
changed = 0
|
||||
for row in db.query(BizMonitorTemplate).all():
|
||||
out = []
|
||||
touched = False
|
||||
for ov in list(row.sheet_overrides_json or []):
|
||||
if (isinstance(ov, dict) and "scope_by_port_map" not in ov
|
||||
and _scope_by_map_default_off(f"{ov.get('metric_id') or ''} {ov.get('sheet_id') or ''}")):
|
||||
ov = {**ov, "scope_by_port_map": False}
|
||||
touched = True
|
||||
out.append(ov)
|
||||
if touched:
|
||||
row.sheet_overrides_json = out
|
||||
row.updated_at = _utcnow()
|
||||
changed += 1
|
||||
if changed:
|
||||
db.commit()
|
||||
return changed
|
||||
|
||||
|
||||
def upgrade_legacy_port_overrides(db: Session) -> int:
|
||||
"""Rewrite stored port overrides that are exactly the pre-A4 preset (AND semantics).
|
||||
|
||||
Field-by-field equality only: anything a user customised is left alone and
|
||||
surfaces ``port_success_legacy_and_semantics`` in ``override_warnings``.
|
||||
"""
|
||||
changed = 0
|
||||
for row in db.query(BizMonitorTemplate).all():
|
||||
overrides = list(row.sheet_overrides_json or [])
|
||||
out: list[Any] = []
|
||||
touched = False
|
||||
for ov in overrides:
|
||||
if isinstance(ov, dict):
|
||||
mid = str(ov.get("metric_id") or "").strip()
|
||||
sid = str(ov.get("sheet_id") or "").strip()
|
||||
if (mid or sid).split(".", 1)[0] == PORT_METRIC_ID and ov == preset_override_for_metric(
|
||||
mid, sheet_id=sid, legacy_port_and=True,
|
||||
):
|
||||
out.append(preset_override_for_metric(mid, sheet_id=sid))
|
||||
touched = True
|
||||
continue
|
||||
out.append(ov)
|
||||
if touched:
|
||||
row.sheet_overrides_json = out
|
||||
row.updated_at = _utcnow()
|
||||
changed += 1
|
||||
if changed:
|
||||
db.commit()
|
||||
return changed
|
||||
|
||||
|
||||
def default_port_monitor_template_id(db: Session, *, seed: bool = True) -> str:
|
||||
"""Return the default monitor template id, seeding the built-ins first unless ``seed=False``."""
|
||||
if seed:
|
||||
def default_port_monitor_template_id(db: Session) -> str:
|
||||
"""Ensure seeds exist and return the default monitor template id."""
|
||||
ensure_default_monitor_templates(db)
|
||||
for name in (
|
||||
DEFAULT_MONITOR_TEMPLATE_NAME,
|
||||
|
|
@ -553,7 +456,6 @@ def create_monitor_template(db: Session, body: dict[str, Any]) -> dict[str, Any]
|
|||
overrides = body.get("sheet_overrides")
|
||||
if not isinstance(overrides, list):
|
||||
overrides = []
|
||||
validate_monitor_rules(defaults, overrides)
|
||||
row = BizMonitorTemplate(
|
||||
id=uuid4().hex,
|
||||
name=name[:256],
|
||||
|
|
@ -577,10 +479,6 @@ def update_monitor_template(db: Session, template_id: str, body: dict[str, Any])
|
|||
row = db.get(BizMonitorTemplate, template_id)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="monitor_template_not_found")
|
||||
validate_monitor_rules(body.get("defaults", row.defaults_json or {}),
|
||||
body.get("sheet_overrides", row.sheet_overrides_json or []))
|
||||
if body.get("compare_template_id") and not db.get(BizCompareTemplate, body["compare_template_id"]):
|
||||
raise HTTPException(status_code=404, detail="compare_template_not_found")
|
||||
if "name" in body and body["name"] is not None:
|
||||
name = str(body["name"] or "").strip()
|
||||
if not name:
|
||||
|
|
@ -617,7 +515,5 @@ def delete_monitor_template(db: Session, template_id: str) -> None:
|
|||
row = db.get(BizMonitorTemplate, template_id)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="monitor_template_not_found")
|
||||
if db.query(BizMigrationProject.id).filter(BizMigrationProject.monitor_template_id == template_id).first():
|
||||
raise HTTPException(status_code=409, detail="monitor_template_in_use")
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,72 +0,0 @@
|
|||
"""Validate monitor correction rules before they can affect live verdicts."""
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from .evaluate import _groups_for_side, _normalize_cond_type
|
||||
|
||||
|
||||
def validate_monitor_rules(defaults: Any, overrides: Any) -> None:
|
||||
def fail(path: str) -> None:
|
||||
raise HTTPException(status_code=400, detail=f"monitor_rule_invalid:{path}")
|
||||
|
||||
if not isinstance(defaults, dict) or not isinstance(overrides, list):
|
||||
fail("defaults/sheet_overrides")
|
||||
if defaults.get("out_of_expect", "strict") not in ("strict", "warn", "ignore"):
|
||||
fail("defaults.out_of_expect")
|
||||
seen = set()
|
||||
for i, ov in enumerate(overrides):
|
||||
path = f"sheet_overrides[{i}]"
|
||||
if not isinstance(ov, dict) or not (ov.get("sheet_id") or ov.get("metric_id")):
|
||||
fail(path)
|
||||
ident = ov.get("sheet_id") or ov.get("metric_id")
|
||||
if ident in seen:
|
||||
fail(path + ".duplicate")
|
||||
seen.add(ident)
|
||||
if ov.get("scope_by_port_map") is not None and not isinstance(ov["scope_by_port_map"], bool):
|
||||
fail(path + ".scope_by_port_map")
|
||||
for key in ("status_fields", "up_values", "down_values"):
|
||||
if key in ov and (not isinstance(ov[key], list) or any(not str(v).strip() for v in ov[key])):
|
||||
fail(path + "." + key)
|
||||
for key in ("success", "anomaly"):
|
||||
patterns = ov.get(key, [])
|
||||
if not isinstance(patterns, list):
|
||||
fail(path + "." + key)
|
||||
for j, pat in enumerate(patterns):
|
||||
pp = f"{path}.{key}[{j}]"
|
||||
if not isinstance(pat, dict):
|
||||
fail(pp)
|
||||
for side in ("old", "new"):
|
||||
for attr in (side, side + "_conds", side + "_groups"):
|
||||
if attr in pat and not isinstance(pat[attr], list):
|
||||
fail(pp + "." + attr)
|
||||
if any(isinstance(g, list) and not g for g in pat.get(side + "_groups", [])):
|
||||
fail(pp + "." + side + "_groups.empty_group")
|
||||
groups = _groups_for_side(pat, side)
|
||||
if key == "success" and not groups:
|
||||
fail(pp + "." + side + ".required")
|
||||
for group in groups:
|
||||
for cond in group:
|
||||
cp = pp + "." + side
|
||||
if isinstance(cond, str):
|
||||
if cond not in ("added", "removed", "changed", "unchanged", "up", "down", "other", "none") and not (cond.startswith("field:") and cond.count(":") >= 2):
|
||||
fail(cp + ".token")
|
||||
elif isinstance(cond, dict):
|
||||
typ = _normalize_cond_type(cond)
|
||||
if typ not in ("presence", "status", "value"):
|
||||
fail(cp + ".type")
|
||||
val = cond.get("value", cond.get("kind", cond.get("status")))
|
||||
if typ == "presence" and val not in ("added", "removed", "changed", "unchanged"):
|
||||
fail(cp + ".presence")
|
||||
if typ == "status" and val not in ("up", "down", "other", "none"):
|
||||
fail(cp + ".status")
|
||||
if typ == "value":
|
||||
op = cond.get("op", "eq")
|
||||
if op not in ("eq", "ne", "in", "not_in", "empty", "not_empty", "changed", "unchanged", "same", "diff"):
|
||||
fail(cp + ".op")
|
||||
if op in ("eq", "ne", "in", "not_in") and (val is None or val == "" or val == []):
|
||||
fail(cp + ".value")
|
||||
else:
|
||||
fail(cp + ".condition")
|
||||
if key == "anomaly" and not any(_groups_for_side(pat, s) for s in ("old", "new")):
|
||||
fail(pp + ".empty")
|
||||
|
|
@ -151,12 +151,6 @@ def api_baseline_ports(project_id: str, db: Session = Depends(get_db)):
|
|||
return svc.list_baseline_ports(db, project_id)
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/preflight")
|
||||
@router.post("/projects/{project_id}/preflight")
|
||||
def api_project_preflight(project_id: str, db: Session = Depends(get_db)):
|
||||
return svc.preflight_project(db, project_id)
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/baseline-expect")
|
||||
def api_baseline_expect(project_id: str, db: Session = Depends(get_db)):
|
||||
"""Multi-metric baseline keys for expect-set picking (from monitor template sheets)."""
|
||||
|
|
@ -228,13 +222,8 @@ def api_finish_batch(
|
|||
mark_done: bool = False,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""本批完成：终验 + 小结 + 红单（允许带红继续）. Runs in the background; poll the run."""
|
||||
return svc.enqueue_finish(db, batch_id, mark_done=mark_done)
|
||||
|
||||
|
||||
@router.post("/projects/{project_id}/red-tickets/resolve-resolvable")
|
||||
def api_resolve_resolvable(project_id: str, db: Session = Depends(get_db)):
|
||||
return svc.resolve_resolvable_red_tickets(db, project_id)
|
||||
"""本批完成：终验 + 小结 + 红单（允许带红继续）."""
|
||||
return svc.finish_batch(db, batch_id, mark_done=mark_done)
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/red-tickets")
|
||||
|
|
@ -269,10 +258,9 @@ def api_patch_batch(batch_id: str, body: BatchPatchIn, db: Session = Depends(get
|
|||
@router.post("/batches/{batch_id}/evaluate")
|
||||
def api_evaluate(batch_id: str, body: EvaluateIn | None = None, db: Session = Depends(get_db)):
|
||||
payload = body.model_dump() if body else {}
|
||||
# Returns the ``running`` run at once; poll GET /runs/{id} until ``status`` settles.
|
||||
return svc.enqueue_evaluate(
|
||||
return svc.run_evaluate(
|
||||
db,
|
||||
batch_id,
|
||||
batch_id=batch_id,
|
||||
old_batch_id=str(payload.get("old_batch_id") or ""),
|
||||
new_batch_id=str(payload.get("new_batch_id") or ""),
|
||||
)
|
||||
|
|
@ -297,8 +285,6 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)):
|
|||
def api_list_diffs(
|
||||
run_id: str,
|
||||
metric_id: str = "",
|
||||
sheet_id: str = "",
|
||||
only_expect: bool = False,
|
||||
verdict: str = "",
|
||||
color: str = "",
|
||||
kw: str = "",
|
||||
|
|
@ -310,8 +296,6 @@ def api_list_diffs(
|
|||
db,
|
||||
run_id,
|
||||
metric_id=metric_id,
|
||||
sheet_id=sheet_id,
|
||||
only_expect=only_expect,
|
||||
verdict=verdict,
|
||||
color=color,
|
||||
kw=kw,
|
||||
|
|
|
|||
|
|
@ -2,10 +2,8 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
|
|
@ -27,11 +25,6 @@ def max_concurrent_tasks() -> int:
|
|||
return max(1, int(getattr(settings, "biz_state_max_concurrent_tasks", 16) or 16))
|
||||
|
||||
|
||||
def max_sessions_per_ne() -> int:
|
||||
"""Concurrent SSH collections allowed per network element (>= 1)."""
|
||||
return max(1, int(getattr(settings, "biz_state_max_sessions_per_ne", 2) or 2))
|
||||
|
||||
|
||||
def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]:
|
||||
"""Create a queued batch and mark task collect_running. Does not open SSH.
|
||||
|
||||
|
|
@ -46,10 +39,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]:
|
|||
task = db.get(BizStateTask, tid)
|
||||
if not task:
|
||||
return {"ok": False, "queued": False, "reason": "task_not_found", "task_id": tid}
|
||||
if task.collect_group_id:
|
||||
group_id = task.collect_group_id
|
||||
db.close()
|
||||
return enqueue_collect_group(group_id, manual=manual, requested_task_id=tid)
|
||||
if str(task.source or "").strip().lower() == "import":
|
||||
return {
|
||||
"ok": False,
|
||||
|
|
@ -110,7 +99,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]:
|
|||
vendor=task.vendor,
|
||||
status="queued",
|
||||
started_at=now,
|
||||
queued_at=now,
|
||||
message="queued" if not manual else "queued_manual",
|
||||
)
|
||||
db.add(batch)
|
||||
|
|
@ -133,62 +121,6 @@ def enqueue_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]:
|
|||
db.close()
|
||||
|
||||
|
||||
def enqueue_collect_group(group_id: str, *, manual: bool = False,
|
||||
requested_task_id: str = "") -> dict[str, Any]:
|
||||
"""Atomically enqueue both devices with one round identity, or neither."""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
tasks = db.query(BizStateTask).filter(BizStateTask.collect_group_id == group_id).order_by(
|
||||
BizStateTask.id).with_for_update().populate_existing().all()
|
||||
result: dict[str, Any] = {"ok": False, "queued": False, "task_id": requested_task_id,
|
||||
"collect_group_id": group_id}
|
||||
# One task = a single device used as both sides (same-device cutover); two tasks = a pair.
|
||||
if len(tasks) not in (1, 2) or len({t.ne_id for t in tasks}) != len(tasks):
|
||||
return {**result, "reason": "collect_pair_invalid"}
|
||||
if max_concurrent_tasks() < len(tasks):
|
||||
return {**result, "reason": "collect_pair_capacity"}
|
||||
from ..cli_budget import clamp_cli_workers
|
||||
|
||||
worker_cap = clamp_cli_workers(int(getattr(settings, "biz_state_worker_collect_threads", None)
|
||||
or getattr(settings, "biz_state_dispatch_workers", 8) or 8))
|
||||
if worker_cap < len(tasks):
|
||||
return {**result, "reason": "collect_pair_capacity"}
|
||||
if any(t.collect_running for t in tasks):
|
||||
return {**result, "reason": "collect_pair_busy"}
|
||||
for task in tasks:
|
||||
if task.source == "import" or (not manual and (task.status != "running" or task.collect_manual_only)):
|
||||
return {**result, "reason": "not_scheduled"}
|
||||
if task.status in ("", "deleted"):
|
||||
return {**result, "reason": "bad_status"}
|
||||
if not db.query(BizStateTaskItem.id).filter(BizStateTaskItem.task_id == task.id,
|
||||
BizStateTaskItem.enabled.is_(True)).first():
|
||||
return {**result, "reason": "no_enabled_items"}
|
||||
now, round_id = _utcnow(), uuid4().hex
|
||||
batches = []
|
||||
for task in tasks:
|
||||
task.collect_running = True
|
||||
task.collect_queued_at = now
|
||||
task.last_error = ""
|
||||
task.updated_at = now
|
||||
batch = BizStateBatch(id=uuid4().hex, task_id=task.id, source=task.source,
|
||||
ne_id=task.ne_id, ne_name=task.ne_name, vendor=task.vendor, status="queued",
|
||||
started_at=now, queued_at=now, collect_group_id=group_id,
|
||||
collect_round_id=round_id, collect_priority=10 if task.collect_manual_only else 20,
|
||||
message="queued_manual" if manual else "queued")
|
||||
db.add(batch)
|
||||
batches.append({"batch_id": batch.id, "task_id": task.id})
|
||||
db.commit()
|
||||
selected = next((b for b in batches if b["task_id"] == requested_task_id), batches[0])
|
||||
return {**result, **selected, "ok": True, "queued": True, "collect_round_id": round_id,
|
||||
"batches": batches, "manual": manual}
|
||||
except Exception:
|
||||
db.rollback()
|
||||
_log.exception("enqueue collect pair failed group=%s", group_id)
|
||||
return {"ok": False, "queued": False, "reason": "enqueue_error", "task_id": requested_task_id}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _dialect_supports_skip_locked(db: Session) -> bool:
|
||||
try:
|
||||
bind = db.get_bind()
|
||||
|
|
@ -219,16 +151,11 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
|
|||
Global ceiling is always ``max_concurrent_tasks()``; ``limit`` only caps
|
||||
how many this caller wants in one call.
|
||||
"""
|
||||
from sqlalchemy import and_, case, text
|
||||
from sqlalchemy import text
|
||||
|
||||
global_cap = max_concurrent_tasks()
|
||||
db = SessionLocal()
|
||||
try:
|
||||
pg = _dialect_supports_skip_locked(db)
|
||||
if pg:
|
||||
# A short transaction lock makes capacity reservation and pair claims
|
||||
# atomic across worker processes. Never held during device I/O.
|
||||
db.execute(text("SELECT pg_advisory_xact_lock(hashtext('biz-state-claim-capacity'))"))
|
||||
running_n = (
|
||||
db.query(BizStateBatch).filter(BizStateBatch.status == "running").count()
|
||||
)
|
||||
|
|
@ -238,67 +165,58 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
|
|||
if slots <= 0:
|
||||
return []
|
||||
|
||||
# Running collections per NE; an NE accepts new ones until it reaches its session cap.
|
||||
per_ne_cap = max_sessions_per_ne()
|
||||
ne_running: dict[str, int] = defaultdict(int)
|
||||
for (running_ne,) in db.query(BizStateBatch.ne_id).filter(BizStateBatch.status == "running").all():
|
||||
if str(running_ne or "").strip():
|
||||
ne_running[str(running_ne).strip()] += 1
|
||||
busy_nes = {
|
||||
str(r[0] or "").strip()
|
||||
for r in db.query(BizStateBatch.ne_id)
|
||||
.filter(BizStateBatch.status == "running")
|
||||
.all()
|
||||
if str(r[0] or "").strip()
|
||||
}
|
||||
|
||||
# Manual route rounds yield to fresh monitoring, but must eventually
|
||||
# run even when a busy device always has regular metrics queued.
|
||||
priority = case((and_(BizStateBatch.collect_priority == 10,
|
||||
BizStateBatch.queued_at <= _utcnow() - timedelta(seconds=300)), 30),
|
||||
else_=BizStateBatch.collect_priority)
|
||||
q = (
|
||||
db.query(BizStateBatch)
|
||||
.filter(BizStateBatch.status == "queued")
|
||||
.order_by(priority.desc(), BizStateBatch.started_at.asc(), BizStateBatch.id.asc())
|
||||
.order_by(BizStateBatch.started_at.asc())
|
||||
)
|
||||
pg = _dialect_supports_skip_locked(db)
|
||||
if pg:
|
||||
q = q.with_for_update(skip_locked=True)
|
||||
else:
|
||||
q = q.with_for_update()
|
||||
|
||||
candidates = q.limit(max(slots * 8, 64)).all()
|
||||
candidates = q.limit(max(slots * 4, slots)).all()
|
||||
claimed_rows: list[BizStateBatch] = []
|
||||
visited: set[str] = set()
|
||||
for batch in candidates:
|
||||
if batch.id in visited:
|
||||
continue
|
||||
if len(claimed_rows) >= slots:
|
||||
break
|
||||
group = [batch]
|
||||
if batch.collect_round_id:
|
||||
peers = db.query(BizStateBatch).filter(
|
||||
BizStateBatch.collect_round_id == batch.collect_round_id,
|
||||
).with_for_update(skip_locked=pg).all()
|
||||
expected = db.query(BizStateTask.id).filter(
|
||||
BizStateTask.collect_group_id == batch.collect_group_id).count() if batch.collect_group_id else 2
|
||||
if len(peers) != max(1, expected) or any(p.status != "queued" for p in peers):
|
||||
ne = str(batch.ne_id or "").strip()
|
||||
if ne and ne in busy_nes:
|
||||
continue
|
||||
group = peers
|
||||
visited.update(p.id for p in group)
|
||||
if len(group) > slots - len(claimed_rows):
|
||||
# Serialize claims per NE across workers (Postgres).
|
||||
if ne and pg:
|
||||
try:
|
||||
db.execute(
|
||||
text("SELECT pg_advisory_xact_lock(hashtext(:ne))"),
|
||||
{"ne": ne},
|
||||
)
|
||||
except Exception:
|
||||
_log.exception("advisory lock failed ne=%s", ne)
|
||||
conflict = (
|
||||
db.query(BizStateBatch.id)
|
||||
.filter(
|
||||
BizStateBatch.status == "running",
|
||||
BizStateBatch.ne_id == ne,
|
||||
BizStateBatch.id != batch.id,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if conflict:
|
||||
continue
|
||||
wanted: dict[str, int] = defaultdict(int)
|
||||
for p in group:
|
||||
if str(p.ne_id or "").strip():
|
||||
wanted[str(p.ne_id).strip()] += 1
|
||||
if any(ne_running[ne] + n > per_ne_cap for ne, n in wanted.items()):
|
||||
continue
|
||||
now = _utcnow()
|
||||
for member in group:
|
||||
member.status = "running"
|
||||
member.message = ""
|
||||
if member.collect_round_id:
|
||||
member.started_at = now
|
||||
task = db.get(BizStateTask, member.task_id)
|
||||
if task:
|
||||
task.last_collect_started_at = now
|
||||
claimed_rows.append(member)
|
||||
for ne, n in wanted.items():
|
||||
ne_running[ne] += n
|
||||
batch.status = "running"
|
||||
batch.message = ""
|
||||
claimed_rows.append(batch)
|
||||
if ne:
|
||||
busy_nes.add(ne)
|
||||
|
||||
if not claimed_rows:
|
||||
db.rollback()
|
||||
|
|
@ -313,10 +231,7 @@ def claim_queued_batches(limit: int | None = None) -> list[dict[str, Any]]:
|
|||
db.query(BizStateBatch).filter(BizStateBatch.status == "running").count()
|
||||
)
|
||||
while n_running > global_cap and claimed_rows:
|
||||
last = claimed_rows[-1]
|
||||
demoted = [b for b in claimed_rows if b.collect_round_id == last.collect_round_id] if last.collect_round_id else [last]
|
||||
for demote in demoted:
|
||||
claimed_rows.remove(demote)
|
||||
demote = claimed_rows.pop()
|
||||
demote.status = "queued"
|
||||
demote.message = "queued"
|
||||
n_running -= 1
|
||||
|
|
|
|||
|
|
@ -784,12 +784,6 @@ def dispatch_collect(task_id: str, *, manual: bool = False) -> dict[str, Any]:
|
|||
result = enqueue_collect(task_id, manual=manual)
|
||||
if not result.get("queued"):
|
||||
return result
|
||||
if result.get("collect_group_id"):
|
||||
if _should_execute_inline():
|
||||
from ..biz_state_scheduler import _claim_and_dispatch
|
||||
|
||||
_claim_and_dispatch()
|
||||
return result
|
||||
batch_id = str(result.get("batch_id") or "")
|
||||
if not batch_id:
|
||||
return result
|
||||
|
|
@ -876,11 +870,6 @@ def execute_claimed_batch(
|
|||
task = db.get(BizStateTask, task_id) if task_id else None
|
||||
if not batch:
|
||||
return
|
||||
if batch.collect_round_id:
|
||||
batch.started_at = _utcnow()
|
||||
if task:
|
||||
task.last_collect_started_at = batch.started_at
|
||||
db.commit()
|
||||
if not task_id:
|
||||
task_id = str(batch.task_id or "")
|
||||
task = db.get(BizStateTask, task_id) if task_id else None
|
||||
|
|
@ -1731,14 +1720,6 @@ def _finalize_batch_status(
|
|||
batch.message = ""
|
||||
status = str(batch.status or "")
|
||||
db.commit()
|
||||
# Failed/partial receipts also update cutover coverage; never keep an old
|
||||
# success looking healthy after the latest collection has failed.
|
||||
try:
|
||||
from ..biz_migration.auto_monitor import schedule_auto_monitor_for_task
|
||||
|
||||
schedule_auto_monitor_for_task(task_id)
|
||||
except Exception:
|
||||
_log.exception("cutover monitor schedule failed task=%s", task_id)
|
||||
# Only full success triggers auto compare; never block the collect thread.
|
||||
if status == "success" and not stopped:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ def _force_close_holders(batch_id: str) -> int:
|
|||
|
||||
|
||||
def request_stop_collect(task_id: str) -> dict[str, Any]:
|
||||
"""Stop a task, including its paired collector, without stranding a half round."""
|
||||
"""Cancel queued batches and signal running collect for this task to abort."""
|
||||
tid = str(task_id or "").strip()
|
||||
if not tid:
|
||||
return {"ok": False, "reason": "missing_task_id"}
|
||||
|
|
@ -141,36 +141,25 @@ def request_stop_collect(task_id: str) -> dict[str, Any]:
|
|||
signaled_ids: list[str] = []
|
||||
closed = 0
|
||||
try:
|
||||
from sqlalchemy import text
|
||||
from .claim import _dialect_supports_skip_locked
|
||||
|
||||
if _dialect_supports_skip_locked(db):
|
||||
db.execute(text("SELECT pg_advisory_xact_lock(hashtext('biz-state-claim-capacity'))"))
|
||||
task = db.get(BizStateTask, tid)
|
||||
if not task:
|
||||
return {"ok": False, "reason": "task_not_found", "task_id": tid}
|
||||
|
||||
task_ids = [tid]
|
||||
if task.collect_group_id:
|
||||
task_ids = [r[0] for r in db.query(BizStateTask.id).filter(
|
||||
BizStateTask.collect_group_id == task.collect_group_id).all()]
|
||||
active = (
|
||||
db.query(BizStateBatch)
|
||||
.filter(
|
||||
BizStateBatch.task_id.in_(task_ids),
|
||||
BizStateBatch.task_id == tid,
|
||||
BizStateBatch.status.in_(("queued", "running")),
|
||||
)
|
||||
.order_by(BizStateBatch.id).with_for_update().all()
|
||||
.all()
|
||||
)
|
||||
tasks = db.query(BizStateTask).filter(BizStateTask.id.in_(task_ids)).order_by(
|
||||
BizStateTask.id).with_for_update().populate_existing().all()
|
||||
if not active:
|
||||
# Nothing to stop — clear sticky collect_running if orphaned.
|
||||
for member in tasks:
|
||||
if member.collect_running:
|
||||
member.collect_running = False
|
||||
member.collect_queued_at = None
|
||||
member.updated_at = utcnow_naive()
|
||||
if bool(task.collect_running):
|
||||
task.collect_running = False
|
||||
if hasattr(task, "collect_queued_at"):
|
||||
task.collect_queued_at = None
|
||||
task.updated_at = utcnow_naive()
|
||||
db.commit()
|
||||
return {
|
||||
"ok": True,
|
||||
|
|
@ -182,7 +171,7 @@ def request_stop_collect(task_id: str) -> dict[str, Any]:
|
|||
}
|
||||
|
||||
now = utcnow_naive()
|
||||
running_tasks: set[str] = set()
|
||||
still_running = False
|
||||
for batch in active:
|
||||
bid = str(batch.id)
|
||||
mark_stop_requested(bid)
|
||||
|
|
@ -194,16 +183,16 @@ def request_stop_collect(task_id: str) -> dict[str, Any]:
|
|||
else:
|
||||
batch.message = STOP_REQUEST_TOKEN
|
||||
signaled_ids.append(bid)
|
||||
running_tasks.add(batch.task_id)
|
||||
still_running = True
|
||||
closed += _force_close_holders(bid)
|
||||
|
||||
for member in tasks:
|
||||
if member.id not in running_tasks:
|
||||
member.collect_running = False
|
||||
member.collect_queued_at = None
|
||||
member.last_collect_ended_at = now
|
||||
member.last_error = STOP_USER_MESSAGE
|
||||
member.updated_at = now
|
||||
if not still_running:
|
||||
task.collect_running = False
|
||||
if hasattr(task, "collect_queued_at"):
|
||||
task.collect_queued_at = None
|
||||
task.last_collect_ended_at = now
|
||||
task.last_error = STOP_USER_MESSAGE
|
||||
task.updated_at = now
|
||||
|
||||
db.commit()
|
||||
_log.info(
|
||||
|
|
|
|||
|
|
@ -2,81 +2,15 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from heapq import heappush, heapreplace
|
||||
from itertools import chain
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from .compare_rules import field_rule_map, values_equal, explain_diff, scalar_text
|
||||
from .compare_rules import field_rule_map, values_equal, explain_diff
|
||||
from .iface_normalize import (
|
||||
apply_iface_normalize_rows,
|
||||
normalize_iface_rules,
|
||||
resolve_mapped_iface,
|
||||
)
|
||||
|
||||
# Prefer these fields when stratifying success samples (BGP multipath / multi-cmd).
|
||||
_STRATUM_FIELDS = ("neighbor", "direction", "afi", "vrf")
|
||||
|
||||
|
||||
def stratum_key(row: Mapping[str, Any] | None) -> str:
|
||||
"""Bucket key for stratified unchanged sampling."""
|
||||
if not isinstance(row, Mapping):
|
||||
return "_"
|
||||
parts: list[str] = []
|
||||
for f in _STRATUM_FIELDS:
|
||||
v = str(row.get(f) or "").strip()
|
||||
if v:
|
||||
parts.append(f"{f}={v}")
|
||||
return "|".join(parts) if parts else "_"
|
||||
|
||||
|
||||
class _StratifiedSample:
|
||||
"""Keep the first N round-robin ranks in O(N) space, even with many strata."""
|
||||
|
||||
def __init__(self, limit: int) -> None:
|
||||
self.limit = max(0, int(limit))
|
||||
self.count = 0
|
||||
self.strata: dict[str, tuple[int, int]] = {}
|
||||
self.heap: list[tuple[int, int, int, Any]] = []
|
||||
|
||||
def add(self, item: Any, key: str) -> None:
|
||||
serial = self.count
|
||||
self.count += 1
|
||||
if not self.limit:
|
||||
return
|
||||
state = self.strata.get(key)
|
||||
if state is None:
|
||||
# Later strata cannot beat the first item of N earlier strata.
|
||||
if len(self.strata) >= self.limit:
|
||||
return
|
||||
index, round_n = len(self.strata), 0
|
||||
else:
|
||||
index, round_n = state
|
||||
self.strata[key] = (index, round_n + 1)
|
||||
entry = (-round_n, -index, serial, item)
|
||||
if len(self.heap) < self.limit:
|
||||
heappush(self.heap, entry)
|
||||
elif entry[:2] > self.heap[0][:2]:
|
||||
heapreplace(self.heap, entry)
|
||||
|
||||
def picked(self) -> list[Any]:
|
||||
# The legacy helper preserves encounter order when no truncation occurs.
|
||||
if self.count <= self.limit:
|
||||
entries = sorted(self.heap, key=lambda e: e[2])
|
||||
else:
|
||||
entries = sorted(self.heap, key=lambda e: (-e[0], -e[1]))
|
||||
return [entry[3] for entry in entries]
|
||||
|
||||
|
||||
def stratify_take(items: Iterable[Any], limit: int, *, key_fn) -> list[Any]:
|
||||
"""Round-robin across strata without retaining every candidate."""
|
||||
sample = _StratifiedSample(limit)
|
||||
if not sample.limit:
|
||||
return []
|
||||
for item in items:
|
||||
sample.add(item, str(key_fn(item) or "_"))
|
||||
return sample.picked()
|
||||
|
||||
|
||||
def apply_port_map(
|
||||
row: dict[str, Any],
|
||||
|
|
@ -93,17 +27,8 @@ def apply_port_map(
|
|||
return out
|
||||
|
||||
|
||||
def key_tuple(row: Mapping[str, Any], fields: Sequence[str]) -> tuple[str, ...]:
|
||||
"""The one canonical match-key form: ``scalar_text(...).strip()`` per field.
|
||||
|
||||
Every place that builds, stores or looks up a key string must go through
|
||||
this (``None`` → ``""``, ``True`` → ``"true"``, surrounding blanks trimmed).
|
||||
"""
|
||||
return tuple(scalar_text(row.get(k)).strip() for k in fields)
|
||||
|
||||
|
||||
def row_key(row: dict[str, Any], key_fields: list[str]) -> tuple[str, ...]:
|
||||
return key_tuple(row, key_fields)
|
||||
return tuple(str(row.get(k) or "").strip() for k in key_fields)
|
||||
|
||||
|
||||
def mapping_stats(
|
||||
|
|
@ -118,8 +43,8 @@ def mapping_stats(
|
|||
``hit_before``: map key appears as a normalized before iface, or as the
|
||||
parent of a before subinterface (so main-port-only maps still validate).
|
||||
|
||||
Map keys are matched against **normalized** iface values; callers pass a
|
||||
port map already normalized with the template iface rules.
|
||||
Map keys are matched against **normalized** iface values (same pipeline as
|
||||
compare). Enter mapping keys in normalized form.
|
||||
"""
|
||||
before_ifaces: set[str] = set()
|
||||
after_ifaces: set[str] = set()
|
||||
|
|
@ -165,7 +90,7 @@ def mapping_stats(
|
|||
"miss_after": miss_after,
|
||||
"unused_before_keys": unused,
|
||||
"ok": not miss_before and not miss_after,
|
||||
"hint": "map keys/values are normalized with the template iface rules before matching",
|
||||
"hint": "map keys must match normalized iface names (post iface_normalize)",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -180,32 +105,12 @@ def compare_rows(
|
|||
field_rules: Sequence[Mapping[str, Any]] | None = None,
|
||||
iface_normalize_rules: Sequence[Mapping[str, str]] | None = None,
|
||||
ignore_port_changes: bool | None = None,
|
||||
include_unchanged: bool = False,
|
||||
unchanged_limit: int | None = None,
|
||||
compact_unchanged: bool = False,
|
||||
pipeline: Any = None,
|
||||
match_keys_override: Sequence[str] | None = None,
|
||||
prepared: tuple[list, list] | None = None,
|
||||
duplicate_pairing: str = "ordered",
|
||||
) -> dict[str, Any]:
|
||||
"""Return summary + diffs list.
|
||||
|
||||
Diff kinds: added | removed | changed | unchanged
|
||||
Diff kinds: added | removed | changed | unchanged | duplicate
|
||||
|
||||
Pipeline: iface normalize (both sides) → port map (before) → ordered
|
||||
same-key pairing (load / list order within each match key).
|
||||
|
||||
Same match key with N before and M after rows: zip by index
|
||||
``0..min(N,M)-1`` for field compare; extras become ``removed`` (before)
|
||||
or ``added`` (after). Example: 5 vs 2 → 2 compared + 3 removed.
|
||||
|
||||
``include_unchanged``: when False, matching rows still increment
|
||||
``summary.unchanged`` but are omitted from ``diffs``.
|
||||
|
||||
``unchanged_limit``: max unchanged diffs to emit (None = no cap). Use with
|
||||
large sheets so the UI can browse a sample without writing millions of rows.
|
||||
|
||||
``compact_unchanged``: emit key only (empty before/after) to cut storage.
|
||||
Pipeline: iface normalize (both sides) → port map (before) → match.
|
||||
|
||||
``ignore_port_changes``:
|
||||
- ``None`` (default): auto — drop iface from match key only when remaining
|
||||
|
|
@ -213,165 +118,97 @@ def compare_rows(
|
|||
- ``True``: force drop iface from match key when a non-empty candidate exists.
|
||||
- ``False``: never drop iface from match key.
|
||||
|
||||
``pipeline`` (``identity_pipeline.Pipeline``) owns normalization, filtering,
|
||||
port mapping and key building; when omitted a legacy pipeline is derived
|
||||
from the loose arguments (callers have already filtered the rows).
|
||||
``match_keys_override`` pins the match key (re-running a stored result).
|
||||
|
||||
``summary.duplicate`` is always 0. ``duplicate_keys_*`` count match keys
|
||||
that appear more than once on a side (diagnostic only).
|
||||
Duplicate match keys are not silently discarded: extras become ``duplicate``
|
||||
diffs and ``summary.duplicate_key_list`` lists the colliding keys.
|
||||
|
||||
``field_rules`` drives normalize / numeric tolerance / per-field compare mode
|
||||
(template-driven; no metric-specific branches here).
|
||||
"""
|
||||
if not key_fields:
|
||||
raise ValueError("key_fields required")
|
||||
if pipeline is None:
|
||||
from .identity_pipeline import legacy_pipeline
|
||||
|
||||
pipeline = legacy_pipeline(
|
||||
key_fields=key_fields, iface_fields=iface_fields, compare_fields=compare_fields,
|
||||
field_rules=field_rules, port_map=port_map, iface_normalize_rules=iface_normalize_rules,
|
||||
ignore_port_changes=ignore_port_changes, duplicate_pairing=duplicate_pairing,
|
||||
)
|
||||
pmap = dict(pipeline.port_map)
|
||||
rules = field_rule_map(field_rules if field_rules is not None else pipeline.field_rules)
|
||||
iface_list = list(pipeline.iface_fields)
|
||||
pmap = dict(port_map or {})
|
||||
rules = field_rule_map(field_rules)
|
||||
norm_rules = normalize_iface_rules(iface_normalize_rules)
|
||||
iface_list = [str(f) for f in (iface_fields or []) if str(f).strip()]
|
||||
iface_set = set(iface_list)
|
||||
|
||||
if prepared is not None:
|
||||
before_pairs, after_pairs = prepared
|
||||
else:
|
||||
before_pairs = pipeline.prepare(before_rows, "before")
|
||||
after_pairs = pipeline.prepare(after_rows, "after")
|
||||
before_rows = [orig for orig, _ in before_pairs]
|
||||
after_rows = [orig for orig, _ in after_pairs]
|
||||
before_norm = [norm for _, norm in before_pairs]
|
||||
after_norm = [norm for _, norm in after_pairs]
|
||||
before_norm = apply_iface_normalize_rows(
|
||||
before_rows, iface_fields=iface_list, rules=norm_rules
|
||||
)
|
||||
after_norm = apply_iface_normalize_rows(
|
||||
after_rows, iface_fields=iface_list, rules=norm_rules
|
||||
)
|
||||
|
||||
ignore_ports = False
|
||||
if match_keys_override is not None:
|
||||
match_keys = [str(k) for k in match_keys_override]
|
||||
ignore_ports = match_keys != list(key_fields)
|
||||
else:
|
||||
# No map → optionally ignore port renames by dropping iface from match key.
|
||||
policy = pipeline.ignore_port_changes
|
||||
unique = None
|
||||
if not pmap and iface_set and policy is None:
|
||||
if not pmap and iface_set:
|
||||
candidate = [k for k in key_fields if k not in iface_set]
|
||||
if not candidate:
|
||||
match_keys = list(key_fields)
|
||||
elif ignore_port_changes is True:
|
||||
match_keys = candidate
|
||||
ignore_ports = True
|
||||
elif ignore_port_changes is False:
|
||||
match_keys = list(key_fields)
|
||||
else:
|
||||
# Auto heuristic (legacy default)
|
||||
before_c = [row_key(r, candidate) for r in before_norm]
|
||||
after_c = [row_key(r, candidate) for r in after_norm]
|
||||
if len(before_c) == len(set(before_c)) and len(after_c) == len(set(after_c)):
|
||||
match_keys = candidate
|
||||
ignore_ports = True
|
||||
else:
|
||||
match_keys = list(key_fields)
|
||||
else:
|
||||
match_keys = list(key_fields)
|
||||
|
||||
def unique_keys(rows: list[dict[str, Any]]) -> bool:
|
||||
seen: set[tuple[str, ...]] = set()
|
||||
for row in rows:
|
||||
key = row_key(row, candidate)
|
||||
if key in seen:
|
||||
return False
|
||||
seen.add(key)
|
||||
return True
|
||||
before_mapped: list[dict[str, Any]] = [
|
||||
apply_port_map(r, iface_fields=iface_list, port_map=pmap) for r in before_norm
|
||||
]
|
||||
|
||||
unique = bool(candidate) and unique_keys(before_norm) and unique_keys(after_norm)
|
||||
match_keys = pipeline.effective_match_keys(unique_without_iface=unique)
|
||||
ignore_ports = match_keys != list(key_fields)
|
||||
|
||||
before_groups: dict[tuple[str, ...], list[tuple[dict[str, Any], dict[str, Any]]]] = (
|
||||
defaultdict(list)
|
||||
)
|
||||
after_groups: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list)
|
||||
for orig, norm in zip(before_rows, before_norm):
|
||||
mapped, k = pipeline.key_of(norm, "before", match_keys)
|
||||
before_groups[k].append((orig, mapped))
|
||||
for norm in after_norm:
|
||||
after_groups[pipeline.key_of(norm, "after", match_keys)[1]].append(norm)
|
||||
sort_pairs = pipeline.duplicate_pairing == "sorted"
|
||||
after_index: dict[tuple[str, ...], dict[str, Any]] = {}
|
||||
after_dup = 0
|
||||
after_dup_keys: list[tuple[str, ...]] = []
|
||||
after_dup_rows: list[tuple[tuple[str, ...], dict[str, Any]]] = []
|
||||
for r in after_norm:
|
||||
k = row_key(r, match_keys)
|
||||
if k in after_index:
|
||||
after_dup += 1
|
||||
after_dup_keys.append(k)
|
||||
after_dup_rows.append((k, r))
|
||||
continue # first wins — do not overwrite
|
||||
after_index[k] = r
|
||||
|
||||
before_keys: set[tuple[str, ...]] = set()
|
||||
before_dup = 0
|
||||
before_dup_keys: list[tuple[str, ...]] = []
|
||||
diffs: list[dict[str, Any]] = []
|
||||
added = removed = changed = unchanged = 0
|
||||
unchanged_listed = 0
|
||||
limit_n = None if unchanged_limit is None else max(0, int(unchanged_limit))
|
||||
multi_before_keys: list[tuple[str, ...]] = []
|
||||
multi_after_keys: list[tuple[str, ...]] = []
|
||||
unchanged_candidates: list[tuple[dict[str, Any], dict[str, Any], dict[str, Any]]] = []
|
||||
sample = _StratifiedSample(limit_n) if include_unchanged and limit_n is not None else None
|
||||
added = removed = changed = unchanged = duplicate = 0
|
||||
|
||||
def _key_obj(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return dict(zip(key_fields, key_tuple(row, key_fields)))
|
||||
return {f: row.get(f, "") for f in key_fields}
|
||||
|
||||
def _row_id(row: dict[str, Any] | None) -> str:
|
||||
if not isinstance(row, dict):
|
||||
return ""
|
||||
netx = row.get("_netx")
|
||||
if isinstance(netx, dict):
|
||||
return str(netx.get("row_id") or "")
|
||||
return ""
|
||||
|
||||
def _append_unchanged_diff(
|
||||
orig: dict[str, Any], mapped: dict[str, Any], after_row: dict[str, Any]
|
||||
) -> None:
|
||||
nonlocal unchanged_listed
|
||||
unchanged_listed += 1
|
||||
before_rid = _row_id(orig)
|
||||
after_rid = _row_id(after_row)
|
||||
if compact_unchanged:
|
||||
for orig, mapped in zip(before_rows, before_mapped):
|
||||
k = row_key(mapped, match_keys)
|
||||
if k in before_keys:
|
||||
before_dup += 1
|
||||
before_dup_keys.append(k)
|
||||
duplicate += 1
|
||||
diffs.append(
|
||||
{
|
||||
"kind": "unchanged",
|
||||
"key": _key_obj(mapped),
|
||||
"before": {},
|
||||
"after": {},
|
||||
"mapped_before": {},
|
||||
"changes": {},
|
||||
"compact": True,
|
||||
"before_row_id": before_rid,
|
||||
"after_row_id": after_rid,
|
||||
}
|
||||
)
|
||||
else:
|
||||
diffs.append(
|
||||
{
|
||||
"kind": "unchanged",
|
||||
"kind": "duplicate",
|
||||
"side": "before",
|
||||
"key": _key_obj(mapped),
|
||||
"before": orig,
|
||||
"after": after_row,
|
||||
"after": after_index.get(k),
|
||||
"mapped_before": mapped,
|
||||
"changes": {},
|
||||
"before_row_id": before_rid,
|
||||
"after_row_id": after_rid,
|
||||
}
|
||||
)
|
||||
|
||||
# Stable key order: before encounter order, then after-only keys
|
||||
ordered_keys = chain(before_groups, (k for k in after_groups if k not in before_groups))
|
||||
for k in ordered_keys:
|
||||
b_list = before_groups.get(k) or []
|
||||
a_list = after_groups.get(k) or []
|
||||
if len(b_list) > 1:
|
||||
multi_before_keys.append(k)
|
||||
if len(a_list) > 1:
|
||||
multi_after_keys.append(k)
|
||||
if sort_pairs and (len(b_list) > 1 or len(a_list) > 1):
|
||||
# Same-key rows pair by value, not by CLI output order (ECMP, multipath).
|
||||
b_list = sorted(b_list, key=lambda pair: pipeline.pair_sort_key(pair[1]))
|
||||
a_list = sorted(a_list, key=pipeline.pair_sort_key)
|
||||
n = max(len(b_list), len(a_list))
|
||||
for i in range(n):
|
||||
if i >= len(b_list):
|
||||
after = a_list[i]
|
||||
added += 1
|
||||
diffs.append(
|
||||
{
|
||||
"kind": "added",
|
||||
"key": _key_obj(after),
|
||||
"before": None,
|
||||
"after": after,
|
||||
"mapped_before": None,
|
||||
"changes": {},
|
||||
"before_row_id": "",
|
||||
"after_row_id": _row_id(after),
|
||||
}
|
||||
)
|
||||
continue
|
||||
if i >= len(a_list):
|
||||
orig, mapped = b_list[i]
|
||||
continue # only first before row participates in match
|
||||
before_keys.add(k)
|
||||
after = after_index.get(k)
|
||||
if after is None:
|
||||
removed += 1
|
||||
diffs.append(
|
||||
{
|
||||
|
|
@ -381,13 +218,11 @@ def compare_rows(
|
|||
"after": None,
|
||||
"mapped_before": mapped,
|
||||
"changes": {},
|
||||
"before_row_id": _row_id(orig),
|
||||
"after_row_id": "",
|
||||
}
|
||||
)
|
||||
continue
|
||||
orig, mapped = b_list[i]
|
||||
after = a_list[i]
|
||||
# Empty compare_fields = presence-only: keyed rows that exist on both
|
||||
# sides are unchanged (no value checks).
|
||||
field_changes: dict[str, dict[str, Any]] = {}
|
||||
for f in compare_fields:
|
||||
bv = mapped.get(f, "")
|
||||
|
|
@ -409,23 +244,49 @@ def compare_rows(
|
|||
"after": after,
|
||||
"mapped_before": mapped,
|
||||
"changes": field_changes,
|
||||
"before_row_id": _row_id(orig),
|
||||
"after_row_id": _row_id(after),
|
||||
}
|
||||
)
|
||||
else:
|
||||
unchanged += 1
|
||||
if include_unchanged:
|
||||
if sample is not None:
|
||||
if sample.limit:
|
||||
sample.add((orig, mapped, after), stratum_key(mapped))
|
||||
else:
|
||||
unchanged_candidates.append((orig, mapped, after))
|
||||
diffs.append(
|
||||
{
|
||||
"kind": "unchanged",
|
||||
"key": _key_obj(mapped),
|
||||
"before": orig,
|
||||
"after": after,
|
||||
"mapped_before": mapped,
|
||||
"changes": {},
|
||||
}
|
||||
)
|
||||
|
||||
if include_unchanged:
|
||||
picked = sample.picked() if sample is not None else unchanged_candidates
|
||||
for orig, mapped, after_row in picked:
|
||||
_append_unchanged_diff(orig, mapped, after_row)
|
||||
for k, after in after_index.items():
|
||||
if k in before_keys:
|
||||
continue
|
||||
added += 1
|
||||
diffs.append(
|
||||
{
|
||||
"kind": "added",
|
||||
"key": _key_obj(after),
|
||||
"before": None,
|
||||
"after": after,
|
||||
"mapped_before": None,
|
||||
"changes": {},
|
||||
}
|
||||
)
|
||||
|
||||
for k, after in after_dup_rows:
|
||||
duplicate += 1
|
||||
diffs.append(
|
||||
{
|
||||
"kind": "duplicate",
|
||||
"side": "after",
|
||||
"key": _key_obj(after),
|
||||
"before": None,
|
||||
"after": after,
|
||||
"mapped_before": None,
|
||||
"changes": {},
|
||||
}
|
||||
)
|
||||
|
||||
def _fmt_keys(keys: list[tuple[str, ...]]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
|
|
@ -435,7 +296,7 @@ def compare_rows(
|
|||
if s not in seen:
|
||||
seen.add(s)
|
||||
out.append(s)
|
||||
return out[:64]
|
||||
return out
|
||||
|
||||
stats = mapping_stats(
|
||||
before_rows=before_norm,
|
||||
|
|
@ -453,21 +314,11 @@ def compare_rows(
|
|||
"removed": removed,
|
||||
"changed": changed,
|
||||
"unchanged": unchanged,
|
||||
"duplicate": 0,
|
||||
"duplicate": duplicate,
|
||||
"match_key_fields": match_keys,
|
||||
"duplicate_keys_before": len(multi_before_keys),
|
||||
"duplicate_keys_after": len(multi_after_keys),
|
||||
"duplicate_key_list": _fmt_keys(multi_before_keys + multi_after_keys),
|
||||
"unchanged_listed": unchanged_listed,
|
||||
"unchanged_truncated": bool(
|
||||
include_unchanged and limit_n is not None and unchanged > unchanged_listed
|
||||
),
|
||||
"unchanged_compact": bool(compact_unchanged and unchanged_listed > 0),
|
||||
"unchanged_sample_mode": (
|
||||
"stratified"
|
||||
if include_unchanged and limit_n is not None and unchanged > unchanged_listed
|
||||
else ("full" if include_unchanged else "none")
|
||||
),
|
||||
"duplicate_keys_before": before_dup,
|
||||
"duplicate_keys_after": after_dup,
|
||||
"duplicate_key_list": _fmt_keys(before_dup_keys + after_dup_keys),
|
||||
},
|
||||
"diffs": diffs,
|
||||
"mapping_stats": stats,
|
||||
|
|
|
|||
|
|
@ -1,194 +0,0 @@
|
|||
"""Pre-flight check of a port mapping against the two batches it will be applied to.
|
||||
|
||||
The same function backs the compare "validate mapping" button and the cutover
|
||||
project pre-flight, so both modes judge mapping coverage the same way. It answers
|
||||
before anyone runs a comparison:
|
||||
|
||||
- which *before* interfaces are mapped (exact / by parent), unmapped but present on
|
||||
the *after* side by the same name, or **suspect** (not mapped, not on *after*)
|
||||
- which map keys / targets match nothing, which targets receive several sources
|
||||
- whether each sheet's two sides were actually collected (``data_state``)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Mapping, Sequence
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .compare_quality import assess_sheet, command_coverage, collect_quality, side_receipts
|
||||
from .identity_pipeline import build_pipeline
|
||||
from .iface_normalize import normalize_port_map, resolve_mapped_iface
|
||||
|
||||
LIST_CAP = 200
|
||||
# Million-row tables are never loaded just to look at interface names.
|
||||
HEAVY_METRICS = frozenset({"bgp_route", "ip_route", "ipv6_route", "l2vpn_mac", "evpn_mac", "interface_detail"})
|
||||
|
||||
RowsLoader = Callable[[str, Mapping[str, Any]], list[dict[str, Any]]]
|
||||
|
||||
|
||||
def _parents(name: str) -> list[str]:
|
||||
parts = name.split(".")
|
||||
return [".".join(parts[:i]) for i in range(len(parts) - 1, 0, -1)]
|
||||
|
||||
|
||||
def _classify(name: str, pmap: Mapping[str, str], after: set[str]) -> str:
|
||||
if name in pmap:
|
||||
return "mapped_exact"
|
||||
if any(p in pmap for p in _parents(name)):
|
||||
return "mapped_by_parent"
|
||||
if name in after:
|
||||
return "identity_same_name"
|
||||
return "suspect_unmapped"
|
||||
|
||||
|
||||
def _present(name: str, names: set[str]) -> bool:
|
||||
"""``name`` exists, or is the parent of an existing sub-interface."""
|
||||
if name in names:
|
||||
return True
|
||||
prefix = name + "."
|
||||
return any(n.startswith(prefix) for n in names)
|
||||
|
||||
|
||||
def _top_level(names: Sequence[str]) -> list[str]:
|
||||
"""Drop sub-interfaces whose parent is already in the list (noise)."""
|
||||
pool = set(names)
|
||||
return sorted(n for n in pool if not any(p in pool for p in _parents(n)))
|
||||
|
||||
|
||||
def sheet_mapping_coverage(
|
||||
pipe: Any,
|
||||
before_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]],
|
||||
after_pairs: Sequence[tuple[Mapping[str, Any], Mapping[str, Any]]],
|
||||
) -> dict[str, Any]:
|
||||
"""Classify every *before* interface of a sheet against the map and the *after* side.
|
||||
|
||||
``before_pairs`` / ``after_pairs`` are ``pipeline.prepare`` outputs. The result
|
||||
carries two private sets (``_after_ifaces``, ``_used_keys``) for aggregation.
|
||||
"""
|
||||
b_if = {str(n.get(f) or "").strip() for _, n in before_pairs for f in pipe.iface_fields}
|
||||
a_if = {str(n.get(f) or "").strip() for _, n in after_pairs for f in pipe.iface_fields}
|
||||
b_if.discard("")
|
||||
a_if.discard("")
|
||||
buckets: dict[str, list[str]] = {
|
||||
"mapped_exact": [], "mapped_by_parent": [], "identity_same_name": [], "suspect_unmapped": [],
|
||||
}
|
||||
for name in sorted(b_if):
|
||||
buckets[_classify(name, pipe.port_map, a_if)].append(name)
|
||||
used = {k for name in b_if for k in pipe.port_map if name == k or name.startswith(k + ".")}
|
||||
missing_targets = sorted({
|
||||
resolve_mapped_iface(n, pipe.port_map) for n in buckets["mapped_exact"] + buckets["mapped_by_parent"]
|
||||
if not _present(resolve_mapped_iface(n, pipe.port_map), a_if)
|
||||
})
|
||||
suspects = _top_level(buckets["suspect_unmapped"])
|
||||
return {
|
||||
"counts": {k: len(v) for k, v in buckets.items()},
|
||||
"mapped_exact": buckets["mapped_exact"][:LIST_CAP],
|
||||
"mapped_by_parent": buckets["mapped_by_parent"][:LIST_CAP],
|
||||
"identity_same_name": buckets["identity_same_name"][:LIST_CAP],
|
||||
"suspect_unmapped": suspects[:LIST_CAP],
|
||||
"suspect_unmapped_total": len(suspects),
|
||||
"mapped_target_missing_on_after": missing_targets[:LIST_CAP],
|
||||
"_after_ifaces": a_if,
|
||||
"_used_keys": used,
|
||||
}
|
||||
|
||||
|
||||
def preflight(
|
||||
db: Session,
|
||||
*,
|
||||
sheets: Sequence[Mapping[str, Any]],
|
||||
iface_rules: Sequence[Mapping[str, Any]] | None,
|
||||
raw_port_map: Mapping[str, str] | None,
|
||||
before_batch_id: str,
|
||||
after_batch_id: str,
|
||||
before_vendor: str = "",
|
||||
after_vendor: str = "",
|
||||
load_rows: RowsLoader,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the pre-flight report. ``load_rows(side, sheet)`` returns raw rows."""
|
||||
raw_map = dict(raw_port_map or {})
|
||||
out_sheets: list[dict[str, Any]] = []
|
||||
used_keys: set[str] = set()
|
||||
after_all: set[str] = set()
|
||||
totals = {"mapped": 0, "suspect": 0, "same_name": 0}
|
||||
risk_reasons: list[str] = []
|
||||
full_pmap: dict[str, str] = {}
|
||||
conflicts: list[dict[str, str]] = []
|
||||
|
||||
for sheet in sheets:
|
||||
sid = str(sheet.get("sheet_id") or sheet.get("metric_id") or "")
|
||||
mid = str(sheet.get("metric_id") or "")
|
||||
title = str(sheet.get("title") or sid)
|
||||
pipe = build_pipeline(
|
||||
sheet, iface_rules=iface_rules, port_map=raw_map,
|
||||
before_vendor=before_vendor, after_vendor=after_vendor,
|
||||
)
|
||||
if not full_pmap and pipe.port_map:
|
||||
full_pmap = dict(pipe.port_map)
|
||||
heavy = mid in HEAVY_METRICS
|
||||
card: dict[str, Any] = {"sheet_id": sid, "metric_id": mid, "title": title}
|
||||
if heavy:
|
||||
b_n = collect_quality(side_receipts(db, before_batch_id, mid))["row_count"]
|
||||
a_n = collect_quality(side_receipts(db, after_batch_id, mid))["row_count"]
|
||||
card.update(skipped="heavy_table", before_count=b_n, after_count=a_n)
|
||||
else:
|
||||
b_pairs = pipe.prepare(load_rows("before", sheet), "before")
|
||||
a_pairs = pipe.prepare(load_rows("after", sheet), "after")
|
||||
card.update(before_count=len(b_pairs), after_count=len(a_pairs))
|
||||
if pipe.iface_fields:
|
||||
cov = sheet_mapping_coverage(pipe, b_pairs, a_pairs)
|
||||
after_all |= cov.pop("_after_ifaces")
|
||||
used_keys |= cov.pop("_used_keys")
|
||||
card.update(cov)
|
||||
totals["mapped"] += cov["counts"]["mapped_exact"] + cov["counts"]["mapped_by_parent"]
|
||||
totals["same_name"] += cov["counts"]["identity_same_name"]
|
||||
totals["suspect"] += cov["suspect_unmapped_total"]
|
||||
card.update(assess_sheet(
|
||||
db, sheet, before_batch_id=before_batch_id, after_batch_id=after_batch_id,
|
||||
before_rows=int(card.get("before_count") or 0), after_rows=int(card.get("after_count") or 0),
|
||||
))
|
||||
if card["data_state"] in ("not_collected", "suspect_parse"):
|
||||
risk_reasons.append(f"{sid}:{card['data_state']}")
|
||||
out_sheets.append(card)
|
||||
|
||||
if raw_map and not full_pmap:
|
||||
full_pmap, conflicts = normalize_port_map(raw_map, iface_rules)
|
||||
elif raw_map:
|
||||
_, conflicts = normalize_port_map(raw_map, iface_rules)
|
||||
sources: dict[str, list[str]] = {}
|
||||
for k, v in full_pmap.items():
|
||||
sources.setdefault(v, []).append(k)
|
||||
n_to_1 = [{"target": t, "sources": sorted(src)} for t, src in sorted(sources.items()) if len(src) > 1]
|
||||
unused = sorted(k for k in full_pmap if k not in used_keys)
|
||||
targets_missing = sorted(v for v in set(full_pmap.values()) if after_all and not _present(v, after_all))
|
||||
|
||||
judged = totals["mapped"] + totals["suspect"]
|
||||
coverage = {
|
||||
"mapped": totals["mapped"], "suspect_unmapped": totals["suspect"],
|
||||
"identity_same_name": totals["same_name"],
|
||||
"rate": round(totals["mapped"] / judged * 100, 1) if judged else None,
|
||||
}
|
||||
mapping = {
|
||||
"pairs": len(full_pmap), "conflicts": conflicts[:LIST_CAP],
|
||||
"unused_map_keys": unused[:LIST_CAP], "unused_map_keys_total": len(unused),
|
||||
"map_targets_missing_on_after": targets_missing[:LIST_CAP],
|
||||
"map_targets_missing_on_after_total": len(targets_missing),
|
||||
"n_to_1_targets": n_to_1[:LIST_CAP],
|
||||
}
|
||||
if totals["suspect"]:
|
||||
risk_reasons.append("suspect_unmapped")
|
||||
if unused:
|
||||
risk_reasons.append("unused_map_keys")
|
||||
if targets_missing:
|
||||
risk_reasons.append("map_targets_missing_on_after")
|
||||
if conflicts:
|
||||
risk_reasons.append("map_conflicts")
|
||||
return {
|
||||
"verdict": "risk" if risk_reasons else "ready",
|
||||
"risk_reasons": sorted(set(risk_reasons)),
|
||||
"coverage": coverage,
|
||||
"mapping": mapping,
|
||||
"sheets": out_sheets,
|
||||
"command_coverage": command_coverage(db, before_batch_id=before_batch_id, after_batch_id=after_batch_id),
|
||||
}
|
||||
|
|
@ -1,198 +0,0 @@
|
|||
"""Collection quality of a compare side: is a missing row a business loss or a data problem?
|
||||
|
||||
Compare results are only as good as the two batches behind them. This module
|
||||
reads *receipts* (``BizStateBatchCommand``: counts and status, never ``raw_text``)
|
||||
to tell apart:
|
||||
|
||||
- ``not_collected`` – a side never ran a command (or the slice) for the sheet
|
||||
- ``suspect_parse`` – the device printed lines but the parser produced no rows
|
||||
- ``no_data`` – both sides ran fine and are really empty
|
||||
- ``ok`` – ordinary comparison
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import BizStateBatchCommand
|
||||
from .compare_rules import scalar_text
|
||||
|
||||
# More than this many CLI lines with zero parsed rows is not "an empty table".
|
||||
SUSPECT_RAW_LINES = 5
|
||||
DATA_STATES = ("ok", "no_data", "suspect_parse", "not_collected")
|
||||
UNJUDGED_STATES = ("no_data", "not_collected")
|
||||
|
||||
|
||||
def _has_receipts(db: Session, batch_id: str) -> bool:
|
||||
return bool(batch_id) and db.query(BizStateBatchCommand.id).filter(
|
||||
BizStateBatchCommand.batch_id == batch_id).limit(1).first() is not None
|
||||
|
||||
|
||||
def side_receipts(db: Session, batch_id: str, metric_id: str) -> list[dict[str, Any]]:
|
||||
if not batch_id or not metric_id:
|
||||
return []
|
||||
rows = (
|
||||
db.query(
|
||||
BizStateBatchCommand.profile_id, BizStateBatchCommand.parse_status,
|
||||
BizStateBatchCommand.row_count, BizStateBatchCommand.raw_line_count,
|
||||
BizStateBatchCommand.declared_total,
|
||||
)
|
||||
.filter(BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id == metric_id)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"profile_id": str(r[0] or ""), "parse_status": str(r[1] or ""),
|
||||
"row_count": int(r[2] or 0), "raw_line_count": int(r[3] or 0),
|
||||
"declared_total": int(r[4] or 0),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def collect_quality(receipts: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
statuses: dict[str, int] = {}
|
||||
for r in receipts:
|
||||
key = str(r.get("parse_status") or "unknown")
|
||||
statuses[key] = statuses.get(key, 0) + 1
|
||||
return {
|
||||
"commands": len(receipts),
|
||||
"parse_status": statuses,
|
||||
"row_count": sum(int(r.get("row_count") or 0) for r in receipts),
|
||||
"raw_line_count": sum(int(r.get("raw_line_count") or 0) for r in receipts),
|
||||
"declared_total": sum(int(r.get("declared_total") or 0) for r in receipts),
|
||||
"profile_ids": sorted({str(r.get("profile_id") or "") for r in receipts if r.get("profile_id")}),
|
||||
}
|
||||
|
||||
|
||||
def sheet_slice(sheet: Mapping[str, Any]) -> dict[str, str]:
|
||||
"""``{field: value}`` the sheet pins with top-level ``eq`` row filters."""
|
||||
out: dict[str, str] = {}
|
||||
for f in sheet.get("row_filters") or []:
|
||||
if not isinstance(f, Mapping) or "any" in f or "all" in f:
|
||||
continue
|
||||
if str(f.get("op") or "eq").strip().lower() not in ("eq", "==", "ci_eq"):
|
||||
continue
|
||||
field = str(f.get("field") or "").strip()
|
||||
value = scalar_text(f.get("value")).strip().lower()
|
||||
if field and value:
|
||||
out[field] = value
|
||||
return out
|
||||
|
||||
|
||||
def covers_slice(
|
||||
receipts: Sequence[Mapping[str, Any]], wanted: Mapping[str, str], *, profile_slice: Any = None,
|
||||
) -> bool:
|
||||
"""True when at least one successful command could have produced ``wanted``.
|
||||
|
||||
A command without a declared slice (or unknown profile) covers everything.
|
||||
"""
|
||||
if profile_slice is None:
|
||||
from .profiles import get_profile
|
||||
|
||||
def profile_slice(pid: str) -> dict[str, str]: # noqa: F811
|
||||
prof = get_profile(pid) if pid else None
|
||||
return dict(prof.slice) if prof is not None and prof.slice else {}
|
||||
|
||||
for r in receipts:
|
||||
if str(r.get("parse_status") or "") in ("failed", "skipped_custom"):
|
||||
continue
|
||||
declared = profile_slice(str(r.get("profile_id") or ""))
|
||||
if all(
|
||||
f not in declared or str(declared[f]).strip().lower() == v
|
||||
for f, v in wanted.items()
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def classify_sheet(
|
||||
*,
|
||||
before: Sequence[Mapping[str, Any]] | None,
|
||||
after: Sequence[Mapping[str, Any]] | None,
|
||||
before_rows: int,
|
||||
after_rows: int,
|
||||
wanted_slice: Mapping[str, str] | None = None,
|
||||
) -> tuple[str, list[str]]:
|
||||
"""``(data_state, reasons)`` for one sheet; reasons name the offending side.
|
||||
|
||||
``None`` receipts mean the batch carries no receipts at all (imported or
|
||||
legacy data): that side cannot be judged and is trusted as-is.
|
||||
"""
|
||||
wanted = dict(wanted_slice or {})
|
||||
reasons: list[str] = []
|
||||
for side, receipts in (("before", before), ("after", after)):
|
||||
if receipts is None:
|
||||
continue
|
||||
if not receipts:
|
||||
reasons.append(f"{side}:no_command")
|
||||
elif wanted and not covers_slice(receipts, wanted):
|
||||
reasons.append(f"{side}:slice_not_collected")
|
||||
if reasons:
|
||||
return "not_collected", reasons
|
||||
for side, receipts in (("before", before), ("after", after)):
|
||||
if receipts is None:
|
||||
continue
|
||||
q = collect_quality(receipts)
|
||||
if q["row_count"] == 0 and q["raw_line_count"] > SUSPECT_RAW_LINES:
|
||||
reasons.append(f"{side}:parsed_0_of_{q['raw_line_count']}_lines")
|
||||
elif q["commands"] and q["parse_status"].get("failed", 0) == q["commands"]:
|
||||
reasons.append(f"{side}:all_commands_failed")
|
||||
if reasons:
|
||||
return "suspect_parse", reasons
|
||||
if before_rows == 0 and after_rows == 0:
|
||||
return "no_data", ["both_sides_empty"]
|
||||
return "ok", []
|
||||
|
||||
|
||||
def assess_sheet(
|
||||
db: Session, sheet: Mapping[str, Any], *, before_batch_id: str, after_batch_id: str,
|
||||
before_rows: int, after_rows: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Quality block stored on a sheet summary."""
|
||||
metric_id = str(sheet.get("metric_id") or "")
|
||||
before = side_receipts(db, before_batch_id, metric_id) if _has_receipts(db, before_batch_id) else None
|
||||
after = side_receipts(db, after_batch_id, metric_id) if _has_receipts(db, after_batch_id) else None
|
||||
state, reasons = classify_sheet(
|
||||
before=before, after=after, before_rows=before_rows, after_rows=after_rows,
|
||||
wanted_slice=sheet_slice(sheet),
|
||||
)
|
||||
return {
|
||||
"data_state": state,
|
||||
"data_state_reasons": reasons,
|
||||
"collect_quality": {
|
||||
"before": collect_quality(before or []), "after": collect_quality(after or []),
|
||||
"unknown_sides": [n for n, r in (("before", before), ("after", after)) if r is None],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def command_coverage(
|
||||
db: Session, *, before_batch_id: str, after_batch_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Collected "metric{slice}" labels per side and their difference.
|
||||
|
||||
Profile ids differ per vendor (``zte.x`` vs ``huawei.x``), so the comparison
|
||||
is on what a command *produces*: its metric and slice.
|
||||
"""
|
||||
from .profiles import get_profile
|
||||
|
||||
def labels(batch_id: str) -> set[str]:
|
||||
if not batch_id:
|
||||
return set()
|
||||
rows = db.query(BizStateBatchCommand.metric_id, BizStateBatchCommand.profile_id).filter(
|
||||
BizStateBatchCommand.batch_id == batch_id, BizStateBatchCommand.metric_id != "",
|
||||
).distinct().all()
|
||||
out: set[str] = set()
|
||||
for metric_id, profile_id in rows:
|
||||
prof = get_profile(str(profile_id or "")) if profile_id else None
|
||||
sl = dict(prof.slice) if prof is not None and prof.slice else {}
|
||||
suffix = ",".join(f"{k}={v}" for k, v in sorted(sl.items()))
|
||||
out.add(f"{metric_id}{{{suffix}}}" if suffix else str(metric_id))
|
||||
return out
|
||||
|
||||
b, a = labels(before_batch_id), labels(after_batch_id)
|
||||
return {"before": sorted(b), "after": sorted(a),
|
||||
"only_before": sorted(b - a), "only_after": sorted(a - b)}
|
||||
|
|
@ -7,21 +7,10 @@ All metric-specific compare behavior belongs in the sheet template
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import math
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
|
||||
_AGE_TIMER_RE = re.compile(r"^\d{1,2}:\d{2}:\d{2}$")
|
||||
NUMBER_PATTERN = r"^[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?$"
|
||||
|
||||
|
||||
def scalar_text(value: Any) -> str:
|
||||
"""Match JSON text extraction: only null is blank; keep zero and booleans."""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
return str(value)
|
||||
|
||||
|
||||
def _as_list(raw: Any) -> list[Any]:
|
||||
|
|
@ -33,7 +22,7 @@ def _as_list(raw: Any) -> list[Any]:
|
|||
|
||||
|
||||
def _field_val(row: Mapping[str, Any], field: str) -> str:
|
||||
return scalar_text((row or {}).get(field)).strip()
|
||||
return str((row or {}).get(field) or "").strip()
|
||||
|
||||
|
||||
def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool:
|
||||
|
|
@ -46,17 +35,17 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool:
|
|||
expect = filt.get("value")
|
||||
|
||||
if op in ("eq", "=="):
|
||||
return raw.lower() == scalar_text(expect).strip().lower()
|
||||
return raw.lower() == str(expect or "").strip().lower()
|
||||
if op in ("ne", "!="):
|
||||
return raw.lower() != scalar_text(expect).strip().lower()
|
||||
return raw.lower() != str(expect or "").strip().lower()
|
||||
if op == "in":
|
||||
opts = {scalar_text(x).strip().lower() for x in _as_list(expect) if scalar_text(x).strip()}
|
||||
opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()}
|
||||
return raw.lower() in opts
|
||||
if op in ("not_in", "nin"):
|
||||
opts = {scalar_text(x).strip().lower() for x in _as_list(expect) if scalar_text(x).strip()}
|
||||
opts = {str(x).strip().lower() for x in _as_list(expect) if str(x).strip()}
|
||||
return raw.lower() not in opts
|
||||
if op == "contains":
|
||||
needle = scalar_text(expect).strip().lower()
|
||||
needle = str(expect or "").strip().lower()
|
||||
return bool(needle) and needle in raw.lower()
|
||||
if op == "empty":
|
||||
return not raw
|
||||
|
|
@ -74,7 +63,7 @@ def eval_leaf_filter(row: Mapping[str, Any], filt: Mapping[str, Any]) -> bool:
|
|||
# HH:MM:SS dynamic ARP age
|
||||
return bool(_AGE_TIMER_RE.match(raw))
|
||||
if op == "ci_eq":
|
||||
return raw.lower() == scalar_text(expect).strip().lower()
|
||||
return raw.lower() == str(expect or "").strip().lower()
|
||||
return True
|
||||
|
||||
|
||||
|
|
@ -110,7 +99,7 @@ def apply_row_filters(
|
|||
|
||||
|
||||
def normalize_value(value: Any, how: str) -> str:
|
||||
text = scalar_text(value).strip()
|
||||
text = str(value if value is not None else "").strip()
|
||||
mode = str(how or "").strip().lower()
|
||||
if not mode or mode in ("none", "strip"):
|
||||
return text
|
||||
|
|
@ -137,8 +126,6 @@ def field_rule_map(rules: Sequence[Mapping[str, Any]] | None) -> dict[str, dict[
|
|||
name = str(raw.get("field") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
if str(raw.get("side") or "").strip():
|
||||
continue # side-scoped rules feed the identity pipeline, not the compare
|
||||
out[name] = dict(raw)
|
||||
return out
|
||||
|
||||
|
|
@ -203,59 +190,12 @@ def effective_display_fields(
|
|||
|
||||
|
||||
def _parse_float(text: str) -> float | None:
|
||||
if not re.fullmatch(NUMBER_PATTERN, text):
|
||||
return None
|
||||
try:
|
||||
value = float(text)
|
||||
return value if math.isfinite(value) else None
|
||||
return float(text) if text else 0.0
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
DIRECTIONAL_MODES = ("not_worse", "numeric_min", "numeric_max")
|
||||
|
||||
|
||||
def _rank_index(value: str, rank: Sequence[Any]) -> int | None:
|
||||
wanted = value.strip().lower()
|
||||
for i, item in enumerate(rank or []):
|
||||
if scalar_text(item).strip().lower() == wanted:
|
||||
return i
|
||||
return None
|
||||
|
||||
|
||||
def _tolerance(rule: Mapping[str, Any]) -> float:
|
||||
try:
|
||||
return float(rule.get("tolerance") or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def _directional_verdict(bv: str, av: str, rule: Mapping[str, Any]) -> tuple[bool, str]:
|
||||
"""``(equal_or_better, reason)`` for not_worse / numeric_min / numeric_max."""
|
||||
mode = str(rule.get("compare") or "").strip().lower()
|
||||
if mode == "not_worse":
|
||||
rank = rule.get("rank") or []
|
||||
bi, ai = _rank_index(bv, rank), _rank_index(av, rank)
|
||||
if bi is None or ai is None:
|
||||
return bv == av, "neq"
|
||||
return ai <= bi, (f"worse: {bv}→{av}" if ai > bi else "")
|
||||
bn, an = _parse_float(bv), _parse_float(av)
|
||||
if bn is None or an is None:
|
||||
return bv == av, "neq"
|
||||
tol = _tolerance(rule)
|
||||
if mode == "numeric_min":
|
||||
floor = bn * (1 - tol / 100.0)
|
||||
if an >= floor:
|
||||
return True, ""
|
||||
drop = (bn - an) / abs(bn) * 100.0 if bn else 0.0
|
||||
return False, f"drop {drop:.0f}% > {tol:g}%"
|
||||
ceil = bn * (1 + tol / 100.0)
|
||||
if an <= ceil:
|
||||
return True, ""
|
||||
rise = (an - bn) / abs(bn) * 100.0 if bn else 0.0
|
||||
return False, f"rise {rise:.0f}% > {tol:g}%"
|
||||
|
||||
|
||||
def explain_diff(
|
||||
before: Any,
|
||||
after: Any,
|
||||
|
|
@ -268,8 +208,6 @@ def explain_diff(
|
|||
bv = normalize_value(before, norm)
|
||||
av = normalize_value(after, norm)
|
||||
mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
|
||||
if mode in DIRECTIONAL_MODES:
|
||||
return _directional_verdict(bv, av, rule)[1]
|
||||
if mode in ("percent", "pct", "rel"):
|
||||
bn = _parse_float(bv)
|
||||
an = _parse_float(av)
|
||||
|
|
@ -310,8 +248,6 @@ def values_equal(
|
|||
mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
|
||||
if mode in ("ignore", "skip", "off"):
|
||||
return True
|
||||
if mode in DIRECTIONAL_MODES:
|
||||
return _directional_verdict(bv, av, rule)[0]
|
||||
if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"):
|
||||
bn = _parse_float(bv)
|
||||
an = _parse_float(av)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,801 +0,0 @@
|
|||
"""PostgreSQL-backed sheet compare (FULL OUTER JOIN) for pushdown-safe sheets.
|
||||
|
||||
Falls back to the Python engine when port-map / complex rules cannot be expressed
|
||||
in SQL. See ``can_sql_compare``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Callable, Mapping, Sequence
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .compare_engine import key_tuple
|
||||
from .compare_rules import effective_compare_fields, field_rule_map, NUMBER_PATTERN, scalar_text
|
||||
|
||||
_log = logging.getLogger("netx.biz_state.compare_sql")
|
||||
|
||||
# Below this, Python hash-join is faster and avoids TEMP CTAS / progress races.
|
||||
_SQL_MIN_ROWS = 20_000
|
||||
# Cap a single SQL statement so a stuck planner/lock surfaces as fallback.
|
||||
_SQL_STATEMENT_TIMEOUT_MS = 180_000
|
||||
|
||||
_FIELD_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
class SqlCompareSkip(Exception):
|
||||
"""Soft skip — caller should use the Python engine (not an error)."""
|
||||
|
||||
def __init__(self, reason: str) -> None:
|
||||
super().__init__(reason)
|
||||
self.reason = str(reason or "skip")
|
||||
_SQL_FILTER_OPS = frozenset(
|
||||
{"eq", "==", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "ci_eq"}
|
||||
)
|
||||
_SQL_NORMALIZE = frozenset({"", "none", "strip", "lower", "upper", "empty_as_blank"})
|
||||
_SQL_COMPARE_MODES = frozenset(
|
||||
{
|
||||
"",
|
||||
"eq",
|
||||
"ignore",
|
||||
"skip",
|
||||
"off",
|
||||
"numeric",
|
||||
"number",
|
||||
"int",
|
||||
"float",
|
||||
"percent",
|
||||
"pct",
|
||||
"rel",
|
||||
}
|
||||
)
|
||||
_EMPTY_AS_BLANK = ("n/a", "na", "-", "--", "none", "null")
|
||||
_NUM_RE_SQL = NUMBER_PATTERN
|
||||
|
||||
|
||||
def _dialect_is_postgres(db: Session) -> bool:
|
||||
bind = db.get_bind()
|
||||
if bind is None:
|
||||
return False
|
||||
return str(getattr(bind.dialect, "name", "") or "").lower() in ("postgresql", "postgres")
|
||||
|
||||
|
||||
def _safe_field(name: str) -> str:
|
||||
f = str(name or "").strip()
|
||||
if not f or not _FIELD_RE.match(f):
|
||||
raise ValueError(f"unsafe_json_field:{name!r}")
|
||||
return f
|
||||
|
||||
|
||||
def _json_text_expr(alias: str, field: str) -> str:
|
||||
"""SQL expression: trimmed text from JSONB column ``alias.data`` / ``data_json``."""
|
||||
f = _safe_field(field)
|
||||
# alias is caller-controlled (_b / data_json) — not user input
|
||||
return f"trim(both from coalesce({alias}->>'{f}', ''))"
|
||||
|
||||
|
||||
def _norm_expr(alias: str, field: str, normalize: str) -> str:
|
||||
base = _json_text_expr(alias, field)
|
||||
mode = str(normalize or "strip").strip().lower() or "strip"
|
||||
if mode in ("", "none", "strip"):
|
||||
return base
|
||||
if mode == "lower":
|
||||
return f"lower({base})"
|
||||
if mode == "upper":
|
||||
return f"upper({base})"
|
||||
if mode == "empty_as_blank":
|
||||
opts = ", ".join(f"'{x}'" for x in _EMPTY_AS_BLANK)
|
||||
return (
|
||||
f"CASE WHEN lower({base}) IN ({opts}) THEN '' ELSE {base} END"
|
||||
)
|
||||
raise ValueError(f"unsupported_normalize:{mode}")
|
||||
|
||||
|
||||
def _filters_sql_compatible(filters: Sequence[Mapping[str, Any]] | None) -> bool:
|
||||
fl = [f for f in (filters or []) if isinstance(f, dict)]
|
||||
return all(_filter_node_compatible(f) for f in fl)
|
||||
|
||||
|
||||
def _filter_node_compatible(filt: Mapping[str, Any]) -> bool:
|
||||
if "any" in filt:
|
||||
kids = filt.get("any") or []
|
||||
return isinstance(kids, list) and all(
|
||||
isinstance(k, dict) and _filter_node_compatible(k) for k in kids
|
||||
)
|
||||
if "all" in filt:
|
||||
kids = filt.get("all") or []
|
||||
return isinstance(kids, list) and all(
|
||||
isinstance(k, dict) and _filter_node_compatible(k) for k in kids
|
||||
)
|
||||
field = str(filt.get("field") or "").strip()
|
||||
op = str(filt.get("op") or "eq").strip().lower()
|
||||
if op not in _SQL_FILTER_OPS:
|
||||
return False
|
||||
if op in ("empty", "not_empty", "nonempty"):
|
||||
return bool(field) and bool(_FIELD_RE.match(field))
|
||||
if not field or not _FIELD_RE.match(field):
|
||||
return False
|
||||
if op in ("in", "not_in", "nin"):
|
||||
vals = filt.get("value")
|
||||
if vals is None:
|
||||
return True
|
||||
if not isinstance(vals, (list, tuple)):
|
||||
vals = [vals]
|
||||
return all(isinstance(x, (str, int, float, bool)) or x is None for x in vals)
|
||||
return True
|
||||
|
||||
|
||||
def _field_rules_sql_compatible(rules: Sequence[Mapping[str, Any]] | None) -> bool:
|
||||
for raw in rules or []:
|
||||
if not isinstance(raw, dict):
|
||||
return False
|
||||
name = str(raw.get("field") or "").strip()
|
||||
if name and not _FIELD_RE.match(name):
|
||||
return False
|
||||
mode = str(raw.get("compare") or "eq").strip().lower() or "eq"
|
||||
if mode not in _SQL_COMPARE_MODES:
|
||||
return False
|
||||
if raw.get("ignore") is True:
|
||||
continue
|
||||
if mode in ("ignore", "skip", "off"):
|
||||
continue
|
||||
norm = str(raw.get("normalize") or "strip").strip().lower() or "strip"
|
||||
if norm not in _SQL_NORMALIZE:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def sql_compare_skip_reason(
|
||||
db: Session,
|
||||
sheet: Mapping[str, Any],
|
||||
*,
|
||||
port_map: Mapping[str, str] | None = None,
|
||||
iface_normalize_rules: Sequence[Mapping[str, str]] | None = None,
|
||||
pipeline: Any = None,
|
||||
) -> str:
|
||||
"""Empty string when SQL is allowed; otherwise a short reason for progress/logs.
|
||||
|
||||
Simple ``row_filters`` (eq/in/contains/…) used for BGP sheet splits are fine —
|
||||
they do **not** force Python by themselves.
|
||||
|
||||
A port map only blocks SQL when the sheet actually reads an interface field
|
||||
in its key or compare columns; a BGP route table with no interface fields is
|
||||
unaffected by it. ``pipeline`` (v2 identity rules) adds its own blockers.
|
||||
"""
|
||||
if not _dialect_is_postgres(db):
|
||||
return "not_postgres"
|
||||
if pipeline is not None:
|
||||
port_map = pipeline.port_map
|
||||
if iface_normalize_rules is None and (pipeline.before.has_iface_rules or pipeline.after.has_iface_rules):
|
||||
iface_normalize_rules = [{"from": "*", "to": "*"}]
|
||||
key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()]
|
||||
if not key_fields:
|
||||
return "no_key_fields"
|
||||
if any(not _FIELD_RE.match(k) for k in key_fields):
|
||||
return "unsafe_key_field"
|
||||
iface_fields = [str(f).strip() for f in (sheet.get("iface_fields") or []) if str(f).strip()]
|
||||
iface_set = set(iface_fields)
|
||||
if port_map and iface_set:
|
||||
early_cmp = effective_compare_fields(
|
||||
list(sheet.get("compare_fields") or []), list(sheet.get("field_rules") or []),
|
||||
)
|
||||
if iface_set & (set(key_fields) | {str(f).strip() for f in early_cmp}):
|
||||
return "port_map"
|
||||
ignore_ports = sheet.get("ignore_port_changes")
|
||||
if ignore_ports is True:
|
||||
return "ignore_port_changes"
|
||||
if ignore_ports is None and iface_set and any(k in iface_set for k in key_fields):
|
||||
return "auto_ignore_ports"
|
||||
field_rules = list(sheet.get("field_rules") or [])
|
||||
if any(
|
||||
isinstance(r, dict) and str(r.get("compare") or "").strip().lower() in
|
||||
("not_worse", "numeric_min", "numeric_max")
|
||||
for r in field_rules
|
||||
):
|
||||
return "directional_compare"
|
||||
if not _field_rules_sql_compatible(field_rules):
|
||||
return "field_rules"
|
||||
compare_fields = effective_compare_fields(
|
||||
list(sheet.get("compare_fields") or []),
|
||||
field_rules,
|
||||
)
|
||||
if any(not _FIELD_RE.match(str(f).strip()) for f in compare_fields if str(f).strip()):
|
||||
return "unsafe_compare_field"
|
||||
used = set(key_fields) | {str(f).strip() for f in compare_fields if str(f).strip()}
|
||||
if port_map and (used & iface_set):
|
||||
return "port_map"
|
||||
if iface_normalize_rules and (used & iface_set):
|
||||
return "iface_normalize"
|
||||
if pipeline is not None:
|
||||
blocked = pipeline.sql_blockers(used)
|
||||
if blocked:
|
||||
return blocked
|
||||
if not _filters_sql_compatible(list(sheet.get("row_filters") or [])):
|
||||
return "row_filters"
|
||||
if not str(sheet.get("metric_id") or "").strip():
|
||||
return "no_metric_id"
|
||||
return ""
|
||||
|
||||
|
||||
def can_sql_compare(
|
||||
db: Session,
|
||||
sheet: Mapping[str, Any],
|
||||
*,
|
||||
port_map: Mapping[str, str] | None = None,
|
||||
iface_normalize_rules: Sequence[Mapping[str, str]] | None = None,
|
||||
) -> bool:
|
||||
"""True when this sheet can run entirely as a PostgreSQL JOIN."""
|
||||
return not bool(
|
||||
sql_compare_skip_reason(
|
||||
db,
|
||||
sheet,
|
||||
port_map=port_map,
|
||||
iface_normalize_rules=iface_normalize_rules,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def compile_row_filters_sql(
|
||||
filters: Sequence[Mapping[str, Any]] | None,
|
||||
*,
|
||||
json_col: str = "data_json",
|
||||
param_prefix: str = "f",
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
"""Compile template row_filters to SQL AND-clause + bind params.
|
||||
|
||||
Returns ``(sql_fragment, params)``. Empty filters → ``(\"TRUE\", {})``.
|
||||
``json_col`` is the JSONB column/expression name (trusted).
|
||||
"""
|
||||
fl = [f for f in (filters or []) if isinstance(f, dict)]
|
||||
if not fl:
|
||||
return "TRUE", {}
|
||||
params: dict[str, Any] = {}
|
||||
counter = {"n": 0}
|
||||
|
||||
def _next(name: str) -> str:
|
||||
counter["n"] += 1
|
||||
return f"{param_prefix}_{counter['n']}_{name}"
|
||||
|
||||
def _node(filt: Mapping[str, Any]) -> str:
|
||||
if "any" in filt:
|
||||
kids = [k for k in (filt.get("any") or []) if isinstance(k, dict)]
|
||||
if not kids:
|
||||
return "TRUE"
|
||||
return "(" + " OR ".join(_node(k) for k in kids) + ")"
|
||||
if "all" in filt:
|
||||
kids = [k for k in (filt.get("all") or []) if isinstance(k, dict)]
|
||||
if not kids:
|
||||
return "TRUE"
|
||||
return "(" + " AND ".join(_node(k) for k in kids) + ")"
|
||||
field = _safe_field(str(filt.get("field") or ""))
|
||||
op = str(filt.get("op") or "eq").strip().lower()
|
||||
expr = _json_text_expr(json_col, field)
|
||||
# _json_text_expr uses alias->> ; for bare column use data_json directly
|
||||
if json_col == "data_json":
|
||||
expr = f"trim(both from coalesce(data_json->>'{field}', ''))"
|
||||
if op in ("empty",):
|
||||
return f"({expr} = '')"
|
||||
if op in ("not_empty", "nonempty"):
|
||||
return f"({expr} <> '')"
|
||||
expect = filt.get("value")
|
||||
if op in ("eq", "==", "ci_eq"):
|
||||
key = _next("v")
|
||||
params[key] = scalar_text(expect).strip()
|
||||
if op == "ci_eq" or op in ("eq", "=="):
|
||||
# Python eq is case-insensitive via .lower()
|
||||
params[key] = scalar_text(expect).strip().lower()
|
||||
return f"(lower({expr}) = :{key})"
|
||||
if op in ("ne", "!="):
|
||||
key = _next("v")
|
||||
params[key] = scalar_text(expect).strip().lower()
|
||||
return f"(lower({expr}) <> :{key})"
|
||||
if op == "contains":
|
||||
key = _next("v")
|
||||
needle = scalar_text(expect).strip().lower()
|
||||
if not needle:
|
||||
return "FALSE"
|
||||
escaped = needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
params[key] = f"%{escaped}%"
|
||||
return f"(lower({expr}) LIKE :{key} ESCAPE E'\\\\')"
|
||||
if op in ("in", "not_in", "nin"):
|
||||
vals = expect
|
||||
if vals is None:
|
||||
vals = []
|
||||
if not isinstance(vals, (list, tuple)):
|
||||
vals = [vals]
|
||||
clean = [scalar_text(x).strip().lower() for x in vals if scalar_text(x).strip()]
|
||||
if not clean:
|
||||
return "FALSE" if op == "in" else "TRUE"
|
||||
keys = []
|
||||
for i, v in enumerate(clean):
|
||||
k = _next(f"in{i}")
|
||||
params[k] = v
|
||||
keys.append(f":{k}")
|
||||
inside = f"lower({expr}) IN ({', '.join(keys)})"
|
||||
return f"({inside})" if op == "in" else f"(NOT {inside})"
|
||||
raise ValueError(f"unsupported_filter_op:{op}")
|
||||
|
||||
parts = [_node(f) for f in fl]
|
||||
return "(" + " AND ".join(parts) + ")", params
|
||||
|
||||
|
||||
def _rk_sql(key_fields: list[str], *, json_col: str = "data_json") -> str:
|
||||
parts = []
|
||||
for k in key_fields:
|
||||
f = _safe_field(k)
|
||||
if json_col == "data_json":
|
||||
parts.append(f"trim(both from coalesce(data_json->>'{f}', ''))")
|
||||
else:
|
||||
parts.append(f"trim(both from coalesce({json_col}->>'{f}', ''))")
|
||||
if len(parts) == 1:
|
||||
return parts[0]
|
||||
# Delimiter concatenation conflates ('a|b', 'c') with ('a', 'b|c').
|
||||
return "jsonb_build_array(" + ", ".join(parts) + ")::text"
|
||||
|
||||
|
||||
def _field_differs_sql(bv: str, av: str, rule: Mapping[str, Any]) -> str:
|
||||
"""SQL boolean: True when before/after values differ under the field rule."""
|
||||
mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
|
||||
if mode in ("ignore", "skip", "off") or rule.get("ignore") is True:
|
||||
return "FALSE"
|
||||
try:
|
||||
tol = float(rule.get("tolerance") or 0)
|
||||
except (TypeError, ValueError):
|
||||
tol = 0.0
|
||||
if mode in ("numeric", "number", "int", "float", "percent", "pct", "rel"):
|
||||
# Match Python values_equal: parseable → numeric compare; else string eq
|
||||
num_b = f"(({bv}) ~ '{_NUM_RE_SQL}')"
|
||||
num_a = f"(({av}) ~ '{_NUM_RE_SQL}')"
|
||||
bn = f"({bv})::double precision"
|
||||
an = f"({av})::double precision"
|
||||
if mode in ("percent", "pct", "rel"):
|
||||
# differ when relative % > tol (before==0 → after must be 0)
|
||||
num_diff = (
|
||||
f"(CASE WHEN {bn} = 0 THEN {an} IS DISTINCT FROM 0 "
|
||||
f"ELSE (abs({an} - {bn}) / abs({bn}) * 100.0) > {tol} END)"
|
||||
)
|
||||
else:
|
||||
num_diff = f"(abs({an} - {bn}) > {tol})"
|
||||
return (
|
||||
f"(CASE WHEN {num_b} AND {num_a} THEN {num_diff} "
|
||||
f"ELSE ({bv} IS DISTINCT FROM {av}) END)"
|
||||
)
|
||||
return f"({bv} IS DISTINCT FROM {av})"
|
||||
|
||||
|
||||
def _changed_predicate(
|
||||
compare_fields: list[str],
|
||||
rules: dict[str, dict[str, Any]],
|
||||
*,
|
||||
before_alias: str = "b",
|
||||
after_alias: str = "a",
|
||||
) -> str:
|
||||
"""SQL boolean: True when any compare field differs (IS DISTINCT FROM)."""
|
||||
if not compare_fields:
|
||||
return "FALSE"
|
||||
clauses: list[str] = []
|
||||
for f in compare_fields:
|
||||
name = str(f).strip()
|
||||
if not name:
|
||||
continue
|
||||
rule = rules.get(name) or {}
|
||||
mode = str(rule.get("compare") or "eq").strip().lower() or "eq"
|
||||
if mode in ("ignore", "skip", "off") or rule.get("ignore") is True:
|
||||
continue
|
||||
norm = str(rule.get("normalize") or "strip").strip().lower() or "strip"
|
||||
bv = _norm_expr(f"{before_alias}.data", name, norm)
|
||||
av = _norm_expr(f"{after_alias}.data", name, norm)
|
||||
clauses.append(_field_differs_sql(bv, av, rule))
|
||||
if not clauses:
|
||||
return "FALSE"
|
||||
return "(" + " OR ".join(clauses) + ")"
|
||||
|
||||
|
||||
def _key_obj_from_data(data: dict[str, Any] | None, key_fields: list[str]) -> dict[str, Any]:
|
||||
row = data if isinstance(data, dict) else {}
|
||||
return dict(zip(key_fields, key_tuple(row, key_fields)))
|
||||
|
||||
|
||||
def _changes_from_rows(
|
||||
before: dict[str, Any] | None,
|
||||
after: dict[str, Any] | None,
|
||||
compare_fields: list[str],
|
||||
rules: dict[str, dict[str, Any]],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
from .compare_rules import explain_diff, values_equal
|
||||
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
b = before or {}
|
||||
a = after or {}
|
||||
for f in compare_fields:
|
||||
rule = rules.get(f)
|
||||
bv = b.get(f, "")
|
||||
av = a.get(f, "")
|
||||
if values_equal(bv, av, rule=rule):
|
||||
continue
|
||||
entry: dict[str, Any] = {"before": bv, "after": av}
|
||||
reason = explain_diff(bv, av, rule=rule)
|
||||
if reason:
|
||||
entry["reason"] = reason
|
||||
out[f] = entry
|
||||
return out
|
||||
|
||||
|
||||
def dup_pairing_order_sql(compare_fields: Sequence[str], rules: Mapping[str, Mapping[str, Any]]) -> str:
|
||||
"""``ORDER BY`` prefix that pairs same-key rows by value (mirrors ``Pipeline.pair_sort_key``).
|
||||
|
||||
Byte-order collation matches Python's code-point string order, so both
|
||||
engines choose the same pairs. Returns '' (or a trailing-comma list).
|
||||
"""
|
||||
parts = []
|
||||
for f in compare_fields:
|
||||
name = str(f).strip()
|
||||
if not name:
|
||||
continue
|
||||
mode = str((rules.get(name) or {}).get("normalize") or "strip").strip().lower() or "strip"
|
||||
parts.append(f'({_norm_expr("data_json", name, mode)}) COLLATE "C" ASC')
|
||||
return ", ".join(parts) + ", " if parts else ""
|
||||
|
||||
|
||||
def run_sql_sheet_compare(
|
||||
db: Session,
|
||||
*,
|
||||
sheet: Mapping[str, Any],
|
||||
before_batch_id: str,
|
||||
after_batch_id: str,
|
||||
store_unchanged: str = "auto",
|
||||
on_progress: Callable[..., None] | None = None,
|
||||
duplicate_pairing: str = "ordered",
|
||||
) -> dict[str, Any]:
|
||||
"""Compare one sheet via PostgreSQL TEMP tables + FULL OUTER JOIN.
|
||||
|
||||
Same-key multipath: rows get ``dup_rn`` by ``seq,id`` and join on
|
||||
``(rk, dup_rn)`` (ordered zip). Extras are added/removed, not duplicate.
|
||||
|
||||
Returns the same ``{summary, diffs, mapping_stats}`` shape as ``compare_rows``.
|
||||
``on_progress(side, n, *, engine=\"sql\", note=..., phase=...)``.
|
||||
"""
|
||||
if not _dialect_is_postgres(db):
|
||||
raise RuntimeError("sql_compare_requires_postgres")
|
||||
|
||||
def _prog(side: str, n: int, *, phase: str = "sql_count", note: str = "") -> None:
|
||||
if not on_progress:
|
||||
return
|
||||
try:
|
||||
on_progress(side, n, engine="sql", note=note, phase=phase)
|
||||
except TypeError:
|
||||
on_progress(side, n)
|
||||
|
||||
key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()]
|
||||
field_rules = list(sheet.get("field_rules") or [])
|
||||
rules = field_rule_map(field_rules)
|
||||
compare_fields = effective_compare_fields(
|
||||
list(sheet.get("compare_fields") or []),
|
||||
field_rules,
|
||||
)
|
||||
row_filters = list(sheet.get("row_filters") or [])
|
||||
mid = str(sheet.get("metric_id") or "").strip()
|
||||
bid_b = str(before_batch_id or "").strip()
|
||||
bid_a = str(after_batch_id or "").strip()
|
||||
if not mid or not bid_b or not bid_a or not key_fields:
|
||||
raise ValueError("sql_compare_missing_args")
|
||||
|
||||
filter_sql, filter_params = compile_row_filters_sql(row_filters)
|
||||
rk = _rk_sql(key_fields)
|
||||
dup_order = dup_pairing_order_sql(compare_fields, rules) if duplicate_pairing == "sorted" else ""
|
||||
tag = uuid4().hex[:8]
|
||||
tb = f"_netx_cmp_b_{tag}"
|
||||
ta = f"_netx_cmp_a_{tag}"
|
||||
|
||||
# Keep worker transaction open across CTAS — progress must NOT commit this session.
|
||||
db.execute(text(f"SET LOCAL statement_timeout = '{int(_SQL_STATEMENT_TIMEOUT_MS)}'"))
|
||||
|
||||
_prog("before", 0, phase="sql_count", note="count")
|
||||
|
||||
# Raw counts (no row_filters)
|
||||
raw_b = int(
|
||||
db.execute(
|
||||
text(
|
||||
"SELECT count(*) FROM biz_state_metric_row "
|
||||
"WHERE batch_id = :bid AND metric_id = :mid"
|
||||
),
|
||||
{"bid": bid_b, "mid": mid},
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
_prog("before", raw_b, phase="sql_count")
|
||||
raw_a = int(
|
||||
db.execute(
|
||||
text(
|
||||
"SELECT count(*) FROM biz_state_metric_row "
|
||||
"WHERE batch_id = :bid AND metric_id = :mid"
|
||||
),
|
||||
{"bid": bid_a, "mid": mid},
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
_prog("after", raw_a, phase="sql_count")
|
||||
|
||||
if max(raw_b, raw_a) < int(_SQL_MIN_ROWS):
|
||||
raise SqlCompareSkip("small_sheet")
|
||||
|
||||
base_params = {"bid": bid_b, "mid": mid, **filter_params}
|
||||
# Build TEMP sides (one transaction — no mid-flight commits on ``db``)
|
||||
_prog("before", raw_b, phase="sql_project", note="temp_before")
|
||||
for tname, batch_id, side, note in (
|
||||
(tb, bid_b, "before", "temp_before"),
|
||||
(ta, bid_a, "after", "temp_after"),
|
||||
):
|
||||
db.execute(text(f"DROP TABLE IF EXISTS {tname}"))
|
||||
params = {**base_params, "bid": batch_id}
|
||||
if side == "after":
|
||||
_prog(side, raw_a, phase="sql_project", note=note)
|
||||
db.execute(
|
||||
text(
|
||||
f"""
|
||||
CREATE TEMP TABLE {tname} ON COMMIT PRESERVE ROWS AS
|
||||
SELECT
|
||||
id,
|
||||
({rk}) AS rk,
|
||||
data_json AS data,
|
||||
row_number() OVER (
|
||||
PARTITION BY ({rk})
|
||||
ORDER BY {dup_order}seq ASC, id ASC
|
||||
) AS dup_rn
|
||||
FROM biz_state_metric_row
|
||||
WHERE batch_id = :bid
|
||||
AND metric_id = :mid
|
||||
AND ({filter_sql})
|
||||
"""
|
||||
),
|
||||
params,
|
||||
)
|
||||
db.execute(text(f"CREATE INDEX IF NOT EXISTS {tname}_rk ON {tname} (rk, dup_rn)"))
|
||||
|
||||
before_n = int(
|
||||
db.execute(text(f"SELECT count(*) FROM {tb}")).scalar() or 0
|
||||
)
|
||||
after_n = int(
|
||||
db.execute(text(f"SELECT count(*) FROM {ta}")).scalar() or 0
|
||||
)
|
||||
_prog("after", after_n, phase="sql_join", note="join")
|
||||
|
||||
changed_pred = _changed_predicate(compare_fields, rules)
|
||||
kind_expr = f"""
|
||||
CASE
|
||||
WHEN b.id IS NULL THEN 'added'
|
||||
WHEN a.id IS NULL THEN 'removed'
|
||||
WHEN {changed_pred} THEN 'changed'
|
||||
ELSE 'unchanged'
|
||||
END
|
||||
"""
|
||||
|
||||
# Ordered same-key pairing: join on (rk, dup_rn) so 5 vs 2 → 2 compared + 3 removed
|
||||
agg_rows = db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT {kind_expr} AS kind, count(*)::bigint AS n
|
||||
FROM {tb} b
|
||||
FULL OUTER JOIN {ta} a
|
||||
ON b.rk = a.rk AND b.dup_rn = a.dup_rn
|
||||
GROUP BY 1
|
||||
"""
|
||||
)
|
||||
).mappings().all()
|
||||
counts = {str(r["kind"]): int(r["n"] or 0) for r in agg_rows}
|
||||
added = counts.get("added", 0)
|
||||
removed = counts.get("removed", 0)
|
||||
changed = counts.get("changed", 0)
|
||||
unchanged = counts.get("unchanged", 0)
|
||||
|
||||
# Diagnostic: count of match keys with >1 row (not fail rows)
|
||||
multi_b = int(
|
||||
db.execute(
|
||||
text(f"SELECT count(*) FROM (SELECT rk FROM {tb} GROUP BY rk HAVING count(*) > 1) t")
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
multi_a = int(
|
||||
db.execute(
|
||||
text(f"SELECT count(*) FROM (SELECT rk FROM {ta} GROUP BY rk HAVING count(*) > 1) t")
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
|
||||
# Lazy import — avoid circular import with compare_service
|
||||
from .compare_service import resolve_unchanged_policy
|
||||
|
||||
policy = resolve_unchanged_policy(
|
||||
store_unchanged, before_n=before_n, after_n=after_n
|
||||
)
|
||||
diffs: list[dict[str, Any]] = []
|
||||
|
||||
# Fail rows (stream into Python — should be << million)
|
||||
fail_sql = text(
|
||||
f"""
|
||||
SELECT
|
||||
{kind_expr} AS kind,
|
||||
b.id AS before_row_id,
|
||||
a.id AS after_row_id,
|
||||
b.data AS before_data,
|
||||
a.data AS after_data,
|
||||
COALESCE(b.rk, a.rk) AS rk
|
||||
FROM {tb} b
|
||||
FULL OUTER JOIN {ta} a
|
||||
ON b.rk = a.rk AND b.dup_rn = a.dup_rn
|
||||
WHERE {kind_expr} IN ('added', 'removed', 'changed')
|
||||
"""
|
||||
)
|
||||
fail_n = 0
|
||||
_prog("after", after_n, phase="sql_fail_fetch", note="fetch_fails")
|
||||
for row in db.execute(fail_sql).mappings():
|
||||
kind = str(row["kind"] or "")
|
||||
before = dict(row["before_data"] or {}) if row["before_data"] is not None else None
|
||||
after = dict(row["after_data"] or {}) if row["after_data"] is not None else None
|
||||
key_src = after if kind == "added" else (before or after or {})
|
||||
item: dict[str, Any] = {
|
||||
"kind": kind,
|
||||
"key": _key_obj_from_data(key_src, key_fields),
|
||||
"before": before,
|
||||
"after": after,
|
||||
"mapped_before": before,
|
||||
"changes": {},
|
||||
"before_row_id": str(row["before_row_id"] or ""),
|
||||
"after_row_id": str(row["after_row_id"] or ""),
|
||||
}
|
||||
if kind == "changed":
|
||||
item["changes"] = _changes_from_rows(before, after, compare_fields, rules)
|
||||
diffs.append(item)
|
||||
fail_n += 1
|
||||
if fail_n == 1 or fail_n % 25_000 == 0:
|
||||
_prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails")
|
||||
if fail_n:
|
||||
_prog("fail", fail_n, phase="sql_fail_fetch", note="fetch_fails")
|
||||
|
||||
unchanged_listed = 0
|
||||
include_u = bool(policy.get("include"))
|
||||
compact = bool(policy.get("compact"))
|
||||
limit_n = policy.get("limit")
|
||||
if include_u and unchanged > 0:
|
||||
params_u: dict[str, Any] = {}
|
||||
# Stratified sample: round-robin by neighbor|direction|afi|vrf so one
|
||||
# BGP command cannot consume the whole success sample.
|
||||
stratum_expr = """
|
||||
concat_ws('|',
|
||||
coalesce(nullif(trim(both from coalesce(b.data->>'neighbor','')), ''), '_'),
|
||||
coalesce(nullif(trim(both from coalesce(b.data->>'direction','')), ''), '_'),
|
||||
coalesce(nullif(trim(both from coalesce(b.data->>'afi','')), ''), '_'),
|
||||
coalesce(nullif(trim(both from coalesce(b.data->>'vrf','')), ''), '_')
|
||||
)
|
||||
"""
|
||||
if limit_n is not None:
|
||||
params_u["lim"] = max(0, int(limit_n))
|
||||
u_sql = text(
|
||||
f"""
|
||||
WITH u AS (
|
||||
SELECT
|
||||
b.id AS before_row_id,
|
||||
a.id AS after_row_id,
|
||||
b.data AS before_data,
|
||||
a.data AS after_data,
|
||||
{stratum_expr} AS stratum
|
||||
FROM {tb} b
|
||||
INNER JOIN {ta} a
|
||||
ON b.rk = a.rk AND b.dup_rn = a.dup_rn
|
||||
WHERE NOT ({changed_pred})
|
||||
),
|
||||
ranked AS (
|
||||
SELECT *,
|
||||
row_number() OVER (
|
||||
PARTITION BY stratum ORDER BY before_row_id ASC
|
||||
) AS rn_in
|
||||
FROM u
|
||||
),
|
||||
picked AS (
|
||||
SELECT *,
|
||||
row_number() OVER (
|
||||
ORDER BY rn_in ASC, stratum ASC, before_row_id ASC
|
||||
) AS pick_ord
|
||||
FROM ranked
|
||||
)
|
||||
SELECT before_row_id, after_row_id, before_data, after_data
|
||||
FROM picked
|
||||
WHERE pick_ord <= :lim
|
||||
"""
|
||||
)
|
||||
else:
|
||||
u_sql = text(
|
||||
f"""
|
||||
SELECT
|
||||
b.id AS before_row_id,
|
||||
a.id AS after_row_id,
|
||||
b.data AS before_data,
|
||||
a.data AS after_data
|
||||
FROM {tb} b
|
||||
INNER JOIN {ta} a
|
||||
ON b.rk = a.rk AND b.dup_rn = a.dup_rn
|
||||
WHERE NOT ({changed_pred})
|
||||
"""
|
||||
)
|
||||
for row in db.execute(u_sql, params_u).mappings():
|
||||
before = dict(row["before_data"] or {})
|
||||
after = dict(row["after_data"] or {})
|
||||
unchanged_listed += 1
|
||||
if compact:
|
||||
diffs.append(
|
||||
{
|
||||
"kind": "unchanged",
|
||||
"key": _key_obj_from_data(before, key_fields),
|
||||
"before": {},
|
||||
"after": {},
|
||||
"mapped_before": {},
|
||||
"changes": {},
|
||||
"compact": True,
|
||||
"before_row_id": str(row["before_row_id"] or ""),
|
||||
"after_row_id": str(row["after_row_id"] or ""),
|
||||
}
|
||||
)
|
||||
else:
|
||||
diffs.append(
|
||||
{
|
||||
"kind": "unchanged",
|
||||
"key": _key_obj_from_data(before, key_fields),
|
||||
"before": before,
|
||||
"after": after,
|
||||
"mapped_before": before,
|
||||
"changes": {},
|
||||
"before_row_id": str(row["before_row_id"] or ""),
|
||||
"after_row_id": str(row["after_row_id"] or ""),
|
||||
}
|
||||
)
|
||||
|
||||
# Cleanup (also ON COMMIT DROP)
|
||||
db.execute(text(f"DROP TABLE IF EXISTS {tb}"))
|
||||
db.execute(text(f"DROP TABLE IF EXISTS {ta}"))
|
||||
|
||||
summary = {
|
||||
"before_count": before_n,
|
||||
"after_count": after_n,
|
||||
"added": added,
|
||||
"removed": removed,
|
||||
"changed": changed,
|
||||
"unchanged": unchanged,
|
||||
"duplicate": 0,
|
||||
"match_key_fields": list(key_fields),
|
||||
"duplicate_keys_before": multi_b,
|
||||
"duplicate_keys_after": multi_a,
|
||||
"duplicate_key_list": [],
|
||||
"unchanged_listed": unchanged_listed,
|
||||
"unchanged_truncated": bool(
|
||||
include_u and limit_n is not None and unchanged > unchanged_listed
|
||||
),
|
||||
"unchanged_compact": bool(compact and unchanged_listed > 0),
|
||||
"unchanged_sample_mode": (
|
||||
"stratified"
|
||||
if include_u and limit_n is not None and unchanged > unchanged_listed
|
||||
else ("full" if include_u else "none")
|
||||
),
|
||||
"engine": "sql",
|
||||
"before_raw_count": raw_b,
|
||||
"after_raw_count": raw_a,
|
||||
"row_filters": len(row_filters),
|
||||
"unchanged_policy": policy,
|
||||
}
|
||||
mapping_stats = {
|
||||
"before_iface_count": 0,
|
||||
"after_iface_count": 0,
|
||||
"map_pairs": 0,
|
||||
"hit_before": [],
|
||||
"miss_before": [],
|
||||
"hit_after": [],
|
||||
"miss_after": [],
|
||||
"unused_before_keys": [],
|
||||
"ok": True,
|
||||
"ignore_port_changes": False,
|
||||
"engine": "sql",
|
||||
}
|
||||
return {"summary": summary, "diffs": diffs, "mapping_stats": mapping_stats}
|
||||
|
|
@ -1,168 +0,0 @@
|
|||
"""Validate template writes without silently dropping parts of the comparison."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
FILTER_OPS = {"eq", "==", "ci_eq", "ne", "!=", "in", "not_in", "nin", "contains", "empty", "not_empty", "nonempty", "regex", "age_timer"}
|
||||
COMPARE_MODES = {"", "eq", "ignore", "skip", "off", "numeric", "number", "int", "float", "percent", "pct", "rel",
|
||||
"not_worse", "numeric_min", "numeric_max"}
|
||||
SIDES = {"", "both", "before", "after"}
|
||||
MAX_VALUE_MAP = 500
|
||||
NORMALIZE_MODES = {"", "none", "strip", "lower", "upper", "mac", "empty_as_blank"}
|
||||
|
||||
|
||||
def invalid(path: str, reason: str) -> None:
|
||||
raise HTTPException(status_code=400, detail={"error": "invalid_template", "path": path, "reason": reason})
|
||||
|
||||
|
||||
def _strings(value: Any, path: str, *, required: bool = False) -> None:
|
||||
if not isinstance(value, list) or any(not isinstance(x, str) or not x.strip() for x in value):
|
||||
invalid(path, "nonempty_strings_required")
|
||||
if required and not value:
|
||||
invalid(path, "key_fields_required")
|
||||
if len({x.strip() for x in value}) != len(value):
|
||||
invalid(path, "duplicate_field")
|
||||
|
||||
|
||||
def validate_filters(value: Any, path: str, depth: int = 0) -> None:
|
||||
if not isinstance(value, list):
|
||||
invalid(path, "list_required")
|
||||
if depth > 12:
|
||||
invalid(path, "filter_nesting_too_deep")
|
||||
for i, node in enumerate(value):
|
||||
here = f"{path}[{i}]"
|
||||
if not isinstance(node, dict) or not node:
|
||||
invalid(here, "filter_required")
|
||||
if depth == 0 and str(node.get("side") or "").strip().lower() not in SIDES:
|
||||
invalid(here, "unknown_filter_side")
|
||||
groups = [key for key in ("any", "all") if key in node]
|
||||
if groups:
|
||||
if len(groups) != 1 or any(node.get(key) is not None for key in ("field", "op", "value")):
|
||||
invalid(here, "group_or_leaf_required")
|
||||
kids = node[groups[0]]
|
||||
if not isinstance(kids, list) or not kids:
|
||||
invalid(here, "nonempty_group_required")
|
||||
validate_filters(kids, f"{here}.{groups[0]}", depth + 1)
|
||||
continue
|
||||
if not isinstance(node.get("field"), str) or not node["field"].strip():
|
||||
invalid(here, "filter_field_required")
|
||||
op = str(node.get("op") or "eq").strip().lower()
|
||||
if op not in FILTER_OPS:
|
||||
invalid(here, "unknown_filter_operator")
|
||||
expect = node.get("value")
|
||||
if op in ("in", "not_in", "nin"):
|
||||
if not isinstance(expect, list) or not expect or any(not isinstance(x, (str, int, float, bool)) for x in expect):
|
||||
invalid(here, "nonempty_value_list_required")
|
||||
elif op not in ("empty", "not_empty", "nonempty", "age_timer"):
|
||||
if not isinstance(expect, (str, int, float, bool)):
|
||||
invalid(here, "scalar_value_required")
|
||||
if op in ("contains", "regex") and not str(expect).strip():
|
||||
invalid(here, "value_required")
|
||||
if op == "regex":
|
||||
try:
|
||||
re.compile(str(expect), re.I)
|
||||
except re.error:
|
||||
invalid(here, "invalid_regex")
|
||||
|
||||
|
||||
def validate_sheet(sheet: Any, path: str, *, partial: bool = False) -> None:
|
||||
if not isinstance(sheet, dict):
|
||||
invalid(path, "sheet_required")
|
||||
if not partial or "metric_id" in sheet:
|
||||
if not isinstance(sheet.get("metric_id"), str) or not sheet["metric_id"].strip():
|
||||
invalid(path, "metric_id_required")
|
||||
for key in ("key_fields", "iface_fields", "compare_fields", "display_fields", "ignore_fields"):
|
||||
if key in sheet or (key == "key_fields" and not partial):
|
||||
_strings(sheet.get(key), f"{path}.{key}", required=key == "key_fields")
|
||||
for opt, allowed in (("unmapped_port_policy", {"by_name", "ignore_port"}),
|
||||
("duplicate_pairing", {"sorted", "ordered"})):
|
||||
if str(sheet.get(opt) or "").strip() and str(sheet[opt]).strip().lower() not in allowed:
|
||||
invalid(path, f"unknown_{opt}")
|
||||
if sheet.get("builtin_vendor_canonical") is not None and not isinstance(sheet["builtin_vendor_canonical"], bool):
|
||||
invalid(path, "boolean_builtin_vendor_canonical_required")
|
||||
if "ignore_port_changes" in sheet and sheet["ignore_port_changes"] is not None and not isinstance(sheet["ignore_port_changes"], bool):
|
||||
invalid(path, "boolean_port_policy_required")
|
||||
if "row_filters" in sheet:
|
||||
validate_filters(sheet["row_filters"], f"{path}.row_filters")
|
||||
if "field_rules" in sheet:
|
||||
rules = sheet["field_rules"]
|
||||
if not isinstance(rules, list):
|
||||
invalid(path, "field_rules_list_required")
|
||||
seen = set()
|
||||
for i, rule in enumerate(rules):
|
||||
here = f"{path}.field_rules[{i}]"
|
||||
if not isinstance(rule, dict) or not isinstance(rule.get("field"), str) or not rule["field"].strip():
|
||||
invalid(here, "rule_field_required")
|
||||
field = rule["field"].strip()
|
||||
rule_side = str(rule.get("side") or "").strip().lower()
|
||||
if rule_side not in SIDES:
|
||||
invalid(here, "unknown_rule_side")
|
||||
marker = (field, "" if rule_side == "both" else rule_side)
|
||||
if marker in seen:
|
||||
invalid(here, "duplicate_field_rule")
|
||||
seen.add(marker)
|
||||
vmap = rule.get("value_map")
|
||||
if vmap is not None:
|
||||
if not isinstance(vmap, dict) or any(
|
||||
not isinstance(k, str) or not isinstance(v, str) for k, v in vmap.items()
|
||||
):
|
||||
invalid(here, "value_map_string_pairs_required")
|
||||
if len(vmap) > MAX_VALUE_MAP:
|
||||
invalid(here, "value_map_too_large")
|
||||
if rule.get("apply_to_key") is not None and not isinstance(rule["apply_to_key"], bool):
|
||||
invalid(here, "boolean_apply_to_key_required")
|
||||
mode = str(rule.get("compare") or "").strip().lower()
|
||||
if mode == "not_worse":
|
||||
rank = rule.get("rank")
|
||||
if not isinstance(rank, list) or len(rank) < 2 or any(not isinstance(x, str) or not x.strip() for x in rank):
|
||||
invalid(here, "rank_list_required")
|
||||
elif rule.get("rank") is not None:
|
||||
invalid(here, "rank_only_for_not_worse")
|
||||
if str(rule.get("compare") or "").strip().lower() not in COMPARE_MODES:
|
||||
invalid(here, "unknown_compare_mode")
|
||||
if str(rule.get("normalize") or "").strip().lower() not in NORMALIZE_MODES:
|
||||
invalid(here, "unknown_normalize_mode")
|
||||
tol = rule.get("tolerance")
|
||||
if tol is not None:
|
||||
if isinstance(tol, bool) or not isinstance(tol, (int, float)) or not math.isfinite(tol) or tol < 0:
|
||||
invalid(here, "finite_nonnegative_tolerance_required")
|
||||
|
||||
|
||||
def validate_template_body(body: dict[str, Any], *, partial: bool = False) -> None:
|
||||
metrics = body.get("metrics")
|
||||
if metrics is not None:
|
||||
if not isinstance(metrics, list) or not metrics:
|
||||
invalid("metrics", "metrics_required")
|
||||
seen = set()
|
||||
for i, sheet in enumerate(metrics):
|
||||
path = f"metrics[{i}]"
|
||||
validate_sheet(sheet, path)
|
||||
sid = str(sheet.get("sheet_id") or sheet["metric_id"]).strip()
|
||||
if sid in seen:
|
||||
invalid(path, "duplicate_sheet_id")
|
||||
seen.add(sid)
|
||||
else:
|
||||
validate_sheet(body, "template", partial=partial)
|
||||
if "iface_normalize_rules" in body and body["iface_normalize_rules"] is not None:
|
||||
rules = body["iface_normalize_rules"]
|
||||
if not isinstance(rules, list):
|
||||
invalid("iface_normalize_rules", "list_required")
|
||||
seen = set()
|
||||
for i, rule in enumerate(rules):
|
||||
path = f"iface_normalize_rules[{i}]"
|
||||
if not isinstance(rule, dict) or any(not isinstance(rule.get(k), str) or not rule[k].strip() for k in ("from", "to")):
|
||||
invalid(path, "alias_pair_required")
|
||||
rside = str(rule.get("side") or "").strip().lower()
|
||||
if rside not in SIDES:
|
||||
invalid(path, "unknown_alias_side")
|
||||
if rule.get("vendor") is not None and not isinstance(rule["vendor"], str):
|
||||
invalid(path, "alias_vendor_string_required")
|
||||
key = (rule["from"].strip().lower(), "" if rside == "both" else rside,
|
||||
str(rule.get("vendor") or "").strip().lower())
|
||||
if key in seen:
|
||||
invalid(path, "duplicate_alias")
|
||||
seen.add(key)
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,454 +0,0 @@
|
|||
"""One identity pipeline for every compare path.
|
||||
|
||||
A row's *match key* must come out identical whether it is computed while a run
|
||||
executes, while its results are searched / hydrated / exported, while a mapping
|
||||
is validated, or while a cutover monitor evaluates the same sheet. All of those
|
||||
paths therefore build a :class:`Pipeline` and call the same methods instead of
|
||||
re-implementing "normalize → map → key".
|
||||
|
||||
Stages for one row on one side (``before`` or ``after``)::
|
||||
|
||||
raw row ──normalize──▶ iface aliases (+vendor canonical) ▶ key normalize ▶ value_map
|
||||
──filter────▶ ``row_filters`` (v2: on the normalized row; legacy: on the raw row)
|
||||
──map───────▶ port map (before side only)
|
||||
──key───────▶ ``key_tuple`` over the match key fields
|
||||
|
||||
``legacy=True`` reproduces the pre-pipeline behaviour (snapshot v1 runs): the
|
||||
template alias rules are the only normalization, keys are never normalized,
|
||||
there is no value map and the filters see the raw row.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
from .compare_rules import (
|
||||
normalize_value,
|
||||
row_matches_filter,
|
||||
scalar_text,
|
||||
)
|
||||
from .iface_normalize import (
|
||||
normalize_iface_name,
|
||||
normalize_iface_rules,
|
||||
normalize_port_map,
|
||||
resolve_mapped_iface,
|
||||
rules_for_side,
|
||||
vendor_canonical_rules,
|
||||
)
|
||||
|
||||
SIDES = ("before", "after")
|
||||
DIRECTIONAL_MODES = {"not_worse", "numeric_min", "numeric_max"}
|
||||
UNMAPPED_POLICIES = ("by_name", "ignore_port")
|
||||
DUPLICATE_PAIRING = ("sorted", "ordered")
|
||||
|
||||
|
||||
def filter_fields(filt: Mapping[str, Any] | None) -> set[str]:
|
||||
"""Every field name a (possibly nested) row filter reads."""
|
||||
out: set[str] = set()
|
||||
if not isinstance(filt, Mapping):
|
||||
return out
|
||||
for group in ("any", "all"):
|
||||
if group in filt:
|
||||
for kid in filt.get(group) or []:
|
||||
out |= filter_fields(kid)
|
||||
return out
|
||||
name = str(filt.get("field") or "").strip()
|
||||
if name:
|
||||
out.add(name)
|
||||
return out
|
||||
|
||||
|
||||
def _vendor_key(vendor: str = "", device_type: str = "") -> str:
|
||||
from ..topology_lldp import resolve_vendor_key
|
||||
|
||||
key = resolve_vendor_key(vendor, device_type)
|
||||
return "" if key == "generic" else key
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class SideSpec:
|
||||
side: str
|
||||
vendor: str = ""
|
||||
canonical: list[dict[str, str]] = field(default_factory=list)
|
||||
alias_rules: list[dict[str, str]] = field(default_factory=list)
|
||||
# field -> {lowercased source value: canonical value}
|
||||
value_maps: dict[str, dict[str, str]] = field(default_factory=dict)
|
||||
# key field -> normalize mode (only fields whose rule asks for one)
|
||||
key_norms: dict[str, str] = field(default_factory=dict)
|
||||
strip_inner_space: bool = True
|
||||
|
||||
def iface(self, name: Any) -> str:
|
||||
text = str(name or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
if self.strip_inner_space:
|
||||
text = "".join(text.split())
|
||||
if self.canonical:
|
||||
text = normalize_iface_name(text, self.canonical)
|
||||
if self.alias_rules:
|
||||
text = normalize_iface_name(text, self.alias_rules)
|
||||
return text
|
||||
|
||||
@property
|
||||
def has_iface_rules(self) -> bool:
|
||||
return bool(self.canonical or self.alias_rules)
|
||||
|
||||
def describe(self) -> dict[str, Any]:
|
||||
"""JSON-able record of the rules in force (stored in the run snapshot)."""
|
||||
return {
|
||||
"vendor": self.vendor,
|
||||
"canonical": list(self.canonical),
|
||||
"alias_rules": list(self.alias_rules),
|
||||
"value_maps": {k: dict(v) for k, v in self.value_maps.items()},
|
||||
"key_norms": dict(self.key_norms),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Pipeline:
|
||||
key_fields: list[str]
|
||||
iface_fields: list[str]
|
||||
compare_fields: list[str]
|
||||
field_rules: list[dict[str, Any]]
|
||||
row_filters: list[dict[str, Any]]
|
||||
port_map: dict[str, str]
|
||||
before: SideSpec
|
||||
after: SideSpec
|
||||
legacy: bool = False
|
||||
ignore_port_changes: bool | None = None
|
||||
unmapped_port_policy: str = "by_name"
|
||||
duplicate_pairing: str = "ordered"
|
||||
_iface_set: set[str] = field(default_factory=set, repr=False)
|
||||
_transformed: set[str] = field(default_factory=set, repr=False)
|
||||
_targets: set[str] = field(default_factory=set, repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._iface_set = set(self.iface_fields)
|
||||
self._transformed = set(self._iface_set)
|
||||
for spec in (self.before, self.after):
|
||||
self._transformed |= set(spec.value_maps) | set(spec.key_norms)
|
||||
self._targets = set(self.port_map.values())
|
||||
|
||||
# -- side helpers ----------------------------------------------------
|
||||
def spec(self, side: str) -> SideSpec:
|
||||
return self.before if side == "before" else self.after
|
||||
|
||||
# -- stage 1: normalize ----------------------------------------------
|
||||
def normalize(self, row: Mapping[str, Any], side: str) -> dict[str, Any]:
|
||||
out = dict(row)
|
||||
spec = self.spec(side)
|
||||
for f in self.iface_fields:
|
||||
val = scalar_text(out.get(f)).strip()
|
||||
if val:
|
||||
out[f] = spec.iface(val)
|
||||
if self.legacy:
|
||||
return out
|
||||
for f, mode in spec.key_norms.items():
|
||||
if f in out:
|
||||
out[f] = normalize_value(out[f], mode)
|
||||
for f, vmap in spec.value_maps.items():
|
||||
if f not in out:
|
||||
continue
|
||||
raw = scalar_text(out[f]).strip()
|
||||
hit = vmap.get(raw.lower())
|
||||
if hit is not None:
|
||||
out[f] = hit
|
||||
return out
|
||||
|
||||
def normalize_name(self, name: Any, side: str) -> str:
|
||||
"""Interface-name normalization only (port-map keys / values, selectors)."""
|
||||
return self.spec(side).iface(name)
|
||||
|
||||
# -- stage 2: filter -------------------------------------------------
|
||||
def filters_for(self, side: str) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for f in self.row_filters:
|
||||
fs = str(f.get("side") or "").strip().lower()
|
||||
if not fs or fs == "both" or fs == side:
|
||||
out.append(f)
|
||||
return out
|
||||
|
||||
def keep(self, row: Mapping[str, Any], side: str) -> bool:
|
||||
"""Evaluate row filters. v2 expects the *normalized* row, legacy the raw one."""
|
||||
return all(row_matches_filter(row, f) for f in self.filters_for(side))
|
||||
|
||||
def accepts(self, raw: Mapping[str, Any], norm: Mapping[str, Any], side: str) -> bool:
|
||||
"""Row-filter verdict: legacy reads the raw row, v2 the normalized one."""
|
||||
return self.keep(raw if self.legacy else norm, side)
|
||||
|
||||
def pushdown_filters(self, side: str) -> list[dict[str, Any]]:
|
||||
"""Filters that read only untransformed fields, so they may run in SQL on raw rows."""
|
||||
if self.legacy:
|
||||
return list(self.row_filters)
|
||||
return [
|
||||
f for f in self.row_filters
|
||||
if not str(f.get("side") or "").strip()
|
||||
and not (filter_fields(f) & self._transformed)
|
||||
]
|
||||
|
||||
def prepare(self, rows: Sequence[Mapping[str, Any]], side: str) -> list[tuple[dict[str, Any], dict[str, Any]]]:
|
||||
"""``[(original row, normalized row)]`` after filtering, in input order."""
|
||||
out: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
||||
has_filters = bool(self.filters_for(side))
|
||||
for raw in rows:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
norm = self.normalize(raw, side)
|
||||
if has_filters and not self.accepts(raw, norm, side):
|
||||
continue
|
||||
out.append((raw, norm))
|
||||
return out
|
||||
|
||||
# -- stage 3: port map -----------------------------------------------
|
||||
def apply_port_map(self, norm_row: Mapping[str, Any]) -> dict[str, Any]:
|
||||
out = dict(norm_row)
|
||||
if not self.port_map:
|
||||
return out
|
||||
for f in self.iface_fields:
|
||||
val = scalar_text(out.get(f)).strip()
|
||||
if val:
|
||||
out[f] = resolve_mapped_iface(val, self.port_map)
|
||||
return out
|
||||
|
||||
def is_mapped(self, norm_row: Mapping[str, Any], side: str) -> bool:
|
||||
"""True when every interface value of the row is covered by the port map."""
|
||||
if not self.iface_fields:
|
||||
return True
|
||||
values = [scalar_text(norm_row.get(f)).strip() for f in self.iface_fields]
|
||||
values = [v for v in values if v]
|
||||
if not values:
|
||||
return False
|
||||
for v in values:
|
||||
if side == "before":
|
||||
if resolve_mapped_iface(v, self.port_map) == v and v not in self.port_map:
|
||||
return False
|
||||
else:
|
||||
if not self._after_value_is_target(v):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _after_value_is_target(self, value: str) -> bool:
|
||||
if value in self._targets:
|
||||
return True
|
||||
parts = value.split(".")
|
||||
return any(".".join(parts[:i]) in self._targets for i in range(len(parts) - 1, 0, -1))
|
||||
|
||||
def transform(self, row: Mapping[str, Any], side: str) -> dict[str, Any]:
|
||||
norm = self.normalize(row, side)
|
||||
return self.apply_port_map(norm) if side == "before" else norm
|
||||
|
||||
# -- stage 4: key ----------------------------------------------------
|
||||
def effective_match_keys(self, *, unique_without_iface: bool | None = None) -> list[str]:
|
||||
"""Match-key fields once the port policy is resolved.
|
||||
|
||||
With a port map the full key is used (the map aligns the names). Without
|
||||
one, ``ignore_port_changes`` may drop the interface fields; ``None`` means
|
||||
"auto", resolved by the caller via ``unique_without_iface``.
|
||||
"""
|
||||
keys = list(self.key_fields)
|
||||
if self.port_map or not self._iface_set:
|
||||
return keys
|
||||
candidate = [k for k in keys if k not in self._iface_set]
|
||||
if not candidate:
|
||||
return keys
|
||||
policy = self.ignore_port_changes
|
||||
if policy is True:
|
||||
return candidate
|
||||
if policy is False:
|
||||
return keys
|
||||
return candidate if unique_without_iface else keys
|
||||
|
||||
def ignores_ports_by_policy(self) -> bool:
|
||||
return bool(self.port_map) and self.unmapped_port_policy == "ignore_port" and bool(self._iface_set)
|
||||
|
||||
def key_of(
|
||||
self, norm_row: Mapping[str, Any], side: str, match_keys: Sequence[str],
|
||||
) -> tuple[dict[str, Any], tuple[str, ...]]:
|
||||
"""``(row after port map, match key)`` for a *normalized* row."""
|
||||
row = self.apply_port_map(norm_row) if side == "before" else dict(norm_row)
|
||||
if self.ignores_ports_by_policy():
|
||||
if not self.is_mapped(norm_row, side):
|
||||
reduced = [k for k in match_keys if k not in self._iface_set] or list(match_keys)
|
||||
return row, ("U",) + tuple(scalar_text(row.get(k)).strip() for k in reduced)
|
||||
return row, ("M",) + tuple(scalar_text(row.get(k)).strip() for k in match_keys)
|
||||
return row, tuple(scalar_text(row.get(k)).strip() for k in match_keys)
|
||||
|
||||
def match_key(
|
||||
self, row: Mapping[str, Any], side: str, match_keys: Sequence[str] | None = None,
|
||||
) -> tuple[str, ...]:
|
||||
"""Match key of a *raw* row (normalize → port map → key)."""
|
||||
keys = list(match_keys) if match_keys is not None else self.effective_match_keys()
|
||||
return self.key_of(self.normalize(row, side), side, keys)[1]
|
||||
|
||||
# -- duplicate pairing -----------------------------------------------
|
||||
def pair_sort_key(self, norm_row: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
rules = {str(r.get("field")): r for r in self.field_rules if isinstance(r, dict)}
|
||||
out = []
|
||||
for f in self.compare_fields:
|
||||
mode = str((rules.get(f) or {}).get("normalize") or "strip")
|
||||
out.append(normalize_value(norm_row.get(f), mode))
|
||||
return tuple(out)
|
||||
|
||||
# -- SQL eligibility -------------------------------------------------
|
||||
def sql_blockers(self, used_fields: set[str]) -> str:
|
||||
"""Reason the SQL engine must not run this sheet ('' when it may)."""
|
||||
if self.legacy:
|
||||
return ""
|
||||
for spec in (self.before, self.after):
|
||||
if set(spec.value_maps) & used_fields:
|
||||
return "value_map"
|
||||
if set(spec.key_norms) & set(self.key_fields):
|
||||
return "key_normalize"
|
||||
if used_fields & self._iface_set and (self.before.has_iface_rules or self.after.has_iface_rules):
|
||||
return "iface_normalize"
|
||||
if any(str(f.get("side") or "").strip() for f in self.row_filters):
|
||||
return "row_filters_side"
|
||||
if any(filter_fields(f) & self._transformed for f in self.row_filters):
|
||||
return "row_filters_post_normalize"
|
||||
if any(
|
||||
str(r.get("compare") or "").strip().lower() in DIRECTIONAL_MODES
|
||||
for r in self.field_rules if isinstance(r, dict)
|
||||
):
|
||||
return "directional_compare"
|
||||
return ""
|
||||
|
||||
def describe(self) -> dict[str, Any]:
|
||||
return {
|
||||
"legacy": self.legacy,
|
||||
"before": self.before.describe(),
|
||||
"after": self.after.describe(),
|
||||
"unmapped_port_policy": self.unmapped_port_policy,
|
||||
"duplicate_pairing": self.duplicate_pairing,
|
||||
}
|
||||
|
||||
|
||||
def _field_rule_specs(
|
||||
field_rules: Sequence[Mapping[str, Any]], key_fields: Sequence[str], side: str,
|
||||
) -> tuple[dict[str, dict[str, str]], dict[str, str]]:
|
||||
value_maps: dict[str, dict[str, str]] = {}
|
||||
key_norms: dict[str, str] = {}
|
||||
key_set = set(key_fields)
|
||||
for rule in field_rules or []:
|
||||
if not isinstance(rule, Mapping):
|
||||
continue
|
||||
name = str(rule.get("field") or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
rs = str(rule.get("side") or "").strip().lower()
|
||||
if rs and rs != "both" and rs != side:
|
||||
continue
|
||||
apply_to_key = rule.get("apply_to_key")
|
||||
apply_to_key = (name in key_set) if apply_to_key is None else bool(apply_to_key)
|
||||
is_key = name in key_set
|
||||
if is_key and not apply_to_key:
|
||||
continue
|
||||
mode = str(rule.get("normalize") or "").strip().lower()
|
||||
vmap = rule.get("value_map")
|
||||
if is_key and mode and mode not in ("none", "strip"):
|
||||
key_norms[name] = mode
|
||||
if isinstance(vmap, Mapping) and vmap:
|
||||
pre = mode if (mode and mode not in ("none", "strip")) else "strip"
|
||||
built: dict[str, str] = {}
|
||||
for src, dst in vmap.items():
|
||||
built[normalize_value(src, pre).lower()] = scalar_text(dst).strip()
|
||||
value_maps[name] = built
|
||||
return value_maps, key_norms
|
||||
|
||||
|
||||
def build_pipeline(
|
||||
sheet: Mapping[str, Any],
|
||||
*,
|
||||
iface_rules: Sequence[Mapping[str, Any]] | None,
|
||||
port_map: Mapping[str, str] | None,
|
||||
before_vendor: str = "",
|
||||
after_vendor: str = "",
|
||||
legacy: bool = False,
|
||||
normalize_map: bool = True,
|
||||
) -> Pipeline:
|
||||
"""Build the pipeline for one sheet.
|
||||
|
||||
``port_map`` is the map as the user wrote it. Unless ``normalize_map`` is
|
||||
false (legacy runs, whose maps were written post-normalization) its keys are
|
||||
normalized with the *before* side rules and its values with the *after* side
|
||||
rules.
|
||||
"""
|
||||
key_fields = [str(k).strip() for k in (sheet.get("key_fields") or []) if str(k).strip()]
|
||||
iface_fields = [str(k).strip() for k in (sheet.get("iface_fields") or []) if str(k).strip()]
|
||||
field_rules = [dict(r) for r in (sheet.get("field_rules") or []) if isinstance(r, Mapping)]
|
||||
from .compare_rules import effective_compare_fields
|
||||
|
||||
compare_fields = effective_compare_fields(list(sheet.get("compare_fields") or []), field_rules)
|
||||
canonical_on = sheet.get("builtin_vendor_canonical", True) is not False and not legacy
|
||||
all_rules = normalize_iface_rules(list(iface_rules or []))
|
||||
|
||||
specs: dict[str, SideSpec] = {}
|
||||
for side, vendor_raw in (("before", before_vendor), ("after", after_vendor)):
|
||||
vendor = _vendor_key(vendor_raw) if vendor_raw else ""
|
||||
if legacy:
|
||||
alias = normalize_iface_rules([{"from": r["from"], "to": r["to"]} for r in all_rules])
|
||||
value_maps, key_norms = {}, {}
|
||||
else:
|
||||
alias = rules_for_side(all_rules, side, vendor)
|
||||
value_maps, key_norms = _field_rule_specs(field_rules, key_fields, side)
|
||||
specs[side] = SideSpec(
|
||||
side=side, vendor=vendor,
|
||||
canonical=vendor_canonical_rules(vendor) if canonical_on else [],
|
||||
alias_rules=alias, value_maps=value_maps, key_norms=key_norms,
|
||||
strip_inner_space=not legacy,
|
||||
)
|
||||
|
||||
pmap = dict(port_map or {})
|
||||
if pmap and normalize_map and not legacy:
|
||||
pmap, _ = normalize_port_map(
|
||||
pmap, None,
|
||||
key_normalizer=specs["before"].iface, value_normalizer=specs["after"].iface,
|
||||
)
|
||||
ignore_ports = sheet.get("ignore_port_changes")
|
||||
policy = str(sheet.get("unmapped_port_policy") or "by_name").strip().lower()
|
||||
pairing = str(sheet.get("duplicate_pairing") or ("ordered" if legacy else "sorted")).strip().lower()
|
||||
return Pipeline(
|
||||
key_fields=key_fields,
|
||||
iface_fields=iface_fields,
|
||||
compare_fields=compare_fields,
|
||||
field_rules=field_rules,
|
||||
row_filters=[dict(f) for f in (sheet.get("row_filters") or []) if isinstance(f, Mapping)],
|
||||
port_map=pmap,
|
||||
before=specs["before"],
|
||||
after=specs["after"],
|
||||
legacy=legacy,
|
||||
ignore_port_changes=None if ignore_ports is None else bool(ignore_ports),
|
||||
unmapped_port_policy=policy if policy in UNMAPPED_POLICIES else "by_name",
|
||||
duplicate_pairing="ordered" if legacy else (pairing if pairing in DUPLICATE_PAIRING else "sorted"),
|
||||
)
|
||||
|
||||
|
||||
def legacy_pipeline(
|
||||
*,
|
||||
key_fields: Sequence[str],
|
||||
iface_fields: Sequence[str],
|
||||
compare_fields: Sequence[str],
|
||||
field_rules: Sequence[Mapping[str, Any]] | None = None,
|
||||
port_map: Mapping[str, str] | None = None,
|
||||
iface_normalize_rules: Sequence[Mapping[str, Any]] | None = None,
|
||||
ignore_port_changes: bool | None = None,
|
||||
row_filters: Sequence[Mapping[str, Any]] | None = None,
|
||||
duplicate_pairing: str = "ordered",
|
||||
) -> Pipeline:
|
||||
"""Pipeline with the pre-v2 semantics, for snapshot-v1 runs and bare callers."""
|
||||
alias = normalize_iface_rules(
|
||||
[{"from": r.get("from"), "to": r.get("to")} for r in (iface_normalize_rules or [])]
|
||||
)
|
||||
return Pipeline(
|
||||
key_fields=[str(k) for k in key_fields],
|
||||
iface_fields=[str(f) for f in iface_fields if str(f).strip()],
|
||||
compare_fields=[str(f) for f in compare_fields],
|
||||
field_rules=[dict(r) for r in (field_rules or []) if isinstance(r, Mapping)],
|
||||
row_filters=[dict(f) for f in (row_filters or []) if isinstance(f, Mapping)],
|
||||
port_map=dict(port_map or {}),
|
||||
before=SideSpec(side="before", alias_rules=list(alias), strip_inner_space=False),
|
||||
after=SideSpec(side="after", alias_rules=list(alias), strip_inner_space=False),
|
||||
legacy=True,
|
||||
ignore_port_changes=ignore_port_changes,
|
||||
duplicate_pairing=duplicate_pairing if duplicate_pairing in DUPLICATE_PAIRING else "ordered",
|
||||
)
|
||||
|
|
@ -39,14 +39,7 @@ def normalize_iface_rules(raw: Any) -> list[dict[str, str]]:
|
|||
fr = str(item.get("from") or item.get("src") or "").strip()
|
||||
to = str(item.get("to") or item.get("dst") or "").strip()
|
||||
if fr and to:
|
||||
rule = {"from": fr, "to": to}
|
||||
side = str(item.get("side") or "").strip().lower()
|
||||
if side in ("before", "after"):
|
||||
rule["side"] = side
|
||||
vendor = str(item.get("vendor") or "").strip().lower()
|
||||
if vendor:
|
||||
rule["vendor"] = vendor
|
||||
out.append(rule)
|
||||
out.append({"from": fr, "to": to})
|
||||
# Longest prefix first so XXVGE wins over XGE / GE.
|
||||
out.sort(key=lambda r: len(r["from"]), reverse=True)
|
||||
return out
|
||||
|
|
@ -114,47 +107,6 @@ def apply_iface_normalize_rows(
|
|||
]
|
||||
|
||||
|
||||
def normalize_port_map(
|
||||
port_map: Mapping[str, str] | None,
|
||||
rules: Sequence[Mapping[str, str]] | None,
|
||||
value_rules: Sequence[Mapping[str, str]] | None = None,
|
||||
*,
|
||||
key_normalizer: Any = None,
|
||||
value_normalizer: Any = None,
|
||||
) -> tuple[dict[str, str], list[dict[str, str]]]:
|
||||
"""Normalize port-map keys and values with the same rules as iface fields.
|
||||
|
||||
``value_rules`` (default: ``rules``) normalizes the after-side values, for
|
||||
per-side rules. ``key_normalizer`` / ``value_normalizer`` are callables that
|
||||
replace the rule-based normalization (used by the identity pipeline).
|
||||
|
||||
Returns ``(normalized_map, conflicts)``. A conflict is two raw keys that
|
||||
normalize to the same key but point at different (normalized) targets; the
|
||||
first one wins in ``normalized_map``.
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
origin: dict[str, str] = {}
|
||||
conflicts: list[dict[str, str]] = []
|
||||
for raw_k, raw_v in (port_map or {}).items():
|
||||
k = key_normalizer(raw_k) if key_normalizer else normalize_iface_name(raw_k, rules)
|
||||
v = (
|
||||
value_normalizer(raw_v) if value_normalizer
|
||||
else normalize_iface_name(raw_v, rules if value_rules is None else value_rules)
|
||||
)
|
||||
if not k or not v:
|
||||
continue
|
||||
if k in out:
|
||||
if out[k] != v:
|
||||
conflicts.append({
|
||||
"before_if": k, "kept_from": origin[k], "kept_to": out[k],
|
||||
"dropped_from": str(raw_k), "dropped_to": v,
|
||||
})
|
||||
continue
|
||||
out[k] = v
|
||||
origin[k] = str(raw_k)
|
||||
return out, conflicts
|
||||
|
||||
|
||||
def resolve_mapped_iface(name: str, port_map: Mapping[str, str] | None) -> str:
|
||||
"""Exact map hit, else longest mapped parent + remaining suffix (QinQ-safe).
|
||||
|
||||
|
|
@ -190,53 +142,3 @@ def default_zte_iface_normalize_rules() -> list[dict[str, str]]:
|
|||
{"from": "SG", "to": "smartgroup"},
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in per-vendor canonical names (same-vendor synonyms only; cross-vendor
|
||||
# numbering is the port map's job). Applied before template alias rules.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
VENDOR_CANONICAL: dict[str, list[dict[str, str]]] = {
|
||||
"huawei": [
|
||||
{"from": "XGigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"},
|
||||
{"from": "100GE", "to": "100GE"}, {"from": "40GE", "to": "40GE"}, {"from": "25GE", "to": "25GE"},
|
||||
{"from": "10GE", "to": "10GE"}, {"from": "Eth-Trunk", "to": "Eth-Trunk"},
|
||||
],
|
||||
"cisco": [
|
||||
{"from": "GigabitEthernet", "to": "Gi"}, {"from": "TenGigabitEthernet", "to": "Te"},
|
||||
{"from": "TenGigE", "to": "Te"}, {"from": "HundredGigE", "to": "Hu"}, {"from": "FortyGigE", "to": "Fo"},
|
||||
{"from": "TwentyFiveGigE", "to": "Twe"}, {"from": "FastEthernet", "to": "Fa"},
|
||||
{"from": "Bundle-Ether", "to": "BE"}, {"from": "Loopback", "to": "Lo"},
|
||||
],
|
||||
"h3c": [
|
||||
{"from": "Ten-GigabitEthernet", "to": "XGE"}, {"from": "GigabitEthernet", "to": "GE"},
|
||||
{"from": "FortyGigE", "to": "FGE"}, {"from": "HundredGigE", "to": "HGE"},
|
||||
{"from": "Bridge-Aggregation", "to": "BAGG"},
|
||||
],
|
||||
"zte": [
|
||||
{"from": "xxvgei", "to": "xxvgei"}, {"from": "xgei", "to": "xgei"}, {"from": "cgei", "to": "cgei"},
|
||||
{"from": "gei", "to": "gei"}, {"from": "smartgroup", "to": "smartgroup"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def vendor_canonical_rules(vendor: str) -> list[dict[str, str]]:
|
||||
return normalize_iface_rules(VENDOR_CANONICAL.get(str(vendor or "").strip().lower(), []))
|
||||
|
||||
|
||||
def rules_for_side(
|
||||
rules: Sequence[Mapping[str, str]] | None, side: str, vendor: str = "",
|
||||
) -> list[dict[str, str]]:
|
||||
"""Template alias rules that apply to ``side`` (``before``/``after``) and ``vendor``."""
|
||||
vendor_n = str(vendor or "").strip().lower()
|
||||
out: list[dict[str, str]] = []
|
||||
for r in rules or []:
|
||||
rs = str(r.get("side") or "").strip().lower()
|
||||
if rs and rs != side:
|
||||
continue
|
||||
rv = str(r.get("vendor") or "").strip().lower()
|
||||
if rv and rv != vendor_n:
|
||||
continue
|
||||
out.append({"from": str(r.get("from") or ""), "to": str(r.get("to") or "")})
|
||||
return normalize_iface_rules(out)
|
||||
|
|
|
|||
|
|
@ -85,9 +85,6 @@ class ParseProfile:
|
|||
enrich_joins: list[EnrichJoin] = field(default_factory=list)
|
||||
# light: default shared SSH lane; heavy: dedicated long-timeout connection
|
||||
collect_lane: str = "light"
|
||||
# Which slice of the metric this command produces, e.g. {"afi": "ipv4"}.
|
||||
# Empty = the command covers the whole metric (or its slicing is unknown).
|
||||
slice: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
_LLDP_FIELDS: list[FieldDef] = [
|
||||
|
|
@ -1654,11 +1651,6 @@ _HEAVY_LANE_METRICS = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def metric_lane(metric_id: str) -> str:
|
||||
"""``heavy`` for huge CLI dumps (own SSH connection), else ``light``."""
|
||||
return "heavy" if str(metric_id or "").strip() in _HEAVY_LANE_METRICS else "light"
|
||||
|
||||
|
||||
def _apply_collect_lanes(profiles: list[ParseProfile]) -> list[ParseProfile]:
|
||||
for p in profiles:
|
||||
if str(p.metric_id or "").strip() in _HEAVY_LANE_METRICS:
|
||||
|
|
@ -1760,41 +1752,12 @@ def _validate_profiles(profiles: list[ParseProfile]) -> None:
|
|||
)
|
||||
|
||||
|
||||
# profile_id -> slice. Compare sheets filter on these fields (``afi``/``af``/``direction``),
|
||||
# so a missing command is detected as "slice not collected" instead of "business lost".
|
||||
_PROFILE_SLICES: dict[str, dict[str, str]] = {
|
||||
**{f"zte.bgp_{afi}_summary": {"afi": afi} for afi in ("vpnv4", "ipv4", "vpnv6", "ipv6", "evpn", "vpls")},
|
||||
"zte.bgp_vpnv4_vrf_summary": {"afi": "vpnv4"},
|
||||
"zte.bgp_vpnv6_vrf_summary": {"afi": "vpnv6"},
|
||||
"zte.vrrp_ipv4": {"af": "ipv4"},
|
||||
"zte.vrrp_ipv6": {"af": "ipv6"},
|
||||
**{
|
||||
f"zte.bgp_{afi}_neighbor_{direction}": {"afi": afi, "direction": direction}
|
||||
for afi in ("vpnv4", "vpnv6", "ipv4", "ipv6")
|
||||
for direction in ("in", "out")
|
||||
},
|
||||
**{
|
||||
f"zte.bgp_{afi}_vrf_neighbor_{direction}": {"afi": afi, "direction": direction}
|
||||
for afi in ("vpnv4", "vpnv6")
|
||||
for direction in ("in", "out")
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _apply_profile_slices(profiles: list[ParseProfile]) -> list[ParseProfile]:
|
||||
for p in profiles:
|
||||
declared = _PROFILE_SLICES.get(p.profile_id)
|
||||
if declared and not p.slice:
|
||||
p.slice = dict(declared)
|
||||
return profiles
|
||||
|
||||
|
||||
def _finalize_profiles(profiles: list[ParseProfile]) -> list[ParseProfile]:
|
||||
"""Normalize discover→aux, validate, apply collect lanes."""
|
||||
for p in profiles:
|
||||
_normalize_discover_aux(p)
|
||||
_validate_profiles(profiles)
|
||||
return _apply_profile_slices(_apply_collect_lanes(profiles))
|
||||
return _apply_collect_lanes(profiles)
|
||||
|
||||
|
||||
def all_profiles() -> list[ParseProfile]:
|
||||
|
|
|
|||
|
|
@ -6,17 +6,12 @@ from collections import defaultdict
|
|||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import (
|
||||
BizCompareDiff,
|
||||
BizCompareJob,
|
||||
BizCompareRun,
|
||||
BizMigrationBatch,
|
||||
BizMigrationDiff,
|
||||
BizMigrationProject,
|
||||
BizMigrationRedTicket,
|
||||
BizMigrationRun,
|
||||
BizStateBatch,
|
||||
BizStateBatchCommand,
|
||||
|
|
@ -31,158 +26,48 @@ def _utcnow() -> datetime:
|
|||
return datetime.utcnow()
|
||||
|
||||
|
||||
def protected_batch_map(
|
||||
db: Session, *, task_id: str = "", batch_ids: list[str] | None = None,
|
||||
) -> dict[str, list[str]]:
|
||||
"""Only read reference columns, scoped to the requested task/batches when given."""
|
||||
def protected_batch_map(db: Session, *, task_id: str = "") -> dict[str, list[str]]:
|
||||
"""batch_id → reason codes. Empty task_id = all tasks."""
|
||||
out: dict[str, list[str]] = defaultdict(list)
|
||||
wanted = set(batch_ids) if batch_ids is not None else None
|
||||
if wanted == set():
|
||||
return {}
|
||||
scope = select(BizStateBatch.id)
|
||||
q = db.query(BizStateBatch.id, BizStateBatch.is_baseline, BizStateBatch.status)
|
||||
if task_id:
|
||||
scope = scope.where(BizStateBatch.task_id == task_id)
|
||||
q = q.filter(BizStateBatch.task_id == task_id)
|
||||
if wanted is not None:
|
||||
scope = scope.where(BizStateBatch.id.in_(wanted))
|
||||
q = q.filter(BizStateBatch.id.in_(wanted))
|
||||
batch_rows = q.all() if task_id or wanted is not None else q.filter(or_(
|
||||
BizStateBatch.is_baseline.is_(True), BizStateBatch.status.in_(("queued", "running")),
|
||||
)).all()
|
||||
if task_id:
|
||||
wanted = {row.id for row in batch_rows}
|
||||
|
||||
def add(bid: str, reason: str) -> None:
|
||||
b = str(bid or "").strip()
|
||||
if not b or (wanted is not None and b not in wanted):
|
||||
if not b:
|
||||
return
|
||||
if reason not in out[b]:
|
||||
out[b].append(reason)
|
||||
|
||||
for b in batch_rows:
|
||||
if b.is_baseline:
|
||||
q = db.query(BizStateBatch)
|
||||
if task_id:
|
||||
q = q.filter(BizStateBatch.task_id == task_id)
|
||||
for b in q.filter(BizStateBatch.is_baseline.is_(True)).all():
|
||||
add(b.id, "manual_baseline")
|
||||
if b.status in ("queued", "running"):
|
||||
add(b.id, "active_collection")
|
||||
|
||||
references = (
|
||||
(BizCompareJob.before_batch_id, BizCompareJob.after_batch_id, "compare_job_before", "compare_job_after"),
|
||||
(BizCompareRun.before_batch_id, BizCompareRun.after_batch_id, "compare_run_before", "compare_run_after"),
|
||||
(BizMigrationProject.old_baseline_batch_id, BizMigrationProject.new_baseline_batch_id, "migration_old_baseline", "migration_new_baseline"),
|
||||
(BizMigrationRun.old_batch_id, BizMigrationRun.new_batch_id, "migration_run_old", "migration_run_new"),
|
||||
)
|
||||
for before, after, before_reason, after_reason in references:
|
||||
refs = db.query(before, after)
|
||||
if task_id or batch_ids is not None:
|
||||
refs = refs.filter(or_(before.in_(scope), after.in_(scope)))
|
||||
for before_id, after_id in refs:
|
||||
add(before_id, before_reason)
|
||||
add(after_id, after_reason)
|
||||
for j in db.query(BizCompareJob).all():
|
||||
add(j.before_batch_id, "compare_job_before")
|
||||
add(j.after_batch_id, "compare_job_after")
|
||||
|
||||
# Every batch a *kept* acceptance/manual run (or the newest auto run of a wave)
|
||||
# compared, not only the first binding. Older auto runs are bounded by
|
||||
# ``purge_auto_runs`` and keep protecting just their primary pair above.
|
||||
for run_batches in _kept_run_metric_batches(db):
|
||||
for bid in run_batches:
|
||||
add(bid, "migration_run_metric")
|
||||
for r in db.query(BizCompareRun).all():
|
||||
add(r.before_batch_id, "compare_run_before")
|
||||
add(r.after_batch_id, "compare_run_after")
|
||||
|
||||
for p in db.query(BizMigrationProject).all():
|
||||
add(p.old_baseline_batch_id, "migration_old_baseline")
|
||||
add(p.new_baseline_batch_id, "migration_new_baseline")
|
||||
|
||||
for r in db.query(BizMigrationRun).all():
|
||||
add(r.old_batch_id, "migration_run_old")
|
||||
add(r.new_batch_id, "migration_run_new")
|
||||
|
||||
return dict(out)
|
||||
|
||||
|
||||
def _kept_run_metric_batches(db: Session, *, limit: int = 500) -> list[set[str]]:
|
||||
newest_auto = (
|
||||
db.query(func.max(BizMigrationRun.created_at).label("at"), BizMigrationRun.batch_id.label("bid"))
|
||||
.filter(BizMigrationRun.purpose == "auto").group_by(BizMigrationRun.batch_id).subquery()
|
||||
)
|
||||
latest_auto_ids = [
|
||||
r[0] for r in db.query(BizMigrationRun.id).join(
|
||||
newest_auto,
|
||||
(BizMigrationRun.batch_id == newest_auto.c.bid) & (BizMigrationRun.created_at == newest_auto.c.at),
|
||||
).limit(limit).all()
|
||||
]
|
||||
kept_ids = [
|
||||
r[0] for r in db.query(BizMigrationRun.id).filter(BizMigrationRun.purpose != "auto")
|
||||
.order_by(BizMigrationRun.created_at.desc()).limit(limit).all()
|
||||
] + latest_auto_ids
|
||||
out: list[set[str]] = []
|
||||
for start in range(0, len(kept_ids), 100):
|
||||
for (summary,) in db.query(BizMigrationRun.summary_json).filter(
|
||||
BizMigrationRun.id.in_(kept_ids[start:start + 100])
|
||||
):
|
||||
batches: set[str] = set()
|
||||
for item in ((summary or {}).get("metric_batches") or {}).values():
|
||||
if isinstance(item, dict):
|
||||
batches.update(str(item.get(k) or "") for k in ("old_batch_id", "new_batch_id"))
|
||||
batches.discard("")
|
||||
if batches:
|
||||
out.append(batches)
|
||||
return out
|
||||
|
||||
|
||||
def purge_auto_runs(db: Session, *, migration_keep: int = 300, compare_keep: int = 50) -> dict[str, int]:
|
||||
"""Bound auto-generated run history.
|
||||
|
||||
Cutover monitoring writes a run (+ one diff row per object) every minute and
|
||||
auto-compare writes one per new sample. Acceptance / manual runs and runs a
|
||||
red ticket points at are kept forever; only the newest ``keep`` auto runs of
|
||||
each wave / job survive.
|
||||
"""
|
||||
dropped = {"migration_runs": 0, "compare_runs": 0}
|
||||
ticket_runs = {r[0] for r in db.query(BizMigrationRedTicket.run_id).filter(
|
||||
BizMigrationRedTicket.run_id != "").distinct()}
|
||||
accepted = {r[0] for r in db.query(BizMigrationBatch.accept_run_id).filter(
|
||||
BizMigrationBatch.accept_run_id != "")}
|
||||
keep_forever = ticket_runs | accepted
|
||||
|
||||
def surplus(model, group_col, extra_filter, keep: int) -> list[str]:
|
||||
over = [g for g, n in db.query(group_col, func.count(model.id)).filter(*extra_filter)
|
||||
.group_by(group_col).all() if n > keep]
|
||||
ids: list[str] = []
|
||||
for g in over:
|
||||
rows = db.query(model.id).filter(group_col == g, *extra_filter).order_by(
|
||||
model.created_at.desc(), model.id.desc()).offset(keep).all()
|
||||
ids.extend(r[0] for r in rows if r[0] not in keep_forever)
|
||||
return ids
|
||||
|
||||
mig_ids = surplus(BizMigrationRun, BizMigrationRun.batch_id, [BizMigrationRun.purpose == "auto"],
|
||||
max(1, int(migration_keep)))
|
||||
for start in range(0, len(mig_ids), 500):
|
||||
chunk = mig_ids[start:start + 500]
|
||||
db.query(BizMigrationDiff).filter(BizMigrationDiff.run_id.in_(chunk)).delete(synchronize_session=False)
|
||||
db.query(BizMigrationRun).filter(BizMigrationRun.id.in_(chunk)).delete(synchronize_session=False)
|
||||
dropped["migration_runs"] = len(mig_ids)
|
||||
|
||||
# Auto-mode jobs only. A run's origin lives in summary_json["purpose"]; unmarked runs
|
||||
# are manual and stay. Only the surplus beyond ``compare_keep`` is inspected.
|
||||
cmp_ids: list[str] = []
|
||||
keep_n = max(1, int(compare_keep))
|
||||
for (job_id,) in db.query(BizCompareJob.id).filter(BizCompareJob.mode == "auto").all():
|
||||
surplus_ids = [r[0] for r in db.query(BizCompareRun.id).filter(
|
||||
BizCompareRun.job_id == job_id, BizCompareRun.status.notin_(("queued", "running")),
|
||||
).order_by(BizCompareRun.created_at.desc(), BizCompareRun.id.desc()).offset(keep_n).all()]
|
||||
for start in range(0, len(surplus_ids), 200):
|
||||
for rid, summary in db.query(BizCompareRun.id, BizCompareRun.summary_json).filter(
|
||||
BizCompareRun.id.in_(surplus_ids[start:start + 200])
|
||||
):
|
||||
if (summary or {}).get("purpose") == "auto":
|
||||
cmp_ids.append(rid)
|
||||
for start in range(0, len(cmp_ids), 500):
|
||||
chunk = cmp_ids[start:start + 500]
|
||||
db.query(BizCompareDiff).filter(BizCompareDiff.run_id.in_(chunk)).delete(synchronize_session=False)
|
||||
db.query(BizCompareRun).filter(BizCompareRun.id.in_(chunk)).delete(synchronize_session=False)
|
||||
dropped["compare_runs"] = len(cmp_ids)
|
||||
if mig_ids or cmp_ids:
|
||||
db.commit()
|
||||
return dropped
|
||||
|
||||
|
||||
def protected_batch_ids(db: Session, *, task_id: str = "") -> set[str]:
|
||||
return set(protected_batch_map(db, task_id=task_id).keys())
|
||||
|
||||
|
||||
def batch_protect_info(db: Session, batch_id: str) -> dict[str, Any]:
|
||||
reasons = protected_batch_map(db, batch_ids=[batch_id]).get(batch_id, [])
|
||||
reasons = protected_batch_map(db).get(batch_id, [])
|
||||
b = db.get(BizStateBatch, batch_id)
|
||||
if b and bool(getattr(b, "is_baseline", False)) and "manual_baseline" not in reasons:
|
||||
reasons = ["manual_baseline", *reasons]
|
||||
|
|
|
|||
|
|
@ -23,7 +23,6 @@ def apply_biz_state_schema(conn: Connection) -> None:
|
|||
"CREATE INDEX IF NOT EXISTS ix_biz_state_batch_command_batch_id ON biz_state_batch_command (batch_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_state_lldp_neighbor_batch_id ON biz_state_lldp_neighbor (batch_id)",
|
||||
"ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS enabled_sheet_ids JSON DEFAULT '[]'",
|
||||
"ALTER TABLE biz_compare_job ADD COLUMN IF NOT EXISTS store_unchanged VARCHAR(16) DEFAULT 'auto'",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_job_status ON biz_compare_job (status)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_run_job_id ON biz_compare_run (job_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_state_vrf_route_batch_id ON biz_state_vrf_route_summary (batch_id)",
|
||||
|
|
@ -32,8 +31,6 @@ def apply_biz_state_schema(conn: Connection) -> None:
|
|||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_id ON biz_compare_diff (run_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_kind ON biz_compare_diff (run_id, metric_id, kind)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_compare_diff_run_metric_seq ON biz_compare_diff (run_id, metric_id, seq)",
|
||||
"ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS before_row_id VARCHAR(64) DEFAULT ''",
|
||||
"ALTER TABLE biz_compare_diff ADD COLUMN IF NOT EXISTS after_row_id VARCHAR(64) DEFAULT ''",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_migration_project_status ON biz_migration_project (status)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_migration_batch_project_id ON biz_migration_batch (project_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_migration_run_batch_id ON biz_migration_run (batch_id)",
|
||||
|
|
@ -82,15 +79,6 @@ def apply_biz_state_schema(conn: Connection) -> None:
|
|||
"ALTER TABLE biz_migration_project ADD COLUMN IF NOT EXISTS new_hf_bindings_json JSON DEFAULT '[]'",
|
||||
"ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS purpose VARCHAR(32) DEFAULT ''",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_state_task_purpose ON biz_state_task (purpose)",
|
||||
"ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS collect_group_id VARCHAR(128) DEFAULT ''",
|
||||
"ALTER TABLE biz_state_task ADD COLUMN IF NOT EXISTS collect_manual_only BOOLEAN DEFAULT FALSE",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_state_task_collect_group_id ON biz_state_task (collect_group_id)",
|
||||
"ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_group_id VARCHAR(128) DEFAULT ''",
|
||||
"ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_round_id VARCHAR(64) DEFAULT ''",
|
||||
"ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS collect_priority INTEGER DEFAULT 0",
|
||||
"ALTER TABLE biz_state_batch ADD COLUMN IF NOT EXISTS queued_at TIMESTAMP",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_state_batch_collect_group_id ON biz_state_batch (collect_group_id)",
|
||||
"CREATE INDEX IF NOT EXISTS ix_biz_state_batch_collect_round_id ON biz_state_batch (collect_round_id)",
|
||||
"ALTER TABLE biz_state_batch_command ADD COLUMN IF NOT EXISTS raw_line_count INTEGER DEFAULT 0",
|
||||
"ALTER TABLE biz_state_batch_command ADD COLUMN IF NOT EXISTS declared_total INTEGER DEFAULT 0",
|
||||
"ALTER TABLE biz_migration_red_ticket ADD COLUMN IF NOT EXISTS match_key_str VARCHAR(256) DEFAULT ''",
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ from typing import Any
|
|||
from uuid import uuid4
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import String, case, cast, func, or_
|
||||
from sqlalchemy.orm import Session, defer
|
||||
from sqlalchemy import String, cast, func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..lldp_shared import resolve_vendor_key
|
||||
from ..models import (
|
||||
|
|
@ -354,15 +354,13 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]:
|
|||
.order_by(BizStateTaskItem.sort_order.asc())
|
||||
.all()
|
||||
)
|
||||
bindings_by_item: dict[str, list[Any]] = {}
|
||||
if items:
|
||||
for binding in db.query(BizStateTaskItemBinding).filter(
|
||||
BizStateTaskItemBinding.item_id.in_([it.id for it in items])
|
||||
).all():
|
||||
bindings_by_item.setdefault(binding.item_id, []).append(binding)
|
||||
item_out = []
|
||||
for it in items:
|
||||
binds = bindings_by_item.get(it.id, [])
|
||||
binds = (
|
||||
db.query(BizStateTaskItemBinding)
|
||||
.filter(BizStateTaskItemBinding.item_id == it.id)
|
||||
.all()
|
||||
)
|
||||
item_out.append(
|
||||
{
|
||||
"id": it.id,
|
||||
|
|
@ -375,11 +373,6 @@ def get_task(db: Session, task_id: str) -> dict[str, Any]:
|
|||
"bindings": [{"placeholder": b.placeholder, "value": b.value} for b in binds],
|
||||
}
|
||||
)
|
||||
return {**_task_summary(task), "items": item_out}
|
||||
|
||||
|
||||
def _task_summary(task: BizStateTask) -> dict[str, Any]:
|
||||
"""Task metadata without command items or bindings (safe for frequent polling)."""
|
||||
return {
|
||||
"id": task.id,
|
||||
"source": task.source,
|
||||
|
|
@ -396,8 +389,6 @@ def _task_summary(task: BizStateTask) -> dict[str, Any]:
|
|||
"daily_keep_enabled": bool(getattr(task, "daily_keep_enabled", False)),
|
||||
"daily_keep_count": int(getattr(task, "daily_keep_count", None) or 10),
|
||||
"collect_running": bool(task.collect_running),
|
||||
"collect_group_id": task.collect_group_id or "",
|
||||
"collect_manual_only": bool(task.collect_manual_only),
|
||||
"last_collect_started_at": task.last_collect_started_at.isoformat() + "Z"
|
||||
if task.last_collect_started_at
|
||||
else None,
|
||||
|
|
@ -405,16 +396,10 @@ def _task_summary(task: BizStateTask) -> dict[str, Any]:
|
|||
if task.last_collect_ended_at
|
||||
else None,
|
||||
"last_error": task.last_error,
|
||||
"items": item_out,
|
||||
}
|
||||
|
||||
|
||||
def get_task_progress(db: Session, task_id: str) -> dict[str, Any]:
|
||||
task = db.get(BizStateTask, task_id)
|
||||
if not task:
|
||||
raise HTTPException(status_code=404, detail="task_not_found")
|
||||
return _task_summary(task)
|
||||
|
||||
|
||||
def list_tasks(db: Session, *, purpose: str | None = None) -> list[dict[str, Any]]:
|
||||
from sqlalchemy import or_
|
||||
|
||||
|
|
@ -446,8 +431,6 @@ def list_tasks(db: Session, *, purpose: str | None = None) -> list[dict[str, Any
|
|||
"status": t.status,
|
||||
"interval_sec": t.interval_sec,
|
||||
"collect_running": bool(t.collect_running),
|
||||
"collect_group_id": t.collect_group_id or "",
|
||||
"collect_manual_only": bool(t.collect_manual_only),
|
||||
"last_error": t.last_error,
|
||||
"last_collect_started_at": t.last_collect_started_at.isoformat() + "Z"
|
||||
if t.last_collect_started_at
|
||||
|
|
@ -464,8 +447,6 @@ def delete_task(db: Session, task_id: str) -> None:
|
|||
task = db.get(BizStateTask, task_id)
|
||||
if not task:
|
||||
raise HTTPException(status_code=404, detail="task_not_found")
|
||||
if task.collect_running:
|
||||
raise HTTPException(status_code=409, detail="task_collecting")
|
||||
batches = db.query(BizStateBatch).filter(BizStateBatch.task_id == task_id).all()
|
||||
# Refuse if any batch is still referenced by compare/migration (manual baseline is OK to drop with task)
|
||||
pmap = protected_batch_map(db, task_id=task_id)
|
||||
|
|
@ -519,7 +500,7 @@ def list_batches(db: Session, task_id: str, *, limit: int = 50) -> list[dict[str
|
|||
.limit(max(1, min(500, int(limit))))
|
||||
.all()
|
||||
)
|
||||
pmap = protected_batch_map(db, batch_ids=[b.id for b in rows])
|
||||
pmap = protected_batch_map(db, task_id=task_id)
|
||||
out: list[dict[str, Any]] = []
|
||||
for b in rows:
|
||||
reasons = list(pmap.get(b.id, []))
|
||||
|
|
@ -641,29 +622,12 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
|
|||
b = db.get(BizStateBatch, batch_id)
|
||||
if not b:
|
||||
raise HTTPException(status_code=404, detail="batch_not_found")
|
||||
raw = func.coalesce(BizStateBatchCommand.raw_text, "")
|
||||
raw_len = func.length(raw)
|
||||
# Compute legacy counts in SQL; workbook summaries never transfer raw CLI text.
|
||||
line_count = case(
|
||||
(BizStateBatchCommand.raw_line_count > 0, BizStateBatchCommand.raw_line_count),
|
||||
(raw_len == 0, 0),
|
||||
else_=raw_len - func.length(func.replace(raw, "\n", ""))
|
||||
+ case((func.substr(raw, raw_len, 1) == "\n", 0), else_=1),
|
||||
)
|
||||
whitespace = " \t\r\n\v\f"
|
||||
trimmed = (func.btrim(raw, whitespace) if db.get_bind().dialect.name == "postgresql"
|
||||
else func.trim(raw, whitespace))
|
||||
cmd_raw_stats: dict[str, tuple[int, bool]] = {}
|
||||
cmds = []
|
||||
for c, lines, has_raw in (
|
||||
db.query(BizStateBatchCommand, line_count, func.length(trimmed) > 0)
|
||||
.options(defer(BizStateBatchCommand.raw_text))
|
||||
cmds = (
|
||||
db.query(BizStateBatchCommand)
|
||||
.filter(BizStateBatchCommand.batch_id == batch_id)
|
||||
.order_by(BizStateBatchCommand.created_at.asc(), BizStateBatchCommand.id.asc())
|
||||
.order_by(BizStateBatchCommand.created_at.asc())
|
||||
.all()
|
||||
):
|
||||
cmds.append(c)
|
||||
cmd_raw_stats[c.id] = (int(lines or 0), bool(has_raw))
|
||||
)
|
||||
protect = batch_protect_info(db, batch_id)
|
||||
|
||||
# Per-metric row counts (generic table)
|
||||
|
|
@ -704,14 +668,11 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
|
|||
# Prefer primary collect rows over aux / aux_cached for the same CLI
|
||||
sheet_cmds[id_].append(cmd_info)
|
||||
|
||||
# Aux failures must remain visible even if their primary command was stored later.
|
||||
primary_cmds_by_cli = {
|
||||
normalize_command(str(c.raw_command or "")) for c in cmds
|
||||
if not str(c.parse_status or "").strip().lower().startswith("aux")
|
||||
}
|
||||
primary_cmds_by_cli: dict[str, dict[str, Any]] = {}
|
||||
for c in cmds:
|
||||
status = str(c.parse_status or "").strip().lower()
|
||||
is_aux = status.startswith("aux")
|
||||
raw = c.raw_text or ""
|
||||
info = {
|
||||
"id": c.id,
|
||||
"profile_id": c.profile_id,
|
||||
|
|
@ -721,13 +682,15 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
|
|||
"params": c.params_json or {},
|
||||
"parse_status": c.parse_status,
|
||||
"row_count": c.row_count,
|
||||
"raw_line_count": cmd_raw_stats[c.id][0],
|
||||
"raw_line_count": _cmd_raw_line_count(c),
|
||||
"declared_total": _cmd_declared_total(c),
|
||||
"message": c.message,
|
||||
"has_raw": cmd_raw_stats[c.id][1],
|
||||
"has_raw": bool(str(raw).strip()),
|
||||
"is_aux": is_aux,
|
||||
}
|
||||
cmd_n = normalize_command(str(c.raw_command or ""))
|
||||
if not is_aux and cmd_n:
|
||||
primary_cmds_by_cli.setdefault(cmd_n, info)
|
||||
# Commands sheet: hide successful aux when the same CLI already has a primary row;
|
||||
# keep failed/skipped aux visible so partial reasons are not hidden.
|
||||
if is_aux and cmd_n and cmd_n in primary_cmds_by_cli:
|
||||
|
|
@ -783,10 +746,10 @@ def get_batch(db: Session, batch_id: str) -> dict[str, Any]:
|
|||
"params": c.params_json or {},
|
||||
"parse_status": c.parse_status,
|
||||
"row_count": c.row_count,
|
||||
"raw_line_count": cmd_raw_stats[c.id][0],
|
||||
"raw_line_count": _cmd_raw_line_count(c),
|
||||
"declared_total": _cmd_declared_total(c),
|
||||
"message": c.message,
|
||||
"has_raw": cmd_raw_stats[c.id][1],
|
||||
"has_raw": bool(str(c.raw_text or "").strip()),
|
||||
"is_aux": True,
|
||||
}
|
||||
)
|
||||
|
|
@ -881,7 +844,6 @@ def list_batch_metric_rows(
|
|||
size_n = max(1, min(200, int(page_size or 50)))
|
||||
kw_n = str(kw or "").strip()
|
||||
col_n = str(column or "").strip()
|
||||
like = "%" + kw_n.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + "%"
|
||||
|
||||
fields = metric_field_map().get(mid) or []
|
||||
columns = [
|
||||
|
|
@ -897,30 +859,30 @@ def list_batch_metric_rows(
|
|||
if mid == "lldp_neighbor":
|
||||
q = db.query(BizStateLldpNeighbor).filter(BizStateLldpNeighbor.batch_id == batch_id)
|
||||
if kw_n:
|
||||
like = f"%{kw_n}%"
|
||||
if col_n == "local_if":
|
||||
q = q.filter(BizStateLldpNeighbor.local_if.ilike(like, escape="\\"))
|
||||
q = q.filter(BizStateLldpNeighbor.local_if.ilike(like))
|
||||
elif col_n == "remote_sys":
|
||||
q = q.filter(BizStateLldpNeighbor.remote_sys.ilike(like, escape="\\"))
|
||||
q = q.filter(BizStateLldpNeighbor.remote_sys.ilike(like))
|
||||
elif col_n == "remote_if":
|
||||
q = q.filter(BizStateLldpNeighbor.remote_if.ilike(like, escape="\\"))
|
||||
q = q.filter(BizStateLldpNeighbor.remote_if.ilike(like))
|
||||
elif col_n == "remote_ip":
|
||||
q = q.filter(BizStateLldpNeighbor.remote_ip.ilike(like, escape="\\"))
|
||||
q = q.filter(BizStateLldpNeighbor.remote_ip.ilike(like))
|
||||
elif col_n == "protocol":
|
||||
q = q.filter(BizStateLldpNeighbor.protocol.ilike(like, escape="\\"))
|
||||
q = q.filter(BizStateLldpNeighbor.protocol.ilike(like))
|
||||
else:
|
||||
q = q.filter(
|
||||
or_(
|
||||
BizStateLldpNeighbor.local_if.ilike(like, escape="\\"),
|
||||
BizStateLldpNeighbor.remote_sys.ilike(like, escape="\\"),
|
||||
BizStateLldpNeighbor.remote_if.ilike(like, escape="\\"),
|
||||
BizStateLldpNeighbor.remote_ip.ilike(like, escape="\\"),
|
||||
BizStateLldpNeighbor.protocol.ilike(like, escape="\\"),
|
||||
BizStateLldpNeighbor.local_if.ilike(like),
|
||||
BizStateLldpNeighbor.remote_sys.ilike(like),
|
||||
BizStateLldpNeighbor.remote_if.ilike(like),
|
||||
BizStateLldpNeighbor.remote_ip.ilike(like),
|
||||
BizStateLldpNeighbor.protocol.ilike(like),
|
||||
)
|
||||
)
|
||||
total = int(q.count() or 0)
|
||||
rows_db = (
|
||||
q.order_by(BizStateLldpNeighbor.local_if.asc(), BizStateLldpNeighbor.remote_sys.asc(),
|
||||
BizStateLldpNeighbor.remote_if.asc(), BizStateLldpNeighbor.id.asc())
|
||||
q.order_by(BizStateLldpNeighbor.local_if.asc())
|
||||
.offset((page_n - 1) * size_n)
|
||||
.limit(size_n)
|
||||
.all()
|
||||
|
|
@ -949,11 +911,12 @@ def list_batch_metric_rows(
|
|||
BizStateMetricRow.metric_id == mid,
|
||||
)
|
||||
if kw_n:
|
||||
like = f"%{kw_n}%"
|
||||
if col_n:
|
||||
# JSON path as text — works on Postgres JSONB and SQLite JSON
|
||||
q = q.filter(cast(BizStateMetricRow.data_json[col_n].as_string(), String).ilike(like, escape="\\"))
|
||||
q = q.filter(cast(BizStateMetricRow.data_json[col_n], String).ilike(like))
|
||||
else:
|
||||
q = q.filter(cast(BizStateMetricRow.data_json, String).ilike(like, escape="\\"))
|
||||
q = q.filter(cast(BizStateMetricRow.data_json, String).ilike(like))
|
||||
total = int(q.count() or 0)
|
||||
rows_db = (
|
||||
q.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc())
|
||||
|
|
@ -1056,64 +1019,56 @@ def export_batch_zip(db: Session, batch_id: str) -> bytes:
|
|||
db.query(BizStateLldpNeighbor)
|
||||
.filter(BizStateLldpNeighbor.batch_id == batch_id)
|
||||
.order_by(BizStateLldpNeighbor.local_if.asc())
|
||||
.yield_per(1000)
|
||||
.all()
|
||||
)
|
||||
with zf.open("tables/lldp_neighbor.csv", "w") as dest:
|
||||
dest.write(b"local_if,remote_sys,remote_if,remote_ip,protocol\n")
|
||||
csv_lines = ["local_if,remote_sys,remote_if,remote_ip,protocol"]
|
||||
for n in neighbors:
|
||||
line = ",".join(_csv(v) for v in (
|
||||
n.local_if, n.remote_sys, n.remote_if, n.remote_ip, n.protocol,
|
||||
)) + "\n"
|
||||
dest.write(line.encode("utf-8"))
|
||||
csv_lines.append(
|
||||
",".join(
|
||||
[
|
||||
_csv(n.local_if),
|
||||
_csv(n.remote_sys),
|
||||
_csv(n.remote_if),
|
||||
_csv(n.remote_ip),
|
||||
_csv(n.protocol),
|
||||
]
|
||||
)
|
||||
)
|
||||
zf.writestr("tables/lldp_neighbor.csv", "\n".join(csv_lines) + "\n")
|
||||
|
||||
# Generic metrics CSV (stream by metric_id)
|
||||
for sheet in detail.get("sheets") or []:
|
||||
mid = str(sheet.get("metric_id") or "").strip()
|
||||
if not mid or mid == "lldp_neighbor":
|
||||
continue
|
||||
# First scan discovers the same complete, ordered header as the legacy export.
|
||||
# Second scan writes rows without materializing the table or CSV.
|
||||
col_keys: dict[str, None] = {}
|
||||
has_rows = False
|
||||
for rec in _iter_metric_export_rows(db, batch_id, mid):
|
||||
has_rows = True
|
||||
for k in rec.keys():
|
||||
col_keys.setdefault(str(k), None)
|
||||
if not has_rows:
|
||||
rows = (
|
||||
db.query(BizStateMetricRow)
|
||||
.filter(
|
||||
BizStateMetricRow.batch_id == batch_id,
|
||||
BizStateMetricRow.metric_id == mid,
|
||||
)
|
||||
.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc())
|
||||
.all()
|
||||
)
|
||||
if not rows:
|
||||
continue
|
||||
cols = list(col_keys)
|
||||
recs = [dict(r.data_json or {}) for r in rows]
|
||||
cols: list[str] = []
|
||||
for rec in recs:
|
||||
for k in rec.keys():
|
||||
if k not in cols:
|
||||
cols.append(str(k))
|
||||
out_lines = [",".join(_csv(c) for c in cols)]
|
||||
for rec in recs:
|
||||
out_lines.append(",".join(_csv(str(rec.get(c, "") or "")) for c in cols))
|
||||
safe = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in mid)[:80] or "metric"
|
||||
with zf.open(f"tables/{safe}.csv", "w") as dest:
|
||||
dest.write((",".join(_csv(c) for c in cols) + "\n").encode("utf-8"))
|
||||
for rec in _iter_metric_export_rows(db, batch_id, mid):
|
||||
line = ",".join(_csv(rec.get(c)) for c in cols) + "\n"
|
||||
dest.write(line.encode("utf-8"))
|
||||
zf.writestr(f"tables/{safe}.csv", "\n".join(out_lines) + "\n")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _iter_metric_export_rows(db: Session, batch_id: str, metric_id: str, chunk_size: int = 1000):
|
||||
cursor: tuple[int, str] | None = None
|
||||
while True:
|
||||
query = db.query(BizStateMetricRow.seq, BizStateMetricRow.id, BizStateMetricRow.data_json).filter(
|
||||
BizStateMetricRow.batch_id == batch_id, BizStateMetricRow.metric_id == metric_id,
|
||||
)
|
||||
if cursor is not None:
|
||||
seq, row_id = cursor
|
||||
query = query.filter(or_(
|
||||
BizStateMetricRow.seq > seq,
|
||||
(BizStateMetricRow.seq == seq) & (BizStateMetricRow.id > row_id),
|
||||
))
|
||||
chunk = query.order_by(BizStateMetricRow.seq.asc(), BizStateMetricRow.id.asc()).limit(chunk_size).all()
|
||||
if not chunk:
|
||||
return
|
||||
for row in chunk:
|
||||
yield row.data_json or {}
|
||||
cursor = (chunk[-1].seq, chunk[-1].id)
|
||||
|
||||
|
||||
def _csv(v: Any) -> str:
|
||||
s = "" if v is None else str(v)
|
||||
if any(ch in s for ch in ",\"\r\n"):
|
||||
def _csv(v: str) -> str:
|
||||
s = str(v or "")
|
||||
if any(ch in s for ch in ",\"\n"):
|
||||
return '"' + s.replace('"', '""') + '"'
|
||||
return s
|
||||
|
||||
|
|
|
|||
|
|
@ -173,11 +173,6 @@ def api_get_task(task_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
|||
return svc.get_task(db, task_id)
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}/progress")
|
||||
def api_task_progress(task_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
return svc.get_task_progress(db, task_id)
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}/commands")
|
||||
def api_plan_task_commands(
|
||||
task_id: str,
|
||||
|
|
@ -549,12 +544,6 @@ class FieldRuleIn(BaseModel):
|
|||
normalize: str = ""
|
||||
ignore: bool = False
|
||||
tolerance: float | None = None
|
||||
# Cross-vendor identity rules (see identity_pipeline).
|
||||
value_map: dict[str, str] | None = None
|
||||
side: str = ""
|
||||
apply_to_key: bool | None = None
|
||||
# Order for compare=not_worse: earlier entries are better.
|
||||
rank: list[str] | None = None
|
||||
|
||||
|
||||
class RowFilterIn(BaseModel):
|
||||
|
|
@ -580,10 +569,6 @@ class TemplateMetricIn(BaseModel):
|
|||
display_fields: list[str] = Field(default_factory=list)
|
||||
row_filters: list[dict[str, Any]] = Field(default_factory=list)
|
||||
field_rules: list[FieldRuleIn] = Field(default_factory=list)
|
||||
ignore_port_changes: bool | None = None
|
||||
unmapped_port_policy: str = ""
|
||||
duplicate_pairing: str = ""
|
||||
builtin_vendor_canonical: bool | None = None
|
||||
|
||||
|
||||
class TemplateIn(BaseModel):
|
||||
|
|
@ -602,7 +587,6 @@ class TemplateIn(BaseModel):
|
|||
display_fields: list[str] = Field(default_factory=list)
|
||||
row_filters: list[dict[str, Any]] = Field(default_factory=list)
|
||||
field_rules: list[FieldRuleIn] = Field(default_factory=list)
|
||||
ignore_port_changes: bool | None = None
|
||||
|
||||
|
||||
class TemplatePatchIn(BaseModel):
|
||||
|
|
@ -618,7 +602,6 @@ class TemplatePatchIn(BaseModel):
|
|||
display_fields: list[str] | None = None
|
||||
row_filters: list[dict[str, Any]] | None = None
|
||||
field_rules: list[FieldRuleIn] | None = None
|
||||
ignore_port_changes: bool | None = None
|
||||
|
||||
|
||||
class MappingRowIn(BaseModel):
|
||||
|
|
@ -632,16 +615,6 @@ class MappingIn(BaseModel):
|
|||
rows: list[MappingRowIn] = Field(default_factory=list)
|
||||
|
||||
|
||||
class PreflightIn(BaseModel):
|
||||
template_id: str = ""
|
||||
enabled_sheet_ids: list[str] = Field(default_factory=list)
|
||||
mapping_id: str = ""
|
||||
# Unsaved editor rows win over ``mapping_id`` when given.
|
||||
mapping_rows: list[MappingRowIn] | None = None
|
||||
before_batch_id: str
|
||||
after_batch_id: str
|
||||
|
||||
|
||||
class ValidateMappingIn(BaseModel):
|
||||
mapping_id: str
|
||||
before_batch_id: str
|
||||
|
|
@ -660,8 +633,6 @@ class CompareJobIn(BaseModel):
|
|||
mode: str = "manual"
|
||||
# Empty = all template sheets; non-empty = only these sheet_id values
|
||||
enabled_sheet_ids: list[str] = Field(default_factory=list)
|
||||
# auto|always|never|sample|keys — how to persist matching (success) rows
|
||||
store_unchanged: str = "auto"
|
||||
note: str = ""
|
||||
|
||||
|
||||
|
|
@ -730,11 +701,6 @@ def api_validate_mapping(body: ValidateMappingIn, db: Session = Depends(get_db))
|
|||
)
|
||||
|
||||
|
||||
@router.post("/compare/preflight")
|
||||
def api_compare_preflight(body: PreflightIn, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
return cmp_svc.preflight_compare(db, body.model_dump())
|
||||
|
||||
|
||||
@router.get("/compare/jobs")
|
||||
def api_list_jobs(db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
return {"items": cmp_svc.list_jobs(db)}
|
||||
|
|
@ -757,15 +723,8 @@ def api_delete_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any]
|
|||
|
||||
|
||||
@router.post("/compare/jobs/{job_id}/run")
|
||||
def api_run_job(
|
||||
job_id: str,
|
||||
sync: bool = Query(False, description="If true, block until compare finishes (tests/debug)"),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
"""Start a compare run. Default is async (returns status=running immediately)."""
|
||||
if sync:
|
||||
def api_run_job(job_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
return cmp_svc.run_compare(db, job_id)
|
||||
return cmp_svc.enqueue_compare(db, job_id)
|
||||
|
||||
|
||||
@router.get("/compare/jobs/{job_id}/runs")
|
||||
|
|
@ -778,12 +737,6 @@ def api_get_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
|||
return cmp_svc.get_run(db, run_id)
|
||||
|
||||
|
||||
@router.post("/compare/runs/{run_id}/cancel")
|
||||
def api_cancel_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
"""Cancel a stuck/running compare so a new run can start."""
|
||||
return cmp_svc.cancel_compare_run(db, run_id)
|
||||
|
||||
|
||||
@router.delete("/compare/runs/{run_id}")
|
||||
def api_delete_run(run_id: str, db: Session = Depends(get_db)) -> dict[str, Any]:
|
||||
return cmp_svc.delete_run(db, run_id)
|
||||
|
|
@ -795,7 +748,6 @@ def api_list_run_diffs(
|
|||
metric_id: str = "",
|
||||
kind: str = "diff",
|
||||
kw: str = "",
|
||||
qf: str = "",
|
||||
page: int = 1,
|
||||
page_size: int = 100,
|
||||
db: Session = Depends(get_db),
|
||||
|
|
@ -806,7 +758,6 @@ def api_list_run_diffs(
|
|||
metric_id=metric_id,
|
||||
kind=kind,
|
||||
kw=kw,
|
||||
qf=qf,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -47,15 +47,6 @@ def _maybe_purge_retention() -> None:
|
|||
from .biz_state.retention import purge_all_tasks
|
||||
|
||||
info = purge_all_tasks(db)
|
||||
from .biz_state.retention import purge_auto_runs
|
||||
|
||||
runs = purge_auto_runs(
|
||||
db,
|
||||
migration_keep=int(getattr(settings, "biz_migration_auto_run_keep", 300) or 300),
|
||||
compare_keep=int(getattr(settings, "biz_compare_auto_run_keep", 50) or 50),
|
||||
)
|
||||
if any(runs.values()):
|
||||
_log.info("biz_state periodic run purge %s", runs)
|
||||
if info.get("dropped"):
|
||||
_log.info(
|
||||
"biz_state periodic retention dropped=%s tasks=%s",
|
||||
|
|
@ -119,21 +110,12 @@ def _sync_cutover_hf_windows() -> None:
|
|||
|
||||
projects = db.query(BizMigrationProject).all()
|
||||
for proj in projects:
|
||||
tids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new")
|
||||
if tids and any((t := db.get(BizStateTask, tid)) and not t.collect_group_id for tid in tids):
|
||||
try:
|
||||
from .biz_migration.service import ensure_highfreq
|
||||
|
||||
ensure_highfreq(db, proj.id, collect_now=False)
|
||||
except Exception:
|
||||
_log.exception("upgrade cutover collection pairs failed project=%s", proj.id)
|
||||
window_status = _hf_window_status(proj)
|
||||
want = _hf_window_status(proj)
|
||||
tids = _all_hf_task_ids(proj, "old") + _all_hf_task_ids(proj, "new")
|
||||
for tid in tids:
|
||||
task = db.get(BizStateTask, tid)
|
||||
if not task:
|
||||
continue
|
||||
want = "paused" if task.collect_manual_only else window_status
|
||||
purpose = str(getattr(task, "purpose", None) or "").strip()
|
||||
if purpose and purpose != PURPOSE_CUTOVER_HF:
|
||||
continue
|
||||
|
|
@ -167,26 +149,12 @@ def _enqueue_due_tasks() -> int:
|
|||
.filter(
|
||||
BizStateTask.status == "running",
|
||||
BizStateTask.collect_running.is_(False),
|
||||
BizStateTask.collect_manual_only.is_(False),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
due_ids: list[str] = []
|
||||
seen_groups: set[str] = set()
|
||||
now = _utcnow()
|
||||
for task in tasks:
|
||||
if task.collect_group_id:
|
||||
if task.collect_group_id in seen_groups:
|
||||
continue
|
||||
seen_groups.add(task.collect_group_id)
|
||||
peers = db.query(BizStateTask).filter(BizStateTask.collect_group_id == task.collect_group_id).all()
|
||||
if len(peers) != 2 or any(p.collect_running or p.status != "running" or p.collect_manual_only for p in peers):
|
||||
continue
|
||||
anchors = [p.last_collect_started_at or p.last_collect_ended_at for p in peers]
|
||||
anchor = max((a for a in anchors if a), default=None)
|
||||
if anchor is None or (now - anchor).total_seconds() >= max(p.interval_sec or 60 for p in peers):
|
||||
due_ids.append(task.id)
|
||||
continue
|
||||
interval = max(60, int(task.interval_sec or 300))
|
||||
# Prefer start-based interval to avoid drift when collect duration
|
||||
# approaches the interval (end-based: 40s collect + 60s → 100s cycle).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .cli_schemas import (
|
||||
|
|
@ -15,14 +15,12 @@ from .cli_service import (
|
|||
delete_cli_profile,
|
||||
get_cli_profile,
|
||||
get_ume_cli_override,
|
||||
list_cli_target_filter_options,
|
||||
list_cli_profiles,
|
||||
list_cli_targets,
|
||||
set_default_cli_profile,
|
||||
update_cli_profile,
|
||||
upsert_ume_cli_override,
|
||||
)
|
||||
from .table_filters import TableFilterParams, table_filter_params
|
||||
from .db import get_db
|
||||
from .ne_connect import schedule_ume_connect_tests
|
||||
|
||||
|
|
@ -68,57 +66,11 @@ def api_delete_cli_profile(profile_id: str, db: Session = Depends(get_db)):
|
|||
def api_list_cli_targets(
|
||||
source: str = Query(default="all"),
|
||||
keyword: str | None = Query(default=None),
|
||||
source_filter: list[str] | None = Query(default=None),
|
||||
source_filter_mode: str | None = Query(default=None),
|
||||
name_filter: list[str] | None = Query(default=None),
|
||||
name_filter_mode: str | None = Query(default=None),
|
||||
ip_filter: list[str] | None = Query(default=None),
|
||||
ip_filter_mode: str | None = Query(default=None),
|
||||
vendor_filter: list[str] | None = Query(default=None),
|
||||
vendor_filter_mode: str | None = Query(default=None),
|
||||
device_type_filter: list[str] | None = Query(default=None),
|
||||
device_type_filter_mode: str | None = Query(default=None),
|
||||
connect_status_filter: list[str] | None = Query(default=None),
|
||||
connect_status_filter_mode: str | None = Query(default=None),
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=50, ge=1, le=500),
|
||||
table_filters: TableFilterParams = Depends(table_filter_params),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
if table_filters.field:
|
||||
return list_cli_target_filter_options(db, field=table_filters.field, source=source, keyword=keyword)
|
||||
filters = table_filters.filters
|
||||
if set(filters) - {"source", "name", "ip", "vendor", "deviceType", "connectStatus"}:
|
||||
raise HTTPException(status_code=400, detail="invalid_filter_field")
|
||||
return list_cli_targets(
|
||||
db,
|
||||
source=source,
|
||||
keyword=keyword,
|
||||
source_filter=filters.get("source", {}).get("values", source_filter),
|
||||
source_filter_mode=filters.get("source", {}).get("mode", source_filter_mode),
|
||||
name_filter=filters.get("name", {}).get("values", name_filter),
|
||||
name_filter_mode=filters.get("name", {}).get("mode", name_filter_mode),
|
||||
ip_filter=filters.get("ip", {}).get("values", ip_filter),
|
||||
ip_filter_mode=filters.get("ip", {}).get("mode", ip_filter_mode),
|
||||
vendor_filter=filters.get("vendor", {}).get("values", vendor_filter),
|
||||
vendor_filter_mode=filters.get("vendor", {}).get("mode", vendor_filter_mode),
|
||||
device_type_filter=filters.get("deviceType", {}).get("values", device_type_filter),
|
||||
device_type_filter_mode=filters.get("deviceType", {}).get("mode", device_type_filter_mode),
|
||||
connect_status_filter=filters.get("connectStatus", {}).get("values", connect_status_filter),
|
||||
connect_status_filter_mode=filters.get("connectStatus", {}).get("mode", connect_status_filter_mode),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/targets/filter-options")
|
||||
def api_list_cli_target_filter_options(
|
||||
field: str = Query(),
|
||||
source: str = Query(default="inventory"),
|
||||
keyword: str | None = Query(default=None),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return list_cli_target_filter_options(db, field=field, source=source, keyword=keyword)
|
||||
return list_cli_targets(db, source=source, keyword=keyword, page=page, page_size=page_size)
|
||||
|
||||
|
||||
@router.get("/ume-overrides/{ume_ne_id}")
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from datetime import datetime
|
|||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .cli_schemas import (
|
||||
|
|
@ -280,66 +279,19 @@ def list_cli_targets(
|
|||
*,
|
||||
source: str = "all",
|
||||
keyword: str | None = None,
|
||||
source_filter: list[str] | None = None,
|
||||
source_filter_mode: str | None = None,
|
||||
name_filter: list[str] | None = None,
|
||||
name_filter_mode: str | None = None,
|
||||
ip_filter: list[str] | None = None,
|
||||
ip_filter_mode: str | None = None,
|
||||
vendor_filter: list[str] | None = None,
|
||||
vendor_filter_mode: str | None = None,
|
||||
device_type_filter: list[str] | None = None,
|
||||
device_type_filter_mode: str | None = None,
|
||||
connect_status_filter: list[str] | None = None,
|
||||
connect_status_filter_mode: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
) -> dict[str, Any]:
|
||||
from .device_types import WEBCRT_NE_SOURCE
|
||||
|
||||
src = str(source or "all").strip().lower()
|
||||
if src not in ("managed", "ume", "webcrt", "inventory", "all"):
|
||||
if src not in ("managed", "ume", "webcrt", "all"):
|
||||
raise HTTPException(status_code=400, detail="invalid_source")
|
||||
ready = cli_profile_ready(db)
|
||||
page = max(1, int(page or 1))
|
||||
page_size = max(1, min(500, int(page_size or 50)))
|
||||
offset = (page - 1) * page_size
|
||||
kw = str(keyword or "").strip()
|
||||
column_filters: dict[str, list[str] | None] = {
|
||||
"source": source_filter,
|
||||
"name": name_filter,
|
||||
"ip": ip_filter,
|
||||
"vendor": vendor_filter,
|
||||
"device_type": device_type_filter,
|
||||
"connect_status": connect_status_filter,
|
||||
}
|
||||
filter_modes = {
|
||||
"source": str(source_filter_mode or ""),
|
||||
"name": str(name_filter_mode or ""),
|
||||
"ip": str(ip_filter_mode or ""),
|
||||
"vendor": str(vendor_filter_mode or ""),
|
||||
"device_type": str(device_type_filter_mode or ""),
|
||||
"connect_status": str(connect_status_filter_mode or ""),
|
||||
}
|
||||
|
||||
def _source_allowed(row_source: str) -> bool:
|
||||
mode = filter_modes["source"]
|
||||
values = column_filters["source"] or []
|
||||
if mode == "include":
|
||||
return row_source in values
|
||||
if mode == "exclude":
|
||||
return row_source not in values
|
||||
return True
|
||||
|
||||
def _apply_value_filter(stmt: Any, field: str, expression: Any) -> Any:
|
||||
mode = filter_modes[field]
|
||||
if mode not in ("include", "exclude"):
|
||||
return stmt
|
||||
values = column_filters[field] or []
|
||||
normalized = func.coalesce(expression, "")
|
||||
if mode == "include":
|
||||
return stmt.filter(normalized.in_(values)) if values else stmt.filter(False)
|
||||
return stmt.filter(~normalized.in_(values)) if values else stmt
|
||||
|
||||
default_profile = get_default_profile(db)
|
||||
default_hop = bool(getattr(default_profile, "hop_enabled", False)) if default_profile else False
|
||||
|
|
@ -399,8 +351,7 @@ def list_cli_targets(
|
|||
return stmt
|
||||
like = f"%{kw}%"
|
||||
return stmt.filter(
|
||||
ManagedNE.id.ilike(like)
|
||||
| ManagedNE.name.ilike(like)
|
||||
ManagedNE.name.ilike(like)
|
||||
| ManagedNE.ip_address.ilike(like)
|
||||
| ManagedNE.username.ilike(like)
|
||||
| ManagedNE.tags.ilike(like)
|
||||
|
|
@ -411,42 +362,16 @@ def list_cli_targets(
|
|||
def _inventory_managed_query():
|
||||
# Asset inventory for WebCRT: exclude Quick-Connect (source=webcrt) rows.
|
||||
stmt = db.query(ManagedNE).filter(ManagedNE.source != WEBCRT_NE_SOURCE)
|
||||
stmt = _apply_kw(stmt)
|
||||
if not _source_allowed("managed"):
|
||||
stmt = stmt.filter(False)
|
||||
stmt = _apply_value_filter(
|
||||
stmt,
|
||||
"name",
|
||||
func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address),
|
||||
)
|
||||
stmt = _apply_value_filter(stmt, "ip", ManagedNE.ip_address)
|
||||
stmt = _apply_value_filter(stmt, "vendor", ManagedNE.vendor)
|
||||
stmt = _apply_value_filter(stmt, "device_type", ManagedNE.device_type)
|
||||
stmt = _apply_value_filter(stmt, "connect_status", ManagedNE.connect_status)
|
||||
return stmt.order_by(ManagedNE.updated_at.desc())
|
||||
return _apply_kw(stmt).order_by(ManagedNE.updated_at.desc())
|
||||
|
||||
def _webcrt_query():
|
||||
stmt = db.query(ManagedNE).filter(ManagedNE.source == WEBCRT_NE_SOURCE)
|
||||
stmt = _apply_kw(stmt)
|
||||
if not _source_allowed("webcrt"):
|
||||
stmt = stmt.filter(False)
|
||||
stmt = _apply_value_filter(
|
||||
stmt,
|
||||
"name",
|
||||
func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address),
|
||||
)
|
||||
stmt = _apply_value_filter(stmt, "ip", ManagedNE.ip_address)
|
||||
stmt = _apply_value_filter(stmt, "vendor", ManagedNE.vendor)
|
||||
stmt = _apply_value_filter(stmt, "device_type", ManagedNE.device_type)
|
||||
stmt = _apply_value_filter(stmt, "connect_status", ManagedNE.connect_status)
|
||||
return stmt.order_by(ManagedNE.updated_at.desc())
|
||||
return _apply_kw(stmt).order_by(ManagedNE.updated_at.desc())
|
||||
|
||||
def _ume_query():
|
||||
stmt = db.query(UmeInventoryNE, UmeCliOverride).outerjoin(
|
||||
UmeCliOverride, UmeInventoryNE.ne_id == UmeCliOverride.ume_ne_id
|
||||
)
|
||||
if not _source_allowed("ume"):
|
||||
stmt = stmt.filter(False)
|
||||
if kw:
|
||||
like = f"%{kw}%"
|
||||
stmt = stmt.filter(
|
||||
|
|
@ -456,46 +381,8 @@ def list_cli_targets(
|
|||
| UmeInventoryNE.ip_address.ilike(like)
|
||||
| UmeInventoryNE.host_name.ilike(like)
|
||||
)
|
||||
ume_name = func.coalesce(
|
||||
func.nullif(UmeInventoryNE.user_label, ""),
|
||||
func.nullif(UmeInventoryNE.ne_name, ""),
|
||||
func.nullif(UmeInventoryNE.host_name, ""),
|
||||
func.nullif(UmeInventoryNE.ip_address, ""),
|
||||
UmeInventoryNE.ne_id,
|
||||
)
|
||||
stmt = _apply_value_filter(stmt, "name", ume_name)
|
||||
stmt = _apply_value_filter(stmt, "ip", UmeInventoryNE.ip_address)
|
||||
stmt = _apply_value_filter(stmt, "vendor", UmeInventoryNE.vendor)
|
||||
stmt = _apply_value_filter(stmt, "device_type", UmeInventoryNE.ne_type)
|
||||
stmt = _apply_value_filter(
|
||||
stmt,
|
||||
"connect_status",
|
||||
func.coalesce(UmeCliOverride.connect_status, "unknown"),
|
||||
)
|
||||
return stmt.order_by(UmeInventoryNE.ne_id.asc())
|
||||
|
||||
if src == "inventory":
|
||||
# Network Management inventory includes managed assets and UME inventory,
|
||||
# but not transient WebCRT quick-connect sessions.
|
||||
mq = _inventory_managed_query()
|
||||
uq = _ume_query()
|
||||
m_total = int(mq.count())
|
||||
u_total = int(uq.count())
|
||||
total = m_total + u_total
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
if offset < m_total:
|
||||
take = min(page_size, m_total - offset)
|
||||
items.extend(_managed_item(row, list_source="managed") for row in mq.offset(offset).limit(take).all())
|
||||
remaining = page_size - take
|
||||
if remaining > 0:
|
||||
items.extend(_ume_item(inv, ov) for inv, ov in uq.offset(0).limit(remaining).all())
|
||||
else:
|
||||
ume_offset = max(0, offset - m_total)
|
||||
items.extend(_ume_item(inv, ov) for inv, ov in uq.offset(ume_offset).limit(page_size).all())
|
||||
|
||||
return {"items": items, "total": total, "page": page, "page_size": page_size}
|
||||
|
||||
if src == "managed":
|
||||
mq = _inventory_managed_query()
|
||||
total = int(mq.count())
|
||||
|
|
@ -556,118 +443,6 @@ def list_cli_targets(
|
|||
return {"items": items, "total": total, "page": page, "page_size": page_size}
|
||||
|
||||
|
||||
def list_cli_target_filter_options(
|
||||
db: Session,
|
||||
*,
|
||||
field: str,
|
||||
source: str = "inventory",
|
||||
keyword: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from .device_types import WEBCRT_NE_SOURCE
|
||||
|
||||
field_map = {
|
||||
"source": "source",
|
||||
"name": "name",
|
||||
"ip": "ip",
|
||||
"vendor": "vendor",
|
||||
"deviceType": "device_type",
|
||||
"connectStatus": "connect_status",
|
||||
}
|
||||
if field not in field_map:
|
||||
raise HTTPException(status_code=400, detail="invalid_filter_field")
|
||||
src = str(source or "inventory").strip().lower()
|
||||
if src not in ("managed", "ume", "webcrt", "inventory", "all"):
|
||||
raise HTTPException(status_code=400, detail="invalid_source")
|
||||
|
||||
include_managed = src in ("managed", "inventory", "all")
|
||||
include_webcrt = src in ("webcrt", "all")
|
||||
include_ume = src in ("ume", "inventory", "all")
|
||||
kw = str(keyword or "").strip()
|
||||
|
||||
managed = db.query(ManagedNE)
|
||||
if src in ("managed", "inventory"):
|
||||
managed = managed.filter(ManagedNE.source != WEBCRT_NE_SOURCE)
|
||||
elif src == "webcrt":
|
||||
managed = managed.filter(ManagedNE.source == WEBCRT_NE_SOURCE)
|
||||
elif src == "all":
|
||||
managed = managed.filter(ManagedNE.source != WEBCRT_NE_SOURCE)
|
||||
ume = db.query(UmeInventoryNE)
|
||||
ume_status = db.query(UmeInventoryNE, UmeCliOverride).outerjoin(
|
||||
UmeCliOverride, UmeInventoryNE.ne_id == UmeCliOverride.ume_ne_id
|
||||
)
|
||||
webcrt = db.query(ManagedNE).filter(ManagedNE.source == WEBCRT_NE_SOURCE)
|
||||
|
||||
def _apply_managed_keyword(query: Any) -> Any:
|
||||
if not kw:
|
||||
return query
|
||||
like = f"%{kw}%"
|
||||
return query.filter(
|
||||
ManagedNE.id.ilike(like)
|
||||
| ManagedNE.name.ilike(like)
|
||||
| ManagedNE.ip_address.ilike(like)
|
||||
| ManagedNE.username.ilike(like)
|
||||
| ManagedNE.tags.ilike(like)
|
||||
| ManagedNE.vendor.ilike(like)
|
||||
| ManagedNE.device_type.ilike(like)
|
||||
)
|
||||
|
||||
def _apply_ume_keyword(query: Any) -> Any:
|
||||
if not kw:
|
||||
return query
|
||||
like = f"%{kw}%"
|
||||
return query.filter(
|
||||
UmeInventoryNE.ne_id.ilike(like)
|
||||
| UmeInventoryNE.ne_name.ilike(like)
|
||||
| UmeInventoryNE.user_label.ilike(like)
|
||||
| UmeInventoryNE.ip_address.ilike(like)
|
||||
| UmeInventoryNE.host_name.ilike(like)
|
||||
)
|
||||
|
||||
managed = _apply_managed_keyword(managed)
|
||||
webcrt = _apply_managed_keyword(webcrt)
|
||||
ume = _apply_ume_keyword(ume)
|
||||
ume_status = _apply_ume_keyword(ume_status)
|
||||
|
||||
if field == "source":
|
||||
values: set[str] = set()
|
||||
if include_managed and managed.count():
|
||||
values.add("managed")
|
||||
if include_webcrt and webcrt.count():
|
||||
values.add("webcrt")
|
||||
if include_ume and ume.count():
|
||||
values.add("ume")
|
||||
else:
|
||||
managed_expr = {
|
||||
"name": func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address),
|
||||
"ip": ManagedNE.ip_address,
|
||||
"vendor": ManagedNE.vendor,
|
||||
"device_type": ManagedNE.device_type,
|
||||
"connect_status": ManagedNE.connect_status,
|
||||
}.get(field_map[field])
|
||||
ume_expr = {
|
||||
"name": func.coalesce(
|
||||
func.nullif(UmeInventoryNE.user_label, ""),
|
||||
func.nullif(UmeInventoryNE.ne_name, ""),
|
||||
func.nullif(UmeInventoryNE.host_name, ""),
|
||||
func.nullif(UmeInventoryNE.ip_address, ""),
|
||||
UmeInventoryNE.ne_id,
|
||||
),
|
||||
"ip": UmeInventoryNE.ip_address,
|
||||
"vendor": UmeInventoryNE.vendor,
|
||||
"device_type": UmeInventoryNE.ne_type,
|
||||
"connect_status": func.coalesce(UmeCliOverride.connect_status, "unknown"),
|
||||
}.get(field_map[field])
|
||||
values = set()
|
||||
if include_managed and managed_expr is not None:
|
||||
values.update(str(row[0] or "") for row in managed.with_entities(managed_expr).distinct().all())
|
||||
if include_webcrt and managed_expr is not None:
|
||||
values.update(str(row[0] or "") for row in webcrt.with_entities(managed_expr).distinct().all())
|
||||
if include_ume and ume_expr is not None:
|
||||
values.update(str(row[0] or "") for row in ume_status.with_entities(ume_expr).distinct().all())
|
||||
|
||||
return {"field": field, "options": sorted(values, key=str.casefold)}
|
||||
|
||||
|
||||
def cli_meta(db: Session) -> dict[str, Any]:
|
||||
return {
|
||||
"credentials_configured": credentials_configured(),
|
||||
|
|
|
|||
|
|
@ -80,9 +80,8 @@ class Settings(BaseSettings):
|
|||
ume_topo_nodes_path: str = "/restconf/data/zte-resources-module:TopoNodes"
|
||||
ume_topological_links_path: str = "/restconf/data/zte-resources-module:TopologicalLinks"
|
||||
ume_sync_alarms_history_every_hours: int = 24
|
||||
# Managed NE credentials (Fernet). Empty = auto-generate & persist to credential_secret_file.
|
||||
# Managed NE credentials (Fernet key; generate with cryptography.fernet.Fernet.generate_key())
|
||||
credential_secret_key: str = ""
|
||||
credential_secret_file: str = "data/auth/credential_secret"
|
||||
# Shared-server worker caps (sized for multi-operator use; raise if bastion/DB allow).
|
||||
ne_connect_max_workers: int = 8
|
||||
ne_connect_timeout_sec: int = 30
|
||||
|
|
@ -137,13 +136,6 @@ class Settings(BaseSettings):
|
|||
biz_state_dedicated_workers: bool = True
|
||||
# Global ceiling for simultaneous running batches (across all workers).
|
||||
biz_state_max_concurrent_tasks: int = 16
|
||||
# Auto-generated run history kept per cutover wave / auto compare job (acceptance + manual runs are never purged).
|
||||
biz_migration_auto_run_keep: int = 300
|
||||
biz_compare_auto_run_keep: int = 50
|
||||
# Rows of immutable baseline batches cached across cutover evaluations (0 disables).
|
||||
biz_migration_baseline_cache_rows: int = 500000
|
||||
# Concurrent SSH collections per network element (0 = unlimited).
|
||||
biz_state_max_sessions_per_ne: int = 2
|
||||
biz_state_worker_collect_threads: int = 8
|
||||
biz_state_persist_workers: int = 4
|
||||
# TextFSM / normalize workers (off SSH collect threads). Default 8.
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ from .config_sync_service import (
|
|||
stop_cycle,
|
||||
update_policy,
|
||||
)
|
||||
from .table_filters import TableFilterParams, table_filter_params
|
||||
from .db import get_db
|
||||
|
||||
router = APIRouter(prefix="/v1/config-sync", tags=["config-sync"])
|
||||
|
|
@ -165,11 +164,10 @@ def api_list_snapshots(
|
|||
keyword: str = Query(default=""),
|
||||
source: str = Query(default=""),
|
||||
vendor: str = Query(default=""),
|
||||
table_filters: TableFilterParams = Depends(table_filter_params),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return list_snapshots(
|
||||
db, page=page, page_size=page_size, keyword=keyword, source=source, vendor=vendor, table_filters=table_filters
|
||||
db, page=page, page_size=page_size, keyword=keyword, source=source, vendor=vendor
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import zipfile
|
|||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import func, or_
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .config_sync_codec import decompress_text
|
||||
|
|
@ -16,7 +16,6 @@ from .config_sync_schemas import (
|
|||
NeConfigSnapshotDetailOut,
|
||||
NeConfigSnapshotMetaOut,
|
||||
)
|
||||
from .table_filters import TableFilterParams, filter_query
|
||||
from .models import NeConfigHistory, NeConfigSnapshot
|
||||
|
||||
def _snap_meta(row: NeConfigSnapshot) -> NeConfigSnapshotMetaOut:
|
||||
|
|
@ -49,7 +48,6 @@ def list_snapshots(
|
|||
keyword: str = "",
|
||||
source: str = "",
|
||||
vendor: str = "",
|
||||
table_filters: TableFilterParams | None = None,
|
||||
) -> dict[str, Any]:
|
||||
q = db.query(NeConfigSnapshot)
|
||||
src = str(source or "").strip().lower()
|
||||
|
|
@ -68,14 +66,6 @@ def list_snapshots(
|
|||
NeConfigSnapshot.target_id.ilike(like),
|
||||
)
|
||||
)
|
||||
q, options = filter_query(q, {
|
||||
"name": func.coalesce(func.nullif(NeConfigSnapshot.ne_name, ""), NeConfigSnapshot.target_id),
|
||||
"ip": NeConfigSnapshot.ne_ip, "vendor": NeConfigSnapshot.vendor,
|
||||
"source": NeConfigSnapshot.source, "size": NeConfigSnapshot.plain_size,
|
||||
"collected": NeConfigSnapshot.collected_at,
|
||||
}, table_filters)
|
||||
if options is not None:
|
||||
return options
|
||||
total = int(q.count())
|
||||
rows = q.order_by(NeConfigSnapshot.collected_at.desc()).offset((page - 1) * page_size).limit(page_size).all()
|
||||
return {"total": total, "page": page, "page_size": page_size, "items": [_snap_meta(r) for r in rows]}
|
||||
|
|
|
|||
|
|
@ -1,34 +1,29 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from datetime import datetime, timezone
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .config import settings
|
||||
from .models import AlarmBatch, AlarmNorm, ImportErrorRow
|
||||
from .parser_config import ParserConfig
|
||||
from .tabular import parse_datetime, read_xlsx
|
||||
|
||||
|
||||
def _blank(v: Any) -> bool:
|
||||
if v is None:
|
||||
return True
|
||||
if isinstance(v, float) and math.isnan(v):
|
||||
return True
|
||||
text = str(v).strip()
|
||||
return not text or text.lower() == "nan"
|
||||
|
||||
|
||||
def _dt(v: Any) -> datetime | None:
|
||||
if _blank(v):
|
||||
if v is None:
|
||||
return None
|
||||
dt = parse_datetime(v)
|
||||
if dt is None:
|
||||
text = str(v).strip()
|
||||
if not text or text.lower() == "nan":
|
||||
return None
|
||||
parsed = pd.to_datetime(text, errors="coerce")
|
||||
if pd.isna(parsed):
|
||||
return None
|
||||
dt = parsed.to_pydatetime()
|
||||
# Convention: Excel source time is local timezone time.
|
||||
# Convert to UTC before persisting; keep DB as naive UTC for compatibility.
|
||||
if dt.tzinfo is None:
|
||||
|
|
@ -40,15 +35,20 @@ def _dt(v: Any) -> datetime | None:
|
|||
|
||||
|
||||
def _str(v: Any) -> str:
|
||||
if _blank(v):
|
||||
if v is None:
|
||||
return ""
|
||||
return str(v).strip()
|
||||
text = str(v).strip()
|
||||
if text.lower() == "nan":
|
||||
return ""
|
||||
return text
|
||||
|
||||
|
||||
def _int(v: Any) -> int:
|
||||
if _blank(v):
|
||||
if v is None:
|
||||
return 0
|
||||
text = str(v).strip()
|
||||
if not text or text.lower() == "nan":
|
||||
return 0
|
||||
try:
|
||||
return int(float(text))
|
||||
except Exception:
|
||||
|
|
@ -56,7 +56,8 @@ def _int(v: Any) -> int:
|
|||
|
||||
|
||||
def import_alarm_excel(db: Session, filename: str, content: bytes, parser: ParserConfig) -> AlarmBatch:
|
||||
headers, rows = read_xlsx(content)
|
||||
df = pd.read_excel(BytesIO(content))
|
||||
headers = [str(h) for h in df.columns]
|
||||
|
||||
col_alarm_time = parser.resolve_col(headers, "alarm_time")
|
||||
col_clear_time = parser.resolve_col(headers, "clear_time")
|
||||
|
|
@ -79,7 +80,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
|
|||
source_file=filename,
|
||||
parser_version=parser.parser_version,
|
||||
dict_version=parser.dict_version,
|
||||
total_rows=len(rows),
|
||||
total_rows=int(len(df.index)),
|
||||
status="processing",
|
||||
)
|
||||
db.add(batch)
|
||||
|
|
@ -88,8 +89,8 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
|
|||
success_rows = 0
|
||||
failed_rows = 0
|
||||
|
||||
for row_no, row in rows: # row_no = sheet row (header is row 1)
|
||||
raw_json = json.dumps(row, ensure_ascii=False, default=str)
|
||||
for idx, row in df.iterrows():
|
||||
row_no = int(idx) + 2 # Excel header is row 1
|
||||
alarm_time = _dt(row.get(col_alarm_time)) if col_alarm_time else None
|
||||
raw_severity = _str(row.get(col_severity)) if col_severity else ""
|
||||
alarm_code = _str(row.get(col_alarm_code)) if col_alarm_code else ""
|
||||
|
|
@ -110,7 +111,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
|
|||
batch_id=batch.batch_id,
|
||||
row_no=row_no,
|
||||
reason="missing_or_invalid_alarm_time",
|
||||
raw_json=raw_json,
|
||||
raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str),
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
|
@ -121,7 +122,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
|
|||
batch_id=batch.batch_id,
|
||||
row_no=row_no,
|
||||
reason="missing_ne_name_and_ne_id",
|
||||
raw_json=raw_json,
|
||||
raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str),
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
|
@ -150,7 +151,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
|
|||
vendor=settings.vendor,
|
||||
source_type=settings.source_type,
|
||||
source_file=filename,
|
||||
raw_json=raw_json,
|
||||
raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str),
|
||||
)
|
||||
)
|
||||
success_rows += 1
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from typing import Any
|
|||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .table_filters import TableFilterParams, table_filter_params
|
||||
from .db import get_db
|
||||
from .lldp_collect_schemas import LldpCollectPolicyUpdate, LldpCollectStartBody
|
||||
from .lldp_collect_service import (
|
||||
|
|
@ -70,10 +69,9 @@ def api_list_jobs(
|
|||
page_size: int = Query(default=20, ge=1, le=100),
|
||||
status: str = Query(default=""),
|
||||
keyword: str = Query(default=""),
|
||||
table_filters: TableFilterParams = Depends(table_filter_params),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
return list_jobs(db, page=page, page_size=page_size, status=status, keyword=keyword, table_filters=table_filters)
|
||||
return list_jobs(db, page=page, page_size=page_size, status=status, keyword=keyword)
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}")
|
||||
|
|
@ -83,7 +81,6 @@ def api_get_job(
|
|||
page_size: int = Query(default=20, ge=1, le=100),
|
||||
status: str = Query(default=""),
|
||||
keyword: str = Query(default=""),
|
||||
table_filters: TableFilterParams = Depends(table_filter_params),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
return get_job_detail(
|
||||
|
|
@ -93,5 +90,4 @@ def api_get_job(
|
|||
page_size=page_size,
|
||||
item_status=status,
|
||||
item_keyword=keyword,
|
||||
table_filters=table_filters,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,11 +5,9 @@ from __future__ import annotations
|
|||
from datetime import datetime, timedelta
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import String, cast, or_
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .table_filters import TableFilterParams, filter_query
|
||||
from .topology_discover_common import discover_item_expressions
|
||||
from .lldp_collect_schemas import (
|
||||
LldpCollectDashboardOut,
|
||||
LldpCollectJobSummary,
|
||||
|
|
@ -490,7 +488,6 @@ def list_jobs(
|
|||
page_size: int = 20,
|
||||
status: str = "",
|
||||
keyword: str = "",
|
||||
table_filters: TableFilterParams | None = None,
|
||||
) -> dict:
|
||||
page = max(1, int(page or 1))
|
||||
page_size = max(1, min(100, int(page_size or 20)))
|
||||
|
|
@ -509,15 +506,6 @@ def list_jobs(
|
|||
TopoDiscoverJob.trigger_mode.ilike(like),
|
||||
)
|
||||
)
|
||||
job = TopoDiscoverJob
|
||||
q, options = filter_query(q, {
|
||||
"id": job.id, "trigger": job.trigger_mode, "scope": job.scope, "status": job.status,
|
||||
"progress": cast(job.done, String) + "/" + cast(job.total, String),
|
||||
"edges": cast(job.edges_added, String) + "/" + cast(job.edges_updated, String),
|
||||
"missingDelta": job.edges_stale, "started": job.started_at, "ended": job.ended_at,
|
||||
}, table_filters)
|
||||
if options is not None:
|
||||
return options
|
||||
q = q.order_by(TopoDiscoverJob.created_at.desc())
|
||||
total = int(q.count())
|
||||
rows = q.offset((page - 1) * page_size).limit(page_size).all()
|
||||
|
|
@ -544,14 +532,7 @@ def get_job_detail(
|
|||
page_size: int | None = None,
|
||||
item_status: str = "",
|
||||
item_keyword: str = "",
|
||||
table_filters: TableFilterParams | None = None,
|
||||
) -> dict:
|
||||
if table_filters and table_filters.field:
|
||||
if db.get(TopoDiscoverJob, job_id) is None:
|
||||
raise HTTPException(status_code=404, detail="discover_job_not_found")
|
||||
q = db.query(TopoDiscoverJobItem).filter(TopoDiscoverJobItem.job_id == job_id)
|
||||
_, options = filter_query(q, discover_item_expressions(), table_filters)
|
||||
return options
|
||||
return get_discover_job(
|
||||
db,
|
||||
job_id,
|
||||
|
|
@ -559,5 +540,4 @@ def get_job_detail(
|
|||
page_size=page_size,
|
||||
item_status=item_status,
|
||||
item_keyword=item_keyword,
|
||||
table_filters=table_filters,
|
||||
).model_dump()
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ from .ne_service import (
|
|||
update_managed_ne,
|
||||
)
|
||||
from .models import ManagedNE
|
||||
from .table_filters import TableFilterParams, table_filter_params
|
||||
|
||||
router = APIRouter(prefix="/v1/managed-ne", tags=["managed-ne"])
|
||||
|
||||
|
|
@ -64,7 +63,6 @@ def api_list_managed_ne(
|
|||
connect_status: str | None = Query(default=None),
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=50, ge=1, le=500),
|
||||
table_filters: TableFilterParams = Depends(table_filter_params),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return list_managed_ne(
|
||||
|
|
@ -74,7 +72,6 @@ def api_list_managed_ne(
|
|||
connect_status=connect_status,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
table_filters=table_filters,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -21,8 +21,6 @@ class AppUser(Base):
|
|||
role: Mapped[str] = mapped_column(String(32), default="user", index=True) # admin | user
|
||||
# Optional capability override; empty => role defaults (see auth_scopes).
|
||||
scopes: Mapped[list] = mapped_column(_JsonType, default=list)
|
||||
# Per-user UI preferences (e.g. workbench card layout). Pure client prefs; not audited.
|
||||
prefs: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, index=True)
|
||||
must_change_password: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
created_by: Mapped[str] = mapped_column(String(64), default="")
|
||||
|
|
|
|||
|
|
@ -32,9 +32,6 @@ class BizStateTask(Base):
|
|||
note: Mapped[str] = mapped_column(String(256), default="")
|
||||
# "" | portrait | cutover_hf — filter cutover HF in biz-state list
|
||||
purpose: Mapped[str] = mapped_column(String(32), default="", index=True)
|
||||
# Optional paired sampling; ordinary per-device tasks remain independent.
|
||||
collect_group_id: Mapped[str] = mapped_column(String(128), default="", index=True)
|
||||
collect_manual_only: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|running|paused|stopped
|
||||
interval_sec: Mapped[int] = mapped_column(Integer, default=300)
|
||||
# Keep snapshots for N calendar days (protected baselines/refs never auto-deleted)
|
||||
|
|
@ -89,10 +86,6 @@ class BizStateBatch(Base):
|
|||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True, default=lambda: uuid4().hex)
|
||||
task_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
||||
collect_group_id: Mapped[str] = mapped_column(String(128), default="", index=True)
|
||||
collect_round_id: Mapped[str] = mapped_column(String(64), default="", index=True)
|
||||
collect_priority: Mapped[int] = mapped_column(Integer, default=0)
|
||||
queued_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="managed", index=True)
|
||||
ne_id: Mapped[str] = mapped_column(String(128), default="", index=True)
|
||||
ne_name: Mapped[str] = mapped_column(String(256), default="")
|
||||
|
|
@ -310,8 +303,6 @@ class BizCompareJob(Base):
|
|||
status: Mapped[str] = mapped_column(String(32), default="draft", index=True) # draft|ready|auto
|
||||
# Empty = all template sheets; non-empty = only these sheet_id values
|
||||
enabled_sheet_ids: Mapped[list] = mapped_column(_JsonType, default=list)
|
||||
# auto|always|never|sample — how to persist matching (success) rows
|
||||
store_unchanged: Mapped[str] = mapped_column(String(16), default="auto")
|
||||
note: Mapped[str] = mapped_column(String(512), default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow_naive)
|
||||
|
|
@ -356,7 +347,4 @@ class BizCompareDiff(Base):
|
|||
after_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
mapped_before_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
changes_json: Mapped[dict] = mapped_column(_JsonType, default=dict)
|
||||
# Pointers into BizStateMetricRow / LLDP tables for compact success hydrate
|
||||
before_row_id: Mapped[str] = mapped_column(String(64), default="")
|
||||
after_row_id: Mapped[str] = mapped_column(String(64), default="")
|
||||
search_text: Mapped[str] = mapped_column(Text, default="")
|
||||
|
|
|
|||
|
|
@ -10,9 +10,7 @@ from netmiko import ConnectHandler
|
|||
_CLI_HOP_NESTED_END_RE = re.compile(
|
||||
r"(?is)"
|
||||
r"(?:^|\n)\s*(?:"
|
||||
# Huawei VRP: "Info: The connection was closed by the remote host."
|
||||
r"(?:info:\s*)?(?:the\s+)?connection\s+was\s+closed\s+by\s+(?:the\s+)?(?:foreign|remote)\s+host"
|
||||
r"|connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?"
|
||||
r"connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?"
|
||||
r"|closed\s+by\s+foreign\s+host"
|
||||
r"|connection\s+to\s+\S+\s+closed"
|
||||
r"|%\s*connection\s+closed(?:\s+by\s+(?:foreign|remote)\s+host)?"
|
||||
|
|
|
|||
|
|
@ -1,73 +1,18 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from .config import settings
|
||||
|
||||
_log = logging.getLogger("netx.crypto")
|
||||
_cached_credential_key: str | None = None
|
||||
|
||||
|
||||
class CredentialCryptoError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def credential_secret_file_path() -> Path:
|
||||
raw = str(getattr(settings, "credential_secret_file", None) or "data/auth/credential_secret").strip()
|
||||
path = Path(raw)
|
||||
if not path.is_absolute():
|
||||
path = Path.cwd() / path
|
||||
return path
|
||||
|
||||
|
||||
def ensure_credential_secret_key() -> str:
|
||||
"""Resolve Fernet key: env ``NETX_CREDENTIAL_SECRET_KEY`` > file > generate once.
|
||||
|
||||
Packaged installs should also write the key into ``.env`` via setup_first_run;
|
||||
this startup/path fallback covers upgrades and missed first-run keys.
|
||||
"""
|
||||
global _cached_credential_key
|
||||
configured = str(settings.credential_secret_key or "").strip()
|
||||
if configured:
|
||||
return configured
|
||||
if _cached_credential_key:
|
||||
return _cached_credential_key
|
||||
|
||||
path = credential_secret_file_path()
|
||||
try:
|
||||
if path.is_file():
|
||||
existing = path.read_text(encoding="utf-8").strip()
|
||||
if existing:
|
||||
_cached_credential_key = existing
|
||||
settings.credential_secret_key = existing
|
||||
return existing
|
||||
except Exception:
|
||||
_log.exception("read credential secret file failed path=%s", path)
|
||||
|
||||
generated = Fernet.generate_key().decode("ascii")
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(generated + "\n", encoding="utf-8")
|
||||
try:
|
||||
path.chmod(0o600)
|
||||
except Exception:
|
||||
pass
|
||||
_log.info("wrote per-install credential Fernet key to %s", path)
|
||||
except Exception:
|
||||
_log.exception(
|
||||
"write credential secret file failed path=%s; using in-memory key only",
|
||||
path,
|
||||
)
|
||||
_cached_credential_key = generated
|
||||
settings.credential_secret_key = generated
|
||||
return generated
|
||||
|
||||
|
||||
def _fernet() -> Fernet:
|
||||
key = ensure_credential_secret_key()
|
||||
key = str(settings.credential_secret_key or "").strip()
|
||||
if not key:
|
||||
raise CredentialCryptoError("credential_secret_key_not_configured")
|
||||
try:
|
||||
return Fernet(key.encode("ascii"))
|
||||
except Exception as exc:
|
||||
|
|
@ -92,7 +37,4 @@ def decrypt_secret(value: str) -> str:
|
|||
|
||||
|
||||
def credentials_configured() -> bool:
|
||||
try:
|
||||
return bool(ensure_credential_secret_key())
|
||||
except Exception:
|
||||
return False
|
||||
return bool(str(settings.credential_secret_key or "").strip())
|
||||
|
|
|
|||
|
|
@ -47,9 +47,6 @@ def paging_disable_commands(*, vendor: str = "", device_type: str = "") -> list[
|
|||
dt = str(device_type or "").strip().lower()
|
||||
v = str(vendor or "").strip().lower()
|
||||
blob = f"{dt} {v}"
|
||||
# RouterOS has no pager; Cisco's command leaves an extra error/prompt redraw.
|
||||
if "mikrotik" in blob or "routeros" in blob or "switchos" in blob:
|
||||
return []
|
||||
if "huawei" in blob or "华为" in v:
|
||||
return ["screen-length 0 temporary"]
|
||||
if "hp_comware" in blob or "h3c" in blob or "comware" in blob:
|
||||
|
|
@ -137,23 +134,6 @@ def drain_read_channel(
|
|||
|
||||
|
||||
def _send_command_expect_prompt(conn: Any, cmd: str, *, read_timeout: int) -> str:
|
||||
device_type = getattr(conn, "device_type", "")
|
||||
if isinstance(device_type, str) and device_type.lower().startswith("mikrotik"):
|
||||
# find_prompt sends an extra Enter *after* our channel drain. RouterOS
|
||||
# can repaint that old prompt after send_command starts, ending the read
|
||||
# before the command's output arrives. Use its complete prompt directly.
|
||||
# Keep echo verification off: long RouterOS scripts wrap at PTY width.
|
||||
prompt_pattern = r"(?m)^\[[^\r\n\]]+\](?:[ \t]+/[^\r\n>]*)?[ \t]*>[ \t]*$"
|
||||
return str(
|
||||
conn.send_command(
|
||||
command_string=cmd,
|
||||
read_timeout=read_timeout,
|
||||
auto_find_prompt=False,
|
||||
expect_string=prompt_pattern,
|
||||
cmd_verify=False,
|
||||
)
|
||||
or ""
|
||||
)
|
||||
try:
|
||||
return str(
|
||||
conn.send_command(
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class ManagedNeCreate(BaseModel):
|
|||
vendor: str
|
||||
device_type: str
|
||||
ip_address: str
|
||||
port: int = Field(default=22, ge=1, le=65535)
|
||||
port: int = 22
|
||||
protocol: str = "ssh"
|
||||
username: str
|
||||
password: str = ""
|
||||
|
|
@ -27,7 +27,7 @@ class ManagedNeCreate(BaseModel):
|
|||
hop_enabled: bool = False
|
||||
hop_vendor: str = "zte"
|
||||
hop_host: str = ""
|
||||
hop_port: int = Field(default=22, ge=1, le=65535)
|
||||
hop_port: int = 22
|
||||
hop_protocol: str = "ssh"
|
||||
hop_username: str = ""
|
||||
hop_password: str = ""
|
||||
|
|
@ -63,7 +63,7 @@ class ManagedNeUpdate(BaseModel):
|
|||
vendor: str | None = None
|
||||
device_type: str | None = None
|
||||
ip_address: str | None = None
|
||||
port: int | None = Field(default=None, ge=1, le=65535)
|
||||
port: int | None = None
|
||||
protocol: str | None = None
|
||||
username: str | None = None
|
||||
password: str | None = None
|
||||
|
|
@ -73,7 +73,7 @@ class ManagedNeUpdate(BaseModel):
|
|||
hop_enabled: bool | None = None
|
||||
hop_vendor: str | None = None
|
||||
hop_host: str | None = None
|
||||
hop_port: int | None = Field(default=None, ge=1, le=65535)
|
||||
hop_port: int | None = None
|
||||
hop_protocol: str | None = None
|
||||
hop_username: str | None = None
|
||||
hop_password: str | None = None
|
||||
|
|
@ -194,7 +194,7 @@ class HopProxyConfig(BaseModel):
|
|||
|
||||
hop_vendor: str = "zte"
|
||||
hop_host: str
|
||||
hop_port: int = Field(default=22, ge=1, le=65535)
|
||||
hop_port: int = 22
|
||||
hop_protocol: str = "ssh"
|
||||
hop_username: str
|
||||
hop_password: str = ""
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
import math
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -60,63 +60,6 @@ def _normalize_ip(ip: str) -> str:
|
|||
return str(ip or "").strip()
|
||||
|
||||
|
||||
_HOSTNAME_RE = re.compile(r"^(?=.{1,253}$)[A-Za-z0-9_](?:[A-Za-z0-9_-]{0,62})(?:\.[A-Za-z0-9_](?:[A-Za-z0-9_-]{0,62}))*\.?$")
|
||||
|
||||
|
||||
def _is_valid_mgmt_address(ip: str) -> bool:
|
||||
"""Accept an IPv4/IPv6 literal or a DNS hostname (no spaces, ports or schemes)."""
|
||||
import ipaddress
|
||||
|
||||
text = str(ip or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
try:
|
||||
ipaddress.ip_address(text)
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
# Dotted-quad lookalikes (e.g. 10.0.0.300) are typos, not hostnames.
|
||||
if re.fullmatch(r"[0-9.]+", text):
|
||||
return False
|
||||
return bool(_HOSTNAME_RE.match(text))
|
||||
|
||||
|
||||
def _require_mgmt_address(ip: str) -> str:
|
||||
text = _normalize_ip(ip)
|
||||
if not text:
|
||||
raise HTTPException(status_code=400, detail="ip_address_required")
|
||||
if not _is_valid_mgmt_address(text):
|
||||
raise HTTPException(status_code=400, detail="invalid_ip_address")
|
||||
return text
|
||||
|
||||
|
||||
CONNECT_FIELDS = (
|
||||
"ip_address", "port", "protocol", "username", "password_enc", "device_type",
|
||||
"hop_enabled", "hop_vendor", "hop_host", "hop_port", "hop_protocol", "hop_username",
|
||||
"hop_password_enc", "hop_command_template", "hop_vrf", "hop_target_auth_mode",
|
||||
"hop_enter_system_view",
|
||||
)
|
||||
|
||||
|
||||
def connect_snapshot(row: ManagedNE) -> tuple[Any, ...]:
|
||||
return tuple(getattr(row, f, None) for f in CONNECT_FIELDS)
|
||||
|
||||
|
||||
def reset_connect_status_if_changed(row: ManagedNE, before: tuple[Any, ...]) -> None:
|
||||
"""A previous pass/fail no longer describes the device once login settings change."""
|
||||
if connect_snapshot(row) != before and str(row.connect_status or "") != "unknown":
|
||||
row.connect_status = "unknown"
|
||||
row.connect_message = ""
|
||||
|
||||
|
||||
def hop_password_optional(hop_vendor: str, hop_target_auth_mode: str) -> bool:
|
||||
"""Only a bastion that manages target auth may run without a stored hop password."""
|
||||
return (
|
||||
_normalize_hop_vendor(hop_vendor) == "bastion"
|
||||
and _normalize_hop_target_auth_mode(hop_target_auth_mode) == "bastion_managed"
|
||||
)
|
||||
|
||||
|
||||
def _normalize_protocol(protocol: str) -> str:
|
||||
p = str(protocol or "ssh").strip().lower()
|
||||
return p if p in ("ssh", "telnet") else "ssh"
|
||||
|
|
@ -199,7 +142,7 @@ def _parse_import_bool(value: Any) -> bool:
|
|||
|
||||
|
||||
def _import_cell_str(value: Any) -> str:
|
||||
if value is None or (isinstance(value, float) and math.isnan(value)):
|
||||
if value is None or (isinstance(value, float) and pd.isna(value)):
|
||||
return ""
|
||||
text = str(value).strip()
|
||||
return "" if text.lower() == "nan" else text
|
||||
|
|
@ -270,8 +213,9 @@ def _apply_hop_update(row: ManagedNE, data: dict[str, Any]) -> None:
|
|||
raise HTTPException(status_code=400, detail="hop_host_required")
|
||||
if not str(row.hop_username or "").strip():
|
||||
raise HTTPException(status_code=400, detail="hop_username_required")
|
||||
if not str(row.hop_password_enc or "").strip() and not hop_password_optional(
|
||||
str(row.hop_vendor or ""), str(row.hop_target_auth_mode or "")
|
||||
if (
|
||||
not str(row.hop_password_enc or "").strip()
|
||||
and _normalize_hop_target_auth_mode(row.hop_target_auth_mode) != "bastion_managed"
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="hop_password_required")
|
||||
|
||||
|
|
|
|||
|
|
@ -5,10 +5,10 @@ from typing import Any
|
|||
from uuid import uuid4
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import false, func, literal, or_
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .device_types import SUPPORTED_VENDORS, WEBCRT_DEVICE_TYPES, is_placeholder_ne_source
|
||||
from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS, WEBCRT_DEVICE_TYPES, is_placeholder_ne_source
|
||||
from .models import ManagedNE
|
||||
from .ne_crypto import encrypt_secret
|
||||
from .ne_hop_templates import default_bastion_username_template, default_hop_command_template
|
||||
|
|
@ -24,11 +24,8 @@ from .ne_service_common import (
|
|||
_apply_hop_update,
|
||||
_normalize_hop_target_auth_mode,
|
||||
_normalize_hop_vendor,
|
||||
_normalize_ip,
|
||||
_normalize_protocol,
|
||||
_require_mgmt_address,
|
||||
connect_snapshot,
|
||||
hop_password_optional,
|
||||
reset_connect_status_if_changed,
|
||||
_normalize_saved_hop_endpoint,
|
||||
_normalize_vendor,
|
||||
_now,
|
||||
|
|
@ -36,43 +33,16 @@ from .ne_service_common import (
|
|||
_validate_hop_on_create,
|
||||
row_to_out,
|
||||
)
|
||||
from .table_filters import TableFilterParams, filter_query
|
||||
from .ne_exec_guard import (
|
||||
EXEC_POLICY_READONLY,
|
||||
allows_open_exec_policy,
|
||||
require_exec_policy_writable,
|
||||
)
|
||||
|
||||
# Create and update share the form's device-type list, which includes linux/generic
|
||||
# (Linux hosts and WebCRT/LLDP placeholders promoted to inventory).
|
||||
# Inventory create stays strict; updates must also accept WebCRT/LLDP placeholder types
|
||||
# (generic/linux) so operators can open the form and promote them to zte_zxros etc.
|
||||
_ALLOWED_UPDATE_DEVICE_TYPES = frozenset(WEBCRT_DEVICE_TYPES)
|
||||
|
||||
def managed_ne_filter_expressions() -> dict[str, Any]:
|
||||
"""Column filter fields for the NE management table (tags handled separately)."""
|
||||
return {
|
||||
"name": func.coalesce(func.nullif(ManagedNE.name, ""), ManagedNE.ip_address),
|
||||
"vendor": ManagedNE.vendor,
|
||||
"deviceType": ManagedNE.device_type,
|
||||
"source": func.coalesce(func.nullif(ManagedNE.source, ""), "manual"),
|
||||
"ip": ManagedNE.ip_address,
|
||||
"username": ManagedNE.username,
|
||||
"connectStatus": ManagedNE.connect_status,
|
||||
}
|
||||
|
||||
|
||||
def _filter_by_tags(stmt: Any, value: dict[str, Any]) -> Any:
|
||||
"""Tags are space-separated; a row matches when any selected tag is present ("" = untagged)."""
|
||||
tags_col = func.coalesce(ManagedNE.tags, "")
|
||||
padded = literal(" ") + tags_col + literal(" ")
|
||||
conds = []
|
||||
for tag in value["values"]:
|
||||
tag = str(tag).strip()
|
||||
escaped = tag.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
conds.append(func.trim(tags_col) == "" if not tag else padded.like(f"% {escaped} %", escape="\\"))
|
||||
matched = or_(*conds) if conds else false()
|
||||
return stmt.filter(matched if value["mode"] == "include" else ~matched)
|
||||
|
||||
|
||||
def list_managed_ne(
|
||||
db: Session,
|
||||
*,
|
||||
|
|
@ -81,7 +51,6 @@ def list_managed_ne(
|
|||
connect_status: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
table_filters: TableFilterParams | None = None,
|
||||
) -> dict[str, Any]:
|
||||
stmt = db.query(ManagedNE)
|
||||
kw = str(keyword or "").strip()
|
||||
|
|
@ -103,22 +72,6 @@ def list_managed_ne(
|
|||
stmt = stmt.filter(ManagedNE.vendor == v)
|
||||
if cs:
|
||||
stmt = stmt.filter(ManagedNE.connect_status == cs)
|
||||
params = table_filters or TableFilterParams()
|
||||
tags_filter = params.filters.get("tags")
|
||||
if tags_filter and params.field != "tags":
|
||||
stmt = _filter_by_tags(stmt, tags_filter)
|
||||
if params.field == "tags":
|
||||
stmt, _ = filter_query(stmt, managed_ne_filter_expressions(), TableFilterParams(
|
||||
{k: v for k, v in params.filters.items() if k != "tags"}))
|
||||
options: set[str] = set()
|
||||
for (tags_val,) in stmt.with_entities(ManagedNE.tags).all():
|
||||
parts = str(tags_val or "").split()
|
||||
options.update(parts if parts else [""])
|
||||
return {"field": "tags", "options": sorted(options)}
|
||||
stmt, options = filter_query(stmt, managed_ne_filter_expressions(), TableFilterParams(
|
||||
{k: v for k, v in params.filters.items() if k != "tags"}, params.field))
|
||||
if options is not None:
|
||||
return options
|
||||
total = int(stmt.count())
|
||||
rows = (
|
||||
stmt.order_by(ManagedNE.updated_at.desc())
|
||||
|
|
@ -144,8 +97,10 @@ def get_managed_ne(db: Session, ne_id: str) -> ManagedNeOut:
|
|||
def create_managed_ne(db: Session, body: ManagedNeCreate) -> ManagedNeOut:
|
||||
_require_crypto()
|
||||
_validate_hop_on_create(body)
|
||||
ip = _require_mgmt_address(body.ip_address)
|
||||
if body.device_type not in _ALLOWED_UPDATE_DEVICE_TYPES:
|
||||
ip = _normalize_ip(body.ip_address)
|
||||
if not ip:
|
||||
raise HTTPException(status_code=400, detail="ip_address_required")
|
||||
if body.device_type not in SUPPORTED_DEVICE_TYPES:
|
||||
raise HTTPException(status_code=400, detail="unsupported_device_type")
|
||||
existing = db.query(ManagedNE).filter(ManagedNE.ip_address == ip).first()
|
||||
if existing:
|
||||
|
|
@ -185,9 +140,10 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed
|
|||
if not row:
|
||||
raise HTTPException(status_code=404, detail="managed_ne_not_found")
|
||||
data = body.model_dump(exclude_unset=True)
|
||||
before = connect_snapshot(row)
|
||||
if "ip_address" in data:
|
||||
ip = _require_mgmt_address(data["ip_address"])
|
||||
ip = _normalize_ip(data["ip_address"])
|
||||
if not ip:
|
||||
raise HTTPException(status_code=400, detail="ip_address_required")
|
||||
other = db.query(ManagedNE).filter(ManagedNE.ip_address == ip, ManagedNE.id != ne_id).first()
|
||||
if other:
|
||||
raise HTTPException(status_code=400, detail="ip_address_exists")
|
||||
|
|
@ -201,7 +157,7 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed
|
|||
v = str(data["vendor"] or "").strip()
|
||||
row.vendor = v if v in SUPPORTED_VENDORS else "Other"
|
||||
if "name" in data:
|
||||
row.name = str(data["name"] or "").strip() or str(row.ip_address or "")
|
||||
row.name = str(data["name"] or "").strip()
|
||||
if "port" in data and data["port"] is not None:
|
||||
row.port = int(data["port"])
|
||||
if "protocol" in data and data["protocol"] is not None:
|
||||
|
|
@ -242,7 +198,6 @@ def update_managed_ne(db: Session, ne_id: str, body: ManagedNeUpdate) -> Managed
|
|||
if "ip_address" in data and str(row.ip_address or "").strip():
|
||||
if is_placeholder_ne_source(row.source):
|
||||
row.source = ""
|
||||
reset_connect_status_if_changed(row, before)
|
||||
row.updated_at = _now()
|
||||
# Keep linked fabric node identity in sync (name / IP / vendor).
|
||||
if any(k in data for k in ("ip_address", "name", "vendor", "device_type")):
|
||||
|
|
@ -270,6 +225,8 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d
|
|||
if not hop_user:
|
||||
raise HTTPException(status_code=400, detail="hop_username_required")
|
||||
hop_auth_mode = _normalize_hop_target_auth_mode(hop.hop_target_auth_mode)
|
||||
if not hop_pass and hop_auth_mode != "bastion_managed":
|
||||
raise HTTPException(status_code=400, detail="hop_password_required")
|
||||
|
||||
template = str(hop.hop_command_template or "").strip()
|
||||
if hop_vendor == "bastion" and not template:
|
||||
|
|
@ -286,15 +243,9 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d
|
|||
missing = [x for x in ne_ids if x not in found_ids]
|
||||
if missing:
|
||||
raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing[:5])}")
|
||||
# Blank password keeps each NE's stored hop password; NEs without one cannot hop.
|
||||
if not hop_pass and not hop_password_optional(hop_vendor, hop_auth_mode):
|
||||
lacking = [str(r.name or r.ip_address) for r in rows if not str(r.hop_password_enc or "").strip()]
|
||||
if lacking:
|
||||
raise HTTPException(status_code=400, detail=f"hop_password_required: {','.join(lacking[:5])}")
|
||||
|
||||
now = _now()
|
||||
for row in rows:
|
||||
before = connect_snapshot(row)
|
||||
row.hop_enabled = True
|
||||
row.hop_vendor = hop_vendor
|
||||
row.hop_host = hop_host
|
||||
|
|
@ -307,7 +258,6 @@ def batch_apply_hop_proxy(db: Session, ids: list[str], hop: HopProxyConfig) -> d
|
|||
row.hop_vrf = str(hop.hop_vrf or "").strip()
|
||||
row.hop_target_auth_mode = hop_auth_mode
|
||||
row.hop_enter_system_view = bool(getattr(hop, "hop_enter_system_view", False))
|
||||
reset_connect_status_if_changed(row, before)
|
||||
row.updated_at = now
|
||||
db.commit()
|
||||
return {"ok": True, "updated": len(rows)}
|
||||
|
|
@ -333,12 +283,10 @@ def batch_apply_account(db: Session, ids: list[str], account: BatchAccountConfig
|
|||
raise HTTPException(status_code=404, detail=f"managed_ne_not_found: {','.join(missing[:5])}")
|
||||
now = _now()
|
||||
for row in rows:
|
||||
before = connect_snapshot(row)
|
||||
if user:
|
||||
row.username = user
|
||||
if pwd:
|
||||
row.password_enc = pwd_enc
|
||||
reset_connect_status_if_changed(row, before)
|
||||
row.updated_at = now
|
||||
db.commit()
|
||||
return {"ok": True, "updated": len(rows)}
|
||||
|
|
|
|||
|
|
@ -1,15 +1,16 @@
|
|||
"""Managed NE Excel import and UME inventory sync into managed_ne."""
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .models import ManagedNE, UmeInventoryNE
|
||||
from .tabular import read_table, write_csv, write_xlsx
|
||||
from .ne_crypto import CredentialCryptoError, encrypt_secret
|
||||
from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS, is_placeholder_ne_source
|
||||
from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS
|
||||
from .ne_schemas import (
|
||||
ImportFailure,
|
||||
ImportResult,
|
||||
|
|
@ -22,9 +23,6 @@ from .ne_service_common import (
|
|||
UME_SYNC_TAG,
|
||||
_import_cell_str,
|
||||
_infer_managed_ne_type_vendor,
|
||||
_is_valid_mgmt_address,
|
||||
connect_snapshot,
|
||||
reset_connect_status_if_changed,
|
||||
_merge_tags,
|
||||
_normalize_ip,
|
||||
_normalize_protocol,
|
||||
|
|
@ -150,74 +148,66 @@ def build_managed_ne_import_template(fmt: str = "xlsx") -> tuple[str, bytes, str
|
|||
"remark": "no direct password, use batch proxy",
|
||||
},
|
||||
]
|
||||
headers = list(IMPORT_COLUMNS)
|
||||
table = [[r.get(c, "") for c in headers] for r in rows]
|
||||
df = pd.DataFrame(rows, columns=list(IMPORT_COLUMNS))
|
||||
buf = BytesIO()
|
||||
kind = str(fmt or "xlsx").strip().lower()
|
||||
if kind == "csv":
|
||||
df.to_csv(buf, index=False, encoding="utf-8-sig")
|
||||
return (
|
||||
"managed_ne_import_template.csv",
|
||||
write_csv(headers, table),
|
||||
buf.getvalue(),
|
||||
"text/csv; charset=utf-8",
|
||||
)
|
||||
payload = write_xlsx([
|
||||
("import", headers, table),
|
||||
("device_type", ["device_type"], [[t] for t in SUPPORTED_DEVICE_TYPES]),
|
||||
])
|
||||
device_types_df = pd.DataFrame({"device_type": list(SUPPORTED_DEVICE_TYPES)})
|
||||
with pd.ExcelWriter(buf, engine="openpyxl") as writer:
|
||||
df.to_excel(writer, sheet_name="import", index=False)
|
||||
device_types_df.to_excel(writer, sheet_name="device_type", index=False)
|
||||
return (
|
||||
"managed_ne_import_template.xlsx",
|
||||
payload,
|
||||
buf.getvalue(),
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
)
|
||||
|
||||
|
||||
def import_managed_ne(db: Session, content: bytes, filename: str) -> ImportResult:
|
||||
_require_crypto()
|
||||
name = str(filename or "").lower()
|
||||
try:
|
||||
headers, raw_rows = read_table(content, filename)
|
||||
if name.endswith(".csv"):
|
||||
df = pd.read_csv(BytesIO(content))
|
||||
else:
|
||||
df = pd.read_excel(BytesIO(content))
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=f"import_parse_failed: {exc}") from exc
|
||||
columns = [h.strip().lower() for h in headers]
|
||||
missing = [c for c in _REQUIRED_IMPORT_COLUMNS if c not in columns]
|
||||
df.columns = [str(c).strip().lower() for c in df.columns]
|
||||
missing = [c for c in _REQUIRED_IMPORT_COLUMNS if c not in df.columns]
|
||||
if missing:
|
||||
raise HTTPException(status_code=400, detail=f"import_missing_columns: {','.join(missing)}")
|
||||
inserted = 0
|
||||
updated = 0
|
||||
failed: list[ImportFailure] = []
|
||||
for row_no, cells in raw_rows:
|
||||
row = {k.strip().lower(): v for k, v in cells.items()}
|
||||
# Savepoint per row: a failing row must not leave a half-written NE behind.
|
||||
savepoint = db.begin_nested()
|
||||
for idx, row in df.iterrows():
|
||||
row_no = int(idx) + 2
|
||||
try:
|
||||
ip = _normalize_ip(_import_cell_str(row.get("ip", "")))
|
||||
if not ip:
|
||||
failed.append(ImportFailure(row=row_no, reason="ip_required"))
|
||||
savepoint.rollback()
|
||||
continue
|
||||
if not _is_valid_mgmt_address(ip):
|
||||
failed.append(ImportFailure(row=row_no, reason="invalid_ip_address"))
|
||||
savepoint.rollback()
|
||||
continue
|
||||
device_type = _import_cell_str(row.get("device_type", ""))
|
||||
if device_type not in SUPPORTED_DEVICE_TYPES:
|
||||
failed.append(ImportFailure(row=row_no, reason="unsupported_device_type"))
|
||||
savepoint.rollback()
|
||||
continue
|
||||
username = _import_cell_str(row.get("username", ""))
|
||||
password = _import_cell_str(row.get("password", ""))
|
||||
if not username:
|
||||
failed.append(ImportFailure(row=row_no, reason="username_required"))
|
||||
savepoint.rollback()
|
||||
continue
|
||||
port_raw = row.get("port")
|
||||
port_raw = row.get("port", 22)
|
||||
try:
|
||||
port = int(float(port_raw)) if port_raw is not None else 22
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
port = int(port_raw)
|
||||
except (TypeError, ValueError):
|
||||
port = 22
|
||||
if not 1 <= port <= 65535:
|
||||
failed.append(ImportFailure(row=row_no, reason="invalid_port"))
|
||||
savepoint.rollback()
|
||||
continue
|
||||
protocol = _normalize_protocol(_import_cell_str(row.get("protocol")) or "ssh")
|
||||
protocol = _normalize_protocol(str(row.get("protocol", "ssh")))
|
||||
display_name = _import_cell_str(row.get("name", "")) or ip
|
||||
vendor_raw = _import_cell_str(row.get("vendor", "")) or "Other"
|
||||
vendor = "Other"
|
||||
|
|
@ -227,43 +217,29 @@ def import_managed_ne(db: Session, content: bytes, filename: str) -> ImportResul
|
|||
break
|
||||
existing = db.query(ManagedNE).filter(ManagedNE.ip_address == ip).first()
|
||||
now = _now()
|
||||
is_new = existing is None
|
||||
if existing is None:
|
||||
existing = ManagedNE(ip_address=ip, created_at=now, connect_status="unknown")
|
||||
existing = ManagedNE(ip_address=ip, created_at=now)
|
||||
db.add(existing)
|
||||
before = connect_snapshot(existing)
|
||||
inserted += 1
|
||||
else:
|
||||
updated += 1
|
||||
existing.name = display_name
|
||||
existing.vendor = vendor
|
||||
existing.device_type = device_type
|
||||
existing.port = port
|
||||
existing.protocol = protocol
|
||||
existing.username = username
|
||||
# Blank password on an existing NE keeps the stored one (template row 2 is blank).
|
||||
if password:
|
||||
existing.password_enc = encrypt_secret(password)
|
||||
elif is_new:
|
||||
existing.password_enc = ""
|
||||
existing.password_enc = encrypt_secret(password) if password else ""
|
||||
tags_val = _import_cell_str(row.get("tags", ""))
|
||||
remark_val = _import_cell_str(row.get("remark", ""))
|
||||
if tags_val:
|
||||
existing.tags = tags_val
|
||||
if remark_val:
|
||||
existing.remark = remark_val
|
||||
if is_placeholder_ne_source(existing.source):
|
||||
existing.source = ""
|
||||
if not is_new:
|
||||
reset_connect_status_if_changed(existing, before)
|
||||
existing.updated_at = now
|
||||
savepoint.commit()
|
||||
if is_new:
|
||||
inserted += 1
|
||||
else:
|
||||
updated += 1
|
||||
except CredentialCryptoError as exc:
|
||||
savepoint.rollback()
|
||||
failed.append(ImportFailure(row=row_no, reason=str(exc)))
|
||||
except Exception as exc:
|
||||
savepoint.rollback()
|
||||
failed.append(ImportFailure(row=row_no, reason=str(exc)[:200]))
|
||||
db.commit()
|
||||
return ImportResult(inserted=inserted, updated=updated, failed=failed)
|
||||
|
|
|
|||
|
|
@ -545,26 +545,18 @@ def _connect_direct(
|
|||
|
||||
|
||||
def _read_channel(conn: ConnectHandler, wait: float = 0.5, max_loops: int = 40) -> str:
|
||||
"""Collect device output.
|
||||
|
||||
``wait`` is the longest we park for the *first* bytes; once output has arrived the read
|
||||
returns as soon as the channel is quiet again. (It used to sleep the whole ``wait`` and
|
||||
then poll at 50-150ms, which added ~0.1-0.4s of dead time to every login-prompt step.)
|
||||
"""
|
||||
quiet = 0.04 if wait <= 0.15 else 0.12
|
||||
first_deadline = time.monotonic() + max(0.0, wait)
|
||||
got_at: float | None = None
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
chunks: list[str] = []
|
||||
reads = 0
|
||||
while reads < max_loops:
|
||||
for _ in range(max_loops):
|
||||
try:
|
||||
part = conn.read_channel()
|
||||
except Exception:
|
||||
break
|
||||
if part is None or part is False:
|
||||
part = ""
|
||||
break
|
||||
# MagicMock truthy-but-empty: stop when unit tests leave read_channel unconfigured.
|
||||
elif not isinstance(part, (str, bytes, bytearray)):
|
||||
if not isinstance(part, (str, bytes, bytearray)):
|
||||
text = str(part)
|
||||
# unittest.mock default str looks like "<MagicMock name=...>"
|
||||
if text.startswith("<MagicMock") or text.startswith("<Mock"):
|
||||
|
|
@ -572,18 +564,10 @@ def _read_channel(conn: ConnectHandler, wait: float = 0.5, max_loops: int = 40)
|
|||
part = text
|
||||
elif isinstance(part, (bytes, bytearray)):
|
||||
part = bytes(part).decode("utf-8", errors="replace")
|
||||
now = time.monotonic()
|
||||
if part:
|
||||
if not part:
|
||||
break
|
||||
chunks.append(part)
|
||||
reads += 1
|
||||
got_at = now
|
||||
continue
|
||||
if got_at is None:
|
||||
if now >= first_deadline:
|
||||
break
|
||||
elif now - got_at >= quiet:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
time.sleep(0.05 if wait <= 0.15 else 0.15)
|
||||
return "".join(chunks)
|
||||
|
||||
|
||||
|
|
@ -817,11 +801,11 @@ def _interactive_target_auth(
|
|||
return
|
||||
else:
|
||||
empty_after_pass = 0
|
||||
time.sleep(0.02)
|
||||
time.sleep(0.12)
|
||||
continue
|
||||
|
||||
if not buf.strip():
|
||||
time.sleep(0.02)
|
||||
time.sleep(0.2)
|
||||
continue
|
||||
# Huawei stelnet often reprints the hop ``[sysname]`` after host-key trust and
|
||||
# *before* ``Enter password:``. Do not treat that as target login success.
|
||||
|
|
@ -842,7 +826,7 @@ def _interactive_target_auth(
|
|||
if denied:
|
||||
raise paramiko.AuthenticationException(f"target_auth_rejected: {denied}")
|
||||
return
|
||||
time.sleep(0.02)
|
||||
time.sleep(0.15)
|
||||
denied = _target_auth_failure_snippet(acc, sent_pass=sent_pass)
|
||||
if denied:
|
||||
raise paramiko.AuthenticationException(f"target_auth_rejected: {denied}")
|
||||
|
|
@ -895,7 +879,7 @@ def _huawei_enter_system_view(
|
|||
for _ in range(8):
|
||||
part = _read_channel(conn, wait=0.2, max_loops=10)
|
||||
if not part:
|
||||
time.sleep(0.02)
|
||||
time.sleep(0.15)
|
||||
continue
|
||||
acc += part
|
||||
if emit_raw:
|
||||
|
|
|
|||
|
|
@ -108,7 +108,6 @@ def apply_auth_schema_patches(conn: Connection) -> None:
|
|||
"ALTER TABLE app_user ADD COLUMN IF NOT EXISTS must_change_password BOOLEAN DEFAULT FALSE",
|
||||
)
|
||||
_run_sql(conn, "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS scopes JSON DEFAULT '[]'")
|
||||
_run_sql(conn, "ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'")
|
||||
_run_sql(conn, "ALTER TABLE api_token ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP")
|
||||
_run_sql(conn, "ALTER TABLE api_token ADD COLUMN IF NOT EXISTS scopes JSON DEFAULT '[]'")
|
||||
_run_sql(
|
||||
|
|
@ -355,7 +354,6 @@ def apply_domain_schema_patches(conn: Connection) -> None:
|
|||
"CREATE INDEX IF NOT EXISTS ix_ume_alarms_history_host_name ON ume_alarms_history (host_name)",
|
||||
"ALTER TABLE api_token ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP",
|
||||
"ALTER TABLE app_user ADD COLUMN IF NOT EXISTS must_change_password BOOLEAN DEFAULT FALSE",
|
||||
"ALTER TABLE app_user ADD COLUMN IF NOT EXISTS prefs JSON DEFAULT '{}'",
|
||||
"ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_enabled BOOLEAN DEFAULT FALSE",
|
||||
"ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_vendor VARCHAR(32) DEFAULT 'zte'",
|
||||
"ALTER TABLE managed_ne ADD COLUMN IF NOT EXISTS hop_host VARCHAR(128) DEFAULT ''",
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
"""Shared exact-value column filters, applied before pagination."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException, Query
|
||||
from sqlalchemy import String, cast, func
|
||||
|
||||
|
||||
@dataclass
|
||||
class TableFilterParams:
|
||||
filters: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
field: str = ""
|
||||
|
||||
|
||||
def table_filter_params(
|
||||
column_filters: str = Query(default=""),
|
||||
filter_field: str = Query(default=""),
|
||||
) -> TableFilterParams:
|
||||
try:
|
||||
filters = json.loads(column_filters) if column_filters else {}
|
||||
if not isinstance(filters, dict):
|
||||
raise ValueError()
|
||||
for value in filters.values():
|
||||
if not isinstance(value, dict) or value.get("mode") not in ("include", "exclude"):
|
||||
raise ValueError()
|
||||
values = value.get("values")
|
||||
if not isinstance(values, list) or any(not isinstance(v, str) for v in values):
|
||||
raise ValueError()
|
||||
except (ValueError, TypeError):
|
||||
raise HTTPException(status_code=400, detail="invalid_column_filters") from None
|
||||
return TableFilterParams(filters, filter_field)
|
||||
|
||||
|
||||
def filter_query(query: Any, expressions: dict[str, Any], params: TableFilterParams | None):
|
||||
"""Return the filtered query and, when requested, a distinct-value response."""
|
||||
params = params or TableFilterParams()
|
||||
if (set(params.filters) | ({params.field} if params.field else set())) - expressions.keys():
|
||||
raise HTTPException(status_code=400, detail="invalid_filter_field")
|
||||
for key, value in params.filters.items():
|
||||
if key == params.field:
|
||||
continue
|
||||
expression = func.coalesce(cast(expressions[key], String), "")
|
||||
values = value["values"]
|
||||
if value["mode"] == "include":
|
||||
query = query.filter(expression.in_(values)) if values else query.filter(False)
|
||||
elif values:
|
||||
query = query.filter(~expression.in_(values))
|
||||
if params.field:
|
||||
expression = func.coalesce(cast(expressions[params.field], String), "")
|
||||
rows = query.order_by(None).with_entities(expression).distinct().all()
|
||||
return query, {"field": params.field, "options": sorted({str(row[0]) for row in rows})}
|
||||
return query, None
|
||||
|
|
@ -1,151 +0,0 @@
|
|||
"""Small xlsx/csv reader/writer for imports and templates (replaces pandas).
|
||||
|
||||
Rows come back as ``(row_no, {header: value})`` where ``row_no`` is the 1-based
|
||||
sheet/file row (header = row 1), so import errors point at the row users see.
|
||||
Blank rows are skipped. Excel cells keep their openpyxl types (str / int /
|
||||
float / datetime / bool / None); integral floats become ints. CSV cells are
|
||||
strings, empty cells are None.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from collections.abc import Iterable, Iterator
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from openpyxl import Workbook, load_workbook
|
||||
|
||||
Row = tuple[int, dict[str, Any]]
|
||||
|
||||
|
||||
def _headers(raw: Iterable[Any]) -> list[str]:
|
||||
out: list[str] = []
|
||||
seen: dict[str, int] = {}
|
||||
for i, h in enumerate(raw):
|
||||
name = "" if h is None else str(h).strip()
|
||||
if not name:
|
||||
name = f"Unnamed: {i}"
|
||||
n = seen.get(name, 0)
|
||||
seen[name] = n + 1
|
||||
out.append(name if n == 0 else f"{name}.{n}")
|
||||
return out
|
||||
|
||||
|
||||
def _cell(v: Any) -> Any:
|
||||
if isinstance(v, str):
|
||||
return v if v.strip() else None
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return int(v)
|
||||
return v
|
||||
|
||||
|
||||
def _rows(records: Iterator[tuple[int, list[Any]]]) -> tuple[list[str], list[Row]]:
|
||||
headers: list[str] | None = None
|
||||
rows: list[Row] = []
|
||||
for row_no, values in records:
|
||||
cells = [_cell(v) for v in values]
|
||||
if all(c is None for c in cells):
|
||||
continue
|
||||
if headers is None:
|
||||
headers = _headers(values)
|
||||
continue
|
||||
cells += [None] * (len(headers) - len(cells))
|
||||
rows.append((row_no, dict(zip(headers, cells))))
|
||||
return headers or [], rows
|
||||
|
||||
|
||||
def _decode(content: bytes) -> str:
|
||||
try:
|
||||
return content.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
# CSVs saved by Chinese Excel default to GBK.
|
||||
return content.decode("gb18030")
|
||||
|
||||
|
||||
def read_csv(content: bytes) -> tuple[list[str], list[Row]]:
|
||||
reader = csv.reader(io.StringIO(_decode(content), newline=""))
|
||||
return _rows((i, rec) for i, rec in enumerate(reader, start=1))
|
||||
|
||||
|
||||
def read_xlsx(content: bytes) -> tuple[list[str], list[Row]]:
|
||||
wb = load_workbook(io.BytesIO(content), read_only=True, data_only=True)
|
||||
try:
|
||||
ws = wb.worksheets[0]
|
||||
return _rows((i, list(rec)) for i, rec in enumerate(ws.iter_rows(values_only=True), start=1))
|
||||
finally:
|
||||
wb.close()
|
||||
|
||||
|
||||
def read_table(content: bytes, filename: str = "") -> tuple[list[str], list[Row]]:
|
||||
if str(filename or "").lower().endswith(".csv"):
|
||||
return read_csv(content)
|
||||
return read_xlsx(content)
|
||||
|
||||
|
||||
def write_csv(headers: list[str], rows: Iterable[Iterable[Any]]) -> bytes:
|
||||
buf = io.StringIO()
|
||||
writer = csv.writer(buf, lineterminator="\n")
|
||||
writer.writerow(headers)
|
||||
for r in rows:
|
||||
writer.writerow(["" if v is None else v for v in r])
|
||||
# BOM so Excel opens UTF-8 (Chinese) correctly.
|
||||
return buf.getvalue().encode("utf-8-sig")
|
||||
|
||||
|
||||
def write_xlsx(sheets: list[tuple[str, list[str], Iterable[Iterable[Any]]]]) -> bytes:
|
||||
wb = Workbook()
|
||||
wb.remove(wb.active)
|
||||
for title, headers, rows in sheets:
|
||||
ws = wb.create_sheet(title)
|
||||
ws.append(headers)
|
||||
for r in rows:
|
||||
ws.append(list(r))
|
||||
buf = io.BytesIO()
|
||||
wb.save(buf)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
_DATE_FORMATS = (
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%d",
|
||||
"%Y%m%d%H%M%S",
|
||||
"%Y%m%d",
|
||||
"%m/%d/%Y %H:%M:%S",
|
||||
"%m/%d/%Y %H:%M",
|
||||
"%m/%d/%Y",
|
||||
"%d-%b-%Y %H:%M:%S",
|
||||
"%b %d, %Y %H:%M:%S",
|
||||
)
|
||||
_CN_DATE = re.compile(r"(\d{4})年(\d{1,2})月(\d{1,2})日")
|
||||
|
||||
|
||||
def parse_datetime(v: Any) -> datetime | None:
|
||||
"""Best-effort timestamp parse for spreadsheet cells; None when unparseable."""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, datetime):
|
||||
return v
|
||||
text = str(v).strip()
|
||||
if not text or text.lower() in ("nan", "nat", "none", "null"):
|
||||
return None
|
||||
text = _CN_DATE.sub(r"\1-\2-\3", text)
|
||||
# 2024/1/2 03:04:05 -> 2024-1-2 ... ; zero-pad so fromisoformat/strptime accept it.
|
||||
if re.match(r"^\d{4}/\d{1,2}/\d{1,2}", text):
|
||||
text = text.replace("/", "-", 2)
|
||||
m = re.match(r"^(\d{4})-(\d{1,2})-(\d{1,2})(.*)$", text)
|
||||
if m:
|
||||
text = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}{m.group(4)}"
|
||||
iso = text[:-1] + "+00:00" if text.endswith(("Z", "z")) else text
|
||||
try:
|
||||
return datetime.fromisoformat(iso)
|
||||
except ValueError:
|
||||
pass
|
||||
for fmt in _DATE_FORMATS:
|
||||
try:
|
||||
return datetime.strptime(text, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
|
@ -4,10 +4,9 @@ from __future__ import annotations
|
|||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import String, case, cast, func, or_
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .table_filters import TableFilterParams, filter_query
|
||||
from .cli_resolve import get_default_profile, infer_device_type_vendor
|
||||
from .device_types import is_placeholder_ne_source
|
||||
from .models import ManagedNE, TopoDiscoverJob, TopoDiscoverJobItem, UmeInventoryNE
|
||||
|
|
@ -43,17 +42,6 @@ def _managed_target_dict(ne: ManagedNE) -> dict[str, str]:
|
|||
}
|
||||
|
||||
|
||||
def discover_item_expressions() -> dict[str, Any]:
|
||||
item = TopoDiscoverJobItem
|
||||
failed = or_(item.ok.is_(False), item.parser_stub.is_(True), item.error.in_(["parser_stub", "empty_cli_output", "vendor_or_device_type_required"]))
|
||||
return {
|
||||
"name": func.coalesce(func.nullif(item.ne_name, ""), item.ne_id), "ip": item.ne_ip,
|
||||
"status": case((failed, "fail"), (item.unmatched_count > 0, "warn"), else_="ok"),
|
||||
"neighbors": item.neighbors, "error": item.error,
|
||||
"edges": cast(item.edges_added, String) + "/" + cast(item.edges_updated, String),
|
||||
}
|
||||
|
||||
|
||||
def _job_out(
|
||||
db: Session,
|
||||
job: TopoDiscoverJob,
|
||||
|
|
@ -63,7 +51,6 @@ def _job_out(
|
|||
page_size: int | None = None,
|
||||
item_status: str = "",
|
||||
item_keyword: str = "",
|
||||
table_filters: TableFilterParams | None = None,
|
||||
) -> FabricDiscoverJobOut:
|
||||
items_out: list[FabricDiscoverJobItemOut] = []
|
||||
items_total = 0
|
||||
|
|
@ -106,7 +93,6 @@ def _job_out(
|
|||
TopoDiscoverJobItem.error.ilike(like),
|
||||
)
|
||||
)
|
||||
q, _ = filter_query(q, discover_item_expressions(), table_filters)
|
||||
items_total = int(q.count())
|
||||
if page is not None and page_size is not None:
|
||||
items_page = max(1, int(page or 1))
|
||||
|
|
@ -171,7 +157,6 @@ def get_discover_job(
|
|||
page_size: int | None = None,
|
||||
item_status: str = "",
|
||||
item_keyword: str = "",
|
||||
table_filters: TableFilterParams | None = None,
|
||||
) -> FabricDiscoverJobOut:
|
||||
job = db.get(TopoDiscoverJob, str(job_id or "").strip())
|
||||
if job is None:
|
||||
|
|
@ -183,7 +168,6 @@ def get_discover_job(
|
|||
page_size=page_size,
|
||||
item_status=item_status,
|
||||
item_keyword=item_keyword,
|
||||
table_filters=table_filters,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,11 +8,10 @@ from typing import Any
|
|||
from uuid import uuid4
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import and_, case, func, or_, text
|
||||
from sqlalchemy import and_, func, or_, text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .table_filters import TableFilterParams, filter_query
|
||||
from .cli_resolve import get_default_profile, infer_device_type_vendor
|
||||
from .device_types import LLDP_DISCOVERED_NE_SOURCE, WEBCRT_NE_SOURCE
|
||||
from .models import (
|
||||
|
|
@ -276,7 +275,6 @@ def list_fabric_nodes(
|
|||
region_folder_id: str = "",
|
||||
unmatched: str = "",
|
||||
link_status: str = "",
|
||||
table_filters: TableFilterParams | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = PAGE_DEFAULT,
|
||||
) -> dict[str, Any]:
|
||||
|
|
@ -377,16 +375,6 @@ def list_fabric_nodes(
|
|||
and_(TopoFabricNode.managed_ne_id.isnot(None), TopoFabricNode.managed_ne_id != ""),
|
||||
and_(TopoFabricNode.ume_ne_id.isnot(None), TopoFabricNode.ume_ne_id != ""),
|
||||
)
|
||||
has_managed = func.coalesce(TopoFabricNode.managed_ne_id, "") != ""
|
||||
has_ume = func.coalesce(TopoFabricNode.ume_ne_id, "") != ""
|
||||
q, options = filter_query(q, {
|
||||
"name": func.coalesce(func.nullif(TopoFabricNode.name, ""), TopoFabricNode.id),
|
||||
"ip": TopoFabricNode.ip, "level": TopoFabricNode.level,
|
||||
"region": TopoFabricNode.region_folder_id,
|
||||
"link": case((and_(has_managed, has_ume), "both"), (has_managed, "managed"), (has_ume, "ume"), else_="orphaned"),
|
||||
}, table_filters)
|
||||
if options is not None:
|
||||
return options
|
||||
total = int(q.count())
|
||||
rows = (
|
||||
q.order_by(TopoFabricNode.name.asc())
|
||||
|
|
@ -411,7 +399,6 @@ def list_fabric_edges(
|
|||
status: str = "",
|
||||
source: str = "",
|
||||
keyword: str = "",
|
||||
table_filters: TableFilterParams | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = PAGE_DEFAULT,
|
||||
) -> dict[str, Any]:
|
||||
|
|
@ -451,23 +438,6 @@ def list_fabric_edges(
|
|||
TopoFabricEdge.b_node_id.in_(matched_ids),
|
||||
)
|
||||
)
|
||||
from sqlalchemy.orm import aliased
|
||||
a = aliased(TopoFabricNode)
|
||||
b = aliased(TopoFabricNode)
|
||||
q = q.outerjoin(a, TopoFabricEdge.a_node_id == a.id).outerjoin(b, TopoFabricEdge.b_node_id == b.id)
|
||||
status_expr = case((TopoFabricEdge.status == "stale", "missing"), else_=TopoFabricEdge.status)
|
||||
miss = func.coalesce(TopoFabricEdge.attrs["miss_count"].as_integer(), 0)
|
||||
q, options = filter_query(q, {
|
||||
"status": status_expr,
|
||||
"aSide": func.coalesce(func.nullif(a.name, ""), func.nullif(a.ip, ""), func.substr(TopoFabricEdge.a_node_id, 1, 8)),
|
||||
"bSide": func.coalesce(func.nullif(b.name, ""), func.nullif(b.ip, ""), func.substr(TopoFabricEdge.b_node_id, 1, 8)),
|
||||
"aPort": TopoFabricEdge.a_port, "bPort": TopoFabricEdge.b_port,
|
||||
"source": case((TopoFabricEdge.source == "stale", "lldp"), else_=func.coalesce(func.nullif(TopoFabricEdge.source, ""), "lldp")),
|
||||
"lastSeen": TopoFabricEdge.last_seen_at,
|
||||
"missCount": case((miss != 0, miss), (status_expr == "missing", 1), else_=0),
|
||||
}, table_filters)
|
||||
if options is not None:
|
||||
return options
|
||||
total = int(q.count())
|
||||
rows = (
|
||||
q.order_by(TopoFabricEdge.updated_at.desc())
|
||||
|
|
@ -620,13 +590,9 @@ def find_fabric_paths(
|
|||
if len(edge_path) >= max_hops:
|
||||
continue
|
||||
_ensure_adj({node})
|
||||
# Parallel links (LAG members / dual fibers) between the same two NEs are one hop:
|
||||
# expanding each would return near-identical paths that differ only by port.
|
||||
seen_nbrs: set[str] = set()
|
||||
for nbr, eid in adj_cache.get(node, []):
|
||||
if nbr in visited or nbr in seen_nbrs:
|
||||
if nbr in visited:
|
||||
continue
|
||||
seen_nbrs.add(nbr)
|
||||
explored += 1
|
||||
new_path = edge_path + [eid]
|
||||
if nbr == to_id:
|
||||
|
|
@ -654,15 +620,10 @@ def find_fabric_paths(
|
|||
"vendor": n.vendor or "",
|
||||
}
|
||||
|
||||
def _parallel_count(e: TopoFabricEdge) -> int:
|
||||
# Only expanded nodes have adjacency loaded; the path's last node never is.
|
||||
src, dst = (e.a_node_id, e.b_node_id) if e.a_node_id in adj_loaded else (e.b_node_id, e.a_node_id)
|
||||
return sum(1 for nbr, _ in adj_cache.get(src, []) if nbr == dst) or 1
|
||||
|
||||
def _edge_summary(e: TopoFabricEdge) -> dict[str, Any]:
|
||||
a_node = node_map.get(e.a_node_id)
|
||||
b_node = node_map.get(e.b_node_id)
|
||||
out = {
|
||||
return {
|
||||
"a_name": (a_node.name if a_node else "") or "",
|
||||
"b_name": (b_node.name if b_node else "") or "",
|
||||
"a_ip": (a_node.ip if a_node else "") or "",
|
||||
|
|
@ -671,10 +632,6 @@ def find_fabric_paths(
|
|||
"b_port": e.b_port or "",
|
||||
"status": _normalize_edge_status(e.status or "active"),
|
||||
}
|
||||
parallel = _parallel_count(e)
|
||||
if parallel > 1:
|
||||
out["parallel_links"] = parallel
|
||||
return out
|
||||
|
||||
def _path_node_ids(edge_ids: list[str]) -> list[str]:
|
||||
ids = [from_id]
|
||||
|
|
|
|||
|
|
@ -121,39 +121,6 @@ def place_child_region_on_parent_canvas(
|
|||
return row
|
||||
|
||||
|
||||
def add_missing_region_placement(
|
||||
db: Session, view: TopoView, fabric_node_id: str, *, x: float, y: float
|
||||
) -> TopoViewNode | None:
|
||||
"""Persist a child-region icon that was only synthesized (no placement row yet).
|
||||
|
||||
Returns None unless ``fabric_node_id`` is a manual child region of the view's folder.
|
||||
"""
|
||||
if not is_region_canvas_node_id(fabric_node_id) or _is_world_flat(view):
|
||||
return None
|
||||
folder = db.get(TopoFolder, region_folder_id_from_node(fabric_node_id))
|
||||
if (
|
||||
folder is None
|
||||
or str(folder.kind or "") != "region"
|
||||
or str(folder.parent_id or "") != str(view.folder_id or "")
|
||||
or _is_ume_managed_folder(folder)
|
||||
):
|
||||
return None
|
||||
now = _utcnow()
|
||||
row = TopoViewNode(
|
||||
id=uuid4().hex,
|
||||
view_id=view.id,
|
||||
fabric_node_id=fabric_node_id,
|
||||
x=float(x),
|
||||
y=float(y),
|
||||
label="",
|
||||
locked=False,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
db.add(row)
|
||||
return row
|
||||
|
||||
|
||||
def remove_region_canvas_placements(db: Session, folder_id: str) -> int:
|
||||
"""Drop this folder's icon from any parent canvas."""
|
||||
nid = region_canvas_node_id(folder_id)
|
||||
|
|
@ -172,7 +139,6 @@ def region_node_out(
|
|||
y: float,
|
||||
locked: bool = False,
|
||||
label: str = "",
|
||||
label_override: str | None = None,
|
||||
) -> ViewNodeOut:
|
||||
child_view = _child_primary_view(db, folder.id)
|
||||
kids = (
|
||||
|
|
@ -188,14 +154,12 @@ def region_node_out(
|
|||
.filter(TopoViewNode.view_id == child_view.id)
|
||||
.count()
|
||||
)
|
||||
raw = str(label_override if label_override is not None else label or "")
|
||||
display = (raw or folder.name or folder.id)[:256]
|
||||
display = (label or folder.name or folder.id)[:256]
|
||||
return ViewNodeOut(
|
||||
fabric_node_id=region_canvas_node_id(folder.id),
|
||||
managed_ne_id="",
|
||||
ume_ne_id="",
|
||||
label=display,
|
||||
label_override=raw[:256],
|
||||
x=float(x),
|
||||
y=float(y),
|
||||
locked=bool(locked),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from typing import Any
|
|||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .table_filters import TableFilterParams, table_filter_params
|
||||
from .db import get_db
|
||||
from .topology_classify import (
|
||||
apply_classify,
|
||||
|
|
@ -39,8 +38,6 @@ from .topology_schemas import (
|
|||
TopologyPlaceholderCreate,
|
||||
TopologyViewCreate,
|
||||
TopologyViewUpdate,
|
||||
FabricEdgePortsPatch,
|
||||
ViewCommitIn,
|
||||
ViewEdgeStylePatch,
|
||||
ViewNodesAdd,
|
||||
ViewNodesRemove,
|
||||
|
|
@ -64,7 +61,6 @@ from .topology_fabric import (
|
|||
from .topology_views import (
|
||||
add_nodes_to_view,
|
||||
bootstrap_topology_tree,
|
||||
commit_view_changes,
|
||||
create_folder,
|
||||
create_topology_placeholder_on_view,
|
||||
create_view,
|
||||
|
|
@ -73,7 +69,6 @@ from .topology_views import (
|
|||
get_topology_tree,
|
||||
get_view_graph,
|
||||
list_views,
|
||||
patch_manual_edge_ports,
|
||||
patch_view_edge_style,
|
||||
patch_view_positions,
|
||||
populate_view,
|
||||
|
|
@ -108,7 +103,6 @@ def api_fabric_nodes(
|
|||
),
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=100, ge=1, le=2000),
|
||||
table_filters: TableFilterParams = Depends(table_filter_params),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
return list_fabric_nodes(
|
||||
|
|
@ -120,7 +114,6 @@ def api_fabric_nodes(
|
|||
region_folder_id=region_folder_id,
|
||||
unmatched=unmatched,
|
||||
link_status=link_status,
|
||||
table_filters=table_filters,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
|
@ -135,7 +128,6 @@ def api_fabric_edges(
|
|||
keyword: str = "",
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=100, ge=1, le=2000),
|
||||
table_filters: TableFilterParams = Depends(table_filter_params),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
return list_fabric_edges(
|
||||
|
|
@ -145,7 +137,6 @@ def api_fabric_edges(
|
|||
status=status,
|
||||
source=source,
|
||||
keyword=keyword,
|
||||
table_filters=table_filters,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
|
@ -490,32 +481,6 @@ def api_edge_style(
|
|||
return patch_view_edge_style(db, view_id, body).model_dump()
|
||||
|
||||
|
||||
@router.post("/views/{view_id}/commit")
|
||||
def api_commit_view(
|
||||
view_id: str, body: ViewCommitIn, db: Session = Depends(get_db)
|
||||
) -> dict[str, Any]:
|
||||
return commit_view_changes(db, view_id, body).model_dump()
|
||||
|
||||
|
||||
@router.patch("/fabric/edges/{edge_id}")
|
||||
def api_patch_fabric_edge_ports(
|
||||
edge_id: str, body: FabricEdgePortsPatch, db: Session = Depends(get_db)
|
||||
) -> dict[str, Any]:
|
||||
edge = patch_manual_edge_ports(
|
||||
db, edge_id, a_port=body.a_port, b_port=body.b_port, commit=True
|
||||
)
|
||||
return {
|
||||
"id": edge.id,
|
||||
"a_node_id": edge.a_node_id,
|
||||
"b_node_id": edge.b_node_id,
|
||||
"a_port": edge.a_port or "",
|
||||
"b_port": edge.b_port or "",
|
||||
"source": edge.source or "manual",
|
||||
"status": edge.status or "active",
|
||||
"layer": edge.layer or "physical",
|
||||
}
|
||||
|
||||
|
||||
# --- Classify rules / slices / search ---------------------------------------
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -273,10 +273,8 @@ class ViewNodeIn(BaseModel):
|
|||
fabric_node_id: str = Field(min_length=1, max_length=64)
|
||||
x: float = 0.0
|
||||
y: float = 0.0
|
||||
# None = leave existing override unchanged; "" = clear override (follow device name).
|
||||
label: str | None = None
|
||||
# None = leave lock unchanged; False unlocks (and allows move); True locks after move.
|
||||
locked: bool | None = None
|
||||
label: str = ""
|
||||
locked: bool = False
|
||||
|
||||
|
||||
class ViewNodeOut(BaseModel):
|
||||
|
|
@ -284,8 +282,6 @@ class ViewNodeOut(BaseModel):
|
|||
managed_ne_id: str = ""
|
||||
ume_ne_id: str = ""
|
||||
label: str = ""
|
||||
# Raw placement override (empty = follow fabric name/ip). ``label`` is the display value.
|
||||
label_override: str = ""
|
||||
x: float = 0.0
|
||||
y: float = 0.0
|
||||
locked: bool = False
|
||||
|
|
@ -452,16 +448,6 @@ class ViewNodesRemove(BaseModel):
|
|||
|
||||
|
||||
class ViewEdgeStylePatch(BaseModel):
|
||||
"""Single-edge style patch (legacy). Prefer ``items`` for batch."""
|
||||
|
||||
fabric_edge_id: str = ""
|
||||
stroke_color: str = ""
|
||||
stroke_width: int = Field(default=0, ge=0, le=12)
|
||||
line_style: str = Field(default="", max_length=16)
|
||||
items: list["ViewEdgeStyleItem"] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ViewEdgeStyleItem(BaseModel):
|
||||
fabric_edge_id: str
|
||||
stroke_color: str = ""
|
||||
stroke_width: int = Field(default=0, ge=0, le=12)
|
||||
|
|
@ -475,11 +461,6 @@ class FabricManualEdgeIn(BaseModel):
|
|||
b_port: str = ""
|
||||
|
||||
|
||||
class FabricEdgePortsPatch(BaseModel):
|
||||
a_port: str = ""
|
||||
b_port: str = ""
|
||||
|
||||
|
||||
class FabricEdgesDeleteRequest(BaseModel):
|
||||
edge_ids: list[str] = Field(default_factory=list, min_length=1)
|
||||
|
||||
|
|
@ -488,40 +469,6 @@ class FabricEdgesDeleteOut(BaseModel):
|
|||
deleted: int = 0
|
||||
|
||||
|
||||
class ViewCommitEdgeCreate(BaseModel):
|
||||
client_id: str = ""
|
||||
a_node_id: str = Field(min_length=1, max_length=64)
|
||||
b_node_id: str = Field(min_length=1, max_length=64)
|
||||
a_port: str = ""
|
||||
b_port: str = ""
|
||||
|
||||
|
||||
class ViewCommitEdgePortEdit(BaseModel):
|
||||
id: str = Field(min_length=1, max_length=64)
|
||||
a_port: str = ""
|
||||
b_port: str = ""
|
||||
|
||||
|
||||
class ViewCommitIn(BaseModel):
|
||||
"""Atomic canvas save: all steps succeed or the whole request rolls back."""
|
||||
|
||||
edge_creates: list[ViewCommitEdgeCreate] = Field(default_factory=list)
|
||||
edge_deletes: list[str] = Field(default_factory=list)
|
||||
edge_port_edits: list[ViewCommitEdgePortEdit] = Field(default_factory=list)
|
||||
edge_styles: list[ViewEdgeStyleItem] = Field(default_factory=list)
|
||||
node_adds: list[str] = Field(default_factory=list)
|
||||
node_removes: list[str] = Field(default_factory=list)
|
||||
positions: list[ViewNodeIn] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ViewCommitOut(BaseModel):
|
||||
ok: bool = True
|
||||
view_id: str = ""
|
||||
edge_id_map: dict[str, str] = Field(default_factory=dict)
|
||||
skipped_locked: int = 0
|
||||
graph: TopologyViewGraphOut | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Classify rules + slices + search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@ from .topology_views import (
|
|||
_place_fabric_ids_on_view,
|
||||
add_nodes_to_view,
|
||||
bootstrap_topology_tree,
|
||||
commit_view_changes,
|
||||
create_folder,
|
||||
create_topology_placeholder_on_view,
|
||||
create_view,
|
||||
|
|
@ -49,7 +48,6 @@ from .topology_views import (
|
|||
get_topology_tree,
|
||||
get_view_graph,
|
||||
list_views,
|
||||
patch_manual_edge_ports,
|
||||
patch_view_edge_style,
|
||||
patch_view_positions,
|
||||
populate_view,
|
||||
|
|
@ -70,7 +68,6 @@ __all__ = [
|
|||
"_place_fabric_ids_on_view",
|
||||
"add_nodes_to_view",
|
||||
"bootstrap_topology_tree",
|
||||
"commit_view_changes",
|
||||
"create_folder",
|
||||
"create_topology_placeholder_on_view",
|
||||
"create_view",
|
||||
|
|
@ -91,7 +88,6 @@ __all__ = [
|
|||
"list_fabric_nodes",
|
||||
"list_views",
|
||||
"merge_duplicate_fabric_nodes",
|
||||
"patch_manual_edge_ports",
|
||||
"patch_view_edge_style",
|
||||
"patch_view_positions",
|
||||
"pause_discover_job",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
"""Topology folder tree and leaf view operations (facade)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .topology_views_commit import commit_view_changes, patch_manual_edge_ports
|
||||
from .topology_views_graph import (
|
||||
_place_fabric_ids_on_view,
|
||||
add_nodes_to_view,
|
||||
|
|
@ -30,7 +29,6 @@ __all__ = [
|
|||
"_place_fabric_ids_on_view",
|
||||
"add_nodes_to_view",
|
||||
"bootstrap_topology_tree",
|
||||
"commit_view_changes",
|
||||
"create_folder",
|
||||
"create_topology_placeholder_on_view",
|
||||
"create_view",
|
||||
|
|
@ -40,7 +38,6 @@ __all__ = [
|
|||
"get_topology_tree",
|
||||
"get_view_graph",
|
||||
"list_views",
|
||||
"patch_manual_edge_ports",
|
||||
"patch_view_edge_style",
|
||||
"patch_view_positions",
|
||||
"populate_view",
|
||||
|
|
|
|||
|
|
@ -1,444 +0,0 @@
|
|||
"""Atomic view canvas commit (single transaction)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from .models import (
|
||||
TopoFabricEdge,
|
||||
TopoFabricNode,
|
||||
TopoViewEdgeStyle,
|
||||
TopoViewNode,
|
||||
)
|
||||
from .topology_common import VIEW_GRAPH_NODE_HARD_CAP, _utcnow
|
||||
from .topology_fabric_links import upsert_fabric_edge
|
||||
from .topology_fabric_nodes import _normalize_endpoints, refresh_fabric_stats
|
||||
from .topology_region_canvas import add_missing_region_placement, is_region_canvas_node_id
|
||||
from .topology_schemas import (
|
||||
TopologyViewGraphOut,
|
||||
ViewCommitIn,
|
||||
ViewCommitOut,
|
||||
ViewEdgeStyleItem,
|
||||
ViewNodeIn,
|
||||
)
|
||||
from .topology_views_graph import (
|
||||
_get_view_or_404,
|
||||
_membership_for_view,
|
||||
get_view_graph,
|
||||
)
|
||||
from .ume_topology_world import is_world_flat_view
|
||||
|
||||
_HEX_COLOR_RE = re.compile(r"^#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})$")
|
||||
_LINE_STYLES = {"", "solid", "dashed", "dotted"}
|
||||
|
||||
|
||||
def _fail(step: str, detail: str, *, status: int = 400) -> None:
|
||||
raise HTTPException(
|
||||
status_code=status,
|
||||
detail={"failed_step": step, "detail": detail},
|
||||
)
|
||||
|
||||
|
||||
def _apply_edge_creates(
|
||||
db: Session, creates: list[Any], edge_id_map: dict[str, str]
|
||||
) -> None:
|
||||
for item in creates:
|
||||
a = str(getattr(item, "a_node_id", "") or "").strip()
|
||||
b = str(getattr(item, "b_node_id", "") or "").strip()
|
||||
if not a or not b:
|
||||
_fail("edge_creates", "edge_endpoints_required")
|
||||
edge, action = upsert_fabric_edge(
|
||||
db,
|
||||
a_node_id=a,
|
||||
b_node_id=b,
|
||||
a_port=str(getattr(item, "a_port", "") or ""),
|
||||
b_port=str(getattr(item, "b_port", "") or ""),
|
||||
source="manual",
|
||||
)
|
||||
if edge is None or action == "skipped_self_loop":
|
||||
_fail("edge_creates", "edge_self_loop")
|
||||
client_id = str(getattr(item, "client_id", "") or "").strip()
|
||||
if client_id:
|
||||
edge_id_map[client_id] = str(edge.id)
|
||||
|
||||
|
||||
def _apply_edge_deletes(db: Session, edge_ids: list[str]) -> set[str]:
|
||||
"""Delete edges; return the ids that were requested (for skipping later edits)."""
|
||||
ids = [str(x).strip() for x in (edge_ids or []) if str(x).strip()]
|
||||
if not ids:
|
||||
return set()
|
||||
seen: set[str] = set()
|
||||
ordered: list[str] = []
|
||||
for i in ids:
|
||||
if i in seen:
|
||||
continue
|
||||
seen.add(i)
|
||||
ordered.append(i)
|
||||
rows = db.query(TopoFabricEdge).filter(TopoFabricEdge.id.in_(ordered)).all()
|
||||
found_ids = [str(r.id) for r in rows]
|
||||
if not found_ids:
|
||||
return seen
|
||||
db.query(TopoViewEdgeStyle).filter(
|
||||
TopoViewEdgeStyle.fabric_edge_id.in_(found_ids)
|
||||
).delete(synchronize_session=False)
|
||||
db.query(TopoFabricEdge).filter(TopoFabricEdge.id.in_(found_ids)).delete(
|
||||
synchronize_session=False
|
||||
)
|
||||
return seen
|
||||
|
||||
|
||||
def patch_manual_edge_ports(
|
||||
db: Session,
|
||||
edge_id: str,
|
||||
*,
|
||||
a_port: str,
|
||||
b_port: str,
|
||||
commit: bool = True,
|
||||
) -> TopoFabricEdge:
|
||||
"""Update ports on a manual fabric edge. Raises 400/404/409."""
|
||||
eid = str(edge_id or "").strip()
|
||||
row = db.get(TopoFabricEdge, eid) if eid else None
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="fabric_edge_not_found")
|
||||
if str(row.source or "").strip().lower() != "manual":
|
||||
raise HTTPException(status_code=400, detail="edge_not_manual")
|
||||
_a, _b, ap, bp = _normalize_endpoints(
|
||||
row.a_node_id,
|
||||
row.b_node_id,
|
||||
str(a_port or ""),
|
||||
str(b_port or ""),
|
||||
)
|
||||
conflict = (
|
||||
db.query(TopoFabricEdge)
|
||||
.filter(
|
||||
TopoFabricEdge.id != row.id,
|
||||
TopoFabricEdge.layer == row.layer,
|
||||
TopoFabricEdge.a_node_id == row.a_node_id,
|
||||
TopoFabricEdge.b_node_id == row.b_node_id,
|
||||
TopoFabricEdge.a_port == ap,
|
||||
TopoFabricEdge.b_port == bp,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if conflict is not None:
|
||||
raise HTTPException(status_code=409, detail="edge_port_conflict")
|
||||
row.a_port = ap
|
||||
row.b_port = bp
|
||||
row.updated_at = _utcnow()
|
||||
try:
|
||||
db.flush()
|
||||
except IntegrityError as exc:
|
||||
raise HTTPException(status_code=409, detail="edge_port_conflict") from exc
|
||||
if commit:
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
return row
|
||||
|
||||
|
||||
def _apply_edge_port_edits(db: Session, edits: list[Any], *, deleted: set[str]) -> None:
|
||||
for item in edits:
|
||||
eid = str(getattr(item, "id", "") or "").strip()
|
||||
if not eid:
|
||||
_fail("edge_port_edits", "edge_id_required")
|
||||
if eid in deleted:
|
||||
continue
|
||||
try:
|
||||
patch_manual_edge_ports(
|
||||
db,
|
||||
eid,
|
||||
a_port=str(getattr(item, "a_port", "") or ""),
|
||||
b_port=str(getattr(item, "b_port", "") or ""),
|
||||
commit=False,
|
||||
)
|
||||
except HTTPException as exc:
|
||||
detail = exc.detail if isinstance(exc.detail, str) else "edge_port_edit_failed"
|
||||
_fail("edge_port_edits", str(detail), status=int(exc.status_code or 400))
|
||||
|
||||
|
||||
def _upsert_edge_style(
|
||||
db: Session,
|
||||
view_id: str,
|
||||
item: ViewEdgeStyleItem,
|
||||
*,
|
||||
now,
|
||||
) -> None:
|
||||
eid = str(item.fabric_edge_id or "").strip()
|
||||
if not eid or db.get(TopoFabricEdge, eid) is None:
|
||||
_fail("edge_styles", "fabric_edge_not_found", status=404)
|
||||
color = str(item.stroke_color or "").strip()
|
||||
if color and not _HEX_COLOR_RE.match(color):
|
||||
_fail("edge_styles", "invalid_stroke_color")
|
||||
width = int(item.stroke_width or 0)
|
||||
if width < 0 or width > 12:
|
||||
_fail("edge_styles", "invalid_stroke_width")
|
||||
style = str(item.line_style or "").strip().lower()
|
||||
if style not in _LINE_STYLES:
|
||||
_fail("edge_styles", "invalid_line_style")
|
||||
row = (
|
||||
db.query(TopoViewEdgeStyle)
|
||||
.filter(
|
||||
TopoViewEdgeStyle.view_id == view_id,
|
||||
TopoViewEdgeStyle.fabric_edge_id == eid,
|
||||
)
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None:
|
||||
db.add(
|
||||
TopoViewEdgeStyle(
|
||||
id=uuid4().hex,
|
||||
view_id=view_id,
|
||||
fabric_edge_id=eid,
|
||||
stroke_color=color,
|
||||
stroke_width=width,
|
||||
line_style=style,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
else:
|
||||
row.stroke_color = color
|
||||
row.stroke_width = width
|
||||
row.line_style = style
|
||||
row.updated_at = now
|
||||
|
||||
|
||||
def _apply_edge_styles(
|
||||
db: Session,
|
||||
view_id: str,
|
||||
styles: list[ViewEdgeStyleItem],
|
||||
*,
|
||||
edge_id_map: dict[str, str],
|
||||
deleted: set[str],
|
||||
) -> None:
|
||||
now = _utcnow()
|
||||
for item in styles:
|
||||
eid = str(item.fabric_edge_id or "").strip()
|
||||
if eid in deleted:
|
||||
continue
|
||||
# Styles on a just-created local edge are keyed by its client id.
|
||||
if eid in edge_id_map:
|
||||
item = item.model_copy(update={"fabric_edge_id": edge_id_map[eid]})
|
||||
_upsert_edge_style(db, view_id, item, now=now)
|
||||
|
||||
|
||||
def _apply_node_adds(
|
||||
db: Session,
|
||||
view,
|
||||
fabric_ids: list[str],
|
||||
*,
|
||||
max_nodes: int,
|
||||
existing: dict[str, TopoViewNode],
|
||||
) -> None:
|
||||
if is_world_flat_view(view):
|
||||
_fail("node_adds", "world_map_no_direct_nes")
|
||||
ids = [str(x).strip() for x in (fabric_ids or []) if str(x).strip()]
|
||||
if not ids:
|
||||
return
|
||||
if len(existing) >= max_nodes:
|
||||
_fail("node_adds", "membership_max_nodes")
|
||||
now = _utcnow()
|
||||
room = max(0, max_nodes - len(existing))
|
||||
added = 0
|
||||
for fid in ids:
|
||||
if fid in existing:
|
||||
continue
|
||||
if is_region_canvas_node_id(fid):
|
||||
continue
|
||||
if db.get(TopoFabricNode, fid) is None:
|
||||
_fail("node_adds", f"fabric_node_not_found:{fid}")
|
||||
if added >= room:
|
||||
_fail("node_adds", "membership_max_nodes")
|
||||
row = TopoViewNode(
|
||||
id=uuid4().hex,
|
||||
view_id=view.id,
|
||||
fabric_node_id=fid,
|
||||
x=40.0,
|
||||
y=40.0,
|
||||
label="",
|
||||
locked=False,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
db.add(row)
|
||||
existing[fid] = row
|
||||
added += 1
|
||||
if added:
|
||||
view.updated_at = now
|
||||
|
||||
|
||||
def _apply_node_removes(
|
||||
db: Session,
|
||||
view,
|
||||
fabric_ids: list[str],
|
||||
*,
|
||||
existing: dict[str, TopoViewNode],
|
||||
) -> None:
|
||||
ids = [str(x).strip() for x in (fabric_ids or []) if str(x).strip()]
|
||||
ids = [x for x in ids if x and not is_region_canvas_node_id(x)]
|
||||
if not ids:
|
||||
return
|
||||
db.query(TopoViewNode).filter(
|
||||
TopoViewNode.view_id == view.id,
|
||||
TopoViewNode.fabric_node_id.in_(ids),
|
||||
).delete(synchronize_session=False)
|
||||
for fid in ids:
|
||||
existing.pop(fid, None)
|
||||
view.updated_at = _utcnow()
|
||||
|
||||
|
||||
def _apply_positions(
|
||||
db: Session,
|
||||
view,
|
||||
positions: list[ViewNodeIn],
|
||||
*,
|
||||
existing: dict[str, TopoViewNode],
|
||||
) -> int:
|
||||
"""Apply positions in-session. Returns skipped_locked count."""
|
||||
if len(positions) > VIEW_GRAPH_NODE_HARD_CAP:
|
||||
_fail("positions", "too_many_positions")
|
||||
now = _utcnow()
|
||||
skipped_locked = 0
|
||||
updated = 0
|
||||
fabric_ids = [
|
||||
str(p.fabric_node_id or "").strip()
|
||||
for p in positions
|
||||
if str(p.fabric_node_id or "").strip()
|
||||
and not is_region_canvas_node_id(str(p.fabric_node_id or "").strip())
|
||||
]
|
||||
alive = (
|
||||
{str(r[0]) for r in db.query(TopoFabricNode.id).filter(TopoFabricNode.id.in_(fabric_ids))}
|
||||
if fabric_ids
|
||||
else set()
|
||||
)
|
||||
for p in positions:
|
||||
fid = str(p.fabric_node_id or "").strip()
|
||||
if not fid:
|
||||
continue
|
||||
if not is_region_canvas_node_id(fid) and fid not in alive and fid not in existing:
|
||||
# Missing fabric: skip rather than fail mid-commit when node was just removed.
|
||||
continue
|
||||
row = existing.get(fid)
|
||||
if row is None:
|
||||
# Fabric NEs are placed via node_adds; only synthesized child-region icons are created here.
|
||||
row = add_missing_region_placement(db, view, fid, x=p.x, y=p.y)
|
||||
if row is None:
|
||||
continue
|
||||
existing[fid] = row
|
||||
if row.locked and p.locked is not False:
|
||||
skipped_locked += 1
|
||||
continue
|
||||
row.x = float(p.x or 0)
|
||||
row.y = float(p.y or 0)
|
||||
if p.label is not None:
|
||||
row.label = str(p.label or "")[:256]
|
||||
if p.locked is not None:
|
||||
row.locked = bool(p.locked)
|
||||
row.updated_at = now
|
||||
updated += 1
|
||||
if updated or skipped_locked:
|
||||
view.updated_at = now
|
||||
return skipped_locked
|
||||
|
||||
|
||||
def commit_view_changes(db: Session, view_id: str, body: ViewCommitIn) -> ViewCommitOut:
|
||||
"""Apply canvas mutations atomically; return one graph + client→server edge map."""
|
||||
view = _get_view_or_404(db, view_id)
|
||||
mem = _membership_for_view(view)
|
||||
max_nodes = int(mem.get("max_nodes") or 300)
|
||||
existing = {
|
||||
vn.fabric_node_id: vn
|
||||
for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all()
|
||||
}
|
||||
edge_id_map: dict[str, str] = {}
|
||||
skipped_locked = 0
|
||||
|
||||
try:
|
||||
_apply_edge_creates(db, list(body.edge_creates or []), edge_id_map)
|
||||
deleted = _apply_edge_deletes(db, list(body.edge_deletes or []))
|
||||
_apply_edge_port_edits(db, list(body.edge_port_edits or []), deleted=deleted)
|
||||
_apply_edge_styles(
|
||||
db,
|
||||
view.id,
|
||||
list(body.edge_styles or []),
|
||||
edge_id_map=edge_id_map,
|
||||
deleted=deleted,
|
||||
)
|
||||
_apply_node_adds(
|
||||
db,
|
||||
view,
|
||||
list(body.node_adds or []),
|
||||
max_nodes=max_nodes,
|
||||
existing=existing,
|
||||
)
|
||||
_apply_node_removes(
|
||||
db,
|
||||
view,
|
||||
list(body.node_removes or []),
|
||||
existing=existing,
|
||||
)
|
||||
# Re-load placements after add/remove so positions see current rows.
|
||||
# Flush first: sessions run with autoflush=False, so new placements would be invisible.
|
||||
db.flush()
|
||||
existing = {
|
||||
vn.fabric_node_id: vn
|
||||
for vn in db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all()
|
||||
}
|
||||
skipped_locked = _apply_positions(
|
||||
db,
|
||||
view,
|
||||
list(body.positions or []),
|
||||
existing=existing,
|
||||
)
|
||||
db.commit()
|
||||
except HTTPException:
|
||||
db.rollback()
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
db.rollback()
|
||||
_fail("commit", str(exc)[:500], status=500)
|
||||
|
||||
try:
|
||||
refresh_fabric_stats(db)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
graph: TopologyViewGraphOut = get_view_graph(db, view.id)
|
||||
return ViewCommitOut(
|
||||
ok=True,
|
||||
view_id=view.id,
|
||||
edge_id_map=edge_id_map,
|
||||
skipped_locked=skipped_locked,
|
||||
graph=graph,
|
||||
)
|
||||
|
||||
|
||||
def normalize_edge_style_items(body: Any) -> list[ViewEdgeStyleItem]:
|
||||
"""Accept legacy single-edge body or batch ``items``."""
|
||||
items = list(getattr(body, "items", None) or [])
|
||||
if items:
|
||||
return [
|
||||
ViewEdgeStyleItem(
|
||||
fabric_edge_id=str(i.fabric_edge_id or ""),
|
||||
stroke_color=str(i.stroke_color or ""),
|
||||
stroke_width=int(i.stroke_width or 0),
|
||||
line_style=str(i.line_style or ""),
|
||||
)
|
||||
for i in items
|
||||
]
|
||||
fid = str(getattr(body, "fabric_edge_id", "") or "").strip()
|
||||
if not fid:
|
||||
return []
|
||||
return [
|
||||
ViewEdgeStyleItem(
|
||||
fabric_edge_id=fid,
|
||||
stroke_color=str(getattr(body, "stroke_color", "") or ""),
|
||||
stroke_width=int(getattr(body, "stroke_width", 0) or 0),
|
||||
line_style=str(getattr(body, "line_style", "") or ""),
|
||||
)
|
||||
]
|
||||
|
|
@ -157,12 +157,7 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut:
|
|||
# Save/mutate return_graph must use the same UME synthetic graph as GET.
|
||||
if is_ume_canvas_view(view):
|
||||
return get_ume_canvas_graph(db, view_id)
|
||||
vnodes = (
|
||||
db.query(TopoViewNode)
|
||||
.filter(TopoViewNode.view_id == view.id)
|
||||
.order_by(TopoViewNode.created_at.asc(), TopoViewNode.id.asc())
|
||||
.all()
|
||||
)
|
||||
vnodes = db.query(TopoViewNode).filter(TopoViewNode.view_id == view.id).all()
|
||||
truncated = False
|
||||
reason = ""
|
||||
if len(vnodes) > VIEW_GRAPH_NODE_HARD_CAP:
|
||||
|
|
@ -174,19 +169,6 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut:
|
|||
n.id: n for n in db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(fids)).all()
|
||||
} if fids else {}
|
||||
managed_by_id, ume_by_id = _batch_ne_lookups(db, fabric_nodes)
|
||||
region_folder_ids = [
|
||||
region_folder_id_from_node(vn.fabric_node_id)
|
||||
for vn in vnodes
|
||||
if is_region_canvas_node_id(vn.fabric_node_id)
|
||||
]
|
||||
folders_by_id = {
|
||||
f.id: f
|
||||
for f in (
|
||||
db.query(TopoFolder).filter(TopoFolder.id.in_(region_folder_ids)).all()
|
||||
if region_folder_ids
|
||||
else []
|
||||
)
|
||||
}
|
||||
filt = dict(view.filter or {})
|
||||
layer = str(filt.get("layer") or "physical").strip() or "physical"
|
||||
status = str(filt.get("status") or "").strip().lower()
|
||||
|
|
@ -194,7 +176,7 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut:
|
|||
region_ids_seen: set[str] = set()
|
||||
for vn in vnodes:
|
||||
if is_region_canvas_node_id(vn.fabric_node_id):
|
||||
folder = folders_by_id.get(region_folder_id_from_node(vn.fabric_node_id))
|
||||
folder = db.get(TopoFolder, region_folder_id_from_node(vn.fabric_node_id))
|
||||
if folder is None or str(folder.kind or "") != "region":
|
||||
continue
|
||||
region_ids_seen.add(folder.id)
|
||||
|
|
@ -205,13 +187,12 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut:
|
|||
x=float(vn.x or 0),
|
||||
y=float(vn.y or 0),
|
||||
locked=bool(vn.locked),
|
||||
label_override=str(vn.label or ""),
|
||||
label=str(vn.label or folder.name or ""),
|
||||
)
|
||||
)
|
||||
continue
|
||||
fn = fabric_nodes.get(vn.fabric_node_id)
|
||||
label_override = str(vn.label or "")
|
||||
label = label_override.strip()
|
||||
label = (vn.label or "").strip()
|
||||
if not label and fn is not None:
|
||||
label = (fn.name or fn.ip or vn.fabric_node_id)[:256]
|
||||
connect_status = ""
|
||||
|
|
@ -235,7 +216,6 @@ def get_view_graph(db: Session, view_id: str) -> TopologyViewGraphOut:
|
|||
managed_ne_id=(fn.managed_ne_id if fn else "") or "",
|
||||
ume_ne_id=(fn.ume_ne_id if fn else "") or "",
|
||||
label=label,
|
||||
label_override=label_override[:256],
|
||||
x=float(vn.x or 0),
|
||||
y=float(vn.y or 0),
|
||||
locked=bool(vn.locked),
|
||||
|
|
@ -368,31 +348,22 @@ def _outside_peers_for_view(
|
|||
member_ids: set[str],
|
||||
layer: str,
|
||||
limit: int = 50,
|
||||
edge_scan_limit: int = 5000,
|
||||
) -> list[dict[str, str]]:
|
||||
if not member_ids:
|
||||
return []
|
||||
members = list(member_ids)
|
||||
# Exclude edges whose both ends are already on the view so dense canvases
|
||||
# do not burn the scan budget before any external neighbor is found.
|
||||
both_internal = and_(
|
||||
TopoFabricEdge.a_node_id.in_(members),
|
||||
TopoFabricEdge.b_node_id.in_(members),
|
||||
)
|
||||
edges = (
|
||||
db.query(TopoFabricEdge)
|
||||
.filter(
|
||||
TopoFabricEdge.layer == layer,
|
||||
or_(
|
||||
TopoFabricEdge.a_node_id.in_(members),
|
||||
TopoFabricEdge.b_node_id.in_(members),
|
||||
TopoFabricEdge.a_node_id.in_(list(member_ids)),
|
||||
TopoFabricEdge.b_node_id.in_(list(member_ids)),
|
||||
),
|
||||
~both_internal,
|
||||
)
|
||||
.limit(max(1, int(edge_scan_limit or 5000)))
|
||||
.limit(5000)
|
||||
.all()
|
||||
)
|
||||
peer_via: list[tuple[str, str]] = []
|
||||
out: list[dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for e in edges:
|
||||
for peer_id, local_id in ((e.b_node_id, e.a_node_id), (e.a_node_id, e.b_node_id)):
|
||||
|
|
@ -401,21 +372,7 @@ def _outside_peers_for_view(
|
|||
if peer_id in seen:
|
||||
continue
|
||||
seen.add(peer_id)
|
||||
peer_via.append((peer_id, local_id))
|
||||
if len(peer_via) >= limit:
|
||||
break
|
||||
if len(peer_via) >= limit:
|
||||
break
|
||||
if not peer_via:
|
||||
return []
|
||||
peer_ids = [p for p, _ in peer_via]
|
||||
peers = {
|
||||
n.id: n
|
||||
for n in db.query(TopoFabricNode).filter(TopoFabricNode.id.in_(peer_ids)).all()
|
||||
}
|
||||
out: list[dict[str, str]] = []
|
||||
for peer_id, local_id in peer_via:
|
||||
fn = peers.get(peer_id)
|
||||
fn = db.get(TopoFabricNode, peer_id)
|
||||
out.append(
|
||||
{
|
||||
"fabric_node_id": peer_id,
|
||||
|
|
@ -424,6 +381,8 @@ def _outside_peers_for_view(
|
|||
"via_node_id": local_id,
|
||||
}
|
||||
)
|
||||
if len(out) >= limit:
|
||||
return out
|
||||
return out
|
||||
|
||||
|
||||
|
|
@ -687,7 +646,6 @@ def patch_view_positions(
|
|||
layout = str(body.layout or "").strip().lower()
|
||||
updated = 0
|
||||
skipped_locked = 0
|
||||
skipped_missing = 0
|
||||
matched = 0
|
||||
|
||||
existing = {
|
||||
|
|
@ -748,51 +706,42 @@ def patch_view_positions(
|
|||
row.updated_at = now
|
||||
updated += 1
|
||||
else:
|
||||
from .topology_region_canvas import add_missing_region_placement, is_region_canvas_node_id
|
||||
|
||||
positions = list(body.positions or [])
|
||||
if len(positions) > VIEW_GRAPH_NODE_HARD_CAP:
|
||||
raise HTTPException(status_code=400, detail="too_many_positions")
|
||||
matched = len(positions)
|
||||
# One IN query instead of per-node db.get (N+1).
|
||||
check_ids = [
|
||||
str(p.fabric_node_id or "").strip()
|
||||
for p in positions
|
||||
if str(p.fabric_node_id or "").strip()
|
||||
and not is_region_canvas_node_id(str(p.fabric_node_id or "").strip())
|
||||
]
|
||||
fabric_alive: set[str] = set()
|
||||
if check_ids:
|
||||
fabric_alive = {
|
||||
str(x[0])
|
||||
for x in db.query(TopoFabricNode.id)
|
||||
.filter(TopoFabricNode.id.in_(check_ids))
|
||||
.all()
|
||||
}
|
||||
for p in positions:
|
||||
fid = str(p.fabric_node_id or "").strip()
|
||||
if not fid:
|
||||
continue
|
||||
if not is_region_canvas_node_id(fid) and fid not in fabric_alive:
|
||||
from .topology_region_canvas import is_region_canvas_node_id
|
||||
|
||||
if not is_region_canvas_node_id(fid) and db.get(TopoFabricNode, fid) is None:
|
||||
raise HTTPException(status_code=400, detail=f"fabric_node_not_found:{fid}")
|
||||
row = existing.get(fid)
|
||||
if row is None:
|
||||
# Child-region icons may be synthesized without a placement row; persist those.
|
||||
# Fabric NEs must not bypass membership caps / freeze (MCP adds via /nodes).
|
||||
row = add_missing_region_placement(db, view, fid, x=p.x, y=p.y)
|
||||
if row is None:
|
||||
skipped_missing += 1
|
||||
continue
|
||||
row = TopoViewNode(
|
||||
id=uuid4().hex,
|
||||
view_id=view.id,
|
||||
fabric_node_id=fid,
|
||||
x=float(p.x or 0),
|
||||
y=float(p.y or 0),
|
||||
label=str(p.label or "")[:256],
|
||||
locked=bool(p.locked),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
db.add(row)
|
||||
existing[fid] = row
|
||||
# Locked rows stay put unless the client explicitly unlocks (locked=False).
|
||||
if row.locked and p.locked is not False:
|
||||
updated += 1
|
||||
else:
|
||||
if row.locked and bool(p.locked):
|
||||
skipped_locked += 1
|
||||
continue
|
||||
row.x = float(p.x or 0)
|
||||
row.y = float(p.y or 0)
|
||||
if p.label is not None:
|
||||
row.label = str(p.label or "")[:256]
|
||||
if p.locked is not None:
|
||||
row.locked = bool(p.locked)
|
||||
row.updated_at = now
|
||||
updated += 1
|
||||
|
|
@ -807,7 +756,6 @@ def patch_view_positions(
|
|||
matched=matched,
|
||||
updated=updated,
|
||||
skipped_locked=skipped_locked,
|
||||
skipped_missing=skipped_missing,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1405,26 +1353,20 @@ _LINE_STYLES = {"", "solid", "dashed", "dotted"}
|
|||
def patch_view_edge_style(
|
||||
db: Session, view_id: str, body: ViewEdgeStylePatch
|
||||
) -> TopologyViewGraphOut:
|
||||
from .topology_views_commit import normalize_edge_style_items
|
||||
|
||||
view = _get_view_or_404(db, view_id)
|
||||
items = normalize_edge_style_items(body)
|
||||
if not items:
|
||||
raise HTTPException(status_code=400, detail="fabric_edge_id_required")
|
||||
now = _utcnow()
|
||||
for item in items:
|
||||
eid = str(item.fabric_edge_id or "").strip()
|
||||
eid = str(body.fabric_edge_id or "").strip()
|
||||
if not eid or db.get(TopoFabricEdge, eid) is None:
|
||||
raise HTTPException(status_code=404, detail="fabric_edge_not_found")
|
||||
color = str(item.stroke_color or "").strip()
|
||||
color = str(body.stroke_color or "").strip()
|
||||
if color and not _HEX_COLOR_RE.match(color):
|
||||
raise HTTPException(status_code=400, detail="invalid_stroke_color")
|
||||
width = int(item.stroke_width or 0)
|
||||
width = int(body.stroke_width or 0)
|
||||
if width < 0 or width > 12:
|
||||
raise HTTPException(status_code=400, detail="invalid_stroke_width")
|
||||
style = str(item.line_style or "").strip().lower()
|
||||
style = str(body.line_style or "").strip().lower()
|
||||
if style not in _LINE_STYLES:
|
||||
raise HTTPException(status_code=400, detail="invalid_line_style")
|
||||
now = _utcnow()
|
||||
row = (
|
||||
db.query(TopoViewEdgeStyle)
|
||||
.filter(TopoViewEdgeStyle.view_id == view.id, TopoViewEdgeStyle.fabric_edge_id == eid)
|
||||
|
|
|
|||
|
|
@ -10,8 +10,6 @@ from sqlalchemy import func, or_
|
|||
from sqlalchemy.orm import Session
|
||||
|
||||
from .config import settings
|
||||
from .table_filters import TableFilterParams, filter_query
|
||||
from .table_filters import table_filter_params
|
||||
from .db import get_db
|
||||
from .key_alert_config import (
|
||||
get_key_alert_monitor_config,
|
||||
|
|
@ -80,7 +78,6 @@ def ume_list_alarms(
|
|||
time_to: str | None = Query(default=None, description="Filter by last_seen_at <="),
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=50, ge=1, le=500),
|
||||
table_filters: TableFilterParams = Depends(table_filter_params),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict[str, Any]:
|
||||
stmt = db.query(UmeAlarmCurrent, UmeInventoryNE).outerjoin(
|
||||
|
|
@ -116,15 +113,6 @@ def ume_list_alarms(
|
|||
stmt = stmt.filter(UmeAlarmCurrent.last_seen_at >= dt_from.replace(tzinfo=None))
|
||||
if dt_to:
|
||||
stmt = stmt.filter(UmeAlarmCurrent.last_seen_at <= dt_to.replace(tzinfo=None))
|
||||
stmt, options = filter_query(stmt, {
|
||||
"time": UmeAlarmCurrent.time_created, "severity": UmeAlarmCurrent.perceived_severity,
|
||||
"neId": UmeAlarmCurrent.ne_id,
|
||||
"hostName": func.coalesce(func.nullif(func.trim(UmeAlarmCurrent.host_name), ""), func.trim(UmeInventoryNE.host_name)),
|
||||
"neType": UmeInventoryNE.ne_type, "cause": UmeAlarmCurrent.native_probable_cause,
|
||||
"cleared": UmeAlarmCurrent.is_cleared,
|
||||
}, table_filters)
|
||||
if options is not None:
|
||||
return options
|
||||
total = int(stmt.count())
|
||||
rows = (
|
||||
stmt.order_by(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import queue
|
|||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
|
@ -45,18 +44,6 @@ def uses_network_cli_keymap(device_type: str = "", vendor: str = "") -> bool:
|
|||
return True
|
||||
|
||||
|
||||
_NETWORK_PROMPT_RE = re.compile(
|
||||
r"^(?:<[^<>\s]{1,63}>"
|
||||
r"|\[[~*]?[\w.\-/:+]{1,80}\]"
|
||||
r"|[A-Za-z0-9][\w.\-/:+]{0,62}(?:\([^()\s]{1,60}\))*[#>])$"
|
||||
)
|
||||
|
||||
|
||||
def looks_like_network_prompt(line: str) -> bool:
|
||||
"""``<r1>`` / ``[~r1]`` / ``R2#`` / ``R2(config)#`` — never a Linux ``user@host:~$`` prompt."""
|
||||
return bool(_NETWORK_PROMPT_RE.match(str(line or "").strip()))
|
||||
|
||||
|
||||
def map_network_cli_keys(
|
||||
data: str,
|
||||
*,
|
||||
|
|
@ -69,8 +56,6 @@ def map_network_cli_keys(
|
|||
text = str(data or "")
|
||||
if not text:
|
||||
return text
|
||||
if "\x1b" not in text and "\x7f" not in text:
|
||||
return text # plain printable typing: nothing to rewrite
|
||||
out: list[str] = []
|
||||
i = 0
|
||||
n = len(text)
|
||||
|
|
@ -101,11 +86,13 @@ def channel_return(conn: ConnectHandler | None) -> str:
|
|||
def map_network_cli_enter(data: str, conn: ConnectHandler | None) -> str:
|
||||
"""Map xterm Enter (\\r) to the device's Netmiko RETURN."""
|
||||
text = str(data or "")
|
||||
if not text or ("\r" not in text and "\n" not in text):
|
||||
if not text:
|
||||
return text
|
||||
ret = channel_return(conn)
|
||||
# Replace once: chained replacements expand the CR in Telnet CRLF again.
|
||||
return re.sub(r"\r\n|\r|\n", lambda _match: ret, text)
|
||||
if ret == "\r":
|
||||
return text
|
||||
# Prefer replacing CRLF first so Telnet RETURN \\r\\n does not double-expand.
|
||||
return text.replace("\r\n", ret).replace("\r", ret)
|
||||
|
||||
|
||||
def _drain_channel(conn: ConnectHandler, *, rounds: int = 6, wait: float = 0.06) -> str:
|
||||
|
|
@ -382,10 +369,10 @@ def _encode_text(text: str, encoding: str) -> bytes:
|
|||
|
||||
|
||||
class _BoundedByteQueue:
|
||||
"""Thread-safe chunk queue that drops the oldest chunks when full (backpressure)."""
|
||||
"""Thread-safe queue that drops oldest chunks when full (backpressure)."""
|
||||
|
||||
def __init__(self, maxsize: int = 2000) -> None:
|
||||
self._dq: deque[bytes | None] = deque()
|
||||
self._q: queue.Queue[bytes | None] = queue.Queue()
|
||||
self._max = max(8, int(maxsize or 2000))
|
||||
self._cond = threading.Condition()
|
||||
self.dropped = 0
|
||||
|
|
@ -393,12 +380,13 @@ class _BoundedByteQueue:
|
|||
|
||||
def put(self, item: bytes | None) -> None:
|
||||
with self._cond:
|
||||
dq = self._dq
|
||||
if item is not None:
|
||||
while len(dq) >= self._max:
|
||||
dq.popleft()
|
||||
while self._q.qsize() >= self._max:
|
||||
try:
|
||||
self._q.get_nowait()
|
||||
self.dropped += 1
|
||||
dq.append(item)
|
||||
except queue.Empty:
|
||||
break
|
||||
self._q.put(item)
|
||||
self._cond.notify()
|
||||
|
||||
def put_nowait(self, item: bytes | None) -> None:
|
||||
|
|
@ -406,42 +394,22 @@ class _BoundedByteQueue:
|
|||
|
||||
def get_nowait(self) -> bytes | None:
|
||||
with self._cond:
|
||||
if not self._dq:
|
||||
raise queue.Empty
|
||||
return self._dq.popleft()
|
||||
return self._q.get_nowait()
|
||||
|
||||
def get(self, timeout: float = 0.25) -> bytes | None:
|
||||
"""Block until a chunk is available or timeout (raises queue.Empty)."""
|
||||
deadline = time.monotonic() + max(0.0, float(timeout))
|
||||
deadline = time.time() + max(0.0, float(timeout))
|
||||
with self._cond:
|
||||
while not self._dq:
|
||||
remaining = deadline - time.monotonic()
|
||||
while self._q.empty():
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
raise queue.Empty
|
||||
self._cond.wait(timeout=remaining)
|
||||
return self._dq.popleft()
|
||||
|
||||
def drain(self, max_bytes: int = 262144) -> tuple[list[bytes], bool]:
|
||||
"""Pop every queued chunk (up to ~max_bytes). Returns ``(chunks, saw_eof)``."""
|
||||
out: list[bytes] = []
|
||||
total = 0
|
||||
with self._cond:
|
||||
dq = self._dq
|
||||
while dq:
|
||||
item = dq[0]
|
||||
if item is None:
|
||||
dq.popleft()
|
||||
return out, True
|
||||
if out and total + len(item) > max_bytes:
|
||||
break
|
||||
dq.popleft()
|
||||
out.append(item)
|
||||
total += len(item)
|
||||
return out, False
|
||||
return self._q.get_nowait()
|
||||
|
||||
def qsize(self) -> int:
|
||||
with self._cond:
|
||||
return len(self._dq)
|
||||
return self._q.qsize()
|
||||
|
||||
def take_drop_delta(self) -> int:
|
||||
"""Return newly dropped chunk count since last call (for client notice)."""
|
||||
|
|
@ -1288,9 +1256,6 @@ def feed_command_line_buffer(buf: str, data: str, *, max_line: int = 512) -> tup
|
|||
completed: list[str] = []
|
||||
limit = max(64, min(int(max_line or 512), 4096))
|
||||
raw = str(data or "")
|
||||
if raw.isprintable():
|
||||
# Plain typing (no Enter / BS / ESC): just append.
|
||||
return (cur + raw)[:limit] if len(cur) < limit else cur, completed
|
||||
i = 0
|
||||
while i < len(raw):
|
||||
ch = raw[i]
|
||||
|
|
|
|||
|
|
@ -1,525 +0,0 @@
|
|||
"""Server-side command audit for WebCRT: track the device's own input row.
|
||||
|
||||
Why: inferring commands from browser keystrokes / xterm snapshots races with the
|
||||
device echo (fast typing, Tab, history recall, slow VMs) and records fragments such
|
||||
as ``R2#s`` or ``cddontainer/r/``. The device itself always knows what it executes:
|
||||
when it echoes the line feed for our Enter, the *current input row* is exactly the
|
||||
prompt + command it is about to run.
|
||||
|
||||
``LineScreen`` is a tiny single-row VT100 model (CR/BS/CSI cursor + erase).
|
||||
``CommandTracker`` owns one ``LineScreen`` per session and resolves each Enter we
|
||||
send against the first line feed the device echoes back. Everything here is pure
|
||||
(no I/O) and cheap: output before the last line feed is skipped unless an Enter
|
||||
is waiting for its echo.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Single-row terminal model
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
_TOKEN_RE = re.compile(
|
||||
r"(\x1b\[[0-?]*[ -/]*[@-~]" # CSI
|
||||
r"|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)" # OSC
|
||||
r"|\x1b[()][0-9A-Za-z]" # charset
|
||||
r"|\x1b[@-Z\\-_0-9=>]" # 2-byte ESC
|
||||
r"|[\x00-\x1f\x7f])" # C0 + DEL
|
||||
)
|
||||
# Incomplete escape at the end of a chunk — carry into the next feed.
|
||||
_PARTIAL_ESC_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*|\][^\x07\x1b]*\x1b?|[()])?\Z")
|
||||
_MAX_CARRY = 96
|
||||
|
||||
|
||||
class LineScreen:
|
||||
"""Current input row of a terminal (no scrolling / wrapping)."""
|
||||
|
||||
__slots__ = ("cells", "cur", "_carry")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.cells: list[str] = []
|
||||
self.cur = 0
|
||||
self._carry = ""
|
||||
|
||||
def reset_row(self) -> None:
|
||||
self.cells = []
|
||||
self.cur = 0
|
||||
|
||||
def text(self) -> str:
|
||||
return "".join(self.cells).rstrip()
|
||||
|
||||
def _put(self, run: str) -> None:
|
||||
if not run:
|
||||
return
|
||||
n = len(run)
|
||||
cells = self.cells
|
||||
cur = self.cur
|
||||
if cur == len(cells):
|
||||
cells.extend(run)
|
||||
else:
|
||||
if cur > len(cells):
|
||||
cells.extend(" " * (cur - len(cells)))
|
||||
cells[cur : cur + n] = run
|
||||
self.cur = cur + n
|
||||
|
||||
def apply_token(self, tok: str) -> None:
|
||||
"""Apply one control token (not LF — the tracker owns line feeds)."""
|
||||
c0 = tok[0]
|
||||
if c0 == "\r":
|
||||
self.cur = 0
|
||||
elif c0 == "\b":
|
||||
if self.cur > 0:
|
||||
self.cur -= 1
|
||||
elif c0 == "\t":
|
||||
self._put(" " * (8 - (self.cur % 8)))
|
||||
elif c0 == "\x1b":
|
||||
if len(tok) >= 3 and tok[1] == "[":
|
||||
self._csi(tok[2:-1], tok[-1])
|
||||
elif tok == "\x1bc":
|
||||
self.reset_row()
|
||||
# every other control byte (BEL, NUL, DEL, SO/SI …) is ignored
|
||||
|
||||
def _csi(self, params: str, final: str) -> None:
|
||||
if final not in "DCKP@XGHfJ":
|
||||
return
|
||||
if params and params[0] in "?>=":
|
||||
return
|
||||
try:
|
||||
first = params.split(";", 1)[0]
|
||||
n = int(first) if first else 0
|
||||
except ValueError:
|
||||
n = 0
|
||||
cells = self.cells
|
||||
if final == "D":
|
||||
self.cur = max(0, self.cur - (n or 1))
|
||||
elif final == "C":
|
||||
self.cur += n or 1
|
||||
elif final == "K":
|
||||
if n == 0:
|
||||
del cells[self.cur :]
|
||||
elif n == 1:
|
||||
for i in range(min(self.cur + 1, len(cells))):
|
||||
cells[i] = " "
|
||||
else:
|
||||
self.reset_row()
|
||||
elif final == "P":
|
||||
del cells[self.cur : self.cur + (n or 1)]
|
||||
elif final == "@":
|
||||
if self.cur > len(cells):
|
||||
cells.extend(" " * (self.cur - len(cells)))
|
||||
cells[self.cur : self.cur] = [" "] * (n or 1)
|
||||
elif final == "X":
|
||||
for i in range(self.cur, min(self.cur + (n or 1), len(cells))):
|
||||
cells[i] = " "
|
||||
elif final == "G":
|
||||
self.cur = max(0, (n or 1) - 1)
|
||||
elif final in "Hf":
|
||||
parts = params.split(";")
|
||||
try:
|
||||
col = int(parts[1]) if len(parts) > 1 and parts[1] else 1
|
||||
except ValueError:
|
||||
col = 1
|
||||
self.cur = max(0, col - 1)
|
||||
elif final == "J":
|
||||
if n in (2, 3):
|
||||
self.reset_row()
|
||||
elif n == 0:
|
||||
del cells[self.cur :]
|
||||
|
||||
def feed_row(self, text: str) -> None:
|
||||
"""Apply text that contains no line feed."""
|
||||
if not text:
|
||||
return
|
||||
if not _has_ctl(text):
|
||||
self._put(text)
|
||||
return
|
||||
parts = _TOKEN_RE.split(text)
|
||||
for i, part in enumerate(parts):
|
||||
if not part:
|
||||
continue
|
||||
if i & 1:
|
||||
self.apply_token(part)
|
||||
else:
|
||||
self._put(part)
|
||||
|
||||
def take_carry(self, text: str) -> str:
|
||||
"""Prepend a carried partial escape; stash a new trailing partial."""
|
||||
if self._carry:
|
||||
text = self._carry + text
|
||||
self._carry = ""
|
||||
if text and "\x1b" in text[-_MAX_CARRY:]:
|
||||
m = _PARTIAL_ESC_RE.search(text, max(0, len(text) - _MAX_CARRY))
|
||||
if m and m.group(0):
|
||||
self._carry = m.group(0)
|
||||
text = text[: m.start()]
|
||||
return text
|
||||
|
||||
|
||||
def _has_ctl(text: str) -> bool:
|
||||
# str.isprintable is a fast C scan; only fall back to token split when needed.
|
||||
return not text.isprintable()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Prompt splitting
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
_PROMPT_RES: tuple[re.Pattern[str], ...] = (
|
||||
# Huawei / H3C / ZTE user & system view: <r1> [r1] [~r1] [*r1-GigabitEthernet0/0/1]
|
||||
re.compile(r"^<[^<>\s]{1,63}>"),
|
||||
re.compile(r"^\[[~*]?[A-Za-z0-9][\w.\-/:+]{0,79}\]"),
|
||||
# Cisco / Arista / ZTE / Ruijie: R2# R2> R2(config-if)# SW-1(config)#
|
||||
re.compile(r"^[A-Za-z0-9][\w.\-/:+]{0,62}(?:\([^()\s]{1,60}\))*[#>]"),
|
||||
# MikroTik: [admin@MikroTik] > [admin@MikroTik] /ip>
|
||||
re.compile(r"^\[[\w.\-]+@[\w.\-]+\][^\n]{0,40}?>\s?"),
|
||||
)
|
||||
_LINUX_PROMPT_RES: tuple[re.Pattern[str], ...] = (
|
||||
# [user@host dir]$ user@host:~/dir$ (venv) user@host:~#
|
||||
re.compile(r"^(?:\([^()]{1,40}\)\s*)?\[[^\]]{1,80}\]\s?[$#%>]\s?"),
|
||||
re.compile(r"^(?:\([^()]{1,40}\)\s*)?[\w.\-]+@[\w.\-]+(?::|\s)[^\s$#%>]{0,120}\s?[$#%>]\s?"),
|
||||
re.compile(r"^[^\s$#%>]{1,80}[$#%]\s"),
|
||||
)
|
||||
_NOT_PROMPT_RE = re.compile(r"^\[[YyNn]/[YyNn]\]")
|
||||
_PASSWORD_ROW_RE = re.compile(r"(?:password|passwd|passphrase|密码)\s*[:：>]\s*\**\s*$", re.IGNORECASE)
|
||||
_PAGER_ROW_RE = re.compile(r"-{2,}\s*more\s*-{2,}|--\s*more\s*--|\(END\)|lines?\s+\d+-\d+", re.IGNORECASE)
|
||||
|
||||
|
||||
def split_prompt(row: str, hint: str = "", *, linux_like: bool = False) -> tuple[str, str] | None:
|
||||
"""Split ``row`` into ``(prompt, command)``; None when no CLI prompt is recognised.
|
||||
|
||||
``prompt`` keeps the whitespace before the command, so ``prompt + command`` is the
|
||||
row exactly as the device displayed it.
|
||||
"""
|
||||
s = row.rstrip()
|
||||
if not s or _NOT_PROMPT_RE.match(s) or _PAGER_ROW_RE.search(s[:80]):
|
||||
return None
|
||||
h = hint.rstrip()
|
||||
if h and s.startswith(h):
|
||||
return _cut(s, len(h))
|
||||
pats = _LINUX_PROMPT_RES + _PROMPT_RES if linux_like else _PROMPT_RES + _LINUX_PROMPT_RES
|
||||
for pat in pats:
|
||||
m = pat.match(s)
|
||||
if m:
|
||||
return _cut(s, m.end())
|
||||
return None
|
||||
|
||||
|
||||
def _cut(s: str, end: int) -> tuple[str, str]:
|
||||
rest = s[end:]
|
||||
cmd = rest.lstrip()
|
||||
return s[: len(s) - len(cmd)], cmd.rstrip()
|
||||
|
||||
|
||||
def looks_like_password_row(row: str) -> bool:
|
||||
return bool(_PASSWORD_ROW_RE.search(row.rstrip()))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Command tracker
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@dataclass
|
||||
class AuditCmd:
|
||||
command: str # prompt + command as the device displayed it
|
||||
prompt: str = ""
|
||||
input: str = "" # command only
|
||||
redacted: bool = False
|
||||
source: str = "stdin"
|
||||
via: str = "device_echo"
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Pending:
|
||||
t: float
|
||||
typed: str
|
||||
snapshot: str # input row at the time we sent Enter
|
||||
prompt_hint: str
|
||||
client_hints: list[str]
|
||||
source: str
|
||||
# "bare": Enter with nothing typed in this payload (history recall / empty Enter) —
|
||||
# consumes the first line feed whatever it shows.
|
||||
bare: bool = False
|
||||
# Tab can make the device redraw the line after a CRLF before the Enter echo; keep the
|
||||
# latest prompt+command row seen and settle once the real output starts.
|
||||
has_tab: bool = False
|
||||
cand: tuple[str, str] | None = None
|
||||
cand_t: float = 0.0
|
||||
|
||||
|
||||
_TAB_SETTLE_SEC = 1.2
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommandTracker:
|
||||
"""Per-session: device input row + FIFO of Enters waiting for their echo."""
|
||||
|
||||
linux_like: bool = False
|
||||
expire_sec: float = 3.0
|
||||
max_command: int = 512
|
||||
screen: LineScreen = field(default_factory=LineScreen)
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
pending: deque[_Pending] = field(default_factory=deque)
|
||||
prompt_hint: str = ""
|
||||
line_started: bool = False
|
||||
password_mode: bool = False
|
||||
# Rows abandoned with Ctrl-C: some devices (IOS, bash) echo that row + CRLF, and that
|
||||
# line feed must not be mistaken for the echo of the next Enter.
|
||||
abandoned: list[tuple[str, float]] = field(default_factory=list)
|
||||
_typed: str = ""
|
||||
|
||||
# ----- device output -------------------------------------------------- #
|
||||
def feed(self, text: str) -> list[AuditCmd]:
|
||||
"""Feed decoded device output; returns commands resolved by line-feed echoes."""
|
||||
out: list[AuditCmd] = []
|
||||
with self.lock:
|
||||
scr = self.screen
|
||||
text = scr.take_carry(text)
|
||||
if not text:
|
||||
return out
|
||||
while text:
|
||||
nl = text.find("\n")
|
||||
if nl < 0:
|
||||
scr.feed_row(text)
|
||||
break
|
||||
self.line_started = False
|
||||
if self.pending:
|
||||
scr.feed_row(text[:nl])
|
||||
self._resolve_one(out)
|
||||
scr.reset_row()
|
||||
text = text[nl + 1 :]
|
||||
continue
|
||||
# Nothing waiting: only the last row matters — skip the bulk.
|
||||
last = text.rfind("\n")
|
||||
scr.reset_row()
|
||||
text = text[last + 1 :]
|
||||
scr.feed_row(text)
|
||||
break
|
||||
self.password_mode = looks_like_password_row(scr.text())
|
||||
if self.line_started and not scr.text():
|
||||
# Row wiped by the device (screen clear / new prompt): start over.
|
||||
self.line_started = False
|
||||
return out
|
||||
|
||||
def _resolve_one(self, out: list[AuditCmd]) -> None:
|
||||
row = self.screen.text()
|
||||
p = self.pending[0]
|
||||
if self.abandoned and not p.bare:
|
||||
now = time.monotonic()
|
||||
self.abandoned = [(r, t) for r, t in self.abandoned if now - t < 5.0]
|
||||
typed = p.typed.replace("\t", "").strip()
|
||||
for i, (ab, _t) in enumerate(self.abandoned):
|
||||
# The device reprints the aborted row; ``startswith`` covers a slow echo
|
||||
# (row snapshot was taken before every typed char came back). A VRP that
|
||||
# keeps the text yields ``garbage`` + our command, which contains ``typed``.
|
||||
if row == ab or (row.startswith(ab) and typed and typed not in row):
|
||||
del self.abandoned[i]
|
||||
return # echo of the aborted line, not our Enter
|
||||
parsed = split_prompt(row, p.prompt_hint, linux_like=self.linux_like)
|
||||
has_cmd = bool(parsed and parsed[1])
|
||||
if p.bare:
|
||||
# A bare Enter (empty line, or executing a recalled history line) is echoed by
|
||||
# the first *prompt* row's line feed; output rows of a command that is still
|
||||
# printing must not consume it (Enter typed ahead during output).
|
||||
if parsed is None:
|
||||
return
|
||||
self.pending.popleft()
|
||||
if has_cmd:
|
||||
out.append(self._make(parsed[0], parsed[1], p.source, "device_echo")) # type: ignore[index]
|
||||
# Enter pressed at a pager left an orphan bare Enter in front of the next
|
||||
# typed line: that line feed belongs to both — do not record it twice.
|
||||
nxt = self.pending[0] if self.pending else None
|
||||
if nxt is not None and not nxt.bare:
|
||||
typed = nxt.typed.replace("\t", "").replace(" ", "").lower()
|
||||
if typed and parsed[1].replace(" ", "").lower().startswith(typed[:3]):
|
||||
self.pending.popleft()
|
||||
return
|
||||
if p.has_tab:
|
||||
if has_cmd:
|
||||
p.cand = (parsed[0], parsed[1]) # type: ignore[index]
|
||||
p.cand_t = time.monotonic()
|
||||
elif p.cand is not None:
|
||||
# Prompt / output row after the echo: the last candidate is the executed line.
|
||||
self._finish_candidate(p, out)
|
||||
return
|
||||
if not has_cmd:
|
||||
# Prompt-only / non-CLI row: earlier output, a prompt reprint — not our echo.
|
||||
return
|
||||
self.pending.popleft()
|
||||
out.append(self._make(parsed[0], parsed[1], p.source, "device_echo")) # type: ignore[index]
|
||||
|
||||
def _finish_candidate(self, p: _Pending, out: list[AuditCmd]) -> None:
|
||||
if self.pending and self.pending[0] is p:
|
||||
self.pending.popleft()
|
||||
if p.cand is not None:
|
||||
out.append(self._make(p.cand[0], p.cand[1], p.source, "device_echo"))
|
||||
p.cand = None
|
||||
|
||||
def _make(self, prompt: str, cmd: str, source: str, via: str) -> AuditCmd:
|
||||
cmd = cmd[: self.max_command]
|
||||
base = prompt.rstrip()
|
||||
# Keep one space after user@host style prompts; network CLIs print none.
|
||||
sep = " " if (len(prompt) > len(base) and (self.linux_like or "@" in base)) else ""
|
||||
return AuditCmd(
|
||||
command=(base + sep + cmd)[: self.max_command],
|
||||
prompt=base,
|
||||
input=cmd,
|
||||
source=source,
|
||||
via=via,
|
||||
)
|
||||
|
||||
def expire(self, now: float | None = None) -> list[AuditCmd]:
|
||||
"""Settle Tab candidates and resolve Enters whose echo never arrived."""
|
||||
out: list[AuditCmd] = []
|
||||
if not self.pending:
|
||||
return out
|
||||
t = time.monotonic() if now is None else now
|
||||
with self.lock:
|
||||
while self.pending:
|
||||
p = self.pending[0]
|
||||
if p.cand is not None and (t - p.cand_t) >= _TAB_SETTLE_SEC:
|
||||
self._finish_candidate(p, out)
|
||||
continue
|
||||
if (t - p.t) >= self.expire_sec:
|
||||
self.pending.popleft()
|
||||
cmd = self._fallback(p)
|
||||
if cmd is not None:
|
||||
out.append(cmd)
|
||||
continue
|
||||
break
|
||||
return out
|
||||
|
||||
def _fallback(self, p: _Pending) -> AuditCmd | None:
|
||||
parsed = split_prompt(p.snapshot, p.prompt_hint, linux_like=self.linux_like)
|
||||
if parsed and parsed[1]:
|
||||
return self._make(parsed[0], parsed[1], p.source, "snapshot")
|
||||
for hint in reversed(p.client_hints):
|
||||
parsed = split_prompt(hint, p.prompt_hint, linux_like=self.linux_like)
|
||||
if parsed and parsed[1]:
|
||||
return self._make(parsed[0], parsed[1], p.source, "client_hint")
|
||||
# Last resort: what the operator typed, under the prompt that was on screen.
|
||||
typed = p.typed.replace("\t", "").strip()
|
||||
if typed and p.prompt_hint:
|
||||
return self._make(p.prompt_hint, typed, p.source, "typed")
|
||||
return None
|
||||
|
||||
def flush_all(self) -> list[AuditCmd]:
|
||||
"""Session closing: resolve every still-pending Enter with the best fallback."""
|
||||
out: list[AuditCmd] = []
|
||||
with self.lock:
|
||||
while self.pending:
|
||||
p = self.pending.popleft()
|
||||
if p.cand is not None:
|
||||
out.append(self._make(p.cand[0], p.cand[1], p.source, "device_echo"))
|
||||
continue
|
||||
cmd = self._fallback(p)
|
||||
if cmd is not None:
|
||||
out.append(cmd)
|
||||
return out
|
||||
|
||||
# ----- operator input -------------------------------------------------- #
|
||||
def on_stdin(
|
||||
self,
|
||||
lines: list[str],
|
||||
*,
|
||||
saw_input: bool,
|
||||
had_enter: bool,
|
||||
client_hints: list[str] | None = None,
|
||||
source: str = "stdin",
|
||||
now: float | None = None,
|
||||
abort: bool = False,
|
||||
abort_text: str = "",
|
||||
) -> list[AuditCmd]:
|
||||
"""Register keystrokes already split into completed ``lines`` by the caller.
|
||||
|
||||
``saw_input``: payload had anything besides bare Enter (captures the prompt).
|
||||
``had_enter``: payload contained at least one line terminator.
|
||||
Returns immediately-final records (password redaction); echo-resolved ones
|
||||
come from ``feed``.
|
||||
"""
|
||||
out: list[AuditCmd] = []
|
||||
t = time.monotonic() if now is None else now
|
||||
with self.lock:
|
||||
row = self.screen.text()
|
||||
if abort:
|
||||
parsed_row = split_prompt(row, self.prompt_hint, linux_like=self.linux_like)
|
||||
if parsed_row and parsed_row[1]:
|
||||
self.abandoned.append((row, t))
|
||||
# What we sent before the Ctrl-C may not have been echoed yet.
|
||||
if abort_text.strip() and self.prompt_hint:
|
||||
self.abandoned.append((self.prompt_hint + abort_text.strip(), t))
|
||||
del self.abandoned[:-6]
|
||||
if saw_input and not self.line_started:
|
||||
# First keystroke of a line: the row is still just the prompt. Prefer the
|
||||
# regex-detected prompt so typed echo never leaks into the hint.
|
||||
if row and not looks_like_password_row(row):
|
||||
known = split_prompt(row, "", linux_like=self.linux_like)
|
||||
if known:
|
||||
self.prompt_hint = known[0]
|
||||
elif row[-1] in "#>$%]":
|
||||
self.prompt_hint = row # exotic CLI prompt
|
||||
else:
|
||||
# "Username:" / "Continue? [y/n]:" … a dialog, not a CLI prompt.
|
||||
self.prompt_hint = ""
|
||||
self.line_started = True
|
||||
if not had_enter:
|
||||
return out
|
||||
hints = [h for h in (client_hints or []) if h]
|
||||
if self.password_mode or looks_like_password_row(row):
|
||||
# Typed secret: never keep it, record a redacted marker.
|
||||
n = max(1, len(lines) or 1)
|
||||
out.extend(AuditCmd(command="***", redacted=True, source=source, via="password") for _ in range(n))
|
||||
self.password_mode = False
|
||||
self.line_started = False
|
||||
return out
|
||||
# No completed text: a bare Enter, possibly executing a recalled history line
|
||||
# whose redraw has not reached us yet.
|
||||
queue_lines = list(lines) or [""]
|
||||
for i, typed in enumerate(queue_lines):
|
||||
self.pending.append(
|
||||
_Pending(
|
||||
t=t,
|
||||
typed=typed,
|
||||
snapshot=row if i == 0 else "",
|
||||
prompt_hint=self.prompt_hint,
|
||||
client_hints=hints[-1:] if i == len(queue_lines) - 1 else [],
|
||||
source=source,
|
||||
bare=not typed,
|
||||
has_tab="\t" in typed,
|
||||
)
|
||||
)
|
||||
self.line_started = False
|
||||
# Bound runaway growth (device never echoing).
|
||||
while len(self.pending) > 64:
|
||||
self.pending.popleft()
|
||||
return out
|
||||
|
||||
def immediate(self, lines: list[str], source: str) -> list[AuditCmd]:
|
||||
"""Trusted non-interactive input (post-login commands): record as typed."""
|
||||
return [
|
||||
AuditCmd(command=ln[: self.max_command], input=ln[: self.max_command], source=source, via="trusted")
|
||||
for ln in lines
|
||||
if ln.strip()
|
||||
]
|
||||
|
||||
def current_row(self) -> str:
|
||||
with self.lock:
|
||||
return self.screen.text()
|
||||
|
||||
def seed(self, text: str) -> None:
|
||||
"""Prime the row from the login transcript (prompt printed before the reader)."""
|
||||
self.feed(text)
|
||||
|
||||
def set_row(self, text: str) -> None:
|
||||
"""Force the current input row (e.g. after a duplicate prompt was dropped)."""
|
||||
with self.lock:
|
||||
self.screen.reset_row()
|
||||
self.screen._carry = ""
|
||||
self.screen.feed_row(text)
|
||||
self.line_started = False
|
||||
|
|
@ -3,7 +3,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import codecs
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
|
@ -209,8 +208,9 @@ def api_quick_connect(
|
|||
user_override = str(body.username or "").strip() or None
|
||||
if str(body.password or "").strip():
|
||||
pwd_override = str(body.password)
|
||||
# Open the terminal immediately; WS reports auth progress and retryable failures.
|
||||
async_connect = bool(body.async_connect)
|
||||
# SSH with password: wait for auth so wrong credentials can re-prompt (SecureCRT-like).
|
||||
wait_for_auth = proto == "ssh" and bool(pwd_override)
|
||||
async_connect = bool(body.async_connect) and not wait_for_auth
|
||||
try:
|
||||
session = create_session(
|
||||
db,
|
||||
|
|
@ -687,7 +687,6 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
|
|||
replay_text = _decode_bytes(raw_boot, "utf-8")
|
||||
sess.bootstrap_replayed = True
|
||||
else:
|
||||
sess.flush_session_log()
|
||||
replay_text = read_session_log_tail(session_id, max_bytes=49152)
|
||||
if replay_text:
|
||||
replay_bytes = _encode_text(replay_text, "utf-8")
|
||||
|
|
@ -701,72 +700,59 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
|
|||
_log.debug("webcrt bootstrap/replay send failed session=%s", session_id, exc_info=True)
|
||||
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
io_pool = webcrt_io_executor()
|
||||
wake = asyncio.Event()
|
||||
# Keystrokes are written as they arrive (no timer delay); whatever piled up while a
|
||||
# write was in flight is coalesced into the next one.
|
||||
stdin_q: asyncio.Queue[tuple[str, str | None]] = asyncio.Queue()
|
||||
stdin_buf: list[str] = []
|
||||
stdin_audit_lines: list[str] = []
|
||||
stdin_flush_task: asyncio.Task[None] | None = None
|
||||
|
||||
async def stdin_writer() -> None:
|
||||
while True:
|
||||
data, audit = await stdin_q.get()
|
||||
parts = [data]
|
||||
audits = [audit] if audit else []
|
||||
taken = 1
|
||||
while True:
|
||||
async def flush_stdin() -> None:
|
||||
nonlocal stdin_buf, stdin_audit_lines
|
||||
if not stdin_buf:
|
||||
return
|
||||
data = "".join(stdin_buf)
|
||||
audit_lines = list(stdin_audit_lines)
|
||||
stdin_buf = []
|
||||
stdin_audit_lines = []
|
||||
try:
|
||||
d2, a2 = stdin_q.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
parts.append(d2)
|
||||
taken += 1
|
||||
if a2:
|
||||
audits.append(a2)
|
||||
blob = "".join(parts)
|
||||
try:
|
||||
await loop.run_in_executor(
|
||||
io_pool,
|
||||
lambda b=blob, al=audits: sess.write_stdin(
|
||||
b,
|
||||
audit_lines=al or None,
|
||||
audit_line=al[-1] if al else None,
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
webcrt_io_executor(),
|
||||
lambda: sess.write_stdin(
|
||||
data,
|
||||
audit_lines=audit_lines or None,
|
||||
audit_line=audit_lines[-1] if audit_lines else None,
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
try:
|
||||
await websocket.send_json(
|
||||
{"type": "status", "state": "error", "message": f"write_failed:{exc}"}
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
stop.set()
|
||||
wake.set()
|
||||
try:
|
||||
await websocket.close(code=4500)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
for _ in range(taken):
|
||||
stdin_q.task_done()
|
||||
|
||||
async def drain_stdin() -> None:
|
||||
try:
|
||||
await asyncio.wait_for(stdin_q.join(), timeout=5.0)
|
||||
except Exception:
|
||||
pass
|
||||
async def schedule_stdin_flush() -> None:
|
||||
await asyncio.sleep(0.008)
|
||||
await flush_stdin()
|
||||
|
||||
async def pump_stdout() -> None:
|
||||
"""Event-driven stdout pump: the reader thread wakes us, we drain and send at once."""
|
||||
sess.set_notifier(loop, wake)
|
||||
dec = None
|
||||
if _normalize_encoding(sess.encoding) != "utf-8":
|
||||
dec = codecs.getincrementaldecoder("gbk")(errors="replace")
|
||||
loop = asyncio.get_running_loop()
|
||||
pending: list[bytes] = []
|
||||
last_flush = loop.time()
|
||||
|
||||
def _to_browser(raw: bytes) -> bytes:
|
||||
if dec is None:
|
||||
def _to_browser_bytes(raw: bytes) -> bytes:
|
||||
if _normalize_encoding(sess.encoding) == "utf-8":
|
||||
return raw
|
||||
return dec.decode(raw).encode("utf-8", errors="replace")
|
||||
return _decode_bytes(raw, sess.encoding).encode("utf-8", errors="replace")
|
||||
|
||||
async def _flush_pending() -> bool:
|
||||
nonlocal pending, last_flush
|
||||
if not pending:
|
||||
return True
|
||||
blob = _to_browser_bytes(b"".join(pending))
|
||||
pending = []
|
||||
last_flush = loop.time()
|
||||
try:
|
||||
await websocket.send_bytes(blob)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def _notify_queue_drops() -> None:
|
||||
try:
|
||||
|
|
@ -787,23 +773,22 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
wake.set() # output queued before the pump started
|
||||
try:
|
||||
while not stop.is_set():
|
||||
await wake.wait()
|
||||
wake.clear()
|
||||
while not stop.is_set():
|
||||
status, data = sess.drain_stdout(attach_gen)
|
||||
if status == "stale":
|
||||
return
|
||||
if data:
|
||||
try:
|
||||
await websocket.send_bytes(_to_browser(data))
|
||||
except Exception:
|
||||
# Longer block is cheap now (Condition wait); cuts executor churn when idle.
|
||||
chunk = await loop.run_in_executor(
|
||||
webcrt_io_executor(), lambda: sess.take_stdout(attach_gen, timeout=0.2)
|
||||
)
|
||||
if chunk == "stale":
|
||||
break
|
||||
if chunk == "empty":
|
||||
if pending and (loop.time() - last_flush) >= 0.016:
|
||||
if not await _flush_pending():
|
||||
stop.set()
|
||||
return
|
||||
break
|
||||
await _notify_queue_drops()
|
||||
if status == "eof":
|
||||
continue
|
||||
if chunk is None:
|
||||
await _flush_pending()
|
||||
await _notify_queue_drops()
|
||||
stop.set()
|
||||
try:
|
||||
|
|
@ -816,19 +801,20 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
|
|||
)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
if status != "data":
|
||||
break
|
||||
finally:
|
||||
sess.clear_notifier(wake)
|
||||
pending.append(chunk)
|
||||
if sum(len(p) for p in pending) >= 8192 or (loop.time() - last_flush) >= 0.016:
|
||||
if not await _flush_pending():
|
||||
stop.set()
|
||||
break
|
||||
await _notify_queue_drops()
|
||||
|
||||
reader_task = asyncio.create_task(pump_stdout())
|
||||
writer_task = asyncio.create_task(stdin_writer())
|
||||
if sess.needs_live_prompt:
|
||||
sess.needs_live_prompt = False
|
||||
try:
|
||||
await loop.run_in_executor(
|
||||
io_pool,
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
webcrt_io_executor(),
|
||||
lambda: sess.write_stdin("\r", audit_source="prompt_sync"),
|
||||
)
|
||||
except Exception:
|
||||
|
|
@ -844,7 +830,9 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
|
|||
text = _decode_bytes(bytes(msg_raw["bytes"]), sess.encoding)
|
||||
except Exception:
|
||||
continue
|
||||
stdin_q.put_nowait((text, None))
|
||||
stdin_buf.append(text)
|
||||
if stdin_flush_task is None or stdin_flush_task.done():
|
||||
stdin_flush_task = asyncio.create_task(schedule_stdin_flush())
|
||||
continue
|
||||
raw = msg_raw.get("text")
|
||||
if raw is None:
|
||||
|
|
@ -859,16 +847,23 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
|
|||
if data is None:
|
||||
continue
|
||||
audit_raw = msg.get("audit_line")
|
||||
audit = str(audit_raw).strip()[:512] if audit_raw is not None and str(audit_raw).strip() else None
|
||||
stdin_q.put_nowait((str(data), audit))
|
||||
if audit_raw is not None and str(audit_raw).strip():
|
||||
stdin_audit_lines.append(str(audit_raw).strip()[:512])
|
||||
stdin_buf.append(str(data))
|
||||
# Coalesce high-frequency keystrokes briefly.
|
||||
if len(stdin_buf) >= 8:
|
||||
if stdin_flush_task and not stdin_flush_task.done():
|
||||
stdin_flush_task.cancel()
|
||||
await flush_stdin()
|
||||
elif stdin_flush_task is None or stdin_flush_task.done():
|
||||
stdin_flush_task = asyncio.create_task(schedule_stdin_flush())
|
||||
elif mtype == "resize":
|
||||
cols = int(msg.get("cols") or sess.cols)
|
||||
rows = int(msg.get("rows") or sess.rows)
|
||||
await loop.run_in_executor(io_pool, sess.resize, cols, rows)
|
||||
await asyncio.get_running_loop().run_in_executor(webcrt_io_executor(), sess.resize, cols, rows)
|
||||
elif mtype == "break":
|
||||
await drain_stdin()
|
||||
try:
|
||||
await loop.run_in_executor(io_pool, sess.send_break)
|
||||
await asyncio.get_running_loop().run_in_executor(webcrt_io_executor(), sess.send_break)
|
||||
except Exception as exc:
|
||||
await websocket.send_json(
|
||||
{"type": "status", "state": "error", "message": f"break_failed:{exc}"}
|
||||
|
|
@ -878,7 +873,7 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
|
|||
await websocket.send_json({"type": "pong"})
|
||||
elif mtype == "close":
|
||||
stop.set()
|
||||
await drain_stdin()
|
||||
await flush_stdin()
|
||||
close_session(
|
||||
session_id,
|
||||
reason="client_close",
|
||||
|
|
@ -891,17 +886,16 @@ async def websocket_session(websocket: WebSocket, session_id: str) -> None:
|
|||
_log.exception("webcrt ws error session=%s", session_id)
|
||||
finally:
|
||||
stop.set()
|
||||
wake.set()
|
||||
if stdin_flush_task and not stdin_flush_task.done():
|
||||
stdin_flush_task.cancel()
|
||||
try:
|
||||
await drain_stdin()
|
||||
await flush_stdin()
|
||||
except Exception:
|
||||
pass
|
||||
writer_task.cancel()
|
||||
reader_task.cancel()
|
||||
for t in (writer_task, reader_task):
|
||||
try:
|
||||
await t
|
||||
except BaseException:
|
||||
await reader_task
|
||||
except Exception:
|
||||
pass
|
||||
# Keep device session so UI reconnect / remount can re-attach.
|
||||
if get_session(session_id) is not None:
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue