From c181158209ed107f63d92e16f75649396437ecb7 Mon Sep 17 00:00:00 2001 From: oliver Date: Tue, 1 Sep 2026 21:41:05 +0800 Subject: [PATCH] Drop HTTP polling noise from operation audit by default. Persist only business events and mutating/failed HTTP calls, and default the audit UI to a business view with an explicit HTTP filter. Co-authored-by: Cursor --- netx_api/audit_async.py | 90 ++++++++++++++++++++++++++++++++----- netx_api/auth_router.py | 2 + netx_api/auth_service.py | 16 +++++++ netx_api/config.py | 3 +- tests/test_audit_noise.py | 57 +++++++++++++++++++++++ web/src/i18n/en.ts | 3 ++ web/src/i18n/zh.ts | 3 ++ web/src/pages/AuditPage.tsx | 46 ++++++++++++++----- 8 files changed, 196 insertions(+), 24 deletions(-) create mode 100644 tests/test_audit_noise.py diff --git a/netx_api/audit_async.py b/netx_api/audit_async.py index 85ecb08..ebd7ba8 100644 --- a/netx_api/audit_async.py +++ b/netx_api/audit_async.py @@ -18,9 +18,32 @@ _lock = threading.Lock() _counter = 0 _dropped = 0 +# Middleware-tagged HTTP wrappers that duplicate semantic business audits. +_MIDDLEWARE_NOISE_ACTIONS = frozenset( + { + "audit.list", + "webcrt.get", + "webcrt.post", + "webcrt.put", + "webcrt.patch", + "webcrt.delete", + "users.get", + "api_tokens.get", + } +) + +_ALWAYS_KEEP_PREFIXES = ( + "auth.", + "users.", + "api_tokens.", + "ne.", + "port_traffic.", + "config_sync.", +) + def _sample_ok() -> bool: - """When sample_n > 1, keep 1/N of http.* audits; always keep auth/security actions.""" + """When sample_n > 1, keep 1/N of leftover generic events.""" global _counter n = int(getattr(settings, "audit_sample_n", 1) or 1) if n <= 1: @@ -29,6 +52,60 @@ def _sample_ok() -> bool: return (_counter % n) == 0 +def audit_should_persist( + *, + action: str, + method: str = "", + status_code: int = 0, + path: str = "", +) -> bool: + """Decide whether a candidate audit event is worth writing. + + Policy: + - Always keep auth / users / tokens / NE / port_traffic / config_sync / semantic webcrt. + - Always keep HTTP failures (status >= 400), except pure list noise. + - Drop successful GET/HEAD/OPTIONS ``http.*`` (page polling). + - Always keep mutating ``http.*`` (POST/PUT/PATCH/DELETE) — no sampling. + - Drop middleware ``webcrt.{method}`` / ``audit.list`` (covered by business events). + """ + del path # reserved for future path allow/deny lists + act = str(action or "") + method_u = str(method or "").upper() + code = int(status_code or 0) + + if act in _MIDDLEWARE_NOISE_ACTIONS: + return False + + if act.startswith("webcrt.session_") or act == "webcrt.command": + return True + + if any(act.startswith(p) for p in _ALWAYS_KEEP_PREFIXES): + return True + + if code >= 400: + return True + + if act.startswith("http."): + if method_u in ("GET", "HEAD", "OPTIONS") or act in ("http.get", "http.head", "http.options"): + return False + if method_u in ("POST", "PUT", "PATCH", "DELETE") or act in ( + "http.post", + "http.put", + "http.patch", + "http.delete", + ): + return True + return _sample_ok() + + # e.g. ume.token.* — keep writes, drop successful reads + if act.startswith("ume."): + if method_u in ("GET", "HEAD", "OPTIONS"): + return False + return True + + return _sample_ok() + + def audit_queue_status() -> dict[str, int]: q = _q return { @@ -99,16 +176,7 @@ def enqueue_audit( ) -> None: global _dropped act = str(action or "") - # Always persist auth / security / device-op events. - if ( - act.startswith("auth.") - or act.startswith("users.") - or act.startswith("api_tokens.") - or act.startswith("webcrt.") - or act.startswith("ne.") - ): - pass - elif act.startswith("http.") and not _sample_ok(): + if not audit_should_persist(action=act, method=method, status_code=status_code, path=path): return payload = { "action": act, diff --git a/netx_api/auth_router.py b/netx_api/auth_router.py index 8d75733..b1fe839 100644 --- a/netx_api/auth_router.py +++ b/netx_api/auth_router.py @@ -394,6 +394,7 @@ def api_audit_logs( page_size: int = Query(default=50, ge=1, le=200), username: str = Query(default=""), action: str = Query(default=""), + exclude_noise: bool = Query(default=True), ) -> dict[str, Any]: return list_audit_logs( db, @@ -402,6 +403,7 @@ def api_audit_logs( page_size=page_size, username=username, action=action, + exclude_noise=exclude_noise, ) diff --git a/netx_api/auth_service.py b/netx_api/auth_service.py index cab6f97..bbb0a08 100644 --- a/netx_api/auth_service.py +++ b/netx_api/auth_service.py @@ -723,6 +723,7 @@ def list_audit_logs( page_size: int = 50, username: str = "", action: str = "", + exclude_noise: bool = True, ) -> dict[str, Any]: page = max(1, int(page or 1)) page_size = max(1, min(200, int(page_size or 50))) @@ -733,6 +734,20 @@ def list_audit_logs( q = q.filter(AuditLog.actor_username == username.strip()) if action.strip(): q = q.filter(AuditLog.action.ilike(f"%{action.strip()}%")) + if exclude_noise: + # Hide historical HTTP polling + middleware wrappers; keep semantic webcrt.*. + noise_actions = ( + "audit.list", + "webcrt.get", + "webcrt.post", + "webcrt.put", + "webcrt.patch", + "webcrt.delete", + "users.get", + "api_tokens.get", + ) + q = q.filter(~AuditLog.action.like("http.%")) + q = q.filter(~AuditLog.action.in_(noise_actions)) total = int(q.count()) rows = ( q.order_by(AuditLog.ts.desc()) @@ -761,4 +776,5 @@ def list_audit_logs( "page": page, "page_size": page_size, "items": items, + "exclude_noise": bool(exclude_noise), } diff --git a/netx_api/config.py b/netx_api/config.py index 1e81650..1968b5a 100644 --- a/netx_api/config.py +++ b/netx_api/config.py @@ -167,7 +167,8 @@ class Settings(BaseSettings): docs_enabled: bool = False # Refuse start when bind host is non-loopback and insecure defaults remain. allow_insecure_defaults: bool = False - # Async audit writer; sample_n>1 keeps 1/N of generic http.* events. + # Async audit writer. Successful GET http.* are dropped; mutating http.* always kept. + # sample_n only applies to leftover unclassified events. audit_async: bool = True audit_sample_n: int = 5 # Prefer Alembic on API start; brownfield patches live in schema_patches + revisions. diff --git a/tests/test_audit_noise.py b/tests/test_audit_noise.py new file mode 100644 index 0000000..e77b76f --- /dev/null +++ b/tests/test_audit_noise.py @@ -0,0 +1,57 @@ +"""Tests for audit noise filtering (persist policy + list exclude_noise).""" + +from __future__ import annotations + +import unittest + +from netx_api.audit_async import audit_should_persist + + +class AuditShouldPersistTests(unittest.TestCase): + def test_drop_successful_http_get(self) -> None: + self.assertFalse( + audit_should_persist(action="http.get", method="GET", status_code=200, path="/v1/topology") + ) + + def test_keep_failed_http_get(self) -> None: + self.assertTrue( + audit_should_persist(action="http.get", method="GET", status_code=500, path="/v1/topology") + ) + + def test_keep_http_mutations_without_sampling(self) -> None: + for method, action in ( + ("POST", "http.post"), + ("PUT", "http.put"), + ("PATCH", "http.patch"), + ("DELETE", "http.delete"), + ): + self.assertTrue( + audit_should_persist(action=action, method=method, status_code=200, path="/v1/x"), + msg=action, + ) + + def test_drop_middleware_noise(self) -> None: + for action in ("audit.list", "webcrt.get", "webcrt.post", "users.get", "api_tokens.get"): + self.assertFalse( + audit_should_persist(action=action, method="GET", status_code=200), + msg=action, + ) + + def test_keep_semantic_webcrt(self) -> None: + self.assertTrue(audit_should_persist(action="webcrt.session_created", status_code=0)) + self.assertTrue(audit_should_persist(action="webcrt.command", status_code=0)) + self.assertTrue(audit_should_persist(action="webcrt.session_closed", status_code=0)) + + def test_keep_business_prefixes(self) -> None: + self.assertTrue(audit_should_persist(action="auth.login", status_code=200)) + self.assertTrue(audit_should_persist(action="ne.exec", status_code=200)) + self.assertTrue(audit_should_persist(action="port_traffic.device.start", status_code=200)) + self.assertTrue(audit_should_persist(action="config_sync.start", status_code=200)) + self.assertTrue(audit_should_persist(action="users.create", status_code=200)) + + def test_drop_ume_token_get(self) -> None: + self.assertFalse(audit_should_persist(action="ume.token.get", method="GET", status_code=200)) + + +if __name__ == "__main__": + unittest.main() diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 78f13e8..75e0740 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -1871,10 +1871,13 @@ const en = { colDetail: "Detail", showDetail: "Expand", hideDetail: "Collapse", + filterBusiness: "Business", filterAll: "All", filterWebcrt: "Device terminal", filterNeExec: "Device exec", filterAuth: "Auth", + filterHttp: "HTTP", + noiseHint: "Page-refresh HTTP noise is hidden by default; writes and failed requests are kept.", summary: { connecting: "Connecting to {{device}}", loginOk: "Logged in to {{device}}", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index abed4f5..4ebf825 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -1849,10 +1849,13 @@ const zh = { colDetail: "详情", showDetail: "展开", hideDetail: "收起", + filterBusiness: "业务", filterAll: "全部", filterWebcrt: "设备终端", filterNeExec: "设备执行", filterAuth: "登录认证", + filterHttp: "HTTP", + noiseHint: "默认隐藏页面刷新类 HTTP 噪声；写操作与失败请求仍会保留。", summary: { connecting: "正在登录 {{device}}", loginOk: "登录 {{device}}", diff --git a/web/src/pages/AuditPage.tsx b/web/src/pages/AuditPage.tsx index 08fe923..b01eabe 100644 --- a/web/src/pages/AuditPage.tsx +++ b/web/src/pages/AuditPage.tsx @@ -17,7 +17,8 @@ type AuditItem = { detail: Record; }; -type QuickFilter = "" | "webcrt." | "ne.exec" | "auth."; +/** Quick filters map to action substring and/or exclude_noise flag. */ +type QuickFilter = "business" | "webcrt." | "ne.exec" | "auth." | "http." | "all"; function statusClass(code: number): string { if (code >= 500) return "pt-list-status--failed"; @@ -43,6 +44,7 @@ export function auditSummary( action: string, detail: Record, t: (key: string, vars?: Record) => string, + row?: Pick, ): string { const d = detail || {}; const device = deviceLabel(d); @@ -87,26 +89,41 @@ export function auditSummary( if (action.startsWith("auth.")) { return action.replace(/^auth\./, ""); } + if (action.startsWith("http.") || action.startsWith("ume.")) { + const method = row?.method || action.replace(/^http\./, "").toUpperCase(); + const path = row?.path || ""; + return path ? `${method} ${path}` : method; + } return ""; } +function quickToQuery(quick: QuickFilter): { action: string; excludeNoise: boolean } { + if (quick === "business") return { action: "", excludeNoise: true }; + if (quick === "all") return { action: "", excludeNoise: false }; + if (quick === "http.") return { action: "http.", excludeNoise: false }; + return { action: quick, excludeNoise: true }; +} + export function AuditPage() { const { t } = useI18n(); const { ready, isAdmin } = useAuth(); const [page, setPage] = useState(1); const [username, setUsername] = useState(""); const [action, setAction] = useState(""); - const [quick, setQuick] = useState(""); + const [quick, setQuick] = useState("business"); const [expanded, setExpanded] = useState(null); - const effectiveAction = action.trim() || quick; + const derived = quickToQuery(quick); + const effectiveAction = action.trim() || derived.action; + const excludeNoise = action.trim() ? false : derived.excludeNoise; const query = useQuery({ - queryKey: ["auditLogs", page, username, effectiveAction], + queryKey: ["auditLogs", page, username, effectiveAction, excludeNoise], queryFn: () => { const p = new URLSearchParams(); p.set("page", String(page)); p.set("page_size", "50"); + p.set("exclude_noise", excludeNoise ? "true" : "false"); if (username.trim()) p.set("username", username.trim()); if (effectiveAction) p.set("action", effectiveAction); return apiGet<{ total: number; page: number; page_size: number; items: AuditItem[] }>( @@ -119,12 +136,13 @@ export function AuditPage() { const items = useMemo(() => query.data?.items || [], [query.data]); const total = query.data?.total || 0; const pages = Math.max(1, Math.ceil(total / 50)); - const hasFilters = Boolean(username.trim() || action.trim() || quick); + const hasFilters = Boolean(username.trim() || action.trim() || quick !== "business"); const setQuickFilter = (next: QuickFilter) => { setPage(1); setQuick(next); - if (next) setAction(""); + if (next !== "all" && next !== "business") setAction(""); + else setAction(""); }; return ( @@ -133,20 +151,24 @@ export function AuditPage() {

{t("audit.logsTitle")}

-

{isAdmin ? t("auth.auditHintAdmin") : t("auth.auditHintUser")}

+

+ {isAdmin ? t("auth.auditHintAdmin") : t("auth.auditHintUser")} {t("audit.noiseHint")} +

{( [ - ["", "audit.filterAll"], + ["business", "audit.filterBusiness"], ["webcrt.", "audit.filterWebcrt"], ["ne.exec", "audit.filterNeExec"], ["auth.", "audit.filterAuth"], + ["http.", "audit.filterHttp"], + ["all", "audit.filterAll"], ] as const ).map(([value, labelKey]) => (