mirror of
https://github.com/hansjone/netx.git
synced 2026-10-19 04:23:23 +08:00
Harden config sync defaults, crash resume, and network nav collapse.
Disable auto-sync by default, enforce single-flight cycles with crash requeue, delay new scheduled runs after restart, and make the network sidebar collapsible. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
0eead0e662
commit
f5b4a14c0f
14 changed files with 387 additions and 99 deletions
|
|
@ -75,6 +75,8 @@ class Settings(BaseSettings):
|
||||||
# Config sync (periodic running-config backup into DB)
|
# Config sync (periodic running-config backup into DB)
|
||||||
config_sync_scheduler_enabled: bool = True
|
config_sync_scheduler_enabled: bool = True
|
||||||
config_sync_scheduler_tick_sec: int = 60
|
config_sync_scheduler_tick_sec: int = 60
|
||||||
|
# After process start / unexpected restart, wait before any scheduled sync.
|
||||||
|
config_sync_startup_grace_sec: int = 3600
|
||||||
# Managed NE exec: max CLI commands per request (lab can raise; hard-capped in ne_exec).
|
# Managed NE exec: max CLI commands per request (lab can raise; hard-capped in ne_exec).
|
||||||
ne_exec_max_commands: int = 5
|
ne_exec_max_commands: int = 5
|
||||||
# WebCRT interactive terminal sessions
|
# WebCRT interactive terminal sessions
|
||||||
|
|
|
||||||
|
|
@ -3,66 +3,112 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from .config_sync_runner import dispatch_cycle
|
from .config_sync_runner import dispatch_cycle
|
||||||
from .config_sync_service import finalize_cycle, sync_cycle_progress
|
from .config_sync_service import finalize_cycle, sync_cycle_progress
|
||||||
from .models import ConfigSyncCycle, ConfigSyncTask
|
from .models import ConfigSyncCycle, ConfigSyncTask
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
_log = logging.getLogger("netx.config_sync.recovery")
|
_log = logging.getLogger("netx.config_sync.recovery")
|
||||||
|
|
||||||
|
_ACTIVE = ("running", "paused", "pending")
|
||||||
|
|
||||||
|
|
||||||
|
def _close_cycle(db: Session, cycle: ConfigSyncCycle, *, reason: str) -> None:
|
||||||
|
"""Fail in-flight work and mark cycle terminal so it cannot block forever."""
|
||||||
|
cycle_id = str(cycle.id)
|
||||||
|
now = datetime.utcnow()
|
||||||
|
for task in (
|
||||||
|
db.query(ConfigSyncTask)
|
||||||
|
.filter(
|
||||||
|
ConfigSyncTask.cycle_id == cycle_id,
|
||||||
|
ConfigSyncTask.status.in_(("running", "pending")),
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
):
|
||||||
|
was_running = str(task.status) == "running"
|
||||||
|
task.status = "fail" if was_running else "cancelled"
|
||||||
|
task.message = reason
|
||||||
|
task.ended_at = now
|
||||||
|
db.commit()
|
||||||
|
sync_cycle_progress(db, cycle_id)
|
||||||
|
db.refresh(cycle)
|
||||||
|
cycle.status = "fail"
|
||||||
|
cycle.error_message = reason
|
||||||
|
cycle.ended_at = now
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
def recover_config_sync_on_startup(db: Session) -> int:
|
def recover_config_sync_on_startup(db: Session) -> int:
|
||||||
"""
|
"""
|
||||||
Mark orphaned running tasks as fail(orphan_recovered), then resume pending
|
Resume at most one interrupted cycle after process restart.
|
||||||
tasks for cycles still marked running/paused.
|
|
||||||
|
Rules:
|
||||||
|
- Only one active cycle (running/pending/paused) may exist; older actives are closed.
|
||||||
|
- Orphan ``running`` tasks are re-queued to ``pending`` and continued (crash 续跑).
|
||||||
|
- ``paused`` cycles stay paused (no auto dispatch) but still occupy the single-flight slot.
|
||||||
|
- New scheduled cycles remain blocked while this active cycle exists.
|
||||||
"""
|
"""
|
||||||
cycles = (
|
cycles = (
|
||||||
db.query(ConfigSyncCycle)
|
db.query(ConfigSyncCycle)
|
||||||
.filter(ConfigSyncCycle.status.in_(("running", "paused", "pending")))
|
.filter(ConfigSyncCycle.status.in_(_ACTIVE))
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
resumed = 0
|
cycles = sorted(cycles, key=lambda c: c.created_at or datetime.min)
|
||||||
for cycle in cycles:
|
if not cycles:
|
||||||
cycle_id = str(cycle.id)
|
return 0
|
||||||
orphans = (
|
|
||||||
db.query(ConfigSyncTask)
|
# Single-flight hygiene: keep newest, close older interrupted cycles.
|
||||||
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status == "running")
|
primary = cycles[-1]
|
||||||
.all()
|
for stale in cycles[:-1]:
|
||||||
|
_log.warning(
|
||||||
|
"config_sync recovery closing older active cycle=%s (keep=%s)",
|
||||||
|
stale.id,
|
||||||
|
primary.id,
|
||||||
)
|
)
|
||||||
for task in orphans:
|
_close_cycle(db, stale, reason="superseded_active_cycle")
|
||||||
task.status = "fail"
|
|
||||||
task.message = "orphan_recovered"
|
|
||||||
task.ended_at = datetime.utcnow()
|
|
||||||
if orphans:
|
|
||||||
db.commit()
|
|
||||||
_log.info("config_sync recovery cycle=%s orphaned_tasks=%s", cycle_id, len(orphans))
|
|
||||||
|
|
||||||
sync_cycle_progress(db, cycle_id)
|
cycle_id = str(primary.id)
|
||||||
db.refresh(cycle)
|
orphans = (
|
||||||
|
db.query(ConfigSyncTask)
|
||||||
|
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status == "running")
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for task in orphans:
|
||||||
|
task.status = "pending"
|
||||||
|
task.message = "requeued_after_restart"
|
||||||
|
task.started_at = None
|
||||||
|
task.ended_at = None
|
||||||
|
if orphans:
|
||||||
|
db.commit()
|
||||||
|
_log.info("config_sync recovery cycle=%s requeued_orphans=%s", cycle_id, len(orphans))
|
||||||
|
|
||||||
if str(cycle.status) == "paused":
|
sync_cycle_progress(db, cycle_id)
|
||||||
continue
|
db.refresh(primary)
|
||||||
|
|
||||||
pending = (
|
if str(primary.status) == "paused":
|
||||||
db.query(ConfigSyncTask)
|
_log.info("config_sync recovery cycle=%s stays paused (blocks new cycles)", cycle_id)
|
||||||
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status == "pending")
|
return 0
|
||||||
.count()
|
|
||||||
)
|
|
||||||
if pending <= 0:
|
|
||||||
if str(cycle.status) in ("running", "pending"):
|
|
||||||
finalize_cycle(db, cycle_id)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if str(cycle.status) == "pending":
|
pending = (
|
||||||
cycle.status = "running"
|
db.query(ConfigSyncTask)
|
||||||
if not cycle.started_at:
|
.filter(ConfigSyncTask.cycle_id == cycle_id, ConfigSyncTask.status == "pending")
|
||||||
cycle.started_at = datetime.utcnow()
|
.all()
|
||||||
db.commit()
|
)
|
||||||
|
if not pending:
|
||||||
|
finalize_cycle(db, cycle_id)
|
||||||
|
_log.info("config_sync recovery cycle=%s finalized (no pending)", cycle_id)
|
||||||
|
return 0
|
||||||
|
|
||||||
n = dispatch_cycle(cycle_id)
|
if str(primary.status) == "pending":
|
||||||
resumed += n
|
primary.status = "running"
|
||||||
_log.info("config_sync recovery resumed cycle=%s pending=%s", cycle_id, n)
|
if not primary.started_at:
|
||||||
return resumed
|
primary.started_at = datetime.utcnow()
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
task_ids = [str(t.id) for t in pending]
|
||||||
|
n = dispatch_cycle(cycle_id)
|
||||||
|
_log.info("config_sync recovery resumed cycle=%s pending=%s dispatched=%s", cycle_id, len(task_ids), n)
|
||||||
|
return n
|
||||||
|
|
|
||||||
|
|
@ -70,7 +70,7 @@ def _update_task(task_id: str, **fields: Any) -> None:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
def _update_task(task_id: str, **fields: Any) -> None:
|
def _claim_task(cycle_id: str, task_id: str) -> bool:
|
||||||
for attempt in range(10):
|
for attempt in range(10):
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from .config import settings
|
from .config import settings
|
||||||
|
|
@ -22,20 +22,50 @@ from .models import ConfigSyncCycle, ConfigSyncTask
|
||||||
_log = logging.getLogger("netx.config_sync.scheduler")
|
_log = logging.getLogger("netx.config_sync.scheduler")
|
||||||
_stop = threading.Event()
|
_stop = threading.Event()
|
||||||
_thread: threading.Thread | None = None
|
_thread: threading.Thread | None = None
|
||||||
|
_BOOT_MONO = time.monotonic()
|
||||||
|
|
||||||
|
|
||||||
def _utcnow() -> datetime:
|
def _utcnow() -> datetime:
|
||||||
return datetime.utcnow()
|
return datetime.utcnow()
|
||||||
|
|
||||||
|
|
||||||
|
def startup_grace_remaining_sec() -> float:
|
||||||
|
grace = max(0, int(settings.config_sync_startup_grace_sec or 0))
|
||||||
|
elapsed = time.monotonic() - _BOOT_MONO
|
||||||
|
return max(0.0, float(grace) - elapsed)
|
||||||
|
|
||||||
|
|
||||||
|
def in_startup_grace() -> bool:
|
||||||
|
return startup_grace_remaining_sec() > 0
|
||||||
|
|
||||||
|
|
||||||
|
def startup_grace_until() -> datetime | None:
|
||||||
|
rem = startup_grace_remaining_sec()
|
||||||
|
if rem <= 0:
|
||||||
|
return None
|
||||||
|
return _utcnow() + timedelta(seconds=rem)
|
||||||
|
|
||||||
|
|
||||||
def try_start_scheduled_cycle() -> str | None:
|
def try_start_scheduled_cycle() -> str | None:
|
||||||
"""Create and dispatch a scheduled cycle if policy is due. Returns cycle id or None."""
|
"""Create and dispatch a scheduled cycle if policy is due. Returns cycle id or None.
|
||||||
|
|
||||||
|
Never starts while another cycle is active (running/pending/paused), including
|
||||||
|
a cycle being resumed after crash. Startup grace only delays *new* scheduled runs.
|
||||||
|
"""
|
||||||
|
if in_startup_grace():
|
||||||
|
return None
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
try:
|
try:
|
||||||
policy = ensure_policy(db)
|
policy = ensure_policy(db)
|
||||||
if not policy.enabled:
|
if not policy.enabled:
|
||||||
return None
|
return None
|
||||||
if has_running_cycle(db):
|
active = has_running_cycle(db)
|
||||||
|
if active:
|
||||||
|
_log.debug(
|
||||||
|
"config_sync schedule skip: active cycle=%s status=%s",
|
||||||
|
active.id,
|
||||||
|
active.status,
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
due = next_due_at(db, policy)
|
due = next_due_at(db, policy)
|
||||||
if due is not None and due > _utcnow():
|
if due is not None and due > _utcnow():
|
||||||
|
|
@ -85,7 +115,8 @@ def try_start_scheduled_cycle() -> str | None:
|
||||||
|
|
||||||
def _loop() -> None:
|
def _loop() -> None:
|
||||||
tick = max(15, int(settings.config_sync_scheduler_tick_sec or 60))
|
tick = max(15, int(settings.config_sync_scheduler_tick_sec or 60))
|
||||||
_log.info("config_sync scheduler started tick=%ss", tick)
|
grace = max(0, int(settings.config_sync_startup_grace_sec or 0))
|
||||||
|
_log.info("config_sync scheduler started tick=%ss startup_grace=%ss", tick, grace)
|
||||||
while not _stop.is_set():
|
while not _stop.is_set():
|
||||||
try:
|
try:
|
||||||
if bool(settings.config_sync_scheduler_enabled):
|
if bool(settings.config_sync_scheduler_enabled):
|
||||||
|
|
|
||||||
|
|
@ -47,7 +47,7 @@ def _utcnow() -> datetime:
|
||||||
def ensure_policy(db: Session) -> ConfigSyncPolicy:
|
def ensure_policy(db: Session) -> ConfigSyncPolicy:
|
||||||
row = db.get(ConfigSyncPolicy, POLICY_ID)
|
row = db.get(ConfigSyncPolicy, POLICY_ID)
|
||||||
if row is None:
|
if row is None:
|
||||||
row = ConfigSyncPolicy(id=POLICY_ID)
|
row = ConfigSyncPolicy(id=POLICY_ID, enabled=False)
|
||||||
db.add(row)
|
db.add(row)
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(row)
|
db.refresh(row)
|
||||||
|
|
@ -202,15 +202,21 @@ def expand_targets(db: Session, policy: ConfigSyncPolicy) -> list[dict[str, str]
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def has_running_cycle(db: Session) -> ConfigSyncCycle | None:
|
def has_active_cycle(db: Session) -> ConfigSyncCycle | None:
|
||||||
|
"""Any non-terminal cycle occupies the single-flight slot (incl. paused)."""
|
||||||
return (
|
return (
|
||||||
db.query(ConfigSyncCycle)
|
db.query(ConfigSyncCycle)
|
||||||
.filter(ConfigSyncCycle.status.in_(("running", "pending")))
|
.filter(ConfigSyncCycle.status.in_(("running", "pending", "paused")))
|
||||||
.order_by(ConfigSyncCycle.created_at.desc())
|
.order_by(ConfigSyncCycle.created_at.desc())
|
||||||
.first()
|
.first()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def has_running_cycle(db: Session) -> ConfigSyncCycle | None:
|
||||||
|
"""Backward-compatible alias: treat paused as active so a new cycle cannot start."""
|
||||||
|
return has_active_cycle(db)
|
||||||
|
|
||||||
|
|
||||||
def last_finished_cycle(db: Session) -> ConfigSyncCycle | None:
|
def last_finished_cycle(db: Session) -> ConfigSyncCycle | None:
|
||||||
return (
|
return (
|
||||||
db.query(ConfigSyncCycle)
|
db.query(ConfigSyncCycle)
|
||||||
|
|
@ -224,6 +230,8 @@ def next_due_at(db: Session, policy: ConfigSyncPolicy | None = None) -> datetime
|
||||||
pol = policy or ensure_policy(db)
|
pol = policy or ensure_policy(db)
|
||||||
if not pol.enabled:
|
if not pol.enabled:
|
||||||
return None
|
return None
|
||||||
|
from .config_sync_scheduler import startup_grace_until
|
||||||
|
|
||||||
last = (
|
last = (
|
||||||
db.query(ConfigSyncCycle)
|
db.query(ConfigSyncCycle)
|
||||||
.filter(ConfigSyncCycle.status == "success", ConfigSyncCycle.ended_at.isnot(None))
|
.filter(ConfigSyncCycle.status == "success", ConfigSyncCycle.ended_at.isnot(None))
|
||||||
|
|
@ -232,8 +240,14 @@ def next_due_at(db: Session, policy: ConfigSyncPolicy | None = None) -> datetime
|
||||||
)
|
)
|
||||||
days = max(1, int(pol.interval_days or 3))
|
days = max(1, int(pol.interval_days or 3))
|
||||||
if last and last.ended_at:
|
if last and last.ended_at:
|
||||||
return last.ended_at + timedelta(days=days)
|
due = last.ended_at + timedelta(days=days)
|
||||||
return _utcnow()
|
else:
|
||||||
|
# Never synced successfully: do not fire immediately on enable / first boot.
|
||||||
|
due = _utcnow() + timedelta(days=days)
|
||||||
|
grace_until = startup_grace_until()
|
||||||
|
if grace_until is not None and due < grace_until:
|
||||||
|
return grace_until
|
||||||
|
return due
|
||||||
|
|
||||||
|
|
||||||
def create_cycle(db: Session, body: ConfigSyncCycleCreate) -> ConfigSyncCycleOut:
|
def create_cycle(db: Session, body: ConfigSyncCycleCreate) -> ConfigSyncCycleOut:
|
||||||
|
|
|
||||||
|
|
@ -840,7 +840,7 @@ def on_startup() -> None:
|
||||||
ensure_policy(db)
|
ensure_policy(db)
|
||||||
cfg_resumed = recover_config_sync_on_startup(db)
|
cfg_resumed = recover_config_sync_on_startup(db)
|
||||||
if cfg_resumed:
|
if cfg_resumed:
|
||||||
_schedule_log.info("startup: resumed %s pending config_sync tasks", cfg_resumed)
|
_schedule_log.info("startup: resumed %s config_sync task(s) from interrupted cycle", cfg_resumed)
|
||||||
except Exception:
|
except Exception:
|
||||||
_schedule_log.exception("startup: ne collection / config_sync recovery failed")
|
_schedule_log.exception("startup: ne collection / config_sync recovery failed")
|
||||||
finally:
|
finally:
|
||||||
|
|
|
||||||
|
|
@ -492,7 +492,7 @@ class ConfigSyncPolicy(Base):
|
||||||
__tablename__ = "config_sync_policy"
|
__tablename__ = "config_sync_policy"
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, default=1)
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
enabled: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||||
interval_days: Mapped[int] = mapped_column(Integer, default=3)
|
interval_days: Mapped[int] = mapped_column(Integer, default=3)
|
||||||
concurrency: Mapped[int] = mapped_column(Integer, default=5)
|
concurrency: Mapped[int] = mapped_column(Integer, default=5)
|
||||||
scope_mode: Mapped[str] = mapped_column(String(32), default="all") # all | selected
|
scope_mode: Mapped[str] = mapped_column(String(32), default="all") # all | selected
|
||||||
|
|
|
||||||
|
|
@ -211,33 +211,81 @@ class ConfigSyncRecoveryTests(unittest.TestCase):
|
||||||
@patch("netx_api.config_sync_recovery.dispatch_cycle", return_value=2)
|
@patch("netx_api.config_sync_recovery.dispatch_cycle", return_value=2)
|
||||||
@patch("netx_api.config_sync_recovery.finalize_cycle")
|
@patch("netx_api.config_sync_recovery.finalize_cycle")
|
||||||
@patch("netx_api.config_sync_recovery.sync_cycle_progress")
|
@patch("netx_api.config_sync_recovery.sync_cycle_progress")
|
||||||
def test_orphan_running_tasks_marked_fail(self, _sync, _fin, dispatch):
|
def test_requeues_orphans_and_resumes(self, _sync, _fin, dispatch):
|
||||||
cycle = MagicMock()
|
cycle = MagicMock()
|
||||||
cycle.id = "c1"
|
cycle.id = "c1"
|
||||||
cycle.status = "running"
|
cycle.status = "running"
|
||||||
|
cycle.created_at = datetime.now(timezone.utc)
|
||||||
cycle.started_at = datetime.now(timezone.utc)
|
cycle.started_at = datetime.now(timezone.utc)
|
||||||
|
cycle.error_message = ""
|
||||||
|
cycle.ended_at = None
|
||||||
|
|
||||||
orphan = MagicMock()
|
orphan = MagicMock()
|
||||||
orphan.status = "running"
|
orphan.status = "running"
|
||||||
orphan.message = ""
|
orphan.message = ""
|
||||||
|
orphan.started_at = datetime.now(timezone.utc)
|
||||||
orphan.ended_at = None
|
orphan.ended_at = None
|
||||||
|
|
||||||
|
pending = MagicMock()
|
||||||
|
pending.status = "pending"
|
||||||
|
pending.id = "t2"
|
||||||
|
|
||||||
db = MagicMock()
|
db = MagicMock()
|
||||||
db.query.side_effect = [
|
db.query.side_effect = [
|
||||||
MagicMock(filter=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[cycle])))),
|
MagicMock(filter=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[cycle])))),
|
||||||
MagicMock(filter=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[orphan])))),
|
MagicMock(filter=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[orphan])))),
|
||||||
MagicMock(filter=MagicMock(return_value=MagicMock(count=MagicMock(return_value=2)))),
|
MagicMock(
|
||||||
|
filter=MagicMock(
|
||||||
|
return_value=MagicMock(all=MagicMock(return_value=[orphan, pending]))
|
||||||
|
)
|
||||||
|
),
|
||||||
]
|
]
|
||||||
db.refresh = MagicMock()
|
db.refresh = MagicMock()
|
||||||
|
|
||||||
resumed = recover_config_sync_on_startup(db)
|
resumed = recover_config_sync_on_startup(db)
|
||||||
|
|
||||||
self.assertEqual(orphan.status, "fail")
|
self.assertEqual(orphan.status, "pending")
|
||||||
self.assertEqual(orphan.message, "orphan_recovered")
|
self.assertEqual(orphan.message, "requeued_after_restart")
|
||||||
self.assertIsNotNone(orphan.ended_at)
|
self.assertIsNone(orphan.started_at)
|
||||||
self.assertEqual(resumed, 2)
|
self.assertEqual(resumed, 2)
|
||||||
dispatch.assert_called_once_with("c1")
|
dispatch.assert_called_once_with("c1")
|
||||||
|
|
||||||
|
@patch("netx_api.config_sync_recovery.dispatch_cycle")
|
||||||
|
@patch("netx_api.config_sync_recovery.finalize_cycle")
|
||||||
|
@patch("netx_api.config_sync_recovery.sync_cycle_progress")
|
||||||
|
def test_closes_older_active_keeps_newest(self, _sync, _fin, dispatch):
|
||||||
|
old = MagicMock()
|
||||||
|
old.id = "old"
|
||||||
|
old.status = "running"
|
||||||
|
old.created_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||||
|
old.error_message = ""
|
||||||
|
old.ended_at = None
|
||||||
|
|
||||||
|
new = MagicMock()
|
||||||
|
new.id = "new"
|
||||||
|
new.status = "running"
|
||||||
|
new.created_at = datetime(2026, 1, 2, tzinfo=timezone.utc)
|
||||||
|
new.started_at = datetime.now(timezone.utc)
|
||||||
|
new.error_message = ""
|
||||||
|
new.ended_at = None
|
||||||
|
|
||||||
|
db = MagicMock()
|
||||||
|
db.query.side_effect = [
|
||||||
|
MagicMock(filter=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[old, new])))),
|
||||||
|
MagicMock(filter=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))),
|
||||||
|
MagicMock(filter=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))),
|
||||||
|
MagicMock(filter=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))),
|
||||||
|
]
|
||||||
|
db.refresh = MagicMock()
|
||||||
|
dispatch.return_value = 0
|
||||||
|
|
||||||
|
recover_config_sync_on_startup(db)
|
||||||
|
|
||||||
|
self.assertEqual(old.status, "fail")
|
||||||
|
self.assertEqual(old.error_message, "superseded_active_cycle")
|
||||||
|
_fin.assert_called()
|
||||||
|
dispatch.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
|
||||||
|
|
@ -98,6 +98,10 @@ src/
|
||||||
- 存储:PostgreSQL `ne_config_snapshot` / `ne_config_history`(zlib BYTEA)
|
- 存储:PostgreSQL `ne_config_snapshot` / `ne_config_history`(zlib BYTEA)
|
||||||
- 范围:ManagedNE + UME CLI 目标;厂商固定只读命令矩阵
|
- 范围:ManagedNE + UME CLI 目标;厂商固定只读命令矩阵
|
||||||
- 调度:`NETX_CONFIG_SYNC_SCHEDULER_ENABLED`(默认开),周期天数策略可配(默认 3 天)
|
- 调度:`NETX_CONFIG_SYNC_SCHEDULER_ENABLED`(默认开),周期天数策略可配(默认 3 天)
|
||||||
|
- 默认策略:`enabled=false`(首次无自动任务,需在页面手动开启周期调度或点「立即同步」)
|
||||||
|
- 单飞:同一时刻只允许一个 `running|pending|paused` 周期;上轮未结束时不会开启新周期
|
||||||
|
- 崩溃续跑:启动时把中断的 `running` 任务重新入队并继续,占用单飞槽位,避免与新周期重叠
|
||||||
|
- 进程启动宽限:`NETX_CONFIG_SYNC_STARTUP_GRACE_SEC`(默认 3600)仅约束**新建**自动周期,不影响续跑
|
||||||
- 前端:`/network/tasks/config-sync`(看板)+ `/network/configs`(查看)
|
- 前端:`/network/tasks/config-sync`(看板)+ `/network/configs`(查看)
|
||||||
|
|
||||||
## WebCRT
|
## WebCRT
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,8 @@ const en = {
|
||||||
configSync: "Config sync",
|
configSync: "Config sync",
|
||||||
portTraffic: "Port traffic",
|
portTraffic: "Port traffic",
|
||||||
},
|
},
|
||||||
|
collapseNav: "Collapse sidebar",
|
||||||
|
expandNav: "Expand sidebar",
|
||||||
placeholder: {
|
placeholder: {
|
||||||
title: "Coming soon",
|
title: "Coming soon",
|
||||||
body: "This capability is not available yet.",
|
body: "This capability is not available yet.",
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,8 @@ const zh = {
|
||||||
configSync: "配置同步",
|
configSync: "配置同步",
|
||||||
portTraffic: "端口流量监控",
|
portTraffic: "端口流量监控",
|
||||||
},
|
},
|
||||||
|
collapseNav: "折叠侧栏",
|
||||||
|
expandNav: "展开侧栏",
|
||||||
placeholder: {
|
placeholder: {
|
||||||
title: "功能规划中",
|
title: "功能规划中",
|
||||||
body: "该能力尚未开放。",
|
body: "该能力尚未开放。",
|
||||||
|
|
|
||||||
|
|
@ -2670,6 +2670,89 @@ pre {
|
||||||
background: #f7f9fc;
|
background: #f7f9fc;
|
||||||
border-right: 1px solid #d0d7e2;
|
border-right: 1px solid #d0d7e2;
|
||||||
min-height: 0;
|
min-height: 0;
|
||||||
|
transition: width 0.18s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-shell.is-nav-collapsed .network-nav {
|
||||||
|
width: 48px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-nav__head {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
padding: 10px 8px 8px 12px;
|
||||||
|
border-bottom: 1px solid #e2e8f0;
|
||||||
|
min-height: 44px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-shell.is-nav-collapsed .network-nav__head {
|
||||||
|
justify-content: center;
|
||||||
|
padding: 10px 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-nav__title {
|
||||||
|
flex: 1;
|
||||||
|
min-width: 0;
|
||||||
|
font-size: 13px;
|
||||||
|
font-weight: 700;
|
||||||
|
color: #0f172a;
|
||||||
|
white-space: nowrap;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
|
||||||
|
.app-main .network-nav__collapse-btn,
|
||||||
|
.network-nav__collapse-btn {
|
||||||
|
flex-shrink: 0;
|
||||||
|
width: 28px;
|
||||||
|
height: 28px;
|
||||||
|
padding: 0;
|
||||||
|
border: 1px solid #cbd5e1;
|
||||||
|
border-radius: 6px;
|
||||||
|
background: #fff;
|
||||||
|
color: #475569;
|
||||||
|
cursor: pointer;
|
||||||
|
line-height: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.app-main .network-nav__collapse-btn:hover,
|
||||||
|
.network-nav__collapse-btn:hover {
|
||||||
|
background: #eef2f7;
|
||||||
|
color: #0f172a;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-nav__rail {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
padding: 10px 4px;
|
||||||
|
overflow: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-nav__rail-link {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
width: 32px;
|
||||||
|
height: 32px;
|
||||||
|
border-radius: 8px;
|
||||||
|
text-decoration: none;
|
||||||
|
color: #475569;
|
||||||
|
font-size: 12px;
|
||||||
|
font-weight: 700;
|
||||||
|
background: transparent;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-nav__rail-link:hover {
|
||||||
|
background: #eef2f7;
|
||||||
|
color: #0f172a;
|
||||||
|
}
|
||||||
|
|
||||||
|
.network-nav__rail-link.is-active {
|
||||||
|
background: #e3effc;
|
||||||
|
color: #1565c0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.network-nav__scroll {
|
.network-nav__scroll {
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,7 @@ export function ConfigSyncPage() {
|
||||||
const [taskStatus, setTaskStatus] = useState("");
|
const [taskStatus, setTaskStatus] = useState("");
|
||||||
const [taskKeyword, setTaskKeyword] = useState("");
|
const [taskKeyword, setTaskKeyword] = useState("");
|
||||||
|
|
||||||
const [enabled, setEnabled] = useState(true);
|
const [enabled, setEnabled] = useState(false);
|
||||||
const [intervalDays, setIntervalDays] = useState(3);
|
const [intervalDays, setIntervalDays] = useState(3);
|
||||||
const [concurrency, setConcurrency] = useState(5);
|
const [concurrency, setConcurrency] = useState(5);
|
||||||
const [scopeMode, setScopeMode] = useState<"all" | "selected">("all");
|
const [scopeMode, setScopeMode] = useState<"all" | "selected">("all");
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,8 @@ import { NavLink, Outlet, useLocation } from "react-router-dom";
|
||||||
import { NETWORK_NAV, type NetworkNavGroupId } from "../../config/networkNav";
|
import { NETWORK_NAV, type NetworkNavGroupId } from "../../config/networkNav";
|
||||||
import { useI18n } from "../../i18n";
|
import { useI18n } from "../../i18n";
|
||||||
|
|
||||||
|
const SIDEBAR_KEY = "netx.network.sidebarCollapsed";
|
||||||
|
|
||||||
function groupContainsPath(groupId: NetworkNavGroupId, pathname: string): boolean {
|
function groupContainsPath(groupId: NetworkNavGroupId, pathname: string): boolean {
|
||||||
const group = NETWORK_NAV.find((g) => g.id === groupId);
|
const group = NETWORK_NAV.find((g) => g.id === groupId);
|
||||||
if (!group) return false;
|
if (!group) return false;
|
||||||
|
|
@ -11,9 +13,18 @@ function groupContainsPath(groupId: NetworkNavGroupId, pathname: string): boolea
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function readSidebarCollapsed(): boolean {
|
||||||
|
try {
|
||||||
|
return localStorage.getItem(SIDEBAR_KEY) === "1";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function NetworkLayout() {
|
export function NetworkLayout() {
|
||||||
const { t } = useI18n();
|
const { t } = useI18n();
|
||||||
const { pathname } = useLocation();
|
const { pathname } = useLocation();
|
||||||
|
const [sidebarCollapsed, setSidebarCollapsed] = useState(readSidebarCollapsed);
|
||||||
const [openGroups, setOpenGroups] = useState<Record<NetworkNavGroupId, boolean>>(() => {
|
const [openGroups, setOpenGroups] = useState<Record<NetworkNavGroupId, boolean>>(() => {
|
||||||
const init = {} as Record<NetworkNavGroupId, boolean>;
|
const init = {} as Record<NetworkNavGroupId, boolean>;
|
||||||
for (const g of NETWORK_NAV) {
|
for (const g of NETWORK_NAV) {
|
||||||
|
|
@ -35,50 +46,95 @@ export function NetworkLayout() {
|
||||||
setOpenGroups((prev) => ({ ...prev, [id]: !prev[id] }));
|
setOpenGroups((prev) => ({ ...prev, [id]: !prev[id] }));
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const toggleSidebar = () => {
|
||||||
|
setSidebarCollapsed((prev) => {
|
||||||
|
const next = !prev;
|
||||||
|
try {
|
||||||
|
localStorage.setItem(SIDEBAR_KEY, next ? "1" : "0");
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="network-shell">
|
<div className={`network-shell${sidebarCollapsed ? " is-nav-collapsed" : ""}`}>
|
||||||
<aside className="network-nav" aria-label={t("network.title")}>
|
<aside className="network-nav" aria-label={t("network.title")}>
|
||||||
<nav className="network-nav__scroll">
|
<div className="network-nav__head">
|
||||||
{NETWORK_NAV.map((group) => {
|
{!sidebarCollapsed ? <span className="network-nav__title">{t("network.title")}</span> : null}
|
||||||
const open = Boolean(openGroups[group.id]);
|
<button
|
||||||
const activeGroup = groupContainsPath(group.id, pathname);
|
type="button"
|
||||||
return (
|
className="network-nav__collapse-btn"
|
||||||
<div
|
title={sidebarCollapsed ? t("network.expandNav") : t("network.collapseNav")}
|
||||||
key={group.id}
|
aria-label={sidebarCollapsed ? t("network.expandNav") : t("network.collapseNav")}
|
||||||
className={`network-nav__group${open ? " is-open" : ""}${activeGroup ? " is-active-group" : ""}`}
|
aria-expanded={!sidebarCollapsed}
|
||||||
>
|
onClick={toggleSidebar}
|
||||||
<button
|
>
|
||||||
type="button"
|
{sidebarCollapsed ? "»" : "«"}
|
||||||
className="network-nav__group-toggle"
|
</button>
|
||||||
aria-expanded={open}
|
</div>
|
||||||
onClick={() => toggleGroup(group.id)}
|
{!sidebarCollapsed ? (
|
||||||
|
<nav className="network-nav__scroll">
|
||||||
|
{NETWORK_NAV.map((group) => {
|
||||||
|
const open = Boolean(openGroups[group.id]);
|
||||||
|
const activeGroup = groupContainsPath(group.id, pathname);
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={group.id}
|
||||||
|
className={`network-nav__group${open ? " is-open" : ""}${activeGroup ? " is-active-group" : ""}`}
|
||||||
>
|
>
|
||||||
<span className="network-nav__chevron" aria-hidden>
|
<button
|
||||||
{open ? "▾" : "▸"}
|
type="button"
|
||||||
</span>
|
className="network-nav__group-toggle"
|
||||||
<span className="network-nav__group-label">{t(group.labelKey)}</span>
|
aria-expanded={open}
|
||||||
</button>
|
onClick={() => toggleGroup(group.id)}
|
||||||
{open ? (
|
>
|
||||||
<ul className="network-nav__list">
|
<span className="network-nav__chevron" aria-hidden>
|
||||||
{group.items.map((item) => (
|
{open ? "▾" : "▸"}
|
||||||
<li key={item.id}>
|
</span>
|
||||||
<NavLink
|
<span className="network-nav__group-label">{t(group.labelKey)}</span>
|
||||||
to={item.path}
|
</button>
|
||||||
className={({ isActive }) =>
|
{open ? (
|
||||||
`network-nav__link${isActive ? " is-active" : ""}`
|
<ul className="network-nav__list">
|
||||||
}
|
{group.items.map((item) => (
|
||||||
end={item.path === "/network/devices"}
|
<li key={item.id}>
|
||||||
>
|
<NavLink
|
||||||
{t(item.labelKey)}
|
to={item.path}
|
||||||
</NavLink>
|
className={({ isActive }) =>
|
||||||
</li>
|
`network-nav__link${isActive ? " is-active" : ""}`
|
||||||
))}
|
}
|
||||||
</ul>
|
end={item.path === "/network/devices"}
|
||||||
) : null}
|
>
|
||||||
</div>
|
{t(item.labelKey)}
|
||||||
);
|
</NavLink>
|
||||||
})}
|
</li>
|
||||||
</nav>
|
))}
|
||||||
|
</ul>
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</nav>
|
||||||
|
) : (
|
||||||
|
<nav className="network-nav__rail" aria-label={t("network.title")}>
|
||||||
|
{NETWORK_NAV.flatMap((group) =>
|
||||||
|
group.items.map((item) => (
|
||||||
|
<NavLink
|
||||||
|
key={item.id}
|
||||||
|
to={item.path}
|
||||||
|
title={t(item.labelKey)}
|
||||||
|
className={({ isActive }) =>
|
||||||
|
`network-nav__rail-link${isActive ? " is-active" : ""}`
|
||||||
|
}
|
||||||
|
end={item.path === "/network/devices"}
|
||||||
|
>
|
||||||
|
{t(item.labelKey).slice(0, 1)}
|
||||||
|
</NavLink>
|
||||||
|
)),
|
||||||
|
)}
|
||||||
|
</nav>
|
||||||
|
)}
|
||||||
</aside>
|
</aside>
|
||||||
<div className="network-main">
|
<div className="network-main">
|
||||||
<Outlet />
|
<Outlet />
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue