Replace pandas with a small openpyxl/csv reader for imports.

pandas + numpy were ~60MB of the shipped venv for three call sites (alarm
Excel import, managed-NE import, import template). netx_api/tabular.py
reads/writes xlsx and csv and parses spreadsheet timestamps.

Checked against the pandas version on the same inputs: timestamp parsing
identical for 24 formats, managed-NE csv/xlsx import, stored rows and both
templates identical. Differences, all fixes:
- alarm codes in mixed text/number columns are "1001", not "1001.0"
- error row numbers are the real sheet rows (blank rows no longer shift them)
- GBK-encoded CSVs (Chinese Excel "Save as CSV") import instead of failing
- "2024年1月2日 03:04:05" style timestamps parse

Also: conftest compares parsed DB URLs (newer SQLAlchemy renders "C:" as
"C%3A").

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

netx_api/importer.py

@@ -1,29 +1,34 @@
 from __future__ import annotations
 
 import json
+import math
 from datetime import datetime, timezone
-from io import BytesIO
 from typing import Any
 
-import pandas as pd
 from sqlalchemy import func, select
 from sqlalchemy.orm import Session
 
 from .config import settings
 from .models import AlarmBatch, AlarmNorm, ImportErrorRow
 from .parser_config import ParserConfig
+from .tabular import parse_datetime, read_xlsx
+
+
+def _blank(v: Any) -> bool:
+    if v is None:
+        return True
+    if isinstance(v, float) and math.isnan(v):
+        return True
+    text = str(v).strip()
+    return not text or text.lower() == "nan"
 
 
 def _dt(v: Any) -> datetime | None:
-    if v is None:
+    if _blank(v):
         return None
-    text = str(v).strip()
-    if not text or text.lower() == "nan":
+    dt = parse_datetime(v)
+    if dt is None:
         return None
-    parsed = pd.to_datetime(text, errors="coerce")
-    if pd.isna(parsed):
-        return None
-    dt = parsed.to_pydatetime()
     # Convention: Excel source time is local timezone time.
     # Convert to UTC before persisting; keep DB as naive UTC for compatibility.
     if dt.tzinfo is None:
@@ -35,20 +40,15 @@ def _dt(v: Any) -> datetime | None:
 
 
 def _str(v: Any) -> str:
-    if v is None:
+    if _blank(v):
         return ""
-    text = str(v).strip()
-    if text.lower() == "nan":
-        return ""
-    return text
+    return str(v).strip()
 
 
 def _int(v: Any) -> int:
-    if v is None:
+    if _blank(v):
         return 0
     text = str(v).strip()
-    if not text or text.lower() == "nan":
-        return 0
     try:
         return int(float(text))
     except Exception:
@@ -56,8 +56,7 @@ def _int(v: Any) -> int:
 
 
 def import_alarm_excel(db: Session, filename: str, content: bytes, parser: ParserConfig) -> AlarmBatch:
-    df = pd.read_excel(BytesIO(content))
-    headers = [str(h) for h in df.columns]
+    headers, rows = read_xlsx(content)
 
     col_alarm_time = parser.resolve_col(headers, "alarm_time")
     col_clear_time = parser.resolve_col(headers, "clear_time")
@@ -80,7 +79,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
         source_file=filename,
         parser_version=parser.parser_version,
         dict_version=parser.dict_version,
-        total_rows=int(len(df.index)),
+        total_rows=len(rows),
         status="processing",
     )
     db.add(batch)
@@ -89,8 +88,8 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
     success_rows = 0
     failed_rows = 0
 
-    for idx, row in df.iterrows():
-        row_no = int(idx) + 2  # Excel header is row 1
+    for row_no, row in rows:  # row_no = sheet row (header is row 1)
+        raw_json = json.dumps(row, ensure_ascii=False, default=str)
         alarm_time = _dt(row.get(col_alarm_time)) if col_alarm_time else None
         raw_severity = _str(row.get(col_severity)) if col_severity else ""
         alarm_code = _str(row.get(col_alarm_code)) if col_alarm_code else ""
@@ -111,7 +110,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
                     batch_id=batch.batch_id,
                     row_no=row_no,
                     reason="missing_or_invalid_alarm_time",
-                    raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str),
+                    raw_json=raw_json,
                 )
             )
             continue
@@ -122,7 +121,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
                     batch_id=batch.batch_id,
                     row_no=row_no,
                     reason="missing_ne_name_and_ne_id",
-                    raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str),
+                    raw_json=raw_json,
                 )
             )
             continue
@@ -151,7 +150,7 @@ def import_alarm_excel(db: Session, filename: str, content: bytes, parser: Parse
                 vendor=settings.vendor,
                 source_type=settings.source_type,
                 source_file=filename,
-                raw_json=json.dumps(row.to_dict(), ensure_ascii=False, default=str),
+                raw_json=raw_json,
             )
         )
         success_rows += 1

netx_api/ne_service_common.py

@@ -2,10 +2,10 @@
 from __future__ import annotations
 
 from datetime import datetime
+import math
 import re
 from typing import Any
 
-import pandas as pd
 from fastapi import HTTPException
 from sqlalchemy.orm import Session
 
@@ -199,7 +199,7 @@ def _parse_import_bool(value: Any) -> bool:
 
 
 def _import_cell_str(value: Any) -> str:
-    if value is None or (isinstance(value, float) and pd.isna(value)):
+    if value is None or (isinstance(value, float) and math.isnan(value)):
         return ""
     text = str(value).strip()
     return "" if text.lower() == "nan" else text

netx_api/ne_service_import.py

@@ -1,14 +1,13 @@
 """Managed NE Excel import and UME inventory sync into managed_ne."""
 from __future__ import annotations
 
-from io import BytesIO
 from typing import Any
 
-import pandas as pd
 from fastapi import HTTPException
 from sqlalchemy.orm import Session
 
 from .models import ManagedNE, UmeInventoryNE
+from .tabular import read_table, write_csv, write_xlsx
 from .ne_crypto import CredentialCryptoError, encrypt_secret
 from .device_types import SUPPORTED_DEVICE_TYPES, SUPPORTED_VENDORS, is_placeholder_ne_source
 from .ne_schemas import (
@@ -151,46 +150,41 @@ def build_managed_ne_import_template(fmt: str = "xlsx") -> tuple[str, bytes, str
             "remark": "no direct password, use batch proxy",
         },
     ]
-    df = pd.DataFrame(rows, columns=list(IMPORT_COLUMNS))
-    buf = BytesIO()
+    headers = list(IMPORT_COLUMNS)
+    table = [[r.get(c, "") for c in headers] for r in rows]
     kind = str(fmt or "xlsx").strip().lower()
     if kind == "csv":
-        df.to_csv(buf, index=False, encoding="utf-8-sig")
         return (
             "managed_ne_import_template.csv",
-            buf.getvalue(),
+            write_csv(headers, table),
             "text/csv; charset=utf-8",
         )
-    device_types_df = pd.DataFrame({"device_type": list(SUPPORTED_DEVICE_TYPES)})
-    with pd.ExcelWriter(buf, engine="openpyxl") as writer:
-        df.to_excel(writer, sheet_name="import", index=False)
-        device_types_df.to_excel(writer, sheet_name="device_type", index=False)
+    payload = write_xlsx([
+        ("import", headers, table),
+        ("device_type", ["device_type"], [[t] for t in SUPPORTED_DEVICE_TYPES]),
+    ])
     return (
         "managed_ne_import_template.xlsx",
-        buf.getvalue(),
+        payload,
         "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
     )
 
 
 def import_managed_ne(db: Session, content: bytes, filename: str) -> ImportResult:
     _require_crypto()
-    name = str(filename or "").lower()
     try:
-        if name.endswith(".csv"):
-            df = pd.read_csv(BytesIO(content))
-        else:
-            df = pd.read_excel(BytesIO(content))
+        headers, raw_rows = read_table(content, filename)
     except Exception as exc:
         raise HTTPException(status_code=400, detail=f"import_parse_failed: {exc}") from exc
-    df.columns = [str(c).strip().lower() for c in df.columns]
-    missing = [c for c in _REQUIRED_IMPORT_COLUMNS if c not in df.columns]
+    columns = [h.strip().lower() for h in headers]
+    missing = [c for c in _REQUIRED_IMPORT_COLUMNS if c not in columns]
     if missing:
         raise HTTPException(status_code=400, detail=f"import_missing_columns: {','.join(missing)}")
     inserted = 0
     updated = 0
     failed: list[ImportFailure] = []
-    for idx, row in df.iterrows():
-        row_no = int(idx) + 2
+    for row_no, cells in raw_rows:
+        row = {k.strip().lower(): v for k, v in cells.items()}
         # Savepoint per row: a failing row must not leave a half-written NE behind.
         savepoint = db.begin_nested()
         try:
@@ -214,16 +208,16 @@ def import_managed_ne(db: Session, content: bytes, filename: str) -> ImportResul
                 failed.append(ImportFailure(row=row_no, reason="username_required"))
                 savepoint.rollback()
                 continue
-            port_raw = row.get("port", 22)
+            port_raw = row.get("port")
             try:
-                port = int(port_raw)
-            except (TypeError, ValueError):
+                port = int(float(port_raw)) if port_raw is not None else 22
+            except (TypeError, ValueError, OverflowError):
                 port = 22
             if not 1 <= port <= 65535:
                 failed.append(ImportFailure(row=row_no, reason="invalid_port"))
                 savepoint.rollback()
                 continue
-            protocol = _normalize_protocol(str(row.get("protocol", "ssh")))
+            protocol = _normalize_protocol(_import_cell_str(row.get("protocol")) or "ssh")
             display_name = _import_cell_str(row.get("name", "")) or ip
             vendor_raw = _import_cell_str(row.get("vendor", "")) or "Other"
             vendor = "Other"

netx_api/tabular.py

@@ -0,0 +1,151 @@
+"""Small xlsx/csv reader/writer for imports and templates (replaces pandas).
+
+Rows come back as ``(row_no, {header: value})`` where ``row_no`` is the 1-based
+sheet/file row (header = row 1), so import errors point at the row users see.
+Blank rows are skipped. Excel cells keep their openpyxl types (str / int /
+float / datetime / bool / None); integral floats become ints. CSV cells are
+strings, empty cells are None.
+"""
+from __future__ import annotations
+
+import csv
+import io
+import re
+from collections.abc import Iterable, Iterator
+from datetime import datetime
+from typing import Any
+
+from openpyxl import Workbook, load_workbook
+
+Row = tuple[int, dict[str, Any]]
+
+
+def _headers(raw: Iterable[Any]) -> list[str]:
+    out: list[str] = []
+    seen: dict[str, int] = {}
+    for i, h in enumerate(raw):
+        name = "" if h is None else str(h).strip()
+        if not name:
+            name = f"Unnamed: {i}"
+        n = seen.get(name, 0)
+        seen[name] = n + 1
+        out.append(name if n == 0 else f"{name}.{n}")
+    return out
+
+
+def _cell(v: Any) -> Any:
+    if isinstance(v, str):
+        return v if v.strip() else None
+    if isinstance(v, float) and v.is_integer():
+        return int(v)
+    return v
+
+
+def _rows(records: Iterator[tuple[int, list[Any]]]) -> tuple[list[str], list[Row]]:
+    headers: list[str] | None = None
+    rows: list[Row] = []
+    for row_no, values in records:
+        cells = [_cell(v) for v in values]
+        if all(c is None for c in cells):
+            continue
+        if headers is None:
+            headers = _headers(values)
+            continue
+        cells += [None] * (len(headers) - len(cells))
+        rows.append((row_no, dict(zip(headers, cells))))
+    return headers or [], rows
+
+
+def _decode(content: bytes) -> str:
+    try:
+        return content.decode("utf-8-sig")
+    except UnicodeDecodeError:
+        # CSVs saved by Chinese Excel default to GBK.
+        return content.decode("gb18030")
+
+
+def read_csv(content: bytes) -> tuple[list[str], list[Row]]:
+    reader = csv.reader(io.StringIO(_decode(content), newline=""))
+    return _rows((i, rec) for i, rec in enumerate(reader, start=1))
+
+
+def read_xlsx(content: bytes) -> tuple[list[str], list[Row]]:
+    wb = load_workbook(io.BytesIO(content), read_only=True, data_only=True)
+    try:
+        ws = wb.worksheets[0]
+        return _rows((i, list(rec)) for i, rec in enumerate(ws.iter_rows(values_only=True), start=1))
+    finally:
+        wb.close()
+
+
+def read_table(content: bytes, filename: str = "") -> tuple[list[str], list[Row]]:
+    if str(filename or "").lower().endswith(".csv"):
+        return read_csv(content)
+    return read_xlsx(content)
+
+
+def write_csv(headers: list[str], rows: Iterable[Iterable[Any]]) -> bytes:
+    buf = io.StringIO()
+    writer = csv.writer(buf, lineterminator="\n")
+    writer.writerow(headers)
+    for r in rows:
+        writer.writerow(["" if v is None else v for v in r])
+    # BOM so Excel opens UTF-8 (Chinese) correctly.
+    return buf.getvalue().encode("utf-8-sig")
+
+
+def write_xlsx(sheets: list[tuple[str, list[str], Iterable[Iterable[Any]]]]) -> bytes:
+    wb = Workbook()
+    wb.remove(wb.active)
+    for title, headers, rows in sheets:
+        ws = wb.create_sheet(title)
+        ws.append(headers)
+        for r in rows:
+            ws.append(list(r))
+    buf = io.BytesIO()
+    wb.save(buf)
+    return buf.getvalue()
+
+
+_DATE_FORMATS = (
+    "%Y-%m-%d %H:%M:%S",
+    "%Y-%m-%d %H:%M",
+    "%Y-%m-%d",
+    "%Y%m%d%H%M%S",
+    "%Y%m%d",
+    "%m/%d/%Y %H:%M:%S",
+    "%m/%d/%Y %H:%M",
+    "%m/%d/%Y",
+    "%d-%b-%Y %H:%M:%S",
+    "%b %d, %Y %H:%M:%S",
+)
+_CN_DATE = re.compile(r"(\d{4})年(\d{1,2})月(\d{1,2})日")
+
+
+def parse_datetime(v: Any) -> datetime | None:
+    """Best-effort timestamp parse for spreadsheet cells; None when unparseable."""
+    if v is None:
+        return None
+    if isinstance(v, datetime):
+        return v
+    text = str(v).strip()
+    if not text or text.lower() in ("nan", "nat", "none", "null"):
+        return None
+    text = _CN_DATE.sub(r"\1-\2-\3", text)
+    # 2024/1/2 03:04:05 -> 2024-1-2 ... ; zero-pad so fromisoformat/strptime accept it.
+    if re.match(r"^\d{4}/\d{1,2}/\d{1,2}", text):
+        text = text.replace("/", "-", 2)
+    m = re.match(r"^(\d{4})-(\d{1,2})-(\d{1,2})(.*)$", text)
+    if m:
+        text = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}{m.group(4)}"
+    iso = text[:-1] + "+00:00" if text.endswith(("Z", "z")) else text
+    try:
+        return datetime.fromisoformat(iso)
+    except ValueError:
+        pass
+    for fmt in _DATE_FORMATS:
+        try:
+            return datetime.strptime(text, fmt)
+        except ValueError:
+            continue
+    return None