hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-10 09:50:44 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 1

Improve WebCRT SFTP transfers and file-panel UX.

Browse source 
Stream large uploads/downloads with progress, raise the size cap, and support drag-drop folders plus fixed/resizable columns.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-08-02 01:10:30 +08:00
parent 7d8b698b91
commit d10ffed277
9 changed files with 769 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

3
netx_api/config.py
View file

@ -101,6 +101,9 @@ class Settings(BaseSettings):
webcrt_session_log_enabled: bool = True
# Reader: short blocking wait instead of fixed 40ms spin (seconds).
webcrt_reader_poll_sec: float = 0.01
# WebCRT SFTP transfer limits (streamed; default 512 MiB per file).
webcrt_sftp_max_file_bytes: int = 512 * 1024 * 1024
webcrt_sftp_chunk_bytes: int = 64 * 1024
# Local app login / audit (lab defaults; override in production)
auth_enabled: bool = True
# Stable default so JWT survives restarts without .env. Override in production.

48
netx_api/webcrt_router.py
View file

@ -213,19 +213,26 @@ def api_sftp_list(body: WebcrtSftpListBody, db: Session = Depends(get_db)) -> di
@router.post("/sftp/download")
def api_sftp_download(body: WebcrtSftpDownloadBody, db: Session = Depends(get_db)) -> Any:
from fastapi.responses import Response
from fastapi.responses import StreamingResponse
from .webcrt_sftp import sftp_download
from .webcrt_sftp import SftpDownloadStream
mid = str(body.ne_id or "").strip()
uid = str(body.ume_ne_id or "").strip()
if bool(mid) == bool(uid):
raise HTTPException(status_code=400, detail="exactly_one_of_ne_id_or_ume_ne_id_required")
data, filename = sftp_download(db, managed_ne_id=mid or None, ume_ne_id=uid or None, path=body.path)
return Response(
content=data,
stream = SftpDownloadStream(
db, managed_ne_id=mid or None, ume_ne_id=uid or None, path=body.path
).open()
headers = {
"Content-Disposition": stream.content_disposition(),
"Content-Length": str(int(stream.size)),
"Cache-Control": "no-store",
}
return StreamingResponse(
stream,
media_type="application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
headers=headers,
)
@ -237,22 +244,31 @@ async def api_sftp_upload(
remote_path: str = Form(...),
file: UploadFile = File(...),
) -> dict[str, Any]:
from .webcrt_sftp import sftp_upload
from starlette.concurrency import run_in_threadpool
from .config import settings
from .webcrt_sftp import sftp_upload_stream
mid = str(ne_id or "").strip()
uid = str(ume_ne_id or "").strip()
if bool(mid) == bool(uid):
raise HTTPException(status_code=400, detail="exactly_one_of_ne_id_or_ume_ne_id_required")
content = await file.read()
if len(content) > 8 * 1024 * 1024:
max_bytes = max(1, int(settings.webcrt_sftp_max_file_bytes or (512 * 1024 * 1024)))
expected = getattr(file, "size", None)
if expected is not None and int(expected) > max_bytes:
raise HTTPException(status_code=413, detail="sftp_file_too_large")
return sftp_upload(
db,
managed_ne_id=mid or None,
ume_ne_id=uid or None,
remote_path=remote_path,
data=content,
)
def _upload() -> dict[str, Any]:
return sftp_upload_stream(
db,
managed_ne_id=mid or None,
ume_ne_id=uid or None,
remote_path=remote_path,
reader=file.file,
expected_size=int(expected) if expected is not None else None,
)
return await run_in_threadpool(_upload)
@router.websocket("/sessions/{session_id}/ws")

215
netx_api/webcrt_sftp.py
View file

@ -7,9 +7,10 @@ import posixpath
import stat as statmod
import threading
import time
from contextlib import contextmanager
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass, field
from typing import Any, Iterator
from typing import Any, BinaryIO, Iterator
from urllib.parse import quote
import paramiko
from fastapi import HTTPException
@ -201,6 +202,43 @@ def _owner_group(attr: Any) -> tuple[str, str]:
return ("" if uid is None else str(uid), "" if gid is None else str(gid))
def _mkdir_p(sftp: Any, remote: str) -> None:
"""Create remote directory and parents (like mkdir -p)."""
path = _normalize_remote(remote, allow_dot=False)
if not path or path in (".", "/"):
return
to_create: list[str] = []
cur = path
while cur and cur not in (".", "/"):
to_create.append(cur)
parent = posixpath.dirname(cur)
if parent == cur:
break
cur = parent
for candidate in reversed(to_create):
try:
st = sftp.stat(candidate)
mode = int(getattr(st, "st_mode", 0) or 0)
if not statmod.S_ISDIR(mode):
raise HTTPException(status_code=400, detail=f"sftp_not_a_directory:{candidate}")
continue
except HTTPException:
raise
except Exception:
pass
try:
sftp.mkdir(candidate)
except Exception as exc:
# Race: another client created it; accept if it is now a directory.
try:
st = sftp.stat(candidate)
if statmod.S_ISDIR(int(getattr(st, "st_mode", 0) or 0)):
continue
except Exception:
pass
raise HTTPException(status_code=502, detail=f"sftp_mkdir_failed:{candidate}:{exc}") from exc
def sftp_list(
db: Session,
*,
@ -244,27 +282,148 @@ def sftp_list(
raise HTTPException(status_code=502, detail=f"sftp_list_failed:{exc}") from exc
def sftp_download(
def _sftp_max_file_bytes() -> int:
return max(1, int(settings.webcrt_sftp_max_file_bytes or (512 * 1024 * 1024)))
def _sftp_chunk_bytes() -> int:
return max(4 * 1024, int(settings.webcrt_sftp_chunk_bytes or (64 * 1024)))
def _content_disposition(filename: str) -> str:
name = str(filename or "download.bin").replace('"', "").replace("\r", "").replace("\n", "")
if not name:
name = "download.bin"
ascii_name = name.encode("ascii", "replace").decode("ascii") or "download.bin"
return f"attachment; filename=\"{ascii_name}\"; filename*=UTF-8''{quote(name)}"
class SftpDownloadStream:
"""Hold the SFTP channel open while StreamingResponse consumes chunks."""
def __init__(
self,
db: Session,
*,
managed_ne_id: str | None,
ume_ne_id: str | None,
path: str,
) -> None:
self._db = db
self._managed_ne_id = managed_ne_id
self._ume_ne_id = ume_ne_id
self.remote = _normalize_remote(path, allow_dot=False)
self.filename = "download.bin"
self.size = 0
self._stack: ExitStack | None = None
self._fh: Any = None
self._chunk = _sftp_chunk_bytes()
def open(self) -> "SftpDownloadStream":
if not self.remote or self.remote.endswith("/"):
raise HTTPException(status_code=400, detail="sftp_path_required")
stack = ExitStack()
try:
sftp, _device = stack.enter_context(
_sftp_client(self._db, managed_ne_id=self._managed_ne_id, ume_ne_id=self._ume_ne_id)
)
try:
st = sftp.stat(self.remote)
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_download_failed:{exc}") from exc
mode = int(getattr(st, "st_mode", 0) or 0)
if statmod.S_ISDIR(mode):
raise HTTPException(status_code=400, detail="sftp_path_is_directory")
size = int(getattr(st, "st_size", 0) or 0)
if size > _sftp_max_file_bytes():
raise HTTPException(status_code=413, detail="sftp_file_too_large")
self.size = max(0, size)
self.filename = posixpath.basename(self.remote) or "download.bin"
try:
self._fh = stack.enter_context(sftp.open(self.remote, "rb"))
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_download_failed:{exc}") from exc
except Exception:
stack.close()
raise
self._stack = stack
return self
def __iter__(self) -> Iterator[bytes]:
fh = self._fh
if fh is None:
return
try:
while True:
chunk = fh.read(self._chunk)
if not chunk:
break
yield chunk
finally:
self.close()
def close(self) -> None:
stack = self._stack
self._stack = None
self._fh = None
if stack is not None:
try:
stack.close()
except Exception:
pass
def content_disposition(self) -> str:
return _content_disposition(self.filename)
def sftp_upload_stream(
db: Session,
*,
managed_ne_id: str | None,
ume_ne_id: str | None,
path: str,
) -> tuple[bytes, str]:
remote = _normalize_remote(path, allow_dot=False)
if not remote or remote.endswith("/"):
remote_path: str,
reader: BinaryIO,
expected_size: int | None = None,
) -> dict[str, Any]:
remote = _normalize_remote(remote_path, allow_dot=False)
if not remote:
raise HTTPException(status_code=400, detail="sftp_path_required")
max_bytes = _sftp_max_file_bytes()
if expected_size is not None and int(expected_size) > max_bytes:
raise HTTPException(status_code=413, detail="sftp_file_too_large")
chunk_size = _sftp_chunk_bytes()
try:
with _sftp_client(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id) as (sftp, _device):
with sftp.open(remote, "rb") as fh:
data = fh.read(8 * 1024 * 1024 + 1)
if len(data) > 8 * 1024 * 1024:
raise HTTPException(status_code=413, detail="sftp_file_too_large")
return data, posixpath.basename(remote) or "download.bin"
with _sftp_client(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id) as (sftp, device):
parent = posixpath.dirname(remote)
if parent and parent not in (".", "/"):
_mkdir_p(sftp, parent)
written = 0
try:
with sftp.open(remote, "wb") as fh:
while True:
buf = reader.read(chunk_size)
if not buf:
break
written += len(buf)
if written > max_bytes:
raise HTTPException(status_code=413, detail="sftp_file_too_large")
fh.write(buf)
except HTTPException:
try:
sftp.remove(remote)
except Exception:
pass
raise
return {
"ok": True,
"ne_id": str(device.get("id") or ""),
"path": remote,
"size": written,
}
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_download_failed:{exc}") from exc
raise HTTPException(status_code=502, detail=f"sftp_upload_failed:{exc}") from exc
def sftp_upload(
@ -275,20 +434,14 @@ def sftp_upload(
remote_path: str,
data: bytes,
) -> dict[str, Any]:
remote = _normalize_remote(remote_path, allow_dot=False)
if not remote:
raise HTTPException(status_code=400, detail="sftp_path_required")
try:
with _sftp_client(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id) as (sftp, device):
with sftp.open(remote, "wb") as fh:
fh.write(data)
return {
"ok": True,
"ne_id": str(device.get("id") or ""),
"path": remote,
"size": len(data),
}
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_upload_failed:{exc}") from exc
"""Compatibility helper for tests — wraps streamed upload over an in-memory buffer."""
import io
return sftp_upload_stream(
db,
managed_ne_id=managed_ne_id,
ume_ne_id=ume_ne_id,
remote_path=remote_path,
reader=io.BytesIO(data or b""),
expected_size=len(data or b""),
)