hansjone/netx
1
0
Fork 0
mirror of https://github.com/hansjone/netx.git synced 2026-10-09 04:20:45 +08:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions 2

Expand WebCRT SFTP into a fuller file manager.

Browse source 
Add mkdir/rename/delete/chmod, multi-select batch ops, cancelable transfers with retries, per-tab paths, sorting, and safer listing limits.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-08-02 01:33:37 +08:00
parent d10ffed277
commit 9674de221d
10 changed files with 1146 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

3
netx_api/config.py
View file

@ -104,6 +104,9 @@ class Settings(BaseSettings):
# WebCRT SFTP transfer limits (streamed; default 512 MiB per file).
webcrt_sftp_max_file_bytes: int = 512 * 1024 * 1024
webcrt_sftp_chunk_bytes: int = 64 * 1024
# Cap directory listings so huge folders cannot pin the API/UI.
webcrt_sftp_list_max_entries: int = 5000
webcrt_sftp_list_timeout_sec: float = 30.0
# Local app login / audit (lab defaults; override in production)
auth_enabled: bool = True
# Stable default so JWT survives restarts without .env. Override in production.

104
netx_api/webcrt_router.py
View file

@ -80,6 +80,33 @@ class WebcrtSftpDownloadBody(BaseModel):
path: str
class WebcrtSftpMkdirBody(BaseModel):
ne_id: str | None = Field(default=None)
ume_ne_id: str | None = Field(default=None)
path: str
class WebcrtSftpRemoveBody(BaseModel):
ne_id: str | None = Field(default=None)
ume_ne_id: str | None = Field(default=None)
path: str
recursive: bool = False
class WebcrtSftpRenameBody(BaseModel):
ne_id: str | None = Field(default=None)
ume_ne_id: str | None = Field(default=None)
old_path: str
new_path: str
class WebcrtSftpChmodBody(BaseModel):
ne_id: str | None = Field(default=None)
ume_ne_id: str | None = Field(default=None)
path: str
mode: str
def _client_label(request: Request | None = None, websocket: WebSocket | None = None) -> str:
host = ""
if request is not None:
@ -200,15 +227,64 @@ def api_close_session(session_id: str, request: Request) -> dict[str, Any]:
return close_session(session_id, reason="client_delete", client=_client_label(request=request))
def _sftp_ne_ids(ne_id: str | None, ume_ne_id: str | None) -> tuple[str | None, str | None]:
mid = str(ne_id or "").strip()
uid = str(ume_ne_id or "").strip()
if bool(mid) == bool(uid):
raise HTTPException(status_code=400, detail="exactly_one_of_ne_id_or_ume_ne_id_required")
return (mid or None, uid or None)
@router.post("/sftp/list")
def api_sftp_list(body: WebcrtSftpListBody, db: Session = Depends(get_db)) -> dict[str, Any]:
from .webcrt_sftp import sftp_list
mid = str(body.ne_id or "").strip()
uid = str(body.ume_ne_id or "").strip()
if bool(mid) == bool(uid):
raise HTTPException(status_code=400, detail="exactly_one_of_ne_id_or_ume_ne_id_required")
return sftp_list(db, managed_ne_id=mid or None, ume_ne_id=uid or None, path=body.path)
mid, uid = _sftp_ne_ids(body.ne_id, body.ume_ne_id)
return sftp_list(db, managed_ne_id=mid, ume_ne_id=uid, path=body.path)
@router.post("/sftp/mkdir")
def api_sftp_mkdir(body: WebcrtSftpMkdirBody, db: Session = Depends(get_db)) -> dict[str, Any]:
from .webcrt_sftp import sftp_mkdir
mid, uid = _sftp_ne_ids(body.ne_id, body.ume_ne_id)
return sftp_mkdir(db, managed_ne_id=mid, ume_ne_id=uid, path=body.path)
@router.post("/sftp/remove")
def api_sftp_remove(body: WebcrtSftpRemoveBody, db: Session = Depends(get_db)) -> dict[str, Any]:
from .webcrt_sftp import sftp_remove
mid, uid = _sftp_ne_ids(body.ne_id, body.ume_ne_id)
return sftp_remove(
db,
managed_ne_id=mid,
ume_ne_id=uid,
path=body.path,
recursive=bool(body.recursive),
)
@router.post("/sftp/rename")
def api_sftp_rename(body: WebcrtSftpRenameBody, db: Session = Depends(get_db)) -> dict[str, Any]:
from .webcrt_sftp import sftp_rename
mid, uid = _sftp_ne_ids(body.ne_id, body.ume_ne_id)
return sftp_rename(
db,
managed_ne_id=mid,
ume_ne_id=uid,
old_path=body.old_path,
new_path=body.new_path,
)
@router.post("/sftp/chmod")
def api_sftp_chmod(body: WebcrtSftpChmodBody, db: Session = Depends(get_db)) -> dict[str, Any]:
from .webcrt_sftp import sftp_chmod
mid, uid = _sftp_ne_ids(body.ne_id, body.ume_ne_id)
return sftp_chmod(db, managed_ne_id=mid, ume_ne_id=uid, path=body.path, mode=body.mode)
@router.post("/sftp/download")
@ -217,13 +293,8 @@ def api_sftp_download(body: WebcrtSftpDownloadBody, db: Session = Depends(get_db
from .webcrt_sftp import SftpDownloadStream
mid = str(body.ne_id or "").strip()
uid = str(body.ume_ne_id or "").strip()
if bool(mid) == bool(uid):
raise HTTPException(status_code=400, detail="exactly_one_of_ne_id_or_ume_ne_id_required")
stream = SftpDownloadStream(
db, managed_ne_id=mid or None, ume_ne_id=uid or None, path=body.path
).open()
mid, uid = _sftp_ne_ids(body.ne_id, body.ume_ne_id)
stream = SftpDownloadStream(db, managed_ne_id=mid, ume_ne_id=uid, path=body.path).open()
headers = {
"Content-Disposition": stream.content_disposition(),
"Content-Length": str(int(stream.size)),
@ -249,10 +320,7 @@ async def api_sftp_upload(
from .config import settings
from .webcrt_sftp import sftp_upload_stream
mid = str(ne_id or "").strip()
uid = str(ume_ne_id or "").strip()
if bool(mid) == bool(uid):
raise HTTPException(status_code=400, detail="exactly_one_of_ne_id_or_ume_ne_id_required")
mid, uid = _sftp_ne_ids(ne_id, ume_ne_id)
max_bytes = max(1, int(settings.webcrt_sftp_max_file_bytes or (512 * 1024 * 1024)))
expected = getattr(file, "size", None)
if expected is not None and int(expected) > max_bytes:
@ -261,8 +329,8 @@ async def api_sftp_upload(
def _upload() -> dict[str, Any]:
return sftp_upload_stream(
db,
managed_ne_id=mid or None,
ume_ne_id=uid or None,
managed_ne_id=mid,
ume_ne_id=uid,
remote_path=remote_path,
reader=file.file,
expected_size=int(expected) if expected is not None else None,

62
netx_api/webcrt_service.py
View file

@ -519,25 +519,14 @@ class WebcrtSession:
except Exception:
pass
def _ensure_sftp_unlocked(self) -> Any:
"""Caller must hold ``_sftp_lock``."""
import paramiko
def _ssh_transport_unlocked(self) -> Any:
"""Caller must hold ``_sftp_lock``. Returns an active Paramiko Transport."""
if self.closed or self.conn is None:
raise RuntimeError("session_closed")
if str(self.protocol or "ssh").lower() != "ssh":
raise RuntimeError("sftp_requires_ssh")
if self.cli_hop_guard:
raise RuntimeError("sftp_hop_not_supported")
if self._sftp is not None:
sock = getattr(self._sftp, "sock", None)
if sock is not None and not bool(getattr(sock, "closed", False)):
return self._sftp
try:
self._sftp.close()
except Exception:
pass
self._sftp = None
channel = getattr(self.conn, "remote_conn", None)
transport = None
if channel is not None and hasattr(channel, "get_transport"):
@ -547,6 +536,22 @@ class WebcrtSession:
transport = None
if transport is None or not bool(getattr(transport, "is_active", lambda: False)()):
raise RuntimeError("ssh_transport_unavailable")
return transport
def _ensure_sftp_unlocked(self) -> Any:
"""Caller must hold ``_sftp_lock``. Shared probe client (sftp_ready)."""
import paramiko
if self._sftp is not None:
sock = getattr(self._sftp, "sock", None)
if sock is not None and not bool(getattr(sock, "closed", False)):
return self._sftp
try:
self._sftp.close()
except Exception:
pass
self._sftp = None
transport = self._ssh_transport_unlocked()
self._sftp = paramiko.SFTPClient.from_transport(transport)
if self._sftp is None:
raise RuntimeError("sftp_open_failed")
@ -558,11 +563,36 @@ class WebcrtSession:
with self._sftp_lock:
return self._ensure_sftp_unlocked()
def run_sftp(self, fn: Any) -> Any:
"""Run ``fn(sftp)`` while holding the session SFTP lock."""
def open_ephemeral_sftp(self) -> Any:
"""Open a dedicated SFTP channel for one operation; caller must ``close()`` it.
Only holds ``_sftp_lock`` briefly while resolving the SSH transport, so long
list/upload/download work does not block other SFTP ops on the same session.
"""
import paramiko
with self._sftp_lock:
sftp = self._ensure_sftp_unlocked()
transport = self._ssh_transport_unlocked()
# Keep probe client warm for UI sftp_ready without sharing it for I/O.
try:
self._ensure_sftp_unlocked()
except Exception:
pass
sftp = paramiko.SFTPClient.from_transport(transport)
if sftp is None:
raise RuntimeError("sftp_open_failed")
return sftp
def run_sftp(self, fn: Any) -> Any:
"""Run ``fn(sftp)`` on an ephemeral channel (does not hold the lock during ``fn``)."""
sftp = self.open_ephemeral_sftp()
try:
return fn(sftp)
finally:
try:
sftp.close()
except Exception:
pass
def try_attach_sftp(self) -> bool:
"""Best-effort SFTP channel open after SSH login (does not fail the shell)."""

242
netx_api/webcrt_sftp.py
View file

@ -150,39 +150,65 @@ def _sftp_client(
managed_ne_id: str | None,
ume_ne_id: str | None,
) -> Iterator[tuple[Any, dict[str, Any]]]:
"""Yield ``(sftp, device)`` — prefers live WebCRT SSH session channel."""
"""Yield ``(sftp, device)`` — prefers a short-lived channel on the live SSH session."""
creds, device = _resolve(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id)
ne_key = str(device.get("id") or managed_ne_id or ume_ne_id or "").strip()
sess = find_ssh_session_for_ne(ne_key) if ne_key else None
if sess is not None:
opened = False
sftp = None
try:
with sess._sftp_lock:
sftp = sess._ensure_sftp_unlocked()
opened = True
yield sftp, device
sftp = sess.open_ephemeral_sftp()
yield sftp, device
return
except HTTPException:
raise
except Exception as exc:
if opened:
# Operation failed on an already-open session channel — don't double-yield.
if sftp is not None:
raise HTTPException(status_code=502, detail=f"sftp_failed:{exc}") from exc
_log.debug("session sftp open failed ne=%s: %s — pool fallback", ne_key, exc)
finally:
if sftp is not None:
try:
sftp.close()
except Exception:
pass
key = _pool_key(managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id)
entry = _get_pooled(key, creds)
# Borrow transport under a short lock, then use a dedicated SFTP channel for I/O.
with entry.lock:
entry.last_used = time.time()
transport = None
try:
yield entry.sftp, device
transport = entry.client.get_transport()
except Exception:
# Drop broken pooled socket so the next call reconnects once.
transport = None
if transport is None or not bool(getattr(transport, "is_active", lambda: False)()):
with _pool_lock:
cur = _pool.pop(key, None)
if cur is not None:
_close_pooled(cur)
raise
raise HTTPException(status_code=502, detail="sftp_pool_transport_unavailable")
sftp = None
try:
sftp = paramiko.SFTPClient.from_transport(transport)
if sftp is None:
raise HTTPException(status_code=502, detail="sftp_open_failed")
yield sftp, device
except HTTPException:
raise
except Exception:
with _pool_lock:
cur = _pool.pop(key, None)
if cur is not None:
_close_pooled(cur)
raise
finally:
if sftp is not None:
try:
sftp.close()
except Exception:
pass
def _filemode(mode: int) -> str:
@ -239,6 +265,188 @@ def _mkdir_p(sftp: Any, remote: str) -> None:
raise HTTPException(status_code=502, detail=f"sftp_mkdir_failed:{candidate}:{exc}") from exc
def _rmtree(sftp: Any, remote: str) -> None:
path = _normalize_remote(remote, allow_dot=False)
if not path or path in (".", "/"):
raise HTTPException(status_code=400, detail="sftp_path_required")
try:
for attr in sftp.listdir_attr(path):
name = str(getattr(attr, "filename", "") or "")
if not name or name in (".", ".."):
continue
child = posixpath.join(path, name)
mode = int(getattr(attr, "st_mode", 0) or 0)
if statmod.S_ISDIR(mode):
_rmtree(sftp, child)
else:
sftp.remove(child)
sftp.rmdir(path)
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_remove_failed:{exc}") from exc
def sftp_mkdir(
db: Session,
*,
managed_ne_id: str | None,
ume_ne_id: str | None,
path: str,
) -> dict[str, Any]:
remote = _normalize_remote(path, allow_dot=False)
if not remote or remote in (".", "/"):
raise HTTPException(status_code=400, detail="sftp_path_required")
try:
with _sftp_client(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id) as (sftp, device):
_mkdir_p(sftp, remote)
return {
"ok": True,
"ne_id": str(device.get("id") or ""),
"path": remote,
}
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_mkdir_failed:{exc}") from exc
def sftp_remove(
db: Session,
*,
managed_ne_id: str | None,
ume_ne_id: str | None,
path: str,
recursive: bool = False,
) -> dict[str, Any]:
remote = _normalize_remote(path, allow_dot=False)
if not remote or remote in (".", "/"):
raise HTTPException(status_code=400, detail="sftp_path_required")
try:
with _sftp_client(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id) as (sftp, device):
try:
st = sftp.stat(remote)
except Exception as exc:
raise HTTPException(status_code=404, detail=f"sftp_not_found:{exc}") from exc
mode = int(getattr(st, "st_mode", 0) or 0)
if statmod.S_ISDIR(mode):
if recursive:
_rmtree(sftp, remote)
else:
try:
sftp.rmdir(remote)
except Exception as exc:
raise HTTPException(status_code=409, detail="sftp_dir_not_empty") from exc
else:
sftp.remove(remote)
return {
"ok": True,
"ne_id": str(device.get("id") or ""),
"path": remote,
}
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_remove_failed:{exc}") from exc
def sftp_rename(
db: Session,
*,
managed_ne_id: str | None,
ume_ne_id: str | None,
old_path: str,
new_path: str,
) -> dict[str, Any]:
src = _normalize_remote(old_path, allow_dot=False)
dst = _normalize_remote(new_path, allow_dot=False)
if not src or not dst or src in (".", "/") or dst in (".", "/"):
raise HTTPException(status_code=400, detail="sftp_path_required")
if src == dst:
return {"ok": True, "ne_id": "", "old_path": src, "new_path": dst}
try:
with _sftp_client(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id) as (sftp, device):
parent = posixpath.dirname(dst)
if parent and parent not in (".", "/"):
_mkdir_p(sftp, parent)
try:
sftp.rename(src, dst)
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_rename_failed:{exc}") from exc
return {
"ok": True,
"ne_id": str(device.get("id") or ""),
"old_path": src,
"new_path": dst,
}
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_rename_failed:{exc}") from exc
def _parse_chmod_mode(mode: str | int) -> int:
if isinstance(mode, int):
return int(mode) & 0o7777
raw = str(mode or "").strip().lower()
if not raw:
raise HTTPException(status_code=400, detail="sftp_chmod_invalid_mode")
if raw.startswith("0o"):
raw = raw[2:]
if raw.isdigit():
try:
return int(raw, 8) & 0o7777
except ValueError as exc:
raise HTTPException(status_code=400, detail="sftp_chmod_invalid_mode") from exc
sym = raw.lstrip("d").lstrip("-")
if len(sym) != 9 or any(c not in "rwx-" for c in sym):
raise HTTPException(status_code=400, detail="sftp_chmod_invalid_mode")
bits = {"r": 4, "w": 2, "x": 1, "-": 0}
value = 0
for i in range(3):
trip = sym[i * 3 : (i + 1) * 3]
value = (value << 3) | (bits[trip[0]] + bits[trip[1]] + bits[trip[2]])
return value & 0o7777
def sftp_chmod(
db: Session,
*,
managed_ne_id: str | None,
ume_ne_id: str | None,
path: str,
mode: str | int,
) -> dict[str, Any]:
remote = _normalize_remote(path, allow_dot=False)
if not remote or remote in (".", "/"):
raise HTTPException(status_code=400, detail="sftp_path_required")
mode_int = _parse_chmod_mode(mode)
try:
with _sftp_client(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id) as (sftp, device):
try:
sftp.chmod(remote, mode_int)
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_chmod_failed:{exc}") from exc
return {
"ok": True,
"ne_id": str(device.get("id") or ""),
"path": remote,
"mode": f"{mode_int:04o}",
}
except HTTPException:
raise
except Exception as exc:
raise HTTPException(status_code=502, detail=f"sftp_chmod_failed:{exc}") from exc
def _list_max_entries() -> int:
return max(100, int(settings.webcrt_sftp_list_max_entries or 5000))
def _list_timeout_sec() -> float:
return max(1.0, float(settings.webcrt_sftp_list_timeout_sec or 30.0))
def sftp_list(
db: Session,
*,
@ -247,10 +455,17 @@ def sftp_list(
path: str = ".",
) -> dict[str, Any]:
remote = _normalize_remote(path, allow_dot=True) or "."
max_entries = _list_max_entries()
timeout_sec = _list_timeout_sec()
try:
with _sftp_client(db, managed_ne_id=managed_ne_id, ume_ne_id=ume_ne_id) as (sftp, device):
entries = []
truncated = False
deadline = time.monotonic() + timeout_sec
for attr in sftp.listdir_attr(remote):
if time.monotonic() > deadline:
truncated = True
break
mode = int(getattr(attr, "st_mode", 0) or 0)
name = str(attr.filename or "")
if not name or name in (".", ".."):
@ -269,12 +484,17 @@ def sftp_list(
"gid": int(getattr(attr, "st_gid", 0) or 0),
}
)
if len(entries) >= max_entries:
truncated = True
break
entries.sort(key=lambda x: (not x["is_dir"], str(x["name"]).lower()))
return {
"ne_id": str(device.get("id") or ""),
"ne_name": str(device.get("name") or ""),
"path": remote,
"items": entries,
"truncated": truncated,
"max_entries": max_entries,
}
except HTTPException:
raise