mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-17 03:13:33 +08:00
feat(persistence): PostgreSQL assistant store, chat persist fixes, gateway scripts
- Add SQLAlchemy Core repos, pg adapter/compat, assistant_store factory, Alembic bootstrap and migration/cutover scripts. - Harden chat_message writes (NUL scrub for PG), turn_uuid on attempt failure, WS turn_runner fallbacks and gateway executed_turn_uuid init. - start_gateway: log paths, PS7 stderr handling via cmd, background stdout/stderr redirect; runtime assistant_runtime_log_dir export. - Ops: clear_all_chat_sessions with PG-only --postgresql and env-gated wipe; clear_postgres_chat_sessions.ps1. - Tests: SA repos, pg compat, persist fallback, smoke env isolation; CI and docs touch-ups. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
2b32d11f43
commit
d14e9d3596
103 changed files with 7574 additions and 1641 deletions
43
svc/persistence/db/__init__.py
Normal file
43
svc/persistence/db/__init__.py
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
"""Persistence DB helpers."""
|
||||
|
||||
from svc.persistence.db.engine import (
|
||||
clear_assistant_engine_cache,
|
||||
engine_for_sqlite_file,
|
||||
get_assistant_engine,
|
||||
)
|
||||
from svc.persistence.db.tables import (
|
||||
app_setting,
|
||||
app_user,
|
||||
auth_session,
|
||||
bind_code,
|
||||
channel_identity,
|
||||
channel_identity_v2,
|
||||
channel_session_v2,
|
||||
chat_message,
|
||||
chat_session,
|
||||
metadata,
|
||||
tenant,
|
||||
tool_log,
|
||||
trace_event,
|
||||
ui_session_owner,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"app_user",
|
||||
"app_setting",
|
||||
"auth_session",
|
||||
"bind_code",
|
||||
"channel_identity",
|
||||
"channel_identity_v2",
|
||||
"channel_session_v2",
|
||||
"chat_message",
|
||||
"chat_session",
|
||||
"clear_assistant_engine_cache",
|
||||
"engine_for_sqlite_file",
|
||||
"get_assistant_engine",
|
||||
"metadata",
|
||||
"tenant",
|
||||
"tool_log",
|
||||
"trace_event",
|
||||
"ui_session_owner",
|
||||
]
|
||||
72
svc/persistence/db/engine.py
Normal file
72
svc/persistence/db/engine.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
"""SQLAlchemy Engine factory for assistant DB (SQLite or PostgreSQL)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from svc.config.database import assistant_sqlalchemy_url
|
||||
|
||||
|
||||
def _sqlite_sa_url_from_os_path(path: str) -> str:
|
||||
"""Build the same ``sqlite+pysqlite:///...`` URL shape as :func:`svc.config.database.assistant_sqlalchemy_url`."""
|
||||
p = Path(path).resolve().as_posix()
|
||||
return f"sqlite+pysqlite:///{p}"
|
||||
|
||||
|
||||
def _register_sqlite_pragmas(eng: Engine) -> None:
|
||||
@event.listens_for(eng, "connect")
|
||||
def _sqlite_pragmas(dbapi_conn: Any, _record: Any) -> None:
|
||||
cur = dbapi_conn.cursor()
|
||||
cur.execute("PRAGMA foreign_keys = ON;")
|
||||
cur.execute("PRAGMA journal_mode = WAL;")
|
||||
cur.execute("PRAGMA synchronous = NORMAL;")
|
||||
cur.execute("PRAGMA busy_timeout = 30000;")
|
||||
cur.close()
|
||||
|
||||
|
||||
@lru_cache(maxsize=64)
|
||||
def _engine_for_url(url: str) -> Engine:
|
||||
"""One engine per URL (process-wide)."""
|
||||
pool_kw: dict[str, Any] = {}
|
||||
if url.startswith("sqlite"):
|
||||
pool_kw["poolclass"] = NullPool
|
||||
else:
|
||||
pool_kw["pool_pre_ping"] = True
|
||||
eng = create_engine(url, future=True, **pool_kw)
|
||||
if url.startswith("sqlite"):
|
||||
_register_sqlite_pragmas(eng)
|
||||
return eng
|
||||
|
||||
|
||||
def get_assistant_engine() -> Engine:
|
||||
"""Engine for the current env-selected assistant DB (Alembic, ``get_assistant_store`` SQLite path)."""
|
||||
return _engine_for_url(assistant_sqlalchemy_url())
|
||||
|
||||
|
||||
def engine_for_sqlite_file(path: str) -> Engine:
|
||||
"""Engine for a specific SQLite file (e.g. ``SqliteStore('/tmp/x.sqlite')`` without env ``DB_PATH``)."""
|
||||
return _engine_for_url(_sqlite_sa_url_from_os_path(path))
|
||||
|
||||
|
||||
def clear_assistant_engine_cache() -> None:
|
||||
"""Drop cached engines (e.g. tests that delete temp DB files must call this before removing the directory)."""
|
||||
_engine_for_url.cache_clear()
|
||||
try:
|
||||
from svc.persistence.assistant_store import reset_assistant_store_singleton
|
||||
|
||||
reset_assistant_store_singleton()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
__all__ = [
|
||||
"clear_assistant_engine_cache",
|
||||
"engine_for_sqlite_file",
|
||||
"get_assistant_engine",
|
||||
]
|
||||
171
svc/persistence/db/tables.py
Normal file
171
svc/persistence/db/tables.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
"""SQLAlchemy Core table objects for incremental persistence migration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import BigInteger, Column, ForeignKey, Integer, MetaData, Table, Text
|
||||
|
||||
metadata = MetaData()
|
||||
|
||||
tenant = Table(
|
||||
"tenant",
|
||||
metadata,
|
||||
Column("id", Text, primary_key=True),
|
||||
Column("name", Text, nullable=False),
|
||||
Column("created_at", Text, nullable=False),
|
||||
)
|
||||
|
||||
bind_code = Table(
|
||||
"bind_code",
|
||||
metadata,
|
||||
Column("code", Text, primary_key=True),
|
||||
Column("tenant_id", Text, nullable=False),
|
||||
Column("role", Text, nullable=False),
|
||||
Column("created_at", Text, nullable=False),
|
||||
Column("used_at", Text, nullable=True),
|
||||
Column("used_by_external_user_id", Text, nullable=True),
|
||||
)
|
||||
|
||||
app_user = Table(
|
||||
"app_user",
|
||||
metadata,
|
||||
Column("id", Text, primary_key=True),
|
||||
Column("tenant_id", Text, nullable=False),
|
||||
Column("username", Text, nullable=True),
|
||||
Column("display_name", Text, nullable=False),
|
||||
Column("role", Text, nullable=False),
|
||||
Column("password_hash", Text, nullable=True),
|
||||
Column("is_active", Integer, nullable=False, server_default="1"),
|
||||
Column("created_at", Text, nullable=False),
|
||||
Column("avatar_attachment_id", Text, nullable=True),
|
||||
)
|
||||
|
||||
app_setting = Table(
|
||||
"app_setting",
|
||||
metadata,
|
||||
Column("key", Text, primary_key=True),
|
||||
Column("value", Text, nullable=False),
|
||||
Column("is_secret", Integer, nullable=False, server_default="0"),
|
||||
Column("updated_at", Text, nullable=False),
|
||||
)
|
||||
|
||||
auth_session = Table(
|
||||
"auth_session",
|
||||
metadata,
|
||||
Column("session_token_hash", Text, primary_key=True),
|
||||
Column("tenant_id", Text, nullable=False),
|
||||
Column("user_id", Text, nullable=False),
|
||||
Column("role", Text, nullable=False),
|
||||
Column("created_at", Text, nullable=False),
|
||||
Column("expires_at", Text, nullable=False),
|
||||
Column("last_seen_at", Text, nullable=False),
|
||||
Column("revoked_at", Text, nullable=True),
|
||||
)
|
||||
|
||||
chat_session = Table(
|
||||
"chat_session",
|
||||
metadata,
|
||||
Column("id", Text, primary_key=True),
|
||||
Column("title", Text, nullable=False),
|
||||
Column("created_at", Text, nullable=False),
|
||||
Column("last_message_at", Text, nullable=True),
|
||||
)
|
||||
|
||||
channel_identity_v2 = Table(
|
||||
"channel_identity_v2",
|
||||
metadata,
|
||||
Column("tenant_id", Text, primary_key=True),
|
||||
Column("channel", Text, primary_key=True),
|
||||
Column("account_id", Text, primary_key=True),
|
||||
Column("external_user_id", Text, primary_key=True),
|
||||
Column("user_id", Text, nullable=False),
|
||||
Column("created_at", Text, nullable=False),
|
||||
)
|
||||
|
||||
channel_identity = Table(
|
||||
"channel_identity",
|
||||
metadata,
|
||||
Column("tenant_id", Text, primary_key=True),
|
||||
Column("channel", Text, primary_key=True),
|
||||
Column("external_user_id", Text, primary_key=True),
|
||||
Column("user_id", Text, nullable=False),
|
||||
Column("created_at", Text, nullable=False),
|
||||
)
|
||||
|
||||
channel_session_v2 = Table(
|
||||
"channel_session_v2",
|
||||
metadata,
|
||||
Column("tenant_id", Text, primary_key=True),
|
||||
Column("channel", Text, primary_key=True),
|
||||
Column("account_id", Text, primary_key=True),
|
||||
Column("external_chat_id", Text, primary_key=True),
|
||||
Column("external_user_id", Text, primary_key=True),
|
||||
Column("session_id", Text, nullable=False),
|
||||
Column("created_at", Text, nullable=False),
|
||||
)
|
||||
|
||||
ui_session_owner = Table(
|
||||
"ui_session_owner",
|
||||
metadata,
|
||||
Column("session_id", Text, primary_key=True),
|
||||
Column("tenant_id", Text, nullable=False),
|
||||
Column("user_id", Text, nullable=False),
|
||||
Column("created_at", Text, nullable=False),
|
||||
)
|
||||
|
||||
chat_message = Table(
|
||||
"chat_message",
|
||||
metadata,
|
||||
Column("id", BigInteger, primary_key=True, autoincrement=True),
|
||||
Column("session_id", Text, ForeignKey("chat_session.id", ondelete="CASCADE"), nullable=False),
|
||||
Column("role", Text, nullable=False),
|
||||
Column("content", Text, nullable=False),
|
||||
Column("tool_calls", Text, nullable=True),
|
||||
Column("attachments", Text, nullable=True),
|
||||
Column("turn_uuid", Text, nullable=True),
|
||||
Column("event_type", Text, nullable=True),
|
||||
Column("event_payload", Text, nullable=True),
|
||||
Column("timestamp", Text, nullable=False),
|
||||
)
|
||||
|
||||
tool_log = Table(
|
||||
"tool_log",
|
||||
metadata,
|
||||
Column("id", BigInteger, primary_key=True, autoincrement=True),
|
||||
Column("session_id", Text, ForeignKey("chat_session.id", ondelete="CASCADE"), nullable=False),
|
||||
Column("tool_name", Text, nullable=False),
|
||||
Column("specialist", Text, nullable=False, server_default=""),
|
||||
Column("args", Text, nullable=False),
|
||||
Column("result", Text, nullable=False),
|
||||
Column("timestamp", Text, nullable=False),
|
||||
Column("duration_ms", Integer, nullable=True),
|
||||
)
|
||||
|
||||
trace_event = Table(
|
||||
"trace_event",
|
||||
metadata,
|
||||
Column("id", BigInteger, primary_key=True, autoincrement=True),
|
||||
Column("session_id", Text, nullable=False),
|
||||
Column("trace_id", Text, nullable=False),
|
||||
Column("span_id", Text, nullable=False),
|
||||
Column("parent_span_id", Text, nullable=True),
|
||||
Column("event_type", Text, nullable=False),
|
||||
Column("payload", Text, nullable=False, server_default="{}"),
|
||||
Column("timestamp", Text, nullable=False),
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"app_user",
|
||||
"app_setting",
|
||||
"auth_session",
|
||||
"bind_code",
|
||||
"channel_identity",
|
||||
"channel_identity_v2",
|
||||
"channel_session_v2",
|
||||
"chat_message",
|
||||
"chat_session",
|
||||
"metadata",
|
||||
"tenant",
|
||||
"tool_log",
|
||||
"trace_event",
|
||||
"ui_session_owner",
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue