Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions apps/scut-senior/api/src/scut_senior_api/adapters/sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,20 @@ def cleanup_auth_records(self) -> AuthCleanupCounts:
"DELETE FROM auth_sessions WHERE revoked_at IS NOT NULL OR expires_at <= ?",
(now,),
).rowcount
tables = {
row["name"]
for row in connection.execute(
"SELECT name FROM sqlite_master WHERE type = 'table'"
)
}
if "model_credentials" in tables:
# BYOK credentials are account-bound, so session cleanup cannot
# cascade to them. Remove expired ciphertext on the same
# scheduled path instead of merely hiding it at read time.
connection.execute(
"DELETE FROM model_credentials WHERE expires_at <= ?",
(now,),
)
return AuthCleanupCounts(states, sessions)

def cleanup_history_records(self) -> HistoryCleanupCounts:
Expand Down Expand Up @@ -505,6 +519,13 @@ def cleanup_material_records(self) -> MaterialCleanupCounts:
""",
(now, now),
).rowcount
if "contribution_attachments" in tables:
# Attachment payloads have their own TTL and are not covered
# by clearing a contribution's text snapshot.
connection.execute(
"DELETE FROM contribution_attachments WHERE expires_at <= ?",
(now,),
)
return MaterialCleanupCounts(materials, cleared)

# ------------------------------------------------------------------
Expand Down Expand Up @@ -649,6 +670,10 @@ def delete_account(self, user_id: str) -> dict[str, int]:
"DELETE FROM temporary_materials WHERE user_id = ?",
(normalized_user_id,),
).rowcount,
"private_knowledge_items": connection.execute(
"DELETE FROM private_knowledge_items WHERE user_id = ?",
(normalized_user_id,),
).rowcount,
"contributions": connection.execute(
"DELETE FROM contributions WHERE user_id = ?",
(normalized_user_id,),
Expand Down Expand Up @@ -1969,6 +1994,12 @@ def record_exam_plan_decision(
if decision not in {"confirmed", "edited", "rejected"}:
raise ValueError("invalid exam plan decision")
with self._connect() as connection:
owner = connection.execute(
"SELECT 1 FROM conversations WHERE conversation_id = ? AND user_id = ?",
(str(conversation_id), user_id),
).fetchone()
if owner is None:
raise LookupError("conversation not found")
connection.execute(
"INSERT INTO exam_plan_decisions "
"(decision_id, conversation_id, user_id, decision, plan_json, created_at) "
Expand Down
1 change: 1 addition & 0 deletions apps/scut-senior/api/src/scut_senior_api/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -873,6 +873,7 @@ class AccountDeletionSummary(ContractModel):
workflow_runs: int
feedback: int
temporary_materials: int
private_knowledge_items: int
contributions: int
model_credentials: int
auth_sessions: int
Expand Down
4 changes: 3 additions & 1 deletion apps/scut-senior/api/src/scut_senior_api/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,9 @@ def _find_stream_session(


OAUTH_STATE_COOKIE_NAME = "__Host-scut_senior_oauth_state"
MAX_REQUEST_BODY_BYTES = 2 * 1024 * 1024
# Attachments are capped at 10 MiB. Leave room for multipart boundaries and
# headers so a valid maximum-size upload reaches the endpoint's own check.
MAX_REQUEST_BODY_BYTES = 11 * 1024 * 1024
MAX_BUFFERED_REQUEST_MESSAGES = 4096


Expand Down
1 change: 1 addition & 0 deletions apps/scut-senior/api/src/scut_senior_api/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,7 @@ def delete_account(self, user: AuthenticatedPrincipal) -> AccountDeletionSummary
workflow_runs=counts["workflow_runs"],
feedback=counts["feedback"],
temporary_materials=counts["temporary_materials"],
private_knowledge_items=counts["private_knowledge_items"],
contributions=counts["contributions"],
model_credentials=counts["model_credentials"],
auth_sessions=counts["auth_sessions"],
Expand Down
63 changes: 39 additions & 24 deletions apps/scut-senior/api/src/scut_senior_api/vector_index.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@

import argparse
import json
import os
import shutil
from collections import defaultdict
from pathlib import Path
from tempfile import mkdtemp
from typing import Iterable

from .adapters.onnx import OnnxEmbeddingProvider
Expand All @@ -19,13 +22,18 @@ def build_candidate_vectors(
*,
batch_size: int = 32,
) -> int:
"""Write one vector SQLite file per course and return vector count."""
"""Atomically publish one vector SQLite file per inactive candidate course."""
if isinstance(batch_size, bool) or batch_size < 1:
raise ValueError("vector batch_size must be positive")
candidate = candidate_path.resolve()
metadata = json.loads((candidate / "metadata.json").read_text(encoding="utf-8"))
if metadata.get("embedding_model_id") != embedding.model_id:
raise ValueError("candidate embedding_model_id does not match provider")
active_path = candidate.parent.parent / "active.json"
if active_path.is_file():
active = json.loads(active_path.read_text(encoding="utf-8"))
if active.get("active_corpus_version") == metadata.get("corpus_version"):
raise ValueError("cannot build vectors for the active corpus candidate")
total = 0
by_course: dict[str, list[dict[str, object]]] = defaultdict(list)
for course_file in sorted((candidate / "courses").glob("*.json")):
Expand All @@ -35,30 +43,37 @@ def build_candidate_vectors(
if not isinstance(course_id, str) or not isinstance(chunks, list):
raise ValueError(f"invalid course payload: {course_file.name}")
by_course[course_id].extend(chunk for chunk in chunks if isinstance(chunk, dict))
for course_id, chunks in by_course.items():
vector_path = candidate / "vectors" / f"{course_id}.db"
vector_path.parent.mkdir(parents=True, exist_ok=True)
store = VectorStore(
vector_path, dimensions=embedding.dimensions, model_id=embedding.model_id
)
try:
for offset in range(0, len(chunks), batch_size):
batch = chunks[offset : offset + batch_size]
texts = [_chunk_text(chunk) for chunk in batch]
vectors = embedding.embed(texts)
if len(vectors) != len(batch):
raise ValueError("embedding provider returned an unexpected batch size")
store.bulk_upsert(
(
str(chunk["chunk_id"]),
course_id,
vector,
staging = Path(mkdtemp(prefix=".vectors-", dir=candidate))
try:
for course_id, chunks in by_course.items():
vector_path = staging / f"{course_id}.db"
store = VectorStore(
vector_path, dimensions=embedding.dimensions, model_id=embedding.model_id
)
try:
for offset in range(0, len(chunks), batch_size):
batch = chunks[offset : offset + batch_size]
texts = [_chunk_text(chunk) for chunk in batch]
vectors = embedding.embed(texts)
if len(vectors) != len(batch):
raise ValueError("embedding provider returned an unexpected batch size")
store.bulk_upsert(
(
str(chunk["chunk_id"]),
course_id,
vector,
)
for chunk, vector in zip(batch, vectors)
)
for chunk, vector in zip(batch, vectors)
)
total += len(batch)
finally:
store.close()
total += len(batch)
finally:
store.close()
vectors_root = candidate / "vectors"
vectors_root.mkdir(parents=True, exist_ok=True)
for vector_path in staging.glob("*.db"):
os.replace(vector_path, vectors_root / vector_path.name)
finally:
shutil.rmtree(staging, ignore_errors=True)
return total


Expand Down
16 changes: 15 additions & 1 deletion apps/scut-senior/api/src/scut_senior_api/vector_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,22 @@ def upsert(
self._connection.commit()

def bulk_upsert(self, records: Iterable[tuple[str, str, Sequence[float]]]) -> None:
prepared: list[tuple[str, str, bytes]] = []
for chunk_id, course_id, vector in records:
self.upsert(chunk_id, course_id, vector)
values = list(vector)
if len(values) != self.dimensions:
raise ValueError(
f"vector dimension {len(values)} != store dimension {self.dimensions}"
)
prepared.append(
(chunk_id, course_id, struct.pack(f"{self.dimensions}f", *values))
)
with self._connection:
self._connection.executemany(
"INSERT OR REPLACE INTO vectors (chunk_id, course_id, vector) "
"VALUES (?, ?, ?)",
prepared,
)

def search(
self,
Expand Down
11 changes: 11 additions & 0 deletions apps/scut-senior/tests/python/test_account_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@ def seed_account_data(app, client: TestClient, *, with_credential: bool) -> None
"SELECT user_id FROM users WHERE github_user_id = 123456"
).fetchone()
alice_user_id = row["user_id"]
repository.save_private_knowledge(
user_id=alice_user_id,
course_id="linear_algebra",
title="注销测试私有知识",
content="该内容必须与账户一起物理删除。",
)

# 贡献待审副本:直接按迁移 schema 插入一行 submitted 记录。
now = datetime.now(UTC)
Expand Down Expand Up @@ -211,6 +217,7 @@ def test_delete_account_wipes_data_blocks_relogin(tmp_path: Path) -> None:
"workflow_runs",
"feedback",
"temporary_materials",
"private_knowledge_items",
"contributions",
"model_credentials",
"auth_sessions",
Expand All @@ -220,6 +227,7 @@ def test_delete_account_wipes_data_blocks_relogin(tmp_path: Path) -> None:
assert before["workflow_runs"] >= 1
assert before["contributions"] >= 1
assert before["temporary_materials"] >= 1
assert before["private_knowledge_items"] >= 1
assert before["model_credentials"] >= 1

old_cookie = client.cookies.get(SESSION_COOKIE_NAME)
Expand All @@ -229,6 +237,7 @@ def test_delete_account_wipes_data_blocks_relogin(tmp_path: Path) -> None:
assert summary["conversations"] >= 1
assert summary["workflow_runs"] >= 1
assert summary["auth_sessions"] >= 1
assert summary["private_knowledge_items"] >= 1
assert summary["login_blocked"] is True

with sqlite3.connect(database_path) as connection:
Expand All @@ -242,6 +251,7 @@ def test_delete_account_wipes_data_blocks_relogin(tmp_path: Path) -> None:
"workflow_runs",
"feedback",
"temporary_materials",
"private_knowledge_items",
"contributions",
"model_credentials",
"auth_sessions",
Expand All @@ -253,6 +263,7 @@ def test_delete_account_wipes_data_blocks_relogin(tmp_path: Path) -> None:
assert after["workflow_runs"] == 0
assert after["feedback"] == 0
assert after["temporary_materials"] == 0
assert after["private_knowledge_items"] == 0
assert after["contributions"] == 0
assert after["model_credentials"] == 0
assert after["auth_sessions"] == 0
Expand Down
28 changes: 28 additions & 0 deletions apps/scut-senior/tests/python/test_sqlite_auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,34 @@ def test_cleanup_removes_dead_auth_records_but_preserves_users(tmp_path: Path) -
assert repository.authenticate_session(expired_session.token) is None


def test_auth_cleanup_physically_removes_expired_byok_credentials(tmp_path: Path) -> None:
clock = MutableClock(datetime(2026, 8, 15, 10, 0, tzinfo=UTC))
repository = SQLiteWorkflowRepository(tmp_path / "expired-byok.db", clock=clock)
user_id = repository.upsert_github_user(
GitHubUserProfile(111112, "expired-byok-user", None)
)
repository.upsert_model_credential(
user_id=user_id,
provider_id="openrouter",
display_name="OpenRouter",
base_url="https://openrouter.ai/api/v1",
model_id="deepseek/deepseek-v4-flash-0731",
protocol="openai_chat_completions",
ciphertext=b"x" * 32,
nonce=b"y" * 12,
algorithm="AES-256-GCM",
key_version=1,
)
clock.advance(timedelta(days=366))

repository.cleanup_auth_records()

with connect(repository.database_path) as connection:
assert connection.execute(
"SELECT COUNT(*) FROM model_credentials"
).fetchone()[0] == 0


def test_auth_cleanup_runs_on_startup_and_before_new_state_or_session(
tmp_path: Path,
) -> None:
Expand Down
38 changes: 38 additions & 0 deletions apps/scut-senior/tests/python/test_vector_index.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from pathlib import Path

import pytest

from scut_senior_api.vector_index import build_candidate_vectors
from scut_senior_api.vector_store import VectorStore
from scut_senior_worker.corpus_builder import _candidate_directory
Expand All @@ -17,6 +19,7 @@ def embed(self, texts):
def test_build_candidate_vectors_writes_course_scoped_sqlite_files(tmp_path: Path) -> None:
store, version, _ = _build_store(tmp_path, embedding_model_id="bge-small-zh-v1.5")
candidate = _candidate_directory(store.resolve(), version)
(store / "active.json").unlink()

count = build_candidate_vectors(candidate, _FakeEmbedder(), batch_size=1)

Expand All @@ -32,3 +35,38 @@ def test_build_candidate_vectors_writes_course_scoped_sqlite_files(tmp_path: Pat
)
finally:
vector_store.close()


class _FailingSecondBatchEmbedder(_FakeEmbedder):
def __init__(self) -> None:
self.calls = 0

def embed(self, texts):
self.calls += 1
if self.calls == 2:
raise RuntimeError("simulated embedding interruption")
return super().embed(texts)


def test_vector_build_leaves_no_partial_files_when_embedding_fails(tmp_path: Path) -> None:
store, version, _ = _build_store(tmp_path, embedding_model_id="bge-small-zh-v1.5")
candidate = _candidate_directory(store.resolve(), version)
(store / "active.json").unlink()

with pytest.raises(RuntimeError, match="simulated embedding interruption"):
build_candidate_vectors(candidate, _FailingSecondBatchEmbedder(), batch_size=1)

assert not (candidate / "vectors").exists()
assert not list(candidate.glob(".vectors-*"))


def test_vector_build_rejects_active_candidate(tmp_path: Path) -> None:
store, version, _ = _build_store(tmp_path, embedding_model_id="bge-small-zh-v1.5")
candidate = _candidate_directory(store.resolve(), version)
(store / "active.json").unlink()
(store / "active.json").write_text(
'{"active_corpus_version": "' + version + '"}', encoding="utf-8"
)

with pytest.raises(ValueError, match="active corpus"):
build_candidate_vectors(candidate, _FakeEmbedder())
17 changes: 16 additions & 1 deletion apps/scut-senior/web/src/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type {
AuthUser,
ByokCredentialStatus,
ByokProviderId,
ContributionAttachmentRecord,
ContributionConfirmations,
ContributionPreview,
ContributionRecord,
Expand Down Expand Up @@ -52,7 +53,9 @@ export class ApiError extends Error {
async function apiRequest<T>(path: string, init?: RequestInit): Promise<T> {
const headers = new Headers(init?.headers);
headers.set("Accept", "application/json");
if (init?.body) headers.set("Content-Type", "application/json");
if (init?.body && !(init.body instanceof FormData)) {
headers.set("Content-Type", "application/json");
}

let response: Response;
try {
Expand Down Expand Up @@ -336,6 +339,18 @@ export async function getMaintainerContribution(contributionId: string): Promise
return apiRequest<MaintainerContributionDetail>(`/api/v1/maintainer/contributions/${encodeURIComponent(contributionId)}`);
}

export async function uploadMaintainerContributionAttachment(
contributionId: string,
file: File,
): Promise<ContributionAttachmentRecord> {
const form = new FormData();
form.append("file", file, file.name);
return apiRequest<ContributionAttachmentRecord>(
`/api/v1/maintainer/contributions/${encodeURIComponent(contributionId)}/attachments`,
{ method: "POST", body: form },
);
}

export async function listMaintainerFeedback(): Promise<FeedbackRecord[]> {
return apiRequest<FeedbackRecord[]>("/api/v1/maintainer/feedback");
}
Expand Down
Loading
Loading