Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions apps/scut-senior/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,18 @@ SCUT_SENIOR_STORAGE_MODE=sqlite_mock
# fixed at 7 days, and the database stores token digests rather than raw tokens.
# SCUT_SENIOR_STORAGE_MODE=sqlite
SCUT_SENIOR_DATABASE_PATH=.local/iteration-zero.db
# The default remains the synthetic Fixture. local_corpus is only valid after
# a candidate source commit has reached trusted master and a separate guarded
# activation created the store; the store path must be absolute.
SCUT_SENIOR_RETRIEVAL_MODE=fixture
# SCUT_SENIOR_RETRIEVAL_MODE=local_corpus
# Use the activated real corpus by default. Fixture mode is reserved for
# isolated tests and must be explicitly selected when needed.
SCUT_SENIOR_RETRIEVAL_MODE=local_corpus
# SCUT_SENIOR_RETRIEVAL_MODE=fixture
# Dense retrieval is enabled by default; it activates when this local ONNX
# directory exists and contains model.onnx + tokenizer.json.
# SCUT_SENIOR_ONNX_MODEL_PATH=/absolute/path/to/bge-small-zh-v1.5
# SCUT_SENIOR_ONNX_MODEL_ID=bge-small-zh-v1.5
# SCUT_SENIOR_ONNX_EMBEDDING_DIMENSIONS=512
# SCUT_SENIOR_ONNX_MAX_LENGTH=512
# SCUT_SENIOR_CORPUS_STORE_PATH=/absolute/path/to/corpus-store
SCUT_SENIOR_CROSS_COURSE_ENABLED=false
SCUT_SENIOR_CROSS_COURSE_ENABLED=true
# GitHub logins allowed to review/transition public contributions.
SCUT_SENIOR_MAINTAINER_GITHUB_LOGINS= 维护人员
SCUT_SENIOR_BILIBILI_RESOURCES_ENABLED=true
19 changes: 19 additions & 0 deletions apps/scut-senior/api/migrations/0016_private_knowledge.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
-- PLAN-3 C-2: cross-conversation, user-bound private knowledge.
-- These entries are never public corpus candidates and expire physically after 7 days.
CREATE TABLE IF NOT EXISTS private_knowledge_items (
knowledge_id TEXT PRIMARY KEY,
user_id TEXT NOT NULL,
course_id TEXT NOT NULL,
title TEXT,
content TEXT NOT NULL,
content_sha256 TEXT NOT NULL,
char_count INTEGER NOT NULL,
visibility TEXT NOT NULL CHECK (visibility = 'private'),
created_at TEXT NOT NULL,
expires_at TEXT NOT NULL
);

CREATE INDEX IF NOT EXISTS idx_private_knowledge_retrieval
ON private_knowledge_items (user_id, course_id, expires_at);
CREATE INDEX IF NOT EXISTS idx_private_knowledge_expiry
ON private_knowledge_items (expires_at);
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
-- PLAN-3 C-1 contribution metadata and private attachment payloads.
ALTER TABLE contributions ADD COLUMN github_email TEXT;
ALTER TABLE contributions ADD COLUMN workflow_type TEXT;
ALTER TABLE contributions ADD COLUMN run_id TEXT;
ALTER TABLE contributions ADD COLUMN supplementary_text TEXT;
ALTER TABLE contributions ADD COLUMN citation_metadata_json TEXT NOT NULL DEFAULT '[]';
ALTER TABLE contributions ADD COLUMN corpus_metadata_json TEXT NOT NULL DEFAULT '{}';

CREATE INDEX IF NOT EXISTS idx_contributions_run ON contributions (run_id);

CREATE TABLE IF NOT EXISTS contribution_attachments (
attachment_id TEXT PRIMARY KEY,
contribution_id TEXT NOT NULL REFERENCES contributions(contribution_id) ON DELETE CASCADE,
original_filename TEXT NOT NULL,
content_type TEXT NOT NULL,
byte_size INTEGER NOT NULL,
sha256 TEXT NOT NULL,
payload BLOB NOT NULL,
created_at TEXT NOT NULL,
expires_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_contribution_attachments_owner
ON contribution_attachments (contribution_id, created_at);
CREATE INDEX IF NOT EXISTS idx_contribution_attachments_expiry
ON contribution_attachments (expires_at);
1 change: 1 addition & 0 deletions apps/scut-senior/api/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ dependencies = [
"idna>=3.18,<4",
"jsonschema>=4.25,<5",
"pydantic>=2.11,<3",
"python-multipart>=0.0.20,<1",
"PyYAML>=6,<7",
"scut-senior-worker==0.1.0",
"uvicorn[standard]>=0.35,<1",
Expand Down
23 changes: 21 additions & 2 deletions apps/scut-senior/api/src/scut_senior_api/adapters/local_corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,29 @@ def is_course_available(self, course_id: str) -> bool:
return True

def search(self, course_ids: list[str], query: str) -> RetrievalBatch:
if len(course_ids) != 1 or not course_ids[0]:
if not course_ids or len(course_ids) != len(set(course_ids)) or any(not course_id for course_id in course_ids):
raise CapabilityUnavailable(
"retrieval",
"local corpus retrieval requires exactly one explicit course",
"local corpus retrieval requires a non-empty unique course set",
)
if len(course_ids) > 1:
batches = [self.search([course_id], query) for course_id in course_ids]
# Do not let the first course consume the global limit. Round-robin
# preserves each course's independently thresholded candidates so a
# cross-course run is visibly and fairly represented downstream.
merged: list[RetrievedSource] = []
for index in range(max((len(batch.sources) for batch in batches), default=0)):
for batch in batches:
if index < len(batch.sources):
merged.append(batch.sources[index])
if len(merged) >= self.limit:
break
if len(merged) >= self.limit:
break
return RetrievalBatch(
tuple(merged),
batches[0].corpus_version,
batches[0].course_pack_version,
)
course_id = course_ids[0]
try:
Expand Down
6 changes: 3 additions & 3 deletions apps/scut-senior/api/src/scut_senior_api/adapters/mock.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ def is_course_available(self, course_id: str) -> bool:
return self.registry.get(course_id).fixture_available

def search(self, course_ids: list[str], query: str) -> RetrievalBatch:
del query # Iteration 0 proves filtering/contracts, not retrieval quality.
if len(course_ids) != 1:
del query # Fixture retrieval proves filtering/contracts, not ranking quality.
if not course_ids or len(course_ids) != len(set(course_ids)):
raise FixtureContractViolation(
"synthetic fixture retrieval requires exactly one course"
"synthetic fixture retrieval requires a non-empty unique course set"
)
if not self.manifest_path.exists():
return RetrievalBatch((), "fixture-corpus-v1")
Expand Down
124 changes: 116 additions & 8 deletions apps/scut-senior/api/src/scut_senior_api/adapters/sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@
)
from ..agent_loop import replay_agent_events
from ..contracts import (
ContributionAttachmentRecord,
ContributionRecord,
ContributionState,
ConversationDetail,
ConversationSummary,
FeedbackRecord,
PrivateKnowledgeRecord,
TemporaryMaterialDetail,
TemporaryMaterialRecord,
WorkflowAttempt,
Expand All @@ -45,7 +47,7 @@
)
from ..credentials import CREDENTIAL_ALGORITHM
from ..paths import MIGRATION_ROOT
from ..ports import StoredModelCredential
from ..ports import RetrievedSource, StoredModelCredential


HISTORY_TTL = timedelta(days=30)
Expand Down Expand Up @@ -474,12 +476,21 @@ def cleanup_material_records(self) -> MaterialCleanupCounts:
"SELECT name FROM sqlite_master WHERE type = 'table'"
)
}
if "temporary_materials" not in tables:
if "temporary_materials" not in tables and "contributions" not in tables and "private_knowledge_items" not in tables:
return MaterialCleanupCounts(0, 0)
materials = connection.execute(
"DELETE FROM temporary_materials WHERE expires_at <= ?",
(now,),
).rowcount
materials = 0
if "temporary_materials" in tables:
materials = connection.execute(
"DELETE FROM temporary_materials WHERE expires_at <= ?",
(now,),
).rowcount
if "private_knowledge_items" in tables:
# Private knowledge has the same physical TTL guarantee but is
# deliberately not counted as temporary conversation material.
connection.execute(
"DELETE FROM private_knowledge_items WHERE expires_at <= ?",
(now,),
)
cleared = connection.execute(
"""
UPDATE contributions
Expand Down Expand Up @@ -967,6 +978,50 @@ def delete_temporary_material(self, user_id: str, material_id: UUID) -> bool:
).rowcount
return deleted > 0

def save_private_knowledge(
self, *, user_id: str, course_id: str, title: str | None, content: str
) -> PrivateKnowledgeRecord:
now = self._now()
knowledge_id = uuid4()
expires_at = now + timedelta(days=TEMPORARY_MATERIAL_TTL_DAYS)
digest = hashlib.sha256(content.encode("utf-8")).hexdigest()
with self._connect() as connection:
connection.execute(
"""
INSERT INTO private_knowledge_items (
knowledge_id, user_id, course_id, title, content, content_sha256,
char_count, visibility, created_at, expires_at
) VALUES (?, ?, ?, ?, ?, ?, ?, 'private', ?, ?)
""",
(str(knowledge_id), user_id, course_id, title, content, digest, len(content), now.isoformat(), expires_at.isoformat()),
)
return PrivateKnowledgeRecord(
knowledge_id=knowledge_id, course_id=course_id, title=title,
char_count=len(content), content_sha256=digest, created_at=now, expires_at=expires_at,
)

def list_private_knowledge_sources(
self, *, user_id: str, course_ids: list[str]
) -> list[RetrievedSource]:
if not course_ids or len(course_ids) != len(set(course_ids)):
return []
placeholders = ", ".join("?" for _ in course_ids)
with self._connect() as connection:
rows = connection.execute(
f"""SELECT * FROM private_knowledge_items
WHERE user_id = ? AND visibility = 'private' AND expires_at > ?
AND course_id IN ({placeholders}) ORDER BY created_at DESC""",
(user_id, self._now().isoformat(), *course_ids),
).fetchall()
return [
RetrievedSource(
chunk_id=f"private:{row['knowledge_id']}", course_id=row["course_id"],
source_id=f"private:{row['knowledge_id']}", source_title=row["title"] or "私人知识",
text=row["content"], locator_type="private_knowledge", locator_start=None,
locator_end=None, question_id=None, heading_path=(),
) for row in rows
]

@staticmethod
def _contribution_record(row: sqlite3.Row) -> ContributionRecord:
pr_url = row["pr_url"]
Expand All @@ -991,6 +1046,13 @@ def _contribution_record(row: sqlite3.Row) -> ContributionRecord:
created_at=datetime.fromisoformat(row["created_at"]),
updated_at=datetime.fromisoformat(row["updated_at"]),
expires_at=datetime.fromisoformat(row["expires_at"]),
github_email=row["github_email"] if "github_email" in keys else None,
workflow_type=row["workflow_type"] if "workflow_type" in keys else None,
run_id=UUID(row["run_id"]) if "run_id" in keys and row["run_id"] else None,
supplementary_text=row["supplementary_text"] if "supplementary_text" in keys else None,
citation_metadata=json.loads(row["citation_metadata_json"] or "[]") if "citation_metadata_json" in keys else [],
corpus_metadata=json.loads(row["corpus_metadata_json"] or "{}") if "corpus_metadata_json" in keys else {},
has_attachments=False,
)

def create_contribution(
Expand All @@ -1004,6 +1066,12 @@ def create_contribution(
content_snapshot: str,
state: ContributionState,
proposed_repo_path: str = "",
github_email: str | None = None,
workflow_type: str | None = None,
run_id: UUID | None = None,
supplementary_text: str | None = None,
citation_metadata: list[dict[str, object]] | None = None,
corpus_metadata: dict[str, object] | None = None,
) -> ContributionRecord:
"""创建贡献记录。

Expand All @@ -1029,8 +1097,10 @@ def create_contribution(
proposed_source_id, proposed_repo_path, title,
content_snapshot, state,
pr_url, maintainer_note, char_count,
created_at, updated_at, expires_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, ?, ?)
created_at, updated_at, expires_at,
github_email, workflow_type, run_id, supplementary_text,
citation_metadata_json, corpus_metadata_json
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
str(contribution_id),
Expand All @@ -1046,6 +1116,12 @@ def create_contribution(
created_at,
updated_at,
expires_at,
github_email,
workflow_type,
str(run_id) if run_id is not None else None,
supplementary_text,
json.dumps(citation_metadata or [], ensure_ascii=False),
json.dumps(corpus_metadata or {}, ensure_ascii=False),
),
)
record = self.get_contribution(user_id, contribution_id)
Expand Down Expand Up @@ -1075,6 +1151,27 @@ def list_contributions(self, user_id: str) -> list[ContributionRecord]:
).fetchall()
return [self._contribution_record(row) for row in rows]

def create_contribution_attachment(
self, contribution_id: UUID, original_filename: str, content_type: str, payload: bytes
) -> ContributionAttachmentRecord:
now = self._now(); attachment_id = uuid4(); expires_at = now + timedelta(days=CONTRIBUTION_REVIEW_COPY_TTL_DAYS)
digest = hashlib.sha256(payload).hexdigest()
with self._connect() as connection:
connection.execute("INSERT INTO contribution_attachments (attachment_id, contribution_id, original_filename, content_type, byte_size, sha256, payload, created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (str(attachment_id), str(contribution_id), original_filename, content_type, len(payload), digest, payload, now.isoformat(), expires_at.isoformat()))
return ContributionAttachmentRecord(attachment_id=attachment_id, contribution_id=contribution_id, original_filename=original_filename, content_type=content_type, byte_size=len(payload), sha256=digest, created_at=now, expires_at=expires_at)

def list_contribution_attachments(self, contribution_id: UUID) -> list[ContributionAttachmentRecord]:
with self._connect() as connection:
rows = connection.execute("SELECT attachment_id, contribution_id, original_filename, content_type, byte_size, sha256, created_at, expires_at FROM contribution_attachments WHERE contribution_id = ? AND expires_at > ? ORDER BY created_at", (str(contribution_id), self._now().isoformat())).fetchall()
return [ContributionAttachmentRecord(attachment_id=UUID(row["attachment_id"]), contribution_id=UUID(row["contribution_id"]), original_filename=row["original_filename"], content_type=row["content_type"], byte_size=int(row["byte_size"]), sha256=row["sha256"], created_at=datetime.fromisoformat(row["created_at"]), expires_at=datetime.fromisoformat(row["expires_at"])) for row in rows]

def get_contribution_attachment(self, contribution_id: UUID, attachment_id: UUID) -> tuple[ContributionAttachmentRecord, bytes] | None:
with self._connect() as connection:
row = connection.execute("SELECT * FROM contribution_attachments WHERE contribution_id = ? AND attachment_id = ? AND expires_at > ?", (str(contribution_id), str(attachment_id), self._now().isoformat())).fetchone()
if row is None: return None
record = ContributionAttachmentRecord(attachment_id=attachment_id, contribution_id=contribution_id, original_filename=row["original_filename"], content_type=row["content_type"], byte_size=int(row["byte_size"]), sha256=row["sha256"], created_at=datetime.fromisoformat(row["created_at"]), expires_at=datetime.fromisoformat(row["expires_at"]))
return record, bytes(row["payload"])

def get_contribution_with_payload(
self, contribution_id: UUID
) -> tuple[ContributionRecord, str] | None:
Expand Down Expand Up @@ -1493,6 +1590,17 @@ def save_feedback(self, user_id: str, record: FeedbackRecord) -> None:
),
)

def list_all_feedback(self) -> list[FeedbackRecord]:
self.cleanup_history_records()
with self._connect() as connection:
rows = connection.execute(
"""SELECT feedback_id, user_id, run_id, conversation_id, course_id,
workflow_type, feedback_type, note, answer_status,
created_at, expires_at
FROM feedback ORDER BY created_at DESC, feedback_id DESC"""
).fetchall()
return [_feedback_record(row) for row in rows]

def list_feedback(self, user_id: str) -> list[FeedbackRecord]:
self.cleanup_history_records()
with self._connect() as connection:
Expand Down
29 changes: 27 additions & 2 deletions apps/scut-senior/api/src/scut_senior_api/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ class Settings:
onnx_embedding_max_length: int = 512
database_path: Path = APP_ROOT / ".local" / "iteration-zero.db"
corpus_store_path: Path = APP_ROOT / ".local" / "corpus-store"
cross_course_enabled: bool = False
# Enabled for the local fixture profile so the shipped cross-course UI is
# immediately testable; production deployments can explicitly disable it.
cross_course_enabled: bool = True
bilibili_resources_enabled: bool = True
# Iteration 5 (SOP §10): deterministic exam-review planning. The flag
# only gates the plan node, appendix and past-exam-first retrieval query;
Expand All @@ -59,6 +61,12 @@ class Settings:
github_client_secret: str | None = field(default=None, repr=False)
github_callback_url: str | None = None
post_login_redirect_url: str | None = None
maintainer_github_user_ids: tuple[int, ...] = ()
maintainer_github_logins: tuple[str, ...] = (
"AlexBybye",
"DevilSean",
)
# 维护者在上方添加,再在下方补充

@classmethod
def from_env(cls) -> "Settings":
Expand All @@ -67,7 +75,10 @@ def from_env(cls) -> "Settings":
identity_mode=os.getenv("SCUT_SENIOR_IDENTITY_MODE", "mock"),
model_mode=os.getenv("SCUT_SENIOR_MODEL_MODE", "mock"),
storage_mode=os.getenv("SCUT_SENIOR_STORAGE_MODE", "sqlite_mock"),
retrieval_mode=os.getenv("SCUT_SENIOR_RETRIEVAL_MODE", "fixture"),
# The running application uses the activated corpus by default;
# fixture retrieval remains available for isolated tests via an
# explicit Settings(retrieval_mode="fixture") or environment override.
retrieval_mode=os.getenv("SCUT_SENIOR_RETRIEVAL_MODE", "local_corpus"),
retrieval_min_score=_env_positive_float(
"SCUT_SENIOR_RETRIEVAL_MIN_SCORE", 1.0
),
Expand Down Expand Up @@ -128,7 +139,21 @@ def from_env(cls) -> "Settings":
post_login_redirect_url=os.getenv(
"SCUT_SENIOR_POST_LOGIN_REDIRECT_URL"
),
maintainer_github_user_ids=tuple(
int(value.strip())
for value in os.getenv("SCUT_SENIOR_MAINTAINER_GITHUB_USER_IDS", "").split(",")
if value.strip().isdigit()
),
maintainer_github_logins=tuple(
value.strip()
for value in os.getenv(
"SCUT_SENIOR_MAINTAINER_GITHUB_LOGINS",
"AlexBybye,DevilSean",
).split(",")
if value.strip()
),
)
# 就是上方这里补充

def assert_safe(self) -> None:
if self.app_env not in {"development", "test", "production"}:
Expand Down
Loading
Loading