Add transactional document deletion and replacement

README.md

@@ -135,25 +135,36 @@ buffers are reconstructed rather than serialized. The service wraps that binary
 snapshot with deterministic JSON metadata in a versioned `RAGSTATE` container and
 a BLAKE2b checksum.
 
-Each persistent insertion is staged against a cloned index while holding the store
-lock. The complete container is written to a sibling temporary file, flushed and
+Each persistent corpus mutation is staged while holding the store lock. The
+complete container is written to a sibling temporary file, flushed and
 `fsync`ed, atomically replaced, and its parent directory is `fsync`ed before the
 new in-memory state is published and HTTP success is returned. A failed commit
-therefore retains the previous file and previous live corpus. Corrupt state causes
+therefore retains the previous file and previous live corpus. This transactional
+stage → persist → publish rule applies equally to add, delete, and replace.
+Corrupt state causes
 startup to fail; it is never silently discarded. Stored index settings, embedding
 dimension, backend class, and model name (where applicable) must match the runtime.
 
+Deletion and replacement compact the corpus with a full deterministic HNSW rebuild,
+ordered by ascending document id and then chunk ordinal. Surviving chunks reuse
+their vectors through the index `vector(id)` API; only replacement text is embedded.
+Document ids are stable (and deleted ids are never reused), while internal chunk ids
+may be reassigned so they remain dense and exactly aligned with HNSW vector ids.
+This rebuild is **O(number of live vectors)** and is deliberately chosen for
+correctness and simplicity at the current portfolio/demo scale, not as
+production-scale constant-time deletion.
+
 Limitations: snapshots coordinate one service process only; do not point multiple
 workers or hosts at the same file. Durability ultimately depends on the filesystem's
-`fsync` and atomic-replace semantics. There is no deletion, migration between
+`fsync` and atomic-replace semantics. There is no migration between
 embedder configurations, or automatic recovery of a corrupt snapshot.
 
 ### Public API hardening
 
 | Variable | Default | Behavior |
 |----------|---------|----------|
 | `RAG_STATE_PATH` | empty | Enables the versioned durable corpus snapshot at the configured file. Empty preserves in-memory behavior. |
-| `RAG_UPLOADS_ENABLED` | `0` | Enables `POST /documents` only when explicitly set to `1`, `true`, `yes`, or `on`. |
+| `RAG_UPLOADS_ENABLED` | `0` | Enables corpus mutations (`POST`, `PUT`, and `DELETE /documents`) only when explicitly set to `1`, `true`, `yes`, or `on`. |
 | `RAG_MAX_TITLE_CHARS` | `200` | Maximum document title length. |
 | `RAG_MAX_DOCUMENT_CHARS` | `1000000` | Maximum document text length. |
 | `RAG_MAX_QUESTION_CHARS` | `2000` | Maximum query length. |
@@ -165,7 +176,7 @@ embedder configurations, or automatic recovery of a corrupt snapshot.
 Chunking accepts `max_words` from 1 through 1000 and `overlap` from 0
 through 999, with overlap strictly smaller than the chunk size. Query `k`
 remains 1 through 50 and `ef_search` is limited to 1 through 2000. Oversized
-bodies return `413`, invalid fields return `422`, disabled uploads return
+bodies return `413`, invalid fields return `422`, disabled corpus mutations return
 `403`, and Claude timeouts return `504`. The Next.js same-origin proxy
 preserves these status codes and service messages.
 

app/app/api/documents/[id]/route.ts

@@ -0,0 +1,44 @@
+import { NextResponse } from "next/server";
+import { deleteDocument, replaceDocument } from "@/app/lib/rag";
+import { INPUT_LIMITS, readLimitedJson } from "@/app/lib/limits";
+import { apiErrorResponse } from "@/app/lib/api-errors";
+
+type Context = { params: Promise<{ id: string }> };
+
+function documentId(raw: string): number | null {
+  const id = Number(raw);
+  return Number.isSafeInteger(id) && id >= 0 ? id : null;
+}
+
+export async function DELETE(_: Request, context: Context) {
+  const id = documentId((await context.params).id);
+  if (id === null) return NextResponse.json({ error: "invalid document id" }, { status: 422 });
+  try {
+    return NextResponse.json(await deleteDocument(id));
+  } catch (err) {
+    return apiErrorResponse(err);
+  }
+}
+
+export async function PUT(req: Request, context: Context) {
+  const id = documentId((await context.params).id);
+  if (id === null) return NextResponse.json({ error: "invalid document id" }, { status: 422 });
+  try {
+    const body = await readLimitedJson<{ title?: unknown; text?: unknown }>(req);
+    if (typeof body.title !== "string" || !body.title.trim()) {
+      return NextResponse.json({ error: "title is required" }, { status: 422 });
+    }
+    if (body.title.trim().length > INPUT_LIMITS.titleChars) {
+      return NextResponse.json({ error: `title must be at most ${INPUT_LIMITS.titleChars} characters` }, { status: 422 });
+    }
+    if (typeof body.text !== "string" || !body.text.trim()) {
+      return NextResponse.json({ error: "document text is required" }, { status: 422 });
+    }
+    if (body.text.length > INPUT_LIMITS.documentChars) {
+      return NextResponse.json({ error: `document text must be at most ${INPUT_LIMITS.documentChars} characters` }, { status: 422 });
+    }
+    return NextResponse.json(await replaceDocument(id, body.title.trim(), body.text));
+  } catch (err) {
+    return apiErrorResponse(err);
+  }
+}

app/app/globals.css

@@ -165,6 +165,21 @@ button {
   font-weight: 700;
 }
 
+.text-button:disabled {
+  cursor: not-allowed;
+  opacity: 0.45;
+}
+
+.danger-button {
+  color: var(--danger);
+}
+
+.document-row > .document-actions {
+  display: flex;
+  gap: 12px;
+  margin-left: auto;
+}
+
 .metrics-grid {
   display: grid;
   grid-template-columns: repeat(4, 1fr);

app/app/lib/rag.ts

@@ -124,6 +124,21 @@ export function addDocument(title: string, text: string): Promise<DocumentInfo>
   });
 }
 
+export function deleteDocument(id: number): Promise<DocumentInfo> {
+  return call<DocumentInfo>(`/documents/${id}`, { method: "DELETE" });
+}
+
+export function replaceDocument(
+  id: number,
+  title: string,
+  text: string,
+): Promise<DocumentInfo> {
+  return call<DocumentInfo>(`/documents/${id}`, {
+    method: "PUT",
+    body: JSON.stringify({ title, text }),
+  });
+}
+
 export function listDocuments(): Promise<DocumentInfo[]> {
   return call<DocumentInfo[]>("/documents");
 }

app/app/page.tsx

@@ -68,6 +68,8 @@ export default function Home() {
   const [uploadError, setUploadError] = useState<string | null>(null);
   const [uploadNotice, setUploadNotice] = useState<string | null>(null);
   const [fileInputKey, setFileInputKey] = useState(0);
+  const [editingDocumentId, setEditingDocumentId] = useState<number | null>(null);
+  const [mutationDocumentId, setMutationDocumentId] = useState<number | null>(null);
 
   const uploadsEnabled = stats?.uploads_enabled ?? false;
   const titleLimit = stats?.limits.title_chars ?? 200;
@@ -149,20 +151,26 @@ export default function Home() {
     setUploadError(null);
     setUploadNotice(null);
     try {
-      const created = await requestJson<DocumentInfo>("/api/documents", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({
-          title: documentTitle.trim(),
-          text: documentText,
-        }),
-      });
+      const created = await requestJson<DocumentInfo>(
+        editingDocumentId === null
+          ? "/api/documents"
+          : `/api/documents/${editingDocumentId}`,
+        {
+          method: editingDocumentId === null ? "POST" : "PUT",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({
+            title: documentTitle.trim(),
+            text: documentText,
+          }),
+        },
+      );
       setUploadNotice(
-        `Indexed “${created.title}” as ${created.n_chunks} ${created.n_chunks === 1 ? "chunk" : "chunks"}.`,
+        `${editingDocumentId === null ? "Indexed" : "Replaced"} “${created.title}” as ${created.n_chunks} ${created.n_chunks === 1 ? "chunk" : "chunks"}.`,
       );
       setDocumentTitle("");
       setDocumentText("");
       setFileInputKey((key) => key + 1);
+      setEditingDocumentId(null);
       await loadWorkspace();
     } catch (err) {
       setUploadError(err instanceof Error ? err.message : "Unable to index document");
@@ -171,6 +179,39 @@ export default function Home() {
     }
   }
 
+  function beginReplacement(document: DocumentInfo) {
+    setEditingDocumentId(document.id);
+    setDocumentTitle(document.title);
+    setDocumentText("");
+    setUploadError(null);
+    setUploadNotice(`Paste the replacement text for “${document.title}”.`);
+  }
+
+  async function removeDocument(document: DocumentInfo) {
+    if (
+      !uploadsEnabled ||
+      !window.confirm(`Delete “${document.title}” and all of its chunks?`)
+    )
+      return;
+    setMutationDocumentId(document.id);
+    setWorkspaceError(null);
+    try {
+      await requestJson<DocumentInfo>(`/api/documents/${document.id}`, {
+        method: "DELETE",
+      });
+      if (editingDocumentId === document.id) {
+        setEditingDocumentId(null);
+        setDocumentTitle("");
+        setDocumentText("");
+      }
+      await loadWorkspace();
+    } catch (err) {
+      setWorkspaceError(err instanceof Error ? err.message : "Unable to delete document");
+    } finally {
+      setMutationDocumentId(null);
+    }
+  }
+
   const hashedRetrieval = stats?.embedder === "HashedEmbedder";
   const retrievalLabel = stats
     ? hashedRetrieval
@@ -356,8 +397,14 @@ export default function Home() {
           >
             <div className="panel-heading compact">
               <div>
-                <p className="section-kicker">ADD SOURCE</p>
-                <h3>Index a document</h3>
+                <p className="section-kicker">
+                  {editingDocumentId === null ? "ADD SOURCE" : "REPLACE SOURCE"}
+                </p>
+                <h3>
+                  {editingDocumentId === null
+                    ? "Index a document"
+                    : `Replace document ${editingDocumentId}`}
+                </h3>
               </div>
               <span className={`upload-status ${uploadsEnabled ? "enabled" : "locked"}`}>
                 {stats ? (uploadsEnabled ? "Enabled" : "Locked") : "Loading"}
@@ -429,7 +476,11 @@ export default function Home() {
                   !documentText.trim()
                 }
               >
-                {uploadLoading ? "Indexing…" : "Add to index"}
+                {uploadLoading
+                  ? "Saving…"
+                  : editingDocumentId === null
+                    ? "Add to index"
+                    : "Replace document"}
               </button>
             </div>
 
@@ -470,7 +521,24 @@ export default function Home() {
                       {document.n_chunks} {document.n_chunks === 1 ? "chunk" : "chunks"}
                     </span>
                   </div>
-                  <span className="indexed-mark">Indexed</span>
+                  <div className="document-actions">
+                    <button
+                      type="button"
+                      className="text-button"
+                      disabled={!uploadsEnabled || mutationDocumentId !== null}
+                      onClick={() => beginReplacement(document)}
+                    >
+                      Replace
+                    </button>
+                    <button
+                      type="button"
+                      className="text-button danger-button"
+                      disabled={!uploadsEnabled || mutationDocumentId !== null}
+                      onClick={() => void removeDocument(document)}
+                    >
+                      {mutationDocumentId === document.id ? "Deleting…" : "Delete"}
+                    </button>
+                  </div>
                 </div>
               ))}
             </div>

service/rag_service/app.py

@@ -4,6 +4,7 @@
 talks to this. Endpoints:
 
     POST /documents   upload text → chunk → embed → insert into HNSW
+    PUT/DELETE /documents/{id} replace or delete through HNSW compaction
     POST /query       embed question → HNSW search → Claude answer + sources
     GET  /stats       index size / config
     GET  /documents   list indexed documents
@@ -28,7 +29,7 @@
 from starlette.types import ASGIApp, Message, Receive, Scope, Send
 
 from .generation import GenerationTimeoutError, generate_answer
-from .store import DocumentStore
+from .store import DocumentNotFoundError, DocumentStore
 from hnsw_rag import get_embedder
 
 log = logging.getLogger(__name__)
@@ -79,7 +80,7 @@ def _add_cors_middleware(target: FastAPI, origins: List[str]) -> None:
     target.add_middleware(
         CORSMiddleware,
         allow_origins=origins,
-        allow_methods=["GET", "POST", "OPTIONS"],
+        allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
         allow_headers=["Content-Type"],
     )
 
@@ -305,6 +306,38 @@ def add_document(doc: DocumentIn) -> DocumentOut:
     return DocumentOut(id=result.id, title=result.title, n_chunks=result.n_chunks)
 
 
+@app.delete(
+    "/documents/{document_id}",
+    response_model=DocumentOut,
+    dependencies=[Depends(_require_uploads_enabled)],
+)
+def delete_document(document_id: int) -> DocumentOut:
+    try:
+        result = _store.delete_document(document_id)
+    except DocumentNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+    return DocumentOut(id=result.id, title=result.title, n_chunks=result.n_chunks)
+
+
+@app.put(
+    "/documents/{document_id}",
+    response_model=DocumentOut,
+    dependencies=[Depends(_require_uploads_enabled)],
+)
+def replace_document(document_id: int, doc: DocumentIn) -> DocumentOut:
+    try:
+        result = _store.replace_document(
+            document_id,
+            doc.title,
+            doc.text,
+            max_words=doc.max_words,
+            overlap=doc.overlap,
+        )
+    except DocumentNotFoundError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+    return DocumentOut(id=result.id, title=result.title, n_chunks=result.n_chunks)
+
+
 @app.post("/query", response_model=QueryOut)
 def query(q: QueryIn) -> QueryOut:
     chunks = _store.retrieve(q.question, k=q.k, ef_search=q.ef_search)