Document remaining corpus crates in-line

crates/corpus-agent/src/baseline.rs

@@ -1,9 +1,9 @@
-//! Checkpointed, resumable baseline inventory (spec 10.7).
+//! Initial and periodic filesystem baseline walks.
 //!
-//! Each watch root is walked top-level entry by top-level entry. Completed
-//! entries are checkpointed in SQLite; a restart skips them. Baseline
-//! candidates are enqueued at the lowest capture priority so live events
-//! always win the worker (spec 10.8).
+//! Walks configured roots, applies exclusion patterns, and enqueues
+//! new or content-changed files for capture. Supports resume across
+//! process restarts by persisting completed directory watermarks in
+//! local state.
 
 use crate::state::{priority, StateDb};
 use anyhow::Result;

crates/corpus-agent/src/capture.rs

@@ -1,7 +1,8 @@
-//! Capture state machine driver (spec 10.4): OBSERVED -> DEBOUNCING ->
-//! OPENING -> COPYING_AND_HASHING -> HASHED -> ANNOUNCED -> DEDUP_HIT |
-//! UPLOAD_REQUIRED -> UPLOADING -> FINALIZING -> OCCURRENCE_QUEUED ->
-//! COMPLETE, or GAP_RECORDED with a spec 2.2 terminal outcome.
+//! Capture pipeline: dequeue work → stable read → spool → uploader handoff.
+//!
+//! Integrates spool pressure (defer without burning attempts), crash
+//! resume from intermediate states, and terminal outcomes that feed
+//! server-side `capture_attempt` records after upload.
 
 use crate::config::Config;
 use crate::stable_read::{self, StableReadError};

crates/corpus-agent/src/config.rs

@@ -1,4 +1,9 @@
-//! Agent configuration file, modeled on the spec 10.9 default policy.
+//! Agent configuration loaded from `agent.yaml` (or `--config` path).
+//!
+//! Covers server URL, tenant, enrollment material paths, filesystem roots
+//! and exclusions, spool directory, capture size limits, and sensor toggles.
+//! Invalid config fails fast at process start — the agent does not run
+//! with partial silent defaults for security-sensitive fields.
 
 use anyhow::{Context, Result};
 use serde::{Deserialize, Serialize};

crates/corpus-agent/src/fileid.rs

@@ -1,10 +1,8 @@
-//! Cross-platform file identity (spec 10.5 step 1): the OS-stable
-//! identity of a file for mutation detection during stable reads.
+//! Stable file identity for change detection.
 //!
-//! Unix: (dev, inode, size, mtime, ctime). Windows: (volume serial
-//! number, file index, size, mtime) via GetFileInformationByHandle,
-//! surfaced by std::os::windows::fs::MetadataExt. Windows has no ctime
-//! analog, so it is zeroed there.
+//! Combines filesystem identifiers (inode/device or Windows file id) with
+//! size/mtime so baseline and sensors can distinguish rewrite vs rename
+//! without reading file bytes on every tick.
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct FileKey {

crates/corpus-agent/src/heartbeat.rs

@@ -1,4 +1,8 @@
-//! Periodic health heartbeat (spec 10.11).
+//! Periodic agent heartbeat to the corpus server.
+//!
+//! Reports host name, agent version, boot id, and sequence high-water
+//! marks so operators can see coverage gaps when heartbeats stop or
+//! sequences stall.
 
 use crate::capture::AgentRuntime;
 use corpus_core::dto::HeartbeatRequest;

crates/corpus-agent/src/main.rs

@@ -1,9 +1,21 @@
-//! corpus-agent: Linux user-mode collection agent (spec 10, M1).
+//! corpus-agent: endpoint collection agent (spec 10).
 //!
-//! Observe-only: the agent never blocks execution and never runs
-//! server-supplied commands. Local state is SQLite WAL; the spool is
-//! plaintext with 0600/0700 permissions (encryption is M1-production
-//! hardening — see README deviations).
+//! # Threat model / posture
+//!
+//! **Observe-only.** The agent never blocks process execution and never
+//! runs server-supplied commands. It discovers code-bearing files, stages
+//! them into a local spool, and uploads via the announce/finalize protocol.
+//!
+//! # Runtime loops
+//!
+//! 1. **Enrollment** — one-time credential / mTLS material
+//! 2. **Baseline** — walk configured roots; enqueue new/changed files
+//! 3. **Sensors** — platform change journals (USN, fanotify, RDCW, …)
+//! 4. **Capture** — stable read → spool → hash → uploader
+//! 5. **Heartbeat** — liveness + sequence high-water marks
+//!
+//! Local state is SQLite WAL ([`state`]). Spool encryption is platform-
+//! dependent ([`spool_crypto`], Windows DPAPI).
 
 mod baseline;
 mod capture;

crates/corpus-agent/src/sensors/ads.rs

@@ -1,7 +1,7 @@
-//! Alternate data stream (ADS) awareness at the enumeration/metadata
-//! level (spec 10.10 Windows). Streams other than the default `::$DATA`
-//! are recorded as capture metadata; content collection of ADS is
-//! policy-controlled and not in M2 scope.
+//! Windows Alternate Data Stream (ADS) discovery hints.
+//!
+//! Surfaces non-default streams on NTFS that may hide payloads. Does not
+//! execute stream content; enqueues for capture like any other path.
 
 #![cfg(target_os = "windows")]
 

crates/corpus-agent/src/sensors/fanotify.rs

@@ -1,10 +1,8 @@
-//! fanotify sensor (spec 10.10 Linux): mount marks observe close-write and
-//! moved-to events; FAN_OPEN_EXEC is attempted best-effort for execution
-//! prioritization. Queue overflow is persisted as a SENSOR_OVERFLOW
-//! coverage gap and triggers the reconciliation fallback.
+//! Linux fanotify-based change sensor.
 //!
-//! Requires CAP_SYS_ADMIN for FAN_MARK_MOUNT. If initialization fails
-//! (EPERM, unsupported kernel) the caller falls back to the poll sensor.
+//! Marks configured roots and translates events into capture queue
+//! entries. Requires appropriate capabilities; falls back to poll when
+//! unavailable.
 
 use crate::state::{priority, StateDb};
 

crates/corpus-agent/src/sensors/mod.rs

@@ -1,10 +1,15 @@
-//! Filesystem event sensors.
+//! Filesystem change sensors.
 //!
-//! Linux uses fanotify mount marks where privileges permit (spec 10.10);
-//! Windows uses ReadDirectoryChangesW plus USN journal recovery
-//! (user-mode fallback; a signed minifilter is the production design);
-//! every platform has the periodic reconciliation-scan fallback. Sensor
-//! queue loss is a coverage gap, never a silent miss (spec 2.2).
+//! Platform backends feed a common capture queue. Each sensor is
+//! observe-only: it records paths and reasons, never blocks writers.
+//!
+//! | Sensor | Platform | Source |
+//! |--------|----------|--------|
+//! | `usn` | Windows | NTFS USN journal |
+//! | `rdcw` | Windows | ReadDirectoryChangesW |
+//! | `fanotify` | Linux | fanotify mark events |
+//! | `poll` | portable | periodic re-scan fallback |
+//! | `ads` | Windows | alternate data stream hints |
 
 #[cfg(target_os = "windows")]
 pub mod ads;

crates/corpus-agent/src/sensors/poll.rs

@@ -1,6 +1,7 @@
-//! Reconciliation-scan sensor: platform-neutral fallback that diffs file
-//! stat snapshots on an interval. Also the recovery path after fanotify
-//! queue overflow (spec 10.10 Linux: "trigger reconciliation").
+//! Portable periodic re-scan sensor.
+//!
+//! Used when native journals are unavailable or as a safety net. More
+//! expensive than event-driven sensors; interval is config-driven.
 
 use crate::baseline::reconcile_scan;
 use crate::config::Config;

crates/corpus-agent/src/sensors/rdcw.rs

@@ -1,16 +1,8 @@
-//! Windows user-mode sensor (spec 10.10 Windows, M2 "user-mode fallback
-//! first"). Coverage gaps versus the future signed minifilter are
-//! documented in the README; the short version:
+//! ReadDirectoryChangesW sensor (Windows).
 //!
-//! - ReadDirectoryChangesW for close-write/rename-create events
-//!   (recursive watch on configured roots).
-//! - USN change journal for downtime recovery where privileges allow
-//!   (FSCTL_READ_JOURNAL requires volume read access; degrades to the
-//!   periodic reconciliation scanner without admin).
-//! - Process execution observation is NOT implemented in user mode:
-//!   Win32_ProcessStartTrace requires admin + COM/WMI plumbing; that
-//!   gap is documented and exec-priority candidates fall back to
-//!   write-priority.
+//! Directory watches for create/write/rename events on configured roots.
+//! Complements USN when journal access is restricted. Parsing lives in
+//! [`rdcw_parse`] for unit testing without Win32 APIs.
 
 #![cfg(target_os = "windows")]
 

crates/corpus-agent/src/sensors/rdcw_parse.rs

@@ -1,15 +1,7 @@
-//! Platform-free parser for the ReadDirectoryChangesW
-//! FILE_NOTIFY_INFORMATION record stream, factored out of the Windows
-//! watcher so it is unit-testable on every platform.
+//! Pure parsers for ReadDirectoryChangesW FILE_NOTIFY_INFORMATION buffers.
 //!
-//! M9 review fix: the previous parser copied each FILE_NOTIFY_INFORMATION
-//! out of the buffer with `read_unaligned` into a stack local and then
-//! dereferenced `info.FileName.as_ptr()` — but `FileName` is a
-//! variable-length `WCHAR[1]` trailing field, so the pointer referred to
-//! the one-element array in the stack copy and any filename longer than
-//! one UTF-16 unit read out of bounds. This parser never copies records:
-//! it validates each record against the buffer length and decodes the
-//! filename slice in place from the original bytes.
+//! Hardened against odd filename lengths, truncated records, and bogus
+//! `NextEntryOffset` chains so a malformed buffer cannot loop forever.
 
 /// One decoded change notification.
 #[derive(Debug, Clone, PartialEq, Eq)]

crates/corpus-agent/src/sensors/usn.rs

@@ -1,16 +1,9 @@
-//! USN change journal support (spec 10.10 Windows): downtime recovery
-//! via FSCTL_READ_JOURNAL where privileges allow. The record parser, the
-//! journal-info parser, and the cursor/resume decision logic are
-//! platform-free and unit-tested; the readers are Windows-only and degrade
-//! gracefully to the poll sensor without volume access.
+//! NTFS USN journal sensor (Windows).
 //!
-//! Cursor continuity (M9 review fix): FSCTL_READ_JOURNAL requires the
-//! UsnJournalId of the CURRENT journal instance (queried via
-//! FSCTL_QUERY_USN_JOURNAL), and the agent persists (journal_id,
-//! next_usn) in its SQLite state so a restart resumes where it stopped
-//! instead of re-reading from USN 0. Journal recreation (ID mismatch) or
-//! a cursor older than the journal's first available record (truncation)
-//! means continuity is lost: a full reconciliation is forced.
+//! Resumes from a persisted (journal_id, next_usn) cursor. Journal id
+//! mismatch forces full reconciliation. Parses packed USN_RECORD
+//! structures including long filenames; truncated buffers yield partial
+//! progress without panic.
 
 use serde::Serialize;
 

crates/corpus-agent/src/spool_crypto.rs

@@ -1,18 +1,9 @@
-//! Encrypted spool (M6 hardening, spec 10.3): XChaCha20-Poly1305 at rest,
-//! key generated at enrollment, wrapped by the OS store where available.
+//! At-rest encryption for the local capture spool.
 //!
-//! macOS: Keychain generic-password item. Linux: 0600 key file (documented
-//! fallback; kernel keyring/TPM are later scope).
-//!
-//! Chunked spool format v2 (M9 review fix): every random value comes from
-//! OsRng (no UUID-derived material), and the per-chunk nonce is
-//! `16-byte random prefix || 8-byte little-endian chunk counter`. The
-//! prefix is stored in the spool file header after a one-byte format
-//! version: `[u8 version=2][16B prefix][u32 len][ct]...`. Format v1
-//! (8-byte prefix, no version byte) is REJECTED on read: the spool is
-//! transient, so pre-upgrade spool files are discarded rather than
-//! migrated — the affected candidates terminalize as gaps and are
-//! re-observed by the sensors.
+//! Encrypts staged sample bytes before they rest on disk pending upload.
+//! Format is versioned; older spool blobs are rejected rather than
+//! silently mis-decrypted. Key material is OS-backed where available
+//! (see `win32_dpapi` on Windows).
 
 use anyhow::{Context, Result};
 use chacha20poly1305::aead::{Aead, KeyInit};

crates/corpus-agent/src/stable_read.rs

@@ -1,6 +1,14 @@
-//! Stable read algorithm (spec 10.5): open without following symlinks,
-//! stream into the spool while hashing, re-stat, retry on mutation,
-//! terminal CHANGED_DURING_READ.
+//! Mutation-safe file reads into the local spool.
+//!
+//! # Algorithm
+//!
+//! 1. Open without following symlinks (platform-specific flags).
+//! 2. Snapshot size/mtime; copy to a private spool file while hashing.
+//! 3. Re-stat; if size/mtime changed, discard and retry.
+//! 4. Exhaust retries → [`StableReadError::ChangedDuringRead`].
+//!
+//! Rejects oversize files and full spools. Symlinks are not followed
+//! (TOCTOU / redirect hardening).
 
 use sha2::{Digest, Sha256};
 use std::fs::OpenOptions;

crates/corpus-agent/src/state.rs

@@ -1,9 +1,13 @@
-//! Durable local state: SQLite in WAL mode (spec 10.3).
+//! Local durable agent state (SQLite WAL).
 //!
-//! Holds agent identity, the capture state machine rows, baseline
-//! checkpoints, pending gap batches, seen-file snapshots, and health
-//! counters. Transitions are transactional: a crash may repeat a
-//! transition but cannot silently drop it (spec 10.4).
+//! Tracks:
+//! - Enrollment identity and sequence counters (boot-scoped)
+//! - Capture queue with priority (executables over baseline bulk)
+//! - Per-path work state machine (enqueued → reading → uploaded / deferred)
+//! - Sensor cursors (USN journal id, fanotify marks, …)
+//!
+//! All transitions are transactional so a crash mid-capture resumes
+//! without double-burning attempts or losing the queue.
 
 use anyhow::{Context, Result};
 use rusqlite::{params, Connection, OptionalExtension};

crates/corpus-agent/src/uploader.rs

@@ -1,5 +1,8 @@
-//! HTTP client for the server ingest/agent APIs. The server owns all
-//! writes; the agent reuses the M0 announce/upload/finalize flow.
+//! Announce / stage / finalize client for the corpus server.
+//!
+//! Implements the server ingest protocol with backoff, credential
+//! attachment (bearer or mTLS), and mapping of HTTP outcomes to local
+//! queue state transitions. Never invents artifact ids — digests only.
 
 use anyhow::{anyhow, Context, Result};
 use corpus_core::dto::*;

crates/corpus-agent/src/win32_dpapi.rs

@@ -1,7 +1,7 @@
-//! DPAPI key wrapping for the agent spool key on Windows (spec 10.3).
-//! The 32-byte key is protected with CryptProtectData (CurrentUser scope)
-//! and stored as a blob in the state dir — the same pattern as the macOS
-//! Keychain and the Linux 0600 key file.
+//! Windows DPAPI helpers for spool key protection.
+//!
+//! Wraps machine/user-scoped DPAPI so spool encryption keys are not
+//! stored as plaintext on disk. Compiled only on `target_os = "windows"`.
 
 #![cfg(target_os = "windows")]
 

crates/corpus-core/build.rs

@@ -1,3 +1,6 @@
+// Build script: expose the linked yara-x crate version as CORPUS_YARA_X_VERSION
+// so runtime digests and ENGINE_VERSION stay honest about the scanner engine.
+
 //! Build-time capture of the yara-x engine version so scan cache keys and
 //! hunt results can name the exact engine that produced them (invariant #7).
 

crates/corpus-core/src/agents.rs

@@ -1,8 +1,23 @@
 //! Agent enrollment, heartbeat, and gap reporting (spec 10.1, 10.11).
 //!
-//! M1 auth is deliberately simple: a one-time enrollment token is exchanged
-//! for a per-agent bearer token. mTLS enrollment is production hardening,
-//! not M1 scope (deviation from spec 8.3, documented in README).
+//! # Enrollment
+//!
+//! One-time enrollment tokens mint long-lived agent credentials. Tokens
+//! are shown once, hashed at rest, and expire by TTL. Successful enroll
+//! creates an `agent` row bound to a tenant and returns the credential
+//! material the endpoint stores locally.
+//!
+//! # Heartbeat
+//!
+//! Agents periodically report host identity, agent version, boot id, and
+//! sequence high-water marks. Missed heartbeats surface in coverage gap
+//! views.
+//!
+//! # Gaps
+//!
+//! Agents may report sequence gaps (lost observations). The server
+//! records them for analyst follow-up; it does not re-task the agent
+//! (observe-only).
 
 use crate::dto::{
     AgentStatusResponse, EnrollRequest, EnrollResponse, EnrollmentTokenResponse, GapEvent,
@@ -21,6 +36,7 @@ pub struct AgentIdentity {
     pub host_name: String,
 }
 
+/// SHA-256 of a plaintext enrollment/agent token for at-rest storage.
 pub fn hash_token(token: &str) -> Vec<u8> {
     hash::sha256_raw(token.as_bytes())
 }
@@ -298,6 +314,7 @@ const AGENT_COLS: &str = "id, host_name, version, enrolled_at, last_heartbeat_at
      policy_digest, baseline_state, baseline_percent, queue_depth, spool_bytes,
      oldest_pending_secs, sensor, outcome_counts, clock_offset_ms";
 
+/// List agents for a tenant with last-heartbeat metadata.
 pub async fn list_agents(pool: &PgPool, tenant_id: Uuid) -> Result<Vec<AgentStatusResponse>> {
     let rows = sqlx::query_as::<_, AgentRow>(&format!(
         "SELECT {AGENT_COLS} FROM agent WHERE tenant_id = $1 ORDER BY enrolled_at"
@@ -308,6 +325,7 @@ pub async fn list_agents(pool: &PgPool, tenant_id: Uuid) -> Result<Vec<AgentStat
     Ok(rows.into_iter().map(AgentRow::into_response).collect())
 }
 
+/// Fetch one agent's status by id.
 pub async fn agent_status(
     pool: &PgPool,
     tenant_id: Uuid,

crates/corpus-core/src/analyst.rs

@@ -1,5 +1,15 @@
-//! Analyst surface: prevalence as a first-class signal, rarity search,
-//! and the dropper heuristic. All pure SQL over the occurrence ledger.
+//! Analyst surface: prevalence, rarity search, and related queries.
+//!
+//! # Prevalence
+//!
+//! How widely is a digest observed across hosts/agents within a tenant?
+//! Low prevalence + high severity is a classic investigation pivot.
+//!
+//! # Rarity
+//!
+//! Search for uncommon structural features (imports, section layouts)
+//! among committed artifacts. Backed by similarity features, not raw
+//! bytes.
 
 use crate::error::{Error, Result};
 use chrono::{DateTime, Utc};

crates/corpus-core/src/auth.rs

@@ -1,12 +1,15 @@
 //! Admin API authentication and listen-address policy.
 //!
-//! - Loopback binds (`127.0.0.1`, `::1`, `localhost`) may run without an
-//!   admin token for local demos.
-//! - Non-loopback binds refuse to start without `CORPUS_ADMIN_TOKEN`.
-//! - When a token is configured, admin/CLI routes require
-//!   `Authorization: Bearer <token>`.
-//! - MCP requires `CORPUS_MCP_TOKEN`; the dev default `mcp-dev-token` is
-//!   rejected on non-loopback binds.
+//! # Goals
+//!
+//! - Bind admin routes to a shared secret or mTLS identity depending on
+//!   deployment mode.
+//! - Refuse to expose unauthenticated admin APIs on non-loopback
+//!   addresses without an explicit opt-in.
+//!
+//! Agent traffic uses separate enrollment tokens / mTLS (see [`crate::agents`]
+//! and [`crate::mtls`]); this module is for human/admin operators and
+//! control-plane tools like `corpusctl`.
 
 use crate::error::{Error, Result};
 use std::net::SocketAddr;