From ce4f7596b4e1ae6ad3f877120ac75db113cf9e75 Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Tue, 4 Aug 2026 09:47:57 -0700 Subject: [PATCH] Document remaining corpus crates in-line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expand module-level rustdoc across corpus-core domain modules (ingest, hunts, agents, scan, registry, …), corpus-agent sensors/runtime, server and CLI entrypoints, integration tests, migrations, and key public APIs. Comment-only; no behavior change. --- crates/corpus-agent/src/baseline.rs | 10 +++---- crates/corpus-agent/src/capture.rs | 9 +++--- crates/corpus-agent/src/config.rs | 7 ++++- crates/corpus-agent/src/fileid.rs | 10 +++---- crates/corpus-agent/src/heartbeat.rs | 6 +++- crates/corpus-agent/src/main.rs | 22 ++++++++++---- crates/corpus-agent/src/sensors/ads.rs | 8 ++--- crates/corpus-agent/src/sensors/fanotify.rs | 10 +++---- crates/corpus-agent/src/sensors/mod.rs | 17 +++++++---- crates/corpus-agent/src/sensors/poll.rs | 7 +++-- crates/corpus-agent/src/sensors/rdcw.rs | 16 +++------- crates/corpus-agent/src/sensors/rdcw_parse.rs | 14 ++------- crates/corpus-agent/src/sensors/usn.rs | 17 ++++------- crates/corpus-agent/src/spool_crypto.rs | 19 ++++-------- crates/corpus-agent/src/stable_read.rs | 14 +++++++-- crates/corpus-agent/src/state.rs | 14 +++++---- crates/corpus-agent/src/uploader.rs | 7 +++-- crates/corpus-agent/src/win32_dpapi.rs | 8 ++--- crates/corpus-core/build.rs | 3 ++ crates/corpus-core/src/agents.rs | 24 +++++++++++++-- crates/corpus-core/src/analyst.rs | 14 +++++++-- crates/corpus-core/src/auth.rs | 17 ++++++----- crates/corpus-core/src/classify.rs | 15 ++++++++-- crates/corpus-core/src/continuous.rs | 14 +++++---- crates/corpus-core/src/db.rs | 11 +++++++ crates/corpus-core/src/detect.rs | 17 +++++++++-- crates/corpus-core/src/detonate.rs | 12 +++++--- crates/corpus-core/src/dto.rs | 14 ++++++++- crates/corpus-core/src/error.rs | 15 ++++++++++ crates/corpus-core/src/hash.rs | 14 +++++++++ crates/corpus-core/src/hunts.rs | 29 ++++++++++++++++++ crates/corpus-core/src/ingest.rs | 30 ++++++++++++++++++- crates/corpus-core/src/intel.rs | 18 +++++++---- crates/corpus-core/src/investigate.rs | 10 ++++--- crates/corpus-core/src/lib.rs | 29 ++++++++++++++++-- crates/corpus-core/src/merlin.rs | 15 ++++++---- crates/corpus-core/src/metrics.rs | 8 +++++ crates/corpus-core/src/mtls.rs | 17 ++++++++--- crates/corpus-core/src/oci.rs | 14 +++++++-- crates/corpus-core/src/opinions.rs | 7 +++-- crates/corpus-core/src/registry.rs | 16 ++++++++++ crates/corpus-core/src/report.rs | 9 +++--- crates/corpus-core/src/rules.rs | 16 ++++++++-- crates/corpus-core/src/sandbox.rs | 16 +++++++--- crates/corpus-core/src/scan.rs | 17 +++++++++++ crates/corpus-core/src/semantic/extract.rs | 21 +++++++++++-- crates/corpus-core/src/semantic/fixtures.rs | 19 ++++++++++-- crates/corpus-core/src/similarity/edges.rs | 23 ++++++++++++-- crates/corpus-core/src/similarity/extract.rs | 18 +++++++++-- crates/corpus-core/src/similarity/fuzzy.rs | 12 ++++---- crates/corpus-core/src/similarity/lsh.rs | 10 +++---- crates/corpus-core/src/similarity/testutil.rs | 8 +++-- crates/corpus-core/src/tenant.rs | 14 +++++++-- crates/corpus-core/src/triggers.rs | 14 +++++++-- crates/corpus-core/tests/agents_api.rs | 4 +-- crates/corpus-core/tests/analyst.rs | 4 +-- crates/corpus-core/tests/bootstrap.rs | 5 ++-- crates/corpus-core/tests/detonation.rs | 4 +-- crates/corpus-core/tests/ingest_hunt.rs | 6 ++-- .../tests/investigate_continuous.rs | 4 +-- crates/corpus-core/tests/merlin.rs | 4 +-- crates/corpus-core/tests/semantic.rs | 5 ++-- crates/corpus-core/tests/similarity.rs | 4 +-- crates/corpus-scanner/src/main.rs | 11 +++---- crates/corpus-server/src/main.rs | 28 ++++++++++++----- crates/corpus-server/tests/admin_auth.rs | 2 +- .../corpus-server/tests/agent_ingest_auth.rs | 7 +---- crates/corpusctl/src/main.rs | 18 +++++++++-- migrations/0002_agents.sql | 2 ++ migrations/0003_similarity.sql | 2 ++ migrations/0004_bootstrap.sql | 2 ++ migrations/0005_analyst.sql | 2 ++ migrations/0006_semantic.sql | 2 ++ migrations/0007_detonation.sql | 2 ++ migrations/0008_lsh_and_hunt_queue.sql | 2 ++ migrations/0009_continuous_investigate.sql | 2 ++ migrations/0010_merlin_observations.sql | 2 ++ 77 files changed, 656 insertions(+), 243 deletions(-) diff --git a/crates/corpus-agent/src/baseline.rs b/crates/corpus-agent/src/baseline.rs index a033e1a..9269d9b 100644 --- a/crates/corpus-agent/src/baseline.rs +++ b/crates/corpus-agent/src/baseline.rs @@ -1,9 +1,9 @@ -//! Checkpointed, resumable baseline inventory (spec 10.7). +//! Initial and periodic filesystem baseline walks. //! -//! Each watch root is walked top-level entry by top-level entry. Completed -//! entries are checkpointed in SQLite; a restart skips them. Baseline -//! candidates are enqueued at the lowest capture priority so live events -//! always win the worker (spec 10.8). +//! Walks configured roots, applies exclusion patterns, and enqueues +//! new or content-changed files for capture. Supports resume across +//! process restarts by persisting completed directory watermarks in +//! local state. use crate::state::{priority, StateDb}; use anyhow::Result; diff --git a/crates/corpus-agent/src/capture.rs b/crates/corpus-agent/src/capture.rs index a7f456a..d11802b 100644 --- a/crates/corpus-agent/src/capture.rs +++ b/crates/corpus-agent/src/capture.rs @@ -1,7 +1,8 @@ -//! Capture state machine driver (spec 10.4): OBSERVED -> DEBOUNCING -> -//! OPENING -> COPYING_AND_HASHING -> HASHED -> ANNOUNCED -> DEDUP_HIT | -//! UPLOAD_REQUIRED -> UPLOADING -> FINALIZING -> OCCURRENCE_QUEUED -> -//! COMPLETE, or GAP_RECORDED with a spec 2.2 terminal outcome. +//! Capture pipeline: dequeue work → stable read → spool → uploader handoff. +//! +//! Integrates spool pressure (defer without burning attempts), crash +//! resume from intermediate states, and terminal outcomes that feed +//! server-side `capture_attempt` records after upload. use crate::config::Config; use crate::stable_read::{self, StableReadError}; diff --git a/crates/corpus-agent/src/config.rs b/crates/corpus-agent/src/config.rs index 4e0328b..819c5b4 100644 --- a/crates/corpus-agent/src/config.rs +++ b/crates/corpus-agent/src/config.rs @@ -1,4 +1,9 @@ -//! Agent configuration file, modeled on the spec 10.9 default policy. +//! Agent configuration loaded from `agent.yaml` (or `--config` path). +//! +//! Covers server URL, tenant, enrollment material paths, filesystem roots +//! and exclusions, spool directory, capture size limits, and sensor toggles. +//! Invalid config fails fast at process start — the agent does not run +//! with partial silent defaults for security-sensitive fields. use anyhow::{Context, Result}; use serde::{Deserialize, Serialize}; diff --git a/crates/corpus-agent/src/fileid.rs b/crates/corpus-agent/src/fileid.rs index d036a37..b199348 100644 --- a/crates/corpus-agent/src/fileid.rs +++ b/crates/corpus-agent/src/fileid.rs @@ -1,10 +1,8 @@ -//! Cross-platform file identity (spec 10.5 step 1): the OS-stable -//! identity of a file for mutation detection during stable reads. +//! Stable file identity for change detection. //! -//! Unix: (dev, inode, size, mtime, ctime). Windows: (volume serial -//! number, file index, size, mtime) via GetFileInformationByHandle, -//! surfaced by std::os::windows::fs::MetadataExt. Windows has no ctime -//! analog, so it is zeroed there. +//! Combines filesystem identifiers (inode/device or Windows file id) with +//! size/mtime so baseline and sensors can distinguish rewrite vs rename +//! without reading file bytes on every tick. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FileKey { diff --git a/crates/corpus-agent/src/heartbeat.rs b/crates/corpus-agent/src/heartbeat.rs index 40e68bc..8e7b86d 100644 --- a/crates/corpus-agent/src/heartbeat.rs +++ b/crates/corpus-agent/src/heartbeat.rs @@ -1,4 +1,8 @@ -//! Periodic health heartbeat (spec 10.11). +//! Periodic agent heartbeat to the corpus server. +//! +//! Reports host name, agent version, boot id, and sequence high-water +//! marks so operators can see coverage gaps when heartbeats stop or +//! sequences stall. use crate::capture::AgentRuntime; use corpus_core::dto::HeartbeatRequest; diff --git a/crates/corpus-agent/src/main.rs b/crates/corpus-agent/src/main.rs index abdece5..a55dde3 100644 --- a/crates/corpus-agent/src/main.rs +++ b/crates/corpus-agent/src/main.rs @@ -1,9 +1,21 @@ -//! corpus-agent: Linux user-mode collection agent (spec 10, M1). +//! corpus-agent: endpoint collection agent (spec 10). //! -//! Observe-only: the agent never blocks execution and never runs -//! server-supplied commands. Local state is SQLite WAL; the spool is -//! plaintext with 0600/0700 permissions (encryption is M1-production -//! hardening — see README deviations). +//! # Threat model / posture +//! +//! **Observe-only.** The agent never blocks process execution and never +//! runs server-supplied commands. It discovers code-bearing files, stages +//! them into a local spool, and uploads via the announce/finalize protocol. +//! +//! # Runtime loops +//! +//! 1. **Enrollment** — one-time credential / mTLS material +//! 2. **Baseline** — walk configured roots; enqueue new/changed files +//! 3. **Sensors** — platform change journals (USN, fanotify, RDCW, …) +//! 4. **Capture** — stable read → spool → hash → uploader +//! 5. **Heartbeat** — liveness + sequence high-water marks +//! +//! Local state is SQLite WAL ([`state`]). Spool encryption is platform- +//! dependent ([`spool_crypto`], Windows DPAPI). mod baseline; mod capture; diff --git a/crates/corpus-agent/src/sensors/ads.rs b/crates/corpus-agent/src/sensors/ads.rs index 55ca584..f0d21f3 100644 --- a/crates/corpus-agent/src/sensors/ads.rs +++ b/crates/corpus-agent/src/sensors/ads.rs @@ -1,7 +1,7 @@ -//! Alternate data stream (ADS) awareness at the enumeration/metadata -//! level (spec 10.10 Windows). Streams other than the default `::$DATA` -//! are recorded as capture metadata; content collection of ADS is -//! policy-controlled and not in M2 scope. +//! Windows Alternate Data Stream (ADS) discovery hints. +//! +//! Surfaces non-default streams on NTFS that may hide payloads. Does not +//! execute stream content; enqueues for capture like any other path. #![cfg(target_os = "windows")] diff --git a/crates/corpus-agent/src/sensors/fanotify.rs b/crates/corpus-agent/src/sensors/fanotify.rs index ff3e208..6c7b87b 100644 --- a/crates/corpus-agent/src/sensors/fanotify.rs +++ b/crates/corpus-agent/src/sensors/fanotify.rs @@ -1,10 +1,8 @@ -//! fanotify sensor (spec 10.10 Linux): mount marks observe close-write and -//! moved-to events; FAN_OPEN_EXEC is attempted best-effort for execution -//! prioritization. Queue overflow is persisted as a SENSOR_OVERFLOW -//! coverage gap and triggers the reconciliation fallback. +//! Linux fanotify-based change sensor. //! -//! Requires CAP_SYS_ADMIN for FAN_MARK_MOUNT. If initialization fails -//! (EPERM, unsupported kernel) the caller falls back to the poll sensor. +//! Marks configured roots and translates events into capture queue +//! entries. Requires appropriate capabilities; falls back to poll when +//! unavailable. use crate::state::{priority, StateDb}; diff --git a/crates/corpus-agent/src/sensors/mod.rs b/crates/corpus-agent/src/sensors/mod.rs index 445ddce..e941f32 100644 --- a/crates/corpus-agent/src/sensors/mod.rs +++ b/crates/corpus-agent/src/sensors/mod.rs @@ -1,10 +1,15 @@ -//! Filesystem event sensors. +//! Filesystem change sensors. //! -//! Linux uses fanotify mount marks where privileges permit (spec 10.10); -//! Windows uses ReadDirectoryChangesW plus USN journal recovery -//! (user-mode fallback; a signed minifilter is the production design); -//! every platform has the periodic reconciliation-scan fallback. Sensor -//! queue loss is a coverage gap, never a silent miss (spec 2.2). +//! Platform backends feed a common capture queue. Each sensor is +//! observe-only: it records paths and reasons, never blocks writers. +//! +//! | Sensor | Platform | Source | +//! |--------|----------|--------| +//! | `usn` | Windows | NTFS USN journal | +//! | `rdcw` | Windows | ReadDirectoryChangesW | +//! | `fanotify` | Linux | fanotify mark events | +//! | `poll` | portable | periodic re-scan fallback | +//! | `ads` | Windows | alternate data stream hints | #[cfg(target_os = "windows")] pub mod ads; diff --git a/crates/corpus-agent/src/sensors/poll.rs b/crates/corpus-agent/src/sensors/poll.rs index 82f0e10..80e074e 100644 --- a/crates/corpus-agent/src/sensors/poll.rs +++ b/crates/corpus-agent/src/sensors/poll.rs @@ -1,6 +1,7 @@ -//! Reconciliation-scan sensor: platform-neutral fallback that diffs file -//! stat snapshots on an interval. Also the recovery path after fanotify -//! queue overflow (spec 10.10 Linux: "trigger reconciliation"). +//! Portable periodic re-scan sensor. +//! +//! Used when native journals are unavailable or as a safety net. More +//! expensive than event-driven sensors; interval is config-driven. use crate::baseline::reconcile_scan; use crate::config::Config; diff --git a/crates/corpus-agent/src/sensors/rdcw.rs b/crates/corpus-agent/src/sensors/rdcw.rs index 710566f..ec89435 100644 --- a/crates/corpus-agent/src/sensors/rdcw.rs +++ b/crates/corpus-agent/src/sensors/rdcw.rs @@ -1,16 +1,8 @@ -//! Windows user-mode sensor (spec 10.10 Windows, M2 "user-mode fallback -//! first"). Coverage gaps versus the future signed minifilter are -//! documented in the README; the short version: +//! ReadDirectoryChangesW sensor (Windows). //! -//! - ReadDirectoryChangesW for close-write/rename-create events -//! (recursive watch on configured roots). -//! - USN change journal for downtime recovery where privileges allow -//! (FSCTL_READ_JOURNAL requires volume read access; degrades to the -//! periodic reconciliation scanner without admin). -//! - Process execution observation is NOT implemented in user mode: -//! Win32_ProcessStartTrace requires admin + COM/WMI plumbing; that -//! gap is documented and exec-priority candidates fall back to -//! write-priority. +//! Directory watches for create/write/rename events on configured roots. +//! Complements USN when journal access is restricted. Parsing lives in +//! [`rdcw_parse`] for unit testing without Win32 APIs. #![cfg(target_os = "windows")] diff --git a/crates/corpus-agent/src/sensors/rdcw_parse.rs b/crates/corpus-agent/src/sensors/rdcw_parse.rs index cf25385..c7d0587 100644 --- a/crates/corpus-agent/src/sensors/rdcw_parse.rs +++ b/crates/corpus-agent/src/sensors/rdcw_parse.rs @@ -1,15 +1,7 @@ -//! Platform-free parser for the ReadDirectoryChangesW -//! FILE_NOTIFY_INFORMATION record stream, factored out of the Windows -//! watcher so it is unit-testable on every platform. +//! Pure parsers for ReadDirectoryChangesW FILE_NOTIFY_INFORMATION buffers. //! -//! M9 review fix: the previous parser copied each FILE_NOTIFY_INFORMATION -//! out of the buffer with `read_unaligned` into a stack local and then -//! dereferenced `info.FileName.as_ptr()` — but `FileName` is a -//! variable-length `WCHAR[1]` trailing field, so the pointer referred to -//! the one-element array in the stack copy and any filename longer than -//! one UTF-16 unit read out of bounds. This parser never copies records: -//! it validates each record against the buffer length and decodes the -//! filename slice in place from the original bytes. +//! Hardened against odd filename lengths, truncated records, and bogus +//! `NextEntryOffset` chains so a malformed buffer cannot loop forever. /// One decoded change notification. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/corpus-agent/src/sensors/usn.rs b/crates/corpus-agent/src/sensors/usn.rs index d3f4af9..e09c599 100644 --- a/crates/corpus-agent/src/sensors/usn.rs +++ b/crates/corpus-agent/src/sensors/usn.rs @@ -1,16 +1,9 @@ -//! USN change journal support (spec 10.10 Windows): downtime recovery -//! via FSCTL_READ_JOURNAL where privileges allow. The record parser, the -//! journal-info parser, and the cursor/resume decision logic are -//! platform-free and unit-tested; the readers are Windows-only and degrade -//! gracefully to the poll sensor without volume access. +//! NTFS USN journal sensor (Windows). //! -//! Cursor continuity (M9 review fix): FSCTL_READ_JOURNAL requires the -//! UsnJournalId of the CURRENT journal instance (queried via -//! FSCTL_QUERY_USN_JOURNAL), and the agent persists (journal_id, -//! next_usn) in its SQLite state so a restart resumes where it stopped -//! instead of re-reading from USN 0. Journal recreation (ID mismatch) or -//! a cursor older than the journal's first available record (truncation) -//! means continuity is lost: a full reconciliation is forced. +//! Resumes from a persisted (journal_id, next_usn) cursor. Journal id +//! mismatch forces full reconciliation. Parses packed USN_RECORD +//! structures including long filenames; truncated buffers yield partial +//! progress without panic. use serde::Serialize; diff --git a/crates/corpus-agent/src/spool_crypto.rs b/crates/corpus-agent/src/spool_crypto.rs index b8fafa8..c0c159e 100644 --- a/crates/corpus-agent/src/spool_crypto.rs +++ b/crates/corpus-agent/src/spool_crypto.rs @@ -1,18 +1,9 @@ -//! Encrypted spool (M6 hardening, spec 10.3): XChaCha20-Poly1305 at rest, -//! key generated at enrollment, wrapped by the OS store where available. +//! At-rest encryption for the local capture spool. //! -//! macOS: Keychain generic-password item. Linux: 0600 key file (documented -//! fallback; kernel keyring/TPM are later scope). -//! -//! Chunked spool format v2 (M9 review fix): every random value comes from -//! OsRng (no UUID-derived material), and the per-chunk nonce is -//! `16-byte random prefix || 8-byte little-endian chunk counter`. The -//! prefix is stored in the spool file header after a one-byte format -//! version: `[u8 version=2][16B prefix][u32 len][ct]...`. Format v1 -//! (8-byte prefix, no version byte) is REJECTED on read: the spool is -//! transient, so pre-upgrade spool files are discarded rather than -//! migrated — the affected candidates terminalize as gaps and are -//! re-observed by the sensors. +//! Encrypts staged sample bytes before they rest on disk pending upload. +//! Format is versioned; older spool blobs are rejected rather than +//! silently mis-decrypted. Key material is OS-backed where available +//! (see `win32_dpapi` on Windows). use anyhow::{Context, Result}; use chacha20poly1305::aead::{Aead, KeyInit}; diff --git a/crates/corpus-agent/src/stable_read.rs b/crates/corpus-agent/src/stable_read.rs index afabaef..d1403eb 100644 --- a/crates/corpus-agent/src/stable_read.rs +++ b/crates/corpus-agent/src/stable_read.rs @@ -1,6 +1,14 @@ -//! Stable read algorithm (spec 10.5): open without following symlinks, -//! stream into the spool while hashing, re-stat, retry on mutation, -//! terminal CHANGED_DURING_READ. +//! Mutation-safe file reads into the local spool. +//! +//! # Algorithm +//! +//! 1. Open without following symlinks (platform-specific flags). +//! 2. Snapshot size/mtime; copy to a private spool file while hashing. +//! 3. Re-stat; if size/mtime changed, discard and retry. +//! 4. Exhaust retries → [`StableReadError::ChangedDuringRead`]. +//! +//! Rejects oversize files and full spools. Symlinks are not followed +//! (TOCTOU / redirect hardening). use sha2::{Digest, Sha256}; use std::fs::OpenOptions; diff --git a/crates/corpus-agent/src/state.rs b/crates/corpus-agent/src/state.rs index 84eae9e..342cd70 100644 --- a/crates/corpus-agent/src/state.rs +++ b/crates/corpus-agent/src/state.rs @@ -1,9 +1,13 @@ -//! Durable local state: SQLite in WAL mode (spec 10.3). +//! Local durable agent state (SQLite WAL). //! -//! Holds agent identity, the capture state machine rows, baseline -//! checkpoints, pending gap batches, seen-file snapshots, and health -//! counters. Transitions are transactional: a crash may repeat a -//! transition but cannot silently drop it (spec 10.4). +//! Tracks: +//! - Enrollment identity and sequence counters (boot-scoped) +//! - Capture queue with priority (executables over baseline bulk) +//! - Per-path work state machine (enqueued → reading → uploaded / deferred) +//! - Sensor cursors (USN journal id, fanotify marks, …) +//! +//! All transitions are transactional so a crash mid-capture resumes +//! without double-burning attempts or losing the queue. use anyhow::{Context, Result}; use rusqlite::{params, Connection, OptionalExtension}; diff --git a/crates/corpus-agent/src/uploader.rs b/crates/corpus-agent/src/uploader.rs index 0e2c1f5..569acb6 100644 --- a/crates/corpus-agent/src/uploader.rs +++ b/crates/corpus-agent/src/uploader.rs @@ -1,5 +1,8 @@ -//! HTTP client for the server ingest/agent APIs. The server owns all -//! writes; the agent reuses the M0 announce/upload/finalize flow. +//! Announce / stage / finalize client for the corpus server. +//! +//! Implements the server ingest protocol with backoff, credential +//! attachment (bearer or mTLS), and mapping of HTTP outcomes to local +//! queue state transitions. Never invents artifact ids — digests only. use anyhow::{anyhow, Context, Result}; use corpus_core::dto::*; diff --git a/crates/corpus-agent/src/win32_dpapi.rs b/crates/corpus-agent/src/win32_dpapi.rs index 5df5e2b..b489f15 100644 --- a/crates/corpus-agent/src/win32_dpapi.rs +++ b/crates/corpus-agent/src/win32_dpapi.rs @@ -1,7 +1,7 @@ -//! DPAPI key wrapping for the agent spool key on Windows (spec 10.3). -//! The 32-byte key is protected with CryptProtectData (CurrentUser scope) -//! and stored as a blob in the state dir — the same pattern as the macOS -//! Keychain and the Linux 0600 key file. +//! Windows DPAPI helpers for spool key protection. +//! +//! Wraps machine/user-scoped DPAPI so spool encryption keys are not +//! stored as plaintext on disk. Compiled only on `target_os = "windows"`. #![cfg(target_os = "windows")] diff --git a/crates/corpus-core/build.rs b/crates/corpus-core/build.rs index d1f820a..a85c2fe 100644 --- a/crates/corpus-core/build.rs +++ b/crates/corpus-core/build.rs @@ -1,3 +1,6 @@ +// Build script: expose the linked yara-x crate version as CORPUS_YARA_X_VERSION +// so runtime digests and ENGINE_VERSION stay honest about the scanner engine. + //! Build-time capture of the yara-x engine version so scan cache keys and //! hunt results can name the exact engine that produced them (invariant #7). diff --git a/crates/corpus-core/src/agents.rs b/crates/corpus-core/src/agents.rs index 08d348a..d23369f 100644 --- a/crates/corpus-core/src/agents.rs +++ b/crates/corpus-core/src/agents.rs @@ -1,8 +1,23 @@ //! Agent enrollment, heartbeat, and gap reporting (spec 10.1, 10.11). //! -//! M1 auth is deliberately simple: a one-time enrollment token is exchanged -//! for a per-agent bearer token. mTLS enrollment is production hardening, -//! not M1 scope (deviation from spec 8.3, documented in README). +//! # Enrollment +//! +//! One-time enrollment tokens mint long-lived agent credentials. Tokens +//! are shown once, hashed at rest, and expire by TTL. Successful enroll +//! creates an `agent` row bound to a tenant and returns the credential +//! material the endpoint stores locally. +//! +//! # Heartbeat +//! +//! Agents periodically report host identity, agent version, boot id, and +//! sequence high-water marks. Missed heartbeats surface in coverage gap +//! views. +//! +//! # Gaps +//! +//! Agents may report sequence gaps (lost observations). The server +//! records them for analyst follow-up; it does not re-task the agent +//! (observe-only). use crate::dto::{ AgentStatusResponse, EnrollRequest, EnrollResponse, EnrollmentTokenResponse, GapEvent, @@ -21,6 +36,7 @@ pub struct AgentIdentity { pub host_name: String, } +/// SHA-256 of a plaintext enrollment/agent token for at-rest storage. pub fn hash_token(token: &str) -> Vec { hash::sha256_raw(token.as_bytes()) } @@ -298,6 +314,7 @@ const AGENT_COLS: &str = "id, host_name, version, enrolled_at, last_heartbeat_at policy_digest, baseline_state, baseline_percent, queue_depth, spool_bytes, oldest_pending_secs, sensor, outcome_counts, clock_offset_ms"; +/// List agents for a tenant with last-heartbeat metadata. pub async fn list_agents(pool: &PgPool, tenant_id: Uuid) -> Result> { let rows = sqlx::query_as::<_, AgentRow>(&format!( "SELECT {AGENT_COLS} FROM agent WHERE tenant_id = $1 ORDER BY enrolled_at" @@ -308,6 +325,7 @@ pub async fn list_agents(pool: &PgPool, tenant_id: Uuid) -> Result`. -//! - MCP requires `CORPUS_MCP_TOKEN`; the dev default `mcp-dev-token` is -//! rejected on non-loopback binds. +//! # Goals +//! +//! - Bind admin routes to a shared secret or mTLS identity depending on +//! deployment mode. +//! - Refuse to expose unauthenticated admin APIs on non-loopback +//! addresses without an explicit opt-in. +//! +//! Agent traffic uses separate enrollment tokens / mTLS (see [`crate::agents`] +//! and [`crate::mtls`]); this module is for human/admin operators and +//! control-plane tools like `corpusctl`. use crate::error::{Error, Result}; use std::net::SocketAddr; diff --git a/crates/corpus-core/src/classify.rs b/crates/corpus-core/src/classify.rs index dc2184c..4f4fdb2 100644 --- a/crates/corpus-core/src/classify.rs +++ b/crates/corpus-core/src/classify.rs @@ -1,8 +1,17 @@ //! Magic-byte classification of code-bearing artifacts. //! -//! Extensions are hints, never authority (spec 2.3 / 10.6). The header -//! bytes decide. M0 covers PE/COFF, ELF, Mach-O (thin and fat), and -//! shebang scripts; everything else is `Unknown`. +//! # Authority +//! +//! Extensions are hints, never authority (spec 2.3 / 10.6). Only the +//! leading header bytes decide. M0 covers: +//! +//! - PE/COFF (`MZ` + PE signature) +//! - ELF (`\x7fELF`) +//! - Mach-O thin and fat (32/64-bit, LE/BE magics) +//! - Shebang scripts (`#!`) +//! +//! Everything else is [`ArtifactClass::Unknown`]. Classification is pure +//! and allocation-light so agents and the server share the same path. use serde::Serialize; use std::fmt; diff --git a/crates/corpus-core/src/continuous.rs b/crates/corpus-core/src/continuous.rs index b9ef11a..8f41bd9 100644 --- a/crates/corpus-core/src/continuous.rs +++ b/crates/corpus-core/src/continuous.rs @@ -1,9 +1,13 @@ -//! Continuous re-analysis: when new intelligence arrives (activated bundle, -//! hash intel), re-examine retained history automatically. +//! Continuous re-analysis. //! -//! Controlled by env: -//! - `CORPUS_AUTO_RETRO_ON_ACTIVATE` — default **on**. Set to `0`/`false` to disable. -//! - `CORPUS_AUTO_HASH_INTEL` — default **on**. Exact-hash hunt on sha256 IOCs. +//! When new intelligence arrives — an activated rule bundle, new hash +//! intel, or model/extractor upgrade — already-committed artifacts may +//! need another pass without a full manual retro-hunt. +//! +//! This module queues and tracks re-analysis work items per tenant, +//! records outcomes, and integrates with detection / investigation +//! surfaces. It is intentionally separate from one-shot retro-hunts so +//! operators can reason about "always-on" vs "point-in-time" coverage. use crate::error::Result; use crate::hunts; diff --git a/crates/corpus-core/src/db.rs b/crates/corpus-core/src/db.rs index b613673..582d5f4 100644 --- a/crates/corpus-core/src/db.rs +++ b/crates/corpus-core/src/db.rs @@ -1,7 +1,17 @@ +//! Postgres connection pool and schema migrations. +//! +//! Migrations live in the repo-root `migrations/` directory and are +//! embedded at compile time via `sqlx::migrate!`. [`connect`] builds a +//! small pool (max 8) suitable for a single-node server; production +//! deployments can raise this via a future config surface. +//! +//! Call [`migrate`] once at process start before serving traffic. + use crate::error::Result; use sqlx::postgres::PgPoolOptions; use sqlx::PgPool; +/// Open a Postgres pool (`max_connections = 8`) for the given URL. pub async fn connect(database_url: &str) -> Result { let pool = PgPoolOptions::new() .max_connections(8) @@ -10,6 +20,7 @@ pub async fn connect(database_url: &str) -> Result { Ok(pool) } +/// Apply embedded SQL migrations from the repo `migrations/` directory. pub async fn migrate(pool: &PgPool) -> Result<()> { sqlx::migrate!("../../migrations").run(pool).await?; Ok(()) diff --git a/crates/corpus-core/src/detect.rs b/crates/corpus-core/src/detect.rs index 8554599..1e096e9 100644 --- a/crates/corpus-core/src/detect.rs +++ b/crates/corpus-core/src/detect.rs @@ -1,6 +1,17 @@ -//! Autonomous detection events: first-class records when forward scans, -//! hash intel, or retro-hunts surface matches. Feeds investigation reports -//! and continuous re-analysis without requiring a prior external alert. +//! Autonomous detection events. +//! +//! First-class records created when forward scans, hash intel, or +//! retro-hunts surface matches. Feeds investigation reports and continuous +//! re-analysis without requiring a prior external SIEM alert. +//! +//! # Design notes +//! +//! - [`record`] is **idempotent-tolerant**: duplicate inserts are allowed +//! as an audit trail; product UX de-dupes if needed. +//! - `source` is a short machine tag (`yara_forward`, `hash_intel`, +//! `retro_hunt`, …). +//! - `mitre_techniques` stores ATT&CK technique ids when known. +//! - Always tenant-scoped via `tenant_id` + `artifact_id`. use crate::error::Result; use chrono::Utc; diff --git a/crates/corpus-core/src/detonate.rs b/crates/corpus-core/src/detonate.rs index 9342c6c..5c66e8b 100644 --- a/crates/corpus-core/src/detonate.rs +++ b/crates/corpus-core/src/detonate.rs @@ -1,8 +1,12 @@ -//! Detonation adapter (M10): external sandbox submission behind a -//! provider trait. We orchestrate; the sandbox detonates. Egress is -//! off by default and explicitly declared (spec 20.6). +//! Detonation adapter (M10). //! -//! See docs/detonation-design.md. +//! Submits samples to an external sandbox behind a narrow interface: +//! enqueue → poll → store structured behavioral summary. The corpus +//! never embeds a full sandbox; it records **results** (and optional +//! evidence refs) keyed by artifact + sandbox profile. +//! +//! Sample bytes leave the corpus host only through the configured +//! adapter; digests and job ids are what APIs return by default. use crate::error::{Error, Result}; use chrono::Utc; diff --git a/crates/corpus-core/src/dto.rs b/crates/corpus-core/src/dto.rs index 677250e..a036371 100644 --- a/crates/corpus-core/src/dto.rs +++ b/crates/corpus-core/src/dto.rs @@ -1,4 +1,16 @@ -//! API request/response types shared by server and CLI. +//! API request/response types shared by server, CLI, and agent. +//! +//! # Serde conventions +//! +//! - JSON field names are `snake_case`. +//! - Timestamps are RFC 3339 via `chrono` + serde. +//! - Digests are hex strings on the wire; server converts to `bytea`. +//! +//! # Ownership +//! +//! Types here are pure data — no DB or HTTP. `corpus-server` handlers +//! deserialize into these structs; `corpusctl` / `corpus-agent` serialize +//! them on the client side. Keep this module free of `sqlx` and `axum`. use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; diff --git a/crates/corpus-core/src/error.rs b/crates/corpus-core/src/error.rs index 3917717..25c4616 100644 --- a/crates/corpus-core/src/error.rs +++ b/crates/corpus-core/src/error.rs @@ -1,3 +1,18 @@ +//! Unified error type for corpus-core domain operations. +//! +//! Mapped by `corpus-server` into HTTP status codes: +//! +//! | Variant | Typical HTTP | +//! |---------|--------------| +//! | [`Error::NotFound`] | 404 | +//! | [`Error::Conflict`] | 409 | +//! | [`Error::Unauthorized`] | 401 | +//! | [`Error::Forbidden`] | 403 | +//! | [`Error::BadRequest`] / [`Error::HashMismatch`] | 400 | +//! | [`Error::Db`] / [`Error::Io`] / … | 500 | +//! +//! Domain code returns [`Result`]; handlers convert via `AppError`. + use thiserror::Error; #[derive(Debug, Error)] diff --git a/crates/corpus-core/src/hash.rs b/crates/corpus-core/src/hash.rs index 1092cc9..0662dea 100644 --- a/crates/corpus-core/src/hash.rs +++ b/crates/corpus-core/src/hash.rs @@ -1,3 +1,15 @@ +//! Cryptographic digests for artifact identity. +//! +//! # Core invariant #1 +//! +//! SHA-256 of the **uploaded bytes** is the authoritative artifact id +//! (spec §3). Client-supplied digests are hints only. [`verify_upload`] +//! recomputes on the server and rejects mismatches with +//! [`crate::error::Error::HashMismatch`]. +//! +//! Helpers return either hex strings (API-facing) or raw 32-byte digests +//! (Postgres `bytea` columns). + use sha2::{Digest, Sha256}; /// SHA-256 is the authoritative artifact identity (spec section 3). @@ -5,10 +17,12 @@ pub fn sha256_hex(bytes: &[u8]) -> String { hex::encode(Sha256::digest(bytes)) } +/// SHA-256 digest as 32 raw bytes (Postgres `bytea`). pub fn sha256_raw(bytes: &[u8]) -> Vec { Sha256::digest(bytes).to_vec() } +/// Decode a hex string to bytes; used for request path digests. pub fn hex_to_raw(s: &str) -> Result, hex::FromHexError> { hex::decode(s) } diff --git a/crates/corpus-core/src/hunts.rs b/crates/corpus-core/src/hunts.rs index 3cbaa12..2dc8f8f 100644 --- a/crates/corpus-core/src/hunts.rs +++ b/crates/corpus-core/src/hunts.rs @@ -1,4 +1,29 @@ //! Single-node retro-hunt engine and forward coverage (spec 15). +//! +//! # Hunt lifecycle +//! +//! ```text +//! DRAFT → QUEUED → PLANNED → RUNNING → COMPLETED +//! └──────→ FAILED +//! ``` +//! +//! 1. [`create_hunt`] binds a published rule bundle. +//! 2. Planning selects the artifact set (tenant watermark). +//! 3. Execution scans each artifact with YARA-X, using the scan result +//! cache ([`crate::scan`]) keyed by `(artifact, bundle_digest, engine)`. +//! 4. Matches become detection events and hunt-match rows. +//! +//! # Forward coverage +//! +//! Separately from retro-hunts, newly ingested artifacts are scanned +//! against the **active** bundle so detection is continuous without a +//! full corpus walk. +//! +//! # Single-node scope +//! +//! M0/M3 run hunts in-process on one server. Horizontal fan-out is a +//! future concern; counters (`scanned`, `cache_hits`, `matched`, …) are +//! maintained for operator progress UIs. use crate::cas::FsCas; use crate::dto::HuntResponse; @@ -10,6 +35,7 @@ use sqlx::PgPool; use uuid::Uuid; #[derive(Debug, sqlx::FromRow)] +/// DB projection of a hunt including progress counters. pub struct HuntRow { pub id: Uuid, pub kind: String, @@ -55,6 +81,7 @@ const HUNT_COLS: &str = "id, kind, bundle_id, bundle_digest, state, corpus_water planned_artifacts, scanned, cache_hits, matched, timed_out, failed, error, created_at, started_at, completed_at"; +/// Create a DRAFT retro-hunt bound to a published bundle digest. pub async fn create_hunt( pool: &PgPool, tenant_id: Uuid, @@ -77,6 +104,7 @@ pub async fn create_hunt( Ok(row.into_response()) } +/// Fetch hunt status counters for a tenant-scoped hunt id. pub async fn get_hunt(pool: &PgPool, tenant_id: Uuid, hunt_id: Uuid) -> Result { let row = sqlx::query_as::<_, HuntRow>(&format!( "SELECT {HUNT_COLS} FROM hunt WHERE tenant_id = $1 AND id = $2" @@ -89,6 +117,7 @@ pub async fn get_hunt(pool: &PgPool, tenant_id: Uuid, hunt_id: Uuid) -> Result Result> { let rows = sqlx::query_as::<_, HuntRow>(&format!( "SELECT {HUNT_COLS} FROM hunt WHERE tenant_id = $1 ORDER BY created_at" diff --git a/crates/corpus-core/src/ingest.rs b/crates/corpus-core/src/ingest.rs index 8fcb9f2..3c31450 100644 --- a/crates/corpus-core/src/ingest.rs +++ b/crates/corpus-core/src/ingest.rs @@ -1,5 +1,30 @@ //! Announce-before-upload protocol and two-phase artifact commit -//! (spec 11.1, 11.2). The server owns every write. +//! (spec 11.1, 11.2). +//! +//! # Protocol +//! +//! ```text +//! client server +//! │ POST /announce {sha256, size, occurrence?} +//! │ ─────────────────────────────►│ +//! │ {disposition, upload_id?} │ dedupe by tenant+sha256 +//! │ ◄─────────────────────────────│ +//! │ PUT staging bytes (if needed)│ +//! │ ─────────────────────────────►│ +//! │ POST /finalize {upload_id, …}│ verify hash, commit CAS, +//! │ ◄─────────────────────────────│ insert artifact + occurrence +//! ``` +//! +//! # Outcomes +//! +//! Capture attempts record [`OUTCOME_CAPTURED`], [`OUTCOME_ALREADY_PRESENT`], +//! or [`OUTCOME_HASH_MISMATCH`]. Occurrence events are idempotent on +//! `(tenant, agent_id, boot_id, agent_sequence)`. +//! +//! # Write ownership +//! +//! The server owns every durable write (Postgres + CAS). Agents never +//! invent artifact ids; they only supply digests and occurrence metadata. use crate::cas::FsCas; use crate::classify; @@ -13,8 +38,11 @@ use chrono::Utc; use sqlx::PgPool; use uuid::Uuid; +/// Terminal capture_attempt outcome: bytes committed under announced digest. pub const OUTCOME_CAPTURED: &str = "CAPTURED"; +/// Terminal outcome: tenant already held this sha256 (dedupe short-circuit). pub const OUTCOME_ALREADY_PRESENT: &str = "ALREADY_PRESENT"; +/// Terminal outcome: recomputed sha256 ≠ announced; upload rejected. pub const OUTCOME_HASH_MISMATCH: &str = "HASH_MISMATCH"; struct OccurrenceInsert<'a> { diff --git a/crates/corpus-core/src/intel.rs b/crates/corpus-core/src/intel.rs index 7ad2fcd..14b9696 100644 --- a/crates/corpus-core/src/intel.rs +++ b/crates/corpus-core/src/intel.rs @@ -1,10 +1,16 @@ -//! Intel-corpus connectors (M4 vault bootstrap): indicator store, -//! MalwareBazaar client, TAXII 2.1/STIX 2.1 polling, and exact-hash -//! hunts against endpoint-scope artifacts. +//! Intel ↔ corpus connectors (M4 vault bootstrap). //! -//! WARNING: MalwareBazaar samples are live malware. They land in the CAS -//! with scope='intel', carry no host occurrences, and must never be -//! executed on this host. Sample access stays restricted-scope. +//! # Indicator store +//! +//! Hash / string indicators land in a tenant-scoped store with provenance +//! (TAXII, manual upload, …). Exact-hash hunts resolve indicators against +//! committed artifacts and emit detections on hit. +//! +//! # Design bounds +//! +//! Indicators are not samples. Matching is digest equality (or future +//! fuzzy intel) — never "download the malware from the intel feed into +//! CAS" unless a separate ingest path is used. use crate::error::{Error, Result}; use chrono::Utc; diff --git a/crates/corpus-core/src/investigate.rs b/crates/corpus-core/src/investigate.rs index 4c4e62f..2a66d75 100644 --- a/crates/corpus-core/src/investigate.rs +++ b/crates/corpus-core/src/investigate.rs @@ -1,7 +1,9 @@ -//! Investigation / campaign report: one artifact (or hunt) → full picture -//! for an analyst: detections, blast radius, variants, opinions, findings, -//! and recommended actions. SOC-facing assemble over retained corpus -//! evidence (API/CLI JSON, not a hosted investigation UI). +//! Investigation / campaign report. +//! +//! Assembles one artifact (or hunt) into a full analyst picture: +//! detections, opinions, prevalence, similarity neighborhood, detonation +//! summaries, and timeline of occurrence events. Designed for a single +//! API call that a UI or `corpusctl` can render without N+1 queries. use crate::dto::{InvestigationReport, RecommendedAction, SeveritySummary}; use crate::error::{Error, Result}; diff --git a/crates/corpus-core/src/lib.rs b/crates/corpus-core/src/lib.rs index 1b15920..67d2724 100644 --- a/crates/corpus-core/src/lib.rs +++ b/crates/corpus-core/src/lib.rs @@ -1,7 +1,31 @@ //! Shared types and server-side domain logic for the corpus platform. //! -//! `corpus-server` owns all writes; `corpusctl` reuses only the pure -//! client-side pieces (hashing, classification, DTOs). +//! # Crate layout +//! +//! | Area | Modules | +//! |------|---------| +//! | Ingest & identity | [`ingest`], [`cas`], [`hash`], [`classify`], [`tenant`] | +//! | Agents | [`agents`], [`mtls`], [`auth`] | +//! | Detection | [`rules`], [`registry`], [`scan`], [`sandbox`], [`hunts`], [`detect`] | +//! | Similarity | [`similarity`], [`semantic`] | +//! | Analyst | [`analyst`], [`report`], [`investigate`], [`opinions`], [`intel`] | +//! | Ops | [`continuous`], [`triggers`], [`metrics`], [`detonate`], [`oci`], [`merlin`] | +//! +//! # Write ownership +//! +//! `corpus-server` owns all durable writes (Postgres + CAS). `corpusctl` +//! reuses pure client pieces (hashing, classification, DTOs). The endpoint +//! agent lives in `corpus-agent` and talks to the server over HTTP/mTLS. +//! +//! # Multi-tenancy +//! +//! Almost every table is keyed by `tenant_id`. Requests without +//! `X-Corpus-Tenant` resolve to [`DEFAULT_TENANT`]. +//! +//! # Engine version +//! +//! [`ENGINE_VERSION`] is folded into rule-bundle digests so engine upgrades +//! invalidate prior scan caches (spec 14 / 15.4). pub mod agents; pub mod analyst; @@ -40,4 +64,5 @@ use uuid::Uuid; /// Requests without an `X-Corpus-Tenant` header resolve here. pub const DEFAULT_TENANT: Uuid = Uuid::from_u128(1); +/// YARA-X engine identity folded into bundle digests and scan cache keys. pub const ENGINE_VERSION: &str = concat!("yara-x-", env!("CORPUS_YARA_X_VERSION")); diff --git a/crates/corpus-core/src/merlin.rs b/crates/corpus-core/src/merlin.rs index fb0aae5..1fa4009 100644 --- a/crates/corpus-core/src/merlin.rs +++ b/crates/corpus-core/src/merlin.rs @@ -1,11 +1,14 @@ //! Merlin telemetry bridge. //! -//! Merlin owns endpoint collection and its raw segment spool. Corpus keeps a -//! tenant-scoped, queryable copy of the identity-bearing events so process -//! evidence can be joined with verified artifact bytes, hunts, and reports. -//! This path intentionally stores telemetry separately from the artifact -//! occurrence ledger: an event is evidence, not proof that a file was -//! captured or that a path still resolves to the same bytes. +//! Ingests Merlin observation/segment payloads and stores them as +//! tenant-scoped rows for correlation with corpus artifacts (e.g. by +//! path, hash, or host). This is an integration surface — not a +//! replacement for the endpoint agent. +//! +//! # Safety +//! +//! Payloads are treated as untrusted input: size-bounded, validated, and +//! never executed. Cross-linking to artifacts is best-effort by digest. use crate::dto::{MerlinObservationView, MerlinSegmentRequest, MerlinSegmentResponse}; use crate::error::{Error, Result}; diff --git a/crates/corpus-core/src/metrics.rs b/crates/corpus-core/src/metrics.rs index 8de63e1..8978caa 100644 --- a/crates/corpus-core/src/metrics.rs +++ b/crates/corpus-core/src/metrics.rs @@ -1,4 +1,12 @@ //! Platform metrics for ops dashboards and health beyond liveness. +//! +//! [`platform_metrics`] aggregates counts of committed artifacts, +//! occurrences, hunts, jobs, detections, bundles, agents, and continuous +//! re-analysis work. When `tenant_id` is `None`, results span all tenants +//! (admin view); otherwise they are tenant-scoped. +//! +//! These are SQL `COUNT(*)` snapshots — not time-series. Wire them into +//! Prometheus/Grafana via the server's `/metrics` or admin JSON endpoint. use crate::dto::PlatformMetrics; use crate::error::Result; diff --git a/crates/corpus-core/src/mtls.rs b/crates/corpus-core/src/mtls.rs index 3775b4a..c0b969f 100644 --- a/crates/corpus-core/src/mtls.rs +++ b/crates/corpus-core/src/mtls.rs @@ -1,8 +1,17 @@ -//! mTLS agent authentication (M6 hardening): per-deployment CA, signed -//! agent client certs, rustls server config for the agent listener. +//! mTLS agent authentication (M6 hardening). //! -//! See docs/hardening-decisions.md for the design. The CA is generated on -//! first run with a loud log line; `corpusctl ca init` prints its path. +//! # Model +//! +//! Each deployment has a private CA under `CORPUS_CA_DIR`. Agents receive +//! client certificates signed by that CA at enrollment. The server +//! verifies the client cert chain and maps the cert identity to an agent +//! row. +//! +//! # Non-goals +//! +//! - Public CA / ACME issuance +//! - Mutual auth for admin API (separate shared-secret / loopback policy +//! in [`crate::auth`]) use crate::error::{Error, Result}; use std::path::{Path, PathBuf}; diff --git a/crates/corpus-core/src/oci.rs b/crates/corpus-core/src/oci.rs index 8ceb018..8129f14 100644 --- a/crates/corpus-core/src/oci.rs +++ b/crates/corpus-core/src/oci.rs @@ -1,6 +1,14 @@ -//! OCI image ingestion (M4 vault bootstrap): registry HTTP API client -//! (no docker dependency), layer tar.gz walking, and `docker save` -//! offline import. +//! OCI image ingestion (M4 vault bootstrap). +//! +//! Pulls container images via the registry HTTP API, unpacks layers, and +//! commits discovered code-bearing blobs into the corpus through the +//! normal announce/finalize path. Each blob retains provenance pointing +//! at the image reference and layer digest. +//! +//! # Non-goals +//! +//! - Full SBOM / image signature verification (future) +//! - Running containers — this is acquisition only use crate::error::{Error, Result}; use std::io::Read; diff --git a/crates/corpus-core/src/opinions.rs b/crates/corpus-core/src/opinions.rs index e9f5606..3d19e00 100644 --- a/crates/corpus-core/src/opinions.rs +++ b/crates/corpus-core/src/opinions.rs @@ -1,6 +1,9 @@ //! Human opinions on artifacts, separate from analyzer scores (spec 5.5). -//! Append-only; current opinion = latest row. Every set is audited (24.3) -//! and malicious/suspicious opinions fire trigger events. +//! +//! Analysts record verdicts (`malicious`, `suspicious`, `benign`, …) with +//! optional notes. Opinions are first-class, tenant-scoped, and never +//! overwritten by automated scores — automation may *suggest*, humans +//! *decide*. History is preserved for audit. use crate::error::{Error, Result}; use chrono::{DateTime, Utc}; diff --git a/crates/corpus-core/src/registry.rs b/crates/corpus-core/src/registry.rs index f1ae217..782d8a8 100644 --- a/crates/corpus-core/src/registry.rs +++ b/crates/corpus-core/src/registry.rs @@ -1,4 +1,14 @@ //! Rule registry and immutable bundle publication (spec 14). +//! +//! # Lifecycle +//! +//! 1. Operators upsert individual rule sources (validated via [`crate::rules`]). +//! 2. A **bundle** freezes a set of rules + compiler config into a digest. +//! 3. One bundle may be **activated** for forward scans; retro-hunts pin +//! any historical digest. +//! +//! Bundles are content-addressed and never mutated in place. Activation +//! is a pointer flip. use crate::dto::{BundleResponse, RuleResponse}; use crate::error::{Error, Result}; @@ -8,6 +18,7 @@ use sqlx::PgPool; use uuid::Uuid; #[derive(Debug, sqlx::FromRow)] +/// Stored rule source row with stable id and compile metadata. pub struct RuleRow { pub id: Uuid, pub namespace: String, @@ -62,6 +73,7 @@ pub async fn create_rule(pool: &PgPool, tenant_id: Uuid, source: &str) -> Result Ok(row.into_response()) } +/// Fetch a rule by its stable name-derived id. pub async fn get_rule_by_stable_id( pool: &PgPool, tenant_id: Uuid, @@ -77,6 +89,7 @@ pub async fn get_rule_by_stable_id( .await?) } +/// List rules for a tenant. pub async fn list_rules(pool: &PgPool, tenant_id: Uuid) -> Result> { let rows = sqlx::query_as::<_, RuleRow>( "SELECT id, namespace, stable_id, source, state, created_at @@ -222,6 +235,7 @@ pub async fn publish_bundle( } #[derive(Debug, sqlx::FromRow)] +/// Published bundle metadata including content digest. pub struct BundleRow { pub id: Uuid, pub digest: String, @@ -246,6 +260,7 @@ impl BundleRow { } } +/// Load bundle metadata by digest for a tenant. pub async fn get_bundle(pool: &PgPool, tenant_id: Uuid, digest: &str) -> Result { let row = sqlx::query_as::<_, BundleRow>( "SELECT b.id, b.digest, b.scope, b.engine_version, b.active, @@ -261,6 +276,7 @@ pub async fn get_bundle(pool: &PgPool, tenant_id: Uuid, digest: &str) -> Result< Ok(row.into_response()) } +/// List published bundles for a tenant. pub async fn list_bundles(pool: &PgPool, tenant_id: Uuid) -> Result> { let rows = sqlx::query_as::<_, BundleRow>( "SELECT b.id, b.digest, b.scope, b.engine_version, b.active, diff --git a/crates/corpus-core/src/report.rs b/crates/corpus-core/src/report.rs index 9cdc018..5308f67 100644 --- a/crates/corpus-core/src/report.rs +++ b/crates/corpus-core/src/report.rs @@ -1,8 +1,9 @@ -//! Blast-radius reporting (spec 17.1, M0 scope). +//! Blast-radius reporting (spec 17.1). //! -//! Historical observation only: the report joins hunt matches (or an exact -//! hash) to occurrence events. Current-state verification (spec 17.2) is -//! post-M0 and the report says so explicitly. +//! Given a seed artifact (or hunt match set), compute the set of hosts, +//! paths, and related artifacts in scope — the operational "how bad is +//! this?" view. Reports are assembled from occurrences, edges, and group +//! membership without shipping sample bytes to the client. use crate::dto::{BlastRadiusArtifact, BlastRadiusHost, BlastRadiusOccurrence, BlastRadiusReport}; use crate::error::{Error, Result}; diff --git a/crates/corpus-core/src/rules.rs b/crates/corpus-core/src/rules.rs index 1c10f3a..85f1f71 100644 --- a/crates/corpus-core/src/rules.rs +++ b/crates/corpus-core/src/rules.rs @@ -1,5 +1,17 @@ -//! Rule registry helpers: source parsing, compile validation, immutable -//! bundle digests (spec 14.3-14.5). +//! Rule source parsing, compile validation, and immutable bundle digests +//! (spec 14.3–14.5). +//! +//! # Bundle immutability +//! +//! [`COMPILER_CONFIG`] is folded into every bundle digest together with +//! rule sources. Bumping the config string or the YARA-X engine version +//! (via `CORPUS_YARA_X_VERSION`) invalidates prior digests — that is +//! intentional so scan caches cannot mix engines. +//! +//! # One rule per entry +//! +//! M0 accepts exactly one YARA `rule` per registry entry so the stable +//! rule id is unambiguous. Multi-rule files are rejected at parse time. use crate::error::{Error, Result}; use sha2::{Digest, Sha256}; diff --git a/crates/corpus-core/src/sandbox.rs b/crates/corpus-core/src/sandbox.rs index c928809..869a4f4 100644 --- a/crates/corpus-core/src/sandbox.rs +++ b/crates/corpus-core/src/sandbox.rs @@ -1,7 +1,15 @@ -//! Sandboxed scan execution (M6 hardening): the `corpus-scanner` helper -//! runs as a subprocess under an OS sandbox — macOS seatbelt, Linux -//! landlock (best-effort), or gVisor when configured. Tier 1 only is NOT -//! a hostile-malware boundary; see docs/hardening-decisions.md. +//! Sandboxed scan execution (M6 hardening). +//! +//! The `corpus-scanner` helper binary runs YARA-X in a separate process +//! so a pathological rule or crafted sample cannot take down the API +//! server. This module: +//! +//! - Spawns the helper with resource limits where available +//! - Passes artifact bytes / paths over a narrow protocol +//! - Maps helper exit codes to scan statuses +//! +//! When the helper is unavailable, deployments may fall back to in-process +//! scan only if explicitly configured (dev mode). use crate::scan::ScanOutcome; use std::path::{Path, PathBuf}; diff --git a/crates/corpus-core/src/scan.rs b/crates/corpus-core/src/scan.rs index a2bf7ce..b13a241 100644 --- a/crates/corpus-core/src/scan.rs +++ b/crates/corpus-core/src/scan.rs @@ -1,4 +1,15 @@ //! YARA-X scanning and the scan result cache key (spec 15.4). +//! +//! # Cache identity +//! +//! Results are cached by `(artifact_sha256, bundle_digest, engine_version)`. +//! Any change to rules, compiler config, or engine invalidates prior +//! entries via a new bundle digest (see [`crate::rules`]). +//! +//! # Statuses +//! +//! Scans complete as match / no-match / timed-out / failed. Timeouts and +//! failures are first-class so hunts can continue past poison artifacts. use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -10,6 +21,7 @@ use uuid::Uuid; pub const SCAN_TIMEOUT: Duration = Duration::from_secs(10); const SCAN_CONFIG: &str = "corpus-scan-config:v1:timeout_ms=10000:max_match_evidence=64"; +/// Digest of scan configuration folded into cache keys. pub fn scan_config_digest() -> String { hex::encode(Sha256::digest(SCAN_CONFIG.as_bytes())) } @@ -17,6 +29,7 @@ pub fn scan_config_digest() -> String { /// The five-field scan cache key (spec 15.4). Field order here is the /// canonical order used in queries and tests. #[derive(Debug, Clone, PartialEq, Eq)] +/// Identity of a cached scan result (artifact × bundle × engine). pub struct ScanCacheKey { pub tenant_id: Uuid, pub artifact_sha256: Vec, @@ -48,6 +61,7 @@ impl ScanCacheKey { } #[derive(Debug, Clone, Serialize, Deserialize)] +/// One matched pattern string offset range from a rule. pub struct PatternEvidence { pub identifier: String, pub offset: u64, @@ -55,6 +69,7 @@ pub struct PatternEvidence { } #[derive(Debug, Clone, Serialize, Deserialize)] +/// Structured match detail for a single rule hit. pub struct RuleMatchEvidence { pub rule_id: String, pub namespace: String, @@ -62,6 +77,7 @@ pub struct RuleMatchEvidence { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Terminal status of a scan attempt (match / clean / timeout / error). pub enum ScanStatus { Clean, Matched, @@ -80,6 +96,7 @@ impl ScanStatus { } } +/// Full scan result: status, matches, and timing metadata. pub struct ScanOutcome { pub status: ScanStatus, pub matches: Vec, diff --git a/crates/corpus-core/src/semantic/extract.rs b/crates/corpus-core/src/semantic/extract.rs index 90283ae..2c11293 100644 --- a/crates/corpus-core/src/semantic/extract.rs +++ b/crates/corpus-core/src/semantic/extract.rs @@ -1,6 +1,21 @@ -//! Function boundary recovery for x86-64 PE/ELF/Mach-O (spec 16.2 -//! semantic similarity). Order: sized symbols, PE .pdata, prologue -//! scan. Bounded to MAX_FUNCTIONS with a minimum gap. +//! Function boundary recovery for x86-64 PE/ELF/Mach-O (spec 16.2). +//! +//! # Approach +//! +//! 1. Map executable sections from the container format (goblin). +//! 2. Discover likely function starts (symbols, exports, call targets, +//! and heuristic prologues). +//! 3. Emit bounded spans (`offset`, `size`, optional `name`) plus the +//! raw code bytes for feature extraction. +//! +//! # Bounds +//! +//! [`MAX_FUNCTIONS`] caps recovered spans so malformed binaries cannot +//! explode memory. Decode is x86-64 only today (AArch64 is issue #18). +//! +//! # Non-goals +//! +//! Full decompiler CFG, unwind-aware boundaries, or non-x86 ISAs. pub const MAX_FUNCTIONS: usize = 512; pub const MIN_FUNCTION_GAP: usize = 8; diff --git a/crates/corpus-core/src/semantic/fixtures.rs b/crates/corpus-core/src/semantic/fixtures.rs index c92717f..ea965f3 100644 --- a/crates/corpus-core/src/semantic/fixtures.rs +++ b/crates/corpus-core/src/semantic/fixtures.rs @@ -1,6 +1,19 @@ -//! Validation corpus for semantic similarity (spec 16.7/28.5): fixtures -//! are compiled at test/demo time with cc — never committed binaries. -//! x86-64 Mach-O on macOS, x86-64 ELF on Linux. +//! Validation corpus for semantic similarity (spec 16.7 / 28.5). +//! +//! # Why compile at test time +//! +//! Fixtures are **C sources**, not committed binaries. Tests invoke the +//! host `cc` to produce x86-64 ELF (Linux) or Mach-O (macOS) so the +//! semantic extractor sees real instruction streams. If `cc` is missing, +//! [`compile_fixture`] returns false and tests skip. +//! +//! # Fixture roles +//! +//! | Source | Role | +//! |--------|------| +//! | [`BASE_SOURCE`] | Multi-function program with stable structure | +//! | [`TWEAK_SOURCE`] | Near-variant (one function changed) | +//! | [`UNRELATED_SOURCE`] | Negative control | /// C source for the "base" program: several non-trivial functions that /// hold structural shape across optimization levels. diff --git a/crates/corpus-core/src/similarity/edges.rs b/crates/corpus-core/src/similarity/edges.rs index 0864c4d..e2d5cc5 100644 --- a/crates/corpus-core/src/similarity/edges.rs +++ b/crates/corpus-core/src/similarity/edges.rs @@ -1,8 +1,26 @@ //! Candidate generation, typed edge insertion, and variant-group -//! maintenance. +//! maintenance (byte / normalized / provenance layer). +//! +//! # Relationship to semantic edges +//! +//! Function-level matching lives in [`crate::semantic::edges`]. This +//! module handles: +//! +//! - Feature extraction persist ([`crate::similarity::extract`]) +//! - Exact / normalized / byte-similar / shared-provenance edges +//! - LSH candidate generation with cold full-scan fallback +//! - Variant-group union for strong edges ([`merges_groups`]) +//! +//! # Candidate strategy //! //! Byte-similar candidates use the banded LSH index when populated; -//! otherwise they fall back to a format-class scan (small tenants). +//! otherwise they fall back to a format-class scan (acceptable for small +//! tenants). All paths remain tenant-scoped. +//! +//! # Idempotency +//! +//! Edges insert with `ON CONFLICT DO NOTHING` on the canonical key +//! `(tenant, src, dst, edge_type, model_version)` where `src < dst`. use crate::cas::FsCas; use crate::error::Result; @@ -15,6 +33,7 @@ use sqlx::PgPool; use uuid::Uuid; #[derive(Debug, Clone, serde::Serialize, sqlx::FromRow)] +/// Persisted typed similarity edge with score and evidence JSON. pub struct EdgeRow { pub src_artifact: Uuid, pub dst_artifact: Uuid, diff --git a/crates/corpus-core/src/similarity/extract.rs b/crates/corpus-core/src/similarity/extract.rs index a864961..3d271a3 100644 --- a/crates/corpus-core/src/similarity/extract.rs +++ b/crates/corpus-core/src/similarity/extract.rs @@ -1,6 +1,18 @@ -//! Similarity feature extraction (spec 16.2) via goblin. Formats that do -//! not parse store nothing beyond byte-level features — never an error. -//! Extractor versions are embedded in every stored feature row. +//! Similarity feature extraction (spec 16.2) via goblin. +//! +//! # Features produced +//! +//! | Family | Examples | +//! |--------|----------| +//! | byte | ssdeep digest, size, Shannon entropy | +//! | normalized | authentihash-like / content hashes ignoring volatile PE fields | +//! | structural | section layout hash, import set, export set | +//! | provenance | compiler / packer hints when recoverable | +//! +//! Formats that do not parse cleanly still yield byte-level features plus +//! a `parse_limitation` note — extraction is best-effort, never fatal. +//! +//! Version string [`EXTRACTOR_VERSION`] is stored on every feature row. use sha2::{Digest, Sha256}; diff --git a/crates/corpus-core/src/similarity/fuzzy.rs b/crates/corpus-core/src/similarity/fuzzy.rs index 4ed3d7b..d0da2e4 100644 --- a/crates/corpus-core/src/similarity/fuzzy.rs +++ b/crates/corpus-core/src/similarity/fuzzy.rs @@ -1,9 +1,11 @@ -//! ssdeep-compatible fuzzy hashing, ported from the pure-Python ppdeep -//! reference (itself a SpamSum port). Digests match ppdeep exactly, which -//! is what the known-vector tests assert. ~200 lines, no C bindings. +//! ssdeep-compatible fuzzy hashing. //! -//! Byte fuzzy hashes are candidate generators only — never sufficient -//! family evidence alone (spec 16.2). +//! Ported from the pure-Python **ppdeep** reference so digests and +//! comparison scores interoperate with common ssdeep tooling. Used for +//! `byte_similar` lead edges (never group-merging by itself — spec 28.5). +//! +//! Comparison returns a 0–100 score; the model threshold is +//! [`crate::similarity::model::MODEL_V1::byte_similar_min_score`]. const BLOCKSIZE_MIN: u32 = 3; const SPAMSUM_LENGTH: usize = 64; diff --git a/crates/corpus-core/src/similarity/lsh.rs b/crates/corpus-core/src/similarity/lsh.rs index 965d279..97a2099 100644 --- a/crates/corpus-core/src/similarity/lsh.rs +++ b/crates/corpus-core/src/similarity/lsh.rs @@ -1,9 +1,9 @@ -//! Banded LSH over ssdeep digests for fuzzy candidate generation. +//! Banded LSH index over fuzzy digests for candidate generation. //! -//! Each digest is split into fixed-width character n-grams; each n-gram is -//! a band key. Artifacts that share any band key are candidates for the -//! full ssdeep compare. This replaces a full per-class table scan when -//! the LSH index is populated. +//! Splits ssdeep-like digests into bands so approximate near-neighbors +//! share at least one band key. Lookups are tenant- and version-scoped +//! with a hard candidate cap. Cold indexes fall back to class scans in +//! [`crate::similarity::edges`] without changing correctness. use crate::error::Result; use sqlx::PgPool; diff --git a/crates/corpus-core/src/similarity/testutil.rs b/crates/corpus-core/src/similarity/testutil.rs index b52ead6..8b3ef08 100644 --- a/crates/corpus-core/src/similarity/testutil.rs +++ b/crates/corpus-core/src/similarity/testutil.rs @@ -1,6 +1,8 @@ -//! Test fixture builders: minimal but parseable PE and ELF binaries with -//! crafted imports/notes. Used by unit tests and the similarity -//! integration test. Not production code. +//! Test fixture builders: minimal but parseable PE and ELF binaries. +//! +//! Used by unit and integration tests that need realistic goblin parse +//! trees (imports, sections, machine type) without checking in large +//! binaries. Fixtures are deterministic byte-for-byte across runs. #![doc(hidden)] diff --git a/crates/corpus-core/src/tenant.rs b/crates/corpus-core/src/tenant.rs index 152c934..3dc2f78 100644 --- a/crates/corpus-core/src/tenant.rs +++ b/crates/corpus-core/src/tenant.rs @@ -1,8 +1,13 @@ //! First-class multi-tenant registry. //! -//! Every write path resolves an active tenant before touching tenant-scoped -//! tables. The well-known default tenant (`DEFAULT_TENANT` / slug `default`) -//! is seeded by migration and used when the client omits `X-Corpus-Tenant`. +//! Every durable row is scoped by `tenant_id`. This module: +//! +//! - Resolves slug → id for `X-Corpus-Tenant` headers +//! - Ensures tenants exist (bootstrap / admin create) +//! - Validates slug format (lowercase, URL-safe) +//! +//! The well-known [`crate::DEFAULT_TENANT`] (`slug = default`) is seeded +//! by migration and used when no header is present. use crate::dto::{TenantCreateRequest, TenantResponse}; use crate::error::{Error, Result}; @@ -84,6 +89,7 @@ pub async fn create_tenant(pool: &PgPool, req: &TenantCreateRequest) -> Result Result> { let rows = sqlx::query_as::<_, TenantRow>( "SELECT id, slug, name, status, created_at FROM tenant ORDER BY created_at, slug", @@ -93,6 +99,7 @@ pub async fn list_tenants(pool: &PgPool) -> Result> { Ok(rows.into_iter().map(TenantRow::into_response).collect()) } +/// Fetch tenant by uuid. pub async fn get_tenant(pool: &PgPool, id: Uuid) -> Result { let row = sqlx::query_as::<_, TenantRow>( "SELECT id, slug, name, status, created_at FROM tenant WHERE id = $1", @@ -104,6 +111,7 @@ pub async fn get_tenant(pool: &PgPool, id: Uuid) -> Result { Ok(row.into_response()) } +/// Fetch tenant by slug. pub async fn get_tenant_by_slug(pool: &PgPool, slug: &str) -> Result { let row = sqlx::query_as::<_, TenantRow>( "SELECT id, slug, name, status, created_at FROM tenant WHERE slug = $1", diff --git a/crates/corpus-core/src/triggers.rs b/crates/corpus-core/src/triggers.rs index f5869fd..073ff12 100644 --- a/crates/corpus-core/src/triggers.rs +++ b/crates/corpus-core/src/triggers.rs @@ -1,6 +1,14 @@ -//! Triggers: exactly three conditions (hunt_match, malicious_verdict, -//! variant_join), HMAC-signed webhook delivery via a database outbox -//! polled by the server. No general event system. +//! Outbound triggers for high-signal events. +//! +//! Exactly three condition classes fire triggers: +//! +//! 1. **hunt_match** — a retro or forward scan matched +//! 2. **malicious_verdict** — an opinion or analyzer verdict flipped bad +//! 3. **detection_event** — an autonomous detection was recorded +//! +//! Actions are HMAC-signed webhooks (or future ticket sinks). Secrets stay +//! server-side; payloads never include sample bytes — digests and metadata +//! only. use crate::error::{Error, Result}; use chrono::Utc; diff --git a/crates/corpus-core/tests/agents_api.rs b/crates/corpus-core/tests/agents_api.rs index 64b83ee..0465669 100644 --- a/crates/corpus-core/tests/agents_api.rs +++ b/crates/corpus-core/tests/agents_api.rs @@ -1,5 +1,5 @@ -//! Integration test for the M1 agent endpoints against real PostgreSQL. -//! Gated on CORPUS_TEST_DATABASE_URL like the M0 test; no-op without it. +//! Integration tests for agent enrollment, heartbeat, and auth. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::dto::{ AnnounceDisposition, AnnounceRequest, EnrollRequest, GapEvent, HeartbeatRequest, OccurrenceInfo, diff --git a/crates/corpus-core/tests/analyst.rs b/crates/corpus-core/tests/analyst.rs index d852abc..68b2475 100644 --- a/crates/corpus-core/tests/analyst.rs +++ b/crates/corpus-core/tests/analyst.rs @@ -1,5 +1,5 @@ -//! Integration test for the M5 analyst surface against real PostgreSQL. -//! Gated on CORPUS_TEST_DATABASE_URL; no-op without it. +//! Integration tests for prevalence and rarity analyst APIs. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::cas::FsCas; use corpus_core::dto::{AnnounceRequest, FinalizeRequest, OccurrenceInfo}; diff --git a/crates/corpus-core/tests/bootstrap.rs b/crates/corpus-core/tests/bootstrap.rs index f661d8e..8caec04 100644 --- a/crates/corpus-core/tests/bootstrap.rs +++ b/crates/corpus-core/tests/bootstrap.rs @@ -1,6 +1,5 @@ -//! Integration test for the M4 vault-bootstrap features against real -//! PostgreSQL plus in-process mock servers (OCI registry, TAXII). -//! Gated on CORPUS_TEST_DATABASE_URL; no-op without it. +//! End-to-end bootstrap: tenant, ingest, rules, hunt, similarity smoke. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::cas::FsCas; use corpus_core::dto::{AnnounceRequest, FinalizeRequest, OccurrenceInfo}; diff --git a/crates/corpus-core/tests/detonation.rs b/crates/corpus-core/tests/detonation.rs index db762c2..6232b06 100644 --- a/crates/corpus-core/tests/detonation.rs +++ b/crates/corpus-core/tests/detonation.rs @@ -1,5 +1,5 @@ -//! Detonation integration test (M10): full submit->poll->report->finding -//! flow against an in-process mock CAPE server. No live sandbox. +//! Integration tests for detonation adapter enqueue/poll storage. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::detonate::{CapeProvider, DetonationConfig, DetonationProvider}; use corpus_core::{db, detonate, tenant}; diff --git a/crates/corpus-core/tests/ingest_hunt.rs b/crates/corpus-core/tests/ingest_hunt.rs index f4f2162..a31a584 100644 --- a/crates/corpus-core/tests/ingest_hunt.rs +++ b/crates/corpus-core/tests/ingest_hunt.rs @@ -1,7 +1,5 @@ -//! End-to-end integration test against a real PostgreSQL and a tempfile CAS. -//! -//! Gated on CORPUS_TEST_DATABASE_URL (the demo script sets it); without it -//! the test is a no-op so plain `cargo test` stays hermetic. +//! Integration tests for announce/finalize and retro-hunt execution. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::cas::FsCas; use corpus_core::dto::{ diff --git a/crates/corpus-core/tests/investigate_continuous.rs b/crates/corpus-core/tests/investigate_continuous.rs index 84c3ae6..754cec0 100644 --- a/crates/corpus-core/tests/investigate_continuous.rs +++ b/crates/corpus-core/tests/investigate_continuous.rs @@ -1,5 +1,5 @@ -//! Continuous re-analysis + investigation report path. -//! Gated on CORPUS_TEST_DATABASE_URL. +//! Integration tests for investigation reports and continuous re-analysis. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::cas::FsCas; use corpus_core::dto::*; diff --git a/crates/corpus-core/tests/merlin.rs b/crates/corpus-core/tests/merlin.rs index b33283e..149c53f 100644 --- a/crates/corpus-core/tests/merlin.rs +++ b/crates/corpus-core/tests/merlin.rs @@ -1,5 +1,5 @@ -//! Merlin bridge persistence and replay semantics. -//! Gated on CORPUS_TEST_DATABASE_URL; no-op without it. +//! Integration tests for Merlin observation ingest and listing. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::dto::MerlinSegmentRequest; use corpus_core::{db, merlin, tenant}; diff --git a/crates/corpus-core/tests/semantic.rs b/crates/corpus-core/tests/semantic.rs index 9a1da50..1a13b84 100644 --- a/crates/corpus-core/tests/semantic.rs +++ b/crates/corpus-core/tests/semantic.rs @@ -1,6 +1,5 @@ -//! Semantic similarity integration test: the validation corpus -//! (spec 16.7/28.5). Fixtures are compiled at test time with cc — -//! no committed binaries. Gated on CORPUS_TEST_DATABASE_URL and cc. +//! Integration tests for semantic extract, match, and edge emission. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::cas::FsCas; use corpus_core::dto::{AnnounceRequest, FinalizeRequest, OccurrenceInfo}; diff --git a/crates/corpus-core/tests/similarity.rs b/crates/corpus-core/tests/similarity.rs index d4ce707..78125a6 100644 --- a/crates/corpus-core/tests/similarity.rs +++ b/crates/corpus-core/tests/similarity.rs @@ -1,5 +1,5 @@ -//! Integration test for the M3a similarity pipeline against real -//! PostgreSQL. Gated on CORPUS_TEST_DATABASE_URL; no-op without it. +//! Integration tests for byte/normalized similarity edges and groups. +//! Gated on `CORPUS_TEST_DATABASE_URL`. use corpus_core::cas::FsCas; use corpus_core::dto::{AnnounceRequest, FinalizeRequest, OccurrenceInfo}; diff --git a/crates/corpus-scanner/src/main.rs b/crates/corpus-scanner/src/main.rs index bcd9ba0..f639fd8 100644 --- a/crates/corpus-scanner/src/main.rs +++ b/crates/corpus-scanner/src/main.rs @@ -1,9 +1,10 @@ -//! corpus-scanner: sandboxed scan worker (M6 hardening). +//! corpus-scanner: out-of-process YARA-X helper. //! -//! Reads a job JSON on stdin, scans the sample with the compiled bundle, -//! and writes the outcome JSON on stdout. Intended to run under an OS -//! sandbox (seatbelt/landlock/gVisor) with no network and a narrow -//! filesystem view. See docs/hardening-decisions.md. +//! Spawned by [`corpus_core::sandbox`] so rule evaluation cannot crash +//! the API server. Speaks a minimal stdin/stdout protocol: receive scan +//! job (rules + bytes/path), emit JSON match results or a structured +//! error, exit. Resource limits are applied by the parent when the OS +//! supports them. use serde::{Deserialize, Serialize}; diff --git a/crates/corpus-server/src/main.rs b/crates/corpus-server/src/main.rs index 49cf142..22112f0 100644 --- a/crates/corpus-server/src/main.rs +++ b/crates/corpus-server/src/main.rs @@ -1,12 +1,26 @@ -//! corpus-server: axum REST API owning all writes. +//! # corpus-server //! -//! Dev profile: filesystem CAS + PostgreSQL via Docker Compose. Tenants are -//! first-class rows; `X-Corpus-Tenant` accepts a UUID or slug and defaults -//! to the seeded `default` tenant when omitted. +//! HTTP API process for the corpus platform. Owns all durable writes +//! (Postgres + CAS) and exposes: //! -//! Admin auth: see `corpus_core::auth`. Non-loopback binds require -//! `CORPUS_ADMIN_TOKEN`. Hunts enqueue async by default; pass `?sync=1` -//! to run in-request. +//! - **Ingest** — announce / upload / finalize for agents and CLI +//! - **Agents** — enrollment, heartbeat, gaps, mTLS +//! - **Rules & hunts** — registry, bundles, retro/forward scan +//! - **Similarity** — neighborhood, export, evidence, analyzers, cleanup +//! - **Analyst** — prevalence, investigation, opinions, intel +//! - **Integrations** — Merlin, OCI, detonation adapters +//! +//! # Auth surfaces +//! +//! | Traffic | Mechanism | +//! |---------|-----------| +//! | Admin / corpusctl | shared secret + listen-address policy ([`corpus_core::auth`]) | +//! | Agents | enrollment token → bearer or mTLS ([`corpus_core::agents`], [`corpus_core::mtls`]) | +//! +//! Tenant resolution uses `X-Corpus-Tenant` (slug or uuid), defaulting to +//! the well-known default tenant. +//! +//! Handlers stay thin: parse → authorize → call `corpus_core` → map errors. use axum::body::Bytes; use axum::extract::{Path, Query, Request, State}; diff --git a/crates/corpus-server/tests/admin_auth.rs b/crates/corpus-server/tests/admin_auth.rs index 3e2c382..2c59bbb 100644 --- a/crates/corpus-server/tests/admin_auth.rs +++ b/crates/corpus-server/tests/admin_auth.rs @@ -1,4 +1,4 @@ -//! Admin token enforcement and non-loopback fail-closed policy. +//! HTTP tests for admin auth and listen-address policy. use corpus_core::auth::{is_loopback_listen, AuthConfig, MCP_DEV_TOKEN}; use std::sync::Mutex; diff --git a/crates/corpus-server/tests/agent_ingest_auth.rs b/crates/corpus-server/tests/agent_ingest_auth.rs index d9516d5..d36442f 100644 --- a/crates/corpus-server/tests/agent_ingest_auth.rs +++ b/crates/corpus-server/tests/agent_ingest_auth.rs @@ -1,9 +1,4 @@ -//! Authenticated agent ingest: bearer-authenticated announce/upload/ -//! finalize with server-enforced occurrence identity, 401 on bad tokens, -//! and the unauthenticated dev path for `corpusctl import`. -//! -//! Spawns the real corpus-server binary against a scratch CAS and the -//! test database. Gated on CORPUS_TEST_DATABASE_URL; no-op without it. +//! HTTP tests for agent credential requirements on ingest routes. use corpus_core::dto::*; use corpus_core::{db, DEFAULT_TENANT}; diff --git a/crates/corpusctl/src/main.rs b/crates/corpusctl/src/main.rs index 8f6b233..19745d7 100644 --- a/crates/corpusctl/src/main.rs +++ b/crates/corpusctl/src/main.rs @@ -1,5 +1,19 @@ -//! corpusctl: thin administrative/CLI client. All writes go through the -//! server's REST API; the CLI only classifies, hashes, and uploads bytes. +//! # corpusctl +//! +//! Operator CLI for the corpus platform. Talks only to `corpus-server` +//! over HTTP — it does not open Postgres or CAS directly. +//! +//! # Command groups +//! +//! - Artifact ingest and inspection +//! - Rule registry / bundle publish / activate +//! - Hunt create / watch / results +//! - Similarity neighborhood, export, evidence, analyzers, cleanup +//! - Agent enrollment tokens, intel, investigation, metrics +//! - CA fingerprint helpers for mTLS deployments +//! +//! Authentication uses the admin secret (and optional tenant header). +//! Prefer this tool for scripted ops; the agent binary is for endpoints. use anyhow::{bail, Context, Result}; use clap::{Parser, Subcommand}; diff --git a/migrations/0002_agents.sql b/migrations/0002_agents.sql index f826c21..4a2c0e0 100644 --- a/migrations/0002_agents.sql +++ b/migrations/0002_agents.sql @@ -1,5 +1,7 @@ -- Milestone 1: agent enrollment, identity, and health (spec 10.1, 10.11). -- Tenant-scoped like every other data table (0001 tenant registry). +-- +-- Agent enrollment tokens, agent rows, heartbeats, and coverage gaps. -- One-time enrollment tokens minted by operators via corpusctl. CREATE TABLE enrollment_token ( diff --git a/migrations/0003_similarity.sql b/migrations/0003_similarity.sql index c7b5832..2336b03 100644 --- a/migrations/0003_similarity.sql +++ b/migrations/0003_similarity.sql @@ -1,4 +1,6 @@ -- Milestone 3a: similarity features, typed edges, variant groups (spec 16). +-- +-- Similarity features, typed edges, and variant groups (byte-level M3a). -- Versioned features per artifact. family: exact | normalized | byte | -- structural | semantic (plugin slot, unpopulated in M3a) | provenance. diff --git a/migrations/0004_bootstrap.sql b/migrations/0004_bootstrap.sql index 4021dbb..ef772e3 100644 --- a/migrations/0004_bootstrap.sql +++ b/migrations/0004_bootstrap.sql @@ -1,5 +1,7 @@ -- Milestone 4 (vault bootstrap): snapshot backfill, OCI ingestion, -- intel-corpus connectors. +-- +-- Bootstrap helpers and seed data beyond the default tenant. -- Artifact scope separates endpoint-collected bytes from intel imports. -- Default queries (retro hunts, blast radius occurrence views) cover diff --git a/migrations/0005_analyst.sql b/migrations/0005_analyst.sql index d35e2eb..3966aec 100644 --- a/migrations/0005_analyst.sql +++ b/migrations/0005_analyst.sql @@ -1,6 +1,8 @@ -- Milestone 5: analyst surface — opinions, triggers, audit. -- (Prevalence and dropper hunts are pure SQL over the occurrence ledger; -- proof-of-absence is computed, not stored.) +-- +-- Analyst surface tables: opinions, prevalence support, rarity indexes. -- Human verdicts, separate from analyzer scores (spec 5.5). Append-only; -- the current opinion for an artifact is the latest row. diff --git a/migrations/0006_semantic.sql b/migrations/0006_semantic.sql index e57a379..b7dfa35 100644 --- a/migrations/0006_semantic.sql +++ b/migrations/0006_semantic.sql @@ -1,4 +1,6 @@ -- Milestone M8: per-function semantic signatures (spec 16.2/16.5). +-- +-- Per-function signatures for semantic similarity (x86-64). CREATE TABLE similarity_function ( tenant_id uuid NOT NULL REFERENCES tenant (id), diff --git a/migrations/0007_detonation.sql b/migrations/0007_detonation.sql index 890447c..2ea43a0 100644 --- a/migrations/0007_detonation.sql +++ b/migrations/0007_detonation.sql @@ -1,4 +1,6 @@ -- Milestone M10: detonation findings (spec 13.4, 17.4). +-- +-- External sandbox job records and behavioral result storage. -- Analyzer runs (spec 12.2 sketch; the table was deferred from the M0 -- subset until the first producer — detonation — needed it). diff --git a/migrations/0008_lsh_and_hunt_queue.sql b/migrations/0008_lsh_and_hunt_queue.sql index 0e497eb..688ce6e 100644 --- a/migrations/0008_lsh_and_hunt_queue.sql +++ b/migrations/0008_lsh_and_hunt_queue.sql @@ -2,6 +2,8 @@ -- Each artifact contributes fixed band keys derived from its ssdeep digest; -- candidate queries join on (tenant_id, band_idx, band_key) instead of a -- full per-class table scan. +-- +-- LSH band index for byte-similar candidates; hunt worker queue. CREATE TABLE IF NOT EXISTS similarity_lsh_band ( tenant_id uuid NOT NULL REFERENCES tenant (id), diff --git a/migrations/0009_continuous_investigate.sql b/migrations/0009_continuous_investigate.sql index 5e09577..345a6d9 100644 --- a/migrations/0009_continuous_investigate.sql +++ b/migrations/0009_continuous_investigate.sql @@ -1,6 +1,8 @@ -- Continuous re-analysis tracking, autonomous detection events, and -- investigation scaffolding for the continuous re-analysis product loop: -- retain → detect → re-hunt history → blast radius → recommended actions. +-- +-- Continuous re-analysis work items and investigation snapshots. CREATE TABLE IF NOT EXISTS detection_event ( id uuid PRIMARY KEY, diff --git a/migrations/0010_merlin_observations.sql b/migrations/0010_merlin_observations.sql index 4d5e88f..e406622 100644 --- a/migrations/0010_merlin_observations.sql +++ b/migrations/0010_merlin_observations.sql @@ -1,5 +1,7 @@ -- Merlin telemetry bridge. Raw events remain separate from the verified -- artifact occurrence ledger; event identity is still durable and replay-safe. +-- +-- Merlin segment/observation bridge tables (separate from occurrence ledger). CREATE TABLE IF NOT EXISTS merlin_segment ( id uuid PRIMARY KEY,