feat(native-sidecar): add scope-based Fault isolation primitives

.gitignore

@@ -81,6 +81,7 @@ scripts/ralph/.codex-last-msg-*
 .claude/scheduled_tasks.lock
 
 # Local caches and stray outputs
+.artifacts/
 crates/execution/assets/v8-bridge.js
 crates/execution/assets/v8-bridge-zlib.js
 crates/execution/.agentos-pyodide-cache/

CLAUDE.md

@@ -4,6 +4,10 @@ AgentOS owns the runtime, kernel, VFS, language execution, registry packages,
 ACP/session layer, AgentOS client APIs, docs, and publish machinery. The
 `secure-exec` repository is now a generated compatibility mirror only.
 
+For future RivetKit work, start with the current documentation index at
+https://rivet.dev/llms.txt and follow the linked page for the surface being
+changed.
+
 ## Boundaries
 
 - Keep AgentOS product versions pinned at `0.0.1` in committed files. Release

crates/bridge/src/lib.rs

@@ -254,6 +254,13 @@ pub trait PersistenceBridge: BridgeTypes {
         &mut self,
         request: FlushFilesystemStateRequest,
     ) -> Result<(), Self::Error>;
+    /// Release any state the bridge retains for `vm_id` once that VM is
+    /// disposed. VM ids are monotonic and never reissued, so a disposed VM's
+    /// snapshot can never be loaded again; a bridge that caches snapshots in
+    /// memory MUST drop them here or it grows once per VM lifecycle for the
+    /// process lifetime (LT-022). Defaults to a no-op for bridges that persist
+    /// externally and hold nothing in memory.
+    fn forget_filesystem_state(&mut self, _vm_id: &str) {}
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]

crates/native-sidecar/src/execution/javascript/rpc.rs

@@ -223,9 +223,24 @@ pub(crate) fn deferred_kernel_wait_request_for_process(
         return Ok(None);
     }
     let fd = javascript_sync_rpc_arg_u32(&request.args, 0, "filesystem write fd")?;
-    let stat = kernel
-        .fd_stat(EXECUTION_DRIVER_NAME, kernel_pid, fd)
-        .map_err(kernel_error)?;
+    // Mapped host fds live in the per-process map, NOT the kernel fd table, so
+    // stat'ing one here raised `EBADF: bad file descriptor 1000000000` on EVERY
+    // guest file write. That error escaped the process-event pump: it used to
+    // kill the whole sidecar (LT-011 cross-tenant DoS) and, once the pump
+    // stopped propagating it, left the guest's sync-RPC response undelivered
+    // until the 31 s bridge deadline (LT-024). A mapped host fd is host-backed
+    // and can never be a kernel pipe, so deferral never applies to it.
+    if fd >= crate::state::MAPPED_HOST_FD_START {
+        return Ok(None);
+    }
+    let stat = match kernel.fd_stat(EXECUTION_DRIVER_NAME, kernel_pid, fd) {
+        Ok(stat) => stat,
+        // Defense in depth for the whole class: any fd this kernel does not own
+        // is by definition not a kernel pipe, so it is not deferrable. Report
+        // "no deferral" rather than failing the write with an unrelated errno.
+        Err(error) if error.code() == "EBADF" => return Ok(None),
+        Err(error) => return Err(kernel_error(error)),
+    };
     if stat.filetype != agentos_kernel::fd_table::FILETYPE_PIPE {
         return Ok(None);
     }

crates/native-sidecar/src/execution/launch.rs

@@ -1196,6 +1196,27 @@ fn sync_host_directory_tree_to_kernel_inner(
                 // just unlinked by the guest — vim's swap-file dance). The
                 // entry is mid-churn; skip it rather than failing the VM.
                 Err(error) if error.code() == "ENOENT" => continue,
+                // The guest exhausting its OWN configured filesystem budget is
+                // a guest-caused condition, not a sidecar fault: the offending
+                // write was already correctly rejected with a typed ENOSPC at
+                // the syscall, and the shadow mirror simply cannot hold the
+                // excess. Escalating it to a fatal error here killed the whole
+                // shared sidecar — and every co-located VM/actor with it — from
+                // one guest filling its quota (the LT-011 blast-radius class).
+                // Warn and skip the entry, matching the ENOENT/EPERM tolerance
+                // above, so the failure stays host-visible but bounded.
+                Err(error)
+                    if matches!(error.code(), "ENOSPC" | "EDQUOT" | "EFBIG") =>
+                {
+                    tracing::warn!(
+                        path = %host_path.display(),
+                        guest_path = %guest_path,
+                        error = %error.code(),
+                        "skipping host shadow file that exceeds the VM filesystem budget \
+                         (limits.resources.maxFilesystemBytes)"
+                    );
+                    continue;
+                }
                 Err(error) => {
                     return Err(SidecarError::InvalidState(format!(
                         "failed to sync host shadow file {} to guest {}: {}",

crates/native-sidecar/src/fault.rs

@@ -0,0 +1,198 @@
+//! Scope-based fault handling for the sidecar's inline serve path.
+//!
+//! The sidecar is the trusted enforcement point shared by every VM in the
+//! process — and, on Rivet Compute, by every co-located actor on the runner.
+//! Historically any error raised while servicing guest-driven work propagated
+//! with `?` out of the serve loop to `main`, which exited(1): one guest's errno
+//! killed every co-located tenant (LT-011).
+//!
+//! The failure was not that the wrong *kind* of error escaped. It was that the
+//! error type carried no notion of **what is actually broken**, so `?` — the
+//! shortest, most idiomatic thing to write — implicitly meant "kill the
+//! process". Blame ("was this the guest's fault?") is the wrong axis: a host
+//! error that only damaged one VM's state should still take down only that VM.
+//! Scope is the right axis.
+//!
+//! So the default direction is inverted here. [`Fault`] converts from ordinary
+//! errors as [`FaultScope::Isolated`], meaning "confine this to the current unit
+//! of work". Escalating to [`FaultScope::Fatal`] must be written out explicitly
+//! at the call site, so the failure mode of carelessness is "one VM died"
+//! rather than "the runner died".
+//!
+//! Ownership/scope reuses [`TaskOwner`] from `agentos-runtime` rather than
+//! introducing a second scope enum, so supervised spawned tasks and inline
+//! serve-loop work share one ownership model.
+
+use std::any::Any;
+use std::panic::{catch_unwind, AssertUnwindSafe};
+
+use agentos_runtime::{TaskOwner, TaskTerminalReason};
+
+/// How far a fault must propagate before the system is consistent again.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum FaultScope {
+    /// Confine to the current unit of work: fail the request, tear down the
+    /// owning VM/session if its state may be inconsistent, keep serving
+    /// everyone else. This is the default for any converted error.
+    Isolated,
+    /// The substrate shared by ALL work is unusable, so continuing would serve
+    /// wrong answers rather than fewer answers. Reserved for a deliberately
+    /// short list: the host transport is gone, framing on the shared channel is
+    /// corrupt, or allocation failed. "A host-side error occurred" is NOT on
+    /// its own a reason to be fatal.
+    Fatal,
+}
+
+/// An abnormal end to a unit of sidecar work, tagged with how far it must
+/// propagate.
+#[derive(Clone, Debug)]
+pub(crate) struct Fault {
+    pub(crate) scope: FaultScope,
+    pub(crate) reason: TaskTerminalReason,
+    pub(crate) cause: String,
+}
+
+impl Fault {
+    /// Confine to the current unit of work. Prefer the `From` conversion; this
+    /// exists for call sites that build a cause string directly.
+    pub(crate) fn isolated(cause: impl Into<String>) -> Self {
+        Self {
+            scope: FaultScope::Isolated,
+            reason: TaskTerminalReason::Failed,
+            cause: cause.into(),
+        }
+    }
+
+    /// Take down the process. Only for shared-substrate failures — see
+    /// [`FaultScope::Fatal`]. Every use should be obvious on inspection.
+    pub(crate) fn fatal(cause: impl Into<String>) -> Self {
+        Self {
+            scope: FaultScope::Fatal,
+            reason: TaskTerminalReason::Failed,
+            cause: cause.into(),
+        }
+    }
+
+    /// A caught panic. Always isolated, never fatal — but the caller MUST reap
+    /// the owning fault domain rather than resuming into it (see
+    /// [`catch_faults`]).
+    pub(crate) fn panicked(payload: &(dyn Any + Send)) -> Self {
+        Self {
+            scope: FaultScope::Isolated,
+            reason: TaskTerminalReason::Panicked,
+            cause: panic_message(payload),
+        }
+    }
+
+    pub(crate) fn is_fatal(&self) -> bool {
+        matches!(self.scope, FaultScope::Fatal)
+    }
+
+    /// True when the owning fault domain's state may be torn and must be
+    /// destroyed rather than reused. A panic can unwind out of a half-finished
+    /// mutation, so resuming would serve from inconsistent state.
+    pub(crate) fn requires_teardown(&self) -> bool {
+        matches!(self.reason, TaskTerminalReason::Panicked)
+    }
+
+    /// Escalate an isolated fault after a circuit breaker trips.
+    pub(crate) fn escalated(mut self) -> Self {
+        self.scope = FaultScope::Fatal;
+        self
+    }
+}
+
+/// Isolation is the default: any ordinary error confined to the current unit of
+/// work. This is the inversion that makes `?` safe in the serve loop.
+impl<E> From<E> for Fault
+where
+    E: std::fmt::Display,
+{
+    fn from(error: E) -> Self {
+        Self::isolated(error.to_string())
+    }
+}
+
+/// Run `body` with panics converted into isolated [`Fault`]s.
+///
+/// A caught panic must never simply resume: it can unwind out of a partially
+/// applied mutation, so the owning domain's state may be torn. Callers are
+/// expected to check [`Fault::requires_teardown`] and destroy `owner` rather
+/// than continuing to serve it. That teardown is also what makes the
+/// `AssertUnwindSafe` here honest — the possibly-inconsistent state is
+/// discarded, not reused.
+pub(crate) fn catch_faults<T>(
+    owner: TaskOwner,
+    body: impl FnOnce() -> Result<T, Fault>,
+) -> Result<T, Fault> {
+    match catch_unwind(AssertUnwindSafe(body)) {
+        Ok(result) => result,
+        Err(payload) => {
+            let fault = Fault::panicked(payload.as_ref());
+            tracing::error!(
+                owner = %owner,
+                cause = %fault.cause,
+                "panic escaped sidecar work; isolating and reaping the owning scope"
+            );
+            Err(fault)
+        }
+    }
+}
+
+/// Guards against per-VM isolation silently masking a genuinely broken host.
+///
+/// Isolating every fault is right until the host itself is the problem — then
+/// the sidecar would cheerfully fail every VM in a loop forever. After
+/// `threshold` consecutive faults sharing a cause, escalate to fatal so the
+/// supervisor can restart the process instead.
+pub(crate) struct FaultBreaker {
+    threshold: u32,
+    consecutive: u32,
+    last_cause: Option<String>,
+}
+
+impl FaultBreaker {
+    pub(crate) fn new(threshold: u32) -> Self {
+        Self {
+            threshold: threshold.max(1),
+            consecutive: 0,
+            last_cause: None,
+        }
+    }
+
+    /// Record a fault; returns the fault, escalated to fatal if this cause has
+    /// now repeated `threshold` times without an intervening success.
+    pub(crate) fn record(&mut self, fault: Fault) -> Fault {
+        if self.last_cause.as_deref() == Some(fault.cause.as_str()) {
+            self.consecutive = self.consecutive.saturating_add(1);
+        } else {
+            self.last_cause = Some(fault.cause.clone());
+            self.consecutive = 1;
+        }
+        if self.consecutive >= self.threshold && !fault.is_fatal() {
+            tracing::error!(
+                cause = %fault.cause,
+                consecutive = self.consecutive,
+                "same fault repeated across units of work; escalating to fatal"
+            );
+            return fault.escalated();
+        }
+        fault
+    }
+
+    /// Any successful unit of work clears the run.
+    pub(crate) fn record_success(&mut self) {
+        self.consecutive = 0;
+        self.last_cause = None;
+    }
+}
+
+fn panic_message(payload: &(dyn Any + Send)) -> String {
+    if let Some(message) = payload.downcast_ref::<&'static str>() {
+        (*message).to_string()
+    } else if let Some(message) = payload.downcast_ref::<String>() {
+        message.clone()
+    } else {
+        String::from("panic with non-string payload")
+    }
+}

crates/native-sidecar/src/filesystem.rs

@@ -1564,9 +1564,11 @@ pub(crate) fn service_javascript_fs_sync_rpc(
             )?;
             record_fs_sync_subphase(request.method.as_str(), "parse", phase_start);
             let phase_start = Instant::now();
+            // Read the byte budget before taking the mutable borrow on `process`.
+            let max_filesystem_bytes = kernel.resource_limits().max_filesystem_bytes;
             if let Some(mapped) = process.mapped_host_fd_mut(fd) {
                 record_fs_sync_subphase(request.method.as_str(), "mapped_fd_match", phase_start);
-                return write_mapped_host_fd(mapped, fd, &contents, position);
+                return write_mapped_host_fd(mapped, fd, &contents, position, max_filesystem_bytes);
             }
             record_fs_sync_subphase(request.method.as_str(), "mapped_fd_none", phase_start);
             let phase_start = Instant::now();
@@ -1616,11 +1618,14 @@ pub(crate) fn service_javascript_fs_sync_rpc(
             record_fs_sync_subphase(request.method.as_str(), "parse", phase_start);
 
             let mut total_written = 0usize;
+            // Read the byte budget before taking the mutable borrow on `process`.
+            let max_filesystem_bytes = kernel.resource_limits().max_filesystem_bytes;
             if let Some(mapped) = process.mapped_host_fd_mut(fd) {
                 record_fs_sync_subphase(request.method.as_str(), "mapped_fd_match", phase_start);
                 let mut next_position = position;
                 for buffer in buffers {
-                    let written = write_all_mapped_host_fd(mapped, fd, buffer, next_position)?;
+                    let written =
+                        write_all_mapped_host_fd(mapped, fd, buffer, next_position, max_filesystem_bytes)?;
                     total_written = total_written.saturating_add(written);
                     if let Some(position) = &mut next_position {
                         *position = position.saturating_add(written as u64);
@@ -4286,12 +4291,45 @@ fn read_mapped_host_fd(
     Ok(javascript_sync_rpc_bytes_value(&bytes))
 }
 
+/// Enforce the VM's filesystem byte budget on a host-mapped write.
+///
+/// Writes through a mapped host fd go straight to the host file and never reach
+/// the kernel VFS, so `check_filesystem_usage` never sees them. Without this a
+/// guest writing to a host-backed path (`/tmp`, `/workspace`) could exceed
+/// `limits.resources.maxFilesystemBytes` without bound and fill the host disk
+/// (LT-011 / LT-017). Only runs when a limit is configured, so the common
+/// unlimited case keeps the original fast path.
+fn check_mapped_host_fd_write_budget(
+    mapped: &crate::state::ActiveMappedHostFd,
+    fd: u32,
+    len: usize,
+    position: Option<u64>,
+    max_filesystem_bytes: Option<u64>,
+) -> Result<(), SidecarError> {
+    let Some(limit) = max_filesystem_bytes else {
+        return Ok(());
+    };
+    let current = mapped.file.metadata().map(|meta| meta.len()).unwrap_or(0);
+    let start = position.unwrap_or(current);
+    let resulting = start.saturating_add(len as u64).max(current);
+    if resulting > limit {
+        return Err(SidecarError::Execution(format!(
+            "ENOSPC: writing {len} byte(s) to mapped guest fd {fd} would reach {resulting} bytes, \
+             exceeding the VM filesystem limit of {limit} (limits.resources.maxFilesystemBytes); \
+             raise the limit to store more data"
+        )));
+    }
+    Ok(())
+}
+
 fn write_mapped_host_fd(
     mapped: &mut crate::state::ActiveMappedHostFd,
     fd: u32,
     contents: &[u8],
     position: Option<u64>,
+    max_filesystem_bytes: Option<u64>,
 ) -> Result<Value, SidecarError> {
+    check_mapped_host_fd_write_budget(mapped, fd, contents.len(), position, max_filesystem_bytes)?;
     let written = match position {
         Some(offset) => mapped.file.write_at(contents, offset),
         None => mapped.file.write(contents),
@@ -4310,7 +4348,9 @@ fn write_all_mapped_host_fd(
     fd: u32,
     contents: &[u8],
     position: Option<u64>,
+    max_filesystem_bytes: Option<u64>,
 ) -> Result<usize, SidecarError> {
+    check_mapped_host_fd_write_budget(mapped, fd, contents.len(), position, max_filesystem_bytes)?;
     let mut total = 0usize;
     while total < contents.len() {
         let write_position = position.map(|offset| offset.saturating_add(total as u64));

crates/native-sidecar/src/lib.rs

@@ -9,6 +9,7 @@ pub(crate) mod bindings;
 pub(crate) mod crypto_cipher;
 pub(crate) mod execution;
 pub mod extension;
+pub(crate) mod fault;
 pub(crate) mod filesystem;
 #[allow(dead_code)]
 pub(crate) mod json_rpc;