feat(container-runner): report actors as crashed on unexpected platform SIGTERM

container-runner/README.md

@@ -8,7 +8,8 @@ This README covers development, the example projects, and the local test harness
 spawning a child game-server process per actor and proxying Rivet's tunneled
 HTTP/WebSocket traffic to it. Each child gets its own port; the pool's request
 concurrency decides how many actors share a container (one, in the recommended
-game-server setup), and the process exits when the last actor stops. Wrap any dedicated server (Unity, Godot, a plain Node process) in a
+game-server setup). The instance stays warm after its last actor stops; the engine
+reaps it by draining the `/start` connection after the request lifespan. Wrap any dedicated server (Unity, Godot, a plain Node process) in a
 container with this binary as the entrypoint and Rivet Compute can cold-start and route
 to it.
 

container-runner/src/actor.rs

@@ -4,22 +4,29 @@
 //! readiness (so the actor is never reported ready before the child listens),
 //! `run` is a watchdog that reports unexpected child exits,
 //! `on_fetch`/`on_websocket` proxy tunneled traffic to the child's port, and
-//! `on_destroy` stops the child, exiting the process once no actors remain.
+//! `on_destroy` stops the child while the instance stays warm for the next
+//! placement.
 
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use rivetkit::{Actor, ActorKeySegment, Ctx, Request, Response, WebSocket, action};
 use tokio::sync::Mutex as TokioMutex;
 
 use crate::child::{ChildProcess, SpawnSpec, log_prefix};
 use crate::input::ActorInput;
 use crate::{
-	children, effective_stop_grace, release_child_port, request_exit, reserve_child_port,
+	children, effective_stop_grace, release_child_port, reserve_child_port,
 	runner_config,
 };
 
+/// Live actor contexts on this instance, keyed by actor id. Lets the process
+/// shutdown path report actors as crashed when the platform reclaims the
+/// container out from under them.
+static ACTOR_CTXS: LazyLock<scc::HashMap<String, Ctx<GameServer>>> =
+	LazyLock::new(scc::HashMap::new);
+
 pub struct GameServer {
 	child: TokioMutex<Option<Arc<ChildProcess>>>,
 }
@@ -34,17 +41,19 @@
 		// deliberate, then stop. `stop` is idempotent if the process shutdown
 		// sweep already stopped this child.
 		children().remove_async(actor_id).await;
+		ACTOR_CTXS.remove_async(actor_id).await;
 		let child = self.child.lock().await.take();
 		if let Some(child) = child {
 			child.stop(effective_stop_grace()).await;
 			release_child_port(child.child_port).await;
 		}
 
-		// Once the last actor is gone the instance drains rather than
-		// lingering for the next placement.
-		if children().is_empty() {
-			request_exit(actor_id, reason);
-		}
+		// The instance stays alive and warm after its last actor stops, ready to
+		// host the next placement. It is reaped by the platform's own shutdown
+		// signal, not by self-exit. This keeps the serverless container long
+		// lived enough for the log agent to drain its stderr, which a fast
+		// self-exit could otherwise lose.
+		tracing::info!(actor_id = %actor_id, reason, "actor stopped, keeping instance warm");
 	}
 }
 
@@ -85,6 +94,7 @@
 					"{} runner: actor already running, ignoring duplicate start",
 					log_prefix(&actor_id, existing.key.as_deref())
 				);
+				register_ctx(&actor_id, &ctx).await;
 				*self.child.lock().await = Some(existing);
 				return Ok(());
 			}
@@ -129,12 +139,10 @@
 			Ok(child) => Arc::new(child),
 			Err(err) => {
 				release_child_port(child_port).await;
-				// A failed start on an otherwise idle instance poisons it;
-				// don't let it serve the next placement. With other actors
-				// running, the failure is this actor's alone.
-				if children().is_empty() {
-					request_exit(&actor_id, "child failed to start");
-				}
+				// A failed start is this actor's alone and does not take the
+				// instance down. The container stays warm and ready for the next
+				// placement, and stays alive long enough for the log agent to
+				// drain the failure logs before the platform reaps it.
 				return Err(err);
 			}
 		};
@@ -152,6 +160,10 @@
 			release_child_port(child_port).await;
 			anyhow::bail!("a child for actor {actor_id} is already registered");
 		}
+		// Register only now that startup has succeeded. Registering earlier would
+		// leak an entry for any generation whose start failed, since a failed
+		// start never runs on_destroy/on_sleep to remove it.
+		register_ctx(&actor_id, &ctx).await;
 		*self.child.lock().await = Some(child);
 		Ok(())
 	}
@@ -236,6 +248,38 @@
 	}
 }
 
+/// Register an actor context for crash-on-shutdown reporting. Overwrites any
+/// stale entry left by a prior generation with the same id.
+async fn register_ctx(actor_id: &str, ctx: &Ctx<GameServer>) {
+	ACTOR_CTXS.remove_async(actor_id).await;
+	let _ = ACTOR_CTXS
+		.insert_async(actor_id.to_string(), ctx.clone())
+		.await;
+}
+
+/// Report every live actor on this instance as crashed. Called when the
+/// platform reclaims the container (an unexpected SIGTERM) so the reclaim
+/// surfaces as a crash on the engine instead of a silent reallocation. Runs
+/// while the envoy is still connected so the crash reaches the engine.
+pub async fn crash_all_actors(message: &str) {
+	let mut ctxs = Vec::new();
+	ACTOR_CTXS
+		.retain_async(|_, ctx| {
+			ctxs.push(ctx.clone());
+			false
+		})
+		.await;
+	for ctx in ctxs {
+		if let Err(err) = ctx.stop_with_error(message) {
+			tracing::debug!(
+				actor_id = %ctx.actor_id(),
+				error = ?err,
+				"crash-on-shutdown stop_with_error failed"
+			);
+		}
+	}
+}
+
 fn actor_key_string(ctx: &Ctx<GameServer>) -> Option<String> {
 	let key = ctx.key();
 	if key.is_empty() {

container-runner/src/child.rs

@@ -86,7 +86,7 @@ impl ChildProcess {
 				"child port {child_port} is already in use before spawning `{program}`: a \
                  previous game server is still running in this container. container-runner \
                  hosts one actor per container — configure the serverless runner with \
-                 max_concurrent_actors=1 and Cloud Run request concurrency=1."
+                 max_concurrent_actors=1 and platform request concurrency=1."
 			);
 		}
 

container-runner/src/main.rs

@@ -12,8 +12,9 @@
 //! The runner hosts as many concurrent actors as the engine places on it,
 //! each with its own child process on its own port; the pool's request
 //! concurrency decides how many that is (1 in the recommended game-server
-//! setup). When the last actor stops the process exits so the platform reaps
-//! the instance.
+//! setup). The instance stays warm after its last actor stops and never
+//! self-exits; the engine reaps it by draining the `/start` connection once
+//! the request lifespan elapses, or the platform sends a SIGTERM.
 
 mod actor;
 mod child;
@@ -68,15 +69,20 @@ static RESERVED_PORTS: LazyLock<scc::HashSet<u16>> = LazyLock::new(scc::HashSet:
 static EXIT: LazyLock<CancellationToken> = LazyLock::new(CancellationToken::new);
 
 /// Set when the process is shutting down because the PLATFORM sent a signal.
-/// Cloud Run gives a container roughly 10 seconds between SIGTERM and SIGKILL,
-/// so every grace period on this path must fit that budget; engine-initiated
-/// stops keep the full configured grace (their budget is the pool's drain
-/// grace period instead).
+/// The hosting platform gives a container only a bounded window (often ~10
+/// seconds) between SIGTERM and SIGKILL, so every grace period on this path must
+/// fit that budget; engine-initiated stops keep the full configured grace
+/// (their budget is the pool's drain grace period instead).
 static SIGNAL_SHUTDOWN: AtomicBool = AtomicBool::new(false);
 
+/// Set when shutdown was triggered by a platform SIGTERM (an instance reclaim),
+/// as opposed to a local SIGINT (developer Ctrl-C). Distinguishes a reclaim
+/// from a manual stop for downstream shutdown handling.
+static PLATFORM_RECLAIM: AtomicBool = AtomicBool::new(false);
+
 /// How long the platform gives this container between SIGTERM and SIGKILL.
-/// Cloud Run defaults to 10 seconds (configurable up to 60 on the service);
-/// keep this in sync with the platform setting via RIVET_SIGTERM_BUDGET_SECS.
+/// Defaults to 10 seconds; keep this in sync with the platform's actual budget
+/// via RIVET_SIGTERM_BUDGET_SECS.
 /// The signal-path teardown splits the budget: ~60% for the engine drain
 /// (whose per-actor stops SIGTERM children with a grace capped at ~40%), 1s
 /// for the straggler sweep, and the rest as margin.
@@ -167,12 +173,12 @@ pub fn effective_stop_grace() -> Duration {
 	}
 }
 
-/// End the process. Called when the LAST actor on this instance is gone (or a
-/// failed start poisoned an otherwise idle instance): the instance drains
-/// instead of lingering for the next placement. The runner is PID 1 in the
+/// End the process. Only the platform shutdown signal drives this now: actors
+/// stopping or failing to start no longer exit the instance, so it stays warm
+/// and reusable and its logs have time to drain. The runner is PID 1 in the
 /// image, so exiting stops the container and the platform reaps the instance.
 pub fn request_exit(actor_id: &str, reason: &str) {
-	tracing::info!(actor_id = %actor_id, reason, "actor finished, exiting container");
+	tracing::info!(actor_id = %actor_id, reason, "shutting down container");
 	EXIT.cancel();
 }
 
@@ -222,8 +228,9 @@ fn base64url_nopad(input: &[u8]) -> String {
     long_about = None,
 )]
 struct Args {
-	/// Serverless HTTP front-door port. Rivet Compute injects RIVET_PORT (plain Cloud Run
-	/// uses PORT); resolved in `main` as --port > RIVET_PORT > PORT > 8080.
+	/// Serverless HTTP front-door port. Rivet Compute injects RIVET_PORT (other
+	/// serverless platforms use the conventional PORT); resolved in `main` as
+	/// --port > RIVET_PORT > PORT > 8080.
 	#[arg(long)]
 	port: Option<u16>,
 
@@ -244,7 +251,7 @@ struct Args {
 	base_path: String,
 
 	/// SIGTERM→SIGKILL grace period (seconds) when stopping the child.
-	#[arg(long, env = "RIVET_STOP_GRACE_SECS", default_value_t = 25)]
+	#[arg(long, env = "RIVET_STOP_GRACE_SECS", default_value_t = 10)]
 	stop_grace_secs: u64,
 
 	/// How long (seconds) to wait for the child's port to open before failing start.
@@ -285,7 +292,7 @@ async fn async_main() -> Result<()> {
 	let boot_id = boot_id();
 	tracing::info!(?args, %boot_id, "starting container-runner");
 
-	// Front-door port: Rivet Compute injects RIVET_PORT; plain Cloud Run uses PORT.
+	// Front-door port: Rivet Compute injects RIVET_PORT; other serverless platforms use the conventional PORT.
 	let port = args
 		.port
 		.or_else(|| env_u16("RIVET_PORT"))
@@ -345,20 +352,34 @@ async fn async_main() -> Result<()> {
 	));
 	tracing::info!(port, "container-runner serverless front door listening");
 
-	// Wait for an exit request, then tear down. Two orders depending on why:
+	// Wait for an exit request, then tear down. Only the signal path is live
+	// today: nothing calls `request_exit` except `spawn_signal_handler`, which
+	// sets `SIGNAL_SHUTDOWN` before cancelling `EXIT`, so the `else` branch is
+	// currently unreachable and kept only as a fallback for a future
+	// actor-driven exit.
 	//
 	// Signal (platform is reclaiming the instance): tell the engine FIRST so
 	// it can start re-placing actors immediately. Its per-actor stops run our
 	// on_destroy hooks, which SIGTERM children with the capped signal grace.
 	// The drain is bounded so an unreachable engine cannot eat the whole
 	// platform budget; the sweep then catches any child whose hooks never ran.
 	//
-	// Actor-driven exit (last actor stopped or a failed start poisoned an
-	// idle instance): no platform deadline. Children are already reaped by
-	// the hooks (the sweep is a no-op backstop), and the runtime drains
-	// unbounded so the /start SSE flushes its stopping frame cleanly.
+	// Fallback actor-driven exit (unreachable today): no platform deadline.
+	// Children are already reaped by the hooks (the sweep is a no-op backstop),
+	// and the runtime drains unbounded so the /start SSE flushes cleanly.
 	EXIT.cancelled().await;
 	if SIGNAL_SHUTDOWN.load(Ordering::Acquire) {
+		// A platform SIGTERM reclaims this instance. Report every actor as crashed
+		// before draining so an unexpected SIGTERM (OOM or the ~60 minute request
+		// cap) surfaces as a crash on the engine instead of a silent reallocation.
+		// This runs while the envoy is still connected so the crash reaches the
+		// engine. A local SIGINT (Ctrl-C) drains gracefully without a crash.
+		if PLATFORM_RECLAIM.load(Ordering::Acquire) {
+			crate::actor::crash_all_actors(
+				"runner received unexpected platform SIGTERM, likely OOM or running longer than 60 minutes",
+			)
+			.await;
+		}
 		if tokio::time::timeout(signal_drain_timeout(), runtime.shutdown())
 			.await
 			.is_err()
@@ -413,7 +434,30 @@ fn spawn_signal_handler() {
 		let mut sigterm = signal(SignalKind::terminate()).expect("install SIGTERM handler");
 		let mut sigint = signal(SignalKind::interrupt()).expect("install SIGINT handler");
 		tokio::select! {
-			_ = sigterm.recv() => tracing::info!("received SIGTERM"),
+			_ = sigterm.recv() => {
+				PLATFORM_RECLAIM.store(true, Ordering::Release);
+				// Attribute the reclaim to each running actor so it is visible in
+				// actor-scoped logs, not only the process-level log stream.
+				let mut actor_ids = Vec::new();
+				CHILDREN
+					.retain_async(|actor_id, _| {
+						actor_ids.push(actor_id.clone());
+						true
+					})
+					.await;
+				if actor_ids.is_empty() {
+					tracing::error!(
+						"unexpected platform SIGTERM received, likely hitting OOM or running longer than 60 minutes"
+					);
+				} else {
+					for actor_id in actor_ids {
+						tracing::error!(
+							actor_id = %actor_id,
+							"unexpected platform SIGTERM received, likely hitting OOM or running longer than 60 minutes"
+						);
+					}
+				}
+			}
 			_ = sigint.recv() => tracing::info!("received SIGINT"),
 		}
 		SIGNAL_SHUTDOWN.store(true, Ordering::Release);

engine/sdks/rust/data/Cargo.toml

@@ -10,7 +10,6 @@ edition.workspace = true
 anyhow.workspace = true
 gas.workspace = true
 rivet-runner-protocol.workspace = true
-rivet-util.workspace = true
 serde_bare.workspace = true
 serde.workspace = true
 vbare.workspace = true