V3: Recover interrupted runs when the built server starts

docs/deployment.md

@@ -214,6 +214,33 @@ directory's `auth.ts`, with `LOUSHO_API_TOKEN` appended to it: see
 
 SQLite is one file on one disk, so run a single instance per database file.
 
+### Crash recovery
+
+When the `node-server` or `docker` server starts listening, it finishes the
+runs that a crash or a kill left unfinished in the store. It calls
+[`agent.recover()`](./durable-execution.md), which resumes every
+`in-progress` run and session turn from its last checkpoint. Steps that
+completed are not run again. A tool call that was in flight when the process
+died runs again, so make such tools idempotent. The server logs
+`lousho server: resuming N interrupted run(s)` when it finds any, and one
+`recovered interrupted run '<id>'` line per run it finishes. It then checks
+the store again every 30 seconds until it shuts down.
+
+- Runs paused on an approval or a question are left alone. Decide them
+  through `POST /chat/:sessionId/approvals/:id`.
+- A run whose [run lease](./durable-execution.md#run-leases) another live
+  process holds is skipped and retried on a later check. So is a run whose
+  crashed process's lease has not expired yet.
+- With the default `memory` store there is nothing to recover after a
+  restart, and the check finds nothing.
+- A resend of the same turn that arrives while the server is still resuming
+  it runs after the resumed turn, as a new turn. Send an `Idempotency-Key`
+  ([Resending a turn](#resending-a-turn)) to get the resumed turn's reply
+  instead.
+
+Set `LOUSHO_RECOVER_ON_START=0` to turn this off, for example when another
+process already runs `agent.recover()` for the same store.
+
 ## `node-server`
 
 `dist/server.js` is a single self-contained bundle (the SDK and its

llms-full.txt

@@ -7263,6 +7263,33 @@ directory's `auth.ts`, with `LOUSHO_API_TOKEN` appended to it: see
 
 SQLite is one file on one disk, so run a single instance per database file.
 
+### Crash recovery
+
+When the `node-server` or `docker` server starts listening, it finishes the
+runs that a crash or a kill left unfinished in the store. It calls
+[`agent.recover()`](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/durable-execution.md), which resumes every
+`in-progress` run and session turn from its last checkpoint. Steps that
+completed are not run again. A tool call that was in flight when the process
+died runs again, so make such tools idempotent. The server logs
+`lousho server: resuming N interrupted run(s)` when it finds any, and one
+`recovered interrupted run '<id>'` line per run it finishes. It then checks
+the store again every 30 seconds until it shuts down.
+
+- Runs paused on an approval or a question are left alone. Decide them
+  through `POST /chat/:sessionId/approvals/:id`.
+- A run whose [run lease](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/durable-execution.md#run-leases) another live
+  process holds is skipped and retried on a later check. So is a run whose
+  crashed process's lease has not expired yet.
+- With the default `memory` store there is nothing to recover after a
+  restart, and the check finds nothing.
+- A resend of the same turn that arrives while the server is still resuming
+  it runs after the resumed turn, as a new turn. Send an `Idempotency-Key`
+  ([Resending a turn](#resending-a-turn)) to get the resumed turn's reply
+  instead.
+
+Set `LOUSHO_RECOVER_ON_START=0` to turn this off, for example when another
+process already runs `agent.recover()` for the same store.
+
 ## `node-server`
 
 `dist/server.js` is a single self-contained bundle (the SDK and its

src/deploy/adapters/node-server.ts

@@ -209,6 +209,11 @@ function serverSource(variant: ServerVariant): string {
  * LOUSHO_ALLOWED_HOSTS (comma-separated) and LOUSHO_REQUIRE_JSON_BODY=false
  * set the routes' cross-site, Host and body checks (docs/auth.md).
  *
+ * Crash recovery: once listening, the server resumes the runs a crash left
+ * in-progress in the store (agent.recover(): completed steps do not run again,
+ * runs paused on an approval are left alone) and logs how many it found.
+ * LOUSHO_RECOVER_ON_START=0 turns it off.
+ *
  * Binds to 127.0.0.1 by default; pass --host=<h> (or HOST=<h>) to opt in to
  * listening on other interfaces. Port: --port=<n> / PORT, default 3000.
  */

src/deploy/nodeServer.ts

@@ -9,11 +9,14 @@
  * `/health` requires it, including a channel route, unless that channel
  * authenticates its own requests (it has a `verify`: a signed platform channel,
  * a webhook with a secret, an `httpChannel({ verify })`).
+ * Once it listens it finishes the runs a crash left interrupted
+ * (`agent.recover()`), unless `LOUSHO_RECOVER_ON_START=0`.
  *
  * Node-only (the Worker target has its own runtime, runtime.worker.ts).
  */
 import * as http from 'node:http';
 import type { SimpleAgent } from '../createAgent';
+import type { RunRecovery } from '../agentRecover';
 import { relayFetch } from '../server/chatRoutes';
 import { serveFetch, type ChatRoutesAccess, type ChatRoutesContext, type ServeAuth } from '../server/fetchRoutes';
 import { startSchedules, type StartSchedulesOptions } from '../schedules/startSchedules';
@@ -34,6 +37,8 @@ import { routeAuth } from '../auth/routeAuth';
 const API_TOKEN_ENV = 'LOUSHO_API_TOKEN';
 /** Environment variable choosing where sessions live: `memory` (default) or `sqlite:<path>`. */
 const STORE_ENV = 'LOUSHO_STORE';
+/** Environment variable that turns off crash recovery at start-up when `0` or `false`. */
+const RECOVER_ENV = 'LOUSHO_RECOVER_ON_START';
 
 /**
  * `authorizeSession`, `authorizeApproval` and `exposeErrors` work as on
@@ -58,6 +63,14 @@ export interface DeployedServerOptions extends ChatRoutesAccess {
   scheduler?: StartSchedulesOptions;
   /** Channels of the served agent directory (`resolveAgentDir()`'s `channels`): mounted under `/channels` next to the chat routes. */
   channels?: readonly Channel[];
+  /**
+   * Crash recovery: once the server listens, `agent.recover()` resumes the
+   * runs a crash left `in-progress` in the store (runs paused on an approval
+   * are left alone), and keeps sweeping every `intervalMs` (default 30 s) until
+   * the server shuts down. On by default; `false`, or `LOUSHO_RECOVER_ON_START=0`
+   * in `env`, turns it off. `intervalMs` is mainly for tests.
+   */
+  recovery?: false | { intervalMs?: number };
 }
 
 /** The store `LOUSHO_STORE` names: `memory` (the default, lost on restart) or `sqlite:<path>` (Node >= 22). */
@@ -139,6 +152,38 @@ async function channelRequestAllowed(req: http.IncomingMessage, res: http.Server
   return false;
 }
 
+/** Whether crash recovery runs at start-up: not with `recovery: false` or `LOUSHO_RECOVER_ON_START=0` / `false`. */
+function recoveryEnabled(options: DeployedServerOptions): boolean {
+  const value = (options.env ?? process.env)[RECOVER_ENV]?.trim().toLowerCase();
+  return options.recovery !== false && value !== '0' && value !== 'false';
+}
+
+/** Codes of `agent.pending()` / `agent.recover()` for a store that keeps no listable checkpoints: nothing to recover. */
+const NOT_RECOVERABLE = new Set(['LOUSHO_CONFIG_MISSING_CHECKPOINT_STORE', 'LOUSHO_CONFIG_INVALID']);
+
+/**
+ * Starts `agent.recover()` (stopped by `signal`), after logging how many
+ * interrupted runs it found. Never throws: a store that cannot list its
+ * checkpoints (none, or a custom one without `list()`) has nothing to
+ * recover, and any other failure is logged without stopping the server.
+ */
+async function startRunRecovery(agent: SimpleAgent, signal: AbortSignal, intervalMs: number | undefined): Promise<RunRecovery | undefined> {
+  try {
+    const interrupted = (await agent.pending()).filter((run) => run.status === 'in-progress' && run.lastError?.retryable !== false);
+    if (signal.aborted) return undefined;
+    if (interrupted.length > 0) console.log(`lousho server: resuming ${interrupted.length} interrupted run(s)`);
+    return agent.recover({
+      signal,
+      ...(intervalMs !== undefined && { intervalMs }),
+      onRecovered: (run) => console.log(`lousho server: recovered interrupted run '${run.sessionId}'`),
+      onError: (error, run) => console.error(`[lousho server] recovery${run ? ` of run '${run.sessionId}'` : ''} failed:`, error),
+    });
+  } catch (error) {
+    if (!NOT_RECOVERABLE.has((error as { code?: string } | null)?.code ?? '')) console.error('[lousho server] recovery failed to start:', error);
+    return undefined;
+  }
+}
+
 /** Default bound of {@link DeployedServer.shutdown}'s wait for in-flight turns; `LOUSHO_SHUTDOWN_TIMEOUT_MS` overrides it in the generated server. */
 const DEFAULT_SHUTDOWN_TIMEOUT_MS = 10_000;
 
@@ -151,7 +196,9 @@ export interface DeployedServer {
   channelRoutes: { name: string; auth: ChannelRouteAuth }[];
   /**
    * A graceful stop, for SIGTERM. Stops accepting connections
-   * (`server.close()`), stops the schedules (no new fires), then waits for
+   * (`server.close()`), stops the schedules (no new fires) and crash recovery
+   * (its resumes in flight are aborted; their checkpoints stay for the next
+   * start), then waits for
    * what is in flight - requests and their turns (a channel turn, a stream),
    * turns that outlived a disconnected client, and schedule fires - for at
    * most `timeoutMs` (default 10 s), and finally closes the connections left.
@@ -216,13 +263,26 @@ export function createDeployedServer(agent: SimpleAgent, options: DeployedServer
     server.on('listening', () => (running = startSchedules(agent, options.schedules ?? [], options.scheduler)));
     server.on('close', () => void stopSchedules());
   }
+  // Crash recovery: the runs a crash left in-progress are resumed once the server listens.
+  const recoveryAbort = new AbortController();
+  let recovering: Promise<RunRecovery | undefined> | undefined;
+  const stopRecovery = async (): Promise<void> => {
+    recoveryAbort.abort();
+    await (await recovering)?.done;
+  };
+  if (recoveryEnabled(options)) {
+    const intervalMs = options.recovery === false ? undefined : options.recovery?.intervalMs;
+    server.on('listening', () => (recovering ??= startRunRecovery(agent, recoveryAbort.signal, intervalMs)));
+    server.on('close', () => void stopRecovery());
+  }
   const shutdown = async ({ timeoutMs = DEFAULT_SHUTDOWN_TIMEOUT_MS }: { timeoutMs?: number } = {}): Promise<void> => {
     if (server.listening) server.close();
     server.closeIdleConnections();
     let timer: ReturnType<typeof setTimeout> | undefined;
     const timedOut = new Promise<'timeout'>((done) => (timer = setTimeout(() => done('timeout'), timeoutMs)));
     const drained = (async () => {
       await stopSchedules();
+      await stopRecovery();
       // A request can start a detached turn while it ends: wait until nothing is left.
       while (inFlight.size > 0) await Promise.all(inFlight);
     })();