feat(gateway): update one worker from the gateway (#838)

docs/internal/flows/worker-update.md

@@ -0,0 +1,15 @@
+# Worker update
+
+Modules: [gateway](../../../src/gateway/AGENTS.md).
+
+`worker.update` sends `daemon.update` over that worker's uplink, bounded at 90 s
+(`UPDATE_UPLINK_BOUND_MS`, `src/gateway/dispatcher.ts`; `WORKER_NOT_CONNECTED` at the limit), then
+refreshes the view. An unknown id → `UNKNOWN_WORKER`; a disconnected one →
+`WORKER_NOT_CONNECTED`. An `incompatible` worker gets the same frozen call.
+
+- A view with `update` set is skipped by routing (`updating`): the request waits, it does
+  not end `NO_CAPACITY`.
+- The update and the drain set are separate: an update never calls `setDrained`, and
+  `update: null` from the next status clears the view.
+- `daemon.update` on the gateway itself goes through the `update` option
+  (`src/daemon/update-coordinator.ts`): no lease wait, straight to installing.

src/admin/index.ts

@@ -84,6 +84,8 @@ export type {
   WorkerListOutput,
   WorkerRemoveOutput,
   WorkerUndrainOutput,
+  WorkerUpdateInput,
+  WorkerUpdateOutput,
   WorkerView,
 } from "../simlock-client/client.js";
 export { isSimlockError, SimlockError } from "../simlock-client/client.js";

src/cli/commands/list.ts

@@ -73,6 +73,7 @@ const GATEWAY_REASON_TEXT: Readonly<
   "not-ready": () => "Not ready yet",
   "runtime-missing": () => "OS not installed",
   unhealthy: () => "Not running",
+  updating: () => "Updating",
   "unknown-model": () => "Model not offered",
 };
 

src/cli/commands/worker.ts

@@ -15,7 +15,8 @@ const WORKER_USAGE =
   "Usage: simlock worker list [--json]\n" +
   "       simlock worker drain <worker-id>\n" +
   "       simlock worker undrain <worker-id>\n" +
-  "       simlock worker remove <worker-id>\n";
+  "       simlock worker remove <worker-id>\n" +
+  "       simlock worker update <worker-id> [--version <version>] [--force] [--detach]\n";
 
 async function run(argv: readonly string[], context: CommandContext): Promise<number> {
   const { environment, token } = context;
@@ -28,6 +29,7 @@ async function run(argv: readonly string[], context: CommandContext): Promise<nu
     if (command === "drain" || command === "undrain" || command === "remove") {
       return await runWorkerAction(command, flagArgs, client, environment);
     }
+    if (command === "update") return await runWorkerUpdate(flagArgs, client, environment);
     throw new UsageError(withHelpHint(`Unknown worker command: ${String(command)}`));
   } finally {
     await client.close();
@@ -78,4 +80,112 @@ async function runWorkerAction(
   return 0;
 }
 
+const FOLLOW_POLL_MS = 2_000;
+
+type WorkerView = Awaited<ReturnType<SimlockAdminClient["listWorkers"]>>["workers"][number];
+
+function workerName(view: WorkerView | undefined, workerId: string): string {
+  return view?.label === undefined ? workerId : `${view.label} (${workerId})`;
+}
+
+// why: the follow ends on the worker's own view: its version is the target with no update running, or its last failure names the target.
+function followOutcome(
+  view: WorkerView | undefined,
+  to: string,
+  name: string,
+): { readonly code: number; readonly text: string } | undefined {
+  const failure = view?.lastUpdateFailure;
+  if (failure?.to === to) {
+    return { code: 1, text: `Update failed (${failure.stage}): ${failure.message}` };
+  }
+  if (view?.version === to && view.update == null) {
+    return { code: 0, text: `${name} is running ${to}` };
+  }
+  return undefined;
+}
+
+function updateEnds(answer: Awaited<ReturnType<SimlockAdminClient["updateWorker"]>>): {
+  readonly from: string;
+  readonly to: string;
+} {
+  return {
+    from: answer.version,
+    to: answer.update?.to ?? answer.resolved ?? "",
+  };
+}
+
+async function runWorkerUpdate(
+  argv: readonly string[],
+  client: SimlockAdminClient,
+  environment: CommandContext["environment"],
+): Promise<number> {
+  const values = commandArgs(argv, {
+    detach: { type: "boolean" },
+    force: { type: "boolean" },
+    help: { type: "boolean", short: "h" },
+    version: { type: "string" },
+  });
+  if (values.help) {
+    environment.stdout.write(
+      "Usage: simlock worker update <worker-id> [--version <version>] [--force] [--detach]\n",
+    );
+    return 0;
+  }
+  const workerId = requiredPositional(values.positionals, "worker-id");
+  const answer = await client.updateWorker({
+    workerId,
+    ...(typeof values.version === "string" ? { version: values.version } : {}),
+    ...(values.force === true ? { force: true } : {}),
+  });
+  const find = async (): Promise<WorkerView | undefined> =>
+    (await client.listWorkers()).workers.find((worker) => worker.id === workerId);
+  const first = await find();
+  const name = workerName(first, workerId);
+  if (answer.upToDate) {
+    environment.stdout.write(
+      `${upToDateMessage(name, answer, typeof values.version === "string" ? values.version : undefined)}\n`,
+    );
+    return 0;
+  }
+  const { from, to } = updateEnds(answer);
+  if (values.detach === true) {
+    environment.stdout.write(
+      `Updating ${name} ${from} → ${to}. Follow it with simlock worker list.\n`,
+    );
+    return 0;
+  }
+  return follow({ environment, find, first, name, to });
+}
+
+// why: the three reasons a worker is up to date, as `simlock update` tells them apart (update.ts upToDateMessage).
+function upToDateMessage(
+  name: string,
+  answer: { readonly version: string; readonly resolved?: string | null },
+  requested: string | undefined,
+): string {
+  if (requested !== undefined) return `${name} is already running ${answer.version}`;
+  if (answer.resolved === answer.version) {
+    return `${name} is already on the latest (${answer.version})`;
+  }
+  return `${name} is newer than the latest (${answer.resolved}). Use --version to install another version.`;
+}
+
+async function follow(options: {
+  readonly environment: CommandContext["environment"];
+  readonly find: () => Promise<WorkerView | undefined>;
+  readonly first: WorkerView | undefined;
+  readonly name: string;
+  readonly to: string;
+}): Promise<number> {
+  const { environment, find, name, to } = options;
+  for (let view = options.first; ; view = await find()) {
+    const outcome = followOutcome(view, to, name);
+    if (outcome !== undefined) {
+      environment.stdout.write(`${outcome.text}\n`);
+      return outcome.code;
+    }
+    await environment.sleep(FOLLOW_POLL_MS);
+  }
+}
+
 export { run };

src/cli/format.ts

@@ -37,7 +37,7 @@ export function formatWorkers(
             : ` -- ${entry.name} ${entry.state}`,
         )
         .join("");
-      return `${label}: ${state} -- ${capacity}, ${leaseCount(worker)}${skew}${host}${capabilities}${load}${installs}`;
+      return `${label}: ${state} -- ${capacity}, ${leaseCount(worker)}${skew}${workerVersion(worker)}${host}${capabilities}${load}${installs}`;
     })
     .join("\n");
 }
@@ -57,6 +57,21 @@ function workerState(worker: WorkerView): string {
   ].join(", ");
 }
 
+function workerVersion(worker: WorkerView): string {
+  const { latestVersion, update, version } = worker;
+  const available =
+    version !== undefined && latestVersion != null && isNewerVersion(latestVersion, version)
+      ? ` (${latestVersion} available)`
+      : "";
+  const running = version === undefined ? "" : ` -- simlock ${version}${available}`;
+  return `${running}${update == null ? "" : ` -- updating to ${update.to}: ${updateStage(update)}`}`;
+}
+
+function updateStage(update: NonNullable<WorkerView["update"]>): string {
+  if (update.stage !== "waiting") return update.stage;
+  return `waiting for ${update.leases} ${update.leases === 1 ? "lease" : "leases"}`;
+}
+
 function leaseCount(worker: WorkerView): string {
   if (worker.leases === undefined) return "leases unknown";
   return `${String(worker.leases.length)} lease(s)`;