From be96323a0ea9cb3f0c079fd0c78399f86b26a232 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 15:58:23 +0200 Subject: [PATCH 01/42] test: operator calls worker.update on the gateway and the worker's view shows the update (#838) --- src/contract/operations.ts | 15 +++++++++++++++ src/contract/schemas.ts | 3 +++ src/daemon/dispatcher.ts | 1 + src/daemon/server.ts | 2 ++ src/gateway/dispatcher.test.ts | 27 +++++++++++++++++++++++++++ src/gateway/dispatcher.ts | 13 +++++++++++++ src/gateway/test-support.ts | 21 +++++++++++++++++++++ 7 files changed, 82 insertions(+) diff --git a/src/contract/operations.ts b/src/contract/operations.ts index b1e737b6..fe34a38e 100644 --- a/src/contract/operations.ts +++ b/src/contract/operations.ts @@ -827,6 +827,20 @@ export const workerRemove = defineOperation({ output: z.object({ workerId: z.string(), removed: z.boolean() }), }); +// why: ADR 0042 §2: the gateway sends `daemon.update` over that worker's uplink and answers with the worker's own answer plus the id. +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const workerUpdate = defineOperation({ + name: "worker.update", + role: "admin", + effect: "write", + input: z.object({ + workerId: z.string().min(1), + version: updateRequestedVersionSchema.optional(), + force: z.boolean().optional(), + }), + output: daemonUpdate.output.extend({ workerId: z.string() }), +}); + // ---- component.install (ADR 0010 §6) -------------------------------------------------------- /** @@ -1131,6 +1145,7 @@ export const OPERATIONS = { "worker.drain": workerDrain, "worker.undrain": workerUndrain, "worker.remove": workerRemove, + "worker.update": workerUpdate, "component.install": componentInstall, "component.list": componentList, "worker.install-component": workerInstallComponent, diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 7c7a2e46..4945ec0e 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -1374,6 +1374,9 @@ export const workerViewSchema = z.object({ host: hostFactsSchema.optional(), installs: workerInstallsSchema.optional(), waiting: z.array(waitingRequestSchema).optional(), + update: daemonUpdateFields.update.optional(), + lastUpdateFailure: daemonUpdateFields.lastUpdateFailure.optional(), + latestVersion: daemonUpdateFields.latestVersion.optional(), }); export const statusLeaseSchema = leaseRecordSchema.extend({ diff --git a/src/daemon/dispatcher.ts b/src/daemon/dispatcher.ts index 291e4104..17419b42 100644 --- a/src/daemon/dispatcher.ts +++ b/src/daemon/dispatcher.ts @@ -332,6 +332,7 @@ export class Dispatcher { "worker.drain": unsupportedOnWorker("worker.drain"), "worker.undrain": unsupportedOnWorker("worker.undrain"), "worker.remove": unsupportedOnWorker("worker.remove"), + "worker.update": unsupportedOnWorker("worker.update"), "worker.install-component": unsupportedOnWorker( "worker.install-component", "worker.install-component installs on a gateway's workers, and this daemon is not a gateway; install on this host without naming workers", diff --git a/src/daemon/server.ts b/src/daemon/server.ts index 1fba05f0..02f40b3b 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -1177,6 +1177,8 @@ export class DaemonServer { ); case "worker.remove": return this.#dispatcher.dispatch("worker.remove", frame.payload, this.#session(connection)); + case "worker.update": + return this.#dispatcher.dispatch("worker.update", frame.payload, this.#session(connection)); case "alerts.test": return this.#dispatcher.dispatch("alerts.test", frame.payload, this.#session(connection)); case "component.install": diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 2c837b8b..23991d5d 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -617,6 +617,32 @@ describe("GatewayDispatcher", () => { ).resolves.toEqual({ removed: true, workerId: "wrk_1" }); }); + it("Operator calls worker.update on the gateway: the worker gets daemon.update and its view shows the update", async () => { + const { dispatcher, directory, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + const update = { + forced: false, + from: "0.3.0", + leases: 2, + stage: "waiting", + startedAt: 1_000, + to: "0.4.0", + } as const; + client.updateAnswer = { ...client.updateAnswer, latestVersion: "0.4.0", started: true, update }; + + const answer = await dispatcher.dispatch( + "worker.update", + { version: "0.4.0", workerId: "wrk_1" }, + session(), + ); + + expect(client.updateInputs).toEqual([{ version: "0.4.0" }]); + expect(answer).toMatchObject({ started: true, update, workerId: "wrk_1" }); + expect(workers.view("wrk_1")).toMatchObject({ latestVersion: "0.4.0", update }); + }); + it("refuses to remove a connected worker with WORKER_CONNECTED, naming it", async () => { const { dispatcher, workers } = harness(); workers.connected("wrk_1", undefined, undefined); @@ -1798,6 +1824,7 @@ describe("GatewayDispatcher", () => { "worker.drain": { workerId: "wrk_1" }, "worker.remove": { workerId: "wrk_1" }, "worker.undrain": { workerId: "wrk_1" }, + "worker.update": { workerId: "wrk_1" }, }; const { dispatcher, workers } = harness(); workers.connected("wrk_1", undefined, undefined); diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 745ffa62..a51030ee 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -233,6 +233,7 @@ export class GatewayDispatcher { "worker.drain": this.#workerDrain, "worker.undrain": this.#workerUndrain, "worker.remove": this.#workerRemove, + "worker.update": this.#workerUpdate, "alerts.test": this.#alertsTest, "daemon.update": unsupportedByDesign( "daemon.update", @@ -411,6 +412,18 @@ export class GatewayDispatcher { return { drained: false, workerId: input.workerId }; }; + #workerUpdate: Handler<"worker.update"> = async (input) => ({ + health: "running", + lastUpdateFailure: null, + latestVersion: null, + resolved: null, + started: false, + update: null, + upToDate: false, + version: "", + workerId: input.workerId, + }); + #alertsTest: Handler<"alerts.test"> = async (input) => { const { destination } = input; const result = (await this.options.testDestination?.(destination)) ?? UNKNOWN_DESTINATION; diff --git a/src/gateway/test-support.ts b/src/gateway/test-support.ts index 76e1ecff..d4f0ae73 100644 --- a/src/gateway/test-support.ts +++ b/src/gateway/test-support.ts @@ -11,6 +11,8 @@ import type { z } from "zod"; import type { ComponentInstallInput, ComponentInstallOutput, + DaemonUpdateInput, + DaemonUpdateOutput, ExecInput, ExecOptions, ExecOutput, @@ -400,6 +402,25 @@ export class ScriptedWorkerClient { }; } + readonly updateInputs: DaemonUpdateInput[] = []; + updateAnswer: DaemonUpdateOutput = { + started: false, + upToDate: false, + resolved: null, + version: "0.3.0", + health: "running", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }; + + // why: the degraded client lets `daemon.update` through (ADR 0042 §1), so this answers even when `failWith` is set. + async updateDaemon(input: DaemonUpdateInput = {}): Promise { + this.calls.push("daemon.update"); + this.updateInputs.push(input); + return this.updateAnswer; + } + async getStatus(): Promise { this.calls.push("status.get"); if (this.hangingCalls.has("status.get")) return new Promise(() => {}); From 68798d6deedde1d539487da5554a59e08900a161 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 15:59:03 +0200 Subject: [PATCH 02/42] feat(gateway): worker.update sends daemon.update to the worker and shows its update (#838) --- src/gateway/dispatcher.ts | 29 +++++++++++++++++------------ src/gateway/worker-registry.ts | 3 +++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index a51030ee..078fa166 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -59,7 +59,7 @@ import { NoopLogger } from "../ports/index.js"; import { aggregateCatalog, aggregateStatus, type AggregateStatusOptions } from "./aggregate.js"; import { relayComponentInstall } from "./component-relay.js"; import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; -import type { WorkerDirectory } from "./fleet-ports.js"; +import { liveClient, type WorkerDirectory } from "./fleet-ports.js"; import type { FleetLeaseIndex } from "./lease-index.js"; import type { WorkerRegistry } from "./worker-registry.js"; @@ -412,17 +412,22 @@ export class GatewayDispatcher { return { drained: false, workerId: input.workerId }; }; - #workerUpdate: Handler<"worker.update"> = async (input) => ({ - health: "running", - lastUpdateFailure: null, - latestVersion: null, - resolved: null, - started: false, - update: null, - upToDate: false, - version: "", - workerId: input.workerId, - }); + #workerUpdate: Handler<"worker.update"> = async (input) => { + const target = this.options.directory.target(input.workerId); + const client = liveClient(target); + if (client === undefined || target === undefined) { + throw new DispatchError("INTERNAL", `worker ${input.workerId} has no session`); + } + const { workerId, ...request } = input; + const answer = await client.updateDaemon(request); + this.options.workers.refresh(workerId, { + lastUpdateFailure: answer.lastUpdateFailure, + latestVersion: answer.latestVersion, + update: answer.update, + }); + await target.refresh(); + return { ...answer, workerId }; + }; #alertsTest: Handler<"alerts.test"> = async (input) => { const { destination } = input; diff --git a/src/gateway/worker-registry.ts b/src/gateway/worker-registry.ts index cdfc444b..dda0040f 100644 --- a/src/gateway/worker-registry.ts +++ b/src/gateway/worker-registry.ts @@ -36,10 +36,13 @@ export type WorkerViewSnapshot = Pick< | "health" | "host" | "installs" + | "lastUpdateFailure" + | "latestVersion" | "lease" | "leases" | "load" | "queueDepth" + | "update" | "version" | "waiting" | "warmPool" From e31c323561f5d18272a9c28a71c52295865526e0 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:00:03 +0200 Subject: [PATCH 03/42] test: worker.update on a disconnected or unknown worker is refused (#838) --- src/contract/errors.ts | 9 +++++++++ src/gateway/dispatcher.test.ts | 13 ++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/contract/errors.ts b/src/contract/errors.ts index d54e4bbf..f7b0595a 100644 --- a/src/contract/errors.ts +++ b/src/contract/errors.ts @@ -126,6 +126,8 @@ export interface ErrorDetailsMap { * answers `{removed: false}` instead: forgetting something already forgotten is done, not an * error.) */ UNKNOWN_WORKER: { readonly workerId: string }; + // why: ADR 0042 §2: `worker.update` names a worker the gateway knows but has no open uplink to. + WORKER_NOT_CONNECTED: { readonly workerId: string }; /** * ADR 0005 §28: the worker that owns this lease (or would serve this request) has no open * uplink. `kind: "transport"`, because it says the same thing `DAEMON_CONNECTION_LOST` says @@ -200,6 +202,7 @@ const CODES_WITH_DECLARED_DETAILS_BY_CODE: Record UNSUPPORTED_IN_WORKER_MODE: true, WORKER_CONNECTED: true, UNKNOWN_WORKER: true, + WORKER_NOT_CONNECTED: true, WORKER_UNREACHABLE: true, ALERT_SEND_FAILED: true, UNKNOWN_ALERT_DESTINATION: true, @@ -436,6 +439,12 @@ export const ERROR_TABLE: { readonly [Code in SimlockErrorCode]: ErrorTableEntry // Exit 12, beside `UNKNOWN_MODEL`: the "what you named does not exist" class, which a script // branches on differently from "the fleet is busy" or "that cannot apply here". UNKNOWN_WORKER: { code: "UNKNOWN_WORKER", kind: "domain", cliExitCode: 12, httpStatus: 404 }, + WORKER_NOT_CONNECTED: { + code: "WORKER_NOT_CONNECTED", + kind: "domain", + cliExitCode: 2, + httpStatus: 409, + }, WORKER_UNREACHABLE: { code: "WORKER_UNREACHABLE", kind: "transport", diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 23991d5d..ed9077a6 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -617,7 +617,7 @@ describe("GatewayDispatcher", () => { ).resolves.toEqual({ removed: true, workerId: "wrk_1" }); }); - it("Operator calls worker.update on the gateway: the worker gets daemon.update and its view shows the update", async () => { + it("Operator calls worker.update on the gateway: connected, disconnected and unknown workers", async () => { const { dispatcher, directory, workers } = harness(); workers.connected("wrk_1", "mac-mini-1", "0.3.0"); const client = new ScriptedWorkerClient(); @@ -641,6 +641,17 @@ describe("GatewayDispatcher", () => { expect(client.updateInputs).toEqual([{ version: "0.4.0" }]); expect(answer).toMatchObject({ started: true, update, workerId: "wrk_1" }); expect(workers.view("wrk_1")).toMatchObject({ latestVersion: "0.4.0", update }); + + workers.connected("wrk_2", "mac-mini-2", "0.3.0"); + workers.disconnected("wrk_2"); + for (const { code, message, workerId } of [ + { code: "WORKER_NOT_CONNECTED", message: "worker wrk_2 is not connected", workerId: "wrk_2" }, + { code: "UNKNOWN_WORKER", message: "Unknown worker: wrk_ghost", workerId: "wrk_ghost" }, + ]) { + await expect( + dispatcher.dispatch("worker.update", { workerId }, session()), + ).rejects.toMatchObject({ code, details: { workerId }, message }); + } }); it("refuses to remove a connected worker with WORKER_CONNECTED, naming it", async () => { From 4d1efa3115a261396c97fb6383f7fe23d626d809 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:00:23 +0200 Subject: [PATCH 04/42] feat(gateway): worker.update refuses an unknown or disconnected worker (#838) --- src/gateway/dispatcher.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 078fa166..1fd9c70e 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -413,10 +413,18 @@ export class GatewayDispatcher { }; #workerUpdate: Handler<"worker.update"> = async (input) => { + const view = this.options.workers.view(input.workerId); + if (view === undefined) { + throw new DispatchError("UNKNOWN_WORKER", `Unknown worker: ${input.workerId}`, { + workerId: input.workerId, + }); + } const target = this.options.directory.target(input.workerId); const client = liveClient(target); - if (client === undefined || target === undefined) { - throw new DispatchError("INTERNAL", `worker ${input.workerId} has no session`); + if (view.connection === "disconnected" || client === undefined || target === undefined) { + throw new DispatchError("WORKER_NOT_CONNECTED", `worker ${input.workerId} is not connected`, { + workerId: input.workerId, + }); } const { workerId, ...request } = input; const answer = await client.updateDaemon(request); From 8e90ddc0f0fe0806852ca949808d419e86552e09 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:00:40 +0200 Subject: [PATCH 05/42] test(contract): worker.update in the role and effect tables (#838) --- src/contract/operations.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 9ada2f16..20dc2367 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -58,6 +58,7 @@ const ROLE_MATRIX: ReadonlyArray<{ { name: "worker.drain", input: { workerId: "wrk_1" }, role: "admin" }, { name: "worker.undrain", input: { workerId: "wrk_1" }, role: "admin" }, { name: "worker.remove", input: { workerId: "wrk_1" }, role: "admin" }, + { name: "worker.update", input: { workerId: "wrk_1" }, role: "admin" }, // ADR 0010 §6: an operator's consent to a download. { name: "component.install", input: { platform: "ios", version: "26.4" }, role: "admin" }, { name: "component.list", input: {}, role: "agent" }, @@ -146,6 +147,7 @@ const EFFECT_MATRIX: ReadonlyArray<{ { name: "worker.drain", input: { workerId: "wrk_1" }, effect: "write" }, { name: "worker.undrain", input: { workerId: "wrk_1" }, effect: "write" }, { name: "worker.remove", input: { workerId: "wrk_1" }, effect: "write" }, + { name: "worker.update", input: { workerId: "wrk_1" }, effect: "write" }, { name: "component.install", input: { platform: "ios", version: "26.4" }, effect: "write" }, { name: "component.list", input: {}, effect: "read" }, { From e88371667b7d0833732baa8537bee7e5908aea5a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:01:46 +0200 Subject: [PATCH 06/42] test: a tick reads an incompatible worker's update with daemon.update query (#838) --- src/gateway/service.test.ts | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index 425bb044..7550988b 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -1201,6 +1201,37 @@ describe("GatewayService", () => { await harness.service.stop(); }); + it("reads an incompatible worker's update with daemon.update query on the tick", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.failWith = protocolMismatchError({ min: 5, max: 5 }); + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + const update = { + forced: false, + from: "0.2.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + worker.updateAnswer = { ...worker.updateAnswer, latestVersion: "0.4.0", update }; + + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + latestVersion: "0.4.0", + update, + }), + ); + expect(worker.updateInputs).toEqual([{ query: true }]); + + await harness.service.stop(); + }); + it("a worker advertising protocol 10 is incompatible on the gateway and nothing it pushes reaches the gateway's bus", async () => { const worker10 = { min: 10, max: 10 }; expect(negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, worker10)).toBeUndefined(); From a11b9e7c3fb245a8a06baded7bc1668f3e0d31e3 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:02:10 +0200 Subject: [PATCH 07/42] feat(gateway): an incompatible worker's refresh reads its update with daemon.update query (#838) --- src/gateway/worker-link.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/gateway/worker-link.ts b/src/gateway/worker-link.ts index 8c678126..c1cc3598 100644 --- a/src/gateway/worker-link.ts +++ b/src/gateway/worker-link.ts @@ -134,6 +134,8 @@ export class WorkerLink { * (inside `#rebuildView`) rather than only once a whole refresh completes -- see * `MAX_CONSECUTIVE_REFRESH_TIMEOUTS`'s comment for why liveness is judged on that one call. */ #consecutiveRefreshTimeouts = 0; + // why: an incompatible worker is asked only the frozen `daemon.update` (ADR 0005 §31, ADR 0042 §1), so its refresh is that query. + #incompatible = false; constructor(private readonly options: WorkerLinkOptions) { this.workerId = options.uplink.workerId; @@ -210,6 +212,7 @@ export class WorkerLink { await this.#withTimeout(client.getStatus(), "status.get"); } catch (error: unknown) { if (isSimlockError(error) && error.code === "PROTOCOL_VERSION_UNSUPPORTED") { + this.#incompatible = true; // ADR 0005 §31: marked `incompatible`, with both ranges, and never asked anything else. // The uplink stays open on purpose -- the worker is running and will reconnect on its // own schedule after an upgrade; closing it here would just make it redial. @@ -306,6 +309,7 @@ export class WorkerLink { async refresh(options: RefreshOptions = {}): Promise { const client = this.#client; if (client === undefined || this.#closed) return; + if (this.#incompatible) return this.#queryUpdate(client); if (this.#refreshing) { this.#queuedRefresh = queueRefresh(this.#queuedRefresh, options); return this.#queuedRefresh.done; @@ -357,6 +361,24 @@ export class WorkerLink { } } + // why: a failed query keeps the last view (ADR 0042 §2), so its error is dropped. + async #queryUpdate(client: GatewayWorkerClient): Promise { + try { + const answer = await this.#withTimeout(client.updateDaemon({ query: true }), "daemon.update"); + if (this.#superseded()) return; + this.options.registry.refresh(this.workerId, { + lastUpdateFailure: answer.lastUpdateFailure, + latestVersion: answer.latestVersion, + update: answer.update, + }); + } catch (error: unknown) { + this.#logger.debug("Incompatible worker's update query failed", { + workerId: this.workerId, + message: errorMessage(error), + }); + } + } + /** The round trips one refresh makes, and what they become in the view. Split out of * `refresh` so that method is only the coalescing rule and this one is only the reads. * Bounded as one unit per call (D3): a worker that never answers one of these calls must not From 90f3ff942ec185d62de1131906eb6a0e1fa43084 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:02:37 +0200 Subject: [PATCH 08/42] test: a failed update query keeps an incompatible worker's last view (#838) --- src/gateway/service.test.ts | 7 ++++++- src/gateway/test-support.ts | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index 7550988b..9c016bc8 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -1201,7 +1201,7 @@ describe("GatewayService", () => { await harness.service.stop(); }); - it("reads an incompatible worker's update with daemon.update query on the tick", async () => { + it("reads an incompatible worker's update with daemon.update query on the tick and keeps the last view when the query fails", async () => { const harness = fleet(); await harness.service.start(); const worker = new ScriptedWorkerClient(); @@ -1229,6 +1229,11 @@ describe("GatewayService", () => { ); expect(worker.updateInputs).toEqual([{ query: true }]); + worker.updateFailWith = new Error("worker went quiet"); + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(worker.updateInputs).toHaveLength(2)); + expect(harness.service.workers.view("wrk_1")).toMatchObject({ latestVersion: "0.4.0", update }); + await harness.service.stop(); }); diff --git a/src/gateway/test-support.ts b/src/gateway/test-support.ts index d4f0ae73..24de3108 100644 --- a/src/gateway/test-support.ts +++ b/src/gateway/test-support.ts @@ -415,9 +415,12 @@ export class ScriptedWorkerClient { }; // why: the degraded client lets `daemon.update` through (ADR 0042 §1), so this answers even when `failWith` is set. + updateFailWith: unknown; + async updateDaemon(input: DaemonUpdateInput = {}): Promise { this.calls.push("daemon.update"); this.updateInputs.push(input); + if (this.updateFailWith !== undefined) throw this.updateFailWith; return this.updateAnswer; } From 3d305f99beca95d0694bbead34d8924d71edded9 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:03:40 +0200 Subject: [PATCH 09/42] test: the gateway skips an updating worker and the request waits until the update ends (#838) --- src/cli/commands/list.ts | 1 + src/contract/schemas.ts | 1 + src/gateway/fleet-coordinator.test.ts | 32 +++++++++++++++++++++++++++ src/gateway/routing/pipeline.ts | 1 + ui/src/model/waiting.ts | 1 + 5 files changed, 36 insertions(+) diff --git a/src/cli/commands/list.ts b/src/cli/commands/list.ts index 7e43fbe5..61bb621b 100644 --- a/src/cli/commands/list.ts +++ b/src/cli/commands/list.ts @@ -73,6 +73,7 @@ const GATEWAY_REASON_TEXT: Readonly< "not-ready": () => "Not ready yet", "runtime-missing": () => "OS not installed", unhealthy: () => "Not running", + updating: () => "Updating", "unknown-model": () => "Model not offered", }; diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 4945ec0e..3c8ca510 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -393,6 +393,7 @@ const gatewayWaitingReasonSchema = z.enum([ "not-ready", "unhealthy", "drained", + "updating", "busy", "no-driver", "unknown-model", diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index 8751b329..dc1d7f6a 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -2193,6 +2193,38 @@ describe("FleetLeaseCoordinator fails a request no worker can serve at once (ADR }, ); + it("Gateway routes a lease while a worker is updating: it skips the worker with reason updating, the request waits, and the worker is picked once the update ends", async () => { + const fleetHarness = harness(); + const { coordinator, directory, workers } = fleetHarness; + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + connectWorker(workers, "wrk_a"); + workers.refresh("wrk_a", { + update: { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1, + to: "0.4.0", + }, + }); + + const { outcome, state } = lease(fleetHarness, REQUEST); + await tick(); + + expect(state.state).toBe("pending"); + expect(coordinator.waitingRequests()[0]).toMatchObject({ + workers: [{ reason: "updating", workerId: "wrk_a" }], + }); + expect(leaseRequests(client)).toEqual([]); + + workers.refresh("wrk_a", { update: null }); + await tick(); + + expect((await outcome).lease.worker?.id).toBe("wrk_a"); + }); + it.each([ { change: (workers: WorkerRegistry) => workers.disconnected("wrk_a"), diff --git a/src/gateway/routing/pipeline.ts b/src/gateway/routing/pipeline.ts index 6d940267..f268db3b 100644 --- a/src/gateway/routing/pipeline.ts +++ b/src/gateway/routing/pipeline.ts @@ -25,6 +25,7 @@ export type WorkerReasonCode = | "not-ready" | "unhealthy" | "drained" + | "updating" | "busy" | "no-driver" | "unknown-model" diff --git a/ui/src/model/waiting.ts b/ui/src/model/waiting.ts index da49bdb0..eaffce18 100644 --- a/ui/src/model/waiting.ts +++ b/ui/src/model/waiting.ts @@ -106,6 +106,7 @@ const WORKER_WHY_TEXT: Readonly< "not-ready": () => "Not ready yet", "runtime-missing": () => "OS not installed", unhealthy: () => "Not running", + updating: () => "Updating", "unknown-model": () => "Model not offered", }; From 0febb69dcbd65ca72babffd0b7282d6c83e5f733 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:04:51 +0200 Subject: [PATCH 10/42] feat(gateway): routing skips an updating worker and keeps the request waiting (#838) --- src/gateway/fleet-coordinator.ts | 1 + src/gateway/routing/serviceability.ts | 6 +++++- src/gateway/routing/stages/takes-requests.ts | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/gateway/fleet-coordinator.ts b/src/gateway/fleet-coordinator.ts index c3899240..e379a8ea 100644 --- a/src/gateway/fleet-coordinator.ts +++ b/src/gateway/fleet-coordinator.ts @@ -119,6 +119,7 @@ const COULD_TAKE_LATER: ReadonlySet = new Set([ "full", "not-ready", "unhealthy", + "updating", ]); const CANNOT_SERVE_CODES: ReadonlySet = new Set([ diff --git a/src/gateway/routing/serviceability.ts b/src/gateway/routing/serviceability.ts index 931c7717..2d09a7a1 100644 --- a/src/gateway/routing/serviceability.ts +++ b/src/gateway/routing/serviceability.ts @@ -60,7 +60,11 @@ export function assess(request: RoutableRequest, views: readonly WorkerView[]): } function assessTable(request: RoutableRequest, views: readonly WorkerView[]): Assessment { - const takers = views.filter((worker) => takesRequests.check(worker, request) === undefined); + // why: an updating worker comes back by itself (ADR 0042 §3), so a request that only it can serve waits instead of failing. + const takers = views.filter((worker) => { + const refusal = takesRequests.check(worker, request); + return refusal === undefined || refusal.reason === "updating"; + }); if (takers.length === 0) return noWorker(); const known = views.filter(isKnown); diff --git a/src/gateway/routing/stages/takes-requests.ts b/src/gateway/routing/stages/takes-requests.ts index 4629a387..1203dd51 100644 --- a/src/gateway/routing/stages/takes-requests.ts +++ b/src/gateway/routing/stages/takes-requests.ts @@ -13,6 +13,7 @@ export const takesRequests: FilterStage = { check(worker) { if (worker.connection !== "connected") return { reason: worker.connection }; if (worker.drained) return { reason: "drained" }; + if (worker.update != null) return { reason: "updating" }; const ready = worker.health === "starting" ? worker.catalog !== undefined From de4d38a8e624168eacff67e687f4a82852cb6c9e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:06:44 +0200 Subject: [PATCH 11/42] test: a worker's update fields reach its view and clear with the next status (#838) --- src/gateway/service.test.ts | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index 9c016bc8..d20c098e 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -1237,6 +1237,51 @@ describe("GatewayService", () => { await harness.service.stop(); }); + it("shows a worker's update, last update failure and latest version from its status, and a later status without an update clears it", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + const update = { + forced: false, + from: "0.3.0", + leases: 2, + stage: "waiting", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 500, + from: "0.2.0", + message: "npm failed", + stage: "installing", + to: "0.3.0", + } as const; + const base = statusFixture(); + worker.status = { + ...base, + daemon: { ...base.daemon, lastUpdateFailure: failure, latestVersion: "0.4.0", update }, + }; + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + latestVersion: "0.4.0", + update, + }), + ); + + worker.status = { ...worker.status, daemon: { ...worker.status.daemon, update: null } }; + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.update).toBeNull()); + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + latestVersion: "0.4.0", + }); + + await harness.service.stop(); + }); + it("a worker advertising protocol 10 is incompatible on the gateway and nothing it pushes reaches the gateway's bus", async () => { const worker10 = { min: 10, max: 10 }; expect(negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, worker10)).toBeUndefined(); From 4f509e9e2654078b3c8fcb413fc6ff64a1fd117d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:08:03 +0200 Subject: [PATCH 12/42] feat(gateway): a worker's view carries its update, last failure and latest version (#838) --- src/contract/worker-view.test.ts | 13 ++++++++++++- src/contract/worker-view.ts | 6 ++++++ src/gateway/service.test.ts | 3 +++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/contract/worker-view.test.ts b/src/contract/worker-view.test.ts index a8ccb07f..9eb14781 100644 --- a/src/contract/worker-view.test.ts +++ b/src/contract/worker-view.test.ts @@ -303,8 +303,11 @@ describe("workerViewFields", () => { health: "running", host: HOST, installs: [], + lastUpdateFailure: null, + latestVersion: null, leases: [], queueDepth: 3, + update: null, waiting: [], }); }); @@ -349,7 +352,15 @@ describe("workerViewFields", () => { }, }); - expect(Object.keys(view).sort()).toEqual(["health", "host", "installs", "waiting"]); + expect(Object.keys(view).sort()).toEqual([ + "health", + "host", + "installs", + "lastUpdateFailure", + "latestVersion", + "update", + "waiting", + ]); }); describe("a worker's load", () => { const LOAD = { diff --git a/src/contract/worker-view.ts b/src/contract/worker-view.ts index 450977c6..818f14df 100644 --- a/src/contract/worker-view.ts +++ b/src/contract/worker-view.ts @@ -55,11 +55,14 @@ export type WorkerViewReport = Pick & | "devices" | "downloads" | "installs" + | "lastUpdateFailure" + | "latestVersion" | "lease" | "leases" | "load" | "physicalQueueDepth" | "queueDepth" + | "update" | "waiting" | "warmPool" > @@ -87,6 +90,9 @@ export function workerViewFields(reads: WorkerReads): WorkerViewReport { host: status.host, ...load, installs: status.installs ?? [], + lastUpdateFailure: status.daemon.lastUpdateFailure, + latestVersion: status.daemon.latestVersion, + update: status.daemon.update, waiting: status.waiting ?? [], ...(status.capabilities === undefined ? {} : { capabilities: status.capabilities }), ...(status.capacity === undefined ? {} : { capacity: status.capacity }), diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index d20c098e..7637094f 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -445,9 +445,12 @@ describe("GatewayService", () => { installs: [install], label: "mac-mini-1", lastSeenAt: 1_000, + lastUpdateFailure: null, + latestVersion: null, lease: { maxTtlMs: 6 * 60 * 60_000 }, leases: [leaseFixture("lease_1", "dev_1")], queueDepth: 3, + update: null, version: "9.8.7", waiting: [waiting], }); From 4aa74f5581e2ca8f2fcabd02791c56056ab94c8b Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:08:26 +0200 Subject: [PATCH 13/42] test: a worker's daemon.update events re-read its view at once (#838) --- src/gateway/service.test.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index 7637094f..d7a7daf5 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -1000,6 +1000,41 @@ describe("GatewayService", () => { await harness.service.stop(); }); + it("shows a worker's update after daemon.update-started and its failure after daemon.update-failed, without waiting for the periodic refresh", async () => { + const worker = new ScriptedWorkerClient(); + const harness = await joined(worker); + const base = statusFixture(); + const update = { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 2_000, + from: "0.3.0", + message: "npm failed", + stage: "installing", + to: "0.4.0", + } as const; + + worker.status = { ...base, daemon: { ...base.daemon, update } }; + worker.pushEvent({ event: "daemon.update-started" }); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.update).toEqual(update)); + + worker.status = { ...base, daemon: { ...base.daemon, lastUpdateFailure: failure } }; + worker.pushEvent({ event: "daemon.update-failed" }); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + update: null, + }), + ); + + await harness.service.stop(); + }); + it("drops a worker's install after component.install-failed, without waiting for the periodic refresh", async () => { const worker = new ScriptedWorkerClient(); worker.status = statusFixture({ installs: [install] }); From 7cb5a065b394beffff9f73c059cc5f95de8a4a2c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:09:00 +0200 Subject: [PATCH 14/42] feat(gateway): daemon.update events make a gateway read the worker's status again (#838) --- src/contract/worker-view.ts | 3 +++ src/gateway/worker-link.ts | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/contract/worker-view.ts b/src/contract/worker-view.ts index 818f14df..c12c99a7 100644 --- a/src/contract/worker-view.ts +++ b/src/contract/worker-view.ts @@ -31,6 +31,9 @@ export const WORKER_VIEW_REFRESH_INTERVAL_MS = 30_000; */ export const WORKER_VIEW_CATALOG_EVENTS = ["component.installed"] as const; +// why: a worker's `update` and `lastUpdateFailure` change on these two events (ADR 0042 §2), so a gateway reads its status again at once. +export const WORKER_VIEW_UPDATE_EVENTS = ["daemon.update-started", "daemon.update-failed"] as const; + type Output = z.infer<(typeof OPERATIONS)[Name]["output"]>; type WorkerView = z.infer; diff --git a/src/gateway/worker-link.ts b/src/gateway/worker-link.ts index c1cc3598..c37e0368 100644 --- a/src/gateway/worker-link.ts +++ b/src/gateway/worker-link.ts @@ -20,6 +20,7 @@ import { grantedDeviceSchema, isSimlockError, WORKER_VIEW_CATALOG_EVENTS, + WORKER_VIEW_UPDATE_EVENTS, workerViewFields, } from "../contract/index.js"; import type { StatusGetOutput } from "../admin/index.js"; @@ -603,7 +604,12 @@ const GATEWAY_OWN_EVENTS: ReadonlySet = new Set([ * progress, and an install that starts or ends changes that list. */ function changesCapacityOrLeases(event: string): boolean { - return event.startsWith("lease.") || event.startsWith("device.") || INSTALL_EVENTS.has(event); + return ( + event.startsWith("lease.") || + event.startsWith("device.") || + INSTALL_EVENTS.has(event) || + (WORKER_VIEW_UPDATE_EVENTS as readonly string[]).includes(event) + ); } const INSTALL_EVENTS: ReadonlySet = new Set([ From aa3b427ef0e9328f867baab2743d6e3a906ef5d5 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:09:40 +0200 Subject: [PATCH 15/42] test: a DAEMON_UPDATING answer to a dispatched request is handled as NO_CAPACITY (#838) --- src/gateway/fleet-coordinator.test.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index dc1d7f6a..e79adbfe 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -1043,6 +1043,31 @@ describe("FleetLeaseCoordinator dispatch", () => { expect(coordinator.queueDepth).toBe(1); }); + it("handles a worker's DAEMON_UPDATING answer as NO_CAPACITY: the view is read again, the request stays queued and is placed once the view changes", async () => { + const { coordinator, directory, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + error: new SimlockError("DAEMON_UPDATING", "domain", "The daemon is updating", {}), + kind: "error", + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const outcome = coordinator.request(REQUEST, requestOptions()); + const state = promiseState(outcome); + await tick(); + + expect(state.state).toBe("pending"); + expect(coordinator.queueDepth).toBe(1); + expect(directory.refreshCalls).toEqual(["wrk_a"]); + + workers.refresh("wrk_a", { update: null }); + await tick(); + + expect((await outcome).lease.worker?.id).toBe("wrk_a"); + }); + it("forwards device.exec with the namespaced requesterId reaching the scripted worker client", async () => { const { coordinator, directory, workers } = harness(); const client = new ScriptedWorkerClient(); From e1f84507d9ae1f73199b05911376ad26b1c76485 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:11:11 +0200 Subject: [PATCH 16/42] feat(gateway): a DAEMON_UPDATING refusal re-queues the request like NO_CAPACITY (#838) --- src/gateway/fleet-coordinator.test.ts | 6 ++++++ src/gateway/fleet-coordinator.ts | 5 ++++- src/gateway/routing/view-state.ts | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index e79adbfe..e85243a8 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -1062,6 +1062,12 @@ describe("FleetLeaseCoordinator dispatch", () => { expect(coordinator.queueDepth).toBe(1); expect(directory.refreshCalls).toEqual(["wrk_a"]); + workers.refresh("wrk_a", { + update: { forced: false, from: "0.3.0", stage: "installing", startedAt: 1, to: "0.4.0" }, + }); + await tick(); + expect(state.state).toBe("pending"); + workers.refresh("wrk_a", { update: null }); await tick(); diff --git a/src/gateway/fleet-coordinator.ts b/src/gateway/fleet-coordinator.ts index e379a8ea..787580c9 100644 --- a/src/gateway/fleet-coordinator.ts +++ b/src/gateway/fleet-coordinator.ts @@ -122,6 +122,9 @@ const COULD_TAKE_LATER: ReadonlySet = new Set([ "updating", ]); +// why: ADR 0042 §3: a worker that began its update after the gateway's last read refuses with `DAEMON_UPDATING`, and its view is as stale as after `NO_CAPACITY`. +const STALE_VIEW_CODES: ReadonlySet = new Set(["NO_CAPACITY", "DAEMON_UPDATING"]); + const CANNOT_SERVE_CODES: ReadonlySet = new Set([ "UNKNOWN_MODEL", "RUNTIME_MISSING", @@ -1051,7 +1054,7 @@ export class FleetLeaseCoordinator { // that is this request's own terminal failure -- not a stale view to retry, because device // work already started means the request was this worker's (§11's own wording). Re-queuing // it anyway silently reversed a dispatch the caller had already been told about. - if (isSimlockError(error) && error.code === "NO_CAPACITY" && !announced) { + if (isSimlockError(error) && STALE_VIEW_CODES.has(error.code) && !announced) { this.#rememberRefusal(waiter, workerId, { capacityReason: refusedCapacityReason(error.details), }); diff --git a/src/gateway/routing/view-state.ts b/src/gateway/routing/view-state.ts index 1b3b01dc..a0db24b3 100644 --- a/src/gateway/routing/view-state.ts +++ b/src/gateway/routing/view-state.ts @@ -12,6 +12,7 @@ export function viewLoadKey(view: WorkerView): string { leases: (view.leases ?? []).map((lease) => lease.id), physicalQueueDepth: view.physicalQueueDepth, queueDepth: view.queueDepth, + updating: view.update != null, }); } From c9f58b929cf85cc68a77a73d723c94c4e3504437 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:11:34 +0200 Subject: [PATCH 17/42] feat(gateway): a DAEMON_UPDATING refusal re-queues the request like NO_CAPACITY (#838) --- src/gateway/routing/view-state.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gateway/routing/view-state.ts b/src/gateway/routing/view-state.ts index a0db24b3..87b5906b 100644 --- a/src/gateway/routing/view-state.ts +++ b/src/gateway/routing/view-state.ts @@ -12,7 +12,7 @@ export function viewLoadKey(view: WorkerView): string { leases: (view.leases ?? []).map((lease) => lease.id), physicalQueueDepth: view.physicalQueueDepth, queueDepth: view.queueDepth, - updating: view.update != null, + ...(view.update == null ? {} : { updating: true }), }); } From 34fa696a4462d974565f477582093f469155428a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:12:00 +0200 Subject: [PATCH 18/42] test: an operator-drained worker stays drained through its update (#838) --- src/gateway/dispatcher.test.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index ed9077a6..2ab7a7c5 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -654,6 +654,32 @@ describe("GatewayDispatcher", () => { } }); + it("an operator-drained worker stays drained through its update, and a drain during an update leaves the update alone", async () => { + const { dispatcher, directory, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + const update = { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + await dispatcher.dispatch("worker.drain", { workerId: "wrk_1" }, session()); + client.updateAnswer = { ...client.updateAnswer, started: true, update }; + + await dispatcher.dispatch("worker.update", { workerId: "wrk_1" }, session()); + expect(workers.view("wrk_1")).toMatchObject({ drained: true, update }); + workers.refresh("wrk_1", { update: null }); + expect(workers.view("wrk_1")).toMatchObject({ drained: true, update: null }); + + await dispatcher.dispatch("worker.undrain", { workerId: "wrk_1" }, session()); + workers.refresh("wrk_1", { update }); + await dispatcher.dispatch("worker.drain", { workerId: "wrk_1" }, session()); + expect(workers.view("wrk_1")).toMatchObject({ drained: true, update }); + }); + it("refuses to remove a connected worker with WORKER_CONNECTED, naming it", async () => { const { dispatcher, workers } = harness(); workers.connected("wrk_1", undefined, undefined); From de69942965abf1e6c8ecd5120b708bda3dfbb238 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:12:31 +0200 Subject: [PATCH 19/42] test: POST /v1/workers/{id}/update answers 202, 200 and 403 (#838) --- src/http/app.test.ts | 57 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/http/app.test.ts b/src/http/app.test.ts index 976ea182..1dfa51d3 100644 --- a/src/http/app.test.ts +++ b/src/http/app.test.ts @@ -2629,6 +2629,63 @@ describe("POST /v1/update", () => { }); }); +describe("POST /v1/workers/{id}/update", () => { + const answer = (overrides: Record = {}) => ({ + started: false, + upToDate: false, + resolved: null, + version: "0.3.0", + health: "running", + update: null, + lastUpdateFailure: null, + latestVersion: null, + workerId: "wrk_1", + ...overrides, + }); + + it("answers 202 when the call began the update and 200 when it followed one, dispatches worker.update with the id from the path and the body, refuses an agent and a bad body", async () => { + const { app, dispatcher } = buildHarness({ config: testConfig({}, "gateway") }); + let reply: Record = answer({ started: true, resolved: "0.4.0" }); + dispatcher.handlers["worker.update"] = (input, session) => + runDispatch("worker.update", input, session, { + handlers: { "worker.update": () => reply }, + }); + const post = (body?: unknown, headers: Record = operatorAuth) => + Promise.resolve( + app.request("/v1/workers/wrk_1/update", { + headers: { + ...headers, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + method: "POST", + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }), + ); + + const started = await post({ version: "0.4.0", force: true }); + expect(started.status).toBe(202); + expect(await started.json()).toEqual(reply); + expect(dispatcher.calls.map(({ operation, input }) => ({ operation, input }))).toEqual([ + { operation: "worker.update", input: { force: true, version: "0.4.0", workerId: "wrk_1" } }, + ]); + + reply = answer({ upToDate: true, resolved: "0.3.0" }); + const following = await post(); + expect(following.status).toBe(200); + expect(dispatcher.calls.at(-1)?.input).toEqual({ workerId: "wrk_1" }); + + const asAgent = await post({}, agentAuth); + expect(asAgent.status).toBe(403); + + const callsBefore = dispatcher.calls.length; + for (const body of [{ version: "latest" }, { query: true }, { force: "yes" }]) { + const refused = await post(body); + expect({ body, status: refused.status }).toEqual({ body, status: 400 }); + } + expect(dispatcher.calls).toHaveLength(callsBefore); + }); +}); + describe("worker routes", () => { function gatewayHarness() { return buildHarness({ config: testConfig({}, "gateway") }); From 9176731d2a94b48073a9298e5346f4f721b95804 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:13:17 +0200 Subject: [PATCH 20/42] feat(http): POST /v1/workers/{id}/update (#838) --- src/http/admin-declarations.ts | 21 +++++++++++++++++++++ src/http/app.ts | 10 ++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/http/admin-declarations.ts b/src/http/admin-declarations.ts index 7257dbde..e0ea1343 100644 --- a/src/http/admin-declarations.ts +++ b/src/http/admin-declarations.ts @@ -78,6 +78,26 @@ export const updateRoute = declareRoute({ }, }); +export const workerUpdateRoute = declareRoute({ + method: "post", + path: "/v1/workers/{id}/update", + tag: "Workers", + summary: "Update one worker to a Simlock version", + role: "admin", + notes: + "The body is optional, as for `POST /v1/update`. The gateway asks the worker to update; the worker takes no new leases while it waits for its leases to end. 202 when this call began the update; 200 when it followed one that was running, or nothing needed updating (`upToDate`). A worker that shows `incompatible` can be updated too. The gateway's drain set is not touched. Gateway only: a worker answers 501 UNSUPPORTED_IN_WORKER_MODE.", + request: { params: workerParamsSchema, optionalJson: updateBodySchema }, + reply: OPERATIONS["worker.update"].output, + status: 202, + alsoStatus: 200, + errors: { + 404: ["UNKNOWN_WORKER"], + 409: ["WORKER_NOT_CONNECTED", "UPDATE_UNSUPPORTED_INSTALL", "UPDATE_SHARED_INSTALL"], + 502: ["UPDATE_RESOLVE_FAILED"], + 501: ["UNSUPPORTED_IN_WORKER_MODE"], + }, +}); + export const componentListRoute = declareRoute({ method: "get", path: "/v1/components", @@ -205,6 +225,7 @@ export const ADMIN_ROUTES = [ workerListRoute, workerDrainRoute, workerUndrainRoute, + workerUpdateRoute, workerRemoveRoute, componentListRoute, componentRemoveRoute, diff --git a/src/http/app.ts b/src/http/app.ts index ad6b2e32..30159f4c 100644 --- a/src/http/app.ts +++ b/src/http/app.ts @@ -28,6 +28,7 @@ import { workerListRoute, workerRemoveRoute, workerUndrainRoute, + workerUpdateRoute, } from "./admin-declarations.js"; import { badRequest, @@ -645,6 +646,15 @@ export function createHttpApp(deps: HttpGatewayDeps): OpenAPIHono & HttpApp return c.json(update, update.started ? 202 : 200); }); + app.openapi({ ...workerUpdateRoute, middleware: [agentAuth, emptyBodyAsNone] }, async (c) => { + const update = await deps.dispatch( + "worker.update", + { ...c.req.valid("json"), workerId: c.req.valid("param").id }, + buildHttpSession(c.get("identity")), + ); + return c.json(update, update.started ? 202 : 200); + }); + app.openapi({ ...workerDrainRoute, middleware: [agentAuth] }, async (c) => c.json( await deps.dispatch( From 3e86d6f5588ee1062cab4cfa5bef435d8f00b250 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:17:16 +0200 Subject: [PATCH 21/42] test: worker.update gives up on a worker that does not answer within 90 s (#838) --- src/gateway/dispatcher.test.ts | 26 ++++++++++++++++++++++++++ src/gateway/test-support.ts | 1 + 2 files changed, 27 insertions(+) diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 2ab7a7c5..c1dd5c0b 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -654,6 +654,32 @@ describe("GatewayDispatcher", () => { } }); + it("worker.update gives up on a worker that does not answer within 90 s with WORKER_NOT_CONNECTED", async () => { + const { clock, dispatcher, directory, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + const client = new ScriptedWorkerClient(); + client.hangingCalls.add("daemon.update"); + directory.add("wrk_1", client); + + let outcome: unknown = "pending"; + void dispatcher.dispatch("worker.update", { workerId: "wrk_1" }, session()).then( + (answer) => (outcome = answer), + (error: unknown) => (outcome = error), + ); + await vi.waitFor(() => expect(client.updateInputs).toHaveLength(1)); + clock.advance(89_999); + await Promise.resolve(); + expect(outcome).toBe("pending"); + clock.advance(1); + + await vi.waitFor(() => expect(outcome).not.toBe("pending")); + expect(outcome).toMatchObject({ + code: "WORKER_NOT_CONNECTED", + details: { workerId: "wrk_1" }, + message: "worker wrk_1 did not answer within 90 s; check simlock worker list", + }); + }); + it("an operator-drained worker stays drained through its update, and a drain during an update leaves the update alone", async () => { const { dispatcher, directory, workers } = harness(); workers.connected("wrk_1", "mac-mini-1", "0.3.0"); diff --git a/src/gateway/test-support.ts b/src/gateway/test-support.ts index 24de3108..191de632 100644 --- a/src/gateway/test-support.ts +++ b/src/gateway/test-support.ts @@ -420,6 +420,7 @@ export class ScriptedWorkerClient { async updateDaemon(input: DaemonUpdateInput = {}): Promise { this.calls.push("daemon.update"); this.updateInputs.push(input); + if (this.hangingCalls.has("daemon.update")) return new Promise(() => {}); if (this.updateFailWith !== undefined) throw this.updateFailWith; return this.updateAnswer; } From 566e1001f880761c62337afefd87f94278151f70 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:17:25 +0200 Subject: [PATCH 22/42] feat(gateway): worker.update is bounded at 90 s on the uplink (#838) --- src/gateway/dispatcher.ts | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 1fd9c70e..19d8fa1d 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -193,6 +193,8 @@ function alertTestError( } } +const UPDATE_UPLINK_BOUND_MS = 90_000; + export class GatewayDispatcher { readonly #logger: Logger; readonly #dispatchLogger: Logger; @@ -427,7 +429,7 @@ export class GatewayDispatcher { }); } const { workerId, ...request } = input; - const answer = await client.updateDaemon(request); + const answer = await this.#withinUplinkBound(client.updateDaemon(request), workerId); this.options.workers.refresh(workerId, { lastUpdateFailure: answer.lastUpdateFailure, latestVersion: answer.latestVersion, @@ -437,6 +439,31 @@ export class GatewayDispatcher { return { ...answer, workerId }; }; + // why: above the worker's own npm `view` and `root -g` (30 s each); at the limit the view catches up on the next read (ADR 0042 §2). + #withinUplinkBound(call: Promise, workerId: string): Promise { + return new Promise((resolve, reject) => { + const timer = this.options.clock.setTimer(UPDATE_UPLINK_BOUND_MS, () => { + reject( + new DispatchError( + "WORKER_NOT_CONNECTED", + `worker ${workerId} did not answer within 90 s; check simlock worker list`, + { workerId }, + ), + ); + }); + call.then( + (value) => { + this.options.clock.cancel(timer); + resolve(value); + }, + (error: unknown) => { + this.options.clock.cancel(timer); + reject(error); + }, + ); + }); + } + #alertsTest: Handler<"alerts.test"> = async (input) => { const { destination } = input; const result = (await this.options.testDestination?.(destination)) ?? UNKNOWN_DESTINATION; From 18dcc2e73bf1a97259ba24aa1170148e63b87086 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:22:45 +0200 Subject: [PATCH 23/42] test: worker.update reaches an incompatible worker with the frozen daemon.update (#838) --- src/daemon/gateway-fleet.e2e.test.ts | 49 +++++++++++++++++++++++++++- src/daemon/main.ts | 5 ++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/daemon/gateway-fleet.e2e.test.ts b/src/daemon/gateway-fleet.e2e.test.ts index 6cef9ad5..7553ae39 100644 --- a/src/daemon/gateway-fleet.e2e.test.ts +++ b/src/daemon/gateway-fleet.e2e.test.ts @@ -30,11 +30,13 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ProtocolRange } from "../contract/index.js"; import type { AlertDestination, PassthroughCommand } from "../core/index.js"; import { FakeDriver, fakeDeviceDataTables } from "../core/testing.js"; -import type { Filesystem, IpcConnector, IpcListenerFactory } from "../ports/index.js"; +import type { Filesystem, IpcConnector, IpcListenerFactory, NpmCli } from "../ports/index.js"; import { FakeHostInfo, + FakeNpmCli, MemoryFilesystem, NodeIpcTransport, NoopLogger, @@ -207,6 +209,9 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { readonly ipc?: IpcConnector & IpcListenerFactory; readonly alerts?: { destinations: readonly AlertDestination[]; poster: FakeWebhookPoster }; readonly onDriver?: (driver: FakeDriver) => void; + /** why: a worker on another protocol range is the gateway's `incompatible` worker. */ + readonly protocolVersion?: ProtocolRange; + readonly npm?: NpmCli; }): Promise<{ daemon: DaemonServer; filesystem: Filesystem; directory: string }> { const directory = options.existing?.directory ?? @@ -245,6 +250,10 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { filesystem, ...(options.ipc === undefined ? {} : { ipc: options.ipc }), logger: new NoopLogger(), + ...(options.npm === undefined ? {} : { npm: options.npm }), + ...(options.protocolVersion === undefined + ? {} + : { protocolVersion: options.protocolVersion }), version: "1.0.0-e2e", ...(options.alerts === undefined ? {} : { webhookPoster: options.alerts.poster }), }); @@ -252,6 +261,44 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { return { daemon, directory, filesystem }; } + it("worker.update reaches an incompatible worker with the frozen daemon.update and shows its answer", async () => { + const gateway = await startGateway(); + const { secret: token } = await gateway.dispatch( + "token.create", + { role: "worker", label: "old-worker" }, + adminSession(), + ); + await startWorker({ + label: "old-worker", + model: "Pixel-A", + // why: a version already on npm's latest answers `upToDate`, so no install runs in this test. + npm: new FakeNpmCli({ view: [{ ok: true, output: "0.4.0" }] }), + protocolVersion: { max: 55, min: 55 }, + stdout: "old", + token, + }); + const incompatible = await vi.waitFor(async () => { + const { workers } = await gateway.dispatch("worker.list", {}, adminSession()); + const found = workers.find((worker) => worker.connection === "incompatible"); + if (found === undefined) throw new Error("the old worker never showed incompatible"); + return found; + }); + + const answer = await gateway.dispatch( + "worker.update", + { workerId: incompatible.id }, + adminSession(), + ); + + expect(answer).toMatchObject({ started: false, upToDate: true, workerId: incompatible.id }); + const { workers } = await gateway.dispatch("worker.list", {}, adminSession()); + expect(workers.find((worker) => worker.id === incompatible.id)).toMatchObject({ + connection: "incompatible", + latestVersion: "0.4.0", + update: null, + }); + }); + it("emits queue.changed once when the gateway has started, with depth 0, and no capacity.changed of its own", async () => { const gateway = await startGateway(); diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 69aa873f..2d608222 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -4,7 +4,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { type Alerts, createAlerts } from "../alerts/index.js"; import { EventBus, type EventBusLogger } from "../bus/index.js"; -import { fitHostFacts, fitHostLoad } from "../contract/index.js"; +import { fitHostFacts, fitHostLoad, type ProtocolRange } from "../contract/index.js"; import { HostLoadSampler } from "../host-load/index.js"; import { createEventHistory, @@ -175,6 +175,8 @@ export interface StartDaemonOptions { /** why: test-only, replaces the npm CLI the version check and the update coordinator run. */ readonly npm?: NpmCli; readonly processRunner?: ProcessRunner; + /** why: test-only, the protocol range the worker daemon announces. */ + readonly protocolVersion?: number | ProtocolRange; readonly processSupervisor?: ProcessSupervisor; readonly socketPath?: string; readonly systemStats?: SystemStats; @@ -605,6 +607,7 @@ async function composeDaemon( }), leases: leasing, logger: logger.child("server"), + ...(options.protocolVersion === undefined ? {} : { protocolVersion: options.protocolVersion }), passthrough: core.catalog, processRunner, execEnv: process.env, From a8bbbdf6db934f843b41a42f97849c784dbc6a31 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:23:26 +0200 Subject: [PATCH 24/42] test: the protocol version is 57 (#838) --- src/daemon-protocol/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon-protocol/index.test.ts b/src/daemon-protocol/index.test.ts index 15ddebee..ecd7507c 100644 --- a/src/daemon-protocol/index.test.ts +++ b/src/daemon-protocol/index.test.ts @@ -9,7 +9,7 @@ import { describe("daemon protocol", () => { it("keeps the current protocol version and newline framing", () => { - expect(DAEMON_PROTOCOL_VERSION).toBe(56); + expect(DAEMON_PROTOCOL_VERSION).toBe(57); expect(serializeFrame({ id: 1, type: "hello" })).toBe('{"id":1,"type":"hello"}\n'); }); From 480bd3e4ac36a4da3055ab43197a8459300f396b Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:23:43 +0200 Subject: [PATCH 25/42] feat(core): protocol 57 for worker.update and the worker view's update fields (#838) --- src/contract/AGENTS.md | 2 +- src/contract/protocol.ts | 2 +- src/daemon/AGENTS.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/contract/AGENTS.md b/src/contract/AGENTS.md index 4852d97e..a3333fe7 100644 --- a/src/contract/AGENTS.md +++ b/src/contract/AGENTS.md @@ -56,7 +56,7 @@ Three identities: ## Protocol version -- `PROTOCOL_VERSION_RANGE` in `protocol.ts` holds it, now `{ min: 56, max: 56 }`. `DAEMON_PROTOCOL_VERSION` (`src/daemon-protocol/`) is its `max`. +- `PROTOCOL_VERSION_RANGE` in `protocol.ts` holds it, now `{ min: 57, max: 57 }`. `DAEMON_PROTOCOL_VERSION` (`src/daemon-protocol/`) is its `max`. - Every change to a contract shape bumps it by one in the same PR. Move `min` too, unless a compatibility path stays side by side. `.agents/scripts/protocol-bump-check.mjs` checks it on push and in CI; it counts every file here as a shape but tests, `index.ts` and the diff --git a/src/contract/protocol.ts b/src/contract/protocol.ts index d6da8cbf..fff1f508 100644 --- a/src/contract/protocol.ts +++ b/src/contract/protocol.ts @@ -10,7 +10,7 @@ export interface ProtocolRange { const protocolRangeSchema = z.object({ min: z.number().int(), max: z.number().int() }); /** Widen only once a second version is kept alive side by side with the first; a bump that leaves no shim behind moves both ends. */ -export const PROTOCOL_VERSION_RANGE: ProtocolRange = { min: 56, max: 56 }; +export const PROTOCOL_VERSION_RANGE: ProtocolRange = { min: 57, max: 57 }; const LEGACY_DAEMON_PROTOCOL_VERSION = 2; diff --git a/src/daemon/AGENTS.md b/src/daemon/AGENTS.md index 59f16ef7..2ec33cda 100644 --- a/src/daemon/AGENTS.md +++ b/src/daemon/AGENTS.md @@ -58,7 +58,7 @@ Its stdout lines move the state; a failed line, a non-zero exit or a failed laun 2. **Hello** — first frame must be `hello`, else `HANDSHAKE_REQUIRED` and close. Role: the uplink connection is `admin` by acceptance; otherwise the resolver. Principal: `hello.principal`, else `SIMLOCK_AGENT_ID`, else the daemon's pid. Both are fixed for the connection's life. 3. **Version** — ranges overlap → highest common version. `PROTOCOL_VERSION_RANGE` - is `{ min: 56, max: 56 }` (`src/contract/protocol.ts`). No overlap → the + is `{ min: 57, max: 57 }` (`src/contract/protocol.ts`). No overlap → the connection stays open, and every request but `daemon.stop` gets `PROTOCOL_VERSION_UNSUPPORTED`. 4. **Request** — one frame → one `dispatch()` call. While stopping: `DAEMON_STOPPING`. From 0d9d3853767edcda4409b0c0954ed10c995f3e2a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:25:34 +0200 Subject: [PATCH 26/42] feat(client): updateWorker asks a gateway to update one worker (#838) --- src/admin/index.ts | 2 ++ src/cli/index.test.ts | 12 ++++++++++++ src/simlock-client/client.test.ts | 30 ++++++++++++++++++++++++++++++ src/simlock-client/client.ts | 14 ++++++++++++-- src/simlock-client/types.ts | 2 ++ 5 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/admin/index.ts b/src/admin/index.ts index 1715d341..f8ff19c9 100644 --- a/src/admin/index.ts +++ b/src/admin/index.ts @@ -84,6 +84,8 @@ export type { WorkerListOutput, WorkerRemoveOutput, WorkerUndrainOutput, + WorkerUpdateInput, + WorkerUpdateOutput, WorkerView, } from "../simlock-client/client.js"; export { isSimlockError, SimlockError } from "../simlock-client/client.js"; diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 7b75f3fb..9ca7c40f 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -7724,6 +7724,18 @@ function fakeClient(overrides: Partial = {}): SimlockAdminCl undrainWorker: (input) => Promise.resolve({ drained: false as const, workerId: input.workerId }), removeWorker: (input) => Promise.resolve({ removed: true, workerId: input.workerId }), + updateWorker: (input) => + Promise.resolve({ + health: "running" as const, + lastUpdateFailure: null, + latestVersion: null, + resolved: null, + started: false, + update: null, + upToDate: false, + version: "0.3.0", + workerId: input.workerId, + }), testAlertDestination: (input) => Promise.resolve({ destination: input.destination, status: 200 }), installComponent: (input) => diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index d1402351..e9e3b9d2 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -268,6 +268,36 @@ describe("connectSimlock: handshake", () => { }); }); + it("the admin client's updateWorker sends worker.update and returns its output, and refuses an empty worker id before sending", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); + await flushMicrotasks(); + completeHello(connection, { role: "admin" }); + const client = await connectPromise; + const answer = { + health: "running", + lastUpdateFailure: null, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: null, + upToDate: false, + version: "0.3.0", + workerId: "wrk_1", + }; + + const callPromise = client.updateWorker({ force: true, version: "0.4.0", workerId: "wrk_1" }); + await flushMicrotasks(); + const call = connection.lastSentOf("worker.update"); + expect(call?.payload).toEqual({ force: true, version: "0.4.0", workerId: "wrk_1" }); + connection.reply(call!.id, answer); + + await expect(callPromise).resolves.toEqual(answer); + await expect(client.updateWorker({ workerId: "" })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + }); + it("enrolls a device from an admin client with device.add and removes it with device.remove, sending the input and returning the daemon's answer", async () => { const connection = new ScriptedConnection(); const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); diff --git a/src/simlock-client/client.ts b/src/simlock-client/client.ts index b15c895f..81bdef40 100644 --- a/src/simlock-client/client.ts +++ b/src/simlock-client/client.ts @@ -94,6 +94,8 @@ import type { WorkerListOutput, WorkerRemoveOutput, WorkerUndrainOutput, + WorkerUpdateInput, + WorkerUpdateOutput, } from "./types.js"; export type { @@ -167,6 +169,8 @@ export type { WorkerListOutput, WorkerRemoveOutput, WorkerUndrainOutput, + WorkerUpdateInput, + WorkerUpdateOutput, WorkerView, AlertsTestInput, AlertsTestOutput, @@ -268,12 +272,13 @@ export interface SimlockAdminClient extends SimlockClient { listTokens(): Promise; revokeToken(input: TokenRevokeInput): Promise; - /** ADR 0005 §23. A worker lists itself as its only worker; `drainWorker`, `undrainWorker` and - * `removeWorker` reject on a worker with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012). */ + /** why: ADR 0005 §23. A worker lists itself as its only worker; `drainWorker`, `undrainWorker`, `removeWorker` and + * `updateWorker` (a gateway updating one of its workers, ADR 0042 §2) reject on a worker with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012). */ listWorkers(): Promise; drainWorker(input: WorkerDrainInput): Promise; undrainWorker(input: WorkerDrainInput): Promise; removeWorker(input: WorkerDrainInput): Promise; + updateWorker(input: WorkerUpdateInput): Promise; /** * why: ADR 0035 §1: asks a gateway to post one sample message to the named alert destination, once. @@ -447,6 +452,7 @@ function buildDegradedClient( drainWorker: () => rejected(), undrainWorker: () => rejected(), removeWorker: () => rejected(), + updateWorker: () => rejected(), testAlertDestination: () => rejected(), installComponent: () => rejected(), installComponentOnWorkers: () => rejected(), @@ -724,6 +730,10 @@ class SimlockClientImpl { return this.#call("worker.undrain", input); } + updateWorker(input: WorkerUpdateInput): Promise { + return this.#call("worker.update", input); + } + testAlertDestination(input: AlertsTestInput): Promise { return this.#call("alerts.test", input); } diff --git a/src/simlock-client/types.ts b/src/simlock-client/types.ts index f91b733e..705de317 100644 --- a/src/simlock-client/types.ts +++ b/src/simlock-client/types.ts @@ -83,6 +83,8 @@ export type WorkerDrainInput = OpInput<"worker.drain">; export type WorkerDrainOutput = OpOutput<"worker.drain">; export type WorkerUndrainOutput = OpOutput<"worker.undrain">; export type WorkerRemoveOutput = OpOutput<"worker.remove">; +export type WorkerUpdateInput = OpInput<"worker.update">; +export type WorkerUpdateOutput = OpOutput<"worker.update">; export type AlertsTestInput = OpInput<"alerts.test">; export type AlertsTestOutput = OpOutput<"alerts.test">; From 830f083286c11abe31af887ec0ccaf6adb96456c Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:28:27 +0200 Subject: [PATCH 27/42] test: daemon.update on a gateway installs at once with fleet leases held (#838) --- src/daemon/gateway-fleet.e2e.test.ts | 45 +++++++++++++++++++++++++++- src/daemon/main.ts | 7 +++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/daemon/gateway-fleet.e2e.test.ts b/src/daemon/gateway-fleet.e2e.test.ts index 7553ae39..df2d95a7 100644 --- a/src/daemon/gateway-fleet.e2e.test.ts +++ b/src/daemon/gateway-fleet.e2e.test.ts @@ -37,6 +37,7 @@ import type { Filesystem, IpcConnector, IpcListenerFactory, NpmCli } from "../po import { FakeHostInfo, FakeNpmCli, + FakeUpdaterLauncher, MemoryFilesystem, NodeIpcTransport, NoopLogger, @@ -46,7 +47,7 @@ import { import { FakeWebhookPoster } from "../ports/testing.js"; import { connectSimlockAdmin } from "../admin/index.js"; import type { DispatchSession } from "./dispatch.js"; -import { startDaemon } from "./main.js"; +import { startDaemon, type StartDaemonOptions } from "./main.js"; import type { DaemonServer } from "./server.js"; function passthroughOptions(command: PassthroughCommand): object { @@ -136,6 +137,7 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { async function startGateway( alerts: { destinations?: readonly AlertDestination[]; poster?: FakeWebhookPoster } = {}, + extra: Partial = {}, ): Promise { const directory = await mkdtemp(join(tmpdir(), "simlock-e2e-gateway-")); directories.push(directory); @@ -154,6 +156,7 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { logger: new NoopLogger(), version: "1.0.0-e2e", ...(alerts.poster === undefined ? {} : { webhookPoster: alerts.poster }), + ...extra, }); daemons.push(daemon); return daemon; @@ -299,6 +302,46 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { }); }); + it("daemon.update on a gateway goes straight to installing with fleet leases held, and its status.get shows it", async () => { + const launcher = new FakeUpdaterLauncher(); + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/opt/simlock"); + const gateway = await startGateway( + {}, + { + filesystem, + npm: new FakeNpmCli({ + globalRoot: [{ ok: true, output: "/opt" }], + view: [{ ok: true, output: "2.0.0" }], + }), + updater: { launcher, otherRoleRunning: async () => false, packageRoot: "/opt/simlock" }, + }, + ); + const { secret: token } = await gateway.dispatch( + "token.create", + { role: "worker", label: "worker-a" }, + adminSession(), + ); + await startWorker({ label: "worker-a", model: "Pixel-A", stdout: "a", token }); + await vi.waitFor(async () => { + const { workers } = await gateway.dispatch("worker.list", {}, adminSession()); + expect(workerServes(workers, "Pixel-A")).toBe(true); + }); + await gateway.dispatch( + "lease.request", + { model: "Pixel-A", noWait: true, platform: "android" }, + agentSession(), + ); + + const answer = await gateway.dispatch("daemon.update", {}, adminSession()); + + const installing = { forced: false, from: "1.0.0-e2e", stage: "installing", to: "2.0.0" }; + expect(answer).toMatchObject({ resolved: "2.0.0", started: true, update: installing }); + expect(launcher.launches).toMatchObject([{ from: "1.0.0-e2e", role: "gateway", to: "2.0.0" }]); + const status = await gateway.dispatch("status.get", {}, adminSession()); + expect(status.daemon.update).toMatchObject(installing); + }); + it("emits queue.changed once when the gateway has started, with depth 0, and no capacity.changed of its own", async () => { const gateway = await startGateway(); diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 2d608222..b9f1dcbe 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -98,6 +98,7 @@ import { NodeProcessSupervisor, NodeSystemStats, NodeUpdaterLauncher, + type UpdaterLauncher, NodeBytePipeLauncher, NodeTcpProbe, NodeTcpStreamConnector, @@ -175,6 +176,12 @@ export interface StartDaemonOptions { /** why: test-only, replaces the npm CLI the version check and the update coordinator run. */ readonly npm?: NpmCli; readonly processRunner?: ProcessRunner; + /** why: test-only, replaces the launcher, package root and shared-install check of the gateway's update coordinator. */ + readonly updater?: { + readonly launcher: UpdaterLauncher; + readonly otherRoleRunning: () => Promise; + readonly packageRoot: string; + }; /** why: test-only, the protocol range the worker daemon announces. */ readonly protocolVersion?: number | ProtocolRange; readonly processSupervisor?: ProcessSupervisor; From 4a91b10744408ccb2b17b32271fec75b53477e86 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:31:39 +0200 Subject: [PATCH 28/42] feat(daemon): a gateway updates itself with no lease wait (#838) --- src/daemon/main.ts | 97 +++++++++++++++++++++++++------- src/daemon/update-coordinator.ts | 3 +- src/gateway/aggregate.ts | 6 +- src/gateway/dispatcher.test.ts | 62 +++++++++++++++++--- src/gateway/dispatcher.ts | 27 ++++++--- 5 files changed, 155 insertions(+), 40 deletions(-) diff --git a/src/daemon/main.ts b/src/daemon/main.ts index b9f1dcbe..4855cef0 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -73,6 +73,7 @@ import { LoggingProcessRunner, NodeFileLogSink, type Clock, + type DaemonRole, type Filesystem, type HostInfo, type HostLoadProbe, @@ -136,7 +137,7 @@ import { createModuleHost, type OptionalModuleDefinition } from "../modules/inde import { optionalModules } from "../module-registry/index.js"; import { DaemonServer } from "./server.js"; import { socketHeld } from "../updater/index.js"; -import { createUpdateCoordinator } from "./update-coordinator.js"; +import { createUpdateCoordinator, type UpdateCoordinatorOptions } from "./update-coordinator.js"; import { startVersionCheck, type VersionCheck } from "./version-check.js"; import { DaemonAlreadyRunningError, @@ -193,6 +194,47 @@ export interface StartDaemonOptions { readonly webhookPoster?: WebhookPoster; } +interface UpdateOptionsParts { + readonly clock: Clock; + readonly dataDirectory: string; + readonly eventBus: EventBus; + readonly filesystem: Filesystem; + readonly ipc: IpcConnector & IpcListenerFactory; + readonly npm: NpmCli; + readonly role: DaemonRole; + readonly version: string; + readonly versionCheck: VersionCheck; +} + +// why: both roles share these; each adds what it holds leases with. +function updateOptionsFor( + parts: UpdateOptionsParts, +): Omit { + const { dataDirectory, ipc, role } = parts; + return { + clock: parts.clock, + dataDirectory, + eventBus: parts.eventBus, + filesystem: parts.filesystem, + latestVersion: () => parts.versionCheck.latest(), + launcher: new NodeUpdaterLauncher(), + logPath: join(dataDirectory, "daemon.log"), + npm: parts.npm, + otherRoleRunning: () => + socketHeld( + ipc, + resolveDaemonSocketPath( + daemonDataDirectory(resolveSimlockHome(), role === "worker" ? "gateway" : "worker"), + ), + ), + packageRoot: fileURLToPath(new URL("../../", import.meta.url)), + pid: process.pid, + role, + simlockHome: resolveSimlockHome(), + version: parts.version, + }; +} + /** Constructs the daemon's real adapters once; all state remains in the daemon. */ export async function startDaemon(options: StartDaemonOptions = {}): Promise { let opened: StateDatabase | undefined; @@ -338,9 +380,11 @@ async function composeDaemon( idGenerator, ipc, logger, + npm, optionalModules: moduleDefinitions, socketPath, state, + ...(options.updater === undefined ? {} : { updater: options.updater }), version: version, versionCheck, ...(options.webhookPoster === undefined ? {} : { webhookPoster: options.webhookPoster }), @@ -633,28 +677,19 @@ async function composeDaemon( adminSecret: auth.adminSecret, tokens: auth.tokens, update: createUpdateCoordinator({ - npm, - latestVersion: () => versionCheck.latest(), - launcher: new NodeUpdaterLauncher(), + ...updateOptionsFor({ + clock, + dataDirectory, + eventBus, + filesystem, + ipc, + npm, + role, + version, + versionCheck, + }), leasing, registry, - eventBus, - clock, - filesystem, - otherRoleRunning: () => - socketHeld( - ipc, - resolveDaemonSocketPath( - daemonDataDirectory(resolveSimlockHome(), role === "worker" ? "gateway" : "worker"), - ), - ), - version, - packageRoot: fileURLToPath(new URL("../../", import.meta.url)), - simlockHome: resolveSimlockHome(), - dataDirectory, - role, - logPath: join(dataDirectory, "daemon.log"), - pid: process.pid, }), version: version, converge: convergeStartup, @@ -820,9 +855,11 @@ interface GatewayDaemonOptions { readonly idGenerator: IdGenerator; readonly ipc: IpcConnector & IpcListenerFactory; readonly logger: Logger; + readonly npm: NpmCli; readonly optionalModules: readonly OptionalModuleDefinition[]; readonly socketPath: string; readonly state: StateDatabase; + readonly updater?: StartDaemonOptions["updater"]; readonly version: string; readonly versionCheck: VersionCheck; readonly webhookPoster?: WebhookPoster; @@ -960,6 +997,23 @@ async function startGatewayDaemon(options: GatewayDaemonOptions): Promise {}, releaseUpdateHold: async () => {} }, + registry: { snapshot: { leases: [] } }, + }); const gatewayDispatcher = new GatewayDispatcher({ awaitReady: () => Promise.resolve(), clock, @@ -976,6 +1030,7 @@ async function startGatewayDaemon(options: GatewayDaemonOptions): Promise options.versionCheck.latest(), logger: logger.child("gateway"), + update, // why: `alerts` is built after this dispatcher but before start(), and only with destinations; without any, the option stays out. ...(config.gateway.alerts.destinations.length === 0 ? {} diff --git a/src/daemon/update-coordinator.ts b/src/daemon/update-coordinator.ts index aa6a6647..57470e6d 100644 --- a/src/daemon/update-coordinator.ts +++ b/src/daemon/update-coordinator.ts @@ -9,7 +9,6 @@ import { type UpdateFailure, type UpdateState, } from "../contract/index.js"; -import type { Registry } from "../core/index.js"; import type { UpdateHold } from "../leasing/index.js"; import { checkInstall } from "../updater/index.js"; import { DispatchError } from "./dispatch.js"; @@ -53,7 +52,7 @@ export interface UpdateCoordinatorOptions { readonly latestVersion: () => string | null; readonly launcher: UpdaterLauncher; readonly leasing: UpdateHold; - readonly registry: Pick; + readonly registry: { readonly snapshot: { readonly leases: readonly unknown[] } }; readonly eventBus: Pick; readonly clock: Clock; readonly filesystem: Filesystem; diff --git a/src/gateway/aggregate.ts b/src/gateway/aggregate.ts index 32affcc6..5659fd89 100644 --- a/src/gateway/aggregate.ts +++ b/src/gateway/aggregate.ts @@ -47,6 +47,8 @@ export interface AggregateStatusOptions { readonly health: StatusOutput["daemon"]["health"]; readonly version: string; readonly latestVersion?: string | null; + // why: the gateway's own update and its last failure; none when nothing is running. + readonly progress?: Pick; readonly host: StatusOutput["host"]; readonly load?: StatusOutput["load"]; readonly queueDepth: number; @@ -70,8 +72,8 @@ export function aggregateStatus( health: options.health, mode: "gateway", version: options.version, - update: null, - lastUpdateFailure: null, + update: options.progress?.update ?? null, + lastUpdateFailure: options.progress?.lastUpdateFailure ?? null, latestVersion: options.latestVersion ?? null, }, host: options.host, diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index c1dd5c0b..53127f2b 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -29,7 +29,7 @@ import { } from "../ports/index.js"; import { FakeWebhookPoster } from "../ports/testing.js"; import type { TokenOperations } from "../auth/index.js"; -import { GatewayDispatcher } from "./dispatcher.js"; +import { GatewayDispatcher, type GatewayDispatcherOptions } from "./dispatcher.js"; import { MemoryDrainStore } from "./drain-store.js"; import { FleetLeaseCoordinator } from "./fleet-coordinator.js"; import type { WorkerDirectory, WorkerDispatchTarget } from "./fleet-ports.js"; @@ -168,6 +168,25 @@ class FakeDirectory implements WorkerDirectory { } } +// why: the gateway's own update coordinator, scripted. +class ScriptedUpdate { + readonly requests: unknown[] = []; + progress: ReturnType = { + lastUpdateFailure: null, + latestVersion: null, + update: null, + }; + + async request(input: unknown) { + this.requests.push(input); + return { ...this.progress, resolved: "0.4.0", started: true, upToDate: false }; + } + + state() { + return this.progress; + } +} + function harness( options: { readonly eventHistory?: Pick; @@ -235,9 +254,11 @@ function harness( version: "9.9.9", workerLabel: () => undefined, }); + const update = new ScriptedUpdate(); const dispatcher = new GatewayDispatcher({ awaitReady: async () => {}, latestVersion: () => null, + update, clock, closeUplinksForToken: async (tokenId) => { closedUplinkTokens.push(tokenId); @@ -271,6 +292,7 @@ function harness( eventBus, leaseIndex, tokens, + update, workers, }; } @@ -1397,16 +1419,40 @@ describe("GatewayDispatcher", () => { }); describe("daemon.update", () => { - it("answers UNSUPPORTED_IN_GATEWAY_MODE to an admin session, saying a gateway cannot update itself yet", async () => { - const { dispatcher } = harness(); + it("asks the gateway's own update coordinator and answers with its outcome, the gateway's version and its health, and status.get carries the update and its last failure", async () => { + const { dispatcher, update } = harness(); + const run = { + forced: true, + from: "0.0.0-test", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 500, + from: "0.0.0-test", + message: "npm failed", + stage: "installing", + to: "0.3.0", + } as const; + update.progress = { lastUpdateFailure: failure, latestVersion: "0.4.0", update: run }; await expect( - dispatcher.dispatch("daemon.update", {}, session({ role: "admin" })), - ).rejects.toMatchObject({ - code: "UNSUPPORTED_IN_GATEWAY_MODE", - details: { operation: "daemon.update" }, - message: "daemon.update updates one machine's daemon; a gateway cannot update itself yet", + dispatcher.dispatch("daemon.update", { force: true }, session({ role: "admin" })), + ).resolves.toEqual({ + health: "running", + lastUpdateFailure: failure, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: run, + upToDate: false, + version: "0.0.0-test", }); + expect(update.requests).toEqual([{ force: true }]); + + const status = await dispatcher.dispatch("status.get", {}, session({ role: "admin" })); + expect(status.daemon).toMatchObject({ lastUpdateFailure: failure, update: run }); }); }); diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 19d8fa1d..7596a2ba 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -93,6 +93,16 @@ export type AlertTestOutcome = readonly error?: string; }; +type DaemonUpdateOutput = z.infer<(typeof OPERATIONS)["daemon.update"]["output"]>; + +// why: ADR 0042 §4: the gateway's own update has no lease wait and goes straight to installing; `src/daemon` supplies it. +export interface GatewayUpdate { + request( + input: z.infer<(typeof OPERATIONS)["daemon.update"]["input"]>, + ): Promise>; + state(): Pick; +} + export interface GatewayDispatcherOptions { readonly historyPath?: string; /** The gateway's own config -- what `config.get` returns (ADR 0005 §34). */ @@ -126,6 +136,7 @@ export interface GatewayDispatcherOptions { readonly health: () => "starting" | "running" | "failed"; readonly version: string; readonly latestVersion: () => string | null; + readonly update: GatewayUpdate; readonly host: AggregateStatusOptions["host"]; readonly load?: () => AggregateStatusOptions["load"]; readonly awaitReady: () => Promise; @@ -237,10 +248,7 @@ export class GatewayDispatcher { "worker.remove": this.#workerRemove, "worker.update": this.#workerUpdate, "alerts.test": this.#alertsTest, - "daemon.update": unsupportedByDesign( - "daemon.update", - "daemon.update updates one machine's daemon; a gateway cannot update itself yet", - ), + "daemon.update": this.#daemonUpdate, "worker.install-component": this.#workerInstallComponent, "lease.list": this.#leaseList, "list.get": this.#listGet, @@ -329,9 +337,7 @@ export class GatewayDispatcher { health: "starting", mode: "gateway", version: this.options.version, - update: null, - lastUpdateFailure: null, - latestVersion: this.options.latestVersion(), + ...this.options.update.state(), ...consoleUrl, }, history, @@ -343,6 +349,7 @@ export class GatewayDispatcher { health: this.options.health(), version: this.options.version, latestVersion: this.options.latestVersion(), + progress: this.options.update.state(), host: this.options.host, ...(load === undefined ? {} : { load }), physicalQueueDepth: this.options.coordinator.physicalQueueDepth, @@ -414,6 +421,12 @@ export class GatewayDispatcher { return { drained: false, workerId: input.workerId }; }; + #daemonUpdate: Handler<"daemon.update"> = async (input) => ({ + ...(await this.options.update.request(input)), + health: this.options.health(), + version: this.options.version, + }); + #workerUpdate: Handler<"worker.update"> = async (input) => { const view = this.options.workers.view(input.workerId); if (view === undefined) { From 6c54b9e04802ea49bfd5fff4f33e256652e59551 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:32:19 +0200 Subject: [PATCH 29/42] test: a failed install shows in the gateway's status.get (#838) --- src/daemon/main.test.ts | 65 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index 4c4acb92..28bd1519 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -37,6 +37,7 @@ import { FakeHostInfo, FakeHostLoadProbe, FakeNpmCli, + FakeUpdaterLauncher, type NpmCli, type NpmOutcome, type HostLoadCounters, @@ -4270,3 +4271,67 @@ describe("startDaemon version check", () => { await expect.poll(() => latestOf(gateway.daemon)).toBe("0.6.0"); }); }); + +describe("startDaemon gateway update", () => { + const operator = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "test-operator", + role: "admin", + } as const; + + async function startUpdatable( + launcher: FakeUpdaterLauncher, + filesystem = new MemoryFilesystem(), + ) { + await filesystem.mkdirp("/opt/simlock"); + const directory = await mkdtemp(join(tmpdir(), "simlock-main-gw-update-")); + temporaryDirectories.push(directory); + const clock = new FakeClock(1_000); + const daemon = await startDaemon({ + clock, + configOverrides: { + http: { enabled: true, host: "127.0.0.1", port: await freeLoopbackPort() }, + }, + dataDirectory: directory, + drivers: [], + filesystem, + hostLoadProbe: null, + logger: new NoopLogger(), + npm: new FakeNpmCli({ + globalRoot: [{ ok: true, output: "/opt" }], + view: [{ ok: true, output: "0.4.0" }], + }), + role: "gateway", + updater: { launcher, otherRoleRunning: async () => false, packageRoot: "/opt/simlock" }, + version: "0.3.0", + } as StartDaemonOptions); + runningDaemons.push(daemon); + return { daemon, directory, filesystem }; + } + + const daemonOf = async (daemon: DaemonServer) => + ((await daemon.dispatch("status.get", {}, operator)) as { daemon: Record }) + .daemon; + + it("a failed install shows in the gateway's status.get as lastUpdateFailure, and the gateway keeps serving", async () => { + const launcher = new FakeUpdaterLauncher({ + exit: 1, + lines: [JSON.stringify({ failed: { message: "npm exited 1", stage: "installing" } })], + }); + const { daemon } = await startUpdatable(launcher); + + await daemon.dispatch("daemon.update", {}, operator); + + await expect + .poll(async () => (await daemonOf(daemon)).lastUpdateFailure) + .toEqual({ + at: 1_000, + from: "0.3.0", + message: "npm exited 1", + stage: "installing", + to: "0.4.0", + }); + expect(await daemonOf(daemon)).toMatchObject({ health: "running", update: null }); + }); +}); From 9f45e7bf880b5860544efdf807a36f87b15a70ee Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:33:17 +0200 Subject: [PATCH 30/42] test: a rolled-back gateway reports the failure at start (#838) --- src/daemon/main.test.ts | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index 28bd1519..9de36e75 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -4283,10 +4283,11 @@ describe("startDaemon gateway update", () => { async function startUpdatable( launcher: FakeUpdaterLauncher, filesystem = new MemoryFilesystem(), + given?: string, ) { await filesystem.mkdirp("/opt/simlock"); - const directory = await mkdtemp(join(tmpdir(), "simlock-main-gw-update-")); - temporaryDirectories.push(directory); + const directory = given ?? (await mkdtemp(join(tmpdir(), "simlock-main-gw-update-"))); + if (given === undefined) temporaryDirectories.push(directory); const clock = new FakeClock(1_000); const daemon = await startDaemon({ clock, @@ -4334,4 +4335,29 @@ describe("startDaemon gateway update", () => { }); expect(await daemonOf(daemon)).toMatchObject({ health: "running", update: null }); }); + + it("a gateway that starts after a rolled-back update reports the failure in status.get and announces daemon.update-failed", async () => { + const filesystem = new MemoryFilesystem(); + const failure = { + at: 900, + from: "0.3.0", + message: "the new version did not start", + stage: "restarting", + to: "0.4.0", + }; + const directory = await mkdtemp(join(tmpdir(), "simlock-main-gw-rolled-back-")); + temporaryDirectories.push(directory); + await filesystem.mkdirp(directory); + await filesystem.writeFileAtomic( + join(directory, "update-result.json"), + JSON.stringify(failure), + ); + const { daemon } = await startUpdatable(new FakeUpdaterLauncher(), filesystem, directory); + + expect((await daemonOf(daemon)).lastUpdateFailure).toEqual(failure); + await expect( + daemon.dispatch("events.replay", {}, operator) as Promise, + ).resolves.toContainEqual(expect.objectContaining({ event: "daemon.update-failed" })); + await expect(filesystem.exists(join(directory, "update-result.json"))).resolves.toBe(false); + }); }); From 8ce88da0378bd18028f94b058d0a5be9a0b1ee4d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:33:39 +0200 Subject: [PATCH 31/42] feat(daemon): a gateway reads the update result file at start (#838) --- src/daemon/main.ts | 1 + src/daemon/server.ts | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 4855cef0..5a098914 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -1048,6 +1048,7 @@ async function startGatewayDaemon(options: GatewayDaemonOptions): Promise((resolve) => setImmediate(resolve)); - if (!this.#stopping) this.#engine?.update?.announceResultFile(); + if (!this.#stopping) this.options.update?.announceResultFile(); } // why: a stale or unreadable result file must never keep the daemon from starting. async #reportUpdateResult(): Promise { try { - await this.#engine?.update?.readResultFile(); + await this.options.update?.readResultFile(); } catch (error: unknown) { this.#logger.warn("Could not read the update result file", { message: errorMessage(error) }); } From 3875d6db4364e40d1210e679e44b716b9801f462 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:35:47 +0200 Subject: [PATCH 32/42] feat(cli): worker list shows a worker's version, the latest when newer and its update stage (#838) --- src/cli/format.ts | 17 ++++++++++- src/cli/index.test.ts | 66 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src/cli/format.ts b/src/cli/format.ts index 9b6790e6..602c1619 100644 --- a/src/cli/format.ts +++ b/src/cli/format.ts @@ -37,7 +37,7 @@ export function formatWorkers( : ` -- ${entry.name} ${entry.state}`, ) .join(""); - return `${label}: ${state} -- ${capacity}, ${leaseCount(worker)}${skew}${host}${capabilities}${load}${installs}`; + return `${label}: ${state} -- ${capacity}, ${leaseCount(worker)}${skew}${workerVersion(worker)}${host}${capabilities}${load}${installs}`; }) .join("\n"); } @@ -57,6 +57,21 @@ function workerState(worker: WorkerView): string { ].join(", "); } +function workerVersion(worker: WorkerView): string { + const { latestVersion, update, version } = worker; + const available = + version !== undefined && latestVersion != null && isNewerVersion(latestVersion, version) + ? ` (${latestVersion} available)` + : ""; + const running = version === undefined ? "" : ` -- simlock ${version}${available}`; + return `${running}${update == null ? "" : ` -- updating to ${update.to}: ${updateStage(update)}`}`; +} + +function updateStage(update: NonNullable): string { + if (update.stage !== "waiting") return update.stage; + return `waiting for ${update.leases} ${update.leases === 1 ? "lease" : "leases"}`; +} + function leaseCount(worker: WorkerView): string { if (worker.leases === undefined) return "leases unknown"; return `${String(worker.leases.length)} lease(s)`; diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 9ca7c40f..3e4b227d 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -65,7 +65,7 @@ import type { StatusGetOutput, TokenListOutput, } from "../simlock-client/client.js"; -import { connectSimlockAdmin } from "../admin/index.js"; +import { connectSimlockAdmin, type WorkerView } from "../admin/index.js"; import { RELEASE_TIMEOUT_MS } from "../lease-policy/index.js"; import { runCli, type CliEnvironment, type CliEnvironmentPorts } from "./index.js"; import { buildCliEnvironment } from "./environment.js"; @@ -4283,6 +4283,70 @@ describe("CLI: worker commands (ADR 0005 §8/§23)", () => { expect(output.stdout).toContain("protocol 4-4 vs gateway 5-5"); }); + it("worker list shows a worker's version, the latest only when newer, and its update stage", async () => { + const waiting = { + forced: false, + from: "0.3.0", + leases: 2, + stage: "waiting", + startedAt: 1_000, + to: "0.4.0", + } as const; + const cases: readonly { + expected: string; + latestVersion: string | null; + update: WorkerView["update"]; + version: string | undefined; + }[] = [ + { + expected: " -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: waiting for 2 leases", + latestVersion: "0.4.0", + update: waiting, + version: "0.3.0", + }, + { + expected: " -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: installing", + latestVersion: "0.4.0", + update: { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + }, + version: "0.3.0", + }, + { + expected: " -- updating to 0.4.0: waiting for 1 lease", + latestVersion: null, + update: { ...waiting, leases: 1 }, + version: undefined, + }, + { expected: " -- simlock 0.4.0", latestVersion: "0.4.0", update: null, version: "0.4.0" }, + { expected: " -- simlock 0.5.0", latestVersion: "0.4.0", update: null, version: "0.5.0" }, + { expected: " -- simlock 0.4.0", latestVersion: null, update: null, version: "0.4.0" }, + ]; + for (const { expected, latestVersion, update, version } of cases) { + const output = outputCapture(); + const worker = { + ...connectedWorker, + latestVersion, + update, + ...(version === undefined ? {} : { version }), + }; + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ listWorkers: () => Promise.resolve({ workers: [worker] }) }), + }); + + await runCli(["worker", "list"], environment); + + expect(output.stdout, expected).toContain(expected); + if (!expected.includes("available")) expect(output.stdout).not.toContain("available"); + if (update === null) expect(output.stdout).not.toContain("updating"); + } + }); + it("emits the raw contract value under --json", async () => { const output = outputCapture(); const environment = output.environmentWith({ From 046f7e4509ed2879d1fb00ea66b14fe65a5fe789 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:37:27 +0200 Subject: [PATCH 33/42] test: simlock worker update follows the update on that worker (#838) --- src/cli/index.test.ts | 142 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 3e4b227d..2e86cdd7 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -9923,3 +9923,145 @@ describe("CLI: a service manager call fails, or the platform has none (#830)", ( }); }); }); + +describe("simlock worker update", () => { + type UpdateAnswer = Awaited>; + type View = Awaited>["workers"][number]; + const view = (overrides: Partial = {}): View => ({ + connection: "connected", + drained: false, + id: "3f81a2c4", + label: "mac-studio-2", + lastSeenAt: 1_000, + version: "0.3.0", + ...overrides, + }); + const run = { forced: false, from: "0.3.0", startedAt: 1_000, to: "0.4.0" } as const; + const answer = (overrides: Partial = {}): UpdateAnswer => ({ + health: "running", + lastUpdateFailure: null, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: { ...run, stage: "installing" }, + upToDate: false, + version: "0.3.0", + workerId: "3f81a2c4", + ...overrides, + }); + + it("follows the update to is running, reports a failure with exit 1, says when the worker is on the latest, detaches, and refuses on a worker daemon", async () => { + const failure = { + at: 2_000, + from: "0.3.0", + message: "npm exited 1", + stage: "installing", + to: "0.4.0", + } as const; + const cases: readonly { + readonly name: string; + readonly argv: readonly string[]; + readonly answer: UpdateAnswer | SimlockError; + readonly lists?: readonly View[]; + readonly code: number; + readonly stdout?: string; + readonly stderr?: string; + readonly polls?: number; + }[] = [ + { + name: "follows to running", + argv: ["worker", "update", "3f81a2c4", "--version", "0.4.0", "--force"], + answer: answer(), + lists: [ + view({ update: { ...run, leases: 2, stage: "waiting" } }), + view({ update: { ...run, stage: "installing" } }), + view({ connection: "disconnected", update: { ...run, stage: "restarting" } }), + view({ update: null, version: "0.4.0" }), + ], + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is running 0.4.0\n", + polls: 4, + }, + { + name: "fails with the stage and message", + argv: ["worker", "update", "3f81a2c4"], + answer: answer(), + lists: [view({ lastUpdateFailure: failure, update: null })], + code: 1, + stdout: "Update failed (installing): npm exited 1\n", + polls: 1, + }, + { + name: "already on the latest", + argv: ["worker", "update", "3f81a2c4"], + answer: answer({ started: false, update: null, upToDate: true, version: "0.4.0" }), + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)\n", + polls: 1, + }, + { + name: "detach", + argv: ["worker", "update", "3f81a2c4", "--detach"], + answer: answer(), + code: 0, + stdout: + "Updating mac-studio-2 (3f81a2c4) 0.3.0 → 0.4.0. Follow it with simlock worker list.\n", + polls: 1, + }, + { + name: "worker daemon", + argv: ["worker", "update", "3f81a2c4"], + answer: new SimlockError( + "UNSUPPORTED_IN_WORKER_MODE", + "domain", + "worker.update is a gateway operation", + { operation: "worker.update" }, + ), + code: 1, + stderr: "worker.update is a gateway operation", + polls: 0, + }, + ]; + for (const spec of cases) { + const output = outputCapture(); + const clock = new FakeClock(0); + const sent: unknown[] = []; + const lists = [...(spec.lists ?? [view()])]; + let polls = 0; + const environment = output.environmentWith({ + clock, + connectAdmin: async () => + fakeClient({ + listWorkers: async () => { + polls += 1; + const next = lists.length > 1 ? lists.shift() : lists[0]; + return { workers: next === undefined ? [] : [next] }; + }, + updateWorker: async (input) => { + sent.push(input); + if (spec.answer instanceof Error) throw spec.answer; + return spec.answer; + }, + }), + sleep: async (milliseconds) => clock.advance(milliseconds), + }); + + const code = await runCli(spec.argv, environment); + + expect({ code, name: spec.name, stdout: output.stdout }).toEqual({ + code: spec.code, + name: spec.name, + stdout: spec.stdout ?? "", + }); + if (spec.stderr !== undefined) expect(output.stderr).toContain(spec.stderr); + expect({ name: spec.name, polls }).toEqual({ name: spec.name, polls: spec.polls ?? 0 }); + expect(sent).toEqual([ + { + workerId: "3f81a2c4", + ...(spec.argv.includes("--version") ? { version: "0.4.0" } : {}), + ...(spec.argv.includes("--force") ? { force: true } : {}), + }, + ]); + } + }); +}); From e125fe692672fd2d56508fde5ee426816297ae22 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:38:26 +0200 Subject: [PATCH 34/42] feat(cli): simlock worker update follows an update on a worker (#838) --- src/cli/commands/worker.ts | 78 +++++++++++++++++++++++++++++++++++++- src/cli/index.test.ts | 2 +- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/cli/commands/worker.ts b/src/cli/commands/worker.ts index 1a4cf6df..472b5134 100644 --- a/src/cli/commands/worker.ts +++ b/src/cli/commands/worker.ts @@ -15,7 +15,8 @@ const WORKER_USAGE = "Usage: simlock worker list [--json]\n" + " simlock worker drain \n" + " simlock worker undrain \n" + - " simlock worker remove \n"; + " simlock worker remove \n" + + " simlock worker update [--version ] [--force] [--detach]\n"; async function run(argv: readonly string[], context: CommandContext): Promise { const { environment, token } = context; @@ -28,6 +29,7 @@ async function run(argv: readonly string[], context: CommandContext): Promise>["workers"][number]; + +function workerName(view: WorkerView | undefined, workerId: string): string { + return view?.label === undefined ? workerId : `${view.label} (${workerId})`; +} + +// why: the follow ends on the worker's own view: its version is the target with no update running, or its last failure names the target. +function followOutcome( + view: WorkerView | undefined, + to: string, + name: string, +): { readonly code: number; readonly text: string } | undefined { + const failure = view?.lastUpdateFailure; + if (failure?.to === to) { + return { code: 1, text: `Update failed (${failure.stage}): ${failure.message}` }; + } + if (view?.version === to && view.update == null) { + return { code: 0, text: `${name} is running ${to}` }; + } + return undefined; +} + +async function runWorkerUpdate( + argv: readonly string[], + client: SimlockAdminClient, + environment: CommandContext["environment"], +): Promise { + const values = commandArgs(argv, { + detach: { type: "boolean" }, + force: { type: "boolean" }, + help: { type: "boolean", short: "h" }, + version: { type: "string" }, + }); + if (values.help) { + environment.stdout.write( + "Usage: simlock worker update [--version ] [--force] [--detach]\n", + ); + return 0; + } + const workerId = requiredPositional(values.positionals, "worker-id"); + const answer = await client.updateWorker({ + workerId, + ...(typeof values.version === "string" ? { version: values.version } : {}), + ...(values.force === true ? { force: true } : {}), + }); + const find = async (): Promise => + (await client.listWorkers()).workers.find((worker) => worker.id === workerId); + let view = await find(); + const name = workerName(view, workerId); + if (answer.upToDate) { + environment.stdout.write(`${name} is already on the latest (${answer.version})\n`); + return 0; + } + const to = answer.update?.to ?? answer.resolved ?? ""; + if (values.detach === true) { + const from = answer.update?.from ?? answer.version; + environment.stdout.write( + `Updating ${name} ${from} → ${to}. Follow it with simlock worker list.\n`, + ); + return 0; + } + for (;;) { + const outcome = followOutcome(view, to, name); + if (outcome !== undefined) { + environment.stdout.write(`${outcome.text}\n`); + return outcome.code; + } + await environment.sleep(FOLLOW_POLL_MS); + view = await find(); + } +} + export { run }; diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 2e86cdd7..370711fc 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -10017,7 +10017,7 @@ describe("simlock worker update", () => { "worker.update is a gateway operation", { operation: "worker.update" }, ), - code: 1, + code: 2, stderr: "worker.update is a gateway operation", polls: 0, }, From d445be61e1c4254ee9aadfbab8c3c5c0fc7441f7 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:41:46 +0200 Subject: [PATCH 35/42] docs: simlock worker update, the 90 s message, WORKER_NOT_CONNECTED and the gateway's own update (#838) --- src/contract/AGENTS.md | 5 +- src/daemon/AGENTS.md | 2 +- src/gateway/AGENTS.md | 52 +++++++++++-------- src/gateway/routing/AGENTS.md | 22 ++++---- src/gateway/routing/stages/takes-requests.ts | 2 +- src/http/AGENTS.md | 3 +- src/http/admin-declarations.ts | 3 +- src/http/app.test.ts | 11 ---- .../docs/docs/gateway/managing-workers.mdx | 27 +++++++++- .../src/content/docs/docs/reference/cli.mdx | 25 +++++++-- .../content/docs/docs/reference/errors.mdx | 5 +- .../using-devices/programmatic-client.mdx | 2 + 12 files changed, 102 insertions(+), 57 deletions(-) diff --git a/src/contract/AGENTS.md b/src/contract/AGENTS.md index a3333fe7..89032441 100644 --- a/src/contract/AGENTS.md +++ b/src/contract/AGENTS.md @@ -92,9 +92,12 @@ Three identities: - `workerViewFields` builds it from `status.get`, `list.get`, `catalog.get` and `config.get` answers. A gateway's `WorkerLink` and a worker answering about itself both call it; the caller adds `id`, `label`, `connection`, `drained`, `lastSeenAt`, `version`. +- `update`, `lastUpdateFailure` and `latestVersion` come from `status.get`'s `daemon` block; `null` when there + is nothing to show, and absent until the first status read. A status without an update clears `update`. - A `starting` worker's view is `health`, `host` and `load` only. Absent, never empty. - Full re-read every `WORKER_VIEW_REFRESH_INTERVAL_MS` (30 s); catalog at once on - `WORKER_VIEW_CATALOG_EVENTS` (`component.installed`). + `WORKER_VIEW_CATALOG_EVENTS` (`component.installed`); status at once on + `WORKER_VIEW_UPDATE_EVENTS` (`daemon.update-started`, `daemon.update-failed`). ## OS ranges and durations diff --git a/src/daemon/AGENTS.md b/src/daemon/AGENTS.md index 2ec33cda..16ee6a67 100644 --- a/src/daemon/AGENTS.md +++ b/src/daemon/AGENTS.md @@ -48,7 +48,7 @@ An HTTP bind failure after a good start stops the daemon (`http-start-failed`). ## Update (`update-coordinator.ts`) -`daemon.update` (worker `Dispatcher`; a gateway answers `UNSUPPORTED_IN_GATEWAY_MODE`) is `createUpdateCoordinator`'s, in memory only. A call during an update follows it (`force` on `waiting` installs now); `query` starts nothing. +`daemon.update` (the worker `Dispatcher`, and `GatewayDispatcher` through its `update` option) is `createUpdateCoordinator`'s, in memory only. A gateway's coordinator holds no leases: its update never waits and goes straight to `installing` (`startGatewayDaemon` passes it a hold that does nothing and no leases), and `DaemonServer` takes it as the gateway arm's `update` for the result file below. A call during an update follows it (`force` on `waiting` installs now); `query` starts nothing. A new update: `NpmCli.view` resolves the version (`UPDATE_RESOLVE_FAILED`); the running version, or one newer than `latest` with no `version` named (`isNewerVersion`), is `upToDate`; then `checkInstall` (`UPDATE_UNSUPPORTED_INSTALL`) and `socketHeld` on the other role's socket (`UPDATE_SHARED_INSTALL`). Then `waiting`, leasing's update hold and `daemon.update-started`; the updater launches with no leases, with `force`, or when the last `lease.released`/`lease.expired` leaves none. Its stdout lines move the state; a failed line, a non-zero exit or a failed launch sets `lastUpdateFailure`, releases the hold and emits `daemon.update-failed`. `DaemonServer.start` calls `readResultFile()` after convergence and before health is `running`, so a `running` daemon already shows the failure in `lastUpdateFailure`; a rolled back update's `update-result.json` is deleted there, and `announceResultFile()` emits its `daemon.update-failed` once the daemon serves, so a parked `events.subscribe` receives it. diff --git a/src/gateway/AGENTS.md b/src/gateway/AGENTS.md index 9f1b2a64..6c0d9510 100644 --- a/src/gateway/AGENTS.md +++ b/src/gateway/AGENTS.md @@ -11,25 +11,25 @@ Flows: [device selection](../../docs/internal/flows/device-selection.md), [devic ## Files -| File | Role | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| `service.ts` | `GatewayService`: uplink listener, one `WorkerLink` per worker, 30 s tick | -| `worker-link.ts` | One uplink, gateway side: admin client, view reads, event relay | -| `worker-registry.ts` | `WorkerRegistry`: views, drain, retention, `worker.*` events | -| `drain-store.ts` | `StateDrainStore`: the drained ids in the `drained_workers` table of `state.db`; `drainedWorkersOwner` | -| `dispatcher.ts` | `GatewayDispatcher`: the contract's handlers, answered from the fleet; `alerts.test` through the `testDestination` option | -| `fleet-coordinator.ts` | `FleetLeaseCoordinator`: admit, dispatch, retry, forward renew/release/exec | -| `queue.ts` | `FleetQueue`: leasing's `WaitQueue`, two lines | -| `lease-index.ts` | `FleetLeaseIndex`: gateway lease id → worker, worker lease id, owner | -| `routing.ts`, `routing/` | `createRoutingPolicy`: `assess` and the stage list | -| `owner-routed-facts.ts` | Relayed lease facts → `lease-lost` and device pushes to the owner | -| `aggregate.ts` | `aggregateStatus`, `aggregateCatalog`: pure, over views | -| `component-relay.ts` | `relayComponentInstall`: `worker.install-component` fan-out | -| `relay.ts` | `GatewayRelay`: one HTTP exchange to a worker's relay target | -| `fleet-ports.ts` | `WorkerDirectory`, `FleetViews`, `liveClient`, `isUplinkLost` | -| `../daemon/gateway-uplink.ts` | Worker: `GatewayUplink`, dial and redial | -| `../daemon/relay.ts` | Worker: `RelayConnection`, serves `relay.request` | -| `../link/` | Both ends: text messages plus byte channels on one WebSocket | +| File | Role | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `service.ts` | `GatewayService`: uplink listener, one `WorkerLink` per worker, 30 s tick | +| `worker-link.ts` | One uplink, gateway side: admin client, view reads, event relay | +| `worker-registry.ts` | `WorkerRegistry`: views, drain, retention, `worker.*` events | +| `drain-store.ts` | `StateDrainStore`: the drained ids in the `drained_workers` table of `state.db`; `drainedWorkersOwner` | +| `dispatcher.ts` | `GatewayDispatcher`: the contract's handlers, answered from the fleet; `alerts.test` through `testDestination`, `daemon.update` through `update` | +| `fleet-coordinator.ts` | `FleetLeaseCoordinator`: admit, dispatch, retry, forward renew/release/exec | +| `queue.ts` | `FleetQueue`: leasing's `WaitQueue`, two lines | +| `lease-index.ts` | `FleetLeaseIndex`: gateway lease id → worker, worker lease id, owner | +| `routing.ts`, `routing/` | `createRoutingPolicy`: `assess` and the stage list | +| `owner-routed-facts.ts` | Relayed lease facts → `lease-lost` and device pushes to the owner | +| `aggregate.ts` | `aggregateStatus`, `aggregateCatalog`: pure, over views | +| `component-relay.ts` | `relayComponentInstall`: `worker.install-component` fan-out | +| `relay.ts` | `GatewayRelay`: one HTTP exchange to a worker's relay target | +| `fleet-ports.ts` | `WorkerDirectory`, `FleetViews`, `liveClient`, `isUplinkLost` | +| `../daemon/gateway-uplink.ts` | Worker: `GatewayUplink`, dial and redial | +| `../daemon/relay.ts` | Worker: `RelayConnection`, serves `relay.request` | +| `../link/` | Both ends: text messages plus byte channels on one WebSocket | ## How it works @@ -52,11 +52,12 @@ Flows: [device selection](../../docs/internal/flows/device-selection.md), [devic 4. `WorkerLink.start`: `hello` as `gw:`, `status.get`. Protocol mismatch → view `incompatible`; role not `admin` → close. Then `worker.connected`, `events.subscribe`, first refresh with catalog. Each of these reads waits at most 10 s (`WORKER_CALL_TIMEOUT_MS`). -5. Refresh: `status.get`; a `starting` worker stops there. Else `list.get` devices, plus +5. Refresh: an `incompatible` worker is asked `daemon.update {query}` only (its `update`, `lastUpdateFailure` and + `latestVersion`; a failed query keeps the last view). Else `status.get`; a `starting` worker stops there. Else `list.get` devices, plus `catalog.get` and `config.get` on the tick, after `component.installed`, or with no catalog yet. One at a time; refreshes asked meanwhile merge into one. 2 timed-out refreshes in a row → close. 6. Worker events are republished with `workerId`. `lease.*`, `device.*`, `component.install*` - trigger a refresh. A worker event named like a gateway fact (`worker.*`, `request.dispatched`, `request.granted`) is dropped. + trigger a refresh, and so do `daemon.update-started` and `daemon.update-failed`. A worker event named like a gateway fact (`worker.*`, `request.dispatched`, `request.granted`) is dropped. **Fleet request** (`FleetLeaseCoordinator.request`, under its own `SerializedDecision`; routing's `assess` and nine stages are in [routing](routing/AGENTS.md)): @@ -79,7 +80,7 @@ Flows: [device selection](../../docs/internal/flows/device-selection.md), [devic → exclude that worker for this request, refresh its catalog, walk again. `PROVISION_FAILED` → a refusal; every worker that could take it failed so (one that is full, busy, not ready or unhealthy could, later) → reject with the last. After a progress push it is relayed as a failure after progress. Anything else, - or a failure after progress → `lease.rejected` `worker-failed`. + or a failure after progress → `lease.rejected` `worker-failed`. `DAEMON_UPDATING` is handled as `NO_CAPACITY` (the worker began an update the view did not show yet). 7. Grant: a lease id unlike the forwarded `leaseId` → release it on the worker, retry. Gateway lease id = the caller's `leaseId`, else `.`. `leaseIndex.add`, then `request.granted`; a clash → release on worker, `LEASE_ID_TAKEN`. @@ -100,6 +101,12 @@ missing from 2 snapshots in a row → dropped; kept across a disconnect, forgott 4. Worker: `relay.*` only on the uplink session (`FORBIDDEN` elsewhere; a gateway refuses them from everyone). Target from `relayTargets`; unknown → `RELAY_TARGET_MISSING`. +**Updates.** `worker.update` sends `daemon.update` over that worker's uplink, bounded at 90 s (`WORKER_NOT_CONNECTED` at the limit), then +refreshes the view. An unknown id → `UNKNOWN_WORKER`; a disconnected one → `WORKER_NOT_CONNECTED`. An `incompatible` worker gets the same frozen call. +A view with `update` set is skipped by routing (`updating`; the request waits, it does not end `NO_CAPACITY`). The update and the drain set are separate: +an update never calls `setDrained`, and `update: null` from the next status clears the view. `daemon.update` on the gateway itself goes through +the `update` option (`../daemon/update-coordinator.ts`): no lease wait, straight to installing. + **Reads.** `status.get`, `catalog.get`, `list.get`, `lease.list` (admin: all; else own) answer from views. `worker.install-component` fans out. `nuke.run`, `cleanup.run`, `doctor.run`, `driver.passthrough`, `component.*`, `device.add`, `device.remove` → `UNSUPPORTED_IN_GATEWAY_MODE`. `usage.get` reads this gateway's `history.db` rows through `core/usage`: fleet totals, one entry per registry or named worker; `dispose` stops its read thread. @@ -112,6 +119,7 @@ views. `worker.install-component` fans out. `nuke.run`, `cleanup.run`, `doctor.r | -------------------------------------- | ---------------------------------------- | ----------------------------------------------------- | | View refresh tick (+ prune) | 30 s | `WORKER_VIEW_REFRESH_INTERVAL_MS` | | One view read | 10 s | `WORKER_CALL_TIMEOUT_MS` | +| `worker.update` over the uplink | 90 s | `UPDATE_UPLINK_BOUND_MS` (`dispatcher.ts`) | | Forwarded `lease.request` | 5 min | `gateway.leaseRequestTimeoutMs` | | Forwarded `device.exec` | 11 min | `gateway.execTimeoutMs` | | Disconnected view kept | 24 h, held while a gateway lease is live | `gateway.disconnectedRetentionMs` | diff --git a/src/gateway/routing/AGENTS.md b/src/gateway/routing/AGENTS.md index 7ed818f9..4d8a4471 100644 --- a/src/gateway/routing/AGENTS.md +++ b/src/gateway/routing/AGENTS.md @@ -35,17 +35,17 @@ Flows: [device selection](../../../docs/internal/flows/device-selection.md), [ga 4. The reported stage is the last rank that removed a worker, else the last that decided. `reason` is `warm-hit` when that stage is `warm-hit`, else `free-capacity`. -| Stage | Kind | Keeps or prefers | Physical | -| ------------------ | ------------- | ----------------------------------------------------------------------------------------------------- | -------- | -| `takes-requests` | filter | connected, not drained, capacity and catalog read this session (`starting`: a catalog held) | yes | -| `has-capabilities` | filter | every `requires` entry `available` | yes | -| `can-serve` | filter | catalog serves model or class with an installed fitting runtime; physical: an enrolled fitting device | yes | -| `healthy` | filter | health `running` | yes | -| `idle-queue` | filter | the worker's own `queueDepth` is 0 | no | -| `warm-hit` | rank, settles | a `ready` fitting device of the asked mode (`servesDefaultMode` if none); physical: a pickable one | yes | -| `free-slot` | filter | a free or evictable (warm) running slot, platform and global | no | -| `ram-budget` | rank | `atRamBudget` not true | no | -| `free-capacity` | rank | most `maxRunning - running - reserved` | no | +| Stage | Kind | Keeps or prefers | Physical | +| ------------------ | ------------- | --------------------------------------------------------------------------------------------------------- | -------- | +| `takes-requests` | filter | connected, not drained, not updating, capacity and catalog read this session (`starting`: a catalog held) | yes | +| `has-capabilities` | filter | every `requires` entry `available` | yes | +| `can-serve` | filter | catalog serves model or class with an installed fitting runtime; physical: an enrolled fitting device | yes | +| `healthy` | filter | health `running` | yes | +| `idle-queue` | filter | the worker's own `queueDepth` is 0 | no | +| `warm-hit` | rank, settles | a `ready` fitting device of the asked mode (`servesDefaultMode` if none); physical: a pickable one | yes | +| `free-slot` | filter | a free or evictable (warm) running slot, platform and global | no | +| `ram-budget` | rank | `atRamBudget` not true | no | +| `free-capacity` | rank | most `maxRunning - running - reserved` | no | A physical request has one rank, `warm-hit`: with no pickable device nothing is picked, and it waits. diff --git a/src/gateway/routing/stages/takes-requests.ts b/src/gateway/routing/stages/takes-requests.ts index 1203dd51..570dad27 100644 --- a/src/gateway/routing/stages/takes-requests.ts +++ b/src/gateway/routing/stages/takes-requests.ts @@ -1,7 +1,7 @@ import type { FilterStage } from "../pipeline.js"; /** - * Keeps a worker that is connected, not drained, and whose capacity and catalog have been read + * why: keeps a worker that is connected, not drained, not updating (ADR 0042 §3), and whose capacity and catalog have been read * (ADR 0009 §2, stage 1). A policy that cannot see free capacity must not guess it has some, and * a worker whose catalog has not been read since it connected is carrying the last session's. * diff --git a/src/http/AGENTS.md b/src/http/AGENTS.md index 925c073e..643be27f 100644 --- a/src/http/AGENTS.md +++ b/src/http/AGENTS.md @@ -88,7 +88,8 @@ agent-device front and the stream (`/v1/stream`). subscribes to the `EventBus` itself. - `GET /v1/events` — reads `since`, `before` and `limit` through `eventsPageQuery` (contract), refused as 400 before dispatching `events.replay`; `HISTORY_NOT_KEPT` maps to 422 with `oldestTs`, `HISTORY_UNAVAILABLE` to 503 here too. - `GET /v1/stats` — parses `since`/`from`/`to` with `usageWindow` (contract) and `ledger=1`, all refused as 400 before dispatching `usage.get` (the dispatcher answers an agent token 403, `HISTORY_NOT_KEPT` maps to 422 with `oldestTs`, `HISTORY_UNAVAILABLE` to 503). -- `POST /v1/update` (optional body `{version, force}`, strict) → `daemon.update`: 202 when the call began the update (`started`), 200 when it followed one or found nothing to do (`upToDate`). The route declares both (`alsoStatus`). The role check is the dispatcher's; a gateway answers 501. +- `POST /v1/update` (optional body `{version, force}`, strict) → `daemon.update`: 202 when the call began the update (`started`), 200 when it followed one or found nothing to do (`upToDate`). The route declares both (`alsoStatus`). The role check is the dispatcher's; on a gateway it updates the gateway itself, with no lease wait. +- `POST /v1/workers/{id}/update` (optional body `{version, force}`, strict) → `worker.update`: the same 202 and 200; `UNKNOWN_WORKER` 404, `WORKER_NOT_CONNECTED` 409, and a worker daemon answers 501. - `GET /v1/leases/{id}/events` — the lease's notices; ends on `lease_lost`. - `POST /v1/leases/{id}/exec` → `device.exec`. The stream opens on `onStarted`; a failure before that is a plain error response. Events: `output`, then `exit` or `error`. diff --git a/src/http/admin-declarations.ts b/src/http/admin-declarations.ts index e0ea1343..d020161d 100644 --- a/src/http/admin-declarations.ts +++ b/src/http/admin-declarations.ts @@ -66,7 +66,7 @@ export const updateRoute = declareRoute({ summary: "Update this daemon to a Simlock version", role: "admin", notes: - "The body is optional: without a version the newest release is installed. The daemon takes no new leases while it waits for its leases to end, installs and restarts. 202 when this call began the update; 200 when it followed an update that was already running, or nothing needed updating (`upToDate`). Worker only: a gateway answers 501 UNSUPPORTED_IN_GATEWAY_MODE.", + "The body is optional: without a version the newest release is installed. The daemon takes no new leases while it waits for its leases to end, installs and restarts. 202 when this call began the update; 200 when it followed an update that was already running, or nothing needed updating (`upToDate`). On a gateway it updates the gateway itself, with no lease wait: a gateway holds none.", request: { optionalJson: updateBodySchema }, reply: OPERATIONS["daemon.update"].output, status: 202, @@ -74,7 +74,6 @@ export const updateRoute = declareRoute({ errors: { 409: ["UPDATE_UNSUPPORTED_INSTALL", "UPDATE_SHARED_INSTALL"], 502: ["UPDATE_RESOLVE_FAILED"], - 501: ["UNSUPPORTED_IN_GATEWAY_MODE"], }, }); diff --git a/src/http/app.test.ts b/src/http/app.test.ts index 1dfa51d3..3fda43ed 100644 --- a/src/http/app.test.ts +++ b/src/http/app.test.ts @@ -2615,17 +2615,6 @@ describe("POST /v1/update", () => { expect({ body, status: refused.status }).toEqual({ body, status: 400 }); } expect(dispatcher.calls).toHaveLength(callsBefore); - - dispatcher.handlers["daemon.update"] = () => { - throw new DispatchError("UNSUPPORTED_IN_GATEWAY_MODE", "daemon.update is refused", { - operation: "daemon.update", - }); - }; - const onGateway = await post(app, {}); - expect(onGateway.status).toBe(501); - expect(await onGateway.json()).toMatchObject({ - error: { code: "UNSUPPORTED_IN_GATEWAY_MODE", operation: "daemon.update" }, - }); }); }); diff --git a/website/src/content/docs/docs/gateway/managing-workers.mdx b/website/src/content/docs/docs/gateway/managing-workers.mdx index 469d839b..03ba6906 100644 --- a/website/src/content/docs/docs/gateway/managing-workers.mdx +++ b/website/src/content/docs/docs/gateway/managing-workers.mdx @@ -1,6 +1,6 @@ --- title: Manage workers -description: List the workers behind a gateway, take one out of service and bring it back, forget one that is gone, install runtimes on many workers at once, and cut a worker off. +description: List the workers behind a gateway, take one out of service and bring it back, update one, forget one that is gone, install runtimes on many workers at once, and cut a worker off. --- import { Steps, Aside } from '@astrojs/starlight/components'; @@ -14,7 +14,7 @@ simlock --gateway worker list ``` ```text -4b1e9c2a-6d3f-4a8e-9b2c-7f1d0e5a3c68 (mac-mini-2): connected -- ios 2/4, android 1/2, 3 lease(s) -- macOS 15.6 arm64; xcode 16.4 (16F6), emulator 35.4.9 -- agent-device off -- stream off +4b1e9c2a-6d3f-4a8e-9b2c-7f1d0e5a3c68 (mac-mini-2): connected -- ios 2/4, android 1/2, 3 lease(s) -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: waiting for 2 leases -- macOS 15.6 arm64; xcode 16.4 (16F6), emulator 35.4.9 -- agent-device off -- stream off Load: CPU 34% busy (10 cores) · memory 12.3 GiB of 32.0 GiB available · swap 0 B used · pressure normal · disk 210.4 GiB of 926.4 GiB free · read 12s ago 9e3a7c1f-0b5d-4c2e-8f6a-1d4b7e9c2a50 (mac-studio-1): disconnected, drained -- ios 0/8, android 0/4, 0 lease(s) -- macOS 15.6 arm64; xcode 16.4 (16F6) Load: no reading while disconnected @@ -25,6 +25,7 @@ Each worker shows: - its id and, in brackets, its label; - its state: `connected`, `disconnected` or `incompatible`, then `starting` while it starts up and `drained` when it takes no new leases from the gateway; - devices running of the limit per platform, and how many leases it holds; +- its Simlock version, with the newest release in brackets when npm has a newer one, and `updating to : ` while it updates (the stage is `waiting for leases`, `installing` or `restarting`); - its operating system, platform tools and optional modules; - under it, the last load reading, and any runtime or system image being installed. @@ -71,6 +72,28 @@ What to know about draining: An id the gateway does not show fails with `UNKNOWN_WORKER` (exit 12). +## Update a worker + +Update one worker's Simlock from the gateway, and follow it: + +```sh +simlock --gateway worker update 3f81a2c4 +``` + +```text +mac-studio-2 (3f81a2c4) is running 0.4.0 +``` + +The worker waits for its leases to end, then installs and restarts, as `simlock update` does on a single machine. While it waits or installs, the gateway sends it no new leases: a request only this worker could serve stays in the queue until the worker is back, and `worker list` shows `updating`. A drain is left alone. A worker you drained stays drained after its update, and only `undrain` ends it. + +- `--version ` installs that exact version, `--force` does not wait for the leases, and `--detach` returns at once with `Updating mac-studio-2 (3f81a2c4) 0.3.0 → 0.4.0. Follow it with simlock worker list.` +- A worker already on the newest release answers `mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)`. +- A failed update ends with `Update failed (): ` and exit 1. The worker goes on with its old version. +- A worker shown as `incompatible` can be updated too. That is the way to bring a worker with an older protocol back into the fleet. +- A worker that is not connected fails with `WORKER_NOT_CONNECTED`. The gateway does not queue an update for it. + +To update the gateway itself, run `simlock update --gateway` on its machine. It installs and restarts at once, with no wait. + ## Forget a worker that is gone When a machine is gone for good, remove it so the gateway stops showing it: diff --git a/website/src/content/docs/docs/reference/cli.mdx b/website/src/content/docs/docs/reference/cli.mdx index 36cb20b3..c1077bed 100644 --- a/website/src/content/docs/docs/reference/cli.mdx +++ b/website/src/content/docs/docs/reference/cli.mdx @@ -392,7 +392,7 @@ simlock 0.4.0 is running - The daemon does not answer within 15 minutes after it restarts: `The daemon did not come back; run simlock status.`, exit 1. - Simlock must be installed with `npm i -g simlock`. Otherwise the command stops with `UPDATE_UNSUPPORTED_INSTALL`. A worker and a gateway of one user share the install, so while the other runs it stops with `UPDATE_SHARED_INSTALL`. npm failing to find the version is `UPDATE_RESOLVE_FAILED`. See [Errors](/simlock/docs/reference/errors#updating-simlock). -`POST /v1/update` does the same over HTTP. A gateway refuses `update` with `UNSUPPORTED_IN_GATEWAY_MODE` (exit 2). +`POST /v1/update` does the same over HTTP. On a machine that runs a gateway, `simlock update --gateway` updates the gateway itself. A gateway holds no leases of its own, so it does not wait: it installs and restarts at once, and a request in that gap fails as in any gateway restart. To update a worker from a gateway, use [`simlock worker update`](#simlock-worker). ## simlock nuke @@ -499,16 +499,35 @@ simlock worker list [--json] simlock worker drain simlock worker undrain simlock worker remove +simlock worker update [--version ] [--force] [--detach] ``` | Subcommand | What it does | | --- | --- | -| `list` | Show each worker: connection, devices, capacity and installs. On a single machine it shows the machine itself. `--json` prints one JSON line in place of the text view. | +| `list` | Show each worker: connection, devices, capacity and installs. A worker's line also shows its Simlock version, with `( available)` when npm has a newer release, and `updating to : ` while it updates. On a single machine it shows the machine itself. `--json` prints one JSON line in place of the text view. | | `drain` | Stop sending new requests to the worker. Its leases run on. | | `undrain` | Send new requests to the worker again. | | `remove` | Forget a worker that is not connected. A connected worker is refused with `WORKER_CONNECTED`: drain it and let it disconnect first. | +| `update` | Update one worker's Simlock from the gateway and follow it. The worker takes no new leases while it updates. A drain stays as it is. | -All four need the admin role. `drain`, `undrain` and `remove` print one JSON line, and on a worker they fail with `UNSUPPORTED_IN_WORKER_MODE` (exit 2). +All five need the admin role. `drain`, `undrain` and `remove` print one JSON line, and on a worker all but `list` fail with `UNSUPPORTED_IN_WORKER_MODE` (exit 2). + +`update` takes the flags of [`simlock update`](#simlock-update) and one more: + +| Flag | Meaning | Default | +| --- | --- | --- | +| `--version ` | Install this exact version on the worker. | The newest release | +| `--force` | The worker does not wait for its leases to end. | Off | +| `--detach` | Print where the update started and return. Follow it with `simlock worker list`. | Off | + +It follows the worker's line in the list and ends with one of these: + +- `mac-studio-2 (3f81a2c4) is running 0.4.0` +- `Update failed (installing): `, exit 1. The worker keeps running its old version and takes leases again. +- `mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)`. +- With `--detach`: `Updating mac-studio-2 (3f81a2c4) 0.3.0 → 0.4.0. Follow it with simlock worker list.` + +A worker that shows `incompatible` can be updated this way too. A worker that is not connected fails with `WORKER_NOT_CONNECTED`; one that does not answer within 90 seconds fails with the same code. `POST /v1/workers/{id}/update` does the same over HTTP. ## simlock alerts diff --git a/website/src/content/docs/docs/reference/errors.mdx b/website/src/content/docs/docs/reference/errors.mdx index db9989d8..5d50311b 100644 --- a/website/src/content/docs/docs/reference/errors.mdx +++ b/website/src/content/docs/docs/reference/errors.mdx @@ -23,7 +23,7 @@ Branch on `code`, not on the message: the message wording can change, the code d | --- | --- | --- | | 0 | — | Success. For a `simlock lease` that stayed running: the lease ended because you asked. | | 1 | `INTERNAL`, `PROVISION_FAILED`, `UNKNOWN_LEASE`, `FORBIDDEN`, `ADMIN_AUTHENTICATION_FAILED`, `DAEMON_STARTUP_FAILED`, `DAEMON_CONNECTION_LOST`, `DAEMON_STOPPING`, `DAEMON_UPDATING`, `HANDSHAKE_REQUIRED`, `PROTOCOL_VERSION_UNSUPPORTED`, `WORKER_UNREACHABLE`, `RELAY_TARGET_MISSING`, `ALERT_SEND_FAILED`, `UPDATE_RESOLVE_FAILED`, `HISTORY_UNAVAILABLE`, `DOCTOR_UNAVAILABLE`, `NUKE_UNAVAILABLE`, `UNKNOWN_DAEMON_ERROR`, `CANCELLED` | The command failed. Read the code and the message. | -| 2 | `USAGE`, `BAD_REQUEST`, `BAD_FRAME`, `UNKNOWN_REQUEST`, `IDEMPOTENCY_CONFLICT`, `PASSTHROUGH_REFUSED`, `UNKNOWN_PASSTHROUGH_TOOL`, `GATEWAY_NOT_PRESENT`, `UPDATE_UNSUPPORTED_INSTALL`, `UPDATE_SHARED_INSTALL`, `UNSUPPORTED_IN_GATEWAY_MODE`, `UNSUPPORTED_IN_WORKER_MODE`, `WORKER_CONNECTED` | The request was wrong. Running it again unchanged fails again. | +| 2 | `USAGE`, `BAD_REQUEST`, `BAD_FRAME`, `UNKNOWN_REQUEST`, `IDEMPOTENCY_CONFLICT`, `PASSTHROUGH_REFUSED`, `UNKNOWN_PASSTHROUGH_TOOL`, `GATEWAY_NOT_PRESENT`, `UPDATE_UNSUPPORTED_INSTALL`, `UPDATE_SHARED_INSTALL`, `UNSUPPORTED_IN_GATEWAY_MODE`, `UNSUPPORTED_IN_WORKER_MODE`, `WORKER_CONNECTED`, `WORKER_NOT_CONNECTED` | The request was wrong. Running it again unchanged fails again. | | 10 | `QUEUE_TIMEOUT`, `EXEC_TIMEOUT`, `DOWNLOAD_TIMEOUT` | A deadline ran out. | | 11 | `NO_CAPACITY`, `DEVICE_BUSY`, `COMPONENT_BUSY` | Busy right now. The same request can pass later. | | 12 | `NO_DRIVER`, `CAPABILITY_MISSING`, `RUNTIME_MISSING`, `UNKNOWN_MODEL`, `INSUFFICIENT_DISK_SPACE`, `LICENSE_NOT_ACCEPTED`, `DOWNLOADS_DISABLED`, `COMPONENT_NOT_OWNED`, `COMPONENT_IN_USE`, `DEVICE_NOT_ENROLLABLE`, `UNKNOWN_DEVICE`, `HISTORY_NOT_KEPT`, `UNKNOWN_WORKER`, `UNKNOWN_ALERT_DESTINATION` | What you named does not exist here, or this machine cannot provide it. | @@ -117,7 +117,8 @@ Branch on `code`, not on the message: the message wording can change, the code d | --- | --- | --- | --- | --- | | `GATEWAY_NOT_PRESENT` | 2 | 404 | `--gateway` on a machine with no gateway. Raised by the CLI and MCP server before they connect, never over HTTP. | Start one with `simlock --gateway daemon start`, or drop `--gateway`. | | `UNSUPPORTED_IN_GATEWAY_MODE` | 2 | 501 | The command acts on one machine's devices or components (`doctor`, `cleanup`, `nuke`, `device`, `component list`, `component remove`, `component install` without a worker flag), and the daemon that answered is a gateway. | Run it against the worker, with `--local` on that machine. For `component install`, name workers with `--worker` or `--all-workers`. | -| `UNSUPPORTED_IN_WORKER_MODE` | 2 | 501 | The command acts on a gateway's workers or alerts (`worker drain`, `undrain`, `remove`, `alerts test`, `component install --worker`), and the daemon that answered is a worker. | Run it against the gateway, with `--gateway` on that machine. | +| `UNSUPPORTED_IN_WORKER_MODE` | 2 | 501 | The command acts on a gateway's workers or alerts (`worker drain`, `undrain`, `remove`, `update`, `alerts test`, `component install --worker`), and the daemon that answered is a worker. | Run it against the gateway, with `--gateway` on that machine. | +| `WORKER_NOT_CONNECTED` | 2 | 409 | `worker update` named a worker the gateway knows but has no open connection to: `worker is not connected`. It is also the answer when the worker does not reply within 90 seconds: `worker did not answer within 90 s; check simlock worker list`. In that case the update may still have started. | Check the worker with `simlock worker list`. Run the update again once it is connected. | | `UNKNOWN_WORKER` | 12 | 404 | The gateway has no worker with this ID. | Check the ID with `simlock worker list`. | | `WORKER_CONNECTED` | 2 | 409 | `worker remove` of a worker that is still connected. | Drain it, stop it or let it disconnect, then remove it. | | `WORKER_UNREACHABLE` | 1 | 503 | The worker that holds this lease or would serve this request is not connected to the gateway. | Check the worker is running and can reach the gateway. | diff --git a/website/src/content/docs/docs/using-devices/programmatic-client.mdx b/website/src/content/docs/docs/using-devices/programmatic-client.mdx index 73acbcdd..b773e877 100644 --- a/website/src/content/docs/docs/using-devices/programmatic-client.mdx +++ b/website/src/content/docs/docs/using-devices/programmatic-client.mdx @@ -195,6 +195,8 @@ const answer = await admin.updateDaemon({ version: "0.4.0" }); - `{ query: true }` starts nothing and returns the same answer: poll it to follow the update. It works while the daemon restarts only by reconnecting, and across a protocol change: like `stopDaemon()`, `updateDaemon()` is accepted by a daemon on another protocol version, so a client that fails every other call with `PROTOCOL_VERSION_UNSUPPORTED` can still update it. - Refusals: `UPDATE_RESOLVE_FAILED`, `UPDATE_UNSUPPORTED_INSTALL` and `UPDATE_SHARED_INSTALL` (see [Errors](/simlock/docs/reference/errors#updating-simlock)). `version` must be an exact version such as `0.4.0` or `0.4.0-rc.1`. +On a gateway, `admin.updateDaemon()` updates the gateway itself: it holds no leases, so it does not wait and goes straight to installing. `admin.updateWorker({ workerId, version?, force? })` asks the gateway to update one of its workers, `simlock worker update` does. It answers like `updateDaemon()`, plus the `workerId`. A worker that is not connected, or that does not answer within 90 seconds, fails it with `WORKER_NOT_CONNECTED`. Poll `admin.listWorkers()` to follow it: the worker's `version`, `latestVersion`, `update` and `lastUpdateFailure` show the update, and `update` is `null` once it ends. On a worker, `updateWorker()` fails with `UNSUPPORTED_IN_WORKER_MODE`. + ## Related - How leases and TTLs work: [Lease lifecycle](/simlock/docs/using-devices/lease-lifecycle). From 22302cb5c384475c497ac9573da361c088b5c294 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:42:22 +0200 Subject: [PATCH 36/42] refactor(gateway): one registry method reads a daemon.update answer into a view (#838) --- src/gateway/dispatcher.ts | 6 +----- src/gateway/worker-link.ts | 6 +----- src/gateway/worker-registry.ts | 12 ++++++++++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 7596a2ba..dfc6b05a 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -443,11 +443,7 @@ export class GatewayDispatcher { } const { workerId, ...request } = input; const answer = await this.#withinUplinkBound(client.updateDaemon(request), workerId); - this.options.workers.refresh(workerId, { - lastUpdateFailure: answer.lastUpdateFailure, - latestVersion: answer.latestVersion, - update: answer.update, - }); + this.options.workers.refreshUpdate(workerId, answer); await target.refresh(); return { ...answer, workerId }; }; diff --git a/src/gateway/worker-link.ts b/src/gateway/worker-link.ts index c37e0368..6d4612a7 100644 --- a/src/gateway/worker-link.ts +++ b/src/gateway/worker-link.ts @@ -367,11 +367,7 @@ export class WorkerLink { try { const answer = await this.#withTimeout(client.updateDaemon({ query: true }), "daemon.update"); if (this.#superseded()) return; - this.options.registry.refresh(this.workerId, { - lastUpdateFailure: answer.lastUpdateFailure, - latestVersion: answer.latestVersion, - update: answer.update, - }); + this.options.registry.refreshUpdate(this.workerId, answer); } catch (error: unknown) { this.#logger.debug("Incompatible worker's update query failed", { workerId: this.workerId, diff --git a/src/gateway/worker-registry.ts b/src/gateway/worker-registry.ts index dda0040f..657fef32 100644 --- a/src/gateway/worker-registry.ts +++ b/src/gateway/worker-registry.ts @@ -384,6 +384,18 @@ export class WorkerRegistry { this.#notifyViewsChanged(); } + // why: both the operator's `worker.update` and an incompatible worker's query read these three fields from a `daemon.update` answer. + refreshUpdate( + workerId: string, + answer: Pick, + ): void { + this.refresh(workerId, { + lastUpdateFailure: answer.lastUpdateFailure, + latestVersion: answer.latestVersion, + update: answer.update, + }); + } + /** * ADR 0005 §15: the gateway's own `lease.maxTtlMs` decides a fleet lease's width at * admission, before any worker is chosen -- but what the gateway dispatches is an ordinary From 8beb1e6d58680870acd1ebd3e616dfbe2499eaa8 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:44:34 +0200 Subject: [PATCH 37/42] refactor: split the worker update follow and the refresh timeout count out of two long functions (#838) --- src/cli/commands/worker.ts | 18 ++++++++++++---- src/gateway/worker-link.ts | 42 ++++++++++++++++++++---------------- ui/src/model/waiting.test.ts | 2 +- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/cli/commands/worker.ts b/src/cli/commands/worker.ts index 472b5134..aebe0643 100644 --- a/src/cli/commands/worker.ts +++ b/src/cli/commands/worker.ts @@ -129,8 +129,8 @@ async function runWorkerUpdate( }); const find = async (): Promise => (await client.listWorkers()).workers.find((worker) => worker.id === workerId); - let view = await find(); - const name = workerName(view, workerId); + const first = await find(); + const name = workerName(first, workerId); if (answer.upToDate) { environment.stdout.write(`${name} is already on the latest (${answer.version})\n`); return 0; @@ -143,14 +143,24 @@ async function runWorkerUpdate( ); return 0; } - for (;;) { + return follow({ environment, find, first, name, to }); +} + +async function follow(options: { + readonly environment: CommandContext["environment"]; + readonly find: () => Promise; + readonly first: WorkerView | undefined; + readonly name: string; + readonly to: string; +}): Promise { + const { environment, find, name, to } = options; + for (let view = options.first; ; view = await find()) { const outcome = followOutcome(view, to, name); if (outcome !== undefined) { environment.stdout.write(`${outcome.text}\n`); return outcome.code; } await environment.sleep(FOLLOW_POLL_MS); - view = await find(); } } diff --git a/src/gateway/worker-link.ts b/src/gateway/worker-link.ts index 6d4612a7..f4a81567 100644 --- a/src/gateway/worker-link.ts +++ b/src/gateway/worker-link.ts @@ -331,25 +331,7 @@ export class WorkerLink { workerId: this.workerId, message: errorMessage(error), }); - // P1: a `WorkerCallTimeoutError` here is not "anything else" -- it is the one failure this - // class already knows means "the socket is not answering", and it used to mean nothing - // happened at all: `registry.refresh` was never reached, so `connection` stayed - // `connected` and `lastSeenAt` stayed stale, forever, on a half-open uplink. After - // `MAX_CONSECUTIVE_REFRESH_TIMEOUTS` in a row, close the link -- which routes through - // `#handleClosed` into `registry.disconnected` -- rather than let it keep confidently - // reporting a machine nothing can reach. `close()` is fire-and-forget here on purpose: it - // flips `#closed` synchronously before its own first `await`, which is what the `finally` - // block below needs to skip queuing another refresh behind a link that is on its way out. - if (error instanceof WorkerCallTimeoutError) { - this.#consecutiveRefreshTimeouts += 1; - if (this.#consecutiveRefreshTimeouts >= MAX_CONSECUTIVE_REFRESH_TIMEOUTS) { - this.#logger.warn("Worker stopped answering; closing its uplink", { - workerId: this.workerId, - consecutiveTimeouts: this.#consecutiveRefreshTimeouts, - }); - void this.close(); - } - } + this.#countTimeout(error); } finally { this.#refreshing = false; const queued = this.#queuedRefresh; @@ -362,6 +344,28 @@ export class WorkerLink { } } + #countTimeout(error: unknown): void { + // why: P1: a `WorkerCallTimeoutError` here is not "anything else" -- it is the one failure this + // class already knows means "the socket is not answering", and it used to mean nothing + // happened at all: `registry.refresh` was never reached, so `connection` stayed + // `connected` and `lastSeenAt` stayed stale, forever, on a half-open uplink. After + // `MAX_CONSECUTIVE_REFRESH_TIMEOUTS` in a row, close the link -- which routes through + // `#handleClosed` into `registry.disconnected` -- rather than let it keep confidently + // reporting a machine nothing can reach. `close()` is fire-and-forget here on purpose: it + // flips `#closed` synchronously before its own first `await`, which is what the `finally` + // block below needs to skip queuing another refresh behind a link that is on its way out. + if (error instanceof WorkerCallTimeoutError) { + this.#consecutiveRefreshTimeouts += 1; + if (this.#consecutiveRefreshTimeouts >= MAX_CONSECUTIVE_REFRESH_TIMEOUTS) { + this.#logger.warn("Worker stopped answering; closing its uplink", { + workerId: this.workerId, + consecutiveTimeouts: this.#consecutiveRefreshTimeouts, + }); + void this.close(); + } + } + } + // why: a failed query keeps the last view (ADR 0042 §2), so its error is dropped. async #queryUpdate(client: GatewayWorkerClient): Promise { try { diff --git a/ui/src/model/waiting.test.ts b/ui/src/model/waiting.test.ts index 1158bd69..f117f427 100644 --- a/ui/src/model/waiting.test.ts +++ b/ui/src/model/waiting.test.ts @@ -280,7 +280,7 @@ describe("the waiting table on a gateway", () => { field as { unwrap(): { element: { shape: { reason: { options: readonly string[] } } } } } ).unwrap().element.shape.reason.options; - expect(listed).toHaveLength(13); + expect(listed).toHaveLength(14); for (const reason of listed) { const text = rowOf({ workers: [{ reason, workerId: "wrk_1" } as never] }).workers?.[0]?.text; expect(text).toMatch(/^[A-Z]/); From 7b482f94877c154ac625f6e28ad6fe4f4df9bd5d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 16:45:10 +0200 Subject: [PATCH 38/42] refactor: split the worker update follow and the refresh timeout count out of two long functions (#838) --- src/cli/commands/worker.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/cli/commands/worker.ts b/src/cli/commands/worker.ts index aebe0643..7c12e9d3 100644 --- a/src/cli/commands/worker.ts +++ b/src/cli/commands/worker.ts @@ -104,6 +104,16 @@ function followOutcome( return undefined; } +function updateEnds(answer: Awaited>): { + readonly from: string; + readonly to: string; +} { + return { + from: answer.update?.from ?? answer.version, + to: answer.update?.to ?? answer.resolved ?? "", + }; +} + async function runWorkerUpdate( argv: readonly string[], client: SimlockAdminClient, @@ -135,9 +145,8 @@ async function runWorkerUpdate( environment.stdout.write(`${name} is already on the latest (${answer.version})\n`); return 0; } - const to = answer.update?.to ?? answer.resolved ?? ""; + const { from, to } = updateEnds(answer); if (values.detach === true) { - const from = answer.update?.from ?? answer.version; environment.stdout.write( `Updating ${name} ${from} → ${to}. Follow it with simlock worker list.\n`, ); From b499f6cbab30ad591cf0b37d4901c56e13fdf01d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 17:10:09 +0200 Subject: [PATCH 39/42] test: close the audit's probe gaps for worker update (#838) --- src/cli/commands/worker.ts | 2 +- src/cli/index.test.ts | 13 +++++-- src/daemon/dispatcher.test.ts | 7 ++-- src/daemon/server.test.ts | 6 ++- src/gateway/dispatcher.test.ts | 9 ++++- src/gateway/fleet-coordinator.test.ts | 16 ++++++++ src/gateway/service.test.ts | 55 ++++++++++++++++++++++++++- ui/src/model/waiting.test.ts | 1 + 8 files changed, 99 insertions(+), 10 deletions(-) diff --git a/src/cli/commands/worker.ts b/src/cli/commands/worker.ts index 7c12e9d3..52707469 100644 --- a/src/cli/commands/worker.ts +++ b/src/cli/commands/worker.ts @@ -109,7 +109,7 @@ function updateEnds(answer: Awaited { await runCli(["worker", "list"], environment); - expect(output.stdout, expected).toContain(expected); + expect(output.stdout.trimEnd().slice(-expected.length)).toBe(expected); if (!expected.includes("available")) expect(output.stdout).not.toContain("available"); if (update === null) expect(output.stdout).not.toContain("updating"); } @@ -8482,6 +8482,7 @@ describe("simlock list --requests", () => { ["incompatible", "ios", undefined, "protocol mismatch"], ["not-ready", "ios", undefined, "not ready yet"], ["unhealthy", "ios", undefined, "not running"], + ["updating", "ios", undefined, "updating"], ["drained", "ios", undefined, "drained"], ["busy", "ios", undefined, "busy with its own queue"], ["no-driver", "ios", undefined, "no iOS support"], @@ -9976,11 +9977,12 @@ describe("simlock worker update", () => { view({ update: { ...run, leases: 2, stage: "waiting" } }), view({ update: { ...run, stage: "installing" } }), view({ connection: "disconnected", update: { ...run, stage: "restarting" } }), + view({ update: { ...run, stage: "restarting" }, version: "0.4.0" }), view({ update: null, version: "0.4.0" }), ], code: 0, stdout: "mac-studio-2 (3f81a2c4) is running 0.4.0\n", - polls: 4, + polls: 5, }, { name: "fails with the stage and message", @@ -10043,7 +10045,12 @@ describe("simlock worker update", () => { return spec.answer; }, }), - sleep: async (milliseconds) => clock.advance(milliseconds), + // why: a follow that never reaches an outcome fails on this line, not by timing out. + sleep: async (milliseconds) => { + if (polls > (spec.lists ?? []).length + 2) + throw new Error(`${spec.name}: the follow never ended`); + clock.advance(milliseconds); + }, }); const code = await runCli(spec.argv, environment); diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 3c8ca418..8e3bdae1 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -793,13 +793,14 @@ describe("Dispatcher: the fleet operations on a worker", () => { expect(workers[0]?.devices?.[0]).not.toHaveProperty("driverData"); }); - it("worker.drain, worker.undrain, worker.remove, worker.install-component and alerts.test on a worker fail with UNSUPPORTED_IN_WORKER_MODE", async () => { + it("worker.drain, worker.undrain, worker.remove, worker.update, worker.install-component and alerts.test on a worker fail with UNSUPPORTED_IN_WORKER_MODE", async () => { const { components, dispatcher } = await buildDispatcher(); const install = vi.spyOn(components, "install"); const calls = [ ["worker.drain", { workerId: "instance-1" }], ["worker.undrain", { workerId: "instance-1" }], ["worker.remove", { workerId: "instance-1" }], + ["worker.update", { workerId: "instance-1" }], ["worker.install-component", { platform: "ios", version: "27.0", workers: "all" }], ["alerts.test", { destination: "ops" }], ] as const; @@ -825,10 +826,10 @@ describe("Dispatcher: the fleet operations on a worker", () => { // The refusal is the whole answer: nothing was installed on this host instead, and the // message says how to install here. expect(install).not.toHaveBeenCalled(); - expect(String((refusals[3] as { error?: unknown } | undefined)?.error)).toContain( + expect(String((refusals[4] as { error?: unknown } | undefined)?.error)).toContain( "install on this host without naming workers", ); - expect((refusals[4] as { error?: Error } | undefined)?.error?.message).toBe( + expect((refusals[5] as { error?: Error } | undefined)?.error?.message).toBe( "alerts.test sends to a gateway's alert destinations, and this daemon is not a gateway; run it against the gateway", ); }); diff --git a/src/daemon/server.test.ts b/src/daemon/server.test.ts index 75efd38e..b8c85a35 100644 --- a/src/daemon/server.test.ts +++ b/src/daemon/server.test.ts @@ -2154,7 +2154,7 @@ describe("DaemonServer lease liveness (ADR 0004)", () => { dispose.mockRestore(); }); - it("answers worker.install-component with UNSUPPORTED_IN_WORKER_MODE: a worker has no workers to install on", async () => { + it("answers worker.install-component and worker.update with UNSUPPORTED_IN_WORKER_MODE: a worker has no workers to install on or update", async () => { const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); const client = await createClient(harness.socketPath); await hello(client); @@ -2172,6 +2172,10 @@ describe("DaemonServer lease liveness (ADR 0004)", () => { }, ok: false, }); + await expect(client.request("worker.update", { workerId: "wrk_1" })).resolves.toMatchObject({ + error: { code: "UNSUPPORTED_IN_WORKER_MODE", details: { operation: "worker.update" } }, + ok: false, + }); await client.close(); }); diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 53127f2b..2674629b 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -156,6 +156,7 @@ const GATEWAY_HOST = { arch: "x64", os: "Linux", osVersion: "6.8.0", tools: [] } * for the fuller doc comment; this file only ever needs one worker at a time. */ class FakeDirectory implements WorkerDirectory { readonly clients = new Map(); + readonly refreshCalls: string[] = []; add(workerId: string, client: ScriptedWorkerClient): void { this.clients.set(workerId, client); @@ -164,7 +165,12 @@ class FakeDirectory implements WorkerDirectory { target(workerId: string): WorkerDispatchTarget | undefined { const client = this.clients.get(workerId); if (client === undefined) return undefined; - return { client: () => client.asClient(), reachable: true, refresh: async () => {}, workerId }; + return { + client: () => client.asClient(), + reachable: true, + refresh: async () => void this.refreshCalls.push(workerId), + workerId, + }; } } @@ -663,6 +669,7 @@ describe("GatewayDispatcher", () => { expect(client.updateInputs).toEqual([{ version: "0.4.0" }]); expect(answer).toMatchObject({ started: true, update, workerId: "wrk_1" }); expect(workers.view("wrk_1")).toMatchObject({ latestVersion: "0.4.0", update }); + expect(directory.refreshCalls).toEqual(["wrk_1"]); workers.connected("wrk_2", "mac-mini-2", "0.3.0"); workers.disconnected("wrk_2"); diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index e85243a8..fae0112f 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -4849,6 +4849,22 @@ describe("FleetLeaseCoordinator lists why each worker cannot take a waiting requ build: () => fleet(["wrk_a", "wrk_b"], (id) => (id === "wrk_b" ? { queueDepth: 1 } : {})), row: "busy with a queue of its own", }, + { + build: () => { + const built = fleet(["wrk_a", "wrk_b"]); + built.workers.refresh("wrk_b", { + update: { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1, + to: "0.4.0", + }, + }); + return built; + }, + row: "updating", + }, ])("leaves the request queued when the other worker is $row", async ({ build }) => { const { clients, coordinator } = build(); clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index d7a7daf5..41330438 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -1255,12 +1255,25 @@ describe("GatewayService", () => { startedAt: 1_000, to: "0.4.0", } as const; - worker.updateAnswer = { ...worker.updateAnswer, latestVersion: "0.4.0", update }; + const lastUpdateFailure = { + at: 500, + from: "0.1.0", + message: "npm failed", + stage: "installing", + to: "0.2.0", + } as const; + worker.updateAnswer = { + ...worker.updateAnswer, + lastUpdateFailure, + latestVersion: "0.4.0", + update, + }; harness.clock.advance(REFRESH_MS); await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure, latestVersion: "0.4.0", update, }), @@ -1275,6 +1288,46 @@ describe("GatewayService", () => { await harness.service.stop(); }); + it("does not let a replaced incompatible link's late update query overwrite its successor's view", async () => { + const harness = fleet(); + await harness.service.start(); + const stale = new ScriptedWorkerClient(); + stale.failWith = protocolMismatchError({ min: 5, max: 5 }); + await harness.join("wrk_1", stale); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + let answerQuery: (() => void) | undefined; + stale.updateDaemon = () => + new Promise((resolve) => { + answerQuery = () => + resolve({ + ...stale.updateAnswer, + update: { + forced: false, + from: "0.2.0", + stage: "installing", + startedAt: 1, + to: "0.4.0", + }, + }); + }); + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(answerQuery).toBeDefined()); + + await harness.join("wrk_1", new ScriptedWorkerClient("admin", "9.9.9")); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); + answerQuery?.(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + connection: "connected", + update: null, + version: "9.9.9", + }); + await harness.service.stop(); + }); + it("shows a worker's update, last update failure and latest version from its status, and a later status without an update clears it", async () => { const harness = fleet(); await harness.service.start(); diff --git a/ui/src/model/waiting.test.ts b/ui/src/model/waiting.test.ts index f117f427..3572408f 100644 --- a/ui/src/model/waiting.test.ts +++ b/ui/src/model/waiting.test.ts @@ -253,6 +253,7 @@ describe("the waiting table on a gateway", () => { ["incompatible", undefined, "Protocol mismatch"], ["not-ready", undefined, "Not ready yet"], ["unhealthy", undefined, "Not running"], + ["updating", undefined, "Updating"], ["drained", undefined, "Drained"], ["busy", undefined, "Busy with its own queue"], ["no-driver", undefined, "No iOS support"], From e9c26f8a0dbbf93dbc5601c0cb24b8dc111bfeb8 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 17:13:02 +0200 Subject: [PATCH 40/42] docs: move the worker update paragraph to a flow doc, keep the gateway manual within 150 lines (#838) --- docs/internal/flows/worker-update.md | 15 +++++++++++++++ src/gateway/AGENTS.md | 16 ++++------------ 2 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 docs/internal/flows/worker-update.md diff --git a/docs/internal/flows/worker-update.md b/docs/internal/flows/worker-update.md new file mode 100644 index 00000000..1a789316 --- /dev/null +++ b/docs/internal/flows/worker-update.md @@ -0,0 +1,15 @@ +# Worker update + +Modules: [gateway](../../../src/gateway/AGENTS.md). + +`worker.update` sends `daemon.update` over that worker's uplink, bounded at 90 s +(`UPDATE_UPLINK_BOUND_MS`, `src/gateway/dispatcher.ts`; `WORKER_NOT_CONNECTED` at the limit), then +refreshes the view. An unknown id → `UNKNOWN_WORKER`; a disconnected one → +`WORKER_NOT_CONNECTED`. An `incompatible` worker gets the same frozen call. + +- A view with `update` set is skipped by routing (`updating`): the request waits, it does + not end `NO_CAPACITY`. +- The update and the drain set are separate: an update never calls `setDrained`, and + `update: null` from the next status clears the view. +- `daemon.update` on the gateway itself goes through the `update` option + (`src/daemon/update-coordinator.ts`): no lease wait, straight to installing. diff --git a/src/gateway/AGENTS.md b/src/gateway/AGENTS.md index 6c0d9510..560569df 100644 --- a/src/gateway/AGENTS.md +++ b/src/gateway/AGENTS.md @@ -7,7 +7,7 @@ Uses: auth (`TokenOperations`), bus, history (type), contract, ports, link, admi core and leasing only the names `.oxlintrc.json` allows (request book, in-memory store, wait queue). Never imports: drivers, the rest of daemon, core's registry, capacity or lifecycle. Imported by: `../daemon/main.ts`. This manual also covers the worker's end of the uplink. -Flows: [device selection](../../docs/internal/flows/device-selection.md), [device.exec](../../docs/internal/flows/device-exec.md), [leasing](../leasing/AGENTS.md). +Flows: [device selection](../../docs/internal/flows/device-selection.md), [device.exec](../../docs/internal/flows/device-exec.md), [worker update](../../docs/internal/flows/worker-update.md), [leasing](../leasing/AGENTS.md). ## Files @@ -52,12 +52,10 @@ Flows: [device selection](../../docs/internal/flows/device-selection.md), [devic 4. `WorkerLink.start`: `hello` as `gw:`, `status.get`. Protocol mismatch → view `incompatible`; role not `admin` → close. Then `worker.connected`, `events.subscribe`, first refresh with catalog. Each of these reads waits at most 10 s (`WORKER_CALL_TIMEOUT_MS`). -5. Refresh: an `incompatible` worker is asked `daemon.update {query}` only (its `update`, `lastUpdateFailure` and - `latestVersion`; a failed query keeps the last view). Else `status.get`; a `starting` worker stops there. Else `list.get` devices, plus - `catalog.get` and `config.get` on the tick, after `component.installed`, or with no catalog yet. - One at a time; refreshes asked meanwhile merge into one. 2 timed-out refreshes in a row → close. +5. Refresh: an `incompatible` worker is asked `daemon.update {query}` only (its `update`, `lastUpdateFailure`, + `latestVersion`; a failed query keeps the last view). Else `status.get`; a `starting` worker stops there. Else `list.get` devices, plus `catalog.get` and `config.get` on the tick, after `component.installed`, or with no catalog yet. One at a time; refreshes asked meanwhile merge into one. 2 timed-out refreshes in a row → close. 6. Worker events are republished with `workerId`. `lease.*`, `device.*`, `component.install*` - trigger a refresh, and so do `daemon.update-started` and `daemon.update-failed`. A worker event named like a gateway fact (`worker.*`, `request.dispatched`, `request.granted`) is dropped. + and `daemon.update-started`/`-failed` trigger a refresh. A worker event named like a gateway fact (`worker.*`, `request.dispatched`, `request.granted`) is dropped. **Fleet request** (`FleetLeaseCoordinator.request`, under its own `SerializedDecision`; routing's `assess` and nine stages are in [routing](routing/AGENTS.md)): @@ -101,12 +99,6 @@ missing from 2 snapshots in a row → dropped; kept across a disconnect, forgott 4. Worker: `relay.*` only on the uplink session (`FORBIDDEN` elsewhere; a gateway refuses them from everyone). Target from `relayTargets`; unknown → `RELAY_TARGET_MISSING`. -**Updates.** `worker.update` sends `daemon.update` over that worker's uplink, bounded at 90 s (`WORKER_NOT_CONNECTED` at the limit), then -refreshes the view. An unknown id → `UNKNOWN_WORKER`; a disconnected one → `WORKER_NOT_CONNECTED`. An `incompatible` worker gets the same frozen call. -A view with `update` set is skipped by routing (`updating`; the request waits, it does not end `NO_CAPACITY`). The update and the drain set are separate: -an update never calls `setDrained`, and `update: null` from the next status clears the view. `daemon.update` on the gateway itself goes through -the `update` option (`../daemon/update-coordinator.ts`): no lease wait, straight to installing. - **Reads.** `status.get`, `catalog.get`, `list.get`, `lease.list` (admin: all; else own) answer from views. `worker.install-component` fans out. `nuke.run`, `cleanup.run`, `doctor.run`, `driver.passthrough`, `component.*`, `device.add`, `device.remove` → `UNSUPPORTED_IN_GATEWAY_MODE`. `usage.get` reads this gateway's `history.db` rows through `core/usage`: fleet totals, one entry per registry or named worker; `dispose` stops its read thread. From 1948466190bb01a0368984359de147cf44565170 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 17:34:14 +0200 Subject: [PATCH 41/42] fix(gateway): worker.update keeps a worker's refusal code and answers a lost link WORKER_NOT_CONNECTED; worker update says why it is up to date; ADR cites (#838) --- src/cli/commands/worker.ts | 17 ++++++++++++- src/cli/index.test.ts | 25 ++++++++++++++++++- src/contract/worker-view.ts | 2 +- src/daemon/main.ts | 2 +- src/gateway/dispatcher.test.ts | 44 +++++++++++++++++++++++++++++++++- src/gateway/dispatcher.ts | 26 ++++++++++++++++---- src/gateway/worker-link.ts | 2 +- src/http/app.test.ts | 17 ++++++++++++- 8 files changed, 124 insertions(+), 11 deletions(-) diff --git a/src/cli/commands/worker.ts b/src/cli/commands/worker.ts index 52707469..90838cb7 100644 --- a/src/cli/commands/worker.ts +++ b/src/cli/commands/worker.ts @@ -142,7 +142,9 @@ async function runWorkerUpdate( const first = await find(); const name = workerName(first, workerId); if (answer.upToDate) { - environment.stdout.write(`${name} is already on the latest (${answer.version})\n`); + environment.stdout.write( + `${upToDateMessage(name, answer, typeof values.version === "string" ? values.version : undefined)}\n`, + ); return 0; } const { from, to } = updateEnds(answer); @@ -155,6 +157,19 @@ async function runWorkerUpdate( return follow({ environment, find, first, name, to }); } +// why: the three reasons a worker is up to date, as `simlock update` tells them apart (update.ts upToDateMessage). +function upToDateMessage( + name: string, + answer: { readonly version: string; readonly resolved?: string | null }, + requested: string | undefined, +): string { + if (requested !== undefined) return `${name} is already running ${answer.version}`; + if (answer.resolved === answer.version) { + return `${name} is already on the latest (${answer.version})`; + } + return `${name} is newer than the latest (${answer.resolved}). Use --version to install another version.`; +} + async function follow(options: { readonly environment: CommandContext["environment"]; readonly find: () => Promise; diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 2368ff74..20a22edc 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -9951,7 +9951,7 @@ describe("simlock worker update", () => { ...overrides, }); - it("follows the update to is running, reports a failure with exit 1, says when the worker is on the latest, detaches, and refuses on a worker daemon", async () => { + it("follows the update to is running, reports a failure with exit 1, says why the worker is up to date, detaches, and refuses on a worker daemon", async () => { const failure = { at: 2_000, from: "0.3.0", @@ -10001,6 +10001,29 @@ describe("simlock worker update", () => { stdout: "mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)\n", polls: 1, }, + { + name: "already running the requested version", + argv: ["worker", "update", "3f81a2c4", "--version", "0.4.0"], + answer: answer({ started: false, update: null, upToDate: true, version: "0.4.0" }), + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is already running 0.4.0\n", + polls: 1, + }, + { + name: "newer than the latest", + argv: ["worker", "update", "3f81a2c4"], + answer: answer({ + resolved: "0.3.0", + started: false, + update: null, + upToDate: true, + version: "0.4.0", + }), + code: 0, + stdout: + "mac-studio-2 (3f81a2c4) is newer than the latest (0.3.0). Use --version to install another version.\n", + polls: 1, + }, { name: "detach", argv: ["worker", "update", "3f81a2c4", "--detach"], diff --git a/src/contract/worker-view.ts b/src/contract/worker-view.ts index c12c99a7..d96c3b43 100644 --- a/src/contract/worker-view.ts +++ b/src/contract/worker-view.ts @@ -31,7 +31,7 @@ export const WORKER_VIEW_REFRESH_INTERVAL_MS = 30_000; */ export const WORKER_VIEW_CATALOG_EVENTS = ["component.installed"] as const; -// why: a worker's `update` and `lastUpdateFailure` change on these two events (ADR 0042 §2), so a gateway reads its status again at once. +// why: a worker's `update` and `lastUpdateFailure` change on these two events (ADR 0042 §11), so a gateway reads its status again at once. export const WORKER_VIEW_UPDATE_EVENTS = ["daemon.update-started", "daemon.update-failed"] as const; type Output = z.infer<(typeof OPERATIONS)[Name]["output"]>; diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 5a098914..deac8aa0 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -997,7 +997,7 @@ async function startGatewayDaemon(options: GatewayDaemonOptions): Promise { }); }); + it("worker.update answers what the worker refused with its own code, and a link that closes mid-call with WORKER_NOT_CONNECTED", async () => { + const { dispatcher, directory, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + + for (const { code, kind, expected, message } of [ + { + code: "UPDATE_RESOLVE_FAILED", + kind: "domain", + expected: "UPDATE_RESOLVE_FAILED", + message: "npm view failed", + }, + { + code: "UPDATE_UNSUPPORTED_INSTALL", + kind: "domain", + expected: "UPDATE_UNSUPPORTED_INSTALL", + message: "not npm", + }, + { + code: "UPDATE_SHARED_INSTALL", + kind: "domain", + expected: "UPDATE_SHARED_INSTALL", + message: "shared", + }, + { + code: "DAEMON_CONNECTION_LOST", + kind: "transport", + expected: "WORKER_NOT_CONNECTED", + message: "worker wrk_1 is not connected", + }, + ] as const) { + client.updateFailWith = new SimlockError(code, kind, "worker's words", {}); + await expect( + dispatcher.dispatch("worker.update", { workerId: "wrk_1" }, session()), + ).rejects.toMatchObject({ + code: expected, + message: expected === code ? "worker's words" : message, + }); + } + }); + it("an operator-drained worker stays drained through its update, and a drain during an update leaves the update alone", async () => { const { dispatcher, directory, workers } = harness(); workers.connected("wrk_1", "mac-mini-1", "0.3.0"); diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index dfc6b05a..8e201e71 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -39,6 +39,7 @@ import { createUsageReader, type UsageReader } from "../core/index.js"; import type { EventHistoryHandle } from "../history/index.js"; import { consoleUrlField, + isSimlockError, EVENTS_PAGE_DEFAULT, OPERATIONS, requestedDevice, @@ -59,7 +60,7 @@ import { NoopLogger } from "../ports/index.js"; import { aggregateCatalog, aggregateStatus, type AggregateStatusOptions } from "./aggregate.js"; import { relayComponentInstall } from "./component-relay.js"; import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; -import { liveClient, type WorkerDirectory } from "./fleet-ports.js"; +import { isUplinkLost, liveClient, type WorkerDirectory } from "./fleet-ports.js"; import type { FleetLeaseIndex } from "./lease-index.js"; import type { WorkerRegistry } from "./worker-registry.js"; @@ -95,7 +96,7 @@ export type AlertTestOutcome = type DaemonUpdateOutput = z.infer<(typeof OPERATIONS)["daemon.update"]["output"]>; -// why: ADR 0042 §4: the gateway's own update has no lease wait and goes straight to installing; `src/daemon` supplies it. +// why: ADR 0042 §6: the gateway's own update has no lease wait and goes straight to installing; `src/daemon` supplies it. export interface GatewayUpdate { request( input: z.infer<(typeof OPERATIONS)["daemon.update"]["input"]>, @@ -442,13 +443,18 @@ export class GatewayDispatcher { }); } const { workerId, ...request } = input; - const answer = await this.#withinUplinkBound(client.updateDaemon(request), workerId); + const answer = await this.#withinUplinkBound( + client.updateDaemon(request).catch((error: unknown) => { + throw relayedUpdateError(error, workerId); + }), + workerId, + ); this.options.workers.refreshUpdate(workerId, answer); await target.refresh(); return { ...answer, workerId }; }; - // why: above the worker's own npm `view` and `root -g` (30 s each); at the limit the view catches up on the next read (ADR 0042 §2). + // why: above the worker's own npm `view` and `root -g` (30 s each); at the limit the view catches up on the next read. #withinUplinkBound(call: Promise, workerId: string): Promise { return new Promise((resolve, reject) => { const timer = this.options.clock.setTimer(UPDATE_UPLINK_BOUND_MS, () => { @@ -722,3 +728,15 @@ function unsupportedByDesign( function relayRefused(): never { throw new DispatchError("FORBIDDEN", "A gateway does not accept relay operations"); } + +// why: a worker's refusal keeps its code and message so HTTP and the CLI answer it as the worker would (409, 502); a link that closed under the call +// never answered, so it is WORKER_NOT_CONNECTED like the 90 s limit. +function relayedUpdateError(error: unknown, workerId: string): unknown { + if (!isSimlockError(error)) return error; + if (isUplinkLost(error)) { + return new DispatchError("WORKER_NOT_CONNECTED", `worker ${workerId} is not connected`, { + workerId, + }); + } + return new DispatchError(error.code, error.message, error.details); +} diff --git a/src/gateway/worker-link.ts b/src/gateway/worker-link.ts index f4a81567..c95ba7b8 100644 --- a/src/gateway/worker-link.ts +++ b/src/gateway/worker-link.ts @@ -366,7 +366,7 @@ export class WorkerLink { } } - // why: a failed query keeps the last view (ADR 0042 §2), so its error is dropped. + // why: a failed query keeps the last view, so its error is dropped. async #queryUpdate(client: GatewayWorkerClient): Promise { try { const answer = await this.#withTimeout(client.updateDaemon({ query: true }), "daemon.update"); diff --git a/src/http/app.test.ts b/src/http/app.test.ts index 3fda43ed..9e63e620 100644 --- a/src/http/app.test.ts +++ b/src/http/app.test.ts @@ -2563,7 +2563,7 @@ describe("POST /v1/update", () => { }), ); - it("answers 202 when the call began the update and 200 when it followed one or nothing was needed, dispatches daemon.update with the body, refuses an agent and a bad body, and passes a gateway's refusal on", async () => { + it("answers 202 when the call began the update and 200 when it followed one or nothing was needed, dispatches daemon.update with the body, refuses an agent and a bad body", async () => { const { app, dispatcher } = buildHarness(); // why: `daemon.update`'s own role check runs through the contract's real pipeline; the route adds none. let reply: Record = answer({ started: true, resolved: "0.4.0" }); @@ -2672,6 +2672,21 @@ describe("POST /v1/workers/{id}/update", () => { expect({ body, status: refused.status }).toEqual({ body, status: 400 }); } expect(dispatcher.calls).toHaveLength(callsBefore); + + for (const [code, status] of [ + ["WORKER_NOT_CONNECTED", 409], + ["UPDATE_UNSUPPORTED_INSTALL", 409], + ["UPDATE_SHARED_INSTALL", 409], + ["UPDATE_RESOLVE_FAILED", 502], + ["UNKNOWN_WORKER", 404], + ] as const) { + dispatcher.handlers["worker.update"] = () => { + throw new DispatchError(code, "refused"); + }; + const refusal = await post(); + expect({ code, status: refusal.status }).toEqual({ code, status }); + expect(((await refusal.json()) as { error: { code: string } }).error.code).toBe(code); + } }); }); From b59cebaf8e15d8dcf6d54071db3d4e07784ecf10 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Thu, 22 Oct 2026 17:44:40 +0200 Subject: [PATCH 42/42] test(gateway): worker.update's refusal arrives as a DispatchError, the form HTTP classifies (#838) --- src/gateway/dispatcher.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index b7535bfd..2fea31b6 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -742,9 +742,15 @@ describe("GatewayDispatcher", () => { }, ] as const) { client.updateFailWith = new SimlockError(code, kind, "worker's words", {}); - await expect( - dispatcher.dispatch("worker.update", { workerId: "wrk_1" }, session()), - ).rejects.toMatchObject({ + // why: HTTP maps a rejection through classifyError, which knows DispatchError and not a raw SimlockError. + const refusal = await dispatcher + .dispatch("worker.update", { workerId: "wrk_1" }, session()) + .then( + () => undefined, + (error: unknown) => error, + ); + expect(refusal).toBeInstanceOf(DispatchError); + expect(refusal).toMatchObject({ code: expected, message: expected === code ? "worker's words" : message, });