diff --git a/docs/internal/flows/worker-update.md b/docs/internal/flows/worker-update.md new file mode 100644 index 000000000..1a7893168 --- /dev/null +++ b/docs/internal/flows/worker-update.md @@ -0,0 +1,15 @@ +# Worker update + +Modules: [gateway](../../../src/gateway/AGENTS.md). + +`worker.update` sends `daemon.update` over that worker's uplink, bounded at 90 s +(`UPDATE_UPLINK_BOUND_MS`, `src/gateway/dispatcher.ts`; `WORKER_NOT_CONNECTED` at the limit), then +refreshes the view. An unknown id → `UNKNOWN_WORKER`; a disconnected one → +`WORKER_NOT_CONNECTED`. An `incompatible` worker gets the same frozen call. + +- A view with `update` set is skipped by routing (`updating`): the request waits, it does + not end `NO_CAPACITY`. +- The update and the drain set are separate: an update never calls `setDrained`, and + `update: null` from the next status clears the view. +- `daemon.update` on the gateway itself goes through the `update` option + (`src/daemon/update-coordinator.ts`): no lease wait, straight to installing. diff --git a/src/admin/index.ts b/src/admin/index.ts index 1715d3414..f8ff19c94 100644 --- a/src/admin/index.ts +++ b/src/admin/index.ts @@ -84,6 +84,8 @@ export type { WorkerListOutput, WorkerRemoveOutput, WorkerUndrainOutput, + WorkerUpdateInput, + WorkerUpdateOutput, WorkerView, } from "../simlock-client/client.js"; export { isSimlockError, SimlockError } from "../simlock-client/client.js"; diff --git a/src/cli/commands/list.ts b/src/cli/commands/list.ts index 7e43fbe52..61bb621b8 100644 --- a/src/cli/commands/list.ts +++ b/src/cli/commands/list.ts @@ -73,6 +73,7 @@ const GATEWAY_REASON_TEXT: Readonly< "not-ready": () => "Not ready yet", "runtime-missing": () => "OS not installed", unhealthy: () => "Not running", + updating: () => "Updating", "unknown-model": () => "Model not offered", }; diff --git a/src/cli/commands/worker.ts b/src/cli/commands/worker.ts index 1a4cf6df8..90838cb7b 100644 --- a/src/cli/commands/worker.ts +++ b/src/cli/commands/worker.ts @@ -15,7 +15,8 @@ const WORKER_USAGE = "Usage: simlock worker list [--json]\n" + " simlock worker drain \n" + " simlock worker undrain \n" + - " simlock worker remove \n"; + " simlock worker remove \n" + + " simlock worker update [--version ] [--force] [--detach]\n"; async function run(argv: readonly string[], context: CommandContext): Promise { const { environment, token } = context; @@ -28,6 +29,7 @@ async function run(argv: readonly string[], context: CommandContext): Promise>["workers"][number]; + +function workerName(view: WorkerView | undefined, workerId: string): string { + return view?.label === undefined ? workerId : `${view.label} (${workerId})`; +} + +// why: the follow ends on the worker's own view: its version is the target with no update running, or its last failure names the target. +function followOutcome( + view: WorkerView | undefined, + to: string, + name: string, +): { readonly code: number; readonly text: string } | undefined { + const failure = view?.lastUpdateFailure; + if (failure?.to === to) { + return { code: 1, text: `Update failed (${failure.stage}): ${failure.message}` }; + } + if (view?.version === to && view.update == null) { + return { code: 0, text: `${name} is running ${to}` }; + } + return undefined; +} + +function updateEnds(answer: Awaited>): { + readonly from: string; + readonly to: string; +} { + return { + from: answer.version, + to: answer.update?.to ?? answer.resolved ?? "", + }; +} + +async function runWorkerUpdate( + argv: readonly string[], + client: SimlockAdminClient, + environment: CommandContext["environment"], +): Promise { + const values = commandArgs(argv, { + detach: { type: "boolean" }, + force: { type: "boolean" }, + help: { type: "boolean", short: "h" }, + version: { type: "string" }, + }); + if (values.help) { + environment.stdout.write( + "Usage: simlock worker update [--version ] [--force] [--detach]\n", + ); + return 0; + } + const workerId = requiredPositional(values.positionals, "worker-id"); + const answer = await client.updateWorker({ + workerId, + ...(typeof values.version === "string" ? { version: values.version } : {}), + ...(values.force === true ? { force: true } : {}), + }); + const find = async (): Promise => + (await client.listWorkers()).workers.find((worker) => worker.id === workerId); + const first = await find(); + const name = workerName(first, workerId); + if (answer.upToDate) { + environment.stdout.write( + `${upToDateMessage(name, answer, typeof values.version === "string" ? values.version : undefined)}\n`, + ); + return 0; + } + const { from, to } = updateEnds(answer); + if (values.detach === true) { + environment.stdout.write( + `Updating ${name} ${from} → ${to}. Follow it with simlock worker list.\n`, + ); + return 0; + } + return follow({ environment, find, first, name, to }); +} + +// why: the three reasons a worker is up to date, as `simlock update` tells them apart (update.ts upToDateMessage). +function upToDateMessage( + name: string, + answer: { readonly version: string; readonly resolved?: string | null }, + requested: string | undefined, +): string { + if (requested !== undefined) return `${name} is already running ${answer.version}`; + if (answer.resolved === answer.version) { + return `${name} is already on the latest (${answer.version})`; + } + return `${name} is newer than the latest (${answer.resolved}). Use --version to install another version.`; +} + +async function follow(options: { + readonly environment: CommandContext["environment"]; + readonly find: () => Promise; + readonly first: WorkerView | undefined; + readonly name: string; + readonly to: string; +}): Promise { + const { environment, find, name, to } = options; + for (let view = options.first; ; view = await find()) { + const outcome = followOutcome(view, to, name); + if (outcome !== undefined) { + environment.stdout.write(`${outcome.text}\n`); + return outcome.code; + } + await environment.sleep(FOLLOW_POLL_MS); + } +} + export { run }; diff --git a/src/cli/format.ts b/src/cli/format.ts index 9b6790e6e..602c16190 100644 --- a/src/cli/format.ts +++ b/src/cli/format.ts @@ -37,7 +37,7 @@ export function formatWorkers( : ` -- ${entry.name} ${entry.state}`, ) .join(""); - return `${label}: ${state} -- ${capacity}, ${leaseCount(worker)}${skew}${host}${capabilities}${load}${installs}`; + return `${label}: ${state} -- ${capacity}, ${leaseCount(worker)}${skew}${workerVersion(worker)}${host}${capabilities}${load}${installs}`; }) .join("\n"); } @@ -57,6 +57,21 @@ function workerState(worker: WorkerView): string { ].join(", "); } +function workerVersion(worker: WorkerView): string { + const { latestVersion, update, version } = worker; + const available = + version !== undefined && latestVersion != null && isNewerVersion(latestVersion, version) + ? ` (${latestVersion} available)` + : ""; + const running = version === undefined ? "" : ` -- simlock ${version}${available}`; + return `${running}${update == null ? "" : ` -- updating to ${update.to}: ${updateStage(update)}`}`; +} + +function updateStage(update: NonNullable): string { + if (update.stage !== "waiting") return update.stage; + return `waiting for ${update.leases} ${update.leases === 1 ? "lease" : "leases"}`; +} + function leaseCount(worker: WorkerView): string { if (worker.leases === undefined) return "leases unknown"; return `${String(worker.leases.length)} lease(s)`; diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 7b75f3fb9..20a22edc9 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -65,7 +65,7 @@ import type { StatusGetOutput, TokenListOutput, } from "../simlock-client/client.js"; -import { connectSimlockAdmin } from "../admin/index.js"; +import { connectSimlockAdmin, type WorkerView } from "../admin/index.js"; import { RELEASE_TIMEOUT_MS } from "../lease-policy/index.js"; import { runCli, type CliEnvironment, type CliEnvironmentPorts } from "./index.js"; import { buildCliEnvironment } from "./environment.js"; @@ -4283,6 +4283,70 @@ describe("CLI: worker commands (ADR 0005 §8/§23)", () => { expect(output.stdout).toContain("protocol 4-4 vs gateway 5-5"); }); + it("worker list shows a worker's version, the latest only when newer, and its update stage", async () => { + const waiting = { + forced: false, + from: "0.3.0", + leases: 2, + stage: "waiting", + startedAt: 1_000, + to: "0.4.0", + } as const; + const cases: readonly { + expected: string; + latestVersion: string | null; + update: WorkerView["update"]; + version: string | undefined; + }[] = [ + { + expected: " -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: waiting for 2 leases", + latestVersion: "0.4.0", + update: waiting, + version: "0.3.0", + }, + { + expected: " -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: installing", + latestVersion: "0.4.0", + update: { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + }, + version: "0.3.0", + }, + { + expected: " -- updating to 0.4.0: waiting for 1 lease", + latestVersion: null, + update: { ...waiting, leases: 1 }, + version: undefined, + }, + { expected: " -- simlock 0.4.0", latestVersion: "0.4.0", update: null, version: "0.4.0" }, + { expected: " -- simlock 0.5.0", latestVersion: "0.4.0", update: null, version: "0.5.0" }, + { expected: " -- simlock 0.4.0", latestVersion: null, update: null, version: "0.4.0" }, + ]; + for (const { expected, latestVersion, update, version } of cases) { + const output = outputCapture(); + const worker = { + ...connectedWorker, + latestVersion, + update, + ...(version === undefined ? {} : { version }), + }; + const environment = output.environmentWith({ + connectAdmin: async () => + fakeClient({ listWorkers: () => Promise.resolve({ workers: [worker] }) }), + }); + + await runCli(["worker", "list"], environment); + + expect(output.stdout.trimEnd().slice(-expected.length)).toBe(expected); + if (!expected.includes("available")) expect(output.stdout).not.toContain("available"); + if (update === null) expect(output.stdout).not.toContain("updating"); + } + }); + it("emits the raw contract value under --json", async () => { const output = outputCapture(); const environment = output.environmentWith({ @@ -7724,6 +7788,18 @@ function fakeClient(overrides: Partial = {}): SimlockAdminCl undrainWorker: (input) => Promise.resolve({ drained: false as const, workerId: input.workerId }), removeWorker: (input) => Promise.resolve({ removed: true, workerId: input.workerId }), + updateWorker: (input) => + Promise.resolve({ + health: "running" as const, + lastUpdateFailure: null, + latestVersion: null, + resolved: null, + started: false, + update: null, + upToDate: false, + version: "0.3.0", + workerId: input.workerId, + }), testAlertDestination: (input) => Promise.resolve({ destination: input.destination, status: 200 }), installComponent: (input) => @@ -8406,6 +8482,7 @@ describe("simlock list --requests", () => { ["incompatible", "ios", undefined, "protocol mismatch"], ["not-ready", "ios", undefined, "not ready yet"], ["unhealthy", "ios", undefined, "not running"], + ["updating", "ios", undefined, "updating"], ["drained", "ios", undefined, "drained"], ["busy", "ios", undefined, "busy with its own queue"], ["no-driver", "ios", undefined, "no iOS support"], @@ -9847,3 +9924,174 @@ describe("CLI: a service manager call fails, or the platform has none (#830)", ( }); }); }); + +describe("simlock worker update", () => { + type UpdateAnswer = Awaited>; + type View = Awaited>["workers"][number]; + const view = (overrides: Partial = {}): View => ({ + connection: "connected", + drained: false, + id: "3f81a2c4", + label: "mac-studio-2", + lastSeenAt: 1_000, + version: "0.3.0", + ...overrides, + }); + const run = { forced: false, from: "0.3.0", startedAt: 1_000, to: "0.4.0" } as const; + const answer = (overrides: Partial = {}): UpdateAnswer => ({ + health: "running", + lastUpdateFailure: null, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: { ...run, stage: "installing" }, + upToDate: false, + version: "0.3.0", + workerId: "3f81a2c4", + ...overrides, + }); + + it("follows the update to is running, reports a failure with exit 1, says why the worker is up to date, detaches, and refuses on a worker daemon", async () => { + const failure = { + at: 2_000, + from: "0.3.0", + message: "npm exited 1", + stage: "installing", + to: "0.4.0", + } as const; + const cases: readonly { + readonly name: string; + readonly argv: readonly string[]; + readonly answer: UpdateAnswer | SimlockError; + readonly lists?: readonly View[]; + readonly code: number; + readonly stdout?: string; + readonly stderr?: string; + readonly polls?: number; + }[] = [ + { + name: "follows to running", + argv: ["worker", "update", "3f81a2c4", "--version", "0.4.0", "--force"], + answer: answer(), + lists: [ + view({ update: { ...run, leases: 2, stage: "waiting" } }), + view({ update: { ...run, stage: "installing" } }), + view({ connection: "disconnected", update: { ...run, stage: "restarting" } }), + view({ update: { ...run, stage: "restarting" }, version: "0.4.0" }), + view({ update: null, version: "0.4.0" }), + ], + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is running 0.4.0\n", + polls: 5, + }, + { + name: "fails with the stage and message", + argv: ["worker", "update", "3f81a2c4"], + answer: answer(), + lists: [view({ lastUpdateFailure: failure, update: null })], + code: 1, + stdout: "Update failed (installing): npm exited 1\n", + polls: 1, + }, + { + name: "already on the latest", + argv: ["worker", "update", "3f81a2c4"], + answer: answer({ started: false, update: null, upToDate: true, version: "0.4.0" }), + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)\n", + polls: 1, + }, + { + name: "already running the requested version", + argv: ["worker", "update", "3f81a2c4", "--version", "0.4.0"], + answer: answer({ started: false, update: null, upToDate: true, version: "0.4.0" }), + code: 0, + stdout: "mac-studio-2 (3f81a2c4) is already running 0.4.0\n", + polls: 1, + }, + { + name: "newer than the latest", + argv: ["worker", "update", "3f81a2c4"], + answer: answer({ + resolved: "0.3.0", + started: false, + update: null, + upToDate: true, + version: "0.4.0", + }), + code: 0, + stdout: + "mac-studio-2 (3f81a2c4) is newer than the latest (0.3.0). Use --version to install another version.\n", + polls: 1, + }, + { + name: "detach", + argv: ["worker", "update", "3f81a2c4", "--detach"], + answer: answer(), + code: 0, + stdout: + "Updating mac-studio-2 (3f81a2c4) 0.3.0 → 0.4.0. Follow it with simlock worker list.\n", + polls: 1, + }, + { + name: "worker daemon", + argv: ["worker", "update", "3f81a2c4"], + answer: new SimlockError( + "UNSUPPORTED_IN_WORKER_MODE", + "domain", + "worker.update is a gateway operation", + { operation: "worker.update" }, + ), + code: 2, + stderr: "worker.update is a gateway operation", + polls: 0, + }, + ]; + for (const spec of cases) { + const output = outputCapture(); + const clock = new FakeClock(0); + const sent: unknown[] = []; + const lists = [...(spec.lists ?? [view()])]; + let polls = 0; + const environment = output.environmentWith({ + clock, + connectAdmin: async () => + fakeClient({ + listWorkers: async () => { + polls += 1; + const next = lists.length > 1 ? lists.shift() : lists[0]; + return { workers: next === undefined ? [] : [next] }; + }, + updateWorker: async (input) => { + sent.push(input); + if (spec.answer instanceof Error) throw spec.answer; + return spec.answer; + }, + }), + // why: a follow that never reaches an outcome fails on this line, not by timing out. + sleep: async (milliseconds) => { + if (polls > (spec.lists ?? []).length + 2) + throw new Error(`${spec.name}: the follow never ended`); + clock.advance(milliseconds); + }, + }); + + const code = await runCli(spec.argv, environment); + + expect({ code, name: spec.name, stdout: output.stdout }).toEqual({ + code: spec.code, + name: spec.name, + stdout: spec.stdout ?? "", + }); + if (spec.stderr !== undefined) expect(output.stderr).toContain(spec.stderr); + expect({ name: spec.name, polls }).toEqual({ name: spec.name, polls: spec.polls ?? 0 }); + expect(sent).toEqual([ + { + workerId: "3f81a2c4", + ...(spec.argv.includes("--version") ? { version: "0.4.0" } : {}), + ...(spec.argv.includes("--force") ? { force: true } : {}), + }, + ]); + } + }); +}); diff --git a/src/contract/AGENTS.md b/src/contract/AGENTS.md index 4852d97e0..890324419 100644 --- a/src/contract/AGENTS.md +++ b/src/contract/AGENTS.md @@ -56,7 +56,7 @@ Three identities: ## Protocol version -- `PROTOCOL_VERSION_RANGE` in `protocol.ts` holds it, now `{ min: 56, max: 56 }`. `DAEMON_PROTOCOL_VERSION` (`src/daemon-protocol/`) is its `max`. +- `PROTOCOL_VERSION_RANGE` in `protocol.ts` holds it, now `{ min: 57, max: 57 }`. `DAEMON_PROTOCOL_VERSION` (`src/daemon-protocol/`) is its `max`. - Every change to a contract shape bumps it by one in the same PR. Move `min` too, unless a compatibility path stays side by side. `.agents/scripts/protocol-bump-check.mjs` checks it on push and in CI; it counts every file here as a shape but tests, `index.ts` and the @@ -92,9 +92,12 @@ Three identities: - `workerViewFields` builds it from `status.get`, `list.get`, `catalog.get` and `config.get` answers. A gateway's `WorkerLink` and a worker answering about itself both call it; the caller adds `id`, `label`, `connection`, `drained`, `lastSeenAt`, `version`. +- `update`, `lastUpdateFailure` and `latestVersion` come from `status.get`'s `daemon` block; `null` when there + is nothing to show, and absent until the first status read. A status without an update clears `update`. - A `starting` worker's view is `health`, `host` and `load` only. Absent, never empty. - Full re-read every `WORKER_VIEW_REFRESH_INTERVAL_MS` (30 s); catalog at once on - `WORKER_VIEW_CATALOG_EVENTS` (`component.installed`). + `WORKER_VIEW_CATALOG_EVENTS` (`component.installed`); status at once on + `WORKER_VIEW_UPDATE_EVENTS` (`daemon.update-started`, `daemon.update-failed`). ## OS ranges and durations diff --git a/src/contract/errors.ts b/src/contract/errors.ts index d54e4bbf4..f7b0595a5 100644 --- a/src/contract/errors.ts +++ b/src/contract/errors.ts @@ -126,6 +126,8 @@ export interface ErrorDetailsMap { * answers `{removed: false}` instead: forgetting something already forgotten is done, not an * error.) */ UNKNOWN_WORKER: { readonly workerId: string }; + // why: ADR 0042 §2: `worker.update` names a worker the gateway knows but has no open uplink to. + WORKER_NOT_CONNECTED: { readonly workerId: string }; /** * ADR 0005 §28: the worker that owns this lease (or would serve this request) has no open * uplink. `kind: "transport"`, because it says the same thing `DAEMON_CONNECTION_LOST` says @@ -200,6 +202,7 @@ const CODES_WITH_DECLARED_DETAILS_BY_CODE: Record UNSUPPORTED_IN_WORKER_MODE: true, WORKER_CONNECTED: true, UNKNOWN_WORKER: true, + WORKER_NOT_CONNECTED: true, WORKER_UNREACHABLE: true, ALERT_SEND_FAILED: true, UNKNOWN_ALERT_DESTINATION: true, @@ -436,6 +439,12 @@ export const ERROR_TABLE: { readonly [Code in SimlockErrorCode]: ErrorTableEntry // Exit 12, beside `UNKNOWN_MODEL`: the "what you named does not exist" class, which a script // branches on differently from "the fleet is busy" or "that cannot apply here". UNKNOWN_WORKER: { code: "UNKNOWN_WORKER", kind: "domain", cliExitCode: 12, httpStatus: 404 }, + WORKER_NOT_CONNECTED: { + code: "WORKER_NOT_CONNECTED", + kind: "domain", + cliExitCode: 2, + httpStatus: 409, + }, WORKER_UNREACHABLE: { code: "WORKER_UNREACHABLE", kind: "transport", diff --git a/src/contract/operations.test.ts b/src/contract/operations.test.ts index 9ada2f163..20dc2367b 100644 --- a/src/contract/operations.test.ts +++ b/src/contract/operations.test.ts @@ -58,6 +58,7 @@ const ROLE_MATRIX: ReadonlyArray<{ { name: "worker.drain", input: { workerId: "wrk_1" }, role: "admin" }, { name: "worker.undrain", input: { workerId: "wrk_1" }, role: "admin" }, { name: "worker.remove", input: { workerId: "wrk_1" }, role: "admin" }, + { name: "worker.update", input: { workerId: "wrk_1" }, role: "admin" }, // ADR 0010 §6: an operator's consent to a download. { name: "component.install", input: { platform: "ios", version: "26.4" }, role: "admin" }, { name: "component.list", input: {}, role: "agent" }, @@ -146,6 +147,7 @@ const EFFECT_MATRIX: ReadonlyArray<{ { name: "worker.drain", input: { workerId: "wrk_1" }, effect: "write" }, { name: "worker.undrain", input: { workerId: "wrk_1" }, effect: "write" }, { name: "worker.remove", input: { workerId: "wrk_1" }, effect: "write" }, + { name: "worker.update", input: { workerId: "wrk_1" }, effect: "write" }, { name: "component.install", input: { platform: "ios", version: "26.4" }, effect: "write" }, { name: "component.list", input: {}, effect: "read" }, { diff --git a/src/contract/operations.ts b/src/contract/operations.ts index b1e737b69..fe34a38e0 100644 --- a/src/contract/operations.ts +++ b/src/contract/operations.ts @@ -827,6 +827,20 @@ export const workerRemove = defineOperation({ output: z.object({ workerId: z.string(), removed: z.boolean() }), }); +// why: ADR 0042 §2: the gateway sends `daemon.update` over that worker's uplink and answers with the worker's own answer plus the id. +// fallow-ignore-next-line unused-export -- consumed only through the OPERATIONS registry, not by name; still public contract surface. +export const workerUpdate = defineOperation({ + name: "worker.update", + role: "admin", + effect: "write", + input: z.object({ + workerId: z.string().min(1), + version: updateRequestedVersionSchema.optional(), + force: z.boolean().optional(), + }), + output: daemonUpdate.output.extend({ workerId: z.string() }), +}); + // ---- component.install (ADR 0010 §6) -------------------------------------------------------- /** @@ -1131,6 +1145,7 @@ export const OPERATIONS = { "worker.drain": workerDrain, "worker.undrain": workerUndrain, "worker.remove": workerRemove, + "worker.update": workerUpdate, "component.install": componentInstall, "component.list": componentList, "worker.install-component": workerInstallComponent, diff --git a/src/contract/protocol.ts b/src/contract/protocol.ts index d6da8cbf4..fff1f508e 100644 --- a/src/contract/protocol.ts +++ b/src/contract/protocol.ts @@ -10,7 +10,7 @@ export interface ProtocolRange { const protocolRangeSchema = z.object({ min: z.number().int(), max: z.number().int() }); /** Widen only once a second version is kept alive side by side with the first; a bump that leaves no shim behind moves both ends. */ -export const PROTOCOL_VERSION_RANGE: ProtocolRange = { min: 56, max: 56 }; +export const PROTOCOL_VERSION_RANGE: ProtocolRange = { min: 57, max: 57 }; const LEGACY_DAEMON_PROTOCOL_VERSION = 2; diff --git a/src/contract/schemas.ts b/src/contract/schemas.ts index 7c7a2e465..3c8ca510e 100644 --- a/src/contract/schemas.ts +++ b/src/contract/schemas.ts @@ -393,6 +393,7 @@ const gatewayWaitingReasonSchema = z.enum([ "not-ready", "unhealthy", "drained", + "updating", "busy", "no-driver", "unknown-model", @@ -1374,6 +1375,9 @@ export const workerViewSchema = z.object({ host: hostFactsSchema.optional(), installs: workerInstallsSchema.optional(), waiting: z.array(waitingRequestSchema).optional(), + update: daemonUpdateFields.update.optional(), + lastUpdateFailure: daemonUpdateFields.lastUpdateFailure.optional(), + latestVersion: daemonUpdateFields.latestVersion.optional(), }); export const statusLeaseSchema = leaseRecordSchema.extend({ diff --git a/src/contract/worker-view.test.ts b/src/contract/worker-view.test.ts index a8ccb07f6..9eb147810 100644 --- a/src/contract/worker-view.test.ts +++ b/src/contract/worker-view.test.ts @@ -303,8 +303,11 @@ describe("workerViewFields", () => { health: "running", host: HOST, installs: [], + lastUpdateFailure: null, + latestVersion: null, leases: [], queueDepth: 3, + update: null, waiting: [], }); }); @@ -349,7 +352,15 @@ describe("workerViewFields", () => { }, }); - expect(Object.keys(view).sort()).toEqual(["health", "host", "installs", "waiting"]); + expect(Object.keys(view).sort()).toEqual([ + "health", + "host", + "installs", + "lastUpdateFailure", + "latestVersion", + "update", + "waiting", + ]); }); describe("a worker's load", () => { const LOAD = { diff --git a/src/contract/worker-view.ts b/src/contract/worker-view.ts index 450977c6c..d96c3b432 100644 --- a/src/contract/worker-view.ts +++ b/src/contract/worker-view.ts @@ -31,6 +31,9 @@ export const WORKER_VIEW_REFRESH_INTERVAL_MS = 30_000; */ export const WORKER_VIEW_CATALOG_EVENTS = ["component.installed"] as const; +// why: a worker's `update` and `lastUpdateFailure` change on these two events (ADR 0042 §11), so a gateway reads its status again at once. +export const WORKER_VIEW_UPDATE_EVENTS = ["daemon.update-started", "daemon.update-failed"] as const; + type Output = z.infer<(typeof OPERATIONS)[Name]["output"]>; type WorkerView = z.infer; @@ -55,11 +58,14 @@ export type WorkerViewReport = Pick & | "devices" | "downloads" | "installs" + | "lastUpdateFailure" + | "latestVersion" | "lease" | "leases" | "load" | "physicalQueueDepth" | "queueDepth" + | "update" | "waiting" | "warmPool" > @@ -87,6 +93,9 @@ export function workerViewFields(reads: WorkerReads): WorkerViewReport { host: status.host, ...load, installs: status.installs ?? [], + lastUpdateFailure: status.daemon.lastUpdateFailure, + latestVersion: status.daemon.latestVersion, + update: status.daemon.update, waiting: status.waiting ?? [], ...(status.capabilities === undefined ? {} : { capabilities: status.capabilities }), ...(status.capacity === undefined ? {} : { capacity: status.capacity }), diff --git a/src/daemon-protocol/index.test.ts b/src/daemon-protocol/index.test.ts index 15ddebeee..ecd7507c2 100644 --- a/src/daemon-protocol/index.test.ts +++ b/src/daemon-protocol/index.test.ts @@ -9,7 +9,7 @@ import { describe("daemon protocol", () => { it("keeps the current protocol version and newline framing", () => { - expect(DAEMON_PROTOCOL_VERSION).toBe(56); + expect(DAEMON_PROTOCOL_VERSION).toBe(57); expect(serializeFrame({ id: 1, type: "hello" })).toBe('{"id":1,"type":"hello"}\n'); }); diff --git a/src/daemon/AGENTS.md b/src/daemon/AGENTS.md index 59f16ef7b..16ee6a679 100644 --- a/src/daemon/AGENTS.md +++ b/src/daemon/AGENTS.md @@ -48,7 +48,7 @@ An HTTP bind failure after a good start stops the daemon (`http-start-failed`). ## Update (`update-coordinator.ts`) -`daemon.update` (worker `Dispatcher`; a gateway answers `UNSUPPORTED_IN_GATEWAY_MODE`) is `createUpdateCoordinator`'s, in memory only. A call during an update follows it (`force` on `waiting` installs now); `query` starts nothing. +`daemon.update` (the worker `Dispatcher`, and `GatewayDispatcher` through its `update` option) is `createUpdateCoordinator`'s, in memory only. A gateway's coordinator holds no leases: its update never waits and goes straight to `installing` (`startGatewayDaemon` passes it a hold that does nothing and no leases), and `DaemonServer` takes it as the gateway arm's `update` for the result file below. A call during an update follows it (`force` on `waiting` installs now); `query` starts nothing. A new update: `NpmCli.view` resolves the version (`UPDATE_RESOLVE_FAILED`); the running version, or one newer than `latest` with no `version` named (`isNewerVersion`), is `upToDate`; then `checkInstall` (`UPDATE_UNSUPPORTED_INSTALL`) and `socketHeld` on the other role's socket (`UPDATE_SHARED_INSTALL`). Then `waiting`, leasing's update hold and `daemon.update-started`; the updater launches with no leases, with `force`, or when the last `lease.released`/`lease.expired` leaves none. Its stdout lines move the state; a failed line, a non-zero exit or a failed launch sets `lastUpdateFailure`, releases the hold and emits `daemon.update-failed`. `DaemonServer.start` calls `readResultFile()` after convergence and before health is `running`, so a `running` daemon already shows the failure in `lastUpdateFailure`; a rolled back update's `update-result.json` is deleted there, and `announceResultFile()` emits its `daemon.update-failed` once the daemon serves, so a parked `events.subscribe` receives it. @@ -58,7 +58,7 @@ Its stdout lines move the state; a failed line, a non-zero exit or a failed laun 2. **Hello** — first frame must be `hello`, else `HANDSHAKE_REQUIRED` and close. Role: the uplink connection is `admin` by acceptance; otherwise the resolver. Principal: `hello.principal`, else `SIMLOCK_AGENT_ID`, else the daemon's pid. Both are fixed for the connection's life. 3. **Version** — ranges overlap → highest common version. `PROTOCOL_VERSION_RANGE` - is `{ min: 56, max: 56 }` (`src/contract/protocol.ts`). No overlap → the + is `{ min: 57, max: 57 }` (`src/contract/protocol.ts`). No overlap → the connection stays open, and every request but `daemon.stop` gets `PROTOCOL_VERSION_UNSUPPORTED`. 4. **Request** — one frame → one `dispatch()` call. While stopping: `DAEMON_STOPPING`. diff --git a/src/daemon/dispatcher.test.ts b/src/daemon/dispatcher.test.ts index 3c8ca4186..8e3bdae13 100644 --- a/src/daemon/dispatcher.test.ts +++ b/src/daemon/dispatcher.test.ts @@ -793,13 +793,14 @@ describe("Dispatcher: the fleet operations on a worker", () => { expect(workers[0]?.devices?.[0]).not.toHaveProperty("driverData"); }); - it("worker.drain, worker.undrain, worker.remove, worker.install-component and alerts.test on a worker fail with UNSUPPORTED_IN_WORKER_MODE", async () => { + it("worker.drain, worker.undrain, worker.remove, worker.update, worker.install-component and alerts.test on a worker fail with UNSUPPORTED_IN_WORKER_MODE", async () => { const { components, dispatcher } = await buildDispatcher(); const install = vi.spyOn(components, "install"); const calls = [ ["worker.drain", { workerId: "instance-1" }], ["worker.undrain", { workerId: "instance-1" }], ["worker.remove", { workerId: "instance-1" }], + ["worker.update", { workerId: "instance-1" }], ["worker.install-component", { platform: "ios", version: "27.0", workers: "all" }], ["alerts.test", { destination: "ops" }], ] as const; @@ -825,10 +826,10 @@ describe("Dispatcher: the fleet operations on a worker", () => { // The refusal is the whole answer: nothing was installed on this host instead, and the // message says how to install here. expect(install).not.toHaveBeenCalled(); - expect(String((refusals[3] as { error?: unknown } | undefined)?.error)).toContain( + expect(String((refusals[4] as { error?: unknown } | undefined)?.error)).toContain( "install on this host without naming workers", ); - expect((refusals[4] as { error?: Error } | undefined)?.error?.message).toBe( + expect((refusals[5] as { error?: Error } | undefined)?.error?.message).toBe( "alerts.test sends to a gateway's alert destinations, and this daemon is not a gateway; run it against the gateway", ); }); diff --git a/src/daemon/dispatcher.ts b/src/daemon/dispatcher.ts index 291e4104c..17419b428 100644 --- a/src/daemon/dispatcher.ts +++ b/src/daemon/dispatcher.ts @@ -332,6 +332,7 @@ export class Dispatcher { "worker.drain": unsupportedOnWorker("worker.drain"), "worker.undrain": unsupportedOnWorker("worker.undrain"), "worker.remove": unsupportedOnWorker("worker.remove"), + "worker.update": unsupportedOnWorker("worker.update"), "worker.install-component": unsupportedOnWorker( "worker.install-component", "worker.install-component installs on a gateway's workers, and this daemon is not a gateway; install on this host without naming workers", diff --git a/src/daemon/gateway-fleet.e2e.test.ts b/src/daemon/gateway-fleet.e2e.test.ts index 6cef9ad57..df2d95a77 100644 --- a/src/daemon/gateway-fleet.e2e.test.ts +++ b/src/daemon/gateway-fleet.e2e.test.ts @@ -30,11 +30,14 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ProtocolRange } from "../contract/index.js"; import type { AlertDestination, PassthroughCommand } from "../core/index.js"; import { FakeDriver, fakeDeviceDataTables } from "../core/testing.js"; -import type { Filesystem, IpcConnector, IpcListenerFactory } from "../ports/index.js"; +import type { Filesystem, IpcConnector, IpcListenerFactory, NpmCli } from "../ports/index.js"; import { FakeHostInfo, + FakeNpmCli, + FakeUpdaterLauncher, MemoryFilesystem, NodeIpcTransport, NoopLogger, @@ -44,7 +47,7 @@ import { import { FakeWebhookPoster } from "../ports/testing.js"; import { connectSimlockAdmin } from "../admin/index.js"; import type { DispatchSession } from "./dispatch.js"; -import { startDaemon } from "./main.js"; +import { startDaemon, type StartDaemonOptions } from "./main.js"; import type { DaemonServer } from "./server.js"; function passthroughOptions(command: PassthroughCommand): object { @@ -134,6 +137,7 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { async function startGateway( alerts: { destinations?: readonly AlertDestination[]; poster?: FakeWebhookPoster } = {}, + extra: Partial = {}, ): Promise { const directory = await mkdtemp(join(tmpdir(), "simlock-e2e-gateway-")); directories.push(directory); @@ -152,6 +156,7 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { logger: new NoopLogger(), version: "1.0.0-e2e", ...(alerts.poster === undefined ? {} : { webhookPoster: alerts.poster }), + ...extra, }); daemons.push(daemon); return daemon; @@ -207,6 +212,9 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { readonly ipc?: IpcConnector & IpcListenerFactory; readonly alerts?: { destinations: readonly AlertDestination[]; poster: FakeWebhookPoster }; readonly onDriver?: (driver: FakeDriver) => void; + /** why: a worker on another protocol range is the gateway's `incompatible` worker. */ + readonly protocolVersion?: ProtocolRange; + readonly npm?: NpmCli; }): Promise<{ daemon: DaemonServer; filesystem: Filesystem; directory: string }> { const directory = options.existing?.directory ?? @@ -245,6 +253,10 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { filesystem, ...(options.ipc === undefined ? {} : { ipc: options.ipc }), logger: new NoopLogger(), + ...(options.npm === undefined ? {} : { npm: options.npm }), + ...(options.protocolVersion === undefined + ? {} + : { protocolVersion: options.protocolVersion }), version: "1.0.0-e2e", ...(options.alerts === undefined ? {} : { webhookPoster: options.alerts.poster }), }); @@ -252,6 +264,84 @@ describe("gateway fleet smoke (ADR 0005 §35)", () => { return { daemon, directory, filesystem }; } + it("worker.update reaches an incompatible worker with the frozen daemon.update and shows its answer", async () => { + const gateway = await startGateway(); + const { secret: token } = await gateway.dispatch( + "token.create", + { role: "worker", label: "old-worker" }, + adminSession(), + ); + await startWorker({ + label: "old-worker", + model: "Pixel-A", + // why: a version already on npm's latest answers `upToDate`, so no install runs in this test. + npm: new FakeNpmCli({ view: [{ ok: true, output: "0.4.0" }] }), + protocolVersion: { max: 55, min: 55 }, + stdout: "old", + token, + }); + const incompatible = await vi.waitFor(async () => { + const { workers } = await gateway.dispatch("worker.list", {}, adminSession()); + const found = workers.find((worker) => worker.connection === "incompatible"); + if (found === undefined) throw new Error("the old worker never showed incompatible"); + return found; + }); + + const answer = await gateway.dispatch( + "worker.update", + { workerId: incompatible.id }, + adminSession(), + ); + + expect(answer).toMatchObject({ started: false, upToDate: true, workerId: incompatible.id }); + const { workers } = await gateway.dispatch("worker.list", {}, adminSession()); + expect(workers.find((worker) => worker.id === incompatible.id)).toMatchObject({ + connection: "incompatible", + latestVersion: "0.4.0", + update: null, + }); + }); + + it("daemon.update on a gateway goes straight to installing with fleet leases held, and its status.get shows it", async () => { + const launcher = new FakeUpdaterLauncher(); + const filesystem = new MemoryFilesystem(); + await filesystem.mkdirp("/opt/simlock"); + const gateway = await startGateway( + {}, + { + filesystem, + npm: new FakeNpmCli({ + globalRoot: [{ ok: true, output: "/opt" }], + view: [{ ok: true, output: "2.0.0" }], + }), + updater: { launcher, otherRoleRunning: async () => false, packageRoot: "/opt/simlock" }, + }, + ); + const { secret: token } = await gateway.dispatch( + "token.create", + { role: "worker", label: "worker-a" }, + adminSession(), + ); + await startWorker({ label: "worker-a", model: "Pixel-A", stdout: "a", token }); + await vi.waitFor(async () => { + const { workers } = await gateway.dispatch("worker.list", {}, adminSession()); + expect(workerServes(workers, "Pixel-A")).toBe(true); + }); + await gateway.dispatch( + "lease.request", + { model: "Pixel-A", noWait: true, platform: "android" }, + agentSession(), + ); + + const answer = await gateway.dispatch("daemon.update", {}, adminSession()); + + const installing = { forced: false, from: "1.0.0-e2e", stage: "installing", to: "2.0.0" }; + expect(answer).toMatchObject({ resolved: "2.0.0", started: true, update: installing }); + expect(launcher.launches).toMatchObject([{ from: "1.0.0-e2e", role: "gateway", to: "2.0.0" }]); + const status = await gateway.dispatch("status.get", {}, adminSession()); + expect(status.daemon.update).toMatchObject(installing); + }); + it("emits queue.changed once when the gateway has started, with depth 0, and no capacity.changed of its own", async () => { const gateway = await startGateway(); diff --git a/src/daemon/main.test.ts b/src/daemon/main.test.ts index 4c4acb922..9de36e753 100644 --- a/src/daemon/main.test.ts +++ b/src/daemon/main.test.ts @@ -37,6 +37,7 @@ import { FakeHostInfo, FakeHostLoadProbe, FakeNpmCli, + FakeUpdaterLauncher, type NpmCli, type NpmOutcome, type HostLoadCounters, @@ -4270,3 +4271,93 @@ describe("startDaemon version check", () => { await expect.poll(() => latestOf(gateway.daemon)).toBe("0.6.0"); }); }); + +describe("startDaemon gateway update", () => { + const operator = { + manageEventSubscription: () => undefined, + via: "socket", + principal: "test-operator", + role: "admin", + } as const; + + async function startUpdatable( + launcher: FakeUpdaterLauncher, + filesystem = new MemoryFilesystem(), + given?: string, + ) { + await filesystem.mkdirp("/opt/simlock"); + const directory = given ?? (await mkdtemp(join(tmpdir(), "simlock-main-gw-update-"))); + if (given === undefined) temporaryDirectories.push(directory); + const clock = new FakeClock(1_000); + const daemon = await startDaemon({ + clock, + configOverrides: { + http: { enabled: true, host: "127.0.0.1", port: await freeLoopbackPort() }, + }, + dataDirectory: directory, + drivers: [], + filesystem, + hostLoadProbe: null, + logger: new NoopLogger(), + npm: new FakeNpmCli({ + globalRoot: [{ ok: true, output: "/opt" }], + view: [{ ok: true, output: "0.4.0" }], + }), + role: "gateway", + updater: { launcher, otherRoleRunning: async () => false, packageRoot: "/opt/simlock" }, + version: "0.3.0", + } as StartDaemonOptions); + runningDaemons.push(daemon); + return { daemon, directory, filesystem }; + } + + const daemonOf = async (daemon: DaemonServer) => + ((await daemon.dispatch("status.get", {}, operator)) as { daemon: Record }) + .daemon; + + it("a failed install shows in the gateway's status.get as lastUpdateFailure, and the gateway keeps serving", async () => { + const launcher = new FakeUpdaterLauncher({ + exit: 1, + lines: [JSON.stringify({ failed: { message: "npm exited 1", stage: "installing" } })], + }); + const { daemon } = await startUpdatable(launcher); + + await daemon.dispatch("daemon.update", {}, operator); + + await expect + .poll(async () => (await daemonOf(daemon)).lastUpdateFailure) + .toEqual({ + at: 1_000, + from: "0.3.0", + message: "npm exited 1", + stage: "installing", + to: "0.4.0", + }); + expect(await daemonOf(daemon)).toMatchObject({ health: "running", update: null }); + }); + + it("a gateway that starts after a rolled-back update reports the failure in status.get and announces daemon.update-failed", async () => { + const filesystem = new MemoryFilesystem(); + const failure = { + at: 900, + from: "0.3.0", + message: "the new version did not start", + stage: "restarting", + to: "0.4.0", + }; + const directory = await mkdtemp(join(tmpdir(), "simlock-main-gw-rolled-back-")); + temporaryDirectories.push(directory); + await filesystem.mkdirp(directory); + await filesystem.writeFileAtomic( + join(directory, "update-result.json"), + JSON.stringify(failure), + ); + const { daemon } = await startUpdatable(new FakeUpdaterLauncher(), filesystem, directory); + + expect((await daemonOf(daemon)).lastUpdateFailure).toEqual(failure); + await expect( + daemon.dispatch("events.replay", {}, operator) as Promise, + ).resolves.toContainEqual(expect.objectContaining({ event: "daemon.update-failed" })); + await expect(filesystem.exists(join(directory, "update-result.json"))).resolves.toBe(false); + }); +}); diff --git a/src/daemon/main.ts b/src/daemon/main.ts index 69aa873f4..deac8aa0c 100644 --- a/src/daemon/main.ts +++ b/src/daemon/main.ts @@ -4,7 +4,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { type Alerts, createAlerts } from "../alerts/index.js"; import { EventBus, type EventBusLogger } from "../bus/index.js"; -import { fitHostFacts, fitHostLoad } from "../contract/index.js"; +import { fitHostFacts, fitHostLoad, type ProtocolRange } from "../contract/index.js"; import { HostLoadSampler } from "../host-load/index.js"; import { createEventHistory, @@ -73,6 +73,7 @@ import { LoggingProcessRunner, NodeFileLogSink, type Clock, + type DaemonRole, type Filesystem, type HostInfo, type HostLoadProbe, @@ -98,6 +99,7 @@ import { NodeProcessSupervisor, NodeSystemStats, NodeUpdaterLauncher, + type UpdaterLauncher, NodeBytePipeLauncher, NodeTcpProbe, NodeTcpStreamConnector, @@ -135,7 +137,7 @@ import { createModuleHost, type OptionalModuleDefinition } from "../modules/inde import { optionalModules } from "../module-registry/index.js"; import { DaemonServer } from "./server.js"; import { socketHeld } from "../updater/index.js"; -import { createUpdateCoordinator } from "./update-coordinator.js"; +import { createUpdateCoordinator, type UpdateCoordinatorOptions } from "./update-coordinator.js"; import { startVersionCheck, type VersionCheck } from "./version-check.js"; import { DaemonAlreadyRunningError, @@ -175,6 +177,14 @@ export interface StartDaemonOptions { /** why: test-only, replaces the npm CLI the version check and the update coordinator run. */ readonly npm?: NpmCli; readonly processRunner?: ProcessRunner; + /** why: test-only, replaces the launcher, package root and shared-install check of the gateway's update coordinator. */ + readonly updater?: { + readonly launcher: UpdaterLauncher; + readonly otherRoleRunning: () => Promise; + readonly packageRoot: string; + }; + /** why: test-only, the protocol range the worker daemon announces. */ + readonly protocolVersion?: number | ProtocolRange; readonly processSupervisor?: ProcessSupervisor; readonly socketPath?: string; readonly systemStats?: SystemStats; @@ -184,6 +194,47 @@ export interface StartDaemonOptions { readonly webhookPoster?: WebhookPoster; } +interface UpdateOptionsParts { + readonly clock: Clock; + readonly dataDirectory: string; + readonly eventBus: EventBus; + readonly filesystem: Filesystem; + readonly ipc: IpcConnector & IpcListenerFactory; + readonly npm: NpmCli; + readonly role: DaemonRole; + readonly version: string; + readonly versionCheck: VersionCheck; +} + +// why: both roles share these; each adds what it holds leases with. +function updateOptionsFor( + parts: UpdateOptionsParts, +): Omit { + const { dataDirectory, ipc, role } = parts; + return { + clock: parts.clock, + dataDirectory, + eventBus: parts.eventBus, + filesystem: parts.filesystem, + latestVersion: () => parts.versionCheck.latest(), + launcher: new NodeUpdaterLauncher(), + logPath: join(dataDirectory, "daemon.log"), + npm: parts.npm, + otherRoleRunning: () => + socketHeld( + ipc, + resolveDaemonSocketPath( + daemonDataDirectory(resolveSimlockHome(), role === "worker" ? "gateway" : "worker"), + ), + ), + packageRoot: fileURLToPath(new URL("../../", import.meta.url)), + pid: process.pid, + role, + simlockHome: resolveSimlockHome(), + version: parts.version, + }; +} + /** Constructs the daemon's real adapters once; all state remains in the daemon. */ export async function startDaemon(options: StartDaemonOptions = {}): Promise { let opened: StateDatabase | undefined; @@ -329,9 +380,11 @@ async function composeDaemon( idGenerator, ipc, logger, + npm, optionalModules: moduleDefinitions, socketPath, state, + ...(options.updater === undefined ? {} : { updater: options.updater }), version: version, versionCheck, ...(options.webhookPoster === undefined ? {} : { webhookPoster: options.webhookPoster }), @@ -605,6 +658,7 @@ async function composeDaemon( }), leases: leasing, logger: logger.child("server"), + ...(options.protocolVersion === undefined ? {} : { protocolVersion: options.protocolVersion }), passthrough: core.catalog, processRunner, execEnv: process.env, @@ -623,28 +677,19 @@ async function composeDaemon( adminSecret: auth.adminSecret, tokens: auth.tokens, update: createUpdateCoordinator({ - npm, - latestVersion: () => versionCheck.latest(), - launcher: new NodeUpdaterLauncher(), + ...updateOptionsFor({ + clock, + dataDirectory, + eventBus, + filesystem, + ipc, + npm, + role, + version, + versionCheck, + }), leasing, registry, - eventBus, - clock, - filesystem, - otherRoleRunning: () => - socketHeld( - ipc, - resolveDaemonSocketPath( - daemonDataDirectory(resolveSimlockHome(), role === "worker" ? "gateway" : "worker"), - ), - ), - version, - packageRoot: fileURLToPath(new URL("../../", import.meta.url)), - simlockHome: resolveSimlockHome(), - dataDirectory, - role, - logPath: join(dataDirectory, "daemon.log"), - pid: process.pid, }), version: version, converge: convergeStartup, @@ -810,9 +855,11 @@ interface GatewayDaemonOptions { readonly idGenerator: IdGenerator; readonly ipc: IpcConnector & IpcListenerFactory; readonly logger: Logger; + readonly npm: NpmCli; readonly optionalModules: readonly OptionalModuleDefinition[]; readonly socketPath: string; readonly state: StateDatabase; + readonly updater?: StartDaemonOptions["updater"]; readonly version: string; readonly versionCheck: VersionCheck; readonly webhookPoster?: WebhookPoster; @@ -950,6 +997,23 @@ async function startGatewayDaemon(options: GatewayDaemonOptions): Promise {}, releaseUpdateHold: async () => {} }, + registry: { snapshot: { leases: [] } }, + }); const gatewayDispatcher = new GatewayDispatcher({ awaitReady: () => Promise.resolve(), clock, @@ -966,6 +1030,7 @@ async function startGatewayDaemon(options: GatewayDaemonOptions): Promise options.versionCheck.latest(), logger: logger.child("gateway"), + update, // why: `alerts` is built after this dispatcher but before start(), and only with destinations; without any, the option stays out. ...(config.gateway.alerts.destinations.length === 0 ? {} @@ -983,6 +1048,7 @@ async function startGatewayDaemon(options: GatewayDaemonOptions): Promise { dispose.mockRestore(); }); - it("answers worker.install-component with UNSUPPORTED_IN_WORKER_MODE: a worker has no workers to install on", async () => { + it("answers worker.install-component and worker.update with UNSUPPORTED_IN_WORKER_MODE: a worker has no workers to install on or update", async () => { const harness = await createHarness({ resolveRole: { resolve: () => "admin" } }); const client = await createClient(harness.socketPath); await hello(client); @@ -2172,6 +2172,10 @@ describe("DaemonServer lease liveness (ADR 0004)", () => { }, ok: false, }); + await expect(client.request("worker.update", { workerId: "wrk_1" })).resolves.toMatchObject({ + error: { code: "UNSUPPORTED_IN_WORKER_MODE", details: { operation: "worker.update" } }, + ok: false, + }); await client.close(); }); diff --git a/src/daemon/server.ts b/src/daemon/server.ts index 1fba05f0d..6ca208275 100644 --- a/src/daemon/server.ts +++ b/src/daemon/server.ts @@ -271,6 +271,7 @@ export type DaemonServerOptions = | (DaemonServerCommonOptions & DaemonServerEngineOptions & { readonly dispatcher?: undefined }) | (DaemonServerCommonOptions & { readonly dispatcher: ContractDispatcher; + readonly update?: UpdateCoordinator; }); type DaemonHealth = "starting" | "running" | "failed"; @@ -577,13 +578,13 @@ export class DaemonServer { this.#engine?.physicalDeviceWatcher?.start(); // why: a gateway's `events.subscribe` parked on readiness resumes only after this continuation, so the rollback's `daemon.update-failed` waits one turn of the event loop for it to subscribe. await new Promise((resolve) => setImmediate(resolve)); - if (!this.#stopping) this.#engine?.update?.announceResultFile(); + if (!this.#stopping) this.options.update?.announceResultFile(); } // why: a stale or unreadable result file must never keep the daemon from starting. async #reportUpdateResult(): Promise { try { - await this.#engine?.update?.readResultFile(); + await this.options.update?.readResultFile(); } catch (error: unknown) { this.#logger.warn("Could not read the update result file", { message: errorMessage(error) }); } @@ -1177,6 +1178,8 @@ export class DaemonServer { ); case "worker.remove": return this.#dispatcher.dispatch("worker.remove", frame.payload, this.#session(connection)); + case "worker.update": + return this.#dispatcher.dispatch("worker.update", frame.payload, this.#session(connection)); case "alerts.test": return this.#dispatcher.dispatch("alerts.test", frame.payload, this.#session(connection)); case "component.install": diff --git a/src/daemon/update-coordinator.ts b/src/daemon/update-coordinator.ts index aa6a66472..57470e6df 100644 --- a/src/daemon/update-coordinator.ts +++ b/src/daemon/update-coordinator.ts @@ -9,7 +9,6 @@ import { type UpdateFailure, type UpdateState, } from "../contract/index.js"; -import type { Registry } from "../core/index.js"; import type { UpdateHold } from "../leasing/index.js"; import { checkInstall } from "../updater/index.js"; import { DispatchError } from "./dispatch.js"; @@ -53,7 +52,7 @@ export interface UpdateCoordinatorOptions { readonly latestVersion: () => string | null; readonly launcher: UpdaterLauncher; readonly leasing: UpdateHold; - readonly registry: Pick; + readonly registry: { readonly snapshot: { readonly leases: readonly unknown[] } }; readonly eventBus: Pick; readonly clock: Clock; readonly filesystem: Filesystem; diff --git a/src/gateway/AGENTS.md b/src/gateway/AGENTS.md index 9f1b2a64e..560569dfb 100644 --- a/src/gateway/AGENTS.md +++ b/src/gateway/AGENTS.md @@ -7,29 +7,29 @@ Uses: auth (`TokenOperations`), bus, history (type), contract, ports, link, admi core and leasing only the names `.oxlintrc.json` allows (request book, in-memory store, wait queue). Never imports: drivers, the rest of daemon, core's registry, capacity or lifecycle. Imported by: `../daemon/main.ts`. This manual also covers the worker's end of the uplink. -Flows: [device selection](../../docs/internal/flows/device-selection.md), [device.exec](../../docs/internal/flows/device-exec.md), [leasing](../leasing/AGENTS.md). +Flows: [device selection](../../docs/internal/flows/device-selection.md), [device.exec](../../docs/internal/flows/device-exec.md), [worker update](../../docs/internal/flows/worker-update.md), [leasing](../leasing/AGENTS.md). ## Files -| File | Role | -| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| `service.ts` | `GatewayService`: uplink listener, one `WorkerLink` per worker, 30 s tick | -| `worker-link.ts` | One uplink, gateway side: admin client, view reads, event relay | -| `worker-registry.ts` | `WorkerRegistry`: views, drain, retention, `worker.*` events | -| `drain-store.ts` | `StateDrainStore`: the drained ids in the `drained_workers` table of `state.db`; `drainedWorkersOwner` | -| `dispatcher.ts` | `GatewayDispatcher`: the contract's handlers, answered from the fleet; `alerts.test` through the `testDestination` option | -| `fleet-coordinator.ts` | `FleetLeaseCoordinator`: admit, dispatch, retry, forward renew/release/exec | -| `queue.ts` | `FleetQueue`: leasing's `WaitQueue`, two lines | -| `lease-index.ts` | `FleetLeaseIndex`: gateway lease id → worker, worker lease id, owner | -| `routing.ts`, `routing/` | `createRoutingPolicy`: `assess` and the stage list | -| `owner-routed-facts.ts` | Relayed lease facts → `lease-lost` and device pushes to the owner | -| `aggregate.ts` | `aggregateStatus`, `aggregateCatalog`: pure, over views | -| `component-relay.ts` | `relayComponentInstall`: `worker.install-component` fan-out | -| `relay.ts` | `GatewayRelay`: one HTTP exchange to a worker's relay target | -| `fleet-ports.ts` | `WorkerDirectory`, `FleetViews`, `liveClient`, `isUplinkLost` | -| `../daemon/gateway-uplink.ts` | Worker: `GatewayUplink`, dial and redial | -| `../daemon/relay.ts` | Worker: `RelayConnection`, serves `relay.request` | -| `../link/` | Both ends: text messages plus byte channels on one WebSocket | +| File | Role | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `service.ts` | `GatewayService`: uplink listener, one `WorkerLink` per worker, 30 s tick | +| `worker-link.ts` | One uplink, gateway side: admin client, view reads, event relay | +| `worker-registry.ts` | `WorkerRegistry`: views, drain, retention, `worker.*` events | +| `drain-store.ts` | `StateDrainStore`: the drained ids in the `drained_workers` table of `state.db`; `drainedWorkersOwner` | +| `dispatcher.ts` | `GatewayDispatcher`: the contract's handlers, answered from the fleet; `alerts.test` through `testDestination`, `daemon.update` through `update` | +| `fleet-coordinator.ts` | `FleetLeaseCoordinator`: admit, dispatch, retry, forward renew/release/exec | +| `queue.ts` | `FleetQueue`: leasing's `WaitQueue`, two lines | +| `lease-index.ts` | `FleetLeaseIndex`: gateway lease id → worker, worker lease id, owner | +| `routing.ts`, `routing/` | `createRoutingPolicy`: `assess` and the stage list | +| `owner-routed-facts.ts` | Relayed lease facts → `lease-lost` and device pushes to the owner | +| `aggregate.ts` | `aggregateStatus`, `aggregateCatalog`: pure, over views | +| `component-relay.ts` | `relayComponentInstall`: `worker.install-component` fan-out | +| `relay.ts` | `GatewayRelay`: one HTTP exchange to a worker's relay target | +| `fleet-ports.ts` | `WorkerDirectory`, `FleetViews`, `liveClient`, `isUplinkLost` | +| `../daemon/gateway-uplink.ts` | Worker: `GatewayUplink`, dial and redial | +| `../daemon/relay.ts` | Worker: `RelayConnection`, serves `relay.request` | +| `../link/` | Both ends: text messages plus byte channels on one WebSocket | ## How it works @@ -52,11 +52,10 @@ Flows: [device selection](../../docs/internal/flows/device-selection.md), [devic 4. `WorkerLink.start`: `hello` as `gw:`, `status.get`. Protocol mismatch → view `incompatible`; role not `admin` → close. Then `worker.connected`, `events.subscribe`, first refresh with catalog. Each of these reads waits at most 10 s (`WORKER_CALL_TIMEOUT_MS`). -5. Refresh: `status.get`; a `starting` worker stops there. Else `list.get` devices, plus - `catalog.get` and `config.get` on the tick, after `component.installed`, or with no catalog yet. - One at a time; refreshes asked meanwhile merge into one. 2 timed-out refreshes in a row → close. +5. Refresh: an `incompatible` worker is asked `daemon.update {query}` only (its `update`, `lastUpdateFailure`, + `latestVersion`; a failed query keeps the last view). Else `status.get`; a `starting` worker stops there. Else `list.get` devices, plus `catalog.get` and `config.get` on the tick, after `component.installed`, or with no catalog yet. One at a time; refreshes asked meanwhile merge into one. 2 timed-out refreshes in a row → close. 6. Worker events are republished with `workerId`. `lease.*`, `device.*`, `component.install*` - trigger a refresh. A worker event named like a gateway fact (`worker.*`, `request.dispatched`, `request.granted`) is dropped. + and `daemon.update-started`/`-failed` trigger a refresh. A worker event named like a gateway fact (`worker.*`, `request.dispatched`, `request.granted`) is dropped. **Fleet request** (`FleetLeaseCoordinator.request`, under its own `SerializedDecision`; routing's `assess` and nine stages are in [routing](routing/AGENTS.md)): @@ -79,7 +78,7 @@ Flows: [device selection](../../docs/internal/flows/device-selection.md), [devic → exclude that worker for this request, refresh its catalog, walk again. `PROVISION_FAILED` → a refusal; every worker that could take it failed so (one that is full, busy, not ready or unhealthy could, later) → reject with the last. After a progress push it is relayed as a failure after progress. Anything else, - or a failure after progress → `lease.rejected` `worker-failed`. + or a failure after progress → `lease.rejected` `worker-failed`. `DAEMON_UPDATING` is handled as `NO_CAPACITY` (the worker began an update the view did not show yet). 7. Grant: a lease id unlike the forwarded `leaseId` → release it on the worker, retry. Gateway lease id = the caller's `leaseId`, else `.`. `leaseIndex.add`, then `request.granted`; a clash → release on worker, `LEASE_ID_TAKEN`. @@ -112,6 +111,7 @@ views. `worker.install-component` fans out. `nuke.run`, `cleanup.run`, `doctor.r | -------------------------------------- | ---------------------------------------- | ----------------------------------------------------- | | View refresh tick (+ prune) | 30 s | `WORKER_VIEW_REFRESH_INTERVAL_MS` | | One view read | 10 s | `WORKER_CALL_TIMEOUT_MS` | +| `worker.update` over the uplink | 90 s | `UPDATE_UPLINK_BOUND_MS` (`dispatcher.ts`) | | Forwarded `lease.request` | 5 min | `gateway.leaseRequestTimeoutMs` | | Forwarded `device.exec` | 11 min | `gateway.execTimeoutMs` | | Disconnected view kept | 24 h, held while a gateway lease is live | `gateway.disconnectedRetentionMs` | diff --git a/src/gateway/aggregate.ts b/src/gateway/aggregate.ts index 32affcc64..5659fd891 100644 --- a/src/gateway/aggregate.ts +++ b/src/gateway/aggregate.ts @@ -47,6 +47,8 @@ export interface AggregateStatusOptions { readonly health: StatusOutput["daemon"]["health"]; readonly version: string; readonly latestVersion?: string | null; + // why: the gateway's own update and its last failure; none when nothing is running. + readonly progress?: Pick; readonly host: StatusOutput["host"]; readonly load?: StatusOutput["load"]; readonly queueDepth: number; @@ -70,8 +72,8 @@ export function aggregateStatus( health: options.health, mode: "gateway", version: options.version, - update: null, - lastUpdateFailure: null, + update: options.progress?.update ?? null, + lastUpdateFailure: options.progress?.lastUpdateFailure ?? null, latestVersion: options.latestVersion ?? null, }, host: options.host, diff --git a/src/gateway/dispatcher.test.ts b/src/gateway/dispatcher.test.ts index 2c837b8ba..2fea31b6e 100644 --- a/src/gateway/dispatcher.test.ts +++ b/src/gateway/dispatcher.test.ts @@ -18,7 +18,7 @@ import { T0, usageRig, } from "../history/testing.js"; -import { OPERATIONS, type OperationName } from "../contract/index.js"; +import { OPERATIONS, SimlockError, type OperationName } from "../contract/index.js"; import { DispatchError, runDispatch, type DispatchSession } from "../daemon/dispatch.js"; import { FakeClock, @@ -29,7 +29,7 @@ import { } from "../ports/index.js"; import { FakeWebhookPoster } from "../ports/testing.js"; import type { TokenOperations } from "../auth/index.js"; -import { GatewayDispatcher } from "./dispatcher.js"; +import { GatewayDispatcher, type GatewayDispatcherOptions } from "./dispatcher.js"; import { MemoryDrainStore } from "./drain-store.js"; import { FleetLeaseCoordinator } from "./fleet-coordinator.js"; import type { WorkerDirectory, WorkerDispatchTarget } from "./fleet-ports.js"; @@ -156,6 +156,7 @@ const GATEWAY_HOST = { arch: "x64", os: "Linux", osVersion: "6.8.0", tools: [] } * for the fuller doc comment; this file only ever needs one worker at a time. */ class FakeDirectory implements WorkerDirectory { readonly clients = new Map(); + readonly refreshCalls: string[] = []; add(workerId: string, client: ScriptedWorkerClient): void { this.clients.set(workerId, client); @@ -164,7 +165,31 @@ class FakeDirectory implements WorkerDirectory { target(workerId: string): WorkerDispatchTarget | undefined { const client = this.clients.get(workerId); if (client === undefined) return undefined; - return { client: () => client.asClient(), reachable: true, refresh: async () => {}, workerId }; + return { + client: () => client.asClient(), + reachable: true, + refresh: async () => void this.refreshCalls.push(workerId), + workerId, + }; + } +} + +// why: the gateway's own update coordinator, scripted. +class ScriptedUpdate { + readonly requests: unknown[] = []; + progress: ReturnType = { + lastUpdateFailure: null, + latestVersion: null, + update: null, + }; + + async request(input: unknown) { + this.requests.push(input); + return { ...this.progress, resolved: "0.4.0", started: true, upToDate: false }; + } + + state() { + return this.progress; } } @@ -235,9 +260,11 @@ function harness( version: "9.9.9", workerLabel: () => undefined, }); + const update = new ScriptedUpdate(); const dispatcher = new GatewayDispatcher({ awaitReady: async () => {}, latestVersion: () => null, + update, clock, closeUplinksForToken: async (tokenId) => { closedUplinkTokens.push(tokenId); @@ -271,6 +298,7 @@ function harness( eventBus, leaseIndex, tokens, + update, workers, }; } @@ -617,6 +645,144 @@ describe("GatewayDispatcher", () => { ).resolves.toEqual({ removed: true, workerId: "wrk_1" }); }); + it("Operator calls worker.update on the gateway: connected, disconnected and unknown workers", async () => { + const { dispatcher, directory, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + const update = { + forced: false, + from: "0.3.0", + leases: 2, + stage: "waiting", + startedAt: 1_000, + to: "0.4.0", + } as const; + client.updateAnswer = { ...client.updateAnswer, latestVersion: "0.4.0", started: true, update }; + + const answer = await dispatcher.dispatch( + "worker.update", + { version: "0.4.0", workerId: "wrk_1" }, + session(), + ); + + expect(client.updateInputs).toEqual([{ version: "0.4.0" }]); + expect(answer).toMatchObject({ started: true, update, workerId: "wrk_1" }); + expect(workers.view("wrk_1")).toMatchObject({ latestVersion: "0.4.0", update }); + expect(directory.refreshCalls).toEqual(["wrk_1"]); + + workers.connected("wrk_2", "mac-mini-2", "0.3.0"); + workers.disconnected("wrk_2"); + for (const { code, message, workerId } of [ + { code: "WORKER_NOT_CONNECTED", message: "worker wrk_2 is not connected", workerId: "wrk_2" }, + { code: "UNKNOWN_WORKER", message: "Unknown worker: wrk_ghost", workerId: "wrk_ghost" }, + ]) { + await expect( + dispatcher.dispatch("worker.update", { workerId }, session()), + ).rejects.toMatchObject({ code, details: { workerId }, message }); + } + }); + + it("worker.update gives up on a worker that does not answer within 90 s with WORKER_NOT_CONNECTED", async () => { + const { clock, dispatcher, directory, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + const client = new ScriptedWorkerClient(); + client.hangingCalls.add("daemon.update"); + directory.add("wrk_1", client); + + let outcome: unknown = "pending"; + void dispatcher.dispatch("worker.update", { workerId: "wrk_1" }, session()).then( + (answer) => (outcome = answer), + (error: unknown) => (outcome = error), + ); + await vi.waitFor(() => expect(client.updateInputs).toHaveLength(1)); + clock.advance(89_999); + await Promise.resolve(); + expect(outcome).toBe("pending"); + clock.advance(1); + + await vi.waitFor(() => expect(outcome).not.toBe("pending")); + expect(outcome).toMatchObject({ + code: "WORKER_NOT_CONNECTED", + details: { workerId: "wrk_1" }, + message: "worker wrk_1 did not answer within 90 s; check simlock worker list", + }); + }); + + it("worker.update answers what the worker refused with its own code, and a link that closes mid-call with WORKER_NOT_CONNECTED", async () => { + const { dispatcher, directory, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + + for (const { code, kind, expected, message } of [ + { + code: "UPDATE_RESOLVE_FAILED", + kind: "domain", + expected: "UPDATE_RESOLVE_FAILED", + message: "npm view failed", + }, + { + code: "UPDATE_UNSUPPORTED_INSTALL", + kind: "domain", + expected: "UPDATE_UNSUPPORTED_INSTALL", + message: "not npm", + }, + { + code: "UPDATE_SHARED_INSTALL", + kind: "domain", + expected: "UPDATE_SHARED_INSTALL", + message: "shared", + }, + { + code: "DAEMON_CONNECTION_LOST", + kind: "transport", + expected: "WORKER_NOT_CONNECTED", + message: "worker wrk_1 is not connected", + }, + ] as const) { + client.updateFailWith = new SimlockError(code, kind, "worker's words", {}); + // why: HTTP maps a rejection through classifyError, which knows DispatchError and not a raw SimlockError. + const refusal = await dispatcher + .dispatch("worker.update", { workerId: "wrk_1" }, session()) + .then( + () => undefined, + (error: unknown) => error, + ); + expect(refusal).toBeInstanceOf(DispatchError); + expect(refusal).toMatchObject({ + code: expected, + message: expected === code ? "worker's words" : message, + }); + } + }); + + it("an operator-drained worker stays drained through its update, and a drain during an update leaves the update alone", async () => { + const { dispatcher, directory, workers } = harness(); + workers.connected("wrk_1", "mac-mini-1", "0.3.0"); + const client = new ScriptedWorkerClient(); + directory.add("wrk_1", client); + const update = { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + await dispatcher.dispatch("worker.drain", { workerId: "wrk_1" }, session()); + client.updateAnswer = { ...client.updateAnswer, started: true, update }; + + await dispatcher.dispatch("worker.update", { workerId: "wrk_1" }, session()); + expect(workers.view("wrk_1")).toMatchObject({ drained: true, update }); + workers.refresh("wrk_1", { update: null }); + expect(workers.view("wrk_1")).toMatchObject({ drained: true, update: null }); + + await dispatcher.dispatch("worker.undrain", { workerId: "wrk_1" }, session()); + workers.refresh("wrk_1", { update }); + await dispatcher.dispatch("worker.drain", { workerId: "wrk_1" }, session()); + expect(workers.view("wrk_1")).toMatchObject({ drained: true, update }); + }); + it("refuses to remove a connected worker with WORKER_CONNECTED, naming it", async () => { const { dispatcher, workers } = harness(); workers.connected("wrk_1", undefined, undefined); @@ -1308,16 +1474,40 @@ describe("GatewayDispatcher", () => { }); describe("daemon.update", () => { - it("answers UNSUPPORTED_IN_GATEWAY_MODE to an admin session, saying a gateway cannot update itself yet", async () => { - const { dispatcher } = harness(); + it("asks the gateway's own update coordinator and answers with its outcome, the gateway's version and its health, and status.get carries the update and its last failure", async () => { + const { dispatcher, update } = harness(); + const run = { + forced: true, + from: "0.0.0-test", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 500, + from: "0.0.0-test", + message: "npm failed", + stage: "installing", + to: "0.3.0", + } as const; + update.progress = { lastUpdateFailure: failure, latestVersion: "0.4.0", update: run }; await expect( - dispatcher.dispatch("daemon.update", {}, session({ role: "admin" })), - ).rejects.toMatchObject({ - code: "UNSUPPORTED_IN_GATEWAY_MODE", - details: { operation: "daemon.update" }, - message: "daemon.update updates one machine's daemon; a gateway cannot update itself yet", + dispatcher.dispatch("daemon.update", { force: true }, session({ role: "admin" })), + ).resolves.toEqual({ + health: "running", + lastUpdateFailure: failure, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: run, + upToDate: false, + version: "0.0.0-test", }); + expect(update.requests).toEqual([{ force: true }]); + + const status = await dispatcher.dispatch("status.get", {}, session({ role: "admin" })); + expect(status.daemon).toMatchObject({ lastUpdateFailure: failure, update: run }); }); }); @@ -1798,6 +1988,7 @@ describe("GatewayDispatcher", () => { "worker.drain": { workerId: "wrk_1" }, "worker.remove": { workerId: "wrk_1" }, "worker.undrain": { workerId: "wrk_1" }, + "worker.update": { workerId: "wrk_1" }, }; const { dispatcher, workers } = harness(); workers.connected("wrk_1", undefined, undefined); diff --git a/src/gateway/dispatcher.ts b/src/gateway/dispatcher.ts index 745ffa624..8e201e71b 100644 --- a/src/gateway/dispatcher.ts +++ b/src/gateway/dispatcher.ts @@ -39,6 +39,7 @@ import { createUsageReader, type UsageReader } from "../core/index.js"; import type { EventHistoryHandle } from "../history/index.js"; import { consoleUrlField, + isSimlockError, EVENTS_PAGE_DEFAULT, OPERATIONS, requestedDevice, @@ -59,7 +60,7 @@ import { NoopLogger } from "../ports/index.js"; import { aggregateCatalog, aggregateStatus, type AggregateStatusOptions } from "./aggregate.js"; import { relayComponentInstall } from "./component-relay.js"; import type { FleetLeaseCoordinator } from "./fleet-coordinator.js"; -import type { WorkerDirectory } from "./fleet-ports.js"; +import { isUplinkLost, liveClient, type WorkerDirectory } from "./fleet-ports.js"; import type { FleetLeaseIndex } from "./lease-index.js"; import type { WorkerRegistry } from "./worker-registry.js"; @@ -93,6 +94,16 @@ export type AlertTestOutcome = readonly error?: string; }; +type DaemonUpdateOutput = z.infer<(typeof OPERATIONS)["daemon.update"]["output"]>; + +// why: ADR 0042 §6: the gateway's own update has no lease wait and goes straight to installing; `src/daemon` supplies it. +export interface GatewayUpdate { + request( + input: z.infer<(typeof OPERATIONS)["daemon.update"]["input"]>, + ): Promise>; + state(): Pick; +} + export interface GatewayDispatcherOptions { readonly historyPath?: string; /** The gateway's own config -- what `config.get` returns (ADR 0005 §34). */ @@ -126,6 +137,7 @@ export interface GatewayDispatcherOptions { readonly health: () => "starting" | "running" | "failed"; readonly version: string; readonly latestVersion: () => string | null; + readonly update: GatewayUpdate; readonly host: AggregateStatusOptions["host"]; readonly load?: () => AggregateStatusOptions["load"]; readonly awaitReady: () => Promise; @@ -193,6 +205,8 @@ function alertTestError( } } +const UPDATE_UPLINK_BOUND_MS = 90_000; + export class GatewayDispatcher { readonly #logger: Logger; readonly #dispatchLogger: Logger; @@ -233,11 +247,9 @@ export class GatewayDispatcher { "worker.drain": this.#workerDrain, "worker.undrain": this.#workerUndrain, "worker.remove": this.#workerRemove, + "worker.update": this.#workerUpdate, "alerts.test": this.#alertsTest, - "daemon.update": unsupportedByDesign( - "daemon.update", - "daemon.update updates one machine's daemon; a gateway cannot update itself yet", - ), + "daemon.update": this.#daemonUpdate, "worker.install-component": this.#workerInstallComponent, "lease.list": this.#leaseList, "list.get": this.#listGet, @@ -326,9 +338,7 @@ export class GatewayDispatcher { health: "starting", mode: "gateway", version: this.options.version, - update: null, - lastUpdateFailure: null, - latestVersion: this.options.latestVersion(), + ...this.options.update.state(), ...consoleUrl, }, history, @@ -340,6 +350,7 @@ export class GatewayDispatcher { health: this.options.health(), version: this.options.version, latestVersion: this.options.latestVersion(), + progress: this.options.update.state(), host: this.options.host, ...(load === undefined ? {} : { load }), physicalQueueDepth: this.options.coordinator.physicalQueueDepth, @@ -411,6 +422,63 @@ export class GatewayDispatcher { return { drained: false, workerId: input.workerId }; }; + #daemonUpdate: Handler<"daemon.update"> = async (input) => ({ + ...(await this.options.update.request(input)), + health: this.options.health(), + version: this.options.version, + }); + + #workerUpdate: Handler<"worker.update"> = async (input) => { + const view = this.options.workers.view(input.workerId); + if (view === undefined) { + throw new DispatchError("UNKNOWN_WORKER", `Unknown worker: ${input.workerId}`, { + workerId: input.workerId, + }); + } + const target = this.options.directory.target(input.workerId); + const client = liveClient(target); + if (view.connection === "disconnected" || client === undefined || target === undefined) { + throw new DispatchError("WORKER_NOT_CONNECTED", `worker ${input.workerId} is not connected`, { + workerId: input.workerId, + }); + } + const { workerId, ...request } = input; + const answer = await this.#withinUplinkBound( + client.updateDaemon(request).catch((error: unknown) => { + throw relayedUpdateError(error, workerId); + }), + workerId, + ); + this.options.workers.refreshUpdate(workerId, answer); + await target.refresh(); + return { ...answer, workerId }; + }; + + // why: above the worker's own npm `view` and `root -g` (30 s each); at the limit the view catches up on the next read. + #withinUplinkBound(call: Promise, workerId: string): Promise { + return new Promise((resolve, reject) => { + const timer = this.options.clock.setTimer(UPDATE_UPLINK_BOUND_MS, () => { + reject( + new DispatchError( + "WORKER_NOT_CONNECTED", + `worker ${workerId} did not answer within 90 s; check simlock worker list`, + { workerId }, + ), + ); + }); + call.then( + (value) => { + this.options.clock.cancel(timer); + resolve(value); + }, + (error: unknown) => { + this.options.clock.cancel(timer); + reject(error); + }, + ); + }); + } + #alertsTest: Handler<"alerts.test"> = async (input) => { const { destination } = input; const result = (await this.options.testDestination?.(destination)) ?? UNKNOWN_DESTINATION; @@ -660,3 +728,15 @@ function unsupportedByDesign( function relayRefused(): never { throw new DispatchError("FORBIDDEN", "A gateway does not accept relay operations"); } + +// why: a worker's refusal keeps its code and message so HTTP and the CLI answer it as the worker would (409, 502); a link that closed under the call +// never answered, so it is WORKER_NOT_CONNECTED like the 90 s limit. +function relayedUpdateError(error: unknown, workerId: string): unknown { + if (!isSimlockError(error)) return error; + if (isUplinkLost(error)) { + return new DispatchError("WORKER_NOT_CONNECTED", `worker ${workerId} is not connected`, { + workerId, + }); + } + return new DispatchError(error.code, error.message, error.details); +} diff --git a/src/gateway/fleet-coordinator.test.ts b/src/gateway/fleet-coordinator.test.ts index 8751b329a..fae0112f0 100644 --- a/src/gateway/fleet-coordinator.test.ts +++ b/src/gateway/fleet-coordinator.test.ts @@ -1043,6 +1043,37 @@ describe("FleetLeaseCoordinator dispatch", () => { expect(coordinator.queueDepth).toBe(1); }); + it("handles a worker's DAEMON_UPDATING answer as NO_CAPACITY: the view is read again, the request stays queued and is placed once the view changes", async () => { + const { coordinator, directory, workers } = harness(); + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + connectWorker(workers, "wrk_a"); + client.requestLeaseQueue.push({ + error: new SimlockError("DAEMON_UPDATING", "domain", "The daemon is updating", {}), + kind: "error", + }); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + + const outcome = coordinator.request(REQUEST, requestOptions()); + const state = promiseState(outcome); + await tick(); + + expect(state.state).toBe("pending"); + expect(coordinator.queueDepth).toBe(1); + expect(directory.refreshCalls).toEqual(["wrk_a"]); + + workers.refresh("wrk_a", { + update: { forced: false, from: "0.3.0", stage: "installing", startedAt: 1, to: "0.4.0" }, + }); + await tick(); + expect(state.state).toBe("pending"); + + workers.refresh("wrk_a", { update: null }); + await tick(); + + expect((await outcome).lease.worker?.id).toBe("wrk_a"); + }); + it("forwards device.exec with the namespaced requesterId reaching the scripted worker client", async () => { const { coordinator, directory, workers } = harness(); const client = new ScriptedWorkerClient(); @@ -2193,6 +2224,38 @@ describe("FleetLeaseCoordinator fails a request no worker can serve at once (ADR }, ); + it("Gateway routes a lease while a worker is updating: it skips the worker with reason updating, the request waits, and the worker is picked once the update ends", async () => { + const fleetHarness = harness(); + const { coordinator, directory, workers } = fleetHarness; + const client = new ScriptedWorkerClient(); + directory.add("wrk_a", client); + client.requestLeaseQueue.push({ grant: grantFixture(), kind: "grant" }); + connectWorker(workers, "wrk_a"); + workers.refresh("wrk_a", { + update: { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1, + to: "0.4.0", + }, + }); + + const { outcome, state } = lease(fleetHarness, REQUEST); + await tick(); + + expect(state.state).toBe("pending"); + expect(coordinator.waitingRequests()[0]).toMatchObject({ + workers: [{ reason: "updating", workerId: "wrk_a" }], + }); + expect(leaseRequests(client)).toEqual([]); + + workers.refresh("wrk_a", { update: null }); + await tick(); + + expect((await outcome).lease.worker?.id).toBe("wrk_a"); + }); + it.each([ { change: (workers: WorkerRegistry) => workers.disconnected("wrk_a"), @@ -4786,6 +4849,22 @@ describe("FleetLeaseCoordinator lists why each worker cannot take a waiting requ build: () => fleet(["wrk_a", "wrk_b"], (id) => (id === "wrk_b" ? { queueDepth: 1 } : {})), row: "busy with a queue of its own", }, + { + build: () => { + const built = fleet(["wrk_a", "wrk_b"]); + built.workers.refresh("wrk_b", { + update: { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1, + to: "0.4.0", + }, + }); + return built; + }, + row: "updating", + }, ])("leaves the request queued when the other worker is $row", async ({ build }) => { const { clients, coordinator } = build(); clients["wrk_a"]?.requestLeaseQueue.push(provisionFailed("worker a: boom")); diff --git a/src/gateway/fleet-coordinator.ts b/src/gateway/fleet-coordinator.ts index c38992405..787580c9b 100644 --- a/src/gateway/fleet-coordinator.ts +++ b/src/gateway/fleet-coordinator.ts @@ -119,8 +119,12 @@ const COULD_TAKE_LATER: ReadonlySet = new Set([ "full", "not-ready", "unhealthy", + "updating", ]); +// why: ADR 0042 §3: a worker that began its update after the gateway's last read refuses with `DAEMON_UPDATING`, and its view is as stale as after `NO_CAPACITY`. +const STALE_VIEW_CODES: ReadonlySet = new Set(["NO_CAPACITY", "DAEMON_UPDATING"]); + const CANNOT_SERVE_CODES: ReadonlySet = new Set([ "UNKNOWN_MODEL", "RUNTIME_MISSING", @@ -1050,7 +1054,7 @@ export class FleetLeaseCoordinator { // that is this request's own terminal failure -- not a stale view to retry, because device // work already started means the request was this worker's (§11's own wording). Re-queuing // it anyway silently reversed a dispatch the caller had already been told about. - if (isSimlockError(error) && error.code === "NO_CAPACITY" && !announced) { + if (isSimlockError(error) && STALE_VIEW_CODES.has(error.code) && !announced) { this.#rememberRefusal(waiter, workerId, { capacityReason: refusedCapacityReason(error.details), }); diff --git a/src/gateway/routing/AGENTS.md b/src/gateway/routing/AGENTS.md index 7ed818f96..4d8a4471e 100644 --- a/src/gateway/routing/AGENTS.md +++ b/src/gateway/routing/AGENTS.md @@ -35,17 +35,17 @@ Flows: [device selection](../../../docs/internal/flows/device-selection.md), [ga 4. The reported stage is the last rank that removed a worker, else the last that decided. `reason` is `warm-hit` when that stage is `warm-hit`, else `free-capacity`. -| Stage | Kind | Keeps or prefers | Physical | -| ------------------ | ------------- | ----------------------------------------------------------------------------------------------------- | -------- | -| `takes-requests` | filter | connected, not drained, capacity and catalog read this session (`starting`: a catalog held) | yes | -| `has-capabilities` | filter | every `requires` entry `available` | yes | -| `can-serve` | filter | catalog serves model or class with an installed fitting runtime; physical: an enrolled fitting device | yes | -| `healthy` | filter | health `running` | yes | -| `idle-queue` | filter | the worker's own `queueDepth` is 0 | no | -| `warm-hit` | rank, settles | a `ready` fitting device of the asked mode (`servesDefaultMode` if none); physical: a pickable one | yes | -| `free-slot` | filter | a free or evictable (warm) running slot, platform and global | no | -| `ram-budget` | rank | `atRamBudget` not true | no | -| `free-capacity` | rank | most `maxRunning - running - reserved` | no | +| Stage | Kind | Keeps or prefers | Physical | +| ------------------ | ------------- | --------------------------------------------------------------------------------------------------------- | -------- | +| `takes-requests` | filter | connected, not drained, not updating, capacity and catalog read this session (`starting`: a catalog held) | yes | +| `has-capabilities` | filter | every `requires` entry `available` | yes | +| `can-serve` | filter | catalog serves model or class with an installed fitting runtime; physical: an enrolled fitting device | yes | +| `healthy` | filter | health `running` | yes | +| `idle-queue` | filter | the worker's own `queueDepth` is 0 | no | +| `warm-hit` | rank, settles | a `ready` fitting device of the asked mode (`servesDefaultMode` if none); physical: a pickable one | yes | +| `free-slot` | filter | a free or evictable (warm) running slot, platform and global | no | +| `ram-budget` | rank | `atRamBudget` not true | no | +| `free-capacity` | rank | most `maxRunning - running - reserved` | no | A physical request has one rank, `warm-hit`: with no pickable device nothing is picked, and it waits. diff --git a/src/gateway/routing/pipeline.ts b/src/gateway/routing/pipeline.ts index 6d9402670..f268db3b9 100644 --- a/src/gateway/routing/pipeline.ts +++ b/src/gateway/routing/pipeline.ts @@ -25,6 +25,7 @@ export type WorkerReasonCode = | "not-ready" | "unhealthy" | "drained" + | "updating" | "busy" | "no-driver" | "unknown-model" diff --git a/src/gateway/routing/serviceability.ts b/src/gateway/routing/serviceability.ts index 931c7717d..2d09a7a1a 100644 --- a/src/gateway/routing/serviceability.ts +++ b/src/gateway/routing/serviceability.ts @@ -60,7 +60,11 @@ export function assess(request: RoutableRequest, views: readonly WorkerView[]): } function assessTable(request: RoutableRequest, views: readonly WorkerView[]): Assessment { - const takers = views.filter((worker) => takesRequests.check(worker, request) === undefined); + // why: an updating worker comes back by itself (ADR 0042 §3), so a request that only it can serve waits instead of failing. + const takers = views.filter((worker) => { + const refusal = takesRequests.check(worker, request); + return refusal === undefined || refusal.reason === "updating"; + }); if (takers.length === 0) return noWorker(); const known = views.filter(isKnown); diff --git a/src/gateway/routing/stages/takes-requests.ts b/src/gateway/routing/stages/takes-requests.ts index 4629a3877..570dad273 100644 --- a/src/gateway/routing/stages/takes-requests.ts +++ b/src/gateway/routing/stages/takes-requests.ts @@ -1,7 +1,7 @@ import type { FilterStage } from "../pipeline.js"; /** - * Keeps a worker that is connected, not drained, and whose capacity and catalog have been read + * why: keeps a worker that is connected, not drained, not updating (ADR 0042 §3), and whose capacity and catalog have been read * (ADR 0009 §2, stage 1). A policy that cannot see free capacity must not guess it has some, and * a worker whose catalog has not been read since it connected is carrying the last session's. * @@ -13,6 +13,7 @@ export const takesRequests: FilterStage = { check(worker) { if (worker.connection !== "connected") return { reason: worker.connection }; if (worker.drained) return { reason: "drained" }; + if (worker.update != null) return { reason: "updating" }; const ready = worker.health === "starting" ? worker.catalog !== undefined diff --git a/src/gateway/routing/view-state.ts b/src/gateway/routing/view-state.ts index 1b3b01dcb..87b5906b8 100644 --- a/src/gateway/routing/view-state.ts +++ b/src/gateway/routing/view-state.ts @@ -12,6 +12,7 @@ export function viewLoadKey(view: WorkerView): string { leases: (view.leases ?? []).map((lease) => lease.id), physicalQueueDepth: view.physicalQueueDepth, queueDepth: view.queueDepth, + ...(view.update == null ? {} : { updating: true }), }); } diff --git a/src/gateway/service.test.ts b/src/gateway/service.test.ts index 425bb0443..413304386 100644 --- a/src/gateway/service.test.ts +++ b/src/gateway/service.test.ts @@ -445,9 +445,12 @@ describe("GatewayService", () => { installs: [install], label: "mac-mini-1", lastSeenAt: 1_000, + lastUpdateFailure: null, + latestVersion: null, lease: { maxTtlMs: 6 * 60 * 60_000 }, leases: [leaseFixture("lease_1", "dev_1")], queueDepth: 3, + update: null, version: "9.8.7", waiting: [waiting], }); @@ -997,6 +1000,41 @@ describe("GatewayService", () => { await harness.service.stop(); }); + it("shows a worker's update after daemon.update-started and its failure after daemon.update-failed, without waiting for the periodic refresh", async () => { + const worker = new ScriptedWorkerClient(); + const harness = await joined(worker); + const base = statusFixture(); + const update = { + forced: false, + from: "0.3.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 2_000, + from: "0.3.0", + message: "npm failed", + stage: "installing", + to: "0.4.0", + } as const; + + worker.status = { ...base, daemon: { ...base.daemon, update } }; + worker.pushEvent({ event: "daemon.update-started" }); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.update).toEqual(update)); + + worker.status = { ...base, daemon: { ...base.daemon, lastUpdateFailure: failure } }; + worker.pushEvent({ event: "daemon.update-failed" }); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + update: null, + }), + ); + + await harness.service.stop(); + }); + it("drops a worker's install after component.install-failed, without waiting for the periodic refresh", async () => { const worker = new ScriptedWorkerClient(); worker.status = statusFixture({ installs: [install] }); @@ -1201,6 +1239,140 @@ describe("GatewayService", () => { await harness.service.stop(); }); + it("reads an incompatible worker's update with daemon.update query on the tick and keeps the last view when the query fails", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + worker.failWith = protocolMismatchError({ min: 5, max: 5 }); + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + const update = { + forced: false, + from: "0.2.0", + stage: "installing", + startedAt: 1_000, + to: "0.4.0", + } as const; + const lastUpdateFailure = { + at: 500, + from: "0.1.0", + message: "npm failed", + stage: "installing", + to: "0.2.0", + } as const; + worker.updateAnswer = { + ...worker.updateAnswer, + lastUpdateFailure, + latestVersion: "0.4.0", + update, + }; + + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure, + latestVersion: "0.4.0", + update, + }), + ); + expect(worker.updateInputs).toEqual([{ query: true }]); + + worker.updateFailWith = new Error("worker went quiet"); + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(worker.updateInputs).toHaveLength(2)); + expect(harness.service.workers.view("wrk_1")).toMatchObject({ latestVersion: "0.4.0", update }); + + await harness.service.stop(); + }); + + it("does not let a replaced incompatible link's late update query overwrite its successor's view", async () => { + const harness = fleet(); + await harness.service.start(); + const stale = new ScriptedWorkerClient(); + stale.failWith = protocolMismatchError({ min: 5, max: 5 }); + await harness.join("wrk_1", stale); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")?.connection).toBe("incompatible"), + ); + let answerQuery: (() => void) | undefined; + stale.updateDaemon = () => + new Promise((resolve) => { + answerQuery = () => + resolve({ + ...stale.updateAnswer, + update: { + forced: false, + from: "0.2.0", + stage: "installing", + startedAt: 1, + to: "0.4.0", + }, + }); + }); + harness.clock.advance(REFRESH_MS); + await vi.waitFor(() => expect(answerQuery).toBeDefined()); + + await harness.join("wrk_1", new ScriptedWorkerClient("admin", "9.9.9")); + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.version).toBe("9.9.9")); + answerQuery?.(); + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + connection: "connected", + update: null, + version: "9.9.9", + }); + await harness.service.stop(); + }); + + it("shows a worker's update, last update failure and latest version from its status, and a later status without an update clears it", async () => { + const harness = fleet(); + await harness.service.start(); + const worker = new ScriptedWorkerClient(); + const update = { + forced: false, + from: "0.3.0", + leases: 2, + stage: "waiting", + startedAt: 1_000, + to: "0.4.0", + } as const; + const failure = { + at: 500, + from: "0.2.0", + message: "npm failed", + stage: "installing", + to: "0.3.0", + } as const; + const base = statusFixture(); + worker.status = { + ...base, + daemon: { ...base.daemon, lastUpdateFailure: failure, latestVersion: "0.4.0", update }, + }; + await harness.join("wrk_1", worker); + await vi.waitFor(() => + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + latestVersion: "0.4.0", + update, + }), + ); + + worker.status = { ...worker.status, daemon: { ...worker.status.daemon, update: null } }; + harness.clock.advance(REFRESH_MS); + + await vi.waitFor(() => expect(harness.service.workers.view("wrk_1")?.update).toBeNull()); + expect(harness.service.workers.view("wrk_1")).toMatchObject({ + lastUpdateFailure: failure, + latestVersion: "0.4.0", + }); + + await harness.service.stop(); + }); + it("a worker advertising protocol 10 is incompatible on the gateway and nothing it pushes reaches the gateway's bus", async () => { const worker10 = { min: 10, max: 10 }; expect(negotiateProtocolVersion(PROTOCOL_VERSION_RANGE, worker10)).toBeUndefined(); diff --git a/src/gateway/test-support.ts b/src/gateway/test-support.ts index 76e1ecff2..191de6320 100644 --- a/src/gateway/test-support.ts +++ b/src/gateway/test-support.ts @@ -11,6 +11,8 @@ import type { z } from "zod"; import type { ComponentInstallInput, ComponentInstallOutput, + DaemonUpdateInput, + DaemonUpdateOutput, ExecInput, ExecOptions, ExecOutput, @@ -400,6 +402,29 @@ export class ScriptedWorkerClient { }; } + readonly updateInputs: DaemonUpdateInput[] = []; + updateAnswer: DaemonUpdateOutput = { + started: false, + upToDate: false, + resolved: null, + version: "0.3.0", + health: "running", + update: null, + lastUpdateFailure: null, + latestVersion: null, + }; + + // why: the degraded client lets `daemon.update` through (ADR 0042 §1), so this answers even when `failWith` is set. + updateFailWith: unknown; + + async updateDaemon(input: DaemonUpdateInput = {}): Promise { + this.calls.push("daemon.update"); + this.updateInputs.push(input); + if (this.hangingCalls.has("daemon.update")) return new Promise(() => {}); + if (this.updateFailWith !== undefined) throw this.updateFailWith; + return this.updateAnswer; + } + async getStatus(): Promise { this.calls.push("status.get"); if (this.hangingCalls.has("status.get")) return new Promise(() => {}); diff --git a/src/gateway/worker-link.ts b/src/gateway/worker-link.ts index 8c678126a..c95ba7b85 100644 --- a/src/gateway/worker-link.ts +++ b/src/gateway/worker-link.ts @@ -20,6 +20,7 @@ import { grantedDeviceSchema, isSimlockError, WORKER_VIEW_CATALOG_EVENTS, + WORKER_VIEW_UPDATE_EVENTS, workerViewFields, } from "../contract/index.js"; import type { StatusGetOutput } from "../admin/index.js"; @@ -134,6 +135,8 @@ export class WorkerLink { * (inside `#rebuildView`) rather than only once a whole refresh completes -- see * `MAX_CONSECUTIVE_REFRESH_TIMEOUTS`'s comment for why liveness is judged on that one call. */ #consecutiveRefreshTimeouts = 0; + // why: an incompatible worker is asked only the frozen `daemon.update` (ADR 0005 §31, ADR 0042 §1), so its refresh is that query. + #incompatible = false; constructor(private readonly options: WorkerLinkOptions) { this.workerId = options.uplink.workerId; @@ -210,6 +213,7 @@ export class WorkerLink { await this.#withTimeout(client.getStatus(), "status.get"); } catch (error: unknown) { if (isSimlockError(error) && error.code === "PROTOCOL_VERSION_UNSUPPORTED") { + this.#incompatible = true; // ADR 0005 §31: marked `incompatible`, with both ranges, and never asked anything else. // The uplink stays open on purpose -- the worker is running and will reconnect on its // own schedule after an upgrade; closing it here would just make it redial. @@ -306,6 +310,7 @@ export class WorkerLink { async refresh(options: RefreshOptions = {}): Promise { const client = this.#client; if (client === undefined || this.#closed) return; + if (this.#incompatible) return this.#queryUpdate(client); if (this.#refreshing) { this.#queuedRefresh = queueRefresh(this.#queuedRefresh, options); return this.#queuedRefresh.done; @@ -326,25 +331,7 @@ export class WorkerLink { workerId: this.workerId, message: errorMessage(error), }); - // P1: a `WorkerCallTimeoutError` here is not "anything else" -- it is the one failure this - // class already knows means "the socket is not answering", and it used to mean nothing - // happened at all: `registry.refresh` was never reached, so `connection` stayed - // `connected` and `lastSeenAt` stayed stale, forever, on a half-open uplink. After - // `MAX_CONSECUTIVE_REFRESH_TIMEOUTS` in a row, close the link -- which routes through - // `#handleClosed` into `registry.disconnected` -- rather than let it keep confidently - // reporting a machine nothing can reach. `close()` is fire-and-forget here on purpose: it - // flips `#closed` synchronously before its own first `await`, which is what the `finally` - // block below needs to skip queuing another refresh behind a link that is on its way out. - if (error instanceof WorkerCallTimeoutError) { - this.#consecutiveRefreshTimeouts += 1; - if (this.#consecutiveRefreshTimeouts >= MAX_CONSECUTIVE_REFRESH_TIMEOUTS) { - this.#logger.warn("Worker stopped answering; closing its uplink", { - workerId: this.workerId, - consecutiveTimeouts: this.#consecutiveRefreshTimeouts, - }); - void this.close(); - } - } + this.#countTimeout(error); } finally { this.#refreshing = false; const queued = this.#queuedRefresh; @@ -357,6 +344,42 @@ export class WorkerLink { } } + #countTimeout(error: unknown): void { + // why: P1: a `WorkerCallTimeoutError` here is not "anything else" -- it is the one failure this + // class already knows means "the socket is not answering", and it used to mean nothing + // happened at all: `registry.refresh` was never reached, so `connection` stayed + // `connected` and `lastSeenAt` stayed stale, forever, on a half-open uplink. After + // `MAX_CONSECUTIVE_REFRESH_TIMEOUTS` in a row, close the link -- which routes through + // `#handleClosed` into `registry.disconnected` -- rather than let it keep confidently + // reporting a machine nothing can reach. `close()` is fire-and-forget here on purpose: it + // flips `#closed` synchronously before its own first `await`, which is what the `finally` + // block below needs to skip queuing another refresh behind a link that is on its way out. + if (error instanceof WorkerCallTimeoutError) { + this.#consecutiveRefreshTimeouts += 1; + if (this.#consecutiveRefreshTimeouts >= MAX_CONSECUTIVE_REFRESH_TIMEOUTS) { + this.#logger.warn("Worker stopped answering; closing its uplink", { + workerId: this.workerId, + consecutiveTimeouts: this.#consecutiveRefreshTimeouts, + }); + void this.close(); + } + } + } + + // why: a failed query keeps the last view, so its error is dropped. + async #queryUpdate(client: GatewayWorkerClient): Promise { + try { + const answer = await this.#withTimeout(client.updateDaemon({ query: true }), "daemon.update"); + if (this.#superseded()) return; + this.options.registry.refreshUpdate(this.workerId, answer); + } catch (error: unknown) { + this.#logger.debug("Incompatible worker's update query failed", { + workerId: this.workerId, + message: errorMessage(error), + }); + } + } + /** The round trips one refresh makes, and what they become in the view. Split out of * `refresh` so that method is only the coalescing rule and this one is only the reads. * Bounded as one unit per call (D3): a worker that never answers one of these calls must not @@ -581,7 +604,12 @@ const GATEWAY_OWN_EVENTS: ReadonlySet = new Set([ * progress, and an install that starts or ends changes that list. */ function changesCapacityOrLeases(event: string): boolean { - return event.startsWith("lease.") || event.startsWith("device.") || INSTALL_EVENTS.has(event); + return ( + event.startsWith("lease.") || + event.startsWith("device.") || + INSTALL_EVENTS.has(event) || + (WORKER_VIEW_UPDATE_EVENTS as readonly string[]).includes(event) + ); } const INSTALL_EVENTS: ReadonlySet = new Set([ diff --git a/src/gateway/worker-registry.ts b/src/gateway/worker-registry.ts index cdfc444b3..657fef32e 100644 --- a/src/gateway/worker-registry.ts +++ b/src/gateway/worker-registry.ts @@ -36,10 +36,13 @@ export type WorkerViewSnapshot = Pick< | "health" | "host" | "installs" + | "lastUpdateFailure" + | "latestVersion" | "lease" | "leases" | "load" | "queueDepth" + | "update" | "version" | "waiting" | "warmPool" @@ -381,6 +384,18 @@ export class WorkerRegistry { this.#notifyViewsChanged(); } + // why: both the operator's `worker.update` and an incompatible worker's query read these three fields from a `daemon.update` answer. + refreshUpdate( + workerId: string, + answer: Pick, + ): void { + this.refresh(workerId, { + lastUpdateFailure: answer.lastUpdateFailure, + latestVersion: answer.latestVersion, + update: answer.update, + }); + } + /** * ADR 0005 §15: the gateway's own `lease.maxTtlMs` decides a fleet lease's width at * admission, before any worker is chosen -- but what the gateway dispatches is an ordinary diff --git a/src/http/AGENTS.md b/src/http/AGENTS.md index 925c073e8..643be27fd 100644 --- a/src/http/AGENTS.md +++ b/src/http/AGENTS.md @@ -88,7 +88,8 @@ agent-device front and the stream (`/v1/stream`). subscribes to the `EventBus` itself. - `GET /v1/events` — reads `since`, `before` and `limit` through `eventsPageQuery` (contract), refused as 400 before dispatching `events.replay`; `HISTORY_NOT_KEPT` maps to 422 with `oldestTs`, `HISTORY_UNAVAILABLE` to 503 here too. - `GET /v1/stats` — parses `since`/`from`/`to` with `usageWindow` (contract) and `ledger=1`, all refused as 400 before dispatching `usage.get` (the dispatcher answers an agent token 403, `HISTORY_NOT_KEPT` maps to 422 with `oldestTs`, `HISTORY_UNAVAILABLE` to 503). -- `POST /v1/update` (optional body `{version, force}`, strict) → `daemon.update`: 202 when the call began the update (`started`), 200 when it followed one or found nothing to do (`upToDate`). The route declares both (`alsoStatus`). The role check is the dispatcher's; a gateway answers 501. +- `POST /v1/update` (optional body `{version, force}`, strict) → `daemon.update`: 202 when the call began the update (`started`), 200 when it followed one or found nothing to do (`upToDate`). The route declares both (`alsoStatus`). The role check is the dispatcher's; on a gateway it updates the gateway itself, with no lease wait. +- `POST /v1/workers/{id}/update` (optional body `{version, force}`, strict) → `worker.update`: the same 202 and 200; `UNKNOWN_WORKER` 404, `WORKER_NOT_CONNECTED` 409, and a worker daemon answers 501. - `GET /v1/leases/{id}/events` — the lease's notices; ends on `lease_lost`. - `POST /v1/leases/{id}/exec` → `device.exec`. The stream opens on `onStarted`; a failure before that is a plain error response. Events: `output`, then `exit` or `error`. diff --git a/src/http/admin-declarations.ts b/src/http/admin-declarations.ts index 7257dbde9..d020161dd 100644 --- a/src/http/admin-declarations.ts +++ b/src/http/admin-declarations.ts @@ -66,7 +66,7 @@ export const updateRoute = declareRoute({ summary: "Update this daemon to a Simlock version", role: "admin", notes: - "The body is optional: without a version the newest release is installed. The daemon takes no new leases while it waits for its leases to end, installs and restarts. 202 when this call began the update; 200 when it followed an update that was already running, or nothing needed updating (`upToDate`). Worker only: a gateway answers 501 UNSUPPORTED_IN_GATEWAY_MODE.", + "The body is optional: without a version the newest release is installed. The daemon takes no new leases while it waits for its leases to end, installs and restarts. 202 when this call began the update; 200 when it followed an update that was already running, or nothing needed updating (`upToDate`). On a gateway it updates the gateway itself, with no lease wait: a gateway holds none.", request: { optionalJson: updateBodySchema }, reply: OPERATIONS["daemon.update"].output, status: 202, @@ -74,7 +74,26 @@ export const updateRoute = declareRoute({ errors: { 409: ["UPDATE_UNSUPPORTED_INSTALL", "UPDATE_SHARED_INSTALL"], 502: ["UPDATE_RESOLVE_FAILED"], - 501: ["UNSUPPORTED_IN_GATEWAY_MODE"], + }, +}); + +export const workerUpdateRoute = declareRoute({ + method: "post", + path: "/v1/workers/{id}/update", + tag: "Workers", + summary: "Update one worker to a Simlock version", + role: "admin", + notes: + "The body is optional, as for `POST /v1/update`. The gateway asks the worker to update; the worker takes no new leases while it waits for its leases to end. 202 when this call began the update; 200 when it followed one that was running, or nothing needed updating (`upToDate`). A worker that shows `incompatible` can be updated too. The gateway's drain set is not touched. Gateway only: a worker answers 501 UNSUPPORTED_IN_WORKER_MODE.", + request: { params: workerParamsSchema, optionalJson: updateBodySchema }, + reply: OPERATIONS["worker.update"].output, + status: 202, + alsoStatus: 200, + errors: { + 404: ["UNKNOWN_WORKER"], + 409: ["WORKER_NOT_CONNECTED", "UPDATE_UNSUPPORTED_INSTALL", "UPDATE_SHARED_INSTALL"], + 502: ["UPDATE_RESOLVE_FAILED"], + 501: ["UNSUPPORTED_IN_WORKER_MODE"], }, }); @@ -205,6 +224,7 @@ export const ADMIN_ROUTES = [ workerListRoute, workerDrainRoute, workerUndrainRoute, + workerUpdateRoute, workerRemoveRoute, componentListRoute, componentRemoveRoute, diff --git a/src/http/app.test.ts b/src/http/app.test.ts index 976ea1824..9e63e6206 100644 --- a/src/http/app.test.ts +++ b/src/http/app.test.ts @@ -2563,7 +2563,7 @@ describe("POST /v1/update", () => { }), ); - it("answers 202 when the call began the update and 200 when it followed one or nothing was needed, dispatches daemon.update with the body, refuses an agent and a bad body, and passes a gateway's refusal on", async () => { + it("answers 202 when the call began the update and 200 when it followed one or nothing was needed, dispatches daemon.update with the body, refuses an agent and a bad body", async () => { const { app, dispatcher } = buildHarness(); // why: `daemon.update`'s own role check runs through the contract's real pipeline; the route adds none. let reply: Record = answer({ started: true, resolved: "0.4.0" }); @@ -2615,17 +2615,78 @@ describe("POST /v1/update", () => { expect({ body, status: refused.status }).toEqual({ body, status: 400 }); } expect(dispatcher.calls).toHaveLength(callsBefore); + }); +}); - dispatcher.handlers["daemon.update"] = () => { - throw new DispatchError("UNSUPPORTED_IN_GATEWAY_MODE", "daemon.update is refused", { - operation: "daemon.update", +describe("POST /v1/workers/{id}/update", () => { + const answer = (overrides: Record = {}) => ({ + started: false, + upToDate: false, + resolved: null, + version: "0.3.0", + health: "running", + update: null, + lastUpdateFailure: null, + latestVersion: null, + workerId: "wrk_1", + ...overrides, + }); + + it("answers 202 when the call began the update and 200 when it followed one, dispatches worker.update with the id from the path and the body, refuses an agent and a bad body", async () => { + const { app, dispatcher } = buildHarness({ config: testConfig({}, "gateway") }); + let reply: Record = answer({ started: true, resolved: "0.4.0" }); + dispatcher.handlers["worker.update"] = (input, session) => + runDispatch("worker.update", input, session, { + handlers: { "worker.update": () => reply }, }); - }; - const onGateway = await post(app, {}); - expect(onGateway.status).toBe(501); - expect(await onGateway.json()).toMatchObject({ - error: { code: "UNSUPPORTED_IN_GATEWAY_MODE", operation: "daemon.update" }, - }); + const post = (body?: unknown, headers: Record = operatorAuth) => + Promise.resolve( + app.request("/v1/workers/wrk_1/update", { + headers: { + ...headers, + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + method: "POST", + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }), + ); + + const started = await post({ version: "0.4.0", force: true }); + expect(started.status).toBe(202); + expect(await started.json()).toEqual(reply); + expect(dispatcher.calls.map(({ operation, input }) => ({ operation, input }))).toEqual([ + { operation: "worker.update", input: { force: true, version: "0.4.0", workerId: "wrk_1" } }, + ]); + + reply = answer({ upToDate: true, resolved: "0.3.0" }); + const following = await post(); + expect(following.status).toBe(200); + expect(dispatcher.calls.at(-1)?.input).toEqual({ workerId: "wrk_1" }); + + const asAgent = await post({}, agentAuth); + expect(asAgent.status).toBe(403); + + const callsBefore = dispatcher.calls.length; + for (const body of [{ version: "latest" }, { query: true }, { force: "yes" }]) { + const refused = await post(body); + expect({ body, status: refused.status }).toEqual({ body, status: 400 }); + } + expect(dispatcher.calls).toHaveLength(callsBefore); + + for (const [code, status] of [ + ["WORKER_NOT_CONNECTED", 409], + ["UPDATE_UNSUPPORTED_INSTALL", 409], + ["UPDATE_SHARED_INSTALL", 409], + ["UPDATE_RESOLVE_FAILED", 502], + ["UNKNOWN_WORKER", 404], + ] as const) { + dispatcher.handlers["worker.update"] = () => { + throw new DispatchError(code, "refused"); + }; + const refusal = await post(); + expect({ code, status: refusal.status }).toEqual({ code, status }); + expect(((await refusal.json()) as { error: { code: string } }).error.code).toBe(code); + } }); }); diff --git a/src/http/app.ts b/src/http/app.ts index ad6b2e32b..30159f4cf 100644 --- a/src/http/app.ts +++ b/src/http/app.ts @@ -28,6 +28,7 @@ import { workerListRoute, workerRemoveRoute, workerUndrainRoute, + workerUpdateRoute, } from "./admin-declarations.js"; import { badRequest, @@ -645,6 +646,15 @@ export function createHttpApp(deps: HttpGatewayDeps): OpenAPIHono & HttpApp return c.json(update, update.started ? 202 : 200); }); + app.openapi({ ...workerUpdateRoute, middleware: [agentAuth, emptyBodyAsNone] }, async (c) => { + const update = await deps.dispatch( + "worker.update", + { ...c.req.valid("json"), workerId: c.req.valid("param").id }, + buildHttpSession(c.get("identity")), + ); + return c.json(update, update.started ? 202 : 200); + }); + app.openapi({ ...workerDrainRoute, middleware: [agentAuth] }, async (c) => c.json( await deps.dispatch( diff --git a/src/simlock-client/client.test.ts b/src/simlock-client/client.test.ts index d1402351c..e9e3b9d2c 100644 --- a/src/simlock-client/client.test.ts +++ b/src/simlock-client/client.test.ts @@ -268,6 +268,36 @@ describe("connectSimlock: handshake", () => { }); }); + it("the admin client's updateWorker sends worker.update and returns its output, and refuses an empty worker id before sending", async () => { + const connection = new ScriptedConnection(); + const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); + await flushMicrotasks(); + completeHello(connection, { role: "admin" }); + const client = await connectPromise; + const answer = { + health: "running", + lastUpdateFailure: null, + latestVersion: "0.4.0", + resolved: "0.4.0", + started: true, + update: null, + upToDate: false, + version: "0.3.0", + workerId: "wrk_1", + }; + + const callPromise = client.updateWorker({ force: true, version: "0.4.0", workerId: "wrk_1" }); + await flushMicrotasks(); + const call = connection.lastSentOf("worker.update"); + expect(call?.payload).toEqual({ force: true, version: "0.4.0", workerId: "wrk_1" }); + connection.reply(call!.id, answer); + + await expect(callPromise).resolves.toEqual(answer); + await expect(client.updateWorker({ workerId: "" })).rejects.toMatchObject({ + code: "BAD_REQUEST", + }); + }); + it("enrolls a device from an admin client with device.add and removes it with device.remove, sending the input and returning the daemon's answer", async () => { const connection = new ScriptedConnection(); const connectPromise = connectSimlockAdmin({ connection, credential: "operator-secret" }); diff --git a/src/simlock-client/client.ts b/src/simlock-client/client.ts index b15c895f3..81bdef409 100644 --- a/src/simlock-client/client.ts +++ b/src/simlock-client/client.ts @@ -94,6 +94,8 @@ import type { WorkerListOutput, WorkerRemoveOutput, WorkerUndrainOutput, + WorkerUpdateInput, + WorkerUpdateOutput, } from "./types.js"; export type { @@ -167,6 +169,8 @@ export type { WorkerListOutput, WorkerRemoveOutput, WorkerUndrainOutput, + WorkerUpdateInput, + WorkerUpdateOutput, WorkerView, AlertsTestInput, AlertsTestOutput, @@ -268,12 +272,13 @@ export interface SimlockAdminClient extends SimlockClient { listTokens(): Promise; revokeToken(input: TokenRevokeInput): Promise; - /** ADR 0005 §23. A worker lists itself as its only worker; `drainWorker`, `undrainWorker` and - * `removeWorker` reject on a worker with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012). */ + /** why: ADR 0005 §23. A worker lists itself as its only worker; `drainWorker`, `undrainWorker`, `removeWorker` and + * `updateWorker` (a gateway updating one of its workers, ADR 0042 §2) reject on a worker with `UNSUPPORTED_IN_WORKER_MODE` (ADR 0012). */ listWorkers(): Promise; drainWorker(input: WorkerDrainInput): Promise; undrainWorker(input: WorkerDrainInput): Promise; removeWorker(input: WorkerDrainInput): Promise; + updateWorker(input: WorkerUpdateInput): Promise; /** * why: ADR 0035 §1: asks a gateway to post one sample message to the named alert destination, once. @@ -447,6 +452,7 @@ function buildDegradedClient( drainWorker: () => rejected(), undrainWorker: () => rejected(), removeWorker: () => rejected(), + updateWorker: () => rejected(), testAlertDestination: () => rejected(), installComponent: () => rejected(), installComponentOnWorkers: () => rejected(), @@ -724,6 +730,10 @@ class SimlockClientImpl { return this.#call("worker.undrain", input); } + updateWorker(input: WorkerUpdateInput): Promise { + return this.#call("worker.update", input); + } + testAlertDestination(input: AlertsTestInput): Promise { return this.#call("alerts.test", input); } diff --git a/src/simlock-client/types.ts b/src/simlock-client/types.ts index f91b733e5..705de3177 100644 --- a/src/simlock-client/types.ts +++ b/src/simlock-client/types.ts @@ -83,6 +83,8 @@ export type WorkerDrainInput = OpInput<"worker.drain">; export type WorkerDrainOutput = OpOutput<"worker.drain">; export type WorkerUndrainOutput = OpOutput<"worker.undrain">; export type WorkerRemoveOutput = OpOutput<"worker.remove">; +export type WorkerUpdateInput = OpInput<"worker.update">; +export type WorkerUpdateOutput = OpOutput<"worker.update">; export type AlertsTestInput = OpInput<"alerts.test">; export type AlertsTestOutput = OpOutput<"alerts.test">; diff --git a/ui/src/model/waiting.test.ts b/ui/src/model/waiting.test.ts index 1158bd690..3572408f2 100644 --- a/ui/src/model/waiting.test.ts +++ b/ui/src/model/waiting.test.ts @@ -253,6 +253,7 @@ describe("the waiting table on a gateway", () => { ["incompatible", undefined, "Protocol mismatch"], ["not-ready", undefined, "Not ready yet"], ["unhealthy", undefined, "Not running"], + ["updating", undefined, "Updating"], ["drained", undefined, "Drained"], ["busy", undefined, "Busy with its own queue"], ["no-driver", undefined, "No iOS support"], @@ -280,7 +281,7 @@ describe("the waiting table on a gateway", () => { field as { unwrap(): { element: { shape: { reason: { options: readonly string[] } } } } } ).unwrap().element.shape.reason.options; - expect(listed).toHaveLength(13); + expect(listed).toHaveLength(14); for (const reason of listed) { const text = rowOf({ workers: [{ reason, workerId: "wrk_1" } as never] }).workers?.[0]?.text; expect(text).toMatch(/^[A-Z]/); diff --git a/ui/src/model/waiting.ts b/ui/src/model/waiting.ts index da49bdb00..eaffce18e 100644 --- a/ui/src/model/waiting.ts +++ b/ui/src/model/waiting.ts @@ -106,6 +106,7 @@ const WORKER_WHY_TEXT: Readonly< "not-ready": () => "Not ready yet", "runtime-missing": () => "OS not installed", unhealthy: () => "Not running", + updating: () => "Updating", "unknown-model": () => "Model not offered", }; diff --git a/website/src/content/docs/docs/gateway/managing-workers.mdx b/website/src/content/docs/docs/gateway/managing-workers.mdx index 469d839b9..03ba6906e 100644 --- a/website/src/content/docs/docs/gateway/managing-workers.mdx +++ b/website/src/content/docs/docs/gateway/managing-workers.mdx @@ -1,6 +1,6 @@ --- title: Manage workers -description: List the workers behind a gateway, take one out of service and bring it back, forget one that is gone, install runtimes on many workers at once, and cut a worker off. +description: List the workers behind a gateway, take one out of service and bring it back, update one, forget one that is gone, install runtimes on many workers at once, and cut a worker off. --- import { Steps, Aside } from '@astrojs/starlight/components'; @@ -14,7 +14,7 @@ simlock --gateway worker list ``` ```text -4b1e9c2a-6d3f-4a8e-9b2c-7f1d0e5a3c68 (mac-mini-2): connected -- ios 2/4, android 1/2, 3 lease(s) -- macOS 15.6 arm64; xcode 16.4 (16F6), emulator 35.4.9 -- agent-device off -- stream off +4b1e9c2a-6d3f-4a8e-9b2c-7f1d0e5a3c68 (mac-mini-2): connected -- ios 2/4, android 1/2, 3 lease(s) -- simlock 0.3.0 (0.4.0 available) -- updating to 0.4.0: waiting for 2 leases -- macOS 15.6 arm64; xcode 16.4 (16F6), emulator 35.4.9 -- agent-device off -- stream off Load: CPU 34% busy (10 cores) · memory 12.3 GiB of 32.0 GiB available · swap 0 B used · pressure normal · disk 210.4 GiB of 926.4 GiB free · read 12s ago 9e3a7c1f-0b5d-4c2e-8f6a-1d4b7e9c2a50 (mac-studio-1): disconnected, drained -- ios 0/8, android 0/4, 0 lease(s) -- macOS 15.6 arm64; xcode 16.4 (16F6) Load: no reading while disconnected @@ -25,6 +25,7 @@ Each worker shows: - its id and, in brackets, its label; - its state: `connected`, `disconnected` or `incompatible`, then `starting` while it starts up and `drained` when it takes no new leases from the gateway; - devices running of the limit per platform, and how many leases it holds; +- its Simlock version, with the newest release in brackets when npm has a newer one, and `updating to : ` while it updates (the stage is `waiting for leases`, `installing` or `restarting`); - its operating system, platform tools and optional modules; - under it, the last load reading, and any runtime or system image being installed. @@ -71,6 +72,28 @@ What to know about draining: An id the gateway does not show fails with `UNKNOWN_WORKER` (exit 12). +## Update a worker + +Update one worker's Simlock from the gateway, and follow it: + +```sh +simlock --gateway worker update 3f81a2c4 +``` + +```text +mac-studio-2 (3f81a2c4) is running 0.4.0 +``` + +The worker waits for its leases to end, then installs and restarts, as `simlock update` does on a single machine. While it waits or installs, the gateway sends it no new leases: a request only this worker could serve stays in the queue until the worker is back, and `worker list` shows `updating`. A drain is left alone. A worker you drained stays drained after its update, and only `undrain` ends it. + +- `--version ` installs that exact version, `--force` does not wait for the leases, and `--detach` returns at once with `Updating mac-studio-2 (3f81a2c4) 0.3.0 → 0.4.0. Follow it with simlock worker list.` +- A worker already on the newest release answers `mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)`. +- A failed update ends with `Update failed (): ` and exit 1. The worker goes on with its old version. +- A worker shown as `incompatible` can be updated too. That is the way to bring a worker with an older protocol back into the fleet. +- A worker that is not connected fails with `WORKER_NOT_CONNECTED`. The gateway does not queue an update for it. + +To update the gateway itself, run `simlock update --gateway` on its machine. It installs and restarts at once, with no wait. + ## Forget a worker that is gone When a machine is gone for good, remove it so the gateway stops showing it: diff --git a/website/src/content/docs/docs/reference/cli.mdx b/website/src/content/docs/docs/reference/cli.mdx index 36cb20b3b..c1077bed2 100644 --- a/website/src/content/docs/docs/reference/cli.mdx +++ b/website/src/content/docs/docs/reference/cli.mdx @@ -392,7 +392,7 @@ simlock 0.4.0 is running - The daemon does not answer within 15 minutes after it restarts: `The daemon did not come back; run simlock status.`, exit 1. - Simlock must be installed with `npm i -g simlock`. Otherwise the command stops with `UPDATE_UNSUPPORTED_INSTALL`. A worker and a gateway of one user share the install, so while the other runs it stops with `UPDATE_SHARED_INSTALL`. npm failing to find the version is `UPDATE_RESOLVE_FAILED`. See [Errors](/simlock/docs/reference/errors#updating-simlock). -`POST /v1/update` does the same over HTTP. A gateway refuses `update` with `UNSUPPORTED_IN_GATEWAY_MODE` (exit 2). +`POST /v1/update` does the same over HTTP. On a machine that runs a gateway, `simlock update --gateway` updates the gateway itself. A gateway holds no leases of its own, so it does not wait: it installs and restarts at once, and a request in that gap fails as in any gateway restart. To update a worker from a gateway, use [`simlock worker update`](#simlock-worker). ## simlock nuke @@ -499,16 +499,35 @@ simlock worker list [--json] simlock worker drain simlock worker undrain simlock worker remove +simlock worker update [--version ] [--force] [--detach] ``` | Subcommand | What it does | | --- | --- | -| `list` | Show each worker: connection, devices, capacity and installs. On a single machine it shows the machine itself. `--json` prints one JSON line in place of the text view. | +| `list` | Show each worker: connection, devices, capacity and installs. A worker's line also shows its Simlock version, with `( available)` when npm has a newer release, and `updating to : ` while it updates. On a single machine it shows the machine itself. `--json` prints one JSON line in place of the text view. | | `drain` | Stop sending new requests to the worker. Its leases run on. | | `undrain` | Send new requests to the worker again. | | `remove` | Forget a worker that is not connected. A connected worker is refused with `WORKER_CONNECTED`: drain it and let it disconnect first. | +| `update` | Update one worker's Simlock from the gateway and follow it. The worker takes no new leases while it updates. A drain stays as it is. | -All four need the admin role. `drain`, `undrain` and `remove` print one JSON line, and on a worker they fail with `UNSUPPORTED_IN_WORKER_MODE` (exit 2). +All five need the admin role. `drain`, `undrain` and `remove` print one JSON line, and on a worker all but `list` fail with `UNSUPPORTED_IN_WORKER_MODE` (exit 2). + +`update` takes the flags of [`simlock update`](#simlock-update) and one more: + +| Flag | Meaning | Default | +| --- | --- | --- | +| `--version ` | Install this exact version on the worker. | The newest release | +| `--force` | The worker does not wait for its leases to end. | Off | +| `--detach` | Print where the update started and return. Follow it with `simlock worker list`. | Off | + +It follows the worker's line in the list and ends with one of these: + +- `mac-studio-2 (3f81a2c4) is running 0.4.0` +- `Update failed (installing): `, exit 1. The worker keeps running its old version and takes leases again. +- `mac-studio-2 (3f81a2c4) is already on the latest (0.4.0)`. +- With `--detach`: `Updating mac-studio-2 (3f81a2c4) 0.3.0 → 0.4.0. Follow it with simlock worker list.` + +A worker that shows `incompatible` can be updated this way too. A worker that is not connected fails with `WORKER_NOT_CONNECTED`; one that does not answer within 90 seconds fails with the same code. `POST /v1/workers/{id}/update` does the same over HTTP. ## simlock alerts diff --git a/website/src/content/docs/docs/reference/errors.mdx b/website/src/content/docs/docs/reference/errors.mdx index db9989d8d..5d50311ba 100644 --- a/website/src/content/docs/docs/reference/errors.mdx +++ b/website/src/content/docs/docs/reference/errors.mdx @@ -23,7 +23,7 @@ Branch on `code`, not on the message: the message wording can change, the code d | --- | --- | --- | | 0 | — | Success. For a `simlock lease` that stayed running: the lease ended because you asked. | | 1 | `INTERNAL`, `PROVISION_FAILED`, `UNKNOWN_LEASE`, `FORBIDDEN`, `ADMIN_AUTHENTICATION_FAILED`, `DAEMON_STARTUP_FAILED`, `DAEMON_CONNECTION_LOST`, `DAEMON_STOPPING`, `DAEMON_UPDATING`, `HANDSHAKE_REQUIRED`, `PROTOCOL_VERSION_UNSUPPORTED`, `WORKER_UNREACHABLE`, `RELAY_TARGET_MISSING`, `ALERT_SEND_FAILED`, `UPDATE_RESOLVE_FAILED`, `HISTORY_UNAVAILABLE`, `DOCTOR_UNAVAILABLE`, `NUKE_UNAVAILABLE`, `UNKNOWN_DAEMON_ERROR`, `CANCELLED` | The command failed. Read the code and the message. | -| 2 | `USAGE`, `BAD_REQUEST`, `BAD_FRAME`, `UNKNOWN_REQUEST`, `IDEMPOTENCY_CONFLICT`, `PASSTHROUGH_REFUSED`, `UNKNOWN_PASSTHROUGH_TOOL`, `GATEWAY_NOT_PRESENT`, `UPDATE_UNSUPPORTED_INSTALL`, `UPDATE_SHARED_INSTALL`, `UNSUPPORTED_IN_GATEWAY_MODE`, `UNSUPPORTED_IN_WORKER_MODE`, `WORKER_CONNECTED` | The request was wrong. Running it again unchanged fails again. | +| 2 | `USAGE`, `BAD_REQUEST`, `BAD_FRAME`, `UNKNOWN_REQUEST`, `IDEMPOTENCY_CONFLICT`, `PASSTHROUGH_REFUSED`, `UNKNOWN_PASSTHROUGH_TOOL`, `GATEWAY_NOT_PRESENT`, `UPDATE_UNSUPPORTED_INSTALL`, `UPDATE_SHARED_INSTALL`, `UNSUPPORTED_IN_GATEWAY_MODE`, `UNSUPPORTED_IN_WORKER_MODE`, `WORKER_CONNECTED`, `WORKER_NOT_CONNECTED` | The request was wrong. Running it again unchanged fails again. | | 10 | `QUEUE_TIMEOUT`, `EXEC_TIMEOUT`, `DOWNLOAD_TIMEOUT` | A deadline ran out. | | 11 | `NO_CAPACITY`, `DEVICE_BUSY`, `COMPONENT_BUSY` | Busy right now. The same request can pass later. | | 12 | `NO_DRIVER`, `CAPABILITY_MISSING`, `RUNTIME_MISSING`, `UNKNOWN_MODEL`, `INSUFFICIENT_DISK_SPACE`, `LICENSE_NOT_ACCEPTED`, `DOWNLOADS_DISABLED`, `COMPONENT_NOT_OWNED`, `COMPONENT_IN_USE`, `DEVICE_NOT_ENROLLABLE`, `UNKNOWN_DEVICE`, `HISTORY_NOT_KEPT`, `UNKNOWN_WORKER`, `UNKNOWN_ALERT_DESTINATION` | What you named does not exist here, or this machine cannot provide it. | @@ -117,7 +117,8 @@ Branch on `code`, not on the message: the message wording can change, the code d | --- | --- | --- | --- | --- | | `GATEWAY_NOT_PRESENT` | 2 | 404 | `--gateway` on a machine with no gateway. Raised by the CLI and MCP server before they connect, never over HTTP. | Start one with `simlock --gateway daemon start`, or drop `--gateway`. | | `UNSUPPORTED_IN_GATEWAY_MODE` | 2 | 501 | The command acts on one machine's devices or components (`doctor`, `cleanup`, `nuke`, `device`, `component list`, `component remove`, `component install` without a worker flag), and the daemon that answered is a gateway. | Run it against the worker, with `--local` on that machine. For `component install`, name workers with `--worker` or `--all-workers`. | -| `UNSUPPORTED_IN_WORKER_MODE` | 2 | 501 | The command acts on a gateway's workers or alerts (`worker drain`, `undrain`, `remove`, `alerts test`, `component install --worker`), and the daemon that answered is a worker. | Run it against the gateway, with `--gateway` on that machine. | +| `UNSUPPORTED_IN_WORKER_MODE` | 2 | 501 | The command acts on a gateway's workers or alerts (`worker drain`, `undrain`, `remove`, `update`, `alerts test`, `component install --worker`), and the daemon that answered is a worker. | Run it against the gateway, with `--gateway` on that machine. | +| `WORKER_NOT_CONNECTED` | 2 | 409 | `worker update` named a worker the gateway knows but has no open connection to: `worker is not connected`. It is also the answer when the worker does not reply within 90 seconds: `worker did not answer within 90 s; check simlock worker list`. In that case the update may still have started. | Check the worker with `simlock worker list`. Run the update again once it is connected. | | `UNKNOWN_WORKER` | 12 | 404 | The gateway has no worker with this ID. | Check the ID with `simlock worker list`. | | `WORKER_CONNECTED` | 2 | 409 | `worker remove` of a worker that is still connected. | Drain it, stop it or let it disconnect, then remove it. | | `WORKER_UNREACHABLE` | 1 | 503 | The worker that holds this lease or would serve this request is not connected to the gateway. | Check the worker is running and can reach the gateway. | diff --git a/website/src/content/docs/docs/using-devices/programmatic-client.mdx b/website/src/content/docs/docs/using-devices/programmatic-client.mdx index 73acbcdd8..b773e877f 100644 --- a/website/src/content/docs/docs/using-devices/programmatic-client.mdx +++ b/website/src/content/docs/docs/using-devices/programmatic-client.mdx @@ -195,6 +195,8 @@ const answer = await admin.updateDaemon({ version: "0.4.0" }); - `{ query: true }` starts nothing and returns the same answer: poll it to follow the update. It works while the daemon restarts only by reconnecting, and across a protocol change: like `stopDaemon()`, `updateDaemon()` is accepted by a daemon on another protocol version, so a client that fails every other call with `PROTOCOL_VERSION_UNSUPPORTED` can still update it. - Refusals: `UPDATE_RESOLVE_FAILED`, `UPDATE_UNSUPPORTED_INSTALL` and `UPDATE_SHARED_INSTALL` (see [Errors](/simlock/docs/reference/errors#updating-simlock)). `version` must be an exact version such as `0.4.0` or `0.4.0-rc.1`. +On a gateway, `admin.updateDaemon()` updates the gateway itself: it holds no leases, so it does not wait and goes straight to installing. `admin.updateWorker({ workerId, version?, force? })` asks the gateway to update one of its workers, `simlock worker update` does. It answers like `updateDaemon()`, plus the `workerId`. A worker that is not connected, or that does not answer within 90 seconds, fails it with `WORKER_NOT_CONNECTED`. Poll `admin.listWorkers()` to follow it: the worker's `version`, `latestVersion`, `update` and `lastUpdateFailure` show the update, and `update` is `null` once it ends. On a worker, `updateWorker()` fails with `UNSUPPORTED_IN_WORKER_MODE`. + ## Related - How leases and TTLs work: [Lease lifecycle](/simlock/docs/using-devices/lease-lifecycle).