feat(config): fan cluster config saves out with replicated set_configuration on Harper 5.2+

DESIGN.md

@@ -44,6 +44,14 @@ The wide hero artwork spans both columns beneath the form card in three independ
 - **Only query errors are muted, and only connectivity-class ones.** Mutations still toast; a 403 or a Harper 500 during a restart still toasts.
 - **Detection is per tab.** A page learns of a container op started elsewhere only from a cluster-record fetch, and the in-cluster studio pages don't poll the cluster, so they don't notice one started in another tab.
 
+## Cluster config updates — when an instance may skip its own `set_configuration`
+
+[`useRollingConfigUpdate`](src/hooks/useRollingConfigUpdate.tsx) saves the config editor's change by restarting a cluster's RUNNING instances one at a time, and every instance it restarts must already have the new config on disk: `set_configuration` only writes the file, and nothing applies it until a restart.
+
+- **Pre-5.2, mixed-version, or unknown-version clusters write each instance before restarting it.** `replicated: true` on `set_configuration` exists from Harper 5.2.0 ([`shouldFanOutSetConfiguration`](src/integrations/api/instance/status/setConfiguration.ts)); 5.0 and 5.1 apply the change locally and report no peers.
+- **On 5.2+ the first instance is still written and restarted alone.** Only once it is back up on the new config is the write fanned out with `replicated: true` from that instance — even when it was the last instance to restart, because the fan-out also reaches live members Studio does not restart (not RUNNING, or failed earlier), which the per-instance path leaves on the old config. Fanning out first would leave an unproven config pending on every node, which a Cancel could no longer contain.
+- **A later instance skips its write only when the fan-out acknowledged that exact node** — its CM `replicationFqdn` (lower-cased) appears among the successful `replicated[]` entries. Fabric names each Harper node after that FQDN (host-manager sets `NODE_HOSTNAME` and `REPLICATION_HOSTNAME` to it). A count of acknowledgements is not proof: the origin's replication peers are not CM's restart targets, so a stale peer can stand in for a missing one. A target with no `replicationFqdn`, a rejected fan-out (the instance client rejects any response with a failed peer), or a response with no usable `replicated[]` all keep the per-instance write.
+
 ## Cluster upgrade picker — which versions an existing cluster is offered
 
 [`buildUpgradeVersionOptions`](src/features/clusters/upsert/lib/buildUpgradeVersionOptions.ts) is the one place that decides, and its test file is the spec. Two rules, and they differ on purpose:

src/hooks/getClusterRestartTargets.test.ts

@@ -0,0 +1,70 @@
+import type { Cluster, Instance } from '@/integrations/api/api.patch';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+const { fabricConnect } = vi.hoisted(() => ({ fabricConnect: { current: false } }));
+vi.mock('@/config/getInstanceClient', () => ({
+	getInstanceClient: vi.fn((config: unknown) => ({ config })),
+}));
+vi.mock('@/features/auth/store/authStore', () => ({
+	authStore: { checkForFabricConnect: vi.fn(() => fabricConnect.current) },
+}));
+vi.mock('@/features/cluster/queries/getClusterInfoQuery', () => ({ getClusterInfo: vi.fn() }));
+
+import { getInstanceClient } from '@/config/getInstanceClient';
+import { authStore } from '@/features/auth/store/authStore';
+import { getClusterInfo } from '@/features/cluster/queries/getClusterInfoQuery';
+import { getClusterRestartTargets } from './getClusterRestartTargets';
+
+function instance(id: string, status: Instance['status'] = 'RUNNING'): Instance {
+	return {
+		id,
+		status,
+		instanceFqdn: `${id}.example.com`,
+		operationsApiPort: 9925,
+		operationsApiSecure: true,
+	} as Instance;
+}
+
+afterEach(() => {
+	fabricConnect.current = false;
+	vi.clearAllMocks();
+});
+
+describe('getClusterRestartTargets', () => {
+	it('builds a client for each RUNNING instance, last instance first, and keeps every instance', async () => {
+		const instances = [instance('ins-a'), instance('ins-b', 'STOPPED'), instance('ins-c')];
+		vi.mocked(getClusterInfo).mockResolvedValue({ id: 'clu-1', instances } as Cluster);
+
+		const { allInstances, targets } = await getClusterRestartTargets('clu-1');
+
+		expect(getClusterInfo).toHaveBeenCalledWith('clu-1');
+		expect(allInstances).toBe(instances);
+		expect(targets.map(({ instance }) => instance.id)).toEqual(['ins-c', 'ins-a']);
+		expect(targets[0].instanceClient).toEqual({
+			config: { id: 'ins-c', forceFabricConnect: false, operationsUrl: 'https://ins-c.example.com:9925/' },
+		});
+	});
+
+	it("forwards the cluster's Fabric Connect mode to every client", async () => {
+		fabricConnect.current = true;
+		vi.mocked(getClusterInfo).mockResolvedValue({ id: 'clu-1', instances: [instance('ins-a')] } as Cluster);
+
+		await getClusterRestartTargets('clu-1');
+
+		expect(authStore.checkForFabricConnect).toHaveBeenCalledWith('clu-1');
+		expect(getInstanceClient).toHaveBeenCalledWith(expect.objectContaining({ forceFabricConnect: true }));
+	});
+
+	it('treats a cluster without an instance list as empty', async () => {
+		vi.mocked(getClusterInfo).mockResolvedValue({ id: 'clu-1' } as Cluster);
+
+		expect(await getClusterRestartTargets('clu-1')).toEqual({ allInstances: [], targets: [] });
+	});
+
+	it('rejects when the cluster cannot be fetched, so callers can report it', async () => {
+		vi.mocked(getClusterInfo).mockRejectedValue(new Error('503'));
+
+		await expect(getClusterRestartTargets('clu-1')).rejects.toThrow('503');
+		expect(getInstanceClient).not.toHaveBeenCalled();
+	});
+});

src/hooks/getClusterRestartTargets.ts

@@ -0,0 +1,35 @@
+import { getInstanceClient } from '@/config/getInstanceClient';
+import { authStore } from '@/features/auth/store/authStore';
+import { getClusterInfo } from '@/features/cluster/queries/getClusterInfoQuery';
+import { Instance } from '@/integrations/api/api.patch';
+import { getOperationsUrlForInstance } from '@/lib/urls/getOperationsUrlForInstance';
+import type { AxiosInstance } from 'axios';
+
+export interface ClusterRestartTarget {
+	instance: Instance;
+	instanceClient: AxiosInstance;
+}
+
+export interface ClusterRestartTargets {
+	allInstances: Instance[];
+	/** A client for each RUNNING instance, in the order a rolling restart visits them. */
+	targets: ClusterRestartTarget[];
+}
+
+export async function getClusterRestartTargets(clusterId: string): Promise<ClusterRestartTargets> {
+	const cluster = await getClusterInfo(clusterId);
+	const allInstances = cluster?.instances ?? [];
+	const clusterUsesFabricConnect = !!cluster && authStore.checkForFabricConnect(cluster.id);
+	const targets = allInstances
+		.filter(instance => instance.status === 'RUNNING')
+		.map(instance => ({
+			instance,
+			instanceClient: getInstanceClient({
+				id: instance.id,
+				forceFabricConnect: clusterUsesFabricConnect,
+				operationsUrl: getOperationsUrlForInstance(instance),
+			}),
+		}))
+		.reverse();
+	return { allInstances, targets };
+}

src/hooks/useApplyLicensesClick.test.tsx

@@ -0,0 +1,78 @@
+/**
+ * @vitest-environment jsdom
+ */
+import type { SchemaLicense } from '@/integrations/api/api.gen';
+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
+import { act, renderHook } from '@testing-library/react';
+import { PropsWithChildren } from 'react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('@/config/useInstanceClient', () => {
+	const instanceClient = { target: 'ins-a' };
+	const params = { entityType: 'instance', entityId: 'ins-a', instanceClient };
+	return {
+		useInstanceClient: () => instanceClient,
+		useInstanceClientIdParams: () => params,
+	};
+});
+vi.mock('@/integrations/api/instance/status/getInstanceUserInfo', () => ({ getInstanceUserInfo: vi.fn() }));
+vi.mock('@/integrations/api/instance/auth/installUsageLicense', () => ({ installUsageLicense: vi.fn() }));
+vi.mock('@/lib/sleep', () => ({ sleep: vi.fn(() => Promise.resolve()) }));
+vi.mock('sonner', () => ({
+	toast: { loading: vi.fn(() => 'toast-1'), error: vi.fn(), success: vi.fn(), dismiss: vi.fn() },
+}));
+
+import { installUsageLicense } from '@/integrations/api/instance/auth/installUsageLicense';
+import { getInstanceUserInfo } from '@/integrations/api/instance/status/getInstanceUserInfo';
+import { toast } from 'sonner';
+import { useApplyLicensesClick } from './useApplyLicensesClick';
+
+const LICENSES = [{ license: 'lic-1' }, { license: 'lic-2' }] as SchemaLicense[];
+
+function wrapper() {
+	const queryClient = new QueryClient();
+	return ({ children }: PropsWithChildren) => <QueryClientProvider client={queryClient}>{children}
+	</QueryClientProvider>;
+}
+
+async function applyLicenses() {
+	const { result } = renderHook(() => useApplyLicensesClick({ licenses: LICENSES }), { wrapper: wrapper() });
+	await act(async () => {
+		await (result.current.onApplyLicensesClick() as unknown as Promise<void>);
+	});
+	return result;
+}
+
+beforeEach(() => {
+	vi.mocked(getInstanceUserInfo).mockResolvedValue({} as never);
+	vi.mocked(installUsageLicense).mockResolvedValue({ message: 'ok' } as never);
+});
+
+afterEach(() => vi.clearAllMocks());
+
+describe('useApplyLicensesClick', () => {
+	it('installs every license and reports success', async () => {
+		const result = await applyLicenses();
+
+		expect(vi.mocked(installUsageLicense).mock.calls.map(([body]) => body.license)).toEqual(['lic-1', 'lic-2']);
+		expect(result.current.isApplyLicensesPending).toBe(false);
+		expect(toast.success).toHaveBeenCalled();
+	});
+
+	it('does not install a license when Cancel is clicked while the instance is being checked', async () => {
+		vi.mocked(getInstanceUserInfo).mockImplementation(async () => {
+			cancel();
+			return {} as never;
+		});
+
+		await applyLicenses();
+
+		expect(installUsageLicense).not.toHaveBeenCalled();
+		expect(toast.error).toHaveBeenCalledWith('Cancelled', expect.anything());
+	});
+});
+
+function cancel() {
+	const [, options] = vi.mocked(toast.loading).mock.calls.at(-1)!;
+	(options as unknown as { action: { onClick: () => void } }).action.onClick();
+}

src/hooks/useApplyLicensesClick.tsx

@@ -74,6 +74,9 @@ export function useApplyLicensesClick({ licenses }: ApplyLicensesClickParams): A
 					await getInstanceUserInfo({
 						instanceClient,
 					});
+					if (canceled) {
+						break;
+					}
 					// Then install the usage license to it.
 					await installUsageLicense({
 						license: license.license,

src/hooks/useRestartClusterClick.test.tsx

@@ -0,0 +1,171 @@
+/**
+ * @vitest-environment jsdom
+ */
+import type { Cluster, Instance } from '@/integrations/api/api.patch';
+import { getRestartState } from '@/lib/restart/restartTracker';
+import { act, renderHook } from '@testing-library/react';
+import type { AxiosInstance } from 'axios';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('@tanstack/react-router', () => ({
+	useParams: () => ({ clusterId: 'clu-1' }),
+}));
+vi.mock('@/config/getInstanceClient', () => ({
+	getInstanceClient: vi.fn(({ id }: { id: string }) => ({ target: id })),
+}));
+vi.mock('@/features/auth/store/authStore', () => ({
+	authStore: { checkForFabricConnect: () => false },
+}));
+vi.mock('@/features/cluster/queries/getClusterInfoQuery', () => ({ getClusterInfo: vi.fn() }));
+vi.mock('@/integrations/api/instance/status/getInstanceUserInfo', () => ({ getInstanceUserInfo: vi.fn() }));
+vi.mock('@/integrations/api/instance/status/restartInstance', () => ({ restartInstance: vi.fn() }));
+vi.mock('@/lib/sleep', () => ({ sleep: vi.fn(() => Promise.resolve()) }));
+vi.mock('sonner', () => ({
+	toast: { loading: vi.fn(() => 'toast-1'), error: vi.fn(), success: vi.fn(), dismiss: vi.fn() },
+}));
+
+import { getInstanceClient } from '@/config/getInstanceClient';
+import { getClusterInfo } from '@/features/cluster/queries/getClusterInfoQuery';
+import { getInstanceUserInfo } from '@/integrations/api/instance/status/getInstanceUserInfo';
+import { restartInstance } from '@/integrations/api/instance/status/restartInstance';
+import { toast } from 'sonner';
+import { useRestartClusterClick } from './useRestartClusterClick';
+
+function instance(id: string, overrides: Partial<Instance> = {}): Instance {
+	return {
+		id,
+		status: 'RUNNING',
+		instanceFqdn: `${id}.example.com`,
+		operationsApiPort: 9925,
+		operationsApiSecure: true,
+		...overrides,
+	} as Instance;
+}
+
+function targetOf(client: AxiosInstance): string {
+	return (client as unknown as { target: string }).target;
+}
+
+let restarted: string[] = [];
+
+async function clickRestart() {
+	const { result } = renderHook(() => useRestartClusterClick());
+	await act(async () => {
+		await (result.current.onRestartClick() as unknown as Promise<void>);
+	});
+	return result;
+}
+
+beforeEach(() => {
+	restarted = [];
+	vi.mocked(getInstanceUserInfo).mockResolvedValue({} as never);
+	vi.mocked(restartInstance).mockImplementation(async ({ instanceClient }) => {
+		restarted.push(targetOf(instanceClient));
+		return { message: 'ok' };
+	});
+});
+
+afterEach(() => vi.clearAllMocks());
+
+describe('useRestartClusterClick', () => {
+	it('restarts only the RUNNING instances, last first, and reports the one it could not restart', async () => {
+		vi.mocked(getClusterInfo).mockResolvedValue(
+			{
+				id: 'clu-1',
+				instances: [instance('ins-a'), instance('ins-b', { status: 'STOPPED' }), instance('ins-c')],
+			} as Cluster,
+		);
+
+		const result = await clickRestart();
+
+		expect(restarted).toEqual(['ins-c', 'ins-a']);
+		expect(result.current.isRestartPending).toBe(false);
+		expect(toast.error).toHaveBeenCalledWith(
+			'Error',
+			expect.objectContaining({ description: expect.stringContaining('Only 2 "RUNNING" instances were restarted') }),
+		);
+	});
+
+	it('reports success once every instance restarted', async () => {
+		vi.mocked(getClusterInfo).mockResolvedValue({ id: 'clu-1', instances: [instance('ins-a')] } as Cluster);
+
+		await clickRestart();
+
+		expect(restarted).toEqual(['ins-a']);
+		expect(toast.success).toHaveBeenCalledWith('Success', expect.objectContaining({ id: 'toast-1' }));
+		expect(getRestartState('clu-1')).toBeUndefined();
+	});
+
+	it('does not restart an instance when Cancel is clicked while it is being checked', async () => {
+		vi.mocked(getClusterInfo).mockResolvedValue(
+			{ id: 'clu-1', instances: [instance('ins-a'), instance('ins-b')] } as Cluster,
+		);
+		vi.mocked(getInstanceUserInfo).mockImplementation(async () => {
+			cancel();
+			return {} as never;
+		});
+
+		await clickRestart();
+
+		expect(restartInstance).not.toHaveBeenCalled();
+		expect(toast.error).toHaveBeenCalledWith('Cancelled', expect.anything());
+	});
+
+	it('says the RUNNING instances failed to restart, not that none were RUNNING', async () => {
+		vi.mocked(getClusterInfo).mockResolvedValue(
+			{ id: 'clu-1', instances: [instance('ins-a'), instance('ins-b')] } as Cluster,
+		);
+		vi.mocked(restartInstance).mockRejectedValue(new Error('did not come back'));
+
+		await clickRestart();
+
+		const [, options] = vi.mocked(toast.error).mock.calls.at(-1)!;
+		expect(options?.description).toContain('Only 0 "RUNNING" instances were restarted of 2 instances.');
+		expect(options?.description).not.toContain('No instances were in a "RUNNING" state');
+	});
+
+	it('reports a cluster with no instances as a failure, not a success', async () => {
+		vi.mocked(getClusterInfo).mockResolvedValue({ id: 'clu-1', instances: [] } as unknown as Cluster);
+
+		await clickRestart();
+
+		expect(toast.success).not.toHaveBeenCalled();
+		expect(toast.error).toHaveBeenCalledWith(
+			'Error',
+			expect.objectContaining({ description: expect.stringContaining('No instances were found within the cluster') }),
+		);
+	});
+
+	describe('when the cluster cannot be loaded', () => {
+		it.each([
+			['the cluster fetch rejects', () => vi.mocked(getClusterInfo).mockRejectedValue(new Error('503'))],
+			['an instance client cannot be built', () => {
+				vi.mocked(getClusterInfo).mockResolvedValue({ id: 'clu-1', instances: [instance('ins-a')] } as Cluster);
+				vi.mocked(getInstanceClient).mockImplementationOnce(() => {
+					throw new TypeError('Invalid URL');
+				});
+			}],
+		])('when %s, it reports it on the restart toast, restarts nothing, and stops pending', async (_label, arrange) => {
+			arrange();
+
+			const result = await clickRestart();
+
+			expect(restartInstance).not.toHaveBeenCalled();
+			expect(result.current.isRestartPending).toBe(false);
+			expect(getRestartState('clu-1')).toBeUndefined();
+			expect(toast.success).not.toHaveBeenCalled();
+			expect(toast.error).toHaveBeenCalledWith(
+				'Error',
+				expect.objectContaining({
+					id: 'toast-1',
+					description: "Could not load the cluster's instances, so nothing was restarted.",
+				}),
+			);
+		});
+	});
+});
+
+function cancel() {
+	const [, options] = vi.mocked(toast.loading).mock.calls.at(-1)!;
+	(options as unknown as { action: { onClick: () => void } }).action.onClick();
+}