fix: only wait on the machines an update brings up

__main__.py

@@ -25,34 +25,33 @@
     "rocky-8": "rocky",
     "rocky-9": "rocky",
 }
-# Held until every machine answers SSH and cloud-init is done with it, so that
-# a finished update means a platform that can actually be used.
+# Held until a machine answers SSH and cloud-init is done with it, so that a
+# finished update means a platform that can actually be used.
 BOOT_WAIT_TIMEOUT = 600
 BOOT_WAIT_SCRIPT = """set -eu
-for host in {hosts}; do
-    give_up=$(( $(date +%s) + {timeout} ))
-    until ssh -F {config} -o ConnectTimeout=10 "$host" true 2>/dev/null; do
-        if [ "$(date +%s)" -ge "$give_up" ]; then
-            echo "$host never answered ssh" >&2
-            exit 1
-        fi
-        sleep 5
-    done
-
-    # NOTE: Bounded like the wait above. `cloud-init status --wait` holds for
-    # as long as it takes, and a boot that never ends would otherwise hold the
-    # update with it. 124 is what the timeout itself exits with.
-    booted=0
-    timeout {timeout} ssh -F {config} "$host" cloud-init status --wait || booted=$?
-    if [ "$booted" -eq 124 ]; then
-        echo "$host was still booting after {timeout}s" >&2
-        exit 1
-    fi
-    if [ "$booted" -ne 0 ]; then
-        echo "$host booted badly, cloud-init exited $booted" >&2
+host={host}
+give_up=$(( $(date +%s) + {timeout} ))
+until ssh -F {config} -o ConnectTimeout=10 "$host" true 2>/dev/null; do
+    if [ "$(date +%s)" -ge "$give_up" ]; then
+        echo "$host never answered ssh" >&2
         exit 1
     fi
+    sleep 5
 done
+
+# NOTE: Bounded like the wait above. `cloud-init status --wait` holds for as
+# long as it takes, and a boot that never ends would otherwise hold the update
+# with it. 124 is what the timeout itself exits with.
+booted=0
+timeout {timeout} ssh -F {config} "$host" cloud-init status --wait || booted=$?
+if [ "$booted" -eq 124 ]; then
+    echo "$host was still booting after {timeout}s" >&2
+    exit 1
+fi
+if [ "$booted" -ne 0 ]; then
+    echo "$host booted badly, cloud-init exited $booted" >&2
+    exit 1
+fi
 """
 # What the bastion is handed, so that it reaches the nodes on its own
 BASTION_SETUP_SCRIPT = """set -eu
@@ -580,27 +579,39 @@ def _wait_for_boot(
     config_file: pulumi.Output[str],
     hosts: list[str],
     instances: list[pulumi.CustomResource],
-) -> pulumi_command.local.Command:
-    """Hold the update until every machine is done booting."""
-    return pulumi_command.local.Command(
-        "wait-for-boot",
-        create=config_file.apply(
-            lambda config: BOOT_WAIT_SCRIPT.format(
-                hosts=" ".join(hosts),
-                config=config,
-                timeout=BOOT_WAIT_TIMEOUT,
-            )
-        ),
-        # Wait again whenever a machine is replaced under us
-        triggers=[instance.id for instance in instances],
-        opts=pulumi.ResourceOptions(depends_on=instances),
-    )
+) -> list[pulumi_command.local.Command]:
+    """
+    Hold the update until every new machine is done booting.
+
+    NOTE: One wait per machine, so that an update only waits on the machines
+    it replaced. The ones left alone have been in use since, and asking
+    cloud-init about them again can only go wrong: once something has run the
+    tmpfiles rules again, as installing a package does, Rocky keeps
+    /run/cloud-init to root and `cloud-init status` fails for the SSH user,
+    which may well not be allowed to run it as root.
+    """
+    return [
+        pulumi_command.local.Command(
+            f"wait-for-boot-{host}",
+            create=config_file.apply(
+                lambda config, host=host: BOOT_WAIT_SCRIPT.format(
+                    host=host,
+                    config=config,
+                    timeout=BOOT_WAIT_TIMEOUT,
+                )
+            ),
+            # Wait again whenever this machine is replaced under us
+            triggers=[instance.id],
+            opts=pulumi.ResourceOptions(depends_on=[instance]),
+        )
+        for host, instance in zip(hosts, instances, strict=True)
+    ]
 
 
 def _equip_bastion(
     config_file: pulumi.Output[str],
     bastion_key: pulumi_tls.PrivateKey,
-    booted: pulumi_command.local.Command,
+    booted: list[pulumi_command.local.Command],
     instances: list[pulumi.CustomResource],
 ) -> None:
     """
@@ -624,7 +635,7 @@ def _equip_bastion(
         # Handed again whenever a machine changes. The config names the
         # nodes, and a replaced bastion starts with neither key nor config.
         triggers=[instance.id for instance in instances],
-        opts=pulumi.ResourceOptions(depends_on=[booted]),
+        opts=pulumi.ResourceOptions(depends_on=booted),
     )