feat: run a user-data script on the nodes

Pulumi.yaml

@@ -94,6 +94,10 @@ config:
     type: integer
     description: "Root disk size for the instance in GiB (AWS only, set by the flavor on OpenStack)"
     default: 50
+  instance_user_data:
+    type: string
+    description: "Script every node runs in the background once booted (see docs/PLATFORM.md)"
+    default: ""
 
   # Bastion host configuration
   bastion_image:

README.md

@@ -56,6 +56,7 @@ A platform is described by the configuration of its stack. Set a key with
 | instance_flavor | Flavor of the instance, a portable size or a name from the cloud catalog | string | `medium` | no |
 | instance_ssh_user | SSH user for the instances (defaults to the one the image is known to use) | string | `` | no |
 | instance_root_disk_size | Root disk size for the instance in GiB (AWS only, set by the flavor on OpenStack) | integer | `50` | no |
+| instance_user_data | Script every node runs in the background once booted (see docs/PLATFORM.md) | string | `` | no |
 | bastion_image | Image for the bastion host | string | `rocky-9` | no |
 | bastion_flavor | Flavor of the bastion, a portable size or a name from the cloud catalog | string | `small` | no |
 | bastion_ssh_user | SSH user for the bastion (defaults to the one the image is known to use) | string | `` | no |

__main__.py

@@ -60,6 +60,22 @@
 printf '%s' "$BASTION_PRIVATE_KEY" | ssh -F {config} bastion "$install"
 scp -F {config} {bastion_config} bastion:{remote_config}
 """
+# Where the user-data of the nodes lands, and the unit it runs as.
+# NOTE: The script goes where local programs do. Under SELinux, systemd runs
+# what it finds there unconfined, while a script left among the configuration
+# runs in the domain of init, which may not even touch /etc/shadow.
+USER_DATA_ENV = pathlib.PurePosixPath("/etc/platform-spawner/node.env")
+USER_DATA_SCRIPT = pathlib.PurePosixPath("/usr/local/libexec/platform-spawner/user-data")
+USER_DATA_UNIT = "platform-spawner-user-data"
+# Relative to the home of the SSH user, which needs no privilege to write to
+NODES_KEY = ".ssh/platform-spawner-nodes"
+# What the nodes are handed when they run user-data, so they reach each other
+NODES_SETUP_SCRIPT = """set -eu
+install="install -d -m 700 ~/.ssh && cat > ~/{key} && chmod 600 ~/{key}"
+for host in {hosts}; do
+    printf '%s' "$NODES_PRIVATE_KEY" | ssh -F {config} "$host" "$install"
+done
+"""
 # Address space of the routed planes, needed both to carve the subnets out
 # and to tell the bastion whose clock it is allowed to serve
 CONTROL_PLANE_CIDR = "172.30.100.0/24"
@@ -229,6 +245,10 @@ def __main__() -> None:
         "user": _ssh_user(config, "bastion"),
     }
     nodes_info = {}
+    user_data = config.require("instance_user_data")
+    present_nodes = [
+        index for index in range(1, instance_count + 1) if f"node-{index}" not in absent_nodes
+    ]
 
     for node_index in range(1, instance_count + 1):
         # NOTE: The interfaces of an absent node are kept. They hold its
@@ -266,6 +286,13 @@ def __main__() -> None:
             cloud_config={
                 "ssh_authorized_keys": [bastion_key.public_key_openssh],
                 **_ntp_config(servers=[bastion_cp_iface.ip, bastion_wp_iface.ip]),
+                **(
+                    _user_data_config(
+                        user_data, node_index, present_nodes, _ssh_user(config, "instance")
+                    )
+                    if user_data
+                    else {}
+                ),
             },
         )
         instances.append(node)
@@ -294,6 +321,8 @@ def __main__() -> None:
 
     booted = _wait_for_boot(ssh_config_ready, ["bastion", *nodes_info], instances)
     _equip_bastion(ssh_config_ready, bastion_key, booted, instances)
+    if user_data:
+        _equip_nodes(ssh_config_ready, list(nodes_info), bastion_key, booted, instances)
     _clean_up_ssh_files(ssh_config_file)
 
 
@@ -402,6 +431,56 @@ def _ntp_config(**settings: object) -> dict:
     return {"ntp": {"enabled": True, "ntp_client": NTP_CLIENT, **settings}}
 
 
+def _user_data_config(
+    script_path: str,
+    node_index: int,
+    present_nodes: list[int],
+    ssh_user: str,
+) -> dict:
+    """
+    Render the cloud-init configuration running a node's user-data.
+
+    NOTE: Run as a unit of its own rather than by cloud-init. The update waits
+    for cloud-init to be done with every machine, and what a script is handed
+    to do, installing a product for instance, easily outlasts that wait.
+
+    The script is told where the node stands through an environment file. Its
+    addresses are fixed, so they are known before any machine exists.
+    """
+    env = {
+        "NODE_NAME": f"node-{node_index}",
+        "NODE_INDEX": node_index,
+        "NODE_COUNT": len(present_nodes),
+        "NODES_CONTROL_PLANE_IPS": " ".join(f"172.30.100.{100 + i}" for i in present_nodes),
+        "NODES_WORKLOAD_PLANE_IPS": " ".join(f"172.30.200.{100 + i}" for i in present_nodes),
+        "CONTROL_PLANE_CIDR": CONTROL_PLANE_CIDR,
+        "WORKLOAD_PLANE_CIDR": WORKLOAD_PLANE_CIDR,
+        "SSH_USER": ssh_user,
+        "NODES_KEY": NODES_KEY,
+    }
+    return {
+        "write_files": [
+            {
+                "path": str(USER_DATA_ENV),
+                "content": "".join(f'{name}="{value}"\n' for name, value in env.items()),
+            },
+            {
+                "path": str(USER_DATA_SCRIPT),
+                "content": pathlib.Path(script_path).read_text(),
+                "permissions": "0755",
+            },
+        ],
+        "runcmd": [
+            [
+                "systemd-run",
+                f"--unit={USER_DATA_UNIT}",
+                f"--property=EnvironmentFile={USER_DATA_ENV}",
+                str(USER_DATA_SCRIPT),
+            ],
+        ],
+    }
+
+
 def _extra_network_cidr(index: int) -> str:
     """
     Return the address space of the `index`th extra network.
@@ -628,5 +707,35 @@ def _equip_bastion(
     )
 
 
+def _equip_nodes(
+    config_file: pulumi.Output[str],
+    nodes: list[str],
+    bastion_key: pulumi_tls.PrivateKey,
+    booted: pulumi_command.local.Command,
+    instances: list[pulumi.CustomResource],
+) -> None:
+    """
+    Hand the nodes running user-data a key, so they reach each other.
+
+    NOTE: The key of the bastion rather than one more. Every node trusts it
+    already, and it opens nothing but the nodes, which is all a node driving
+    the others needs. It lands in the home of the SSH user: a node whose
+    product has taken sudo away still takes it on the next update.
+    """
+    pulumi_command.local.Command(
+        "nodes-ssh-key",
+        create=config_file.apply(
+            lambda config: NODES_SETUP_SCRIPT.format(
+                config=config,
+                key=NODES_KEY,
+                hosts=" ".join(nodes),
+            )
+        ),
+        environment={"NODES_PRIVATE_KEY": bastion_key.private_key_openssh},
+        triggers=[instance.id for instance in instances],
+        opts=pulumi.ResourceOptions(depends_on=[booted]),
+    )
+
+
 if __name__ == "__main__":
     __main__()

docs/PLATFORM.md

@@ -68,6 +68,34 @@ platform, and is not what a platform is spawned to test.
 From a workflow, the same goes through the
 [`update` action](GITHUB_ACTION.md#updating).
 
+### Running a script on the nodes
+
+`instance_user_data` names a script every node runs once it has booted,
+installing a product for instance. It runs as root, in a unit of its own,
+`platform-spawner-user-data`, rather than inside cloud-init: the update only
+waits for the machines to boot, and the script outlasts it as long as it
+needs. Follow it with:
+
+```bash
+ssh -F ssh_config node-1 sudo journalctl -fu platform-spawner-user-data
+```
+
+The script learns where it stands from its environment:
+
+| Variable | What it holds |
+|----------|---------------|
+| `NODE_NAME`, `NODE_INDEX` | Which node runs it, `node-1` and `1` for instance |
+| `NODE_COUNT` | How many nodes are there, absent ones left out |
+| `NODES_CONTROL_PLANE_IPS`, `NODES_WORKLOAD_PLANE_IPS` | The addresses of every node, itself included |
+| `CONTROL_PLANE_CIDR`, `WORKLOAD_PLANE_CIDR` | The address spaces of the planes |
+| `SSH_USER` | The user the nodes are reached with |
+| `NODES_KEY` | Where a key opening the other nodes lands, under the home of `SSH_USER` |
+
+The key arrives once every machine has booted, so a script needing the other
+nodes waits for it. It is the key of the bastion, which opens the nodes and
+nothing else. [`examples/user-data/artesca-install.sh`](../examples/user-data/artesca-install.sh)
+installs ARTESCA that way, on one node or three.
+
 ## Networks
 
 | Network | CIDR | What it is |

docs/QUICKSTART.md

@@ -142,7 +142,7 @@ The ones worth knowing early:
 
 | Key | Default | What it does |
 |-----|---------|--------------|
-| `instance_flavor` | `medium` | `small`, `medium`, `large` or `xlarge`. |
+| `instance_flavor` | `medium` | `small`, `medium`, `large` or `xlarge`, or any name of the cloud catalog (`c5.4xlarge`). |
 | `instance_root_disk_size` | `50` | Root disk of a node, in GiB. AWS only, since the flavor decides it on OpenStack. |
 | `bastion_flavor` | `small` | Same sizes as `instance_flavor`. |
 | `authorized_cidrs` | `['__my_ip__']` | Who may reach the bastion. `__my_ip__` stands for the public address of the machine spawning. |
@@ -238,6 +238,80 @@ for someone to decide, so remove it once you are sure:
 rm -f ssh_<product>_<stack> ssh_<product>_<stack>.pub
 ```
 
+## Example: ARTESCA from its AMI
+
+One platform per architecture, both from the same AMI, here
+`artesca-4.4.0-preview.32` in `eu-north-1`. The AMI is shared with us, so it is
+named by its id. Every command names its stack with `-s`, which lets both
+live side by side without `pulumi stack select`:
+
+```bash
+for stack in <you>-1node:1 <you>-3node:3; do
+    name=${stack%%:*} count=${stack##*:}
+    pulumi stack init "$name"
+    pulumi -s "$name" config set provider aws
+    pulumi -s "$name" config set aws:region eu-north-1
+    pulumi -s "$name" config set product artesca
+    pulumi -s "$name" config set instance_image ami-0cc90c0853c34e7c4
+    pulumi -s "$name" config set instance_count "$count"
+    pulumi -s "$name" config set instance_flavor c5.4xlarge
+    pulumi -s "$name" config set instance_root_disk_size 300
+    pulumi -s "$name" config set instance_ssh_user artesca-os
+    pulumi -s "$name" config set ssh_private_key_create true
+    pulumi -s "$name" config set instance_user_data examples/user-data/artesca-install.sh
+done
+```
+
+| Key | Why |
+|-----|-----|
+| `instance_flavor` | `medium` is far too small for ARTESCA. `c5.4xlarge` is what the CI demo uses. |
+| `instance_root_disk_size` | The `demo` profile keeps its volumes on the root disk. |
+| `instance_ssh_user` | The AMI is none of the `rocky-*` images, so nothing else names its user. |
+| `instance_user_data` | Installs ARTESCA once the nodes are up, see below. Leave it out for bare nodes. |
+
+Both spawn at once, from two shells or in the background:
+
+```bash
+pulumi -s <you>-1node up --yes &
+pulumi -s <you>-3node up --yes &
+wait
+```
+
+> **Warning**
+> The AMI expires the password of `artesca-os`, and an expired password turns
+> SSH away without a terminal to change it, keys included. The update then
+> fails with `node-1 never answered ssh`.
+> [`artesca-install.sh`](../examples/user-data/artesca-install.sh) lifts the
+> expiry, so a stack without `instance_user_data` needs a script that does at
+> least that much. A node already spawned without one takes an interactive
+> `ssh -tt`, and the initial password of the AMI, to change it by hand.
+
+`pulumi up` returns once the nodes have booted, and the install carries on on
+its own on node-1, which drives the other nodes on three. It takes a while.
+Follow it, then reach the UI once it is done:
+
+```bash
+ssh -F ssh_config-<you>-3node node-1 sudo journalctl -fu platform-spawner-user-data
+ssh -F ssh_config-<you>-3node -L 8443:172.30.100.101:8443 node-1   # then https://localhost:8443
+```
+
+Installer logs are in `/var/log/artesca/` on node-1, and the configuration it
+was given in `~/config.yaml`. The other nodes are one hop away, through the
+bastion like any other:
+
+```bash
+ssh -F ssh_config-<you>-1node node-1
+ssh -F ssh_config-<you>-3node node-3
+```
+
+Destroy both the same way as any other:
+
+```bash
+for name in <you>-1node <you>-3node; do
+    pulumi -s "$name" destroy --yes && pulumi stack rm -s "$name" --yes
+done
+```
+
 ## Cheat sheet
 
 ```bash

examples/user-data/artesca-install.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# Install ARTESCA from an ARTESCA AMI, on one node or three.
+#
+# Meant for `instance_user_data`. node-1 drives the install and reaches the
+# other nodes with the key platform-spawner hands it once they have booted.
+#
+# NOTE: AWS has no layer 2 between the nodes and checks the source of every
+# packet, so there are no virtual IPs and pods speak through IP-in-IP.
+set -euo pipefail
+
+# NOTE: The AMI expires the password of its user, and an expired password
+# refuses SSH without a TTY to change it, keys included. Every node needs it
+# lifted: platform-spawner waits on SSH, and node-1 drives the others with it.
+# The AMI expires it from cloud-init, which started this script and may not
+# have got there yet.
+cloud-init status --wait >/dev/null || true
+chage --lastday "$(date +%F)" "$SSH_USER"
+
+[ "$NODE_INDEX" = 1 ] || exit 0
+
+home=$(getent passwd "$SSH_USER" | cut -d: -f6)
+key=/etc/metalk8s/pki/bastion
+
+echo "Waiting for the nodes key"
+until [ -s "$home/$NODES_KEY" ]; do sleep 10; done
+install -D -m 600 "$home/$NODES_KEY" "$key"
+
+nodes=()
+for ip in $NODES_CONTROL_PLANE_IPS; do
+    index=${ip##*.}
+    index=$((index - 100))
+    [ "$index" = 1 ] && continue
+
+    echo "Waiting for node-$index ($ip)"
+    until ssh -i "$key" -o StrictHostKeyChecking=no -o ConnectTimeout=10 \
+        "$SSH_USER@$ip" true; do
+        sleep 10
+    done
+    nodes+=(--node "{name: node-$index, host: $ip, key: $key}")
+done
+
+if [ "$NODE_COUNT" = 1 ]; then
+    architecture=(--architecture single-node)
+else
+    architecture=(
+        --architecture 3-nodes
+        --no-managed-management-vip
+        --no-data-vip
+    )
+fi
+
+cd "$home"
+rm -f config.yaml
+su "$SSH_USER" -c "$(printf '%q ' sudo ./artesca-installer config generate \
+    --output config.yaml \
+    "${architecture[@]}" \
+    --profile demo \
+    --use-case generic \
+    --username scality \
+    --base-domain workloadplane.scality.local \
+    --local-node-name node-1 \
+    --ipip-always \
+    --workload-plane-cidr "$WORKLOAD_PLANE_CIDR" \
+    --control-plane-cidr "$CONTROL_PLANE_CIDR" \
+    --management-ip '' \
+    "${nodes[@]}")"
+su "$SSH_USER" -c "sudo ./artesca-installer config validate --config config.yaml"
+
+echo "Configuration generated, deploying"
+su "$SSH_USER" -c "sudo ./artesca-installer deploy --allow-unverified-raid -c config.yaml"
+echo "Deployment finished"