feat: add armhf (32-bit ARM) architecture support [ISD-5856]

.github/workflows/integration_test_app.yaml

@@ -15,18 +15,22 @@ jobs:
       fail-fast: false
       matrix:
         image: [focal, jammy, noble, resolute]
-        arch: [amd64, arm64, s390x, ppc64le]
+        arch: [amd64, arm64, armhf, s390x, ppc64le]
         exclude:
           - image: focal
             arch: ppc64le
           - image: focal
             arch: s390x
+          - image: focal
+            arch: armhf
           - image: jammy
             arch: arm64
           - image: jammy
             arch: ppc64le
           - image: jammy
             arch: s390x
+          - image: jammy
+            arch: armhf
           - image: resolute
             arch: arm64
           - image: resolute
@@ -59,6 +63,13 @@ jobs:
         run: |
           tox -e integration -- --arch arm64 --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS_APP_ARM64 }}
         working-directory: app
+      - name: Run integration tests (armhf)
+        if: matrix.arch == 'armhf'
+        env:
+          OPENSTACK_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD_ARM64 }}
+        run: |
+          tox -e integration -- --arch armhf --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS_APP_ARM64 }}
+        working-directory: app
       - name: Run integration tests (s390x)
         if: matrix.arch == 's390x'
         env:

.gitignore

@@ -30,3 +30,6 @@ testing_key.pem
 .vale/styles/config/vocabularies/*
 !.vale/styles/config/vocabularies/local
 # END VALE WORKFLOW IGNORE
+
+# Superpowers internal planning artifacts (not product docs)
+docs/superpowers/

app/conftest.py

@@ -41,7 +41,7 @@ def pytest_addoption(parser: Parser):
         "--arch",
         action="store",
         help="The architecture to build for.",
-        choices=["amd64", "arm64", "s390x", "ppc64le"],
+        choices=["amd64", "arm64", "armhf", "s390x", "ppc64le"],
     )
     parser.addoption(
         "--openstack-network-name",

app/src/github_runner_image_builder/cloud_image.py

@@ -20,7 +20,7 @@
 
 logger = logging.getLogger(__name__)
 
-SupportedBaseImageArch = typing.Literal["amd64", "arm64", "s390x", "ppc64el"]
+SupportedBaseImageArch = typing.Literal["amd64", "arm64", "s390x", "ppc64el", "armhf"]
 
 CHECKSUM_BUF_SIZE = 65536  # 65kb
 
@@ -95,6 +95,11 @@ def _get_supported_runner_arch(arch: Arch) -> SupportedBaseImageArch:
             return "s390x"
         case Arch.PPC64LE:
             return "ppc64el"  # cloud-images.ubuntu.com uses ppc64el instead of ppc64le
+        case Arch.ARM:
+            # Download the arm64 (64-bit) base cloud image, not armhf. The armhf runner image is
+            # an arm64 boot image (64-bit kernel) that runs the 32-bit linux-arm runner via
+            # multiarch; a native armhf (32-bit kernel) image does not boot on the aarch64 hosts.
+            return "arm64"
         case _:
             raise UnsupportedArchitectureError(f"Detected system arch: {arch} is unsupported.")
 

app/src/github_runner_image_builder/config.py

@@ -20,12 +20,14 @@ class Arch(str, Enum):
         X64: Represents an X64/AMD64 system architecture.
         S390X: Represents an S390X system architecture.
         PPC64LE: Represents a PPC64LE system architecture.
+        ARM: Represents an ARM (32-bit armhf) system architecture.
     """
 
     ARM64 = "arm64"
     X64 = "x64"
     S390X = "s390x"
     PPC64LE = "ppc64le"
+    ARM = "arm"
 
     def to_openstack(self) -> str:
         """Convert the architecture to OpenStack compatible arch string.
@@ -42,6 +44,13 @@ def to_openstack(self) -> str:
                 return "s390x"
             case Arch.PPC64LE:
                 return "ppc64le"
+            case Arch.ARM:
+                # The armhf runner image is an arm64 boot image (64-bit kernel) that carries a
+                # 32-bit linux-arm runner payload. Native armhf images (32-bit kernel) do not boot
+                # on the aarch64 "virt" machine, so the glance architecture is aarch64 so the base
+                # image and snapshot schedule and boot on arm64 hosts. The 32-bit runner is
+                # installed via multiarch (see cloud-init.sh.j2 and ARM_ADDITIONAL_APT_PACKAGES).
+                return "aarch64"
         raise ValueError  # pragma: nocover
 
 
@@ -127,6 +136,26 @@ def from_str(cls, tag_or_name: str) -> "BaseImage":
     "wget",
 ]
 S390X_PPC64LE_ADDITIONAL_APT_PACKAGES = ["dotnet-runtime-8.0"]
+# The 32-bit linux-arm runner agent runs via the host's native AArch32 support on the arm64
+# image. It needs the armhf loader (ld-linux-armhf.so.3 from libc6:armhf) and the armhf build of
+# libatomic (a .NET runtime dependency). rustup provides the armhf/armv7 Rust toolchain and
+# docker-buildx enables arm32 container builds. libicu (the other .NET runtime dependency) is
+# release-specific and handled by ARM_LIBICU_APT_PACKAGE_BY_BASE below.
+ARM_ADDITIONAL_APT_PACKAGES = [
+    "libc6:armhf",
+    "libatomic1:armhf",
+    "rustup",
+    "docker-buildx",
+]
+# rustup (installed on armhf images) conflicts with the distro cargo and rustc packages, so these
+# are dropped from the default apt package set on armhf; rustup provides cargo and rustc instead.
+ARM_EXCLUDED_DEFAULT_APT_PACKAGES = ("cargo", "rustc")
+# The linux-arm runner's bundled .NET runtime dlopens libicu, whose soname is release-specific
+# (noble ships libicu74, resolute ships libicu78). Install the armhf build matching the base image.
+ARM_LIBICU_APT_PACKAGE_BY_BASE = {
+    BaseImage.NOBLE: "libicu74:armhf",
+    BaseImage.RESOLUTE: "libicu78:armhf",
+}
 
 _LOG_LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR)
 LOG_LEVELS = tuple(

app/src/github_runner_image_builder/openstack_builder.py

@@ -40,6 +40,9 @@
 import github_runner_image_builder.errors
 from github_runner_image_builder import cloud_image, config, store
 from github_runner_image_builder.config import (
+    ARM_ADDITIONAL_APT_PACKAGES,
+    ARM_EXCLUDED_DEFAULT_APT_PACKAGES,
+    ARM_LIBICU_APT_PACKAGE_BY_BASE,
     FORK_RUNNER_BINARY_REPO,
     IMAGE_DEFAULT_APT_PACKAGES,
     S390X_PPC64LE_ADDITIONAL_APT_PACKAGES,
@@ -60,6 +63,10 @@
 SHARED_SECURITY_GROUP_NAME = "github-runner-image-builder-v1"
 EXTERNAL_SCRIPT_PATH = pathlib.Path("/root/external.sh")
 
+# armhf additional apt packages (libicu74, rustup, docker-buildx) are only available from the
+# noble (24.04) archive onwards, so armhf images can only be built on these base images.
+ARM_SUPPORTED_BASE_IMAGES = (BaseImage.NOBLE, BaseImage.RESOLUTE)
+
 # Server operation timeout constants (in seconds)
 CREATE_SERVER_TIMEOUT = 20 * 60  # 20 minutes
 DELETE_SERVER_TIMEOUT = 20 * 60  # 20 minutes
@@ -540,6 +547,10 @@ def _generate_cloud_init_script(
         image_config: The target image configuration values.
         proxy: The proxy to enable while setting up the VM.
 
+    Raises:
+        UnsupportedArchitectureError: If an armhf image is requested on a base image older than
+            the supported ARM base images (noble and newer).
+
     Returns:
         The cloud-init script to create snapshot image.
     """
@@ -552,6 +563,29 @@ def _generate_cloud_init_script(
     apt_packages = IMAGE_DEFAULT_APT_PACKAGES
     if image_config.arch in (Arch.S390X, Arch.PPC64LE):
         apt_packages = IMAGE_DEFAULT_APT_PACKAGES + S390X_PPC64LE_ADDITIONAL_APT_PACKAGES
+    elif image_config.arch == Arch.ARM:
+        # The armhf additional apt packages (rustup, docker-buildx, the armhf multiarch libs) and
+        # the release-specific libicu are only available from the noble (24.04) archive onwards,
+        # so fail fast on older bases rather than letting apt-get fail midway through the build.
+        if image_config.base not in ARM_SUPPORTED_BASE_IMAGES:
+            raise github_runner_image_builder.errors.UnsupportedArchitectureError(
+                f"armhf images require a base image of "
+                f"{[base.value for base in ARM_SUPPORTED_BASE_IMAGES]} or newer, "
+                f"got: {image_config.base.value}."
+            )
+        # rustup provides the armhf/armv7 Rust toolchain and conflicts with the distro cargo/rustc
+        # packages, so drop those from the default set. libicu's soname is release-specific, so
+        # add the armhf build matching the base image.
+        default_packages = [
+            package
+            for package in IMAGE_DEFAULT_APT_PACKAGES
+            if package not in ARM_EXCLUDED_DEFAULT_APT_PACKAGES
+        ]
+        apt_packages = (
+            default_packages
+            + ARM_ADDITIONAL_APT_PACKAGES
+            + [ARM_LIBICU_APT_PACKAGE_BY_BASE[image_config.base]]
+        )
     return template.render(
         PROXY=proxy,
         APT_PACKAGES=" ".join(apt_packages),

app/src/github_runner_image_builder/store.py

@@ -77,12 +77,13 @@ def upload_image(
     with openstack.connect(cloud=cloud_name) as connection:
         try:
             logger.info("Uploading image %s.", image_name)
+            image_properties = {"architecture": arch.to_openstack()}
             # ignore type since the library does not provide correct type hinting but the docstring
             # does define the return type.
             image: Image = connection.create_image(
                 name=image_name,
                 filename=str(image_path),
-                properties={"architecture": arch.to_openstack()},
+                properties=image_properties,
                 allow_duplicates=True,
                 wait=True,
             )  # type: ignore

app/src/github_runner_image_builder/templates/cloud-init.sh.j2

@@ -53,6 +53,34 @@ EOF
     sleep 5
 }
 
+function enable_armhf_multiarch() {
+    echo "Enabling armhf multiarch"
+    native_arch=$(/usr/bin/dpkg --print-architecture)
+    codename=$(. /etc/os-release && echo "$VERSION_CODENAME")
+    /usr/bin/dpkg --add-architecture armhf
+    # The arm64 cloud image's apt mirrors only carry the native architecture; armhf is a "ports"
+    # architecture served from ports.ubuntu.com. Pin the stock sources to the native architecture
+    # and add a ports.ubuntu.com source for armhf so the 32-bit runner's dependencies resolve.
+    /usr/bin/sed -i "/^Types: deb/a Architectures: $native_arch" /etc/apt/sources.list.d/ubuntu.sources
+    /usr/bin/tee /etc/apt/sources.list.d/armhf-ports.sources >/dev/null <<EOF
+Types: deb
+URIs: http://ports.ubuntu.com/ubuntu-ports
+Suites: $codename $codename-updates $codename-security
+Components: main restricted universe multiverse
+Architectures: armhf
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+EOF
+    # The runner VM boots from a snapshot whose cloud-init state was cleared (see the
+    # "cloud-init clean" before snapshot), so cloud-init regenerates the stock apt sources on
+    # first boot and drops the architecture pin above while the armhf dpkg architecture persists,
+    # breaking "apt-get update". Tell cloud-init to preserve the pinned sources baked into the
+    # image so the 32-bit multiarch configuration survives on the runner VM.
+    /usr/bin/tee /etc/cloud/cloud.cfg.d/99-armhf-preserve-apt-sources.cfg >/dev/null <<EOF
+apt:
+  preserve_sources_list: true
+EOF
+}
+
 function install_apt_packages() {
     local packages="$1"
     local hwe_version="$2"
@@ -205,6 +233,11 @@ github_runner_arch="{{ RUNNER_ARCH }}"
 runner_binary_repo="{{ RUNNER_BINARY_REPO }}"
 
 configure_proxy "$proxy"
+# Enable armhf multiarch before installing packages so the 32-bit linux-arm runner agent and
+# its armhf runtime dependencies can be installed and executed via native AArch32.
+if [ "$github_runner_arch" == "arm" ]; then
+    enable_armhf_multiarch
+fi
 install_apt_packages "$apt_packages" "$hwe_version"
 disable_unattended_upgrades
 enable_network_fair_queuing_congestion
@@ -218,3 +251,9 @@ install_opentelemetry_collector_snap
 install_github_runner "$github_runner_version" "$github_runner_arch"
 chown_home
 configure_system_users
+
+# Set the default Rust toolchain for the ubuntu user. rustup is only installed on
+# armhf images, so this block is a no-op on other architectures.
+if [ "$github_runner_arch" == "arm" ]; then
+    sudo -u ubuntu rustup default stable
+fi

app/tests/integration/commands.py

@@ -5,7 +5,12 @@
 
 import dataclasses
 
-from tests.integration.helpers import TESTDATA_TEST_SCRIPT_URL
+from github_runner_image_builder.config import Arch
+
+TESTDATA_TEST_SCRIPT_URL = (
+    "https://raw.githubusercontent.com/canonical/github-runner-image-builder-operator/"
+    "be135aa505b37aae29aec0ab13805909c46b7903/app/tests/integration/testdata/test_script.sh"
+)
 
 
 @dataclasses.dataclass
@@ -125,3 +130,34 @@ class Commands:
         "! grep '/home/ubuntu/secret.txt' /var/log/auth.log*",
     ),
 )
+
+
+# armhf-specific assertions. These packages/binaries are only installed on armhf images
+# (see ARM_ADDITIONAL_APT_PACKAGES and the rustup cloud-init step), so they must not run on
+# other architectures.
+ARM_RUNNER_COMMANDS = (
+    Commands(name="rustc version (rustup default stable)", command="rustc --version"),
+    Commands(name="cargo version", command="cargo --version"),
+    Commands(name="docker buildx version", command="docker buildx version"),
+    Commands(
+        name="github runner binary is 32-bit ARM",
+        command=(
+            "file /home/ubuntu/actions-runner/bin/Runner.Listener | grep -i 'ELF 32-bit' | "
+            "grep -i 'ARM'"
+        ),
+    ),
+)
+
+
+def commands_for_arch(arch: Arch) -> tuple[Commands, ...]:
+    """Return the test commands to run for the given architecture.
+
+    Args:
+        arch: The architecture under test.
+
+    Returns:
+        The base test commands, plus armhf-specific commands when arch is ARM.
+    """
+    if arch == Arch.ARM:
+        return TEST_RUNNER_COMMANDS + ARM_RUNNER_COMMANDS
+    return TEST_RUNNER_COMMANDS

app/tests/integration/conftest.py

@@ -33,6 +33,8 @@ def arch_fixture(pytestconfig: pytest.Config):
     match arch:
         case "arm64":
             return config.Arch.ARM64
+        case "armhf":
+            return config.Arch.ARM
         case "amd64":
             return config.Arch.X64
         case "s390x":

app/tests/integration/helpers.py

@@ -32,17 +32,12 @@
 from pylxd.models.instance import Instance, InstanceState
 from requests_toolbelt import MultipartEncoder
 
+from github_runner_image_builder.config import Arch
 from tests.integration import commands, types
 
 logger = logging.getLogger(__name__)
 
 
-TESTDATA_TEST_SCRIPT_URL = (
-    "https://raw.githubusercontent.com/canonical/github-runner-image-builder-operator/"
-    "be135aa505b37aae29aec0ab13805909c46b7903/app/tests/integration/testdata/test_script.sh"
-)
-
-
 P = ParamSpec("P")
 R = TypeVar("R")
 S = Callable[P, R] | Callable[P, Awaitable[R]]
@@ -410,13 +405,14 @@ def setup_aproxy(ssh_connection: SSHConnection, proxy: str) -> None:
         assert False, "Aproxy did not start up correctly."
 
 
-def run_openstack_tests(ssh_connection: SSHConnection):
+def run_openstack_tests(ssh_connection: SSHConnection, arch: Arch):
     """Run test commands on the openstack instance via ssh.
 
     Args:
         ssh_connection: The SSH connection instance to OpenStack test server.
+        arch: The architecture under test, selecting arch-specific commands.
     """
-    for testcmd in commands.TEST_RUNNER_COMMANDS:
+    for testcmd in commands.commands_for_arch(arch):
         logger.info("Running command: %s", testcmd.command)
         result: Result = ssh_connection.run(testcmd.command, env=testcmd.env)
         logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr)

app/tests/integration/test_openstack_builder.py

@@ -16,7 +16,7 @@
 import pytest
 import pytest_asyncio
 from fabric.connection import Connection as SSHConnection
-from integration.helpers import TESTDATA_TEST_SCRIPT_URL
+from integration.commands import TESTDATA_TEST_SCRIPT_URL
 from openstack.compute.v2.image import Image
 from openstack.compute.v2.server import Server
 from openstack.connection import Connection
@@ -219,6 +219,7 @@ async def ssh_connection_fixture(
 def test_run(
     ssh_connection: SSHConnection,
     proxy: types.ProxyConfig,
+    arch: config.Arch,
 ):
     """
     arrange: given openstack cloud instance.
@@ -227,7 +228,7 @@ def test_run(
     """
     if proxy.http is not None:
         helpers.setup_aproxy(ssh_connection, proxy.http)
-    helpers.run_openstack_tests(ssh_connection=ssh_connection)
+    helpers.run_openstack_tests(ssh_connection=ssh_connection, arch=arch)
 
 
 def test_openstack_state(

app/tests/unit/test_cloud_image.py

@@ -179,6 +179,7 @@ def test__get_supported_runner_arch_unsupported_error():
         pytest.param(Arch.X64, "amd64", id="AMD64"),
         pytest.param(Arch.S390X, "s390x", id="S390X"),
         pytest.param(Arch.PPC64LE, "ppc64el", id="PPC64LE"),
+        pytest.param(Arch.ARM, "arm64", id="ARM"),
     ],
 )
 def test__get_supported_runner_arch(arch: Arch, expected: SupportedBaseImageArch):