From a5cb5daf376fdf85edf4ae4ae8df42037bc9c11d Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 2 Jul 2026 13:49:52 +0700 Subject: [PATCH 01/27] feat: add armhf (32-bit ARM) architecture support Add "armhf"/"arm" as a buildable image architecture. armhf images are built on arm64 (aarch64) hosts running a 32-bit userspace and use the GitHub runner binary published by the canonical/github-actions-runner fork (actions-runner-linux-arm-.tar.gz). Changes: - charm state.py: add ARM to the Arch enum and map the "armhf"/"arm"/"armv7l" config values to it in Arch.from_charm. - app config.py: add ARM to the Arch enum (drives the runner download arch "arm") with to_openstack() -> "armhf"; add ARM_ADDITIONAL_APT_PACKAGES (libicu74, libatomic1) required at runtime by the self-contained .NET runtime bundled in the linux-arm runner tarball. - app cloud_image.py: map Arch.ARM to the "armhf" Ubuntu cloud-image arch. - app openstack_builder.py: install the armhf runtime apt packages when building for Arch.ARM. - charmcraft.yaml: document the "armhf" architecture option. Unit tests added for every new arch mapping (TDD). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/cloud_image.py | 4 +++- app/src/github_runner_image_builder/config.py | 7 +++++++ app/src/github_runner_image_builder/openstack_builder.py | 3 +++ app/tests/unit/test_cloud_image.py | 1 + app/tests/unit/test_config.py | 1 + app/tests/unit/test_openstack_builder.py | 5 +++++ charmcraft.yaml | 6 ++++-- src/state.py | 5 +++++ tests/unit/test_state.py | 2 ++ 9 files changed, 31 insertions(+), 3 deletions(-) diff --git a/app/src/github_runner_image_builder/cloud_image.py b/app/src/github_runner_image_builder/cloud_image.py index b16725c5..a35a57f7 100644 --- a/app/src/github_runner_image_builder/cloud_image.py +++ b/app/src/github_runner_image_builder/cloud_image.py @@ -20,7 +20,7 @@ logger = logging.getLogger(__name__) -SupportedBaseImageArch = typing.Literal["amd64", "arm64", "s390x", "ppc64el"] +SupportedBaseImageArch = typing.Literal["amd64", "arm64", "s390x", "ppc64el", "armhf"] CHECKSUM_BUF_SIZE = 65536 # 65kb @@ -95,6 +95,8 @@ def _get_supported_runner_arch(arch: Arch) -> SupportedBaseImageArch: return "s390x" case Arch.PPC64LE: return "ppc64el" # cloud-images.ubuntu.com uses ppc64el instead of ppc64le + case Arch.ARM: + return "armhf" # cloud-images.ubuntu.com uses armhf for 32-bit arm case _: raise UnsupportedArchitectureError(f"Detected system arch: {arch} is unsupported.") diff --git a/app/src/github_runner_image_builder/config.py b/app/src/github_runner_image_builder/config.py index 579fe295..7a87a41e 100644 --- a/app/src/github_runner_image_builder/config.py +++ b/app/src/github_runner_image_builder/config.py @@ -20,12 +20,14 @@ class Arch(str, Enum): X64: Represents an X64/AMD64 system architecture. S390X: Represents an S390X system architecture. PPC64LE: Represents a PPC64LE system architecture. + ARM: Represents an ARM (32-bit armhf) system architecture. """ ARM64 = "arm64" X64 = "x64" S390X = "s390x" PPC64LE = "ppc64le" + ARM = "arm" def to_openstack(self) -> str: """Convert the architecture to OpenStack compatible arch string. @@ -42,6 +44,8 @@ def to_openstack(self) -> str: return "s390x" case Arch.PPC64LE: return "ppc64le" + case Arch.ARM: + return "armhf" raise ValueError # pragma: nocover @@ -127,6 +131,9 @@ def from_str(cls, tag_or_name: str) -> "BaseImage": "wget", ] S390X_PPC64LE_ADDITIONAL_APT_PACKAGES = ["dotnet-runtime-8.0"] +# The linux-arm runner tarball is self-contained but its bundled .NET runtime needs libicu and +# libatomic present on the system at runtime. +ARM_ADDITIONAL_APT_PACKAGES = ["libicu74", "libatomic1"] _LOG_LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR) LOG_LEVELS = tuple( diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 093abe32..8ab834c9 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -40,6 +40,7 @@ import github_runner_image_builder.errors from github_runner_image_builder import cloud_image, config, store from github_runner_image_builder.config import ( + ARM_ADDITIONAL_APT_PACKAGES, FORK_RUNNER_BINARY_REPO, IMAGE_DEFAULT_APT_PACKAGES, S390X_PPC64LE_ADDITIONAL_APT_PACKAGES, @@ -552,6 +553,8 @@ def _generate_cloud_init_script( apt_packages = IMAGE_DEFAULT_APT_PACKAGES if image_config.arch in (Arch.S390X, Arch.PPC64LE): apt_packages = IMAGE_DEFAULT_APT_PACKAGES + S390X_PPC64LE_ADDITIONAL_APT_PACKAGES + elif image_config.arch == Arch.ARM: + apt_packages = IMAGE_DEFAULT_APT_PACKAGES + ARM_ADDITIONAL_APT_PACKAGES return template.render( PROXY=proxy, APT_PACKAGES=" ".join(apt_packages), diff --git a/app/tests/unit/test_cloud_image.py b/app/tests/unit/test_cloud_image.py index 794d546e..eefb821f 100644 --- a/app/tests/unit/test_cloud_image.py +++ b/app/tests/unit/test_cloud_image.py @@ -179,6 +179,7 @@ def test__get_supported_runner_arch_unsupported_error(): pytest.param(Arch.X64, "amd64", id="AMD64"), pytest.param(Arch.S390X, "s390x", id="S390X"), pytest.param(Arch.PPC64LE, "ppc64el", id="PPC64LE"), + pytest.param(Arch.ARM, "armhf", id="ARM"), ], ) def test__get_supported_runner_arch(arch: Arch, expected: SupportedBaseImageArch): diff --git a/app/tests/unit/test_config.py b/app/tests/unit/test_config.py index dac4243d..9b2f206c 100644 --- a/app/tests/unit/test_config.py +++ b/app/tests/unit/test_config.py @@ -22,6 +22,7 @@ pytest.param(Arch.X64, "x86_64", id="amd64"), pytest.param(Arch.S390X, "s390x", id="s390x"), pytest.param(Arch.PPC64LE, "ppc64le", id="ppc64le"), + pytest.param(Arch.ARM, "armhf", id="arm"), ], ) def test_arch_openstack_conversion(arch: Arch, expected: str): diff --git a/app/tests/unit/test_openstack_builder.py b/app/tests/unit/test_openstack_builder.py index 90beda21..dd33dbc9 100644 --- a/app/tests/unit/test_openstack_builder.py +++ b/app/tests/unit/test_openstack_builder.py @@ -676,6 +676,11 @@ def test__determine_network(network_name: str | None): ["dotnet-runtime-8.0"], id="ppc64le", ), + pytest.param( + openstack_builder.Arch.ARM, + ["libicu74", "libatomic1"], + id="arm", + ), ], ) def test__generate_cloud_init_script( diff --git a/charmcraft.yaml b/charmcraft.yaml index 3b14d47e..93810a49 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -73,8 +73,10 @@ config: type: string description: | The image architecture to build for using external builder. Can be one of "amd64", "arm64", "s390x", - "ppc64le" (alternatively "ppc64el"). - Support for "s390x" and "ppc64el" is considered experimental. + "ppc64le" (alternatively "ppc64el") or "armhf" (alternatively "arm", 32-bit ARM). + Support for "s390x", "ppc64el" and "armhf" is considered experimental. + Note: "armhf" images are built on arm64 (aarch64) hardware running a 32-bit userspace, and use + the GitHub runner binary published by the canonical/github-actions-runner fork. base-image: type: string default: noble diff --git a/src/state.py b/src/state.py index 3de0a606..8f8db9db 100644 --- a/src/state.py +++ b/src/state.py @@ -17,6 +17,7 @@ logger = logging.getLogger(__name__) ARCHITECTURES_ARM64 = {"aarch64", "arm64"} +ARCHITECTURES_ARM = {"armhf", "arm", "armv7l"} ARCHITECTURES_S390X = {"s390x"} ARCHITECTURES_PPC64LE = {"ppc64le", "ppc64el"} ARCHITECTURES_X86 = {"x86_64", "amd64", "x64"} @@ -80,6 +81,7 @@ class Arch(str, Enum): X64: Represents an X64/AMD64 system architecture. S390X: Represents an S390X system architecture. PPC64LE: Represents an PPC64LE system architecture. + ARM: Represents an ARM (32-bit armhf) system architecture. """ def __str__(self) -> str: @@ -94,6 +96,7 @@ def __str__(self) -> str: X64 = "x64" S390X = "s390x" PPC64LE = "ppc64le" + ARM = "arm" @classmethod def from_charm(cls, charm: ops.CharmBase) -> "Arch": @@ -114,6 +117,8 @@ def from_charm(cls, charm: ops.CharmBase) -> "Arch": match architecture: case arch if arch in ARCHITECTURES_ARM64: return Arch.ARM64 + case arch if arch in ARCHITECTURES_ARM: + return Arch.ARM case arch if arch in ARCHITECTURES_X86: return Arch.X64 case arch if arch in ARCHITECTURES_S390X: diff --git a/tests/unit/test_state.py b/tests/unit/test_state.py index 3c635178..4bf268db 100644 --- a/tests/unit/test_state.py +++ b/tests/unit/test_state.py @@ -41,6 +41,8 @@ def patch_juju_version_29_fixture(monkeypatch: pytest.MonkeyPatch): pytest.param("amd64", state.Arch.X64, id="amd64"), pytest.param("s390x", state.Arch.S390X, id="s390x"), pytest.param("ppc64le", state.Arch.PPC64LE, id="ppc64le"), + pytest.param("armhf", state.Arch.ARM, id="armhf"), + pytest.param("arm", state.Arch.ARM, id="arm"), ], ) def test_arch_from_charm(arch: str, expected: state.Arch): From 30daea5f218bc288867abb5641f4754e8afb0716 Mon Sep 17 00:00:00 2001 From: florentianayuwono <76247368+florentianayuwono@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:21:40 +0700 Subject: [PATCH 02/27] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- charmcraft.yaml | 2 +- tests/unit/test_state.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/charmcraft.yaml b/charmcraft.yaml index 93810a49..26a2fd44 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -72,7 +72,7 @@ config: architecture: type: string description: | - The image architecture to build for using external builder. Can be one of "amd64", "arm64", "s390x", + The image architecture to build for using external builder. Can be one of "amd64", "arm64", "s390x", "ppc64le" (alternatively "ppc64el") or "armhf" (alternatively "arm", 32-bit ARM). Support for "s390x", "ppc64el" and "armhf" is considered experimental. Note: "armhf" images are built on arm64 (aarch64) hardware running a 32-bit userspace, and use diff --git a/tests/unit/test_state.py b/tests/unit/test_state.py index 4bf268db..2a245da7 100644 --- a/tests/unit/test_state.py +++ b/tests/unit/test_state.py @@ -43,6 +43,7 @@ def patch_juju_version_29_fixture(monkeypatch: pytest.MonkeyPatch): pytest.param("ppc64le", state.Arch.PPC64LE, id="ppc64le"), pytest.param("armhf", state.Arch.ARM, id="armhf"), pytest.param("arm", state.Arch.ARM, id="arm"), + pytest.param("armv7l", state.Arch.ARM, id="armv7l"), ], ) def test_arch_from_charm(arch: str, expected: state.Arch): From 0c47f67479ba44937951ab4f6c6e0a0f1e2f3667 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 2 Jul 2026 16:48:59 +0700 Subject: [PATCH 03/27] feat: pre-install rustup and docker-buildx on armhf images Add rustup and docker-buildx to the armhf-only additional apt packages and set the default Rust toolchain for the ubuntu user during image build. - config.py: ARM_ADDITIONAL_APT_PACKAGES gains rustup and docker-buildx (docker.io is already in the default package set). - cloud-init.sh.j2: after configuring system users, run `sudo -u ubuntu rustup default stable`, guarded on the arm arch so it is a no-op on other architectures (rustup is only installed on armhf images). Test updated: the armhf cloud-init assertion now expects the extra packages and the rustup toolchain step. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/config.py | 5 +++-- .../templates/cloud-init.sh.j2 | 6 ++++++ app/tests/unit/test_openstack_builder.py | 10 ++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/app/src/github_runner_image_builder/config.py b/app/src/github_runner_image_builder/config.py index 7a87a41e..2762432c 100644 --- a/app/src/github_runner_image_builder/config.py +++ b/app/src/github_runner_image_builder/config.py @@ -132,8 +132,9 @@ def from_str(cls, tag_or_name: str) -> "BaseImage": ] S390X_PPC64LE_ADDITIONAL_APT_PACKAGES = ["dotnet-runtime-8.0"] # The linux-arm runner tarball is self-contained but its bundled .NET runtime needs libicu and -# libatomic present on the system at runtime. -ARM_ADDITIONAL_APT_PACKAGES = ["libicu74", "libatomic1"] +# libatomic present on the system at runtime. rustup and docker-buildx are additionally +# pre-installed on armhf images for arm32 build workloads. +ARM_ADDITIONAL_APT_PACKAGES = ["libicu74", "libatomic1", "rustup", "docker-buildx"] _LOG_LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR) LOG_LEVELS = tuple( diff --git a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 index e5991eb0..c8d898cb 100644 --- a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 +++ b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 @@ -218,3 +218,9 @@ install_opentelemetry_collector_snap install_github_runner "$github_runner_version" "$github_runner_arch" chown_home configure_system_users + +# Set the default Rust toolchain for the ubuntu user. rustup is only installed on +# armhf images, so this block is a no-op on other architectures. +if [ "$github_runner_arch" == "arm" ]; then + sudo -u ubuntu rustup default stable +fi diff --git a/app/tests/unit/test_openstack_builder.py b/app/tests/unit/test_openstack_builder.py index dd33dbc9..ac7c1437 100644 --- a/app/tests/unit/test_openstack_builder.py +++ b/app/tests/unit/test_openstack_builder.py @@ -678,7 +678,7 @@ def test__determine_network(network_name: str | None): ), pytest.param( openstack_builder.Arch.ARM, - ["libicu74", "libatomic1"], + ["libicu74", "libatomic1", "rustup", "docker-buildx"], id="arm", ), ], @@ -934,7 +934,13 @@ def test__generate_cloud_init_script( install_opentelemetry_collector_snap install_github_runner "$github_runner_version" "$github_runner_arch" chown_home -configure_system_users\ +configure_system_users + +# Set the default Rust toolchain for the ubuntu user. rustup is only installed on +# armhf images, so this block is a no-op on other architectures. +if [ "$github_runner_arch" == "arm" ]; then + sudo -u ubuntu rustup default stable +fi\ """ # nosec # noqa: E501 ) # pylint: enable=R0801 From 857cfb1073cad2027f3b0c03e9896ccbf1d19ea4 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 2 Jul 2026 17:33:05 +0700 Subject: [PATCH 04/27] feat: restrict armhf builds to noble+ base images The armhf additional apt packages (libicu74, rustup, docker-buildx) are only available from the noble (24.04) archive onwards. On focal/jammy these package names do not exist, so an apt-get install would fail midway through the image build with a confusing error. Fail fast in _generate_cloud_init_script: when arch == ARM and the requested base image is older than noble, raise UnsupportedArchitectureError with a clear message instead of producing a cloud-init script that is guaranteed to fail. Tests: the cloud-init render test now exercises armhf on a supported (noble) base, and a new test asserts UnsupportedArchitectureError is raised for armhf on focal/jammy. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openstack_builder.py | 13 ++++++ app/tests/unit/test_openstack_builder.py | 40 ++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 8ab834c9..54269449 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -61,6 +61,10 @@ SHARED_SECURITY_GROUP_NAME = "github-runner-image-builder-v1" EXTERNAL_SCRIPT_PATH = pathlib.Path("/root/external.sh") +# armhf additional apt packages (libicu74, rustup, docker-buildx) are only available from the +# noble (24.04) archive onwards, so armhf images can only be built on these base images. +ARM_SUPPORTED_BASE_IMAGES = (BaseImage.NOBLE, BaseImage.RESOLUTE) + # Server operation timeout constants (in seconds) CREATE_SERVER_TIMEOUT = 20 * 60 # 20 minutes DELETE_SERVER_TIMEOUT = 20 * 60 # 20 minutes @@ -554,6 +558,15 @@ def _generate_cloud_init_script( if image_config.arch in (Arch.S390X, Arch.PPC64LE): apt_packages = IMAGE_DEFAULT_APT_PACKAGES + S390X_PPC64LE_ADDITIONAL_APT_PACKAGES elif image_config.arch == Arch.ARM: + # The armhf additional apt packages (libicu74, rustup, docker-buildx) are only available + # from the noble (24.04) archive onwards, so fail fast on older bases rather than letting + # apt-get fail midway through the image build. + if image_config.base not in ARM_SUPPORTED_BASE_IMAGES: + raise github_runner_image_builder.errors.UnsupportedArchitectureError( + f"armhf images require a base image of " + f"{[base.value for base in ARM_SUPPORTED_BASE_IMAGES]} or newer, " + f"got: {image_config.base.value}." + ) apt_packages = IMAGE_DEFAULT_APT_PACKAGES + ARM_ADDITIONAL_APT_PACKAGES return template.render( PROXY=proxy, diff --git a/app/tests/unit/test_openstack_builder.py b/app/tests/unit/test_openstack_builder.py index ac7c1437..45c1aee5 100644 --- a/app/tests/unit/test_openstack_builder.py +++ b/app/tests/unit/test_openstack_builder.py @@ -692,11 +692,17 @@ def test__generate_cloud_init_script( act: when _generate_cloud_init_script is run. assert: expected cloud init template is generated. """ + base = ( + openstack_builder.BaseImage.NOBLE + if arch == openstack_builder.Arch.ARM + else openstack_builder.BaseImage.JAMMY + ) + expected_hwe = openstack_builder.BaseImage.get_version(base) assert ( openstack_builder._generate_cloud_init_script( image_config=openstack_builder.config.ImageConfig( arch=arch, - base=openstack_builder.BaseImage.JAMMY, + base=base, runner_version="", name="test-image", script_config=openstack_builder.config.ScriptConfig( @@ -916,7 +922,7 @@ def test__generate_cloud_init_script( proxy="test.proxy.internal:3128" apt_packages="build-essential cargo docker.io gh jq npm pkg-config python-is-python3 python3-dev python3-pip rustc shellcheck socat tar time unzip wget{(' ' + ' '.join(additional_apt_packages)) if additional_apt_packages else ''}" -hwe_version="22.04" +hwe_version="{expected_hwe}" github_runner_version="" github_runner_arch="{arch.value}" runner_binary_repo="canonical/github-actions-runner" @@ -946,6 +952,36 @@ def test__generate_cloud_init_script( # pylint: enable=R0801 +@pytest.mark.parametrize( + "base", + [ + pytest.param(openstack_builder.BaseImage.FOCAL, id="focal"), + pytest.param(openstack_builder.BaseImage.JAMMY, id="jammy"), + ], +) +def test__generate_cloud_init_script_arm_unsupported_base(base: openstack_builder.BaseImage): + """ + arrange: ARM architecture paired with a pre-noble base image. + act: when _generate_cloud_init_script is run. + assert: UnsupportedArchitectureError is raised because the armhf apt packages \ + (e.g. libicu74, rustup, docker-buildx) are only available from noble onwards. + """ + with pytest.raises(errors.UnsupportedArchitectureError): + openstack_builder._generate_cloud_init_script( + image_config=openstack_builder.config.ImageConfig( + arch=openstack_builder.Arch.ARM, + base=base, + runner_version="", + name="test-image", + script_config=openstack_builder.config.ScriptConfig( + script_url=None, + script_secrets={}, + ), + ), + proxy="test.proxy.internal:3128", + ) + + def test__wait_for_cloud_init_complete_fail(): """ arrange: given a monkeypatched _get_ssh_connection and connection.run functions that raises an\ From 63a87cc4f42b2a95a6874a49c216a11c4083a182 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Mon, 6 Jul 2026 16:50:30 +0700 Subject: [PATCH 05/27] docs: document UnsupportedArchitectureError in _generate_cloud_init_script Add the missing Raises: section for the fail-fast guard that rejects armhf images on pre-noble base images, addressing review feedback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/openstack_builder.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 54269449..3ba03552 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -545,6 +545,10 @@ def _generate_cloud_init_script( image_config: The target image configuration values. proxy: The proxy to enable while setting up the VM. + Raises: + UnsupportedArchitectureError: If an armhf image is requested on a base image older than + the supported ARM base images (noble and newer). + Returns: The cloud-init script to create snapshot image. """ From c1cda23e155f907ffa3977c82485948fcb128e76 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Mon, 6 Jul 2026 19:19:47 +0700 Subject: [PATCH 06/27] refactor(tests): move TESTDATA_TEST_SCRIPT_URL to commands.py Breaks the commands.py <-> helpers.py circular import so command constants can be imported without the heavy OpenStack/fabric deps, enabling unit tests for arch-conditional command selection. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/tests/integration/commands.py | 7 ++++++- app/tests/integration/helpers.py | 6 ------ app/tests/integration/test_openstack_builder.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 9b327140..17e349f6 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -5,7 +5,12 @@ import dataclasses -from tests.integration.helpers import TESTDATA_TEST_SCRIPT_URL +from github_runner_image_builder.config import Arch + +TESTDATA_TEST_SCRIPT_URL = ( + "https://raw.githubusercontent.com/canonical/github-runner-image-builder-operator/" + "be135aa505b37aae29aec0ab13805909c46b7903/app/tests/integration/testdata/test_script.sh" +) @dataclasses.dataclass diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 4438fadc..8b9d3ecf 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -37,12 +37,6 @@ logger = logging.getLogger(__name__) -TESTDATA_TEST_SCRIPT_URL = ( - "https://raw.githubusercontent.com/canonical/github-runner-image-builder-operator/" - "be135aa505b37aae29aec0ab13805909c46b7903/app/tests/integration/testdata/test_script.sh" -) - - P = ParamSpec("P") R = TypeVar("R") S = Callable[P, R] | Callable[P, Awaitable[R]] diff --git a/app/tests/integration/test_openstack_builder.py b/app/tests/integration/test_openstack_builder.py index 29f2bc40..f3e2a95d 100644 --- a/app/tests/integration/test_openstack_builder.py +++ b/app/tests/integration/test_openstack_builder.py @@ -16,7 +16,7 @@ import pytest import pytest_asyncio from fabric.connection import Connection as SSHConnection -from integration.helpers import TESTDATA_TEST_SCRIPT_URL +from integration.commands import TESTDATA_TEST_SCRIPT_URL from openstack.compute.v2.image import Image from openstack.compute.v2.server import Server from openstack.connection import Connection From 05bca9ee5bcc3e32d81aa090dea721a7c5f6dfc4 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Mon, 6 Jul 2026 19:20:34 +0700 Subject: [PATCH 07/27] test: add arch-conditional armhf runner assertions Add ARM_RUNNER_COMMANDS (rustc/cargo, docker buildx, 32-bit Runner.Listener) and a pure commands_for_arch() selector, with unit tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/tests/integration/commands.py | 31 +++++++++++++++++++ app/tests/unit/test_integration_commands.py | 34 +++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 app/tests/unit/test_integration_commands.py diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 17e349f6..7b02109c 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -130,3 +130,34 @@ class Commands: "! grep '/home/ubuntu/secret.txt' /var/log/auth.log*", ), ) + + +# armhf-specific assertions. These packages/binaries are only installed on armhf images +# (see ARM_ADDITIONAL_APT_PACKAGES and the rustup cloud-init step), so they must not run on +# other architectures. +ARM_RUNNER_COMMANDS = ( + Commands(name="rustc version (rustup default stable)", command="rustc --version"), + Commands(name="cargo version", command="cargo --version"), + Commands(name="docker buildx version", command="docker buildx version"), + Commands( + name="github runner binary is 32-bit ARM", + command=( + "file /home/ubuntu/actions-runner/bin/Runner.Listener | grep -i 'ELF 32-bit' | " + "grep -i 'ARM'" + ), + ), +) + + +def commands_for_arch(arch: Arch) -> tuple[Commands, ...]: + """Return the test commands to run for the given architecture. + + Args: + arch: The architecture under test. + + Returns: + The base test commands, plus armhf-specific commands when arch is ARM. + """ + if arch == Arch.ARM: + return TEST_RUNNER_COMMANDS + ARM_RUNNER_COMMANDS + return TEST_RUNNER_COMMANDS diff --git a/app/tests/unit/test_integration_commands.py b/app/tests/unit/test_integration_commands.py new file mode 100644 index 00000000..328f7464 --- /dev/null +++ b/app/tests/unit/test_integration_commands.py @@ -0,0 +1,34 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Unit tests for arch-conditional integration test command selection.""" + +from github_runner_image_builder.config import Arch +from tests.integration import commands + + +def test_commands_for_arch_includes_arm_commands_for_arm(): + """ + arrange: given the ARM architecture. + act: when commands_for_arch is called. + assert: the returned commands include every ARM_RUNNER_COMMANDS entry and all base commands. + """ + result = commands.commands_for_arch(Arch.ARM) + + for arm_command in commands.ARM_RUNNER_COMMANDS: + assert arm_command in result + for base_command in commands.TEST_RUNNER_COMMANDS: + assert base_command in result + + +def test_commands_for_arch_excludes_arm_commands_for_non_arm(): + """ + arrange: given a non-ARM architecture. + act: when commands_for_arch is called. + assert: the returned commands are exactly the base commands (no ARM_RUNNER_COMMANDS). + """ + result = commands.commands_for_arch(Arch.X64) + + assert tuple(result) == tuple(commands.TEST_RUNNER_COMMANDS) + for arm_command in commands.ARM_RUNNER_COMMANDS: + assert arm_command not in result From 9887b4d32896ee0aad19e613108c1d33639bbac7 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Mon, 6 Jul 2026 19:21:34 +0700 Subject: [PATCH 08/27] test: run arch-specific commands in openstack integration test Thread arch through run_openstack_tests so armhf images additionally assert rustup, docker-buildx, and the 32-bit runner binary. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/tests/integration/helpers.py | 6 ++++-- app/tests/integration/test_openstack_builder.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 8b9d3ecf..4755c94f 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -32,6 +32,7 @@ from pylxd.models.instance import Instance, InstanceState from requests_toolbelt import MultipartEncoder +from github_runner_image_builder.config import Arch from tests.integration import commands, types logger = logging.getLogger(__name__) @@ -404,13 +405,14 @@ def setup_aproxy(ssh_connection: SSHConnection, proxy: str) -> None: assert False, "Aproxy did not start up correctly." -def run_openstack_tests(ssh_connection: SSHConnection): +def run_openstack_tests(ssh_connection: SSHConnection, arch: Arch): """Run test commands on the openstack instance via ssh. Args: ssh_connection: The SSH connection instance to OpenStack test server. + arch: The architecture under test, selecting arch-specific commands. """ - for testcmd in commands.TEST_RUNNER_COMMANDS: + for testcmd in commands.commands_for_arch(arch): logger.info("Running command: %s", testcmd.command) result: Result = ssh_connection.run(testcmd.command, env=testcmd.env) logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) diff --git a/app/tests/integration/test_openstack_builder.py b/app/tests/integration/test_openstack_builder.py index f3e2a95d..84c93a81 100644 --- a/app/tests/integration/test_openstack_builder.py +++ b/app/tests/integration/test_openstack_builder.py @@ -219,6 +219,7 @@ async def ssh_connection_fixture( def test_run( ssh_connection: SSHConnection, proxy: types.ProxyConfig, + arch: config.Arch, ): """ arrange: given openstack cloud instance. @@ -227,7 +228,7 @@ def test_run( """ if proxy.http is not None: helpers.setup_aproxy(ssh_connection, proxy.http) - helpers.run_openstack_tests(ssh_connection=ssh_connection) + helpers.run_openstack_tests(ssh_connection=ssh_connection, arch=arch) def test_openstack_state( From 6f1dce32f3920fe63c094dc6f05784f66ebd051f Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Mon, 6 Jul 2026 19:21:54 +0700 Subject: [PATCH 09/27] test: map --arch armhf to Arch.ARM in integration conftest Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/tests/integration/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/tests/integration/conftest.py b/app/tests/integration/conftest.py index 36ffebcc..d21075b3 100644 --- a/app/tests/integration/conftest.py +++ b/app/tests/integration/conftest.py @@ -33,6 +33,8 @@ def arch_fixture(pytestconfig: pytest.Config): match arch: case "arm64": return config.Arch.ARM64 + case "armhf": + return config.Arch.ARM case "amd64": return config.Arch.X64 case "s390x": From e73ce4c6a1b62762c42e002a559565d2991ab08c Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Mon, 6 Jul 2026 19:22:29 +0700 Subject: [PATCH 10/27] ci: add armhf integration-test leg on noble+resolute Reuses the arm64 OpenStack tenant (creds/network/flavor) with --arch armhf. armhf is restricted to noble and resolute to match ARM_SUPPORTED_BASE_IMAGES. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/integration_test_app.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index 4182b5a9..5e2866ac 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -15,18 +15,22 @@ jobs: fail-fast: false matrix: image: [focal, jammy, noble, resolute] - arch: [amd64, arm64, s390x, ppc64le] + arch: [amd64, arm64, armhf, s390x, ppc64le] exclude: - image: focal arch: ppc64le - image: focal arch: s390x + - image: focal + arch: armhf - image: jammy arch: arm64 - image: jammy arch: ppc64le - image: jammy arch: s390x + - image: jammy + arch: armhf - image: resolute arch: arm64 - image: resolute @@ -59,6 +63,13 @@ jobs: run: | tox -e integration -- --arch arm64 --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS_APP_ARM64 }} working-directory: app + - name: Run integration tests (armhf) + if: matrix.arch == 'armhf' + env: + OPENSTACK_PASSWORD: ${{ secrets.OPENSTACK_PASSWORD_ARM64 }} + run: | + tox -e integration -- --arch armhf --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS_APP_ARM64 }} + working-directory: app - name: Run integration tests (s390x) if: matrix.arch == 's390x' env: From 66c11f3e2bf2d511305a169b10914f4b05324771 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Mon, 6 Jul 2026 19:47:09 +0700 Subject: [PATCH 11/27] test: allow --arch armhf in integration pytest option The pytest --arch option restricted choices to amd64/arm64/s390x/ppc64le, so pytest rejected 'armhf' before arch_fixture could map it. Add armhf to the allowed choices. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/conftest.py b/app/conftest.py index 35cb4663..5268b9f4 100644 --- a/app/conftest.py +++ b/app/conftest.py @@ -41,7 +41,7 @@ def pytest_addoption(parser: Parser): "--arch", action="store", help="The architecture to build for.", - choices=["amd64", "arm64", "s390x", "ppc64le"], + choices=["amd64", "arm64", "armhf", "s390x", "ppc64le"], ) parser.addoption( "--openstack-network-name", From dec30d23a97fb4b2e9c86bd024ece2288eeb3cbe Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Mon, 6 Jul 2026 20:13:02 +0700 Subject: [PATCH 12/27] fix: use armv7l as OpenStack architecture for armhf Nova validates the glance image 'architecture' property against its architecture enum, which uses 'armv7l' for 32-bit ARM; 'armhf' (the Ubuntu userland ABI name) is rejected with 'Architecture name armhf is not valid'. The cloud-image download filename still uses 'armhf' via cloud_image.py, which is unaffected. Surfaced by the new armhf integration test when booting an OpenStack server from the built image. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/config.py | 5 ++++- app/tests/unit/test_config.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/src/github_runner_image_builder/config.py b/app/src/github_runner_image_builder/config.py index 2762432c..b200dfb1 100644 --- a/app/src/github_runner_image_builder/config.py +++ b/app/src/github_runner_image_builder/config.py @@ -45,7 +45,10 @@ def to_openstack(self) -> str: case Arch.PPC64LE: return "ppc64le" case Arch.ARM: - return "armhf" + # Nova/libvirt use the canonical "armv7l" for 32-bit ARM as the image + # architecture property; "armhf" (the Ubuntu userland ABI name) is rejected by + # Nova. The cloud-image download filename still uses "armhf" (see cloud_image.py). + return "armv7l" raise ValueError # pragma: nocover diff --git a/app/tests/unit/test_config.py b/app/tests/unit/test_config.py index 9b2f206c..b9e26146 100644 --- a/app/tests/unit/test_config.py +++ b/app/tests/unit/test_config.py @@ -22,7 +22,7 @@ pytest.param(Arch.X64, "x86_64", id="amd64"), pytest.param(Arch.S390X, "s390x", id="s390x"), pytest.param(Arch.PPC64LE, "ppc64le", id="ppc64le"), - pytest.param(Arch.ARM, "armhf", id="arm"), + pytest.param(Arch.ARM, "armv7l", id="arm"), ], ) def test_arch_openstack_conversion(arch: Arch, expected: str): From 56387619abb4bc6d3f3a8e93a2aa802e8127307f Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Tue, 7 Jul 2026 16:14:53 +0700 Subject: [PATCH 13/27] fix(armhf): set virtio image properties so 32-bit ARM images boot The armhf base image is tagged architecture=armv7l and booted on aarch64 compute hosts, which use the QEMU "virt" machine type. That machine type has no IDE bus, so libvirt defaulted to an IDE controller for the root disk and the config-drive CD-ROM and rejected the domain with "IDE controllers are unsupported for this QEMU binary or machine type", leaving the builder VM in ERROR. Set hw_machine_type=virt and virtio/scsi disk/cdrom buses on ARM image uploads so the guest boots. amd64 keeps the default PC machine type. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/store.py | 17 ++++++- app/tests/unit/test_store.py | 48 ++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/app/src/github_runner_image_builder/store.py b/app/src/github_runner_image_builder/store.py index 648d6099..076b046a 100644 --- a/app/src/github_runner_image_builder/store.py +++ b/app/src/github_runner_image_builder/store.py @@ -18,6 +18,18 @@ logger = logging.getLogger(__name__) +# The 32-bit ARM (armhf) base images are booted on aarch64 compute hosts, which use the +# QEMU "virt" machine type. That machine type has no IDE bus, so without these properties +# libvirt defaults to an IDE controller for the root disk and the config-drive CD-ROM and +# rejects the domain with "IDE controllers are unsupported for this QEMU binary or machine +# type". Forcing virtio/scsi buses and the virt machine type lets the guest boot. +_ARM_IMAGE_PROPERTIES = { + "hw_machine_type": "virt", + "hw_disk_bus": "virtio", + "hw_cdrom_bus": "scsi", + "hw_scsi_model": "virtio-scsi", +} + # Timeout constants (in seconds) SNAPSHOT_CREATION_TIMEOUT = 60 * 30 # 30 minutes @@ -77,12 +89,15 @@ def upload_image( with openstack.connect(cloud=cloud_name) as connection: try: logger.info("Uploading image %s.", image_name) + image_properties = {"architecture": arch.to_openstack()} + if arch == Arch.ARM: + image_properties.update(_ARM_IMAGE_PROPERTIES) # ignore type since the library does not provide correct type hinting but the docstring # does define the return type. image: Image = connection.create_image( name=image_name, filename=str(image_path), - properties={"architecture": arch.to_openstack()}, + properties=image_properties, allow_duplicates=True, wait=True, ) # type: ignore diff --git a/app/tests/unit/test_store.py b/app/tests/unit/test_store.py index 66c4c6b3..9771c001 100644 --- a/app/tests/unit/test_store.py +++ b/app/tests/unit/test_store.py @@ -12,6 +12,7 @@ from openstack.connection import Connection from github_runner_image_builder import store +from github_runner_image_builder.config import Arch from github_runner_image_builder.store import Image, OpenstackError, UploadImageError, openstack from tests.unit.factories import MockOpenstackImageFactory @@ -236,6 +237,53 @@ def test_upload_image(mock_connection: MagicMock): ) +def test_upload_image_arm_uses_virtio_properties(mock_connection: MagicMock): + """ + arrange: given a mocked openstack create_image function. + act: when upload_image is called with the 32-bit ARM architecture. + assert: create_image is called with virtio/scsi hw properties so that the + aarch64 "virt" machine type (which has no IDE bus) can boot the image. + """ + mock_connection.create_image.return_value = MockOpenstackImageFactory(id="1") + + store.upload_image( + arch=Arch.ARM, + cloud_name=MagicMock(), + image_name=MagicMock(), + image_path=MagicMock(), + keep_revisions=MagicMock(), + ) + + properties = mock_connection.create_image.call_args.kwargs["properties"] + assert properties["architecture"] == "armv7l" + assert properties["hw_machine_type"] == "virt" + assert properties["hw_disk_bus"] == "virtio" + assert properties["hw_cdrom_bus"] == "scsi" + + +def test_upload_image_amd64_omits_virtio_properties(mock_connection: MagicMock): + """ + arrange: given a mocked openstack create_image function. + act: when upload_image is called with the amd64 architecture. + assert: create_image is called without the ARM-only virtio/machine-type + properties (amd64 uses the default PC machine type with IDE support). + """ + mock_connection.create_image.return_value = MockOpenstackImageFactory(id="1") + + store.upload_image( + arch=Arch.X64, + cloud_name=MagicMock(), + image_name=MagicMock(), + image_path=MagicMock(), + keep_revisions=MagicMock(), + ) + + properties = mock_connection.create_image.call_args.kwargs["properties"] + assert "hw_machine_type" not in properties + assert "hw_disk_bus" not in properties + assert "hw_cdrom_bus" not in properties + + @pytest.mark.usefixtures("mock_connection") @pytest.mark.parametrize( "images, expected_id", From 1f0d638bc5a66a73cae67cb652889deddf4b2840 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 16:38:20 +0700 Subject: [PATCH 14/27] fix(armhf): put config drive on virtio-blk bus The armhf builder VMs booted on the aarch64 virt machine type but never became SSH-reachable. The root disk used virtio-blk (so the VM booted), but the config drive was attached as a virtio-scsi CD-ROM. The 32-bit armhf guest kernel lacks the virtio-scsi driver, so it could not read the config drive, cloud-init received no network metadata, and networking never came up. Attach the config drive on the same virtio-blk bus as the root disk and drop the now-unneeded virtio-scsi model. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/store.py | 8 +++++--- app/tests/unit/test_store.py | 5 ++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app/src/github_runner_image_builder/store.py b/app/src/github_runner_image_builder/store.py index 076b046a..843c2126 100644 --- a/app/src/github_runner_image_builder/store.py +++ b/app/src/github_runner_image_builder/store.py @@ -22,12 +22,14 @@ # QEMU "virt" machine type. That machine type has no IDE bus, so without these properties # libvirt defaults to an IDE controller for the root disk and the config-drive CD-ROM and # rejects the domain with "IDE controllers are unsupported for this QEMU binary or machine -# type". Forcing virtio/scsi buses and the virt machine type lets the guest boot. +# type". Both the root disk and the config drive are placed on the virtio-blk bus: the +# 32-bit armhf guest kernel has the virtio-blk driver, but not virtio-scsi, so a virtio-scsi +# config-drive CD-ROM would be unreadable and cloud-init would never receive its network +# metadata, leaving the VM unreachable over SSH. _ARM_IMAGE_PROPERTIES = { "hw_machine_type": "virt", "hw_disk_bus": "virtio", - "hw_cdrom_bus": "scsi", - "hw_scsi_model": "virtio-scsi", + "hw_cdrom_bus": "virtio", } # Timeout constants (in seconds) diff --git a/app/tests/unit/test_store.py b/app/tests/unit/test_store.py index 9771c001..4374b47f 100644 --- a/app/tests/unit/test_store.py +++ b/app/tests/unit/test_store.py @@ -258,7 +258,10 @@ def test_upload_image_arm_uses_virtio_properties(mock_connection: MagicMock): assert properties["architecture"] == "armv7l" assert properties["hw_machine_type"] == "virt" assert properties["hw_disk_bus"] == "virtio" - assert properties["hw_cdrom_bus"] == "scsi" + # The config drive must be reachable over virtio-blk (the same bus as the root disk). + # A virtio-scsi CD-ROM is unreadable by the 32-bit armhf guest kernel, which leaves + # cloud-init without network metadata so the VM never becomes SSH-reachable. + assert properties["hw_cdrom_bus"] == "virtio" def test_upload_image_amd64_omits_virtio_properties(mock_connection: MagicMock): From eaf00fe0a0cabac1dcbef58784f2567b593b873b Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 19:29:08 +0700 Subject: [PATCH 15/27] fix(armhf): revert config drive to virtio-scsi CD-ROM bus virtio-blk cannot back ejectable CD-ROM media (libvirt rejects the domain with "disk type of 'vdb' does not support ejectable media"), so the config-drive CD-ROM must ride a virtio-scsi controller. This restores the bus layout that at least boots the guest so its serial console can be inspected to root-cause the SSH-unreachable behaviour on the aarch64 hosts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/store.py | 10 +++++----- app/tests/unit/test_store.py | 7 +++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/app/src/github_runner_image_builder/store.py b/app/src/github_runner_image_builder/store.py index 843c2126..4296a509 100644 --- a/app/src/github_runner_image_builder/store.py +++ b/app/src/github_runner_image_builder/store.py @@ -22,14 +22,14 @@ # QEMU "virt" machine type. That machine type has no IDE bus, so without these properties # libvirt defaults to an IDE controller for the root disk and the config-drive CD-ROM and # rejects the domain with "IDE controllers are unsupported for this QEMU binary or machine -# type". Both the root disk and the config drive are placed on the virtio-blk bus: the -# 32-bit armhf guest kernel has the virtio-blk driver, but not virtio-scsi, so a virtio-scsi -# config-drive CD-ROM would be unreadable and cloud-init would never receive its network -# metadata, leaving the VM unreachable over SSH. +# type". The root disk uses virtio-blk and the config-drive CD-ROM uses a virtio-scsi +# controller (virtio-blk cannot back ejectable CD-ROM media). This matches the buses the +# 64-bit arm64 images boot with on the same hosts. _ARM_IMAGE_PROPERTIES = { "hw_machine_type": "virt", "hw_disk_bus": "virtio", - "hw_cdrom_bus": "virtio", + "hw_cdrom_bus": "scsi", + "hw_scsi_model": "virtio-scsi", } # Timeout constants (in seconds) diff --git a/app/tests/unit/test_store.py b/app/tests/unit/test_store.py index 4374b47f..ece936bf 100644 --- a/app/tests/unit/test_store.py +++ b/app/tests/unit/test_store.py @@ -258,10 +258,9 @@ def test_upload_image_arm_uses_virtio_properties(mock_connection: MagicMock): assert properties["architecture"] == "armv7l" assert properties["hw_machine_type"] == "virt" assert properties["hw_disk_bus"] == "virtio" - # The config drive must be reachable over virtio-blk (the same bus as the root disk). - # A virtio-scsi CD-ROM is unreadable by the 32-bit armhf guest kernel, which leaves - # cloud-init without network metadata so the VM never becomes SSH-reachable. - assert properties["hw_cdrom_bus"] == "virtio" + # virtio-blk cannot back ejectable CD-ROM media, so the config-drive CD-ROM must ride a + # virtio-scsi controller (the same bus the arm64 images boot with on these hosts). + assert properties["hw_cdrom_bus"] == "scsi" def test_upload_image_amd64_omits_virtio_properties(mock_connection: MagicMock): From 62dfb32b87be9fc0b1861b86444b1ffe7ae8d726 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 21:05:36 +0700 Subject: [PATCH 16/27] docs(armhf): add design for arm64 boot image + 32-bit runner payload Native armhf VMs (32-bit kernel) do not boot on the ps7 arm64 hosts (empty console). The proven-working model from runner PRs #155/#169/#171 is an arm64 kernel running 32-bit armhf userland via native AArch32. This spec redefines the image-builder armhf path to build an arm64 boot image carrying the 32-bit linux-arm runner agent plus armhf multiarch userland. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...7-09-armhf-arm64-multiarch-image-design.md | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md diff --git a/docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md b/docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md new file mode 100644 index 00000000..dac556fb --- /dev/null +++ b/docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md @@ -0,0 +1,161 @@ +# armhf Runner Image via arm64 Boot Image + 32-bit Runner Payload + +- **Date:** 2026-07-09 +- **Status:** Approved (design) +- **Related:** PR #233 (image-builder armhf support), runner PRs #155/#169/#171 +- **Spec:** ISD-5856 + +> Note: this design runs 32-bit armhf userland **natively** via the hosts' AArch32 +> support (AArch32@EL0) using Ubuntu multiarch — it does **not** use qemu emulation. + +## Problem + +The image-builder's `Arch.ARM` (armhf) path builds a **native armhf image** — it +downloads `-server-cloudimg-armhf.img` (a 32-bit ARM kernel) and boots it as a +full VM on the ProdStack 7 (ps7) arm64 compute hosts. That VM never boots: Nova marks +it `ACTIVE` and neutron assigns an IP, but the serial console is **completely empty** +and SSH times out forever. + +### Root cause + +Booting a full armhf VM requires a **32-bit kernel** to start under the aarch64 `virt` +machine's 64-bit UEFI firmware (i.e. AArch32 at EL1 and/or a 32-bit boot chain). The +ps7 hosts do **not** provide this, so the 32-bit kernel never executes → empty console. + +This is **not** a hardware limitation for armhf *workloads*. The same ps7 hosts do +support **AArch32 at EL0** (32-bit userland), proven by the merged runner work: + +- PR #155: the runner was "Built and run on an arm64 VM with AArch32 (native 32-bit) + userspace, inside an `arm32v7/ubuntu:noble` container … binaries run natively as + armhf (no qemu)." +- PR #169/#171: the shipped `linux-arm` (armv7l, 32-bit) package is smoke-tested by + running `./config.sh --version` inside an `arm32v7/ubuntu:noble` container on the + arm64 self-hosted pool, "natively via the host's AArch32 support (no qemu)." + +The proven-working model is therefore an **arm64 kernel (64-bit) running 32-bit armhf +userland**, which needs only AArch32@EL0 (available). The image-builder instead builds +a 32-bit-kernel image (needs AArch32@EL1 / 32-bit boot — unavailable). It builds the +wrong kind of image. + +## Goal + +Make the image-builder's "armhf runner" image reproduce the validated model: an +**arm64 boot image** that carries the **32-bit `linux-arm` runner agent** plus the +armhf multiarch userland it needs, running via the hosts' native AArch32. armhf jobs +build/test armhf binaries and run `linux/arm/v7` containers. + +Non-goals: full-system emulation (`qemu-system-arm`); a qemu-user portability fallback +(all target hosts are verified AArch32-capable, so native execution is used). + +## Design + +### Core reframing: boot architecture vs runner architecture + +For every architecture except armhf, the architecture the builder VM **boots** equals +the architecture of the **runner payload**. For `Arch.ARM` they diverge: + +| Aspect | Current (broken) | New | +|---|---|---| +| Base cloud image | `…-cloudimg-armhf.img` (32-bit kernel) | `…-cloudimg-arm64.img` (64-bit kernel) | +| Glance `architecture` property | `armv7l` | `aarch64` | +| Builder / runner flavor | `shared.xsmall.arm64` | unchanged | +| Boots on ps7 hosts | No (empty console) | Yes (normal arm64 boot) | +| Runner agent installed | `linux-arm` (32-bit) | `linux-arm` (32-bit) — unchanged | +| armhf userland | none | armhf multiarch libs | +| virtio glance props | required (still failed) | removed | + +`Arch.ARM` continues to mean "produce a 32-bit ARM runner"; it is now delivered as an +arm64 image carrying a 32-bit runner + armhf multiarch userland, executed via native +AArch32. + +### Component changes + +1. **`config.py`** + - `Arch.ARM.to_openstack()` returns **`aarch64`** (was `armv7l`) so the base image + and snapshot schedule/boot on arm64 hosts. Update the outdated `armv7l` comment. + - `Arch.ARM.value` stays `"arm"` (drives the 32-bit runner tarball download and the + `github_runner_arch` cloud-init variable). + - `ARM_ADDITIONAL_APT_PACKAGES` becomes the **armhf-qualified** runtime deps for the + 32-bit runner agent plus arm64 host tools: + `["libc6:armhf", "libicu74:armhf", "libatomic1:armhf", "rustup", "docker-buildx"]`. + (`libc6:armhf` provides `/lib/ld-linux-armhf.so.3`; `libicu74:armhf` and + `libatomic1:armhf` are the .NET runtime deps the armv7l runner needs — see #169.) + +2. **`cloud_image.py`** + - `_get_supported_runner_arch(Arch.ARM)` returns **`arm64`** (was `armhf`), so the + builder VM boots a 64-bit kernel. This is the direct fix for the empty-console + failure. Update the docstring/comment accordingly. + +3. **`store.py`** + - Remove `_ARM_IMAGE_PROPERTIES` and the `if arch == Arch.ARM` block; revert + `upload_image` to `properties={"architecture": arch.to_openstack()}`. An arm64 + image boots with the cloud's normal defaults, so the IDE/virtio workaround is + obsolete. + +4. **`openstack_builder.py`** + - `_generate_cloud_init_script` keeps `RUNNER_ARCH="arm"` and the noble+ base-image + guard (`ARM_SUPPORTED_BASE_IMAGES`). The armhf multiarch enablement is expressed in + the cloud-init template (below); no change to the flavor logic (already arm64). + +5. **`templates/cloud-init.sh.j2`** + - Add an armhf-only step, guarded by `[ "$github_runner_arch" == "arm" ]`, run + **before** `install_apt_packages`: + - `dpkg --add-architecture armhf` + - Ensure the apt sources serve armhf. arm64 Ubuntu already uses + `ports.ubuntu.com`, which also hosts armhf; add `armhf` to the sources' + architectures (deb822 `Architectures:` on noble+, or an `[arch=armhf]` entry). + - `apt-get update` + - The armhf-qualified libs then install via the normal `install_apt_packages` path; + the existing `install_github_runner` (arch `arm`) downloads the 32-bit tarball; the + existing `rustup default stable` armhf block stays. + +### Data flow + +``` +download arm64 cloud image ──▶ upload base (architecture=aarch64) + │ + ▼ +create builder VM (shared.xsmall.arm64) ── boots 64-bit kernel ✅ + │ + ▼ +cloud-init: + enable armhf multiarch (dpkg --add-architecture armhf; apt update) + install libc6:armhf + armhf runtime deps + host tools + install linux-arm (32-bit) runner agent ── runs via native AArch32@EL0 + │ + ▼ +snapshot ──▶ upload runner image (architecture=aarch64) +``` + +### Error handling + +- Non-noble bases for armhf still fail fast with `UnsupportedArchitectureError` (the + armhf-qualified libs require noble/24.04+). +- If a host genuinely lacked AArch32, the 32-bit runner agent would fail to `exec`; + since all target hosts are verified AArch32-capable, no fallback is provided (matches + #169's stated assumption). + +## Testing + +### Unit +- `test_store.py`: revert/remove the two virtio-property tests; assert `upload_image` + passes only `properties={"architecture": …}`. +- `test_cloud_image.py`: `Arch.ARM` downloads the `arm64` base image filename. +- `test_config.py`: `Arch.ARM.to_openstack()` returns `aarch64`; + `ARM_ADDITIONAL_APT_PACKAGES` contains the armhf-qualified deps. +- `test_openstack_builder.py`: generated cloud-init for `Arch.ARM` enables armhf + multiarch and still sets `RUNNER_ARCH=arm`; non-noble armhf base raises. + +### Integration (ps7 arm64 tenant) +- The armhf leg now **boots** (previously the entire failure mode). +- Assert the produced image is arm64 and the baked 32-bit runner agent is armv7l and + executable — e.g. `file .../bin/Runner.Listener` reports "ARM, EABI5 32-bit" and/or + `./config.sh --version` runs (mirroring #169's container smoke test). + +## Rollback / supersedes + +This supersedes the native-armhf commits on branch `feat/armhf-arch-support-isd-5856` +(virtio glance properties in `store.py`, `armv7l` glance arch, `armhf` base-image +download). Those are reverted/replaced by the arm64-boot + multiarch model here. The +runner-side PRs (#155/#169/#171) are unchanged — this design consumes their 32-bit +`linux-arm` package exactly as they built it. From 9e0e4459fde3026f7ba331f4ed9d5fba09a0b6ab Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 21:19:55 +0700 Subject: [PATCH 17/27] fix(armhf): map Arch.ARM glance architecture to aarch64 The armhf runner image is an arm64 boot image carrying a 32-bit runner payload; native armhf (32-bit kernel) images do not boot on the aarch64 hosts. Schedule/boot the base image and snapshot as aarch64. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/config.py | 10 ++++++---- app/tests/unit/test_config.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/src/github_runner_image_builder/config.py b/app/src/github_runner_image_builder/config.py index b200dfb1..e2af3c6d 100644 --- a/app/src/github_runner_image_builder/config.py +++ b/app/src/github_runner_image_builder/config.py @@ -45,10 +45,12 @@ def to_openstack(self) -> str: case Arch.PPC64LE: return "ppc64le" case Arch.ARM: - # Nova/libvirt use the canonical "armv7l" for 32-bit ARM as the image - # architecture property; "armhf" (the Ubuntu userland ABI name) is rejected by - # Nova. The cloud-image download filename still uses "armhf" (see cloud_image.py). - return "armv7l" + # The armhf runner image is an arm64 boot image (64-bit kernel) that carries a + # 32-bit linux-arm runner payload. Native armhf images (32-bit kernel) do not boot + # on the aarch64 "virt" machine, so the glance architecture is aarch64 so the base + # image and snapshot schedule and boot on arm64 hosts. The 32-bit runner is + # installed via multiarch (see cloud-init.sh.j2 and ARM_ADDITIONAL_APT_PACKAGES). + return "aarch64" raise ValueError # pragma: nocover diff --git a/app/tests/unit/test_config.py b/app/tests/unit/test_config.py index b9e26146..916746e0 100644 --- a/app/tests/unit/test_config.py +++ b/app/tests/unit/test_config.py @@ -22,7 +22,7 @@ pytest.param(Arch.X64, "x86_64", id="amd64"), pytest.param(Arch.S390X, "s390x", id="s390x"), pytest.param(Arch.PPC64LE, "ppc64le", id="ppc64le"), - pytest.param(Arch.ARM, "armv7l", id="arm"), + pytest.param(Arch.ARM, "aarch64", id="arm"), ], ) def test_arch_openstack_conversion(arch: Arch, expected: str): From 7d19b977c56fdefa73366fb1ce23b5de188b1254 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 21:20:36 +0700 Subject: [PATCH 18/27] fix(armhf): download arm64 base cloud image for Arch.ARM The builder VM must boot a 64-bit kernel on the aarch64 hosts. Download the arm64 base cloud image; the 32-bit runner payload is added via multiarch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/cloud_image.py | 5 ++++- app/tests/unit/test_cloud_image.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/src/github_runner_image_builder/cloud_image.py b/app/src/github_runner_image_builder/cloud_image.py index a35a57f7..5ef4af03 100644 --- a/app/src/github_runner_image_builder/cloud_image.py +++ b/app/src/github_runner_image_builder/cloud_image.py @@ -96,7 +96,10 @@ def _get_supported_runner_arch(arch: Arch) -> SupportedBaseImageArch: case Arch.PPC64LE: return "ppc64el" # cloud-images.ubuntu.com uses ppc64el instead of ppc64le case Arch.ARM: - return "armhf" # cloud-images.ubuntu.com uses armhf for 32-bit arm + # Download the arm64 (64-bit) base cloud image, not armhf. The armhf runner image is + # an arm64 boot image (64-bit kernel) that runs the 32-bit linux-arm runner via + # multiarch; a native armhf (32-bit kernel) image does not boot on the aarch64 hosts. + return "arm64" case _: raise UnsupportedArchitectureError(f"Detected system arch: {arch} is unsupported.") diff --git a/app/tests/unit/test_cloud_image.py b/app/tests/unit/test_cloud_image.py index eefb821f..1c498089 100644 --- a/app/tests/unit/test_cloud_image.py +++ b/app/tests/unit/test_cloud_image.py @@ -179,7 +179,7 @@ def test__get_supported_runner_arch_unsupported_error(): pytest.param(Arch.X64, "amd64", id="AMD64"), pytest.param(Arch.S390X, "s390x", id="S390X"), pytest.param(Arch.PPC64LE, "ppc64el", id="PPC64LE"), - pytest.param(Arch.ARM, "armhf", id="ARM"), + pytest.param(Arch.ARM, "arm64", id="ARM"), ], ) def test__get_supported_runner_arch(arch: Arch, expected: SupportedBaseImageArch): From 8a608517145aa20e5701a0a6b164535928491e91 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 21:22:09 +0700 Subject: [PATCH 19/27] fix(armhf): drop obsolete virtio glance properties With the armhf runner image now built from an arm64 base (64-bit kernel), it boots with the cloud's normal defaults. The virtio-scsi/machine-type workaround for booting a native armhf (32-bit kernel) image is no longer needed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/store.py | 16 ------- app/tests/unit/test_store.py | 50 -------------------- 2 files changed, 66 deletions(-) diff --git a/app/src/github_runner_image_builder/store.py b/app/src/github_runner_image_builder/store.py index 4296a509..2674b54b 100644 --- a/app/src/github_runner_image_builder/store.py +++ b/app/src/github_runner_image_builder/store.py @@ -18,20 +18,6 @@ logger = logging.getLogger(__name__) -# The 32-bit ARM (armhf) base images are booted on aarch64 compute hosts, which use the -# QEMU "virt" machine type. That machine type has no IDE bus, so without these properties -# libvirt defaults to an IDE controller for the root disk and the config-drive CD-ROM and -# rejects the domain with "IDE controllers are unsupported for this QEMU binary or machine -# type". The root disk uses virtio-blk and the config-drive CD-ROM uses a virtio-scsi -# controller (virtio-blk cannot back ejectable CD-ROM media). This matches the buses the -# 64-bit arm64 images boot with on the same hosts. -_ARM_IMAGE_PROPERTIES = { - "hw_machine_type": "virt", - "hw_disk_bus": "virtio", - "hw_cdrom_bus": "scsi", - "hw_scsi_model": "virtio-scsi", -} - # Timeout constants (in seconds) SNAPSHOT_CREATION_TIMEOUT = 60 * 30 # 30 minutes @@ -92,8 +78,6 @@ def upload_image( try: logger.info("Uploading image %s.", image_name) image_properties = {"architecture": arch.to_openstack()} - if arch == Arch.ARM: - image_properties.update(_ARM_IMAGE_PROPERTIES) # ignore type since the library does not provide correct type hinting but the docstring # does define the return type. image: Image = connection.create_image( diff --git a/app/tests/unit/test_store.py b/app/tests/unit/test_store.py index ece936bf..66c4c6b3 100644 --- a/app/tests/unit/test_store.py +++ b/app/tests/unit/test_store.py @@ -12,7 +12,6 @@ from openstack.connection import Connection from github_runner_image_builder import store -from github_runner_image_builder.config import Arch from github_runner_image_builder.store import Image, OpenstackError, UploadImageError, openstack from tests.unit.factories import MockOpenstackImageFactory @@ -237,55 +236,6 @@ def test_upload_image(mock_connection: MagicMock): ) -def test_upload_image_arm_uses_virtio_properties(mock_connection: MagicMock): - """ - arrange: given a mocked openstack create_image function. - act: when upload_image is called with the 32-bit ARM architecture. - assert: create_image is called with virtio/scsi hw properties so that the - aarch64 "virt" machine type (which has no IDE bus) can boot the image. - """ - mock_connection.create_image.return_value = MockOpenstackImageFactory(id="1") - - store.upload_image( - arch=Arch.ARM, - cloud_name=MagicMock(), - image_name=MagicMock(), - image_path=MagicMock(), - keep_revisions=MagicMock(), - ) - - properties = mock_connection.create_image.call_args.kwargs["properties"] - assert properties["architecture"] == "armv7l" - assert properties["hw_machine_type"] == "virt" - assert properties["hw_disk_bus"] == "virtio" - # virtio-blk cannot back ejectable CD-ROM media, so the config-drive CD-ROM must ride a - # virtio-scsi controller (the same bus the arm64 images boot with on these hosts). - assert properties["hw_cdrom_bus"] == "scsi" - - -def test_upload_image_amd64_omits_virtio_properties(mock_connection: MagicMock): - """ - arrange: given a mocked openstack create_image function. - act: when upload_image is called with the amd64 architecture. - assert: create_image is called without the ARM-only virtio/machine-type - properties (amd64 uses the default PC machine type with IDE support). - """ - mock_connection.create_image.return_value = MockOpenstackImageFactory(id="1") - - store.upload_image( - arch=Arch.X64, - cloud_name=MagicMock(), - image_name=MagicMock(), - image_path=MagicMock(), - keep_revisions=MagicMock(), - ) - - properties = mock_connection.create_image.call_args.kwargs["properties"] - assert "hw_machine_type" not in properties - assert "hw_disk_bus" not in properties - assert "hw_cdrom_bus" not in properties - - @pytest.mark.usefixtures("mock_connection") @pytest.mark.parametrize( "images, expected_id", From 4f90d51258a0ef1ec8f4d1ee9ef44d14d45df0ef Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 21:22:49 +0700 Subject: [PATCH 20/27] fix(armhf): install armhf-qualified runtime deps for the 32-bit runner The 32-bit linux-arm runner agent needs the armhf loader (libc6:armhf) and armhf builds of its .NET runtime deps (libicu74:armhf, libatomic1:armhf) to exec on the arm64 image via native AArch32. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/config.py | 15 +++++++++++---- app/tests/unit/test_openstack_builder.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/app/src/github_runner_image_builder/config.py b/app/src/github_runner_image_builder/config.py index e2af3c6d..163581b9 100644 --- a/app/src/github_runner_image_builder/config.py +++ b/app/src/github_runner_image_builder/config.py @@ -136,10 +136,17 @@ def from_str(cls, tag_or_name: str) -> "BaseImage": "wget", ] S390X_PPC64LE_ADDITIONAL_APT_PACKAGES = ["dotnet-runtime-8.0"] -# The linux-arm runner tarball is self-contained but its bundled .NET runtime needs libicu and -# libatomic present on the system at runtime. rustup and docker-buildx are additionally -# pre-installed on armhf images for arm32 build workloads. -ARM_ADDITIONAL_APT_PACKAGES = ["libicu74", "libatomic1", "rustup", "docker-buildx"] +# The 32-bit linux-arm runner agent runs via the host's native AArch32 support on the arm64 +# image. It needs the armhf loader (ld-linux-armhf.so.3 from libc6:armhf) and the armhf builds of +# its .NET runtime dependencies (libicu, libatomic). rustup and docker-buildx are additionally +# pre-installed (arm64 host tools) for arm32 build workloads. +ARM_ADDITIONAL_APT_PACKAGES = [ + "libc6:armhf", + "libicu74:armhf", + "libatomic1:armhf", + "rustup", + "docker-buildx", +] _LOG_LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR) LOG_LEVELS = tuple( diff --git a/app/tests/unit/test_openstack_builder.py b/app/tests/unit/test_openstack_builder.py index 45c1aee5..f47058dc 100644 --- a/app/tests/unit/test_openstack_builder.py +++ b/app/tests/unit/test_openstack_builder.py @@ -678,7 +678,7 @@ def test__determine_network(network_name: str | None): ), pytest.param( openstack_builder.Arch.ARM, - ["libicu74", "libatomic1", "rustup", "docker-buildx"], + ["libc6:armhf", "libicu74:armhf", "libatomic1:armhf", "rustup", "docker-buildx"], id="arm", ), ], From b4204e05f893f18e175307ea55a6cc4eb351a4be Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 21:23:32 +0700 Subject: [PATCH 21/27] feat(armhf): enable armhf multiarch in cloud-init Add dpkg --add-architecture armhf (guarded to the arm runner) before apt install so the arm64 image can install and run the 32-bit linux-arm runner and its armhf runtime deps via native AArch32. armhf packages are served by ports.ubuntu.com, which the arm64 base already uses. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/cloud-init.sh.j2 | 10 ++++++++++ app/tests/unit/test_openstack_builder.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 index c8d898cb..2c1b25c1 100644 --- a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 +++ b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 @@ -53,6 +53,11 @@ EOF sleep 5 } +function enable_armhf_multiarch() { + echo "Enabling armhf multiarch" + /usr/bin/dpkg --add-architecture armhf +} + function install_apt_packages() { local packages="$1" local hwe_version="$2" @@ -205,6 +210,11 @@ github_runner_arch="{{ RUNNER_ARCH }}" runner_binary_repo="{{ RUNNER_BINARY_REPO }}" configure_proxy "$proxy" +# Enable armhf multiarch before installing packages so the 32-bit linux-arm runner agent and +# its armhf runtime dependencies can be installed and executed via native AArch32. +if [ "$github_runner_arch" == "arm" ]; then + enable_armhf_multiarch +fi install_apt_packages "$apt_packages" "$hwe_version" disable_unattended_upgrades enable_network_fair_queuing_congestion diff --git a/app/tests/unit/test_openstack_builder.py b/app/tests/unit/test_openstack_builder.py index f47058dc..167d9f2c 100644 --- a/app/tests/unit/test_openstack_builder.py +++ b/app/tests/unit/test_openstack_builder.py @@ -772,6 +772,11 @@ def test__generate_cloud_init_script( sleep 5 }} +function enable_armhf_multiarch() {{ + echo "Enabling armhf multiarch" + /usr/bin/dpkg --add-architecture armhf +}} + function install_apt_packages() {{ local packages="$1" local hwe_version="$2" @@ -928,6 +933,11 @@ def test__generate_cloud_init_script( runner_binary_repo="canonical/github-actions-runner" configure_proxy "$proxy" +# Enable armhf multiarch before installing packages so the 32-bit linux-arm runner agent and +# its armhf runtime dependencies can be installed and executed via native AArch32. +if [ "$github_runner_arch" == "arm" ]; then + enable_armhf_multiarch +fi install_apt_packages "$apt_packages" "$hwe_version" disable_unattended_upgrades enable_network_fair_queuing_congestion From 90e0f06d161c8ea3776e2e44d32c6992a6c40ad5 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 21:24:14 +0700 Subject: [PATCH 22/27] docs(armhf): add arm64 multiarch image implementation plan Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../2026-07-09-armhf-arm64-multiarch-image.md | 442 ++++++++++++++++++ 1 file changed, 442 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-09-armhf-arm64-multiarch-image.md diff --git a/docs/superpowers/plans/2026-07-09-armhf-arm64-multiarch-image.md b/docs/superpowers/plans/2026-07-09-armhf-arm64-multiarch-image.md new file mode 100644 index 00000000..6fcb08bf --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-armhf-arm64-multiarch-image.md @@ -0,0 +1,442 @@ +# armhf Runner Image (arm64 boot + 32-bit runner payload) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the image-builder's `armhf` path build an **arm64 boot image** that boots on the ps7 arm64 hosts and carries the **32-bit `linux-arm` runner agent** plus armhf multiarch userland, so it runs via the hosts' native AArch32. + +**Architecture:** Decouple *boot architecture* from *runner architecture* for `Arch.ARM`: boot/base image becomes arm64 (glance `architecture=aarch64`), while the runner payload stays 32-bit (`RUNNER_ARCH=arm`, armhf multiarch libs). Remove the obsolete virtio glance-property workaround. All changes are TDD with signed commits. + +**Tech Stack:** Python 3.12, pytest, Jinja2 cloud-init template, OpenStack SDK. Unit tests run **from the `app/` directory**: `~/.venv-armhf/bin/python -m pytest tests/unit`. Line length 99 (black / isort profile=black / flake8 --max-line-length=99). Signed commits: `git commit -S`. + +**Repo/branch:** `~/github-runner-image-builder-operator`, branch `feat/armhf-arch-support-isd-5856` (PR #233). + +**Spec:** `docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md` + +**Baseline:** 135 unit tests pass today (`cd app && ~/.venv-armhf/bin/python -m pytest tests/unit -q`). After this plan there are 133 (two obsolete virtio tests removed). + +--- + +## File Structure + +| File | Responsibility | Change | +|---|---|---| +| `app/src/github_runner_image_builder/config.py` | Arch enum, glance arch mapping, apt package lists | `Arch.ARM.to_openstack()` → `aarch64`; `ARM_ADDITIONAL_APT_PACKAGES` → armhf-qualified deps | +| `app/src/github_runner_image_builder/cloud_image.py` | Base cloud-image download | `_get_supported_runner_arch(Arch.ARM)` → `arm64` | +| `app/src/github_runner_image_builder/store.py` | Upload image to glance | Remove `_ARM_IMAGE_PROPERTIES` + ARM virtio block | +| `app/src/github_runner_image_builder/templates/cloud-init.sh.j2` | Runner image provisioning | Add `enable_armhf_multiarch` + guarded call | +| `app/tests/unit/test_config.py` | config unit tests | ARM → `aarch64` | +| `app/tests/unit/test_cloud_image.py` | cloud_image unit tests | ARM → `arm64` | +| `app/tests/unit/test_store.py` | store unit tests | Remove the 2 virtio tests | +| `app/tests/unit/test_openstack_builder.py` | cloud-init render tests | New arm packages + multiarch function in expected string | + +Notes: +- The integration test (`app/tests/integration/commands.py`) already asserts the runner binary is `ELF 32-bit ARM` and runs `rustc`/`cargo`/`docker buildx` for armhf. No integration-code change is required; the armhf leg simply needs to boot now. +- Tasks are ordered so each leaves the unit suite green. +- All commands below assume you start each task at the repo root `~/github-runner-image-builder-operator` unless the command itself `cd`s into `app`. + +--- + +## Task 1: Glance architecture for armhf → aarch64 + +**Files:** +- Modify: `app/src/github_runner_image_builder/config.py` (`Arch.to_openstack`) +- Test: `app/tests/unit/test_config.py` (`test_arch_openstack_conversion`, ARM param ~L25) + +- [ ] **Step 1: Update the failing test** + +In `app/tests/unit/test_config.py`, change the ARM param: + +```python + pytest.param(Arch.ARM, "aarch64", id="arm"), +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_config.py::test_arch_openstack_conversion -q` +Expected: FAIL — `assert 'armv7l' == 'aarch64'` for the `arm` param. + +- [ ] **Step 3: Update `to_openstack`** + +In `app/src/github_runner_image_builder/config.py`, replace the `Arch.ARM` case in `to_openstack`: + +```python + case Arch.ARM: + # The armhf runner image is an arm64 boot image (64-bit kernel) that carries a + # 32-bit linux-arm runner payload. Native armhf images (32-bit kernel) do not boot + # on the aarch64 "virt" machine, so the glance architecture is aarch64 so the base + # image and snapshot schedule and boot on arm64 hosts. The 32-bit runner is + # installed via multiarch (see cloud-init.sh.j2 and ARM_ADDITIONAL_APT_PACKAGES). + return "aarch64" +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_config.py::test_arch_openstack_conversion -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +cd ~/github-runner-image-builder-operator +git add app/src/github_runner_image_builder/config.py app/tests/unit/test_config.py +git commit -S -m "fix(armhf): map Arch.ARM glance architecture to aarch64 + +The armhf runner image is an arm64 boot image carrying a 32-bit runner +payload; native armhf (32-bit kernel) images do not boot on the aarch64 +hosts. Schedule/boot the base image and snapshot as aarch64. + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 2: Base cloud-image download for armhf → arm64 + +**Files:** +- Modify: `app/src/github_runner_image_builder/cloud_image.py` (`_get_supported_runner_arch`) +- Test: `app/tests/unit/test_cloud_image.py` (`test__get_supported_runner_arch`, ARM param ~L182) + +- [ ] **Step 1: Update the failing test** + +In `app/tests/unit/test_cloud_image.py`, change the ARM param of `test__get_supported_runner_arch`: + +```python + pytest.param(Arch.ARM, "arm64", id="ARM"), +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_cloud_image.py::test__get_supported_runner_arch -q` +Expected: FAIL — `assert 'armhf' == 'arm64'` for the `ARM` param. + +- [ ] **Step 3: Update `_get_supported_runner_arch`** + +In `app/src/github_runner_image_builder/cloud_image.py`, replace the `Arch.ARM` case: + +```python + case Arch.ARM: + # Download the arm64 (64-bit) base cloud image, not armhf. The armhf runner image is + # an arm64 boot image (64-bit kernel) that runs the 32-bit linux-arm runner via + # multiarch; a native armhf (32-bit kernel) image does not boot on the aarch64 hosts. + return "arm64" +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_cloud_image.py::test__get_supported_runner_arch -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +cd ~/github-runner-image-builder-operator +git add app/src/github_runner_image_builder/cloud_image.py app/tests/unit/test_cloud_image.py +git commit -S -m "fix(armhf): download arm64 base cloud image for Arch.ARM + +The builder VM must boot a 64-bit kernel on the aarch64 hosts. Download the +arm64 base cloud image; the 32-bit runner payload is added via multiarch. + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 3: Remove obsolete virtio glance properties + +**Files:** +- Modify: `app/src/github_runner_image_builder/store.py` (`_ARM_IMAGE_PROPERTIES` dict + ARM branch in `upload_image`) +- Test: `app/tests/unit/test_store.py` (remove `test_upload_image_arm_uses_virtio_properties` and `test_upload_image_amd64_omits_virtio_properties`, ~L240-286) + +- [ ] **Step 1: Remove the two virtio tests** + +In `app/tests/unit/test_store.py`, delete both functions in their entirety: +- `test_upload_image_arm_uses_virtio_properties` (~L240-263) +- `test_upload_image_amd64_omits_virtio_properties` (~L266-286) + +Delete the blank lines so the file goes directly from the preceding test to `@pytest.mark.usefixtures("mock_connection")` (the `test_get_latest_image_id` block at ~L289). Leave the `from github_runner_image_builder.config import Arch` import in place (other tests still use `Arch`). + +- [ ] **Step 2: Run tests to verify the remaining store tests still pass** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_store.py -q` +Expected: PASS — 13 tests (was 15). + +- [ ] **Step 3: Remove `_ARM_IMAGE_PROPERTIES` and the ARM branch** + +In `app/src/github_runner_image_builder/store.py`: + +Delete the entire comment block and dict definition of `_ARM_IMAGE_PROPERTIES` (the block that begins with the comment about 32-bit ARM base images and ends at the closing `}`). + +Then in `upload_image`, make the properties unconditional. Replace the block that builds `image_properties` and conditionally updates it for `Arch.ARM` with: + +```python + logger.info("Uploading image %s.", image_name) + image_properties = {"architecture": arch.to_openstack()} + # ignore type since the library does not provide correct type hinting but the docstring + # does define the return type. + image: Image = connection.create_image( +``` + +(i.e. remove the `if arch == Arch.ARM: image_properties.update(_ARM_IMAGE_PROPERTIES)` lines. The `properties=image_properties` argument passed to `create_image` stays.) + +- [ ] **Step 4: Run store tests + lint** + +Run: +```bash +cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_store.py -q \ + && ~/.venv-armhf/bin/python -m flake8 --max-line-length=99 \ + src/github_runner_image_builder/store.py tests/unit/test_store.py +``` +Expected: 13 passed; no flake8 output. + +- [ ] **Step 5: Commit** + +```bash +cd ~/github-runner-image-builder-operator +git add app/src/github_runner_image_builder/store.py app/tests/unit/test_store.py +git commit -S -m "fix(armhf): drop obsolete virtio glance properties + +With the armhf runner image now built from an arm64 base (64-bit kernel), it +boots with the cloud's normal defaults. The virtio-scsi/machine-type +workaround for booting a native armhf (32-bit kernel) image is no longer +needed. + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 4: armhf-qualified runtime dependencies + +**Files:** +- Modify: `app/src/github_runner_image_builder/config.py` (`ARM_ADDITIONAL_APT_PACKAGES`) +- Test: `app/tests/unit/test_openstack_builder.py` (`test__generate_cloud_init_script` ARM param, ~L679-683) + +- [ ] **Step 1: Update the failing test** + +In `app/tests/unit/test_openstack_builder.py`, change the ARM param's `additional_apt_packages` list: + +```python + pytest.param( + openstack_builder.Arch.ARM, + ["libc6:armhf", "libicu74:armhf", "libatomic1:armhf", "rustup", "docker-buildx"], + id="arm", + ), +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest "tests/unit/test_openstack_builder.py::test__generate_cloud_init_script[arm]" -q` +Expected: FAIL — the rendered `apt_packages` string still contains `libicu74 libatomic1 rustup docker-buildx` (arm64), not the `:armhf` variants. + +- [ ] **Step 3: Update `ARM_ADDITIONAL_APT_PACKAGES`** + +In `app/src/github_runner_image_builder/config.py`, replace the definition: + +```python +# The 32-bit linux-arm runner agent runs via the host's native AArch32 support on the arm64 +# image. It needs the armhf loader (ld-linux-armhf.so.3 from libc6:armhf) and the armhf builds of +# its .NET runtime dependencies (libicu, libatomic). rustup and docker-buildx are additionally +# pre-installed (arm64 host tools) for arm32 build workloads. +ARM_ADDITIONAL_APT_PACKAGES = [ + "libc6:armhf", + "libicu74:armhf", + "libatomic1:armhf", + "rustup", + "docker-buildx", +] +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest "tests/unit/test_openstack_builder.py::test__generate_cloud_init_script[arm]" -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +cd ~/github-runner-image-builder-operator +git add app/src/github_runner_image_builder/config.py app/tests/unit/test_openstack_builder.py +git commit -S -m "fix(armhf): install armhf-qualified runtime deps for the 32-bit runner + +The 32-bit linux-arm runner agent needs the armhf loader (libc6:armhf) and +armhf builds of its .NET runtime deps (libicu74:armhf, libatomic1:armhf) to +exec on the arm64 image via native AArch32. + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 5: Enable armhf multiarch in cloud-init + +The 32-bit runner and its `:armhf` packages can only be installed after `dpkg --add-architecture armhf`. armhf packages are served by `ports.ubuntu.com`, which the arm64 base image already uses as its apt mirror, so no apt-source edit is needed — just the dpkg foreign architecture plus the `apt-get update` that `install_apt_packages` already runs. + +**Files:** +- Modify: `app/src/github_runner_image_builder/templates/cloud-init.sh.j2` (add `enable_armhf_multiarch` function before `function install_apt_packages()`; add guarded call before the `install_apt_packages "$apt_packages" "$hwe_version"` call) +- Test: `app/tests/unit/test_openstack_builder.py` (`test__generate_cloud_init_script` expected f-string: add function definition ~before L775, add guarded call ~before L931) + +- [ ] **Step 1: Update the failing test — add the function definition** + +In `app/tests/unit/test_openstack_builder.py`, in the big expected f-string of `test__generate_cloud_init_script`, add the new function definition immediately **before** the `function install_apt_packages() {{` line (~L775). Note the doubled braces `{{`/`}}` because this is an f-string: + +```python +function enable_armhf_multiarch() {{ + echo "Enabling armhf multiarch" + /usr/bin/dpkg --add-architecture armhf +}} + +function install_apt_packages() {{ +``` + +- [ ] **Step 2: Update the failing test — add the guarded call** + +In the same expected f-string, change the call sequence (~L930-931) from: + +```python +configure_proxy "$proxy" +install_apt_packages "$apt_packages" "$hwe_version" +``` + +to: + +```python +configure_proxy "$proxy" +# Enable armhf multiarch before installing packages so the 32-bit linux-arm runner agent and +# its armhf runtime dependencies can be installed and executed via native AArch32. +if [ "$github_runner_arch" == "arm" ]; then + enable_armhf_multiarch +fi +install_apt_packages "$apt_packages" "$hwe_version" +``` + +(No brace-escaping needed here — there are no literal `{`/`}` in these lines.) + +- [ ] **Step 3: Run test to verify it fails** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_openstack_builder.py::test__generate_cloud_init_script -q` +Expected: FAIL for all 5 params — the rendered template does not yet contain `enable_armhf_multiarch` (neither the definition nor the guarded call). + +- [ ] **Step 4: Add the function to the template** + +In `app/src/github_runner_image_builder/templates/cloud-init.sh.j2`, add the function immediately **before** `function install_apt_packages() {` (currently ~L55): + +```bash +function enable_armhf_multiarch() { + echo "Enabling armhf multiarch" + /usr/bin/dpkg --add-architecture armhf +} + +function install_apt_packages() { +``` + +- [ ] **Step 5: Add the guarded call in the template** + +In the same file, change the call sequence at the bottom from: + +```bash +configure_proxy "$proxy" +install_apt_packages "$apt_packages" "$hwe_version" +``` + +to: + +```bash +configure_proxy "$proxy" +# Enable armhf multiarch before installing packages so the 32-bit linux-arm runner agent and +# its armhf runtime dependencies can be installed and executed via native AArch32. +if [ "$github_runner_arch" == "arm" ]; then + enable_armhf_multiarch +fi +install_apt_packages "$apt_packages" "$hwe_version" +``` + +- [ ] **Step 6: Run test to verify it passes** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_openstack_builder.py::test__generate_cloud_init_script -q` +Expected: PASS (all 5 params). + +- [ ] **Step 7: Commit** + +```bash +cd ~/github-runner-image-builder-operator +git add app/src/github_runner_image_builder/templates/cloud-init.sh.j2 app/tests/unit/test_openstack_builder.py +git commit -S -m "feat(armhf): enable armhf multiarch in cloud-init + +Add dpkg --add-architecture armhf (guarded to the arm runner) before apt +install so the arm64 image can install and run the 32-bit linux-arm runner +and its armhf runtime deps via native AArch32. armhf packages are served by +ports.ubuntu.com, which the arm64 base already uses. + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +--- + +## Task 6: Full verification and push + +**Files:** none (verification only) + +- [ ] **Step 1: Run the full unit suite** + +Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit -q` +Expected: PASS — 133 tests (135 baseline minus the 2 removed virtio tests). + +- [ ] **Step 2: Format** + +Run: +```bash +cd app && ~/.venv-armhf/bin/python -m black --line-length 99 \ + src/github_runner_image_builder tests/unit \ + && ~/.venv-armhf/bin/python -m isort --profile black \ + src/github_runner_image_builder tests/unit +``` +Expected: "All done" (files unchanged or reformatted). + +- [ ] **Step 3: Lint** + +Run: +```bash +cd app && ~/.venv-armhf/bin/python -m flake8 --max-line-length=99 \ + src/github_runner_image_builder tests/unit +``` +Expected: no output. + +- [ ] **Step 4: Commit any formatting-only changes (if any)** + +```bash +cd ~/github-runner-image-builder-operator +git add -A app +git diff --cached --quiet || git commit -S -m "style(armhf): apply black/isort formatting + +Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" +``` + +- [ ] **Step 5: Push** + +```bash +cd ~/github-runner-image-builder-operator +git push "https://x-access-token:$(gh auth token)@github.com/canonical/github-runner-image-builder-operator.git" feat/armhf-arch-support-isd-5856 +``` +Expected: branch updated on origin; CI "Integration tests for application" starts. + +- [ ] **Step 6: Verify the armhf integration legs boot** + +After CI starts, confirm the armhf legs no longer fail at server-create/SSH. The armhf builder VM should now boot (64-bit kernel), cloud-init should install the 32-bit runner + armhf libs, and the `commands.py` armhf assertions (`file … Runner.Listener` = ELF 32-bit ARM, `rustc`/`cargo`/`docker buildx`) should pass. + +Poll job statuses (replace `` with the run id): +```bash +gh api repos/canonical/github-runner-image-builder-operator/actions/runs//jobs \ + --paginate --jq '.jobs[] | select(.name|test("arm")) | "\(.name)=\(.status)/\(.conclusion)"' +``` +Expected: eventually `completed/success` for the armhf legs. + +--- + +## Rollback / supersedes + +This plan supersedes the two virtio commits already on the branch (`store.py` `_ARM_IMAGE_PROPERTIES`) and the `armv7l` glance arch / `armhf` base-image download. Those are removed/replaced by the arm64-boot + multiarch model. Runner PRs #155/#169/#171 are unchanged; this plan consumes their 32-bit `linux-arm` package. + +## Open risks (validated during CI, not before) + +- **armhf package names on noble arm64:** `libc6:armhf`, `libicu74:armhf`, `libatomic1:armhf` are assumed present on `ports.ubuntu.com` for noble/resolute. If a name differs on resolute, the armhf leg's apt install will fail with a clear "unable to locate package" error — adjust the name in `ARM_ADDITIONAL_APT_PACKAGES` and re-push. +- **apt sources architecture restriction:** if the arm64 base's deb822 sources ever pin `Architectures: arm64`, `dpkg --add-architecture armhf` alone won't fetch armhf lists. The default noble/resolute arm64 sources do NOT pin architectures, so this is expected to work; if apt update errors on missing armhf lists, add an armhf-scoped `ports.ubuntu.com` source in `enable_armhf_multiarch`. From c36f96f1ef5d1b6cf63ee45a5bcc4c186bc9477a Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Thu, 9 Jul 2026 21:57:51 +0700 Subject: [PATCH 23/27] chore: stop tracking superpowers planning artifacts The docs/superpowers plan and spec are internal agent planning artifacts, not product documentation. Untrack them and gitignore the directory so the Vale docs linter does not scan them. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 3 + .../2026-07-09-armhf-arm64-multiarch-image.md | 442 ------------------ ...7-09-armhf-arm64-multiarch-image-design.md | 161 ------- 3 files changed, 3 insertions(+), 603 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-09-armhf-arm64-multiarch-image.md delete mode 100644 docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md diff --git a/.gitignore b/.gitignore index 2f13cc2a..251faf0f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ testing_key.pem .vale/styles/config/vocabularies/* !.vale/styles/config/vocabularies/local # END VALE WORKFLOW IGNORE + +# Superpowers internal planning artifacts (not product docs) +docs/superpowers/ diff --git a/docs/superpowers/plans/2026-07-09-armhf-arm64-multiarch-image.md b/docs/superpowers/plans/2026-07-09-armhf-arm64-multiarch-image.md deleted file mode 100644 index 6fcb08bf..00000000 --- a/docs/superpowers/plans/2026-07-09-armhf-arm64-multiarch-image.md +++ /dev/null @@ -1,442 +0,0 @@ -# armhf Runner Image (arm64 boot + 32-bit runner payload) Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the image-builder's `armhf` path build an **arm64 boot image** that boots on the ps7 arm64 hosts and carries the **32-bit `linux-arm` runner agent** plus armhf multiarch userland, so it runs via the hosts' native AArch32. - -**Architecture:** Decouple *boot architecture* from *runner architecture* for `Arch.ARM`: boot/base image becomes arm64 (glance `architecture=aarch64`), while the runner payload stays 32-bit (`RUNNER_ARCH=arm`, armhf multiarch libs). Remove the obsolete virtio glance-property workaround. All changes are TDD with signed commits. - -**Tech Stack:** Python 3.12, pytest, Jinja2 cloud-init template, OpenStack SDK. Unit tests run **from the `app/` directory**: `~/.venv-armhf/bin/python -m pytest tests/unit`. Line length 99 (black / isort profile=black / flake8 --max-line-length=99). Signed commits: `git commit -S`. - -**Repo/branch:** `~/github-runner-image-builder-operator`, branch `feat/armhf-arch-support-isd-5856` (PR #233). - -**Spec:** `docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md` - -**Baseline:** 135 unit tests pass today (`cd app && ~/.venv-armhf/bin/python -m pytest tests/unit -q`). After this plan there are 133 (two obsolete virtio tests removed). - ---- - -## File Structure - -| File | Responsibility | Change | -|---|---|---| -| `app/src/github_runner_image_builder/config.py` | Arch enum, glance arch mapping, apt package lists | `Arch.ARM.to_openstack()` → `aarch64`; `ARM_ADDITIONAL_APT_PACKAGES` → armhf-qualified deps | -| `app/src/github_runner_image_builder/cloud_image.py` | Base cloud-image download | `_get_supported_runner_arch(Arch.ARM)` → `arm64` | -| `app/src/github_runner_image_builder/store.py` | Upload image to glance | Remove `_ARM_IMAGE_PROPERTIES` + ARM virtio block | -| `app/src/github_runner_image_builder/templates/cloud-init.sh.j2` | Runner image provisioning | Add `enable_armhf_multiarch` + guarded call | -| `app/tests/unit/test_config.py` | config unit tests | ARM → `aarch64` | -| `app/tests/unit/test_cloud_image.py` | cloud_image unit tests | ARM → `arm64` | -| `app/tests/unit/test_store.py` | store unit tests | Remove the 2 virtio tests | -| `app/tests/unit/test_openstack_builder.py` | cloud-init render tests | New arm packages + multiarch function in expected string | - -Notes: -- The integration test (`app/tests/integration/commands.py`) already asserts the runner binary is `ELF 32-bit ARM` and runs `rustc`/`cargo`/`docker buildx` for armhf. No integration-code change is required; the armhf leg simply needs to boot now. -- Tasks are ordered so each leaves the unit suite green. -- All commands below assume you start each task at the repo root `~/github-runner-image-builder-operator` unless the command itself `cd`s into `app`. - ---- - -## Task 1: Glance architecture for armhf → aarch64 - -**Files:** -- Modify: `app/src/github_runner_image_builder/config.py` (`Arch.to_openstack`) -- Test: `app/tests/unit/test_config.py` (`test_arch_openstack_conversion`, ARM param ~L25) - -- [ ] **Step 1: Update the failing test** - -In `app/tests/unit/test_config.py`, change the ARM param: - -```python - pytest.param(Arch.ARM, "aarch64", id="arm"), -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_config.py::test_arch_openstack_conversion -q` -Expected: FAIL — `assert 'armv7l' == 'aarch64'` for the `arm` param. - -- [ ] **Step 3: Update `to_openstack`** - -In `app/src/github_runner_image_builder/config.py`, replace the `Arch.ARM` case in `to_openstack`: - -```python - case Arch.ARM: - # The armhf runner image is an arm64 boot image (64-bit kernel) that carries a - # 32-bit linux-arm runner payload. Native armhf images (32-bit kernel) do not boot - # on the aarch64 "virt" machine, so the glance architecture is aarch64 so the base - # image and snapshot schedule and boot on arm64 hosts. The 32-bit runner is - # installed via multiarch (see cloud-init.sh.j2 and ARM_ADDITIONAL_APT_PACKAGES). - return "aarch64" -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_config.py::test_arch_openstack_conversion -q` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -cd ~/github-runner-image-builder-operator -git add app/src/github_runner_image_builder/config.py app/tests/unit/test_config.py -git commit -S -m "fix(armhf): map Arch.ARM glance architecture to aarch64 - -The armhf runner image is an arm64 boot image carrying a 32-bit runner -payload; native armhf (32-bit kernel) images do not boot on the aarch64 -hosts. Schedule/boot the base image and snapshot as aarch64. - -Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" -``` - ---- - -## Task 2: Base cloud-image download for armhf → arm64 - -**Files:** -- Modify: `app/src/github_runner_image_builder/cloud_image.py` (`_get_supported_runner_arch`) -- Test: `app/tests/unit/test_cloud_image.py` (`test__get_supported_runner_arch`, ARM param ~L182) - -- [ ] **Step 1: Update the failing test** - -In `app/tests/unit/test_cloud_image.py`, change the ARM param of `test__get_supported_runner_arch`: - -```python - pytest.param(Arch.ARM, "arm64", id="ARM"), -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_cloud_image.py::test__get_supported_runner_arch -q` -Expected: FAIL — `assert 'armhf' == 'arm64'` for the `ARM` param. - -- [ ] **Step 3: Update `_get_supported_runner_arch`** - -In `app/src/github_runner_image_builder/cloud_image.py`, replace the `Arch.ARM` case: - -```python - case Arch.ARM: - # Download the arm64 (64-bit) base cloud image, not armhf. The armhf runner image is - # an arm64 boot image (64-bit kernel) that runs the 32-bit linux-arm runner via - # multiarch; a native armhf (32-bit kernel) image does not boot on the aarch64 hosts. - return "arm64" -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_cloud_image.py::test__get_supported_runner_arch -q` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -cd ~/github-runner-image-builder-operator -git add app/src/github_runner_image_builder/cloud_image.py app/tests/unit/test_cloud_image.py -git commit -S -m "fix(armhf): download arm64 base cloud image for Arch.ARM - -The builder VM must boot a 64-bit kernel on the aarch64 hosts. Download the -arm64 base cloud image; the 32-bit runner payload is added via multiarch. - -Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" -``` - ---- - -## Task 3: Remove obsolete virtio glance properties - -**Files:** -- Modify: `app/src/github_runner_image_builder/store.py` (`_ARM_IMAGE_PROPERTIES` dict + ARM branch in `upload_image`) -- Test: `app/tests/unit/test_store.py` (remove `test_upload_image_arm_uses_virtio_properties` and `test_upload_image_amd64_omits_virtio_properties`, ~L240-286) - -- [ ] **Step 1: Remove the two virtio tests** - -In `app/tests/unit/test_store.py`, delete both functions in their entirety: -- `test_upload_image_arm_uses_virtio_properties` (~L240-263) -- `test_upload_image_amd64_omits_virtio_properties` (~L266-286) - -Delete the blank lines so the file goes directly from the preceding test to `@pytest.mark.usefixtures("mock_connection")` (the `test_get_latest_image_id` block at ~L289). Leave the `from github_runner_image_builder.config import Arch` import in place (other tests still use `Arch`). - -- [ ] **Step 2: Run tests to verify the remaining store tests still pass** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_store.py -q` -Expected: PASS — 13 tests (was 15). - -- [ ] **Step 3: Remove `_ARM_IMAGE_PROPERTIES` and the ARM branch** - -In `app/src/github_runner_image_builder/store.py`: - -Delete the entire comment block and dict definition of `_ARM_IMAGE_PROPERTIES` (the block that begins with the comment about 32-bit ARM base images and ends at the closing `}`). - -Then in `upload_image`, make the properties unconditional. Replace the block that builds `image_properties` and conditionally updates it for `Arch.ARM` with: - -```python - logger.info("Uploading image %s.", image_name) - image_properties = {"architecture": arch.to_openstack()} - # ignore type since the library does not provide correct type hinting but the docstring - # does define the return type. - image: Image = connection.create_image( -``` - -(i.e. remove the `if arch == Arch.ARM: image_properties.update(_ARM_IMAGE_PROPERTIES)` lines. The `properties=image_properties` argument passed to `create_image` stays.) - -- [ ] **Step 4: Run store tests + lint** - -Run: -```bash -cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_store.py -q \ - && ~/.venv-armhf/bin/python -m flake8 --max-line-length=99 \ - src/github_runner_image_builder/store.py tests/unit/test_store.py -``` -Expected: 13 passed; no flake8 output. - -- [ ] **Step 5: Commit** - -```bash -cd ~/github-runner-image-builder-operator -git add app/src/github_runner_image_builder/store.py app/tests/unit/test_store.py -git commit -S -m "fix(armhf): drop obsolete virtio glance properties - -With the armhf runner image now built from an arm64 base (64-bit kernel), it -boots with the cloud's normal defaults. The virtio-scsi/machine-type -workaround for booting a native armhf (32-bit kernel) image is no longer -needed. - -Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" -``` - ---- - -## Task 4: armhf-qualified runtime dependencies - -**Files:** -- Modify: `app/src/github_runner_image_builder/config.py` (`ARM_ADDITIONAL_APT_PACKAGES`) -- Test: `app/tests/unit/test_openstack_builder.py` (`test__generate_cloud_init_script` ARM param, ~L679-683) - -- [ ] **Step 1: Update the failing test** - -In `app/tests/unit/test_openstack_builder.py`, change the ARM param's `additional_apt_packages` list: - -```python - pytest.param( - openstack_builder.Arch.ARM, - ["libc6:armhf", "libicu74:armhf", "libatomic1:armhf", "rustup", "docker-buildx"], - id="arm", - ), -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest "tests/unit/test_openstack_builder.py::test__generate_cloud_init_script[arm]" -q` -Expected: FAIL — the rendered `apt_packages` string still contains `libicu74 libatomic1 rustup docker-buildx` (arm64), not the `:armhf` variants. - -- [ ] **Step 3: Update `ARM_ADDITIONAL_APT_PACKAGES`** - -In `app/src/github_runner_image_builder/config.py`, replace the definition: - -```python -# The 32-bit linux-arm runner agent runs via the host's native AArch32 support on the arm64 -# image. It needs the armhf loader (ld-linux-armhf.so.3 from libc6:armhf) and the armhf builds of -# its .NET runtime dependencies (libicu, libatomic). rustup and docker-buildx are additionally -# pre-installed (arm64 host tools) for arm32 build workloads. -ARM_ADDITIONAL_APT_PACKAGES = [ - "libc6:armhf", - "libicu74:armhf", - "libatomic1:armhf", - "rustup", - "docker-buildx", -] -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest "tests/unit/test_openstack_builder.py::test__generate_cloud_init_script[arm]" -q` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -cd ~/github-runner-image-builder-operator -git add app/src/github_runner_image_builder/config.py app/tests/unit/test_openstack_builder.py -git commit -S -m "fix(armhf): install armhf-qualified runtime deps for the 32-bit runner - -The 32-bit linux-arm runner agent needs the armhf loader (libc6:armhf) and -armhf builds of its .NET runtime deps (libicu74:armhf, libatomic1:armhf) to -exec on the arm64 image via native AArch32. - -Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" -``` - ---- - -## Task 5: Enable armhf multiarch in cloud-init - -The 32-bit runner and its `:armhf` packages can only be installed after `dpkg --add-architecture armhf`. armhf packages are served by `ports.ubuntu.com`, which the arm64 base image already uses as its apt mirror, so no apt-source edit is needed — just the dpkg foreign architecture plus the `apt-get update` that `install_apt_packages` already runs. - -**Files:** -- Modify: `app/src/github_runner_image_builder/templates/cloud-init.sh.j2` (add `enable_armhf_multiarch` function before `function install_apt_packages()`; add guarded call before the `install_apt_packages "$apt_packages" "$hwe_version"` call) -- Test: `app/tests/unit/test_openstack_builder.py` (`test__generate_cloud_init_script` expected f-string: add function definition ~before L775, add guarded call ~before L931) - -- [ ] **Step 1: Update the failing test — add the function definition** - -In `app/tests/unit/test_openstack_builder.py`, in the big expected f-string of `test__generate_cloud_init_script`, add the new function definition immediately **before** the `function install_apt_packages() {{` line (~L775). Note the doubled braces `{{`/`}}` because this is an f-string: - -```python -function enable_armhf_multiarch() {{ - echo "Enabling armhf multiarch" - /usr/bin/dpkg --add-architecture armhf -}} - -function install_apt_packages() {{ -``` - -- [ ] **Step 2: Update the failing test — add the guarded call** - -In the same expected f-string, change the call sequence (~L930-931) from: - -```python -configure_proxy "$proxy" -install_apt_packages "$apt_packages" "$hwe_version" -``` - -to: - -```python -configure_proxy "$proxy" -# Enable armhf multiarch before installing packages so the 32-bit linux-arm runner agent and -# its armhf runtime dependencies can be installed and executed via native AArch32. -if [ "$github_runner_arch" == "arm" ]; then - enable_armhf_multiarch -fi -install_apt_packages "$apt_packages" "$hwe_version" -``` - -(No brace-escaping needed here — there are no literal `{`/`}` in these lines.) - -- [ ] **Step 3: Run test to verify it fails** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_openstack_builder.py::test__generate_cloud_init_script -q` -Expected: FAIL for all 5 params — the rendered template does not yet contain `enable_armhf_multiarch` (neither the definition nor the guarded call). - -- [ ] **Step 4: Add the function to the template** - -In `app/src/github_runner_image_builder/templates/cloud-init.sh.j2`, add the function immediately **before** `function install_apt_packages() {` (currently ~L55): - -```bash -function enable_armhf_multiarch() { - echo "Enabling armhf multiarch" - /usr/bin/dpkg --add-architecture armhf -} - -function install_apt_packages() { -``` - -- [ ] **Step 5: Add the guarded call in the template** - -In the same file, change the call sequence at the bottom from: - -```bash -configure_proxy "$proxy" -install_apt_packages "$apt_packages" "$hwe_version" -``` - -to: - -```bash -configure_proxy "$proxy" -# Enable armhf multiarch before installing packages so the 32-bit linux-arm runner agent and -# its armhf runtime dependencies can be installed and executed via native AArch32. -if [ "$github_runner_arch" == "arm" ]; then - enable_armhf_multiarch -fi -install_apt_packages "$apt_packages" "$hwe_version" -``` - -- [ ] **Step 6: Run test to verify it passes** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit/test_openstack_builder.py::test__generate_cloud_init_script -q` -Expected: PASS (all 5 params). - -- [ ] **Step 7: Commit** - -```bash -cd ~/github-runner-image-builder-operator -git add app/src/github_runner_image_builder/templates/cloud-init.sh.j2 app/tests/unit/test_openstack_builder.py -git commit -S -m "feat(armhf): enable armhf multiarch in cloud-init - -Add dpkg --add-architecture armhf (guarded to the arm runner) before apt -install so the arm64 image can install and run the 32-bit linux-arm runner -and its armhf runtime deps via native AArch32. armhf packages are served by -ports.ubuntu.com, which the arm64 base already uses. - -Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" -``` - ---- - -## Task 6: Full verification and push - -**Files:** none (verification only) - -- [ ] **Step 1: Run the full unit suite** - -Run: `cd app && ~/.venv-armhf/bin/python -m pytest tests/unit -q` -Expected: PASS — 133 tests (135 baseline minus the 2 removed virtio tests). - -- [ ] **Step 2: Format** - -Run: -```bash -cd app && ~/.venv-armhf/bin/python -m black --line-length 99 \ - src/github_runner_image_builder tests/unit \ - && ~/.venv-armhf/bin/python -m isort --profile black \ - src/github_runner_image_builder tests/unit -``` -Expected: "All done" (files unchanged or reformatted). - -- [ ] **Step 3: Lint** - -Run: -```bash -cd app && ~/.venv-armhf/bin/python -m flake8 --max-line-length=99 \ - src/github_runner_image_builder tests/unit -``` -Expected: no output. - -- [ ] **Step 4: Commit any formatting-only changes (if any)** - -```bash -cd ~/github-runner-image-builder-operator -git add -A app -git diff --cached --quiet || git commit -S -m "style(armhf): apply black/isort formatting - -Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>" -``` - -- [ ] **Step 5: Push** - -```bash -cd ~/github-runner-image-builder-operator -git push "https://x-access-token:$(gh auth token)@github.com/canonical/github-runner-image-builder-operator.git" feat/armhf-arch-support-isd-5856 -``` -Expected: branch updated on origin; CI "Integration tests for application" starts. - -- [ ] **Step 6: Verify the armhf integration legs boot** - -After CI starts, confirm the armhf legs no longer fail at server-create/SSH. The armhf builder VM should now boot (64-bit kernel), cloud-init should install the 32-bit runner + armhf libs, and the `commands.py` armhf assertions (`file … Runner.Listener` = ELF 32-bit ARM, `rustc`/`cargo`/`docker buildx`) should pass. - -Poll job statuses (replace `` with the run id): -```bash -gh api repos/canonical/github-runner-image-builder-operator/actions/runs//jobs \ - --paginate --jq '.jobs[] | select(.name|test("arm")) | "\(.name)=\(.status)/\(.conclusion)"' -``` -Expected: eventually `completed/success` for the armhf legs. - ---- - -## Rollback / supersedes - -This plan supersedes the two virtio commits already on the branch (`store.py` `_ARM_IMAGE_PROPERTIES`) and the `armv7l` glance arch / `armhf` base-image download. Those are removed/replaced by the arm64-boot + multiarch model. Runner PRs #155/#169/#171 are unchanged; this plan consumes their 32-bit `linux-arm` package. - -## Open risks (validated during CI, not before) - -- **armhf package names on noble arm64:** `libc6:armhf`, `libicu74:armhf`, `libatomic1:armhf` are assumed present on `ports.ubuntu.com` for noble/resolute. If a name differs on resolute, the armhf leg's apt install will fail with a clear "unable to locate package" error — adjust the name in `ARM_ADDITIONAL_APT_PACKAGES` and re-push. -- **apt sources architecture restriction:** if the arm64 base's deb822 sources ever pin `Architectures: arm64`, `dpkg --add-architecture armhf` alone won't fetch armhf lists. The default noble/resolute arm64 sources do NOT pin architectures, so this is expected to work; if apt update errors on missing armhf lists, add an armhf-scoped `ports.ubuntu.com` source in `enable_armhf_multiarch`. diff --git a/docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md b/docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md deleted file mode 100644 index dac556fb..00000000 --- a/docs/superpowers/specs/2026-07-09-armhf-arm64-multiarch-image-design.md +++ /dev/null @@ -1,161 +0,0 @@ -# armhf Runner Image via arm64 Boot Image + 32-bit Runner Payload - -- **Date:** 2026-07-09 -- **Status:** Approved (design) -- **Related:** PR #233 (image-builder armhf support), runner PRs #155/#169/#171 -- **Spec:** ISD-5856 - -> Note: this design runs 32-bit armhf userland **natively** via the hosts' AArch32 -> support (AArch32@EL0) using Ubuntu multiarch — it does **not** use qemu emulation. - -## Problem - -The image-builder's `Arch.ARM` (armhf) path builds a **native armhf image** — it -downloads `-server-cloudimg-armhf.img` (a 32-bit ARM kernel) and boots it as a -full VM on the ProdStack 7 (ps7) arm64 compute hosts. That VM never boots: Nova marks -it `ACTIVE` and neutron assigns an IP, but the serial console is **completely empty** -and SSH times out forever. - -### Root cause - -Booting a full armhf VM requires a **32-bit kernel** to start under the aarch64 `virt` -machine's 64-bit UEFI firmware (i.e. AArch32 at EL1 and/or a 32-bit boot chain). The -ps7 hosts do **not** provide this, so the 32-bit kernel never executes → empty console. - -This is **not** a hardware limitation for armhf *workloads*. The same ps7 hosts do -support **AArch32 at EL0** (32-bit userland), proven by the merged runner work: - -- PR #155: the runner was "Built and run on an arm64 VM with AArch32 (native 32-bit) - userspace, inside an `arm32v7/ubuntu:noble` container … binaries run natively as - armhf (no qemu)." -- PR #169/#171: the shipped `linux-arm` (armv7l, 32-bit) package is smoke-tested by - running `./config.sh --version` inside an `arm32v7/ubuntu:noble` container on the - arm64 self-hosted pool, "natively via the host's AArch32 support (no qemu)." - -The proven-working model is therefore an **arm64 kernel (64-bit) running 32-bit armhf -userland**, which needs only AArch32@EL0 (available). The image-builder instead builds -a 32-bit-kernel image (needs AArch32@EL1 / 32-bit boot — unavailable). It builds the -wrong kind of image. - -## Goal - -Make the image-builder's "armhf runner" image reproduce the validated model: an -**arm64 boot image** that carries the **32-bit `linux-arm` runner agent** plus the -armhf multiarch userland it needs, running via the hosts' native AArch32. armhf jobs -build/test armhf binaries and run `linux/arm/v7` containers. - -Non-goals: full-system emulation (`qemu-system-arm`); a qemu-user portability fallback -(all target hosts are verified AArch32-capable, so native execution is used). - -## Design - -### Core reframing: boot architecture vs runner architecture - -For every architecture except armhf, the architecture the builder VM **boots** equals -the architecture of the **runner payload**. For `Arch.ARM` they diverge: - -| Aspect | Current (broken) | New | -|---|---|---| -| Base cloud image | `…-cloudimg-armhf.img` (32-bit kernel) | `…-cloudimg-arm64.img` (64-bit kernel) | -| Glance `architecture` property | `armv7l` | `aarch64` | -| Builder / runner flavor | `shared.xsmall.arm64` | unchanged | -| Boots on ps7 hosts | No (empty console) | Yes (normal arm64 boot) | -| Runner agent installed | `linux-arm` (32-bit) | `linux-arm` (32-bit) — unchanged | -| armhf userland | none | armhf multiarch libs | -| virtio glance props | required (still failed) | removed | - -`Arch.ARM` continues to mean "produce a 32-bit ARM runner"; it is now delivered as an -arm64 image carrying a 32-bit runner + armhf multiarch userland, executed via native -AArch32. - -### Component changes - -1. **`config.py`** - - `Arch.ARM.to_openstack()` returns **`aarch64`** (was `armv7l`) so the base image - and snapshot schedule/boot on arm64 hosts. Update the outdated `armv7l` comment. - - `Arch.ARM.value` stays `"arm"` (drives the 32-bit runner tarball download and the - `github_runner_arch` cloud-init variable). - - `ARM_ADDITIONAL_APT_PACKAGES` becomes the **armhf-qualified** runtime deps for the - 32-bit runner agent plus arm64 host tools: - `["libc6:armhf", "libicu74:armhf", "libatomic1:armhf", "rustup", "docker-buildx"]`. - (`libc6:armhf` provides `/lib/ld-linux-armhf.so.3`; `libicu74:armhf` and - `libatomic1:armhf` are the .NET runtime deps the armv7l runner needs — see #169.) - -2. **`cloud_image.py`** - - `_get_supported_runner_arch(Arch.ARM)` returns **`arm64`** (was `armhf`), so the - builder VM boots a 64-bit kernel. This is the direct fix for the empty-console - failure. Update the docstring/comment accordingly. - -3. **`store.py`** - - Remove `_ARM_IMAGE_PROPERTIES` and the `if arch == Arch.ARM` block; revert - `upload_image` to `properties={"architecture": arch.to_openstack()}`. An arm64 - image boots with the cloud's normal defaults, so the IDE/virtio workaround is - obsolete. - -4. **`openstack_builder.py`** - - `_generate_cloud_init_script` keeps `RUNNER_ARCH="arm"` and the noble+ base-image - guard (`ARM_SUPPORTED_BASE_IMAGES`). The armhf multiarch enablement is expressed in - the cloud-init template (below); no change to the flavor logic (already arm64). - -5. **`templates/cloud-init.sh.j2`** - - Add an armhf-only step, guarded by `[ "$github_runner_arch" == "arm" ]`, run - **before** `install_apt_packages`: - - `dpkg --add-architecture armhf` - - Ensure the apt sources serve armhf. arm64 Ubuntu already uses - `ports.ubuntu.com`, which also hosts armhf; add `armhf` to the sources' - architectures (deb822 `Architectures:` on noble+, or an `[arch=armhf]` entry). - - `apt-get update` - - The armhf-qualified libs then install via the normal `install_apt_packages` path; - the existing `install_github_runner` (arch `arm`) downloads the 32-bit tarball; the - existing `rustup default stable` armhf block stays. - -### Data flow - -``` -download arm64 cloud image ──▶ upload base (architecture=aarch64) - │ - ▼ -create builder VM (shared.xsmall.arm64) ── boots 64-bit kernel ✅ - │ - ▼ -cloud-init: - enable armhf multiarch (dpkg --add-architecture armhf; apt update) - install libc6:armhf + armhf runtime deps + host tools - install linux-arm (32-bit) runner agent ── runs via native AArch32@EL0 - │ - ▼ -snapshot ──▶ upload runner image (architecture=aarch64) -``` - -### Error handling - -- Non-noble bases for armhf still fail fast with `UnsupportedArchitectureError` (the - armhf-qualified libs require noble/24.04+). -- If a host genuinely lacked AArch32, the 32-bit runner agent would fail to `exec`; - since all target hosts are verified AArch32-capable, no fallback is provided (matches - #169's stated assumption). - -## Testing - -### Unit -- `test_store.py`: revert/remove the two virtio-property tests; assert `upload_image` - passes only `properties={"architecture": …}`. -- `test_cloud_image.py`: `Arch.ARM` downloads the `arm64` base image filename. -- `test_config.py`: `Arch.ARM.to_openstack()` returns `aarch64`; - `ARM_ADDITIONAL_APT_PACKAGES` contains the armhf-qualified deps. -- `test_openstack_builder.py`: generated cloud-init for `Arch.ARM` enables armhf - multiarch and still sets `RUNNER_ARCH=arm`; non-noble armhf base raises. - -### Integration (ps7 arm64 tenant) -- The armhf leg now **boots** (previously the entire failure mode). -- Assert the produced image is arm64 and the baked 32-bit runner agent is armv7l and - executable — e.g. `file .../bin/Runner.Listener` reports "ARM, EABI5 32-bit" and/or - `./config.sh --version` runs (mirroring #169's container smoke test). - -## Rollback / supersedes - -This supersedes the native-armhf commits on branch `feat/armhf-arch-support-isd-5856` -(virtio glance properties in `store.py`, `armv7l` glance arch, `armhf` base-image -download). Those are reverted/replaced by the arm64-boot + multiarch model here. The -runner-side PRs (#155/#169/#171) are unchanged — this design consumes their 32-bit -`linux-arm` package exactly as they built it. From 53c164cded81bba1c6d6e019d12777ebd9fd0de7 Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Fri, 10 Jul 2026 01:18:47 +0700 Subject: [PATCH 24/27] fix(armhf): fetch armhf packages from ports.ubuntu.com The arm64 cloud image's apt mirrors (clouds.archive/security.ubuntu.com) only carry the native architecture; armhf is a ports architecture and its package indices 404 there. Pin the stock sources to the native arch and add a ports.ubuntu.com source for armhf so the 32-bit runner's runtime deps (libc6:armhf, libicu74:armhf, libatomic1:armhf) resolve. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/cloud-init.sh.j2 | 14 ++++++++++++++ app/tests/unit/test_openstack_builder.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 index 2c1b25c1..5e356256 100644 --- a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 +++ b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 @@ -55,7 +55,21 @@ EOF function enable_armhf_multiarch() { echo "Enabling armhf multiarch" + native_arch=$(/usr/bin/dpkg --print-architecture) + codename=$(. /etc/os-release && echo "$VERSION_CODENAME") /usr/bin/dpkg --add-architecture armhf + # The arm64 cloud image's apt mirrors only carry the native architecture; armhf is a "ports" + # architecture served from ports.ubuntu.com. Pin the stock sources to the native architecture + # and add a ports.ubuntu.com source for armhf so the 32-bit runner's dependencies resolve. + /usr/bin/sed -i "/^Types: deb/a Architectures: $native_arch" /etc/apt/sources.list.d/ubuntu.sources + /usr/bin/tee /etc/apt/sources.list.d/armhf-ports.sources >/dev/null </dev/null < Date: Fri, 10 Jul 2026 10:38:26 +0700 Subject: [PATCH 25/27] fix(armhf): resolve armhf apt package conflicts Two armhf package-selection bugs surfaced once the builder VM boots and runs apt: - rustup conflicts with the distro cargo/rustc packages ('held broken packages'). rustup provides the armhf/armv7 Rust toolchain, so drop cargo and rustc from the default apt set on armhf. - libicu's soname is release-specific (noble ships libicu74, resolute ships libicu78); the hardcoded libicu74:armhf is 'Unable to locate' on resolute. Select the armhf libicu matching the base image. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/src/github_runner_image_builder/config.py | 17 +++++++--- .../openstack_builder.py | 22 ++++++++++--- app/tests/unit/test_openstack_builder.py | 33 ++++++++++++------- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/app/src/github_runner_image_builder/config.py b/app/src/github_runner_image_builder/config.py index 163581b9..1261fc7c 100644 --- a/app/src/github_runner_image_builder/config.py +++ b/app/src/github_runner_image_builder/config.py @@ -137,16 +137,25 @@ def from_str(cls, tag_or_name: str) -> "BaseImage": ] S390X_PPC64LE_ADDITIONAL_APT_PACKAGES = ["dotnet-runtime-8.0"] # The 32-bit linux-arm runner agent runs via the host's native AArch32 support on the arm64 -# image. It needs the armhf loader (ld-linux-armhf.so.3 from libc6:armhf) and the armhf builds of -# its .NET runtime dependencies (libicu, libatomic). rustup and docker-buildx are additionally -# pre-installed (arm64 host tools) for arm32 build workloads. +# image. It needs the armhf loader (ld-linux-armhf.so.3 from libc6:armhf) and the armhf build of +# libatomic (a .NET runtime dependency). rustup provides the armhf/armv7 Rust toolchain and +# docker-buildx enables arm32 container builds. libicu (the other .NET runtime dependency) is +# release-specific and handled by ARM_LIBICU_APT_PACKAGE_BY_BASE below. ARM_ADDITIONAL_APT_PACKAGES = [ "libc6:armhf", - "libicu74:armhf", "libatomic1:armhf", "rustup", "docker-buildx", ] +# rustup (installed on armhf images) conflicts with the distro cargo and rustc packages, so these +# are dropped from the default apt package set on armhf; rustup provides cargo and rustc instead. +ARM_EXCLUDED_DEFAULT_APT_PACKAGES = ("cargo", "rustc") +# The linux-arm runner's bundled .NET runtime dlopens libicu, whose soname is release-specific +# (noble ships libicu74, resolute ships libicu78). Install the armhf build matching the base image. +ARM_LIBICU_APT_PACKAGE_BY_BASE = { + BaseImage.NOBLE: "libicu74:armhf", + BaseImage.RESOLUTE: "libicu78:armhf", +} _LOG_LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR) LOG_LEVELS = tuple( diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 3ba03552..a1fd96b1 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -41,6 +41,8 @@ from github_runner_image_builder import cloud_image, config, store from github_runner_image_builder.config import ( ARM_ADDITIONAL_APT_PACKAGES, + ARM_EXCLUDED_DEFAULT_APT_PACKAGES, + ARM_LIBICU_APT_PACKAGE_BY_BASE, FORK_RUNNER_BINARY_REPO, IMAGE_DEFAULT_APT_PACKAGES, S390X_PPC64LE_ADDITIONAL_APT_PACKAGES, @@ -562,16 +564,28 @@ def _generate_cloud_init_script( if image_config.arch in (Arch.S390X, Arch.PPC64LE): apt_packages = IMAGE_DEFAULT_APT_PACKAGES + S390X_PPC64LE_ADDITIONAL_APT_PACKAGES elif image_config.arch == Arch.ARM: - # The armhf additional apt packages (libicu74, rustup, docker-buildx) are only available - # from the noble (24.04) archive onwards, so fail fast on older bases rather than letting - # apt-get fail midway through the image build. + # The armhf additional apt packages (rustup, docker-buildx, the armhf multiarch libs) and + # the release-specific libicu are only available from the noble (24.04) archive onwards, + # so fail fast on older bases rather than letting apt-get fail midway through the build. if image_config.base not in ARM_SUPPORTED_BASE_IMAGES: raise github_runner_image_builder.errors.UnsupportedArchitectureError( f"armhf images require a base image of " f"{[base.value for base in ARM_SUPPORTED_BASE_IMAGES]} or newer, " f"got: {image_config.base.value}." ) - apt_packages = IMAGE_DEFAULT_APT_PACKAGES + ARM_ADDITIONAL_APT_PACKAGES + # rustup provides the armhf/armv7 Rust toolchain and conflicts with the distro cargo/rustc + # packages, so drop those from the default set. libicu's soname is release-specific, so + # add the armhf build matching the base image. + default_packages = [ + package + for package in IMAGE_DEFAULT_APT_PACKAGES + if package not in ARM_EXCLUDED_DEFAULT_APT_PACKAGES + ] + apt_packages = ( + default_packages + + ARM_ADDITIONAL_APT_PACKAGES + + [ARM_LIBICU_APT_PACKAGE_BY_BASE[image_config.base]] + ) return template.render( PROXY=proxy, APT_PACKAGES=" ".join(apt_packages), diff --git a/app/tests/unit/test_openstack_builder.py b/app/tests/unit/test_openstack_builder.py index 407c6993..67916fa5 100644 --- a/app/tests/unit/test_openstack_builder.py +++ b/app/tests/unit/test_openstack_builder.py @@ -661,31 +661,40 @@ def test__determine_network(network_name: str | None): ) +_DEFAULT_APT_PACKAGES = ( + "build-essential cargo docker.io gh jq npm pkg-config python-is-python3 python3-dev " + "python3-pip rustc shellcheck socat tar time unzip wget" +) +# armhf drops the distro cargo/rustc (replaced by rustup) and adds the armhf multiarch runtime +# libs, rustup, docker-buildx and the release-specific libicu (libicu74 on noble). +_ARM_APT_PACKAGES = ( + "build-essential docker.io gh jq npm pkg-config python-is-python3 python3-dev python3-pip " + "shellcheck socat tar time unzip wget libc6:armhf libatomic1:armhf rustup docker-buildx " + "libicu74:armhf" +) + + @pytest.mark.parametrize( - "arch, additional_apt_packages", + "arch, expected_apt_packages", [ - pytest.param(openstack_builder.Arch.X64, [], id="x64"), - pytest.param(openstack_builder.Arch.ARM64, [], id="arm64"), + pytest.param(openstack_builder.Arch.X64, _DEFAULT_APT_PACKAGES, id="x64"), + pytest.param(openstack_builder.Arch.ARM64, _DEFAULT_APT_PACKAGES, id="arm64"), pytest.param( openstack_builder.Arch.S390X, - ["dotnet-runtime-8.0"], + _DEFAULT_APT_PACKAGES + " dotnet-runtime-8.0", id="s390x", ), pytest.param( openstack_builder.Arch.PPC64LE, - ["dotnet-runtime-8.0"], + _DEFAULT_APT_PACKAGES + " dotnet-runtime-8.0", id="ppc64le", ), - pytest.param( - openstack_builder.Arch.ARM, - ["libc6:armhf", "libicu74:armhf", "libatomic1:armhf", "rustup", "docker-buildx"], - id="arm", - ), + pytest.param(openstack_builder.Arch.ARM, _ARM_APT_PACKAGES, id="arm"), ], ) def test__generate_cloud_init_script( arch: openstack_builder.Arch, - additional_apt_packages: list[str], + expected_apt_packages: str, ): """ arrange: A certain architecture. @@ -940,7 +949,7 @@ def test__generate_cloud_init_script( proxy="test.proxy.internal:3128" -apt_packages="build-essential cargo docker.io gh jq npm pkg-config python-is-python3 python3-dev python3-pip rustc shellcheck socat tar time unzip wget{(' ' + ' '.join(additional_apt_packages)) if additional_apt_packages else ''}" +apt_packages="{expected_apt_packages}" hwe_version="{expected_hwe}" github_runner_version="" github_runner_arch="{arch.value}" From 73e40876e12e21300739a40d6d14de1034af115c Mon Sep 17 00:00:00 2001 From: florentianayuwono Date: Fri, 10 Jul 2026 13:19:29 +0700 Subject: [PATCH 26/27] fix(armhf): preserve pinned apt sources on runner VM The runner VM boots from a snapshot whose cloud-init state was cleared via "cloud-init clean --configs all" before snapshotting. On first boot cloud-init regenerates the stock apt sources, dropping the native-arch pin added by enable_armhf_multiarch, while the armhf dpkg architecture persists. apt-get update then requests binary-armhf indexes from the primary/security mirrors (which only serve arm64) and 404s, failing the runner's apt-get update. Bake an /etc/cloud/cloud.cfg.d drop-in with apt.preserve_sources_list: true so cloud-init keeps the pinned sources baked into the image. The drop-in lives outside /var/lib/cloud so it survives cloud-init clean. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../templates/cloud-init.sh.j2 | 9 +++++++++ app/tests/unit/test_openstack_builder.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 index 5e356256..5deb2a28 100644 --- a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 +++ b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 @@ -69,6 +69,15 @@ Suites: $codename $codename-updates $codename-security Components: main restricted universe multiverse Architectures: armhf Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +EOF + # The runner VM boots from a snapshot whose cloud-init state was cleared (see the + # "cloud-init clean" before snapshot), so cloud-init regenerates the stock apt sources on + # first boot and drops the architecture pin above while the armhf dpkg architecture persists, + # breaking "apt-get update". Tell cloud-init to preserve the pinned sources baked into the + # image so the 32-bit multiarch configuration survives on the runner VM. + /usr/bin/tee /etc/cloud/cloud.cfg.d/99-armhf-preserve-apt-sources.cfg >/dev/null </dev/null < Date: Fri, 10 Jul 2026 18:26:04 +0700 Subject: [PATCH 27/27] chore: bump copyright year to 2026 in test_integration_commands Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app/tests/unit/test_integration_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/unit/test_integration_commands.py b/app/tests/unit/test_integration_commands.py index 328f7464..32394868 100644 --- a/app/tests/unit/test_integration_commands.py +++ b/app/tests/unit/test_integration_commands.py @@ -1,4 +1,4 @@ -# Copyright 2025 Canonical Ltd. +# Copyright 2026 Canonical Ltd. # See LICENSE file for licensing details. """Unit tests for arch-conditional integration test command selection."""