diff --git a/.arg.template b/.arg.template index cc400181..1cf1a18c 100644 --- a/.arg.template +++ b/.arg.template @@ -53,3 +53,41 @@ FORCE_INTERACTIVE_INSTALL=false # AUTO_ENROLL_SECUREBOOT_KEYS=false # Set to true to automatically enroll certificates on devices in Setup Mode, useful for flashing devices without user interaction # DRBD_VERSION=9.2.13 # This variable is required for Piraeus pack for drbd module installtion. + +# NVIDIA GPU driver pre-install (for running the NVIDIA GPU Operator in air-gapped +# environments with driver.enabled=false). Bakes the datacenter driver + DKMS +# kernel modules into the Ubuntu base image so GPU nodes need no host-side network. +# See scripts/install-nvidia-drivers.sh for details. +# INSTALL_NVIDIA_GPU_DRIVERS=true +# NVIDIA_DRIVER_BRANCH=580 # Driver branch (check: apt-cache search 'nvidia-headless-.*-server') +# NVIDIA_DRIVER_TYPE=open # open | proprietary. Default "open" (REQUIRED on Hopper/Blackwell, +# safe on Turing/Ampere/Ada). Set "proprietary" only for pre-Turing GPUs. +# NVIDIA_USE_CUDA_REPO=true # Add NVIDIA CUDA network repo at build time (has every -server branch) +# NVIDIA_INSTALL_FABRICMANAGER=false # true for NVSwitch / HGX systems +# NVIDIA_INSTALL_CONTAINER_TOOLKIT=false # true to also pre-install nvidia-container-toolkit on host (then toolkit.enabled=false) +# NVIDIA_REBUILD_INITRD=true # Rebuild initrd so the nouveau blacklist applies at early boot + +# AMD Instinct GPU driver pre-install (for running the AMD GPU Operator in air-gapped +# environments with driver.enable=false). Mutually exclusive with the NVIDIA options +# above. See scripts/install-amdgpu-drivers.sh and docs/amd-gpu-airgapped.md for details. +# INSTALL_AMD_GPU_DRIVERS=true +# AMDGPU_DRIVER_SOURCE=dkms # dkms | inbox. "dkms" bakes AMD's amdgpu-dkms +# # built against the image kernel (recommended +# # for Instinct/MI silicon). "inbox" uses the +# # in-tree amdgpu module from linux-modules-* +# # and skips the AMD apt repo entirely — use +# # when the DKMS build fails against your image +# # kernel (see docs). +# AMDGPU_DRIVER_RELEASE=7.2.1 # Only used with dkms mode. amdgpu-install +# # release marker. Default 7.2.1 pairs with +# # GPU Operator v1.5.0 (ROCm 7.2.1). AMD also +# # publishes driver-release-marker paths like +# # 30.30.1 / 30.30.4 / 31.30; either form works. +# # 31.x is tech-preview -- do not mix with a +# # production operator. See the version- +# # alignment table in docs/amd-gpu-airgapped.md. +# AMDGPU_REBUILD_INITRD=false # Default false. amdgpu is intentionally +# # kept out of the initrd (multi-GPU init +# # emits enough udev events to time out +# # dracut-initqueue). It loads after +# # switch-root via modules-load.d. diff --git a/.earthlyignore b/.earthlyignore index c6fb21c9..486f781e 100644 --- a/.earthlyignore +++ b/.earthlyignore @@ -1,2 +1,6 @@ local/ -build/* \ No newline at end of file +build/* +# Whitelist AMD driver artifacts (produced by scripts/prebuild-amdgpu-artifact.sh +# and consumed by the base-image target via COPY). Without this exception, +# `build/*` above would hide the tarball from Earthly's build context. +!build/amdgpu-artifact-*.tar.gz \ No newline at end of file diff --git a/.github/workflows/base-images.yaml b/.github/workflows/base-images.yaml index 2af0979d..f4e0481d 100644 --- a/.github/workflows/base-images.yaml +++ b/.github/workflows/base-images.yaml @@ -2,11 +2,205 @@ name: Build Kairos Init Base Images on: workflow_dispatch: + inputs: + base_os_image: + description: "Base OS Image" + required: false + type: string + default: "" + kairos_init_image: + description: "Kairos Init Image (its version tags the built base images)" + required: false + type: string + default: "quay.io/kairos/kairos-init:v0.16.2" + arch: + description: "Architecture" + required: false + type: choice + options: + - amd64 + - arm64 + default: "amd64" + model: + description: "Model" + required: false + type: string + default: "generic" + kairos_version: + description: "Kairos Version (passed to kairos-init --version; not used in the image tag)" + required: false + type: string + default: "v4.1.2" + trusted_boot: + description: "Trusted Boot" + required: false + type: boolean + default: false + registry_prefix: + description: "Registry prefix for output images" + required: false + type: string + default: "us-east1-docker.pkg.dev/spectro-images/dev/pe-8787/edge" jobs: generate-matrix: - runs-on: ubuntu-latest + runs-on: Luet-BigRunner + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - name: Generate Matrix + - name: Generate build matrix + id: set-matrix run: | - echo "Hello" + python3 << 'EOF' + import json + import os + + base_os_image = "${{ github.event.inputs.base_os_image }}" + registry_prefix = "${{ github.event.inputs.registry_prefix }}" + arch = "${{ github.event.inputs.arch }}" + model = "${{ github.event.inputs.model }}" + kairos_init_image = "${{ github.event.inputs.kairos_init_image }}" + trusted_boot = "${{ github.event.inputs.trusted_boot }}" == "true" + + def kairos_init_version(image): + """Version component of the kairos-init image reference. + + Base images are tagged with the kairos-init version (not the + Kairos version) because kairos-init is what determines the + layout and contents of the produced image. Earthfile's + KAIROS_INIT_VERSION must match this. + """ + ref = image.rsplit('/', 1)[-1] + if '@' in ref: # digest form: kairos-init@sha256:abc... + return ref.split('@', 1)[1].replace(':', '-') + if ':' in ref: + return ref.split(':', 1)[1] + return 'latest' + + init_version = kairos_init_version(kairos_init_image) + print(f"Tagging with kairos-init version: {init_version}") + + matrix = [] + + if base_os_image: + # Custom base OS image + simple_name = base_os_image.split('/')[-1].replace(':', '-') + tag = f"{registry_prefix}/kairos-custom:{simple_name}-core-{arch}-{model}-{init_version}" + if trusted_boot: + tag += "-uki" + + matrix.append({ + "base_os": base_os_image, + "tag": tag, + }) + + elif trusted_boot: + # UKI builds - only Ubuntu 24.04 + tag = f"{registry_prefix}/kairos-ubuntu:24.04-core-{arch}-{model}-{init_version}-uki" + matrix.append({ + "base_os": "ubuntu:24.04", + "tag": tag, + }) + + else: + # Standard builds - all combinations + combinations = [ + ("ubuntu:20.04", "kairos-ubuntu:20.04-core"), + ("ubuntu:22.04", "kairos-ubuntu:22.04-core"), + ("ubuntu:24.04", "kairos-ubuntu:24.04-core"), + ("opensuse/leap:15.6", "kairos-opensuse:leap-15.6-core"), + ("registry.suse.com/suse/sle-micro-rancher/5.4:latest", "kairos-slem:5.4-core"), + ] + + for base_os, tag_prefix in combinations: + tag = f"{registry_prefix}/{tag_prefix}-{arch}-{model}-{init_version}" + matrix.append({ + "base_os": base_os, + "tag": tag, + }) + + matrix_json = json.dumps(matrix) + print(f"Generated matrix: {matrix_json}") + + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f"matrix={matrix_json}\n") + EOF + + kairosify: + needs: generate-matrix + runs-on: Luet-BigRunner + strategy: + matrix: + include: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to registry + run: echo "${{ secrets.US_EAST_JSON_KEY_B64 }}" | base64 -d | docker login -u _json_key --password-stdin us-east1-docker.pkg.dev + + - name: Build and push kairosify image + uses: docker/bake-action@v6 + with: + files: docker-bake-kairosify.hcl + targets: kairosify + push: true + set: | + kairosify.platform=linux/${{ github.event.inputs.arch }} + kairosify.args.BASE_OS_IMAGE=${{ matrix.base_os }} + kairosify.args.KAIROS_INIT_IMAGE=${{ github.event.inputs.kairos_init_image }} + kairosify.args.KAIROS_VERSION=${{ github.event.inputs.kairos_version }} + kairosify.args.TRUSTED_BOOT=${{ github.event.inputs.trusted_boot }} + kairosify.args.MODEL=${{ github.event.inputs.model }} + kairosify.tags=${{ matrix.tag }} + env: + DOCKER_BUILD_SUMMARY: false + + - name: Save build result + run: | + mkdir -p /tmp/results + echo "${{ matrix.tag }}" >> /tmp/results/built_tags.txt + + - name: Upload build results + uses: actions/upload-artifact@v4 + with: + name: build-result-${{ strategy.job-index }}-${{ hashFiles('**/matrix.*') }} + path: /tmp/results/built_tags.txt + + collect-outputs: + needs: kairosify + runs-on: Luet-BigRunner + outputs: + built_tags: ${{ steps.combine-tags.outputs.tags }} + steps: + - name: Download all build results + uses: actions/download-artifact@v4 + with: + pattern: build-result-* + path: /tmp/results + + - name: Combine all tags + id: combine-tags + run: | + ALL_TAGS=$(find /tmp/results -name "*.txt" -type f -exec cat {} \; | grep -v '^$' | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "tags=$ALL_TAGS" >> $GITHUB_OUTPUT + echo "All built tags: $ALL_TAGS" + + - name: Summary + run: | + echo "## Kairosify Build Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Input Parameters:**" >> $GITHUB_STEP_SUMMARY + echo "- Base OS Image: ${{ github.event.inputs.base_os_image || 'Default combinations' }}" >> $GITHUB_STEP_SUMMARY + echo "- Kairos Init Image: ${{ github.event.inputs.kairos_init_image }} (its version tags the built images)" >> $GITHUB_STEP_SUMMARY + echo "- Architecture: ${{ github.event.inputs.arch }}" >> $GITHUB_STEP_SUMMARY + echo "- Model: ${{ github.event.inputs.model }}" >> $GITHUB_STEP_SUMMARY + echo "- Kairos Version: ${{ github.event.inputs.kairos_version }} (passed to kairos-init --version)" >> $GITHUB_STEP_SUMMARY + echo "- Trusted Boot: ${{ github.event.inputs.trusted_boot }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Built Images:**" >> $GITHUB_STEP_SUMMARY + echo '${{ steps.combine-tags.outputs.tags }}' | jq -r '.[] | "- " + .' >> $GITHUB_STEP_SUMMARY diff --git a/Earthfile b/Earthfile index 560b2a34..a33ea7ce 100644 --- a/Earthfile +++ b/Earthfile @@ -21,19 +21,25 @@ ARG KAIROS_BASE_IMAGE_URL=$SPECTRO_PUB_REPO/edge # Spectro Cloud and Kairos tags. ARG PE_VERSION=v4.9.21 -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 +# Version component of the base image tags produced by .github/workflows/base-images.yaml. +# Those images are tagged with the kairos-init version, so this must track the +# kairos_init_image input of that workflow — NOT KAIROS_VERSION. +ARG KAIROS_INIT_VERSION=v0.16.2 ARG K3S_FLAVOR_TAG=k3s1 ARG RKE2_FLAVOR_TAG=rke2r1 ARG BASE_IMAGE_URL=quay.io/kairos ARG OSBUILDER_VERSION=v0.400.3 ARG OSBUILDER_IMAGE=quay.io/kairos/osbuilder-tools:$OSBUILDER_VERSION -ARG AURORABOOT_VERSION=v0.16.0 +# v0.18.0 is the minimum usable version. v0.16.0 and v0.17.0 do not work for the Hadron + +ARG AURORABOOT_VERSION=v0.18.0 ARG AURORABOOT_IMAGE=quay.io/kairos/auroraboot:$AURORABOOT_VERSION -ARG K3S_PROVIDER_VERSION=v4.9.4 -ARG KUBEADM_PROVIDER_VERSION=v4.9.8 -ARG RKE2_PROVIDER_VERSION=v4.9.3 +ARG K3S_PROVIDER_VERSION=v4.10.0 +ARG KUBEADM_PROVIDER_VERSION=v4.10.0 +ARG RKE2_PROVIDER_VERSION=v4.10.0 ARG NODEADM_PROVIDER_VERSION=v4.9.3 -ARG CANONICAL_PROVIDER_VERSION=v4.9.3 +ARG CANONICAL_PROVIDER_VERSION=v4.10.0 # Variables used in the builds. Update for ADVANCED use cases only. Modify in .arg file or via CLI arguments. ARG OS_DISTRIBUTION @@ -76,6 +82,52 @@ ARG no_proxy=${NO_PROXY} ARG UPDATE_KERNEL=false +# NVIDIA GPU driver pre-install (for air-gapped GPU Operator with driver.enabled=false). +# When true, the NVIDIA data-center driver + DKMS kernel modules are baked into the +# Ubuntu base image so GPU nodes need no host-side network at boot. +ARG INSTALL_NVIDIA_GPU_DRIVERS=false +ARG NVIDIA_DRIVER_BRANCH=580 +ARG NVIDIA_DRIVER_TYPE=open +ARG NVIDIA_USE_CUDA_REPO=true +ARG NVIDIA_INSTALL_FABRICMANAGER=true +ARG NVIDIA_INSTALL_IMEX=true +ARG NVIDIA_INSTALL_CONTAINER_TOOLKIT=false +ARG NVIDIA_REBUILD_INITRD=true + +# AMD Instinct GPU driver pre-install (for air-gapped AMD GPU Operator with +# driver.enable=false). See scripts/install-amdgpu-drivers.sh + docs/amd-gpu-airgapped.md. +ARG INSTALL_AMD_GPU_DRIVERS=false +# dkms | inbox. "dkms" builds AMD's amdgpu-dkms against the image kernel (default, +# recommended for Instinct silicon). "inbox" uses the in-tree amdgpu module shipped +# with linux-modules-* and skips the AMD apt repo — use only when the DKMS build +# fails against your image kernel and you accept the in-tree driver's feature set. +ARG AMDGPU_DRIVER_SOURCE=dkms +# amdgpu-install release marker (URL segment under repo.radeon.com/amdgpu-install//). +# Default 7.2.1 pairs with AMD GPU Operator v1.5.0 (per its release notes) and installs +# amdgpu-dkms 6.16.13 (30.30.1 line). Empirically builds cleanly against Linux kernels +# through 6.17 on Ubuntu 24.04. AMD publishes both ROCm-alias (7.2.1) and driver-release- +# marker (30.30.1, 31.30) URL segments; either form is accepted here. The 31.x line is +# tech-preview -- do not mix with a production operator. See docs/amd-gpu-airgapped.md. +ARG AMDGPU_DRIVER_RELEASE=7.2.1 +# Path to a driver artifact produced by scripts/prebuild-amdgpu-artifact.sh +# on the build host. Threaded in by earthly.sh when INSTALL_AMD_GPU_DRIVERS=true +# and AMDGPU_DRIVER_SOURCE=dkms. When set, the base-image AMD block skips the +# in-buildkit DKMS install (which fails in buildkit's RUN sandbox -- see docs) +# and simply extracts the pre-built modules + firmware + config drop-ins. +ARG AMDGPU_ARTIFACT_PATH="" +# Default false: amdgpu is intentionally omitted from the initrd (see +# scripts/install-amdgpu-drivers.sh -- multi-GPU amdgpu init emits enough +# udev events to blow past dracut-initqueue's udev-settle timeout, dropping +# the node into emergency mode). amdgpu loads after switch-root via +# /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. +ARG AMDGPU_REBUILD_INITRD=false + +# NVIDIA and AMD driver pre-install are mutually exclusive within a single image. +IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] && [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] + RUN echo "ERROR: INSTALL_NVIDIA_GPU_DRIVERS and INSTALL_AMD_GPU_DRIVERS are mutually exclusive. Enable only one." >&2 && \ + exit 1 +END + IF [ "$FIPS_ENABLED" = "true" ] && [ "$UPDATE_KERNEL" = "true" ] RUN echo "ERROR: UPDATE_KERNEL and FIPS_ENABLED are mutually exclusive. Cannot set both to true." >&2 && \ exit 1 @@ -96,6 +148,10 @@ ARG IS_UKI=false ARG INCLUDE_MS_SECUREBOOT_KEYS=true ARG AUTO_ENROLL_SECUREBOOT_KEYS=false ARG UKI_BRING_YOUR_OWN_KEYS=false +# When UKI_BRING_YOUR_OWN_KEYS=true, set false to skip merging Spectro extension cert into db +ARG ENROLL_SPECTRO_EXTENSION_CERT=true +# OCI image (scratch) with palette-sysext-cert.pem; merged into UEFI db during +uki-genkey +ARG SPECTRO_EXTENSION_CERT_IMAGE=us-east1-docker.pkg.dev/spectro-images/dev/arun/sysext/palette-sysext-cert:latest ARG CMDLINE="stylus.registration" ARG BRANDING="Palette eXtended Kubernetes Edge" @@ -109,30 +165,21 @@ ARG EFI_IMG_SIZE=2200 ARG GOLANG_VERSION=1.23 ARG DEBUG=false -# Pin UKI to Kairos v3.5.9: systemd 257.x dropped the boot-assessment -# suffix from sd-boot entry IDs, breaking `bootentry` selection and -# assessment fallback on newer builds (refs: kairos-io/kairos#3831, -# kairos-io/kairos#4046). v3.5.9 ships systemd 256.x where it still works. -IF [ "$IS_UKI" = "true" ] - LET KAIROS_VERSION=v3.5.9 -END - IF [ "$OS_DISTRIBUTION" = "ubuntu" ] && [ "$BASE_IMAGE" = "" ] IF [ "$OS_VERSION" == 22 ] || [ "$OS_VERSION" == 20 ] - ARG BASE_IMAGE_TAG=kairos-$OS_DISTRIBUTION:$OS_VERSION.04-core-$ARCH-generic-$KAIROS_VERSION + ARG BASE_IMAGE_TAG=kairos-$OS_DISTRIBUTION:$OS_VERSION.04-core-$ARCH-generic-$KAIROS_INIT_VERSION ELSE IF [ "$IS_UKI" = "true" ] - ARG BASE_IMAGE_TAG=kairos-$OS_DISTRIBUTION:$OS_VERSION-core-$ARCH-generic-$KAIROS_VERSION-uki + ARG BASE_IMAGE_TAG=kairos-$OS_DISTRIBUTION:$OS_VERSION-core-$ARCH-generic-$KAIROS_INIT_VERSION-uki ELSE - ARG BASE_IMAGE_TAG=kairos-$OS_DISTRIBUTION:$OS_VERSION-core-$ARCH-generic-$KAIROS_VERSION + ARG BASE_IMAGE_TAG=kairos-$OS_DISTRIBUTION:$OS_VERSION-core-$ARCH-generic-$KAIROS_INIT_VERSION END END ARG BASE_IMAGE=$KAIROS_BASE_IMAGE_URL/$BASE_IMAGE_TAG ELSE IF [ "$OS_DISTRIBUTION" = "opensuse-leap" ] && [ "$BASE_IMAGE" = "" ] - ARG BASE_IMAGE_TAG=kairos-opensuse:leap-$OS_VERSION-core-$ARCH-generic-$KAIROS_VERSION + ARG BASE_IMAGE_TAG=kairos-opensuse:leap-$OS_VERSION-core-$ARCH-generic-$KAIROS_INIT_VERSION ARG BASE_IMAGE=$KAIROS_BASE_IMAGE_URL/$BASE_IMAGE_TAG -ELSE IF [ "$OS_DISTRIBUTION" = "rhel" ] || [ "$OS_DISTRIBUTION" = "sles" ] - # Check for default value for rhel +ELSE ARG BASE_IMAGE END @@ -216,6 +263,41 @@ BASE_ALPINE: COPY --if-exists certs/ /etc/ssl/certs/ RUN update-ca-certificates + +# Probe $BASE_IMAGE for systemd >= 255; used only by +CHECK_SYSTEMD_VERSION. +systemd-extensions-support: + ARG ARCH + ARG BASE_IMAGE + FROM --platform=linux/${ARCH} $BASE_IMAGE + # Missing/failed systemctl detection must not fail the RUN under set -e; + # always emit supports-systemd-extensions as "true" or "false". + RUN SYSTEMCTL="" ; \ + for c in systemctl /usr/bin/systemctl /bin/systemctl /usr/local/bin/systemctl /usr/sbin/systemctl /sbin/systemctl /usr/local/sbin/systemctl; do \ + if command -v "$c" >/dev/null 2>&1; then SYSTEMCTL="$c"; break; fi; \ + done ; \ + SYSTEMD_VER=0 ; \ + if [ -n "$SYSTEMCTL" ]; then \ + SYSTEMD_VER=$("$SYSTEMCTL" --version 2>/dev/null | awk 'NR==1{print $2}') || SYSTEMD_VER=0 ; \ + fi ; \ + case "$SYSTEMD_VER" in ''|*[!0-9]*) SYSTEMD_VER=0 ;; esac ; \ + echo "CHECK_SYSTEMD_VERSION: detected systemd version $SYSTEMD_VER (systemctl: ${SYSTEMCTL:-not found})" ; \ + if [ "$SYSTEMD_VER" -ge 255 ]; then \ + echo true > /supports-systemd-extensions ; \ + echo "CHECK_SYSTEMD_VERSION: systemd >= 255 — supports-systemd-extensions=true"; \ + else \ + echo false > /supports-systemd-extensions ; \ + echo "CHECK_SYSTEMD_VERSION: systemd < 255 — supports-systemd-extensions=false"; \ + fi + SAVE ARTIFACT /supports-systemd-extensions + +# Loads true/false into the caller's build env at /tmp/supports-systemd-extensions. +CHECK_SYSTEMD_VERSION: + COMMAND + ARG ARCH + ARG BASE_IMAGE + COPY (+systemd-extensions-support/supports-systemd-extensions --ARCH=$ARCH --BASE_IMAGE=$BASE_IMAGE) /tmp/supports-systemd-extensions + RUN echo "SUPPORTS_SYSTEMD_EXTENSIONS=$(cat /tmp/supports-systemd-extensions)" + iso-image-rootfs: FROM --platform=linux/${ARCH} +iso-image SAVE ARTIFACT --keep-ts --keep-own /. rootfs @@ -237,7 +319,10 @@ uki-provider-image: COPY (+third-party/luet --binary=luet) /usr/bin/luet COPY +kairos-agent/kairos-agent /usr/bin/kairos-agent COPY --platform=linux/${ARCH} +trust-boot-unpack/ /trusted-boot - COPY --keep-ts --platform=linux/${ARCH} +install-k8s/output/ /k8s + DO +CHECK_SYSTEMD_VERSION --ARCH=$ARCH --BASE_IMAGE=$BASE_IMAGE + IF [ "$(cat /tmp/supports-systemd-extensions)" != "true" ] + COPY --keep-ts --platform=linux/${ARCH} +install-k8s/output/ /k8s + END COPY --if-exists "$EDGE_CUSTOM_CONFIG" /oem/.edge_custom_config.yaml COPY --if-exists +stylus-image/etc/kairos/80_stylus.yaml /etc/kairos/80_stylus.yaml SAVE IMAGE --push $IMAGE_PATH @@ -411,7 +496,29 @@ build-iso: rm -f /build/image/opt/spectrocloud/local-ui.tar; \ fi - IF [ "$ARCH" = "arm64" ] + # Hadron uses AuroraBoot instead of osbuilder's enki: enki writes the grub + # stage as EFI/BOOT/grub.efi, but Hadron's shim chainloads grubx64.efi, so an + # enki-built Hadron ISO does not boot on any UEFI firmware. AuroraBoot names + # the file after its source, giving grubx64.efi. The UKI ISO + # boots systemd-boot directly and has no shim->grub chain. + IF [ "$OS_DISTRIBUTION" = "hadron" ] + WITH DOCKER --pull $AURORABOOT_IMAGE + RUN mkdir -p /iso && \ + LOGLEVEL=info && \ + if [ "$DEBUG" = "true" ]; then LOGLEVEL=debug; fi && \ + docker run --rm --privileged \ + -v /build/image:/rootfs \ + -v /overlay:/overlay \ + -v /iso:/aurora \ + $AURORABOOT_IMAGE \ + build-iso \ + --loglevel "$LOGLEVEL" \ + --override-name "$ISO_NAME" \ + --overlay-iso /overlay \ + --output /aurora \ + dir:/rootfs + END + ELSE IF [ "$ARCH" = "arm64" ] RUN CMD="/entrypoint.sh --name $ISO_NAME build-iso --date=false --overlay-iso /overlay dir:/build/image --output /iso/ --arch $ARCH" && \ if [ "$DEBUG" = "true" ]; then CMD="$CMD --debug"; else CMD="$CMD"; fi && \ $CMD @@ -453,6 +560,16 @@ uki-genkey: RUN --no-cache mkdir -p /public-keys RUN --no-cache cd /keys; mv *.key tpm2-pcr-private.pem /private-keys RUN --no-cache cd /keys; mv *.pem /public-keys + # The osbuilder image (openSUSE Leap) does not ship efitools; install it so the + # ENROLL_SPECTRO_EXTENSION_CERT command can re-sign the db when enabled. + IF [ "$ENROLL_SPECTRO_EXTENSION_CERT" = "true" ] + RUN zypper --non-interactive install efitools + DO +ENROLL_SPECTRO_EXTENSION_CERT \ + --ARCH=$ARCH \ + --ENROLLMENT_DIR=/keys \ + --KEK_CERT=/public-keys/KEK.pem \ + --KEK_KEY=/private-keys/KEK.key + END ELSE COPY +uki-byok/ /keys END @@ -469,6 +586,46 @@ download-sbctl: RUN curl -Ls https://github.com/Foxboron/sbctl/releases/download/0.13/sbctl-0.13-linux-amd64.tar.gz | tar -xvzf - && mv sbctl/sbctl /usr/bin/sbctl SAVE ARTIFACT /usr/bin/sbctl +spectro-extension-cert: + ARG ARCH + FROM --platform=linux/${ARCH} $SPECTRO_EXTENSION_CERT_IMAGE + SAVE ARTIFACT /palette-sysext-cert.pem cert.pem + +spectro-extension-cert-esl: + ARG ARCH + FROM --platform=linux/${ARCH} $ALPINE_IMG + DO +BASE_ALPINE + RUN apk add --no-cache efitools + COPY (+spectro-extension-cert/cert.pem --ARCH=$ARCH) /cert/spectro-cert.pem + RUN cert-to-efi-sig-list -g 8be4df61-93ca-11d2-aa0d-00e098032b8c \ + /cert/spectro-cert.pem /cert/spectro-db.esl + SAVE ARTIFACT /cert/spectro-db.esl spectro-db.esl + +# Self-contained merge of the Spectro extension cert into the UEFI db enrollment +# material. Gated on ENROLL_SPECTRO_EXTENSION_CERT: when true, it fetches the ESL, +# appends it to db.esl and (re)generates db.auth/db.der so the db is fully ready; +# when false it is a no-op. +ENROLL_SPECTRO_EXTENSION_CERT: + COMMAND + ARG ARCH + ARG ENROLLMENT_DIR + ARG KEK_CERT + ARG KEK_KEY + # Append the Spectro extension cert to the db signature list and re-sign db.auth + # with the KEK so the resulting db enrolls both the OS db cert and the Spectro + # cert into UEFI firmware. db.der is kept as the OS cert (db-0.der) since the UKI + # itself is signed with the OS db key, not the Spectro cert. + # efitools (sign-efi-sig-list / sig-list-to-certs) must already be present in the + # caller's image: uki-genkey installs it via zypper, uki-byok via apt-get. + # With --arg-scope-and-set, CLI build-arg overrides (e.g. --MY_ORG / --EXPIRATION_IN_DAYS) + # cause COPY +target inside a COMMAND to see only the COMMAND's args — not .arg + # globals like ARCH. Forward ARCH explicitly. + COPY (+spectro-extension-cert-esl/spectro-db.esl --ARCH=$ARCH) /spectro/spectro-db.esl + RUN cat /spectro/spectro-db.esl >> "$ENROLLMENT_DIR/db.esl" && \ + sign-efi-sig-list -c "$KEK_CERT" -k "$KEK_KEY" db "$ENROLLMENT_DIR/db.esl" "$ENROLLMENT_DIR/db.auth" && \ + cd "$ENROLLMENT_DIR" && sig-list-to-certs 'db.esl' 'db' && \ + (cp db-0.der db.der 2>/dev/null || true) + uki-byok: FROM +ubuntu @@ -499,6 +656,14 @@ uki-byok: RUN [ -f /exported-keys/db ] && cat /exported-keys/db >> /output/db.esl || true RUN [ -f /exported-keys/dbx ] && cat /exported-keys/dbx >> /output/dbx.esl || true + IF [ "$ENROLL_SPECTRO_EXTENSION_CERT" = "true" ] + DO +ENROLL_SPECTRO_EXTENSION_CERT \ + --ARCH=$ARCH \ + --ENROLLMENT_DIR=/output \ + --KEK_CERT=/keys/KEK.pem \ + --KEK_KEY=/keys/KEK.key + END + WORKDIR /output RUN sign-efi-sig-list -c /keys/PK.pem -k /keys/PK.key PK PK.esl PK.auth RUN sign-efi-sig-list -c /keys/PK.pem -k /keys/PK.key KEK KEK.esl KEK.auth @@ -599,22 +764,37 @@ provider-image: RUN chmod 644 /etc/logrotate.d/stylus.conf END - COPY --platform=linux/${ARCH} +kairos-provider-image/ / + DO +CHECK_SYSTEMD_VERSION --ARCH=$ARCH --BASE_IMAGE=$BASE_IMAGE + IF [ "$(cat /tmp/supports-systemd-extensions)" != "true" ] + COPY --platform=linux/${ARCH} +kairos-provider-image/ / + # Newer kairos providers place agent-provider-* at /usr/local/system/providers/ + # instead of /system/providers/. Move to /system/providers/ and remove the new + # path so consumers always find the binary at the legacy location. + RUN if ls /usr/local/system/providers/agent-provider-* >/dev/null 2>&1; then \ + mkdir -p /system/providers && \ + mv /usr/local/system/providers/agent-provider-* /system/providers/ && \ + rm -rf /usr/local/system/providers; \ + fi + END COPY +stylus-image/etc/kairos/branding /etc/kairos/branding COPY --if-exists +stylus-image/etc/kairos/80_stylus.yaml /etc/kairos/80_stylus.yaml COPY +stylus-image/oem/stylus_config.yaml /etc/kairos/branding/stylus_config.yaml COPY +stylus-image/etc/elemental/config.yaml /etc/elemental/config.yaml COPY --if-exists "$EDGE_CUSTOM_CONFIG" /oem/.edge_custom_config.yaml - IF [ "$IS_UKI" = "true" ] - COPY +internal-slink/slink /usr/bin/slink - COPY --keep-ts +install-k8s/output/ /k8s - RUN slink --source /k8s/ --target /opt/k8s - RUN rm -f /usr/bin/slink - RUN rm -rf /k8s - RUN ln -sf /opt/spectrocloud/bin/agent-provider-stylus /usr/local/bin/agent-provider-stylus - ELSE - COPY --keep-ts +install-k8s/output/ / + # As part of PE-8315, kairos-provider binaries are in /usr/local/system/providers instead of earlier /system/providers. + # To avoid breaking existing functionality for non systemd extensions supported paths we move the binary back to original path. + IF [ "$(cat /tmp/supports-systemd-extensions)" != "true" ] + IF [ "$IS_UKI" = "true" ] + COPY +internal-slink/slink /usr/bin/slink + COPY --keep-ts +install-k8s/output/ /k8s + RUN slink --source /k8s/ --target /opt/k8s + RUN rm -f /usr/bin/slink + RUN rm -rf /k8s + RUN ln -sf /opt/spectrocloud/bin/agent-provider-stylus /usr/local/bin/agent-provider-stylus + ELSE + COPY --keep-ts +install-k8s/output/ / + END END RUN rm -f /etc/ssh/ssh_host_* /etc/ssh/moduli @@ -649,6 +829,7 @@ provider-image: DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl && \ install -d /usr/share/postgresql-common/pgdg && \ curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y lsb-release && \ echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-16 postgresql-contrib-16 iputils-ping @@ -738,8 +919,17 @@ base-image: COPY cloudconfigs/80_stylus_maas.yaml /system/oem/80_stylus_maas.yaml END + # Ensure the Renesas xHCI (USB 3.0) host controller driver is bundled into the + # initramfs so installation from USB media works on hardware using that chipset. + # Must run before the distro dracut regeneration below so the driver is included. + RUN mkdir -p /etc/dracut.conf.d && \ + printf '%s\n' 'hostonly="no"' 'add_drivers+=" xhci_pci_renesas "' 'force_drivers+=" xhci_pci_renesas "' > /etc/dracut.conf.d/99-usb-media.conf + # OS == Ubuntu IF [ "$OS_DISTRIBUTION" = "ubuntu" ] && [ "$ARCH" = "amd64" ] + RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends snapd kbd zstd vim iputils-ping bridge-utils curl tcpdump ethtool rsyslog logrotate libpam-pwquality -y + IF [ "$UBUNTU_PRO_ATTACH" = "true" ] # The token is mounted via Earthly's secret store as an env var # that lives only for the duration of this RUN. It is materialized @@ -751,7 +941,6 @@ base-image: # is not supplied, Earthly aborts before this RUN is invoked. RUN --secret UBUNTU_PRO_KEY \ sed -i '/^[[:space:]]*$/d' /etc/os-release && \ - apt-get update && apt-get install -y snapd && \ umask 077 && \ printf 'token: %s\n' "$UBUNTU_PRO_KEY" > /tmp/.pro-attach.yaml && \ unset UBUNTU_PRO_KEY && \ @@ -759,9 +948,6 @@ base-image: rm -f /tmp/.pro-attach.yaml END - RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends kbd zstd vim iputils-ping bridge-utils curl tcpdump ethtool rsyslog logrotate -y - LET APT_UPGRADE_FLAGS="-y" IF [ "$UPDATE_KERNEL" = "false" ] RUN if dpkg -l "linux-image-generic-hwe-$OS_VERSION" > /dev/null; then apt-mark hold "linux-image-generic-hwe-$OS_VERSION" "linux-headers-generic-hwe-$OS_VERSION" "linux-generic-hwe-$OS_VERSION" ; fi && \ @@ -823,6 +1009,62 @@ base-image: fi END + # NVIDIA GPU driver + DKMS kernel modules, built against the now-finalized + # image kernel. Runs here (not in the Dockerfile) so the kernel is settled + # first. Reuses install-kernel-headers.sh for ABI-exact headers. + IF [ "$INSTALL_NVIDIA_GPU_DRIVERS" = "true" ] + COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh + COPY scripts/install-nvidia-drivers.sh /tmp/install-nvidia-drivers.sh + RUN chmod 755 /tmp/install-kernel-headers.sh /tmp/install-nvidia-drivers.sh && \ + NVIDIA_DRIVER_BRANCH="$NVIDIA_DRIVER_BRANCH" \ + NVIDIA_DRIVER_TYPE="$NVIDIA_DRIVER_TYPE" \ + NVIDIA_USE_CUDA_REPO="$NVIDIA_USE_CUDA_REPO" \ + NVIDIA_INSTALL_FABRICMANAGER="$NVIDIA_INSTALL_FABRICMANAGER" \ + NVIDIA_INSTALL_IMEX="$NVIDIA_INSTALL_IMEX" \ + NVIDIA_INSTALL_CONTAINER_TOOLKIT="$NVIDIA_INSTALL_CONTAINER_TOOLKIT" \ + NVIDIA_REBUILD_INITRD="$NVIDIA_REBUILD_INITRD" \ + /tmp/install-nvidia-drivers.sh && \ + rm -f /tmp/install-nvidia-drivers.sh /tmp/install-kernel-headers.sh + END + + # AMD Instinct GPU driver (amdgpu-dkms) + kernel module, built against the + # now-finalized image kernel. Mutually exclusive with the NVIDIA block above. + IF [ "$INSTALL_AMD_GPU_DRIVERS" = "true" ] + # dkms mode with a pre-built artifact (default path when earthly.sh + # produced one via scripts/prebuild-amdgpu-artifact.sh). Buildkit's + # RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc probe -- + # see docs/amd-gpu-airgapped.md. The prebuild runs on the host in a + # plain `docker run --privileged` against the same base image, and + # this stage just extracts the resulting modules + firmware + drop-ins. + IF [ "$AMDGPU_DRIVER_SOURCE" = "dkms" ] && [ "$AMDGPU_ARTIFACT_PATH" != "" ] + COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh + COPY "$AMDGPU_ARTIFACT_PATH" /tmp/amdgpu-artifact.tar.gz + RUN --privileged \ + chmod 755 /tmp/install-amdgpu-drivers.sh && \ + AMDGPU_DRIVER_SOURCE=dkms \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ + AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + AMDGPU_ARTIFACT_PATH=/tmp/amdgpu-artifact.tar.gz \ + /tmp/install-amdgpu-drivers.sh && \ + rm -f /tmp/install-amdgpu-drivers.sh /tmp/amdgpu-artifact.tar.gz + ELSE + # inbox mode, OR dkms mode without a pre-built artifact (which + # will fail in buildkit's sandbox, but we let install-amdgpu- + # drivers.sh emit its own clear error rather than short-circuit + # here). install-kernel-headers.sh is only needed for the + # in-buildkit DKMS path; inbox mode doesn't use it. + COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh + COPY scripts/install-amdgpu-drivers.sh /tmp/install-amdgpu-drivers.sh + RUN --privileged \ + chmod 755 /tmp/install-kernel-headers.sh /tmp/install-amdgpu-drivers.sh && \ + AMDGPU_DRIVER_SOURCE="$AMDGPU_DRIVER_SOURCE" \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE" \ + AMDGPU_REBUILD_INITRD="$AMDGPU_REBUILD_INITRD" \ + /tmp/install-amdgpu-drivers.sh && \ + rm -f /tmp/install-amdgpu-drivers.sh /tmp/install-kernel-headers.sh + END + END + IF [ "$CIS_HARDENING" = "true" ] COPY cis-harden/harden.sh /tmp/harden.sh RUN /tmp/harden.sh && rm /tmp/harden.sh @@ -892,6 +1134,25 @@ base-image: RUN if ! grep -Fq "systemd.unified_cgroup_hierarchy=1" /etc/cos/bootargs.cfg; then \ sed -i 's|\(set baseCmd="[^"]*\)"|\1 systemd.unified_cgroup_hierarchy=1"|' /etc/cos/bootargs.cfg; \ fi + + # Block nouveau and qat_4xxx at the kernel command line on every + # build, and pin PCI BAR layout to firmware assignments. + # + # nouveau: modern NVIDIA data-center GPUs (Ada/Hopper/Blackwell) + # hang in GSP init when initramfs udev auto-loads nouveau before + # switchroot, stalling systemd-udev-settle indefinitely. Applied + # unconditionally — the image may be installed onto NVIDIA hardware + # even when INSTALL_NVIDIA_GPU_DRIVERS=false. rd.driver.blacklist= + # is the load-bearing flag (dracut honors it before udev fires); + # modprobe.blacklist= is belt-and-braces for post-switchroot. + # Harmless when no NVIDIA GPU is present. + # + # pci=realloc=off: firmware-assigned PCI resource layout is + # authoritative; kernel-side reallocation has caused BAR conflicts + # on some server platforms. Mirrors the installer ISO cmdline. + RUN if ! grep -Fq "rd.driver.blacklist=nouveau" /etc/cos/bootargs.cfg; then \ + sed -i 's|\(set baseCmd="[^"]*\)"|\1 rd.driver.blacklist=nouveau modprobe.blacklist=nouveau nouveau.modeset=0 pci=realloc=off"|' /etc/cos/bootargs.cfg; \ + fi END KAIROS_RELEASE: @@ -938,7 +1199,8 @@ iso-image: FROM --platform=linux/${ARCH} +base-image ARG IS_CLOUD_IMAGE=false ARG IMAGE_REGISTRY - + + DO +CHECK_SYSTEMD_VERSION --ARCH=$ARCH --BASE_IMAGE=$BASE_IMAGE IF [ "$IS_UKI" = "false" ] COPY --keep-ts --platform=linux/${ARCH} +stylus-image/ / @@ -948,6 +1210,17 @@ iso-image: RUN rm -f /usr/bin/luet END COPY overlay/files/ / + + # Workaround for kairos-agent v2.30.2 install-time mount-lifecycle bug on + # SLE Micro Rancher 5.5: /system/oem/08_grub.yaml's after-install "Grub + # branding" cp writes to a tmpfs path that is lost on reboot, so + # /grubmenu never lands on the persistent state partition and GRUB + # cannot find the Palette Registration menuentry. See + # slem/5.5/oem/09_grub_branding_fixup.yaml for full explanation. + IF [ "$OS_DISTRIBUTION" = "sles" ] && [ "$OS_VERSION" = "5.5" ] + COPY slem/5.5/oem/09_grub_branding_fixup.yaml /system/oem/09_grub_branding_fixup.yaml + END + IF [ "$IS_CLOUD_IMAGE" = "true" ] COPY cloud-images/workaround/grubmenu.cfg /etc/kairos/branding/grubmenu.cfg COPY cloud-images/workaround/custom-post-reset.yaml /system/oem/custom-post-reset.yaml @@ -984,6 +1257,16 @@ iso-image: fi END + + IF [ "$(cat /tmp/supports-systemd-extensions)" = "true" ] && \ + [ "$OS_DISTRIBUTION" = "ubuntu" ] && \ + [ "$ARCH" = "amd64" ] && [ "$IS_UKI" = "true" ] + COPY scripts/install-kernel-headers.sh /tmp/install-kernel-headers.sh + RUN chmod 755 /tmp/install-kernel-headers.sh + RUN /tmp/install-kernel-headers.sh + RUN rm -rf /tmp/install-kernel-headers.sh /var/lib/apt/lists/* /var/cache/apt/archives/*.deb + END + RUN rm -f /etc/ssh/ssh_host_* /etc/ssh/moduli RUN touch /etc/machine-id \ && chmod 444 /etc/machine-id diff --git a/docker-bake-kairosify.hcl b/docker-bake-kairosify.hcl new file mode 100644 index 00000000..18140b18 --- /dev/null +++ b/docker-bake-kairosify.hcl @@ -0,0 +1,41 @@ +variable "KAIROS_INIT_IMAGE" { + default = "quay.io/kairos/kairos-init:v0.16.2" +} + +variable "ARCH" { + default = "amd64" +} + +variable "BASE_OS_IMAGE" { + default = "ubuntu:20.04" +} + +variable "MODEL" { + default = "generic" +} + +variable "KAIROS_VERSION" { + default = "v4.1.2" +} + +variable "TRUSTED_BOOT" { + type = bool + default = false +} + +variable "TAG" { + default = "kairosify:latest" +} + +target "kairosify" { + dockerfile = "dockerfiles/kairosify/Dockerfile.kairosify" + platforms = ["linux/${ARCH}"] + args = { + BASE_OS_IMAGE = BASE_OS_IMAGE + KAIROS_INIT_IMAGE = KAIROS_INIT_IMAGE + KAIROS_VERSION = KAIROS_VERSION + TRUSTED_BOOT = TRUSTED_BOOT + MODEL = MODEL + } + tags = [TAG] +} \ No newline at end of file diff --git a/dockerfiles/kairosify/Dockerfile.kairosify b/dockerfiles/kairosify/Dockerfile.kairosify new file mode 100644 index 00000000..754dffc2 --- /dev/null +++ b/dockerfiles/kairosify/Dockerfile.kairosify @@ -0,0 +1,30 @@ +ARG KAIROS_INIT_IMAGE +ARG BASE_OS_IMAGE + +FROM ${KAIROS_INIT_IMAGE} AS kairos-init +FROM ${BASE_OS_IMAGE} AS baseos + +ARG MODEL +ARG KAIROS_VERSION +ARG TRUSTED_BOOT +ARG BASE_OS_IMAGE + +COPY --from=kairos-init /kairos-init /kairos-init +RUN /kairos-init -l debug -m "${MODEL}" -t "${TRUSTED_BOOT}" --version "${KAIROS_VERSION}" && rm /kairos-init + +# open-vm-tools: kairos-init installs the VMware guest agent for the SUSE family +# but not for Debian/Ubuntu, and the ubuntu-kairos-base Dockerfile that used to +# install it explicitly is no longer part of this build. Without vmtoolsd running, +# vSphere reports blank IP Addresses / DNS Name for the guest (PE-9173). +RUN if echo "${BASE_OS_IMAGE}" | grep -q ubuntu; then \ + apt-get update && apt-get install -y --no-install-recommends \ + dracut dracut-network isc-dhcp-common isc-dhcp-client cloud-guest-utils \ + open-vm-tools \ + $([ "$BASE_OS_IMAGE" != "ubuntu:20.04" ] && echo "dracut-live"); \ + systemctl enable open-vm-tools.service; \ + fi + +RUN if echo "${BASE_OS_IMAGE}" | grep -q opensuse; then \ + zypper refresh && zypper update -y && \ + zypper install -y dracut dhcp-client squashfs; \ + fi \ No newline at end of file diff --git a/docs/amd-gpu-airgapped.md b/docs/amd-gpu-airgapped.md new file mode 100644 index 00000000..4a3c48ae --- /dev/null +++ b/docs/amd-gpu-airgapped.md @@ -0,0 +1,312 @@ +# Pre-installing the AMD Instinct GPU driver for air-gapped GPU Operator + +This guide explains how to pre-provision the AMD **amdgpu** kernel-mode driver +**in a CanvOS Ubuntu base image**, so that AMD Instinct GPU nodes can run the +[AMD GPU Operator](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/specialized_networks/airgapped-install.html) +in a **fully air-gapped** environment — with **no host-side network access** and +**without the operator building/managing the driver**. + +It is the AMD counterpart of [`nvidia-gpu-airgapped.md`](./nvidia-gpu-airgapped.md) +and follows the same "pre-installed driver" model. + +- Script: [`scripts/install-amdgpu-drivers.sh`](../scripts/install-amdgpu-drivers.sh) +- Wired into the `base-image` target in the [`Earthfile`](../Earthfile), + gated by `INSTALL_AMD_GPU_DRIVERS=true`. + +**Supported targets:** Ubuntu **22.04** (jammy) and **24.04** (noble), `amd64`. +The codename is derived from the image at build time. + +## Two driver-source modes (`AMDGPU_DRIVER_SOURCE`) + +| Mode | What ships in the image | When to use | +| --- | --- | --- | +| `dkms` (default) | AMD's `amdgpu-dkms` source is DKMS-built against the image kernel and lands under `/lib/modules//updates/dkms/`. | Recommended for Instinct/MI silicon. The AMD out-of-tree driver typically carries newer SMU firmware interfaces and per-SKU support ahead of what the in-tree amdgpu has. | +| `inbox` | No AMD apt repo is added; the script only ensures the in-tree `amdgpu` module (shipped in `linux-modules-`) autoloads. | Fallback when the DKMS build fails against your image kernel — e.g. AMD hasn't yet published a driver release whose source builds against a very new kernel. Requires accepting the in-tree driver's feature set. | + +Both modes still require `driver.enable=false` at the Helm layer — the operator +does not build a driver either way. A marker at +`/etc/canvos/amdgpu-driver-source` on the booted node records which mode ran. + +> **Mutually exclusive with NVIDIA.** A single image supports one GPU vendor. +> Enabling both `INSTALL_AMD_GPU_DRIVERS` and `INSTALL_NVIDIA_GPU_DRIVERS` fails +> the build. + +--- + +## Split of responsibilities + +| Component | Where it lives | Who installs it | +| --- | --- | --- | +| amdgpu kernel module (`amdgpu`) + firmware | **In the OS image** | **This script (build time)** | +| ROCm user-space, device-plugin, node-labeller, metrics-exporter | Container images | AMD GPU Operator (from your content bundle) | + +The OS carries only the kernel driver; everything else is a container image you +mirror into your Palette content bundle. With `driver.enable=false` the operator +"directly uses inbox or pre-installed AMD GPU drivers" and only deploys the +device-plugin / node-labeller / metrics-exporter. + +At Helm-install time you **must** set: + +``` +--set driver.enable=false # note: "enable", not "enabled" +``` + +--- + +## Relationship to the AMD air-gapped guide + +The AMD guide's `driver.enable=true` path has the operator build the out-of-tree +driver at runtime, which needs build packages and (in restricted networks) a +local package mirror. This integration uses the opposite path +(`driver.enable=false`): the driver is either DKMS-built into the image at +build time (`AMDGPU_DRIVER_SOURCE=dkms`) or the in-tree amdgpu is used as-is +(`AMDGPU_DRIVER_SOURCE=inbox`). Either way, **no host-side mirror or network is +needed at boot**. + +--- + +## The key build-time problem this solves (dkms mode) + +Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, not +the kernel baked into the image. In `dkms` mode the driver must be compiled +against the image kernel's headers. + +We ran into a second problem too: Earthly's buildkit `RUN` sandbox breaks +AMD's `amdgpu-dkms` `./configure` heredoc probe with +`"cannot detect CFLAGS…"`, even though the same script + base image + host +succeed under plain `docker run --privileged`. Rather than debug buildkit +(some seccomp/apparmor/mount detail we don't control from the Earthfile), +`dkms` mode uses a **two-stage build**: + +1. **Prebuild on the host** (via `scripts/prebuild-amdgpu-artifact.sh`, + auto-invoked by `./earthly.sh`). Runs a plain `docker run --privileged` + against the same kairos base image, executes the DKMS install inside, + tars the resulting `/lib/modules//updates/dkms/` + firmware + + drop-ins into `build/amdgpu-artifact---.tar.gz`. + Cached by (release × base-image digest × kver); ~8–10 min the first + time, instant on cache hit. +2. **Consume in Earthly**: the `base-image` target `COPY`s the tarball and + extracts it, runs `depmod` against the image kernel, and rebuilds the + initrd. No `./configure`, no compile inside buildkit. + +Both stages use the same `scripts/install-amdgpu-drivers.sh` — the in-buildkit +step just takes the `AMDGPU_ARTIFACT_PATH` fast path. The result on-node is +identical to a native DKMS install. + +`inbox` mode skips both stages entirely and only ensures the in-tree amdgpu +autoloads. + +> **No blacklist needed.** Unlike NVIDIA (where `nouveau` must be blacklisted), +> the DKMS `amdgpu` module replaces the in-tree one via `depmod`'s `updates/` +> override. The script just autoloads `amdgpu`. + +> **When is the prebuild helper invoked?** `./earthly.sh` triggers it +> automatically when `INSTALL_AMD_GPU_DRIVERS=true` and +> `AMDGPU_DRIVER_SOURCE=dkms` (either from `.arg` or a CLI override). No +> extra command needed. It's skipped for `inbox`, or for any non-AMD build. + +--- + +## Quick start + +1. Edit `.arg` and enable the feature: + + ```sh + OS_DISTRIBUTION=ubuntu + OS_VERSION=22 # or 24 for Ubuntu 24.04 + ARCH=amd64 + + INSTALL_AMD_GPU_DRIVERS=true + AMDGPU_DRIVER_SOURCE=dkms # or "inbox" — see modes above + AMDGPU_DRIVER_RELEASE=7.2.1 # pairs with GPU Operator v1.5.0 (dkms mode only) + ``` + +2. Build as usual, e.g.: + + ```sh + ./earthly.sh +build-all-images --ARCH=amd64 + ``` + + or override on the command line: + + ```sh + ./earthly.sh +base-image --ARCH=amd64 \ + --INSTALL_AMD_GPU_DRIVERS=true \ + --AMDGPU_DRIVER_RELEASE=7.2.1 + ``` + + If the `dkms` build fails on your image kernel (see the mapping table + below), rebuild with `--AMDGPU_DRIVER_SOURCE=inbox` to fall back to the + in-tree driver instead. + +3. Mirror the AMD GPU Operator container images into your Palette content bundle + and install the operator with `driver.enable=false`. + +--- + +## Configuration reference + +| Variable | Default | Description | +| --- | --- | --- | +| `INSTALL_AMD_GPU_DRIVERS` | `false` | Master switch. Enables the AMD pre-install pipeline. | +| `AMDGPU_DRIVER_SOURCE` | `dkms` | `dkms` (build AMD's out-of-tree driver against the image kernel) or `inbox` (skip the AMD repo and use the in-tree amdgpu). See modes above. | +| `AMDGPU_DRIVER_RELEASE` | `7.2.1` | **`dkms` mode only.** `amdgpu-install` URL segment under `repo.radeon.com/amdgpu-install//`. AMD publishes both ROCm-alias paths (e.g. `7.2.1`, `7.2.4`) and driver-release-marker paths (e.g. `30.30.1`, `30.30.4`, `31.30`) — either form works. Default `7.2.1` pairs with **GPU Operator v1.5.0** (per AMD's release notes) → **ROCm 7.2.1** → **amdgpu-dkms 6.16.13** (30.30.1 build). | +| `AMDGPU_REBUILD_INITRD` | `true` | Rebuild the initrd for the image kernel. | + +### Version alignment across the stack (snapshot, 2026-07-11) + +Five things have to line up to have a supportable node. Start from the operator +version you bundle and follow AMD's release notes / compat matrix from there: + +``` + GPU Operator ─┐ AMD release notes pair the operator with a specific + │ ROCm user-space release + ROCm user-space (device-plugin / metrics-exporter / etc) + │ AMD user↔kernel compat matrix pairs ROCm with a + │ driver-release marker + amdgpu driver (amdgpu-dkms) → this is what this script installs + │ The DKMS source has a supported kernel window + Image kernel ─┘ + Kubernetes version — validated per operator release +``` + +**Two parallel driver tracks** — do not mix them: + +| Track | `AMDGPU_DRIVER_RELEASE` values | ROCm user-space | Paired GPU Operator | +| --- | --- | --- | --- | +| **Production** | `7.2.1` (= `30.30.1`), `7.2.4` (= `30.30.4`), etc. | ROCm 7.2.x | **v1.5.0 (what CanvOS bundles)** | +| Tech preview | `31.10` / `31.20` / `31.30` | ROCm 7.13.0 tech-preview | not yet paired with a released operator | + +Authoritative references: +- [AMD GPU Operator v1.5.0 release notes](https://instinct.docs.amd.com/projects/gpu-operator/en/main/releasenotes.html#gpu-operator-v1-5-0-release-notes) +- [ROCm user↔kernel compat matrix](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/user-kernel-space-compat-matrix.html) +- [ROCm on Linux system requirements](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/reference/system-requirements.html) +- Repo index: + +Snapshot of what `repo.radeon.com/amdgpu//ubuntu/dists/noble/…/Packages` +publishes for `amdgpu-dkms`: + +| `AMDGPU_DRIVER_RELEASE` | `amdgpu-dkms` build | Track | Paired with | +| --- | --- | --- | --- | +| **`7.2.1` (= `30.30.1`, default)** | `6.16.13-2303411` | Production | ROCm 7.2.1 → GPU Operator v1.5.0 | +| `7.2.4` (= `30.30.4`) | `6.16.13-2341068` | Production | ROCm 7.2.4 | +| `31.10` | `6.18.4` | Tech preview | ROCm 7.13.0 tech-preview | +| `31.30` | `6.19.4` | Tech preview | ROCm 7.13.0 tech-preview | + +The whole 30.30.x line uses the same driver *source* (`6.16.13`); only the build +number and paired user-space differ. Empirically, this source **builds cleanly +against Ubuntu 24.04's edge kernel 6.17** — the "EFI variables are not supported +on this system" line printed during postinst is a cosmetic mokutil warning +(sign_tool step); DKMS proceeds and lands the module under `updates/dkms/`. + +**How to pick:** default to the release marker paired with the operator you're +bundling. Bump only when you're also moving the operator to a paired version. +Do not switch to `31.x` for kernel-newness alone — that crosses into tech +preview and won't be validated with a production operator. + +### When the DKMS build fails + +Common causes: + +1. **Kernel outside the driver's supported window** — `make.log` shows + `configure: cannot detect CFLAGS` or unresolved kernel symbols. Prefer + moving the image kernel into range (or the operator/ROCm/driver combo up + as a set) over jumping to a tech-preview driver. +2. **`linux-headers-` not installed for the image kernel** — check the + earlier log lines from `install-kernel-headers.sh`. Fix the headers. +3. **DKMS module signing (mokutil) failure in the container** — surfaces as + "EFI variables are not supported on this system / /sys/firmware/efi/efivars + not found, aborting." The script writes + `/etc/dkms/framework.conf.d/canvos-no-mok-signing.conf` (empty `sign_tool`) + before the apt install to sidestep this. In 30.30.x the AMD postinst + already tolerates the missing EFI vars (it prints the warning and + continues); the sign_tool drop-in is defensive belt-and-suspenders for + 31.x and future releases that may treat it as fatal. + +The script prints the last 60 lines of `make.log` on failure. Read it — +the class of failure matters for the fix. Workaround for any of the above: +rerun with `AMDGPU_DRIVER_SOURCE=inbox` to use the in-tree amdgpu (accepts +the caveats above about missing driver-version label + SMU IF mismatch on +newer silicon). + +--- + +## Verify on a booted node + +```sh +lsmod | grep amdgpu +dmesg | grep -i amdgpu +ls /sys/class/kfd 2>/dev/null && echo "KFD present" +cat /etc/canvos/amdgpu-driver-source # which mode ran + release info +# In dkms mode, expect a module under /lib/modules//updates/dkms/ +find /lib/modules/$(uname -r)/updates -name 'amdgpu.ko*' 2>/dev/null +# If you also bundle ROCm user-space tooling: +# rocminfo ; amd-smi list +``` + +--- + +## Building the air-gapped content (which images to bundle) + +Pre-installing the driver in the OS removes the **driver-build** images (KMM & +friends). The operator still deploys the rest as containers, so those images — +plus cert-manager and a couple of non-image steps — must be handled. +**Bundling images alone is not sufficient.** + +### Images to mirror into your content bundle + +| Image | Needed with `driver.enable=false`? | +| --- | --- | +| `rocm/gpu-operator` (controller-manager) | Yes | +| `rocm/gpu-operator-utils` | Yes | +| `rocm/k8s-device-plugin` | Yes | +| `rocm/k8s-device-plugin:labeller-*` (node labeller) | Yes | +| `rocm/device-metrics-exporter` | Yes, if you want metrics | +| `rocm/device-config-manager` | Yes | +| `busybox:1.36` (init container) | Yes | +| `registry.k8s.io/nfd/node-feature-discovery` | Yes — unless the cluster already runs NFD | +| cert-manager (`controller`, `webhook`, `cainjector`, `acmesolver`) | Yes — hard dependency | +| KMM images (operator / webhook / worker / signimage) | **No — skip** | +| `gcr.io/kaniko-project/executor`, `ubuntu:` (driver build) | **No — skip** | +| `rocm/test-runner` | Optional (testing only) | + +Render the exact set from the chart rather than transcribing tags: + +```sh +helm template amd-gpu ./gpu-operator-.tgz -f operator-values.yaml \ + | grep -Eo 'image: *"?[^"]+' | sort -u +``` + +### Non-image steps + +1. **Install cert-manager first** (with its images pulled from your registry) — + the AMD operator will not start without it. +2. In the `DeviceConfig` CR, set `spec.driver.enable: false`. +3. Override every image (`controllerManager.manager.image`, + `commonConfig.initContainerImage`, `utilsContainer.image`, + `devicePlugin.devicePluginImage`, `devicePlugin.nodeLabellerImage`, + `metricsExporter.image`, `configManager.image`, and the NFD image) to your + bundle/registry; set `imagePullSecrets` as needed. +4. Ensure GPU nodes are labelled (via NFD or manually): + `feature.node.kubernetes.io/amd-gpu=true`. + +### Palette content bundle + +Add the AMD GPU Operator (and cert-manager) as Helm packs in the cluster profile, +then build the content bundle so it includes the rendered images above (minus the +KMM/kaniko/ubuntu build images). Images set only via `values.yaml` may need to be +added to the pack's additional-images list if the bundle builder doesn't +auto-detect them. Verify on a node with `lsmod | grep amdgpu` and by checking the +operator pods reach `Ready`. + +## Limitations / caveats + +- **Secure Boot / UKI is not supported by this path** (unsigned DKMS modules + won't load). Use the standard (non-UKI) Ubuntu image for GPU nodes. +- **amd64 / Ubuntu only.** +- **Version alignment is yours to own** — `AMDGPU_DRIVER_RELEASE` must line up + with the ROCm version of the operator images you bundle. +- **`inbox` mode loses the driver-version node label** — the AMD GPU Operator's + node-labeller reads `/sys/class/drm/card*/device/driver/module/version`, + which only exists when the driver was DKMS-installed. Enumeration and + scheduling still work; driver-version-aware policies won't. diff --git a/docs/nvidia-gpu-airgapped.md b/docs/nvidia-gpu-airgapped.md new file mode 100644 index 00000000..a802230f --- /dev/null +++ b/docs/nvidia-gpu-airgapped.md @@ -0,0 +1,291 @@ +# Pre-installing the NVIDIA GPU driver for air-gapped GPU Operator + +This guide explains how to bake the NVIDIA data-center GPU driver and its +kernel modules **into a CanvOS Ubuntu base image**, so that GPU nodes can run +the [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/install-gpu-operator-air-gapped.html) +in a **fully air-gapped** environment — with **no host-side network access** and +**without the operator's driver container**. + +- Script: [`scripts/install-nvidia-drivers.sh`](../scripts/install-nvidia-drivers.sh) +- Wired into the `base-image` target in the [`Earthfile`](../Earthfile), + gated by `INSTALL_NVIDIA_GPU_DRIVERS=true`. + +**Supported targets:** Ubuntu **22.04** and **24.04**, `amd64`. The script is +version-agnostic — it derives the CUDA repo tag (`ubuntu2204` / `ubuntu2404`) and +the kernel codename (`jammy` / `noble`) from the image's `/etc/os-release` at +build time, so the same script works for both without changes. + +> For AMD Instinct GPUs, see [`amd-gpu-airgapped.md`](./amd-gpu-airgapped.md). +> The two are **mutually exclusive** — enabling both `INSTALL_NVIDIA_GPU_DRIVERS` +> and `INSTALL_AMD_GPU_DRIVERS` fails the build. + +--- + +## Why do this (the split of responsibilities) + +The GPU Operator normally deploys the NVIDIA driver as a **driver container** that +downloads and compiles the driver at runtime. That requires network access on the +node and a matching kernel-headers source — neither of which exists on an +air-gapped, immutable (Kairos) edge host. + +The supported alternative is the **pre-installed driver** model: + +| Component | Where it lives | Who installs it | +| --- | --- | --- | +| NVIDIA kernel driver + modules (`nvidia`, `nvidia_uvm`, `nvidia_modeset`, `nvidia_drm`) | **In the OS image** | **This script (build time)** | +| `nvidia-smi` / driver user-space | **In the OS image** | **This script (build time)** | +| nvidia-container-toolkit / runtime class | Container image | GPU Operator (from your content bundle) | +| device-plugin, gpu-feature-discovery, DCGM exporter, MIG manager, validator | Container images | GPU Operator (from your content bundle) | + +So: **the OS carries only the driver + kernel modules**; everything else is a +container image you mirror into your Palette content bundle. At boot the node has a +working driver with zero connectivity, and once your bundled operator images are +present the GPU cluster comes up with no external pulls. + +At Helm-install time you **must** tell the operator the driver is pre-installed: + +``` +--set driver.enabled=false +``` + +If you also opt in to pre-installing the container toolkit on the host +(`NVIDIA_INSTALL_CONTAINER_TOOLKIT=true`, off by default), additionally set: + +``` +--set toolkit.enabled=false +``` + +--- + +## Relationship to NVIDIA's "Local Package Repository" section + +The NVIDIA air-gapped guide lists these Ubuntu packages under +**Local Package Repository → Required Packages**: + +``` +ubuntu: + linux-headers-${KERNEL_VERSION} + linux-image-${KERNEL_VERSION} + linux-modules-${KERNEL_VERSION} +``` + +That list belongs to the **driver-container** strategy: the node runs the GPU +Operator's *driver container*, which compiles the driver **at runtime** and pulls +those OS packages from **a local Ubuntu apt mirror you host**. It requires +`driver.enabled=true` plus a maintained mirror. + +This CanvOS integration deliberately uses the **other** supported strategy — +**pre-installed driver in the OS image** (`driver.enabled=false`) — so **no local +apt mirror is needed**. The substance of those three packages is still satisfied, +just at build time inside the image rather than from a runtime mirror: + +| NVIDIA-required package | How this integration satisfies it | +| --- | --- | +| `linux-headers-${KERNEL_VERSION}` | Installed at build time by `install-kernel-headers.sh` (ABI-exact; DKMS builds against these). | +| `linux-image-${KERNEL_VERSION}` | Already shipped in the Kairos base image (the bootable kernel). | +| `linux-modules-${KERNEL_VERSION}` | Already shipped in the Kairos base image (`/lib/modules/${KERNEL_VERSION}/`). | + +If you specifically want the driver-container + local-mirror model instead, this +script is not the right tool — you would host an apt mirror serving the packages +above and leave `driver.enabled=true`. + +## The key build-time problem this solves + +Inside the Earthly/Docker build, `uname -r` is the **builder host's** kernel, **not** +the kernel baked into the image. If DKMS builds "for the running kernel", you get +modules for the wrong ABI (or the build fails). The script therefore: + +1. derives the **target kernel** from `/lib/modules/*` (the kernel that will boot), +2. installs **ABI-exact kernel headers** for it (reusing + [`install-kernel-headers.sh`](../scripts/install-kernel-headers.sh), which falls + back to `snapshot.ubuntu.com` when Ubuntu rotates the ABI out of the live mirror), +3. forces **DKMS build + install + `depmod`** against that target kernel, and +4. **verifies** the resulting `nvidia*.ko` modules actually landed under + `/lib/modules//` — failing the build loudly if they did not. + +It runs in the `base-image` target **after** the kernel is finalized +(hold/upgrade/purge/dracut), so modules are always built against the settled kernel. + +> **Connectivity note:** the build runs where the builder has internet and bakes +> everything into the image. The resulting image needs no network at boot. + +--- + +## Quick start + +1. Edit `.arg` (copied from `.arg.template`) and enable the feature: + + ```sh + OS_DISTRIBUTION=ubuntu + OS_VERSION=22 # or 24 for Ubuntu 24.04 + ARCH=amd64 + + INSTALL_NVIDIA_GPU_DRIVERS=true + NVIDIA_DRIVER_BRANCH=580 # verify the branch exists (see below) + NVIDIA_DRIVER_TYPE=proprietary # or: open (Turing+ only) + ``` + +2. Build as usual, e.g.: + + ```sh + ./earthly.sh +build-all-images --ARCH=amd64 + ``` + + or override on the command line without touching `.arg`: + + ```sh + ./earthly.sh +base-image --ARCH=amd64 \ + --INSTALL_NVIDIA_GPU_DRIVERS=true \ + --NVIDIA_DRIVER_BRANCH=580 \ + --NVIDIA_DRIVER_TYPE=proprietary + ``` + +3. Mirror the GPU Operator container images into your Palette content bundle + (per the NVIDIA air-gapped guide), and install the operator with + `driver.enabled=false`. + +--- + +## Configuration reference + +All variables are optional and have defaults. Set them in `.arg` or pass as +`--VAR=value` on the `earthly.sh` command line. + +| Variable | Default | Description | +| --- | --- | --- | +| `INSTALL_NVIDIA_GPU_DRIVERS` | `false` | Master switch. When `true`, the driver + DKMS modules are baked into the Ubuntu base image. | +| `NVIDIA_DRIVER_BRANCH` | `580` | Driver **branch** to install (e.g. `550`, `570`, `580`). apt installs the latest patch within the branch — it is not pinned to an exact point release (e.g. `580.159.03`). Must be a real `-server` branch — see [Choosing a driver branch](#choosing-a-driver-branch). | +| `NVIDIA_DRIVER_TYPE` | `open` | `open` or `proprietary`. `open` uses the NVIDIA open GPU kernel modules and is **required** on Hopper (H100/H200) and Blackwell (RTX PRO 6000 Blackwell, B100/B200/GB200); the closed modules fail with `RmInitAdapter (0x22:0x56:897)` on those GPUs. Also safe on Turing/Ampere/Ada. Override to `proprietary` only for pre-Turing hardware (Pascal/Volta). See [Choosing the module flavor](#choosing-the-module-flavor-nvidia_driver_type). | +| `NVIDIA_USE_CUDA_REPO` | `true` | Add the NVIDIA CUDA network repo at build time. It carries every `-server` branch; recommended. `false` uses only Ubuntu's own repos. | +| `NVIDIA_INSTALL_FABRICMANAGER` | `true` | Installs `nvidia-fabricmanager-` + `libnvidia-nscq-` + `nvlsm` (NVIDIA Subnet Manager, from the CUDA repo) + `infiniband-diags` (for `ibstat`) and enables the unit — **required** on NVSwitch systems (HGX H100/H200, HGX B200, DGX, GB200 NVL72) for multi-GPU NVLink to come up. NVIDIA's 570+ shipped unit invokes a wrapper (`/usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh`) that probes `ibstat` + `nvlsm` before starting `nv-fabricmanager`; all four packages must be present or the unit fails before FM is ever invoked. On non-NVSwitch hosts FM exits `"No NvSwitch found"` and the unit stays inactive; no kernel side effect, no restart loop, ~70–110 MB image cost. Set `false` to skip if you want to shave the image and know none of your fleet uses NVSwitch. | +| `NVIDIA_INSTALL_IMEX` | `true` | Installs `nvidia-imex-` (Internode Memory Exchange daemon) and enables the unit — **required** on **GB200 NVL72** for multi-node NVLink Sharp (Blackwell, driver 570+). Not needed for single-node HGX B200 or HGX H100. On non-NVL72 hosts the daemon has no `/etc/nvidia-imex/nodes_config.cfg` and exits cleanly, so the unit stays inactive; ~10–20 MB image cost. Best-effort: older driver branches (pre-570) do not publish the package and the install step is skipped with a warning. | +| `NVIDIA_INSTALL_CONTAINER_TOOLKIT` | `false` | Set `true` to also pre-install `nvidia-container-toolkit` **on the host**. Then set `toolkit.enabled=false` in the operator. Off by default because the operator ships the toolkit. | +| `NVIDIA_REBUILD_INITRD` | `true` | Rebuild the initrd so the `nouveau` blacklist applies during early boot. | + +### Choosing a driver branch + +Only certain branches publish the headless `-server` packages. Inside the base +image (or any Ubuntu 22.04 box with the CUDA repo added) you can list them: + +```sh +apt-cache search 'nvidia-headless-.*-server' +``` + +Pick a branch supported by both your GPU generation and the CUDA/toolkit versions +of the operator images you're bundling. + +### Choosing the module flavor (`NVIDIA_DRIVER_TYPE`) + +The default is `open`. It works on every server GPU generation Turing and newer, +and is **required** for Hopper and Blackwell. Override to `proprietary` only for +pre-Turing hardware. + +| GPU generation | Example cards | Required `NVIDIA_DRIVER_TYPE` | +| -------------- | ------------------------------------------------------ | ----------------------------- | +| Blackwell | RTX PRO 6000 Blackwell, B100, B200, GB200 | `open` (only) | +| Hopper | H100, H200 | `open` (only) | +| Ada Lovelace | L4, L40, L40S, RTX 6000 Ada | either (`open` recommended) | +| Ampere | A100, A10, A30, A40 | either | +| Turing | T4, RTX 20xx | either | +| Pre-Turing | V100, P100, P40 | `proprietary` (only) | + +Symptom of the wrong choice on Hopper/Blackwell: `nvidia-smi` reports +`No devices were found`, and `dmesg` shows one line per GPU of the form +`NVRM: GPU : RmInitAdapter failed! (0x22:0x56:897)`. In that state the +GPU Operator's toolkit init container loops on +`Attempting to validate a driver container installation`, containerd never +registers the `nvidia` runtime handler, the device plugin never advertises +`nvidia.com/gpu`, and workload pods stay `Pending` on +`Insufficient nvidia.com/gpu`. + +--- + +## What the script configures on the host + +- `/etc/modprobe.d/blacklist-nouveau.conf` — blacklists the `nouveau` driver. +- `/etc/modules-load.d/nvidia.conf` — autoloads `nvidia`, `nvidia_uvm`, + `nvidia_modeset`, `nvidia_drm` at boot. +- `/etc/modprobe.d/nvidia.conf` — `NVreg_PreserveVideoMemoryAllocations=1`. +- Enables `nvidia-persistenced.service` (recommended for data-center GPUs). +- Runs `depmod -a ` and rebuilds the initrd for the target kernel. + +Verify on a booted node: + +```sh +nvidia-smi +lsmod | grep nvidia +``` + +--- + +## Building the air-gapped content (which images to bundle) + +Pre-installing the driver in the OS only removes the **driver image**. The +operator still deploys everything else as containers, so those images (and a few +non-image steps) must be handled. **Bundling images alone is not sufficient.** + +### Images to mirror into your content bundle + +| Image | Needed with `driver.enabled=false`? | +| --- | --- | +| `gpu-operator` | Yes | +| `gpu-operator-validator` | Yes | +| `container-toolkit` | Yes — **unless** you also pre-installed it on the host (`NVIDIA_INSTALL_CONTAINER_TOOLKIT=true` → then set `toolkit.enabled=false` and skip this image) | +| `k8s-device-plugin` | Yes | +| `gpu-feature-discovery` | Yes | +| `dcgm` + `dcgm-exporter` | Yes, if you want GPU metrics | +| `node-feature-discovery` | Yes — unless the cluster already runs NFD (`nfd.enabled=false`) | +| CUDA validation image (`nvcr.io/nvidia/cuda:…`) | Yes — used by the validator init container (easy to miss) | +| `k8s-mig-manager` | Only if using MIG | +| **`driver`** | **No — skip it (that's the point of pre-installing)** | + +Don't transcribe tags by hand — they change per operator version. Render the +exact set from the chart and mirror precisely that: + +```sh +helm template gpu-operator nvidia/gpu-operator --version \ + --set driver.enabled=false | grep -Eo 'image: *"?[^"]+' | sort -u +``` + +### Non-image steps + +1. `--set driver.enabled=false`. +2. Override **every** image `repository` to your bundle/registry and set + `imagePullSecrets`. +3. **Palette Edge (k3s / rke2) gotcha:** the container-toolkit defaults assume + stock containerd. On k3s/rke2 you must point it at the right socket and + config, e.g.: + + ``` + --set toolkit.env[0].name=CONTAINERD_CONFIG \ + --set toolkit.env[0].value=/var/lib/rancher/k3s/agent/etc/containerd/config.toml \ + --set toolkit.env[1].name=CONTAINERD_SOCKET \ + --set toolkit.env[1].value=/run/k3s/containerd/containerd.sock \ + --set toolkit.env[2].name=CONTAINERD_RUNTIME_CLASS \ + --set toolkit.env[2].value=nvidia + ``` + + (rke2 paths: `/var/lib/rancher/rke2/agent/etc/containerd/config.toml.tmpl`, + `/run/k3s/containerd/containerd.sock`.) Miss this and workloads never get the + GPU runtime even though the driver is present. + +### Palette content bundle + +Add the GPU Operator as a Helm pack in the cluster profile, then build the +content bundle so it includes the rendered images above (minus `driver`). Images +set only via `values.yaml` may need to be added to the pack's additional-images +list if the bundle builder doesn't auto-detect them. Verify on a node with +`nvidia-smi` and by checking the operator's `*-validator` pods reach `Ready`. + +## Limitations / caveats + +- **Secure Boot / UKI is not supported by this path.** When `IS_UKI=true`, DKMS + modules are unsigned and will not load under Secure Boot; that requires MOK + signing, which this script does **not** implement. Use the standard (non-UKI) + Ubuntu image for GPU nodes. +- **amd64 / Ubuntu only.** The script targets apt-based Ubuntu images on + `x86_64` (with a best-effort `sbsa` path for arm64). Non-Ubuntu distributions + are out of scope. +- **Branch/version alignment is yours to own.** Make sure `NVIDIA_DRIVER_BRANCH` + matches the GPU hardware and the CUDA/toolkit versions expected by the operator + images in your bundle. diff --git a/earthly.sh b/earthly.sh index 0601a04d..8e92dd7f 100755 --- a/earthly.sh +++ b/earthly.sh @@ -370,6 +370,96 @@ if [[ "$1" == "+maas-image" ]]; then exit 0 fi +# --------------------------------------------------------------------------- +# AMD GPU driver prebuild (dkms mode only). +# +# amdgpu-dkms's ./configure fails inside Earthly's buildkit RUN sandbox +# (see docs/amd-gpu-airgapped.md). We pre-compile the module in a plain +# `docker run --privileged` container here on the build host -- the exact +# environment we verified works end-to-end -- and pass the resulting tarball +# to Earthly for a simple COPY + tar-extract + depmod inside the base image. +# --------------------------------------------------------------------------- + +# Read a --FOO=bar override out of $@ without consuming it. Prints the value +# or empty; the arg is still passed through to earthly untouched. +peek_arg() { + local key="$1" + local a + for a in "$@"; do + case "$a" in + --${key}=*) printf '%s' "${a#--${key}=}"; return ;; + esac + done +} + +INSTALL_AMD_GPU_DRIVERS_EFFECTIVE="$(peek_arg INSTALL_AMD_GPU_DRIVERS "$@")" +INSTALL_AMD_GPU_DRIVERS_EFFECTIVE="${INSTALL_AMD_GPU_DRIVERS_EFFECTIVE:-${INSTALL_AMD_GPU_DRIVERS:-false}}" +AMDGPU_DRIVER_SOURCE_EFFECTIVE="$(peek_arg AMDGPU_DRIVER_SOURCE "$@")" +AMDGPU_DRIVER_SOURCE_EFFECTIVE="${AMDGPU_DRIVER_SOURCE_EFFECTIVE:-${AMDGPU_DRIVER_SOURCE:-dkms}}" + +if [ "$INSTALL_AMD_GPU_DRIVERS_EFFECTIVE" = "true" ] && [ "$AMDGPU_DRIVER_SOURCE_EFFECTIVE" = "dkms" ]; then + # Derive the same BASE_IMAGE the Earthfile derives, so we compile against + # the same rootfs Earthly is about to build on. Only Ubuntu is supported + # for AMD driver pre-install; the AMD mutual-exclusion + Ubuntu-only checks + # elsewhere handle other OS_DISTRIBUTIONs. + AMDGPU_BASE_IMAGE="$(peek_arg BASE_IMAGE "$@")" + AMDGPU_BASE_IMAGE="${AMDGPU_BASE_IMAGE:-${BASE_IMAGE:-}}" + if [ -z "$AMDGPU_BASE_IMAGE" ]; then + _os_dist="$(peek_arg OS_DISTRIBUTION "$@")"; _os_dist="${_os_dist:-${OS_DISTRIBUTION:-ubuntu}}" + _os_ver="$(peek_arg OS_VERSION "$@")"; _os_ver="${_os_ver:-${OS_VERSION:-24.04}}" + _arch="$(peek_arg ARCH "$@")"; _arch="${_arch:-${ARCH:-amd64}}" + _kairos_ver="$(peek_arg KAIROS_VERSION "$@")"; _kairos_ver="${_kairos_ver:-${KAIROS_VERSION:-v4.0.4}}" + _kairos_url="$(peek_arg KAIROS_BASE_IMAGE_URL "$@")"; _kairos_url="${_kairos_url:-${KAIROS_BASE_IMAGE_URL:-$SPECTRO_PUB_REPO/edge}}" + _is_uki="$(peek_arg IS_UKI "$@")"; _is_uki="${_is_uki:-${IS_UKI:-false}}" + + if [ "$_os_dist" != "ubuntu" ]; then + echo "AMD GPU driver pre-install requires OS_DISTRIBUTION=ubuntu (got: $_os_dist)." >&2 + exit 1 + fi + # Same tag formula as Earthfile lines ~141-151. + if [ "$_os_ver" = "22" ] || [ "$_os_ver" = "20" ]; then + _tag="kairos-${_os_dist}:${_os_ver}.04-core-${_arch}-generic-${_kairos_ver}" + elif [ "$_is_uki" = "true" ]; then + _tag="kairos-${_os_dist}:${_os_ver}-core-${_arch}-generic-${_kairos_ver}-uki" + else + _tag="kairos-${_os_dist}:${_os_ver}-core-${_arch}-generic-${_kairos_ver}" + fi + AMDGPU_BASE_IMAGE="${_kairos_url}/${_tag}" + fi + + AMDGPU_DRIVER_RELEASE_EFFECTIVE="$(peek_arg AMDGPU_DRIVER_RELEASE "$@")" + AMDGPU_DRIVER_RELEASE_EFFECTIVE="${AMDGPU_DRIVER_RELEASE_EFFECTIVE:-${AMDGPU_DRIVER_RELEASE:-7.2.1}}" + + echo "=== Pre-building AMD amdgpu driver (dkms mode) ===" + echo " BASE_IMAGE: $AMDGPU_BASE_IMAGE" + echo " AMDGPU_DRIVER_RELEASE: $AMDGPU_DRIVER_RELEASE_EFFECTIVE" + prebuild_out="$( + BASE_IMAGE="$AMDGPU_BASE_IMAGE" \ + AMDGPU_DRIVER_RELEASE="$AMDGPU_DRIVER_RELEASE_EFFECTIVE" \ + AMDGPU_ARTIFACT_DIR="$(pwd)/build" \ + bash scripts/prebuild-amdgpu-artifact.sh + )" || { echo "AMD driver pre-build failed. See lines above." >&2; exit 1; } + + # Last line of prebuild output is: AMDGPU_ARTIFACT_PATH= + AMDGPU_ARTIFACT_PATH="$(printf '%s\n' "$prebuild_out" | tail -1 | sed -n 's/^AMDGPU_ARTIFACT_PATH=//p')" + [ -s "$AMDGPU_ARTIFACT_PATH" ] || { echo "Prebuild did not emit AMDGPU_ARTIFACT_PATH; aborting." >&2; exit 1; } + echo " Artifact: $AMDGPU_ARTIFACT_PATH" + + # Earthly's COPY reads from the repo build-context (the directory containing + # the Earthfile), not from the host filesystem, so we must pass a path + # relative to the repo root -- not the absolute host path. + repo_root="$(pwd)" + case "$AMDGPU_ARTIFACT_PATH" in + "$repo_root"/*) AMDGPU_ARTIFACT_REL="${AMDGPU_ARTIFACT_PATH#$repo_root/}" ;; + *) echo "Prebuild artifact '$AMDGPU_ARTIFACT_PATH' is outside repo root '$repo_root'; \ +COPY into Earthly would fail. Move the artifact under the repo tree." >&2 ; exit 1 ;; + esac + + # Thread the (repo-relative) artifact path through to Earthly. Its Earthfile + # ARG (added in the companion commit) picks this up and consumes the tarball. + set -- "$@" "--AMDGPU_ARTIFACT_PATH=$AMDGPU_ARTIFACT_REL" +fi + # Normal build flow for other targets if [ -z "$HTTP_PROXY" ] && [ -z "$HTTPS_PROXY" ] && [ -z "$(find certs -type f ! -name '.*' -print -quit)" ]; then build_without_proxy "$@" diff --git a/hadron/Dockerfile b/hadron/Dockerfile new file mode 100644 index 00000000..0492bb9e --- /dev/null +++ b/hadron/Dockerfile @@ -0,0 +1,30 @@ +ARG HADRON_VERSION=v0.5.1 +ARG KAIROS_INIT_VERSION=v0.16.2 +ARG FIPS=false +ARG IS_UKI=false + +ARG MODULES_IMAGE=us-east1-docker.pkg.dev/spectro-images/dev/arun/hadron/modules:${HADRON_VERSION} + +FROM quay.io/kairos/kairos-init:${KAIROS_INIT_VERSION} AS kairos-init + +FROM ${MODULES_IMAGE} AS modules + +FROM ghcr.io/kairos-io/hadron:${HADRON_VERSION} AS base-fips-false-uki-false +FROM ghcr.io/kairos-io/hadron-fips:${HADRON_VERSION} AS base-fips-true-uki-false +FROM ghcr.io/kairos-io/hadron-trusted:${HADRON_VERSION} AS base-fips-false-uki-true +# base-fips-true-uki-true intentionally omitted: FIPS and UKI cannot be combined. + +FROM base-fips-${FIPS}-uki-${IS_UKI} AS base-kairos +ARG KAIROS_VERSION=v4.1.2 +ARG FIPS=false +ARG IS_UKI=false + +COPY --from=modules / / + +RUN --mount=type=bind,from=kairos-init,src=/kairos-init,dst=/kairos-init \ + FIPS_FLAG=$([ "${FIPS}" = "true" ] && echo "--fips" || true) && \ + /kairos-init -l debug -s install -m "generic" -t "${IS_UKI}" --version "${KAIROS_VERSION}" ${FIPS_FLAG} + +RUN --mount=type=bind,from=kairos-init,src=/kairos-init,dst=/kairos-init \ + FIPS_FLAG=$([ "${FIPS}" = "true" ] && echo "--fips" || true) && \ + /kairos-init -l debug -s init -m "generic" -t "${IS_UKI}" --version "${KAIROS_VERSION}" ${FIPS_FLAG} diff --git a/hadron/Dockerfile.modules b/hadron/Dockerfile.modules new file mode 100644 index 00000000..07b28f63 --- /dev/null +++ b/hadron/Dockerfile.modules @@ -0,0 +1,329 @@ +ARG HADRON_VERSION=v0.5.1 +ARG JQ_VERSION=1.8.1 +ARG DRBD_VERSION=9.3.3 +ARG RSYSLOG_VERSION=8.2606.0 +ARG LIBESTR_VERSION=0.1.11 +ARG LIBFASTJSON_VERSION=1.2304.0 +ARG LIBUCONTEXT_VERSION=1.5.2 +ARG LOGROTATE_VERSION=3.22.0 +ARG POPT_VERSION=1.19 + +# Every source artifact is pinned by SHA-256 so a rebuild either reproduces byte +# for byte or fails loudly. +# +# To bump a version: change the _VERSION arg, run the build, and take the digest +# from the sha256sum failure message. +ARG JQ_SHA256_AMD64=020468de7539ce70ef1bceaf7cde2e8c4f2ca6c3afb84642aabc5c97d9fc2a0d +ARG JQ_SHA256_ARM64=6bc62f25981328edd3cfcfe6fe51b073f2d7e7710d7ef7fcdac28d4e384fc3d4 +ARG LIBESTR_SHA256=46632b2785ff4a231dcf241eeb0dcb5fc0c7d4da8ee49cf5687722cdbe8b2024 +ARG LIBFASTJSON_SHA256=ef30d1e57a18ec770f90056aaac77300270c6203bbe476f4181cc83a2d5dc80c +ARG LIBUCONTEXT_SHA256=b7deadd8d3b9fdb66603e420a99fb12ca170654ee319c37a5fa8b100bf0420a5 +ARG RSYSLOG_SHA256=2574b3f3068e6955eb94ef5643e2b6a5b8585cc8eaa77209ff5cbc1e2e5f71e5 +ARG LOGROTATE_SHA256=93154424e73094d923a54de0d358007457282df7e14ee999a7c10d153e2c347e +ARG POPT_SHA256=c25a4838fc8e4c1c8aacb8bd620edb3084a3d63bf8987fdad3ca2758c63240f9 + +FROM ghcr.io/kairos-io/hadron-layers/drbd:${DRBD_VERSION} AS drbd-layer +FROM ghcr.io/kairos-io/hadron-cloud:${HADRON_VERSION} AS hadron-cloud + +######################################################## +# +# Downloads +# +# Every fetch gets its own stage, following the layout of the upstream Hadron +# Dockerfile. +# +######################################################## + +FROM ghcr.io/kairos-io/hadron-toolchain:${HADRON_VERSION} AS sources-base +RUN mkdir -p /downloads +WORKDIR /downloads + +# jq ships per-architecture release binaries rather than a tarball, so the digest +# is per-architecture too. +FROM sources-base AS jq-download +ARG JQ_VERSION +ARG JQ_SHA256_AMD64 +ARG JQ_SHA256_ARM64 +ARG TARGETARCH +RUN case "${TARGETARCH}" in \ + amd64) JQ_SHA256="${JQ_SHA256_AMD64}" ;; \ + arm64) JQ_SHA256="${JQ_SHA256_ARM64}" ;; \ + *) echo "no pinned jq digest for TARGETARCH=${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + curl -fsSL "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${TARGETARCH}" -o jq && \ + echo "${JQ_SHA256} jq" | sha256sum -c - && \ + chmod +x jq + +FROM sources-base AS libestr-download +ARG LIBESTR_VERSION +ARG LIBESTR_SHA256 +RUN curl -fsSL "https://libestr.adiscon.com/files/download/libestr-${LIBESTR_VERSION}.tar.gz" -o libestr.tar.gz && \ + echo "${LIBESTR_SHA256} libestr.tar.gz" | sha256sum -c - + +FROM sources-base AS libfastjson-download +ARG LIBFASTJSON_VERSION +ARG LIBFASTJSON_SHA256 +RUN curl -fsSL "https://download.rsyslog.com/libfastjson/libfastjson-${LIBFASTJSON_VERSION}.tar.gz" -o libfastjson.tar.gz && \ + echo "${LIBFASTJSON_SHA256} libfastjson.tar.gz" | sha256sum -c - + +# The tarball comes from the author's own distfiles server rather than GitHub's +# archive/refs/tags endpoint: that endpoint generates the tarball on demand, so +# its bytes are not guaranteed stable over time and a pinned digest there can +# start failing without anything upstream having changed. This is the same +# release tarball Alpine consumes. +FROM sources-base AS libucontext-download +ARG LIBUCONTEXT_VERSION +ARG LIBUCONTEXT_SHA256 +RUN curl -fsSL "https://distfiles.ariadne.space/libucontext/libucontext-${LIBUCONTEXT_VERSION}.tar.xz" -o libucontext.tar.xz && \ + echo "${LIBUCONTEXT_SHA256} libucontext.tar.xz" | sha256sum -c - + +FROM sources-base AS rsyslog-download +ARG RSYSLOG_VERSION +ARG RSYSLOG_SHA256 +RUN curl -fsSL "https://www.rsyslog.com/files/download/rsyslog/rsyslog-${RSYSLOG_VERSION}.tar.gz" -o rsyslog.tar.gz && \ + echo "${RSYSLOG_SHA256} rsyslog.tar.gz" | sha256sum -c - + +# The upstream git repo (github.com/rpm-software-management/popt) is deliberately +# not used: it publishes no release assets, and its tag archive ships no configure +# script -- regenerating it needs gettext's AM_GNU_GETTEXT / AM_ICONV_LINK m4 +# macros, which this toolchain does not have, so autoreconf fails outright. The +# release tarball's 17 hand-written source files are byte-identical to the tag. +FROM sources-base AS popt-download +ARG POPT_VERSION +ARG POPT_SHA256 +RUN curl -fsSL "http://ftp.rpm.org/popt/releases/popt-1.x/popt-${POPT_VERSION}.tar.gz" -o popt.tar.gz && \ + echo "${POPT_SHA256} popt.tar.gz" | sha256sum -c - + +FROM sources-base AS logrotate-download +ARG LOGROTATE_VERSION +ARG LOGROTATE_SHA256 +RUN curl -fsSL "https://github.com/logrotate/logrotate/releases/download/${LOGROTATE_VERSION}/logrotate-${LOGROTATE_VERSION}.tar.gz" -o logrotate.tar.gz && \ + echo "${LOGROTATE_SHA256} logrotate.tar.gz" | sha256sum -c - + +######################################################## +# +# Modules +# +# One stage per module, each staging a complete, self-contained tree under +# /output: binaries, libraries, config and unit symlinks. +# +######################################################## + +# jq is a prebuilt binary; it only needs to land in the right place. +FROM scratch AS jq +COPY --from=jq-download /downloads/jq /output/usr/bin/jq + +# Install rsyslog. +FROM ghcr.io/kairos-io/hadron-toolchain:${HADRON_VERSION} AS rsyslog +WORKDIR /sources/libestr +COPY --from=libestr-download /downloads/libestr.tar.gz . +RUN tar -xz --strip-components=1 -f libestr.tar.gz && \ + rm libestr.tar.gz && \ + ./configure ${COMMON_CONFIGURE_ARGS} && \ + make -s -j"$(nproc)" && \ + make -s install && \ + make -s install DESTDIR=/output + +WORKDIR /sources/libfastjson +COPY --from=libfastjson-download /downloads/libfastjson.tar.gz . +RUN tar -xz --strip-components=1 -f libfastjson.tar.gz && \ + rm libfastjson.tar.gz && \ + ./configure ${COMMON_CONFIGURE_ARGS} && \ + make -s -j"$(nproc)" && \ + make -s install && \ + make -s install DESTDIR=/output + +# libucontext provides the POSIX ucontext functions that musl libc omits. The +# toolchain's libsystemd.so was linked against it but does not ship it, so +# without this rsyslogd fails to link with undefined references to +# getcontext/makecontext/swapcontext. +ARG TARGETARCH +WORKDIR /sources/libucontext +COPY --from=libucontext-download /downloads/libucontext.tar.xz . +RUN tar -xJ --strip-components=1 -f libucontext.tar.xz && \ + rm libucontext.tar.xz && \ + case "${TARGETARCH}" in \ + amd64) ARCH=x86_64 ;; \ + arm64) ARCH=aarch64 ;; \ + *) ARCH="${TARGETARCH}" ;; \ + esac && \ + make ARCH="${ARCH}" -j"$(nproc)" && \ + make ARCH="${ARCH}" prefix=/usr install + +WORKDIR /sources/rsyslog +COPY patches/rsyslog-compat-queue-stailq-foreach.patch /patches/ +COPY --from=rsyslog-download /downloads/rsyslog.tar.gz . +# --enable-libsystemd is auto-detected, but is passed explicitly so the build +# fails loudly if libsystemd ever disappears from the toolchain: imuxsock only +# accepts the socket handed over by syslog.socket when compiled with it +# (HAVE_LIBSYSTEMD), and rsyslogd's sd_notify(READY=1) is what makes the +# Type=notify unit work. +# +# libyaml, libgcrypt and impstats-push are features rsyslog enables by default +# whose dependencies are absent from the toolchain. Disabling is preferred over +# building those dependencies because none of them is needed here: +# libyaml - backs only the alternative YAML config format; the classic +# format is what /etc/rsyslog.conf uses. +# libgcrypt - log file encryption (rscryutil). +# impstats-push - pushing rsyslog statistics to a remote endpoint, which +# additionally wants protobuf-c and snappy. +# klog is disabled because systemd-journald owns /dev/kmsg on Hadron. +RUN tar -xz --strip-components=1 -f rsyslog.tar.gz && \ + rm rsyslog.tar.gz && \ + patch -p1 < /patches/rsyslog-compat-queue-stailq-foreach.patch && \ + ./configure ${COMMON_CONFIGURE_ARGS} \ + --enable-libsystemd \ + --disable-klog \ + --disable-libyaml \ + --disable-libgcrypt \ + --disable-impstats-push && \ + make -s -j"$(nproc)" && \ + make -s install DESTDIR=/output + +# rsyslog upstream installs no config file and no unit, so both come from +# overlay/rsyslog/. The unit and the journald drop-in live under /usr/lib/systemd +# rather than /etc/systemd, which is a persistent bind mount on Kairos. +COPY overlay/rsyslog/ /output/ +# Enable rsyslog and make it the syslog implementation that syslog.socket +# activates. The .wants symlink goes under /usr/lib for the same reason as the +# unit itself; systemd reads target .wants directories from there too. +RUN mkdir -p /output/usr/lib/systemd/system/multi-user.target.wants && \ + ln -sf ../rsyslog.service \ + /output/usr/lib/systemd/system/multi-user.target.wants/rsyslog.service && \ + ln -sf rsyslog.service /output/usr/lib/systemd/system/syslog.service && \ + mkdir -p /output/etc/rsyslog.d +# Strip development artifacts so the exported layer only carries runtime files. +RUN find /output -name "*.h" -delete && \ + find /output -name "*.a" -delete && \ + find /output -name "*.la" -delete && \ + find /output -name "*.pc" -delete && \ + rm -rf /output/usr/include \ + /output/usr/share/man \ + /output/usr/share/doc \ + /output/usr/share/info \ + /output/usr/share/aclocal + +# Install logrotate. +# popt is a hard requirement (configure.ac: AC_CHECK_LIB([popt],...)) and is +# present neither in the toolchain nor in the Hadron base image, so it is built +# and shipped, installed twice like libestr/libfastjson above. + +# https://github.com/rpm-software-management/popt +FROM ghcr.io/kairos-io/hadron-toolchain:${HADRON_VERSION} AS logrotate +WORKDIR /sources/popt +COPY --from=popt-download /downloads/popt.tar.gz . +RUN tar -xz --strip-components=1 -f popt.tar.gz && \ + rm popt.tar.gz && \ + ./configure ${COMMON_CONFIGURE_ARGS} --disable-nls && \ + make -s -j"$(nproc)" && \ + make -s install && \ + make -s install DESTDIR=/output + +WORKDIR /sources/logrotate +COPY --from=logrotate-download /downloads/logrotate.tar.gz . +# --without-selinux because the toolchain has no libselinux; ACL support is kept +# because libacl is in both the toolchain and the Hadron base image. The default +# compress command is /bin/gzip, which on Hadron is a busybox applet and works. +# The uncompress default (/bin/gunzip) does not exist on Hadron, but it is only +# reachable via the "mail" directive, which needs a mail command the base image +# also lacks and which no CanvOS drop-in uses. +RUN tar -xz --strip-components=1 -f logrotate.tar.gz && \ + rm logrotate.tar.gz && \ + ./configure ${COMMON_CONFIGURE_ARGS} \ + --without-selinux \ + --with-acl && \ + make -s -j"$(nproc)" && \ + make -s install DESTDIR=/output + +# logrotate upstream installs no config file and no units either, so both come +# from overlay/logrotate/, under /usr/lib/systemd for the same reason as above. +COPY overlay/logrotate/ /output/ +# It is the *timer* that gets enabled: logrotate.service is a Type=oneshot job +# with no [Install] section, and wiring it into multi-user.target.wants would run +# it once at boot and never again. +RUN mkdir -p /output/usr/lib/systemd/system/timers.target.wants && \ + ln -sf ../logrotate.timer \ + /output/usr/lib/systemd/system/timers.target.wants/logrotate.timer && \ + mkdir -p /output/etc/logrotate.d +# Strip development artifacts so the exported layer only carries runtime files. +RUN find /output -name "*.h" -delete && \ + find /output -name "*.a" -delete && \ + find /output -name "*.la" -delete && \ + find /output -name "*.pc" -delete && \ + rm -rf /output/usr/include \ + /output/usr/share/man \ + /output/usr/share/doc \ + /output/usr/share/info \ + /output/usr/share/aclocal + +# Pinned to the same HADRON_VERSION as the base image, so glib and the agent +# stay in step with the libtirpc and musl the base ships. +FROM hadron-cloud AS open-vm-tools +# glib is a hard dependency (open-vm-tools needs glib2/gmodule/gobject/gthread) +# and is absent from the non-cloud base, as are the pcre2 and libffi that glib +# itself links. gio, girepository, the glib tools and the schemas are left +# behind: nothing open-vm-tools installs links them. +# +# libtirpc is *not* copied. Every open-vm-tools library needs it, but the +# non-cloud base already ships libtirpc.so.3 and this layer lands on top of +# that base, so copying ours would replace the base image's copy. +# +# The deployPkg plugin is dropped: it implements vSphere guest-customization +# specs, which is not how these nodes are provisioned, and hadron-cloud builds +# it against a libmspack.so.0 that no Hadron image actually ships -- vmtoolsd +# would only log a failure to load it. +RUN mkdir -p /output/usr/bin /output/usr/lib/udev/rules.d /output/etc && \ + cp -a /usr/bin/vm-support /usr/bin/vmtoolsd /usr/bin/vmware-* /output/usr/bin/ && \ + cp -a /usr/lib/libvmtools.so* \ + /usr/lib/libguestlib.so* \ + /usr/lib/libguestStoreClient.so* \ + /usr/lib/libhgfs.so* \ + /usr/lib/libglib-2.0.so.0* \ + /usr/lib/libgobject-2.0.so.0* \ + /usr/lib/libgmodule-2.0.so.0* \ + /usr/lib/libgthread-2.0.so.0* \ + /usr/lib/libpcre2-8.so.0* \ + /usr/lib/libffi.so.8* \ + /output/usr/lib/ && \ + cp -a /usr/lib/open-vm-tools /output/usr/lib/ && \ + rm -f /output/usr/lib/open-vm-tools/plugins/vmsvc/libdeployPkgPlugin.so && \ + cp -a /usr/lib/udev/rules.d/99-vmware-scsi-udev.rules /output/usr/lib/udev/rules.d/ && \ + cp -a /etc/vmware-tools /output/etc/ + +# open-vm-tools ships no unit of its own either, so it comes from +# overlay/open-vm-tools/. The unit is gated on ConditionVirtualization=vmware, so +# enabling it is harmless everywhere that is not VMware. +COPY overlay/open-vm-tools/ /output/ +RUN mkdir -p /output/usr/lib/systemd/system/multi-user.target.wants && \ + ln -sf ../open-vm-tools.service \ + /output/usr/lib/systemd/system/multi-user.target.wants/open-vm-tools.service + + + # ======================================================== +# qemu-guest-agent, the QEMU/KVM guest agent -- lets the hypervisor query the +# guest and request filesystem freeze/thaw, shutdown and reboot. Needed on +# KubeVirt/VMO, where without it the VM reports no guest info and graceful +# shutdown falls back to ACPI. +FROM hadron-cloud AS qemu-guest-agent +RUN mkdir -p /output/usr/bin /output/usr/lib /output/usr/lib/systemd/system && \ + cp -a /usr/bin/qemu-ga /output/usr/bin/ && \ + cp -a /usr/lib/libglib-2.0.so.0* \ + /usr/lib/libpcre2-8.so.0* \ + /output/usr/lib/ && \ + cp -a /usr/lib/systemd/system/qemu-guest-agent.service \ + /output/usr/lib/systemd/system/ + +COPY overlay/qemu-guest-agent/ /output/ + + +######################################################## +# Export +######################################################## + +FROM scratch AS export +COPY --from=jq /output / +COPY --from=rsyslog /output / +COPY --from=logrotate /output / +COPY --from=drbd-layer / / +COPY --from=open-vm-tools /output / +COPY --from=qemu-guest-agent /output / diff --git a/hadron/build.sh b/hadron/build.sh new file mode 100755 index 00000000..d60819c5 --- /dev/null +++ b/hadron/build.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +TARGET=hadron +FIPS=false +IS_UKI=false +OUTPUT=load +NO_CACHE=false +HADRON_VERSION="${HADRON_VERSION:-v0.5.1}" +KAIROS_VERSION="${KAIROS_VERSION:-v4.1.2}" +KAIROS_INIT_VERSION="${KAIROS_INIT_VERSION:-v0.16.2}" +SPECTRO_REPO="${SPECTRO_REPO:-us-east1-docker.pkg.dev/spectro-images/dev/arun}" +MODULES_IMAGE="" + +default_modules_image() { + echo "$SPECTRO_REPO/base/hadron-modules:${HADRON_VERSION}" +} + +hadron_image_tag() { + local variant="" + if [ "${FIPS}" = "true" ]; then + variant="-fips" + elif [ "${IS_UKI}" = "true" ]; then + variant="-uki" + fi + # Tagged with the kairos-init version, matching the base-images pipeline — + # kairos-init is what determines the layout and contents of the image. + echo "${SPECTRO_REPO}/base/hadron${variant}-${HADRON_VERSION}:${KAIROS_INIT_VERSION}" +} + +platforms() { + # Multi-arch on push, linux/amd64 on local load (BuildKit can't load multi-arch + # images into the local docker daemon). + if [ "${OUTPUT}" = "push" ]; then + echo "linux/amd64,linux/arm64" + else + echo "linux/amd64" + fi +} + +build_modules_image() { + docker buildx build \ + --progress=plain \ + --platform "$(platforms)" \ + "${CACHE_ARGS[@]}" \ + -f "${SCRIPT_DIR}/Dockerfile.modules" \ + -t "${MODULES_IMAGE}" \ + --build-arg HADRON_VERSION="${HADRON_VERSION}" \ + "--${OUTPUT}" \ + "${SCRIPT_DIR}" +} + +build_hadron_image() { + docker buildx build \ + --progress=plain \ + --platform "$(platforms)" \ + "${CACHE_ARGS[@]}" \ + --build-arg KAIROS_VERSION="${KAIROS_VERSION}" \ + --build-arg KAIROS_INIT_VERSION="${KAIROS_INIT_VERSION}" \ + --build-arg HADRON_VERSION="${HADRON_VERSION}" \ + --build-arg FIPS="${FIPS}" \ + --build-arg IS_UKI="${IS_UKI}" \ + --build-arg MODULES_IMAGE="${MODULES_IMAGE}" \ + -f "${SCRIPT_DIR}/Dockerfile" \ + -t "${HADRON_IMAGE}" \ + "--${OUTPUT}" \ + "${SCRIPT_DIR}" +} + + +validate() { + if ! command -v docker >/dev/null 2>&1; then + echo "Error: docker not found on PATH" >&2 + exit 1 + fi + + case "${TARGET}" in + modules|hadron) ;; + *) echo "Invalid target: ${TARGET} (expected modules or hadron)" >&2; usage 1 ;; + esac + + if [ "${FIPS}" = "true" ] && [ "${IS_UKI}" = "true" ]; then + echo "Error: --fips and --uki cannot be combined (UKI is not supported in FIPS mode)" >&2 + exit 1 + fi +} + + +usage() { + cat <<'EOF' +Usage: build.sh [OPTIONS] + +Build the Hadron base image (default) or the Spectro modules image. + +Options: + --target {hadron|modules} Image to build (default: hadron) + --fips Use the FIPS base image and enable FIPS mode + --uki Trusted boot. Not compatible with --fips. + --push Push the resulting image (multi-arch: + linux/amd64,linux/arm64). Default(--load). + --no-cache Pass --no-cache to docker buildx build + --modules-image TAG Override the modules image tag. Defaults to + ${SPECTRO_REPO}/base/hadron-modules:${HADRON_VERSION}. + -h, --help Show this help + +Environment (override defaults; CLI flags always win): + HADRON_VERSION Upstream Hadron version tag (default: v0.5.1) + KAIROS_VERSION Kairos version passed to kairos-init --version + (default: v4.1.2). Not used in the image tag. + KAIROS_INIT_VERSION kairos-init image tag. This is + also the tag of the built Hadron image. + + SPECTRO_REPO Registry + org prefix for all built images + (default: us-east1-docker.pkg.dev/spectro-images/dev/arun) + +Examples: + ./build.sh --fips --push # FIPS variant, pushed + ./build.sh --uki # UKI variant, loaded locally + ./build.sh --target modules --push # build & push modules only + HADRON_VERSION=v0.6.0 ./build.sh --push # override Hadron version via env + SPECTRO_REPO=myrepo.example.com/team ./build.sh --push # publish under a different registry/org +EOF + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --target) + [[ $# -gt 1 ]] || { echo "--target requires an argument" >&2; usage 1; } + TARGET="$2" + shift 2 + ;; + --fips) FIPS=true; shift ;; + --uki) IS_UKI=true; shift ;; + --push) OUTPUT=push; shift ;; + --no-cache) NO_CACHE=true; shift ;; + --modules-image) + [[ $# -gt 1 ]] || { echo "--modules-image requires an argument" >&2; usage 1; } + MODULES_IMAGE="$2"; shift 2 ;; + -h|--help) usage 0 ;; + *) echo "Unknown option: $1" >&2; usage 1 ;; + esac +done + + +validate + +CACHE_ARGS=() +if [ "${NO_CACHE}" = "true" ]; then + CACHE_ARGS=(--no-cache) +fi + +echo "Build configuration:" +echo " Target: ${TARGET}" +echo " FIPS: ${FIPS}" +echo " Trusted Boot (UKI): ${IS_UKI}" +echo " Hadron version: ${HADRON_VERSION}" +echo " Kairos version: ${KAIROS_VERSION}" +echo " kairos-init version: ${KAIROS_INIT_VERSION}" +echo " Output mode: ${OUTPUT}" +echo " No cache: ${NO_CACHE}" + +MODULES_IMAGE="${MODULES_IMAGE:-$(default_modules_image)}" +echo " Modules image: ${MODULES_IMAGE}" + +if [ "${TARGET}" = "modules" ]; then + build_modules_image + exit 0 +fi + +HADRON_IMAGE="$(hadron_image_tag)" +echo " Hadron image: ${HADRON_IMAGE}" + +build_hadron_image diff --git a/hadron/overlay/logrotate/etc/logrotate.conf b/hadron/overlay/logrotate/etc/logrotate.conf new file mode 100644 index 00000000..43c3c753 --- /dev/null +++ b/hadron/overlay/logrotate/etc/logrotate.conf @@ -0,0 +1,20 @@ +# Rotate weekly, keeping four generations. +weekly +rotate 4 + +# Create a fresh empty log after rotating. +create + +# Date-stamp rotated files rather than using .1/.2 suffixes. +dateext + +# Do not fail the whole run when a log listed in a drop-in does not exist yet. +missingok +notifempty + +# Compression is off globally; drop-ins that want it opt in. Note that the +# compress command is /bin/gzip, which on Hadron is a busybox applet. +#compress + +# Drop-ins, e.g. /etc/logrotate.d/stylus.conf. +include /etc/logrotate.d diff --git a/hadron/overlay/logrotate/usr/lib/systemd/system/logrotate.service b/hadron/overlay/logrotate/usr/lib/systemd/system/logrotate.service new file mode 100644 index 00000000..284d6391 --- /dev/null +++ b/hadron/overlay/logrotate/usr/lib/systemd/system/logrotate.service @@ -0,0 +1,45 @@ +# logrotate upstream ships this only as examples/logrotate.service (EXTRA_DIST, +# never installed), so it is maintained here. Copied from upstream 3.22.0 with +# no functional change. +# +# Lives under /usr/lib/systemd/system rather than /etc/systemd/system because +# /etc/systemd is a persistent bind mount on Kairos: a unit written to /etc at +# build time is shadowed once the node has state. +# +# There is deliberately no [Install] section -- this is a Type=oneshot job +# triggered by logrotate.timer, which is the unit that gets enabled. + +[Unit] +Description=Rotate log files +Documentation=man:logrotate(8) man:logrotate.conf(5) +RequiresMountsFor=/var/log +ConditionACPower=true + +[Service] +Type=oneshot +ExecStart=/usr/sbin/logrotate /etc/logrotate.conf + +# performance options +Nice=19 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 + +# hardening options +# details: https://www.freedesktop.org/software/systemd/man/systemd.exec.html +# no ProtectHome for userdir logs +# no PrivateNetwork for mail deliviery +# no NoNewPrivileges for third party rotate scripts +# no RestrictSUIDSGID for creating setgid directories +LockPersonality=true +MemoryDenyWriteExecute=true +PrivateDevices=true +PrivateTmp=true +ProtectClock=true +ProtectControlGroups=true +ProtectHostname=true +ProtectKernelLogs=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectSystem=full +RestrictNamespaces=true +RestrictRealtime=true diff --git a/hadron/overlay/logrotate/usr/lib/systemd/system/logrotate.timer b/hadron/overlay/logrotate/usr/lib/systemd/system/logrotate.timer new file mode 100644 index 00000000..ad151e3c --- /dev/null +++ b/hadron/overlay/logrotate/usr/lib/systemd/system/logrotate.timer @@ -0,0 +1,18 @@ +# Copied from upstream logrotate 3.22.0 examples/logrotate.timer with no +# functional change. This is the unit that gets enabled; logrotate.service is a +# Type=oneshot job with no [Install] section that this timer triggers. +# +# Persistent=true makes a missed run (node powered off at the scheduled time) +# fire on the next boot, which matters for edge hosts that are not always on. + +[Unit] +Description=Daily rotation of log files +Documentation=man:logrotate(8) man:logrotate.conf(5) + +[Timer] +OnCalendar=daily +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/hadron/overlay/open-vm-tools/usr/lib/systemd/system/open-vm-tools.service b/hadron/overlay/open-vm-tools/usr/lib/systemd/system/open-vm-tools.service new file mode 100644 index 00000000..42895f9f --- /dev/null +++ b/hadron/overlay/open-vm-tools/usr/lib/systemd/system/open-vm-tools.service @@ -0,0 +1,29 @@ +# open-vm-tools ships no unit file of its own -- every distribution writes its +# own in packaging -- so it is maintained here. Named open-vm-tools.service to +# match the unit the other CanvOS base images enable. +# +# Lives under /usr/lib/systemd/system rather than /etc/systemd/system because +# /etc/systemd is a persistent bind mount on Kairos: a unit written to /etc at +# build time is shadowed once the node has state. + +[Unit] +Description=Open VM Tools guest agent +Documentation=https://github.com/vmware/open-vm-tools +# These images boot on bare metal and KVM as well as vSphere. vmtoolsd has +# nothing to talk to elsewhere -- it exits immediately with "must be run inside +# a virtual machine" -- so gate the unit instead of letting it fail everywhere +# that is not VMware. +ConditionVirtualization=vmware +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/vmtoolsd +# The power-operation scripts under /etc/vmware-tools run guest shutdown and +# reboot on request from the host; give them room to finish. +TimeoutStopSec=30 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/hadron/overlay/qemu-guest-agent/usr/lib/udev/rules.d/99-qemu-guest-agent.rules b/hadron/overlay/qemu-guest-agent/usr/lib/udev/rules.d/99-qemu-guest-agent.rules new file mode 100644 index 00000000..2a36da0b --- /dev/null +++ b/hadron/overlay/qemu-guest-agent/usr/lib/udev/rules.d/99-qemu-guest-agent.rules @@ -0,0 +1,6 @@ +# Activate qemu-guest-agent when the host exposes the guest-agent channel. +# +# On bare metal and vSphere the port never appears, the +# rule never matches, and the agent stays dormant. +SUBSYSTEM=="virtio-ports", ATTR{name}=="org.qemu.guest_agent.0", \ + TAG+="systemd", ENV{SYSTEMD_WANTS}+="qemu-guest-agent.service" diff --git a/hadron/overlay/rsyslog/etc/rsyslog.conf b/hadron/overlay/rsyslog/etc/rsyslog.conf new file mode 100644 index 00000000..19443ef5 --- /dev/null +++ b/hadron/overlay/rsyslog/etc/rsyslog.conf @@ -0,0 +1,40 @@ +# Minimal rsyslog configuration for Hadron. +# +# rsyslog upstream ships no default configuration (distributions supply their +# own), so this file is maintained here. + + +#### MODULES #### + +# Local system logging. Under systemd the listening socket is passed in by +# syslog.socket; imuxsock detects /run/systemd/journal/syslog and uses the +# handed-over file descriptor instead of creating a socket of its own. +module(load="imuxsock") + +#### GLOBAL DIRECTIVES #### + +# Permissions for files and directories rsyslog creates. +$FileOwner root +$FileGroup root +$FileCreateMode 0640 +$DirCreateMode 0755 +$Umask 0022 + +# Spool and state directory. Created by the unit's StateDirectory=rsyslog so it +# survives the persistent /var bind mount on Kairos. +$WorkDirectory /var/lib/rsyslog + +#### RULES #### + +# Emergency messages to all users and to avoid Rsyslog error 1023 +*.emerg action(type="omusrmsg" users="*") + +# Drop-in configuration, e.g. /etc/rsyslog.d/49-stylus.conf. +# mode="optional" keeps rsyslog quiet when the directory holds no files. +include(file="/etc/rsyslog.d/*.conf" mode="optional") + +# Traditional file logging, disabled by default because journald already keeps +# these messages. Uncomment if you want plain-text log files as well. +#auth,authpriv.* action(type="omfile" file="/var/log/auth.log") +#*.*;auth,authpriv.none action(type="omfile" file="/var/log/syslog") +#kern.* action(type="omfile" file="/var/log/kern.log") diff --git a/hadron/overlay/rsyslog/usr/lib/systemd/journald.conf.d/00-forward-to-syslog.conf b/hadron/overlay/rsyslog/usr/lib/systemd/journald.conf.d/00-forward-to-syslog.conf new file mode 100644 index 00000000..80aabf1b --- /dev/null +++ b/hadron/overlay/rsyslog/usr/lib/systemd/journald.conf.d/00-forward-to-syslog.conf @@ -0,0 +1,8 @@ +# journald's ForwardToSyslog defaults to no, so without this drop-in rsyslog +# would start, own /run/systemd/journal/syslog and never receive a message. +# +# A drop-in under /usr/lib is used rather than editing /etc/systemd/journald.conf +# because /etc/systemd is a persistent bind mount on Kairos and a build-time +# edit there is shadowed once the node has state. +[Journal] +ForwardToSyslog=yes diff --git a/hadron/overlay/rsyslog/usr/lib/systemd/system/rsyslog.service b/hadron/overlay/rsyslog/usr/lib/systemd/system/rsyslog.service new file mode 100644 index 00000000..a9011f36 --- /dev/null +++ b/hadron/overlay/rsyslog/usr/lib/systemd/system/rsyslog.service @@ -0,0 +1,32 @@ + +[Unit] +Description=System Logging Service +Documentation=man:rsyslogd(8) +Documentation=https://www.rsyslog.com/doc/ +# syslog.socket owns /run/systemd/journal/syslog and hands the file descriptor +# to rsyslog, which is how journald forwards messages to it. +Requires=syslog.socket +After=syslog.socket + +[Service] +Type=notify +NotifyAccess=main +ExecStart=/usr/sbin/rsyslogd -n -iNONE +# SIGHUP makes rsyslog reopen its output files. logrotate's postrotate hook +# runs `systemctl try-reload-or-restart rsyslog`, which uses this instead of a +# full restart. +# +# Hadron ships NO kill binary (no /bin/kill, /usr/bin/kill or /sbin/kill). +# kill exists only as a shell builtin, so this has to go through /bin/sh. # The usual `ExecReload=/bin/kill -HUP $MAINPID` fails with 203/EXEC, +# and because logrotate's postrotate ends in `|| true` the failure is silent. +# rsyslog keeps writing to the rotated-and-deleted inode and every message after the first rotation is lost. +ExecReload=/bin/sh -c "kill -HUP $MAINPID" +# Creates and owns /var/lib/rsyslog ($WorkDirectory) at start-up, so it works +# regardless of the persistent /var bind mount. +StateDirectory=rsyslog +Restart=on-failure +StandardOutput=null + +[Install] +WantedBy=multi-user.target +Alias=syslog.service diff --git a/hadron/patches/rsyslog-compat-queue-stailq-foreach.patch b/hadron/patches/rsyslog-compat-queue-stailq-foreach.patch new file mode 100644 index 00000000..597772bf --- /dev/null +++ b/hadron/patches/rsyslog-compat-queue-stailq-foreach.patch @@ -0,0 +1,28 @@ +rsyslog 8.2606.0 added runtime/compat_queue.h so the build works on musl, which +ships no . That fallback is incomplete: runtime/dynstats.c uses +STAILQ_FOREACH, which the header does not define, so the build fails with + + dynstats.c:1333:5: error: implicit declaration of function 'STAILQ_FOREACH' + +Upstream added STAILQ_NEXT and STAILQ_FOREACH to compat_queue.h after the +8.2606.0 release. This patch is those two definitions, copied verbatim from +rsyslog main. Drop it once RSYSLOG_VERSION is bumped to a release that has them. + +--- a/runtime/compat_queue.h ++++ b/runtime/compat_queue.h +@@ -62,6 +62,15 @@ + } \ + } while (0) + #endif ++ ++ #ifndef STAILQ_NEXT ++ #define STAILQ_NEXT(elm, field) ((elm)->field.stqe_next) ++ #endif ++ ++ #ifndef STAILQ_FOREACH ++ #define STAILQ_FOREACH(var, head, field) \ ++ for ((var) = STAILQ_FIRST(head); (var); (var) = STAILQ_NEXT((var), field)) ++ #endif + #endif + + #endif /* #ifndef INCLUDED_COMPAT_QUEUE_H */ diff --git a/overlay/files-iso/boot/grub2/grub.cfg b/overlay/files-iso/boot/grub2/grub.cfg index 448421e7..a3bb3162 100644 --- a/overlay/files-iso/boot/grub2/grub.cfg +++ b/overlay/files-iso/boot/grub2/grub.cfg @@ -13,21 +13,21 @@ if [ -f ${font} ];then fi menuentry "Palette eXtended Kubernetes Edge Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette eXtended Kubernetes Edge Installer (manual)" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 echo Loading initrd... $initrd ($root)/boot/initrd } menuentry "Palette Edge Interactive Installer" --class os --unrestricted { echo Loading kernel... - $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 interactive-install + $linux ($root)/boot/kernel cdroot root=live:CDLABEL=COS_LIVE rd.live.dir=/ rd.live.squashimg=rootfs.squashfs net.ifnames=1 console=tty1 console=ttyS0 rd.cos.disable vga=795 nomodeset nodepair.enable selinux=0 rd.live.overlay.overlayfs rd.immucore.sysrootwait=600 systemd.unified_cgroup_hierarchy=1 pci=realloc=off rd.driver.blacklist=nouveau,qat_4xxx modprobe.blacklist=nouveau,qat_4xxx nouveau.modeset=0 interactive-install echo Loading initrd... $initrd ($root)/boot/initrd } diff --git a/rhel-core-images/Dockerfile.rhel10 b/rhel-core-images/Dockerfile.rhel10 new file mode 100644 index 00000000..d2ddc7ac --- /dev/null +++ b/rhel-core-images/Dockerfile.rhel10 @@ -0,0 +1,105 @@ +# syntax=docker/dockerfile:1 +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init +FROM registry.access.redhat.com/ubi10-init:10.2 + +ARG KAIROS_VERSION=v4.1.2 + +# EPEL: prefer mirror redirector; dl.fedoraproject.org often returns 503 under load. +# EPEL 10 is required here, not optional: systemd-networkd, systemd-timesyncd, +# livecd-tools and haveged are not shipped in RHEL 10 BaseOS/AppStream. +RUN dnf install -y 'https://download.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm' || \ + dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm' + +# Subscription manager in redhat does not run directly in containers unless you run on a redhat host, hence we remove the rhsm-host, login to the redhat subscription and add the repos +# +# No `subscription-manager attach --auto` here, unlike the RHEL 8/9 Dockerfiles: the `attach` +# module was removed in RHEL 10 (it is obsolete under Simple Content Access) and invoking it +# just prints the usage text and exits non-zero. Registration alone is enough — it leaves +# rhel-10-for-x86_64-baseos-rpms and -appstream-rpms already enabled. +RUN --mount=type=secret,id=RHSM_USERNAME,env=RHSM_USERNAME \ + --mount=type=secret,id=RHSM_PASSWORD,env=RHSM_PASSWORD \ + : "${RHSM_USERNAME:?missing --secret id=RHSM_USERNAME,env=RHSM_USERNAME}" \ + && : "${RHSM_PASSWORD:?missing --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD}" \ + && rm /etc/rhsm-host \ + && subscription-manager register --username "${RHSM_USERNAME}" --password "${RHSM_PASSWORD}" \ + && yum repolist \ + && subscription-manager repos --enable rhel-10-for-x86_64-appstream-rpms \ + && yum repolist \ + && dnf clean all + +RUN --mount=type=bind,from=kairos-init,src=/kairos-init,dst=/kairos-init /kairos-init -l debug --version "${KAIROS_VERSION}" -m "generic" -t "false" + +RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf + +# Generate machine-id because https://bugzilla.redhat.com/show_bug.cgi?id=1737355#c6 +# +# Notable RHEL 10 differences: +# - dhclient: dropped, ISC dhcp is gone from RHEL 10. DHCP is handled by systemd-networkd. +# - grub2, iptables and conntrack no longer exist as package names; kairos-init requests +# grub2 and iptables and they resolve via virtual provides (grub2-pc, iptables-nft). +# - systemd-networkd, systemd-timesyncd, livecd-tools and haveged are the EPEL-sourced +# packages RHEL 10 itself does not ship. +RUN uuidgen > /etc/machine-id && dnf install -y \ + livecd-tools \ + efibootmgr \ + systemd-networkd \ + systemd-timesyncd \ + haveged \ + ncurses \ + open-vm-tools \ + iscsi-initiator-utils \ + iproute-tc conntrack-tools ethtool socat \ + parted \ + kbd \ + coreutils-single && dnf clean all + +RUN sed -i 's/\bsource\b/./g' /system/oem/00_rootfs.yaml +RUN sed -i 's/\bsource\b/./g' /system/oem/09_openrc_services.yaml +RUN sed -i 's/\bsource\b/./g' /system/oem/50_recovery.yaml + +RUN mkdir -p /run/lock +RUN touch /usr/libexec/.keep + +# Configure the box. The ubi image masks services for containers, we unmask them +RUN systemctl list-unit-files |grep masked |cut -f 1 -d " " | xargs systemctl unmask +RUN systemctl enable getty@tty1.service +RUN systemctl enable getty@tty2.service +RUN systemctl enable getty@tty3.service +RUN systemctl enable systemd-networkd +RUN systemctl enable systemd-resolved +RUN systemctl enable sshd +RUN systemctl disable selinux-autorelabel-mark.service || true + +# Avoid clashes with systemd-networkd (Kairos/RHEL guidance). +# +# RHEL 9 uninstalls NetworkManager for this. That is not possible on RHEL 10: dracut-network +# requires "NetworkManager >= 1.20" unconditionally, whereas RHEL 9 requires +# "(NetworkManager >= 1.20 or dhclient)" and is satisfied by dhclient. Since dhclient is gone +# from RHEL 10, `dnf remove NetworkManager` would also drag out dracut-network and leave the +# final initramfs with no network module at all. RHEL 10 also dropped dracut's dhclient-based +# network-legacy module, so the only initramfs network paths left are 35network-manager and +# 01systemd-networkd. Mask NetworkManager instead of removing it: the package stays to satisfy +# dracut-network and to give dracut a usable initramfs network module, while systemd-networkd +# owns the network on the real root. Masking NetworkManager.service does not affect the +# initramfs, which runs its own nm-initrd.service. +# +# This must run *after* the unmask-everything step above, otherwise that step unmasks it again. +RUN if rpm -q NetworkManager >/dev/null 2>&1; then \ + systemctl disable NetworkManager.service NetworkManager-wait-online.service NetworkManager-dispatcher.service 2>/dev/null || true; \ + systemctl mask NetworkManager.service NetworkManager-wait-online.service NetworkManager-dispatcher.service; \ + fi + +COPY overlay/rhel10/ / + +RUN kernel=$(ls /boot/vmlinuz-* | head -n1) && \ + ln -sf "${kernel#/boot/}" /boot/vmlinuz +RUN kernel=$(ls /lib/modules | head -n1) && \ + dracut -v -N -f "/boot/initrd-${kernel}" "${kernel}" && \ + ln -sf "initrd-${kernel}" /boot/initrd && depmod -a "${kernel}" +RUN rm -rf /boot/initramfs-* + +RUN mkdir -p /etc/luet/repos.conf.d +## Clear cache +RUN rm -rf /var/cache/* && journalctl --vacuum-size=1K && rm /etc/machine-id + +RUN subscription-manager unregister diff --git a/rhel-core-images/Dockerfile.rhel10.sat b/rhel-core-images/Dockerfile.rhel10.sat new file mode 100644 index 00000000..9f9f7276 --- /dev/null +++ b/rhel-core-images/Dockerfile.rhel10.sat @@ -0,0 +1,100 @@ +# syntax=docker/dockerfile:1 +ARG BASE_IMAGE=registry.access.redhat.com/ubi10-init:10.2 +ARG KAIROS_INIT_IMAGE=quay.io/kairos/kairos-init:v0.16.2 + +FROM $KAIROS_INIT_IMAGE AS kairos-init +FROM $BASE_IMAGE + +ARG ORGNAME +ARG SATHOSTNAME +ARG KAIROS_VERSION=v4.1.2 + +# Unlike Dockerfile.rhel10 there is no EPEL install here — EPEL 10 must already be synced on +# the Satellite and attached to the activation key. It is mandatory, not optional: RHEL 10 +# ships none of systemd-networkd, systemd-timesyncd, livecd-tools or haveged, so a Satellite +# without an EPEL 10 repo fails later in the big dnf install with unresolvable packages. +# +# Disable the UBI repos baked into the base image so everything resolves from Satellite. +# These are the repo IDs ubi10-init actually reports (they are not named like the ubi-9-* ones). +RUN dnf config-manager --disable \ + ubi-10-for-x86_64-baseos-rpms \ + ubi-10-for-x86_64-appstream-rpms \ + codeready-builder-for-ubi-10-x86_64-rpms + +# Subscription manager in redhat does not run directly in containers unless you run on a redhat host, hence we remove the rhsm-host and register against Satellite +# +# The activation key is a secret and is passed as a BuildKit secret sourced from an +# environment variable, so it never becomes a build arg and never lands in an image layer or +# in `docker history`. ORGNAME and SATHOSTNAME are not secret and stay as build args: +RUN rm /etc/rhsm-host +RUN rpm -Uvh http://${SATHOSTNAME}/pub/katello-ca-consumer-latest.noarch.rpm +RUN --mount=type=secret,id=KEYNAME,env=KEYNAME \ + : "${KEYNAME:?missing --secret id=KEYNAME,env=KEYNAME}" \ + && : "${ORGNAME:?missing --build-arg ORGNAME}" \ + && subscription-manager register --org="${ORGNAME}" --activationkey="${KEYNAME}" \ + && yum repolist \ + && dnf clean all + +RUN --mount=type=bind,from=kairos-init,src=/kairos-init,dst=/kairos-init /kairos-init -l debug --version "${KAIROS_VERSION}" -m "generic" -t "false" + +RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf + +# Generate machine-id because https://bugzilla.redhat.com/show_bug.cgi?id=1737355#c6 +RUN uuidgen > /etc/machine-id && dnf install -y \ + livecd-tools \ + efibootmgr \ + systemd-networkd \ + systemd-timesyncd \ + haveged \ + ncurses \ + open-vm-tools \ + iscsi-initiator-utils \ + iproute-tc conntrack-tools ethtool socat \ + parted \ + kbd \ + coreutils-single && dnf clean all + +RUN sed -i 's/\bsource\b/./g' /system/oem/00_rootfs.yaml +RUN sed -i 's/\bsource\b/./g' /system/oem/09_openrc_services.yaml +RUN sed -i 's/\bsource\b/./g' /system/oem/50_recovery.yaml + +RUN mkdir -p /run/lock +RUN touch /usr/libexec/.keep + +# Configure the box. The ubi image masks services for containers, we unmask them +RUN systemctl list-unit-files |grep masked |cut -f 1 -d " " | xargs systemctl unmask +RUN systemctl enable getty@tty1.service +RUN systemctl enable getty@tty2.service +RUN systemctl enable getty@tty3.service +RUN systemctl enable systemd-networkd +RUN systemctl enable systemd-resolved +RUN systemctl enable sshd +RUN systemctl disable selinux-autorelabel-mark.service || true + +# Avoid clashes with systemd-networkd (Kairos/RHEL guidance). +# +# RHEL 9 uninstalls NetworkManager for this. That is not possible on RHEL 10: dracut-network +# requires "NetworkManager >= 1.20" unconditionally, so removing it would also drag out +# dracut-network and leave the final initramfs with no network module. Mask it instead — see +# Dockerfile.rhel10 for the full explanation. +# +# This must run *after* the unmask-everything step above, otherwise that step unmasks it again. +RUN if rpm -q NetworkManager >/dev/null 2>&1; then \ + systemctl disable NetworkManager.service NetworkManager-wait-online.service NetworkManager-dispatcher.service 2>/dev/null || true; \ + systemctl mask NetworkManager.service NetworkManager-wait-online.service NetworkManager-dispatcher.service; \ + fi + +COPY overlay/rhel10/ / + +RUN kernel=$(ls /boot/vmlinuz-* | head -n1) && \ + ln -sf "${kernel#/boot/}" /boot/vmlinuz +RUN kernel=$(ls /lib/modules | head -n1) && \ + dracut -v -N -f "/boot/initrd-${kernel}" "${kernel}" && \ + ln -sf "initrd-${kernel}" /boot/initrd && depmod -a "${kernel}" +RUN rm -rf /boot/initramfs-* + +RUN mkdir -p /etc/luet/repos.conf.d +## Clear cache +RUN rm -rf /var/cache/* && journalctl --vacuum-size=1K && rm /etc/machine-id + +RUN subscription-manager unregister diff --git a/rhel-core-images/Dockerfile.rhel8 b/rhel-core-images/Dockerfile.rhel8 index 8b8a2fec..5c60d0c9 100644 --- a/rhel-core-images/Dockerfile.rhel8 +++ b/rhel-core-images/Dockerfile.rhel8 @@ -1,17 +1,21 @@ # syntax=docker/dockerfile:1 -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM registry.access.redhat.com/ubi8/ubi-init:8.7-10 -ARG USERNAME -ARG PASSWORD -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 # EPEL: prefer mirror redirector; dl.fedoraproject.org often returns 503 under load. RUN dnf install -y 'https://download.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' || \ dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' # Subscription manager in redhat does not run directly in containers unless you run on a redhat host, hence we remove the rhsm-host, login to the redhat subscription and add the repos -RUN rm /etc/rhsm-host && subscription-manager register --username ${USERNAME} --password ${PASSWORD} \ + +RUN --mount=type=secret,id=RHSM_USERNAME,env=RHSM_USERNAME \ + --mount=type=secret,id=RHSM_PASSWORD,env=RHSM_PASSWORD \ + : "${RHSM_USERNAME:?missing --secret id=RHSM_USERNAME,env=RHSM_USERNAME}" \ + && : "${RHSM_PASSWORD:?missing --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD}" \ + && rm /etc/rhsm-host \ + && subscription-manager register --username "${RHSM_USERNAME}" --password "${RHSM_PASSWORD}" \ && yum repolist \ && subscription-manager attach --auto \ && subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms \ diff --git a/rhel-core-images/Dockerfile.rhel8.sat b/rhel-core-images/Dockerfile.rhel8.sat index 370e9454..d19ba182 100644 --- a/rhel-core-images/Dockerfile.rhel8.sat +++ b/rhel-core-images/Dockerfile.rhel8.sat @@ -1,19 +1,22 @@ +# syntax=docker/dockerfile:1 ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi-init:8.7-10 -ARG KAIROS_INIT_IMAGE=quay.io/kairos/kairos-init:v0.8.12 +ARG KAIROS_INIT_IMAGE=quay.io/kairos/kairos-init:v0.16.2 -FROM $KAIROS_INIT_IMAGE as kairos-init +FROM $KAIROS_INIT_IMAGE AS kairos-init FROM $BASE_IMAGE ARG ORGNAME -ARG KEYNAME ARG SATHOSTNAME -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 RUN dnf config-manager --disable ubi-8-appstream-rpms ubi-8-baseos-rpms ubi-8-codeready-builder-rpms RUN rm /etc/rhsm-host RUN rpm -Uvh http://${SATHOSTNAME}/pub/katello-ca-consumer-latest.noarch.rpm -RUN subscription-manager register --org=${ORGNAME} --activationkey=${KEYNAME} +RUN --mount=type=secret,id=KEYNAME,env=KEYNAME \ + : "${KEYNAME:?missing --secret id=KEYNAME,env=KEYNAME}" \ + && : "${ORGNAME:?missing --build-arg ORGNAME}" \ + && subscription-manager register --org="${ORGNAME}" --activationkey="${KEYNAME}" RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf diff --git a/rhel-core-images/Dockerfile.rhel9 b/rhel-core-images/Dockerfile.rhel9 index c26eaacc..7988707d 100644 --- a/rhel-core-images/Dockerfile.rhel9 +++ b/rhel-core-images/Dockerfile.rhel9 @@ -1,17 +1,20 @@ # syntax=docker/dockerfile:1 -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM registry.access.redhat.com/ubi9-init:9.4-6 -ARG USERNAME -ARG PASSWORD -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 # EPEL: prefer mirror redirector; dl.fedoraproject.org often returns 503 under load. RUN dnf install -y 'https://download.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm' || \ dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm' # Subscription manager in redhat does not run directly in containers unless you run on a redhat host, hence we remove the rhsm-host, login to the redhat subscription and add the repos -RUN rm /etc/rhsm-host && subscription-manager register --username ${USERNAME} --password ${PASSWORD} \ +RUN --mount=type=secret,id=RHSM_USERNAME,env=RHSM_USERNAME \ + --mount=type=secret,id=RHSM_PASSWORD,env=RHSM_PASSWORD \ + : "${RHSM_USERNAME:?missing --secret id=RHSM_USERNAME,env=RHSM_USERNAME}" \ + && : "${RHSM_PASSWORD:?missing --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD}" \ + && rm /etc/rhsm-host \ + && subscription-manager register --username "${RHSM_USERNAME}" --password "${RHSM_PASSWORD}" \ && yum repolist \ && subscription-manager attach --auto \ && subscription-manager repos --enable rhel-9-for-x86_64-appstream-rpms \ diff --git a/rhel-core-images/Dockerfile.rhel9.sat b/rhel-core-images/Dockerfile.rhel9.sat index 167cb18d..3c90c3de 100644 --- a/rhel-core-images/Dockerfile.rhel9.sat +++ b/rhel-core-images/Dockerfile.rhel9.sat @@ -1,19 +1,23 @@ +# syntax=docker/dockerfile:1 ARG BASE_IMAGE=registry.access.redhat.com/ubi9-init:9.4-6 -ARG KAIROS_INIT_IMAGE=quay.io/kairos/kairos-init:v0.8.12 +ARG KAIROS_INIT_IMAGE=quay.io/kairos/kairos-init:v0.16.2 -FROM $KAIROS_INIT_IMAGE as kairos-init +FROM $KAIROS_INIT_IMAGE AS kairos-init FROM $BASE_IMAGE ARG ORGNAME -ARG KEYNAME ARG SATHOSTNAME -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 RUN dnf config-manager --disable ubi-9-appstream-rpms ubi-9-baseos-rpms ubi-9-codeready-builder-rpms RUN rm /etc/rhsm-host RUN rpm -Uvh http://${SATHOSTNAME}/pub/katello-ca-consumer-latest.noarch.rpm -RUN subscription-manager register --org=${ORGNAME} --activationkey=${KEYNAME} + +RUN --mount=type=secret,id=KEYNAME,env=KEYNAME \ + : "${KEYNAME:?missing --secret id=KEYNAME,env=KEYNAME}" \ + && : "${ORGNAME:?missing --build-arg ORGNAME}" \ + && subscription-manager register --org="${ORGNAME}" --activationkey="${KEYNAME}" RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf diff --git a/rhel-core-images/README.md b/rhel-core-images/README.md index ba59f40b..ef6e32e4 100644 --- a/rhel-core-images/README.md +++ b/rhel-core-images/README.md @@ -1,26 +1,103 @@ -# Kairos RHEL 8 and RHEL 9 images +# Kairos RHEL 8, RHEL 9 and RHEL 10 images ## Build the image using Red Hat Subscription Follow steps below to execute the build process on the host with access to Red Hat Subscription Management system (redhat.com) and by using Red Hat username and password. -To build the image provide username and password for Red Hat Subscription Manager to register the system and install packages during the build process. +### Quick start (recommended): `build.sh` + +```bash +export RHSM_USERNAME='' +export RHSM_PASSWORD='' + +bash build.sh --ver 8 # -> palette-rhel8:latest +bash build.sh --ver 9 # -> palette-rhel9:latest +bash build.sh --ver 10 # -> palette-rhel10:latest + +# custom name, and push to a registry in one step +bash build.sh --ver 10 --tag /: --push +``` + +| flag | | +|---|---| +| `--ver <8\|9\|10>` | RHEL major version; selects `Dockerfile.rhel` (required) | +| `--tag ` | image name to build (default `palette-rhel:latest`) | +| `--push` | `docker push` the image after a successful build; requires `--tag` | + +`--push` refuses to run without `--tag`, since the default name is unqualified and pushing it +would fail or silently target Docker Hub. Log in to the registry (`docker login`) first. + +**Credentials are exported, not passed as `--build-arg`.** The build fails immediately naming the missing variable if either is unset. + +The raw `docker build` invocations below are equivalent — use them if you need to pass extra +flags. Note they now require `--secret` rather than `--build-arg USERNAME=... PASSWORD=...`. To build RHEL 8 Kairos Image, execute: ``` -docker build -t /: --build-arg USERNAME= --build-arg PASSWORD='' -f Dockerfile.rhel8. +docker build -t /: --secret id=RHSM_USERNAME,env=RHSM_USERNAME --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD -f Dockerfile.rhel8 . ``` To build RHEL 9 Kairos Image, execute: ``` -docker build -t /: --build-arg USERNAME= --build-arg PASSWORD='' -f Dockerfile.rhel9 . +docker build -t /: --secret id=RHSM_USERNAME,env=RHSM_USERNAME --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD -f Dockerfile.rhel9 . +``` + +To build RHEL 10 Kairos Image, execute: +``` +docker build -t /: --secret id=RHSM_USERNAME,env=RHSM_USERNAME --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD -f Dockerfile.rhel10 . ``` + +### RHEL 10 notes + +Base image is `registry.access.redhat.com/ubi10-init:10.2`. The package list differs from RHEL 9 because of upstream changes in RHEL 10: + +* `dhclient` was removed — ISC dhcp is no longer shipped in RHEL 10. DHCP is provided by `systemd-networkd`. +* `grub2`, `iptables` and `conntrack` no longer exist as package names. kairos-init requests `grub2` and `iptables` and they resolve through virtual provides (`grub2-pc`, `iptables-nft`), so the explicit list does not repeat them. +* The explicit `dnf install` list is much shorter than the RHEL 8/9 ones — 15 packages instead of 46. kairos-init already installs 128 packages, and a real build showed 34 of the 46 that `Dockerfile.rhel9` lists were reported by dnf as `already installed`. `parted`, `kbd` and `coreutils-single` are kept in the list even though they are redundant today, because they only arrive as transitive dependencies or from the base image and a version bump could drop them silently. +* `subscription-manager attach --auto` was removed in RHEL 10 — the `attach` module no longer exists, since it is obsolete under Simple Content Access. `Dockerfile.rhel10` therefore only registers; that already leaves `rhel-10-for-x86_64-baseos-rpms` and `-appstream-rpms` enabled. +* EPEL 10 is mandatory, not just convenient: `systemd-networkd`, `systemd-timesyncd`, `livecd-tools` and `haveged` are not in RHEL 10 BaseOS/AppStream. +* NetworkManager is **masked, not uninstalled**. On RHEL 9 `dracut-network` requires `(NetworkManager >= 1.20 or dhclient)`, so `dhclient` satisfies it and NetworkManager can be removed. On RHEL 10 the requirement is an unconditional `NetworkManager >= 1.20`, so removing NetworkManager would also remove `dracut-network` and leave the final initramfs without the `network` module that kairos-init asks for in `/etc/dracut.conf.d/kairos-network.conf`. The mask step therefore runs *after* the "unmask everything" step, which would otherwise undo it. + +**RHEL 10 requires an `x86-64-v3` CPU to boot.** Red Hat raised the RHEL 10 baseline from `x86-64-v2` to `x86-64-v3`, so nothing older than roughly Intel Haswell / AMD Excavator (2013+) will run it. The guest's CPU model must expose v3, not just the host's — a v3-capable host still fails if the hypervisor presents an older model. + +It surfaces two different ways depending on how far the boot gets: + +* **Silent death** — GRUB prints `Loading kernel...` / `Loading initrd...` and then nothing at all, no panic and no kernel log, because the check happens before any console exists. Seen under SeaBIOS with QEMU's default `qemu64` model. +* **glibc abort** — the kernel starts, then: + + ``` + Fatal glibc error: CPU does not support x86-64-v3 + Kernel panic - not syncing: Attempted to kill init! exitcode=0x00007f00 + ``` + + RHEL 10's glibc is compiled for v3 and aborts as PID 1, which panics the kernel. Seen under edk2/UEFI on KubeVirt. + +Set the CPU model explicitly: QEMU `-cpu host` (or `-cpu max` without KVM), libvirt `host-passthrough`, and for KubeVirt/VMO `spec.template.spec.domain.cpu.model: host-passthrough`. RHEL 8/9 images are unaffected, so "rhel9 boots, rhel10 doesn't" points here rather than at the image. + +The build must run on an `x86_64` host. Emulating `linux/amd64` on Apple Silicon does not work for RHEL 10 — its OpenSSL fails its provider integrity self-check under Rosetta, so every HTTPS request (including `dnf makecache`) fails with `error:030000EA:digital envelope routines::provider signature failure`. + **In case of any errors during package installation steps - these errors might be caused by previous build attempts. Execute `docker build` command again by providing argument `--no-cache` to build the image from scratch** ## Build the image using Red Hat Satellite and mirrored repositories -This scenario is for the environment where Red Hat Satellite must be used and access to public Red Hat repositories is not possible. For this case use Dockerfiles `Dockerfile.rhel9.sat` and `Dockerfile.rhel8.sat` - these files are modified to use Red Hat Satellite Activation key to register host and install all required packages. +This scenario is for the environment where Red Hat Satellite must be used and access to public Red Hat repositories is not possible. For this case use Dockerfiles `Dockerfile.rhel8.sat`, `Dockerfile.rhel9.sat` and `Dockerfile.rhel10.sat` - these files are modified to use Red Hat Satellite Activation key to register host and install all required packages. + +### Quick start (recommended): `build-sat.sh` + +```bash +export KEYNAME='' + +bash build-sat.sh 9 --org --satellite \ + --base-image \ + --kairos-init \ + --tag /: +``` + +Only `--org` and `--satellite` are required; `--base-image`, `--kairos-init` and `--tag` fall +back to the Dockerfile defaults and `palette-rhel:latest`. + +**The activation key is exported, not passed as `--build-arg`.** The build fails immediately if `KEYNAME` is unset or `--org`/`--satellite` are missing. ### Prerequisites @@ -40,6 +117,11 @@ For RHEL8: * rhel-8-for-x86_64-baseos-rpms * EPEL8 (upstream URL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/) +For RHEL10: +* rhel-10-for-x86_64-appstream-rpms +* rhel-10-for-x86_64-baseos-rpms +* EPEL10 (upstream URL https://dl.fedoraproject.org/pub/epel/10/Everything/x86_64/) — **mandatory** for RHEL 10, not optional: `systemd-networkd`, `systemd-timesyncd`, `livecd-tools` and `haveged` are not in RHEL 10 BaseOS/AppStream, so a Satellite without EPEL 10 fails in the package install step. + 4. Create Activation Key in RH Satellite and add corresponding repositories listed above. Make these repositories enabled by default (set `Override Enabled` for these repositories in the Activation Key configuration). Provide Activation Key for the build process by using argument `KEYNAME`. @@ -59,23 +141,44 @@ KEYNAME - Name of the Activation key with repositories attached, for example `rh To build RHEL 8 Kairos Image, execute: ``` -docker build -t /: --build-arg BASE_IMAGE= --build-arg KAIROS_FRAMEWORK_IMAGE='' --build-arg SATHOSTNAME= --build-arg ORGNAME= --build-arg KEYNAME= -f Dockerfile.rhel8.sat . +docker build -t /: --secret id=KEYNAME,env=KEYNAME --build-arg BASE_IMAGE= --build-arg KAIROS_FRAMEWORK_IMAGE='' --build-arg SATHOSTNAME= --build-arg ORGNAME= -f Dockerfile.rhel8.sat . ``` To build RHEL 9 Kairos Image, execute: ``` -docker build -t /: --build-arg BASE_IMAGE= --build-arg KAIROS_FRAMEWORK_IMAGE='' --build-arg SATHOSTNAME= --build-arg ORGNAME= --build-arg KEYNAME= -f Dockerfile.rhel9.sat . +docker build -t /: --secret id=KEYNAME,env=KEYNAME --build-arg BASE_IMAGE= --build-arg KAIROS_FRAMEWORK_IMAGE='' --build-arg SATHOSTNAME= --build-arg ORGNAME= -f Dockerfile.rhel9.sat . ``` For example, to build RHEL9 image: ``` -docker build -t localhost/palette-rhel9:latest --build-arg BASE_IMAGE=redhat.spectrocloud.dev/ubi9-init:9.4-6 --build-arg KAIROS_FRAMEWORK_IMAGE=quay.spectrocloud.dev/kairos/framework:v2.7.33 --build-arg SATHOSTNAME=katello.spectrocloud.dev --build-arg ORGNAME=test-org --build-arg KEYNAME=rhel9-canvos-key -f Dockerfile.rhel9.sat . +docker build -t localhost/palette-rhel9:latest --secret id=KEYNAME,env=KEYNAME --build-arg BASE_IMAGE=redhat.spectrocloud.dev/ubi9-init:9.4-6 --build-arg KAIROS_FRAMEWORK_IMAGE=quay.spectrocloud.dev/kairos/framework:v2.7.33 --build-arg SATHOSTNAME=katello.spectrocloud.dev --build-arg ORGNAME=test-org -f Dockerfile.rhel9.sat . ``` For example, to build RHEL8 image: ``` -docker build -t localhost/palette-rhel8:latest --build-arg BASE_IMAGE=redhat.spectrocloud.dev/ubi8/ubi-init:8.7-10 --build-arg KAIROS_FRAMEWORK_IMAGE=quay.spectrocloud.dev/kairos/framework:v2.7.33 --build-arg SATHOSTNAME=katello.spectrocloud.dev --build-arg ORGNAME=test-org --build-arg KEYNAME=rhel8-canvos-key -f Dockerfile.rhel8.sat . +docker build -t localhost/palette-rhel8:latest --secret id=KEYNAME,env=KEYNAME --build-arg BASE_IMAGE=redhat.spectrocloud.dev/ubi8/ubi-init:8.7-10 --build-arg KAIROS_FRAMEWORK_IMAGE=quay.spectrocloud.dev/kairos/framework:v2.7.33 --build-arg SATHOSTNAME=katello.spectrocloud.dev --build-arg ORGNAME=test-org -f Dockerfile.rhel8.sat . +``` + +To build RHEL 10 Kairos Image via Satellite, execute: +``` +export KEYNAME='' + +docker build -t /: \ + --secret id=KEYNAME,env=KEYNAME \ + --build-arg ORGNAME= \ + --build-arg SATHOSTNAME= \ + --build-arg BASE_IMAGE= \ + -f Dockerfile.rhel10.sat . ``` +`Dockerfile.rhel10.sat` differs from the RHEL 8/9 Satellite files in two ways, both forced by RHEL 10: + +* **No `subscription-manager attach --auto`** — the `attach` module was removed in RHEL 10. +* **No `KAIROS_FRAMEWORK_IMAGE`** — the RHEL 10 files use `kairos-init` (`KAIROS_INIT_IMAGE`), not the older framework image. Mirror `quay.io/kairos/kairos-init:v0.16.2` instead and pass it via `--build-arg KAIROS_INIT_IMAGE=`. + +It also mirrors `Dockerfile.rhel10` rather than `Dockerfile.rhel9.sat`, so `kairos-init` runs *before* the extra package install — that is the ordering verified end to end on RHEL 10. The resulting package set is identical either way. + + +> **Note:** a Red Hat Satellite variant for RHEL 10 (`Dockerfile.rhel10.sat`) has not been added yet. Only the direct-subscription `Dockerfile.rhel10` exists. diff --git a/rhel-core-images/build-sat.sh b/rhel-core-images/build-sat.sh new file mode 100755 index 00000000..eab032ec --- /dev/null +++ b/rhel-core-images/build-sat.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Build a RHEL Kairos base image registered against a Red Hat Satellite. +# For a direct Red Hat Subscription, use build.sh instead. +# +# Usage: +# export KEYNAME='' +# bash build-sat.sh <8|9|10> --org --satellite \ +# [--base-image ] [--kairos-init ] \ +# [--tag ] + +set -euo pipefail + +usage() { + echo "usage: KEYNAME= bash build-sat.sh <8|9|10> --org --satellite [--base-image X] [--kairos-init Y] [--tag Z]" >&2 + exit 1 +} + +VER="${1:-}"; shift || usage +case "$VER" in + 8|9|10) ;; + *) echo "ERROR: unsupported RHEL version '$VER' (expected 8, 9 or 10)" >&2; usage ;; +esac + +ORGNAME=""; SATHOSTNAME=""; BASE_IMAGE=""; KAIROS_INIT_IMAGE=""; IMAGE="" +while [ $# -gt 0 ]; do + case "$1" in + --org) ORGNAME="${2:?--org needs a value}"; shift 2 ;; + --satellite) SATHOSTNAME="${2:?--satellite needs a value}"; shift 2 ;; + --base-image) BASE_IMAGE="${2:?--base-image needs a value}"; shift 2 ;; + --kairos-init) KAIROS_INIT_IMAGE="${2:?--kairos-init needs a value}"; shift 2 ;; + --tag) IMAGE="${2:?--tag needs a value}"; shift 2 ;; + *) echo "ERROR: unknown option '$1'" >&2; usage ;; + esac +done + +DOCKERFILE="Dockerfile.rhel${VER}.sat" +[ -f "$DOCKERFILE" ] || { echo "ERROR: $DOCKERFILE not found (run from rhel-core-images/)" >&2; exit 1; } + +: "${KEYNAME:?export KEYNAME before running (Satellite activation key)}" +[ -n "$ORGNAME" ] || { echo "ERROR: --org is required" >&2; usage; } +[ -n "$SATHOSTNAME" ] || { echo "ERROR: --satellite is required" >&2; usage; } + +IMAGE="${IMAGE:-palette-rhel${VER}:latest}" + +BUILD_ARGS=(--build-arg "ORGNAME=$ORGNAME" --build-arg "SATHOSTNAME=$SATHOSTNAME") +[ -n "$BASE_IMAGE" ] && BUILD_ARGS+=(--build-arg "BASE_IMAGE=$BASE_IMAGE") +[ -n "$KAIROS_INIT_IMAGE" ] && BUILD_ARGS+=(--build-arg "KAIROS_INIT_IMAGE=$KAIROS_INIT_IMAGE") + + +echo "Building $IMAGE from $DOCKERFILE (org=$ORGNAME satellite=$SATHOSTNAME) ..." +docker build \ + --secret id=KEYNAME,env=KEYNAME \ + "${BUILD_ARGS[@]}" \ + -t "$IMAGE" \ + -f "$DOCKERFILE" . diff --git a/rhel-core-images/build.sh b/rhel-core-images/build.sh new file mode 100755 index 00000000..aae122f0 --- /dev/null +++ b/rhel-core-images/build.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Build a RHEL Kairos base image using a direct Red Hat Subscription (subscription.rhsm.redhat.com). +# For Red Hat Satellite, use build-sat.sh instead. +# +# Usage: +# export RHSM_USERNAME='you@example.com' +# export RHSM_PASSWORD='...' +# bash build.sh --ver <8|9|10> [--tag ] [--push] +# +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: RHSM_USERNAME=... RHSM_PASSWORD=... bash build.sh --ver <8|9|10> [--tag ] [--push] + + --ver <8|9|10> RHEL major version; selects Dockerfile.rhel (required) + --tag image name to build (default: palette-rhel:latest) + --push docker push the image after a successful build; requires --tag +EOF + exit 1 +} + +VER=""; IMAGE=""; PUSH=false +while [ $# -gt 0 ]; do + case "$1" in + --ver) VER="${2:?--ver needs a value}"; shift 2 ;; + --tag) IMAGE="${2:?--tag needs a value}"; shift 2 ;; + --push) PUSH=true; shift ;; + -h|--help) usage ;; + *) echo "ERROR: unknown option '$1'" >&2; usage ;; + esac +done + +[ -n "$VER" ] || { echo "ERROR: --ver is required" >&2; usage; } +case "$VER" in + 8|9|10) ;; + *) echo "ERROR: unsupported RHEL version '$VER' (expected 8, 9 or 10)" >&2; exit 1 ;; +esac + +DOCKERFILE="Dockerfile.rhel${VER}" +[ -f "$DOCKERFILE" ] || { echo "ERROR: $DOCKERFILE not found (run from rhel-core-images/)" >&2; exit 1; } + +# --push needs a registry-qualified name. The default is a bare local name, so pushing it +# would either fail or, worse, target Docker Hub — refuse rather than guess. +if [ "$PUSH" = true ] && [ -z "$IMAGE" ]; then + echo "ERROR: --push requires --tag with a registry path (e.g. --tag registry.example.com/palette-rhel${VER}:v1)" >&2 + exit 1 +fi + +IMAGE="${IMAGE:-palette-rhel${VER}:latest}" + +: "${RHSM_USERNAME:?export RHSM_USERNAME before running (Red Hat Subscription Manager username)}" +: "${RHSM_PASSWORD:?export RHSM_PASSWORD before running (Red Hat Subscription Manager password)}" + +PUSH_ARG="" +if [ "$PUSH" = true ]; then + PUSH_ARG="--push" +fi + +echo "Building $IMAGE from $DOCKERFILE ..." +docker build \ + --secret id=RHSM_USERNAME,env=RHSM_USERNAME \ + --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD \ + -t "$IMAGE" \ + -f "$DOCKERFILE" \ + $PUSH_ARG . diff --git a/rhel-core-images/overlay/rhel10/system/oem/33_tmp_mount.yaml b/rhel-core-images/overlay/rhel10/system/oem/33_tmp_mount.yaml new file mode 100644 index 00000000..09a5f98d --- /dev/null +++ b/rhel-core-images/overlay/rhel10/system/oem/33_tmp_mount.yaml @@ -0,0 +1,10 @@ +name: " tmp layout setup" +stages: + initramfs.after: + - name: mount tmp + commands: + - systemctl enable tmp.mount + fs.before: + - name: start tmp + commands: + - systemctl start tmp.mount diff --git a/rhel-fips/Dockerfile.rhel10 b/rhel-fips/Dockerfile.rhel10 new file mode 100644 index 00000000..13aadf60 --- /dev/null +++ b/rhel-fips/Dockerfile.rhel10 @@ -0,0 +1,135 @@ +# syntax=docker/dockerfile:1 +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init + +FROM registry.access.redhat.com/ubi10-init:10.2 + +ARG KAIROS_VERSION=v4.1.2 + +# EPEL: prefer mirror redirector; dl.fedoraproject.org often returns 503 under load. +# EPEL 10 is required, not optional: systemd-networkd, systemd-timesyncd, livecd-tools and +# haveged are not shipped in RHEL 10 BaseOS/AppStream. +RUN dnf install -y 'https://download.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm' || \ + dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm' + +# Subscription manager in redhat does not run directly in containers unless you run on a redhat host, hence we remove the rhsm-host, login to the redhat subscription and add the repos + +RUN --mount=type=secret,id=RHSM_USERNAME,env=RHSM_USERNAME \ + --mount=type=secret,id=RHSM_PASSWORD,env=RHSM_PASSWORD \ + : "${RHSM_USERNAME:?missing --secret id=RHSM_USERNAME,env=RHSM_USERNAME}" \ + && : "${RHSM_PASSWORD:?missing --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD}" \ + && rm /etc/rhsm-host \ + && subscription-manager register --username "${RHSM_USERNAME}" --password "${RHSM_PASSWORD}" \ + && yum repolist \ + && subscription-manager repos --enable rhel-10-for-x86_64-appstream-rpms \ + && yum repolist \ + && dnf clean all +RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf + +# overlay/rhel10 deliberately does NOT ship copies of 60-persistent-storage.rules or +# 13-dm-disk.rules, unlike overlay/rhel8 and overlay/rhel9. RHEL 10 already provides both +# (from systemd-udev and device-mapper respectively), so the install_items entries in +# dracut.conf pick up the distro's own copies rather than stale RHEL 9 versions. +COPY overlay/rhel10/ / + +COPY dracut.conf /etc/dracut.conf.d/kairos-fips.conf + +# Bind-mounted rather than COPYed: the RHEL 8/9 FIPS Dockerfiles `COPY` the kairos-init +# binary in and never delete it, leaving ~78 MB of build tooling in the shipped image. +RUN --mount=type=bind,from=kairos-init,src=/kairos-init,dst=/kairos-init \ + /kairos-init -l debug -s install --fips --version "${KAIROS_VERSION}" + +# Generate machine-id because https://bugzilla.redhat.com/show_bug.cgi?id=1737355#c6 +# +# Short list by design. kairos-init above already installs ~128 packages, so on RHEL 10 only +# these add anything; the equivalent RHEL 9 list is mostly redundant names. The last three are +# redundant today but kept explicit because parted and kbd arrive only as transitive +# dependencies of kairos-init's set and coreutils-single comes from the base image. +# +# RHEL 10 package deltas vs RHEL 9: dhclient is gone (ISC dhcp removed), and grub2, iptables +# and conntrack no longer exist as package names — kairos-init requests grub2 and iptables and +# they resolve through virtual provides (grub2-pc, iptables-nft). +RUN uuidgen > /etc/machine-id && dnf install -y \ + livecd-tools \ + efibootmgr \ + systemd-networkd \ + systemd-timesyncd \ + haveged \ + ncurses \ + open-vm-tools \ + iscsi-initiator-utils \ + iproute-tc conntrack-tools ethtool socat \ + parted \ + kbd \ + coreutils-single && dnf clean all + +RUN sed -i 's/\bsource\b/./g' /system/oem/00_rootfs.yaml +RUN sed -i 's/\bsource\b/./g' /system/oem/09_openrc_services.yaml +RUN sed -i 's/\bsource\b/./g' /system/oem/50_recovery.yaml + +RUN mkdir -p /run/lock +RUN touch /usr/libexec/.keep + +# Configure the box. The ubi image masks services for containers, we unmask them +RUN systemctl list-unit-files |grep masked |cut -f 1 -d " " | xargs systemctl unmask +RUN systemctl enable getty@tty1.service +RUN systemctl enable getty@tty2.service +RUN systemctl enable getty@tty3.service +RUN systemctl enable systemd-networkd +RUN systemctl enable systemd-resolved +RUN systemctl enable sshd +RUN systemctl disable selinux-autorelabel-mark.service || true +RUN systemctl unmask systemd-udevd +RUN systemctl enable systemd-udevd +RUN systemctl unmask systemd-logind.service +RUN systemctl enable systemd-logind.service + +COPY overlay/rhel10/ / + +COPY dracut.conf /etc/dracut.conf.d/kairos-fips.conf + +RUN --mount=type=bind,from=kairos-init,src=/kairos-init,dst=/kairos-init \ + /kairos-init -l debug -s init --fips --version "${KAIROS_VERSION}" + +# Avoid clashes with systemd-networkd (Kairos/RHEL guidance). +# +# The RHEL 8/9 FIPS Dockerfiles run `dnf remove -y NetworkManager` for this. That is not +# possible on RHEL 10: dracut-network requires "NetworkManager >= 1.20" unconditionally, +# whereas RHEL 9 requires "(NetworkManager >= 1.20 or dhclient)" and is satisfied by dhclient. +# Since dhclient is gone from RHEL 10, removing NetworkManager would also drag out +# dracut-network and leave the initramfs with no network module. Mask it instead: the package +# stays to satisfy dracut-network, while systemd-networkd owns the network on the real root. +# +# Placement is load-bearing and differs from Dockerfile.rhel10 in rhel-core-images. This must +# run AFTER `kairos-init -s init`, because that stage contains a step "Enable NetworkManager +# for RHEL if binary is available" which runs `systemctl enable NetworkManager` and fails hard +# on a masked unit ("Unit ... is masked" -> kairos-init aborts). Masking afterwards is also +# what we want for the initramfs: `-s init` runs dracut while NetworkManager is still enabled, +# so the initrd gets the 35network-manager module, and only the real root has it masked. +RUN if rpm -q NetworkManager >/dev/null 2>&1; then \ + systemctl disable NetworkManager.service NetworkManager-wait-online.service NetworkManager-dispatcher.service 2>/dev/null || true; \ + systemctl mask NetworkManager.service NetworkManager-wait-online.service NetworkManager-dispatcher.service; \ + fi + +# dhcp-client is intentionally absent compared with the RHEL 8/9 FIPS Dockerfiles — it does +# not exist on RHEL 10. +RUN dnf install -y dracut dracut-network dracut-live dracut-squash \ + && dnf clean all + +COPY dracut.conf /etc/dracut.conf.d/kairos-fips.conf + +RUN kernel=$(ls /boot/vmlinuz-* | head -n1) && ln -sf ."${kernel#/boot/}".hmac /boot/.vmlinuz.hmac + +# Disable SELinux +RUN echo "SELINUX=disabled" > /etc/selinux/config + +RUN rm -rf /boot/initramfs-* + +COPY overlay/rhel10/ / + +# Guard is a no-op on RHEL 10 (ubi10-init already ships shim.efi alongside shimx64.efi) but is +# kept for parity with the RHEL 8/9 files, where it fixes UEFI ISO boot. +RUN if [ ! -f /boot/efi/EFI/redhat/shim.efi ]; then cp /boot/efi/EFI/redhat/shimx64.efi /boot/efi/EFI/redhat/shim.efi; fi + +# Release the subscription so the built image carries no entitlements and the build host does +# not accumulate orphan registrations. The RHEL 8/9 FIPS Dockerfiles omit this. +RUN subscription-manager unregister diff --git a/rhel-fips/Dockerfile.rhel8 b/rhel-fips/Dockerfile.rhel8 index ecd93174..326dfec3 100644 --- a/rhel-fips/Dockerfile.rhel8 +++ b/rhel-fips/Dockerfile.rhel8 @@ -1,22 +1,21 @@ # syntax=docker/dockerfile:1 # Kairos init image -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM registry.access.redhat.com/ubi8/ubi-init:8.7-10 AS base -ARG USERNAME -ARG PASSWORD - # Generate os-release file -ARG KAIROS_VERSION=v4.0.4 - -# Don't get asked while running apt commands -ENV DEBIAN_FRONTEND=noninteractive +ARG KAIROS_VERSION=v4.1.2 RUN dnf install -y 'https://download.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' || \ dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm' # Subscription manager in redhat does not run directly in containers unless you run on a redhat host, hence we remove the rhsm-host, login to the redhat subscription and add the repos -RUN rm /etc/rhsm-host && subscription-manager register --username ${USERNAME} --password ${PASSWORD} \ +RUN --mount=type=secret,id=RHSM_USERNAME,env=RHSM_USERNAME \ + --mount=type=secret,id=RHSM_PASSWORD,env=RHSM_PASSWORD \ + : "${RHSM_USERNAME:?missing --secret id=RHSM_USERNAME,env=RHSM_USERNAME}" \ + && : "${RHSM_PASSWORD:?missing --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD}" \ + && rm /etc/rhsm-host \ + && subscription-manager register --username "${RHSM_USERNAME}" --password "${RHSM_PASSWORD}" \ && yum repolist \ && subscription-manager attach --auto \ && subscription-manager repos --enable rhel-8-for-x86_64-appstream-rpms \ diff --git a/rhel-fips/Dockerfile.rhel9 b/rhel-fips/Dockerfile.rhel9 index fce58ce4..373e810f 100644 --- a/rhel-fips/Dockerfile.rhel9 +++ b/rhel-fips/Dockerfile.rhel9 @@ -1,24 +1,30 @@ # syntax=docker/dockerfile:1 # Kairos init image -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM registry.access.redhat.com/ubi9-init:9.4-6 AS base -ARG USERNAME -ARG PASSWORD - # Generate os-release file -ARG KAIROS_VERSION=v4.0.4 - -# Don't get asked while running apt commands -ENV DEBIAN_FRONTEND=noninteractive +ARG KAIROS_VERSION=v4.1.2 # EPEL: prefer mirror redirector; dl.fedoraproject.org often returns 503 under load. RUN dnf install -y 'https://download.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm' || \ dnf install -y 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm' # Subscription manager in redhat does not run directly in containers unless you run on a redhat host, hence we remove the rhsm-host, login to the redhat subscription and add the repos -RUN rm /etc/rhsm-host && subscription-manager register --username ${USERNAME} --password ${PASSWORD} \ +# +# Credentials are BuildKit secrets sourced from environment variables and exposed as env only +# for the duration of this RUN, so they never become build args and never land in an image +# layer or in `docker history`. Use build.sh, which wires this up: +# +# export RHSM_USERNAME='you@example.com' RHSM_PASSWORD='...' +# bash build.sh --ver 9 +RUN --mount=type=secret,id=RHSM_USERNAME,env=RHSM_USERNAME \ + --mount=type=secret,id=RHSM_PASSWORD,env=RHSM_PASSWORD \ + : "${RHSM_USERNAME:?missing --secret id=RHSM_USERNAME,env=RHSM_USERNAME}" \ + && : "${RHSM_PASSWORD:?missing --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD}" \ + && rm /etc/rhsm-host \ + && subscription-manager register --username "${RHSM_USERNAME}" --password "${RHSM_PASSWORD}" \ && yum repolist \ && subscription-manager attach --auto \ && subscription-manager repos --enable rhel-9-for-x86_64-appstream-rpms \ diff --git a/rhel-fips/README.md b/rhel-fips/README.md index 89cc8e4a..1af04fe4 100644 --- a/rhel-fips/README.md +++ b/rhel-fips/README.md @@ -1,14 +1,53 @@ -# Kairos RHEL8 and RHEL9 FIPS +# Kairos RHEL8, RHEL9 and RHEL10 FIPS -## Build RHEL 8 FIPS Image -- run `bash build.sh.rhel8 []` -- use the generated base image as input in installer generation with `earthly +iso` +## Build a RHEL FIPS Image (8, 9 or 10) -## Build RHEL 9 FIPS Image -- run `bash build.sh.rhel9 []` -- use the generated base image as input in installer generation with `earthly +iso` +One script builds all three versions — pass the RHEL major version and it selects the matching +`Dockerfile.rhel`: -**Note**: Red Hat subscription credentials are required to build these images as RHEL8/RHEL9 FIPS packages are only available through Red Hat repositories. +```bash +export RHSM_USERNAME='' +export RHSM_PASSWORD='' + +bash build.sh --ver 8 # -> rhel8-byoi-fips +bash build.sh --ver 9 # -> rhel9-byoi-fips +bash build.sh --ver 10 # -> rhel10-byoi-fips + +# custom name, and push to a registry in one step +bash build.sh --ver 10 --tag /: --push +``` + +| flag | | +|---|---| +| `--ver <8\|9\|10>` | RHEL major version; selects `Dockerfile.rhel` (required) | +| `--tag ` | image name to build (default `rhel-byoi-fips`) | +| `--push` | `docker push` the image after a successful build; requires `--tag` | + +`--push` refuses to run without `--tag`, since the default name is unqualified and pushing it +would fail or silently target Docker Hub. Log in to the registry (`docker login`) first. + +Then use the generated base image as input in installer generation with `earthly +iso`. + +**Credentials are exported, not passed as arguments.** The build fails immediately with a message +naming the missing variable if either is unset. + +Note the old `build.sh.rhel8` defaulted its image name to `rhel-byoi-fips` (no `8`); the +unified script uses `rhel-byoi-fips` consistently. + +### RHEL 10 differences + +`Dockerfile.rhel10` diverges from the RHEL 8/9 FIPS files in ways forced by RHEL 10: + +* **`dhclient` / `dhcp-client` are gone** — ISC dhcp was removed. DHCP is handled by `systemd-networkd`. +* **NetworkManager is masked, not uninstalled.** On RHEL 10 `dracut-network` requires `NetworkManager >= 1.20` unconditionally (RHEL 9 accepts `dhclient` as an alternative), so removing it would also remove `dracut-network` and leave the initramfs with no network module. The mask step runs *after* the "unmask everything" step, which would otherwise undo it. +* **No `subscription-manager attach --auto`** — the `attach` module was removed in RHEL 10. +* **The package list is much shorter** (15 entries vs ~46). `kairos-init` already installs ~128 packages; on RHEL 10 only these add anything. +* **`overlay/rhel10/` ships no udev rules.** RHEL 10 already provides `60-persistent-storage.rules` (systemd-udev) and `13-dm-disk.rules` (device-mapper), so `dracut.conf`'s `install_items` picks up the distro's own copies instead of stale RHEL 9 ones. +* **An `x86-64-v3` CPU is required to boot RHEL 10** — see `rhel-core-images/README.md` for the two failure signatures. This applies to the FIPS image too. + +`dracut.conf` is shared across all three versions and needs no RHEL 10 changes: the `01fips` dracut module is present, and both `install_items` paths exist in the RHEL 10 image. + +**Note**: Red Hat subscription credentials are required to build these images as RHEL8/RHEL9/RHEL10 FIPS packages are only available through Red Hat repositories. The system is not enabling FIPS by default in kernel space. diff --git a/rhel-fips/build.sh b/rhel-fips/build.sh new file mode 100755 index 00000000..a28fadd4 --- /dev/null +++ b/rhel-fips/build.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Build a RHEL FIPS Kairos base image. +# +# Usage: +# export RHSM_USERNAME='you@example.com' +# export RHSM_PASSWORD='...' +# bash build.sh --ver <8|9|10> [--tag ] [--push] +# +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: RHSM_USERNAME=... RHSM_PASSWORD=... bash build.sh --ver <8|9|10> [--tag ] [--push] + + --ver <8|9|10> RHEL major version; selects Dockerfile.rhel (required) + --tag image name to build (default: rhel-byoi-fips) + --push docker push the image after a successful build; requires --tag +EOF + exit 1 +} + +VER=""; IMAGE=""; PUSH=false +while [ $# -gt 0 ]; do + case "$1" in + --ver) VER="${2:?--ver needs a value}"; shift 2 ;; + --tag) IMAGE="${2:?--tag needs a value}"; shift 2 ;; + --push) PUSH=true; shift ;; + -h|--help) usage ;; + *) echo "ERROR: unknown option '$1'" >&2; usage ;; + esac +done + +[ -n "$VER" ] || { echo "ERROR: --ver is required" >&2; usage; } +case "$VER" in + 8|9|10) ;; + *) echo "ERROR: unsupported RHEL version '$VER' (expected 8, 9 or 10)" >&2; exit 1 ;; +esac + +DOCKERFILE="Dockerfile.rhel${VER}" +[ -f "$DOCKERFILE" ] || { echo "ERROR: $DOCKERFILE not found (run from rhel-fips/)" >&2; exit 1; } + +# --push needs a registry-qualified name. The default is a bare local name, so pushing it +# would either fail or, worse, target Docker Hub — refuse rather than guess. +if [ "$PUSH" = true ] && [ -z "$IMAGE" ]; then + echo "ERROR: --push requires --tag with a registry path (e.g. --tag registry.example.com/rhel${VER}-byoi-fips:v1)" >&2 + exit 1 +fi + +# Consistent naming across versions. build.sh.rhel8 used to default to "rhel-byoi-fips" +# without the 8, so building 8 and then 9 produced inconsistently named images. +IMAGE="${IMAGE:-rhel${VER}-byoi-fips}" + +: "${RHSM_USERNAME:?export RHSM_USERNAME before running (Red Hat Subscription Manager username)}" +: "${RHSM_PASSWORD:?export RHSM_PASSWORD before running (Red Hat Subscription Manager password)}" + + +PUSH_ARG="" +if [ "$PUSH" = true ]; then + PUSH_ARG="--push" +fi + +echo "Building $IMAGE from $DOCKERFILE ..." +docker build \ + --secret id=RHSM_USERNAME,env=RHSM_USERNAME \ + --secret id=RHSM_PASSWORD,env=RHSM_PASSWORD \ + -t "$IMAGE" \ + -f "$DOCKERFILE" \ + $PUSH_ARG . diff --git a/rhel-fips/build.sh.rhel8 b/rhel-fips/build.sh.rhel8 deleted file mode 100755 index 2ec606d2..00000000 --- a/rhel-fips/build.sh.rhel8 +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - - -USERNAME=$1 -PASSWORD=$2 -BASE_IMAGE="${3:-rhel-byoi-fips}" - -# Build the container image -docker build --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD" -t "$BASE_IMAGE" -f Dockerfile.rhel8 . diff --git a/rhel-fips/build.sh.rhel9 b/rhel-fips/build.sh.rhel9 deleted file mode 100755 index 72e7a63d..00000000 --- a/rhel-fips/build.sh.rhel9 +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - - -USERNAME=$1 -PASSWORD=$2 -BASE_IMAGE="${3:-rhel9-byoi-fips}" - -# Build the container image -docker build --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD" -t "$BASE_IMAGE" -f Dockerfile.rhel9 . \ No newline at end of file diff --git a/rhel-fips/overlay/rhel10/etc/ssh/sshd_config.d/100_fips_crypto.conf b/rhel-fips/overlay/rhel10/etc/ssh/sshd_config.d/100_fips_crypto.conf new file mode 100644 index 00000000..717dc6dd --- /dev/null +++ b/rhel-fips/overlay/rhel10/etc/ssh/sshd_config.d/100_fips_crypto.conf @@ -0,0 +1,6 @@ +# Default algorithms favoring higher-performance FIPS algorithms +# in most cases. +Ciphers ^aes256-gcm@openssh.com,aes256-ctr,aes128-gcm@openssh.com,aes128-ctr +KexAlgorithms ^ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521 +MACs ^hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha2-256,hmac-sha2-512 +HostKeyAlgorithms ^ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,rsa-sha2-256,rsa-sha2-512 \ No newline at end of file diff --git a/rhel-fips/overlay/rhel10/system/oem/33_tmp_mount.yaml b/rhel-fips/overlay/rhel10/system/oem/33_tmp_mount.yaml new file mode 100644 index 00000000..09a5f98d --- /dev/null +++ b/rhel-fips/overlay/rhel10/system/oem/33_tmp_mount.yaml @@ -0,0 +1,10 @@ +name: " tmp layout setup" +stages: + initramfs.after: + - name: mount tmp + commands: + - systemctl enable tmp.mount + fs.before: + - name: start tmp + commands: + - systemctl start tmp.mount diff --git a/rhel-stig/Dockerfile.rhel9 b/rhel-stig/Dockerfile.rhel9 index 503cafa7..b2e962b6 100644 --- a/rhel-stig/Dockerfile.rhel9 +++ b/rhel-stig/Dockerfile.rhel9 @@ -1,7 +1,7 @@ -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM registry.access.redhat.com/ubi9-init:9.4-6 -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 RUN dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm -y # Credentials come from a BuildKit secret (never in image layers, build args, or docker history). diff --git a/rhel-stig/Dockerfile.rhel9-fips b/rhel-stig/Dockerfile.rhel9-fips index c8922922..2446aa7c 100644 --- a/rhel-stig/Dockerfile.rhel9-fips +++ b/rhel-stig/Dockerfile.rhel9-fips @@ -1,7 +1,7 @@ -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM registry.access.redhat.com/ubi9-init:9.4-6 -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 RUN dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm -y # Credentials come from a BuildKit secret (never in image layers, build args, or docker history). diff --git a/scripts/install-amdgpu-drivers.sh b/scripts/install-amdgpu-drivers.sh new file mode 100755 index 00000000..6d2889e8 --- /dev/null +++ b/scripts/install-amdgpu-drivers.sh @@ -0,0 +1,533 @@ +#!/usr/bin/env bash +# +# install-amdgpu-drivers.sh +# +# Pre-provision the AMD Instinct GPU kernel-mode driver INTO a CanvOS / Kairos +# Ubuntu base image, so a node booted from the image can run the AMD GPU Operator +# with `driver.enable=false` in a fully air-gapped environment. +# +# MODES (AMDGPU_DRIVER_SOURCE) +# ---------------------------- +# dkms (default) -- install AMD's amdgpu-dkms. Two execution paths: +# (a) if AMDGPU_ARTIFACT_PATH is set (Earthly build via +# earthly.sh's prebuild helper), extract the pre- +# compiled modules + firmware + drop-ins from the +# tarball, depmod, rebuild initrd. Fast. +# (b) otherwise download from repo.radeon.com and +# DKMS-build against the image kernel in-place. +# Works under `docker run --privileged`; FAILS in +# Earthly's buildkit RUN sandbox at AMD's ./configure +# step -- use path (a) for Earthly. +# inbox -- do NOT install the AMD apt repo or amdgpu-dkms. Rely on +# the in-tree `amdgpu` module that Ubuntu ships with +# linux-modules-$(uname -r) and the firmware blobs in +# linux-firmware. Only ensures amdgpu autoloads. Choose +# this when you accept the in-tree driver's feature set +# (may miss recent SMU / per-SKU support). +# +# WHAT THIS COVERS (dkms mode, OS side only) +# ------------------------------------------ +# * build toolchain (gcc, make, dkms, kmod, libc headers) +# * kernel headers matching the image kernel (via install-kernel-headers.sh) +# * linux-modules-extra for the image kernel +# * amdgpu-dkms + amdgpu-dkms-firmware, built against the IMAGE kernel +# * amdgpu module autoload + initrd refresh +# * a marker at /etc/canvos/amdgpu-driver-source recording which mode ran +# +# WHAT THIS DOES *NOT* COVER (both modes) -- ship these as container images in +# your content bundle, deployed by the AMD GPU Operator itself: +# * ROCm user-space, device-plugin, node-labeller, metrics exporter, etc. +# +# At Helm-install time you MUST tell the operator the driver is pre-installed: +# --set driver.enable=false # note: "enable", not "enabled" +# +# WHY THE DKMS DANCE (same rationale as install-nvidia-drivers.sh) +# --------------------------------------------------------------- +# In an Earthly/Docker build `uname -r` is the BUILD HOST kernel, not the kernel +# baked into the image. We derive the target kernel from /lib/modules (the +# kernel that will actually boot) and force the DKMS build + module install + +# depmod against THAT kernel. +# +# TUNABLES (environment variables; all optional) +# AMDGPU_DRIVER_SOURCE dkms | inbox. Default: dkms. +# AMDGPU_DRIVER_RELEASE amdgpu-install release marker (URL segment under +# repo.radeon.com/amdgpu-install//). AMD publishes +# both ROCm-alias paths (7.2.1, 7.2.4) and driver- +# release-marker paths (30.30.1, 30.30.4, 31.30); +# either form is accepted. Default: 7.2.1 -- pairs +# with GPU Operator v1.5.0 per its release notes. +# The 31.x line is tech-preview and pairs only with +# ROCm 7.13.0 tech-preview; do not mix with a +# production operator. Ignored in inbox mode. See +# docs/amd-gpu-airgapped.md for the compat matrix. +# AMDGPU_REBUILD_INITRD "true" to rebuild the initrd for the image kernel. +# Default: false. amdgpu is intentionally NOT +# included in the initrd (see the dracut omit +# drop-in the script writes). The base image's +# existing initrd already handles rootfs mount; +# amdgpu loads after switch-root via +# /etc/modules-load.d/amdgpu.conf. +# +set -eo pipefail +set -u + +log() { echo "[install-amdgpu-drivers] $*"; } +warn() { echo "[install-amdgpu-drivers] WARNING: $*" >&2; } +die() { echo "[install-amdgpu-drivers] ERROR: $*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +AMDGPU_DRIVER_SOURCE="${AMDGPU_DRIVER_SOURCE:-dkms}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-7.2.1}" +AMDGPU_REBUILD_INITRD="${AMDGPU_REBUILD_INITRD:-false}" + +case "${AMDGPU_DRIVER_SOURCE}" in + dkms|inbox) ;; + *) die "AMDGPU_DRIVER_SOURCE must be 'dkms' or 'inbox' (got: '${AMDGPU_DRIVER_SOURCE}')." ;; +esac + +export DEBIAN_FRONTEND=noninteractive + +command -v apt-get >/dev/null 2>&1 || die "this script only supports apt-based (Ubuntu/Debian) images." + +# --------------------------------------------------------------------------- +# 1. Identify the kernel shipped in the image (NOT the build host kernel) +# --------------------------------------------------------------------------- +KVER="$(printf '%s\n' /lib/modules/* 2>/dev/null | xargs -n1 basename 2>/dev/null | sort -V | tail -1)" +[ -n "${KVER}" ] || die "could not determine target kernel from /lib/modules." +log "Target (image) kernel: ${KVER}" +log "Driver source mode: ${AMDGPU_DRIVER_SOURCE}" +[ "${AMDGPU_DRIVER_SOURCE}" = "dkms" ] && log "AMD driver release: ${AMDGPU_DRIVER_RELEASE}" + +# Ubuntu release codename (jammy / noble) read from the image itself. +codename="" +osid="ubuntu" +if [ -r /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release + codename="${VERSION_CODENAME:-}" + osid="${ID:-ubuntu}" +fi +[ -n "${codename}" ] || die "could not determine Ubuntu codename from /etc/os-release." + +mkdir -p /etc/canvos + +# --------------------------------------------------------------------------- +# INBOX MODE: skip the AMD apt repo and DKMS entirely. Rely on the in-tree +# amdgpu module shipped with the image's linux-modules-* package. Only ensure +# the module autoloads at boot, then rebuild initrd if requested. +# --------------------------------------------------------------------------- +if [ "${AMDGPU_DRIVER_SOURCE}" = "inbox" ]; then + log "inbox mode: verifying in-tree amdgpu module is present under /lib/modules/${KVER}/kernel/..." + if ! find "/lib/modules/${KVER}" -path '*/kernel/drivers/gpu/drm/amd/amdgpu/amdgpu.ko*' 2>/dev/null | grep -q .; then + die "inbox mode selected but no in-tree amdgpu module found under /lib/modules/${KVER}/kernel/. \ +This image kernel does not ship an in-tree amdgpu driver -- switch to \ +AMDGPU_DRIVER_SOURCE=dkms or pick a different image kernel." + fi + + log "Configuring amdgpu module autoload ..." + cat > /etc/modules-load.d/amdgpu.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh (inbox mode) +# Load the in-tree AMD GPU driver at boot so the AMD GPU Operator sees a ready driver. +amdgpu +EOF + + log "Running depmod -a ${KVER} ..." + depmod -a "${KVER}" + + # Even if AMDGPU_REBUILD_INITRD=true, we explicitly OMIT amdgpu from the + # initrd. On multi-GPU MI systems (e.g. 8x MI325X, 8 XCP partitions each) + # amdgpu init emits so many udev events that dracut's initqueue times out + # waiting for udev-settle before rootfs pivot, leaving the node in + # dracut-emergency. amdgpu isn't needed to mount root (NVMe/SATA use their + # own drivers) so it's safe to load it *after* switch-root via + # /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. + log "Configuring dracut to OMIT amdgpu from initrd (avoid init-time udev storm) ..." + mkdir -p /etc/dracut.conf.d + cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. amdgpu emits enough +# udev events at load time (per-XCP-partition, per-ring) to blow past dracut's +# initqueue timeout on multi-GPU systems, dropping the node into emergency +# mode. amdgpu is not required to mount the rootfs; systemd loads it via +# modules-load.d after switch-root. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + + if [ "${AMDGPU_REBUILD_INITRD}" = "true" ]; then + if command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." + dracut -f "/boot/initrd-${KVER}" "${KVER}" + ln -sf "initrd-${KVER}" /boot/initrd + elif command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" + else + warn "no dracut or update-initramfs found; skipping initrd rebuild." + fi + fi + + printf 'AMDGPU_DRIVER_SOURCE=inbox\nKVER=%s\n' "${KVER}" > /etc/canvos/amdgpu-driver-source + + log "Done. Using in-tree amdgpu driver for kernel ${KVER}." + log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." + exit 0 +fi + +# --------------------------------------------------------------------------- +# DKMS MODE with a pre-built artifact (produced by +# scripts/prebuild-amdgpu-artifact.sh on the build host). +# +# Buildkit's RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc probe, +# so we compile outside Earthly in `docker run --privileged` and consume the +# resulting tarball here. Structurally this branch just extracts the tarball, +# runs depmod against the target kernel, and rebuilds the initrd. +# --------------------------------------------------------------------------- +AMDGPU_ARTIFACT_PATH="${AMDGPU_ARTIFACT_PATH:-}" +if [ "${AMDGPU_DRIVER_SOURCE}" = "dkms" ] && [ -n "${AMDGPU_ARTIFACT_PATH}" ]; then + log "dkms mode: consuming pre-built artifact ${AMDGPU_ARTIFACT_PATH}" + [ -s "${AMDGPU_ARTIFACT_PATH}" ] || die "AMDGPU_ARTIFACT_PATH='${AMDGPU_ARTIFACT_PATH}' \ +is not a non-empty file inside the build container. Verify the earthly.sh \ +wrapper produced it and Earthly COPYed it in." + + log "Extracting artifact into root filesystem ..." + tar -xzf "${AMDGPU_ARTIFACT_PATH}" -C / \ + || die "tar extraction of ${AMDGPU_ARTIFACT_PATH} failed." + + MODDIR="/lib/modules/${KVER}" + if ! find "${MODDIR}/updates/dkms" -name 'amdgpu.ko*' 2>/dev/null | grep -q .; then + find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' >&2 || true + die "amdgpu module missing under ${MODDIR}/updates/dkms after extract. \ +Artifact was built for a different kernel? Delete build/amdgpu-artifact-*.tar.gz \ +and rebuild (AMDGPU_FORCE_REBUILD=1) or verify BASE_IMAGE matches." + fi + + log "Running depmod -a ${KVER} ..." + depmod -a "${KVER}" || die "depmod failed for ${KVER}." + + # Even if AMDGPU_REBUILD_INITRD=true, we explicitly OMIT amdgpu from the + # initrd. On multi-GPU MI systems (e.g. 8x MI325X, 8 XCP partitions each) + # amdgpu init emits so many udev events that dracut's initqueue times out + # waiting for udev-settle before rootfs pivot, leaving the node in + # dracut-emergency. amdgpu isn't needed to mount root (NVMe/SATA use their + # own drivers) so it's safe to load it *after* switch-root via + # /etc/modules-load.d/amdgpu.conf where there is no timeout pressure. + log "Configuring dracut to OMIT amdgpu from initrd (avoid init-time udev storm) ..." + mkdir -p /etc/dracut.conf.d + cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. amdgpu emits enough +# udev events at load time (per-XCP-partition, per-ring) to blow past dracut's +# initqueue timeout on multi-GPU systems, dropping the node into emergency +# mode. amdgpu is not required to mount the rootfs; systemd loads it via +# modules-load.d after switch-root. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + + if [ "${AMDGPU_REBUILD_INITRD}" = "true" ]; then + if command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." + dracut -f "/boot/initrd-${KVER}" "${KVER}" + ln -sf "initrd-${KVER}" /boot/initrd + elif command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" + else + warn "no dracut or update-initramfs found; skipping initrd rebuild." + fi + fi + + # Marker written by the prebuild is preserved from the tar. Overwrite + # any prebuild-mode marker with the final in-image reality. + printf 'AMDGPU_DRIVER_SOURCE=dkms (artifact)\nAMDGPU_DRIVER_RELEASE=%s\nKVER=%s\n' \ + "${AMDGPU_DRIVER_RELEASE}" "${KVER}" > /etc/canvos/amdgpu-driver-source + + log "Done. AMD amdgpu driver (release ${AMDGPU_DRIVER_RELEASE}, artifact) baked in for kernel ${KVER}." + log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." + exit 0 +fi + +# --------------------------------------------------------------------------- +# DKMS MODE in-buildkit (fallback). Runs the full apt + DKMS build inside +# the image. This path fails inside Earthly's buildkit RUN sandbox at AMD's +# ./configure step (see docs) but is retained for: +# - direct `docker run --privileged` invocations (proven working), +# - the scripts/prebuild-amdgpu-artifact.sh helper, which uses this same +# script inside the container it spawns. +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# 2. Build toolchain +# --------------------------------------------------------------------------- +log "Installing build toolchain ..." +apt-get update || true +# The full Kbuild bootstrap: gcc/make/libc from build-essential PLUS the +# tools that recent kernel Makefiles pull in unconditionally. Missing any of +# these fails AMD's amdgpu-dkms ./configure at "cannot detect CFLAGS..." -- +# the failure mode is silent because CFLAGS-detection just runs `make -f -` +# and swallows stderr. We enumerate them explicitly instead of relying on +# --install-recommends (which would also pull other unwanted docs/data). +# bc, bison, flex : referenced by kernel Kbuild machinery +# libelf-dev : module utilities (modpost) + BPF +# libssl-dev : signing certificates / hash routines +# dwarves : pahole for BTF debuginfo (amdgpu-dkms explicitly Recommends this) +# cpio, xz-utils : initramfs assembly (may be needed by initramfs-tools trigger) +apt-get install -y --no-install-recommends \ + ca-certificates curl wget gnupg \ + build-essential gcc make \ + dkms kmod libc6-dev initramfs-tools \ + bc bison flex libelf-dev libssl-dev dwarves \ + cpio xz-utils \ + || die "failed to install build toolchain." + +# --------------------------------------------------------------------------- +# 3. Kernel headers + modules-extra matching the image kernel +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +HEADERS_HELPER="" +for cand in "${SCRIPT_DIR}/install-kernel-headers.sh" /tmp/install-kernel-headers.sh; do + [ -r "${cand}" ] && { HEADERS_HELPER="${cand}"; break; } +done + +if [ -n "${HEADERS_HELPER}" ]; then + log "Installing kernel headers via ${HEADERS_HELPER} ..." + bash "${HEADERS_HELPER}" || warn "kernel-headers helper returned non-zero; continuing." +else + log "Header helper not found; attempting a direct header install ..." + apt-get install -y "linux-headers-${KVER}" || \ + apt-get install -y linux-headers-generic || \ + warn "could not install linux-headers-${KVER}." +fi + +# amdgpu depends on modules that live in linux-modules-extra (e.g. for some +# PCIe / crypto / networking helpers). Best-effort -- may be absent if Ubuntu +# rotated the ABI out of the live mirror. +log "Installing linux-modules-extra-${KVER} (best-effort) ..." +apt-get install -y "linux-modules-extra-${KVER}" || \ + warn "linux-modules-extra-${KVER} not available; continuing." + +# DKMS needs /lib/modules//build to point at the headers source tree. +if [ ! -e "/lib/modules/${KVER}/build" ]; then + src="$(ls -d /usr/src/linux-headers-${KVER} 2>/dev/null | head -1)" + if [ -n "${src}" ]; then + ln -sfn "${src}" "/lib/modules/${KVER}/build" + log "Linked /lib/modules/${KVER}/build -> ${src}" + else + warn "no /usr/src/linux-headers-${KVER}; DKMS build will likely fail." + fi +fi + +# --------------------------------------------------------------------------- +# 4. Register the AMD driver repo via the amdgpu-install package +# The amdgpu-install .deb (AMD's blessed entry point) configures the correct +# versioned driver apt repo + GPG key for the requested release. Its +# filename carries a build number, so we auto-discover it from the directory +# listing rather than hardcoding it. +# --------------------------------------------------------------------------- +inst_dir="https://repo.radeon.com/amdgpu-install/${AMDGPU_DRIVER_RELEASE}/${osid}/${codename}" +log "Locating amdgpu-install package under ${inst_dir}/ ..." +deb_name="$(curl -fsSL "${inst_dir}/" 2>/dev/null \ + | grep -oE 'amdgpu-install_[0-9A-Za-z._-]+_all\.deb' | sort -u | tail -1)" +[ -n "${deb_name}" ] || die "could not find an amdgpu-install package for driver \ +release '${AMDGPU_DRIVER_RELEASE}' on ${codename} at ${inst_dir}/. Check available \ +releases at https://repo.radeon.com/amdgpu-install/ and pick one that supports \ +your image kernel (${KVER}); see docs/amd-gpu-airgapped.md for the mapping." + +log "Installing ${deb_name} (configures the AMD driver apt repo) ..." +wget -qO /tmp/amdgpu-install.deb "${inst_dir}/${deb_name}" \ + || die "failed to download ${deb_name}." +apt-get install -y /tmp/amdgpu-install.deb || die "failed to install amdgpu-install." +rm -f /tmp/amdgpu-install.deb +apt-get update || warn "apt-get update after adding the AMD repo failed." + +# --------------------------------------------------------------------------- +# 4b. Disable DKMS module signing before installing amdgpu-dkms. +# +# amdgpu-dkms (>= 6.18 range, and observed on 31.x releases) invokes mokutil +# from the DKMS sign_tool hook to enroll a Machine Owner Key, which reads +# /sys/firmware/efi/efivars. Docker/Earthly build containers don't expose +# efivars, so mokutil aborts with: +# "EFI variables are not supported on this system / +# /sys/firmware/efi/efivars not found, aborting." +# and the amdgpu-dkms postinst returns non-zero. Empty sign_tool tells DKMS +# to skip signing entirely, sidestepping the mokutil invocation. +# +# CAVEAT: modules produced this way are unsigned -- consistent with the +# Secure Boot / UKI limitation already documented in docs/amd-gpu-airgapped.md. +# --------------------------------------------------------------------------- +log "Disabling DKMS module signing (container has no UEFI efivars) ..." +mkdir -p /etc/dkms/framework.conf.d +cat > /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Empty sign_tool tells DKMS to skip module signing. Required for building +# amdgpu-dkms inside container image builds where /sys/firmware/efi/efivars +# is not available. Modules are unsigned; this path does not support Secure Boot. +sign_tool="" +EOF + +# --------------------------------------------------------------------------- +# 5. Install the kernel-mode driver (amdgpu-dkms + firmware). +# Any apt/postinst failure surfaces here -- DO NOT swallow errors; a broken +# DKMS build must fail the image build so the user can fix AMDGPU_DRIVER_RELEASE +# or fall back to AMDGPU_DRIVER_SOURCE=inbox. +# --------------------------------------------------------------------------- +log "Installing amdgpu-dkms + amdgpu-dkms-firmware ..." +if ! apt-get install -y --no-install-recommends amdgpu-dkms amdgpu-dkms-firmware; then + # apt/postinst failure -- dump the DKMS build artifacts so root-causing + # doesn't require an interactive session (Earthly's -i tty is often broken). + log "apt install failed. Dumping DKMS build artifacts for diagnosis:" + log "--- dkms status ---" + dkms status 2>&1 | sed 's/^/ /' || true + for f in /var/lib/dkms/amdgpu/*/build/make.log; do + [ -r "$f" ] || continue + log "--- ${f} (tail -150) ---" + tail -n 150 "$f" | sed 's/^/ /' || true + done + log "--- environment probes ---" + log " kernel: $(uname -r); target KVER: ${KVER}" + log " linux-headers pkg: $(dpkg -l "linux-headers-${KVER}" 2>/dev/null | awk '/^ii/{print $2, $3}')" + log " /lib/modules/${KVER}/build: $(readlink -f "/lib/modules/${KVER}/build" 2>/dev/null || echo MISSING)" + log " /usr/src/linux-headers-${KVER}/Module.symvers: $(test -s "/usr/src/linux-headers-${KVER}/Module.symvers" && echo present || echo missing/empty)" + log " sign_tool drop-in: $(test -r /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf && grep -E '^sign_tool' /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf || echo MISSING)" + log " memory: $(awk '/MemAvailable/{print $2/1024" MiB avail"}' /proc/meminfo)" + die "failed to install amdgpu-dkms (release '${AMDGPU_DRIVER_RELEASE}') for \ +kernel ${KVER}. See make.log tail above. Common causes: (1) the AMD driver source \ +in this release does not support this kernel -- bump AMDGPU_DRIVER_RELEASE (see \ +docs/amd-gpu-airgapped.md); (2) linux-headers-${KVER} not installed / Module.symvers \ +empty; (3) DKMS module signing failed reaching /sys/firmware/efi/efivars -- normally \ +handled by the sign_tool='' drop-in above. Workaround: rerun with \ +AMDGPU_DRIVER_SOURCE=inbox to use the in-tree amdgpu." +fi + +# --------------------------------------------------------------------------- +# 6. Build the DKMS module against the IMAGE kernel (not the build host). +# apt-get's postinst may have already tried against $KVER; we re-run +# explicitly and let failures propagate (no `|| true`). +# --------------------------------------------------------------------------- +command -v dkms >/dev/null 2>&1 || die "dkms binary not found after installing amdgpu-dkms." + +log "Building amdgpu DKMS module for kernel ${KVER} ..." +# `dkms status` differs across versions: +# dkms 2.x: "amdgpu, 6.19.4, 6.14.0-36-generic, x86_64: installed" +# dkms 3.x: "amdgpu/6.19.4, 6.14.0-36-generic, x86_64: installed" +# We want the module-name and source-version (not the kernel). +dkms_line="$(dkms status 2>/dev/null | grep -iE '^amdgpu[/,]' | head -1 || true)" +[ -n "${dkms_line}" ] || die "dkms status does not know about the amdgpu module \ +after apt install -- driver package is broken or DKMS registration failed." + +mod="$(printf '%s\n' "${dkms_line}" | sed -E 's/[,/:].*//' | tr -d ' ')" +ver="$(printf '%s\n' "${dkms_line}" | sed -E 's/^[^/,]+[/,] *//' | sed -E 's/[,:].*//' | tr -d ' ')" +# Fallback: grep any version-looking token if the second column wasn't the version. +if ! printf '%s' "${ver}" | grep -qE '^[0-9]+\.[0-9]+'; then + ver="$(printf '%s\n' "${dkms_line}" | grep -oE '[0-9]+\.[0-9]+[0-9.]*' | head -1)" +fi +[ -n "${mod}" ] && [ -n "${ver}" ] || die "could not parse dkms status line: '${dkms_line}'" + +log " dkms build ${mod}/${ver} -k ${KVER}" +if ! dkms build -m "${mod}" -v "${ver}" -k "${KVER}"; then + log "DKMS build failed. Full make.log tail:" + tail -n 60 "/var/lib/dkms/${mod}/${ver}/build/make.log" 2>&1 | sed 's/^/ /' || true + die "DKMS build of ${mod}/${ver} against kernel ${KVER} failed. AMD driver \ +release '${AMDGPU_DRIVER_RELEASE}' likely does not support this kernel. Either \ +bump AMDGPU_DRIVER_RELEASE (see docs/amd-gpu-airgapped.md) or rerun with \ +AMDGPU_DRIVER_SOURCE=inbox." +fi + +log " dkms install ${mod}/${ver} -k ${KVER}" +dkms install -m "${mod}" -v "${ver}" -k "${KVER}" --force \ + || die "dkms install of ${mod}/${ver} against kernel ${KVER} failed." + +log "DKMS status:"; dkms status 2>&1 | sed 's/^/ /' || true + +# --------------------------------------------------------------------------- +# 7. Verify the DKMS-built module actually landed under updates/dkms and that +# dkms considers it installed for the target kernel. The in-tree amdgpu +# that Ubuntu ships under kernel/... does NOT count -- we're only satisfied +# if the OOT driver made it in. +# --------------------------------------------------------------------------- +MODDIR="/lib/modules/${KVER}" +dkms_mod_found="" +if find "${MODDIR}/updates" -name 'amdgpu.ko*' 2>/dev/null | grep -q .; then + dkms_mod_found="yes" +fi + +dkms_installed="" +if dkms status 2>/dev/null \ + | grep -iE "^amdgpu[/,][^,]*,[[:space:]]*${KVER}[,]" \ + | grep -q ': installed'; then + dkms_installed="yes" +fi + +if [ -z "${dkms_mod_found}" ] || [ -z "${dkms_installed}" ]; then + log "Verification failed:" + log " updates/dkms module present under ${MODDIR}/updates: ${dkms_mod_found:-no}" + log " dkms status shows 'installed' for kernel ${KVER}: ${dkms_installed:-no}" + find "${MODDIR}" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' || true + die "amdgpu DKMS module was NOT built+installed for kernel ${KVER}. The \ +in-tree amdgpu (if any) is NOT sufficient in dkms mode -- rerun with \ +AMDGPU_DRIVER_SOURCE=inbox if that is what you want." +fi + +log "Verified: amdgpu DKMS module installed for ${KVER}." +find "${MODDIR}/updates" -name 'amdgpu.ko*' 2>/dev/null | sed 's/^/ /' + +# --------------------------------------------------------------------------- +# 8. Autoload amdgpu at boot (no blacklist needed -- dkms replaces the in-tree +# module of the same name via depmod's updates/ override). +# --------------------------------------------------------------------------- +log "Configuring amdgpu module autoload ..." +cat > /etc/modules-load.d/amdgpu.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh (dkms mode) +# Load the AMD GPU driver at boot so the AMD GPU Operator sees a ready driver. +amdgpu +EOF + +# --------------------------------------------------------------------------- +# 9. depmod for the target kernel so modprobe resolves amdgpu at boot +# --------------------------------------------------------------------------- +log "Running depmod -a ${KVER} ..." +depmod -a "${KVER}" || warn "depmod reported an error." + +# --------------------------------------------------------------------------- +# 10. Drop dracut config that OMITS amdgpu from any rebuilt initrd. See the +# equivalent block in the artifact / inbox branches for full rationale -- +# multi-GPU amdgpu init blows past initqueue's timeout when loaded early. +# --------------------------------------------------------------------------- +mkdir -p /etc/dracut.conf.d +cat > /etc/dracut.conf.d/98-canvos-amdgpu-omit.conf <<'EOF' +# Managed by CanvOS install-amdgpu-drivers.sh +# Keep amdgpu (and its DKMS helpers) OUT of the initrd. See install script. +omit_drivers+=" amdgpu amdttm amdkcl amd-sched amddrm_ttm_helper amddrm_buddy amddrm_exec amdxcp " +EOF + +# --------------------------------------------------------------------------- +# 11. Rebuild the initrd for the target kernel (amdgpu omitted per above). +# --------------------------------------------------------------------------- +if [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut, amdgpu omitted) ..." + if dracut -f "/boot/initrd-${KVER}" "${KVER}"; then + ln -sf "initrd-${KVER}" /boot/initrd + else + warn "dracut initrd rebuild failed." + fi +elif [ "${AMDGPU_REBUILD_INITRD}" = "true" ] && command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" || warn "update-initramfs failed." +fi + +# --------------------------------------------------------------------------- +# 11. Record what we did so ops can query it on-node. +# --------------------------------------------------------------------------- +printf 'AMDGPU_DRIVER_SOURCE=dkms\nAMDGPU_DRIVER_RELEASE=%s\nAMDGPU_DKMS_MODULE=%s/%s\nKVER=%s\n' \ + "${AMDGPU_DRIVER_RELEASE}" "${mod}" "${ver}" "${KVER}" \ + > /etc/canvos/amdgpu-driver-source + +# --------------------------------------------------------------------------- +# 12. Cleanup apt caches to keep the image lean +# --------------------------------------------------------------------------- +apt-get clean +rm -rf /var/lib/apt/lists/* + +log "Done. AMD amdgpu driver (release ${AMDGPU_DRIVER_RELEASE}) baked in for kernel ${KVER}." +log "Reminder: install the AMD GPU Operator with 'driver.enable=false'." diff --git a/scripts/install-nvidia-drivers.sh b/scripts/install-nvidia-drivers.sh new file mode 100755 index 00000000..0acde144 --- /dev/null +++ b/scripts/install-nvidia-drivers.sh @@ -0,0 +1,446 @@ +#!/usr/bin/env bash +# +# install-nvidia-drivers.sh +# +# Pre-install the NVIDIA data-center GPU driver and build its kernel modules +# INTO a CanvOS / Kairos Ubuntu base image, so that a node booted from the +# image can run the NVIDIA GPU Operator in a fully air-gapped environment +# WITHOUT any host-side network access and WITHOUT the operator's driver +# container. +# +# WHAT THIS COVERS (OS side only) +# ------------------------------- +# * build toolchain (gcc, make, dkms, kmod, libc headers) +# * kernel headers that match the kernel shipped in the image +# (delegated to scripts/install-kernel-headers.sh) +# * the NVIDIA driver user-space + `nvidia-smi` (nvidia-utils-*-server) +# * the NVIDIA kernel modules (nvidia, nvidia-uvm, nvidia-modeset, +# nvidia-drm, nvidia-peermem) built with DKMS against the IMAGE kernel +# * nouveau blacklist + nvidia module autoload + nvidia-persistenced +# * nvidia-fabricmanager + libnvidia-nscq for HGX / NVSwitch systems +# (HGX H100/H200, HGX B200, DGX, GB200) -- required for multi-GPU NVLink +# * nvidia-imex for GB200 NVL72 multi-node NVLink Sharp (Blackwell) +# +# WHAT THIS DOES *NOT* COVER (ships as container images in your content bundle, +# deployed by the GPU Operator itself): +# * nvidia-container-toolkit / runtime class +# * k8s-device-plugin, gpu-feature-discovery, DCGM exporter, MIG manager, ... +# +# At Helm-install time you MUST tell the operator the driver is pre-installed: +# --set driver.enabled=false +# (and, if you also pre-install the toolkit below, --set toolkit.enabled=false) +# +# WHY THE DKMS DANCE +# ------------------ +# In an Earthly/Docker build `uname -r` is the BUILD HOST kernel, not the kernel +# baked into the image. If we let apt/DKMS build "for the running kernel" the +# modules would target the wrong ABI (or fail). We therefore derive the target +# kernel from /lib/modules (the kernel that will actually boot) and force every +# DKMS build + module install + depmod against THAT kernel. +# +# CONNECTIVITY +# ------------ +# This script runs at BUILD time, where the builder has internet. It bakes +# everything into the image. The resulting image needs no network at boot. +# +# TUNABLES (environment variables; all optional) +# NVIDIA_DRIVER_BRANCH Driver branch to install (e.g. 550, 570, 580). +# Default: 580 (a data-center production branch) +# NVIDIA_DRIVER_TYPE "proprietary" | "open" Default: open +# "open" is REQUIRED on Hopper (H100/H200) and +# Blackwell (RTX PRO 6000 Blackwell, B100, B200, +# GB200) — the closed modules fail with +# "RmInitAdapter (0x22:0x56:897)" on those GPUs +# and `nvidia-smi` reports "No devices were found". +# Also safe on Turing/Ampere/Ada. Override to +# "proprietary" only for pre-Turing hardware +# (Pascal/Volta). +# NVIDIA_USE_CUDA_REPO "true" to add developer.download.nvidia.com CUDA +# repo (recommended, has every -server branch). +# "false" to use only Ubuntu's own repos. +# Default: true +# NVIDIA_INSTALL_FABRICMANAGER "true" for NVSwitch/HGX boxes (H100/B200 HGX, +# DGX, GB200). Default: true. Harmless on non- +# NVSwitch hosts: the unit exits early and stays +# inactive; no restart loop, no kernel effect, +# ~60-90 MB image cost. Also installs the +# matching libnvidia-nscq- explicitly. +# See docs/nvidia-gpu-airgapped.md. +# NVIDIA_INSTALL_IMEX "true" to also install nvidia-imex-, +# the Internode Memory Exchange daemon required +# for GB200 NVL72 multi-node NVLink Sharp +# (Blackwell, driver 570+). Default: true. +# Harmless on non-NVL72 boxes: without +# /etc/nvidia-imex/nodes_config.cfg the daemon +# exits and the unit stays inactive. Best-effort +# -- skipped with a warning on branches that +# predate IMEX (pre-570). +# NVIDIA_INSTALL_CONTAINER_TOOLKIT "true" to ALSO pre-install +# nvidia-container-toolkit on the host (then set +# toolkit.enabled=false in the operator). +# Default: false (operator ships it) +# NVIDIA_REBUILD_INITRD "true" to rebuild the initrd so the nouveau +# blacklist takes effect in early boot. +# Default: true +# +set -u + +log() { echo "[install-nvidia-drivers] $*"; } +warn() { echo "[install-nvidia-drivers] WARNING: $*" >&2; } +die() { echo "[install-nvidia-drivers] ERROR: $*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +NVIDIA_DRIVER_BRANCH="${NVIDIA_DRIVER_BRANCH:-580}" +NVIDIA_DRIVER_TYPE="${NVIDIA_DRIVER_TYPE:-open}" +NVIDIA_USE_CUDA_REPO="${NVIDIA_USE_CUDA_REPO:-true}" +NVIDIA_INSTALL_FABRICMANAGER="${NVIDIA_INSTALL_FABRICMANAGER:-true}" +NVIDIA_INSTALL_IMEX="${NVIDIA_INSTALL_IMEX:-true}" +NVIDIA_INSTALL_CONTAINER_TOOLKIT="${NVIDIA_INSTALL_CONTAINER_TOOLKIT:-false}" +NVIDIA_REBUILD_INITRD="${NVIDIA_REBUILD_INITRD:-true}" + +export DEBIAN_FRONTEND=noninteractive + +command -v apt-get >/dev/null 2>&1 || die "this script only supports apt-based (Ubuntu/Debian) images." + +# --------------------------------------------------------------------------- +# 1. Identify the kernel shipped in the image (NOT the build host kernel) +# --------------------------------------------------------------------------- +KVER="$(printf '%s\n' /lib/modules/* 2>/dev/null | xargs -n1 basename 2>/dev/null | sort -V | tail -1)" +[ -n "${KVER}" ] || die "could not determine target kernel from /lib/modules." +log "Target (image) kernel: ${KVER}" +log "Driver branch: ${NVIDIA_DRIVER_BRANCH} (${NVIDIA_DRIVER_TYPE})" + +# --------------------------------------------------------------------------- +# 2. Build toolchain +# --------------------------------------------------------------------------- +log "Installing build toolchain ..." +apt-get update || true +apt-get install -y --no-install-recommends \ + ca-certificates curl wget gnupg \ + build-essential gcc make \ + dkms kmod libc6-dev pkg-config \ + || die "failed to install build toolchain." + +# --------------------------------------------------------------------------- +# 3. Kernel headers matching the image kernel +# Reuse the repo's ABI-exact / snapshot-aware header installer if present. +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +HEADERS_HELPER="" +for cand in "${SCRIPT_DIR}/install-kernel-headers.sh" /tmp/install-kernel-headers.sh; do + [ -r "${cand}" ] && { HEADERS_HELPER="${cand}"; break; } +done + +if [ -n "${HEADERS_HELPER}" ]; then + log "Installing kernel headers via ${HEADERS_HELPER} ..." + bash "${HEADERS_HELPER}" || warn "kernel-headers helper returned non-zero; continuing." +else + log "Header helper not found; attempting a direct header install ..." + apt-get install -y "linux-headers-${KVER}" || \ + apt-get install -y linux-headers-generic || \ + warn "could not install linux-headers-${KVER}." +fi + +# DKMS needs /lib/modules//build to point at the headers source tree. +if [ ! -e "/lib/modules/${KVER}/build" ]; then + # Find the header tree that matches our kernel and symlink it. + src="$(ls -d /usr/src/linux-headers-${KVER} 2>/dev/null | head -1)" + if [ -n "${src}" ]; then + ln -sfn "${src}" "/lib/modules/${KVER}/build" + log "Linked /lib/modules/${KVER}/build -> ${src}" + else + warn "no /usr/src/linux-headers-${KVER}; DKMS build will likely fail." + fi +fi + +# --------------------------------------------------------------------------- +# 4. NVIDIA package repo (CUDA network repo — has every *-server branch) +# --------------------------------------------------------------------------- +if [ "${NVIDIA_USE_CUDA_REPO}" = "true" ]; then + # Derive the CUDA repo "distro" tag from the image (e.g. 22.04 -> ubuntu2204) + osid=""; osver="" + if [ -r /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release + osid="${ID:-ubuntu}" + osver="${VERSION_ID:-22.04}" + fi + distro="${osid}$(echo "${osver}" | tr -d '.')" # ubuntu2204, ubuntu2004, ... + case "$(uname -m)" in + x86_64) cudaarch="x86_64" ;; + aarch64) cudaarch="sbsa" ;; + *) cudaarch="x86_64" ;; + esac + repo_base="https://developer.download.nvidia.com/compute/cuda/repos/${distro}/${cudaarch}" + log "Adding NVIDIA CUDA repo: ${repo_base}" + if wget -qO /tmp/cuda-keyring.deb "${repo_base}/cuda-keyring_1.1-1_all.deb"; then + dpkg -i /tmp/cuda-keyring.deb || warn "cuda-keyring install failed." + rm -f /tmp/cuda-keyring.deb + apt-get update || warn "apt-get update after adding CUDA repo failed." + else + warn "could not download cuda-keyring; falling back to Ubuntu repos." + fi +fi + +# --------------------------------------------------------------------------- +# 5. Choose driver packages +# Headless server packages (no Xorg / GUI). nvidia-utils gives nvidia-smi. +# --------------------------------------------------------------------------- +if [ "${NVIDIA_DRIVER_TYPE}" = "open" ]; then + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-server-open" +else + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-server" +fi +UTILS_PKG="nvidia-utils-${NVIDIA_DRIVER_BRANCH}-server" + +log "Installing NVIDIA driver packages: ${HEADLESS_PKG} ${UTILS_PKG}" +if ! apt-get install -y --no-install-recommends "${HEADLESS_PKG}" "${UTILS_PKG}"; then + warn "'${HEADLESS_PKG}' not available; retrying with generic (non-server) branch." + if [ "${NVIDIA_DRIVER_TYPE}" = "open" ]; then + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}-open" + else + HEADLESS_PKG="nvidia-headless-${NVIDIA_DRIVER_BRANCH}" + fi + UTILS_PKG="nvidia-utils-${NVIDIA_DRIVER_BRANCH}" + apt-get install -y --no-install-recommends "${HEADLESS_PKG}" "${UTILS_PKG}" \ + || die "failed to install NVIDIA driver packages for branch ${NVIDIA_DRIVER_BRANCH}. \ +Check available branches with: apt-cache search 'nvidia-headless-.*-server'" +fi + +# --------------------------------------------------------------------------- +# 6. Build the DKMS modules against the IMAGE kernel (not the build host) +# --------------------------------------------------------------------------- +# The apt postinst runs `dkms autoinstall`, which only builds for kernels that +# have headers present -- i.e. our target kernel, since the build host kernel's +# headers are absent in the image. We still force it explicitly to be safe. +if command -v dkms >/dev/null 2>&1; then + log "Building NVIDIA DKMS modules for kernel ${KVER} ..." + # Explicitly (re)build every registered nvidia dkms module for the target. + # `dkms status` output differs across versions: + # dkms 2.x: "nvidia, 580.159.03, 6.14.0-36-generic, x86_64: installed" + # dkms 3.x: "nvidia/580.159.03, 6.14.0-36-generic, x86_64: installed" + # Extract the module name (up to the first , / or :) and the first + # version-looking token, which works for both formats. + dkms status 2>/dev/null | grep -i nvidia | while read -r line; do + mod="$(printf '%s\n' "${line}" | sed -E 's/[,/:].*//' | tr -d ' ')" + ver="$(printf '%s\n' "${line}" | grep -oE '[0-9]+\.[0-9]+[0-9.]*' | head -1)" + case "${mod}" in nvidia*) ;; *) continue ;; esac + [ -n "${mod}" ] && [ -n "${ver}" ] || continue + log " dkms install ${mod}/${ver} -k ${KVER}" + dkms build -m "${mod}" -v "${ver}" -k "${KVER}" 2>/dev/null || true + dkms install -m "${mod}" -v "${ver}" -k "${KVER}" --force 2>/dev/null || true + done + # Belt-and-suspenders: try the autoinstaller pinned to the target kernel + # (ignored gracefully by older dkms that lack the -k flag). + dkms autoinstall -k "${KVER}" 2>/dev/null || true + log "DKMS status:"; dkms status 2>/dev/null || true +else + warn "dkms not found; relying on apt postinst build." +fi + +# --------------------------------------------------------------------------- +# 7. Verify the modules actually landed in the image kernel's module tree +# --------------------------------------------------------------------------- +MODDIR="/lib/modules/${KVER}" +if ls "${MODDIR}"/updates/dkms/nvidia*.ko* >/dev/null 2>&1 || \ + ls "${MODDIR}"/kernel/drivers/video/nvidia*.ko* >/dev/null 2>&1 || \ + find "${MODDIR}" -name 'nvidia*.ko*' 2>/dev/null | grep -q .; then + log "Verified: nvidia kernel modules present under ${MODDIR}." + find "${MODDIR}" -name 'nvidia*.ko*' 2>/dev/null | sed 's/^/ /' +else + die "no nvidia*.ko modules found under ${MODDIR} -- DKMS build did not \ +produce modules for the image kernel. Check that linux-headers-${KVER} and a \ +matching gcc are installed." +fi + +# --------------------------------------------------------------------------- +# 8. NVIDIA Fabric Manager + libnvidia-nscq + nvlsm (HGX / NVSwitch systems) +# Required on HGX H100/H200, HGX B200, DGX, GB200 for multi-GPU NVLink. +# libnvidia-nscq- is a fabricmanager dep and gets pulled in +# transitively -- listed explicitly so the install fails loudly if the +# CUDA repo ever drops the auto-dep. +# +# NVIDIA 570+ ships the FM unit with ExecStart wrapped in +# /usr/share/nvidia/fabricmanager/nvidia-fabricmanager-start.sh, which +# probes `ibstat` (infiniband-diags) and `nvlsm` (NVIDIA Subnet Manager) +# before invoking nv-fabricmanager -- needed so the NVLink subnet is up +# on GB200 NVL72. Both binaries must be present or the unit dies: +# "ibstat command not found! Please install ibstat." +# "nvlsm command not found! Please install nvlsm." +# Empirically the wrapper succeeds on standalone HGX topologies once +# both are installed (verified on HGX with nvlsm 2025.10.14-1 from the +# NVIDIA CUDA repo we already added in step 4), so we install them +# alongside FM and let the vendor wrapper stay in charge. +# +# Package sources: nvlsm ships in the CUDA repo under an unversioned +# package name (not nvlsm-). infiniband-diags is Ubuntu-native. +# On non-NVSwitch hosts nv-fabricmanager still exits "No NvSwitch found" +# and the unit stays inactive; no kernel side effect, no restart loop. +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_FABRICMANAGER}" = "true" ]; then + FM_PKG="nvidia-fabricmanager-${NVIDIA_DRIVER_BRANCH}" + NSCQ_PKG="libnvidia-nscq-${NVIDIA_DRIVER_BRANCH}" + log "Installing ${FM_PKG} + ${NSCQ_PKG} + nvlsm + infiniband-diags ..." + if apt-get install -y --no-install-recommends \ + "${FM_PKG}" "${NSCQ_PKG}" nvlsm infiniband-diags; then + systemctl enable nvidia-fabricmanager.service 2>/dev/null || true + + # --- ib_umad autoload for fabricmanager precheck -------------------- + # NVIDIA 570+ nvidia-fabricmanager-start.sh --mode precheck (invoked + # by the systemd unit's ExecStartPre) takes the "Detected NVL5+ + # system" branch on HGX B200 / GB200 hardware and requires the + # ib_umad kernel module to be loaded before nv-fabricmanager can + # start -- the wrapper opens /dev/infiniband/umad* to send MADs to + # the NVSwitch fabric (NVSwitch reuses the InfiniBand management- + # datagram shape). Kairos edge images ship the module (it's in the + # kernel-modules-extra set) but don't auto-load it, so precheck + # dies with: + # Detected NVL5+ system + # Kernel module "ib_umad" has not been loaded, + # fabric manager cannot be started + # Please run "modprobe ib_umad" before starting fabric manager + # Load it at boot so the service comes up on NVSwitch hardware + # without any operator intervention. Verified live on 8x HGX B200 + # (driver 580.159.03): with this in place nvidia-fabricmanager.service + # goes active in ~3s at boot and downstream nvidia-cuda-validator + # exits 0 on the next GPU-operator reconcile. + # + # On non-NVSwitch hosts the module load costs ~10 KiB of RSS and has + # no other side effect; fabricmanager still exits "No NvSwitch found" + # and the unit stays inactive as before. + cat > /etc/modules-load.d/nvidia-fabricmanager.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +# Required by nvidia-fabricmanager-start.sh --mode precheck on NVL5+ +# systems (HGX B200, GB200) -- see /usr/bin/nvidia-fabricmanager-start.sh. +ib_umad +EOF + else + warn "could not install ${FM_PKG} / ${NSCQ_PKG} / nvlsm / infiniband-diags; skipping fabric manager." + fi +fi + +# --------------------------------------------------------------------------- +# 8b. NVIDIA IMEX -- Internode Memory Exchange daemon +# Required for GB200 NVL72 multi-node NVLink Sharp (Blackwell, driver +# 570+). Not needed for single-node HGX B200 or HGX H100. Package is +# part of the CUDA repo; older driver branches (pre-570) do not publish +# it, so this is best-effort. On single-node boxes the daemon has no +# /etc/nvidia-imex/nodes_config.cfg and exits cleanly -- unit stays +# inactive, ~10-20 MB image cost. +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_IMEX}" = "true" ]; then + IMEX_PKG="nvidia-imex-${NVIDIA_DRIVER_BRANCH}" + log "Installing ${IMEX_PKG} (GB200 NVL72 multi-node NVLink Sharp) ..." + if apt-get install -y --no-install-recommends "${IMEX_PKG}"; then + systemctl enable nvidia-imex.service 2>/dev/null || true + else + warn "${IMEX_PKG} not available (branch ${NVIDIA_DRIVER_BRANCH} may predate IMEX -- 570+ only). Skipping." + fi +fi + +# --------------------------------------------------------------------------- +# 9. Optional: nvidia-container-toolkit on the host +# (default OFF -- the GPU Operator ships and configures the toolkit) +# --------------------------------------------------------------------------- +if [ "${NVIDIA_INSTALL_CONTAINER_TOOLKIT}" = "true" ]; then + log "Installing nvidia-container-toolkit on host ..." + install -d -m 0755 /usr/share/keyrings + if curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ + | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg; then + curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ + | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ + > /etc/apt/sources.list.d/nvidia-container-toolkit.list + apt-get update && apt-get install -y --no-install-recommends nvidia-container-toolkit \ + || warn "nvidia-container-toolkit install failed." + else + warn "could not fetch nvidia-container-toolkit gpg key; skipping." + fi +fi + +# --------------------------------------------------------------------------- +# 10. Host module configuration: blacklist nouveau + autoload nvidia +# --------------------------------------------------------------------------- +log "Configuring nouveau blacklist and nvidia module autoload ..." +cat > /etc/modprobe.d/blacklist-nouveau.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +blacklist nouveau +blacklist lbm-nouveau +options nouveau modeset=0 +alias nouveau off +alias lbm-nouveau off +EOF + +cat > /etc/modules-load.d/nvidia.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +# Load the NVIDIA stack at boot so the GPU Operator sees a ready driver. +# nvidia_drm is intentionally omitted: it grabs KMS/DRM and on headless GPU +# nodes can wedge early boot. It loads on demand if anything wants it. +nvidia +nvidia_uvm +nvidia_modeset +EOF + +# NVIDIA driver run-time module options recommended for datacenter use: +# NVreg_OpenRmEnableUnsupportedGpus is only relevant for the open modules. +cat > /etc/modprobe.d/nvidia.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +options nvidia NVreg_PreserveVideoMemoryAllocations=1 +EOF + +# Enable the persistence daemon (recommended for datacenter GPUs). +systemctl enable nvidia-persistenced.service 2>/dev/null || true + +# --------------------------------------------------------------------------- +# 11. depmod for the target kernel so modprobe can resolve nvidia at boot +# --------------------------------------------------------------------------- +log "Running depmod -a ${KVER} ..." +depmod -a "${KVER}" || warn "depmod reported an error." + +# --------------------------------------------------------------------------- +# 12. Make sure the initrd honors the nouveau blacklist and does NOT ship +# nouveau.ko. The /etc/modprobe.d/blacklist-nouveau.conf we wrote above +# lives on the rootfs and is only consulted AFTER switchroot; by then +# nouveau has already been auto-loaded by initramfs udev on modern +# NVIDIA data-center GPUs (Ada/Hopper/Blackwell), where nouveau's GSP-RM +# support hangs on device init and stalls udev-settle forever. +# +# Fix: write a dracut.conf.d snippet so BOTH this script's dracut +# rebuild AND the later Earthfile-driven dracut rebuild produce an +# initrd that (a) omits nouveau entirely and (b) carries the modprobe +# blacklist file, so initramfs modprobe honors it too. +# --------------------------------------------------------------------------- +mkdir -p /etc/dracut.conf.d +cat > /etc/dracut.conf.d/95-blacklist-nouveau.conf <<'EOF' +# Managed by CanvOS install-nvidia-drivers.sh +omit_drivers+=" nouveau lbm-nouveau " +install_items+=" /etc/modprobe.d/blacklist-nouveau.conf " +EOF + +# --------------------------------------------------------------------------- +# 12b. Rebuild the initrd so the nouveau blacklist applies in early boot +# --------------------------------------------------------------------------- +if [ "${NVIDIA_REBUILD_INITRD}" = "true" ] && command -v dracut >/dev/null 2>&1; then + log "Rebuilding initrd for ${KVER} (dracut) ..." + if dracut -f "/boot/initrd-${KVER}" "${KVER}"; then + ln -sf "initrd-${KVER}" /boot/initrd + else + warn "dracut initrd rebuild failed; nouveau blacklist still applies post-switchroot." + fi +elif [ "${NVIDIA_REBUILD_INITRD}" = "true" ] && command -v update-initramfs >/dev/null 2>&1; then + log "Rebuilding initramfs for ${KVER} (update-initramfs) ..." + update-initramfs -u -k "${KVER}" || warn "update-initramfs failed." +fi + +# --------------------------------------------------------------------------- +# 13. Cleanup apt caches to keep the image lean +# --------------------------------------------------------------------------- +apt-get clean +rm -rf /var/lib/apt/lists/* + +log "Done. NVIDIA driver ${NVIDIA_DRIVER_BRANCH} (${NVIDIA_DRIVER_TYPE}) baked in for kernel ${KVER}." +log "Reminder: install the GPU Operator with 'driver.enabled=false'." +if [ "${NVIDIA_INSTALL_CONTAINER_TOOLKIT}" = "true" ]; then + log "Reminder: you pre-installed the container toolkit -> also set 'toolkit.enabled=false'." +fi diff --git a/scripts/prebuild-amdgpu-artifact.sh b/scripts/prebuild-amdgpu-artifact.sh new file mode 100755 index 00000000..e341d173 --- /dev/null +++ b/scripts/prebuild-amdgpu-artifact.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# +# prebuild-amdgpu-artifact.sh +# +# Compile AMD's amdgpu-dkms module against the kairos base image's kernel in a +# host-side `docker run --privileged` container, then tar the resulting kernel +# module + firmware + support files into an artifact the main Earthly build +# COPYs in. +# +# WHY THIS EXISTS +# --------------- +# Earthly's buildkit RUN sandbox breaks AMD's amdgpu-dkms ./configure heredoc +# probe (fails at "cannot detect CFLAGS..."), despite the same script + same +# base image + same host succeeding under plain `docker run --privileged`. +# The specific buildkit-vs-docker sandbox difference is not something we +# control from the Earthfile. Instead of fighting it, we run the DKMS build +# outside Earthly, in the environment we know works, and let Earthly consume +# the produced artifact via COPY. +# +# WHAT ENDS UP IN THE ARTIFACT +# /lib/modules//updates/dkms/.ko* +# /lib/firmware/amdgpu/* (firmware blobs) +# /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf (defensive) +# /etc/modules-load.d/amdgpu.conf (autoload) +# /etc/canvos/amdgpu-driver-source (on-node marker) +# +# Extracting the tar into the image (via the Earthfile) + running depmod on +# the target kernel is functionally equivalent to running install-amdgpu- +# drivers.sh directly in the image. +# +# CACHING +# Artifacts are stored at build/amdgpu-artifact---.tar.gz +# Cache hits when release + base-image digest + kver match. Docker image +# pulls are hit via the local docker daemon's own cache. +# +# INPUTS (env vars; defaults mirror the Earthfile / .arg.template) +# BASE_IMAGE kairos base image ref (REQUIRED) +# AMDGPU_DRIVER_RELEASE default: 7.2.1 (pairs with GPU Operator v1.5.0) +# AMDGPU_ARTIFACT_DIR default: ./build +# AMDGPU_FORCE_REBUILD set to 1 to bypass cache +# +# OUTPUT (stdout) +# Absolute path to the produced .tar.gz on the last line, prefixed by +# "AMDGPU_ARTIFACT_PATH=" so callers can `eval "$(prebuild-amdgpu-artifact.sh)"` +# or just take the last line. +# +set -euo pipefail + +log() { echo "[prebuild-amdgpu] $*" >&2; } +die() { echo "[prebuild-amdgpu] ERROR: $*" >&2; exit 1; } + +# --- inputs --------------------------------------------------------------- +: "${BASE_IMAGE:?BASE_IMAGE must be set (kairos base image ref, e.g. us-docker.pkg.dev/palette-images/edge/kairos-ubuntu:24.04-core-amd64-generic-v4.0.4)}" +AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE:-7.2.1}" +AMDGPU_ARTIFACT_DIR="${AMDGPU_ARTIFACT_DIR:-./build}" +AMDGPU_FORCE_REBUILD="${AMDGPU_FORCE_REBUILD:-0}" + +command -v docker >/dev/null 2>&1 || die "docker must be available on the build host." + +# The install script we'll run inside the container. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +INSTALL_SCRIPT="${SCRIPT_DIR}/install-amdgpu-drivers.sh" +HEADERS_SCRIPT="${SCRIPT_DIR}/install-kernel-headers.sh" +[ -r "${INSTALL_SCRIPT}" ] || die "cannot find install-amdgpu-drivers.sh at ${INSTALL_SCRIPT}" +[ -r "${HEADERS_SCRIPT}" ] || die "cannot find install-kernel-headers.sh at ${HEADERS_SCRIPT}" + +mkdir -p "${AMDGPU_ARTIFACT_DIR}" + +# --- discover target kernel + base image digest --------------------------- +log "Pulling base image (may be cached): ${BASE_IMAGE}" +docker pull "${BASE_IMAGE}" >/dev/null || die "failed to pull ${BASE_IMAGE}" + +BASE_DIGEST="$(docker image inspect -f '{{.Id}}' "${BASE_IMAGE}" | sed 's/^sha256://' | cut -c1-12)" +[ -n "${BASE_DIGEST}" ] || die "could not read digest of ${BASE_IMAGE}" + +KVER="$(docker run --rm --entrypoint /bin/sh "${BASE_IMAGE}" -c 'ls /lib/modules | sort -V | tail -1' 2>/dev/null)" +[ -n "${KVER}" ] || die "could not discover kernel from /lib/modules inside ${BASE_IMAGE}" + +ARTIFACT_NAME="amdgpu-artifact-${AMDGPU_DRIVER_RELEASE}-${KVER}-${BASE_DIGEST}.tar.gz" +ARTIFACT_PATH="$(cd "${AMDGPU_ARTIFACT_DIR}" && pwd)/${ARTIFACT_NAME}" + +log "Base image digest : ${BASE_DIGEST}" +log "Target kernel : ${KVER}" +log "Driver release : ${AMDGPU_DRIVER_RELEASE}" +log "Artifact path : ${ARTIFACT_PATH}" + +# --- cache check ---------------------------------------------------------- +if [ "${AMDGPU_FORCE_REBUILD}" != "1" ] && [ -s "${ARTIFACT_PATH}" ]; then + log "Cache hit -- reusing existing artifact. Set AMDGPU_FORCE_REBUILD=1 to override." + echo "AMDGPU_ARTIFACT_PATH=${ARTIFACT_PATH}" + exit 0 +fi + +# --- build ---------------------------------------------------------------- +# Run the install script inside a privileged container against the same base +# image the Earthfile will use, then tar out the produced files. We stream +# the tar over stdout to avoid needing an intermediate volume mount that some +# rootless docker setups can't do cleanly. +STAGE_DIR="$(mktemp -d "${AMDGPU_ARTIFACT_DIR}/.amdgpu-build.XXXXXX")" +trap 'rm -rf "${STAGE_DIR}"' EXIT + +log "Compiling amdgpu-dkms in container. This takes ~8-10 min the first time." +docker run --rm --privileged \ + -e AMDGPU_DRIVER_SOURCE=dkms \ + -e AMDGPU_DRIVER_RELEASE="${AMDGPU_DRIVER_RELEASE}" \ + -e AMDGPU_REBUILD_INITRD=false \ + -e KVER_EXPECTED="${KVER}" \ + -v "${INSTALL_SCRIPT}:/tmp/install-amdgpu-drivers.sh:ro" \ + -v "${HEADERS_SCRIPT}:/tmp/install-kernel-headers.sh:ro" \ + --entrypoint /bin/bash \ + "${BASE_IMAGE}" \ + -c ' + set -eo pipefail + # Scripts are bind-mounted read-only from the host; invoke via `bash` + # so we don t need chmod +x (which would fail on the RO mount). + bash /tmp/install-amdgpu-drivers.sh 1>&2 + + # Verify the module actually landed. + MODDIR="/lib/modules/${KVER_EXPECTED}/updates/dkms" + if ! find "${MODDIR}" -name "amdgpu.ko*" 2>/dev/null | grep -q .; then + echo "prebuild: no amdgpu module under ${MODDIR}" >&2 + exit 1 + fi + + # Build the tar to stdout. Paths must exist to be included; the tar + # is anchored at / so extraction inside the image lands under the + # same absolute paths. + TAR_INPUTS=( + "/lib/modules/${KVER_EXPECTED}/updates/dkms" + "/etc/modules-load.d/amdgpu.conf" + "/etc/canvos/amdgpu-driver-source" + ) + # Firmware + framework drop-in are optional but helpful; skip silently if absent. + [ -d /lib/firmware/amdgpu ] && TAR_INPUTS+=("/lib/firmware/amdgpu") + [ -r /etc/dkms/framework.conf.d/canvos-no-mok-signing.conf ] && \ + TAR_INPUTS+=("/etc/dkms/framework.conf.d/canvos-no-mok-signing.conf") + + tar -czf - "${TAR_INPUTS[@]}" + ' > "${STAGE_DIR}/artifact.tar.gz" + +# Move the artifact to its final path IMMEDIATELY so a subsequent verify +# failure leaves a diagnosable file behind (STAGE_DIR is wiped by the trap). +[ -s "${STAGE_DIR}/artifact.tar.gz" ] || die "prebuild produced an empty artifact." +mv "${STAGE_DIR}/artifact.tar.gz" "${ARTIFACT_PATH}" + +# Sanity: tar file must be a valid gzip and contain the amdgpu module. +tar_listing="$(tar -tzf "${ARTIFACT_PATH}" 2>&1)" \ + || die "artifact ${ARTIFACT_PATH} is not a valid gzipped tar. Head of output: $(printf '%s\n' "${tar_listing}" | head -3)" +if ! printf '%s\n' "${tar_listing}" | grep -qE "updates/dkms/.*amdgpu\.ko"; then + log "Artifact contents (first 40 entries):" + printf '%s\n' "${tar_listing}" | head -40 | sed 's/^/ /' >&2 + die "artifact ${ARTIFACT_PATH} does not contain an amdgpu module under updates/dkms/. \ +The tarball is preserved for inspection. Delete it and rerun with \ +AMDGPU_FORCE_REBUILD=1 to try again." +fi + +log "Artifact produced ($(du -h "${ARTIFACT_PATH}" | awk '{print $1}'))." + +# --- output --------------------------------------------------------------- +echo "AMDGPU_ARTIFACT_PATH=${ARTIFACT_PATH}" diff --git a/slem/Dockerfile b/slem/5.4/Dockerfile similarity index 71% rename from slem/Dockerfile rename to slem/5.4/Dockerfile index 36d53d98..eba9e457 100644 --- a/slem/Dockerfile +++ b/slem/5.4/Dockerfile @@ -1,8 +1,8 @@ -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM registry.suse.com/suse/sle-micro-rancher/5.4:latest -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 COPY --from=kairos-init /kairos-init /kairos-init RUN /kairos-init -l debug -m "generic" -t false --version "${KAIROS_VERSION}" && rm /kairos-init diff --git a/slem/README.md b/slem/5.4/README.md similarity index 100% rename from slem/README.md rename to slem/5.4/README.md diff --git a/slem/build.sh b/slem/5.4/build.sh old mode 100644 new mode 100755 similarity index 100% rename from slem/build.sh rename to slem/5.4/build.sh diff --git a/slem/5.5/Dockerfile b/slem/5.5/Dockerfile new file mode 100644 index 00000000..6c73c305 --- /dev/null +++ b/slem/5.5/Dockerfile @@ -0,0 +1,75 @@ +# syntax=docker/dockerfile:1.4 +ARG KAIROS_INIT_IMAGE=quay.io/kairos/kairos-init:v0.16.2 +FROM ${KAIROS_INIT_IMAGE} AS kairos-init + +# --------------------------------------------------------------------------- +# Stage 1: register against SCC to obtain the entitled, version-matched +# SLE 15 SP5 repositories. The SLE Micro image ships no SUSEConnect, so we use +# a BCI base (which can install it) purely to produce the RIS service + +# credentials that libzypp can then consume natively. +# --------------------------------------------------------------------------- +FROM registry.suse.com/bci/bci-base:15.5 AS scc +RUN zypper -n install -y suseconnect-ng +RUN --mount=type=secret,id=SUSE_REGCODE \ + SUSEConnect -r "$(cat /run/secrets/SUSE_REGCODE)" && \ + SUSEConnect -p sle-module-basesystem/15.5/x86_64 || true; \ + SUSEConnect -p PackageHub/15.5/x86_64 || true + +# --------------------------------------------------------------------------- +# Stage 2: SLE Micro (for Rancher) 5.5 base built against the entitled repos. +# Using version-matched SP5 packages means kairos-init reinstalls a kernel + +# dracut that agree with the base image, so the initrd it rebuilds is bootable +# (the openSUSE Leap OSS mix downgraded dracut and produced an unbootable initrd). +# --------------------------------------------------------------------------- +FROM registry.suse.com/suse/sle-micro/5.5:latest + +ARG KAIROS_VERSION=v4.1.2 +ARG TRUSTED_BOOT=false + +# Bring over the SCC registration so zypper here can pull the entitled repos. +COPY --from=scc /etc/zypp/credentials.d/ /etc/zypp/credentials.d/ +COPY --from=scc /etc/zypp/services.d/ /etc/zypp/services.d/ +COPY --from=scc /etc/zypp/repos.d/ /etc/zypp/repos.d/ +COPY suse-build-key.asc /tmp/suse-build-key.asc + +# refresh-services processes the copied RIS service (hits SCC with the +# credentials) and materialises the entitled repos; a plain refresh alone does +# not. Fail loudly if no enabled repos result. +# kairos-init's SLE Micro (Rancher) path also adds the openSUSE Leap OSS repo, +# whose signing key expired 2026-06-19, so a later `zypper refresh` across all +# repos aborts (exit 4). Disable repo-metadata gpg check globally so that stale +# repo does not break the build; package signatures are still verified (SUSE key +# imported above), and the real packages come from the valid SCC SP5 repos. +RUN rpm --import /tmp/suse-build-key.asc && rm -f /tmp/suse-build-key.asc && \ + sed -i '/^\[main\]/a repo_gpgcheck = off' /etc/zypp/zypp.conf && \ + rm -f /etc/zypp/repos.d/SLE_BCI.repo && \ + zypper --non-interactive --gpg-auto-import-keys refresh-services --force || true; \ + echo "=== enabled repositories ===" && zypper --non-interactive lr --uri && \ + zypper --non-interactive --gpg-auto-import-keys refresh + +# SLE Micro for Rancher ships Rancher Elemental, a competing immutable-OS stack. +# Its dracut modules, systemd services and /system/oem layout files race with +# Kairos immucore and prevent sysroot.mount from landing. Remove before kairos-init. +RUN zypper --non-interactive rm -y --clean-deps \ + elemental elemental-register1.5 elemental-support1.5 \ + elemental-system-agent elemental-toolkit elemental-updater \ + 2>/dev/null || true; \ + for svc in elemental-setup-reconcile elemental-register-reset \ + elemental-register elemental-setup-network \ + elemental-setup-fs elemental-populate-node-labels \ + elemental-system-agent elemental-setup-rootfs \ + elemental-setup-boot elemental-setup-initramfs \ + elemental-register-install elemental-immutable-rootfs; do \ + systemctl mask "${svc}.service" 2>/dev/null || true; \ + done; \ + rm -f /system/oem/*_elemental-* \ + /etc/dracut.conf.d/99-elemental-systemd.conf \ + /etc/dracut.conf.d/02-elemental-immutable-rootfs.conf \ + /etc/dracut.conf.d/02-elemental-setup-initramfs.conf \ + /etc/dracut.conf.d/50-elemental-initrd.conf + +COPY --from=kairos-init /kairos-init /kairos-init +RUN /kairos-init -l debug -m "generic" -t "${TRUSTED_BOOT}" --version "${KAIROS_VERSION}" && rm /kairos-init + +# Scrub subscription data so credentials do not ship in the final image. +RUN rm -rf /etc/zypp/credentials.d/* /etc/zypp/services.d/* /etc/SUSEConnect 2>/dev/null || true diff --git a/slem/5.5/README.md b/slem/5.5/README.md new file mode 100644 index 00000000..12821edf --- /dev/null +++ b/slem/5.5/README.md @@ -0,0 +1,9 @@ +# SUSE Linux Enterprise Micro + +## Pre-requisites : +* A host with SLES Micro distribution installed +* Registration code to register with SUSEConnect +* If you wish to override the BASE_IMAGE, make sure to use a container image that has zypper installed in it + +## Steps to build the image: +`./build.sh []` \ No newline at end of file diff --git a/slem/5.5/build.sh b/slem/5.5/build.sh new file mode 100755 index 00000000..dc57e6f1 --- /dev/null +++ b/slem/5.5/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Build a bootable Kairos SLE Micro (for Rancher) 5.5 base image on any host +# with Docker + BuildKit. +# +# registry.suse.com/suse/sle-micro/5.5 is the "SLE Micro for Rancher 5.5" image. +# To get a *bootable* Kairos image, kairos-init must install a kernel + dracut +# that match the base (mixing in older openSUSE Leap packages produces an +# unbootable initrd). We therefore register against SCC at build time to pull +# the version-matched SLE 15 SP5 repos. +# +# Usage: ./build.sh [] + +set -euo pipefail + +if [[ -z "${1:-}" ]]; then + echo "ERROR : Registration code is empty !" + echo "Re-run with a SUSE registration code, e.g.: ./build.sh 1234567890 [slem-kairos:5.5]" + exit 1 +fi + +REGISTRATION_CODE="$1" +OUTPUT_TAG="${2:-slem-kairos:5.5}" + +cd "$(dirname "$0")" + +# Pass the regcode as a BuildKit secret so it never lands in an image layer. +REGCODE_FILE="$(mktemp)" +trap 'rm -f "${REGCODE_FILE}"' EXIT +printf '%s' "${REGISTRATION_CODE}" > "${REGCODE_FILE}" + +echo "==> Building ${OUTPUT_TAG} (SCC-registered SLE Micro 5.5) ..." +DOCKER_BUILDKIT=1 docker build \ + --secret "id=SUSE_REGCODE,src=${REGCODE_FILE}" \ + -t "${OUTPUT_TAG}" \ + . + +echo "==> Done: ${OUTPUT_TAG}" diff --git a/slem/5.5/oem/09_grub_branding_fixup.yaml b/slem/5.5/oem/09_grub_branding_fixup.yaml new file mode 100644 index 00000000..7acdfca8 --- /dev/null +++ b/slem/5.5/oem/09_grub_branding_fixup.yaml @@ -0,0 +1,27 @@ +name: "SLE Micro 5.5 grubmenu fixup" +# Workaround for a kairos-agent v2.30.2 install-time mount-lifecycle bug on +# SLE Micro Rancher 5.5: /system/oem/08_grub.yaml's after-install block tries +# to mount COS_STATE at /tmp/mnt/STATE, but on SUSE the STATE partition is +# already mounted at /run/cos/state, so the mount fails (exit 32). The stage +# "Grub branding" then cp's /etc/kairos/branding/grubmenu.cfg to +# /tmp/mnt/STATE/grubmenu -- a tmpfs directory -- and the file is lost on +# reboot. GRUB never finds a `--id registration` menuentry and boots the +# plain active entry instead, leaving stylus-agent in a crash loop for lack +# of /oem/80_stylus.yaml. +# +# GRUB searches /grubmenu on any partition (see /etc/cos/grub.cfg: +# `search --no-floppy --file --set=menu_blk "/grubmenu"`), and COS_OEM is +# already where it finds /grubenv, so writing to /oem/grubmenu is +# functionally equivalent to writing to cos-state/grubmenu. +# after-install-chroot bind-mounts COS_OEM at /oem inside the target chroot +# and is confirmed to fire cleanly on SUSE. +# +# This file is COPY-ed only into SLE Micro 5.5 builds by the Earthfile +# iso-image target -- do not add a runtime OS guard here. +stages: + after-install-chroot: + - name: "Copy branding grubmenu to OEM" + if: '[ -e /etc/kairos/branding/grubmenu.cfg ] && [ -d /oem ]' + commands: + - cp -f /etc/kairos/branding/grubmenu.cfg /oem/grubmenu + - sync diff --git a/slem/5.5/suse-build-key.asc b/slem/5.5/suse-build-key.asc new file mode 100644 index 00000000..263044ae --- /dev/null +++ b/slem/5.5/suse-build-key.asc @@ -0,0 +1,78 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: Hockeypuck 2.2 +Comment: Hostname: + +xsBNBFEKlmsBCADbpZZbbSC5Zi+HxCR/ynYsVxU5JNNiSSZabN5GMgc9Z0hxeXxp +YWvFoE/4n0+IXIsp83iKvxf06Eu8je/DXp0lMqDZu7WiT3XXAlkOPSNV4akHTDoY +91SJaZCpgUJ7K1QXOPABNbREsAMN1a7rxBowjNjBUyiTJ2YuvQRLtGdK1kExsVma +hieh/QxpoDyYd5w/aky3z23erCoEd+OPfAqEHd5tQIa6LOosa63BSCEl3milJ7J9 +vDmoGPAoS6ui7S2R5X4/+PLN8Mm2kOBrFjhmL93LX0mrGCMxsNsKgP6zabYKQEb8 +L028SXvl7EGoA+Vw5Vd3wIGbM73PfbgNrXjfABEBAAHNKFN1U0UgUGFja2FnZSBT +aWduaW5nIEtleSA8YnVpbGRAc3VzZS5kZT7CwJMEEwEIAD0CGwMGCwkIBwMCBBUC +CAMEFgIDAQIeAQIXgBYhBP6rUCU52EbbLAlhynCvnoE523yCBQJmxdkaBQkdeMEv +AAoJEHCvnoE523yCsyEH/1NZhXtgIa4kFCZdWhPhXPvqz7IkIm62yXpS3Iseivbm +rxzQNXNlQVLnaOOKZX4nEUyh1lr+w18PGlb1yIdMjQqt04hwFgCU+q99cTfrAHG5 +jzirSq9I2iBjn+zARCjLzJsD+dH7JGfEMm0lxtPyMRoNJ6bq8eEkjEtKxDOg0iTE +vQ4eboRlR0a8hH06tauPfeWx6Ri6hIobN3TNdCY/RQe4WeyYL8vEog3c7uYYag/V +iMFfj8QzRHgkkcCE9W3TTfr1K/h8AGZTW0uJH4YQhl2HqUsspKmicZIbK/W9M87l +HUyO8EgreF1MuKsg1GWxV2OikZAJKMcNs6EhzLWUWHvCwHwEEwECACYCGwMGCwkI +BwMCBBUCCAMEFgIDAQIeAQIXgAUCX2himwUJFeKaMAAKCRBwr56BOdt8glQcCACX +QAkHKf0y7EPlayuX/EHc8sro4IAJDZqQFiPaJh8F+5HWD36+iw3D/HlOlzbd2y9o +VqtbVDZVamOJ0KV+l2oxPbMVg32plYGXLlXh1Gwp7/lLWieceXVzf3AbleejgXfa +fUyiCuvjVaQyPNcGlEXIjiwi3qulw0+2rYAiUAf6KEq4wM9a/KLTLMhvxi2NigC+ +MiIbZtmtHlpFhMkp+Bdpdcqtz6cAucE3dSpVQcvkfNLOkZgrtkZfzkNBPaWnyZLt +WdSrQah2vdIQX+RvMYVAQP4x+aL44ALlhnyeUST0wIX4AH82ifSnpvz/rEb1kpER +j3XhwW+NNPBstHtGpA+lwsB8BBMBAgAmAhsDBgsJCAcDAgQVAggDBBYCAwECHgEC +F4AFAlhH6x8FCQ7CIrQACgkQcK+egTnbfIKalAf8D0kkuQXA7jouJ9NnI+B3be1i +EygY46NJRtX1xtaacdcDkk/rPBXq7TFRWOd01uhRo3GLcFTiKq7DGIlepH9TcVFf +43AMu1ZguRGZztEfw97E+DoeYbC7REWrmaJSuQWZndEudUGvHk8WeZl/Ty9ot7P0 +oRE3N9Ha3/N7oHOweMmTzUFlzQz4BnAvoRHguasOPEcx2CKVLUcx5nn0eWd4qE1G +w5c8vkhj9rgO5oUeC35P3X+X2MVnwqCpcho7a7wOuXAfj5LMp8awHgbgfrwTPipw +68Vtkp+AG1pMx//3X8/TiQybuFDPdYW+6sTNDpK1x0Fd4jas4DAxIZMv/TajMsLA +fAQTAQIAJgUCUQqWawIbAwUJB4TOAAYLCQgHAwIEFQIIAwQWAgMBAh4BAheAAAoJ +EHCvnoE523yCiYYIALXc+LrQjo7wYnTcFCuZXLImsNzAnHGrDQeYgQ1/esMsxhMG +524zeY2bkXLYaXYi5YP2Ye9yRCzTFr4DuYDAdeEhdYUKtlbJEfXrD3cHOOMsaIww +qKYTyaDhdxpEvUUmHEaqrsWkuBpelCh+TDTIVxq+65sROkNHfDiqJFWw0qrhEzTS +kwKlOkkNZqYUmb+g5SYETjlDTvW/0LihAc+wE0r/Iy2VGyx6XzXlfHiVskikoCDg +FG3IEdYLMf+BVEaFgDzEOz8MS+5n2azAF2eI/4rVdKSv423EW+2QDL8LZDaqTeyq +kIw63cwMZPIGCoiimUT/hdEKKz13tvW/XO/j+E3CRgQQEQIABgUCUvi8gAAKCRDq +e/OXAXViPiBTAJ4gzDSlVX2Emz3UnlJ2JUhrpFIlHACglmkcO30WUcvUWoo78HSG +rjNG8Y/CwXMEEAEKAB0WIQRV0WEVtpb9nRcG0MoZ13/QS82iRgUCW7UjswAKCRAZ +13/QS82iRmLJEACQzUTfHn7rUu8X0T4DuHTjxePf/raUK6fnlGaQVm6Waqy3DqVC +FapYYFMJr3UBSb3xGjemA6LpQbL5gef+VyMo9MZ+45DoRvl0lI2fxmLAHGoPCRCo +lvUbt2GE6rKnpJZiGEPAOHghtiIPXi3wk7cB+AWzG8rGtuzcKxY1Vk0Bp4zJXeOz +esDMgsdXNMIqqUsfQfOettkTjXGZ3+sHtf4OR0F6QQEvDjpGRacOUmGvclG42HRI +ue1NZxVr/oxkAgt2hSF2vmYZ7ZZI0n3Rsc+DLWCx0ATy/Spaxbx5c19wAONI+bGO +ur+byYs9lRFGlcIQFfDKbXIMjjUyEAL53v3rh/o/h8exzuVGlLl1Y9I9zjVFOSyM +ZzqM1k1VMmQ3yFfAQo2IaqXTyyR89QT6uqXiUORszjK6PAq9/ohPszfME+Fl/yp6 +WVIdLMAcddoqMhaHVMWuF6GPx1YDEzLJOVIyrwS/Y2EGQao19ibkC5us4FKaeZoE +ApfqYYpMUEXCaTNIo6h6XgoGxAhx+f9MhnpmroWgWe9JrhtJ03xun3qpRAaCJpZ3 +OozL1Px6XkMPyuZW7nGxMerdyPBL5siZBo82Tza3jCGWv0EpzM+Kv9doUWBWHROj +jD+O8Hwbj4jCOx5EBKlk4C7luc+r4hs/dec++Rcw3NgFXgv1G1AWCXhnEsLAcwQQ +AQgAHRYhBA3A9Vhi9VeoC+fUxISbtLYWQU4ZBQJd+qx+AAoJEISbtLYWQU4ZWaQH +/jeE0z/4sVtm3iZw8mYnblAUGdbvY6ItUGupGszUDF06oMFySpWJKfzjL0VEb7a/ +VR1l9TpDY0ywDZyZmmk8/VVTmzgzu+YOh2xSjGaYb17b1aWY+RFVotcWnsBJOfKa +E5M960JUHjJu5Z63QlF2NzNcudww4yT0kheXxQwvm8v4C39kpoIgLw6VAVnMi5vp +bwvZ/OVQc6X704NLcDgLTTJ/HPFl/GAf89F5WqwhKaA9jPDYJCLUYG823vQ83p9j +PEuSS1x13O4fYGwQcZPDa7bxEAx6hikduwsW8U9BouEcSM+nKIuZ/vEIre+QSQ/K +6Zd/JmoIhSMazLxVDK4P4RrCwHMEEAEIAB0WIQQCigXbqynsNnTiTRfKGWH4ObpA +6wUCXwcONwAKCRDKGWH4ObpA6/5tB/9qW7XU4ACHdZwvw89pwi8KfKDDyiYtsl42 +R8MEiQ6hvd2wuEvrUJ7/ItsN0DZJmvn+qfN+0QgDEq4TLZ9KT8vFnaEX62VIHthZ +UilVTnu2Ikmu6X9pDSBZrupAT7WZNLZHtTrDWDOyBLGpeDByh8BvH8b0nnaUCyqV +DymVgkbGReZS2badkSWHGdYvYb51iXDBVog7eXT3cNLJFxkgauJnSx06xqEi40sT +ogemnE2HIcz5FPBwH0UZIgSqSwVWAk7TKSG08jTFrrYJ24+r7NAb1ly1LMFauF+L +YOREIxrOY3Ffr5Km7o1ISY2gHTJkv89kcvYG6ScK3XyTRWuA/xeKwsFcBBABCgAG +BQJf14i8AAoJECIJ1pAvlpyVc/QP/05zjnz55Gx5chi2ClpKWlZKaauYwPpk9tE0 +/3h0UL2YLfepftnSFoQCEaWdHQ4dCWFdEodpQqjxFve8ljGMFhtiNGaaF7Ct5Xpt +Qe47x3CIy1KT/F+IfCS+iVQ/uBoeyKBSxGHD5h0HO2asdLePZw8DlBbna7A9x6Pn +B89jaL40zEYiEwykuCVuxG4nB63/I4lH8pXV63zUZUmsqzFdRl5tYNdoF+LKTrd5 +EqccuCFYVMkG3JGT/odSZTxe+8ekektuudTf6L9etMr3MiocyoSEX5tcwvIshG4t +21HeDdpQX8dGRoYMjzV3KLG9elzKn5mz4eXJey68+xo67lvDTof07JNPgr36yp1q +xXl2YNPIYAyvJrAPy1bY01tWJPu6OOvbwiH64RksISyMH0u04f9R5mTIr+oo4oCr +yyF/9JCj6eLkG66SvGH1XZvPVbLxhZfHD0lS3t19AUdLw5CSoVGBvrzZzkEjlORW +YDXDloqTMNmy7WdD257WzS8bkc/NWPrvKWlHwFjT2RKTzBU3/25sRi9haRrXqOeZ +dosXYKatPMFaeGSdtwKSfxUWCAwPQ6G77Q4+174wVZg9VXeJq1FQc5a7zz97cQiT +ve+tslg2GBXU8znWYBLr8RbXnxRo/VbH3CkuqsSKcMh7lxMDXxIuTLNtamuK2nis +3TAFLqEL +=5DiY +-----END PGP PUBLIC KEY BLOCK----- diff --git a/ubuntu-fips/20.04/99-usb-media.conf b/ubuntu-fips/20.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/20.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/20.04/Dockerfile b/ubuntu-fips/20.04/Dockerfile index 2cf14ee2..667dae97 100644 --- a/ubuntu-fips/20.04/Dockerfile +++ b/ubuntu-fips/20.04/Dockerfile @@ -1,11 +1,11 @@ # Kairos init image -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init # Base ubuntu image (focal) FROM ubuntu:focal AS base -ARG KAIROS_VERSION=v4.0.4 +ARG KAIROS_VERSION=v4.1.2 # Don't get asked while running apt commands ENV DEBIAN_FRONTEND=noninteractive @@ -112,6 +112,11 @@ RUN cd /usr/lib/dracut/modules.d/95iscsi && patch < /dracut-broken-iscsi-ubuntu- COPY dracut.conf /etc/dracut.conf.d/kairos-fips.conf +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + # Copy the custom dracut modules.fips that includes 2 missing modules COPY modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | grep fips | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips diff --git a/ubuntu-fips/22.04/99-usb-media.conf b/ubuntu-fips/22.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/22.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips b/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips index 5d4c06be..0c90c30a 100644 --- a/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips +++ b/ubuntu-fips/22.04/Dockerfile.ubuntu22.04-fips @@ -1,7 +1,7 @@ -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM ubuntu:22.04 -ARG VERSION=v4.0.4 +ARG VERSION=v4.1.2 ARG ENABLE_STIG=1 ARG SKIP_STIG_BANNER=0 ENV DEBIAN_FRONTEND=noninteractive @@ -32,6 +32,11 @@ RUN --mount=type=secret,id=pro-attach-config \ COPY 22.04/dracut.conf /etc/dracut.conf.d/kairos-fips.conf +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 22.04/99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + # Copy the custom dracut modules.fips that includes 2 missing modules COPY 22.04/modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips diff --git a/ubuntu-fips/24.04/99-usb-media.conf b/ubuntu-fips/24.04/99-usb-media.conf new file mode 100644 index 00000000..af815ee8 --- /dev/null +++ b/ubuntu-fips/24.04/99-usb-media.conf @@ -0,0 +1,3 @@ +hostonly="no" +add_drivers+=" xhci_pci_renesas " +force_drivers+=" xhci_pci_renesas " diff --git a/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips b/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips index a9d1d6de..5d6d8bc5 100644 --- a/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips +++ b/ubuntu-fips/24.04/Dockerfile.ubuntu24.04-fips @@ -1,7 +1,7 @@ -FROM quay.io/kairos/kairos-init:v0.8.12 AS kairos-init +FROM quay.io/kairos/kairos-init:v0.16.2 AS kairos-init FROM ubuntu:24.04 -ARG VERSION=v4.0.4 +ARG VERSION=v4.1.2 ARG ENABLE_STIG=1 ARG SKIP_STIG_BANNER=0 ENV DEBIAN_FRONTEND=noninteractive @@ -41,6 +41,11 @@ RUN --mount=type=secret,id=pro-attach-config \ COPY 24.04/modules.fips /tmp/modules.fips RUN kernel=$(ls /lib/modules | head -n1) && mv /tmp/modules.fips /lib/modules/${kernel}/modules.fips +# Bundle the Renesas xHCI (USB 3.0) host controller driver into the initramfs so +# installation from USB media works on hardware using that chipset. Consumed by the +# `kairos-init -s init` dracut run below (the Earthfile skips dracut for FIPS builds). +COPY 24.04/99-usb-media.conf /etc/dracut.conf.d/99-usb-media.conf + COPY 24.04/fix.sh /tmp/fix.sh COPY stig-remediate.sh /tmp/stig-remediate.sh COPY restore-ubuntu-default-banners.sh /tmp/restore-ubuntu-default-banners.sh