deploy: Pull bound images before staging via composefs mount

.github/workflows/ci.yml

@@ -302,9 +302,6 @@ jobs:
         seal_state: ["sealed", "unsealed"]
 
         exclude:
-          # https://github.com/bootc-dev/bootc/issues/1812
-          - test_os: centos-9
-            variant: composefs
           - seal_state: "sealed"
             boot_type: bls
           - seal_state: "sealed"

Makefile

@@ -29,7 +29,15 @@ prefix ?= /usr
 # We may in the future also want to include Fedora+derivatives as
 # the code is really tiny.
 # (Note we should also make installation of the units conditional on the rhsm feature)
-CARGO_FEATURES_DEFAULT ?= $(shell . /usr/lib/os-release; if echo "$$ID_LIKE" |grep -qF rhel; then echo rhsm; fi)
+#
+# Enable the rhel9 feature on RHEL/CentOS Stream 9, which runs kernel 5.14.
+# That kernel cannot mount an erofs image directly from a file descriptor;
+# composefs-ctl's rhel9 feature activates a loopback-device fallback instead.
+CARGO_FEATURES_DEFAULT ?= $(shell . /usr/lib/os-release; \
+  features=""; \
+  if echo "$$ID_LIKE" | grep -qF rhel; then features="$$features rhsm"; fi; \
+  if echo "$$ID_LIKE" | grep -qF rhel && [ "$$VERSION_ID" = "9" ]; then features="$$features rhel9"; fi; \
+  echo $$features)
 # You can set this to override all cargo features, including the defaults
 CARGO_FEATURES ?= $(CARGO_FEATURES_DEFAULT)
 

contrib/packaging/bootc.spec

@@ -12,6 +12,14 @@
     %bcond_with rhsm
 %endif
 
+# kernel 5.14 (RHEL/CentOS 9) cannot mount an erofs image directly from a file
+# descriptor; composefs-ctl's rhel9 feature enables a loopback-device fallback.
+%if 0%{?rhel} == 9
+    %bcond_without rhel9
+%else
+    %bcond_with rhel9
+%endif
+
 %global rust_minor %(rustc --version | cut -f2 -d" " | cut -f2 -d".")
 
 # https://github.com/bootc-dev/bootc/issues/1640
@@ -132,13 +140,16 @@ make manpages
 # Build all binaries
 %if 0%{?container_build}
 # Container build: use cargo directly with cached dependencies to avoid RPM macro overhead
-cargo build -j%{_smp_build_ncpus} --release %{?with_rhsm:--features rhsm} --bins
+cargo build -j%{_smp_build_ncpus} --release %{?with_rhsm:--features rhsm} %{?with_rhel9:--features rhel9} --bins
 %else
 # Non-container build: use RPM macros for proper dependency tracking
 %if %new_cargo_macros
-    %cargo_build %{?with_rhsm:-f rhsm} -- --bins
+    # Note: %%cargo_build's own -f option only accepts a single value, so a
+    # second -f would silently clobber the first; pass extra features as
+    # plain --features args after -- instead, which cargo unions correctly.
+    %cargo_build -- %{?with_rhsm:--features rhsm} %{?with_rhel9:--features rhel9} --bins
 %else
-    %cargo_build %{?with_rhsm:--features rhsm} -- --bins
+    %cargo_build %{?with_rhsm:--features rhsm} %{?with_rhel9:--features rhel9} -- --bins
 %endif
 %endif
 
@@ -152,7 +163,7 @@ sed -i -e '/https:\/\//d' cargo-vendor.txt
 
 %install
 # Pass CARGO_FEATURES explicitly to prevent auto-detection rebuild in install environment
-%make_install INSTALL="install -p -c" CARGO_FEATURES="%{?with_rhsm:rhsm}"
+%make_install INSTALL="install -p -c" CARGO_FEATURES="%{?with_rhsm:rhsm} %{?with_rhel9:rhel9}"
 %if %{with ostree_ext}
 make install-ostree-hooks DESTDIR=%{?buildroot}
 %endif

crates/lib/src/cli.rs

@@ -1210,7 +1210,7 @@ struct ApplyFromDownloadedOpts {
     apply: bool,
 }
 
-fn apply_from_downloaded_ostree(
+async fn apply_from_downloaded_ostree(
     storage: &Storage,
     booted_ostree: &BootedOstree<'_>,
     host: &crate::spec::Host,
@@ -1222,6 +1222,7 @@ fn apply_from_downloaded_ostree(
         .ok_or_else(|| anyhow::anyhow!("No staged deployment found"))?;
 
     if staged_deployment.is_finalization_locked() {
+        crate::boundimage::pull_bound_images(storage, &staged_deployment).await?;
         ostree.change_finalization(&staged_deployment)?;
         println!("Staged deployment will now be applied on reboot");
     } else {
@@ -1298,7 +1299,8 @@ async fn upgrade(
                 soft_reboot: opts.soft_reboot,
                 apply: opts.apply,
             },
-        );
+        )
+        .await;
     }
 
     // Ensure the bootc storage directory is initialized; the --check path
@@ -1479,7 +1481,8 @@ async fn switch_ostree(
                 soft_reboot: opts.soft_reboot,
                 apply: opts.apply,
             },
-        );
+        )
+        .await;
     }
 
     let target = imgref_for_switch(&opts)?;

crates/lib/src/deploy.rs

@@ -46,7 +46,7 @@
 
 use std::collections::HashSet;
 use std::io::{BufRead, Write};
-use std::os::fd::AsFd;
+use std::os::fd::{AsFd, AsRawFd};
 use std::process::Command;
 
 use anyhow::{Context, Result, anyhow};
@@ -1007,6 +1007,38 @@ impl MergeState {
     }
 }
 
+/// Pull the bound images referenced by an imported commit before staging it.
+#[context("Pulling bound images for ostree commit {commit}")]
+async fn pull_bound_images_for_commit(sysroot: &Storage, commit: &str) -> Result<()> {
+    let repo = sysroot.get_ostree()?.repo();
+    let repo_dir = Dir::reopen_dir(&repo.dfd_borrow())?;
+    let repo_tmp = repo_dir
+        .open_dir("tmp")
+        .context("Opening ostree repo tmp/")?;
+    let td = cap_std_ext::cap_tempfile::TempDir::new_in(&repo_tmp)?;
+    let checkout_mode = if repo.mode() == ostree::RepoMode::Bare {
+        ostree::RepoCheckoutMode::None
+    } else {
+        ostree::RepoCheckoutMode::User
+    };
+    let checkout_opts = ostree::RepoCheckoutAtOptions {
+        mode: checkout_mode,
+        ..Default::default()
+    };
+    let root_name = "root";
+    repo.checkout_at(
+        Some(&checkout_opts),
+        td.as_raw_fd(),
+        root_name,
+        commit,
+        gio::Cancellable::NONE,
+    )
+    .context("Checking out imported commit")?;
+    let root = td.open_dir(root_name)?;
+    let bound_images = crate::boundimage::query_bound_images(&root)?;
+    crate::boundimage::pull_images(sysroot, bound_images).await
+}
+
 /// Stage (queue deployment of) a fetched container image.
 #[context("Staging")]
 pub(crate) async fn stage(
@@ -1054,9 +1086,9 @@ pub(crate) async fn stage(
 
     subtask.completed = true;
     subtasks.push(subtask.clone());
-    subtask.subtask = "deploying".into();
-    subtask.id = "deploying".into();
-    subtask.description = "Deploying Image".into();
+    subtask.subtask = "bound_images".into();
+    subtask.id = "bound_images".into();
+    subtask.description = "Pulling Bound Images".into();
     subtask.completed = false;
     prog.send(Event::ProgressSteps {
         task: "staging".into(),
@@ -1072,15 +1104,13 @@ pub(crate) async fn stage(
             .collect(),
     })
     .await;
-    let origin = origin_from_imageref(spec.image)?;
-    let deployment =
-        crate::deploy::deploy(sysroot, from, image, &origin, lock_finalization).await?;
+    pull_bound_images_for_commit(sysroot, &image.ostree_commit).await?;
 
     subtask.completed = true;
     subtasks.push(subtask.clone());
-    subtask.subtask = "bound_images".into();
-    subtask.id = "bound_images".into();
-    subtask.description = "Pulling Bound Images".into();
+    subtask.subtask = "deploying".into();
+    subtask.id = "deploying".into();
+    subtask.description = "Deploying Image".into();
     subtask.completed = false;
     prog.send(Event::ProgressSteps {
         task: "staging".into(),
@@ -1096,7 +1126,8 @@ pub(crate) async fn stage(
             .collect(),
     })
     .await;
-    crate::boundimage::pull_bound_images(sysroot, &deployment).await?;
+    let origin = origin_from_imageref(spec.image)?;
+    crate::deploy::deploy(sysroot, from, image, &origin, lock_finalization).await?;
 
     subtask.completed = true;
     subtasks.push(subtask.clone());

tmt/tests/booted/test-image-pushpull-upgrade.nu

@@ -130,9 +130,12 @@ def sanity_check_switch_progress_json [data] {
     assert equal $deploy.steps 3
     assert equal $deploy.stepsTotal 3
     let deploy_tasks = $deploy.subtasks
+    # Bound images are now pulled before staging (see deploy::stage), so
+    # the "bound_images" subtask now comes before "deploying" instead of
+    # after it.
     assert equal ($deploy_tasks | length) 5
     let deploy_names = $deploy_tasks | get subtask
-    assert equal $deploy_names ["merging", "deploying", "bound_images", "cleanup", "cleanup"]
+    assert equal $deploy_names ["merging", "bound_images", "deploying", "cleanup", "cleanup"]
 }
 
 # The second boot; verify we're in the derived image

tmt/tests/booted/test-logically-bound-switch.nu

@@ -102,6 +102,18 @@ def first_boot [] {
     }]
 
     let image_name = "localhost/bootc-bound"
+    let unavailable_images = [{
+        "bound": true,
+        "image": "invalid.invalid/bootc-bound-image-does-not-exist:latest",
+        "name": "unavailable"
+    }]
+    build_image $image_name $unavailable_images []
+    let failed_switch = do { bootc switch --transport containers-storage $image_name } | complete
+    assert ($failed_switch.exit_code != 0) "switch should fail when a bound image cannot be pulled"
+    assert ((bootc status --json | from json | get status.staged) == null) "no deployment should be staged after a bound image pull failure"
+
+    # Rebuilding the tag verifies that a failed pull leaves no partial staged
+    # deployment which would prevent a successful retry.
     build_image $image_name $images $containers
     bootc switch --transport containers-storage $image_name
     verify_images $images $containers