Composefs test parity with ostree

.github/workflows/ci.yml

@@ -160,8 +160,8 @@ jobs:
       fail-fast: false
       matrix:
         # No fedora-44 due to https://bugzilla.redhat.com/show_bug.cgi?id=2429501
-        test_os: [fedora-43, centos-9, centos-10]
-        variant: [ostree, composefs-sealeduki-sdboot]
+        test_os: [fedora-43, centos-10]
+        variant: [composefs-sdboot, composefs-grub]
         exclude:
           # centos-9 UKI is experimental/broken (https://github.com/bootc-dev/bootc/issues/1812)
           - test_os: centos-9
@@ -182,7 +182,30 @@ jobs:
         run: |
           BASE=$(just pullspec-for-os base ${{ matrix.test_os }})
           echo "BOOTC_base=${BASE}" >> $GITHUB_ENV
-          echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
+          echo "RUST_BACKTRACE=full" >> $GITHUB_ENV
+          echo "RUST_LOG=trace" >> $GITHUB_ENV
+
+          case "${{ matrix.variant }}" in
+            composefs-grub)
+              echo "BOOTC_variant=composefs" >> $GITHUB_ENV
+              echo "BOOTC_bootloader=grub" >> $GITHUB_ENV
+            ;;
+
+            composefs-sdboot)
+              echo "BOOTC_variant=composefs" >> $GITHUB_ENV
+              echo "BOOTC_bootloader=systemd" >> $GITHUB_ENV
+            ;;
+
+            composefs-sealeduki-sdboot)
+              echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
+              echo "BOOTC_bootloader=systemd" >> $GITHUB_ENV
+            ;;
+
+            ostree)
+              echo "BOOTC_variant=${{ matrix.variant }}" >> $GITHUB_ENV
+              echo "BOOTC_bootloader=grub" >> $GITHUB_ENV
+            ;;
+          esac
 
           if [ "${{ matrix.variant }}" = "composefs-sealeduki-sdboot" ]; then
             BUILDROOTBASE=$(just pullspec-for-os buildroot-base ${{ matrix.test_os }})
@@ -197,7 +220,8 @@ jobs:
 
       - name: Build container
         run: |
-          BOOTC_SKIP_PACKAGE=1 just build
+          BOOTC_SKIP_PACKAGE=1 just bootloader=$BOOTC_bootloader build
+
           # Extra cross-check (duplicating the integration test) that we're using the right base
           used_vid=$(podman run --rm localhost/bootc bash -c '. /usr/lib/os-release && echo ${ID}-${VERSION_ID}')
           test ${{ matrix.test_os }} = "${used_vid}"
@@ -211,11 +235,12 @@ jobs:
 
       - name: Run TMT integration tests
         run: |
-          if [ "${{ matrix.variant }}" = "composefs-sealeduki-sdboot" ]; then
-            just test-composefs
+          if [[ "${{ matrix.variant }}" = composefs* ]]; then
+            just "test-${{ matrix.variant }}"
           else
             just test-tmt integration
           fi
+
           just clean-local-images
 
       - name: Archive TMT logs

Dockerfile

@@ -34,7 +34,8 @@ WORKDIR /src
 # We aren't using the full recommendations there, just the simple bits.
 # First we download all of our Rust dependencies
 # Note: Local path dependencies (from [patch] sections) are auto-detected and bind-mounted by the Justfile
-RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome cargo fetch
+RUN --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome \
+    rm -rf /var/roothome/.cargo/registry; cargo fetch
 
 # We always do a "from scratch" build
 # https://docs.fedoraproject.org/en-US/bootc/building-from-scratch/
@@ -143,12 +144,13 @@ RUN --network=none --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp
 # Perform all filesystem transformations except generating the sealed UKI (if configured)
 FROM base as base-penultimate
 ARG variant
-# Switch to a signed systemd-boot, if configured
+ARG bootloader
+# Switch to systemd-boot (signed or unsigned), if configured
 RUN --network=none --mount=type=tmpfs,target=/run --mount=type=tmpfs,target=/tmp \
     --mount=type=bind,from=packaging,src=/,target=/run/packaging \
     --mount=type=bind,from=sdboot-signed,src=/,target=/run/sdboot-signed <<EORUN
 set -xeuo pipefail
-if test "${variant}" = "composefs-sealeduki-sdboot"; then
+if [[ "${bootloader}" == "systemd" ]]; then
   /run/packaging/switch-to-sdboot /run/sdboot-signed
 fi
 EORUN

Justfile

@@ -20,6 +20,7 @@ upgrade_img := base_img + "-upgrade"
 
 # Build variant: ostree (default) or composefs-sealeduki-sdboot (sealed UKI)
 variant := env("BOOTC_variant", "ostree")
+bootloader := env("BOOTC_bootloader", "grub")
 # Base container image to build from
 base := env("BOOTC_base", "quay.io/centos-bootc/centos-bootc:stream10")
 # Buildroot base image
@@ -38,7 +39,7 @@ lbi_images := "quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.a
 fedora-coreos := "quay.io/fedora/fedora-coreos:testing-devel"
 generic_buildargs := ""
 _extra_src_args := if extra_src != "" { "-v " + extra_src + ":/run/extra-src:ro --security-opt=label=disable" } else { "" }
-base_buildargs := generic_buildargs + " " + _extra_src_args + " --build-arg=base=" + base + " --build-arg=variant=" + variant
+base_buildargs := generic_buildargs + " " + _extra_src_args + " --build-arg=base=" + base + " --build-arg=variant=" + variant + " --build-arg=bootloader=" + bootloader
 buildargs := base_buildargs \
              + " --cap-add=all --security-opt=label=type:container_runtime_t --device /dev/fuse" \
              + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt"
@@ -105,9 +106,33 @@ test-container: build build-units
 
 # Build and test sealed composefs images
 [group('core')]
-test-composefs:
+test-composefs-sealeduki-sdboot:
     just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot
 
+[group('core')]
+test-composefs bootloader:
+    just variant=composefs bootloader={{bootloader}} \
+        test-tmt --composefs-backend --bootloader {{bootloader}} \
+        readonly \
+        bib-build \
+        download-only \
+        image-pushpull-upgrade \
+        image-upgrade-reboot \
+        install-outside-container \
+        install-to-filesystem-var-mount \
+        soft-reboot \
+        usroverlay
+
+# Build and test composefs images booted using Type1 boot entries and systemd-boot as the bootloader
+[group('core')]
+test-composefs-sdboot:
+    just test-composefs systemd
+
+# Build and test composefs images booted using Type1 boot entries and grub as the bootloader
+[group('core')]
+test-composefs-grub:
+    just test-composefs grub
+
 # Run cargo fmt and clippy checks in container
 [group('core')]
 validate:
@@ -220,6 +245,7 @@ clean-local-images:
     podman image prune -f
     podman rmi {{fedora-coreos}} -f
 
+
 # Build packages (RPM) into target/packages/
 [group('maintenance')]
 package:

contrib/packaging/install-unsigned-sdboot

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Install unsigned systemd-boot RPM (already downloaded by tools stage)
+set -xeuo pipefail
+
+# Uninstall bootupd if present (we're switching to sd-boot managed differently)
+if rpm -q bootupd &>/dev/null; then
+  rpm -e bootupd
+  rm -vrf /usr/lib/bootupd/updates
+fi
+
+# Install the unsigned systemd-boot RPM that was downloaded by the tools stage
+# The RPM is available in /run/sdboot-signed/out (copied from tools stage)
+rpm -Uvh /run/sdboot-signed/out/*.rpm

crates/lib/src/bootc_composefs/boot.rs

@@ -1248,6 +1248,7 @@ pub(crate) async fn setup_composefs_boot(
             &state.config_opts,
             None,
             get_secureboot_keys(&mounted_fs, BOOTC_AUTOENROLL_PATH)?,
+            &mounted_fs,
         )?;
     }
 

crates/lib/src/bootloader.rs

@@ -165,6 +165,7 @@ pub(crate) fn install_systemd_boot(
     _configopts: &crate::install::InstallConfigOpts,
     _deployment_path: Option<&str>,
     autoenroll: Option<SecurebootKeys>,
+    mounted_erofs: &Dir,
 ) -> Result<()> {
     let esp_part = device
         .find_partition_of_type(discoverable_partition_specification::ESP)
@@ -180,6 +181,73 @@ pub(crate) fn install_systemd_boot(
         .log_debug()
         .run_inherited_with_cmd_context()?;
 
+    // Check for systemd-boot binaries in the EFI/systemd directory
+    // systemd v258 won't copy the binary if an EFI booted system is not detected
+    let systemd_dir = esp_mount.fd.open_dir_optional("EFI/systemd")?;
+
+    let systemd_boot_found = if let Some(dir) = systemd_dir {
+        let mut found = false;
+        for entry in dir.entries()? {
+            let entry = entry?;
+            let name = entry.file_name();
+
+            if let Some(name_str) = name.to_str() {
+                if name_str.starts_with("systemd-boot") && name_str.ends_with(".efi") {
+                    found = true;
+                    break;
+                }
+            }
+        }
+
+        found
+    } else {
+        false
+    };
+
+    if !systemd_boot_found {
+        println!("Copying systemd-boot binary manually");
+
+        // Find the systemd-boot binary in the source directory
+        let boot_dir = mounted_erofs.open_dir("usr/lib/systemd/boot/efi")?;
+        let mut systemd_boot_binary = None;
+
+        for entry in boot_dir.entries()? {
+            let entry = entry?;
+            let name = entry.file_name();
+            if let Some(name_str) = name.to_str() {
+                if name_str.starts_with("systemd-boot") && name_str.ends_with(".efi") {
+                    systemd_boot_binary = Some(name_str.to_string());
+                    break;
+                }
+            }
+        }
+
+        let binary_name = systemd_boot_binary
+            .ok_or_else(|| anyhow::anyhow!("No systemd-boot binary found in source"))?;
+
+        let src_path = format!("usr/lib/systemd/boot/efi/{}", binary_name);
+        let systemd_dest_path = format!("EFI/systemd/{}", binary_name);
+
+        // Determine the appropriate BOOT binary name based on architecture
+        let boot_binary_name = if binary_name.contains("x64") {
+            "BOOTX64.EFI"
+        } else if binary_name.contains("ia32") {
+            "BOOTIA32.EFI"
+        } else if binary_name.contains("aa64") {
+            "BOOTAA64.EFI"
+        } else {
+            "BOOTX64.EFI" // Default fallback
+        };
+
+        mounted_erofs.copy(&src_path, &esp_mount.fd, &systemd_dest_path)?;
+
+        mounted_erofs.copy(
+            &src_path,
+            &esp_mount.fd,
+            &format!("EFI/BOOT/{}", boot_binary_name),
+        )?;
+    }
+
     if let Some(SecurebootKeys { dir, keys }) = autoenroll {
         let path = esp_path.join(SYSTEMD_KEY_DIR);
         create_dir_all(&path)?;

crates/tests-integration/src/container.rs

@@ -52,8 +52,8 @@ pub(crate) fn test_bootc_container_inspect() -> Result<()> {
         .expect("kernel.unified should be a boolean");
     if let Some(variant) = std::env::var("BOOTC_variant").ok() {
         match variant.as_str() {
-            "ostree" => {
-                assert!(!unified, "Expected unified=false for ostree variant");
+            v @ "ostree" | v @ "composefs" => {
+                assert!(!unified, "Expected unified=false for variant {v}");
                 // For traditional kernels, version should look like a uname (contains digits)
                 assert!(
                     version.chars().any(|c| c.is_ascii_digit()),
@@ -159,7 +159,7 @@ fn test_variant_base_crosscheck() -> Result<()> {
         // TODO add this to `bootc status` or so?
         let boot_efi = Utf8Path::new("/boot/EFI");
         match variant.as_str() {
-            "ostree" => {
+            "composefs" | "ostree" => {
                 assert!(!boot_efi.try_exists()?);
             }
             "composefs-sealeduki-sdboot" => {

crates/xtask/src/tmt.rs

@@ -29,6 +29,8 @@ const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image";
 // Distro identifiers
 const DISTRO_CENTOS_9: &str = "centos-9";
 
+const COMPOSEFS_KERNEL_ARGS: [&str; 1] = ["--karg=enforcing=0"];
+
 // Import the argument types from xtask.rs
 use crate::{RunTmtArgs, TmtProvisionArgs};
 
@@ -430,6 +432,17 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
                     opts.push("--filesystem=xfs".to_string());
                 }
             }
+
+            if args.composefs_backend {
+                opts.push("--filesystem=ext4".into());
+                opts.push("--composefs-backend".into());
+                opts.extend(COMPOSEFS_KERNEL_ARGS.map(|x| x.into()));
+            }
+
+            if let Some(b) = &args.bootloader {
+                opts.push(format!("--bootloader={b}"));
+            }
+
             opts
         };
 

crates/xtask/src/xtask.rs

@@ -5,13 +5,14 @@
 //! end up as a lot of nontrivial bash code.
 
 use std::borrow::Cow;
+use std::fmt::Display;
 use std::fs::File;
 use std::io::{BufRead, BufReader, BufWriter, Write};
 use std::process::Command;
 
 use anyhow::{Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
-use clap::{Args, Parser, Subcommand};
+use clap::{Args, Parser, Subcommand, ValueEnum};
 use fn_error_context::context;
 use xshell::{Shell, cmd};
 
@@ -76,6 +77,25 @@ pub(crate) struct LocalRustDepsArgs {
     pub(crate) format: String,
 }
 
+/// Bootloader passed as --bootloader param for composefs builds
+// TODO: Find a better way to share this Enum between this and crates/lib
+#[derive(Debug, Clone, ValueEnum)]
+pub enum Bootloader {
+    /// grub as bootloader
+    Grub,
+    /// systemd-boot as bootloader
+    Systemd,
+}
+
+impl Display for Bootloader {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Bootloader::Grub => f.write_str("grub"),
+            Bootloader::Systemd => f.write_str("systemd"),
+        }
+    }
+}
+
 /// Arguments for run-tmt command
 #[derive(Debug, Args)]
 pub(crate) struct RunTmtArgs {
@@ -101,6 +121,12 @@ pub(crate) struct RunTmtArgs {
     /// Preserve VMs after test completion (useful for debugging)
     #[arg(long)]
     pub(crate) preserve_vm: bool,
+
+    #[arg(long)]
+    pub(crate) composefs_backend: bool,
+
+    #[arg(long, requires = "composefs_backend")]
+    pub(crate) bootloader: Option<Bootloader>,
 }
 
 /// Arguments for tmt-provision command

tmt/tests/booted/readonly/001-test-status.nu

@@ -14,7 +14,7 @@ assert ($opts | any { |o| $o == "ro" }) "/sysroot should be mounted read-only"
 
 let st = bootc status --json | from json
 # Detect composefs by checking if composefs field is present
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 
 assert equal $st.apiVersion org.containers.bootc/v1
 

tmt/tests/booted/readonly/010-test-bootc-container-store.nu

@@ -5,7 +5,7 @@ tap begin "verify bootc-owned container storage"
 
 # Detect composefs by checking if composefs field is present
 let st = bootc status --json | from json
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 
 if $is_composefs {
     print "# TODO composefs: skipping test - /usr/lib/bootc/storage doesn't exist with composefs"

tmt/tests/booted/readonly/011-test-ostree-ext-cli.nu

@@ -8,7 +8,7 @@ tap begin "verify bootc wrapping ostree-ext"
 # Parse the status and get the booted image
 let st = bootc status --json | from json
 # Detect composefs by checking if composefs field is present
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 if $is_composefs {
     print "# TODO composefs: skipping test - ostree-container commands don't work with composefs"
 } else {

tmt/tests/booted/readonly/011-test-resolvconf.nu

@@ -6,7 +6,7 @@ tap begin "verify there's not an empty /etc/resolv.conf in the image"
 let st = bootc status --json | from json
 
 # Detect composefs by checking if composefs field is present
-let is_composefs = ($st.status.booted.composefs? != null)
+let is_composefs = (tap is_composefs)
 if $is_composefs {
     print "# TODO composefs: skipping test - ostree commands don't work with composefs"
 } else {