ci: Unify more of hack/ and tests/

A key thing for me is that the `Justfile` should be a one-stop
shop for development of the project. It can't have everything but
it should answer the basic questions of "how do I build and test
this project".

This aligns the recently added tmt-on-GHA flow a *bit* more closely
with some of that. Biggest is to use the `just build-integration-test-image` as the canonical
way to build a container image with our testing stuff in it;
which uses our main Dockerfile

Other cleanups:
- Change test script to move into tests/tmt/ as a workaround for
  https://github.com/teemtee/tmt/pull/3037#issuecomment-3259585271
- Change the qemu logic to use SMBIOS credentials so we don't
  have to carry around both a disk image and a SSH key
- Change qemu to use `-snapshot` so we can reuse disks
- Change the scripts to accept data via argv[1] and not environment
- Drop the hardcoded testing directory and use `target/` as
  a generic build artifact dir

Signed-off-by: Colin Walters <walters@verbum.org>

.github/workflows/ci.yml

@@ -56,7 +56,9 @@ jobs:
         run: sudo apt update && sudo apt install just
       - uses: actions/checkout@v4
       - name: Build and run container integration tests
-        run: sudo just run-container-integration run-container-external-tests
+        run: |
+          sudo just build
+          sudo just run-container-integration run-container-external-tests
   container-continuous:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'control/skip-ci') }}
     runs-on: ubuntu-24.04
@@ -105,7 +107,8 @@ jobs:
           set -xeu
           # Build images to test; TODO investigate doing single container builds
           # via GHA and pushing to a temporary registry to share among workflows?
-          sudo just build-integration-test-image
+          sudo just build
+          sudo just build-install-test-image
           sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity
 
           # TODO move into a container, and then have this tool run other containers
@@ -120,9 +123,9 @@ jobs:
           sudo podman run --privileged --pid=host -v /:/run/host -v $(pwd):/src:ro -v /var/tmp:/var/tmp \
             -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd localhost/bootc /src/crates/ostree-ext/ci/priv-integration.sh
           # Nondestructive but privileged tests
-          sudo bootc-integration-tests host-privileged localhost/bootc-integration
+          sudo bootc-integration-tests host-privileged localhost/bootc-integration-install
           # Install tests
-          sudo bootc-integration-tests install-alongside localhost/bootc-integration
+          sudo bootc-integration-tests install-alongside localhost/bootc-integration-install
 
           # system-reinstall-bootc tests
           cargo build --release -p system-reinstall-bootc

.github/workflows/integration.yml

@@ -1,84 +1,81 @@
-name: bootc integration test
+# This workflow builds a container across a matrix of OSes,
+# generates a disk image from that, and runs integration tests
+# using tmt + libvirt (using nested virt support in the default GHA runners).
+name: Build+TMT
 on:
   pull_request:
-    branches: [main]
+    branches: [main]  
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
-        test_os: [fedora-41, fedora-42, fedora-43, centos-9]
+        test_os: [fedora-42, fedora-43, centos-9, centos-10]
-        test_runner: [ubuntu-latest, ubuntu-24.04-arm]
 
-    runs-on: ${{ matrix.test_runner }}
+    runs-on: ubuntu-24.04
 
     steps:
-      - name: Install podman for heredoc support
+      - name: Install dependencies
         run: |
           set -eux
           echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
           sudo apt update
-          sudo apt install -y crun/testing podman/testing
+          sudo apt install -y crun/testing podman/testing just qemu-utils
 
       - uses: actions/checkout@v4
 
-      - name: Build bootc and bootc image
+      - name: Set architecture variable
-        env:
+        id: set_arch
-          TEST_OS: ${{ matrix.test_os }}
+        run: echo "ARCH=$(arch)" >> $GITHUB_ENV
-        run: sudo -E TEST_OS=$TEST_OS tests/build.sh
 
-      - name: Grant sudo user permission to archive files
+      - name: Build container and disk image
         run: |
-          sudo chmod 0755 /tmp/tmp-bootc-build/id_rsa
+          sudo tests/build.sh ${{ matrix.test_os }}
 
-      - name: Archive bootc disk image - disk.raw
+      - name: Archive disk image
-        if: matrix.test_runner == 'ubuntu-latest'
         uses: actions/upload-artifact@v4
         with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-disk
-          path: /tmp/tmp-bootc-build/disk.raw
+          path: target/bootc-integration-test.qcow2
-          retention-days: 1
-
-      - name: Archive SSH private key - id_rsa
-        if: matrix.test_runner == 'ubuntu-latest'
-        uses: actions/upload-artifact@v4
-        with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa
-          path: /tmp/tmp-bootc-build/id_rsa
           retention-days: 1
 
   test:
     needs: build
     strategy:
+      fail-fast: false
       matrix:
-        test_os: [fedora-41, fedora-42, fedora-43, centos-9]
+        test_os: [fedora-42, fedora-43, centos-9, centos-10]
-        tmt_plan: [test-01-readonly, test-20-local-upgrade, test-21-logically-bound-switch, test-22-logically-bound-install, test-23-install-outside-container, test-24-local-upgrade-reboot]
 
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install dependence
+      - name: Set architecture variable
+        id: set_arch
+        run: echo "ARCH=$(arch)" >> $GITHUB_ENV
+
+      - name: Install deps
         run: |
           sudo apt-get update
-          sudo apt install -y qemu-kvm qemu-system
+          # see https://tmt.readthedocs.io/en/stable/overview.html#install
-          pip install --user tmt
+          sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-kvm qemu-utils libvirt-daemon-system
+          pip install --user "tmt[provision-virtual]"
 
       - name: Create folder to save disk image
-        run: mkdir -p /tmp/tmp-bootc-build
+        run: mkdir -p target
 
       - name: Download disk.raw
         uses: actions/download-artifact@v4
         with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-disk
+          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-disk
-          path: /tmp/tmp-bootc-build
+          path: target
-
-      - name: Download id_rsa
-        uses: actions/download-artifact@v4
-        with:
-          name: PR-${{ github.event.number }}-${{ matrix.test_os }}-id_rsa
-          path: /tmp/tmp-bootc-build
 
       - name: Enable KVM group perms
         run: |
@@ -87,14 +84,16 @@ jobs:
           sudo udevadm trigger --name-match=kvm
           ls -l /dev/kvm
 
+      - name: Workaround https://github.com/teemtee/testcloud/issues/18
+        run: sudo rm -f /usr/bin/chcon && sudo ln -sr /usr/bin/true /usr/bin/chcon
+
       - name: Run test
-        env:
+        run: |
-          TMT_PLAN_NAME: ${{ matrix.tmt_plan }}
+          tests/run-tmt.sh
-        run: chmod 600 /tmp/tmp-bootc-build/id_rsa && tests/test.sh
 
       - name: Archive TMT logs
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ matrix.tmt_plan }}
+          name: tmt-log-PR-${{ github.event.number }}-${{ matrix.test_os }}-${{ env.ARCH }}-${{ matrix.tmt_plan }}
           path: /var/tmp/tmt

.packit.yaml

@@ -59,24 +59,26 @@ jobs:
     owner: rhcontainerbot
     project: bootc
     enable_net: true
+    # TODO
     notifications:
       failure_comment:
         message: "bootc Copr build failed for {commit_sha}. @admin check logs {logs_url} and packit dashboard {packit_dashboard_url}"
 
-  - job: tests
+  # TODO: Readd some tmt tests that install the built RPM and e.g. test out system-reinstall-bootc
-    trigger: pull_request
+  # - job: tests
-    targets:
+  #   trigger: pull_request
-      - centos-stream-9-x86_64
+  #   targets:
-      - centos-stream-9-aarch64
+  #     - centos-stream-9-x86_64
-      - centos-stream-10-x86_64
+  #     - centos-stream-9-aarch64
-      - centos-stream-10-aarch64
+  #     - centos-stream-10-x86_64
-      - fedora-42-x86_64
+  #     - centos-stream-10-aarch64
-      - fedora-42-aarch64
+  #     - fedora-42-x86_64
-      - fedora-rawhide-x86_64
+  #     - fedora-42-aarch64
-      - fedora-rawhide-aarch64
+  #     - fedora-rawhide-x86_64
-    tmt_plan: /integration
+  #     - fedora-rawhide-aarch64
-    skip_build: true
+  #   tmt_plan: /integration
-    identifier: integration-test
+  #   skip_build: true
+  #   identifier: integration-test
 
   - job: propose_downstream
     trigger: release

Justfile

@@ -3,8 +3,14 @@ build *ARGS:
     podman build --jobs=4 -t localhost/bootc {{ARGS}} .
 
 # This container image has additional testing content and utilities
-build-integration-test-image *ARGS: build
+build-integration-test-image *ARGS:
     podman build --jobs=4 -t localhost/bootc-integration -f hack/Containerfile {{ARGS}} .
+    # Keep these in sync with what's used in hack/lbi
+    podman pull -q --retry 5 --retry-delay 5s quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest
+
+# Only used by ci.yml right now
+build-install-test-image: build-integration-test-image
+    cd hack && podman build -t localhost/bootc-integration-install -f Containerfile.drop-lbis
 
 # Run container integration tests
 run-container-integration: build-integration-test-image

Makefile

@@ -96,9 +96,6 @@ bin-archive: all
 test-bin-archive: all
 	$(MAKE) install-all DESTDIR=tmp-install && $(TAR_REPRODUCIBLE) --zstd -C tmp-install -cf target/bootc.tar.zst . && rm tmp-install -rf
 
-test-tmt:
-	cargo xtask test-tmt
-
 test:
 	tests/build.sh && tests/test.sh
 

crates/tests-integration/src/install.rs

@@ -26,6 +26,10 @@ pub(crate) fn delete_ostree(sh: &Shell) -> Result<(), anyhow::Error> {
     if !Path::new("/ostree/").exists() {
         return Ok(());
     }
+    // TODO: This shouldn't be leaking out of installs
+    cmd!(sh, "sudo umount -Rl /ostree/bootc/storage/overlay")
+        .ignore_status()
+        .run()?;
     cmd!(sh, "sudo /bin/sh -c 'rm -rf /ostree/'").run()?;
     Ok(())
 }

crates/xtask/src/xtask.rs

@@ -13,11 +13,6 @@ use xshell::{cmd, Shell};
 mod man;
 
 const NAME: &str = "bootc";
-const TEST_IMAGES: &[&str] = &[
-    "quay.io/curl/curl-base:latest",
-    "quay.io/curl/curl:latest",
-    "registry.access.redhat.com/ubi9/podman:latest",
-];
 const TAR_REPRODUCIBLE_OPTS: &[&str] = &[
     "--sort=name",
     "--owner=0",
@@ -43,7 +38,6 @@ const TASKS: &[(&str, fn(&Shell) -> Result<()>)] = &[
     ("package", package),
     ("package-srpm", package_srpm),
     ("spec", spec),
-    ("test-tmt", test_tmt),
 ];
 
 fn try_main() -> Result<()> {
@@ -100,78 +94,6 @@ fn gitrev(sh: &Shell) -> Result<String> {
     }
 }
 
-#[context("test-integration")]
-fn all_plan_files(sh: &Shell) -> Result<Vec<(u32, String)>> {
-    // We need to split most of our tests into separate plans because tmt doesn't
-    // support automatic isolation. (xref)
-    let mut all_plan_files =
-        sh.read_dir("plans")?
-            .into_iter()
-            .try_fold(Vec::new(), |mut acc, ent| -> Result<_> {
-                let path = Utf8PathBuf::try_from(ent)?;
-                let Some(ext) = path.extension() else {
-                    return Ok(acc);
-                };
-                if ext != "fmf" {
-                    return Ok(acc);
-                }
-                let stem = path.file_stem().expect("file stem");
-                let Some((prefix, suffix)) = stem.split_once('-') else {
-                    return Ok(acc);
-                };
-                if prefix != "test" {
-                    return Ok(acc);
-                }
-                let Some((priority, _)) = suffix.split_once('-') else {
-                    anyhow::bail!("Invalid test {path}");
-                };
-                let priority: u32 = priority
-                    .parse()
-                    .with_context(|| format!("Parsing {path}"))?;
-                acc.push((priority, stem.to_string()));
-                Ok(acc)
-            })?;
-    all_plan_files.sort_by_key(|v| v.0);
-    println!("Discovered plans: {all_plan_files:?}");
-    Ok(all_plan_files)
-}
-
-#[context("test-integration")]
-fn test_tmt(sh: &Shell) -> Result<()> {
-    let mut tests = all_plan_files(sh)?;
-    if let Ok(name) = std::env::var("TMT_TEST") {
-        tests.retain(|x| x.1.as_str() == name);
-        if tests.is_empty() {
-            anyhow::bail!("Failed to match test: {name}");
-        }
-    }
-
-    // pull some small images that are used for LBI installation tests
-    cmd!(sh, "podman pull {TEST_IMAGES...}").run()?;
-
-    for (_prio, name) in tests {
-        // cc https://pagure.io/testcloud/pull-request/174
-        cmd!(sh, "rm -vf /var/tmp/tmt/testcloud/images/disk.qcow2").run()?;
-        let verbose_enabled = std::env::var("TMT_VERBOSE")
-            .ok()
-            .and_then(|s| s.parse::<u32>().ok())
-            .unwrap_or(0);
-
-        let verbose = if verbose_enabled == 1 {
-            Some("-vvvvv".to_string())
-        } else {
-            None
-        };
-
-        if let Err(e) = cmd!(sh, "tmt {verbose...} run plans -n {name}").run() {
-            // tmt annoyingly does not output errors by default
-            let _ = cmd!(sh, "tmt run -l report -vvv").run();
-            return Err(e.into());
-        }
-    }
-    Ok(())
-}
-
 /// Return a string formatted version of the git commit timestamp, up to the minute
 /// but not second because, well, we're not going to build more than once a second.
 #[context("Finding git timestamp")]

hack/Containerfile

@@ -1,4 +1,7 @@
-# This injects some extra testing stuff into our image
+# Build a container image that has extra testing stuff in it, such
+# as nushell, some preset logically bound images, etc. This expects
+# to create an image derived FROM localhost/bootc which was created
+# by the Dockerfile at top.
 
 FROM scratch as context
 # We only need this stuff in the initial context
@@ -11,7 +14,15 @@ ARG variant=
 # And this layer has additional stuff for testing, such as nushell etc.
 RUN --mount=type=bind,from=context,target=/run/context <<EORUN
 set -xeuo pipefail
-/run/context/hack/provision-derived.sh "$variant"
+cd /run/context/hack
+./provision-derived.sh "$variant"
+
+# For test-22-logically-bound-install
+cp -a lbi/usr/. /usr
+for x in curl.container curl-base.image podman.image; do
+    ln -s /usr/share/containers/systemd/$x /usr/lib/bootc/bound-images.d/$x
+done
+
 # Add some testing kargs into our dev builds
 install -D -t /usr/lib/bootc/kargs.d /run/context/hack/test-kargs/*
 # Also copy in some default install configs we use for testing

hack/Containerfile.drop-lbis

@@ -0,0 +1,3 @@
+FROM localhost/bootc-integration
+# Workaround for https://github.com/bootc-dev/bootc/issues/1618
+RUN rm -rf /usr/lib/bootc/bound-images.d/*

hack/lbi/usr/share/containers/systemd/curl-base.image

@@ -0,0 +1,2 @@
+[Image]
+Image=quay.io/curl/curl-base:latest

hack/lbi/usr/share/containers/systemd/curl.container

@@ -0,0 +1,3 @@
+[Container]
+Image=quay.io/curl/curl:latest
+GlobalArgs=--storage-opt=additionalimagestore=/usr/lib/bootc/storage

hack/lbi/usr/share/containers/systemd/jboss-webserver-5.image

@@ -0,0 +1,6 @@
+# This is not symlinked to bound-images.d so it should not be pulled.
+# It's here to represent an app image that exists
+# in a bootc image but is not logically bound.
+[Image]
+Image=registry.redhat.io/jboss-webserver-5/jws5-rhel8-operator:latest
+AuthFile=/root/auth.json

hack/lbi/usr/share/containers/systemd/podman.image

@@ -0,0 +1,2 @@
+[Image]
+Image=registry.access.redhat.com/ubi9/podman:latest

hack/packages.txt

@@ -0,0 +1,5 @@
+# Needed by tmt
+rsync
+cloud-init
+/usr/bin/flock
+/usr/bin/awk

hack/provision-derived.sh

@@ -16,30 +16,45 @@ case "${ID}-${VERSION_ID}" in
         dnf config-manager --set-enabled crb
         dnf -y install epel-release epel-next-release
         dnf -y install nu
-        dnf clean all
         ;;
     "rhel-9."*)
         dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
         dnf -y install nu
-        dnf clean all
         ;;
     "centos-10"|"rhel-10."*)
         # nu is not available in CS10
+        td=$(mktemp -d)
+        cd $td
         curl -kL "https://github.com/nushell/nushell/releases/download/0.103.0/nu-0.103.0-$(uname -m)-unknown-linux-gnu.tar.gz" --output nu.tar.gz
         mkdir -p nu && tar zvxf nu.tar.gz --strip-components=1 -C nu
         mv nu/nu /usr/bin/nu
         rm -rf nu nu.tar.gz
+        cd -
+        rm -rf "${td}"
         ;;
     "fedora-"*)
         dnf -y install nu
-        dnf clean all
         ;;
 esac
 
+# Extra packages we install
+grep -Ev -e '^#' packages.txt | xargs dnf -y install
+dnf clean all
+
+# Cloud bits
+cat <<KARGEOF >> /usr/lib/bootc/kargs.d/20-console.toml
+kargs = ["console=ttyS0,115200n8"]
+KARGEOF
+# And cloud-init stuff
+ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants
+
 # Stock extra cleaning of logs and caches in general (mostly dnf)
 rm /var/log/* /var/cache /var/lib/{dnf,rpm-state,rhsm} -rf
 # And clean root's homedir
 rm /var/roothome/.config -rf
+cat >/usr/lib/tmpfiles.d/bootc-cloud-init.conf <<'EOF'
+d /var/lib/cloud 0755 root root - -
+EOF
 
 # Fast track tmpfiles.d content from the base image, xref
 # https://gitlab.com/fedora/bootc/base-images/-/merge_requests/92
@@ -65,3 +80,23 @@ if ! grep -q -r sudo /usr/lib/sysusers.d; then
 g sudo 16
 EOF
 fi
+
+# dhcpcd
+if rpm -q dhcpcd &>/dev/null; then
+if ! grep -q -r dhcpcd /usr/lib/sysusers.d; then
+  cat >/usr/lib/sysusers.d/bootc-dhcpcd-workaround.conf <<'EOF'
+u dhcpcd - 'Minimalistic DHCP client' /var/lib/dhcpcd
+EOF
+fi
+cat >/usr/lib/tmpfiles.d/bootc-dhcpd.conf <<'EOF'
+d /var/lib/dhcpcd 0755 root dhcpcd - -
+EOF
+  rm -rf /var/lib/dhcpcd
+fi
+# dhclient
+if test -d /var/lib/dhclient; then
+  cat >/usr/lib/tmpfiles.d/bootc-dhclient.conf <<'EOF'
+d /var/lib/dhclient 0755 root root - -
+EOF
+  rm -rf /var/lib/dhclient
+fi

tests/build.sh

@@ -4,116 +4,66 @@ set -exuo pipefail
 # This script basically builds bootc from source using the provided base image,
 # then runs the target tests.
 
-mkdir -p /tmp/tmp-bootc-build
+# If provided should be of the form fedora-42 or centos-10
-BOOTC_TEMPDIR="/tmp/tmp-bootc-build"
+target=${1:-}
 
-# Get OS info from TEST_OS env
+bcvk=$(which bcvk 2>/dev/null || true)
-OS_ID=$(echo "$TEST_OS" | cut -d '-' -f 1)
+if test -z "${bcvk}" && test "$(id -u)" != 0; then
-OS_VERSION_ID=$(echo "$TEST_OS" | cut -d '-' -f 2)
+    echo "This script currently requires full root"; exit 1
+fi
 
-# Base image
+build_args=()
-case "$OS_ID" in
+if test -n "${target:-}"; then
-    "centos")
+    shift
-        TIER1_IMAGE_URL="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}"
+    # Get OS info from TEST_OS env
+    OS_ID=$(echo "$target" | cut -d '-' -f 1)
+    OS_VERSION_ID=$(echo "$target" | cut -d '-' -f 2)
+
+    # Base image
+    case "$OS_ID" in
+        "centos")
+            BASE="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}"
         ;;
-    "fedora")
+        "fedora")
-        TIER1_IMAGE_URL="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}"
+            BASE="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}"
         ;;
-esac
+        *) echo "Unknown OS: ${OS_ID}" 1>&2; exit 1
+        ;;
+    esac
+    build_args+=("--build-arg=base=$BASE")
+fi
 
-CONTAINERFILE="${BOOTC_TEMPDIR}/Containerfile"
+just build ${build_args[@]}
-tee "$CONTAINERFILE" > /dev/null << CONTAINERFILEOF
+just build-integration-test-image
-FROM $TIER1_IMAGE_URL as build
 
-WORKDIR /code
+# Host builds will have this already, but we use it as a general dumping space
+# for output artifacts
+mkdir -p target
 
-RUN <<EORUN
+SIZE=10G
-set -xeuo pipefail
+DISK=target/bootc-integration-test.qcow2
-. /usr/lib/os-release
+rm -vf "${DISK}"
-case \$ID in
+# testcloud barfs on .raw
-    centos|rhel) dnf config-manager --set-enabled crb;;
+if test -n "${bcvk}"; then
-    fedora) dnf -y install dnf-utils 'dnf5-command(builddep)';;
+  bcvk to-disk --format=qcow2 --disk-size "${SIZE}" localhost/bootc-integration "${DISK}"
-esac
+else
-dnf -y distro-sync ostree{,-libs} systemd
+  TMPDISK=target/bootc-integration-test.raw
-dnf -y builddep contrib/packaging/bootc.spec
+  truncate -s "${SIZE}" "${TMPDISK}"
-dnf -y install git-core
+  podman run \
-EORUN
+    --rm \
-
+    --privileged \
-RUN mkdir -p /build/target/dev-rootfs
+    --pid=host \
-# git config --global --add safe.directory /code to fix "fatal: detected dubious ownership in repository at '/code'" error
+    --security-opt label=type:unconfined_t \
-RUN --mount=type=cache,target=/build/target --mount=type=cache,target=/var/roothome git config --global --add safe.directory /code && make test-bin-archive && mkdir -p /out && cp target/bootc.tar.zst /out
+    -v /var/lib/containers:/var/lib/containers \
-
+    -v /dev:/dev \
-FROM $TIER1_IMAGE_URL
+    -v $(pwd)/target:/target \
-
+    localhost/bootc-integration \
-# Inject our built code
+    bootc install to-disk \
-COPY --from=build /out/bootc.tar.zst /tmp
+    --filesystem "xfs" \
-RUN tar -C / --zstd -xvf /tmp/bootc.tar.zst && rm -vrf /tmp/*
+    --karg=console=ttyS0,115200n8 \
-
+    --generic-image \
-RUN <<EORUN
+    --via-loopback \
-set -xeuo pipefail
+    /target/$(basename ${TMPDISK})
-
+  qemu-img convert -f raw -O qcow2 ${TMPDISK} ${DISK}
-# Provision test requirement
+  rm -f "${TMPDISK}"
-/code/hack/provision-derived.sh
+fi
-# Also copy in some default install configs we use for testing
-cp -a /code/hack/install-test-configs/* /usr/lib/bootc/install/
-# And some test kargs
-cp -a /code/hack/test-kargs/* /usr/lib/bootc/kargs.d/
-
-# For testing farm
-mkdir -p -m 0700 /var/roothome
-
-# Enable ttyS0 console
-mkdir -p /usr/lib/bootc/kargs.d/
-cat <<KARGEOF >> /usr/lib/bootc/kargs.d/20-console.toml
-kargs = ["console=ttyS0,115200n8"]
-KARGEOF
-
-# For test-22-logically-bound-install
-cp -a /code/tmt/tests/lbi/usr/. /usr
-ln -s /usr/share/containers/systemd/curl.container /usr/lib/bootc/bound-images.d/curl.container
-ln -s /usr/share/containers/systemd/curl-base.image /usr/lib/bootc/bound-images.d/curl-base.image
-ln -s /usr/share/containers/systemd/podman.image /usr/lib/bootc/bound-images.d/podman.image
-
-# Install rsync which is required by tmt
-dnf -y install cloud-init rsync
-dnf -y clean all
-
-rm -rf /var/cache /var/lib/dnf
-EORUN
-CONTAINERFILEOF
-
-LOCAL_IMAGE="localhost/bootc:test"
-podman build \
-    --retry 5 \
-    --retry-delay 5s \
-    -v "$(pwd)":/code:z \
-    -t "$LOCAL_IMAGE" \
-    -f "$CONTAINERFILE" \
-    "$BOOTC_TEMPDIR"
-
-SSH_KEY=${BOOTC_TEMPDIR}/id_rsa
-ssh-keygen -f "${SSH_KEY}" -N "" -q -t rsa-sha2-256 -b 2048
-
-truncate -s 10G "${BOOTC_TEMPDIR}/disk.raw"
-
-# For test-22-logically-bound-install
-podman pull --retry 5 --retry-delay 5s quay.io/curl/curl:latest
-podman pull --retry 5 --retry-delay 5s quay.io/curl/curl-base:latest
-podman pull --retry 5 --retry-delay 5s registry.access.redhat.com/ubi9/podman:latest
-
-podman run \
-  --rm \
-  --privileged \
-  --pid=host \
-  --security-opt label=type:unconfined_t \
-  -v /var/lib/containers:/var/lib/containers \
-  -v /dev:/dev \
-  -v "$BOOTC_TEMPDIR":/output \
-  "$LOCAL_IMAGE" \
-  bootc install to-disk \
-  --filesystem "xfs" \
-  --root-ssh-authorized-keys "/output/id_rsa.pub" \
-  --karg=console=ttyS0,115200n8 \
-  --generic-image \
-  --via-loopback \
-  /output/disk.raw

tests/run-tmt.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -exuo pipefail
+
+# You must have invoked test/build.sh before running this.
+# This is basically a wrapper for tmt which sets up context
+# (to point to our disk image) and works around bugs in
+# tmt and testcloud.
+# Use e.g. `./tests/run-tmt.sh plan --name test-21-logically-bound-switch`
+# to run an individual test.
+
+# Ensure we're in the topdir canonically
+cd $(git rev-parse --show-toplevel)
+
+DISK=$(pwd)/target/bootc-integration-test.qcow2
+test -f "${DISK}"
+
+# Move the tmt bits to a subdirectory to work around https://github.com/teemtee/tmt/issues/4062
+mkdir -p target/tmt-workdir
+rsync -a --delete --force .fmf tmt target/tmt-workdir/
+
+# Hack around https://github.com/teemtee/testcloud/issues/17
+rm -vrf /var/tmp/tmt/testcloud/images/bootc-integration-test.qcow2
+
+cd target/tmt-workdir
+# TMT will rsync tmt-* scripts to TMT_SCRIPTS_DIR=/var/lib/tmt/scripts
+exec tmt --context "test_disk_image=${DISK}" run --all -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts "$@"

tests/test.sh

@@ -1,70 +0,0 @@
-#!/bin/bash
-set -exuo pipefail
-
-# This script runs disk image with qemu-system and run tmt against this vm.
-
-BOOTC_TEMPDIR="/tmp/tmp-bootc-build"
-SSH_OPTIONS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5)
-SSH_KEY=${BOOTC_TEMPDIR}/id_rsa
-
-ARCH=$(uname -m)
-case "$ARCH" in
-"aarch64")
-  qemu-system-aarch64 \
-    -name bootc-vm \
-    -enable-kvm \
-    -machine virt \
-    -cpu host \
-    -m 2G \
-    -bios /usr/share/AAVMF/AAVMF_CODE.fd \
-    -drive file="${BOOTC_TEMPDIR}/disk.raw",if=virtio,format=raw \
-    -net nic,model=virtio \
-    -net user,hostfwd=tcp::2222-:22 \
-    -display none \
-    -daemonize
-  ;;
-"x86_64")
-  qemu-system-x86_64 \
-    -name bootc-vm \
-    -enable-kvm \
-    -cpu host \
-    -m 2G \
-    -drive file="${BOOTC_TEMPDIR}/disk.raw",if=virtio,format=raw \
-    -net nic,model=virtio \
-    -net user,hostfwd=tcp::2222-:22 \
-    -display none \
-    -daemonize
-  ;;
-*)
-  echo "Only support x86_64 and aarch64" >&2
-  exit 1
-  ;;
-esac
-
-wait_for_ssh_up() {
-  SSH_STATUS=$(ssh "${SSH_OPTIONS[@]}" -i "$SSH_KEY" -p 2222 root@"${1}" '/bin/bash -c "echo -n READY"')
-  if [[ $SSH_STATUS == READY ]]; then
-    echo 1
-  else
-    echo 0
-  fi
-}
-
-for _ in $(seq 0 30); do
-  RESULT=$(wait_for_ssh_up "localhost")
-  if [[ $RESULT == 1 ]]; then
-    echo "SSH is ready now! 🥳"
-    break
-  fi
-  sleep 10
-done
-
-# Make sure VM is ready for testing
-ssh "${SSH_OPTIONS[@]}" \
-  -i "$SSH_KEY" \
-  -p 2222 \
-  root@localhost \
-  "bootc status"
-
-# TMT will rsync tmt-* scripts to TMT_SCRIPTS_DIR=/var/lib/tmt/scripts
-tmt run --all --verbose -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision --how connect --guest localhost --port 2222 --user root --key "$SSH_KEY" plan --name "/tmt/plans/bootc-integration/${TMT_PLAN_NAME}"

tmt/plans/bootc-integration.fmf

@@ -1,46 +0,0 @@
-execute:
-  how: tmt
-
-/test-01-readonly:
-  summary: Execute booted readonly/nondestructive tests
-  discover:
-    how: fmf
-    test:
-        - /tmt/tests/test-01-readonly
-
-/test-20-local-upgrade:
-  summary: Execute local upgrade tests
-  discover:
-    how: fmf
-    test:
-        - /tmt/tests/test-20-local-upgrade
-
-/test-21-logically-bound-switch:
-  summary: Execute logically bound images tests for switching images
-  discover:
-    how: fmf
-    test:
-        - /tmt/tests/test-21-logically-bound-switch
-
-/test-22-logically-bound-install:
-  summary: Execute logically bound images tests for switching images
-  environment+:
-    LBI: enabled
-  discover:
-    how: fmf
-    test:
-        - /tmt/tests/test-22-logically-bound-install
-
-/test-23-install-outside-container:
-  summary: Execute tests for installing outside of a container
-  discover:
-    how: fmf
-    test:
-        - /tmt/tests/test-23-install-outside-container
-
-/test-24-local-upgrade-reboot:
-  summary: Execute local upgrade tests with automated reboot
-  discover:
-    how: fmf
-    test:
-        - /tmt/tests/test-24-local-upgrade-reboot

tmt/plans/integration.fmf

@@ -1,24 +1,15 @@
-# Please change the image when you run this plan locally with tmt run
-# tmt run -vvvvv plan -n /integration/test-01-readonly
-# local image: file:///home/foobar/image.qcow2
 provision:
   how: virtual
-  image: fedora-rawhide
+  # Build via `./tests/build.sh`
-prepare:
+  image: $@{test_disk_image}
-  - how: install
-    package:
-      - podman
-      - skopeo
-      - jq
 execute:
   how: tmt
 
-/test-01-readonly:
+/readonly-tests:
   summary: Execute booted readonly/nondestructive tests
   discover:
     how: fmf
     test:
-        - /tmt/tests/bootc-install-provision
         - /tmt/tests/test-01-readonly
 
 /test-20-local-upgrade:
@@ -26,7 +17,6 @@ execute:
   discover:
     how: fmf
     test:
-        - /tmt/tests/bootc-install-provision
         - /tmt/tests/test-20-local-upgrade
 
 /test-21-logically-bound-switch:
@@ -34,17 +24,13 @@ execute:
   discover:
     how: fmf
     test:
-        - /tmt/tests/bootc-install-provision
         - /tmt/tests/test-21-logically-bound-switch
 
 /test-22-logically-bound-install:
   summary: Execute logically bound images tests for switching images
-  environment+:
-    LBI: enabled
   discover:
     how: fmf
     test:
-        - /tmt/tests/bootc-install-provision
         - /tmt/tests/test-22-logically-bound-install
 
 /test-23-install-outside-container:
@@ -52,7 +38,6 @@ execute:
   discover:
     how: fmf
     test:
-        - /tmt/tests/bootc-install-provision
         - /tmt/tests/test-23-install-outside-container
 
 /test-24-local-upgrade-reboot:
@@ -60,7 +45,6 @@ execute:
   discover:
     how: fmf
     test:
-        - /tmt/tests/bootc-install-provision
         - /tmt/tests/test-24-local-upgrade-reboot
 
 /test-25-soft-reboot:
@@ -68,5 +52,4 @@ execute:
   discover:
     how: fmf
     test:
-        - /tmt/tests/bootc-install-provision
         - /tmt/tests/test-25-soft-reboot

tmt/tests/bootc-install-provision.fmf

@@ -1,3 +0,0 @@
-summary: Run bootc install to-existing-root to deploy bootc image
-test: ./bootc-install-provision.sh
-duration: 30m

tmt/tests/bootc-install-provision.sh

@@ -1,215 +0,0 @@
-#!/bin/bash
-set -exuo pipefail
-
-# This script basically builds bootc from source using the provided base image,
-# then runs the target tests. We need to do this because at the moment
-# packit/tmt/testing-farm effectively only support RPMs, not container images.
-# https://issues.redhat.com/browse/TFT-2751
-
-BOOTC_TEMPDIR=$(mktemp -d)
-trap 'rm -rf -- "$BOOTC_TEMPDIR"' EXIT
-
-# LBI only enabled for test-22-logically-bound-install
-LBI="${LBI:-disabled}"
-
-# Get OS info
-source /etc/os-release
-case "$ID" in
-    "centos")
-        TIER1_IMAGE_URL="${TIER1_IMAGE_URL:-quay.io/centos-bootc/centos-bootc:stream${VERSION_ID}}"
-        ;;
-    "fedora")
-        TIER1_IMAGE_URL="${TIER1_IMAGE_URL:-quay.io/fedora/fedora-bootc:${VERSION_ID}}"
-        ;;
-esac
-
-if [ "$TMT_REBOOT_COUNT" -eq 0 ]; then
-    # Let's move to bootc root folder
-    cd ../..
-    # Fedora CI: https://github.com/fedora-ci/dist-git-pipeline/blob/master/Jenkinsfile#L145
-    # OSCI: https://gitlab.cee.redhat.com/osci-pipelines/dist-git-pipeline/-/blob/master/Jenkinsfile?ref_type=heads#L93
-    if [[ -v KOJI_TASK_ID ]] || [[ -v CI_KOJI_TASK_ID ]]; then
-        # Just left those ls commands here to ring the bell for me when something changed
-        echo "$TMT_SOURCE_DIR"
-        ls -al "$TMT_SOURCE_DIR"
-        ls -al "$TMT_SOURCE_DIR/SRPMS"
-        ls -al /etc/yum.repos.d
-        cat /etc/yum.repos.d/test-artifacts.repo
-        ls -al /var/share/test-artifacts
-    fi
-
-    # TMT needs this key
-    cp -r /root/.ssh "$BOOTC_TEMPDIR"
-
-    # Running on Testing Farm
-    if [[ -d "/var/ARTIFACTS" ]]; then
-        cp -r /var/ARTIFACTS "$BOOTC_TEMPDIR"
-    # Running on local machine with tmt run
-    else
-        cp -r /var/tmp/tmt "$BOOTC_TEMPDIR"
-    fi
-
-    # Some rhts-*, rstrnt-* and tmt-* commands are in /usr/local/bin
-    cp -r /usr/local/bin "$BOOTC_TEMPDIR"
-
-    # Check image building folder content
-    ls -al "$BOOTC_TEMPDIR"
-
-    CONTAINERFILE=${BOOTC_TEMPDIR}/Containerfile
-
-    COMMON_CONTAINERFILE="${BOOTC_TEMPDIR}/common_containerfile"
-    tee "$COMMON_CONTAINERFILE" > /dev/null << COMMONEOF
-RUN <<EORUN
-set -xeuo pipefail
-
-# Provision test requirement
-/code/hack/provision-derived.sh
-# Also copy in some default install configs we use for testing
-cp -a /code/hack/install-test-configs/* /usr/lib/bootc/install/
-# And some test kargs
-cp -a /code/hack/test-kargs/* /usr/lib/bootc/kargs.d/
-
-# For testing farm
-mkdir -p -m 0700 /var/roothome
-
-# Enable ttyS0 console
-mkdir -p /usr/lib/bootc/kargs.d/
-cat <<KARGEOF >> /usr/lib/bootc/kargs.d/20-console.toml
-kargs = ["console=ttyS0,115200n8"]
-KARGEOF
-
-# cloud-init and rsync are required by TMT
-dnf -y install cloud-init rsync
-ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants
-dnf -y clean all
-
-rm -rf /var/cache /var/lib/dnf
-EORUN
-
-# Some rhts-*, rstrnt-* and tmt-* commands are in /usr/local/bin
-COPY bin /usr/local/bin
-
-# In Testing Farm, all ssh things should be reserved for ssh command run after reboot
-COPY .ssh /var/roothome/.ssh
-COMMONEOF
-
-    if [[ -v KOJI_TASK_ID ]] || [[ -v CI_KOJI_TASK_ID ]]; then
-        FEDORA_CI_CONTAINERFILE="${BOOTC_TEMPDIR}/fedora_ci_containerfile"
-        tee "$FEDORA_CI_CONTAINERFILE" > /dev/null << FEDORACIEOF
-FROM $TIER1_IMAGE_URL
-
-RUN dnf -y upgrade /rpms/*.rpm
-FEDORACIEOF
-        cat >"$CONTAINERFILE" <<REALEOF
-$(cat "$FEDORA_CI_CONTAINERFILE")
-$(cat "$COMMON_CONTAINERFILE")
-
-REALEOF
-    else
-        BOOTC_CI_CONTAINERFILE="${BOOTC_TEMPDIR}/bootc_ci_containerfile"
-        # TODO use the default Dockerfile here instead of a copy of it
-        tee "$BOOTC_CI_CONTAINERFILE" > /dev/null <<BOOTCCIEOF
-FROM $TIER1_IMAGE_URL as build
-
-WORKDIR /code
-RUN <<EORUN
-set -xeuo pipefail
-. /usr/lib/os-release
-case $ID in
-  centos|rhel) dnf config-manager --set-enabled crb;;
-  fedora) dnf -y install dnf-utils 'dnf5-command(builddep)';;
-esac
-# Handle version skew
-dnf -y distro-sync ostree{,-libs} systemd
-dnf -y builddep contrib/packaging/bootc.spec
-# Extra dependencies
-dnf -y install git-core
-EORUN
-
-RUN mkdir -p /build/target/dev-rootfs
-RUN --mount=type=cache,target=/build/target --mount=type=cache,target=/var/roothome make test-bin-archive && mkdir -p /out && cp target/bootc.tar.zst /out
-
-FROM $TIER1_IMAGE_URL
-
-# Inject our built code
-COPY --from=build /out/bootc.tar.zst /tmp
-RUN tar -C / --zstd -xvf /tmp/bootc.tar.zst && rm -vrf /tmp/*
-BOOTCCIEOF
-        cat >"$CONTAINERFILE" <<REALEOF
-$(cat "$BOOTC_CI_CONTAINERFILE")
-$(cat "$COMMON_CONTAINERFILE")
-REALEOF
-    fi
-
-    if [[ -d "/var/ARTIFACTS" ]]; then
-        # In Testing Farm, TMT work dir /var/ARTIFACTS should be reserved
-        echo "COPY ARTIFACTS /var/ARTIFACTS" >> "$CONTAINERFILE"
-    else
-        # In local machine, TMT work dir /var/tmp/tmt should be reserved
-        echo "COPY tmt /var/tmp/tmt" >> "$CONTAINERFILE"
-    fi
-
-    # For test-22-logically-bound-install
-    if [[ "$LBI" == "enabled" ]]; then
-        echo "RUN cp -a /code/tmt/tests/lbi/usr/. /usr && ln -s /usr/share/containers/systemd/curl.container /usr/lib/bootc/bound-images.d/curl.container && ln -s /usr/share/containers/systemd/curl-base.image /usr/lib/bootc/bound-images.d/curl-base.image && ln -s /usr/share/containers/systemd/podman.image /usr/lib/bootc/bound-images.d/podman.image" >> "$CONTAINERFILE"
-        podman pull --retry 5 --retry-delay 5s quay.io/curl/curl:latest
-        podman pull --retry 5 --retry-delay 5s quay.io/curl/curl-base:latest
-        podman pull --retry 5 --retry-delay 5s registry.access.redhat.com/ubi9/podman:latest
-    fi
-
-    cat "$CONTAINERFILE"
-    # Retry here to avoid quay.io "502 Bad Gateway"
-    # bind mount bootc source code folder for bootc binary building and run test provision
-    # bind mount /var/share/test-artifacts for bootc RPM package installation in Fedora CI and OSCI
-    if [[ -v KOJI_TASK_ID ]] || [[ -v CI_KOJI_TASK_ID ]]; then
-        podman build \
-            --retry 5 \
-            --retry-delay 5s \
-            --tls-verify=false \
-            -v /var/share/test-artifacts:/rpms:z \
-            -v "$(pwd)":/code:z \
-            -t localhost/bootc:tmt \
-            -f "$CONTAINERFILE" \
-            "$BOOTC_TEMPDIR"
-    else
-        podman build \
-            --retry 5 \
-            --retry-delay 5s \
-            --tls-verify=false \
-            -v "$(pwd)":/code:z \
-            -t localhost/bootc:tmt \
-            -f "$CONTAINERFILE" \
-            "$BOOTC_TEMPDIR"
-    fi
-
-    podman images
-    podman run \
-        --rm \
-        --tls-verify=false \
-        --privileged \
-        --pid=host \
-        -v /:/target \
-        -v /dev:/dev \
-        -v /var/lib/containers:/var/lib/containers \
-        -v /root/.ssh:/output \
-        --security-opt label=type:unconfined_t \
-        "localhost/bootc:tmt" \
-        bootc install to-existing-root --target-transport containers-storage --acknowledge-destructive
-
-    # Reboot
-    tmt-reboot
-elif [ "$TMT_REBOOT_COUNT" -eq 1 ]; then
-    # Some simple and fast checkings
-    bootc status
-    echo "$PATH"
-    printenv
-    if [[ -d "/var/ARTIFACTS" ]]; then
-        ls -al /var/ARTIFACTS
-    else
-        ls -al /var/tmp/tmt
-    fi
-    ls -al /usr/local/bin
-    echo "Bootc system on TMT/TF runner"
-
-    exit 0
-fi

tmt/tests/booted/README.md

@@ -1,4 +1,3 @@
 # Booted tests
 
-These are intended to run via tmt; use e.g.
+These are intended to run via tmt.
-`make test-tmt`.