ci: Rework Dockerfile, add Justfile and improved testing

- Move the Dockerfile to the toplevel as a primary dev entrypoint
- The Justfile is intended especially for agentic AI like
  block/goose or Claude Code as an allowlistable-command entrypoint
- Include attempt at incremental build caching, partially defeated
  by autotools
- Add new tests-unit-container that tests ostree-prepare-root in
  a container

Signed-off-by: Colin Walters <walters@verbum.org>

.dockerignore

@@ -1,4 +1,7 @@
+# Don't need these in the container
 Dockerfile
+Justfile
+.github
 # We put most binaries under here
 target/
 # We don't have a lockfile by default

.github/workflows/bootc.yaml

@@ -15,18 +15,36 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  c9s-bootc-e2e:
+  c9s-e2e:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - name: Installdeps
+        run: sudo apt update && sudo apt install just
+      - name: Get a newer podman for heredoc support (from debian testing)
+        run: |
+          set -eux
+          echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
+          sudo apt update
+          sudo apt install -y crun/testing podman/testing skopeo/testing
       - name: build
-        run: sudo podman build -t localhost/test:latest -f ci/Containerfile.c9s .
+        run: sudo just build
+      - name: unitcontainer
+        run: sudo just unitcontainer
+      - name: unittest
+        run: sudo just unittest
       - name: bootc install
         run: |
           set -xeuo pipefail
           sudo podman run --env BOOTC_SKIP_SELINUX_HOST_CHECK=1 --rm -ti --privileged -v /:/target --pid=host --security-opt label=disable \
             -v /dev:/dev -v /var/lib/containers:/var/lib/containers \
-            localhost/test:latest bootc install to-filesystem --skip-fetch-check \
+            localhost/ostree:latest bootc install to-filesystem --skip-fetch-check \
              --replace=alongside /target
           # Verify labeling for /etc
           sudo ls -dZ /ostree/deploy/default/deploy/*.0/etc |grep :etc_t:
+      - name: Upload test logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-suite-log
+          path: target/unittest

Dockerfile

@@ -0,0 +1,49 @@
+
+ARG base=quay.io/centos-bootc/centos-bootc:stream9
+
+FROM $base as buildroot
+# This installs our package dependencies, and we want to cache it independently of the rest.
+# Basically we don't want changing a .rs file to blow out the cache of packages.
+COPY ci /ci
+RUN /ci/installdeps.sh
+
+# This image holds the source code
+FROM $base as src
+COPY . /src
+
+# This image holds only the main program sources, helping ensure that
+# when one edits the tests it doesn't recompile the whole program
+FROM src as binsrc
+RUN --network=none rm tests-unit-container -rf && touch -r src .
+
+FROM buildroot as build
+COPY --from=binsrc /src /build
+WORKDIR /build
+RUN --mount=type=cache,target=/ccache <<EORUN
+set -xeuo pipefail
+mkdir -p /var/roothome
+env NOCONFIGURE=1 ./autogen.sh
+export CC="ccache gcc" CCACHE_DIR=/ccache
+env ./configure \
+    --sysconfdir=/etc --prefix=/usr --libdir=/usr/lib64 \
+    --with-openssl --with-selinux --with-composefs \
+    --with-dracut=yesbutnoconf \
+    --disable-gtk-doc --with-curl --without-soup
+make -j $(nproc)
+make install DESTDIR=/out
+EORUN
+
+# This image holds both the main binary and the tests
+FROM $base as bin-and-test
+RUN rpm -e --nodeps ostree{,-libs}
+COPY --from=build /out/ /
+COPY --from=src /src/tests-unit-container /tests
+
+# The default final container
+FROM $base
+RUN rpm -e --nodeps ostree{,-libs}
+COPY --from=build /out/ /
+# https://docs.fedoraproject.org/en-US/bootc/initramfs/#_regenerating_the_initrd
+# since we have ostree-prepare-root there
+RUN set -x; kver=$(cd /usr/lib/modules && echo *); dracut -vf /usr/lib/modules/$kver/initramfs.img $kver
+

Justfile

@@ -0,0 +1,37 @@
+# Detect the os for a workaround below
+osid := `. /usr/lib/os-release && echo $ID`
+
+# Build the container image from current sources
+build:
+    podman build --jobs=4 -t localhost/ostree .
+
+build-unittest:
+    podman build --jobs=4 --target build -t localhost/ostree-buildroot .
+
+# We need a filesystem that supports O_TMPFILE right now (i.e. not overlayfs)
+# or ostree hard crashes in the http code =/
+unittest_args := "--pids-limit=-1 --tmpfs /var/tmp --tmpfs /tmp"
+
+# Build and then run unit tests. If this fails, it will try to print
+# the errors to stderr. However, the full unabridged test log can
+# be found in target/unittest/test-suite.log.
+unittest *ARGS: build-unittest
+    rm -rf target/unittest && mkdir -p target/unittest
+    podman run --net=none {{unittest_args}} --security-opt=label=disable --rm \
+        -v $(pwd)/target/unittest:/run/output --env=ARTIFACTS=/run/output \
+        --env=OSTREE_TEST_SKIP=known-xfail-docker \
+        localhost/ostree-buildroot  ./tests/makecheck.py {{ARGS}}
+
+# Start an interactive shell in the unittest container
+unittest-shell: build-unittest
+    podman run --rm -ti {{unittest_args}} "--env=PS1=unittests> " localhost/ostree-buildroot  bash
+
+# For some reason doing the bind mount isn't working on at least the GHA Ubuntu 24.04 runner
+# without --privileged. I think it may be apparmor?
+unitpriv := if osid == "ubuntu" { "--privileged" } else { "" }
+unitcontainer-build:
+    podman build --jobs=4 --target bin-and-test -t localhost/ostree-bintest .
+unitcontainer: unitcontainer-build 
+    # need cap-add=all for mounting
+    podman run --rm --net=none {{unitpriv}} {{unittest_args}} --cap-add=all --env=TEST_CONTAINER=1 localhost/ostree-bintest /tests/run.sh
+    

Makefile-tests.am

@@ -139,7 +139,6 @@ _installed_or_uninstalled_test_scripts = \
 	tests/test-concurrency.py \
 	tests/test-refs.sh \
 	tests/test-demo-buildsystem.sh \
-	tests/test-switchroot.sh \
 	tests/test-pull-contenturl.sh \
 	tests/test-pull-mirrorlist.sh \
 	tests/test-summary-update.sh \

ci/Containerfile.c9s

@@ -1,12 +0,0 @@
-FROM quay.io/centos/centos:stream9 as build
-COPY ci/c9s-buildroot.repo /etc/yum.repos.d
-RUN dnf -y install dnf-utils zstd && dnf config-manager --enable crb && dnf builddep -y ostree
-COPY . /build
-WORKDIR /build
-RUN env NOCONFIGURE=1 ./autogen.sh && \
-    ./configure --prefix=/usr --libdir=/usr/lib64 --sysconfdir=/etc --with-curl --with-selinux --with-dracut=yesbutnoconf && \
-    make -j 8 && \
-    make install DESTDIR=$(pwd)/target/inst
-
-FROM quay.io/centos-bootc/centos-bootc-dev:stream9
-COPY --from=build /build/target/inst/ /

tests-unit-container/README.md

@@ -0,0 +1,5 @@
+# Tests which are designed to be run in a container with user namespacing
+
+These tests are mainly designed to cover ostree-prepare-root.
+
+Run them with `just unitcontainer`.

tests-unit-container/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -euo pipefail
+dn=$(dirname $0)
+n=0
+for case in ${dn}/test-*; do
+  echo "Running: $case"
+  $case
+  echo "ok $case"
+  n=$(($n+1))
+done
+echo "Executed tests: $n"
+exit 0
+

tests-unit-container/test-prepare-root.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# This script tests ostree-prepare-root.service. It expects to run in
+# a podman container. See the `privunit` job in Justfile.
+# Here we're treating the podman container like an initramfs.
+
+set -xeuo pipefail
+
+# Ensure this isn't run accidentally
+test "${TEST_CONTAINER}" = 1
+
+cleanup() {
+	if mountpoint /target-sysroot &>/dev/null; then
+		umount -lR /target-sysroot
+	fi
+	rm -rf /run/ostree-booted /run/ostree
+}
+trap cleanup EXIT
+
+test '!' -f /run/ostree-booted
+
+mkdir /target-sysroot
+# Needs to be a mount point
+mount --bind /target-sysroot /target-sysroot
+ostree admin init-fs --epoch=1 /target-sysroot
+cd /target-sysroot
+ostree admin --sysroot=. stateroot-init default
+# now we just fake out a deployment
+mkdir -p ostree/deploy/default/deploy/1234/{etc,usr,sysroot}
+
+ln -sr ostree/deploy/default/deploy/1234 boot/ostree.0
+t=$(mktemp)
+# Need to disable composefs in an unprivileged container
+echo "root=UUID=cafebabe ostree.prepare-root.composefs=0 ostree=/boot/ostree.0" > ${t}
+mount --bind $t /proc/cmdline
+
+cd /
+/usr/lib/ostree/ostree-prepare-root /target-sysroot
+
+findmnt -R /target-sysroot
+
+# Verify we have this stamp file
+test -f /run/ostree-booted
+
+# Note that usr is a bind mount in legacy mode without compsoefs
+for d in etc usr; do
+	mountpoint /target-sysroot/${d}
+done
+
+# Default is ro in our images
+grep -q 'readonly.*true' /usr/lib/ostree/prepare-root.conf
+[[ "$(findmnt -n -o OPTIONS /target-sysroot/sysroot)" == *ro* ]]
+
+cleanup
+test '!' -f /run/ostree-booted
+
+mv /usr/lib/ostree/prepare-root.conf{,.orig}
+
+mount --bind /target-sysroot /target-sysroot
+/usr/lib/ostree/ostree-prepare-root /target-sysroot
+findmnt -R /target-sysroot
+[[ "$(findmnt -n -o OPTIONS /target-sysroot/sysroot)" == *rw* ]]
+
+echo "ok verified default prepare-root"

tests/libtest.sh

@@ -379,8 +379,8 @@ setup_fake_remote_repo2() {
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir}/ostree-srv ostree
-    run_webserver
-    cd ${oldpwd} 
+    run_webserver $args
+    cd ${oldpwd}
     export OSTREE="${CMD_PREFIX} ostree --repo=repo"
 }
 
@@ -427,11 +427,7 @@ setup_os_repository () {
     shift
     bootmode=$1
     shift
-    bootdir=usr/lib/modules/3.6.0
-    if test "$#" -gt 0; then
-        bootdir=$1
-        shift
-    fi
+    bootdir=${1:-usr/lib/modules/3.6.0}
 
     oldpwd=`pwd`
 
@@ -547,7 +543,7 @@ EOF
     mkdir ${test_tmpdir}/httpd
     cd httpd
     ln -s ${test_tmpdir} ostree
-    run_webserver "$@"
+    run_webserver
     cd ${oldpwd} 
 }
 
@@ -638,6 +634,12 @@ skip_without_ostree_httpd () {
     fi
 }
 
+skip_known_xfail_docker() {
+    if test "${OSTREE_TEST_SKIP:-}" = known-xfail-docker; then
+        skip "This test was explicitly skipped via OSTREE_TEST_SKIP=known-xfail-docker"
+    fi
+}
+
 skip_without_user_xattrs () {
     if ! have_user_xattrs; then
         skip "this test requires xattr support"

tests/makecheck.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+import os
+import argparse
+import shutil
+import re
+
+HEADERS = ["PASS", "SKIP", "XFAIL", "FAIL", "XPASS", "ERROR"]
+
+def is_header(line) -> bool:
+    return line.startswith("========")
+
+def run_make_check():
+    """
+    Runs 'make check' with optional additional arguments.
+    Returns True if 'make check' succeeds, False otherwise.
+    """
+    command = ['make', 'check', '-j', '6'] + sys.argv[1:]
+    print(f"Running '{' '.join(command)}'...")
+    try:
+        result = subprocess.run(command, check=False) # check=False to handle return code manually
+    except FileNotFoundError:
+        print(f"Error: 'make' command not found. Is it in your PATH?", file=sys.stderr)
+        return False # Indicate failure
+
+    if result.returncode == 0:
+        return True
+    return False
+    
+def print_truncated(lines):
+    if len(lines) == 0:
+        return
+    print()
+    print(lines[0])
+    print("(skipped %d lines)" % max(len(lines) - 20, 0))
+    print(os.linesep.join(lines[-20:]))
+    print("-" * 20)
+
+def get_failed_test_output(lines):
+    """
+    Parses test-suite.log to find failed tests and print the last 20 lines
+    of their output.
+    """
+    in_error_section = None
+    prevline = None
+    errlines = []
+    for line in lines:
+        line = line.strip()
+        if is_header(line) and prevline != None:
+            (k, v) = prevline.split(':')
+            print("%s %s" % (k, v))
+            if k in ('ERROR', 'FAIL'):
+                if in_error_section:
+                    print_truncated(errlines)
+                    in_error_section = None
+                else:
+                    in_error_section = v
+                errlines = []
+        prevline = line
+        if in_error_section:
+            errlines.append(line)
+    print_truncated(errlines)
+
+if __name__== "__main__":
+    if len(sys.argv) > 1 and sys.argv[1] == "analyze":
+        get_failed_test_output(open(sys.argv[2]).readlines())
+        sys.exit(0)
+
+    if run_make_check():
+        print("make check passed successfully.")
+        sys.exit(0)
+    else:
+        print("make check failed. Attempting to extract failed test output.")
+        get_failed_test_output(open('test-suite.log').readlines())
+        artifacts = os.environ.get('ARTIFACTS')
+        if artifacts is not None:
+            shutil.move('test-suite.log', os.path.join(artifacts, 'test-suite.log'))
+            print("Saved test-suite.log to artifacts directory.")
+        sys.exit(1)

tests/test-pull-summary-sigs.sh

@@ -21,6 +21,8 @@ set -euo pipefail
 
 . $(dirname $0)/libtest.sh
 
+skip_known_xfail_docker
+
 # Ensure repo caching is in use.
 unset OSTREE_SKIP_CACHE
 

tests/test-signed-pull-summary.sh

@@ -23,6 +23,8 @@ set -euo pipefail
 
 . $(dirname $0)/libtest.sh
 
+skip_known_xfail_docker
+
 echo "1..14"
 
 # Ensure repo caching is in use.

tests/test-switchroot.sh

@@ -1,154 +0,0 @@
-#!/bin/bash -ex
-
-this_script="${BASH_SOURCE:-$(readlink -f "$0")}"
-
-OSTREE_PREPARE_ROOT=$(dirname "${this_script}")/../ostree-prepare-root
-if [ ! -x "${OSTREE_PREPARE_ROOT}" ]; then
-	# ostree-prepare-root is in $libdir by default, assume we can find it
-	# based on our test directory, if not we'll have to skip this test.
-	OSTREE_PREPARE_ROOT=$(dirname "${this_script}")/../../../lib/ostree/ostree-prepare-root
-	if [ ! -x "${OSTREE_PREPARE_ROOT}" ]; then
-		OSTREE_PREPARE_ROOT=""
-	fi
-fi
-
-setup_bootfs() {
-	mkdir -p "$1/proc" "$1/bin"
-
-	# We need the real /proc mounted here so musl's realpath will work, but we
-	# want to be able to override /proc/cmdline, so bind mount.
-	mount -t proc proc "$1/proc"
-
-	echo "quiet ostree=/ostree/boot.0 ro" >"$1/override_cmdline"
-	mount --bind "$1/override_cmdline" "$1/proc/cmdline"
-
-	touch "$1/this_is_bootfs"
-	cp "${OSTREE_PREPARE_ROOT}" "$1/bin"
-}
-
-setup_rootfs() {
-	mkdir -p "$1/ostree/deploy/linux/deploy/1334/sysroot" \
-	         "$1/ostree/deploy/linux/deploy/1334/var" \
-	         "$1/ostree/deploy/linux/deploy/1334/usr" \
-	         "$1/ostree/deploy/linux/var" \
-	         "$1/bin"
-	ln -s "deploy/linux/deploy/1334" "$1/ostree/boot.0"
-	ln -s . "$1/sysroot"
-	touch "$1/ostree/deploy/linux/deploy/1334/this_is_ostree_root" \
-	      "$1/ostree/deploy/linux/var/this_is_ostree_var" \
-	      "$1/ostree/deploy/linux/deploy/1334/usr/this_is_ostree_usr" \
-	      "$1/this_is_real_root"
-	cp /bin/busybox "$1/bin"
-	busybox --list | xargs -n1 -I '{}' ln -s busybox "$1/bin/{}"
-	cp -r "$1/bin" "$1/ostree/deploy/linux/deploy/1334/"
-}
-
-setup_overlay() {
-	mkdir -p "$1/ostree/deploy/linux/deploy/1334/.usr-ovl-work" \
-	         "$1/ostree/deploy/linux/deploy/1334/.usr-ovl-upper"
-}
-
-enter_fs() {
-	cd "$1"
-	mkdir testroot
-	pivot_root . testroot
-	export PATH=$PATH:/sysroot/bin
-	cd /
-	umount -l testroot
-	rmdir testroot
-}
-
-find_in_env() {
-	tmpdir="$(mktemp -dt ostree-test-switchroot.XXXXXX)"
-	unshare -m <<-EOF
-		set -e
-		. "$this_script"
-		"$1" "$tmpdir"
-		enter_fs "$tmpdir"
-		ostree-prepare-root /sysroot
-		find / \( -path /proc -o -path /sysroot/proc \) -prune -o -print
-		touch /usr/usr_writable 2>/null \
-			&& echo "/usr is writable" \
-			|| echo "/usr is not writable"
-		touch /sysroot/usr/sysroot_usr_writable 2>/null \
-			&& echo "/sysroot/usr is writable" \
-			|| echo "/sysroot/usr is not writable"
-		EOF
-	(cd $tmpdir && find) >permanent_files
-	rm -rf "$tmpdir"
-}
-
-setup_initrd_env() {
-	mount -t tmpfs tmpfs "$1"
-	setup_bootfs "$1"
-	mkdir "$1/sysroot"
-	mount -t tmpfs tmpfs "$1/sysroot"
-	setup_rootfs "$1/sysroot"
-}
-
-test_that_prepare_root_sets_sysroot_up_correctly_with_initrd() {
-	find_in_env setup_initrd_env >files
-
-	grep -qx "/this_is_bootfs" files
-	grep -qx "/sysroot/this_is_ostree_root" files
-	grep -qx "/sysroot/sysroot/this_is_real_root" files
-	if ! have_systemd_and_libmount; then
-		grep -qx "/sysroot/var/this_is_ostree_var" files
-	fi
-	grep -qx "/sysroot/usr/this_is_ostree_usr" files
-
-	grep -qx "/sysroot/usr is not writable" files
-	echo "ok ostree-prepare-root sets sysroot up correctly with initrd"
-}
-
-setup_no_initrd_env() {
-	mount --bind "$1" "$1"
-	setup_rootfs "$1"
-	setup_bootfs "$1"
-}
-
-test_that_prepare_root_sets_root_up_correctly_with_no_initrd() {
-	find_in_env setup_no_initrd_env >files
-
-	grep -qx "/this_is_ostree_root" files
-	grep -qx "/sysroot/this_is_bootfs" files
-	grep -qx "/sysroot/this_is_real_root" files
-	if ! have_systemd_and_libmount; then
-		grep -qx "/var/this_is_ostree_var" files
-	fi
-	grep -qx "/usr/this_is_ostree_usr" files
-
-	grep -qx "/usr is not writable" files
-	echo "ok ostree-prepare-root sets root up correctly with no initrd"
-}
-
-setup_no_initrd_with_overlay() {
-	setup_no_initrd_env "$1"
-	setup_overlay "$1"
-}
-
-test_that_prepare_root_provides_overlay_over_usr_if__usr_ovl_work_exists() {
-	find_in_env setup_no_initrd_with_overlay >files
-
-	grep -qx "/usr is writable" files
-	grep -qx "./ostree/deploy/linux/deploy/1334/.usr-ovl-upper/usr_writable" permanent_files
-	! grep -qx "./ostree/deploy/linux/deploy/1334/usr/usr_writable" permanent_files || exit 1
-	echo "ok ostree-prepare-root sets root up correctly with writable usr overlay"
-}
-
-# This script sources itself so we only want to run tests if we're the parent:
-if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
-	. $(dirname $0)/libtest.sh
-	unshare -m true || \
-	    skip "this test needs to set up mount namespaces, rerun as root"
-	[ -f /bin/busybox ] || \
-	    skip "this test needs busybox"
-
-	[ -n "${OSTREE_PREPARE_ROOT}" ] || \
-	    skip "this test needs ostree-prepare-root"
-
-	echo "1..3"
-	test_that_prepare_root_sets_sysroot_up_correctly_with_initrd
-	test_that_prepare_root_sets_root_up_correctly_with_no_initrd
-	test_that_prepare_root_provides_overlay_over_usr_if__usr_ovl_work_exists
-fi