From 62e7b69654cd891b865fbf634030af5a2f3b1908 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 16 Jan 2026 08:57:17 -0500 Subject: [PATCH] docs: Improve Justfile with groups and self-documenting targets Justfile changes: - Organize targets into groups (core, testing, docs, debugging, maintenance) - Add `list-variants` target to show available build variants - Simplify comments to be concise single-line descriptions - Move composefs targets (build-sealed, test-composefs) into core group CONTRIBUTING.md changes: - Reference `just --list` and `just list-variants` instead of duplicating - Remove tables that duplicate Justfile information - Fix broken link to cli.rs The Justfile is now self-documenting via `just --list` (grouped targets) and `just list-variants` (build configuration options). Assisted-by: OpenCode (Claude Sonnet 4) Signed-off-by: Colin Walters --- CONTRIBUTING.md | 49 +++++- Justfile | 407 +++++++++++++++++++++++++----------------------- 2 files changed, 256 insertions(+), 200 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b395bd58..31d217f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,11 +41,20 @@ accepted! Worth stating: before you start diving into the code you should understand using the system as a user and how it works. See the user documentation for that. -## Understanding the Justfile +## The Justfile + +The [Justfile](Justfile) is the primary interface for building and testing bootc. + +```bash +just --list # Show all targets organized by group +just list-variants # Show available build variants and current config +``` + +### Building from source Edit the source code; a simple thing to do is add e.g. -`eprintln!("hello world");` into `run_from_opt` in [crates/lib/src/cli.rs](cli.rs). -You can run `make` or `cargo build` to build that locally. However, a key +`eprintln!("hello world");` into `run_from_opt` in [crates/lib/src/cli.rs](crates/lib/src/cli.rs). +You can run `make` or `cargo build` to build that locally. However, a key next step is to get that binary into a bootc container image. Running `just` defaults to `just build` which will build a container @@ -58,14 +67,24 @@ and try running `bootc`. ### Running container-oriented integration tests -`just test-container` +```bash +just test-container +``` ### Running (TMT) integration tests A common cycle here is you'll edit e.g. `deploy.rs` and want to run the tests that perform an upgrade: -`just test-tmt-one test-20-local-upgrade` +```bash +just test-tmt local-upgrade-reboot +``` + +To run a specific test: + +```bash +just test-tmt readonly +``` ### Faster iteration cycles @@ -96,6 +115,26 @@ then you can `cargo b --release` directly in a Fedora 42 container or even on your host system, and then directly run e.g. `./target/release/bootc upgrade` etc. +### Testing with composefs (sealed images) + +To build and test with the experimental composefs backend: + +```bash +# Build a sealed image with auto-generated test Secure Boot keys +just build-sealed + +# Run composefs-specific tests +just test-composefs + +# Validate that composefs digests match between build and install views +# (useful for debugging mtime/metadata issues) +just validate-composefs-digest +``` + +The `build-sealed` target generates test Secure Boot keys in `target/test-secureboot/` +and builds a complete sealed image with UKI. See [experimental-composefs.md](docs/src/experimental-composefs.md) +for more information on sealed images. + ### Debugging via lldb diff --git a/Justfile b/Justfile index 22a2ebc8..53253e40 100644 --- a/Justfile +++ b/Justfile @@ -1,111 +1,222 @@ # The default entrypoint to working on this project. -# Commands here typically wrap e.g. `podman build` or -# other tools like `bcvk` which might launch local virtual machines. -# +# Run `just --list` to see available targets organized by group. +# # See also `Makefile` and `xtask.rs`. Commands which end in `-local` # skip containerization or virtualization (and typically just proxy `make`). # -# Rules written here are *often* used by the Github Action flows, -# and should support being configurable where that makes sense (e.g. -# the `build` rule supports being provided a base image). -# -# By default the layering should be thus: +# By default the layering is: # Github Actions -> Justfile -> podman -> make -> rustc -# -> podman -> dnf|apt ... +# -> podman -> package manager # -> cargo xtask # -------------------------------------------------------------------- -# This image is just the base image plus our updated bootc binary +# Configuration variables (override via environment or command line) +# Example: BOOTC_base=quay.io/fedora/fedora-bootc:42 just build + +# Output image name base_img := "localhost/bootc" -# Has a synthetic upgrade +# Synthetic upgrade image for testing upgrade_img := base_img + "-upgrade" -# ostree: The default -# composefs-sealeduki-sdboot: A system with a sealed composefs using systemd-boot +# Build variant: ostree (default) or composefs-sealeduki-sdboot (sealed UKI) variant := env("BOOTC_variant", "ostree") +# Base container image to build from base := env("BOOTC_base", "quay.io/centos-bootc/centos-bootc:stream10") +# Buildroot base image buildroot_base := env("BOOTC_buildroot_base", "quay.io/centos/centos:stream10") - -testimage_label := "bootc.testimage=1" -# Images used by hack/lbi; keep in sync -lbi_images := "quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest" -# We used to have --jobs=4 here but sometimes that'd hit this -# ``` -# [2/3] STEP 2/2: RUN --mount=type=bind,from=context,target=/run/context < Using cache b068d42ac7491067cf5fafcaaf2f09d348e32bb752a22c85bbb87f266409554d -# --> b068d42ac749 -# + cd /run/context/ -# /bin/sh: line 3: cd: /run/context/: Permission denied -# ``` -# TODO: Gather more info and file a buildah bug -generic_buildargs := "" -# Optional: path to extra source directory (e.g. composefs-rs) to bind mount into builds. -# Usage: BOOTC_extra_src=$HOME/src/github/containers/composefs-rs just build -# The directory will be mounted at /run/extra-src inside the container. -# When using this, you must also patch Cargo.toml to use path dependencies: -# composefs = { path = "/run/extra-src/crates/composefs", ... } -# Note: This disables SELinux labeling for the mount. +# Optional: path to extra source (e.g. composefs-rs) for local development extra_src := env("BOOTC_extra_src", "") -# Generate podman args for extra source mount if configured + +# Internal variables +testimage_label := "bootc.testimage=1" +lbi_images := "quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.access.redhat.com/ubi9/podman:latest" +fedora-coreos := "quay.io/fedora/fedora-coreos:testing-devel" +generic_buildargs := "" _extra_src_args := if extra_src != "" { "-v " + extra_src + ":/run/extra-src:ro --security-opt=label=disable" } else { "" } -# Args for package building (no secrets needed, just builds RPMs) base_buildargs := generic_buildargs + " " + _extra_src_args + " --build-arg=base=" + base + " --build-arg=variant=" + variant -# - scratch builds need extra perms per https://docs.fedoraproject.org/en-US/bootc/building-from-scratch/ -# - we do secure boot signing here, so provide the keys buildargs := base_buildargs \ + " --cap-add=all --security-opt=label=type:container_runtime_t --device /dev/fuse" \ + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt" -# The default target: build the container image from current sources. -# Note commonly you might want to override the base image via e.g. -# `just build --build-arg=base=quay.io/fedora/fedora-bootc:42` -# into the container image. -# -# Note you can set `BOOTC_SKIP_PACKAGE=1` in the environment to bypass this stage. + +# ============================================================================ +# Core workflows - the main targets most developers will use +# ============================================================================ + +# Build container image from current sources (default target) +[group('core')] build: package _keygen && _pull-lbi-images #!/bin/bash set -xeuo pipefail test -d target/packages - # Use --build-context to pass packages instead of -v to avoid mount stubs in /run pkg_path=$(realpath target/packages) podman build --build-context "packages=${pkg_path}" -t {{base_img}} {{buildargs}} . -# Pull images used by hack/lbi -_pull-lbi-images: - podman pull -q --retry 5 --retry-delay 5s {{lbi_images}} - -# Compute SOURCE_DATE_EPOCH and VERSION from git for reproducible builds. -# Outputs shell variable assignments that can be eval'd. -_git-build-vars: +# Show available build variants and current configuration +[group('core')] +list-variants: #!/bin/bash - set -euo pipefail - SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - # Compute version from git (matching xtask.rs gitrev logic) - if VERSION=$(git describe --tags --exact-match 2>/dev/null); then - VERSION="${VERSION#v}" - VERSION="${VERSION//-/.}" - else - COMMIT=$(git rev-parse HEAD | cut -c1-10) - COMMIT_TS=$(git show -s --format=%ct) - TIMESTAMP=$(date -u -d @${COMMIT_TS} +%Y%m%d%H%M) - VERSION="${TIMESTAMP}.g${COMMIT}" - fi - echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" - echo "VERSION=${VERSION}" + cat <<'EOF' + Build Variants (set via BOOTC_variant= or variant=) + ==================================================== -# Needed by bootc install on ostree -fedora-coreos := "quay.io/fedora/fedora-coreos:testing-devel" -# Generate Secure Boot keys (only for our own CI/testing) -_keygen: - ./hack/generate-secureboot-keys + ostree (default) + Standard bootc image using ostree backend. + This is the traditional, production-ready configuration. -# Build a sealed image from current sources. + composefs-sealeduki-sdboot + Sealed composefs image with: + - Unified Kernel Image (UKI) containing kernel + initramfs + cmdline + - Secure Boot signing (using keys in target/test-secureboot/) + - systemd-boot bootloader + - composefs digest embedded in kernel cmdline for verified boot + + Use `just build-sealed` as a shortcut, or: + just variant=composefs-sealeduki-sdboot build + + Current Configuration + ===================== + EOF + echo " BOOTC_variant={{variant}}" + echo " BOOTC_base={{base}}" + echo " BOOTC_extra_src={{extra_src}}" + echo "" + +# Build a sealed composefs image (alias for variant=composefs-sealeduki-sdboot) +[group('core')] build-sealed: @just --justfile {{justfile()}} variant=composefs-sealeduki-sdboot build -# Build packages (e.g. RPM) into target/packages/ -# Any old packages will be removed. -# Set BOOTC_SKIP_PACKAGE=1 in the environment to bypass this stage. We don't -# yet have an accurate ability to avoid rebuilding this in CI yet. +# Run tmt integration tests in VMs (e.g. `just test-tmt readonly`) +[group('core')] +test-tmt *ARGS: build + @just _build-upgrade-image + @just test-tmt-nobuild {{ARGS}} + +# Run containerized unit and integration tests +[group('core')] +test-container: build build-units + podman run --rm --read-only localhost/bootc-units /usr/bin/bootc-units + podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} {{base_img}} bootc-integration-tests container + +# Build and test sealed composefs images +[group('core')] +test-composefs: + just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot + +# Run cargo fmt and clippy checks in container +[group('core')] +validate: + podman build {{base_buildargs}} --target validate . + +# ============================================================================ +# Testing variants and utilities +# ============================================================================ + +# Run tmt tests without rebuilding (for fast iteration) +[group('testing')] +test-tmt-nobuild *ARGS: + cargo xtask run-tmt --env=BOOTC_variant={{variant}} --upgrade-image={{upgrade_img}} {{base_img}} {{ARGS}} + +# Run tmt tests on Fedora CoreOS +[group('testing')] +test-tmt-on-coreos *ARGS: + cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_target={{base_img}}-coreos:latest {{fedora-coreos}} {{ARGS}} + +# Run external container tests against localhost/bootc +[group('testing')] +run-container-external-tests: + ./tests/container/run {{base_img}} + +# Remove all test VMs created by tmt tests +[group('testing')] +tmt-vm-cleanup: + bcvk libvirt rm --stop --force --label bootc.test=1 + +# Build test image for Fedora CoreOS testing +[group('testing')] +build-testimage-coreos PATH: _keygen + #!/bin/bash + set -xeuo pipefail + pkg_path=$(realpath "{{PATH}}") + podman build --build-context "packages=${pkg_path}" \ + --build-arg SKIP_CONFIGS=1 \ + -t {{base_img}}-coreos {{buildargs}} . + +# Build test image for install tests (used by CI) +[group('testing')] +build-install-test-image: build + cd hack && podman build {{base_buildargs}} -t {{base_img}}-install -f Containerfile.drop-lbis + +# ============================================================================ +# Documentation +# ============================================================================ + +# Serve docs locally (prints URL) +[group('docs')] +mdbook-serve: build-mdbook + #!/bin/bash + set -xeuo pipefail + podman run --init --replace -d --name bootc-mdbook --rm --publish 127.0.0.1::8000 localhost/bootc-mdbook + echo http://$(podman port bootc-mdbook 8000/tcp) + +# Build the documentation (mdbook) +[group('docs')] +build-mdbook: + #!/bin/bash + set -xeuo pipefail + secret_arg="" + if test -n "${GH_TOKEN:-}"; then + secret_arg="--secret=id=GH_TOKEN,env=GH_TOKEN" + fi + podman build {{generic_buildargs}} ${secret_arg} -t localhost/bootc-mdbook -f docs/Dockerfile.mdbook . + +# Build docs and extract to DIR +[group('docs')] +build-mdbook-to DIR: build-mdbook + #!/bin/bash + set -xeuo pipefail + container_id=$(podman create localhost/bootc-mdbook) + podman cp ${container_id}:/src/docs/book {{DIR}} + podman rm -f ${container_id} + +# ============================================================================ +# Debugging and validation +# ============================================================================ + +# Validate composefs digests match between build and install views +[group('debugging')] +validate-composefs-digest: + cargo xtask validate-composefs-digest {{base_img}} + +# Verify reproducible builds (runs package twice, compares output) +[group('debugging')] +check-buildsys: + cargo run -p xtask check-buildsys + +# Get container image pullspec for a given OS (e.g. `pullspec-for-os base fedora-42`) +[group('debugging')] +pullspec-for-os TYPE NAME: + @jq -r --arg v "{{NAME}}" '."{{TYPE}}"[$v]' < hack/os-image-map.json + +# ============================================================================ +# Maintenance +# ============================================================================ + +# Update generated files (man pages, JSON schemas) +[group('maintenance')] +update-generated: + cargo run -p xtask update-generated + +# Remove all locally-built test container images +[group('maintenance')] +clean-local-images: + podman images --filter "label={{testimage_label}}" + podman images --filter "label={{testimage_label}}" --format "{{{{.ID}}" | xargs -r podman rmi -f + podman image prune -f + podman rmi {{fedora-coreos}} -f + +# Build packages (RPM) into target/packages/ +[group('maintenance')] package: #!/bin/bash set -xeuo pipefail @@ -124,134 +235,40 @@ package: podman run --rm localhost/bootc-pkg tar -C /out/ -cf - . | tar -C "${packages}"/ -xvf - chmod a+rx target "${packages}" chmod a+r "${packages}"/*.rpm - # Keep localhost/bootc-pkg for layer caching; use `just clean-local-images` to reclaim space -# Build+test using the `composefs-sealeduki-sdboot` variant. -test-composefs: - just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot - -# Validate composefs digests match between build-time and install-time views. -# This catches mtime/metadata issues that cause sealed boot failures. -# Note: This requires a locally-built image with the compute-composefs-digest commands. -validate-composefs-digest: - cargo xtask validate-composefs-digest {{base_img}} - -# Only used by ci.yml right now -build-install-test-image: build - cd hack && podman build {{base_buildargs}} -t {{base_img}}-install -f Containerfile.drop-lbis - -# These tests accept the container image as input, and may spawn it. -run-container-external-tests: - ./tests/container/run {{base_img}} - -# We build the unit tests into a container image +# Build unit tests into a container image +[group('maintenance')] build-units: #!/bin/bash set -xeuo pipefail eval $(just _git-build-vars) podman build {{base_buildargs}} --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} --build-arg=pkgversion=${VERSION} --target units -t localhost/bootc-units . -# Perform validation (build, linting) in a container build environment -validate: - podman build {{base_buildargs}} --target validate . +# ============================================================================ +# Internal helpers (prefixed with _) +# ============================================================================ -# Run tmt-based test suites using local virtual machines with -# bcvk. -# -# To run an individual test, pass it as an argument like: -# `just test-tmt readonly` -test-tmt *ARGS: build - @just _build-upgrade-image - @just test-tmt-nobuild {{ARGS}} +_pull-lbi-images: + podman pull -q --retry 5 --retry-delay 5s {{lbi_images}} + +_git-build-vars: + #!/bin/bash + set -euo pipefail + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + if VERSION=$(git describe --tags --exact-match 2>/dev/null); then + VERSION="${VERSION#v}" + VERSION="${VERSION//-/.}" + else + COMMIT=$(git rev-parse HEAD | cut -c1-10) + COMMIT_TS=$(git show -s --format=%ct) + TIMESTAMP=$(date -u -d @${COMMIT_TS} +%Y%m%d%H%M) + VERSION="${TIMESTAMP}.g${COMMIT}" + fi + echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}" + echo "VERSION=${VERSION}" + +_keygen: + ./hack/generate-secureboot-keys -# Generate a local synthetic upgrade _build-upgrade-image: cat tmt/tests/Dockerfile.upgrade | podman build -t {{upgrade_img}} --from={{base_img}} - - -# Assume the localhost/bootc image is up to date, and just run tests. -# Useful for iterating on tests quickly. -test-tmt-nobuild *ARGS: - cargo xtask run-tmt --env=BOOTC_variant={{variant}} --upgrade-image={{upgrade_img}} {{base_img}} {{ARGS}} - -# Build test container image for testing on coreos with SKIP_CONFIGS=1, -# which skips LBIs, test kargs, and install configs that would conflict with FCOS. -build-testimage-coreos PATH: _keygen - #!/bin/bash - set -xeuo pipefail - pkg_path=$(realpath "{{PATH}}") - # Use --build-context to pass packages instead of -v to avoid mount stubs in /run - podman build --build-context "packages=${pkg_path}" \ - --build-arg SKIP_CONFIGS=1 \ - -t {{base_img}}-coreos {{buildargs}} . - -# Run test bootc install on FCOS -# BOOTC_target is `bootc-coreos`, it will be used for bootc install. -# Run `just build-testimage-coreos target/packages` to build test image firstly, -# then run `just test-tmt-on-coreos plan-bootc-install-on-coreos` -test-tmt-on-coreos *ARGS: - cargo xtask run-tmt --env=BOOTC_variant={{variant}} --env=BOOTC_target={{base_img}}-coreos:latest {{fedora-coreos}} {{ARGS}} - -# Cleanup all test VMs created by tmt tests -tmt-vm-cleanup: - bcvk libvirt rm --stop --force --label bootc.test=1 - -# Run tests (unit and integration) that are containerized -test-container: build build-units - podman run --rm --read-only localhost/bootc-units /usr/bin/bootc-units - # Pass these through for cross-checking - podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} {{base_img}} bootc-integration-tests container - -# Remove all container images built (locally) via this Justfile, by matching a label -clean-local-images: - podman images --filter "label={{testimage_label}}" - podman images --filter "label={{testimage_label}}" --format "{{{{.ID}}" | xargs -r podman rmi -f - podman image prune -f - podman rmi {{fedora-coreos}} -f - -# Print the container image reference for a given short $ID-VERSION_ID for NAME -# and 'base' or 'buildroot-base' for TYPE (base image type) -pullspec-for-os TYPE NAME: - @jq -r --arg v "{{NAME}}" '."{{TYPE}}"[$v]' < hack/os-image-map.json - -build-mdbook: - #!/bin/bash - set -xeuo pipefail - secret_arg="" - # Pass GH_TOKEN to avoid API rate limits when cargo-binstall fetches binaries - if test -n "${GH_TOKEN:-}"; then - secret_arg="--secret=id=GH_TOKEN,env=GH_TOKEN" - fi - podman build {{generic_buildargs}} ${secret_arg} -t localhost/bootc-mdbook -f docs/Dockerfile.mdbook . - -# Generate the rendered HTML to the target DIR directory -build-mdbook-to DIR: build-mdbook - #!/bin/bash - set -xeuo pipefail - # Create a temporary container to extract the built docs - container_id=$(podman create localhost/bootc-mdbook) - podman cp ${container_id}:/src/docs/book {{DIR}} - podman rm -f ${container_id} - -mdbook-serve: build-mdbook - #!/bin/bash - set -xeuo pipefail - podman run --init --replace -d --name bootc-mdbook --rm --publish 127.0.0.1::8000 localhost/bootc-mdbook - echo http://$(podman port bootc-mdbook 8000/tcp) - -# Update all generated files (man pages and JSON schemas) -# -# This is the unified command that: -# - Auto-discovers new CLI commands and creates man page templates -# - Syncs CLI options from Rust code to existing man page templates -# - Updates JSON schema files -# -# Use this after adding, removing, or modifying CLI options or schemas. -update-generated: - cargo run -p xtask update-generated - -# Verify build system properties (reproducible builds) -# -# This runs `just package` twice and verifies that the resulting RPMs -# are bit-for-bit identical, confirming SOURCE_DATE_EPOCH is working. -check-buildsys: - cargo run -p xtask check-buildsys