diff --git a/.dockerignore b/.dockerignore
index 1f557997..5f13236d 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -15,6 +15,8 @@
 !docs/
 # We use the spec file
 !contrib/
+# This is used to add content on top of our default base
+!hack/
 # The systemd units and baseimage bits end up in installs
 !systemd/
 !baseimage/
diff --git a/Dockerfile b/Dockerfile
index 410b06e5..ef12cede 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -49,6 +49,8 @@ RUN /usr/libexec/bootc-base-imagectl build-rootfs --manifest=standard /target-ro
 
 FROM scratch as base
 COPY --from=target-base /target-rootfs/ /
+COPY --from=src /src/hack/ /run/hack/
+RUN cd /run/hack/ && ./provision-derived.sh
 # Note we don't do any customization here yet
 # Mark this as a test image
 LABEL bootc.testimage="1"
diff --git a/hack/Containerfile b/hack/Containerfile
index 5ec8ab7f..13dc120b 100644
--- a/hack/Containerfile
+++ b/hack/Containerfile
@@ -7,37 +7,10 @@ FROM scratch as context
 # We only need this stuff in the initial context
 COPY . /
 
-# An intermediate layer which caches the extended RPMS
-FROM localhost/bootc as extended
-# And this layer has additional stuff for testing, such as nushell etc.
-RUN --mount=type=bind,from=context,target=/run/context <<EORUN
-set -xeuo pipefail
-cd /run/context/
-./provision-derived.sh
-EORUN
-
 # And the configs
-FROM extended
-ARG SKIP_CONFIGS
+FROM localhost/bootc
 RUN --mount=type=bind,from=context,target=/run/context <<EORUN
-# Do not need the configs when testing installation on ostree
-echo "SKIP_CONFIGS=${SKIP_CONFIGS:-<unset>}"
-if [ -n "${SKIP_CONFIGS:-}" ]; then
-    echo "Skipping configs installation"
-    exit 0
-fi
 set -xeuo pipefail
 cd /run/context
-# For test-22-logically-bound-install
-cp -a lbi/usr/. /usr
-for x in curl.container curl-base.image podman.image; do
-    ln -s /usr/share/containers/systemd/$x /usr/lib/bootc/bound-images.d/$x
-done
-
-# Add some testing kargs into our dev builds
-install -D -t /usr/lib/bootc/kargs.d test-kargs/*
-# Also copy in some default install configs we use for testing
-install -D -t /usr/lib/bootc/install/ install-test-configs/*
-# Finally, test our own linting
 bootc container lint --fatal-warnings
 EORUN
diff --git a/hack/Containerfile.packit b/hack/Containerfile.packit
index 87071ed1..6f1d4a52 100644
--- a/hack/Containerfile.packit
+++ b/hack/Containerfile.packit
@@ -29,18 +29,7 @@ cp test-artifacts.repo /etc/yum.repos.d/
 dnf -y update bootc
 # Required by tmt avc checking after test
 dnf -y install audit
-./provision-derived.sh cloudinit
-
-# For test-22-logically-bound-install
-cp -a lbi/usr/. /usr
-for x in curl.container curl-base.image podman.image; do
-    ln -s /usr/share/containers/systemd/$x /usr/lib/bootc/bound-images.d/$x
-done
-
-# Add some testing kargs into our dev builds
-install -D -t /usr/lib/bootc/kargs.d test-kargs/*
-# Also copy in some default install configs we use for testing
-install -D -t /usr/lib/bootc/install/ install-test-configs/*
+./provision-derived.sh
 
 # Remove bootc repo, bootc updated already
 rm -rf /var/share/test-artifacts /etc/yum.repos.d/test-artifacts.repo
diff --git a/hack/provision-derived.sh b/hack/provision-derived.sh
index af5a1273..5b1e109d 100755
--- a/hack/provision-derived.sh
+++ b/hack/provision-derived.sh
@@ -4,13 +4,6 @@ set -xeu
 # using it in our test suite because it's better than bash. First,
 # enable EPEL to get it.
 
-cloudinit=0
-case ${1:-} in
-  cloudinit) cloudinit=1 ;;
-  "") ;;
-  *) echo "Unhandled flag: ${1:-}" 1>&2; exit 1 ;;
-esac
-
 # Ensure this is pre-created
 mkdir -p -m 0700 /var/roothome
 mkdir -p ~/.config/nushell
@@ -51,9 +44,8 @@ grep -Ev -e '^#' packages.txt | xargs dnf -y install
 cat <<KARGEOF >> /usr/lib/bootc/kargs.d/20-console.toml
 kargs = ["console=ttyS0,115200n8"]
 KARGEOF
-if test $cloudinit = 1; then
-  dnf -y install cloud-init
-  ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants
+dnf -y install cloud-init
+ln -s ../cloud-init.target /usr/lib/systemd/system/default.target.wants
   # Allow root SSH login for testing with bcvk/tmt
 mkdir -p /etc/cloud/cloud.cfg.d
 cat > /etc/cloud/cloud.cfg.d/80-enable-root.cfg <<'CLOUDEOF'
@@ -67,7 +59,6 @@ growpart:
   devices: ["/sysroot"]
 resize_rootfs: false
 CLOUDEOF
-fi
 
 dnf clean all
 # Stock extra cleaning of logs and caches in general (mostly dnf)
@@ -122,3 +113,14 @@ d /var/lib/dhclient 0755 root root - -
 EOF
   rm -rf /var/lib/dhclient
 fi
+
+# For test-22-logically-bound-install
+cp -a lbi/usr/. /usr
+for x in curl.container curl-base.image podman.image; do
+    ln -s /usr/share/containers/systemd/$x /usr/lib/bootc/bound-images.d/$x
+done
+
+# Add some testing kargs into our dev builds
+install -D -t /usr/lib/bootc/kargs.d test-kargs/*
+# Also copy in some default install configs we use for testing
+install -D -t /usr/lib/bootc/install/ install-test-configs/*