tree: import changes from testing-devel at 1bb1b0987b

.github/workflows/container-native.yml

@@ -0,0 +1,35 @@
+name: Container-native
+
+on:
+  pull_request:
+    branches: [testing-devel]
+
+permissions:
+  contents: read
+
+jobs:
+  podman_build:
+    name: Podman Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Validate synced Containerfile
+        run: |
+          set -xeuo pipefail
+          cat Containerfile.* > Containerfile
+          git diff --exit-code
+      # cargo-culted from rpm-ostree CI
+      # https://github.com/containers/podman/discussions/17362
+      - name: Get newer podman
+        run: |
+          set -xeuo pipefail
+          echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
+          sudo apt update
+          sudo apt install -y crun/testing podman/testing skopeo/testing
+          # Something is confused in latest GHA here
+          sudo rm /var/lib/containers -rf
+      - name: Build
+        run: podman build --security-opt=label=disable --cap-add=all --device /dev/fuse . -t localhost/fedora-coreos
+      - name: Sanity-check
+        run: podman run --rm -ti localhost/fedora-coreos echo hello

Containerfile

@@ -0,0 +1,50 @@
+# DO NOT EDIT. This Containerfile is produced by the concatenation of:
+# - Containerfile.advisory: contains this advisory
+# - Containerfile.args: contains stream-specific build args
+# - Containerfile.base: actual build logic
+# Rebuild it using `cat Containerfile.* > Containerfile`.
+
+### Containerfile.args
+
+# This is the developer default version. In pipelines, this is driven by versionary.
+ARG VERSION="42"
+# XXX: This uses the Konflux-built version until it takes over from pungi and
+# shows up at the official quay.io/fedora/fedora-bootc endpoint since the latter
+# doesn't yet have bootc-base-imagectl.
+# https://gitlab.com/fedora/bootc/base-images/-/issues/44
+# XXX: Note also this should be a digested pull that gets bumped.
+# https://gitlab.com/fedora/bootc/tracker/-/issues/34
+ARG BUILDER_IMG=quay.io/bootc-devel/fedora-bootc-42-standard:latest
+ARG MANIFEST=manifest.yaml
+
+### Containerfile.base
+
+FROM ${BUILDER_IMG} as builder
+
+ARG VERSION
+ARG MANIFEST
+
+# useful if you're hacking on rpm-ostree/bootc-base-imagectl
+# COPY rpm-ostree /usr/bin/
+# COPY bootc-base-imagectl /usr/libexec/
+
+# Note: once we can rely on https://github.com/coreos/rpm-ostree/pull/5391,
+# add this bit to the RUN command to make the developer path less painful.
+# --mount=type=cache,rw,id=coreos-build-cache,target=/cache
+RUN --mount=type=bind,target=/run/src /run/src/build-rootfs "${MANIFEST}" "${VERSION}" /target-rootfs
+
+FROM scratch
+ARG VERSION
+COPY --from=builder /target-rootfs/ /
+RUN <<EOF
+set -xeuo pipefail
+for script in /usr/libexec/coreos-postprocess-*; do
+  $script; rm $script
+done
+EOF
+
+LABEL containers.bootc=1
+LABEL ostree.bootable=1
+LABEL org.opencontainers.image.version=$VERSION
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]

Containerfile.advisory

@@ -0,0 +1,6 @@
+# DO NOT EDIT. This Containerfile is produced by the concatenation of:
+# - Containerfile.advisory: contains this advisory
+# - Containerfile.args: contains stream-specific build args
+# - Containerfile.base: actual build logic
+# Rebuild it using `cat Containerfile.* > Containerfile`.
+

Containerfile.args

@@ -0,0 +1,13 @@
+### Containerfile.args
+
+# This is the developer default version. In pipelines, this is driven by versionary.
+ARG VERSION="42"
+# XXX: This uses the Konflux-built version until it takes over from pungi and
+# shows up at the official quay.io/fedora/fedora-bootc endpoint since the latter
+# doesn't yet have bootc-base-imagectl.
+# https://gitlab.com/fedora/bootc/base-images/-/issues/44
+# XXX: Note also this should be a digested pull that gets bumped.
+# https://gitlab.com/fedora/bootc/tracker/-/issues/34
+ARG BUILDER_IMG=quay.io/bootc-devel/fedora-bootc-42-standard:latest
+ARG MANIFEST=manifest.yaml
+

Containerfile.base

@@ -0,0 +1,31 @@
+### Containerfile.base
+
+FROM ${BUILDER_IMG} as builder
+
+ARG VERSION
+ARG MANIFEST
+
+# useful if you're hacking on rpm-ostree/bootc-base-imagectl
+# COPY rpm-ostree /usr/bin/
+# COPY bootc-base-imagectl /usr/libexec/
+
+# Note: once we can rely on https://github.com/coreos/rpm-ostree/pull/5391,
+# add this bit to the RUN command to make the developer path less painful.
+# --mount=type=cache,rw,id=coreos-build-cache,target=/cache
+RUN --mount=type=bind,target=/run/src /run/src/build-rootfs "${MANIFEST}" "${VERSION}" /target-rootfs
+
+FROM scratch
+ARG VERSION
+COPY --from=builder /target-rootfs/ /
+RUN <<EOF
+set -xeuo pipefail
+for script in /usr/libexec/coreos-postprocess-*; do
+  $script; rm $script
+done
+EOF
+
+LABEL containers.bootc=1
+LABEL ostree.bootable=1
+LABEL org.opencontainers.image.version=$VERSION
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]

build-rootfs

@@ -0,0 +1,236 @@
+#!/usr/bin/python3
+
+# This script is called by the Containerfile to build FCOS. Here's what it does at a high-level:
+# 1. It gathers the list of FCOS-specific packages using the manifests.
+# 2. It gathers the list of FCOS-specific overlays using the manifests.
+# 3. It runs `bootc-base-imagectl rebuild`, passing in the packages and overlays.
+# 4. It injects various metadata (e.g. image.json, live/ bits, and platforms.json).
+# 5. It runs the postprocess scripts defined in the manifest.
+
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import yaml
+
+
+ARCH = os.uname().machine
+CONTEXTDIR = '/run/src'
+
+
+def main():
+    manifest_name = sys.argv[1]
+    version = sys.argv[2]
+    target_rootfs = sys.argv[3]
+
+    manifest_path = os.path.join(CONTEXTDIR, manifest_name)
+    manifest = get_treefile(manifest_path)
+
+    packages = list(manifest['packages'])
+
+    locked_nevras = get_locked_nevras()
+    if locked_nevras:
+        inject_pool_repo(locked_nevras)
+
+    packages.extend(locked_nevras)
+    overlays = gather_overlays(manifest)
+    build_rootfs(target_rootfs, manifest_path, packages, overlays)
+
+    inject_live(target_rootfs)
+    inject_image_json(target_rootfs, manifest_path)
+    inject_platforms_json(target_rootfs)
+
+    if version != "":
+        inject_version_info(target_rootfs, manifest['mutate-os-release'], version)
+    inject_postprocess_scripts(target_rootfs, manifest)
+
+
+def get_treefile(manifest_path):
+    with tempfile.NamedTemporaryFile(suffix='.json', mode='w') as tmp_manifest:
+        # This ensures that the treefile represents only the CoreOS bits and
+        # doesn't recurse into fedora-bootc. We can drop this once we've fully
+        # cut over to derivation.
+        json.dump({
+            "variables": {
+                "deriving": True
+            },
+            "include": manifest_path
+        }, tmp_manifest)
+        tmp_manifest.flush()
+        data = subprocess.check_output(['rpm-ostree', 'compose', 'tree',
+                                        '--print-only', tmp_manifest.name])
+    return json.loads(data)
+
+
+def build_rootfs(target_rootfs, manifest_path, packages, overlays):
+    with tempfile.NamedTemporaryFile(mode='w') as argsfile:
+        for pkg in packages:
+            argsfile.write(f"--install={pkg}\n")
+        for overlay in overlays:
+            argsfile.write(f"--add-dir={overlay}\n")
+        argsfile.flush()
+        cache_arg = ['--cachedir=/cache'] if os.path.isdir('/cache') else []
+        subprocess.check_call(["/usr/libexec/bootc-base-imagectl",
+                               "--args-file", argsfile.name, "build-rootfs",
+                               "--manifest", 'fedora-minimal-plus',
+                               target_rootfs] + cache_arg)
+
+
+def inject_postprocess_scripts(rootfs, manifest):
+    # since we have the derive-only manifest handy, just inject a script now
+    # that we can execute in the second stage. we could of course use e.g.
+    # bwrap to run those now, but... it's just cleaner to use multi-stage build
+    # semantics
+    for i, script in enumerate(manifest.get('postprocess', [])):
+        with open(os.path.join(rootfs, f'usr/libexec/coreos-postprocess-{i}'), mode='w') as f:
+            os.fchmod(f.fileno(), 0o755)
+            f.write(script)
+
+
+def get_locked_nevras():
+    lockfile_path = os.path.join(CONTEXTDIR, f"manifest-lock.{ARCH}.json")
+    overrides_path = os.path.join(CONTEXTDIR, "manifest-lock.overrides.yaml")
+    overrides_arch_path = os.path.join(CONTEXTDIR, f"manifest-lock.overrides.{ARCH}.yaml")
+
+    locks = {}
+    for path in [lockfile_path, overrides_path, overrides_arch_path]:
+        if os.path.exists(path):
+            with open(path) as f:
+                if path.endswith('.yaml'):
+                    data = yaml.safe_load(f)
+                else:
+                    data = json.load(f)
+                # this essentially re-implements the merge semantics of rpm-ostree
+                locks.update({pkgname: v['evra'] if 'evra' in v else v['evr']
+                              for (pkgname, v) in data['packages'].items()})
+    return [f'{k}-{v}' for (k, v) in locks.items()]
+
+
+def inject_pool_repo(locked_nevras):
+    # When adding the pool, we only want to _filter in_ locked packages;
+    # matching `lockfile-repos` semantics. This is abusing pretty hard the
+    # `includepkgs=` semantic but... it works.
+    shutil.copyfile(os.path.join(CONTEXTDIR, "fedora-coreos-pool.repo"),
+                    "/etc/yum.repos.d/pool.repo")
+    packages = ','.join(locked_nevras)
+    with open("/etc/yum.repos.d/pool.repo", 'a') as f:
+        f.write(f"\nincludepkgs={packages}\n")
+
+
+# This re-implements rpm-ostree's mutate-os-release to preserve the historical
+# /usr/lib/os-release API, but we may in the future completely sever off of that
+# and only rely on org.opencontainers.image.version as argued in:
+# https://gitlab.com/fedora/bootc/base-images/-/issues/40
+# https://gitlab.com/fedora/bootc/base-images/-/issues/46
+def inject_version_info(rootfs, base_version, version):
+    os_release_path = os.path.join(rootfs, 'usr/lib/os-release')
+    with open(os_release_path) as f:
+        from collections import OrderedDict
+        os_release = OrderedDict([line.strip().split('=', 1) for line in f])
+
+    for key in ['VERSION', 'PRETTY_NAME']:
+        os_release[key] = os_release[key].replace(base_version, version)
+    os_release['OSTREE_VERSION'] = f"'{version}'"
+    os_release['IMAGE_VERSION'] = f"'{version}'"
+
+    with open(os_release_path, mode='w', encoding='utf-8') as f:
+        for (k, v) in os_release.items():
+            f.write(f'{k}={v}\n')
+
+
+# This re-implements cosa's overlay logic.
+def gather_overlays(manifest):
+    overlays = []
+    for layer in manifest.get('ostree-layers', []):
+        assert layer.startswith('overlay/')
+        overlays.append(os.path.join(CONTEXTDIR, 'overlay.d', layer[len('overlay/'):]))
+    return overlays
+
+
+# Inject live/ bits.
+def inject_live(rootfs):
+    target_path = os.path.join(rootfs, 'usr/share/coreos-assembler/live')
+    shutil.copytree(os.path.join(CONTEXTDIR, "live"), target_path)
+
+
+def inject_image_json(rootfs, manifest_path):
+    manifest_vars = yaml.safe_load(open(manifest_path))['variables']
+    image = flatten_image_yaml(os.path.join(CONTEXTDIR, 'image.yaml'),
+                               format_args=manifest_vars)
+    fn = os.path.join(rootfs, 'usr/share/coreos-assembler/image.json')
+    with open(fn, 'w') as f:
+        json.dump(image, f, sort_keys=True)
+
+
+# Imported from cosa
+# Merge two lists, avoiding duplicates. Exact duplicate kargs could be valid
+# but we have no use case for them right now in our official images.
+def merge_lists(x, y, k):
+    x[k] = x.get(k, [])
+    assert isinstance(x[k], list)
+    y[k] = y.get(k, [])
+    assert isinstance(y[k], list)
+    x[k].extend([i for i in y[k] if i not in x[k]])
+
+
+# Imported from cosa
+def flatten_image_yaml(srcfile, base=None, format_args={}):
+    if base is None:
+        base = {}
+
+    with open(srcfile) as f:
+        contents = f.read()
+        srcyaml = yaml.safe_load(contents.format(**format_args))
+
+    # first, special-case list values
+    merge_lists(base, srcyaml, 'extra-kargs')
+
+    # then handle all the non-list values
+    base = merge_dicts(base, srcyaml)
+
+    if 'include' not in srcyaml:
+        return base
+
+    fn = os.path.join(os.path.dirname(srcfile), srcyaml['include'])
+    del base['include']
+    return flatten_image_yaml(fn, base=base, format_args=format_args)
+
+
+# Imported from cosa
+# Credit to @arithx
+def merge_dicts(x, y):
+    """
+    Merge two dicts recursively, but based on the difference.
+    """
+    sd = set(x.keys()).symmetric_difference(y.keys())
+    ret = {}
+    for d in [x, y]:
+        for k, v in d.items():
+            if k in sd:
+                # the key is only present in one dict, add it directly
+                ret.update({k: v})
+            elif isinstance(x[k], dict) and isinstance(y[k], dict):
+                # recursively merge
+                ret.update({k: merge_dicts(x[k], y[k])})
+            elif isinstance(x[k], list) and isinstance(y[k], list):
+                ret.update({k: x[k]})
+                merge_lists(ret, y, k)
+            else:
+                # first dictionary always takes precedence
+                ret.update({k: x[k]})
+    return ret
+
+
+def inject_platforms_json(rootfs):
+    with open(os.path.join(CONTEXTDIR, 'platforms.yaml')) as f:
+        platforms = yaml.safe_load(f)
+    fn = os.path.join(rootfs, 'usr/share/coreos-assembler/platforms.json')
+    if ARCH in platforms:
+        with open(fn, 'w') as f:
+            json.dump(platforms[ARCH], f, sort_keys=True, indent=2)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

manifests/fedora-coreos.yaml

@@ -167,7 +167,7 @@ postprocess:
 
   # Delete documentation. (Note that fedora-bootc doesn't explicitly set
   # `documentation`, and neither do we, so the rpm-ostree default is true. We
-  # historically have effectively done `the equivalent of `documentation: false`
+  # historically have effectively done the equivalent of `documentation: false`
   # though.)
   - |
     #!/usr/bin/env bash