From b274b315c24de933e67a538a5fccbcf904854af8 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Wed, 18 Jan 2023 17:39:03 -0500 Subject: [PATCH] Add `cargo xtask` and packaging infrastructure First, this adds `cargo xtask` following https://github.com/matklad/cargo-xtask/ We use this to write "external glue scripts" in Rust, not bash. Specifically we now have e.g. `cargo xtask vendor` which just wraps running `cargo vendor-filterer`. Then build on that and add `cargo xtask package-srpm` which generates a `.src.rpm`. And build on that by adding the requisite glue to have Fedora's COPR be able to understand it, so that we can get auto-built and shipped packages there. This will make trying out bootc a bit easier. Signed-off-by: Colin Walters --- .cargo/config.toml | 2 + .copr/Makefile | 7 ++ .github/workflows/packaging.yml | 36 +++++++ Cargo.toml | 17 ++- Makefile | 8 ++ cli/Cargo.toml | 3 +- contrib/packaging/bootc.spec | 53 ++++++++++ xtask/Cargo.toml | 19 ++++ xtask/src/xtask.rs | 176 ++++++++++++++++++++++++++++++++ 9 files changed, 318 insertions(+), 3 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 .copr/Makefile create mode 100644 .github/workflows/packaging.yml create mode 100644 contrib/packaging/bootc.spec create mode 100644 xtask/Cargo.toml create mode 100644 xtask/src/xtask.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..35049cbc --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --package xtask --" diff --git a/.copr/Makefile b/.copr/Makefile new file mode 100644 index 00000000..011fb2ad --- /dev/null +++ b/.copr/Makefile @@ -0,0 +1,7 @@ +srpm: + dnf -y install cargo git openssl-devel + # similar to https://github.com/actions/checkout/issues/760, but for COPR + git config --global --add safe.directory '*' + cargo install cargo-vendor-filterer + cargo xtask package-srpm + mv target/*.src.rpm $$outdir diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml new file mode 100644 index 00000000..7b40f945 --- /dev/null +++ b/.github/workflows/packaging.yml @@ -0,0 +1,36 @@ +# This is an unused WIP. Maybe we'll use it in the future +# name: Packaging + +# permissions: +# actions: read + +# on: +# push: +# branches: [main] +# pull_request: +# branches: [main] +# types: [labeled, opened, synchronize, reopened] +# workflow_dispatch: {} + +# jobs: +# srpm: +# if: ${{ contains(github.event.pull_request.labels.*.name, 'ci/full') }} +# runs-on: ubuntu-latest +# container: quay.io/coreos-assembler/fcos-buildroot:testing-devel +# steps: +# - uses: actions/checkout@v3 +# - name: Mark git checkout as safe +# run: git config --global --add safe.directory "$GITHUB_WORKSPACE" +# - name: Cache Dependencies +# uses: Swatinem/rust-cache@v2 +# with: +# key: "srpm" +# - name: Install vendor tool +# run: cargo install cargo-vendor-filterer +# - name: Build +# run: cargo xtask package-srpm +# - name: Upload +# uses: actions/upload-artifact@v2 +# with: +# name: bootc-srpm +# path: target/*.src.rpm diff --git a/Cargo.toml b/Cargo.toml index c10e2754..e643649c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,21 @@ [workspace] -members = ["cli", "lib"] +members = ["cli", "lib", "xtask"] [profile.dev] opt-level = 1 # No optimizations are too slow for us. [profile.release] -lto = "thin" +# RPMs/debs/etc want debuginfo by default +debug = true + +# See https://github.com/coreos/cargo-vendor-filterer +[workspace.metadata.vendor-filter] +platforms = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", "s390x-unknown-linux-gnu"] +all-features = true +exclude-crate-paths = [ { name = "libz-sys", exclude = "src/zlib" }, + { name = "libz-sys", exclude = "src/zlib-ng" }, + # rustix includes pre-generated assembly for linux_raw, which we don't use + { name = "rustix", exclude = "src/imp/linux_raw" }, + # Test files that include binaries + { name = "system-deps", exclude = "src/tests" }, + ] diff --git a/Makefile b/Makefile index 84eff857..7a0edf42 100644 --- a/Makefile +++ b/Makefile @@ -11,3 +11,11 @@ bin-archive: all install-kola-tests: install -D -t $(DESTDIR)$(prefix)/lib/coreos-assembler/tests/kola/bootc tests/kolainst/basic + +vendor: + cargo xtask $@ +.PHONY: vendor + +package-rpm: + cargo xtask $@ +.PHONY: package-rpm diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ff17eff1..ddf68855 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -7,6 +7,7 @@ repository = "https://github.com/cgwalters/bootc" readme = "README.md" publish = false rust-version = "1.63.0" +default-run = "bootc" # See https://github.com/coreos/cargo-vendor-filterer [package.metadata.vendor-filter] @@ -17,7 +18,7 @@ platforms = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", "powerpc64 [dependencies] anyhow = "1.0" -bootc-lib = { path = "../lib" } +bootc-lib = { version = "0.1", path = "../lib" } clap = "3.2" libc = "0.2.92" tokio = { version = "1", features = ["macros"] } diff --git a/contrib/packaging/bootc.spec b/contrib/packaging/bootc.spec new file mode 100644 index 00000000..bf7d4bc1 --- /dev/null +++ b/contrib/packaging/bootc.spec @@ -0,0 +1,53 @@ +%bcond_without check + +Name: bootc +Version: 0.1 +Release: 1%{?dist} +Summary: Boot containers + +License: ASL 2.0 +URL: https://github.com/containers/bootc +Source0: https://github.com/containers/bootc/releases/download/v%{version}/bootc-%{version}.tar.zstd +Source1: https://github.com/containers/bootc/releases/download/v%{version}/bootc-%{version}-vendor.tar.zstd + +BuildRequires: make +BuildRequires: openssl-devel +BuildRequires: cargo +BuildRequires: systemd +# For autosetup -Sgit +BuildRequires: git +BuildRequires: zlib-devel +BuildRequires: ostree-devel +BuildRequires: openssl-devel +BuildRequires: systemd-devel + +%description +%{summary} + +%files +%license LICENSE-APACHE LICENSE-MIT +%doc README.md +%{_bindir}/bootc + +%prep +%autosetup -p1 -Sgit +tar -xv -f %{SOURCE1} +mkdir -p .cargo +cat >>.cargo/config.toml << EOF +[source.crates-io] +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "vendor" +EOF + +%build +make + +%install +%make_install INSTALL="install -p -c" + +%changelog +* Tue Oct 18 2022 Colin Walters +- Dummy changelog + diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 00000000..e258c647 --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,19 @@ +# See https://github.com/matklad/cargo-xtask +# This is an implementation detail of bootc +[package] +name = "xtask" +version = "0.1.0" +license = "MIT OR Apache-2.0" +edition = "2021" +publish = false + +[[bin]] +name = "xtask" +path = "src/xtask.rs" + +[dependencies] +anyhow = "1.0.68" +camino = "1.0" +fn-error-context = "0.2.0" +tempfile = "3.3" +xshell = { version = "0.2" } diff --git a/xtask/src/xtask.rs b/xtask/src/xtask.rs new file mode 100644 index 00000000..dc4f923a --- /dev/null +++ b/xtask/src/xtask.rs @@ -0,0 +1,176 @@ +use std::fs::File; +use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::process::{Command, Stdio}; + +use anyhow::{Context, Result}; +use camino::{Utf8Path, Utf8PathBuf}; +use fn_error_context::context; +use xshell::{cmd, Shell}; + +const NAME: &str = "bootc"; +const VENDORPATH: &str = "target/vendor.tar.zstd"; + +fn main() { + if let Err(e) = try_main() { + eprintln!("{e:?}"); + std::process::exit(1); + } +} + +fn try_main() -> Result<()> { + let task = std::env::args().nth(1); + let sh = xshell::Shell::new()?; + if let Some(cmd) = task.as_deref() { + let f = match cmd { + "vendor" => vendor, + "package" => package, + "package-srpm" => package_srpm, + _ => print_help, + }; + f(&sh)?; + } else { + print_help(&sh)?; + } + Ok(()) +} + +fn vendor(sh: &Shell) -> Result<()> { + let target = VENDORPATH; + cmd!( + sh, + "cargo vendor-filterer --prefix=vendor --format=tar.zstd {target}" + ) + .run()?; + Ok(()) +} + +fn gitrev_to_version(v: &str) -> String { + let v = v.trim().trim_start_matches('v'); + v.replace('-', ".") +} + +#[context("Finding gitrev")] +fn gitrev(sh: &Shell) -> Result { + if let Ok(rev) = cmd!(sh, "git describe --tags").ignore_stderr().read() { + Ok(gitrev_to_version(&rev)) + } else { + let mut desc = cmd!(sh, "git describe --tags --always").read()?; + desc.insert_str(0, "0."); + Ok(desc) + } +} + +struct Package { + version: String, + srcpath: Utf8PathBuf, +} + +#[context("Packaging")] +fn impl_package(sh: &Shell) -> Result { + let v = gitrev(sh)?; + let namev = format!("{NAME}-{v}"); + let p = Utf8Path::new("target").join(format!("{namev}.tar.zstd")); + let o = File::create(&p)?; + let prefix = format!("{namev}/"); + let st = Command::new("git") + .args([ + "archive", + "--format=tar", + "--prefix", + prefix.as_str(), + "HEAD", + ]) + .stdout(Stdio::from(o)) + .status()?; + if !st.success() { + anyhow::bail!("Failed to run {st:?}"); + } + Ok(Package { + version: v, + srcpath: p, + }) +} + +fn package(sh: &Shell) -> Result<()> { + let p = impl_package(sh)?.srcpath; + println!("Generated: {p}"); + Ok(()) +} + +fn impl_srpm(sh: &Shell) -> Result { + let pkg = impl_package(sh)?; + vendor(sh)?; + let td = tempfile::tempdir_in("target").context("Allocating tmpdir")?; + let td = td.into_path(); + let td: &Utf8Path = td.as_path().try_into().unwrap(); + let srcpath = td.join(pkg.srcpath.file_name().unwrap()); + std::fs::rename(pkg.srcpath, srcpath)?; + let v = pkg.version; + let vendorpath = td.join(format!("{NAME}-{v}-vendor.tar.zstd")); + std::fs::rename(VENDORPATH, vendorpath)?; + { + let specin = File::open(format!("contrib/packaging/{NAME}.spec")) + .map(BufReader::new) + .context("Opening spec")?; + let mut o = File::create(td.join(format!("{NAME}.spec"))).map(BufWriter::new)?; + for line in specin.lines() { + let line = line?; + if line.starts_with("Version:") { + writeln!(o, "# Replaced by cargo xtask package-srpm")?; + writeln!(o, "Version: {v}")?; + } else { + writeln!(o, "{}", line)?; + } + } + } + let d = sh.push_dir(td); + let mut cmd = cmd!(sh, "rpmbuild"); + for k in [ + "_sourcedir", + "_specdir", + "_builddir", + "_srcrpmdir", + "_rpmdir", + ] { + cmd = cmd.arg("--define"); + cmd = cmd.arg(format!("{k} {td}")); + } + cmd.arg("--define") + .arg(format!("_buildrootdir {td}/.build")) + .args(["-bs", "bootc.spec"]) + .run()?; + drop(d); + let mut srpm = None; + for e in std::fs::read_dir(td)? { + let e = e?; + let n = e.file_name(); + let n = if let Some(n) = n.to_str() { + n + } else { + continue; + }; + if n.ends_with(".src.rpm") { + srpm = Some(td.join(n)); + break; + } + } + let srpm = srpm.ok_or_else(|| anyhow::anyhow!("Failed to find generated .src.rpm"))?; + let dest = Utf8Path::new("target").join(srpm.file_name().unwrap()); + std::fs::rename(&srpm, &dest)?; + Ok(dest) +} + +fn package_srpm(sh: &Shell) -> Result<()> { + let srpm = impl_srpm(sh)?; + println!("Generated: {srpm}"); + Ok(()) +} + +fn print_help(_sh: &Shell) -> Result<()> { + eprintln!( + "Tasks: + - vendor +" + ); + Ok(()) +}