incus/.github/workflows/tests.yml

name: Tests
on:
  push:
    branches:
      - main
      - stable-*
  pull_request:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  code-tests:
    name: Code
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        go:
          - oldstable
          - stable
          - tip
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # Differential ShellCheck requires full git history
          fetch-depth: 0

      - name: Dependency Review
        uses: actions/dependency-review-action@v4
        if: github.event_name == 'pull_request'

      - id: ShellCheck
        name: Differential ShellCheck
        uses: redhat-plumbers-in-action/differential-shellcheck@v5
        env:
          SHELLCHECK_OPTS: --shell sh
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
        if: github.event_name == 'pull_request' && matrix.go == 'stable'

      - name: Upload artifact with ShellCheck defects in SARIF format
        uses: actions/upload-artifact@v4
        with:
          name: Differential ShellCheck SARIF
          path: ${{ steps.ShellCheck.outputs.sarif }}
        if: github.event_name == 'pull_request' && matrix.go == 'stable'

      - name: Install Go (${{ matrix.go }})
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go }}
        if: matrix.go != 'tip'

      - name: Install Go (stable)
        uses: actions/setup-go@v5
        with:
          go-version: stable
        if: matrix.go == 'tip'

      - name: Install Go (tip)
        run: |
          go install golang.org/dl/gotip@latest
          gotip download
          ~/sdk/gotip/bin/go version
          echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV
        if: matrix.go == 'tip'

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install --no-install-recommends -y \
            curl \
            gettext \
            git \
            libacl1-dev \
            libcap-dev \
            libdbus-1-dev \
            libcowsql-dev \
            liblxc-dev \
            lxc-templates \
            libseccomp-dev \
            libselinux-dev \
            libsqlite3-dev \
            libtool \
            libudev-dev \
            make \
            pipx \
            pkg-config \
            shellcheck

          # With pipx >= 1.5.0, we could use pipx --global instead.
          PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin \
            pipx install codespell flake8

      - name: Fix repository permissions
        run: |
          sudo chown -R runner:docker .

      - name: Check compatible min Go version
        run: |
          go mod tidy

      - name: Download go dependencies
        run: |
          go mod download

      - name: Run Incus build
        run: |
          make

      - name: Run static analysis
        env:
          GITHUB_BEFORE: ${{ github.event.before }}
        run: |
          make static-analysis

      - name: Unit tests (all)
        run: |
          sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" go test ./...

  system-tests:
    name: System
    strategy:
      fail-fast: false
      matrix:
        go:
          - stable
        suite:
          - cluster
          - standalone
        backend:
          - dir
          - btrfs
          - lvm
          - zfs
          - ceph
          - linstor
          - random
        os:
          - ubuntu-24.04
        include:
          - go: oldstable
            suite: cluster
            backend: dir
            os: ubuntu-24.04
          - go: oldstable
            suite: standalone
            backend: dir
            os: ubuntu-24.04
          - go: tip
            suite: cluster
            backend: dir
            os: ubuntu-24.04
          - go: tip
            suite: standalone
            backend: dir
            os: ubuntu-24.04

          - go: oldstable
            suite: cluster
            backend: dir
            os: ubuntu-24.04-arm
          - go: oldstable
            suite: standalone
            backend: dir
            os: ubuntu-24.04-arm
          - go: stable
            suite: cluster
            backend: dir
            os: ubuntu-24.04-arm
          - go: stable
            suite: standalone
            backend: dir
            os: ubuntu-24.04-arm
          - go: tip
            suite: cluster
            backend: dir
            os: ubuntu-24.04-arm
          - go: tip
            suite: standalone
            backend: dir
            os: ubuntu-24.04-arm
    runs-on: ${{ matrix.os }}

    steps:
      - name: Performance tuning
        run: |
          set -eux
          # optimize ext4 FSes for performance, not reliability
          for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do
            # nombcache and data=writeback cannot be changed on remount
            sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true
          done

          # disable dpkg from calling sync()
          echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io

      - name: Reclaim some space
        run: |
          set -eux

          sudo snap remove lxd --purge
          # Purge older snap revisions that are disabled/superseded by newer revisions of the same snap
          snap list --all | while read -r name _ rev _ _ notes _; do
            [ "${notes}" = "disabled" ] && snap remove "${name}" --revision "${rev}" --purge
          done || true

          # This was inspired from https://github.com/easimon/maximize-build-space
          df -h /
          # dotnet
          sudo rm -rf /usr/share/dotnet
          # android
          sudo rm -rf /usr/local/lib/android
          # haskell
          sudo rm -rf /opt/ghc
          df -h /

      - name: Remove docker
        run: |
          set -eux
          sudo apt-get autopurge -y moby-containerd docker uidmap
          sudo ip link delete docker0
          sudo nft flush ruleset

      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Go (${{ matrix.go }})
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go }}
        if: matrix.go != 'tip'

      - name: Install Go (stable)
        uses: actions/setup-go@v5
        with:
          go-version: stable
        if: matrix.go == 'tip'

      - name: Install Go (tip)
        run: |
          go install golang.org/dl/gotip@latest
          gotip download
          ~/sdk/gotip/bin/go version
          echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV
        if: matrix.go == 'tip'

      - name: Install dependencies
        run: |
          set -x
          sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update
          sudo add-apt-repository ppa:cowsql/stable -y --no-update
          sudo apt-get update

          sudo systemctl mask lxc.service lxc-net.service

          sudo apt-get install --no-install-recommends -y \
            apparmor \
            bsdextrautils \
            bzip2 \
            curl \
            dosfstools \
            git \
            libacl1-dev \
            libcap-dev \
            libdbus-1-dev \
            libcowsql-dev \
            liblxc-dev \
            libseccomp-dev \
            libselinux-dev \
            libsqlite3-dev \
            libtool \
            libudev-dev \
            linux-modules-extra-$(uname -r) \
            make \
            pkg-config\
            acl \
            attr \
            bind9-dnsutils \
            btrfs-progs \
            busybox-static \
            dnsmasq-base \
            easy-rsa \
            gettext \
            jq \
            lxc-utils \
            lvm2 \
            nftables \
            quota \
            rsync \
            s3cmd \
            socat \
            sqlite3 \
            squashfs-tools \
            tar \
            tcl \
            thin-provisioning-tools \
            uuid-runtime \
            xfsprogs \
            xz-utils \
            zfsutils-linux

          # Make sure all AppArmor profiles are loaded.
          sudo systemctl start apparmor

          # Reclaim some space
          sudo apt-get clean

          # Download minio.
          curl -sSfL https://dl.min.io/server/minio/release/linux-$(dpkg --print-architecture)/archive/minio_20240116160738.0.0_$(dpkg --print-architecture).deb --output /tmp/minio.deb
          sudo apt-get install /tmp/minio.deb --yes

          # Download MinIO client
          curl -sSfL https://dl.min.io/client/mc/release/linux-$(dpkg --print-architecture)/archive/mc.RELEASE.2024-01-16T16-06-34Z --output /tmp/mc
          sudo mv /tmp/mc /usr/local/bin/
          sudo chmod +x /usr/local/bin/mc

          # Download latest release of openfga server.
          mkdir -p "$(go env GOPATH)/bin/"
          curl -sSfL https://api.github.com/repos/openfga/openfga/releases/latest | jq -r ".assets | .[] | .browser_download_url | select(. | test(\"_linux_$(dpkg --print-architecture).tar.gz$\"))" | xargs -I {} curl -sSfL {} -o openfga.tar.gz
          tar -xzf openfga.tar.gz -C "$(go env GOPATH)/bin/"

          # Download latest release of openfga cli.
          curl -sSfL https://api.github.com/repos/openfga/cli/releases/latest | jq -r ".assets | .[] | .browser_download_url | select(. | test(\"_linux_$(dpkg --print-architecture).tar.gz$\"))" | xargs -I {} curl -sSfL {} -o fga.tar.gz
          tar -xzf fga.tar.gz -C "$(go env GOPATH)/bin/"

      - name: Download go dependencies
        run: |
          go mod download

      - name: Run Incus build
        env:
          CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
        run: |
          make

      - name: Setup MicroCeph
        if: ${{ matrix.backend == 'ceph' }}
        run: |
          set -x

          # If the rootfs and the ephemeral part are on the same physical disk, giving the whole
          # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action
          # runners to have a single disk, we immediately bail rather than trying to gracefully
          # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150,
          # we will be able to stop worrying about that special case.
          if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
            echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
            exit 1
          fi

          sudo apt-get install --no-install-recommends -y snapd
          sudo snap install microceph --channel=quincy/stable
          sudo apt-get install --no-install-recommends -y ceph-common
          sudo microceph cluster bootstrap
          sudo microceph.ceph config set global osd_pool_default_size 1
          sudo microceph.ceph config set global mon_allow_pool_delete true
          sudo microceph.ceph config set global osd_memory_target 939524096
          sudo microceph.ceph osd crush rule rm replicated_rule
          sudo microceph.ceph osd crush rule create-replicated replicated default osd
          for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do
              sudo microceph.ceph osd set $flag
          done
          # Repurpose the ephemeral disk for ceph OSD.
          sudo swapoff /mnt/swapfile
          ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
          sudo umount /mnt
          sudo microceph disk add --wipe "${ephemeral_disk}"
          sudo rm -rf /etc/ceph
          sudo ln -s /var/snap/microceph/current/conf/ /etc/ceph
          sudo microceph enable rgw
          sudo microceph.ceph osd pool create cephfs_meta 32
          sudo microceph.ceph osd pool create cephfs_data 32
          sudo microceph.ceph fs new cephfs cephfs_meta cephfs_data
          sudo microceph.ceph fs ls
          sleep 30
          sudo microceph.ceph status
          # Wait until there are no more "unkowns" pgs
          for _ in $(seq 60); do
            if sudo microceph.ceph pg stat | grep -wF unknown; then
              sleep 1
            else
              break
            fi
          done
          sudo microceph.ceph status
          sudo rm -f /snap/bin/rbd

      - name: Setup LINSTOR
        if: ${{ matrix.backend == 'linstor' }}
        run: |
          set -x

          # As with Ceph, we hope for a spare disk.
          if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
            echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
            exit 1
          fi

          sudo add-apt-repository ppa:linbit/linbit-drbd9-stack -y

          # Install everything required to compile DRBD and run LINSTOR tools.
          sudo apt-get install --no-install-recommends -y \
            drbd-dkms \
            linstor-client \
            linstor-controller \
            linstor-satellite \
            linux-headers-generic \
            python3-setuptools

          # Enable DRBD.
          sudo modprobe -r drbd
          sudo modprobe drbd

          # Get the runner IP.
          runner_ip="$(hostname -I | cut -d' ' -f1)"

          # Create a single local node.
          sudo linstor node create local "${runner_ip}" --node-type combined

          # Repurpose the ephemeral disk for LINSTOR physical storage.
          sudo swapoff /mnt/swapfile
          ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
          sudo umount /mnt
          sudo wipefs -a "${ephemeral_disk}"
          sudo linstor physical-storage create-device-pool --storage-pool incus --pool-name linstor-incus zfsthin local "${ephemeral_disk}"

          # Update the runner env.
          echo "INCUS_LINSTOR_CLUSTER=${runner_ip}" >> "$GITHUB_ENV"

      - name: "Ensure offline mode (block image server)"
        run: |
          sudo nft add table inet filter
          sudo nft add chain 'inet filter output { type filter hook output priority 10 ; }'
          sudo nft add rule inet filter output ip daddr 45.45.148.8 reject
          sudo nft add rule inet filter output ip6 daddr 2602:fc62:a:1::8 reject

      - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }}, ${{ matrix.backend }})"
        env:
          CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
          INCUS_CEPH_CLUSTER: "ceph"
          INCUS_CEPH_CEPHFS: "cephfs"
          INCUS_CEPH_CEPHOBJECT_RADOSGW: "http://127.0.0.1"
          INCUS_LINSTOR_LOCAL_SATELLITE: "local"
          INCUS_CONCURRENT: "1"
          INCUS_VERBOSE: "1"
          INCUS_OFFLINE: "1"
          INCUS_TMPFS: "1"
          INCUS_REQUIRED_TESTS: "test_storage_buckets"
        run: |
          chmod +x ~
          echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
          cd test
          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_BACKEND,INCUS_CEPH_CLUSTER,INCUS_CEPH_CEPHFS,INCUS_CEPH_CEPHOBJECT_RADOSGW,INCUS_LINSTOR_LOCAL_SATELLITE,INCUS_LINSTOR_CLUSTER,INCUS_OFFLINE,INCUS_SKIP_TESTS,INCUS_REQUIRED_TESTS, INCUS_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}

  client:
    name: Client
    strategy:
      fail-fast: false
      matrix:
        go:
          - oldstable
          - stable
        os:
          - ubuntu-latest
          - macos-latest
          - windows-latest
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go }}

      - name: Create build directory
        run: |
            mkdir bin

      - name: Build static incus (x86_64)
        env:
          CGO_ENABLED: 0
          GOARCH: amd64
        run: |
          go build -o bin/incus.x86_64 ./cmd/incus

      - name: Build static incus (aarch64)
        env:
          CGO_ENABLED: 0
          GOARCH: arm64
        run: |
          go build -o bin/incus.aarch64 ./cmd/incus

      - name: Build static incus-agent (x86_64)
        if: runner.os == 'Linux' || runner.os == 'Windows'
        env:
          CGO_ENABLED: 0
          GOARCH: amd64
        run: |
          go build -o bin/incus-agent.x86_64 ./cmd/incus-agent

      - name: Build static incus-agent (aarch64)
        if: runner.os == 'Linux' || runner.os == 'Windows'
        env:
          CGO_ENABLED: 0
          GOARCH: arm64
        run: |
          go build -o bin/incus-agent.aarch64 ./cmd/incus-agent

      - name: Build static incus-migrate
        if: runner.os == 'Linux'
        env:
          CGO_ENABLED: 0
        run: |
          GOARCH=amd64 go build -o bin/incus-migrate.x86_64 ./cmd/incus-migrate
          GOARCH=arm64 go build -o bin/incus-migrate.aarch64 ./cmd/incus-migrate

      - name: Build static lxd-to-incus
        if: runner.os == 'Linux'
        env:
          CGO_ENABLED: 0
        run: |
          GOARCH=amd64 go build -o bin/lxd-to-incus.x86_64 ./cmd/lxd-to-incus
          GOARCH=arm64 go build -o bin/lxd-to-incus.aarch64 ./cmd/lxd-to-incus

      - name: Unit tests (client)
        env:
          CGO_ENABLED: 0
        run: go test -v ./client/...

      - name: Unit tests (incus)
        env:
          CGO_ENABLED: 0
        run: go test -v ./cmd/incus/...

      - name: Unit tests (shared)
        env:
          CGO_ENABLED: 0
        run: go test -v ./shared/...

      - name: Upload incus client artifacts
        if: matrix.go == 'stable'
        uses: actions/upload-artifact@v4
        continue-on-error: true
        with:
          name: ${{ runner.os }}
          path: bin/

  documentation:
    name: Documentation
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Go
        uses: actions/setup-go@v5
        with:
          go-version: stable

      - name: Install dependencies
        run: |
          sudo apt-get install aspell aspell-en
          sudo snap install mdl

      - name: Run markdown linter
        run: |
          make doc-lint

      - name: Run spell checker
        run: |
          make doc-spellcheck

      - name: Run inclusive naming checker
        uses: get-woke/woke-action@v0
        with:
          fail-on-error: true
          woke-args: "*.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml"

      - name: Run link checker
        run: |
          make doc-linkcheck

      - name: Build docs (Sphinx)
        run: make doc

      - name: Print warnings (Sphinx)
        run: if [ -s doc/.sphinx/warnings.txt ]; then cat doc/.sphinx/warnings.txt; exit 1; fi

      - name: Upload documentation artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: documentation
          path: doc/html