diff --git a/.github/.jira_sync_config.yaml b/.github/.jira_sync_config.yaml new file mode 100644 index 000000000..c7abc3156 --- /dev/null +++ b/.github/.jira_sync_config.yaml @@ -0,0 +1,39 @@ +settings: + # Jira project key to create the issue in + jira_project_key: "LXD" + + # Dictionary mapping GitHub issue status to Jira issue status + status_mapping: + opened: Untriaged + closed: done + + # (Optional) Jira project components that should be attached to the created issue + # Component names are case-sensitive + # components: + # - LXD + # - MicroCloud + + # (Optional) GitHub labels. Only issues with one of those labels will be synchronized. + # If not specified, all issues will be synchronized + labels: + - Jira + + # (Optional) (Default: false) Add a new comment in GitHub with a link to Jira created issue + add_gh_comment: false + + # (Optional) (Default: true) Synchronize issue description from GitHub to Jira + sync_description: true + + # (Optional) (Default: true) Synchronize comments from GitHub to Jira + sync_comments: false + + # (Optional) (Default: None) Parent Epic key to link the issue to + epic_key: "LXD-1251" + + # (Optional) Dictionary mapping GitHub issue labels to Jira issue types. + # If label on the issue is not in specified list, this issue will be created as a Bug + label_mapping: + improvement: Story + feature: Story + investigation: Spike + bug: Bug \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..4520238d4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + labels: [] + schedule: + interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + labels: [] + schedule: + interval: "weekly" diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 000000000..ea476fbf3 --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "enabled": false, + "dependencyDashboard": false, + "extends": [ + "config:recommended", + ":disableDependencyDashboard" + ] +} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..ad306d264 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,84 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main", "*" ] + pull_request: + branches: [ "main", "*" ] + schedule: + - cron: '19 20 * * 5' + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/commits.yml b/.github/workflows/commits.yml index 61e4cfd23..7c6f71fbf 100644 --- a/.github/workflows/commits.yml +++ b/.github/workflows/commits.yml @@ -1,27 +1,36 @@ name: Commits on: - - pull_request + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true permissions: contents: read jobs: - dco-check: - permissions: - pull-requests: read # for tim-actions/get-pr-commits to get list of commits from the PR - name: Signed-off-by (DCO) - runs-on: ubuntu-22.04 + commits: + name: Branch target and CLA + runs-on: ubuntu-latest steps: - - name: Get PR Commits - id: 'get-pr-commits' - uses: tim-actions/get-pr-commits@master - with: - token: ${{ secrets.GITHUB_TOKEN }} + - name: Check branch target + env: + TARGET: ${{ github.event.pull_request.base.ref }} + TITLE: ${{ github.event.pull_request.title }} + if: ${{ github.actor != 'dependabot[bot]' }} + run: | + set -eux + TARGET_FROM_PR_TITLE="$(echo "${TITLE}" | sed -n 's/.*(\(v[0-9]-\(edge\|candidate\)\))$/\1/p')" + if [ -z "${TARGET_FROM_PR_TITLE}" ]; then + TARGET_FROM_PR_TITLE="main" + else + echo "Branch target overridden from PR title" + fi + [ "${TARGET}" = "${TARGET_FROM_PR_TITLE}" ] && exit 0 - - name: Check that all commits are signed-off - uses: tim-actions/dco@master - with: - commits: ${{ steps.get-pr-commits.outputs.commits }} + echo "Invalid branch target: ${TARGET} != ${TARGET_FROM_PR_TITLE}" + exit 1 - - name: Check if CLA signed - uses: canonical/has-signed-canonical-cla@v1 + - name: Check if CLA signed + uses: canonical/has-signed-canonical-cla@046337b42822b7868ad62970988929c79f9c1d40 # 1.2.3 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 000000000..8659625be --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,121 @@ +name: Vulnerability Scanning with Trivy +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' # Test Trivy daily at midnight + +permissions: + contents: read + security-events: write # for uploading SARIF results to the security tab + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + trivy-repo: + name: Trivy - Repository + runs-on: ubuntu-22.04 + if: ${{ github.ref_name == 'main' }} + steps: + - name: Checkout + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + ref: main + + - name: Install Trivy + uses: canonical/lxd/.github/actions/install-trivy@main + + - name: Download Trivy DB + id: db_download + run: trivy fs --download-db-only --cache-dir /home/runner/vuln-cache + continue-on-error: true + + - name: Use previous downloaded database + if: ${{ steps.db_download.outcome == 'failure' }} + uses: actions/cache/restore@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 + with: + path: /home/runner/vuln-cache + key: trivy-latest-cache + + - name: Run Trivy vulnerability scanner + run: | + trivy fs --skip-db-update \ + --scanners vuln,secret,misconfig \ + --format sarif \ + --cache-dir /home/runner/vuln-cache \ + --severity LOW,MEDIUM,HIGH,CRITICAL \ + --output trivy-microcloud-repo-scan-results.sarif . + + - name: Cache Trivy vulnerability database + if: ${{ steps.db_download.outcome == 'success' }} + uses: actions/cache/save@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 + with: + path: /home/runner/vuln-cache + key: trivy-latest-cache + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: "trivy-microcloud-repo-scan-results.sarif" + sha: ${{ github.sha }} + ref: refs/heads/main + + trivy-snap: + name: Trivy - Snap + runs-on: ubuntu-22.04 + if: ${{ github.ref_name == 'main' }} + needs: trivy-repo + strategy: + matrix: + include: + - channel: "3/edge" + branch: "main" + - channel: "2/candidate" + branch: "v2-edge" + - channel: "1/edge" + branch: "v1-edge" + steps: + - name: Checkout + uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + with: + ref: ${{ matrix.branch }} + + - name: Install Trivy + uses: canonical/lxd/.github/actions/install-trivy@main + + - name: Restore cached Trivy vulnerability database + uses: actions/cache/restore@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 + with: + path: /home/runner/vuln-cache + key: trivy-latest-cache + + - name: Download snap for scan + run: | + snap download microcloud --channel=${{ matrix.channel }} + unsquashfs ./microcloud*.snap + + - name: Run Trivy vulnerability scanner + run: | + trivy rootfs --skip-db-update \ + --scanners vuln,secret,misconfig \ + --format sarif \ + --cache-dir /home/runner/vuln-cache \ + --severity LOW,MEDIUM,HIGH,CRITICAL \ + --output snap-scan-results.sarif squashfs-root + + - name: Flag snap scanning alerts + run: | + jq '.runs[].tool.driver.rules[] |= (.shortDescription.text |= "Snap scan - " + .)' snap-scan-results.sarif > tmp.json + mv tmp.json snap-scan-results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@f779452ac5af1c261dce0346a8f964149f49322b # v3.26.13 + with: + sarif_file: "snap-scan-results.sarif" + sha: ${{ github.sha }} + ref: refs/heads/${{ matrix.branch }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 280b006df..c6114cc1b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,67 +1,411 @@ name: Tests on: - - push - - pull_request + schedule: + - cron: '0 0 * * 0' # Weekly on Sunday at 00:00 UTC + push: + branches: + - v1-edge + pull_request: + workflow_dispatch: + +env: + CGO_CFLAGS: -I/home/runner/go/deps/dqlite/include/ + CGO_LDFLAGS: -L/home/runner/go/deps/dqlite/.libs/ + LD_LIBRARY_PATH: /home/runner/go/deps/dqlite/.libs/ + CGO_LDFLAGS_ALLOW: (-Wl,-wrap,pthread_create)|(-Wl,-z,now) + GOCOVERDIR: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && '/home/runner/work/microcloud/microcloud/cover' || '' }} + +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +defaults: + run: + # Make sure bash is always invoked with `-eo pipefail` + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell + shell: bash + jobs: + changes: + name: Changes + runs-on: ubuntu-22.04 + outputs: + except_docs: ${{ steps.filter.outputs.except_docs }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Check for changes + uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + except_docs: + # Match all changes except 'doc/**'. + # If no files outside of 'doc/**' are changed except_docs is set to 'false'. + - '!(doc/**)' + code-tests: - name: Code tests + name: Code runs-on: ubuntu-22.04 + needs: [changes] + if: ${{ needs.changes.outputs.except_docs == 'true' }} steps: - name: Checkout uses: actions/checkout@v4 + with: + # A non-shallow clone is needed for the Differential ShellCheck + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.22.x + + - name: Check compatibility with min Go version + working-directory: './microcloud' + run: | + set -eux + GOMIN="$(sed -n 's/^GOMIN=\([0-9.]\+\)$/\1/p' Makefile)" + go mod tidy -go="${GOMIN}" - name: Dependency Review - uses: actions/dependency-review-action@v3 + uses: actions/dependency-review-action@v4 if: github.event_name == 'pull_request' - id: ShellCheck name: Differential ShellCheck - uses: redhat-plumbers-in-action/differential-shellcheck@v4 + uses: redhat-plumbers-in-action/differential-shellcheck@v5 with: token: ${{ secrets.GITHUB_TOKEN }} + strict-check-on-push: true if: github.event_name == 'pull_request' - name: Upload artifact with ShellCheck defects in SARIF format - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: Differential ShellCheck SARIF path: ${{ steps.ShellCheck.outputs.sarif }} if: github.event_name == 'pull_request' - - name: Install Go - uses: actions/setup-go@v4 - with: - go-version: 1.20.x - - name: Install dependencies run: | sudo add-apt-repository ppa:dqlite/dev -y --no-update sudo apt-get update - sudo apt-get install --no-install-recommends -y libdqlite-dev pkg-config + sudo apt-get install --no-install-recommends -y pkg-config autoconf automake libtool make libuv1-dev libsqlite3-dev liblz4-dev libdqlite-dev + + - name: Build + working-directory: './microcloud' + run: make - name: Run static analysis - working-directory: microcloud + working-directory: './microcloud' run: make check-static + - name: Make GOCOVERDIR + run: mkdir -p "${GOCOVERDIR}" + if: env.GOCOVERDIR != '' + - name: Unit tests - working-directory: microcloud - run: make check-unit + working-directory: './microcloud' + run: | + set -eux + make check-unit + + - name: Upload coverage data + uses: actions/upload-artifact@v4 + with: + name: coverage-unit + path: ${{env.GOCOVERDIR}} + if: env.GOCOVERDIR != '' + + - name: Upload system test dependencies + uses: actions/upload-artifact@v4 + with: + name: system-test-deps + path: | + /home/runner/go/bin/microcloud + /home/runner/go/bin/microcloudd + retention-days: 1 + + system-tests: + env: + DEBUG: "1" + SKIP_VM_LAUNCH: "1" + SNAPSHOT_RESTORE: "1" + name: System + runs-on: ubuntu-22.04 + needs: code-tests + strategy: + fail-fast: false + matrix: + # Test suites that will be combined with the set versions. + # Define this first in the matrix so that it's readable + # after GitHub as formed the name for the respective check. + suite: + - "add" + - "instances" + - "basic" + - "interactive" + - "mismatch" + - "preseed" + # Set of versions to use for the matrix tests. + os: ["24.04"] + go: ["1.22.x"] + microceph: ["reef/candidate"] + microovn: ["22.03/edge"] + lxd: ["5.21/edge"] + # Continue using latest/edge as the default branch because 1/edge does not yet support debug binaries. + microcloud: ["latest/edge"] + steps: + - name: Performance tuning + run: | + set -eux + # optimize ext4 FSes for performance, not reliability + for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do + # nombcache and data=writeback cannot be changed on remount + sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true + done + + # disable dpkg from calling sync() + echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io + + - name: Reclaim some space + run: | + set -eux + + sudo snap remove lxd --purge + # Purge older snap revisions that are disabled/superseded by newer revisions of the same snap + snap list --all | while read -r name _ rev _ _ notes _; do + [[ "${notes}" =~ disabled$ ]] && sudo snap remove "${name}" --revision "${rev}" --purge + done || true + + # This was inspired from https://github.com/easimon/maximize-build-space + df -h / + # dotnet + sudo rm -rf /usr/share/dotnet + # android + sudo rm -rf /usr/local/lib/android + # haskell + sudo rm -rf /opt/ghc + df -h / + + - name: Reclaim some memory + run: | + set -eux + + free -mt + sudo systemctl stop dpkg-db-backup.timer e2scrub_all.timer fstrim.timer logrotate.timer man-db.timer motd-news.timer phpsessionclean.timer update-notifier-download.timer update-notifier-motd.timer + sudo systemctl stop iscsid.socket multipathd.socket + sudo systemctl stop cron.service irqbalance.service mono-xsp4.service multipathd.service networkd-dispatcher.service php8.1-fpm.service + free -mt + + - name: Remove docker + run: | + set -eux + sudo apt-get autopurge -y containerd.io moby-containerd docker docker-ce podman uidmap + sudo ip link delete docker0 + sudo nft flush ruleset + + - name: "Disable br_netfilter" + run: | + # When br_netfilter is enabled, the multicast traffic that passes the native LXD bridge + # will get masqueraded too which breaks the multicast discovery. + sudo rmmod br_netfilter + + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Go (${{ matrix.go }}) + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + + - name: Install dependencies + run: | + sudo add-apt-repository ppa:dqlite/dev -y --no-update + sudo apt-get update + sudo apt-get install --no-install-recommends -y libdqlite-dev pkg-config + + - name: Download system test dependencies + uses: actions/download-artifact@v4 + with: + name: system-test-deps + merge-multiple: true + path: /home/runner/go/bin + + - name: Make GOCOVERDIR + run: mkdir -p "${GOCOVERDIR}" + if: env.GOCOVERDIR != '' + + - name: Sideload debug binaries + run: | + set -eux + + # Binaries to sideload + export MICROCLOUD_DEBUG_PATH=~/go/bin/microcloud + export MICROCLOUDD_DEBUG_PATH=~/go/bin/microcloudd + + # strip debug binaries + strip -s "${MICROCLOUD_DEBUG_PATH}" "${MICROCLOUDD_DEBUG_PATH}" + + echo "MICROCLOUD_DEBUG_PATH=${MICROCLOUD_DEBUG_PATH}" >> "${GITHUB_ENV}" + echo "MICROCLOUDD_DEBUG_PATH=${MICROCLOUDD_DEBUG_PATH}" >> "${GITHUB_ENV}" + + - name: "Free up the ephemeral disk" + run: | + set -eux + + if ! mountpoint --quiet /mnt; then + echo "INFO: no ephemeral disk mounted on /mnt" + mount + exit 0 + fi + + # If the rootfs and the ephemeral part are on the same physical disk, giving the whole + # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action + # runners to have a single disk, we immediately bail rather than trying to gracefully + # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150, + # we will be able to stop worrying about that special case. + if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then + echo "FAIL: rootfs and ephemeral part on the same disk, aborting" + lsblk + blkid + sudo fdisk -l + exit 1 + fi + + # Free-up the ephemeral disk to use it as storage device for LXD. + sudo swapoff /mnt/swapfile + ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')" + sudo umount /mnt + sudo wipefs -a "${ephemeral_disk}" + export TEST_STORAGE_SOURCE="${ephemeral_disk}" + + echo "TEST_STORAGE_SOURCE=${TEST_STORAGE_SOURCE}" >> "${GITHUB_ENV}" + + - name: "Setup host LXD" + run: | + set -eux + sudo snap install lxd --channel 5.21/edge || sudo snap refresh lxd --channel 5.21/edge + sudo lxd init --auto + + - name: "Prepare for system tests" + working-directory: './microcloud' + run: | + set -eux + chmod +x ~ + + export BASE_OS="${{ matrix.os }}" + export LXD_SNAP_CHANNEL="${{ matrix.lxd }}" + export MICROCEPH_SNAP_CHANNEL="${{ matrix.microceph }}" + export MICROOVN_SNAP_CHANNEL="${{ matrix.microovn }}" + export MICROCLOUD_SNAP_CHANNEL="${{ matrix.microcloud }}" + + cd test + sudo --preserve-env=GOCOVERDIR,DEBUG,GITHUB_ACTIONS,MICROCLOUD_DEBUG_PATH,MICROCLOUDD_DEBUG_PATH,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE,TESTBED_READY,BASE_OS,LXD_SNAP_CHANNEL,MICROCEPH_SNAP_CHANNEL,MICROOVN_SNAP_CHANNEL,MICROCLOUD_SNAP_CHANNEL ./main.sh setup + + echo "TESTBED_READY=1" >> "${GITHUB_ENV}" + echo "BASE_OS=${BASE_OS}" >> "${GITHUB_ENV}" + echo "LXD_SNAP_CHANNEL=${LXD_SNAP_CHANNEL}" >> "${GITHUB_ENV}" + echo "MICROCEPH_SNAP_CHANNEL=${MICROCEPH_SNAP_CHANNEL}" >> "${GITHUB_ENV}" + echo "MICROOVN_SNAP_CHANNEL=${MICROOVN_SNAP_CHANNEL}" >> "${GITHUB_ENV}" + echo "MICROCLOUD_SNAP_CHANNEL=${MICROCLOUD_SNAP_CHANNEL}" >> "${GITHUB_ENV}" + + - name: "Run system tests (${{ matrix.suite }})" + working-directory: './microcloud' + run: | + set -eux + chmod +x ~ + cd test + sudo --preserve-env=GOCOVERDIR,DEBUG,GITHUB_ACTIONS,MICROCLOUD_DEBUG_PATH,MICROCLOUDD_DEBUG_PATH,SKIP_VM_LAUNCH,SNAPSHOT_RESTORE,TEST_STORAGE_SOURCE,TESTBED_READY,BASE_OS,LXD_SNAP_CHANNEL,MICROCEPH_SNAP_CHANNEL,MICROOVN_SNAP_CHANNEL,MICROCLOUD_SNAP_CHANNEL ./main.sh ${{ matrix.suite }} + echo "TIMESTAMP=$(date +%Y%m%d_%H%M%S_%N)" >> "${GITHUB_ENV}" + + - name: Upload coverage data + uses: actions/upload-artifact@v4 + with: + name: coverage-${{ matrix.go }}-${{ matrix.suite }}-${{ env.TIMESTAMP }} + path: ${{ env.GOCOVERDIR }} + if: env.GOCOVERDIR != '' + + tics: + name: Tiobe TICS + runs-on: ubuntu-22.04 + needs: system-tests + if: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && github.ref_name == 'main' && github.repository == 'canonical/microcloud' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.23.x + + - name: Download coverage data + uses: actions/download-artifact@v4 + with: + pattern: coverage-* + path: ${{env.GOCOVERDIR}} + merge-multiple: true + + - name: Extract coverage data + run: | + find ${{ env.GOCOVERDIR }}/micro*/cover/ -type f -exec mv {} ${{ env.GOCOVERDIR }} \; + rm -rf ${{ env.GOCOVERDIR }}/micro* + ls -la ${{ env.GOCOVERDIR }} + + - name: Download system test dependencies + uses: actions/download-artifact@v4 + with: + name: system-test-deps + merge-multiple: true + path: /home/runner/go/bin - documentation-checks: + - name: Install dependencies + working-directory: './microcloud' + run: | + go install github.com/axw/gocov/gocov@latest + go install github.com/AlekSi/gocov-xml@latest + go install honnef.co/go/tools/cmd/staticcheck@latest + + - name: Convert coverage files + working-directory: './microcloud' + run: | + go tool covdata textfmt -i="${GOCOVERDIR}" -o "${GOCOVERDIR}"/coverage.out + gocov convert "${GOCOVERDIR}"/coverage.out > "${GOCOVERDIR}"/coverage.json + gocov-xml < "${GOCOVERDIR}"/coverage.json > "${GOCOVERDIR}"/coverage-go.xml + go tool covdata percent -i="${GOCOVERDIR}" + + - name: Run TICS + uses: tiobe/tics-github-action@v3 + with: + mode: qserver + project: microcloud + viewerUrl: https://canonical.tiobe.com/tiobeweb/TICS/api/cfg?name=default + branchdir: ${{ github.workspace }} + ticsAuthToken: ${{ secrets.TICS_AUTH_TOKEN }} + installTics: true + calc: ALL + tmpdir: /tmp/tics + + doc-tests: + name: Documentation uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main with: working-directory: './doc' + makefile: 'Makefile' snap: name: Trigger snap edge build runs-on: ubuntu-22.04 - needs: [code-tests, documentation-checks] - if: ${{ github.repository == 'canonical/microcloud' && github.event_name == 'push'}} + needs: [code-tests, system-tests, doc-tests] + if: ${{ github.repository == 'canonical/microcloud' && github.event_name == 'push' && github.actor != 'dependabot[bot]' }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -71,18 +415,20 @@ jobs: SSH_AUTH_SOCK: /tmp/ssh_agent.sock LAUNCHPAD_LXD_BOT_KEY: ${{ secrets.LAUNCHPAD_LXD_BOT_KEY }} run: | + set -eux + mkdir -m 0700 -p ~/.ssh/ ssh-agent -a "${SSH_AUTH_SOCK}" > /dev/null ssh-add - <<< "${{ secrets.LAUNCHPAD_LXD_BOT_KEY }}" - mkdir -m 0700 -p ~/.ssh/ + ssh-add -L > ~/.ssh/id_ed25519.pub # In ephemeral environments like GitHub Action runners, relying on TOFU isn't providing any security # so require the key obtained by `ssh-keyscan` to match the expected hash from https://help.launchpad.net/SSHFingerprints ssh-keyscan git.launchpad.net >> ~/.ssh/known_hosts ssh-keygen -qlF git.launchpad.net | grep -xF 'git.launchpad.net RSA SHA256:UNOzlP66WpDuEo34Wgs8mewypV0UzqHLsIFoqwe8dYo' - - name: Install Go - uses: actions/setup-go@v4 + - name: Install Go (${{ matrix.go }}) + uses: actions/setup-go@v5 with: - go-version: 1.20.x + go-version: 1.22.x - name: Trigger Launchpad snap build env: @@ -90,15 +436,19 @@ jobs: TARGET: >- ${{ fromJson('{ "main": "latest-edge", - "stable-5.0": "5.0-edge", + "v1-edge": "v1-edge", }')[github.ref_name] }} run: | - set -x + set -eux + git config --global transfer.fsckobjects true git config --global user.name "Canonical LXD Bot" git config --global user.email "lxd@lists.canonical.com" - localRev=$(git rev-parse HEAD) + git config --global commit.gpgsign true + git config --global gpg.format "ssh" + git config --global user.signingkey ~/.ssh/id_ed25519.pub + localRev="$(git rev-parse HEAD)" GOPROXY=direct go install github.com/canonical/lxd-ci/lxd-snapcraft@latest - git clone -b "${TARGET}" git+ssh://lxdbot@git.launchpad.net/~canonical-lxd/microcloud ~/microcloud-pkg-snap-lp + git clone -b "${TARGET}" git+ssh://lxdbot@git.launchpad.net/~microcloud-snap/microcloud ~/microcloud-pkg-snap-lp cd ~/microcloud-pkg-snap-lp lxd-snapcraft -package microcloud -set-version "git-${localRev:0:7}" -set-source-commit "${localRev}" git add --all