.github/workflows/qns.yml

name: QNS
on:
  push:
    branches: ["main"]
    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
  pull_request:
    branches: ["main"]
    types: [opened, synchronize, reopened, ready_for_review]
    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
  merge_group:
  workflow_dispatch:
  schedule:
    # Run at 1 AM each day, so there is a `main`-branch baseline in the cache.
    - cron: '0 1 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  LATEST: neqo-latest
  DELIM: ' vs. '
  TIMEOUT: 20

jobs:
  docker-image:
    name: Build Docker image
    runs-on: ubuntu-latest
    outputs:
      imageID: ${{ steps.docker_build_and_push.outputs.imageID }}
    permissions:
      packages: write
    steps:
      - uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3.3.0
      - uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
        with:
          cache-binary: ${{ github.event_name == 'pull_request' }} # zizmor: ignore[cache-poisoning]

      - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}

      - uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
        id: meta
        with:
          images: ghcr.io/${{ github.repository }}-qns
          tags: |
            # default
            type=schedule
            type=ref,event=branch
            type=ref,event=tag
            type=ref,event=pr
            # set latest tag for default branch
            type=raw,value=latest,enable={{is_default_branch}}

      - uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
        if: ${{ github.event_name != 'pull_request' }}
        with:
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          file: qns/Dockerfile
          build-args: RUST_VERSION=stable
          cache-from: type=gha
          cache-to: type=gha,mode=max
          # FIXME: gcc for arm64 currently segmentation faults :-( recheck periodically
          # platforms: 'linux/amd64, linux/arm64'
          platforms: 'linux/amd64'

      - uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991 # v6.13.0
        id: docker_build_and_push
        with:
          tags: ${{ steps.meta.outputs.tags }}
          file: qns/Dockerfile
          build-args: RUST_VERSION=stable
          cache-from: type=gha
          cache-to: type=gha,mode=max
          platforms: 'linux/amd64'
          outputs: type=docker,dest=/tmp/${{ env.LATEST }}.tar

      - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
        with:
          name: '${{ env.LATEST }} Docker image'
          path: /tmp/${{ env.LATEST }}.tar

  implementations:
    name: Determine interop pairs
    if: ${{ github.event_name != 'push' }}
    needs: docker-image
    runs-on: ubuntu-latest
    outputs:
      pairs: ${{ steps.config.outputs.pairs }}
      implementations: ${{ steps.config.outputs.implementations }}
    env:
      URL: https://github.com/mozilla/neqo
      ROLE: both
      IMAGE: ${{ needs.docker-image.outputs.imageID }}
    steps:
      - id: config
        run: |
          # Add neqo-latest and some non-default implementations.
          # tquic: https://github.com/quic-interop/quic-interop-runner/pull/385/files
          # openssl: https://github.com/quic-interop/quic-interop-runner/pull/402/files
          cat <<EOF > additional.json
          {
            "neqo-latest": {
              "image": "$IMAGE",
              "url": "$URL",
              "role": "$ROLE"
            },
            "tquic": {
              "image": "tquicgroup/tquic:latest",
              "url": "https://github.com/Tencent/tquic/",
              "role": "both"
            },
            "openssl": {
              "image": "quay.io/openssl-ci/openssl-quic-interop",
              "url": "https://github.com/openssl/openssl",
              "role": "client"
            }
          }
          EOF
          curl -o runner.json https://raw.githubusercontent.com/quic-interop/quic-interop-runner/master/implementations.json
          jq '. += input' runner.json additional.json > implementations.json
          {
            echo "implementations<<EOF"
            cat implementations.json
            echo "EOF"
          } >> "$GITHUB_OUTPUT"
          # Determine valid interop pairs that contain $LATEST
          jq < implementations.json "[
              [to_entries[] | select(.value.role==\"server\" or .value.role==\"both\").key] as \$servers |
              [to_entries[] | select(.value.role==\"client\" or .value.role==\"both\").key] as \$clients |
              \$clients[] as \$client |
              \$servers[] as \$server |
              \$client + \"$DELIM\" + \$server |
              select(contains(\"$LATEST\"))
            ]" > pairs.json
          {
            echo "pairs<<EOF"
            cat pairs.json
            echo "EOF"
          } >> "$GITHUB_OUTPUT"

  run-qns:
    name: Run QNS
    if: ${{ github.event_name != 'push' }}
    needs: implementations
    strategy:
      fail-fast: false
      matrix:
        pair: ${{ fromJson(needs.implementations.outputs.pairs) }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: '${{ env.LATEST }} Docker image'
          path: /tmp

      - run: docker load --input /tmp/${{ env.LATEST }}.tar

      - id: depair
        env:
          PAIR: ${{ matrix.pair }}
        run: |
          PAIR="${PAIR/$DELIM/%}"
          echo "client=$(echo "$PAIR" | cut -d% -f1)" >> "$GITHUB_OUTPUT"
          echo "server=$(echo "$PAIR" | cut -d% -f2)" >> "$GITHUB_OUTPUT"

      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: false

      # TODO: Replace once https://github.com/quic-interop/quic-interop-runner/pull/356 is merged.
      - uses: ./.github/actions/quic-interop-runner
        timeout-minutes: ${{ fromJSON(env.TIMEOUT) }}
        with:
          client: ${{ steps.depair.outputs.client }}
          server: ${{ steps.depair.outputs.server }}
          implementations: ${{ needs.implementations.outputs.implementations }}

  report:
    name: Report results
    if: ${{ !cancelled() && github.event_name != 'push' }}
    needs: [run-qns, implementations]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: false
      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: '*results'
          path: results

      - uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
        with:
          path: results-main
          key: qns-${{ github.sha }}
          restore-keys: qns-

      - run: sudo apt-get install -y --no-install-recommends wdiff
      - env:
          PAIRS: ${{ needs.implementations.outputs.pairs }}
        run: |
          # shellcheck disable=SC2153
          mapfile -t LIST < <(echo "$PAIRS" | jq '.[]' | sort)
          for PREFIX in "${LIST[@]}"; do
            PREFIX=$(echo "$PREFIX" | tr -d '"')
            CLIENT=$(echo "$PREFIX" | cut -f1 -d " ")
            SERVER=$(echo "$PREFIX" | cut -f3 -d " ")
            if [ "$CLIENT" == "$LATEST" ]; then
              ROLE=client
            else
              ROLE=server
            fi
            RUN="results/${PREFIX} results"
            PAIR="$CLIENT $DELIM $SERVER"
            if [ ! -e "$RUN/result.json" ]; then
              echo "* $PAIR: run cancelled after $TIMEOUT min" >> "$ROLE.failed.md"
              continue
            fi
            jq < "$RUN/result.json" '
                . as $data |
                .results[][].result //= "failed" |
                {
                  results: [.results[] | group_by(.result)[] | {(.[0].result): [.[] | .abbr]}] |
                  add
                } |
                . + {log_url: $data.log_url}
              ' > "$RUN/$ROLE.grouped.json"
            for ROLE in client server; do
              [ ! -e "$RUN/$ROLE.grouped.json" ] && continue
              for GROUP in $(jq -r < "$RUN/$ROLE.grouped.json" '.results | keys[]'); do
                RESULT=$(jq < "$RUN/$ROLE.grouped.json" -r '.results.'"$GROUP"'[]' | fmt -w 1000)
                LOG=$(jq -r < "$RUN/$ROLE.grouped.json" -r '.log_url')
                BASELINE=$(mktemp)
                if [ -e "results-main/${PREFIX} results/$ROLE.grouped.json" ]; then
                  jq < "results-main/${PREFIX} results/$ROLE.grouped.json" -r '.results.'"$GROUP"'[]' | fmt -w 1000 > "$BASELINE"
                else
                  touch "$BASELINE"
                fi
                [ -n "$RESULT" ] || continue
                DIFF=$(wdiff -n "$BASELINE" - <<< "$RESULT" || true)
                if [ "$GROUP" == "failed" ]; then
                  ADD=":warning:"
                  ADD_DELIM="\*\*"
                  DEL=":rocket:"
                  DEL_DELIM="~~"
                elif [ "$GROUP" == "succeeded" ]; then
                  ADD=":rocket:"
                  ADD_DELIM="~~"
                  DEL=":warning:"
                  DEL_DELIM="\*\*"
                else
                  ADD=""
                  ADD_DELIM=""
                  DEL=""
                  DEL_DELIM=""
                fi
                RESULT=$(echo "$DIFF" | sed -E "s/\[-/ $DEL$DEL_DELIM/g; s/-\]/$DEL_DELIM /g; s/\{\+/ $ADD$ADD_DELIM/g; s/\+\}/$ADD_DELIM /g")
                echo "* [$PAIR]($LOG): $RESULT" >> "$ROLE.$GROUP.md"
              done
            done
          done
          {
            echo "### Failed Interop Tests"
            if [ -e client.failed.md ] || [ -e server.failed.md ]; then
              echo -n "[QUIC Interop Runner](https://github.com/quic-interop/quic-interop-runner), *client* vs. *server*"
              SHA=$(cat results-main/baseline-sha.txt || true)
              if [ -n "$SHA" ]; then
                echo ", differences relative to $SHA."
              fi
              echo
              echo "#### $LATEST as client"
              cat client.failed.md || echo
              echo "#### $LATEST as server"
              cat server.failed.md || echo
            else
              echo -n "None "
              if [ -e "client.succeeded.md" ] || [ -e "server.succeeded.md" ] || [ -e "client.unsupported.md" ] || [ -e "server.unsupported.md" ]; then
                echo ":tada:"
              else
                echo ":question:"
              fi
            fi
            echo "<details><summary>All results</summary>"
            echo
            for GROUP in succeeded unsupported; do
              echo "### ${GROUP^} Interop Tests"
              if [ -e "client.$GROUP.md" ] || [ -e "server.$GROUP.md" ]; then
                echo "[QUIC Interop Runner](https://github.com/quic-interop/quic-interop-runner), *client* vs. *server*"
                echo "#### $LATEST as client"
                cat "client.$GROUP.md"
                echo "#### $LATEST as server"
                cat "server.$GROUP.md"
              else
                echo "None :question:"
              fi
            done
            echo
            echo "</details>"
          } >> comment.md

      - if: ${{ github.ref == 'refs/heads/main' }}
        run: |
          rm -rf results-main || true
          mv results results-main
          echo "${{ github.sha }}" > results-main/baseline-sha.txt

      - if: ${{ github.ref == 'refs/heads/main' }}
        uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
        with:
          path: results-main
          key: qns-${{ github.sha }}

      - uses: ./.github/actions/pr-comment-data-export
        with:
          name: ${{ github.workflow }}
          contents: comment.md