performance-monitoring.yml

# Based on Gen private website workflow
# https://github.com/probcomp/gen-website-private/blob/main/.github/workflows/publish_private_website_example.yml
name: Performance monitoring

on:
  push:
    branches:          [main, performance-monitoring]
  # Disable on other branches for now, because I don't know how to publish artifacts elsewhere.
  # pull_request:
  #   branches:          [main]

jobs:
  prepare:
    permissions:
      id-token: write
      contents: read
    runs-on: ubuntu-latest
    steps:
      # Thanks to https://stackoverflow.com/a/58035262
      - name: Determine branch
        shell: bash
        run: |
          branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
          echo "Running on branch ${branch:?}"
          echo "branch=${branch:?}" >> $GITHUB_OUTPUT
        id: extract_branch

      - uses: actions/checkout@v4
        with:
          fetch-depth: 1

      # Based on https://github.com/probcomp/gen-website-private/blob/807a8d0a912dd24f4b4bb7da2f8dc44c7227d39c/README.md#authentication-notes
      - name: Authenticate with Google Cloud
        uses: 'google-github-actions/auth@v2.1.3'
        with:
          project_id: 'probcomp-caliban'
          workload_identity_provider: 'projects/110275315150/locations/global/workloadIdentityPools/gen-website-private-publishers/providers/github'
          service_account: 'gen-website-private-admin@probcomp-caliban.iam.gserviceaccount.com'
          audience: '//iam.googleapis.com/projects/110275315150/locations/global/workloadIdentityPools/gen-website-private-publishers/providers/github'

      - name: Run benchmark
        run: |
          python -m venv venv
          source venv/bin/activate
          sudo apt-get -y install graphviz
          pip install --upgrade pip "setuptools>=62.4"
          make benchmark

      - name: Fetch historical data
        # Does not yet account for running on pull requests, where we might need to initialize the benchmark data
        # from main
        run: |
          BUCKET_PATH=gs://gen-website-private/gen.dev/genparse-performance-monitoring
          # The benchmarking action should tolerate a nonexistent data file, 
          # but fetch historical data if available.
          ([[ "${{ steps.extract_branch.outputs.branch }}" != main ]] && 
           (gcloud storage cp "${BUCKET_PATH:?}"/"${{ steps.extract_branch.outputs.branch }}"/data.js data.js || 
            echo "No historical data found for ${{ steps.extract_branch.outputs.branch }}, attempting to get main branch data.")
          ) ||
            gcloud storage cp "${BUCKET_PATH:?}"/main/data.js data.js ||
            gcloud storage cp "${BUCKET_PATH:?}"/data.js data.js ||
            echo "No historical data found, skipping."

      - name: Translate from data.js to benchmarks.json
        # Remove it if sed fails because sed failing indicates that data.js does not exist, hence we will want the
        # benchmarking action to generate benchmarks.json from scratch.
        run: sed -e 's/window.BENCHMARK_DATA = //' data.js > benchmarks.json || rm benchmarks.json

      - name: Translate to benchmark-action data file format
        uses: benchmark-action/github-action-benchmark@v1
        with:
          # For a full list of inputs, see:
          # https://github.com/benchmark-action/github-action-benchmark/tree/master?tab=readme-ov-file#action-inputs
          tool: 'pytest'
          output-file-path: output.json
          external-data-json-path: benchmarks.json
          # Unset because I think we don't need a GitHub token except to make comments.
          # github-token:
          comment-always: false
          alert-threshold: '200%'
          alert-comment-cc-users: '@timvieira,@benlebrun'
          comment-on-alert: false
          fail-on-alert: false
          fail-threshold: '300%'
          # If we find the chart gets too busy for us, we can set max-items-in-chart
          # max-items-in-chart: ~

      - name: Translate benchmark data back to data.js
        run: sed '1s/\(.*\)/window.BENCHMARK_DATA = \1/' benchmarks.json > data.js

      - name: Fetch preexisting data
        run: |
          BUCKET_PATH=gs://gen-website-private/gen.dev/genparse-performance-monitoring
          # for pull requests, use the branch name so the data are visualized at
          # genparse-performance-monitoring.gen.dev/BRANCH
          ROOT_WEBSITE_DIR=website
          mkdir -p "$ROOT_WEBSITE_DIR"
          
          gcloud storage rsync --recursive "${BUCKET_PATH:?}" "${ROOT_WEBSITE_DIR:?}" ||
            echo "No preexisting website data, skipping..."

      - name: Update relevant website data
        run: |
          # for pull requests, use the branch name so the data are visualized at
          # genparse-performance-monitoring.gen.dev/BRANCH
          WEBSITE_DIR=website
          if [[ "${{ steps.extract_branch.outputs.branch }}" != main ]]; then
            WEBSITE_DIR="${WEBSITE_DIR:?}"/"${{ steps.extract_branch.outputs.branch }}"
          fi
          echo "website_dir=${WEBSITE_DIR:?}" >> "$GITHUB_OUTPUT"
          
          echo "Generating website directory ${WEBSITE_DIR:?}"
          mkdir -p "${WEBSITE_DIR:?}"
          # Do `|| true` because otherwise this glob will list some directories.
          # This causes an error because we can't delete directories without -r.
          # However, we want to leave directories alone and delete only files,
          # so `|| true` should give the desired behavior.
          rm "$WEBSITE_DIR"/* || true  # Delete any existing data for this branch
          cp .github/workflows/performance-monitoring/index.html "${WEBSITE_DIR:?}"/index.html
          cp data.js "${WEBSITE_DIR:?}"/data.js
        id: update_website_data

      - name: Add timestamp and repo URL to index.html
        run: |
          WEBSITE_DIR="${{ steps.update_website_data.outputs.website_dir }}"
          echo "Generating website directory ${WEBSITE_DIR:?}"
          sed -i -e 's/\$TIMESTAMP/'"$(date -Iseconds)"'/' "${WEBSITE_DIR:?}"/index.html
          sed -i -e 's/\$REPO/'"${{ github.event.repository.name }}"'/' "${WEBSITE_DIR:?}"/index.html

      - name: Create website artifact
        uses: actions/upload-artifact@v4
        with:
          name: perfmon-website
          path: ./website

  # This should update data.js in the appropriate bucket location too,
  publish:
    permissions:
      id-token: write
      contents: read
    needs: prepare
    uses: probcomp/gen-website-private/.github/workflows/publish_private_website.yml@main
    with:
      artifact: perfmon-website
      parent_domain: gen.dev
      subdomain: genparse-performance-monitoring