ci: refactor build to improve build times #2455
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| workflow_call: | |
| pull_request: | |
| push: | |
| branches: | |
| - master | |
| jobs: | |
| build: | |
| permissions: | |
| id-token: write | |
| contents: read | |
| issues: write # This permission shouldn't needed as there is a bug currently for gh pr edit. https://github.com/cli/cli/issues/4631 | |
| pull-requests: write | |
| outputs: | |
| # location of the output configuration generally s3://linz-basemaps-staging/config/config-:hash.json.gz | |
| config_path: ${{ steps.path.outputs.config_path }} | |
| # location of the output asset file, generally s3://linz-basemaps-staging/config/assets.tar.co | |
| assets_path: ${{ steps.path.outputs.assets_path }} | |
| # Hashed filename of the assets eg "assets-FgL4CU4wTd17Y4P7iEgMdG82Vgzj8nXJq12W9A2oPiVf.tar.co" | |
| assets_hash: ${{ steps.path.outputs.assets_hash }} | |
| runs-on: ubuntu-latest | |
| concurrency: build-${{ github.head_ref }} | |
| steps: | |
| - uses: linz/action-typescript@v3 | |
| - name: Prepare path for glyphs | |
| run: | | |
| mkdir -p assets/fonts/ | |
| - name: Build Glyphs | |
| uses: linz/action-build-pbf-glyphs@v1 | |
| with: | |
| source: config/fonts/ | |
| target: assets/fonts/ | |
| - name: Build Sprites | |
| run: | | |
| npx basemaps-sprites $PWD/config/sprites/topographic/ | |
| mkdir -p assets/sprites/ | |
| cp topographic* assets/sprites | |
| - name: Bundle Assets Into Cotar | |
| run: | | |
| ./scripts/bmc.sh bundle-assets --assets $PWD/assets/ --output $PWD/assets.tar.co | |
| ASSETS_HASH=$(ls assets*.tar.co) | |
| echo "ASSETS_HASH=${ASSETS_HASH}" >> $GITHUB_ENV | |
| echo "ASSETS_LOCATION_STAGING=s3://linz-basemaps-staging/assets/${ASSETS_HASH}" >> $GITHUB_ENV | |
| - name: AWS Configure | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: ap-southeast-2 | |
| mask-aws-account-id: true | |
| role-to-assume: ${{ secrets.AWS_ROLE_SCREENSHOT }} | |
| - name: Bundle Config File | |
| run: | | |
| ./scripts/bmc.sh bundle --config $PWD/config --output $PWD/config-staging.json --assets ${ASSETS_LOCATION_STAGING} | |
| CONFIG_HASH_STAGING=$(cat config-staging.json | jq .hash -r) | |
| echo "CONFIG_LOCATION_STAGING=s3://linz-basemaps-staging/config/config-${CONFIG_HASH_STAGING}.json.gz" >> $GITHUB_ENV | |
| - name: Upload Config & Assets | |
| run: | | |
| aws s3 cp ${ASSETS_HASH} ${ASSETS_LOCATION_STAGING} | |
| gzip -9 -k config-staging.json | |
| aws s3 cp config-staging.json.gz ${CONFIG_LOCATION_STAGING} --content-encoding gzip | |
| - id: path | |
| name: Define Outputs for Screenshots | |
| run: | | |
| echo "config_path=${CONFIG_LOCATION_STAGING}" >> "$GITHUB_OUTPUT" | |
| echo "assets_path=${ASSETS_LOCATION_STAGING}" >> "$GITHUB_OUTPUT" | |
| echo "assets_hash=${ASSETS_HASH}" >> "$GITHUB_OUTPUT" | |
| # TODO just diff the json config files | |
| - name: Compare To Production | |
| run: | | |
| aws s3 cp s3://linz-basemaps/config/config-latest.json.gz . | |
| ./scripts/bmc.sh import --config ${CONFIG_LOCATION_STAGING} --output $PWD/changes.md --target $PWD/config-latest.json.gz | |
| # - name: (Pull Request) Update Description | |
| # if: github.ref != 'refs/heads/master' && github.event_name == 'pull_request' | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # run: | | |
| # [ ! -f changes.md ] || gh pr comment ${{ github.event.number }} --body-file changes.md | |
| # Compare and deploy to non-prod | |
| deploy-nonprod: | |
| needs: [build] | |
| concurrency: deploy-nonprod-${{ github.head_ref }} | |
| name: Deploy Nonprod | |
| # if: github.ref == 'refs/heads/master' && github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| env: | |
| CONFIG_PATH: ${{ needs.build.outputs.config_path }} | |
| ASSETS_PATH: ${{ needs.build.outputs.assets_path }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: | |
| name: 'nonprod' | |
| url: https://dev.basemaps.linz.govt.nz | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: AWS Configure | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: ap-southeast-2 | |
| mask-aws-account-id: true | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| - name: Import config | |
| run: | | |
| ./scripts/bmc.sh import --config ${CONFIG_PATH}--commit | |
| # Compare and deploy to non-prod | |
| deploy-prod: | |
| needs: [build, deploy-nonprod] | |
| concurrency: deploy-prod-${{ github.head_ref }} | |
| name: Deploy Production | |
| if: github.ref == 'refs/heads/master' && github.event_name == 'push' | |
| runs-on: ubuntu-latest | |
| env: | |
| CONFIG_PATH: ${{ needs.build.outputs.config_path }} | |
| ASSETS_PATH: ${{ needs.build.outputs.assets_path }} | |
| ASSETS_HASH: ${{ needs.build.outputs.assets_hash }} | |
| permissions: | |
| id-token: write | |
| contents: read | |
| environment: | |
| name: 'prod' | |
| url: https://basemaps.linz.govt.nz | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: AWS Configure | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: ap-southeast-2 | |
| mask-aws-account-id: true | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN }} | |
| - name: Download Config | |
| run: | | |
| aws s3 cp ${CONFIG_PATH} config-current.json.gz | |
| aws s3 cp ${ASSETS_PATH} assets-current.tar.co | |
| - name: Update config to production URLs | |
| run: | | |
| echo "ASSETS_LOCATION_PROD=s3://linz-basemaps/assets/${ASSETS_HASH}" >> "${GITHUB_ENV}" | |
| aws s3 sync assets-current.tar.co ${ASSETS_LOCATION_PROD} | |
| ./scripts/bmc.sh bundle --config $PWD/config-current.json.gz --output $PWD/config-prod.json --assets ${ASSETS_LOCATION_PROD} | |
| CONFIG_HASH_PROD=$(cat config-prod.json | jq .hash -r) | |
| echo "CONFIG_LOCATION_PROD=s3://linz-basemaps/config/config-${CONFIG_HASH_PROD}.json.gz" >> $GITHUB_ENV | |
| - name: Upload config | |
| run: | | |
| gzip -9 -k config-prod.json | |
| aws s3 cp config-prod.json.gz s3://linz-basemaps/config/config-latest.json.gz --content-encoding gzip | |
| aws s3 cp config-prod.json.gz ${CONFIG_LOCATION_PROD} --content-encoding gzip | |
| - name: Import config | |
| run: | | |
| ./scripts/bmc.sh import --config ${CONFIG_LOCATION_PROD} --commit | |
| screenshot: | |
| permissions: | |
| id-token: write | |
| contents: read | |
| needs: [build] | |
| name: taking screenshots | |
| runs-on: ubuntu-latest | |
| env: | |
| SCREENSHOT_CONTAINER: ghcr.io/linz/basemaps-screenshot/cli:v1 | |
| DOCKER_AWS_ENV: -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION -e AWS_DEFAULT_REGION | |
| CONFIG_PATH: ${{ needs.build.outputs.config_path }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: (Screenshot) AWS Configure | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: ap-southeast-2 | |
| mask-aws-account-id: true | |
| role-to-assume: ${{ secrets.AWS_ROLE_SCREENSHOT }} | |
| - name: (Screenshot) Screenshot Pull Request Changes | |
| run: | | |
| ./scripts/bmc.sh serve --config ${CONFIG_PATH} & | |
| # Wait for the server to start | |
| timeout 30 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:5000/v1/version)" != "200" ]]; do sleep 0.5; done' || false | |
| docker run --rm --network="host" -v $PWD:$PWD ${DOCKER_AWS_ENV} ${SCREENSHOT_CONTAINER} --url http://localhost:5000 --output $PWD/.artifacts/visual-snapshots | |
| - name: Save snapshots | |
| uses: getsentry/action-visual-snapshot@v2 | |
| with: | |
| save-only: true | |
| snapshot-path: .artifacts/visual-snapshots | |
| visual-diff: | |
| permissions: | |
| id-token: write | |
| contents: write | |
| checks: write | |
| needs: [build, screenshot] | |
| name: validate screenshots | |
| if: github.ref != 'refs/heads/master' && github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - name: AWS Configure | |
| uses: aws-actions/[email protected] | |
| with: | |
| aws-region: ap-southeast-2 | |
| mask-aws-account-id: true | |
| role-to-assume: ${{ secrets.AWS_ROLE_SCREENSHOT }} | |
| - name: Diff snapshots | |
| id: visual-snapshots-diff | |
| uses: blacha/action-visual-snapshot@v2 | |
| with: | |
| storage-prefix: 's3://linz-basemaps-screenshot' | |
| storage-url: 'https://d25mfjh9syaxsr.cloudfront.net' |