ci: refactor build to improve build times (#827)

#### Motivation The deploy pipeline is currently taking 10+ minutes to complete this affects how quickly we can update the configuration of basemaps #### Modification reconfigure the deploy pipeline into multiple sub steps so it can be more efficient in deploying to nonprod/prod. #### Checklist _If not applicable, provide explanation of why._ - [ ] Tests updated - [ ] Docs updated - [ ] Issue linked in Title
linz · Dec 11, 2023 · 035c231 · 035c231
1 parent 5f4ef8b
commit 035c231
Show file tree

Hide file tree

Showing 4 changed files with 639 additions and 92 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -13,134 +13,177 @@ jobs:
       contents: read
       issues: write # This permission shouldn't needed as there is a bug currently for gh pr edit. https://github.com/cli/cli/issues/4631
       pull-requests: write
+
     outputs:
+      # location of the output configuration generally s3://linz-basemaps-staging/config/config-:hash.json.gz
       config_path: ${{ steps.path.outputs.config_path }}
+      # location of the output asset file, generally s3://linz-basemaps-staging/config/assets.tar.co
+      assets_path: ${{ steps.path.outputs.assets_path }}
+      # Hashed filename of the assets eg "assets-FgL4CU4wTd17Y4P7iEgMdG82Vgzj8nXJq12W9A2oPiVf.tar.co"
+      assets_hash: ${{ steps.path.outputs.assets_hash }}
+
     runs-on: ubuntu-latest
-    concurrency: deploy-${{ github.head_ref }}
-    env:
-      BASEMAPS_BUCKET: linz-basemaps
-      BASEMAPS_STAGING_BUCKET: linz-basemaps-staging
-      BASEMAPS_CONTAINER: ghcr.io/linz/basemaps/cli:v6.44.0-8-g7b293487
-      SCREENSHOT_CONTAINER: ghcr.io/linz/basemaps-screenshot/cli:v1
-      DOCKER_AWS_ENV: -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION -e AWS_DEFAULT_REGION
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      PR_NUMBER: ${{ github.event.number }}
+    concurrency: build-${{ github.head_ref }}
 
     steps:
       - uses: linz/action-typescript@v3
 
-      - name: (Build) Prepare path for glyphs and sprites
+      - name: Prepare path for glyphs
         run: |
           mkdir -p assets/fonts/
-          mkdir -p assets/sprites/
 
-      - name: (Build) Build Glyphs
+      - name: Build Glyphs
         uses: linz/action-build-pbf-glyphs@v1
         with:
           source: config/fonts/
           target: assets/fonts/
-          
-      - name: (Build) Build Sprites
+
+      - name: Build Sprites
         run: |
-          docker run -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} sprites --path $PWD/config/sprites/topographic/ --output $PWD/assets/sprites/
-        
-      - name: (Build) Bundle Assets Into Cotar
+          npx basemaps-sprites $PWD/config/sprites/topographic/
+          mkdir -p assets/sprites/
+          cp topographic* assets/sprites
+
+      - name: Bundle Assets Into Cotar
         run: |
-          docker run -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} bundle-assets --assets $PWD/assets/ --output $PWD/assets.tar.co
+          ./scripts/bmc.sh bundle-assets --assets $PWD/assets/ --output $PWD/assets.tar.co
           ASSETS_HASH=$(ls assets*.tar.co)
           echo "ASSETS_HASH=${ASSETS_HASH}" >> $GITHUB_ENV
-          echo "ASSETS_LOCATION_STAGING=s3://${BASEMAPS_STAGING_BUCKET}/assets/${ASSETS_HASH}" >> $GITHUB_ENV
-          echo "ASSETS_LOCATION=s3://${BASEMAPS_BUCKET}/assets/${ASSETS_HASH}" >> $GITHUB_ENV
+          echo "ASSETS_LOCATION_STAGING=s3://linz-basemaps-staging/assets/${ASSETS_HASH}" >> $GITHUB_ENV
 
-      - name: (Build) AWS Configure
+      - name: AWS Configure
         uses: aws-actions/[email protected]
         with:
           aws-region: ap-southeast-2
           mask-aws-account-id: true
           role-to-assume: ${{ secrets.AWS_ROLE_SCREENSHOT }}
 
-      - name: (NonProd) Bundle Config File
+      - name: Bundle Config File
         run: |
-          docker run -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} bundle --config $PWD/config --output $PWD/config-staging.json --assets ${ASSETS_LOCATION_STAGING}
+          ./scripts/bmc.sh bundle --config $PWD/config --output $PWD/config-staging.json --assets ${ASSETS_LOCATION_STAGING}
           CONFIG_HASH_STAGING=$(cat config-staging.json | jq .hash -r)
-          echo "CONFIG_LOCATION_STAGING=s3://${BASEMAPS_STAGING_BUCKET}/config/config-${CONFIG_HASH_STAGING}.json.gz" >> $GITHUB_ENV
-      
-      - name: (NonProd) Upload Hashed Assets
+          echo "CONFIG_LOCATION_STAGING=s3://linz-basemaps-staging/config/config-${CONFIG_HASH_STAGING}.json.gz" >> $GITHUB_ENV
+
+      - name: Upload Config & Assets
         run: |
           aws s3 cp ${ASSETS_HASH} ${ASSETS_LOCATION_STAGING}
 
-      - name: (NonProd) Upload Hashed Config File
-        run: |
           gzip -9 -k config-staging.json
           aws s3 cp config-staging.json.gz ${CONFIG_LOCATION_STAGING} --content-encoding gzip
 
       - id: path
         name: Define Outputs for Screenshots
         run: |
           echo "config_path=${CONFIG_LOCATION_STAGING}" >> "$GITHUB_OUTPUT"
+          echo "assets_path=${ASSETS_LOCATION_STAGING}" >> "$GITHUB_OUTPUT"
+          echo "assets_hash=${ASSETS_HASH}" >> "$GITHUB_OUTPUT"
+
+      - name: Compare To Production
+        run: |
+          aws s3 cp s3://linz-basemaps/config/config-latest.json.gz .
+          ./scripts/bmc.sh import --config ${CONFIG_LOCATION_STAGING} --output $PWD/changes.md --target $PWD/config-latest.json.gz
+
+      - name: (Pull Request) Update Description
+        if: github.ref != 'refs/heads/master' && github.event_name == 'pull_request'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          [ ! -f changes.md ] || gh pr comment ${{ github.event.number }} --body-file changes.md
+
+  # Compare and deploy to non-prod
+  deploy-nonprod:
+    needs: [build]
+    concurrency: deploy-nonprod-${{ github.head_ref }}
+
+    name: Deploy Nonprod
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    env:
+      CONFIG_PATH: ${{ needs.build.outputs.config_path }}
+      ASSETS_PATH: ${{ needs.build.outputs.assets_path }}
+
+    permissions:
+      id-token: write
+      contents: read
 
-      - name: (NonProd) AWS Configure
+    environment:
+      name: 'nonprod'
+      url: https://dev.basemaps.linz.govt.nz
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: AWS Configure
         uses: aws-actions/[email protected]
         with:
           aws-region: ap-southeast-2
           mask-aws-account-id: true
-          role-to-assume: ${{ secrets.AWS_ROLE_NON_PROD }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
 
-      - name: (NonProd - Config) Compare To DynamoDB
+      - name: Import config
         run: |
-          docker run -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} import --config ${CONFIG_LOCATION_STAGING} --output $PWD/changes.md
+          ./scripts/bmc.sh import --config ${CONFIG_PATH} --commit
 
-      - name: (NonProd - Config) Import Config To DynamoDB
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        run: |
-          docker run -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} import --config ${CONFIG_LOCATION_STAGING} --commit
+  # Compare and deploy to prod
+  deploy-prod:
+    needs: [build, deploy-nonprod]
+    concurrency: deploy-prod-${{ github.head_ref }}
+
+    name: Deploy Production
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    env:
+      CONFIG_PATH: ${{ needs.build.outputs.config_path }}
+      ASSETS_PATH: ${{ needs.build.outputs.assets_path }}
+      ASSETS_HASH: ${{ needs.build.outputs.assets_hash }}
+
+    permissions:
+      id-token: write
+      contents: read
+
+    environment:
+      name: 'prod'
+      url: https://basemaps.linz.govt.nz
 
-      - name: (Prod) AWS Configure
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: AWS Configure
         uses: aws-actions/[email protected]
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
         with:
           aws-region: ap-southeast-2
           mask-aws-account-id: true
-          role-to-assume: ${{ secrets.AWS_ROLE_PROD }}
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
 
-      - name: (Prod) Bundle Config File
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+      - name: Download Config
         run: |
-          docker run -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} bundle --config $PWD/config --output $PWD/config.json --assets ${ASSETS_LOCATION}
-          CONFIG_HASH=$(cat config.json | jq .hash -r)
-          echo "CONFIG_LOCATION=s3://${BASEMAPS_BUCKET}/config/config-${CONFIG_HASH}.json.gz" >> $GITHUB_ENV
+          aws s3 cp ${CONFIG_PATH} config-current.json.gz
+          aws s3 cp ${ASSETS_PATH} assets-current.tar.co
 
-      - name: (Prod - Config) Upload Hashed Assets
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+      - name: Update config to production URLs
         run: |
-          aws s3 cp ${ASSETS_HASH} ${ASSETS_LOCATION}
+          ASSETS_LOCATION_PROD=s3://linz-basemaps/assets/${ASSETS_HASH}
 
-      - name: (Prod - Config) Upload Hashed Config File
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        run: |
-          gzip -9 -k config.json
-          aws s3 cp config.json.gz ${CONFIG_LOCATION} --content-encoding gzip
+          echo ${ASSETS_LOCATION_PROD}
 
-      - name: (Prod - Config) Upload Latest Config File
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        run: |
-          aws s3 cp config.json.gz s3://${BASEMAPS_BUCKET}/config/config-latest.json.gz --content-encoding gzip
-  
-      - name: (Prod - Config) Compare to DynamoDB
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-        run: |
-          docker run -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} import --config ${CONFIG_LOCATION} --output $PWD/changes.md
+          # aws s3 cp assets-current.tar.co ${ASSETS_LOCATION_PROD}
+
+          ./scripts/bmc.sh bundle --config $PWD/config-current.json.gz --output $PWD/config-prod.json --assets ${ASSETS_LOCATION_PROD}
 
-      - name: (Prod - Config) Import Config To DynamoDB
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+          CONFIG_HASH_PROD=$(cat config-prod.json | jq .hash -r)
+          echo "CONFIG_LOCATION_PROD=s3://linz-basemaps/config/config-${CONFIG_HASH_PROD}.json.gz" >> $GITHUB_ENV
+
+      - name: Upload config
         run: |
-          docker run -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} import --config ${CONFIG_LOCATION} --commit
+          gzip -9 -k config-prod.json
 
-      - name: Update Description
-        if: github.ref != 'refs/heads/master' && github.event_name == 'pull_request'
+          # aws s3 cp config-prod.json.gz s3://linz-basemaps/config/config-latest.json.gz --content-encoding gzip
+          aws s3 cp config-prod.json.gz ${CONFIG_LOCATION_PROD} --content-encoding gzip
+
+      - name: Import config
         run: |
-          [ ! -f changes.md ] || gh pr comment ${PR_NUMBER} --body-file changes.md
-        
+          ./scripts/bmc.sh import --config ${CONFIG_LOCATION_PROD}
+
   screenshot:
     permissions:
       id-token: write
@@ -149,12 +192,13 @@ jobs:
     name: taking screenshots
     runs-on: ubuntu-latest
     env:
-      BASEMAPS_CONTAINER: ghcr.io/linz/basemaps/cli:v6.39.0-39-gdbedf1b1
       SCREENSHOT_CONTAINER: ghcr.io/linz/basemaps-screenshot/cli:v1
       DOCKER_AWS_ENV: -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e AWS_REGION -e AWS_DEFAULT_REGION
-      CONFIG_PATH: ${{needs.build.outputs.config_path}}
+      CONFIG_PATH: ${{ needs.build.outputs.config_path }}
 
     steps:
+      - uses: actions/checkout@v4
+
       - name: (Screenshot) AWS Configure
         uses: aws-actions/[email protected]
         with:
@@ -164,8 +208,8 @@ jobs:
 
       - name: (Screenshot) Screenshot Pull Request Changes
         run: |
-          docker run -p 5000:5000 -v $PWD:$PWD ${DOCKER_AWS_ENV} ${BASEMAPS_CONTAINER} serve --config ${CONFIG_PATH} &
-          
+          ./scripts/bmc.sh serve --config ${CONFIG_PATH} &
+
           # Wait for the server to start
           timeout 30 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:5000/v1/version)" !=  "200" ]]; do sleep 0.5; done' || false
 
@@ -199,5 +243,5 @@ jobs:
         id: visual-snapshots-diff
         uses: blacha/action-visual-snapshot@v2
         with:
-          storage-prefix: "s3://linz-basemaps-screenshot"
-          storage-url: "https://d25mfjh9syaxsr.cloudfront.net"
+          storage-prefix: 's3://linz-basemaps-screenshot'
+          storage-url: 'https://d25mfjh9syaxsr.cloudfront.net'