Merge pull request #18 from DataRecce/chore/add-the-workflow-for-recc…

…-cloud [Chore] CI/CD for recce cloud compatible output
DataRecce · Jun 5, 2024 · acdec80 · acdec80
2 parents 17dcbc9 + 8b9f22f
commit acdec80
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 42 deletions.
diff --git a/.github/workflows/dbt_base.yml b/.github/workflows/dbt_base.yml
@@ -1,16 +1,13 @@
 name: DBT Base Environment
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
-
 jobs:
   build:
-    name: DBT Runner
     runs-on: ubuntu-latest
-    permissions:
-      pull-requests: write
 
     steps:
       - uses: actions/checkout@v3
@@ -27,24 +24,18 @@ jobs:
       - name: Run DBT
         run: |
           dbt deps
-          dbt seed --target ${{ env.DBT_BASE_TARGET }} --target-path target-base
-          dbt run --target ${{ env.DBT_BASE_TARGET }} --target-path target-base
-          dbt docs generate --target ${{ env.DBT_BASE_TARGET }} --target-path target-base
+          dbt seed --target ${{ env.DBT_BASE_TARGET }}
+          dbt run --target ${{ env.DBT_BASE_TARGET }}
+          dbt docs generate --target ${{ env.DBT_BASE_TARGET }}
         env:
           DBT_BASE_TARGET: "prod"
 
-      - name: Package DBT artifacts
-        run: |
-          tar -czvf dbt-artifacts.tar.gz target-base ${{ env.DB_FILE }}
-          mv dbt-artifacts.tar.gz $GITHUB_WORKSPACE/${{ github.sha }}.tar.gz
-        env:
-          DB_FILE: "jaffle_shop.duckdb"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: duckdb
+          path: jaffle_shop.duckdb
 
-      - name: Upload to S3
-        run: |
-          aws s3 cp $GITHUB_WORKSPACE/${{ github.sha }}.tar.gz s3://${{ env.AWS_S3_BUCKET }}/${{ github.sha }}.tar.gz
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} # Set these in your repository secrets
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # Set these in your repository secrets
-          AWS_REGION: ${{ secrets.AWS_REGION }} # Set these in your repository secrets
-          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} # Set these in your repository secrets
+      - uses: actions/upload-artifact@v4
+        with:
+          name: target
+          path: target/
diff --git a/.github/workflows/recce_ci.yml b/.github/workflows/recce_ci.yml
@@ -22,30 +22,18 @@ jobs:
           pip install -r requirements.txt
       - name: Install Recce
         run: |
-          pip install recce-nightly
-      - name: Add packages.yml file
-        run: |
-          echo '${{ vars.PACKAGES_YAML }}' > packages.yml
+          pip install recce
       - name: Prepare dbt Base environment
         run: |
-          if aws s3 cp s3://$AWS_S3_BUCKET/${{ github.event.pull_request.base.sha }}.tar.gz .; then
-            echo "Base environment found in S3"
-            tar -xvf ${{ github.event.pull_request.base.sha }}.tar.gz
-          else
-            echo "Base environment not found in S3. Running dbt to create base environment"
-            git checkout ${{ github.event.pull_request.base.sha }}
-            dbt deps
-            dbt seed --target ${{ env.DBT_BASE_TARGET }} --target-path target-base
-            dbt run --target ${{ env.DBT_BASE_TARGET }} --target-path target-base
-            dbt docs generate --target ${{ env.DBT_BASE_TARGET }} --target-path target-base  
-          fi
+          gh repo set-default ${{ github.repository }}
+          base_branch=${{ github.base_ref }}
+          run_id=$(gh run list --workflow ${WORKFLOW_BASE} --branch ${base_branch} --status success --limit 1 --json databaseId --jq '.[0].databaseId')
+          echo "Download artifacts from run $run_id"
+          gh run download ${run_id} -n target -D target-base
+          gh run download ${run_id} -n duckdb
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          AWS_REGION: ${{ secrets.AWS_REGION }}
-          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
-          DBT_BASE_TARGET: "prod"
-
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WORKFLOW_BASE: ".github/workflows/dbt_base.yml"
       - name: Prepare dbt Current environment
         run: |
           git checkout ${{ github.event.pull_request.head.sha }}