diff --git a/.github/workflows/dbt_base.yml b/.github/workflows/dbt_base.yml index 55edbde65..cf68d5650 100644 --- a/.github/workflows/dbt_base.yml +++ b/.github/workflows/dbt_base.yml @@ -1,16 +1,13 @@ name: DBT Base Environment on: + workflow_dispatch: push: branches: - main - jobs: build: - name: DBT Runner runs-on: ubuntu-latest - permissions: - pull-requests: write steps: - uses: actions/checkout@v3 @@ -27,24 +24,18 @@ jobs: - name: Run DBT run: | dbt deps - dbt seed --target ${{ env.DBT_BASE_TARGET }} --target-path target-base - dbt run --target ${{ env.DBT_BASE_TARGET }} --target-path target-base - dbt docs generate --target ${{ env.DBT_BASE_TARGET }} --target-path target-base + dbt seed --target ${{ env.DBT_BASE_TARGET }} + dbt run --target ${{ env.DBT_BASE_TARGET }} + dbt docs generate --target ${{ env.DBT_BASE_TARGET }} env: DBT_BASE_TARGET: "prod" - - name: Package DBT artifacts - run: | - tar -czvf dbt-artifacts.tar.gz target-base ${{ env.DB_FILE }} - mv dbt-artifacts.tar.gz $GITHUB_WORKSPACE/${{ github.sha }}.tar.gz - env: - DB_FILE: "jaffle_shop.duckdb" + - uses: actions/upload-artifact@v4 + with: + name: duckdb + path: jaffle_shop.duckdb - - name: Upload to S3 - run: | - aws s3 cp $GITHUB_WORKSPACE/${{ github.sha }}.tar.gz s3://${{ env.AWS_S3_BUCKET }}/${{ github.sha }}.tar.gz - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} # Set these in your repository secrets - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # Set these in your repository secrets - AWS_REGION: ${{ secrets.AWS_REGION }} # Set these in your repository secrets - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} # Set these in your repository secrets + - uses: actions/upload-artifact@v4 + with: + name: target + path: target/ diff --git a/.github/workflows/recce_ci.yml b/.github/workflows/recce_ci.yml index 6c938025e..74ee87580 100644 --- a/.github/workflows/recce_ci.yml +++ b/.github/workflows/recce_ci.yml @@ -22,30 +22,18 @@ jobs: pip install -r requirements.txt - name: Install Recce run: | - pip install recce-nightly - - name: Add packages.yml file - run: | - echo '${{ vars.PACKAGES_YAML }}' > packages.yml + pip install recce - name: Prepare dbt Base environment run: | - if aws s3 cp s3://$AWS_S3_BUCKET/${{ github.event.pull_request.base.sha }}.tar.gz .; then - echo "Base environment found in S3" - tar -xvf ${{ github.event.pull_request.base.sha }}.tar.gz - else - echo "Base environment not found in S3. Running dbt to create base environment" - git checkout ${{ github.event.pull_request.base.sha }} - dbt deps - dbt seed --target ${{ env.DBT_BASE_TARGET }} --target-path target-base - dbt run --target ${{ env.DBT_BASE_TARGET }} --target-path target-base - dbt docs generate --target ${{ env.DBT_BASE_TARGET }} --target-path target-base - fi + gh repo set-default ${{ github.repository }} + base_branch=${{ github.base_ref }} + run_id=$(gh run list --workflow ${WORKFLOW_BASE} --branch ${base_branch} --status success --limit 1 --json databaseId --jq '.[0].databaseId') + echo "Download artifacts from run $run_id" + gh run download ${run_id} -n target -D target-base + gh run download ${run_id} -n duckdb env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} - AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} - DBT_BASE_TARGET: "prod" - + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WORKFLOW_BASE: ".github/workflows/dbt_base.yml" - name: Prepare dbt Current environment run: | git checkout ${{ github.event.pull_request.head.sha }}