diff --git a/.github/workflows/dbt_base.yml b/.github/workflows/dbt_base.yml new file mode 100644 index 000000000..55edbde65 --- /dev/null +++ b/.github/workflows/dbt_base.yml @@ -0,0 +1,50 @@ +name: DBT Base Environment + +on: + push: + branches: + - main + +jobs: + build: + name: DBT Runner + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.10.x" + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Run DBT + run: | + dbt deps + dbt seed --target ${{ env.DBT_BASE_TARGET }} --target-path target-base + dbt run --target ${{ env.DBT_BASE_TARGET }} --target-path target-base + dbt docs generate --target ${{ env.DBT_BASE_TARGET }} --target-path target-base + env: + DBT_BASE_TARGET: "prod" + + - name: Package DBT artifacts + run: | + tar -czvf dbt-artifacts.tar.gz target-base ${{ env.DB_FILE }} + mv dbt-artifacts.tar.gz $GITHUB_WORKSPACE/${{ github.sha }}.tar.gz + env: + DB_FILE: "jaffle_shop.duckdb" + + - name: Upload to S3 + run: | + aws s3 cp $GITHUB_WORKSPACE/${{ github.sha }}.tar.gz s3://${{ env.AWS_S3_BUCKET }}/${{ github.sha }}.tar.gz + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} # Set these in your repository secrets + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # Set these in your repository secrets + AWS_REGION: ${{ secrets.AWS_REGION }} # Set these in your repository secrets + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} # Set these in your repository secrets diff --git a/.github/workflows/recce_ci.yml b/.github/workflows/recce_ci.yml new file mode 100644 index 000000000..e0fa405b0 --- /dev/null +++ b/.github/workflows/recce_ci.yml @@ -0,0 +1,74 @@ +name: Jaffle Shop Recce CI + +on: + pull_request: + branches: [main] + +jobs: + check-pull-request: + name: Check pull request by Recce CI + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10.x" + - name: Install dependencies + run: | + pip install -r requirements.txt + - name: Install Recce - Nightly + run: | + pip install recce-nightly + - name: Prepare dbt Base environment + run: | + if aws s3 cp s3://$AWS_S3_BUCKET/${{ github.event.pull_request.base.sha }}.tar.gz .; then + echo "Base environment found in S3" + tar -xvf ${{ github.event.pull_request.base.sha }}.tar.gz + else + echo "Base environment not found in S3. Running dbt to create base environment" + git checkout ${{ github.event.pull_request.base.sha }} + dbt deps + dbt seed --target ${{ env.DBT_BASE_TARGET }} --target-path target-base + dbt run --target ${{ env.DBT_BASE_TARGET }} --target-path target-base + dbt docs generate --target ${{ env.DBT_BASE_TARGET }} --target-path target-base + fi + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + DBT_BASE_TARGET: "prod" + + - name: Prepare dbt Current environment + run: | + git checkout ${{ github.event.pull_request.head.sha }} + dbt deps + dbt seed --target ${{ env.DBT_CURRENT_TARGET}} + dbt run --target ${{ env.DBT_CURRENT_TARGET}} + dbt docs generate --target ${{ env.DBT_CURRENT_TARGET}} + env: + DBT_CURRENT_TARGET: "dev" + + - name: Run Recce CI + run: | + recce run --github-pull-request-url ${{ github.event.pull_request.html_url }} + + - name: Archive Recce State File + uses: actions/upload-artifact@v4 + id: recce-artifact-uploader + with: + name: recce-state-file + path: recce_state.json + + - name: Comment on pull request + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + Recce `run` successfully completed. + Please download the [artifact](${{ env.ARTIFACT_URL }}) for the state file. + env: + ARTIFACT_URL: ${{ steps.recce-artifact-uploader.outputs.artifact-url }}