diff --git a/.github/workflows/dbt_alter_gha_tasks.yml b/.github/workflows/dbt_alter_gha_tasks.yml index d138700c1..4655e1565 100644 --- a/.github/workflows/dbt_alter_gha_tasks.yml +++ b/.github/workflows/dbt_alter_gha_tasks.yml @@ -36,7 +36,8 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + uses: | + FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 with: workflow_name: | ${{ inputs.workflow_name }} diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 385051676..fbfa6f627 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -24,5 +24,6 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main + uses: | + FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 8e17661b7..2ede01f16 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -28,16 +28,16 @@ on: required: true env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" WAREHOUSE: "${{ inputs.warehouse }}" - SCHEMA: "${{ secrets.SCHEMA }}" + SCHEMA: "${{ vars.SCHEMA }}" jobs: @@ -52,13 +52,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_batch_backfill.yml b/.github/workflows/dbt_run_batch_backfill.yml index 5b00b75ab..ae74bfac8 100644 --- a/.github/workflows/dbt_run_batch_backfill.yml +++ b/.github/workflows/dbt_run_batch_backfill.yml @@ -8,16 +8,16 @@ on: - cron: '*/7 * * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" WAREHOUSE: DBT_EMERGENCY - SCHEMA: "${{ secrets.SCHEMA }}" + SCHEMA: "${{ vars.SCHEMA }}" DBT_IS_BATCH_LOAD: true concurrency: @@ -32,13 +32,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index 4b1205205..726685b62 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -7,16 +7,16 @@ on: - cron: '35 0 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -30,13 +30,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_deploy_udfs_and_sps.yml b/.github/workflows/dbt_run_deploy_udfs_and_sps.yml index a4bee4cb6..633eafa13 100644 --- a/.github/workflows/dbt_run_deploy_udfs_and_sps.yml +++ b/.github/workflows/dbt_run_deploy_udfs_and_sps.yml @@ -9,16 +9,16 @@ on: - 'macros/**' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -39,7 +39,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index fa531adce..401e468c3 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -38,7 +38,8 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@main + uses: | + FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: | ${{ inputs.dbt_command }} diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 5bc08d20a..5d892ee36 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -8,16 +8,16 @@ on: - cron: '0 6 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -31,13 +31,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index 52e18c6a5..8cadfd74a 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -7,16 +7,16 @@ on: - "main" env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -30,13 +30,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_incremental_non_core.yml b/.github/workflows/dbt_run_incremental_non_core.yml index ac10f8082..67768658d 100644 --- a/.github/workflows/dbt_run_incremental_non_core.yml +++ b/.github/workflows/dbt_run_incremental_non_core.yml @@ -8,16 +8,16 @@ on: # - cron: "20,50 * * * *" env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -31,13 +31,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_macro_get_block_production.yml b/.github/workflows/dbt_run_macro_get_block_production.yml index f1539ab9c..4194c49c4 100644 --- a/.github/workflows/dbt_run_macro_get_block_production.yml +++ b/.github/workflows/dbt_run_macro_get_block_production.yml @@ -7,16 +7,16 @@ on: - cron: '0 */4 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -30,13 +30,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_macro_get_token_data.yml b/.github/workflows/dbt_run_macro_get_token_data.yml index d47eab7a8..66b1d5240 100644 --- a/.github/workflows/dbt_run_macro_get_token_data.yml +++ b/.github/workflows/dbt_run_macro_get_token_data.yml @@ -7,16 +7,16 @@ on: - cron: '*/2 * * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -30,13 +30,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_parser_program.yml b/.github/workflows/dbt_run_parser_program.yml deleted file mode 100644 index cdd1f2072..000000000 --- a/.github/workflows/dbt_run_parser_program.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_parser_program -run-name: dbt_run_parser_program - -on: - schedule: - # Runs every 30 mins (see https://crontab.guru) - - cron: '*/30 * * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s models/streamline/parser/streamline__complete_decoded_instructions.sql - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m ./models/streamline/parser/streamline__all_undecoded_instructions_two_days.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_parser_program_historical.yml b/.github/workflows/dbt_run_parser_program_historical.yml deleted file mode 100644 index eb755aaf8..000000000 --- a/.github/workflows/dbt_run_parser_program_historical.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: dbt_run_parser_program_historical -run-name: dbt_run_parser_program_historical - -on: - workflow_dispatch: - schedule: - # Runs every 30 mins at XX:10, XX:40 (see https://crontab.guru) - - cron: '10,40 * * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s models/streamline/parser/streamline__complete_decoded_instructions.sql - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/streamline/parser/history/streamline__all_undecoded_instructions_history.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_parser_program_historical_build_queue.yml b/.github/workflows/dbt_run_parser_program_historical_build_queue.yml deleted file mode 100644 index b93fbc087..000000000 --- a/.github/workflows/dbt_run_parser_program_historical_build_queue.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_parser_program_historical_build_queue -run-name: dbt_run_parser_program_historical_build_queue - -on: - workflow_dispatch: - schedule: - # Runs every SUNDAY at 22:22 (see https://crontab.guru) - - cron: '22 2 * * SUN' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s models/streamline/parser/history/streamline__all_undecoded_instructions_history_queue.sql diff --git a/.github/workflows/dbt_run_parser_program_historical_clean_queue.yml b/.github/workflows/dbt_run_parser_program_historical_clean_queue.yml deleted file mode 100644 index 2b83b5ef8..000000000 --- a/.github/workflows/dbt_run_parser_program_historical_clean_queue.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: dbt_run_parser_program_historical_clean_queue -run-name: dbt_run_parser_program_historical_clean_queue - -on: - schedule: - # Runs daily at 03:17 (see https://crontab.guru) - - cron: '17 3 * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run-operation run_sp_clean_program_parser_historical_queue \ No newline at end of file diff --git a/.github/workflows/dbt_run_parser_program_retry.yml b/.github/workflows/dbt_run_parser_program_retry.yml deleted file mode 100644 index 745b7bc34..000000000 --- a/.github/workflows/dbt_run_parser_program_retry.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_parser_program_retry -run-name: dbt_run_parser_program_retry - -on: - workflow_dispatch: - schedule: - # Runs every day (see https://crontab.guru) - - cron: '0 2 * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m ./models/streamline/parser/streamline__retry_error_instructions.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_parser_program_unknown.yml b/.github/workflows/dbt_run_parser_program_unknown.yml deleted file mode 100644 index 83cd54d47..000000000 --- a/.github/workflows/dbt_run_parser_program_unknown.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_parser_program_unknown -run-name: dbt_run_parser_program_unknown - -on: - workflow_dispatch: - schedule: - # Runs every day (see https://crontab.guru) - - cron: '0 2 * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m ./models/streamline/parser/streamline__unknown_error_instructions.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_solscan_blocks.yml b/.github/workflows/dbt_run_solscan_blocks.yml index f1c63b79a..9a2896502 100644 --- a/.github/workflows/dbt_run_solscan_blocks.yml +++ b/.github/workflows/dbt_run_solscan_blocks.yml @@ -8,16 +8,16 @@ on: - cron: '*/20 * * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -31,13 +31,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_streamline_get_blocks_real_time.yml b/.github/workflows/dbt_run_streamline_get_blocks_real_time.yml deleted file mode 100644 index 11101578b..000000000 --- a/.github/workflows/dbt_run_streamline_get_blocks_real_time.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: dbt_run_streamline_get_blocks_real_time -run-name: dbt_run_streamline_get_blocks_real_time - -on: - workflow_dispatch: - schedule: - # Runs "every 15mins" (see https://crontab.guru) - - cron: '*/15 * * * *' - -env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" - - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: - group: 'Default Larger Runners' - labels: ubuntu-latest-4-cores - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run -s models/bronze/bronze__blocks.sql - diff --git a/.github/workflows/dbt_run_weekly.yml b/.github/workflows/dbt_run_weekly.yml index 8729165b4..a35780d6c 100644 --- a/.github/workflows/dbt_run_weekly.yml +++ b/.github/workflows/dbt_run_weekly.yml @@ -7,16 +7,16 @@ on: - cron: '0 1 * * 0' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -30,13 +30,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_decode_instructions.yml b/.github/workflows/decoder/dbt_run_decode_instructions.yml similarity index 100% rename from .github/workflows/dbt_run_decode_instructions.yml rename to .github/workflows/decoder/dbt_run_decode_instructions.yml diff --git a/.github/workflows/dbt_run_decode_instructions_backfill.yml b/.github/workflows/decoder/dbt_run_decode_instructions_backfill.yml similarity index 100% rename from .github/workflows/dbt_run_decode_instructions_backfill.yml rename to .github/workflows/decoder/dbt_run_decode_instructions_backfill.yml diff --git a/.github/workflows/dbt_run_idls_history.yml b/.github/workflows/decoder/dbt_run_idls_history.yml similarity index 58% rename from .github/workflows/dbt_run_idls_history.yml rename to .github/workflows/decoder/dbt_run_idls_history.yml index 950990815..c760209ac 100644 --- a/.github/workflows/dbt_run_idls_history.yml +++ b/.github/workflows/decoder/dbt_run_idls_history.yml @@ -7,16 +7,16 @@ on: - cron: '15 */12 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -30,13 +30,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_get_nft_compressed_backfill.yml b/.github/workflows/nft/dbt_run_get_nft_compressed_backfill.yml similarity index 66% rename from .github/workflows/dbt_run_get_nft_compressed_backfill.yml rename to .github/workflows/nft/dbt_run_get_nft_compressed_backfill.yml index 9b6a663ff..1b545efc4 100644 --- a/.github/workflows/dbt_run_get_nft_compressed_backfill.yml +++ b/.github/workflows/nft/dbt_run_get_nft_compressed_backfill.yml @@ -7,16 +7,16 @@ on: - "main" env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -30,13 +30,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_helius_metadata.yml b/.github/workflows/nft/dbt_run_helius_metadata.yml similarity index 60% rename from .github/workflows/dbt_run_helius_metadata.yml rename to .github/workflows/nft/dbt_run_helius_metadata.yml index d9a7b0bb0..5d4056473 100644 --- a/.github/workflows/dbt_run_helius_metadata.yml +++ b/.github/workflows/nft/dbt_run_helius_metadata.yml @@ -8,16 +8,16 @@ on: - cron: '0 3,15 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -31,13 +31,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_macro_get_compressed_nft.yml b/.github/workflows/nft/dbt_run_macro_get_compressed_nft.yml similarity index 60% rename from .github/workflows/dbt_run_macro_get_compressed_nft.yml rename to .github/workflows/nft/dbt_run_macro_get_compressed_nft.yml index 42a6fc8e7..cdc6b9ed6 100644 --- a/.github/workflows/dbt_run_macro_get_compressed_nft.yml +++ b/.github/workflows/nft/dbt_run_macro_get_compressed_nft.yml @@ -7,16 +7,16 @@ on: - cron: '30 2 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -30,13 +30,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/tests/dbt_run_full_observability.yml similarity index 89% rename from .github/workflows/dbt_run_full_observability.yml rename to .github/workflows/tests/dbt_run_full_observability.yml index 018c83d5c..16ed3f756 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/tests/dbt_run_full_observability.yml @@ -31,9 +31,10 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/tests/dbt_test.yml similarity index 54% rename from .github/workflows/dbt_test.yml rename to .github/workflows/tests/dbt_test.yml index 71e04bcb0..850c02ed2 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/tests/dbt_test.yml @@ -8,16 +8,16 @@ on: - cron: '0 9 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -31,13 +31,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_test_hourly.yml b/.github/workflows/tests/dbt_test_hourly.yml similarity index 56% rename from .github/workflows/dbt_test_hourly.yml rename to .github/workflows/tests/dbt_test_hourly.yml index 621b75bc1..a0f2373d3 100644 --- a/.github/workflows/dbt_test_hourly.yml +++ b/.github/workflows/tests/dbt_test_hourly.yml @@ -8,16 +8,16 @@ on: - cron: '20 */1 * * *' env: - DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - ACCOUNT: "${{ secrets.ACCOUNT }}" - ROLE: "${{ secrets.ROLE }}" - USER: "${{ secrets.USER }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ secrets.REGION }}" - DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" - SCHEMA: "${{ secrets.SCHEMA }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" concurrency: group: ${{ github.workflow }} @@ -31,13 +31,14 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/tests/dbt_test_intraday.yml similarity index 89% rename from .github/workflows/dbt_test_intraday.yml rename to .github/workflows/tests/dbt_test_intraday.yml index 5ef04ee4a..fe0e4282f 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/tests/dbt_test_intraday.yml @@ -31,9 +31,10 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" - name: install dependencies run: | diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/tests/dbt_test_tasks.yml similarity index 79% rename from .github/workflows/dbt_test_tasks.yml rename to .github/workflows/tests/dbt_test_tasks.yml index 976440bd6..a30043001 100644 --- a/.github/workflows/dbt_test_tasks.yml +++ b/.github/workflows/tests/dbt_test_tasks.yml @@ -23,5 +23,6 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main + uses: | + FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 secrets: inherit \ No newline at end of file diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 000000000..5dc1fa1f8 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,16 @@ +packages: +- package: calogica/dbt_expectations + version: 0.8.2 +- package: dbt-labs/dbt_external_tables + version: 0.8.2 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: c84d623aa09ce2acb9451e6aedf5fa70c19b0b95 +- package: get-select/dbt_snowflake_query_tags + version: 2.3.1 +- package: calogica/dbt_date + version: 0.7.2 +- git: https://github.com/FlipsideCrypto/livequery-models.git + revision: bca494102fbd2d621d32746e9a7fe780678044f8 +sha1_hash: ce25373a314499826f1bef1d1fca885141e945e9 diff --git a/requirements.txt b/requirements.txt index ec44b0620..59217431d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dbt-snowflake>=1.4,<1.5 \ No newline at end of file +dbt-snowflake>=1.7,<1.8 \ No newline at end of file