diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml new file mode 100644 index 0000000..bf57ca8 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -0,0 +1,45 @@ +name: dbt_run_streamline_history +run-name: dbt_run_streamline_history + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run History Models + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts" diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index cd555a2..48f3053 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,3 +1,4 @@ workflow_name,workflow_schedule dbt_run_streamline_chainhead,"24,54 * * * *" -dbt_run_scheduled_main,"2,32 * * * *" \ No newline at end of file +dbt_run_scheduled_main,"2,32 * * * *" +dbt_run_streamline_history,"12 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 78d36ae..f176649 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -142,13 +142,32 @@ vars: ## REQUIRED ## OPTIONAL - # TRACES_FULL_RELOAD_MODE: True - # TRACES_FULL_RELOAD_START_BLOCK: 2500000 - BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True - RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True + # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 + + BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True + BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 300000 + BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 60000 + BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 12000 + # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 4 + + RECEIPTS_REALTIME_NEW_BUILD: True + # RECEIPTS_REALTIME_TESTING_LIMIT: 3 + + RECEIPTS_HISTORY_NEW_BUILD: True + RECEIPTS_HISTORY_SQL_LIMIT: 300000 + RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 60000 + RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 12000 + # RECEIPTS_HISTORY_TESTING_LIMIT: 4 + TRACES_REALTIME_NEW_BUILD: True - CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True + # TRACES_REALTIME_TESTING_LIMIT: 3 + + TRACES_HISTORY_NEW_BUILD: True + TRACES_HISTORY_SQL_LIMIT: 150000 + TRACES_HISTORY_PRODUCER_BATCH_SIZE: 18750 + TRACES_HISTORY_WORKER_BATCH_SIZE: 3750 + # TRACES_HISTORY_TESTING_LIMIT: 4 # GOLD_FULL_REFRESH: True # SILVER_FULL_REFRESH: True