Merge pull request #6 from FlipsideCrypto/AN-5447/backfill-set-up

AN-5447/backfill-set-up

.github/workflows/dbt_run_streamline_history.yml

@@ -0,0 +1,45 @@
+name: dbt_run_streamline_history
+run-name: dbt_run_streamline_history
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+          
+      - name: Run History Models
+        run: |
+          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts" 

data/github_actions__workflows.csv

@@ -1,3 +1,4 @@
 workflow_name,workflow_schedule
 dbt_run_streamline_chainhead,"24,54 * * * *"
-dbt_run_scheduled_main,"2,32 * * * *"
+dbt_run_scheduled_main,"2,32 * * * *"
+dbt_run_streamline_history,"12 * * * *"

dbt_project.yml

@@ -142,13 +142,32 @@ vars:
   ## REQUIRED
 
   ## OPTIONAL
-  # TRACES_FULL_RELOAD_MODE: True
-  # TRACES_FULL_RELOAD_START_BLOCK: 2500000 
-
   BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True
-  RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True
+  # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3
+
+  BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True
+  BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 300000
+  BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 60000
+  BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 12000
+  # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 4
+
+  RECEIPTS_REALTIME_NEW_BUILD: True
+  # RECEIPTS_REALTIME_TESTING_LIMIT: 3
+
+  RECEIPTS_HISTORY_NEW_BUILD: True
+  RECEIPTS_HISTORY_SQL_LIMIT: 300000
+  RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 60000
+  RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 12000
+  # RECEIPTS_HISTORY_TESTING_LIMIT: 4
+
   TRACES_REALTIME_NEW_BUILD: True
-  CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True
+  # TRACES_REALTIME_TESTING_LIMIT: 3
+
+  TRACES_HISTORY_NEW_BUILD: True
+  TRACES_HISTORY_SQL_LIMIT: 150000
+  TRACES_HISTORY_PRODUCER_BATCH_SIZE: 18750
+  TRACES_HISTORY_WORKER_BATCH_SIZE: 3750
+  # TRACES_HISTORY_TESTING_LIMIT: 4
 
   # GOLD_FULL_REFRESH: True
   # SILVER_FULL_REFRESH: True