An 4429/setup pyth backfill (#460)

* setup pyth backfill

* wip

* remove dev file

* adjust backfill scheduler, add more runs of complete decoded

* adjust backfill rane

.github/workflows/dbt_run_decode_instructions_completed.yml

@@ -0,0 +1,44 @@
+name: dbt_run_decode_instructions_completed
+run-name: dbt_run_decode_instructions_completed
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "${{ vars.PYTHON_VERSION }}"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -s models/streamline/decode_instructions/streamline__complete_decoded_instructions_2.sql 

data/github_actions__workflows.csv

@@ -4,4 +4,5 @@ dbt_run_incremental,"16,46 * * * *"
 dbt_run_incremental_non_core,"1,31 * * * *"
 dbt_test_tasks,"0,30 * * * *"
 dbt_run_decode_instructions,"*/30 * * * *"
-dbt_run_decode_instructions_backfill,"17 */1 * * *"
+dbt_run_decode_instructions_backfill,"20,50 * * * *"
+dbt_run_decode_instructions_completed,"10,40 * * * *"

macros/helpers/decoded_instructions_backfill_helpers.sql

@@ -4,9 +4,9 @@
             default_backfill_start_block_id
         from """ ~ ref('streamline__idls_history') ~ """ 
         where program_id = '""" ~ program_id ~ """';""").columns %}
-    {% set min_block_id = result_cols[0].values()[0] | int %}
-    {% set max_block_id = result_cols[1].values()[0] | int %}
-    {% set step = 1000000 %}
+    {% set min_block_id = 86813920 %}
+    {% set max_block_id = 243627006 %}
+    {% set step = 200000 %}
 
     {% for i in range(min_block_id, max_block_id, step) %}
         {% if i == min_block_id %}
@@ -149,7 +149,7 @@
         from information_schema.views
         where table_name like 'DECODED_INSTRUCTIONS_BACKFILL_%'
         order by 2 desc
-        limit 20;""").columns %}
+        limit 1;""").columns %}
 
     {% set schema_names = results[0].values() %}
     {% set table_names = results[1].values() %}
@@ -163,9 +163,9 @@
 {% endmacro %}
 
 {% macro decoded_instructions_backfill_calls() %}
-    {% set sql_limit = 2500000 %}
-    {% set producer_batch_size = 1000000 %}
-    {% set worker_batch_size = 50000 %}
+    {% set sql_limit = 100000000 %}
+    {% set producer_batch_size = 20000000 %}
+    {% set worker_batch_size = 500000 %}
     {% set batch_call_limit = 1000 %}
 
     {% set results = run_query("""select
@@ -179,7 +179,7 @@
             table_name
         from """ ~ ref('streamline__complete_decoded_instructions_2_backfill') ~ """
         order by 2 desc
-        limit 10;""").columns %}
+        limit 1;""").columns %}
     {% set schema_names = results[0].values() %}
     {% set table_names = results[1].values() %}
     {% for table_name in table_names %}