Create hourly spellbook and move all subprojects in new directory str…

…ucture (duneanalytics#6422)

* split out hourly models

* refactor refs and sources

* fix bitcoin address filter

* move more lineages to hourly

* fix schemas

* move more models and fix all sources/refs

* move models into dbt subproject

* fix legacy tests

* trigger tokens CI

* upload manifest through CI

* fx

* remove temp CI

* move solana models to solana

* move all subprojects in new dir structure

* fix script

* fix dbt compile path

* update package locks

* updated readme for subprojects

* add link to gh discussion for subprojects

* add readme to old models directory

* update PR template

---------

Co-authored-by: jeff-dude <[email protected]>

.github/pull_request_template.md

@@ -1,5 +1,8 @@
 ## Thank you for contributing to Spellbook 🪄
 
+### Update!
+Please build spells in the proper [subproject](../dbt_subprojects/) directory. For more information, please see the main [readme](../README.md), which also links to a GH discussion with the option to ask questions.
+
 ### Contribution type
 Please check the type of contribution this pull request is for:
 

.github/workflows/commit_manifest.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
     strategy:
       matrix:
-        project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft', 'dex', 'solana']
+        project: [ 'tokens', 'daily_spellbook', 'nft', 'dex', 'solana', 'hourly_spellbook']
       max-parallel: 1
 
     steps:
@@ -33,11 +33,8 @@ jobs:
         echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
         echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
         echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{matrix.project}}" >> $GITHUB_ENV
-          PROJECT_DIR=${{ matrix.project }}
-          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
-            PROJECT_DIR="."
-          fi
-          echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
+        PROJECT_DIR=dbt_subprojects/${{ matrix.project }}
+        echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
 
     - name: dbt dependencies
       working-directory: ${{env.PROJECT_DIR}}
@@ -56,4 +53,4 @@ jobs:
       run: "aws s3 cp $S3_LOCATION/manifest.json $S3_LOCATION/manifest_$GIT_SHA.json"
 
     - name: upload manifest
-      run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json"
+      run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json"

.github/workflows/daily_spellbook.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - daily_spellbook/**
+      - dbt_subprojects/daily_spellbook/**
       - .github/workflows/daily_spellbook.yml
       - .github/workflows/dbt_run.yml
 

.github/workflows/dbt_run.yml

@@ -22,10 +22,7 @@ jobs:
           echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
           echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
           echo "S3_LOCATION=s3://manifest-spellbook-dunesql/${{inputs.project}}" >> $GITHUB_ENV
-          PROJECT_DIR=${{ inputs.project }}
-          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
-            PROJECT_DIR="."
-          fi
+          PROJECT_DIR=dbt_subprojects/${{ inputs.project }}
           echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
       - name: Add git_sha to schema
         run: "/runner/change_schema.sh git_dunesql_$GIT_SHA"
@@ -42,7 +39,8 @@ jobs:
         run: "./scripts/ensure_cluster.sh"
 
       - name: dbt compile to create manifest to compare to
-        run: "dbt --warn-error compile --project-dir ${PROJECT_DIR}"
+        working-directory: ${{env.PROJECT_DIR}}
+        run: "dbt --warn-error compile"
 
       - name: check schemas
         run: "./scripts/check_schema.sh"

.github/workflows/dex.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - dex/**
+      - dbt_subprojects/dex/**
       - .github/workflows/dex.yml
       - .github/workflows/dbt_run.yml
 
@@ -16,4 +16,4 @@ jobs:
   dbt-run:
     uses: ./.github/workflows/dbt_run.yml
     with:
-      project: 'dex'
+      project: 'dex'

.github/workflows/hourly_spellbook.yml

@@ -0,0 +1,20 @@
+name: DBT Hourly Spellbook run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dbt_subprojects/hourly_spellbook/**
+      - .github/workflows/hourly_spellbook.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'hourly_spellbook'
+

.github/workflows/nft.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - nft/**
+      - dbt_subprojects/nft/**
       - .github/workflows/nft.yml
       - .github/workflows/dbt_run.yml
 

.github/workflows/prices_check.yml

@@ -5,7 +5,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - 'models/prices/*/*.sql'
+      - 'dbt_subprojects/tokens/models/prices/*/*.sql'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -30,7 +30,7 @@ jobs:
       - name: Get prices diff
         shell: bash
         run: |
-          git diff ${{ github.event.pull_request.base.sha}}..${{ github.event.pull_request.head.sha }} models/prices/**/*.sql | grep '^\+ ' > scripts/new_lines.txt
+          git diff ${{ github.event.pull_request.base.sha}}..${{ github.event.pull_request.head.sha }} dbt_subprojects/tokens/models/prices/**/*.sql | grep '^\+ ' > scripts/new_lines.txt
 
       - name: Run validations
         id: validations

.github/workflows/solana.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - solana/**
+      - dbt_subprojects/solana/**
       - .github/workflows/solana.yml
       - .github/workflows/dbt_run.yml
 

.github/workflows/tokens.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - tokens/**
+      - dbt_subprojects/tokens/**
       - .github/workflows/tokens.yml
       - .github/workflows/dbt_run.yml
 

README.md

@@ -7,7 +7,7 @@ Welcome to [Spellbook](https://youtu.be/o7p0BNt7NHs). Cast a magical incantation
 
 - Have a question on how something works in Spellbook, or why we design spells in a particular way?
   - Please visit the [docs](docs/) directory to find various topics & ideally answers to any question about Spellbook
-- Spellbook has [introduced sub-projects](https://github.com/duneanalytics/spellbook/discussions/5238), with the intention to pilot a path forward for scaling the repo
+- Spellbook has [introduced sub-projects](#sub-projects), with the intention to build a path forward for scaling Spellbook
 - Are you building something new? **Please make sure to open a Draft PR**, so we minimize duplicated work, and other wizards can help you if you need
 - Don't know where to start? The docs below will guide you, but as a summary:
   - Want to make an incremental improvement to one of our spells? (add a new project, fix a bug you found), simply open a PR with your changes.
@@ -45,9 +45,27 @@ Spellbook has a lot of moving parts & specific design principles for contributin
 
 ## Sub-projects
 
-In order to prepare for scaling Spellbook, the repo has started to pilot sub-projects to break out complex DBT lineages a bit & keep focus areas clean. This will also help downstream orchestration to keep spells fresh in production. Please refer to initial [GH discussion](https://github.com/duneanalytics/spellbook/discussions/5238) for high-level info & continue the conversation there with the Dune team.
-
-- First sub-project: erc20 token metadata, found [here](tokens)
+In order to scale Spellbook, the repo has introduced sub-projects to break out complex DBT lineages a bit & keep focus areas clean. This will also help downstream orchestration to keep spells fresh in production. DBT sub-projects in Spellbook are simply multiple DBT projects within one repo. The current structure for projects:
+- `dbt_subprojects`
+  - `daily_spellbook`
+    - *note*: new spells will live here, unless otherwise directed by Dune team
+    - all "other" spells which don't feed into larger sector-wide spells, refreshed on a daily basis
+    - example: project specific, standalone spells
+  - `hourly_spellbook`
+    - "other" spells which have been promoted from daily to hourly, allowing for more frequent refreshes
+    - feed into sector-level spells, with potential to be promoted into it's own project
+    - required to fit latest spellbook best practices
+    - requires approval from Dune team to be hourly
+  - `dex`
+    - all spells which live in the `dex` or `dex_aggregator` schemas, including upstream spells to help build the final sector-level spells
+  - `nft`
+    - all spells which live in the `nft` schema, including upstream spells to help build the final sector-level spells
+  - `solana`
+    - solana specific spells, don't fit into EVM code structure as easily
+  - `tokens`
+    - token metadata, transfers, balances
+
+For further information on sub-projects, please visit [this discussion](https://github.com/duneanalytics/spellbook/discussions/6037) and ask any questions there.
 
 ## Ways to contribute to Spellbook
 

dbt_subprojects/README.md

@@ -0,0 +1 @@
+[//]: # (todo: documentation around subprojects)

dbt_project.yml → dbt_subprojects/daily_spellbook/dbt_project.yml

@@ -1,6 +1,6 @@
 #Welcome to your Spellbook!
 
-name: "spellbook"
+name: "daily_spellbook"
 version: "1.0.0"
 config-version: 2
 
@@ -10,6 +10,7 @@ quoting:
   identifier: false
 
 # This setting configures which "profile" dbt uses for this project.
+# profile: "spellbook-poc-tokens"
 profile: "spellbook-local"
 
 vars:
@@ -20,10 +21,10 @@ vars:
 
 # These configurations specify where dbt should look for different types of files.
 # You don't need to change these!
-model-paths: ["models", "sources"]
+model-paths: ["models", "../../sources"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]
-macro-paths: ["macros"]
+macro-paths: ["macros", "../../macros"]
 
 target-path: "target" # directory which will store compiled SQL files
 clean-targets: # directories to be removed by `dbt clean`
@@ -33,22 +34,26 @@ clean-targets: # directories to be removed by `dbt clean`
 # Configuring tests
 # Full documentation: https://docs.getdbt.com/reference/test-configs
 tests:
-  spellbook:
+  daily_spellbook:
     +store_failures: true  # store failures for all tests
 
 # Configuring seeds
 # Full documentation: https://docs.getdbt.com/reference/seed-configs
 # For configuring individual seeds (e.g. overriding column types) we recommend a yml file nested in the same folder as the seed
 # example: seeds/cryptopunks/schema.yml
 seeds:
-  spellbook:
+  daily_spellbook:
     +schema: test_data
 
+# defining search order of macro invokation
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['trino_utils', 'dbt_utils']
 
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models
 models:
-  spellbook:
+  daily_spellbook:
     +post-hook:
       - sql: "{{ set_trino_session_property(is_materialized(model), 'writer_scaling_min_data_processed', model.config.get('writer_min_size', '500MB')) }}"
         transaction: true
@@ -59,30 +64,32 @@ models:
       - sql: "{{ mark_as_spell(this, model.config.materialized) }}"
         transaction: true
     +materialized: view
-    +schema: no_schema    # this should be overridden in model specific configs
+    +schema: no_schema    # this should be overriden in model specific configs
     +view_security: invoker
 
-    dex:
-      +schema: dex
-
-    cex:
-      +schema: cex
+    chainlink:
+      +schema: chainlink
       arbitrum:
-        +schema: cex_arbitrum
+        +schema: chainlink_arbitrum
       avalanche_c:
-        +schema: cex_avalanche_c
+        +schema: chainlink_avalanche_c
+      base:
+        +schema: chainlink_base
       bnb:
-        +schema: cex_bnb
+        +schema: chainlink_bnb
       ethereum:
-        +schema: cex_ethereum
+        +schema: chainlink_ethereum
+      fantom:
+        +schema: chainlink_fantom
+      gnosis:
+        +schema: chainlink_gnosis
       optimism:
-        +schema: cex_optimism
+        +schema: chainlink_optimism
       polygon:
-        +schema: cex_polygon
-      fantom:
-        +schema: cex_fantom
-      bitcoin:
-        +schema: cex_bitcoin
+        +schema: chainlink_polygon
+
+    dex:
+      +schema: dex
 
     balances:
       +schema: balances
@@ -131,29 +138,6 @@ models:
       ethereum:
         +schema: sudoswap_ethereum
 
-    safe:
-      +schema: safe
-      ethereum:
-        +schema: safe_ethereum
-      optimism:
-        +schema: safe_optimism
-      gnosis:
-        +schema: safe_gnosis
-      polygon:
-        +schema: safe_polygon
-      bnb:
-        +schema: safe_bnb
-      avalanche_c:
-        +schema: safe_avalanche_c
-      fantom:
-        +schema: safe_fantom
-      arbitrum:
-        +schema: safe_arbitrum
-      goerli:
-        +schema: safe_goerli
-      base:
-        +schema: safe_base
-
     aragon:
       +schema: aragon
       ethereum:
@@ -221,12 +205,7 @@ models:
       ethereum:
         +schema: fiat_dao_ethereum
 
-    cow_protocol:
-      +schema: cow_protocol
-      ethereum:
-        +schema: cow_protocol_ethereum
-      gnosis:
-        +schema: cow_protocol_gnosis
+
 
     ironbank:
       +schema: ironbank
@@ -335,17 +314,6 @@ models:
       ethereum:
         +schema: dydx_ethereum
 
-    lido:
-      +schema: lido
-      accounting:
-        +schema: lido_accounting
-        ethereum:
-          +schema: lido_accounting_ethereum
-      liquidity:
-        +schema: lido_liquidity
-        ethereum:
-          +schema: lido_liquidity_ethereum
-
     layerzero:
       +schema: layerzero
       ethereum: