T 13807 fix tests by only querying relevant time range (#1278)

This PR implements changes aim to improve run time of spell tests. => it sucesfully reduces runtime from 24 minutes to 10 minutes without affecting tests quality Brief comments on the purpose of your changes: *For Dune Engine V2* I've checked that: * [x] I tested the query on dune.com after compiling the model with dbt compile (compiled queries are written to the target directory) * [x] I used "refs" to reference other models in this repo and "sources" to reference raw or decoded tables * [x] if adding a new model, I added a test * [x] the filename is unique and ends with .sql * [x] each sql file is a select statement and has only one view, table or function defined * [x] column names are `lowercase_snake_cased` When you are ready for a review, tag duneanalytics/data-experience. We will re-open your forked pull request as an internal pull request. Then your spells will run in dbt and the logs will be avaiable in Github Actions DBT Slim CI. This job will only run the models and tests changed by your PR compared to the production project.
duneanalytics · Jul 12, 2022 · 76de6e0 · 76de6e0
1 parent 20a903f
commit 76de6e0
Show file tree

Hide file tree

Showing 18 changed files with 54 additions and 41 deletions.
diff --git a/spellbook/dbt_project.yml b/spellbook/dbt_project.yml
@@ -115,7 +115,7 @@ seeds:
       solana:
         +enabled: true
         +schema: test_data
-        opensea_solana_trades_postgres:
+        opensea_solana_trades_solscan:
           +column_types:
             block_time: timestamp
             tx_hash: string
@@ -124,7 +124,7 @@ seeds:
       solana:
         +enabled: true
         +schema: test_data
-        magiceden_solana_trades_postgres:
+        magiceden_solana_trades_solscan:
           +column_types:
             block_time: timestamp
             tx_hash: string

diff --git a/spellbook/models/opensea/ethereum/opensea_ethereum_schema.yml b/spellbook/models/opensea/ethereum/opensea_ethereum_schema.yml
@@ -153,11 +153,8 @@ models:
       - &royalty_fee_currency_symbol
         name: royalty_fee_currency_symbol
         description: "Symbol of the token in which fees are paid out"
-      - &unique_trade_id
-        name: unique_trade_id
+      - name: unique_trade_id
         description:  "Unique trade ID"
-        tests:
-          - unique
 
   - name: opensea_ethereum_transactions
     meta:
@@ -203,8 +200,12 @@ models:
       - *royalty_fee_amount_usd
       - *royalty_fee_receive_address
       - *royalty_fee_currency_symbol
-      - *unique_trade_id
-
+      - &unique_trade_id
+        name: unique_trade_id
+        description:  "Unique trade ID"
+        tests:
+          - unique
+
   - name: opensea_ethereum_trades
     meta:
       blockchain: ethereum

diff --git a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_day.sql b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_day.sql
@@ -3,7 +3,7 @@
         materialized ='incremental',
         file_format ='delta',
         incremental_strategy='merge',
-        unique_key='wallet_contract_day'
+        unique_key='unique_transfer_id'
         )
 }}
 
@@ -13,7 +13,7 @@ select
     tr.wallet_address,
     tr.token_address,
     t.symbol,
-    tr.wallet_address || '-' || tr.token_address || '-' || date_trunc('day', tr.evt_block_time) as wallet_contract_day,
+    tr.wallet_address || '-' || tr.token_address || '-' || date_trunc('day', tr.evt_block_time) as unique_transfer_id,
     sum(tr.amount_raw) as amount_raw,
     sum(tr.amount_raw / power(10, t.decimals)) as amount
 from {{ ref('transfers_ethereum_erc20') }} tr

diff --git a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_hour.sql b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_hour.sql
@@ -3,7 +3,7 @@
         materialized ='incremental',
         file_format ='delta',
         incremental_strategy='merge',
-        unique_key='wallet_contract_hour'
+        unique_key='unique_transfer_id'
         )
 }}
 
@@ -13,7 +13,7 @@ select
     tr.wallet_address,
     tr.token_address,
     t.symbol,
-    tr.wallet_address || '-' || tr.token_address || '-' || date_trunc('hour', tr.evt_block_time) as wallet_contract_hour,
+    tr.wallet_address || '-' || tr.token_address || '-' || date_trunc('hour', tr.evt_block_time) as unique_transfer_id,
     sum(tr.amount_raw) as amount_raw,
     sum(tr.amount_raw / power(10, t.decimals)) as amount
 from {{ ref('transfers_ethereum_erc20') }} tr

diff --git a/spellbook/models/transfers/ethereum/transfers_ethereum_schema.yml b/spellbook/models/transfers/ethereum/transfers_ethereum_schema.yml
@@ -13,8 +13,6 @@ models:
     columns:
       - name: unique_transfer_id
         description: "Unique transfer ID (used for testing for duplicates)"
-        tests:
-          - unique
       - &blockchain
         name: blockchain
         description: "Blockchain"
@@ -51,8 +49,10 @@ models:
       - *amount_raw
       - name: amount
         description: "Raw amount of ERC20 token held *after* taking into account token decimals"
-      - name: amount_usd
-        description: "Amount of ERC20 token held in USD (fiat value at time of transaction)"
+      - name: unique_transfer_id
+        description: "Unique ID, used in order to apply the merge into incremental strategy"
+        tests:
+          - unique
 
   - name: transfers_ethereum_erc20_rolling_hour
     meta:
@@ -72,8 +72,6 @@ models:
       - *amount_raw
       - name: amount
         description: "Rolling sum of raw amount of ERC20 token held *after* taking into account token decimals"
-      - name: amount_usd
-        description: "Rolling sum of amount of ERC20 token held in USD (fiat value at time of transaction)"
       - name: updated_at
         description: "UTC timestamp when table was last updated"
       - name: recency_index
@@ -99,8 +97,10 @@ models:
       - *amount_raw
       - name: amount
         description: "Raw amount of ERC20 token held *after* taking into account token decimals"
-      - name: amount_usd
-        description: "Amount of ERC20 token held in USD (fiat value at time of transaction)"
+      - name: unique_transfer_id
+        description: "Unique ID, used in order to apply the merge into incremental strategy"
+        tests:
+          - unique
 
   - name: transfers_ethereum_erc20_rolling_day
     meta:
@@ -120,8 +120,6 @@ models:
       - *amount_raw
       - name: amount
         description: "Rolling sum of raw amount of ERC20 token held *after* taking into account token decimals"
-      - name: amount_usd
-        description: "Rolling sum of amount of ERC20 token held in USD (fiat value at time of transaction)"
       - name: last_updated
         description: "UTC timestamp when table was last updated"
       - name: recency_index

diff --git a/...lana/magiceden_solana_trades_postgres.csv → ...olana/magiceden_solana_trades_solscan.csv b/...lana/magiceden_solana_trades_postgres.csv → ...olana/magiceden_solana_trades_solscan.csv
diff --git a/...solana/opensea_solana_trades_postgres.csv → .../solana/opensea_solana_trades_solscan.csv b/...solana/opensea_solana_trades_postgres.csv → .../solana/opensea_solana_trades_solscan.csv
diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_nonnegative.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_nonnegative.sql
@@ -2,7 +2,9 @@
 -- Some balances are very small negative numbers due to loss of precision from large ints
 
 select amount
-from {{ ref('balances_ethereum_erc20_day') }}
+from {{ ref('balances_ethereum_erc20_day') }} bal
 where round(amount/power(10, 18), 6) < 0
 -- limiting to a selection of tokens because we haven't filtered out all non-compliant tokens
 and symbol in ('AAVE', 'DAI', 'UNI', 'LINK')
+and bal.day > now() - interval 2 days
+
diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_samples.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_samples.sql
@@ -3,8 +3,9 @@
 with sampled_wallets as
  (
      select *
-     from {{ ref('balances_ethereum_erc20_day') }}
-     where wallet_address in (select distinct wallet_address from {{ ref('balances_ethereum_erc20_daily_entries') }})
+     from {{ ref('balances_ethereum_erc20_day') }} bal
+     where wallet_address in (select distinct wallet_address from {{ ref('balances_ethereum_erc20_daily_entries')  }})
+     and bal.day > '2021-12-30' and bal.day < '2022-01-01'
  )
 
 , unit_tests as

diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_nonnegative.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_nonnegative.sql
@@ -2,7 +2,9 @@
 -- Some balances are very small negative numbers due to loss of precision from large ints
 
 select amount
-from {{ ref('balances_ethereum_erc20_hour') }}
+from {{ ref('balances_ethereum_erc20_hour') }} bal
 where round(amount/power(10, 18), 6) < 0
 -- limiting to a selection of tokens because we haven't filtered out all non-compliant tokens
 and symbol in ('AAVE', 'DAI', 'UNI', 'LINK')
+and bal.hour > now() - interval 2 days
+
diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_samples.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_samples.sql
@@ -3,9 +3,10 @@
 with sampled_wallets as
  (
      select *
-     from {{ ref('balances_ethereum_erc20_hour') }}
+     from {{ ref('balances_ethereum_erc20_hour') }} bal
      where wallet_address in (select distinct wallet_address from {{ ref('balances_ethereum_erc20_latest_entries') }})
      and symbol in ('USDT', 'LINK', 'DAI', 'USDC')
+     and bal.hour > '2022-05-04' and bal.hour < '2022-05-06'
  )
 
 , unit_tests as

diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc721_day_assert_nomultipleholders.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc721_day_assert_nomultipleholders.sql
@@ -7,6 +7,6 @@ select blockchain,
     tokenId,
     count(*)
 from {{ ref('balances_ethereum_erc721_day') }}
-where day >= now() - interval '4 days'
+where day >= now() - interval '2 days'
 group by blockchain, day, wallet_address, token_address, tokenId
 having count(*) > 1
diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc721_hour_assert_nomultipleholders.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc721_hour_assert_nomultipleholders.sql
@@ -7,6 +7,6 @@ select blockchain,
     tokenId,
     count(*)
 from {{ ref('balances_ethereum_erc721_hour') }}
-where hour >= now() - interval '4 hours'
+where hour >= now() - interval '2 hours'
 group by blockchain, hour, wallet_address, token_address, tokenId
 having count(*) > 1
diff --git a/spellbook/tests/magiceden/solana/magiceden_solana_assert_samples.sql b/spellbook/tests/magiceden/solana/magiceden_solana_assert_samples.sql
@@ -1,11 +1,13 @@
--- Bootstrapped correctness test against legacy Postgres values.
--- Also manually check solscan info for the first 5 rows
+-- Bootstrapped correctness test against values downloaded from the Dune App 
+-- The first 10 values were also manually checked using Solscan API
 
 WITH unit_tests as
-(SELECT case when test_data.amount = os_trades.amount_original then True else False end as price_test
-FROM {{ ref('magiceden_solana_trades') }} os_trades
-JOIN {{ ref('magiceden_solana_trades_postgres') }} test_data ON test_data.tx_hash = os_trades.tx_hash
-AND test_data.block_time = os_trades.block_time
+(SELECT case when test_data.amount = me_trades.amount_original then True else False end as price_test
+FROM {{ ref('nft_trades') }} me_trades
+JOIN {{ ref('magiceden_solana_trades_solscan') }} test_data ON test_data.tx_hash = me_trades.tx_hash
+AND test_data.block_time = me_trades.block_time
+WHERE me_trades.block_time > '2021-10-23' and me_trades.block_time < '2021-10-25'
+and me_trades.project = 'magiceden' and me_trades.blockchain = 'solana'
 )
 
 select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows

diff --git a/spellbook/tests/opensea/ethereum/opensea_ethereum_assert_samples.sql b/spellbook/tests/opensea/ethereum/opensea_ethereum_assert_samples.sql
@@ -3,8 +3,10 @@
 -- Also manually check etherscan info for the first 5 rows
 WITH unit_tests as
 (SELECT case when test_data.price = os_trades.amount_raw then True else False end as price_test
-FROM {{ ref('opensea_ethereum_trades') }} os_trades
+FROM {{ ref('nft_trades') }} os_trades
 JOIN {{ ref('opensea_ethereum_trades_postgres') }} test_data ON test_data.evt_tx_hash = os_trades.tx_hash
+WHERE os_trades.block_time > '2022-05-22' and os_trades.block_time < '2022-05-24'
+and os_trades.project = 'opensea' and os_trades.blockchain = 'ethereum'
 )
 select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows
 from unit_tests

diff --git a/spellbook/tests/opensea/solana/opensea_solana_assert_samples.sql b/spellbook/tests/opensea/solana/opensea_solana_assert_samples.sql
@@ -1,11 +1,13 @@
--- Bootstrapped correctness test against legacy Postgres values.
--- Also manually check solscan info for the first 5 rows
+-- Bootstrapped correctness test against values downloaded from the Dune App 
+-- The first 10 values were also manually checked using Solscan API
 
 WITH unit_tests as
 (SELECT case when test_data.amount = os_trades.amount_original then True else False end as price_test
 FROM {{ ref('opensea_solana_trades') }} os_trades
-JOIN {{ ref('opensea_solana_trades_postgres') }} test_data ON test_data.tx_hash = os_trades.tx_hash
+JOIN {{ ref('opensea_solana_trades_solscan') }} test_data ON test_data.tx_hash = os_trades.tx_hash
 AND test_data.block_time = os_trades.block_time
+WHERE os_trades.block_time > '2022-05-01' and os_trades.block_time < '2022-05-03'
+and os_trades.project = 'opensea' and os_trades.blockchain = 'solana'
 )
 
 select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows

diff --git a/spellbook/tests/uniswap/ethereum/uniswap_v2_ethereum_assert_samples.sql b/spellbook/tests/uniswap/ethereum/uniswap_v2_ethereum_assert_samples.sql
@@ -6,8 +6,9 @@
 -- Also manually check etherscan info for the first 5 rows
 WITH unit_tests as
 (SELECT case when test_data.token_a_amount = us_trades.token_a_amount AND test_data.token_b_amount = us_trades.token_b_amount then True else False end as price_test
-FROM {{ ref('uniswap_v2_ethereum_trades') }} us_trades
+FROM {{ ref('uniswap_trades') }} us_trades
 JOIN {{ ref('uniswap_v2_ethereum_trades_postgres') }} test_data ON test_data.tx_hash = us_trades.tx_hash
+WHERE us_trades.blockchain = 'ethereum' and us_trades.version = 'v2'
 )
 select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows
 from unit_tests

diff --git a/spellbook/tests/uniswap/ethereum/uniswap_v3_ethereum_assert_samples.sql b/spellbook/tests/uniswap/ethereum/uniswap_v3_ethereum_assert_samples.sql
@@ -6,8 +6,9 @@
 -- Also manually check etherscan info for the first 5 rows
 WITH unit_tests as
 (SELECT case when test_data.token_a_amount = us_trades.token_a_amount AND test_data.token_b_amount = us_trades.token_b_amount then True else False end as price_test
-FROM {{ ref('uniswap_v3_ethereum_trades') }} us_trades
+FROM {{ ref('uniswap_trades') }} us_trades
 JOIN {{ ref('uniswap_v3_ethereum_trades_postgres') }} test_data ON test_data.tx_hash = us_trades.tx_hash
+WHERE us_trades.blockchain = 'ethereum' and us_trades.version = 'v3'
 )
 select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows
 from unit_tests