diff --git a/spellbook/dbt_project.yml b/spellbook/dbt_project.yml index fd5e9b2ae17..38db78ca2e0 100644 --- a/spellbook/dbt_project.yml +++ b/spellbook/dbt_project.yml @@ -115,7 +115,7 @@ seeds: solana: +enabled: true +schema: test_data - opensea_solana_trades_postgres: + opensea_solana_trades_solscan: +column_types: block_time: timestamp tx_hash: string @@ -124,7 +124,7 @@ seeds: solana: +enabled: true +schema: test_data - magiceden_solana_trades_postgres: + magiceden_solana_trades_solscan: +column_types: block_time: timestamp tx_hash: string diff --git a/spellbook/models/opensea/ethereum/opensea_ethereum_schema.yml b/spellbook/models/opensea/ethereum/opensea_ethereum_schema.yml index d57dcd44a23..d9ad25a2093 100644 --- a/spellbook/models/opensea/ethereum/opensea_ethereum_schema.yml +++ b/spellbook/models/opensea/ethereum/opensea_ethereum_schema.yml @@ -153,11 +153,8 @@ models: - &royalty_fee_currency_symbol name: royalty_fee_currency_symbol description: "Symbol of the token in which fees are paid out" - - &unique_trade_id - name: unique_trade_id + - name: unique_trade_id description: "Unique trade ID" - tests: - - unique - name: opensea_ethereum_transactions meta: @@ -203,8 +200,12 @@ models: - *royalty_fee_amount_usd - *royalty_fee_receive_address - *royalty_fee_currency_symbol - - *unique_trade_id - + - &unique_trade_id + name: unique_trade_id + description: "Unique trade ID" + tests: + - unique + - name: opensea_ethereum_trades meta: blockchain: ethereum diff --git a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_day.sql b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_day.sql index 0677115dc8c..4802236a08a 100644 --- a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_day.sql +++ b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_day.sql @@ -3,7 +3,7 @@ materialized ='incremental', file_format ='delta', incremental_strategy='merge', - unique_key='wallet_contract_day' + unique_key='unique_transfer_id' ) }} @@ -13,7 +13,7 @@ select tr.wallet_address, tr.token_address, t.symbol, - tr.wallet_address || '-' || tr.token_address || '-' || date_trunc('day', tr.evt_block_time) as wallet_contract_day, + tr.wallet_address || '-' || tr.token_address || '-' || date_trunc('day', tr.evt_block_time) as unique_transfer_id, sum(tr.amount_raw) as amount_raw, sum(tr.amount_raw / power(10, t.decimals)) as amount from {{ ref('transfers_ethereum_erc20') }} tr diff --git a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_hour.sql b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_hour.sql index a17b1380879..4dde49893d5 100644 --- a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_hour.sql +++ b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_agg_hour.sql @@ -3,7 +3,7 @@ materialized ='incremental', file_format ='delta', incremental_strategy='merge', - unique_key='wallet_contract_hour' + unique_key='unique_transfer_id' ) }} @@ -13,7 +13,7 @@ select tr.wallet_address, tr.token_address, t.symbol, - tr.wallet_address || '-' || tr.token_address || '-' || date_trunc('hour', tr.evt_block_time) as wallet_contract_hour, + tr.wallet_address || '-' || tr.token_address || '-' || date_trunc('hour', tr.evt_block_time) as unique_transfer_id, sum(tr.amount_raw) as amount_raw, sum(tr.amount_raw / power(10, t.decimals)) as amount from {{ ref('transfers_ethereum_erc20') }} tr diff --git a/spellbook/models/transfers/ethereum/transfers_ethereum_schema.yml b/spellbook/models/transfers/ethereum/transfers_ethereum_schema.yml index 1a195177682..371295a033d 100644 --- a/spellbook/models/transfers/ethereum/transfers_ethereum_schema.yml +++ b/spellbook/models/transfers/ethereum/transfers_ethereum_schema.yml @@ -13,8 +13,6 @@ models: columns: - name: unique_transfer_id description: "Unique transfer ID (used for testing for duplicates)" - tests: - - unique - &blockchain name: blockchain description: "Blockchain" @@ -51,8 +49,10 @@ models: - *amount_raw - name: amount description: "Raw amount of ERC20 token held *after* taking into account token decimals" - - name: amount_usd - description: "Amount of ERC20 token held in USD (fiat value at time of transaction)" + - name: unique_transfer_id + description: "Unique ID, used in order to apply the merge into incremental strategy" + tests: + - unique - name: transfers_ethereum_erc20_rolling_hour meta: @@ -72,8 +72,6 @@ models: - *amount_raw - name: amount description: "Rolling sum of raw amount of ERC20 token held *after* taking into account token decimals" - - name: amount_usd - description: "Rolling sum of amount of ERC20 token held in USD (fiat value at time of transaction)" - name: updated_at description: "UTC timestamp when table was last updated" - name: recency_index @@ -99,8 +97,10 @@ models: - *amount_raw - name: amount description: "Raw amount of ERC20 token held *after* taking into account token decimals" - - name: amount_usd - description: "Amount of ERC20 token held in USD (fiat value at time of transaction)" + - name: unique_transfer_id + description: "Unique ID, used in order to apply the merge into incremental strategy" + tests: + - unique - name: transfers_ethereum_erc20_rolling_day meta: @@ -120,8 +120,6 @@ models: - *amount_raw - name: amount description: "Rolling sum of raw amount of ERC20 token held *after* taking into account token decimals" - - name: amount_usd - description: "Rolling sum of amount of ERC20 token held in USD (fiat value at time of transaction)" - name: last_updated description: "UTC timestamp when table was last updated" - name: recency_index diff --git a/spellbook/seeds/magiceden/solana/magiceden_solana_trades_postgres.csv b/spellbook/seeds/magiceden/solana/magiceden_solana_trades_solscan.csv similarity index 100% rename from spellbook/seeds/magiceden/solana/magiceden_solana_trades_postgres.csv rename to spellbook/seeds/magiceden/solana/magiceden_solana_trades_solscan.csv diff --git a/spellbook/seeds/opensea/solana/opensea_solana_trades_postgres.csv b/spellbook/seeds/opensea/solana/opensea_solana_trades_solscan.csv similarity index 100% rename from spellbook/seeds/opensea/solana/opensea_solana_trades_postgres.csv rename to spellbook/seeds/opensea/solana/opensea_solana_trades_solscan.csv diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_nonnegative.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_nonnegative.sql index 756b99d9d79..138925684a1 100644 --- a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_nonnegative.sql +++ b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_nonnegative.sql @@ -2,7 +2,9 @@ -- Some balances are very small negative numbers due to loss of precision from large ints select amount -from {{ ref('balances_ethereum_erc20_day') }} +from {{ ref('balances_ethereum_erc20_day') }} bal where round(amount/power(10, 18), 6) < 0 -- limiting to a selection of tokens because we haven't filtered out all non-compliant tokens and symbol in ('AAVE', 'DAI', 'UNI', 'LINK') +and bal.day > now() - interval 2 days + diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_samples.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_samples.sql index 40453722128..f2979641950 100644 --- a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_samples.sql +++ b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_day_assert_samples.sql @@ -3,8 +3,9 @@ with sampled_wallets as ( select * - from {{ ref('balances_ethereum_erc20_day') }} - where wallet_address in (select distinct wallet_address from {{ ref('balances_ethereum_erc20_daily_entries') }}) + from {{ ref('balances_ethereum_erc20_day') }} bal + where wallet_address in (select distinct wallet_address from {{ ref('balances_ethereum_erc20_daily_entries') }}) + and bal.day > '2021-12-30' and bal.day < '2022-01-01' ) , unit_tests as diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_nonnegative.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_nonnegative.sql index eabeabaa139..c001d74a7f9 100644 --- a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_nonnegative.sql +++ b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_nonnegative.sql @@ -2,7 +2,9 @@ -- Some balances are very small negative numbers due to loss of precision from large ints select amount -from {{ ref('balances_ethereum_erc20_hour') }} +from {{ ref('balances_ethereum_erc20_hour') }} bal where round(amount/power(10, 18), 6) < 0 -- limiting to a selection of tokens because we haven't filtered out all non-compliant tokens and symbol in ('AAVE', 'DAI', 'UNI', 'LINK') +and bal.hour > now() - interval 2 days + diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_samples.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_samples.sql index bc7e55beb22..ad524a9e393 100644 --- a/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_samples.sql +++ b/spellbook/tests/balances/ethereum/balances_ethereum_erc20_hour_assert_samples.sql @@ -3,9 +3,10 @@ with sampled_wallets as ( select * - from {{ ref('balances_ethereum_erc20_hour') }} + from {{ ref('balances_ethereum_erc20_hour') }} bal where wallet_address in (select distinct wallet_address from {{ ref('balances_ethereum_erc20_latest_entries') }}) and symbol in ('USDT', 'LINK', 'DAI', 'USDC') + and bal.hour > '2022-05-04' and bal.hour < '2022-05-06' ) , unit_tests as diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc721_day_assert_nomultipleholders.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc721_day_assert_nomultipleholders.sql index 065fdf53ad2..1d5f18f3d7b 100644 --- a/spellbook/tests/balances/ethereum/balances_ethereum_erc721_day_assert_nomultipleholders.sql +++ b/spellbook/tests/balances/ethereum/balances_ethereum_erc721_day_assert_nomultipleholders.sql @@ -7,6 +7,6 @@ select blockchain, tokenId, count(*) from {{ ref('balances_ethereum_erc721_day') }} -where day >= now() - interval '4 days' +where day >= now() - interval '2 days' group by blockchain, day, wallet_address, token_address, tokenId having count(*) > 1 diff --git a/spellbook/tests/balances/ethereum/balances_ethereum_erc721_hour_assert_nomultipleholders.sql b/spellbook/tests/balances/ethereum/balances_ethereum_erc721_hour_assert_nomultipleholders.sql index 913c9b0617a..0d508ff953e 100644 --- a/spellbook/tests/balances/ethereum/balances_ethereum_erc721_hour_assert_nomultipleholders.sql +++ b/spellbook/tests/balances/ethereum/balances_ethereum_erc721_hour_assert_nomultipleholders.sql @@ -7,6 +7,6 @@ select blockchain, tokenId, count(*) from {{ ref('balances_ethereum_erc721_hour') }} -where hour >= now() - interval '4 hours' +where hour >= now() - interval '2 hours' group by blockchain, hour, wallet_address, token_address, tokenId having count(*) > 1 diff --git a/spellbook/tests/magiceden/solana/magiceden_solana_assert_samples.sql b/spellbook/tests/magiceden/solana/magiceden_solana_assert_samples.sql index 58648673b53..99397a72de9 100644 --- a/spellbook/tests/magiceden/solana/magiceden_solana_assert_samples.sql +++ b/spellbook/tests/magiceden/solana/magiceden_solana_assert_samples.sql @@ -1,11 +1,13 @@ --- Bootstrapped correctness test against legacy Postgres values. --- Also manually check solscan info for the first 5 rows +-- Bootstrapped correctness test against values downloaded from the Dune App +-- The first 10 values were also manually checked using Solscan API WITH unit_tests as -(SELECT case when test_data.amount = os_trades.amount_original then True else False end as price_test -FROM {{ ref('magiceden_solana_trades') }} os_trades -JOIN {{ ref('magiceden_solana_trades_postgres') }} test_data ON test_data.tx_hash = os_trades.tx_hash -AND test_data.block_time = os_trades.block_time +(SELECT case when test_data.amount = me_trades.amount_original then True else False end as price_test +FROM {{ ref('nft_trades') }} me_trades +JOIN {{ ref('magiceden_solana_trades_solscan') }} test_data ON test_data.tx_hash = me_trades.tx_hash +AND test_data.block_time = me_trades.block_time +WHERE me_trades.block_time > '2021-10-23' and me_trades.block_time < '2021-10-25' +and me_trades.project = 'magiceden' and me_trades.blockchain = 'solana' ) select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows diff --git a/spellbook/tests/opensea/ethereum/opensea_ethereum_assert_samples.sql b/spellbook/tests/opensea/ethereum/opensea_ethereum_assert_samples.sql index 418a2bb2bfb..1651312f67f 100644 --- a/spellbook/tests/opensea/ethereum/opensea_ethereum_assert_samples.sql +++ b/spellbook/tests/opensea/ethereum/opensea_ethereum_assert_samples.sql @@ -3,8 +3,10 @@ -- Also manually check etherscan info for the first 5 rows WITH unit_tests as (SELECT case when test_data.price = os_trades.amount_raw then True else False end as price_test -FROM {{ ref('opensea_ethereum_trades') }} os_trades +FROM {{ ref('nft_trades') }} os_trades JOIN {{ ref('opensea_ethereum_trades_postgres') }} test_data ON test_data.evt_tx_hash = os_trades.tx_hash +WHERE os_trades.block_time > '2022-05-22' and os_trades.block_time < '2022-05-24' +and os_trades.project = 'opensea' and os_trades.blockchain = 'ethereum' ) select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows from unit_tests diff --git a/spellbook/tests/opensea/solana/opensea_solana_assert_samples.sql b/spellbook/tests/opensea/solana/opensea_solana_assert_samples.sql index 22ae0522f7c..87a48766c18 100644 --- a/spellbook/tests/opensea/solana/opensea_solana_assert_samples.sql +++ b/spellbook/tests/opensea/solana/opensea_solana_assert_samples.sql @@ -1,11 +1,13 @@ --- Bootstrapped correctness test against legacy Postgres values. --- Also manually check solscan info for the first 5 rows +-- Bootstrapped correctness test against values downloaded from the Dune App +-- The first 10 values were also manually checked using Solscan API WITH unit_tests as (SELECT case when test_data.amount = os_trades.amount_original then True else False end as price_test FROM {{ ref('opensea_solana_trades') }} os_trades -JOIN {{ ref('opensea_solana_trades_postgres') }} test_data ON test_data.tx_hash = os_trades.tx_hash +JOIN {{ ref('opensea_solana_trades_solscan') }} test_data ON test_data.tx_hash = os_trades.tx_hash AND test_data.block_time = os_trades.block_time +WHERE os_trades.block_time > '2022-05-01' and os_trades.block_time < '2022-05-03' +and os_trades.project = 'opensea' and os_trades.blockchain = 'solana' ) select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows diff --git a/spellbook/tests/uniswap/ethereum/uniswap_v2_ethereum_assert_samples.sql b/spellbook/tests/uniswap/ethereum/uniswap_v2_ethereum_assert_samples.sql index f35de973dc4..f4ac1180df2 100644 --- a/spellbook/tests/uniswap/ethereum/uniswap_v2_ethereum_assert_samples.sql +++ b/spellbook/tests/uniswap/ethereum/uniswap_v2_ethereum_assert_samples.sql @@ -6,8 +6,9 @@ -- Also manually check etherscan info for the first 5 rows WITH unit_tests as (SELECT case when test_data.token_a_amount = us_trades.token_a_amount AND test_data.token_b_amount = us_trades.token_b_amount then True else False end as price_test -FROM {{ ref('uniswap_v2_ethereum_trades') }} us_trades +FROM {{ ref('uniswap_trades') }} us_trades JOIN {{ ref('uniswap_v2_ethereum_trades_postgres') }} test_data ON test_data.tx_hash = us_trades.tx_hash +WHERE us_trades.blockchain = 'ethereum' and us_trades.version = 'v2' ) select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows from unit_tests diff --git a/spellbook/tests/uniswap/ethereum/uniswap_v3_ethereum_assert_samples.sql b/spellbook/tests/uniswap/ethereum/uniswap_v3_ethereum_assert_samples.sql index 5aa4c85ad51..c2a03b3f5fd 100644 --- a/spellbook/tests/uniswap/ethereum/uniswap_v3_ethereum_assert_samples.sql +++ b/spellbook/tests/uniswap/ethereum/uniswap_v3_ethereum_assert_samples.sql @@ -6,8 +6,9 @@ -- Also manually check etherscan info for the first 5 rows WITH unit_tests as (SELECT case when test_data.token_a_amount = us_trades.token_a_amount AND test_data.token_b_amount = us_trades.token_b_amount then True else False end as price_test -FROM {{ ref('uniswap_v3_ethereum_trades') }} us_trades +FROM {{ ref('uniswap_trades') }} us_trades JOIN {{ ref('uniswap_v3_ethereum_trades_postgres') }} test_data ON test_data.tx_hash = us_trades.tx_hash +WHERE us_trades.blockchain = 'ethereum' and us_trades.version = 'v3' ) select count(case when price_test = false then 1 else null end)/count(*) as pct_mismatch, count(*) as count_rows from unit_tests