From fe59cdeb0e8b5b81c95a5b28d6bf11f075a7c426 Mon Sep 17 00:00:00 2001 From: Sam <110511194+sam-xyz@users.noreply.github.com> Date: Thu, 30 Jan 2025 01:17:20 +0800 Subject: [PATCH] An 5714/traces2 hotswap bsc (#306) * comment out and delete workflow * add * handle overflow * overflow * updates --------- Co-authored-by: Austin --- .../workflows/dbt_run_overflowed_traces2.yml | 46 -- .github/workflows/dbt_run_traces_fix.yml | 50 -- .../bronze__potential_overflowed_traces.sql | 60 +- .../v2/bronze__overflowed_traces2.sql | 80 --- .../bronze__potential_overflowed_traces2.sql | 77 --- models/gold/core/core__fact_traces.sql | 536 +++++++++++++++++- models/gold/core/core__fact_traces.yml | 63 +- .../tests/test_gold__fact_traces_full.sql | 9 + .../tests/test_gold__fact_traces_full.yml | 120 ++++ .../tests/test_gold__fact_traces_recent.sql | 27 + .../tests/test_gold__fact_traces_recent.yml | 120 ++++ ...ver_observability__traces_completeness.sql | 5 +- models/silver/abis/silver__proxies.sql | 11 +- .../abis/silver__user_verified_abis.sql | 2 +- .../silver/core/silver__created_contracts.sql | 4 +- models/silver/core/silver__fact_traces2.sql | 531 ----------------- .../silver/core/silver__native_transfers.sql | 21 +- .../core/silver__relevant_contracts.sql | 6 +- models/silver/core/silver__traces.sql | 475 ++-------------- models/silver/core/silver__traces2.sql | 14 - .../tests/traces/test_silver__traces_full.sql | 2 +- .../traces/test_silver__traces_recent.sql | 2 +- .../traces2_fix/silver__fact_traces2_fix.sql | 13 +- .../silver_bridge__stargate_createpool.sql | 15 +- .../hashflow/silver_dex__hashflow_pools.sql | 8 +- .../dforce/silver__dforce_asset_details.sql | 21 +- .../nft/sales/silver__element_sales.sql | 8 +- .../nft/sales/silver__tofunft_sales.sql | 6 +- .../silver__overflowed_traces.sql | 158 ++---- .../silver__run_overflowed_traces.sql | 18 +- .../v2/silver__overflowed_traces2.sql | 117 ---- models/sources.yml | 7 +- .../streamline__bnb_balances_history.sql | 4 +- .../streamline__bnb_balances_realtime.sql | 4 +- .../silver/core/retry/_missing_traces.sql | 4 +- tests/bsc/test_silver__missing_traces.sql | 2 +- .../test_silver__recent_missing_traces.sql | 2 +- 37 files changed, 1036 insertions(+), 1612 deletions(-) delete mode 100644 .github/workflows/dbt_run_overflowed_traces2.yml delete mode 100644 .github/workflows/dbt_run_traces_fix.yml delete mode 100644 models/bronze/overflow/v2/bronze__overflowed_traces2.sql delete mode 100644 models/bronze/overflow/v2/bronze__potential_overflowed_traces2.sql create mode 100644 models/gold/tests/test_gold__fact_traces_full.sql create mode 100644 models/gold/tests/test_gold__fact_traces_full.yml create mode 100644 models/gold/tests/test_gold__fact_traces_recent.sql create mode 100644 models/gold/tests/test_gold__fact_traces_recent.yml delete mode 100644 models/silver/core/silver__fact_traces2.sql delete mode 100644 models/silver/core/silver__traces2.sql delete mode 100644 models/silver/overflow/v2/silver__overflowed_traces2.sql diff --git a/.github/workflows/dbt_run_overflowed_traces2.yml b/.github/workflows/dbt_run_overflowed_traces2.yml deleted file mode 100644 index f75c2420..00000000 --- a/.github/workflows/dbt_run_overflowed_traces2.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_run_overflowed_traces2 -run-name: dbt_run_overflowed_traces2 - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - - - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m "bsc_models,tag:overflowed_traces2" --vars '{"OVERFLOWED_TRACES":True}' \ No newline at end of file diff --git a/.github/workflows/dbt_run_traces_fix.yml b/.github/workflows/dbt_run_traces_fix.yml deleted file mode 100644 index 68ae9e5c..00000000 --- a/.github/workflows/dbt_run_traces_fix.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: dbt_run_traces_fix -run-name: dbt_run_traces_fix - -on: - workflow_dispatch: - inputs: - use_backfill_env: - description: "Use the backfill environment" - type: boolean - schedule: - # every 15 minutes (see https://crontab.guru) - - cron: "*/15 * * * *" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: ${{ github.event_name == 'workflow_dispatch' && inputs.use_backfill_env && 'workflow_prod_backfill' || 'workflow_prod_2xl' }} - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: run traces fix model - run: | - dbt run -m "bsc_models,tag:traces_fix" diff --git a/models/bronze/overflow/bronze__potential_overflowed_traces.sql b/models/bronze/overflow/bronze__potential_overflowed_traces.sql index 00cd5869..fd58fab0 100644 --- a/models/bronze/overflow/bronze__potential_overflowed_traces.sql +++ b/models/bronze/overflow/bronze__potential_overflowed_traces.sql @@ -6,50 +6,62 @@ WITH impacted_blocks AS ( SELECT - VALUE :: INT AS block_number + blocks_impacted_array FROM - ( - SELECT - blocks_impacted_array - FROM - {{ ref("silver_observability__traces_completeness") }} - ORDER BY - test_timestamp DESC - LIMIT - 1 - ), LATERAL FLATTEN ( + {{ ref("silver_observability__traces_completeness") }} + ORDER BY + test_timestamp DESC + LIMIT + 1 +), all_missing AS ( + SELECT + DISTINCT VALUE :: INT AS block_number + FROM + impacted_blocks, + LATERAL FLATTEN ( input => blocks_impacted_array ) ), all_txs AS ( SELECT - t.block_number, - t.position, - t.tx_hash + block_number, + POSITION AS tx_position, + tx_hash FROM {{ ref("silver__transactions") }} - t - JOIN impacted_blocks USING (block_number) + JOIN all_missing USING (block_number) ), missing_txs AS ( SELECT - DISTINCT block_number, - POSITION, + DISTINCT txs.block_number, + txs.tx_position, file_name FROM - all_txs - LEFT JOIN {{ ref("silver__traces") }} - tr USING ( + all_txs txs + LEFT JOIN {{ source( + 'bsc_gold', + 'fact_traces' + ) }} + tr2 USING ( block_number, - tx_hash + tx_position ) JOIN {{ ref("streamline__complete_traces") }} USING (block_number) + LEFT JOIN {{ source( + 'bsc_silver', + 'overflowed_traces' + ) }} + ot USING ( + block_number, + tx_position + ) WHERE - tr.tx_hash IS NULL + tr2.block_number IS NULL + AND ot.block_number IS NULL ) SELECT block_number, - POSITION, + tx_position AS POSITION, file_name, build_scoped_file_url( @streamline.bronze.external_tables, diff --git a/models/bronze/overflow/v2/bronze__overflowed_traces2.sql b/models/bronze/overflow/v2/bronze__overflowed_traces2.sql deleted file mode 100644 index 2be73b9e..00000000 --- a/models/bronze/overflow/v2/bronze__overflowed_traces2.sql +++ /dev/null @@ -1,80 +0,0 @@ -{{ config ( - materialized = "view", - tags = ['overflowed_traces2'] -) }} - -{% for item in range( - 1, - 11 - ) %} - - SELECT - o.file_name, - f.block_number, - f.index_vals, - f.path, - f.key, - f.value_ - FROM - ( - SELECT - file_name, - file_url, - index_cols, - [overflowed_block, overflowed_tx] AS index_vals - FROM - ( - SELECT - block_number, - POSITION, - file_name, - file_url, - index_cols, - VALUE [0] AS overflowed_block, - VALUE [1] AS overflowed_tx, - block_number = overflowed_block - AND POSITION = overflowed_tx AS missing - FROM - ( - SELECT - block_number, - POSITION, - file_name, - file_url, - index_cols, - utils.udf_detect_overflowed_responses( - file_url, - index_cols - ) AS index_vals - FROM - {{ ref("bronze__potential_overflowed_traces2") }} - WHERE - row_no = {{ item }} - ), - LATERAL FLATTEN ( - input => index_vals - ) - ) - WHERE - missing = TRUE - ) o, - TABLE( - utils.udtf_flatten_overflowed_responses( - o.file_url, - o.index_cols, - [o.index_vals] - ) - ) f - WHERE - NOT IS_OBJECT( - f.value_ - ) - AND NOT IS_ARRAY( - f.value_ - ) - AND NOT IS_NULL_VALUE( - f.value_ - ) {% if not loop.last %} - UNION ALL - {% endif %} - {% endfor %} diff --git a/models/bronze/overflow/v2/bronze__potential_overflowed_traces2.sql b/models/bronze/overflow/v2/bronze__potential_overflowed_traces2.sql deleted file mode 100644 index fcf7fbb1..00000000 --- a/models/bronze/overflow/v2/bronze__potential_overflowed_traces2.sql +++ /dev/null @@ -1,77 +0,0 @@ -{{ config ( - materialized = "view", - tags = ['overflowed_traces2'] -) }} - -WITH impacted_blocks AS ( - - SELECT - blocks_impacted_array - FROM - {{ ref("silver_observability__traces_completeness") }} - ORDER BY - test_timestamp DESC - LIMIT - 1 -), all_missing AS ( - SELECT - DISTINCT VALUE :: INT AS block_number - FROM - impacted_blocks, - LATERAL FLATTEN ( - input => blocks_impacted_array - ) -), -all_txs AS ( - SELECT - block_number, - POSITION AS tx_position, - tx_hash - FROM - {{ ref("silver__transactions") }} - JOIN all_missing USING (block_number) -), -missing_txs AS ( - SELECT - DISTINCT txs.block_number, - txs.tx_position, - file_name - FROM - all_txs txs - LEFT JOIN {{ ref("silver__traces2") }} - tr2 USING ( - block_number, - tx_position - ) - JOIN {{ ref("streamline__complete_traces") }} USING (block_number) - LEFT JOIN {{ source( - 'bsc_silver', - 'overflowed_traces2' - ) }} - ot USING ( - block_number, - tx_position - ) - WHERE - tr2.block_number IS NULL - AND ot.block_number IS NULL -) -SELECT - block_number, - tx_position AS POSITION, - file_name, - build_scoped_file_url( - @streamline.bronze.external_tables, - file_name - ) AS file_url, - ['block_number', 'array_index'] AS index_cols, - ROW_NUMBER() over ( - ORDER BY - block_number ASC, - POSITION ASC - ) AS row_no -FROM - missing_txs -ORDER BY - block_number ASC, - POSITION ASC diff --git a/models/gold/core/core__fact_traces.sql b/models/gold/core/core__fact_traces.sql index 5b00c8e1..f7ee6cd8 100644 --- a/models/gold/core/core__fact_traces.sql +++ b/models/gold/core/core__fact_traces.sql @@ -1,43 +1,531 @@ -{{ config( - materialized = 'view', - persist_docs ={ "relation": true, - "columns": true } +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = "block_number", + incremental_predicates = [fsc_evm.standard_predicate()], + cluster_by = "block_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", + tags = ['core','non_realtime'], + full_refresh = false ) }} +{# {{ fsc_evm.gold_traces_v1( +full_reload_start_block = 5000000, +full_reload_blocks = 1000000, +uses_overflow_steps = TRUE +) }} +#} +WITH silver_traces AS ( + + SELECT + block_number, + tx_position, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + traces_id, + 'regular' AS source + FROM + {{ ref('silver__traces') }} + WHERE + 1 = 1 +{% if is_incremental() and not full_reload_mode %} +AND modified_timestamp > ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) {% elif is_incremental() and full_reload_mode %} +AND block_number BETWEEN ( + SELECT + MAX( + block_number + ) + FROM + {{ this }} +) +AND ( + SELECT + MAX( + block_number + ) + 1000000 + FROM + {{ this }} +) +{% else %} + AND block_number <= 5000000 +{% endif %} +UNION ALL +SELECT + block_number, + tx_position, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + traces_id, + 'overflow' AS source +FROM + {{ ref('silver__overflowed_traces') }} +WHERE + 1 = 1 + +{% if is_incremental() and not full_reload_mode %} +AND modified_timestamp > ( + SELECT + DATEADD('hour', -2, MAX(modified_timestamp)) + FROM + {{ this }}) {% elif is_incremental() and full_reload_mode %} + AND block_number BETWEEN ( + SELECT + MAX( + block_number + ) + FROM + {{ this }} + ) + AND ( + SELECT + MAX( + block_number + ) + 1000000 + FROM + {{ this }} + ) + {% else %} + AND block_number <= 5000000 + {% endif %} +), +sub_traces AS ( + SELECT + block_number, + tx_position, + parent_trace_address, + COUNT(*) AS sub_traces + FROM + silver_traces + GROUP BY + block_number, + tx_position, + parent_trace_address +), +trace_index_array AS ( + SELECT + block_number, + tx_position, + trace_address, + ARRAY_AGG(flat_value) AS number_array + FROM + ( + SELECT + block_number, + tx_position, + trace_address, + IFF( + VALUE :: STRING = 'ORIGIN', + -1, + VALUE :: INT + ) AS flat_value + FROM + silver_traces, + LATERAL FLATTEN ( + input => trace_address_array + ) + ) + GROUP BY + block_number, + tx_position, + trace_address +), +trace_index_sub_traces AS ( + SELECT + b.block_number, + b.tx_position, + b.trace_address, + IFNULL( + sub_traces, + 0 + ) AS sub_traces, + number_array, + ROW_NUMBER() over ( + PARTITION BY b.block_number, + b.tx_position + ORDER BY + number_array ASC + ) - 1 AS trace_index, + b.trace_json, + b.traces_id, + b.source + FROM + silver_traces b + LEFT JOIN sub_traces s + ON b.block_number = s.block_number + AND b.tx_position = s.tx_position + AND b.trace_address = s.parent_trace_address + JOIN trace_index_array n + ON b.block_number = n.block_number + AND b.tx_position = n.tx_position + AND b.trace_address = n.trace_address +), +errored_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + trace_json + FROM + trace_index_sub_traces + WHERE + trace_json :error :: STRING IS NOT NULL +), +error_logic AS ( + SELECT + b0.block_number, + b0.tx_position, + b0.trace_address, + b0.trace_json :error :: STRING AS error, + b1.trace_json :error :: STRING AS any_error, + b2.trace_json :error :: STRING AS origin_error + FROM + trace_index_sub_traces b0 + LEFT JOIN errored_traces b1 + ON b0.block_number = b1.block_number + AND b0.tx_position = b1.tx_position + AND b0.trace_address RLIKE CONCAT( + '^', + b1.trace_address, + '(_[0-9]+)*$' + ) + LEFT JOIN errored_traces b2 + ON b0.block_number = b2.block_number + AND b0.tx_position = b2.tx_position + AND b2.trace_address = 'ORIGIN' +), +aggregated_errors AS ( + SELECT + block_number, + tx_position, + trace_address, + error, + IFF(MAX(any_error) IS NULL + AND error IS NULL + AND origin_error IS NULL, TRUE, FALSE) AS trace_succeeded + FROM + error_logic + GROUP BY + block_number, + tx_position, + trace_address, + error, + origin_error), + json_traces AS ( + SELECT + block_number, + tx_position, + trace_address, + sub_traces, + number_array, + trace_index, + trace_json AS DATA, + trace_succeeded, + trace_json :error :: STRING AS error_reason, + trace_json :revertReason :: STRING AS revert_reason, + trace_json :from :: STRING AS from_address, + trace_json :to :: STRING AS to_address, + IFNULL( + trace_json :value :: STRING, + '0x0' + ) AS value_hex, + IFNULL( + utils.udf_hex_to_int( + trace_json :value :: STRING + ), + '0' + ) AS value_precise_raw, + utils.udf_decimal_adjust( + value_precise_raw, + 18 + ) AS value_precise, + value_precise :: FLOAT AS VALUE, + utils.udf_hex_to_int( + trace_json :gas :: STRING + ) :: INT AS gas, + utils.udf_hex_to_int( + trace_json :gasUsed :: STRING + ) :: INT AS gas_used, + trace_json :input :: STRING AS input, + trace_json :output :: STRING AS output, + trace_json :type :: STRING AS TYPE, + concat_ws( + '_', + TYPE, + trace_address + ) AS identifier, + IFF( + trace_succeeded, + 'SUCCESS', + 'FAIL' + ) AS trace_status, + traces_id + FROM + trace_index_sub_traces + JOIN aggregated_errors USING ( + block_number, + tx_position, + trace_address + ) + ), + incremental_traces AS ( + SELECT + f.block_number, + t.tx_hash, + t.block_timestamp, + t.origin_function_signature, + t.from_address AS origin_from_address, + t.to_address AS origin_to_address, + t.tx_status, + f.tx_position, + f.trace_index, + f.from_address AS from_address, + f.to_address AS to_address, + f.value_hex, + f.value_precise_raw, + f.value_precise, + f.value, + f.gas, + f.gas_used, + f.input, + f.output, + f.type, + f.identifier, + f.sub_traces, + f.error_reason, + f.revert_reason, + f.trace_status, + f.data, + f.traces_id, + f.trace_succeeded, + f.trace_address, + IFF( + t.tx_status = 'SUCCESS', + TRUE, + FALSE + ) AS tx_succeeded + FROM + json_traces f + LEFT OUTER JOIN {{ ref('silver__transactions') }} + t + ON f.tx_position = t.position + AND f.block_number = t.block_number + +{% if is_incremental() and not full_reload_mode %} +AND t.modified_timestamp >= ( + SELECT + DATEADD('hour', -24, MAX(modified_timestamp)) + FROM + {{ this }}) + {% endif %} +) + +{% if is_incremental() %}, +overflow_blocks AS ( + SELECT + DISTINCT block_number + FROM + silver_traces + WHERE + source = 'overflow' +), +heal_missing_data AS ( + SELECT + t.block_number, + txs.tx_hash, + txs.block_timestamp, + txs.origin_function_signature, + txs.from_address AS origin_from_address, + txs.to_address AS origin_to_address, + txs.tx_status, + t.tx_position, + t.trace_index, + t.from_address, + t.to_address, + t.value_hex, + t.value_precise_raw, + t.value_precise, + t.value, + t.gas, + t.gas_used, + t.input, + t.output, + t.type, + t.identifier, + t.sub_traces, + t.error_reason, + t.revert_reason, + t.trace_status, + t.data, + t.fact_traces_id AS traces_id, + t.trace_succeeded, + t.trace_address, + IFF( + txs.tx_status = 'SUCCESS', + TRUE, + FALSE + ) AS tx_succeeded + FROM + {{ this }} + t + JOIN {{ ref('silver__transactions') }} + txs + ON t.tx_position = txs.position + AND t.block_number = txs.block_number + WHERE + t.tx_hash IS NULL + OR t.block_timestamp IS NULL + OR t.tx_status IS NULL +) +{% endif %}, +all_traces AS ( + SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, + sub_traces, + error_reason, + revert_reason, + trace_status, + DATA, + trace_succeeded, + trace_address, + tx_succeeded + FROM + incremental_traces + +{% if is_incremental() %} +UNION ALL SELECT - tx_hash, block_number, + tx_hash, block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_status, + tx_position, + trace_index, from_address, to_address, - bnb_value AS VALUE, - bnb_value_precise_raw AS value_precise_raw, - bnb_value_precise AS value_precise, + value_hex, + value_precise_raw, + value_precise, + VALUE, gas, gas_used, input, output, TYPE, identifier, + sub_traces, + error_reason, + revert_reason, + trace_status, DATA, + trace_succeeded, + trace_address, + tx_succeeded +FROM + heal_missing_data +UNION ALL +SELECT + block_number, + tx_hash, + block_timestamp, + origin_function_signature, + origin_from_address, + origin_to_address, tx_status, + tx_position, + trace_index, + from_address, + to_address, + value_hex, + value_precise_raw, + value_precise, + VALUE, + gas, + gas_used, + input, + output, + TYPE, + identifier, sub_traces, - trace_status, error_reason, + revert_reason, + trace_status, + DATA, + trace_succeeded, + trace_address, + tx_succeeded +FROM + {{ this }} + JOIN overflow_blocks USING (block_number) +{% endif %} +) +SELECT + block_number, + block_timestamp, + tx_hash, + tx_position, trace_index, - COALESCE ( - traces_id, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] - ) }} - ) AS fact_traces_id, - COALESCE( - inserted_timestamp, - '2000-01-01' - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' - ) AS modified_timestamp + from_address, + to_address, + input, + output, + TYPE, + trace_address, + sub_traces, + VALUE, + value_precise_raw, + value_precise, + value_hex, + gas, + gas_used, + origin_from_address, + origin_to_address, + origin_function_signature, + trace_succeeded, + error_reason, + revert_reason, + tx_succeeded, + identifier, + DATA, + tx_status, + trace_status, + {{ dbt_utils.generate_surrogate_key( + ['tx_hash', 'trace_index'] + ) }} AS fact_traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp FROM - {{ ref('silver__traces') }} + all_traces qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index +ORDER BY + modified_timestamp DESC, block_timestamp DESC nulls last)) = 1 diff --git a/models/gold/core/core__fact_traces.yml b/models/gold/core/core__fact_traces.yml index c6e975b9..3c4bcb20 100644 --- a/models/gold/core/core__fact_traces.yml +++ b/models/gold/core/core__fact_traces.yml @@ -1,55 +1,68 @@ version: 2 models: - name: core__fact_traces - description: '{{ doc("bsc_traces_table_doc") }}' + description: '{{ doc("bsc_traces_table_doc") }}' columns: - name: BLOCK_NUMBER - description: '{{ doc("bsc_traces_block_no") }}' + description: '{{ doc("bsc_traces_block_no") }}' - name: BLOCK_TIMESTAMP description: '{{ doc("bsc_traces_blocktime") }}' - name: TX_HASH - description: '{{ doc("bsc_traces_tx_hash") }}' + description: '{{ doc("bsc_traces_tx_hash") }}' + - name: TX_POSITION + description: The position of the transaction within the block. + - name: TRACE_INDEX + description: The index of the trace within the transaction. - name: FROM_ADDRESS description: '{{ doc("bsc_traces_from") }}' - name: TO_ADDRESS description: '{{ doc("bsc_traces_to") }}' - - name: VALUE - description: '{{ doc("bsc_traces_value") }}' - - name: VALUE_PRECISE_RAW - description: '{{ doc("precise_amount_unadjusted") }}' - - name: VALUE_PRECISE - description: '{{ doc("precise_amount_adjusted") }}' - - name: GAS - description: '{{ doc("bsc_traces_gas") }}' - - name: GAS_USED - description: '{{ doc("bsc_traces_gas_used") }}' - name: INPUT description: '{{ doc("bsc_traces_input") }}' - name: OUTPUT description: '{{ doc("bsc_traces_output") }}' - name: TYPE description: '{{ doc("bsc_traces_type") }}' + - name: TRACE_ADDRESS + description: This field represents the position of the trace within the transaction. + - name: SUB_TRACES + description: '{{ doc("bsc_traces_sub") }}' - name: IDENTIFIER description: '{{ doc("bsc_traces_identifier") }}' - name: DATA description: '{{ doc("bsc_traces_call_data") }}' - - name: TX_STATUS - description: '{{ doc("bsc_tx_status") }}' - - name: SUB_TRACES - description: '{{ doc("bsc_traces_sub") }}' + - name: VALUE + description: '{{ doc("bsc_traces_value") }}' + - name: VALUE_PRECISE_RAW + description: '{{ doc("precise_amount_unadjusted") }}' + - name: VALUE_PRECISE + description: '{{ doc("precise_amount_adjusted") }}' + - name: GAS + description: '{{ doc("bsc_traces_gas") }}' + - name: GAS_USED + description: '{{ doc("bsc_traces_gas_used") }}' + - name: ORIGIN_FROM_ADDRESS + description: The from address at the transaction level. + - name: ORIGIN_TO_ADDRESS + description: The to address at the transaction level. + - name: ORIGIN_FUNCTION_SIGNATURE + description: The function signature at the transaction level. - name: TRACE_STATUS description: The status of the trace, either `SUCCESS` or `FAIL` + - name: TRACE_SUCCEEDED + description: Whether the trace succeeded or failed - name: ERROR_REASON description: The reason for the trace failure, if any. - - name: TRACE_INDEX - description: The index of the trace within the transaction. + - name: REVERT_REASON + description: The reason for the trace revert, if any. + - name: TX_STATUS + description: '{{ doc("bsc_tx_status") }}' + - name: TX_SUCCEEDED + description: Whether the transaction succeeded or failed - name: FACT_TRACES_ID - description: '{{ doc("pk") }}' + description: '{{ doc("pk") }}' - name: INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' + description: '{{ doc("inserted_timestamp") }}' - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' - - - \ No newline at end of file + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/tests/test_gold__fact_traces_full.sql b/models/gold/tests/test_gold__fact_traces_full.sql new file mode 100644 index 00000000..8d9a458b --- /dev/null +++ b/models/gold/tests/test_gold__fact_traces_full.sql @@ -0,0 +1,9 @@ +{{ config ( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref('core__fact_traces') }} diff --git a/models/gold/tests/test_gold__fact_traces_full.yml b/models/gold/tests/test_gold__fact_traces_full.yml new file mode 100644 index 00000000..915e27e5 --- /dev/null +++ b/models/gold/tests/test_gold__fact_traces_full.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold__fact_traces_full + description: "This is a view used to test all of the gold fact traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - fsc_utils.sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/gold/tests/test_gold__fact_traces_recent.sql b/models/gold/tests/test_gold__fact_traces_recent.sql new file mode 100644 index 00000000..5f8db5f1 --- /dev/null +++ b/models/gold/tests/test_gold__fact_traces_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('core__fact_traces') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/gold/tests/test_gold__fact_traces_recent.yml b/models/gold/tests/test_gold__fact_traces_recent.yml new file mode 100644 index 00000000..b79fafdd --- /dev/null +++ b/models/gold/tests/test_gold__fact_traces_recent.yml @@ -0,0 +1,120 @@ +version: 2 +models: + - name: test_gold__fact_traces_recent + description: "This is a view used to test all of the gold fact traces model." + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - TRACE_INDEX + - fsc_utils.sequence_gaps: + partition_by: + - TX_HASH + column_name: TRACE_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 AND TX_HASH IS NOT NULL + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TX_POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRACE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: INPUT + tests: + - not_null + - name: TYPE + tests: + - not_null + - name: TRACE_ADDRESS + tests: + - not_null + - name: SUB_TRACES + tests: + - not_null + - name: VALUE + tests: + - not_null + - name: VALUE_PRECISE_RAW + tests: + - not_null + - name: VALUE_PRECISE + tests: + - not_null + - name: VALUE_HEX + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: TRACE_SUCCEEDED + tests: + - not_null + - name: TX_SUCCEEDED + tests: + - not_null + - name: FACT_TRACES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 2 \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__traces_completeness.sql b/models/silver/_observability/silver_observability__traces_completeness.sql index a4ca260f..18d0f9c6 100644 --- a/models/silver/_observability/silver_observability__traces_completeness.sql +++ b/models/silver/_observability/silver_observability__traces_completeness.sql @@ -81,7 +81,10 @@ broken_blocks AS ( FROM {{ ref("silver__transactions") }} tx - LEFT JOIN {{ ref("silver__traces") }} + LEFT JOIN {{ source( + 'bsc_gold', + 'fact_traces' + ) }} tr USING ( block_number, tx_hash diff --git a/models/silver/abis/silver__proxies.sql b/models/silver/abis/silver__proxies.sql index bb35fa32..3cceeaf1 100644 --- a/models/silver/abis/silver__proxies.sql +++ b/models/silver/abis/silver__proxies.sql @@ -11,9 +11,9 @@ WITH base AS ( from_address, to_address, MIN(block_number) AS start_block, - MAX(_inserted_timestamp) AS _inserted_timestamp + MAX(modified_timestamp) AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE TYPE = 'DELEGATECALL' AND trace_status = 'SUCCESS' @@ -21,7 +21,7 @@ WITH base AS ( AND from_address != to_address -- exclude self-calls {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '24 hours' FROM @@ -99,5 +99,6 @@ FROM FINAL f JOIN {{ ref('silver__created_contracts') }} C ON f.contract_address = C.created_contract_address - JOIN {{ ref('silver__created_contracts') }} p - ON f.proxy_address = p.created_contract_address \ No newline at end of file + JOIN {{ ref('silver__created_contracts') }} + p + ON f.proxy_address = p.created_contract_address diff --git a/models/silver/abis/silver__user_verified_abis.sql b/models/silver/abis/silver__user_verified_abis.sql index 66eb5ffb..2e466578 100644 --- a/models/silver/abis/silver__user_verified_abis.sql +++ b/models/silver/abis/silver__user_verified_abis.sql @@ -473,7 +473,7 @@ valid_traces AS ( SELECT base_address FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} JOIN function_mapping ON function_sig = LEFT( input, diff --git a/models/silver/core/silver__created_contracts.sql b/models/silver/core/silver__created_contracts.sql index a1b2b074..e6b9a5bc 100644 --- a/models/silver/core/silver__created_contracts.sql +++ b/models/silver/core/silver__created_contracts.sql @@ -14,7 +14,7 @@ SELECT to_address AS created_contract_address, from_address AS creator_address, input AS created_contract_input, - _inserted_timestamp, + modified_timestamp AS _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( ['to_address'] ) }} AS created_contracts_id, @@ -22,7 +22,7 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE TYPE ILIKE 'create%' AND to_address IS NOT NULL diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql deleted file mode 100644 index 8ddb020d..00000000 --- a/models/silver/core/silver__fact_traces2.sql +++ /dev/null @@ -1,531 +0,0 @@ -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = "block_number", - incremental_predicates = [fsc_evm.standard_predicate()], - cluster_by = "block_timestamp::date", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['core','non_realtime'], - full_refresh = false -) }} -{# {{ fsc_evm.gold_traces_v1( -full_reload_start_block = 5000000, -full_reload_blocks = 1000000, -uses_overflow_steps = TRUE -) }} -#} -WITH silver_traces AS ( - - SELECT - block_number, - tx_position, - trace_address, - parent_trace_address, - trace_address_array, - trace_json, - traces_id, - 'regular' AS source - FROM - {{ ref('silver__traces2') }} - WHERE - 1 = 1 - -{% if is_incremental() and not full_reload_mode %} -AND modified_timestamp > ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) {% elif is_incremental() and full_reload_mode %} -AND block_number BETWEEN ( - SELECT - MAX( - block_number - ) - FROM - {{ this }} -) -AND ( - SELECT - MAX( - block_number - ) + 1000000 - FROM - {{ this }} -) -{% else %} - AND block_number <= 5000000 -{% endif %} -UNION ALL -SELECT - block_number, - tx_position, - trace_address, - parent_trace_address, - trace_address_array, - trace_json, - traces_id, - 'overflow' AS source -FROM - {{ ref('silver__overflowed_traces2') }} -WHERE - 1 = 1 - -{% if is_incremental() and not full_reload_mode %} -AND modified_timestamp > ( - SELECT - DATEADD('hour', -2, MAX(modified_timestamp)) - FROM - {{ this }}) {% elif is_incremental() and full_reload_mode %} - AND block_number BETWEEN ( - SELECT - MAX( - block_number - ) - FROM - {{ this }} - ) - AND ( - SELECT - MAX( - block_number - ) + 1000000 - FROM - {{ this }} - ) - {% else %} - AND block_number <= 5000000 - {% endif %} -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - silver_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -trace_index_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS number_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - silver_traces, - LATERAL FLATTEN ( - input => trace_address_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -trace_index_sub_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - number_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - number_array ASC - ) - 1 AS trace_index, - b.trace_json, - b.traces_id, - b.source - FROM - silver_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN trace_index_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address -), -errored_traces AS ( - SELECT - block_number, - tx_position, - trace_address, - trace_json - FROM - trace_index_sub_traces - WHERE - trace_json :error :: STRING IS NOT NULL -), -error_logic AS ( - SELECT - b0.block_number, - b0.tx_position, - b0.trace_address, - b0.trace_json :error :: STRING AS error, - b1.trace_json :error :: STRING AS any_error, - b2.trace_json :error :: STRING AS origin_error - FROM - trace_index_sub_traces b0 - LEFT JOIN errored_traces b1 - ON b0.block_number = b1.block_number - AND b0.tx_position = b1.tx_position - AND b0.trace_address RLIKE CONCAT( - '^', - b1.trace_address, - '(_[0-9]+)*$' - ) - LEFT JOIN errored_traces b2 - ON b0.block_number = b2.block_number - AND b0.tx_position = b2.tx_position - AND b2.trace_address = 'ORIGIN' -), -aggregated_errors AS ( - SELECT - block_number, - tx_position, - trace_address, - error, - IFF(MAX(any_error) IS NULL - AND error IS NULL - AND origin_error IS NULL, TRUE, FALSE) AS trace_succeeded - FROM - error_logic - GROUP BY - block_number, - tx_position, - trace_address, - error, - origin_error), - json_traces AS ( - SELECT - block_number, - tx_position, - trace_address, - sub_traces, - number_array, - trace_index, - trace_json AS DATA, - trace_succeeded, - trace_json :error :: STRING AS error_reason, - trace_json :revertReason :: STRING AS revert_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - trace_json :value :: STRING, - '0x0' - ) AS value_hex, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS value_precise_raw, - utils.udf_decimal_adjust( - value_precise_raw, - 18 - ) AS value_precise, - value_precise :: FLOAT AS VALUE, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - IFF( - trace_succeeded, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - traces_id - FROM - trace_index_sub_traces - JOIN aggregated_errors USING ( - block_number, - tx_position, - trace_address - ) - ), - incremental_traces AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.origin_function_signature, - t.from_address AS origin_from_address, - t.to_address AS origin_to_address, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address AS from_address, - f.to_address AS to_address, - f.value_hex, - f.value_precise_raw, - f.value_precise, - f.value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - f.revert_reason, - f.trace_status, - f.data, - f.traces_id, - f.trace_succeeded, - f.trace_address, - IFF( - t.tx_status = 'SUCCESS', - TRUE, - FALSE - ) AS tx_succeeded - FROM - json_traces f - LEFT OUTER JOIN {{ ref('silver__transactions') }} - t - ON f.tx_position = t.position - AND f.block_number = t.block_number - -{% if is_incremental() and not full_reload_mode %} -AND t.modified_timestamp >= ( - SELECT - DATEADD('hour', -24, MAX(modified_timestamp)) - FROM - {{ this }}) - {% endif %} -) - -{% if is_incremental() %}, -overflow_blocks AS ( - SELECT - DISTINCT block_number - FROM - silver_traces - WHERE - source = 'overflow' -), -heal_missing_data AS ( - SELECT - t.block_number, - txs.tx_hash, - txs.block_timestamp, - txs.origin_function_signature, - txs.from_address AS origin_from_address, - txs.to_address AS origin_to_address, - txs.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.value_hex, - t.value_precise_raw, - t.value_precise, - t.value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - t.revert_reason, - t.trace_status, - t.data, - t.fact_traces_id AS traces_id, - t.trace_succeeded, - t.trace_address, - IFF( - txs.tx_status = 'SUCCESS', - TRUE, - FALSE - ) AS tx_succeeded - FROM - {{ this }} - t - JOIN {{ ref('silver__transactions') }} - txs - ON t.tx_position = txs.position - AND t.block_number = txs.block_number - WHERE - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL -) -{% endif %}, -all_traces AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded - FROM - incremental_traces - -{% if is_incremental() %} -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded -FROM - heal_missing_data -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - origin_function_signature, - origin_from_address, - origin_to_address, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - value_hex, - value_precise_raw, - value_precise, - VALUE, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - revert_reason, - trace_status, - DATA, - trace_succeeded, - trace_address, - tx_succeeded -FROM - {{ this }} - JOIN overflow_blocks USING (block_number) -{% endif %} -) -SELECT - block_number, - block_timestamp, - tx_hash, - tx_position, - trace_index, - from_address, - to_address, - input, - output, - TYPE, - trace_address, - sub_traces, - VALUE, - value_precise_raw, - value_precise, - value_hex, - gas, - gas_used, - origin_from_address, - origin_to_address, - origin_function_signature, - trace_succeeded, - error_reason, - revert_reason, - tx_succeeded, - identifier, - DATA, - tx_status, - trace_status, - {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] - ) }} AS fact_traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp -FROM - all_traces qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index -ORDER BY - modified_timestamp DESC, block_timestamp DESC nulls last)) = 1 diff --git a/models/silver/core/silver__native_transfers.sql b/models/silver/core/silver__native_transfers.sql index 4469da88..5073c00a 100644 --- a/models/silver/core/silver__native_transfers.sql +++ b/models/silver/core/silver__native_transfers.sql @@ -16,15 +16,24 @@ WITH bnb_base AS ( identifier, from_address, to_address, - bnb_value, - _call_id, - _inserted_timestamp, - bnb_value_precise_raw, - bnb_value_precise, + VALUE AS bnb_value, + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp, + value_precise_raw AS bnb_value_precise_raw, + value_precise AS bnb_value_precise, tx_position, trace_index FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE bnb_value > 0 AND tx_status = 'SUCCESS' diff --git a/models/silver/core/silver__relevant_contracts.sql b/models/silver/core/silver__relevant_contracts.sql index 85b5b2bc..23aefb70 100644 --- a/models/silver/core/silver__relevant_contracts.sql +++ b/models/silver/core/silver__relevant_contracts.sql @@ -31,10 +31,10 @@ function_calls AS ( SELECT to_address AS contract_address, COUNT(*) AS function_call_count, - MAX(_inserted_timestamp) AS max_inserted_timestamp_traces, + MAX(modified_timestamp) AS max_inserted_timestamp_traces, MAX(block_number) AS latest_call_block FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE tx_status = 'SUCCESS' AND trace_status = 'SUCCESS' @@ -43,7 +43,7 @@ function_calls AS ( AND input <> '0x' {% if is_incremental() %} -AND _inserted_timestamp > ( +AND modified_timestamp > ( SELECT MAX(max_inserted_timestamp_traces) FROM diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index c06f3829..da1f5b58 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -1,26 +1,29 @@ -- depends_on: {{ ref('bronze__streamline_traces') }} -{% set warehouse = 'DBT_SNOWPARK' if var('OVERFLOWED_TRACES') else target.warehouse %} {{ config ( materialized = "incremental", incremental_strategy = 'delete+insert', unique_key = "block_number", - cluster_by = "block_timestamp::date, _inserted_timestamp::date", + cluster_by = ['modified_timestamp::DATE','partition_key'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['core','non_realtime','overflowed_traces'], - full_refresh = false, - snowflake_warehouse = warehouse + tags = ['core','non_realtime'], + full_refresh = false ) }} - +{# {{ fsc_evm.silver_traces_v1( +full_reload_start_block = 2300000, +full_reload_blocks = 1000000 +) }} +#} WITH bronze_traces AS ( SELECT block_number, + _partition_by_block_id AS partition_key, VALUE :array_index :: INT AS tx_position, DATA :result AS full_traces, _inserted_timestamp FROM -{% if is_incremental() %} +{% if is_incremental() and not full_reload_mode %} {{ ref('bronze__streamline_traces') }} WHERE _inserted_timestamp >= ( @@ -29,6 +32,21 @@ WHERE FROM {{ this }} ) + AND DATA :result IS NOT NULL {% elif is_incremental() and full_reload_mode %} + {{ ref('bronze__streamline_fr_traces') }} +WHERE + _partition_by_block_id BETWEEN ( + SELECT + MAX(_partition_by_block_id) - 100000 + FROM + {{ this }} + ) + AND ( + SELECT + MAX(_partition_by_block_id) + 1000000 + FROM + {{ this }} + ) {% else %} {{ ref('bronze__streamline_fr_traces') }} WHERE @@ -43,6 +61,7 @@ flatten_traces AS ( SELECT block_number, tx_position, + partition_key, IFF( path IN ( 'result', @@ -56,6 +75,7 @@ flatten_traces AS ( 'result.output', 'result.error', 'result.revertReason', + 'result.time', 'gasUsed', 'gas', 'type', @@ -65,6 +85,7 @@ flatten_traces AS ( 'input', 'error', 'output', + 'time', 'revertReason' ), 'ORIGIN', @@ -91,7 +112,7 @@ flatten_traces AS ( SPLIT( trace_address, '_' - ) AS str_array + ) AS trace_address_array FROM bronze_traces txs, TABLE( @@ -109,446 +130,26 @@ flatten_traces AS ( GROUP BY block_number, tx_position, + partition_key, trace_address, _inserted_timestamp -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - flatten_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -num_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - flatten_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -cleaned_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - flatten_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address -), -final_traces AS ( - SELECT - tx_position, - trace_index, - block_number, - trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS bnb_value_precise_raw, - utils.udf_decimal_adjust( - bnb_value_precise_raw, - 18 - ) AS bnb_value_precise, - bnb_value_precise :: FLOAT AS bnb_value, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, - _inserted_timestamp, - trace_json AS DATA, - sub_traces - FROM - cleaned_traces -), -new_records AS ( - SELECT - f.block_number, - t.tx_hash, - t.block_timestamp, - t.tx_status, - f.tx_position, - f.trace_index, - f.from_address, - f.to_address, - f.bnb_value_precise_raw, - f.bnb_value_precise, - f.bnb_value, - f.gas, - f.gas_used, - f.input, - f.output, - f.type, - f.identifier, - f.sub_traces, - f.error_reason, - IFF( - f.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - f.data, - IFF( - t.tx_hash IS NULL - OR t.block_timestamp IS NULL - OR t.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - f._call_id, - f._inserted_timestamp - FROM - final_traces f - LEFT OUTER JOIN {{ ref('silver__transactions') }} - t - ON f.tx_position = t.position - AND f.block_number = t.block_number - -{% if is_incremental() %} -AND t._INSERTED_TIMESTAMP >= ( - SELECT - DATEADD('hour', -24, MAX(_inserted_timestamp)) - FROM - {{ this }}) - {% endif %} -) - -{% if is_incremental() %}, -missing_data AS ( - SELECT - t.block_number, - txs.tx_hash, - txs.block_timestamp, - txs.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.bnb_value_precise_raw, - t.bnb_value_precise, - t.bnb_value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - t.trace_status, - t.data, - IFF( - txs.tx_hash IS NULL - OR txs.block_timestamp IS NULL - OR txs.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - t._call_id, - GREATEST( - t._inserted_timestamp, - txs._inserted_timestamp - ) AS _inserted_timestamp - FROM - {{ this }} - t - INNER JOIN {{ ref('silver__transactions') }} - txs - ON t.tx_position = txs.position - AND t.block_number = txs.block_number - WHERE - t.is_pending -) -{% endif %}, - -{% if is_incremental() and var( - 'OVERFLOWED_TRACES', -) %} -overflowed_traces AS ( - SELECT - t.block_number, - txs.tx_hash, - txs.block_timestamp, - txs.tx_status, - t.tx_position, - t.trace_index, - t.from_address, - t.to_address, - t.bnb_value_precise_raw, - t.bnb_value_precise, - t.bnb_value, - t.gas, - t.gas_used, - t.input, - t.output, - t.type, - t.identifier, - t.sub_traces, - t.error_reason, - IFF( - t.error_reason IS NULL, - 'SUCCESS', - 'FAIL' - ) AS trace_status, - t.data, - IFF( - txs.tx_hash IS NULL - OR txs.block_timestamp IS NULL - OR txs.tx_status IS NULL, - TRUE, - FALSE - ) AS is_pending, - t._call_id, - txs._inserted_timestamp AS _inserted_timestamp - FROM - {{ source( - 'bsc_silver', - 'overflowed_traces' - ) }} - t - LEFT JOIN {{ ref('silver__transactions') }} - txs - ON t.tx_position = txs.position - AND t.block_number = txs.block_number -), -{% endif %} - -FINAL AS ( - SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - bnb_value_precise_raw, - bnb_value_precise, - bnb_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp - FROM - new_records - -{% if is_incremental() %} -UNION -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - bnb_value_precise_raw, - bnb_value_precise, - bnb_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp -FROM - missing_data -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - bnb_value_precise_raw, - bnb_value_precise, - bnb_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp -FROM - {{ this }} - INNER JOIN ( - SELECT - DISTINCT block_number - FROM - missing_data - -{% if is_incremental() and var( - 'OVERFLOWED_TRACES', -) %} -UNION -SELECT - DISTINCT block_number -FROM - overflowed_traces -{% endif %} -) USING (block_number) -{% endif %} - -{% if is_incremental() and var( - 'OVERFLOWED_TRACES', -) %} -UNION ALL -SELECT - block_number, - tx_hash, - block_timestamp, - tx_status, - tx_position, - trace_index, - from_address, - to_address, - bnb_value_precise_raw, - bnb_value_precise, - bnb_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, - _inserted_timestamp -FROM - overflowed_traces -{% endif %} ) SELECT block_number, - tx_hash, - block_timestamp, - tx_status, tx_position, - trace_index, - from_address, - to_address, - bnb_value_precise, - bnb_value, - gas, - gas_used, - input, - output, - TYPE, - identifier, - sub_traces, - error_reason, - trace_status, - DATA, - is_pending, - _call_id, + trace_address, + parent_trace_address, + trace_address_array, + trace_json, + partition_key, _inserted_timestamp, - bnb_value_precise_raw, {{ dbt_utils.generate_surrogate_key( - ['tx_hash', 'trace_index'] + ['block_number', 'tx_position', 'trace_address'] ) }} AS traces_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index + flatten_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id ORDER BY - _inserted_timestamp DESC, is_pending ASC)) = 1 + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql deleted file mode 100644 index 91e1b788..00000000 --- a/models/silver/core/silver__traces2.sql +++ /dev/null @@ -1,14 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_traces') }} -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = "block_number", - cluster_by = ['modified_timestamp::DATE','partition_key'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['core','non_realtime'], - full_refresh = false -) }} -{{ fsc_evm.silver_traces_v1( - full_reload_start_block = 2300000, - full_reload_blocks = 1000000 -) }} diff --git a/models/silver/core/tests/traces/test_silver__traces_full.sql b/models/silver/core/tests/traces/test_silver__traces_full.sql index be52e71d..8d9a458b 100644 --- a/models/silver/core/tests/traces/test_silver__traces_full.sql +++ b/models/silver/core/tests/traces/test_silver__traces_full.sql @@ -6,4 +6,4 @@ SELECT * FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} diff --git a/models/silver/core/tests/traces/test_silver__traces_recent.sql b/models/silver/core/tests/traces/test_silver__traces_recent.sql index 1dc3e769..027b0380 100644 --- a/models/silver/core/tests/traces/test_silver__traces_recent.sql +++ b/models/silver/core/tests/traces/test_silver__traces_recent.sql @@ -13,7 +13,7 @@ WITH last_3_days AS ( SELECT * FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE block_number >= ( SELECT diff --git a/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql b/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql index 416d762a..79e9c243 100644 --- a/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql +++ b/models/silver/core/traces2_fix/silver__fact_traces2_fix.sql @@ -1,12 +1,11 @@ -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = ["block_number", "tx_position", "trace_address"], - tags = ['traces_fix'] +{# {{ config ( +materialized = "incremental", +incremental_strategy = 'delete+insert', +unique_key = ["block_number", "tx_position", "trace_address"], +tags = ['traces_fix'] ) }} {% set batch_query %} - SELECT MAX(next_batch_id) AS next_batch_id FROM @@ -161,4 +160,4 @@ aggregated_errors AS ( prod_trace_succeeded FROM batch - CROSS JOIN final_errors + CROSS JOIN final_errors #} diff --git a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql index 089824c6..10ee0a2c 100644 --- a/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql +++ b/models/silver/defi/bridge/stargate/silver_bridge__stargate_createpool.sql @@ -13,10 +13,19 @@ WITH base_contracts AS ( block_timestamp, from_address, to_address AS contract_address, - _call_id, - _inserted_timestamp + concat_ws( + '-', + block_number, + tx_position, + CONCAT( + TYPE, + '_', + trace_address + ) + ) AS _call_id, + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE from_address = LOWER('0xe7Ec689f432f29383f217e36e680B5C855051f25') AND TYPE ILIKE 'create%' diff --git a/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql b/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql index 73fb7cdb..2b515fb0 100644 --- a/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql +++ b/models/silver/defi/dex/hashflow/silver_dex__hashflow_pools.sql @@ -13,9 +13,9 @@ WITH contract_deployments AS ( block_timestamp, from_address AS deployer_address, to_address AS contract_address, - _inserted_timestamp + modified_timestamp AS _inserted_timestamp FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE from_address IN ( '0x63ae536fec0b57bdeb1fd6a893191b4239f61bff', @@ -40,7 +40,6 @@ qualify(ROW_NUMBER() over(PARTITION BY to_address ORDER BY block_timestamp ASC)) = 1 ) - SELECT tx_hash, block_number, @@ -48,4 +47,5 @@ SELECT deployer_address, contract_address AS pool_address, _inserted_timestamp -FROM contract_deployments +FROM + contract_deployments diff --git a/models/silver/defi/lending/dforce/silver__dforce_asset_details.sql b/models/silver/defi/lending/dforce/silver__dforce_asset_details.sql index c34709a5..7c95543f 100644 --- a/models/silver/defi/lending/dforce/silver__dforce_asset_details.sql +++ b/models/silver/defi/lending/dforce/silver__dforce_asset_details.sql @@ -18,8 +18,10 @@ WITH log_pull AS ( FROM {{ ref('silver__logs') }} WHERE - (topics [0] :: STRING = '0x70aea8d848e8a90fb7661b227dc522eb6395c3dac71b63cb59edd5c9899b2364' - AND origin_from_address = LOWER('0x4375c89AF5b4aF46791b05810C4B795A0470207F')) + ( + topics [0] :: STRING = '0x70aea8d848e8a90fb7661b227dc522eb6395c3dac71b63cb59edd5c9899b2364' + AND origin_from_address = LOWER('0x4375c89AF5b4aF46791b05810C4B795A0470207F') + ) OR tx_hash = '0xcf0ea37207cc7f54c90b6dcb8208cdcb247768836ad1d33ccbb6cbe5d13dee80' --edge case where Liqee token is token collateral in liquidations table {% if is_incremental() %} @@ -37,7 +39,7 @@ traces_pull AS ( from_address AS token_address, to_address AS underlying_asset FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE tx_hash IN ( SELECT @@ -46,14 +48,15 @@ traces_pull AS ( log_pull ) AND TYPE = 'STATICCALL' + {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} ) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' +AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' {% endif %} ), contracts AS ( @@ -108,16 +111,16 @@ WHERE underlying_asset IS NOT NULL AND l.token_name IS NOT NULL UNION ALL ---manually adding iBNB, no token creation log + --manually adding iBNB, no token creation log SELECT '0x593fd69bcd788afd9a19adae0783f5c429f821d6c46d20c742e5443b8a067d73' AS tx_hash, 6580133 AS block_number, - '2021-04-15 06:36:40.000' as block_timestamp, - lower('0xd57e1425837567f74a35d07669b23bfb67aa4a93') AS token_address, + '2021-04-15 06:36:40.000' AS block_timestamp, + LOWER('0xd57e1425837567f74a35d07669b23bfb67aa4a93') AS token_address, 'dForce BNB' AS token_name, 'iBNB' AS token_symbol, 18 AS token_decimals, - lower('0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c') AS underlying_asset_address, + LOWER('0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c') AS underlying_asset_address, 'Wrapped BNB' AS underlying_name, 'WBNB' AS underlying_symbol, 18 AS underlying_decimals, diff --git a/models/silver/nft/sales/silver__element_sales.sql b/models/silver/nft/sales/silver__element_sales.sql index b704223d..8ab98871 100644 --- a/models/silver/nft/sales/silver__element_sales.sql +++ b/models/silver/nft/sales/silver__element_sales.sql @@ -184,7 +184,7 @@ old_native_transfers AS ( trace_index, from_address, to_address, - bnb_value, + VALUE AS bnb_value, bnb_value * pow( 10, 18 @@ -200,7 +200,7 @@ old_native_transfers AS ( 0 ) AS intra_grouping FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE block_timestamp :: DATE >= ( SELECT @@ -250,13 +250,13 @@ old_native_transfers AS ( AND trace_status = 'SUCCESS' {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} ) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' +AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' {% endif %} ), old_native_labels AS ( diff --git a/models/silver/nft/sales/silver__tofunft_sales.sql b/models/silver/nft/sales/silver__tofunft_sales.sql index 74046fb8..c252197e 100644 --- a/models/silver/nft/sales/silver__tofunft_sales.sql +++ b/models/silver/nft/sales/silver__tofunft_sales.sql @@ -148,7 +148,7 @@ traces_raw AS ( ) ) :: INT AS bundle_array_size FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE block_timestamp :: DATE >= '2021-10-01' AND to_address = '0x449d05c544601631785a7c062dcdff530330317e' @@ -159,13 +159,13 @@ traces_raw AS ( AND trace_status = 'SUCCESS' {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} ) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' +AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' {% endif %} ), traces_raw_rn AS ( diff --git a/models/silver/overflow/formatted_views/silver__overflowed_traces.sql b/models/silver/overflow/formatted_views/silver__overflowed_traces.sql index 6d361ab7..ee267e0e 100644 --- a/models/silver/overflow/formatted_views/silver__overflowed_traces.sql +++ b/models/silver/overflow/formatted_views/silver__overflowed_traces.sql @@ -1,12 +1,24 @@ +-- depends_on: {{ ref('bronze__overflowed_traces') }} +{% set warehouse = 'DBT_SNOWPARK' if var('OVERFLOWED_TRACES') else target.warehouse %} {{ config ( - materialized = 'view', - tags = ['overflowed_traces'] + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = ['block_number','tx_position'], + cluster_by = ['modified_timestamp::DATE','partition_key'], + tags = ['overflowed_traces'], + full_refresh = false, + snowflake_warehouse = warehouse ) }} +{% if is_incremental() %} WITH bronze_overflowed_traces AS ( SELECT block_number :: INT AS block_number, + ROUND( + block_number, + -3 + ) AS partition_key, index_vals [1] :: INT AS tx_position, IFF( path IN ( @@ -58,7 +70,7 @@ WITH bronze_overflowed_traces AS ( SPLIT( trace_address, '_' - ) AS str_array + ) AS trace_address_array FROM {{ ref("bronze__overflowed_traces") }} GROUP BY @@ -66,118 +78,40 @@ WITH bronze_overflowed_traces AS ( tx_position, trace_address, _inserted_timestamp -), -sub_traces AS ( - SELECT - block_number, - tx_position, - parent_trace_address, - COUNT(*) AS sub_traces - FROM - bronze_overflowed_traces - GROUP BY - block_number, - tx_position, - parent_trace_address -), -num_array AS ( - SELECT - block_number, - tx_position, - trace_address, - ARRAY_AGG(flat_value) AS num_array - FROM - ( - SELECT - block_number, - tx_position, - trace_address, - IFF( - VALUE :: STRING = 'ORIGIN', - -1, - VALUE :: INT - ) AS flat_value - FROM - bronze_overflowed_traces, - LATERAL FLATTEN ( - input => str_array - ) - ) - GROUP BY - block_number, - tx_position, - trace_address -), -cleaned_traces AS ( - SELECT - b.block_number, - b.tx_position, - b.trace_address, - IFNULL( - sub_traces, - 0 - ) AS sub_traces, - num_array, - ROW_NUMBER() over ( - PARTITION BY b.block_number, - b.tx_position - ORDER BY - num_array ASC - ) - 1 AS trace_index, - trace_json, - b._inserted_timestamp - FROM - bronze_overflowed_traces b - LEFT JOIN sub_traces s - ON b.block_number = s.block_number - AND b.tx_position = s.tx_position - AND b.trace_address = s.parent_trace_address - JOIN num_array n - ON b.block_number = n.block_number - AND b.tx_position = n.tx_position - AND b.trace_address = n.trace_address ) SELECT - tx_position, - trace_index, block_number, + tx_position, trace_address, - trace_json :error :: STRING AS error_reason, - trace_json :from :: STRING AS from_address, - trace_json :to :: STRING AS to_address, - IFNULL( - utils.udf_hex_to_int( - trace_json :value :: STRING - ), - '0' - ) AS bnb_value_precise_raw, - utils.udf_decimal_adjust( - bnb_value_precise_raw, - 18 - ) AS bnb_value_precise, - bnb_value_precise :: FLOAT AS bnb_value, - utils.udf_hex_to_int( - trace_json :gas :: STRING - ) :: INT AS gas, - utils.udf_hex_to_int( - trace_json :gasUsed :: STRING - ) :: INT AS gas_used, - trace_json :input :: STRING AS input, - trace_json :output :: STRING AS output, - trace_json :type :: STRING AS TYPE, - concat_ws( - '_', - TYPE, - trace_address - ) AS identifier, - concat_ws( - '-', - block_number, - tx_position, - identifier - ) AS _call_id, + parent_trace_address, + trace_address_array, + trace_json, + partition_key, _inserted_timestamp, - trace_json AS DATA, - sub_traces + {{ dbt_utils.generate_surrogate_key( + ['block_number', 'tx_position', 'trace_address'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM - cleaned_traces + bronze_overflowed_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id +ORDER BY + _inserted_timestamp DESC)) = 1 +{% else %} +SELECT + NULL :: INT AS block_number, + NULL :: INT tx_position, + NULL :: text AS trace_address, + NULL :: text AS parent_trace_address, + NULL :: ARRAY AS trace_address_array, + NULL :: OBJECT AS trace_json, + NULL :: INT AS partition_key, + NULL :: timestamp_ltz AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_number', 'tx_position', 'trace_address'] + ) }} AS traces_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +{% endif %} diff --git a/models/silver/overflow/run_models/silver__run_overflowed_traces.sql b/models/silver/overflow/run_models/silver__run_overflowed_traces.sql index 9771188d..4dc69b16 100644 --- a/models/silver/overflow/run_models/silver__run_overflowed_traces.sql +++ b/models/silver/overflow/run_models/silver__run_overflowed_traces.sql @@ -23,13 +23,7 @@ WITH base AS ( 'bsc-models', 'dbt_run_overflowed_traces.yml', NULL - ) AS run_overflow_models, - github_actions.workflow_dispatches( - 'FlipsideCrypto', - 'bsc-models', - 'dbt_run_overflowed_traces2.yml', - NULL - ) AS run_overflow_models2 + ) AS run_overflow_models FROM base WHERE @@ -47,15 +41,7 @@ WITH base AS ( 'status', 'skipped' ) - ) AS run_overflow_models, - COALESCE( - run_overflow_models2, - OBJECT_CONSTRUCT( - 'status', - 'skipped' - ) - ) AS run_overflow_models2, - SYSDATE() AS test_timestamp + ) AS run_overflow_models SYSDATE() AS test_timestamp FROM ( SELECT diff --git a/models/silver/overflow/v2/silver__overflowed_traces2.sql b/models/silver/overflow/v2/silver__overflowed_traces2.sql deleted file mode 100644 index e5630f64..00000000 --- a/models/silver/overflow/v2/silver__overflowed_traces2.sql +++ /dev/null @@ -1,117 +0,0 @@ --- depends_on: {{ ref('bronze__overflowed_traces2') }} -{% set warehouse = 'DBT_SNOWPARK' if var('OVERFLOWED_TRACES') else target.warehouse %} -{{ config ( - materialized = "incremental", - incremental_strategy = 'delete+insert', - unique_key = ['block_number','tx_position'], - cluster_by = ['modified_timestamp::DATE','partition_key'], - tags = ['overflowed_traces2'], - full_refresh = false, - snowflake_warehouse = warehouse -) }} - -{% if is_incremental() %} -WITH bronze_overflowed_traces AS ( - - SELECT - block_number :: INT AS block_number, - ROUND( - block_number, - -3 - ) AS partition_key, - index_vals [1] :: INT AS tx_position, - IFF( - path IN ( - 'result', - 'result.value', - 'result.type', - 'result.to', - 'result.input', - 'result.gasUsed', - 'result.gas', - 'result.from', - 'result.output', - 'result.error', - 'result.revertReason', - 'result.txHash', - 'gasUsed', - 'gas', - 'type', - 'to', - 'from', - 'value', - 'input', - 'error', - 'output', - 'revertReason', - 'txHash' - ), - 'ORIGIN', - REGEXP_REPLACE(REGEXP_REPLACE(path, '[^0-9]+', '_'), '^_|_$', '') - ) AS trace_address, - SYSDATE() :: timestamp_ltz AS _inserted_timestamp, - OBJECT_AGG( - key, - value_ - ) AS trace_json, - CASE - WHEN trace_address = 'ORIGIN' THEN NULL - WHEN POSITION( - '_' IN trace_address - ) = 0 THEN 'ORIGIN' - ELSE REGEXP_REPLACE( - trace_address, - '_[0-9]+$', - '', - 1, - 1 - ) - END AS parent_trace_address, - SPLIT( - trace_address, - '_' - ) AS trace_address_array - FROM - {{ ref("bronze__overflowed_traces2") }} - GROUP BY - block_number, - tx_position, - trace_address, - _inserted_timestamp -) -SELECT - block_number, - tx_position, - trace_address, - parent_trace_address, - trace_address_array, - trace_json, - partition_key, - _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'tx_position', 'trace_address'] - ) }} AS traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - bronze_overflowed_traces qualify(ROW_NUMBER() over(PARTITION BY traces_id -ORDER BY - _inserted_timestamp DESC)) = 1 -{% else %} -SELECT - NULL :: INT AS block_number, - NULL :: INT tx_position, - NULL :: text AS trace_address, - NULL :: text AS parent_trace_address, - NULL :: ARRAY AS trace_address_array, - NULL :: OBJECT AS trace_json, - NULL :: INT AS partition_key, - NULL :: timestamp_ltz AS _inserted_timestamp, - {{ dbt_utils.generate_surrogate_key( - ['block_number', 'tx_position', 'trace_address'] - ) }} AS traces_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -{% endif %} diff --git a/models/sources.yml b/models/sources.yml index 72e67b0c..ed677642 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -65,4 +65,9 @@ sources: database: bsc_dev schema: silver tables: - - name: overflowed_traces2 \ No newline at end of file + - name: overflowed_traces2 + - name: bsc_gold + database: "{{ 'bsc' if target.database == 'BSC' else 'bsc_dev' }}" + schema: core + tables: + - name: fact_traces diff --git a/models/streamline/silver/balances/history/streamline__bnb_balances_history.sql b/models/streamline/silver/balances/history/streamline__bnb_balances_history.sql index 66bc55fb..e08e9c06 100644 --- a/models/streamline/silver/balances/history/streamline__bnb_balances_history.sql +++ b/models/streamline/silver/balances/history/streamline__bnb_balances_history.sql @@ -20,9 +20,9 @@ traces AS ( from_address, to_address FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE - bnb_value > 0 + VALUE > 0 AND trace_status = 'SUCCESS' AND tx_status = 'SUCCESS' AND block_number < ( diff --git a/models/streamline/silver/balances/realtime/streamline__bnb_balances_realtime.sql b/models/streamline/silver/balances/realtime/streamline__bnb_balances_realtime.sql index b5ed35d8..ec1f7250 100644 --- a/models/streamline/silver/balances/realtime/streamline__bnb_balances_realtime.sql +++ b/models/streamline/silver/balances/realtime/streamline__bnb_balances_realtime.sql @@ -24,9 +24,9 @@ traces AS ( from_address, to_address FROM - {{ ref('silver__traces') }} + {{ ref('core__fact_traces') }} WHERE - bnb_value > 0 + VALUE > 0 AND trace_status = 'SUCCESS' AND tx_status = 'SUCCESS' AND block_number >= ( diff --git a/models/streamline/silver/core/retry/_missing_traces.sql b/models/streamline/silver/core/retry/_missing_traces.sql index 8513fd6c..f4ea89d7 100644 --- a/models/streamline/silver/core/retry/_missing_traces.sql +++ b/models/streamline/silver/core/retry/_missing_traces.sql @@ -14,7 +14,7 @@ SELECT FROM {{ ref("silver__transactions") }} tx - LEFT JOIN {{ ref("silver__traces") }} + LEFT JOIN {{ ref("core__fact_traces") }} tr ON tx.block_number = tr.block_number AND tx.tx_hash = tr.tx_hash @@ -28,4 +28,4 @@ WHERE lookback ) AND tr.block_timestamp >= DATEADD('hour', -84, SYSDATE()) - AND tr.block_timestamp IS NOT NULL \ No newline at end of file + AND tr.block_timestamp IS NOT NULL diff --git a/tests/bsc/test_silver__missing_traces.sql b/tests/bsc/test_silver__missing_traces.sql index c96256f3..514d31cf 100644 --- a/tests/bsc/test_silver__missing_traces.sql +++ b/tests/bsc/test_silver__missing_traces.sql @@ -1,2 +1,2 @@ -- depends_on: {{ ref('test_silver__transactions_full') }} -{{ missing_txs(ref("test_silver__traces_full")) }} +{{ missing_txs(ref("test_gold__fact_traces_full")) }} diff --git a/tests/bsc/test_silver__recent_missing_traces.sql b/tests/bsc/test_silver__recent_missing_traces.sql index 537d8f1f..70e60af7 100644 --- a/tests/bsc/test_silver__recent_missing_traces.sql +++ b/tests/bsc/test_silver__recent_missing_traces.sql @@ -1,2 +1,2 @@ -- depends_on: {{ ref('test_silver__transactions_recent') }} -{{ recent_missing_traces(ref("test_silver__traces_recent"), 10000) }} +{{ recent_missing_traces(ref("test_gold__fact_traces_recent"), 10000) }}