feat(dbt): adds source and staging models for arb1 events (#2197)

* feat(dbt): adds source and staging models for arb1 events

* fix: update int_arbitrum_one_traces

* Update int_arbitrum_one_transactions.sql

* Adds gas_price conversion

* make sure the playgrounds work

* Reduce the need for 80 char line length

---------

Co-authored-by: Reuven V. Gonzales <[email protected]>

.sqlfluff

@@ -2,7 +2,7 @@
 dialect = bigquery
 templater = dbt
 runaway_limit = 10
-max_line_length = 80
+max_line_length = 100
 indent_unit = space
 exclude_rules = AL09, RF02
 

warehouse/dbt/models/arbitrum_one_sources.yml

@@ -0,0 +1,13 @@
+sources:
+  - name: arbitrum_one
+    database: opensource-observer
+    schema: arbitrum_one
+    tables:
+      - name: transactions
+        identifier: arbitrum_transactions
+
+      - name: blocks
+        identifier: arbitrum_blocks
+
+      - name: traces
+        identifier: arbitrum_traces

warehouse/dbt/models/base_playground_sources.yml

@@ -8,6 +8,15 @@ sources:
       - name: arbitrum_deployers
         identifier: stg_arbitrum__deployers
 
+      - name: arbitrum_one_traces
+        identifier: int_arbitrum_one_traces
+
+      - name: arbitrum_one_transactions
+        identifier: int_arbitrum_one_transactions
+
+      - name: arbitrum_one_deployers
+        identifier: stg_arbitrum_one__deployers
+
       - name: base_traces
         identifier: int_base_traces
 

warehouse/dbt/models/intermediate/blockchain_artifacts/int_deployers.sql

@@ -4,7 +4,7 @@
   )
 }}
 
-{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %}
+{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora", "arbitrum_one"] %}
 
 {% set union_queries = [] %}
 

warehouse/dbt/models/intermediate/blockchain_artifacts/int_factories.sql

@@ -4,7 +4,7 @@
   )
 }}
 
-{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %}
+{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora", "arbitrum_one"] %}
 
 {% set union_queries = [] %}
 

warehouse/dbt/models/intermediate/blockchain_artifacts/int_proxies.sql

@@ -1,6 +1,6 @@
 {# any from address coming out of a proxy #}
 
-{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %}
+{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora", "arbitrum_one"] %}
 
 {% set union_queries = [] %}
 

warehouse/dbt/models/intermediate/blockchain_artifacts/int_safes.sql

@@ -4,7 +4,7 @@
   )
 }}
 
-{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %}
+{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora", "arbitrum_one"] %}
 
 {% set union_queries = [] %}
 

warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_contract_invocation_events.sql

@@ -0,0 +1,19 @@
+{{
+  config(
+    materialized='incremental',
+    partition_by={
+      "field": "time",
+      "data_type": "timestamp",
+      "granularity": "day",
+    },
+    unique_id="id",
+    on_schema_change="append_new_columns",
+    incremental_strategy="insert_overwrite"
+  )
+}}
+{% if is_incremental() %}
+  {% set start = "TIMESTAMP_SUB(_dbt_max_partition, INTERVAL 1 DAY)" %}
+{% else %}
+  {% set start = "'1970-01-01'" %}
+{% endif %}
+{{ contract_invocation_events_with_l1("arbitrum_one", start) }}

warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_traces.sql

@@ -0,0 +1,14 @@
+{{
+  config(
+    materialized='incremental',
+    partition_by={
+      "field": "block_timestamp",
+      "data_type": "timestamp",
+      "granularity": "day",
+    },
+    unique_id="id",
+    on_schema_change="append_new_columns",
+    incremental_strategy="insert_overwrite"
+  )
+}}
+{{ filtered_blockchain_events("ARBITRUM_ONE", "arbitrum_one", "traces") }}

warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_transactions.sql

@@ -0,0 +1,82 @@
+{{
+  config(
+    materialized='incremental',
+    partition_by={
+      "field": "block_timestamp",
+      "data_type": "timestamp",
+      "granularity": "day",
+    },
+    unique_id="id",
+    on_schema_change="append_new_columns",
+    incremental_strategy="insert_overwrite",
+    sql_header='''
+CREATE TEMP FUNCTION from_string_to_double(input STRING)
+RETURNS FLOAT64
+LANGUAGE js AS r"""
+    yourNumber = BigInt(input, 10);
+    return parseFloat(yourNumber);
+  """;
+    '''
+  )
+}}
+
+with filtered_transactions as (
+  {{ 
+    filtered_blockchain_events(
+      "ARBITRUM_ONE", 
+      "arbitrum_one", 
+      "transactions"
+    )
+  }}
+)
+
+select
+  id,
+  `hash`,
+  nonce,
+  block_hash,
+  block_number,
+  transaction_index,
+  from_address,
+  to_address,
+  `value`,
+  gas,
+  {% if target.name == 'production' %}
+  {#
+    We need to convert the arbitrum gas_price from bytes to a double. We
+    intentionally lose some precision by doing this. The gas_price is 
+    actually a utf-8 string. So we convert this to a string which is a 
+    string of the integer. This transformation should only happen on 
+    production and not on the playgrounds as the playgrounds source their 
+    data originally from production
+  #}
+  from_string_to_double(CAST(gas_price AS STRING FORMAT 'UTF-8')) as gas_price,
+  {% else %}
+    gas_price,
+  {% endif %}
+  input,
+  max_fee_per_gas,
+  max_priority_fee_per_gas,
+  transaction_type,
+  receipt_cumulative_gas_used,
+  receipt_gas_used,
+  receipt_contract_address,
+  receipt_status,
+  receipt_effective_gas_price,
+  receipt_root_hash,
+  receipt_l1_fee,
+  receipt_l1_gas_used,
+  receipt_l1_gas_price,
+  receipt_l1_fee_scalar,
+  receipt_l1_blob_base_fee,
+  receipt_l1_blob_base_fee_scalar,
+  blob_versioned_hashes,
+  max_fee_per_blob_gas,
+  receipt_l1_block_number,
+  receipt_l1_base_fee_scalar,
+  gateway_fee,
+  fee_currency,
+  gateway_fee_recipient,
+  ingestion_time,
+  block_timestamp
+from filtered_transactions

warehouse/dbt/models/intermediate/events/int_events.sql

@@ -212,6 +212,8 @@ all_events as (
     select * from {{ ref('int_pgn_contract_invocation_events') }}
     union all
     select * from {{ ref('int_zora_contract_invocation_events') }}
+    union all
+    select * from {{ ref('int_arbitrum_one_contract_invocation_events') }}
   )
   union all
   select

warehouse/dbt/models/playground_sources.yml

@@ -12,6 +12,15 @@ sources:
       - name: arbitrum_deployers
         identifier: stg_arbitrum__deployers
 
+      - name: arbitrum_one_traces
+        identifier: int_arbitrum_one_traces
+
+      - name: arbitrum_one_transactions
+        identifier: int_arbitrum_one_transactions
+
+      - name: arbitrum_one_deployers
+        identifier: stg_arbitrum_one__deployers
+
       - name: base_traces
         identifier: int_base_traces
 

warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__deployers.sql

@@ -0,0 +1,14 @@
+{{
+  config(
+    materialized='incremental',
+    partition_by={
+      "field": "block_timestamp",
+      "data_type": "timestamp",
+      "granularity": "day",
+    },
+    unique_id="transaction_hash",
+    on_schema_change="append_new_columns",
+    incremental_strategy="insert_overwrite"
+  ) 
+}}
+{{ transactions_with_receipts_deployers("arbitrum_one") }}

warehouse/dbt/models/staging/arbitrum/stg_arbitrum__deployers.sql → warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__factories.sql

@@ -9,6 +9,6 @@
     unique_id="transaction_hash",
     on_schema_change="append_new_columns",
     incremental_strategy="insert_overwrite"
-  )
+  ) 
 }}
-{{ goog_blockchain_deployers("arbitrum") }}
+{{ factory_deployments("arbitrum_one") }}

warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__proxies.sql

@@ -0,0 +1,19 @@
+{{
+  config(
+    materialized='incremental',
+    partition_by={
+      "field": "block_timestamp",
+      "data_type": "timestamp",
+      "granularity": "day",
+    },
+    unique_key="id",
+    on_schema_change="append_new_columns",
+    incremental_strategy="insert_overwrite"
+  )
+}}
+{% if is_incremental() %}
+  {% set start = "TIMESTAMP_SUB(_dbt_max_partition, INTERVAL 1 DAY)" %}
+{% else %}
+  {% set start = "'1970-01-01'" %}
+{% endif %}
+{{ known_proxies("arbitrum_one", start) }}