AN-4881/agent-activity (#178)

* initial models * docs * docs and tests * overview docs * updates for service reads * coalesce * row num * heal * inc * remove test * rebuild * remove docs
FlipsideCrypto · Jun 25, 2024 · d774949 · d774949
1 parent 9875f3d
commit d774949
Show file tree

Hide file tree

Showing 14 changed files with 875 additions and 0 deletions.
diff --git a/models/silver/protocols/olas/metadata/silver_olas__getservice_reads.sql b/models/silver/protocols/olas/metadata/silver_olas__getservice_reads.sql
@@ -0,0 +1,153 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = 'getservice_reads_id',
+    full_refresh = false,
+    tags = ['curated']
+) }}
+
+WITH service_contracts AS (
+
+    SELECT
+        contract_address,
+        service_id AS registry_id,
+        MAX(block_number) AS block_number
+    FROM
+        {{ ref('silver_olas__service_registrations') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp) - INTERVAL '12 hours'
+        FROM
+            {{ this }}
+    )
+    AND CONCAT(
+        contract_address,
+        '-',
+        registry_id
+    ) NOT IN (
+        SELECT
+            CONCAT(
+                contract_address,
+                '-',
+                function_input
+            )
+        FROM
+            {{ this }}
+    )
+{% endif %}
+GROUP BY
+    1,
+    2
+),
+function_sigs AS (
+    SELECT
+        '0xef0e239b' AS function_sig,
+        'getService' AS function_name
+),
+inputs AS (
+    SELECT
+        contract_address,
+        block_number,
+        function_sig,
+        function_name,
+        registry_id AS function_input,
+        CONCAT(
+            function_sig,
+            LPAD(
+                SUBSTR(utils.udf_int_to_hex(function_input), 3),
+                64,
+                0)
+            ) AS DATA
+            FROM
+                service_contracts
+                JOIN function_sigs
+                ON 1 = 1
+        ),
+        contract_reads AS (
+            SELECT
+                contract_address,
+                block_number,
+                function_sig,
+                function_name,
+                function_input,
+                DATA,
+                utils.udf_json_rpc_call(
+                    'eth_call',
+                    [{ 'to': contract_address, 'from': null, 'data': data }, utils.udf_int_to_hex(block_number) ]
+                ) AS rpc_request,
+                live.udf_api(
+                    'POST',
+                    CONCAT(
+                        '{service}',
+                        '/',
+                        '{Authentication}'
+                    ),{},
+                    rpc_request,
+                    'Vault/prod/base/quicknode/mainnet'
+                ) AS read_output,
+                SYSDATE() AS _inserted_timestamp
+            FROM
+                inputs
+        ),
+        reads_flat AS (
+            SELECT
+                read_output,
+                read_output :data :id :: STRING AS read_id,
+                read_output :data :result :: STRING AS read_result,
+                SPLIT(
+                    read_id,
+                    '-'
+                ) AS read_id_object,
+                regexp_substr_all(SUBSTR(read_result, 3, len(read_result)), '.{64}') AS segmented_read,
+                utils.udf_hex_to_int(
+                    VALUE :: STRING
+                ) AS decoded_read,
+                function_sig,
+                function_name,
+                function_input,
+                DATA,
+                contract_address,
+                block_number,
+                _inserted_timestamp
+            FROM
+                contract_reads,
+                LATERAL FLATTEN(
+                    input => segmented_read
+                )
+        ),
+        reads_final AS (
+            SELECT
+                read_output,
+                read_id,
+                read_result,
+                read_id_object,
+                segmented_read,
+                function_sig,
+                function_name,
+                function_input,
+                DATA,
+                contract_address,
+                block_number,
+                _inserted_timestamp,
+                ARRAY_AGG(TRY_TO_NUMBER(decoded_read)) AS reads_array,
+                ARRAY_SLICE(reads_array, 9, ARRAY_SIZE(reads_array)) AS agent_ids
+            FROM
+                reads_flat
+            GROUP BY
+                ALL)
+            SELECT
+                *,
+                {{ dbt_utils.generate_surrogate_key(
+                    ['contract_address','function_input']
+                ) }} AS getservice_reads_id,
+                SYSDATE() AS inserted_timestamp,
+                SYSDATE() AS modified_timestamp,
+                '{{ invocation_id }}' AS _invocation_id
+            FROM
+                reads_final
+            WHERE
+                agent_ids IS NOT NULL
+                AND agent_ids :: STRING <> '[]'
diff --git a/models/silver/protocols/olas/metadata/silver_olas__getservice_reads.yml b/models/silver/protocols/olas/metadata/silver_olas__getservice_reads.yml
@@ -0,0 +1,17 @@
+version: 2
+models:
+  - name: silver_olas__getservice_reads
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - GETSERVICE_READS_ID
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null
+      - name: FUNCTION_INPUT
+        tests:
+          - not_null
+      - name: AGENT_IDS
+        tests:
+          - not_null
diff --git a/models/silver/protocols/olas/metadata/silver_olas__registry_metadata.sql b/models/silver/protocols/olas/metadata/silver_olas__registry_metadata.sql
@@ -0,0 +1,146 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = 'registry_metadata_id',
+    full_refresh = false,
+    tags = ['curated']
+) }}
+
+WITH new_records AS (
+
+    SELECT
+        block_number,
+        contract_address,
+        function_input AS registry_id,
+        token_uri_link,
+        _inserted_timestamp,
+        ROW_NUMBER() over (
+            ORDER BY
+                contract_address,
+                registry_id
+        ) AS row_num
+    FROM
+        {{ ref('silver_olas__registry_reads') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp > (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+    OR 
+    CONCAT(
+        contract_address,
+        '-',
+        registry_id
+    ) IN (
+        SELECT
+            CONCAT(
+                contract_address,
+                '-',
+                registry_id
+            )
+        FROM
+            {{ this }}
+        WHERE
+            NAME IS NULL
+    )
+{% endif %}
+),
+uri_calls AS (
+    SELECT
+        block_number,
+        contract_address,
+        registry_id,
+        token_uri_link,
+        live.udf_api(token_uri_link) AS resp,
+        _inserted_timestamp
+    FROM
+        new_records
+    WHERE
+        row_num <= 100
+    UNION ALL
+    SELECT
+        block_number,
+        contract_address,
+        registry_id,
+        token_uri_link,
+        live.udf_api(token_uri_link) AS resp,
+        _inserted_timestamp
+    FROM
+        new_records
+    WHERE
+        row_num > 100
+        AND row_num <= 200
+    UNION ALL
+    SELECT
+        block_number,
+        contract_address,
+        registry_id,
+        token_uri_link,
+        live.udf_api(token_uri_link) AS resp,
+        _inserted_timestamp
+    FROM
+        new_records
+    WHERE
+        row_num > 200
+),
+response AS (
+    SELECT
+        resp,
+        block_number,
+        contract_address,
+        registry_id,
+        token_uri_link,
+        resp :data :attributes [0] :trait_type :: STRING AS trait_type,
+        resp :data :attributes [0] :value :: STRING AS trait_value,
+        REPLACE(
+            resp :data :code_uri :: STRING,
+            'ipfs://',
+            'https://gateway.autonolas.tech/ipfs/'
+        ) AS code_uri_link,
+        resp :data :description :: STRING AS description,
+        CASE
+            WHEN resp :data :image :: STRING ILIKE 'ipfs://%' THEN REPLACE(
+                resp :data :image :: STRING,
+                'ipfs://',
+                'https://gateway.autonolas.tech/ipfs/'
+            )
+            WHEN resp :data :image :: STRING NOT ILIKE '%://%' THEN CONCAT(
+                'https://gateway.autonolas.tech/ipfs/',
+                resp :data :image :: STRING
+            )
+            ELSE resp :data :image :: STRING
+        END AS image_link,
+        resp :data :name :: STRING AS NAME,
+        _inserted_timestamp
+    FROM
+        uri_calls
+)
+SELECT
+    resp,
+    block_number,
+    contract_address,
+    registry_id,
+    token_uri_link,
+    trait_type,
+    trait_value,
+    code_uri_link,
+    description,
+    image_link,
+    NAME,
+    _inserted_timestamp,
+    {{ dbt_utils.generate_surrogate_key(
+        ['contract_address','registry_id']
+    ) }} AS registry_metadata_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS _invocation_id
+FROM
+    response
+WHERE
+    resp :: STRING NOT ILIKE '%merkledag: not found%'
+    AND resp :: STRING NOT ILIKE '%tuple index out of range%'
+    AND resp :: STRING NOT ILIKE '%"error":%'
diff --git a/models/silver/protocols/olas/metadata/silver_olas__registry_metadata.yml b/models/silver/protocols/olas/metadata/silver_olas__registry_metadata.yml
@@ -0,0 +1,19 @@
+version: 2
+models:
+  - name: silver_olas__registry_metadata
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - REGISTRY_METADATA_ID
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null
+      - name: CONTRACT_ADDRESS
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: REGISTRY_ID
+        tests:
+          - not_null
diff --git a/models/silver/protocols/olas/metadata/silver_olas__registry_metadata_complete.sql b/models/silver/protocols/olas/metadata/silver_olas__registry_metadata_complete.sql
@@ -0,0 +1,35 @@
+{{ config(
+    materialized = 'view'
+) }}
+
+SELECT
+    m.name,
+    m.description,
+    m.registry_id,
+    m.contract_address,
+    CASE
+        WHEN m.contract_address = '0x3c1ff68f5aa342d296d4dee4bb1cacca912d95fe' THEN 'Service'
+    END AS registry_type,
+    m.trait_type,
+    m.trait_value,
+    m.code_uri_link,
+    m.image_link,
+    s.agent_ids,
+    m.registry_metadata_id,
+    m.inserted_timestamp,
+    GREATEST(
+        COALESCE(
+            m.modified_timestamp,
+            '1970-01-01' :: TIMESTAMP
+        ),
+        COALESCE(
+            s.modified_timestamp,
+            '1970-01-01' :: TIMESTAMP
+        )
+    ) AS modified_timestamp
+FROM
+    {{ ref('silver_olas__registry_metadata') }}
+    m
+    LEFT JOIN {{ ref('silver_olas__getservice_reads') }}
+    s
+    ON m.registry_id = s.function_input
diff --git a/models/silver/protocols/olas/metadata/silver_olas__registry_metadata_complete.yml b/models/silver/protocols/olas/metadata/silver_olas__registry_metadata_complete.yml
@@ -0,0 +1,16 @@
+version: 2
+models:
+  - name: silver_olas__registry_metadata_complete
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - REGISTRY_METADATA_ID
+    columns:
+      - name: CONTRACT_ADDRESS
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: REGISTRY_ID
+        tests:
+          - not_null