Feature/add transformation runs

CHANGELOG.md

@@ -1,3 +1,18 @@
+# dbt_fivetran_log v1.11.0
+[PR #141](https://github.com/fivetran/dbt_fivetran_log/pull/141) includes the following updates:
+
+## Schema Changes: Adding the transformation_runs table
+- We have added the `transformation_runs` source table. This includes the following updates:
+  - Added a new staging `stg_fivetran_platform__transformation_runs` model. Additionally, a new tmp model `stg_fivetran_platform__transformation_runs_tmp` and `get_transformation_runs_columns()` macro to ensure all required columns are present in the new staging model.
+  - Added the following fields to the `fivetran_platform__usage_mar_destination_history` end model for each destination and month:
+    - `paid_model_runs`
+    - `free_model_runs`
+    - `total_model_runs`
+  - Included documentation about the respective fields in the `transformation_runs` source table and the aggregated `*_model_run` fields.
+
+## Under the Hood
+- Added `transformation_runs` seed data in `integration_tests/seeds/`.
+
 # dbt_fivetran_log v1.10.0
 [PR #140](https://github.com/fivetran/dbt_fivetran_log/pull/140) includes the following updates:
 
@@ -10,6 +25,7 @@
 ## Under the Hood (Maintainers Only)
 - Enhanced seed data for integration testing to include the different spellings and ensure compatibility with Redshift.
 
+
 # dbt_fivetran_log v1.9.1
 [PR #138](https://github.com/fivetran/dbt_fivetran_log/pull/138) includes the following updates:
 

README.md

@@ -70,7 +70,7 @@ Include the following Fivetran Platform package version range in your `packages.
 ```yaml
 packages:
   - package: fivetran/fivetran_log
-    version: [">=1.10.0", "<1.11.0"]
+    version: [">=1.11.0", "<1.12.0"]
 ```
 
 > Note that although the source connector is now "Fivetran Platform", the package retains the old name of "fivetran_log".

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'fivetran_log'
-version: '1.10.0'
+version: '1.11.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:
@@ -22,4 +22,5 @@ vars:
     destination_membership: "{{ source('fivetran_platform', 'destination_membership') }}"
     log: "{{ source('fivetran_platform', 'log') }}"
     user: "{{ source('fivetran_platform', 'user') }}"
-    usage_cost: "{{ source('fivetran_platform', 'usage_cost') }}"
+    usage_cost: "{{ source('fivetran_platform', 'usage_cost') }}"
+    transformation_runs: "{{ source('fivetran_platform','transformation_runs') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: fivetran_platform_integration_tests_5 
+      schema: fivetran_platform_integration_tests_6 
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: fivetran_platform_integration_tests_5
+      schema: fivetran_platform_integration_tests_6
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: fivetran_platform_integration_tests_5
+      schema: fivetran_platform_integration_tests_6
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: fivetran_platform_integration_tests_5 
+      schema: fivetran_platform_integration_tests_6 
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: fivetran_platform_integration_tests_5 
+      schema: fivetran_platform_integration_tests_6 
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks
@@ -66,7 +66,7 @@ integration_tests:
       server: "{{ env_var('CI_SQLSERVER_DBT_SERVER') }}"
       port: 1433
       database: "{{ env_var('CI_SQLSERVER_DBT_DATABASE') }}"
-      schema: fivetran_platform_integration_tests_5 
+      schema: fivetran_platform_integration_tests_6 
       user: "{{ env_var('CI_SQLSERVER_DBT_USER') }}"
       password: "{{ env_var('CI_SQLSERVER_DBT_PASS') }}"
       threads: 8

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'fivetran_log_integration_tests'
-version: '1.10.0'
+version: '1.11.0'
 
 config-version: 2
 profile: 'integration_tests'
@@ -10,7 +10,7 @@ dispatch:
 
 vars:
   fivetran_log:
-    fivetran_platform_schema: "fivetran_platform_integration_tests_5"
+    fivetran_platform_schema: "fivetran_platform_integration_tests_6"
     fivetran_platform_account_identifier: "account"
     fivetran_platform_incremental_mar_identifier: "incremental_mar"
     fivetran_platform_connector_identifier: "connector"
@@ -20,6 +20,8 @@ vars:
     fivetran_platform_destination_membership_identifier: "destination_membership"
     fivetran_platform_log_identifier: "log"
     fivetran_platform_user_identifier: "user"
+    fivetran_platform_transformation_runs_identifier: "transformation_runs"
+
 
 models:
   fivetran_log:

integration_tests/seeds/transformation_runs.csv

@@ -0,0 +1,3 @@
+destination_id,job_id,measured_date,project_type,free_type,job_name,updated_at,model_runs,_fivetran_synced
+egg_shell,formosa,2024-12-15 00:00:00.000000,QUICKSTART,PAID,Github/staging,2024-12-15 18:13:59.840000,6,2024-12-16 00:11:52.792000
+egg_shell,macau,2024-12-08 00:00:00.000000,QUICKSTART,PROMOTION,Github/staging,2024-12-09 15:21:02.278000,6,2024-12-09 18:11:46.905000

macros/get_transformation_runs_columns.sql

@@ -0,0 +1,17 @@
+{% macro get_transformation_runs_columns() %}
+
+{% set columns = [
+    {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "destination_id", "datatype": dbt.type_string()},
+    {"name": "free_type", "datatype": dbt.type_string()},
+    {"name": "job_id", "datatype": dbt.type_string()},
+    {"name": "job_name", "datatype": dbt.type_string()},
+    {"name": "measured_date", "datatype": dbt.type_timestamp()},
+    {"name": "model_runs", "datatype": dbt.type_int()},
+    {"name": "project_type", "datatype": dbt.type_string()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+] %}
+
+{{ return(columns) }}
+
+{% endmacro %}

models/fivetran_platform.yml

@@ -163,6 +163,15 @@ models:
       - name: dollars_spent
         description: The dollar amount used by the destination for the given month.
 
+      - name: paid_model_runs
+        description: The number of paid model runs for the destination for the given month.
+
+      - name: free_model_runs
+        description: The number of free model runs for the destination for the given month.
+
+      - name: total_model_runs
+        description: The total number of model runs for the destination for the given month.
+
   - name: fivetran_platform__connector_daily_events
     description: >
       Table of each connector's daily history of the quantity of api calls, schema changes, and records modified,

models/fivetran_platform__usage_mar_destination_history.sql

@@ -10,12 +10,24 @@ credits_used as (
     from {{ ref('stg_fivetran_platform__credits_used') }}
 ),
 
-useage_cost as (
+usage_cost as (
 
     select *
     from {{ ref('stg_fivetran_platform__usage_cost') }}
 ),
 
+transformation_runs as (
+
+    select
+        destination_id,
+        cast(measured_month as date) as measured_month,
+        sum(case when free_type = 'PAID' then model_runs else 0 end) as paid_model_runs,
+        sum(case when free_type != 'PAID' then model_runs else 0 end) as free_model_runs,
+        sum(model_runs) as total_model_runs
+    from {{ ref('stg_fivetran_platform__transformation_runs') }}
+    group by destination_id, measured_month
+),
+
 destination_mar as (
 
     select 
@@ -32,14 +44,14 @@ destination_mar as (
 usage as (
 
     select 
-        coalesce(credits_used.destination_id, useage_cost.destination_id) as destination_id,
+        coalesce(credits_used.destination_id, usage_cost.destination_id) as destination_id,
         credits_used.credits_spent,
-        useage_cost.dollars_spent,
-        cast(concat(coalesce(credits_used.measured_month,useage_cost.measured_month), '-01') as date) as measured_month -- match date format to join with MAR table
+        usage_cost.dollars_spent,
+        cast(concat(coalesce(credits_used.measured_month,usage_cost.measured_month), '-01') as date) as measured_month -- match date format to join with MAR table
     from credits_used
-    full outer join useage_cost
-        on useage_cost.measured_month = credits_used.measured_month
-        and useage_cost.destination_id = credits_used.destination_id
+    full outer join usage_cost
+        on usage_cost.measured_month = credits_used.measured_month
+        and usage_cost.destination_id = credits_used.destination_id
 ),
 
 join_usage_mar as (
@@ -53,6 +65,9 @@ join_usage_mar as (
         destination_mar.free_monthly_active_rows,
         destination_mar.paid_monthly_active_rows,
         destination_mar.total_monthly_active_rows,
+        transformation_runs.paid_model_runs,
+        transformation_runs.free_model_runs,
+        transformation_runs.total_model_runs,
 
         -- credit and usage mar calculations
         round( cast(nullif(usage.credits_spent,0) * 1000000.0 as {{ dbt.type_numeric() }}) / cast(nullif(destination_mar.total_monthly_active_rows,0) as {{ dbt.type_numeric() }}), 2) as credits_spent_per_million_mar,
@@ -63,6 +78,9 @@ join_usage_mar as (
     left join usage 
         on destination_mar.measured_month = cast(usage.measured_month as date)
         and destination_mar.destination_id = usage.destination_id
+    left join transformation_runs
+        on destination_mar.measured_month = cast(transformation_runs.measured_month as date)
+        and destination_mar.destination_id = transformation_runs.destination_id
 )
 
 select * 

models/staging/src_fivetran_platform.yml

@@ -200,5 +200,28 @@ sources:
             description: Phone number associated with user.
           - name: verified 
             description: Boolean that indicates whether the user has verified their email address in the account creation process.
+          - name: _fivetran_synced
+            description: Timestamp of when the record was last synced.
+
+      - name: transformation_runs
+        identifier: '{{ var("fivetran_log_transformation_runs_identifier", "transformation_runs") }}'
+        description: Table of all transformation runs that have been executed by Fivetran.
+        columns:
+          - name: destination_id
+            description: Foreign key referencing the destination for which model runs is calculated.
+          - name: job_id
+            description: Foreign key referencing the job affiliated with the transformation run.
+          - name: measured_date
+            description: The date in UTC format of when models were run.
+          - name: project_type
+            description: The type of transformation project that was run (ie DBT_CORE, QUICKSTART).
+          - name: free_type
+            description: If it is free MAR, the value indicates the type of free MAR. For paid MAR, the value is `PAID`.
+          - name: job_name
+            description: The name of the transformation job that was run.
+          - name: updated_at
+            description: Timestamp of when the record was last updated.
+          - name: model_runs
+            description: The number of models that were run in the transformation.
           - name: _fivetran_synced
             description: Timestamp of when the record was last synced.

models/staging/stg_fivetran_platform.yml

@@ -187,4 +187,29 @@ models:
       - name: phone
         description: Phone number associated with user.
       - name: is_verified
-        description: Boolean that indicates whether the user has verified their email address in the account creation process.
+        description: Boolean that indicates whether the user has verified their email address in the account creation process.
+
+  - name: stg_fivetran_platform__transformation_runs
+    description: Table of all transformation runs that have been executed by Fivetran.
+    columns:
+      - name: destination_id
+        description: Foreign key referencing the destination for which model runs is calculated.
+      - name: job_id
+        description: Foreign key referencing the job affiliated with the transformation run.
+      - name: measured_date
+        description: The date in UTC format of when models were run.
+      - name: measured_month
+        description: The month (yyyy-mm) of the usage activity.
+      - name: project_type
+        description: The type of transformation project that was run (ie DBT_CORE, QUICKSTART).
+      - name: free_type
+        description: If it is free MAR, the value indicates the type of free MAR. For paid MAR, the value is `PAID`.
+      - name: job_name
+        description: The name of the transformation job that was run.
+      - name: updated_at
+        description: Timestamp of when the record was last updated.
+      - name: model_runs
+        description: The number of models that were run in the transformation.
+      - name: _fivetran_synced
+        description: Timestamp of when the record was last synced.
+

models/staging/stg_fivetran_platform__transformation_runs.sql

@@ -0,0 +1,42 @@
+
+with base as (
+
+    select * 
+    from {{ ref('stg_fivetran_platform__transformation_runs_tmp') }}
+),
+
+fields as (
+
+    select
+        {{
+            fivetran_utils.fill_staging_columns(
+                source_columns=adapter.get_columns_in_relation(ref('stg_fivetran_platform__transformation_runs_tmp')),
+                staging_columns=get_transformation_runs_columns()
+            )
+        }}
+        {{ fivetran_utils.source_relation(
+            union_schema_variable='fivetran_platform_union_schemas', 
+            union_database_variable='fivetran_platform_union_databases') 
+        }}
+    from base
+),
+
+final as (
+
+    select
+        _fivetran_synced,
+        destination_id,
+        free_type,
+        job_id,
+        job_name,
+        cast(measured_date as {{ dbt.type_timestamp() }}) as measured_date,
+        model_runs,
+        project_type,
+        updated_at
+    from fields
+)
+
+select
+    *,
+    {{ dbt.date_trunc('month', 'measured_date') }} as measured_month
+from final

models/staging/tmp/stg_fivetran_platform__transformation_runs_tmp.sql

@@ -0,0 +1,2 @@
+select *
+from {{ var('transformation_runs') }}