Merge pull request dbt-labs#359 from dbt-labs/remove-metrics

update metric to new metric spec

.gitignore

@@ -5,4 +5,5 @@ dbt_modules/
 dbt_packages/
 .vscode
 integration_tests/state/
-site/
+site/
+.DS_Store

dbt_project.yml

@@ -2,7 +2,7 @@ name: 'dbt_project_evaluator'
 version: '1.0.0'
 config-version: 2
 
-require-dbt-version: [">=1.3.0", "<2.0.0"]
+require-dbt-version: [">=1.6.0-rc1", "<2.0.0"]
 
 model-paths: ["models"]
 analysis-paths: ["analysis"]

integration_tests/models/marts/exposures.yml

@@ -13,6 +13,7 @@ exposures:
 
     depends_on:
       - ref('fct_model_6')
+      - ref('fct_model_9')
       - ref('dim_model_7')
       - ref('dim_model_7')
       - source('source_1', 'table_1')

integration_tests/models/marts/int_model_5.sql

@@ -1,2 +1,3 @@
-{{ ref('int_model_4') }}
-{{ ref('stg_model_1') }}
+-- {{ ref('int_model_4') }}
+-- {{ ref('stg_model_1') }}
+select 1 as id

integration_tests/models/marts/intermediate/dim_model_7.sql

@@ -1,2 +1,9 @@
-{{ ref('stg_model_4') }}
-{{ ref('int_model_5') }}
+
+{{
+    config(
+        materialized = 'table',
+    )
+}}
+
+select * from {{ ref('stg_model_4') }}
+-- {{ ref('int_model_5') }}

integration_tests/models/marts/metrics.yml

@@ -1,29 +1,37 @@
 version: 2
 
-metrics:
-  - name: new_customers
-    label: New Customers marked 'paying'
-    model: ref('dim_model_7')
-    description: "The number of paid customers using the product"
 
-    calculation_method: count
-    expression: user_id # superfluous here, but shown as an example
+semantic_models:
+  - name: my_model 
+    model: ref('dim_model_7') 
+    description: "Transaction fact table at the transaction level. This table contains one row per transaction and includes the transaction timestamp."
+    defaults:
+      agg_time_dimension: date_day
+
+    entities: # Entities included in the table are defined here. MetricFlow will use these columns as join keys.
+      - name: dim
+        type: primary
+        expr: id
 
-    timestamp: signup_date
-    time_grains: [day, week, month]
+    dimensions: # dimensions are qualitative values such as names, dates, or geographical data. They provide context to metrics and allow "metric by group" data slicing.
+      - name: id
+        type: categorical
 
-    dimensions:
-      - plan
-      - country
+      - name: date_day
+        type: time
+        type_params:
+          time_granularity: day
 
-    filters:
-      - field: is_paying
-        operator: '='
-        value: 'true'
-      - field: company_name
-        operator: '!='
-        value: "Acme', Inc"
+    measures: # Measures are columns we perform an aggregation over. Measures are inputs to metrics.
+      - name: total_count
+        description: "The total count."
+        agg: count
+        expr: 1
 
-    meta: 
-      team: "Finance"
-      refresh_rate: "Bob's weekly run"
+metrics:
+  - name: new_metric
+    label: "New Metric"
+    type: simple 
+    type_params:
+  # Specify the measure you are creating a proxy for. 
+      measure: total_count 

integration_tests/models/utils/_utils_models.yml

@@ -0,0 +1,8 @@
+models:
+  - name: metricflow_time_spine
+    description: default time spine for metricflow 
+    columns:
+      - name: date_day
+        tests:
+          - unique
+          - not_null

integration_tests/models/utils/metricflow_time_spine.sql

@@ -0,0 +1,26 @@
+-- metricflow_time_spine.sql
+{% set date_expr = "current_date()" if target.type in ["duckdb"] else dbt.current_timestamp() %}
+{{
+    config(
+        materialized = 'table',
+    )
+}}
+
+with days as (
+
+    {{
+        dbt_utils.date_spine(
+            'day',
+            date_expr,
+            dbt.dateadd('day', 1, date_expr),
+        )
+    }}
+
+),
+
+final as (
+    select cast(date_day as date) as date_day
+    from days
+)
+
+select * from final

integration_tests/seeds/docs/test_fct_documentation_coverage.csv

@@ -1,2 +1,2 @@
 total_models,documented_models,documentation_coverage_pct,staging_documentation_coverage_pct,intermediate_documentation_coverage_pct,marts_documentation_coverage_pct,other_documentation_coverage_pct
-14,3,21.43,20.00,0.00,0.00,66.67
+15,4,26.67,20.00,0.00,0.00,75.00

integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv

@@ -1,3 +1,3 @@
 parent_resource_type,parent_resource_name,exposure_name,parent_model_materialization
-model,dim_model_7,exposure_1,ephemeral
+model,fct_model_9,exposure_1,ephemeral
 source,source_1.table_1,exposure_1,

integration_tests/seeds/tests/test_fct_test_coverage.csv

@@ -1,2 +1,2 @@
 total_models,total_tests,tested_models,test_coverage_pct,staging_test_coverage_pct,intermediate_test_coverage_pct,marts_test_coverage_pct,other_test_coverage_pct,test_to_model_ratio
-14,11,5,35.71,80.00,50.00,0.00,0.00,0.7857
+15,13,6,40.00,80.00,50.00,0.00,25.00,0.8667

integration_tests/seeds/tests/tests_seeds.yml

@@ -10,6 +10,7 @@ seeds:
   - name: test_fct_test_coverage
     config:
       column_types:
+        test_coverage_pct: float
         staging_test_coverage_pct: float
         intermediate_test_coverage_pct: float
         marts_test_coverage_pct: float

macros/unpack/get_metric_values.sql

@@ -9,17 +9,6 @@
     {%- set values = [] -%}
 
     {%- for node in nodes_list -%}
-
-          {% set metric_filters %}
-            {%- if node.filters|length -%}
-              {%- for filt in node.filters %}
-                '{{ filt.field }}'||'{{ filt.operator }}'||'''{{ dbt.escape_single_quotes(filt.value) }}'''
-                {% if not loop.last %}|| ' - ' ||{% endif %}
-              {% endfor -%}
-            {%- else -%}
-                ''
-            {% endif -%}
-          {% endset %}
 
           {%- set values_line = 
             [
@@ -29,13 +18,16 @@
             wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")),
             "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as boolean)",
             wrap_string_with_quotes(node.type),
-            wrap_string_with_quotes(node.model.identifier),
             wrap_string_with_quotes(dbt.escape_single_quotes(node.label)),
-            wrap_string_with_quotes(node.sql),
-            wrap_string_with_quotes(node.timestamp),
             wrap_string_with_quotes(node.package_name),
-            wrap_string_with_quotes(node.dimensions|join(' - ')),
-            metric_filters,
+            wrap_string_with_quotes(node.filter),
+            wrap_string_with_quotes(node.type_params.measure.name),
+            wrap_string_with_quotes(node.type_params.measure.alias),
+            wrap_string_with_quotes(node.type_params.numerator),
+            wrap_string_with_quotes(node.type_params.denominator),
+            wrap_string_with_quotes(node.type_params.expr),
+            wrap_string_with_quotes(node.type_params.window),
+            wrap_string_with_quotes(node.type_params.grain_to_date),
             wrap_string_with_quotes(node.meta | tojson)
             ]
           %}

models/marts/core/int_all_graph_resources.sql

@@ -87,10 +87,15 @@ joined as (
         unioned_with_calc.meta,
         unioned_with_calc.macro_dependencies,
         unioned_with_calc.metric_type, 
-        unioned_with_calc.model, 
         unioned_with_calc.label, 
-        unioned_with_calc.sql, 
-        unioned_with_calc.timestamp as timestamp,  
+        unioned_with_calc.metric_filter,
+        unioned_with_calc.metric_measure,
+        unioned_with_calc.metric_measure_alias,
+        unioned_with_calc.numerator,
+        unioned_with_calc.denominator,
+        unioned_with_calc.expr,
+        unioned_with_calc.metric_window,
+        unioned_with_calc.grain_to_date,
         unioned_with_calc.source_name, -- NULL for non-source resources
         unioned_with_calc.is_source_described, 
         unioned_with_calc.loaded_at_field, 

models/marts/dag/fct_root_models.sql

@@ -8,6 +8,8 @@ with model_relationships as (
     -- only filter out excluded children nodes
         -- filtering parents could result in incorrectly flagging nodes that depend on excluded nodes
     and not child_is_excluded
+    -- exclude required time spine
+    and child != 'metricflow_time_spine'
 ),
 
 final as (

models/marts/structure/fct_model_naming_conventions.sql

@@ -4,6 +4,8 @@
 with all_graph_resources as (
     select * from {{ ref('int_all_graph_resources') }}
     where not is_excluded
+    -- exclude required metricflow time spine
+    and resource_name != 'metricflow_time_spine'
 ),
 
 naming_convention_prefixes as (

models/staging/graph/stg_metrics.sql

@@ -21,13 +21,16 @@ select
     cast(null as {{ dbt.type_string() }}) as file_path,
     cast(True as boolean) as is_described,
     cast(null as {{ dbt.type_string() }}) as metric_type,
-    cast(null as {{ dbt.type_string() }}) as model,
     cast(null as {{ dbt.type_string() }}) as label,
-    cast(null as {{ dbt.type_string() }}) as sql,
-    cast(null as {{ dbt.type_string() }}) as timestamp,
     cast(null as {{ dbt.type_string() }}) as package_name,
-    cast(null as {{ dbt.type_string() }}) as dimensions,
-    cast(null as {{ dbt.type_string() }}) as filters,
+    cast(null as {{ dbt.type_string() }}) as metric_filter,
+    cast(null as {{ dbt.type_string() }}) as metric_measure,
+    cast(null as {{ dbt.type_string() }}) as metric_measure_alias,
+    cast(null as {{ dbt.type_string() }}) as numerator,
+    cast(null as {{ dbt.type_string() }}) as denominator,
+    cast(null as {{ dbt.type_string() }}) as expr,
+    cast(null as {{ dbt.type_string() }}) as metric_window,
+    cast(null as {{ dbt.type_string() }}) as grain_to_date,
     cast(null as {{ dbt.type_string() }}) as meta
 
 from dummy_cte