From ee8050ee1176ddcf4a075ce40c6a3ea3fb2e6d82 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Thu, 4 May 2023 13:48:42 +0200 Subject: [PATCH 01/37] Update macros and models to add `is_excluded` --- macros/unpack/get_node_values.sql | 14 +++++++++++++- macros/unpack/get_source_values.sql | 14 +++++++++++++- models/staging/graph/stg_nodes.sql | 3 ++- models/staging/graph/stg_sources.sql | 3 ++- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/macros/unpack/get_node_values.sql b/macros/unpack/get_node_values.sql index 34d3d147..3eeb896b 100644 --- a/macros/unpack/get_node_values.sql +++ b/macros/unpack/get_node_values.sql @@ -5,6 +5,7 @@ {%- macro default__get_node_values() -%} {%- if execute -%} + {% set re = modules.re %} {%- set nodes_list = graph.nodes.values() -%} {%- set values = [] -%} @@ -12,6 +13,16 @@ {%- set hard_coded_references = dbt_project_evaluator.find_all_hard_coded_references(node) -%} + {%- set ns = namespace(exclude=false) -%} + {%- set node_package_path = node.package_name ~ ":" ~ node.original_file_path | replace("\\","\\\\") -%} + + {%- for exclude_pattern in var('exclude_packages_and_paths',[]) -%} + {%- set is_match = re.match(exclude_pattern, node_package_path, re.IGNORECASE) -%} + {%- if is_match %} + {% set ns.exclude = true %} + {%- endif -%} + {%- endfor -%} + {%- set values_line = [ wrap_string_with_quotes(node.unique_id), @@ -30,7 +41,8 @@ wrap_string_with_quotes(node.meta | tojson), wrap_string_with_quotes(dbt.escape_single_quotes(hard_coded_references)), wrap_string_with_quotes(node.get('depends_on',{}).get('macros',[]) | tojson), - "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.test_metadata) | trim ~ " as boolean)" + "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.test_metadata) | trim ~ " as boolean)", + "cast(" ~ ns.exclude ~ " as boolean)", ] %} diff --git a/macros/unpack/get_source_values.sql b/macros/unpack/get_source_values.sql index 7a30ac63..4774bb4f 100644 --- a/macros/unpack/get_source_values.sql +++ b/macros/unpack/get_source_values.sql @@ -5,11 +5,22 @@ {%- macro default__get_source_values() -%} {%- if execute -%} + {% set re = modules.re %} {%- set nodes_list = graph.sources.values() -%} {%- set values = [] -%} {%- for node in nodes_list -%} + {%- set ns = namespace(exclude=false) -%} + {%- set node_package_path = node.package_name ~ ":" ~ node.original_file_path | replace("\\","\\\\") ~ ":" ~ node.name -%} + + {%- for exclude_pattern in var('exclude_packages_and_paths',[]) -%} + {%- set is_match = re.match(exclude_pattern, node_package_path, re.IGNORECASE) -%} + {%- if is_match %} + {% set ns.exclude = true %} + {%- endif -%} + {%- endfor -%} + {%- set values_line = [ wrap_string_with_quotes(node.unique_id), @@ -27,7 +38,8 @@ wrap_string_with_quotes(node.package_name), wrap_string_with_quotes(node.loader), wrap_string_with_quotes(node.identifier), - wrap_string_with_quotes(node.meta | tojson) + wrap_string_with_quotes(node.meta | tojson), + "cast(" ~ ns.exclude ~ " as boolean)", ] %} diff --git a/models/staging/graph/stg_nodes.sql b/models/staging/graph/stg_nodes.sql index 6e68d8c0..f241d1e1 100644 --- a/models/staging/graph/stg_nodes.sql +++ b/models/staging/graph/stg_nodes.sql @@ -32,7 +32,8 @@ select cast(null as {{ dbt.type_string() }}) as meta, cast(null as {{ dbt.type_string() }}) as hard_coded_references, cast(null as {{ dbt.type_string() }}) as macro_dependencies, - cast(True as boolean) as is_generic_test + cast(True as boolean) as is_generic_test, + cast(True as boolean) as is_excluded from dummy_cte where false \ No newline at end of file diff --git a/models/staging/graph/stg_sources.sql b/models/staging/graph/stg_sources.sql index 052652ff..26aecd5f 100644 --- a/models/staging/graph/stg_sources.sql +++ b/models/staging/graph/stg_sources.sql @@ -31,7 +31,8 @@ select cast(null as {{ dbt.type_string() }}) as package_name, cast(null as {{ dbt.type_string() }}) as loader, cast(null as {{ dbt.type_string() }}) as identifier, - cast(null as {{ dbt.type_string() }}) as meta + cast(null as {{ dbt.type_string() }}) as meta, + cast(True as boolean) as is_excluded, from dummy_cte where false \ No newline at end of file From d9c2ddff55c0672300e9c7c84fd04a557baabc31 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Thu, 4 May 2023 13:50:30 +0200 Subject: [PATCH 02/37] Add `is_excluded` to the main tables --- macros/recursive_dag.sql | 13 ++++++++++++- models/marts/core/int_all_graph_resources.sql | 3 ++- models/marts/core/int_direct_relationships.sql | 3 ++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/macros/recursive_dag.sql b/macros/recursive_dag.sql index 45861cf2..b14614b9 100644 --- a/macros/recursive_dag.sql +++ b/macros/recursive_dag.sql @@ -25,6 +25,7 @@ all_relationships ( parent_file_path, parent_directory_path, parent_file_name, + parent_is_excluded, child_id, child, child_resource_type, @@ -34,6 +35,7 @@ all_relationships ( child_file_path, child_directory_path, child_file_name, + child_is_excluded, distance, path, is_dependent_on_chain_of_views @@ -49,6 +51,7 @@ all_relationships ( file_path as parent_file_path, directory_path as parent_directory_path, file_name as parent_file_name, + is_excluded as parent_is_excluded, resource_id as child_id, resource_name as child, resource_type as child_resource_type, @@ -58,6 +61,7 @@ all_relationships ( file_path as child_file_path, directory_path as child_directory_path, file_name as child_file_name, + is_excluded as child_is_excluded, 0 as distance, {{ dbt.array_construct(['resource_name']) }} as path, cast(null as boolean) as is_dependent_on_chain_of_views @@ -78,6 +82,7 @@ all_relationships ( all_relationships.parent_file_path as parent_file_path, all_relationships.parent_directory_path as parent_directory_path, all_relationships.parent_file_name as parent_file_name, + all_relationships.parent_is_excluded as parent_is_excluded, direct_relationships.resource_id as child_id, direct_relationships.resource_name as child, direct_relationships.resource_type as child_resource_type, @@ -87,6 +92,7 @@ all_relationships ( direct_relationships.file_path as child_file_path, direct_relationships.directory_path as child_directory_path, direct_relationships.file_name as child_file_name, + direct_relationships.is_excluded as child_is_excluded, all_relationships.distance+1 as distance, {{ dbt.array_append('all_relationships.path', 'direct_relationships.resource_name') }} as path, case @@ -139,7 +145,8 @@ with direct_relationships as ( resource_id as parent_id, resource_id as child_id, resource_name, - materialized as child_materialized + materialized as child_materialized, + is_excluded as child_is_excluded from direct_relationships ) @@ -148,6 +155,7 @@ with direct_relationships as ( parent_id, child_id, child_materialized, + child_is_excluded, 0 as distance, {{ dbt.array_construct(['resource_name']) }} as path, cast(null as boolean) as is_dependent_on_chain_of_views @@ -161,6 +169,7 @@ with direct_relationships as ( cte_{{i - 1}}.parent_id as parent_id, direct_relationships.resource_id as child_id, direct_relationships.materialized as child_materialized, + direct_relationships.is_excluded as child_is_excluded, cte_{{i - 1}}.distance+1 as distance, {{ dbt.array_append(prev_cte_path, 'direct_relationships.resource_name') }} as path, case @@ -200,6 +209,7 @@ with direct_relationships as ( parent.file_path as parent_file_path, parent.directory_path as parent_directory_path, parent.file_name as parent_file_name, + parent.is_excluded as parent_is_excluded, child.resource_id as child_id, child.resource_name as child, child.resource_type as child_resource_type, @@ -209,6 +219,7 @@ with direct_relationships as ( child.file_path as child_file_path, child.directory_path as child_directory_path, child.file_name as child_file_name, + child.is_excluded as child_is_excluded, all_relationships_unioned.distance, all_relationships_unioned.path, all_relationships_unioned.is_dependent_on_chain_of_views diff --git a/models/marts/core/int_all_graph_resources.sql b/models/marts/core/int_all_graph_resources.sql index 0387c7fd..1fa1685e 100644 --- a/models/marts/core/int_all_graph_resources.sql +++ b/models/marts/core/int_all_graph_resources.sql @@ -96,7 +96,8 @@ joined as ( unioned_with_calc.loaded_at_field, unioned_with_calc.loader, unioned_with_calc.identifier, - unioned_with_calc.hard_coded_references -- NULL for non-model resources + unioned_with_calc.hard_coded_references, -- NULL for non-model resources + unioned_with_calc.is_excluded -- NULL for metrics and exposures from unioned_with_calc left join naming_convention_prefixes diff --git a/models/marts/core/int_direct_relationships.sql b/models/marts/core/int_direct_relationships.sql index 302c6e08..e7372f8a 100644 --- a/models/marts/core/int_direct_relationships.sql +++ b/models/marts/core/int_direct_relationships.sql @@ -11,7 +11,8 @@ all_graph_resources as ( file_name, model_type, materialized, - source_name + source_name, + is_excluded from {{ ref('int_all_graph_resources') }} ), From 8f72610b0730d06554ec192f9bc595beb084bd0c Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Thu, 4 May 2023 13:50:48 +0200 Subject: [PATCH 03/37] Add filter on the test models --- models/marts/dag/fct_direct_join_to_source.sql | 2 ++ models/marts/dag/fct_duplicate_sources.sql | 1 + models/marts/dag/fct_hard_coded_references.sql | 1 + .../marts/dag/fct_marts_or_intermediate_dependent_on_source.sql | 2 ++ models/marts/dag/fct_model_fanout.sql | 2 ++ models/marts/dag/fct_multiple_sources_joined.sql | 2 ++ models/marts/dag/fct_rejoining_of_upstream_concepts.sql | 2 ++ models/marts/dag/fct_root_models.sql | 2 ++ models/marts/dag/fct_source_fanout.sql | 2 ++ .../dag/fct_staging_dependent_on_marts_or_intermediate.sql | 2 ++ models/marts/dag/fct_staging_dependent_on_staging.sql | 2 ++ models/marts/dag/fct_unused_sources.sql | 2 ++ models/marts/documentation/fct_documentation_coverage.sql | 1 + models/marts/documentation/fct_undocumented_models.sql | 1 + models/marts/performance/fct_chained_views_dependencies.sql | 2 ++ .../marts/performance/fct_exposure_parents_materializations.sql | 2 ++ models/marts/structure/fct_model_directories.sql | 1 + models/marts/structure/fct_model_naming_conventions.sql | 1 + models/marts/structure/fct_source_directories.sql | 1 + models/marts/structure/fct_test_directories.sql | 1 + models/marts/tests/intermediate/int_model_test_summary.sql | 1 + 21 files changed, 33 insertions(+) diff --git a/models/marts/dag/fct_direct_join_to_source.sql b/models/marts/dag/fct_direct_join_to_source.sql index ebb32cbe..27a0f0c3 100644 --- a/models/marts/dag/fct_direct_join_to_source.sql +++ b/models/marts/dag/fct_direct_join_to_source.sql @@ -6,6 +6,8 @@ with direct_model_relationships as ( from {{ ref('int_all_dag_relationships') }} where child_resource_type = 'model' and distance = 1 + and not parent_is_excluded + and not child_is_excluded ), model_and_source_joined as ( diff --git a/models/marts/dag/fct_duplicate_sources.sql b/models/marts/dag/fct_duplicate_sources.sql index 820a59eb..b0fd5d54 100644 --- a/models/marts/dag/fct_duplicate_sources.sql +++ b/models/marts/dag/fct_duplicate_sources.sql @@ -8,6 +8,7 @@ with sources as ( end as source_db_location from {{ ref('int_all_graph_resources') }} where resource_type = 'source' + and not is_excluded -- we order the CTE so that listagg returns values correctly sorted for some warehouses order by 1, 2 ), diff --git a/models/marts/dag/fct_hard_coded_references.sql b/models/marts/dag/fct_hard_coded_references.sql index a7769453..7876590c 100644 --- a/models/marts/dag/fct_hard_coded_references.sql +++ b/models/marts/dag/fct_hard_coded_references.sql @@ -3,6 +3,7 @@ with models as ( select * from {{ ref('int_all_graph_resources') }} where resource_type = 'model' + and not is_excluded ), final as ( diff --git a/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql b/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql index fa354955..a38bcf19 100644 --- a/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql +++ b/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql @@ -4,6 +4,8 @@ with direct_relationships as ( * from {{ ref('int_all_dag_relationships') }} where distance = 1 + and not parent_is_excluded + and not child_is_excluded ), final as ( select diff --git a/models/marts/dag/fct_model_fanout.sql b/models/marts/dag/fct_model_fanout.sql index 9d1a39db..dc7ff228 100644 --- a/models/marts/dag/fct_model_fanout.sql +++ b/models/marts/dag/fct_model_fanout.sql @@ -2,6 +2,8 @@ with all_dag_relationships as ( select * from {{ ref('int_all_dag_relationships') }} + where not parent_is_excluded + and not child_is_excluded ), -- find all models without children diff --git a/models/marts/dag/fct_multiple_sources_joined.sql b/models/marts/dag/fct_multiple_sources_joined.sql index e2b6bb4a..737d2dfc 100644 --- a/models/marts/dag/fct_multiple_sources_joined.sql +++ b/models/marts/dag/fct_multiple_sources_joined.sql @@ -6,6 +6,8 @@ with direct_source_relationships as ( from {{ ref('int_all_dag_relationships') }} where distance = 1 and parent_resource_type = 'source' + and not parent_is_excluded + and not child_is_excluded -- we order the CTE so that listagg returns values correctly sorted for some warehouses order by 1, 2 ), diff --git a/models/marts/dag/fct_rejoining_of_upstream_concepts.sql b/models/marts/dag/fct_rejoining_of_upstream_concepts.sql index 318ae778..f93af44b 100644 --- a/models/marts/dag/fct_rejoining_of_upstream_concepts.sql +++ b/models/marts/dag/fct_rejoining_of_upstream_concepts.sql @@ -4,6 +4,8 @@ with all_relationships as ( from {{ ref('int_all_dag_relationships') }} where parent_resource_type not in ('exposures', 'metrics') and child_resource_type not in ('exposures', 'metrics') + and not parent_is_excluded + and not child_is_excluded ), -- all parent/child relationships where the parent is BOTH the direct parent of the child and the second level parent of the child diff --git a/models/marts/dag/fct_root_models.sql b/models/marts/dag/fct_root_models.sql index 0ea8e4ff..fa55f9a0 100644 --- a/models/marts/dag/fct_root_models.sql +++ b/models/marts/dag/fct_root_models.sql @@ -5,6 +5,8 @@ with model_relationships as ( * from {{ ref('int_all_dag_relationships') }} where child_resource_type = 'model' + and not parent_is_excluded + and not child_is_excluded ), final as ( diff --git a/models/marts/dag/fct_source_fanout.sql b/models/marts/dag/fct_source_fanout.sql index c2675e43..5ab000d2 100644 --- a/models/marts/dag/fct_source_fanout.sql +++ b/models/marts/dag/fct_source_fanout.sql @@ -6,6 +6,8 @@ with direct_source_relationships as ( where distance = 1 and parent_resource_type = 'source' and child_resource_type = 'model' + and not parent_is_excluded + and not child_is_excluded -- we order the CTE so that listagg returns values correctly sorted for some warehouses order by child ), diff --git a/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql b/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql index 321da88d..85fe138d 100644 --- a/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql +++ b/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql @@ -7,6 +7,8 @@ with direct_model_relationships as ( where distance = 1 and parent_resource_type = 'model' and child_resource_type = 'model' + and not parent_is_excluded + and not child_is_excluded ), final as ( select diff --git a/models/marts/dag/fct_staging_dependent_on_staging.sql b/models/marts/dag/fct_staging_dependent_on_staging.sql index fbbe0e01..439ea7a1 100644 --- a/models/marts/dag/fct_staging_dependent_on_staging.sql +++ b/models/marts/dag/fct_staging_dependent_on_staging.sql @@ -5,6 +5,8 @@ with direct_model_relationships as ( from {{ ref('int_all_dag_relationships') }} where parent_resource_type in ('model', 'snapshot') and child_resource_type in ('model', 'snapshot') + and not parent_is_excluded + and not child_is_excluded and distance = 1 ), diff --git a/models/marts/dag/fct_unused_sources.sql b/models/marts/dag/fct_unused_sources.sql index 6803dbef..987501d4 100644 --- a/models/marts/dag/fct_unused_sources.sql +++ b/models/marts/dag/fct_unused_sources.sql @@ -5,6 +5,8 @@ with source_relationships as ( * from {{ ref('int_all_dag_relationships') }} where parent_resource_type = 'source' + and not parent_is_excluded + and not child_is_excluded ), final as ( diff --git a/models/marts/documentation/fct_documentation_coverage.sql b/models/marts/documentation/fct_documentation_coverage.sql index 1df17322..39dd9ce3 100644 --- a/models/marts/documentation/fct_documentation_coverage.sql +++ b/models/marts/documentation/fct_documentation_coverage.sql @@ -3,6 +3,7 @@ with models as ( select * from {{ ref('int_all_graph_resources') }} where resource_type = 'model' + and not is_excluded ), conversion as ( diff --git a/models/marts/documentation/fct_undocumented_models.sql b/models/marts/documentation/fct_undocumented_models.sql index 60ae4b43..4a0c6277 100644 --- a/models/marts/documentation/fct_undocumented_models.sql +++ b/models/marts/documentation/fct_undocumented_models.sql @@ -2,6 +2,7 @@ with all_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), diff --git a/models/marts/performance/fct_chained_views_dependencies.sql b/models/marts/performance/fct_chained_views_dependencies.sql index 42527100..6473ad87 100644 --- a/models/marts/performance/fct_chained_views_dependencies.sql +++ b/models/marts/performance/fct_chained_views_dependencies.sql @@ -3,6 +3,8 @@ with all_relationships as ( * from {{ ref('int_all_dag_relationships') }} where distance <> 0 + and not parent_is_excluded + and not child_is_excluded ), final as ( diff --git a/models/marts/performance/fct_exposure_parents_materializations.sql b/models/marts/performance/fct_exposure_parents_materializations.sql index 87bd7683..bede7c26 100644 --- a/models/marts/performance/fct_exposure_parents_materializations.sql +++ b/models/marts/performance/fct_exposure_parents_materializations.sql @@ -13,6 +13,8 @@ direct_exposure_relationships as ( parent_resource_type = 'source' ) ) + -- no test on child_is_excluded because exposures are never excluded + and not parent_is_excluded ), final as ( diff --git a/models/marts/structure/fct_model_directories.sql b/models/marts/structure/fct_model_directories.sql index 933f5230..d1863c3c 100644 --- a/models/marts/structure/fct_model_directories.sql +++ b/models/marts/structure/fct_model_directories.sql @@ -55,6 +55,7 @@ innappropriate_subdirectories_non_staging_models as ( on folders.model_type = all_graph_resources.model_type -- either appropriate folder_name is not in the current_directory_path or a inappropriate folder name is closer to the file_name where all_graph_resources.model_type <> all_graph_resources.model_type_folder + and not is_excluded ), unioned as ( diff --git a/models/marts/structure/fct_model_naming_conventions.sql b/models/marts/structure/fct_model_naming_conventions.sql index 8f4bedca..eb3ac94f 100644 --- a/models/marts/structure/fct_model_naming_conventions.sql +++ b/models/marts/structure/fct_model_naming_conventions.sql @@ -3,6 +3,7 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), naming_convention_prefixes as ( diff --git a/models/marts/structure/fct_source_directories.sql b/models/marts/structure/fct_source_directories.sql index a8e3c152..167a2ebe 100644 --- a/models/marts/structure/fct_source_directories.sql +++ b/models/marts/structure/fct_source_directories.sql @@ -6,6 +6,7 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), -- find all sources that are definied in a .yml file NOT in their subdirectory diff --git a/models/marts/structure/fct_test_directories.sql b/models/marts/structure/fct_test_directories.sql index 3ca1c65e..7f768814 100644 --- a/models/marts/structure/fct_test_directories.sql +++ b/models/marts/structure/fct_test_directories.sql @@ -4,6 +4,7 @@ with resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 80ede484..54116d25 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -2,6 +2,7 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), relationships as ( From 5b27fb0d76ddf08c57a4c29718fdcf79d53ee006 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Thu, 4 May 2023 14:06:18 +0200 Subject: [PATCH 04/37] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20Remove=20extra=20com?= =?UTF-8?q?ma?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/staging/graph/stg_sources.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/graph/stg_sources.sql b/models/staging/graph/stg_sources.sql index 26aecd5f..ed018971 100644 --- a/models/staging/graph/stg_sources.sql +++ b/models/staging/graph/stg_sources.sql @@ -32,7 +32,7 @@ select cast(null as {{ dbt.type_string() }}) as loader, cast(null as {{ dbt.type_string() }}) as identifier, cast(null as {{ dbt.type_string() }}) as meta, - cast(True as boolean) as is_excluded, + cast(True as boolean) as is_excluded from dummy_cte where false \ No newline at end of file From 9cbe70b397a332cf331d13354095b9119f95ec6f Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Fri, 5 May 2023 15:41:48 +0200 Subject: [PATCH 05/37] Add integration tests --- integration_tests/dbt_project.yml | 1 + .../models/marts/intermediate/to_exclude/int_model_10.sql | 3 +++ .../models/marts/intermediate/to_exclude/int_model_11.sql | 3 +++ 3 files changed, 7 insertions(+) create mode 100644 integration_tests/models/marts/intermediate/to_exclude/int_model_10.sql create mode 100644 integration_tests/models/marts/intermediate/to_exclude/int_model_11.sql diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 1d5b9c29..c23206dd 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -59,6 +59,7 @@ seeds: vars: # ensure integration tests run successfully when there are 0 of a given model type (extra) + exclude_packages_and_paths: ["^(?!dbt_project_evaluator_integration_tests:).*",".*/to_exclude/.*"] model_types: ['staging', 'intermediate', 'marts', 'other', 'extra', 'new_model_type'] # dummy variable used for testing fct_hard_coded_references my_table_reference: 'grace_table' diff --git a/integration_tests/models/marts/intermediate/to_exclude/int_model_10.sql b/integration_tests/models/marts/intermediate/to_exclude/int_model_10.sql new file mode 100644 index 00000000..7a0669e0 --- /dev/null +++ b/integration_tests/models/marts/intermediate/to_exclude/int_model_10.sql @@ -0,0 +1,3 @@ +-- depends on: {{ ref('fct_model_9') }} + +select 1 as id \ No newline at end of file diff --git a/integration_tests/models/marts/intermediate/to_exclude/int_model_11.sql b/integration_tests/models/marts/intermediate/to_exclude/int_model_11.sql new file mode 100644 index 00000000..7a0669e0 --- /dev/null +++ b/integration_tests/models/marts/intermediate/to_exclude/int_model_11.sql @@ -0,0 +1,3 @@ +-- depends on: {{ ref('fct_model_9') }} + +select 1 as id \ No newline at end of file From 51df6664522cb40f6d790e2ae6ab68c19ceaf3ff Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Fri, 5 May 2023 15:42:07 +0200 Subject: [PATCH 06/37] Update docs --- docs/contributing.md | 1 + docs/customization/customization.md | 14 ++-- .../excluding-packages-and-paths.md | 64 +++++++++++++++++++ mkdocs.yml | 7 +- 4 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 docs/customization/excluding-packages-and-paths.md diff --git a/docs/contributing.md b/docs/contributing.md index cf205dfe..6b0906a5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -24,6 +24,7 @@ Docs are then automatically pushed to the website as part of our CI/CD process. "markdownlint.config": { "ul-indent": {"indent": 4}, "MD036": false, + "MD046": false, } ``` diff --git a/docs/customization/customization.md b/docs/customization/customization.md index 3b6e381b..1b9d5955 100644 --- a/docs/customization/customization.md +++ b/docs/customization/customization.md @@ -1,7 +1,14 @@ -# Disabling Models +# Disabling tests from the package -If there is a particular model or set of models that you *do not want this package to execute*, you can -disable these models as you would any other model in your `dbt_project.yml` file +!!! note + + This section is describing how to completely deactivate tests from the package. + If you are looking to deactivate models/sources from being tested, you can look at [excluding packages and paths](excluding-packages-and-paths.md) + +All the tests done as part of the package are tied to `fct` models. + +If there is a particular test or set of tests that you *do not want this package to execute*, you can +disable the corresponding `fct` models as you would any other model in your `dbt_project.yml` file ``` yaml title="dbt_project.yml" models: @@ -14,5 +21,4 @@ models: # disable single DAG model fct_model_fanout: +enabled: false - ``` diff --git a/docs/customization/excluding-packages-and-paths.md b/docs/customization/excluding-packages-and-paths.md new file mode 100644 index 00000000..6a0fc7aa --- /dev/null +++ b/docs/customization/excluding-packages-and-paths.md @@ -0,0 +1,64 @@ +# Excluding packages or sources/models based on their path + +!!! note + + This section is describing how to entirely exclude models/sources and packages to be evaluated. + If you want to document exception to the rules, see the section [on exceptions](exceptions.md) + and if you want to deactivate entire tests you can follow instructions from [this page](customization.md) + +There might be cases where you might want to exclude models/sources from being tested: + +- they could come from a package for which you have no control over +- you might be refactoring your project and wanting to exclude entire folders to follow best-practices in the new models + +In that case, this package provides the ability to exclude whole packages and/or models and sources based on their path + +## Configuration + +The variable `exclude_packages_and_paths` allows you to define a list of regex patterns to exclude from being reported as errors. + +- **for models**, the regex provided will try to match the pattern in the string `:`, allowing to exclude packages, but also whole folders or individual models +- **for sources**, the regex will try to match the pattern in `::` *(the pattern is different than for models because the path doesn't let us exclude individual sources)* + +### Example to exclude a whole package + +```yaml title="dbt_project.yml" +vars: + exclude_packages_and_paths: [":.*"] +``` + +### Example to exclude a given path + +```yaml title="dbt_project.yml" +vars: + exclude_packages_and_paths: [".*//.*"] +``` + +### Example to exclude both a package and models in a path + +```yaml title="dbt_project.yml" +vars: + exclude_packages_and_paths: [":.*", ".*//.*"] +``` + +## Tips and tricks + +Regular expressions are very powerful but can become complex. After defining your value for `exclude_packages_and_paths`, we recommend running the package and inspecting the model `int_all_graph_resources`, checking if the value in the column `is_disabled` matches your expectation. + +A useful tool to debug regular expression is [regex101](https://regex101.com/). You can provide a pattern and a list of strings to see which ones actually match the pattern. + +### Running the tests only on the current project + +Instead of listing all the packages to exclude, we can use a [regex negative look-ahead](https://www.regular-expressions.info/lookaround.html) expression to filter out all the models and sources that come from a package different than the current project. + +```yaml title="dbt_project.yml" +vars: + exclude_packages_and_paths: ["^(?!:).*"] +``` + +This expression will exclude all the nodes that don't start with `:`, e.g. all the nodes coming from imported packages: + +- `^` : we search for the beginning of a string +- `(?! ... )` : not immediately followed +- by `:` +- `.*` : with whatever characters after diff --git a/mkdocs.yml b/mkdocs.yml index e0bbcd3b..23b0b6be 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,8 @@ theme: # Palette toggle for light mode - media: "(prefers-color-scheme: light)" scheme: default + primary: custom + accent: custom toggle: icon: material/brightness-7 name: Switch to dark mode @@ -16,6 +18,8 @@ theme: # Palette toggle for dark mode - media: "(prefers-color-scheme: dark)" scheme: slate + primary: custom + accent: custom toggle: icon: material/brightness-4 name: Switch to light mode @@ -71,9 +75,10 @@ nav: - Structure: rules/structure.md - Performance: rules/performance.md - Customization: - - Disabling models: customization/customization.md - Overriding variables: customization/overriding-variables.md + - Disabling tests: customization/customization.md - Configuring exceptions to the rules: customization/exceptions.md + - Excluding packages and models/sources based on path: customization/excluding-packages-and-paths.md - Display issues in the logs: customization/issues-in-log.md - Run in CI Check: ci-check.md - Querying the DAG: querying-the-dag.md From 23eacf8bca523a39a7f17db6305e90bd8e4e46ac Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Tue, 9 May 2023 10:41:05 +0200 Subject: [PATCH 07/37] Update logic to use new variables --- macros/unpack/get_node_values.sql | 17 +++++++++++++---- macros/unpack/get_source_values.sql | 17 +++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/macros/unpack/get_node_values.sql b/macros/unpack/get_node_values.sql index 3eeb896b..50ebaa4d 100644 --- a/macros/unpack/get_node_values.sql +++ b/macros/unpack/get_node_values.sql @@ -14,15 +14,24 @@ {%- set hard_coded_references = dbt_project_evaluator.find_all_hard_coded_references(node) -%} {%- set ns = namespace(exclude=false) -%} - {%- set node_package_path = node.package_name ~ ":" ~ node.original_file_path | replace("\\","\\\\") -%} + {%- set node_path = node.original_file_path | replace("\\","\\\\") -%} - {%- for exclude_pattern in var('exclude_packages_and_paths',[]) -%} - {%- set is_match = re.match(exclude_pattern, node_package_path, re.IGNORECASE) -%} - {%- if is_match %} + {#- we exclude the node if it is from the current project and matches the pattern -#} + {%- for exclude_paths_pattern in var('exclude_paths_from_project',[]) -%} + {%- set matched_path = re.search(exclude_paths_pattern, node_path, re.IGNORECASE) -%} + {%- if matched_path and node.package_name == project_name %} {% set ns.exclude = true %} {%- endif -%} {%- endfor -%} + {#- we exclude the node if the package if it is listed in `exclude_packages` or if it is "all" -#} + {%- if ( + node.package_name != project_name) + and (node.package_name in var('exclude_packages',[]) or 'all' in var('exclude_packages',[])) + -%} + {% set ns.exclude = true %} + {%- endif -%} + {%- set values_line = [ wrap_string_with_quotes(node.unique_id), diff --git a/macros/unpack/get_source_values.sql b/macros/unpack/get_source_values.sql index 4774bb4f..9e6bc8e6 100644 --- a/macros/unpack/get_source_values.sql +++ b/macros/unpack/get_source_values.sql @@ -12,15 +12,24 @@ {%- for node in nodes_list -%} {%- set ns = namespace(exclude=false) -%} - {%- set node_package_path = node.package_name ~ ":" ~ node.original_file_path | replace("\\","\\\\") ~ ":" ~ node.name -%} + {%- set node_path_and_name = node.original_file_path | replace("\\","\\\\") ~ ":" ~ node.fqn[-2] ~ "." ~ node.fqn[-1] -%} - {%- for exclude_pattern in var('exclude_packages_and_paths',[]) -%} - {%- set is_match = re.match(exclude_pattern, node_package_path, re.IGNORECASE) -%} - {%- if is_match %} + {#- we exclude the source if it is from the current project and matches the pattern -#} + {%- for exclude_pattern in var('exclude_paths_from_project',[]) -%} + {%- set matched_path = re.search(exclude_pattern, node_path_and_name, re.IGNORECASE) -%} + {%- if matched_path and node.package_name == project_name %} {% set ns.exclude = true %} {%- endif -%} {%- endfor -%} + {#- we exclude the node if the package if it is listed in `exclude_packages` or if it is "all" -#} + {%- if ( + node.package_name != project_name) + and (node.package_name in var('exclude_packages',[]) or 'all' in var('exclude_packages',[])) + -%} + {% set ns.exclude = true %} + {%- endif -%} + {%- set values_line = [ wrap_string_with_quotes(node.unique_id), From 54a01246a0d402505f78733c7f568cd3d44dca6f Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Tue, 9 May 2023 10:41:32 +0200 Subject: [PATCH 08/37] Update config of the integration tests --- integration_tests/dbt_project.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c23206dd..ca3934d5 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -59,7 +59,8 @@ seeds: vars: # ensure integration tests run successfully when there are 0 of a given model type (extra) - exclude_packages_and_paths: ["^(?!dbt_project_evaluator_integration_tests:).*",".*/to_exclude/.*"] + exclude_packages: [] + exclude_paths_from_project: ["/to_exclude/","source_3.table_6"] model_types: ['staging', 'intermediate', 'marts', 'other', 'extra', 'new_model_type'] # dummy variable used for testing fct_hard_coded_references my_table_reference: 'grace_table' From 1618e5393108d7c22a9aad0e1607a713a5b8c692 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Tue, 9 May 2023 10:41:59 +0200 Subject: [PATCH 09/37] Add integration test to exclude a source --- integration_tests/models/staging/source_1/source.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/integration_tests/models/staging/source_1/source.yml b/integration_tests/models/staging/source_1/source.yml index bd2fccc2..289c264c 100644 --- a/integration_tests/models/staging/source_1/source.yml +++ b/integration_tests/models/staging/source_1/source.yml @@ -17,4 +17,9 @@ sources: schema: real_schema_2 # database: real_database tables: - - name: table_3 \ No newline at end of file + - name: table_3 + + - name: source_3 + schema: real_schema_3 + tables: + - name: table_6 \ No newline at end of file From 2db2613bd78a024b459152084013b2ecfd3f8318 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Tue, 9 May 2023 10:42:16 +0200 Subject: [PATCH 10/37] Update docs with the new parameters --- .../excluding-packages-and-paths.md | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/docs/customization/excluding-packages-and-paths.md b/docs/customization/excluding-packages-and-paths.md index 6a0fc7aa..8b1137b3 100644 --- a/docs/customization/excluding-packages-and-paths.md +++ b/docs/customization/excluding-packages-and-paths.md @@ -15,50 +15,37 @@ In that case, this package provides the ability to exclude whole packages and/or ## Configuration -The variable `exclude_packages_and_paths` allows you to define a list of regex patterns to exclude from being reported as errors. +The variables `exclude_packages` and `exclude_paths_from_project` allow you to define a list of regex patterns to exclude from being reported as errors. -- **for models**, the regex provided will try to match the pattern in the string `:`, allowing to exclude packages, but also whole folders or individual models -- **for sources**, the regex will try to match the pattern in `::` *(the pattern is different than for models because the path doesn't let us exclude individual sources)* +- `exclude_packages` accepts a list of package names to exclude from the tool. To exclude all packages except the current project, you can set it to `["all"]` +- `exclude_paths_from_project` accepts a list of regular expressions of paths to exclude for the current project + - **for models**, the regex provided will try to match the pattern in the string ``, allowing to exclude packages, but also whole folders or individual models + - **for sources**, the regex will try to match the pattern in `:.` *(the pattern is different than for models because the path itself doesn't let us exclude individual sources)* ### Example to exclude a whole package ```yaml title="dbt_project.yml" vars: - exclude_packages_and_paths: [":.*"] + exclude_packages: [""] ``` -### Example to exclude a given path +### Example to exclude models/sources in a given path ```yaml title="dbt_project.yml" vars: - exclude_packages_and_paths: [".*//.*"] + exclude_paths_from_project: ["//"] ``` -### Example to exclude both a package and models in a path +### Example to exclude both a package and models/sources in 2 different paths ```yaml title="dbt_project.yml" vars: - exclude_packages_and_paths: [":.*", ".*//.*"] + exclude_packages: [""] + exclude_paths_from_project: ["", ""] ``` ## Tips and tricks -Regular expressions are very powerful but can become complex. After defining your value for `exclude_packages_and_paths`, we recommend running the package and inspecting the model `int_all_graph_resources`, checking if the value in the column `is_disabled` matches your expectation. +Regular expressions are very powerful but can become complex. After defining your value for `exclude_paths_from_project`, we recommend running the package and inspecting the model `int_all_graph_resources`, checking if the value in the column `is_excluded` matches your expectation. A useful tool to debug regular expression is [regex101](https://regex101.com/). You can provide a pattern and a list of strings to see which ones actually match the pattern. - -### Running the tests only on the current project - -Instead of listing all the packages to exclude, we can use a [regex negative look-ahead](https://www.regular-expressions.info/lookaround.html) expression to filter out all the models and sources that come from a package different than the current project. - -```yaml title="dbt_project.yml" -vars: - exclude_packages_and_paths: ["^(?!:).*"] -``` - -This expression will exclude all the nodes that don't start with `:`, e.g. all the nodes coming from imported packages: - -- `^` : we search for the beginning of a string -- `(?! ... )` : not immediately followed -- by `:` -- `.*` : with whatever characters after From f8a6a35fc6dbc23715211a40371ac763afe781a5 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Tue, 9 May 2023 10:46:09 +0200 Subject: [PATCH 11/37] Small update to page title --- docs/customization/customization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/customization/customization.md b/docs/customization/customization.md index 1b9d5955..efa97d69 100644 --- a/docs/customization/customization.md +++ b/docs/customization/customization.md @@ -1,4 +1,4 @@ -# Disabling tests from the package +# Disabling checks from the package !!! note From dda474b7060af95c6d067795646ca47dfe94af3c Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Wed, 10 May 2023 13:11:47 +0200 Subject: [PATCH 12/37] Update wording in docs following PR comments --- .../excluding-packages-and-paths.md | 18 ++++++++++++------ mkdocs.yml | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/customization/excluding-packages-and-paths.md b/docs/customization/excluding-packages-and-paths.md index 8b1137b3..a5e8f835 100644 --- a/docs/customization/excluding-packages-and-paths.md +++ b/docs/customization/excluding-packages-and-paths.md @@ -3,10 +3,10 @@ !!! note This section is describing how to entirely exclude models/sources and packages to be evaluated. - If you want to document exception to the rules, see the section [on exceptions](exceptions.md) + If you want to document exceptions to the rules, see the section [on exceptions](exceptions.md) and if you want to deactivate entire tests you can follow instructions from [this page](customization.md) -There might be cases where you might want to exclude models/sources from being tested: +There might be cases where you want to exclude models/sources from being tested: - they could come from a package for which you have no control over - you might be refactoring your project and wanting to exclude entire folders to follow best-practices in the new models @@ -22,26 +22,32 @@ The variables `exclude_packages` and `exclude_paths_from_project` allow you to d - **for models**, the regex provided will try to match the pattern in the string ``, allowing to exclude packages, but also whole folders or individual models - **for sources**, the regex will try to match the pattern in `:.` *(the pattern is different than for models because the path itself doesn't let us exclude individual sources)* +!!! note + + We currently don't allow excluding metrics and exposures, as if those need to be entirely excluded they could be deactivated from the project. + + If you have a specific use case requiring this ability, please raise a GitHub issue to explain the situation you'd like to solve and we can revisit this decision ! + ### Example to exclude a whole package ```yaml title="dbt_project.yml" vars: - exclude_packages: [""] + exclude_packages: ["upstream_package"] ``` ### Example to exclude models/sources in a given path ```yaml title="dbt_project.yml" vars: - exclude_paths_from_project: ["//"] + exclude_paths_from_project: ["/models/legacy/"] ``` ### Example to exclude both a package and models/sources in 2 different paths ```yaml title="dbt_project.yml" vars: - exclude_packages: [""] - exclude_paths_from_project: ["", ""] + exclude_packages: ["upstream_package"] + exclude_paths_from_project: ["/models/legacy/", "/my_date_spine.sql"] ``` ## Tips and tricks diff --git a/mkdocs.yml b/mkdocs.yml index 23b0b6be..dd0dbd43 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,7 +76,7 @@ nav: - Performance: rules/performance.md - Customization: - Overriding variables: customization/overriding-variables.md - - Disabling tests: customization/customization.md + - Disabling checks: customization/customization.md - Configuring exceptions to the rules: customization/exceptions.md - Excluding packages and models/sources based on path: customization/excluding-packages-and-paths.md - Display issues in the logs: customization/issues-in-log.md From 45c83def57582b40be46e3b812926c468e98c2c3 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Wed, 10 May 2023 13:12:12 +0200 Subject: [PATCH 13/37] Move the logic to exclude to a dedicated macro --- macros/set_is_excluded.sql | 39 +++++++++++++++++++++++++++++ macros/unpack/get_node_values.sql | 21 ++-------------- macros/unpack/get_source_values.sql | 22 ++-------------- 3 files changed, 43 insertions(+), 39 deletions(-) create mode 100644 macros/set_is_excluded.sql diff --git a/macros/set_is_excluded.sql b/macros/set_is_excluded.sql new file mode 100644 index 00000000..f7067da1 --- /dev/null +++ b/macros/set_is_excluded.sql @@ -0,0 +1,39 @@ +{% macro set_is_excluded(resource, resource_type) %} + {{ return(adapter.dispatch('set_is_excluded', 'dbt_project_evaluator')(resource, resource_type)) }} +{% endmacro %} + +{% macro default__set_is_excluded(resource, resource_type) %} + + {% set re = modules.re %} + {%- set ns = namespace(exclude=false) -%} + + {% if resource_type == 'node' %} + {%- set resource_path = resource.original_file_path | replace("\\","\\\\") -%} + {% elif resource_type == 'source' %} + {%- set resource_path = resource.original_file_path | replace("\\","\\\\") ~ ":" ~ resource.fqn[-2] ~ "." ~ resource.fqn[-1] -%} + {% else %} + {{ exceptions.raise_compiler_error( + "`set_is_excluded()` macro does not support resource type: " ~ resource_type + ) }} + {% endif %} + + + {#- we exclude the resource if it is from the current project and matches the pattern -#} + {%- for exclude_paths_pattern in var('exclude_paths_from_project',[]) -%} + {%- set matched_path = re.search(exclude_paths_pattern, resource_path, re.IGNORECASE) -%} + {%- if matched_path and resource.package_name == project_name %} + {% set ns.exclude = true %} + {%- endif -%} + {%- endfor -%} + + {#- we exclude the resource if the package if it is listed in `exclude_packages` or if it is "all" -#} + {%- if ( + resource.package_name != project_name) + and (resource.package_name in var('exclude_packages',[]) or 'all' in var('exclude_packages',[])) + -%} + {% set ns.exclude = true %} + {%- endif -%} + + {{ return(ns.exclude) }} + +{% endmacro %} diff --git a/macros/unpack/get_node_values.sql b/macros/unpack/get_node_values.sql index 50ebaa4d..25e6a90f 100644 --- a/macros/unpack/get_node_values.sql +++ b/macros/unpack/get_node_values.sql @@ -12,25 +12,8 @@ {%- for node in nodes_list -%} {%- set hard_coded_references = dbt_project_evaluator.find_all_hard_coded_references(node) -%} + {%- set exclude_node = dbt_project_evaluator.set_is_excluded(node, resource_type="node") -%} - {%- set ns = namespace(exclude=false) -%} - {%- set node_path = node.original_file_path | replace("\\","\\\\") -%} - - {#- we exclude the node if it is from the current project and matches the pattern -#} - {%- for exclude_paths_pattern in var('exclude_paths_from_project',[]) -%} - {%- set matched_path = re.search(exclude_paths_pattern, node_path, re.IGNORECASE) -%} - {%- if matched_path and node.package_name == project_name %} - {% set ns.exclude = true %} - {%- endif -%} - {%- endfor -%} - - {#- we exclude the node if the package if it is listed in `exclude_packages` or if it is "all" -#} - {%- if ( - node.package_name != project_name) - and (node.package_name in var('exclude_packages',[]) or 'all' in var('exclude_packages',[])) - -%} - {% set ns.exclude = true %} - {%- endif -%} {%- set values_line = [ @@ -51,7 +34,7 @@ wrap_string_with_quotes(dbt.escape_single_quotes(hard_coded_references)), wrap_string_with_quotes(node.get('depends_on',{}).get('macros',[]) | tojson), "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.test_metadata) | trim ~ " as boolean)", - "cast(" ~ ns.exclude ~ " as boolean)", + "cast(" ~ exclude_node ~ " as boolean)", ] %} diff --git a/macros/unpack/get_source_values.sql b/macros/unpack/get_source_values.sql index 9e6bc8e6..b8d3abad 100644 --- a/macros/unpack/get_source_values.sql +++ b/macros/unpack/get_source_values.sql @@ -5,30 +5,12 @@ {%- macro default__get_source_values() -%} {%- if execute -%} - {% set re = modules.re %} {%- set nodes_list = graph.sources.values() -%} {%- set values = [] -%} {%- for node in nodes_list -%} - {%- set ns = namespace(exclude=false) -%} - {%- set node_path_and_name = node.original_file_path | replace("\\","\\\\") ~ ":" ~ node.fqn[-2] ~ "." ~ node.fqn[-1] -%} - - {#- we exclude the source if it is from the current project and matches the pattern -#} - {%- for exclude_pattern in var('exclude_paths_from_project',[]) -%} - {%- set matched_path = re.search(exclude_pattern, node_path_and_name, re.IGNORECASE) -%} - {%- if matched_path and node.package_name == project_name %} - {% set ns.exclude = true %} - {%- endif -%} - {%- endfor -%} - - {#- we exclude the node if the package if it is listed in `exclude_packages` or if it is "all" -#} - {%- if ( - node.package_name != project_name) - and (node.package_name in var('exclude_packages',[]) or 'all' in var('exclude_packages',[])) - -%} - {% set ns.exclude = true %} - {%- endif -%} + {%- set exclude_source = dbt_project_evaluator.set_is_excluded(node, resource_type="source") -%} {%- set values_line = [ @@ -48,7 +30,7 @@ wrap_string_with_quotes(node.loader), wrap_string_with_quotes(node.identifier), wrap_string_with_quotes(node.meta | tojson), - "cast(" ~ ns.exclude ~ " as boolean)", + "cast(" ~ exclude_source ~ " as boolean)", ] %} From fd43832506f34b4c786d45cb3f27e43bc1ca2654 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Wed, 10 May 2023 20:46:51 +0200 Subject: [PATCH 14/37] Remove `re` that moved to the macro --- macros/unpack/get_node_values.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/macros/unpack/get_node_values.sql b/macros/unpack/get_node_values.sql index 25e6a90f..e9c72e0f 100644 --- a/macros/unpack/get_node_values.sql +++ b/macros/unpack/get_node_values.sql @@ -5,7 +5,6 @@ {%- macro default__get_node_values() -%} {%- if execute -%} - {% set re = modules.re %} {%- set nodes_list = graph.nodes.values() -%} {%- set values = [] -%} From 4f0674f7cc5539654073d58ca000252f96a7fb0a Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Mon, 15 May 2023 15:54:57 -0700 Subject: [PATCH 15/37] adjust root models to only filter excluded children nodes --- models/marts/dag/fct_root_models.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/marts/dag/fct_root_models.sql b/models/marts/dag/fct_root_models.sql index fa55f9a0..692302a1 100644 --- a/models/marts/dag/fct_root_models.sql +++ b/models/marts/dag/fct_root_models.sql @@ -5,7 +5,8 @@ with model_relationships as ( * from {{ ref('int_all_dag_relationships') }} where child_resource_type = 'model' - and not parent_is_excluded + -- only filter out excluded children nodes + -- filtering parents could result in incorrectly flagging nodes that depend on excluded nodes and not child_is_excluded ), From 1ef0fd7c0d04b6b4b937740c0eaf848b9016407a Mon Sep 17 00:00:00 2001 From: David Bloss Date: Wed, 17 May 2023 12:06:46 -0500 Subject: [PATCH 16/37] update used gh actions ahead of node12 deprecation --- .github/workflows/mkdocs_main.yml | 6 +++--- .github/workflows/mkdocs_release.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mkdocs_main.yml b/.github/workflows/mkdocs_main.yml index f59086a0..4dc22079 100644 --- a/.github/workflows/mkdocs_main.yml +++ b/.github/workflows/mkdocs_main.yml @@ -10,12 +10,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 @@ -30,4 +30,4 @@ jobs: run: | git config --global user.name "GitHub Actions Bot" git config --global user.email "github-actions[bot]@users.noreply.github.com" - mike deploy --push --message "Deployed by GitHub Actions" main \ No newline at end of file + mike deploy --push --message "Deployed by GitHub Actions" main diff --git a/.github/workflows/mkdocs_release.yml b/.github/workflows/mkdocs_release.yml index 651fd3b9..cf44343f 100644 --- a/.github/workflows/mkdocs_release.yml +++ b/.github/workflows/mkdocs_release.yml @@ -10,12 +10,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 @@ -41,4 +41,4 @@ jobs: git config user.name "GitHub Actions" git config user.email "actions@github.com" mike deploy --push --message "Deploy docs for release ${{ github.event.release.tag_name }}" --update-alias $MAJOR_MINOR_VERSION latest - mike set-default --push latest \ No newline at end of file + mike set-default --push latest From 433ff08f691047a3c04b22549b83d66dbd1714f4 Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Fri, 26 May 2023 13:57:46 +0200 Subject: [PATCH 17/37] =?UTF-8?q?=F0=9F=90=9B=20Use=20`alias`=20rather=20t?= =?UTF-8?q?han=20the=20model=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- macros/on-run-end/print_dbt_project_evaluator_issues.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/on-run-end/print_dbt_project_evaluator_issues.sql b/macros/on-run-end/print_dbt_project_evaluator_issues.sql index 9079d0e2..802c926c 100644 --- a/macros/on-run-end/print_dbt_project_evaluator_issues.sql +++ b/macros/on-run-end/print_dbt_project_evaluator_issues.sql @@ -10,9 +10,9 @@ {{ print("\n-- " ~ result.node.alias ~ " --") }} {% set unique_id_model_checked = result.node.depends_on.nodes[0] %} - {% set name_model_checked = unique_id_model_checked.split('.')[-1] %} {% set model_details = graph["nodes"][unique_id_model_checked] %} + {% set name_model_checked = model_details.alias %} {% set model_schema = model_details.schema %} {% set model_database = model_details.database %} {% set db_schema = model_database ~ "." ~ model_schema if model_database else model_schema %} From 649c3e156aa7aa0dc2b4c91eea8328bd3aec0dbd Mon Sep 17 00:00:00 2001 From: Benoit Perigaud <8754100+b-per@users.noreply.github.com> Date: Fri, 26 May 2023 15:00:51 +0200 Subject: [PATCH 18/37] =?UTF-8?q?=F0=9F=90=9B=20Fix=20exceptions,=20by=20r?= =?UTF-8?q?eading=20`model.name`=20`this.name`=20gives=20the=20table=20nam?= =?UTF-8?q?e=20and=20not=20the=20model=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- macros/filter_exceptions.sql | 8 ++++---- models/marts/dag/fct_direct_join_to_source.sql | 2 +- models/marts/dag/fct_hard_coded_references.sql | 2 +- .../dag/fct_marts_or_intermediate_dependent_on_source.sql | 2 +- models/marts/dag/fct_model_fanout.sql | 2 +- models/marts/dag/fct_multiple_sources_joined.sql | 2 +- models/marts/dag/fct_rejoining_of_upstream_concepts.sql | 2 +- models/marts/dag/fct_root_models.sql | 2 +- models/marts/dag/fct_source_fanout.sql | 2 +- .../fct_staging_dependent_on_marts_or_intermediate.sql | 2 +- models/marts/dag/fct_staging_dependent_on_staging.sql | 2 +- models/marts/dag/fct_unused_sources.sql | 2 +- models/marts/documentation/fct_undocumented_models.sql | 2 +- .../marts/performance/fct_chained_views_dependencies.sql | 2 +- .../performance/fct_exposure_parents_materializations.sql | 2 +- models/marts/structure/fct_model_directories.sql | 2 +- models/marts/structure/fct_model_naming_conventions.sql | 2 +- models/marts/structure/fct_source_directories.sql | 2 +- models/marts/structure/fct_test_directories.sql | 2 +- models/marts/tests/fct_missing_primary_key_tests.sql | 8 +++++++- 20 files changed, 29 insertions(+), 23 deletions(-) diff --git a/macros/filter_exceptions.sql b/macros/filter_exceptions.sql index 304d5e84..9c86109a 100644 --- a/macros/filter_exceptions.sql +++ b/macros/filter_exceptions.sql @@ -1,15 +1,15 @@ -{% macro filter_exceptions(model_ref) -%} - {{ return(adapter.dispatch('filter_exceptions', 'dbt_project_evaluator')(model_ref)) }} +{% macro filter_exceptions(model_name) -%} + {{ return(adapter.dispatch('filter_exceptions', 'dbt_project_evaluator')(model_name)) }} {%- endmacro %} -{% macro default__filter_exceptions(model_ref) %} +{% macro default__filter_exceptions(model_name) %} {% set query_filters %} select column_name, id_to_exclude from {{ ref('dbt_project_evaluator_exceptions') }} - where fct_name = '{{ model_ref.name }}' + where fct_name = '{{ model_name }}' {% endset %} {% if execute %} diff --git a/models/marts/dag/fct_direct_join_to_source.sql b/models/marts/dag/fct_direct_join_to_source.sql index ebb32cbe..52a9f344 100644 --- a/models/marts/dag/fct_direct_join_to_source.sql +++ b/models/marts/dag/fct_direct_join_to_source.sql @@ -35,4 +35,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_hard_coded_references.sql b/models/marts/dag/fct_hard_coded_references.sql index a7769453..7d16e821 100644 --- a/models/marts/dag/fct_hard_coded_references.sql +++ b/models/marts/dag/fct_hard_coded_references.sql @@ -15,4 +15,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql b/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql index fa354955..08d56abf 100644 --- a/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql +++ b/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql @@ -17,4 +17,4 @@ final as ( ) select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_model_fanout.sql b/models/marts/dag/fct_model_fanout.sql index 9d1a39db..4c2e0966 100644 --- a/models/marts/dag/fct_model_fanout.sql +++ b/models/marts/dag/fct_model_fanout.sql @@ -46,4 +46,4 @@ model_fanout_agg as ( select * from model_fanout_agg -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_multiple_sources_joined.sql b/models/marts/dag/fct_multiple_sources_joined.sql index e2b6bb4a..9946aeb7 100644 --- a/models/marts/dag/fct_multiple_sources_joined.sql +++ b/models/marts/dag/fct_multiple_sources_joined.sql @@ -25,4 +25,4 @@ multiple_sources_joined as ( select * from multiple_sources_joined -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_rejoining_of_upstream_concepts.sql b/models/marts/dag/fct_rejoining_of_upstream_concepts.sql index 318ae778..b6e745d2 100644 --- a/models/marts/dag/fct_rejoining_of_upstream_concepts.sql +++ b/models/marts/dag/fct_rejoining_of_upstream_concepts.sql @@ -64,4 +64,4 @@ final_filtered as ( select * from final_filtered -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_root_models.sql b/models/marts/dag/fct_root_models.sql index 0ea8e4ff..01bc36f3 100644 --- a/models/marts/dag/fct_root_models.sql +++ b/models/marts/dag/fct_root_models.sql @@ -17,4 +17,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_source_fanout.sql b/models/marts/dag/fct_source_fanout.sql index c2675e43..423f34a4 100644 --- a/models/marts/dag/fct_source_fanout.sql +++ b/models/marts/dag/fct_source_fanout.sql @@ -25,4 +25,4 @@ source_fanout as ( select * from source_fanout -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql b/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql index 321da88d..f1b18528 100644 --- a/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql +++ b/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql @@ -20,4 +20,4 @@ final as ( ) select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_staging_dependent_on_staging.sql b/models/marts/dag/fct_staging_dependent_on_staging.sql index fbbe0e01..2c71c841 100644 --- a/models/marts/dag/fct_staging_dependent_on_staging.sql +++ b/models/marts/dag/fct_staging_dependent_on_staging.sql @@ -21,4 +21,4 @@ bending_connections as ( select * from bending_connections -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_unused_sources.sql b/models/marts/dag/fct_unused_sources.sql index 6803dbef..6b32cec1 100644 --- a/models/marts/dag/fct_unused_sources.sql +++ b/models/marts/dag/fct_unused_sources.sql @@ -17,4 +17,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/documentation/fct_undocumented_models.sql b/models/marts/documentation/fct_undocumented_models.sql index 60ae4b43..d62186a2 100644 --- a/models/marts/documentation/fct_undocumented_models.sql +++ b/models/marts/documentation/fct_undocumented_models.sql @@ -18,4 +18,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/performance/fct_chained_views_dependencies.sql b/models/marts/performance/fct_chained_views_dependencies.sql index 42527100..e4850bcc 100644 --- a/models/marts/performance/fct_chained_views_dependencies.sql +++ b/models/marts/performance/fct_chained_views_dependencies.sql @@ -18,6 +18,6 @@ final as ( select * from final -{{ filter_exceptions(this) }} +{{ filter_exceptions(model.name) }} order by distance desc \ No newline at end of file diff --git a/models/marts/performance/fct_exposure_parents_materializations.sql b/models/marts/performance/fct_exposure_parents_materializations.sql index 87bd7683..4383494f 100644 --- a/models/marts/performance/fct_exposure_parents_materializations.sql +++ b/models/marts/performance/fct_exposure_parents_materializations.sql @@ -29,4 +29,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/structure/fct_model_directories.sql b/models/marts/structure/fct_model_directories.sql index 933f5230..e122bf7a 100644 --- a/models/marts/structure/fct_model_directories.sql +++ b/models/marts/structure/fct_model_directories.sql @@ -65,6 +65,6 @@ unioned as ( select * from unioned -{{ filter_exceptions(this) }} +{{ filter_exceptions(model.name) }} diff --git a/models/marts/structure/fct_model_naming_conventions.sql b/models/marts/structure/fct_model_naming_conventions.sql index 8f4bedca..85e1233a 100644 --- a/models/marts/structure/fct_model_naming_conventions.sql +++ b/models/marts/structure/fct_model_naming_conventions.sql @@ -51,4 +51,4 @@ inappropriate_model_names as ( select * from inappropriate_model_names -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/structure/fct_source_directories.sql b/models/marts/structure/fct_source_directories.sql index a8e3c152..0ea358e3 100644 --- a/models/marts/structure/fct_source_directories.sql +++ b/models/marts/structure/fct_source_directories.sql @@ -22,4 +22,4 @@ inappropriate_subdirectories_sources as ( select * from inappropriate_subdirectories_sources -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/structure/fct_test_directories.sql b/models/marts/structure/fct_test_directories.sql index 3ca1c65e..c9646923 100644 --- a/models/marts/structure/fct_test_directories.sql +++ b/models/marts/structure/fct_test_directories.sql @@ -84,4 +84,4 @@ different_directories as ( select * from different_directories -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/tests/fct_missing_primary_key_tests.sql b/models/marts/tests/fct_missing_primary_key_tests.sql index 4c55a334..37bd0c29 100644 --- a/models/marts/tests/fct_missing_primary_key_tests.sql +++ b/models/marts/tests/fct_missing_primary_key_tests.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = 'my_alias', + ) +}} + with tests as ( @@ -15,4 +21,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file From 5a0b7f5d82ce2f6515aafab742d047354114d070 Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Fri, 26 May 2023 12:16:45 -0700 Subject: [PATCH 19/37] Fix docs hyperlink path --- docs/rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rules.md b/docs/rules.md index 96046e32..f7bc2154 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -12,7 +12,7 @@ hide: |Modeling |[Rejoining of Upstream Concepts](../rules/modeling/#rejoining-of-upstream-concepts) |`fct_rejoining_of_upstream_concepts`| |Modeling |[Model Fanout](../rules/modeling/#model-fanout) |`fct_model_fanout`| |Modeling |[Downstream Models Dependent on Source](../rules/modeling/#downstream-models-dependent-on-source) |`fct_marts_or_intermediate_dependent_on_source`| -|Modeling |[Direct Join to Source](/..rules/modeling/#direct-join-to-source) |`fct_direct_join_to_source`| +|Modeling |[Direct Join to Source](../rules/modeling/#direct-join-to-source) |`fct_direct_join_to_source`| |Modeling |[Duplicate Sources](../rules/modeling/#duplicate-sources) |`fct_duplicate_sources`| |Modeling |[Hard Coded References](../rules/modeling/#hard-coded-references) |`fct_hard_coded_references`| |Modeling |[Multiple Sources Joined](../rules/modeling/#multiple-sources-joined) |`fct_multiple_sources_joined`| From 13ce0c2c95c59ef4ee2120c2b6efbe252c9a3e10 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Tue, 30 May 2023 13:22:23 -0700 Subject: [PATCH 20/37] select only relevant rows in fct_direct_join_to_source --- models/marts/dag/fct_direct_join_to_source.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/models/marts/dag/fct_direct_join_to_source.sql b/models/marts/dag/fct_direct_join_to_source.sql index 27a0f0c3..4fa1b432 100644 --- a/models/marts/dag/fct_direct_join_to_source.sql +++ b/models/marts/dag/fct_direct_join_to_source.sql @@ -27,7 +27,11 @@ model_and_source_joined as ( final as ( select - direct_model_relationships.* + direct_model_relationships.parent, + direct_model_relationships.parent_resource_type, + direct_model_relationships.child, + direct_model_relationships.child_resource_type, + direct_model_relationships.distance from direct_model_relationships inner join model_and_source_joined on direct_model_relationships.child = model_and_source_joined.child From 761aff1d301942485c872f255a62b563f3560452 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Thu, 8 Jun 2023 13:13:59 -0700 Subject: [PATCH 21/37] updated integration tests for exceptions seed --- integration_tests/seeds/dbt_project_evaluator_exceptions.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/seeds/dbt_project_evaluator_exceptions.csv b/integration_tests/seeds/dbt_project_evaluator_exceptions.csv index 33d8958c..d57e2718 100644 --- a/integration_tests/seeds/dbt_project_evaluator_exceptions.csv +++ b/integration_tests/seeds/dbt_project_evaluator_exceptions.csv @@ -1,2 +1,2 @@ fct_name,column_name,id_to_exclude,comment -fct_direct_join_to_source,parent_id,model.dbt_project_evaluator_integration_tests.stg_model_1,This is actually OK because... \ No newline at end of file +fct_direct_join_to_source,parent,stg_model_1,This is actually OK because... \ No newline at end of file From 8f8d1cf9373e8745413f2ff581e582445278d5c6 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 28 Jun 2023 16:37:56 -0500 Subject: [PATCH 22/37] update metric to new metric spec --- integration_tests/models/marts/exposures.yml | 1 + .../models/marts/int_model_5.sql | 5 ++- .../models/marts/intermediate/dim_model_7.sql | 11 ++++- integration_tests/models/marts/metrics.yml | 44 +++++++++---------- ..._fct_exposure_parents_materializations.csv | 2 +- macros/unpack/get_metric_values.sql | 24 ++++------ models/marts/core/int_all_graph_resources.sql | 11 +++-- models/staging/graph/stg_metrics.sql | 13 +++--- 8 files changed, 60 insertions(+), 51 deletions(-) diff --git a/integration_tests/models/marts/exposures.yml b/integration_tests/models/marts/exposures.yml index a38e7136..1382780d 100644 --- a/integration_tests/models/marts/exposures.yml +++ b/integration_tests/models/marts/exposures.yml @@ -13,6 +13,7 @@ exposures: depends_on: - ref('fct_model_6') + - ref('fct_model_9') - ref('dim_model_7') - ref('dim_model_7') - source('source_1', 'table_1') \ No newline at end of file diff --git a/integration_tests/models/marts/int_model_5.sql b/integration_tests/models/marts/int_model_5.sql index 6d5e4c3e..c2530621 100644 --- a/integration_tests/models/marts/int_model_5.sql +++ b/integration_tests/models/marts/int_model_5.sql @@ -1,2 +1,3 @@ -{{ ref('int_model_4') }} -{{ ref('stg_model_1') }} \ No newline at end of file +-- {{ ref('int_model_4') }} +-- {{ ref('stg_model_1') }} +select 1 as id \ No newline at end of file diff --git a/integration_tests/models/marts/intermediate/dim_model_7.sql b/integration_tests/models/marts/intermediate/dim_model_7.sql index c42daeb9..3ddf87e8 100644 --- a/integration_tests/models/marts/intermediate/dim_model_7.sql +++ b/integration_tests/models/marts/intermediate/dim_model_7.sql @@ -1,2 +1,9 @@ -{{ ref('stg_model_4') }} -{{ ref('int_model_5') }} \ No newline at end of file + +{{ + config( + materialized = 'table', + ) +}} + +select * from {{ ref('stg_model_4') }} +-- {{ ref('int_model_5') }} \ No newline at end of file diff --git a/integration_tests/models/marts/metrics.yml b/integration_tests/models/marts/metrics.yml index 44d21460..3e9903a2 100644 --- a/integration_tests/models/marts/metrics.yml +++ b/integration_tests/models/marts/metrics.yml @@ -1,29 +1,29 @@ version: 2 -metrics: - - name: new_customers - label: New Customers marked 'paying' - model: ref('dim_model_7') - description: "The number of paid customers using the product" - calculation_method: count - expression: user_id # superfluous here, but shown as an example +semantic_models: + - name: my_model # A semantic model with the name Transactions + model: ref('dim_model_7') # References the dbt model named `fact_transactions` + description: "Transaction fact table at the transaction level. This table contains one row per transaction and includes the transaction timestamp." - timestamp: signup_date - time_grains: [day, week, month] + entities: # Entities included in the table are defined here. MetricFlow will use these columns as join keys. + - name: dim + type: primary + expr: id - dimensions: - - plan - - country + dimensions: # dimensions are qualitative values such as names, dates, or geographical data. They provide context to metrics and allow "metric by group" data slicing. + - name: id + type: categorical - filters: - - field: is_paying - operator: '=' - value: 'true' - - field: company_name - operator: '!=' - value: "Acme', Inc" + measures: # Measures are columns we perform an aggregation over. Measures are inputs to metrics. + - name: total_count + description: "The total count." + agg: count - meta: - team: "Finance" - refresh_rate: "Bob's weekly run" +metrics: + - name: new_metric + label: "New Metric" + type: simple + type_params: + # Specify the measure you are creating a proxy for. + measure: total_count \ No newline at end of file diff --git a/integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv b/integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv index ffe826ec..c74b99c6 100644 --- a/integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv +++ b/integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv @@ -1,3 +1,3 @@ parent_resource_type,parent_resource_name,exposure_name,parent_model_materialization -model,dim_model_7,exposure_1,ephemeral +model,fct_model_9,exposure_1,ephemeral source,source_1.table_1,exposure_1, \ No newline at end of file diff --git a/macros/unpack/get_metric_values.sql b/macros/unpack/get_metric_values.sql index 80ac8e8f..f10e9ef9 100644 --- a/macros/unpack/get_metric_values.sql +++ b/macros/unpack/get_metric_values.sql @@ -9,17 +9,6 @@ {%- set values = [] -%} {%- for node in nodes_list -%} - - {% set metric_filters %} - {%- if node.filters|length -%} - {%- for filt in node.filters %} - '{{ filt.field }}'||'{{ filt.operator }}'||'''{{ dbt.escape_single_quotes(filt.value) }}''' - {% if not loop.last %}|| ' - ' ||{% endif %} - {% endfor -%} - {%- else -%} - '' - {% endif -%} - {% endset %} {%- set values_line = [ @@ -29,13 +18,16 @@ wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")), "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as boolean)", wrap_string_with_quotes(node.type), - wrap_string_with_quotes(node.model.identifier), wrap_string_with_quotes(dbt.escape_single_quotes(node.label)), - wrap_string_with_quotes(node.sql), - wrap_string_with_quotes(node.timestamp), wrap_string_with_quotes(node.package_name), - wrap_string_with_quotes(node.dimensions|join(' - ')), - metric_filters, + wrap_string_with_quotes(node.filter), + wrap_string_with_quotes(node.type_params.measure.name), + wrap_string_with_quotes(node.type_params.measure.alias), + wrap_string_with_quotes(node.type_params.numerator), + wrap_string_with_quotes(node.type_params.denominator), + wrap_string_with_quotes(node.type_params.expr), + wrap_string_with_quotes(node.type_params.window), + wrap_string_with_quotes(node.type_params.grain_to_date), wrap_string_with_quotes(node.meta | tojson) ] %} diff --git a/models/marts/core/int_all_graph_resources.sql b/models/marts/core/int_all_graph_resources.sql index 1fa1685e..6ac0083a 100644 --- a/models/marts/core/int_all_graph_resources.sql +++ b/models/marts/core/int_all_graph_resources.sql @@ -87,10 +87,15 @@ joined as ( unioned_with_calc.meta, unioned_with_calc.macro_dependencies, unioned_with_calc.metric_type, - unioned_with_calc.model, unioned_with_calc.label, - unioned_with_calc.sql, - unioned_with_calc.timestamp as timestamp, + unioned_with_calc.metric_filter, + unioned_with_calc.metric_measure, + unioned_with_calc.metric_measure_alias, + unioned_with_calc.numerator, + unioned_with_calc.denominator, + unioned_with_calc.expr, + unioned_with_calc.metric_window, + unioned_with_calc.grain_to_date, unioned_with_calc.source_name, -- NULL for non-source resources unioned_with_calc.is_source_described, unioned_with_calc.loaded_at_field, diff --git a/models/staging/graph/stg_metrics.sql b/models/staging/graph/stg_metrics.sql index fd921892..dd37b9a1 100644 --- a/models/staging/graph/stg_metrics.sql +++ b/models/staging/graph/stg_metrics.sql @@ -21,13 +21,16 @@ select cast(null as {{ dbt.type_string() }}) as file_path, cast(True as boolean) as is_described, cast(null as {{ dbt.type_string() }}) as metric_type, - cast(null as {{ dbt.type_string() }}) as model, cast(null as {{ dbt.type_string() }}) as label, - cast(null as {{ dbt.type_string() }}) as sql, - cast(null as {{ dbt.type_string() }}) as timestamp, cast(null as {{ dbt.type_string() }}) as package_name, - cast(null as {{ dbt.type_string() }}) as dimensions, - cast(null as {{ dbt.type_string() }}) as filters, + cast(null as {{ dbt.type_string() }}) as metric_filter, + cast(null as {{ dbt.type_string() }}) as metric_measure, + cast(null as {{ dbt.type_string() }}) as metric_measure_alias, + cast(null as {{ dbt.type_string() }}) as numerator, + cast(null as {{ dbt.type_string() }}) as denominator, + cast(null as {{ dbt.type_string() }}) as expr, + cast(null as {{ dbt.type_string() }}) as metric_window, + cast(null as {{ dbt.type_string() }}) as grain_to_date, cast(null as {{ dbt.type_string() }}) as meta from dummy_cte From 610885c21b14c14424e58e603010c3f28888b330 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 28 Jun 2023 16:39:40 -0500 Subject: [PATCH 23/37] update required dbt version --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index c4471679..53023e6f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ name: 'dbt_project_evaluator' version: '1.0.0' config-version: 2 -require-dbt-version: [">=1.3.0", "<2.0.0"] +require-dbt-version: [">=1.6.0", "<2.0.0"] model-paths: ["models"] analysis-paths: ["analysis"] From 408d8e140b94e4d20187d06331fde7949bf08c36 Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:41:32 -0600 Subject: [PATCH 24/37] GitHub Action to add/remove triage labels as-needed (#361) --- .github/workflows/triage-labels.yml | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/triage-labels.yml diff --git a/.github/workflows/triage-labels.yml b/.github/workflows/triage-labels.yml new file mode 100644 index 00000000..0f88157b --- /dev/null +++ b/.github/workflows/triage-labels.yml @@ -0,0 +1,31 @@ +# **what?** +# When we triage issues, we sometimes need more information from the issue creator. In +# those cases we remove the `triage` label and add the `awaiting_response` label. Once we +# receive a response in the form of a comment, we want the `awaiting_response` label removed +# in favor of the `triage` label so we are aware that the issue needs action. + +# **why?** +# To help with out team triage issue tracking + +# **when?** +# This will run when a comment is added to an issue and that issue has the `awaiting_response` label. + +name: Update Triage Label + +on: issue_comment + +defaults: + run: + shell: bash + +permissions: + issues: write + +jobs: + triage_label: + if: contains(github.event.issue.labels.*.name, 'awaiting_response') + uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main + with: + add_label: "triage" + remove_label: "awaiting_response" + secrets: inherit From 6db4b252b1932f4338201fb46fea13477967c3da Mon Sep 17 00:00:00 2001 From: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:42:28 -0600 Subject: [PATCH 25/37] GitHub Action to close issues as stale as-needed (#360) --- .github/workflows/stale.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..75a14dd4 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,30 @@ +# **what?** +# For issues that have been open for awhile without activity, label +# them as stale with a warning that they will be closed out. If +# anyone comments to keep the issue open, it will automatically +# remove the stale label and keep it open. + +# Stale label rules: +# awaiting_response, more_information_needed -> 90 days +# good_first_issue, help_wanted -> 360 days (a year) +# tech_debt -> 720 (2 years) +# all else defaults -> 180 days (6 months) + +# **why?** +# To keep the repo in a clean state from issues that aren't relevant anymore + +# **when?** +# Once a day + +name: "Close stale issues and PRs" +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main From 0dfef7d368a43ddc7f3e4bd564f08d3a01b3ec74 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 19 Jul 2023 13:58:45 -0500 Subject: [PATCH 26/37] only consider sql models in find_all_hard_coded_references --- macros/find_all_hard_coded_references.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/macros/find_all_hard_coded_references.sql b/macros/find_all_hard_coded_references.sql index 731ff25c..28fe4a98 100644 --- a/macros/find_all_hard_coded_references.sql +++ b/macros/find_all_hard_coded_references.sql @@ -6,7 +6,8 @@ {%- set all_hard_coded_references_list = [] -%} - {% if node.resource_type == 'model' %} + {% if node.resource_type == 'model' and node.language == "sql" %} + {{ log(node.language, info=True) }} {% if execute %} {%- set model_raw_sql = node.raw_sql or node.raw_code -%} From 772fdd970a60e9fb659d58f047dc98fa41c7a035 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 19 Jul 2023 14:00:25 -0500 Subject: [PATCH 27/37] remove log --- macros/find_all_hard_coded_references.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/macros/find_all_hard_coded_references.sql b/macros/find_all_hard_coded_references.sql index 28fe4a98..d908d806 100644 --- a/macros/find_all_hard_coded_references.sql +++ b/macros/find_all_hard_coded_references.sql @@ -7,7 +7,6 @@ {%- set all_hard_coded_references_list = [] -%} {% if node.resource_type == 'model' and node.language == "sql" %} - {{ log(node.language, info=True) }} {% if execute %} {%- set model_raw_sql = node.raw_sql or node.raw_code -%} From a73f60def8fc6f44cf8b810f23e0e3ca213bb652 Mon Sep 17 00:00:00 2001 From: Grace Goheen Date: Wed, 19 Jul 2023 15:35:58 -0700 Subject: [PATCH 28/37] include 1.6.0-rc1 in require-dbt-version --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 53023e6f..f06ff3e9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ name: 'dbt_project_evaluator' version: '1.0.0' config-version: 2 -require-dbt-version: [">=1.6.0", "<2.0.0"] +require-dbt-version: [">=1.6.0-rc1", "<2.0.0"] model-paths: ["models"] analysis-paths: ["analysis"] From e7b7bdf0ee85696deab5dbcbfa9881e55b57a96b Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 20 Jul 2023 10:07:04 -0500 Subject: [PATCH 29/37] add metricflow spine, adjsut count metrics, add exceptions to tests and global seed --- integration_tests/models/marts/metrics.yml | 8 +++++++ .../models/utils/_utils_models.yml | 8 +++++++ .../models/utils/metricflow_time_spine.sql | 22 +++++++++++++++++++ .../dbt_project_evaluator_exceptions.csv | 4 +++- .../docs/test_fct_documentation_coverage.csv | 2 +- .../seeds/tests/test_fct_test_coverage.csv | 2 +- seeds/dbt_project_evaluator_exceptions.csv | 4 +++- 7 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 integration_tests/models/utils/_utils_models.yml create mode 100644 integration_tests/models/utils/metricflow_time_spine.sql diff --git a/integration_tests/models/marts/metrics.yml b/integration_tests/models/marts/metrics.yml index 3e9903a2..d8b694e7 100644 --- a/integration_tests/models/marts/metrics.yml +++ b/integration_tests/models/marts/metrics.yml @@ -5,6 +5,8 @@ semantic_models: - name: my_model # A semantic model with the name Transactions model: ref('dim_model_7') # References the dbt model named `fact_transactions` description: "Transaction fact table at the transaction level. This table contains one row per transaction and includes the transaction timestamp." + defaults: + agg_time_dimension: date_day entities: # Entities included in the table are defined here. MetricFlow will use these columns as join keys. - name: dim @@ -15,10 +17,16 @@ semantic_models: - name: id type: categorical + - name: date_day + type: time + type_params: + time_granularity: day + measures: # Measures are columns we perform an aggregation over. Measures are inputs to metrics. - name: total_count description: "The total count." agg: count + expr: 1 metrics: - name: new_metric diff --git a/integration_tests/models/utils/_utils_models.yml b/integration_tests/models/utils/_utils_models.yml new file mode 100644 index 00000000..5f897e82 --- /dev/null +++ b/integration_tests/models/utils/_utils_models.yml @@ -0,0 +1,8 @@ +models: + - name: metricflow_time_spine + description: default time spine for metricflow + columns: + - name: date_day + tests: + - unique + - not_null \ No newline at end of file diff --git a/integration_tests/models/utils/metricflow_time_spine.sql b/integration_tests/models/utils/metricflow_time_spine.sql new file mode 100644 index 00000000..aa25fd1b --- /dev/null +++ b/integration_tests/models/utils/metricflow_time_spine.sql @@ -0,0 +1,22 @@ +{{ + config( + materialized = 'table', + ) +}} + +-- metricflow_time_spine.sql +with days as ( + {{ dbt_utils.date_spine('day' + , "to_date('01/01/2000','mm/dd/yyyy')" + , "to_date('01/01/2027','mm/dd/yyyy')" + ) + }} +), + +final as ( + select cast(date_day as date) as date_day + from days +) + +select * +from final \ No newline at end of file diff --git a/integration_tests/seeds/dbt_project_evaluator_exceptions.csv b/integration_tests/seeds/dbt_project_evaluator_exceptions.csv index d57e2718..6c5d791b 100644 --- a/integration_tests/seeds/dbt_project_evaluator_exceptions.csv +++ b/integration_tests/seeds/dbt_project_evaluator_exceptions.csv @@ -1,2 +1,4 @@ fct_name,column_name,id_to_exclude,comment -fct_direct_join_to_source,parent,stg_model_1,This is actually OK because... \ No newline at end of file +fct_direct_join_to_source,parent,stg_model_1,This is actually OK because... +fct_model_naming_conventions,resource_name,metricflow_time_spine,naming convention required by metricflow +fct_root_models,child,metricflow_time_spine,macro time spine \ No newline at end of file diff --git a/integration_tests/seeds/docs/test_fct_documentation_coverage.csv b/integration_tests/seeds/docs/test_fct_documentation_coverage.csv index 9540fae7..14303f45 100644 --- a/integration_tests/seeds/docs/test_fct_documentation_coverage.csv +++ b/integration_tests/seeds/docs/test_fct_documentation_coverage.csv @@ -1,2 +1,2 @@ total_models,documented_models,documentation_coverage_pct,staging_documentation_coverage_pct,intermediate_documentation_coverage_pct,marts_documentation_coverage_pct,other_documentation_coverage_pct -14,3,21.43,20.00,0.00,0.00,66.67 +15,4,26.67,20.00,0.00,0.00,75.00 diff --git a/integration_tests/seeds/tests/test_fct_test_coverage.csv b/integration_tests/seeds/tests/test_fct_test_coverage.csv index b28d53ad..5f10fc8b 100644 --- a/integration_tests/seeds/tests/test_fct_test_coverage.csv +++ b/integration_tests/seeds/tests/test_fct_test_coverage.csv @@ -1,2 +1,2 @@ total_models,total_tests,tested_models,test_coverage_pct,staging_test_coverage_pct,intermediate_test_coverage_pct,marts_test_coverage_pct,other_test_coverage_pct,test_to_model_ratio -14,11,5,35.71,80.00,50.00,0.00,0.00,0.7857 +15,13,6,40.00,80.00,50.00,0.00,25.00,0.8667 diff --git a/seeds/dbt_project_evaluator_exceptions.csv b/seeds/dbt_project_evaluator_exceptions.csv index 0453ef18..8e768eeb 100644 --- a/seeds/dbt_project_evaluator_exceptions.csv +++ b/seeds/dbt_project_evaluator_exceptions.csv @@ -1 +1,3 @@ -fct_name,column_name,id_to_exclude,comment \ No newline at end of file +fct_name,column_name,id_to_exclude,comment +fct_model_naming_conventions,resource_name,metricflow_time_spine,naming convention required by metricflow +fct_root_models,child,metricflow_time_spine,macro time spine \ No newline at end of file From 213fa9ac6ce7489a86425e0adc91fe1688c246f7 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 20 Jul 2023 10:14:55 -0500 Subject: [PATCH 30/37] add docs --- docs/customization/exceptions.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/customization/exceptions.md b/docs/customization/exceptions.md index 5b369bfb..a4cf6374 100644 --- a/docs/customization/exceptions.md +++ b/docs/customization/exceptions.md @@ -39,6 +39,10 @@ seeds: +enabled: false ``` +> The default `dbt_project_evaluator_exceptions.csv` file includes two exceptions for the [`metricflow_time_spine` model required](https://docs.getdbt.com/docs/build/metricflow-time-spine) for dbt metrics users on dbt 1.6+. +> Specifically, it will exclude this datespine model from the root model and naming convention checks. +> When overriding this file, consider keeping these checks in your seed if you are using the dbt semantic layer. + ## 3. Run the seed and the package We then run both the seed and the package by executing the following command: From 97fb3a4b5b39972ec92ba7b113d0275bfc9339a5 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 20 Jul 2023 11:05:05 -0500 Subject: [PATCH 31/37] remove seed exclusions, add hard coded exclusions --- .../seeds/dbt_project_evaluator_exceptions.csv | 4 +--- models/marts/.DS_Store | Bin 0 -> 6148 bytes models/marts/dag/fct_root_models.sql | 2 ++ .../structure/fct_model_naming_conventions.sql | 1 + seeds/dbt_project_evaluator_exceptions.csv | 4 +--- 5 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 models/marts/.DS_Store diff --git a/integration_tests/seeds/dbt_project_evaluator_exceptions.csv b/integration_tests/seeds/dbt_project_evaluator_exceptions.csv index 6c5d791b..d57e2718 100644 --- a/integration_tests/seeds/dbt_project_evaluator_exceptions.csv +++ b/integration_tests/seeds/dbt_project_evaluator_exceptions.csv @@ -1,4 +1,2 @@ fct_name,column_name,id_to_exclude,comment -fct_direct_join_to_source,parent,stg_model_1,This is actually OK because... -fct_model_naming_conventions,resource_name,metricflow_time_spine,naming convention required by metricflow -fct_root_models,child,metricflow_time_spine,macro time spine \ No newline at end of file +fct_direct_join_to_source,parent,stg_model_1,This is actually OK because... \ No newline at end of file diff --git a/models/marts/.DS_Store b/models/marts/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e02ca4979ca761c048867b9ae06685bca4c54469 GIT binary patch literal 6148 zcmeHKJE{Uf5bS;$_%JavoGW;PLG%P(Ku|Feh4m9}^v?B;X7vYARujQQs-e4Pdb(!W z3bwaJM5pKVL}Vl)1Gu65S?HRbn|JIdGYW*`j+LC`ERAd~i%#|PgmHi62wnr`9lzP{ z4*PA~{G7@x6`%rCfC^9nD)5E^)_Y-tVIU(FpaN9jrht7P3f!Bu`B$e#hzg+>KFt-uw9+ZE0L literal 0 HcmV?d00001 diff --git a/models/marts/dag/fct_root_models.sql b/models/marts/dag/fct_root_models.sql index d4c86416..d86d2005 100644 --- a/models/marts/dag/fct_root_models.sql +++ b/models/marts/dag/fct_root_models.sql @@ -8,6 +8,8 @@ with model_relationships as ( -- only filter out excluded children nodes -- filtering parents could result in incorrectly flagging nodes that depend on excluded nodes and not child_is_excluded + -- exclude required time spine + and child != 'metricflow_time_spine' ), final as ( diff --git a/models/marts/structure/fct_model_naming_conventions.sql b/models/marts/structure/fct_model_naming_conventions.sql index 401a59dc..c24ef040 100644 --- a/models/marts/structure/fct_model_naming_conventions.sql +++ b/models/marts/structure/fct_model_naming_conventions.sql @@ -4,6 +4,7 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} where not is_excluded + and resource_name != 'metricflow_time_spine' ), naming_convention_prefixes as ( diff --git a/seeds/dbt_project_evaluator_exceptions.csv b/seeds/dbt_project_evaluator_exceptions.csv index 8e768eeb..0453ef18 100644 --- a/seeds/dbt_project_evaluator_exceptions.csv +++ b/seeds/dbt_project_evaluator_exceptions.csv @@ -1,3 +1 @@ -fct_name,column_name,id_to_exclude,comment -fct_model_naming_conventions,resource_name,metricflow_time_spine,naming convention required by metricflow -fct_root_models,child,metricflow_time_spine,macro time spine \ No newline at end of file +fct_name,column_name,id_to_exclude,comment \ No newline at end of file From 0cbd5948fe73b796bd672bc4fc81e1baf273e5c5 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 20 Jul 2023 11:05:56 -0500 Subject: [PATCH 32/37] remove docs callout --- docs/customization/exceptions.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/customization/exceptions.md b/docs/customization/exceptions.md index a4cf6374..5b369bfb 100644 --- a/docs/customization/exceptions.md +++ b/docs/customization/exceptions.md @@ -39,10 +39,6 @@ seeds: +enabled: false ``` -> The default `dbt_project_evaluator_exceptions.csv` file includes two exceptions for the [`metricflow_time_spine` model required](https://docs.getdbt.com/docs/build/metricflow-time-spine) for dbt metrics users on dbt 1.6+. -> Specifically, it will exclude this datespine model from the root model and naming convention checks. -> When overriding this file, consider keeping these checks in your seed if you are using the dbt semantic layer. - ## 3. Run the seed and the package We then run both the seed and the package by executing the following command: From 1c7f82906e90ba547a67cc8fb97d36d2a454c652 Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 20 Jul 2023 11:11:36 -0500 Subject: [PATCH 33/37] make timespine warehouse agnostic --- .../models/utils/metricflow_time_spine.sql | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/integration_tests/models/utils/metricflow_time_spine.sql b/integration_tests/models/utils/metricflow_time_spine.sql index aa25fd1b..acaea2f9 100644 --- a/integration_tests/models/utils/metricflow_time_spine.sql +++ b/integration_tests/models/utils/metricflow_time_spine.sql @@ -1,16 +1,20 @@ +-- metricflow_time_spine.sql {{ - config( - materialized = 'table', + config( + materialized = 'table', ) }} --- metricflow_time_spine.sql with days as ( - {{ dbt_utils.date_spine('day' - , "to_date('01/01/2000','mm/dd/yyyy')" - , "to_date('01/01/2027','mm/dd/yyyy')" - ) + + {{ + dbt_utils.date_spine( + 'day', + dbt.safe_cast(dbt.current_timestamp(), "date"), + dbt.dateadd('day', 1, dbt.safe_cast(dbt.current_timestamp(), "date")), + ) }} + ), final as ( @@ -18,5 +22,4 @@ final as ( from days ) -select * -from final \ No newline at end of file +select * from final \ No newline at end of file From 4df5c030f4f603ab9e650c72a20cd8cb4eb4684a Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 20 Jul 2023 11:19:47 -0500 Subject: [PATCH 34/37] duckdb compat --- integration_tests/models/utils/metricflow_time_spine.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration_tests/models/utils/metricflow_time_spine.sql b/integration_tests/models/utils/metricflow_time_spine.sql index acaea2f9..dfd9de4f 100644 --- a/integration_tests/models/utils/metricflow_time_spine.sql +++ b/integration_tests/models/utils/metricflow_time_spine.sql @@ -1,4 +1,5 @@ -- metricflow_time_spine.sql +{% set date_expr = "current_date()" if target.type in ["duckdb"] else dbt.current_timestamp() %} {{ config( materialized = 'table', @@ -10,8 +11,8 @@ with days as ( {{ dbt_utils.date_spine( 'day', - dbt.safe_cast(dbt.current_timestamp(), "date"), - dbt.dateadd('day', 1, dbt.safe_cast(dbt.current_timestamp(), "date")), + date_expr, + dbt.dateadd('day', 1, date_expr), ) }} From d814be76fba22958dbee8952a4317a46a00042cf Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Thu, 20 Jul 2023 11:31:22 -0500 Subject: [PATCH 35/37] BQ was mad about a column for some reason --- integration_tests/seeds/tests/tests_seeds.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/seeds/tests/tests_seeds.yml b/integration_tests/seeds/tests/tests_seeds.yml index 6defd088..9d3545a5 100644 --- a/integration_tests/seeds/tests/tests_seeds.yml +++ b/integration_tests/seeds/tests/tests_seeds.yml @@ -10,6 +10,7 @@ seeds: - name: test_fct_test_coverage config: column_types: + test_coverage_pct: float staging_test_coverage_pct: float intermediate_test_coverage_pct: float marts_test_coverage_pct: float From d6c881716af78adb7623af7425ad14902f07f0dc Mon Sep 17 00:00:00 2001 From: dave-connors-3 <73915542+dave-connors-3@users.noreply.github.com> Date: Thu, 20 Jul 2023 13:41:19 -0500 Subject: [PATCH 36/37] Apply suggestions from code review --- integration_tests/models/marts/metrics.yml | 4 ++-- models/marts/structure/fct_model_naming_conventions.sql | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/integration_tests/models/marts/metrics.yml b/integration_tests/models/marts/metrics.yml index d8b694e7..46908c62 100644 --- a/integration_tests/models/marts/metrics.yml +++ b/integration_tests/models/marts/metrics.yml @@ -2,8 +2,8 @@ version: 2 semantic_models: - - name: my_model # A semantic model with the name Transactions - model: ref('dim_model_7') # References the dbt model named `fact_transactions` + - name: my_model + model: ref('dim_model_7') description: "Transaction fact table at the transaction level. This table contains one row per transaction and includes the transaction timestamp." defaults: agg_time_dimension: date_day diff --git a/models/marts/structure/fct_model_naming_conventions.sql b/models/marts/structure/fct_model_naming_conventions.sql index c24ef040..d12b7c6a 100644 --- a/models/marts/structure/fct_model_naming_conventions.sql +++ b/models/marts/structure/fct_model_naming_conventions.sql @@ -4,6 +4,7 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} where not is_excluded + -- exclude required metricflow time spine and resource_name != 'metricflow_time_spine' ), From 053db4d9dcd7b0bec980a352f3f6aa9e22cf08ae Mon Sep 17 00:00:00 2001 From: Dave Connors Date: Wed, 26 Jul 2023 08:19:51 -0500 Subject: [PATCH 37/37] edit gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index aec6153c..7151f9f5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ dbt_modules/ dbt_packages/ .vscode integration_tests/state/ -site/ \ No newline at end of file +site/ +.DS_Store \ No newline at end of file