diff --git a/.github/workflows/mkdocs_main.yml b/.github/workflows/mkdocs_main.yml index f59086a0..4dc22079 100644 --- a/.github/workflows/mkdocs_main.yml +++ b/.github/workflows/mkdocs_main.yml @@ -10,12 +10,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 @@ -30,4 +30,4 @@ jobs: run: | git config --global user.name "GitHub Actions Bot" git config --global user.email "github-actions[bot]@users.noreply.github.com" - mike deploy --push --message "Deployed by GitHub Actions" main \ No newline at end of file + mike deploy --push --message "Deployed by GitHub Actions" main diff --git a/.github/workflows/mkdocs_release.yml b/.github/workflows/mkdocs_release.yml index 651fd3b9..cf44343f 100644 --- a/.github/workflows/mkdocs_release.yml +++ b/.github/workflows/mkdocs_release.yml @@ -10,12 +10,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 @@ -41,4 +41,4 @@ jobs: git config user.name "GitHub Actions" git config user.email "actions@github.com" mike deploy --push --message "Deploy docs for release ${{ github.event.release.tag_name }}" --update-alias $MAJOR_MINOR_VERSION latest - mike set-default --push latest \ No newline at end of file + mike set-default --push latest diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..75a14dd4 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,30 @@ +# **what?** +# For issues that have been open for awhile without activity, label +# them as stale with a warning that they will be closed out. If +# anyone comments to keep the issue open, it will automatically +# remove the stale label and keep it open. + +# Stale label rules: +# awaiting_response, more_information_needed -> 90 days +# good_first_issue, help_wanted -> 360 days (a year) +# tech_debt -> 720 (2 years) +# all else defaults -> 180 days (6 months) + +# **why?** +# To keep the repo in a clean state from issues that aren't relevant anymore + +# **when?** +# Once a day + +name: "Close stale issues and PRs" +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/triage-labels.yml b/.github/workflows/triage-labels.yml new file mode 100644 index 00000000..0f88157b --- /dev/null +++ b/.github/workflows/triage-labels.yml @@ -0,0 +1,31 @@ +# **what?** +# When we triage issues, we sometimes need more information from the issue creator. In +# those cases we remove the `triage` label and add the `awaiting_response` label. Once we +# receive a response in the form of a comment, we want the `awaiting_response` label removed +# in favor of the `triage` label so we are aware that the issue needs action. + +# **why?** +# To help with out team triage issue tracking + +# **when?** +# This will run when a comment is added to an issue and that issue has the `awaiting_response` label. + +name: Update Triage Label + +on: issue_comment + +defaults: + run: + shell: bash + +permissions: + issues: write + +jobs: + triage_label: + if: contains(github.event.issue.labels.*.name, 'awaiting_response') + uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main + with: + add_label: "triage" + remove_label: "awaiting_response" + secrets: inherit diff --git a/.gitignore b/.gitignore index 6dca4b7d..277adc9a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ dbt_packages/ .vscode integration_tests/state/ site/ -env/ \ No newline at end of file +env/ +.DS_Store diff --git a/dbt_project.yml b/dbt_project.yml index 353536c0..f06ff3e9 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ name: 'dbt_project_evaluator' version: '1.0.0' config-version: 2 -require-dbt-version: [">=1.5.0", "<2.0.0"] +require-dbt-version: [">=1.6.0-rc1", "<2.0.0"] model-paths: ["models"] analysis-paths: ["analysis"] diff --git a/docs/contributing.md b/docs/contributing.md index cf205dfe..6b0906a5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -24,6 +24,7 @@ Docs are then automatically pushed to the website as part of our CI/CD process. "markdownlint.config": { "ul-indent": {"indent": 4}, "MD036": false, + "MD046": false, } ``` diff --git a/docs/customization/customization.md b/docs/customization/customization.md index 3b6e381b..efa97d69 100644 --- a/docs/customization/customization.md +++ b/docs/customization/customization.md @@ -1,7 +1,14 @@ -# Disabling Models +# Disabling checks from the package -If there is a particular model or set of models that you *do not want this package to execute*, you can -disable these models as you would any other model in your `dbt_project.yml` file +!!! note + + This section is describing how to completely deactivate tests from the package. + If you are looking to deactivate models/sources from being tested, you can look at [excluding packages and paths](excluding-packages-and-paths.md) + +All the tests done as part of the package are tied to `fct` models. + +If there is a particular test or set of tests that you *do not want this package to execute*, you can +disable the corresponding `fct` models as you would any other model in your `dbt_project.yml` file ``` yaml title="dbt_project.yml" models: @@ -14,5 +21,4 @@ models: # disable single DAG model fct_model_fanout: +enabled: false - ``` diff --git a/docs/customization/excluding-packages-and-paths.md b/docs/customization/excluding-packages-and-paths.md new file mode 100644 index 00000000..a5e8f835 --- /dev/null +++ b/docs/customization/excluding-packages-and-paths.md @@ -0,0 +1,57 @@ +# Excluding packages or sources/models based on their path + +!!! note + + This section is describing how to entirely exclude models/sources and packages to be evaluated. + If you want to document exceptions to the rules, see the section [on exceptions](exceptions.md) + and if you want to deactivate entire tests you can follow instructions from [this page](customization.md) + +There might be cases where you want to exclude models/sources from being tested: + +- they could come from a package for which you have no control over +- you might be refactoring your project and wanting to exclude entire folders to follow best-practices in the new models + +In that case, this package provides the ability to exclude whole packages and/or models and sources based on their path + +## Configuration + +The variables `exclude_packages` and `exclude_paths_from_project` allow you to define a list of regex patterns to exclude from being reported as errors. + +- `exclude_packages` accepts a list of package names to exclude from the tool. To exclude all packages except the current project, you can set it to `["all"]` +- `exclude_paths_from_project` accepts a list of regular expressions of paths to exclude for the current project + - **for models**, the regex provided will try to match the pattern in the string ``, allowing to exclude packages, but also whole folders or individual models + - **for sources**, the regex will try to match the pattern in `:.` *(the pattern is different than for models because the path itself doesn't let us exclude individual sources)* + +!!! note + + We currently don't allow excluding metrics and exposures, as if those need to be entirely excluded they could be deactivated from the project. + + If you have a specific use case requiring this ability, please raise a GitHub issue to explain the situation you'd like to solve and we can revisit this decision ! + +### Example to exclude a whole package + +```yaml title="dbt_project.yml" +vars: + exclude_packages: ["upstream_package"] +``` + +### Example to exclude models/sources in a given path + +```yaml title="dbt_project.yml" +vars: + exclude_paths_from_project: ["/models/legacy/"] +``` + +### Example to exclude both a package and models/sources in 2 different paths + +```yaml title="dbt_project.yml" +vars: + exclude_packages: ["upstream_package"] + exclude_paths_from_project: ["/models/legacy/", "/my_date_spine.sql"] +``` + +## Tips and tricks + +Regular expressions are very powerful but can become complex. After defining your value for `exclude_paths_from_project`, we recommend running the package and inspecting the model `int_all_graph_resources`, checking if the value in the column `is_excluded` matches your expectation. + +A useful tool to debug regular expression is [regex101](https://regex101.com/). You can provide a pattern and a list of strings to see which ones actually match the pattern. diff --git a/docs/rules.md b/docs/rules.md index bc6594d4..1b757207 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -12,7 +12,7 @@ hide: |Modeling |[Rejoining of Upstream Concepts](../rules/modeling/#rejoining-of-upstream-concepts) |`fct_rejoining_of_upstream_concepts`| |Modeling |[Model Fanout](../rules/modeling/#model-fanout) |`fct_model_fanout`| |Modeling |[Downstream Models Dependent on Source](../rules/modeling/#downstream-models-dependent-on-source) |`fct_marts_or_intermediate_dependent_on_source`| -|Modeling |[Direct Join to Source](/..rules/modeling/#direct-join-to-source) |`fct_direct_join_to_source`| +|Modeling |[Direct Join to Source](../rules/modeling/#direct-join-to-source) |`fct_direct_join_to_source`| |Modeling |[Duplicate Sources](../rules/modeling/#duplicate-sources) |`fct_duplicate_sources`| |Modeling |[Hard Coded References](../rules/modeling/#hard-coded-references) |`fct_hard_coded_references`| |Modeling |[Multiple Sources Joined](../rules/modeling/#multiple-sources-joined) |`fct_multiple_sources_joined`| diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 1d5b9c29..ca3934d5 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -59,6 +59,8 @@ seeds: vars: # ensure integration tests run successfully when there are 0 of a given model type (extra) + exclude_packages: [] + exclude_paths_from_project: ["/to_exclude/","source_3.table_6"] model_types: ['staging', 'intermediate', 'marts', 'other', 'extra', 'new_model_type'] # dummy variable used for testing fct_hard_coded_references my_table_reference: 'grace_table' diff --git a/integration_tests/models/marts/exposures.yml b/integration_tests/models/marts/exposures.yml index a38e7136..1382780d 100644 --- a/integration_tests/models/marts/exposures.yml +++ b/integration_tests/models/marts/exposures.yml @@ -13,6 +13,7 @@ exposures: depends_on: - ref('fct_model_6') + - ref('fct_model_9') - ref('dim_model_7') - ref('dim_model_7') - source('source_1', 'table_1') \ No newline at end of file diff --git a/integration_tests/models/marts/int_model_5.sql b/integration_tests/models/marts/int_model_5.sql index 6d5e4c3e..c2530621 100644 --- a/integration_tests/models/marts/int_model_5.sql +++ b/integration_tests/models/marts/int_model_5.sql @@ -1,2 +1,3 @@ -{{ ref('int_model_4') }} -{{ ref('stg_model_1') }} \ No newline at end of file +-- {{ ref('int_model_4') }} +-- {{ ref('stg_model_1') }} +select 1 as id \ No newline at end of file diff --git a/integration_tests/models/marts/intermediate/dim_model_7.sql b/integration_tests/models/marts/intermediate/dim_model_7.sql index c42daeb9..3ddf87e8 100644 --- a/integration_tests/models/marts/intermediate/dim_model_7.sql +++ b/integration_tests/models/marts/intermediate/dim_model_7.sql @@ -1,2 +1,9 @@ -{{ ref('stg_model_4') }} -{{ ref('int_model_5') }} \ No newline at end of file + +{{ + config( + materialized = 'table', + ) +}} + +select * from {{ ref('stg_model_4') }} +-- {{ ref('int_model_5') }} \ No newline at end of file diff --git a/integration_tests/models/marts/intermediate/to_exclude/int_model_10.sql b/integration_tests/models/marts/intermediate/to_exclude/int_model_10.sql new file mode 100644 index 00000000..7a0669e0 --- /dev/null +++ b/integration_tests/models/marts/intermediate/to_exclude/int_model_10.sql @@ -0,0 +1,3 @@ +-- depends on: {{ ref('fct_model_9') }} + +select 1 as id \ No newline at end of file diff --git a/integration_tests/models/marts/intermediate/to_exclude/int_model_11.sql b/integration_tests/models/marts/intermediate/to_exclude/int_model_11.sql new file mode 100644 index 00000000..7a0669e0 --- /dev/null +++ b/integration_tests/models/marts/intermediate/to_exclude/int_model_11.sql @@ -0,0 +1,3 @@ +-- depends on: {{ ref('fct_model_9') }} + +select 1 as id \ No newline at end of file diff --git a/integration_tests/models/marts/metrics.yml b/integration_tests/models/marts/metrics.yml index 44d21460..46908c62 100644 --- a/integration_tests/models/marts/metrics.yml +++ b/integration_tests/models/marts/metrics.yml @@ -1,29 +1,37 @@ version: 2 -metrics: - - name: new_customers - label: New Customers marked 'paying' - model: ref('dim_model_7') - description: "The number of paid customers using the product" - calculation_method: count - expression: user_id # superfluous here, but shown as an example +semantic_models: + - name: my_model + model: ref('dim_model_7') + description: "Transaction fact table at the transaction level. This table contains one row per transaction and includes the transaction timestamp." + defaults: + agg_time_dimension: date_day + + entities: # Entities included in the table are defined here. MetricFlow will use these columns as join keys. + - name: dim + type: primary + expr: id - timestamp: signup_date - time_grains: [day, week, month] + dimensions: # dimensions are qualitative values such as names, dates, or geographical data. They provide context to metrics and allow "metric by group" data slicing. + - name: id + type: categorical - dimensions: - - plan - - country + - name: date_day + type: time + type_params: + time_granularity: day - filters: - - field: is_paying - operator: '=' - value: 'true' - - field: company_name - operator: '!=' - value: "Acme', Inc" + measures: # Measures are columns we perform an aggregation over. Measures are inputs to metrics. + - name: total_count + description: "The total count." + agg: count + expr: 1 - meta: - team: "Finance" - refresh_rate: "Bob's weekly run" +metrics: + - name: new_metric + label: "New Metric" + type: simple + type_params: + # Specify the measure you are creating a proxy for. + measure: total_count \ No newline at end of file diff --git a/integration_tests/models/reports/report_1.sql b/integration_tests/models/reports/report_1.sql index cdf13c62..83180302 100644 --- a/integration_tests/models/reports/report_1.sql +++ b/integration_tests/models/reports/report_1.sql @@ -1 +1,3 @@ -{{ ref('fct_model_6') }} \ No newline at end of file +-- {{ ref('fct_model_6') }} + +select true \ No newline at end of file diff --git a/integration_tests/models/reports/reports.yml b/integration_tests/models/reports/reports.yml index 4b47b272..a86de310 100644 --- a/integration_tests/models/reports/reports.yml +++ b/integration_tests/models/reports/reports.yml @@ -2,8 +2,10 @@ version: 2 models: - name: report_1 - description: tom cruise's minorty report + description: tom cruise's minority report access: public + config: + materialized: table - name: report_2 access: public config: diff --git a/integration_tests/models/staging/source_1/source.yml b/integration_tests/models/staging/source_1/source.yml index bd2fccc2..289c264c 100644 --- a/integration_tests/models/staging/source_1/source.yml +++ b/integration_tests/models/staging/source_1/source.yml @@ -17,4 +17,9 @@ sources: schema: real_schema_2 # database: real_database tables: - - name: table_3 \ No newline at end of file + - name: table_3 + + - name: source_3 + schema: real_schema_3 + tables: + - name: table_6 \ No newline at end of file diff --git a/integration_tests/models/utils/_utils_models.yml b/integration_tests/models/utils/_utils_models.yml new file mode 100644 index 00000000..5f897e82 --- /dev/null +++ b/integration_tests/models/utils/_utils_models.yml @@ -0,0 +1,8 @@ +models: + - name: metricflow_time_spine + description: default time spine for metricflow + columns: + - name: date_day + tests: + - unique + - not_null \ No newline at end of file diff --git a/integration_tests/models/utils/metricflow_time_spine.sql b/integration_tests/models/utils/metricflow_time_spine.sql new file mode 100644 index 00000000..dfd9de4f --- /dev/null +++ b/integration_tests/models/utils/metricflow_time_spine.sql @@ -0,0 +1,26 @@ +-- metricflow_time_spine.sql +{% set date_expr = "current_date()" if target.type in ["duckdb"] else dbt.current_timestamp() %} +{{ + config( + materialized = 'table', + ) +}} + +with days as ( + + {{ + dbt_utils.date_spine( + 'day', + date_expr, + dbt.dateadd('day', 1, date_expr), + ) + }} + +), + +final as ( + select cast(date_day as date) as date_day + from days +) + +select * from final \ No newline at end of file diff --git a/integration_tests/seeds/dbt_project_evaluator_exceptions.csv b/integration_tests/seeds/dbt_project_evaluator_exceptions.csv index 33d8958c..d57e2718 100644 --- a/integration_tests/seeds/dbt_project_evaluator_exceptions.csv +++ b/integration_tests/seeds/dbt_project_evaluator_exceptions.csv @@ -1,2 +1,2 @@ fct_name,column_name,id_to_exclude,comment -fct_direct_join_to_source,parent_id,model.dbt_project_evaluator_integration_tests.stg_model_1,This is actually OK because... \ No newline at end of file +fct_direct_join_to_source,parent,stg_model_1,This is actually OK because... \ No newline at end of file diff --git a/integration_tests/seeds/docs/test_fct_documentation_coverage.csv b/integration_tests/seeds/docs/test_fct_documentation_coverage.csv index 9540fae7..14303f45 100644 --- a/integration_tests/seeds/docs/test_fct_documentation_coverage.csv +++ b/integration_tests/seeds/docs/test_fct_documentation_coverage.csv @@ -1,2 +1,2 @@ total_models,documented_models,documentation_coverage_pct,staging_documentation_coverage_pct,intermediate_documentation_coverage_pct,marts_documentation_coverage_pct,other_documentation_coverage_pct -14,3,21.43,20.00,0.00,0.00,66.67 +15,4,26.67,20.00,0.00,0.00,75.00 diff --git a/integration_tests/seeds/governance/test_fct_exposures_dependent_on_private_models.csv b/integration_tests/seeds/governance/test_fct_exposures_dependent_on_private_models.csv index 33c44893..289d67ad 100644 --- a/integration_tests/seeds/governance/test_fct_exposures_dependent_on_private_models.csv +++ b/integration_tests/seeds/governance/test_fct_exposures_dependent_on_private_models.csv @@ -1,4 +1,5 @@ exposure_name,parent_resource_name,parent_access,parent_resource_type exposure_1,dim_model_7,protected,model exposure_1,fct_model_6,protected,model +exposure_1,fct_model_9,protected,model exposure_1,source_1.table_1,,source diff --git a/integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv b/integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv index ffe826ec..c74b99c6 100644 --- a/integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv +++ b/integration_tests/seeds/performance/test_fct_exposure_parents_materializations.csv @@ -1,3 +1,3 @@ parent_resource_type,parent_resource_name,exposure_name,parent_model_materialization -model,dim_model_7,exposure_1,ephemeral +model,fct_model_9,exposure_1,ephemeral source,source_1.table_1,exposure_1, \ No newline at end of file diff --git a/integration_tests/seeds/tests/test_fct_test_coverage.csv b/integration_tests/seeds/tests/test_fct_test_coverage.csv index b28d53ad..5f10fc8b 100644 --- a/integration_tests/seeds/tests/test_fct_test_coverage.csv +++ b/integration_tests/seeds/tests/test_fct_test_coverage.csv @@ -1,2 +1,2 @@ total_models,total_tests,tested_models,test_coverage_pct,staging_test_coverage_pct,intermediate_test_coverage_pct,marts_test_coverage_pct,other_test_coverage_pct,test_to_model_ratio -14,11,5,35.71,80.00,50.00,0.00,0.00,0.7857 +15,13,6,40.00,80.00,50.00,0.00,25.00,0.8667 diff --git a/integration_tests/seeds/tests/tests_seeds.yml b/integration_tests/seeds/tests/tests_seeds.yml index 6defd088..9d3545a5 100644 --- a/integration_tests/seeds/tests/tests_seeds.yml +++ b/integration_tests/seeds/tests/tests_seeds.yml @@ -10,6 +10,7 @@ seeds: - name: test_fct_test_coverage config: column_types: + test_coverage_pct: float staging_test_coverage_pct: float intermediate_test_coverage_pct: float marts_test_coverage_pct: float diff --git a/macros/filter_exceptions.sql b/macros/filter_exceptions.sql index 304d5e84..9c86109a 100644 --- a/macros/filter_exceptions.sql +++ b/macros/filter_exceptions.sql @@ -1,15 +1,15 @@ -{% macro filter_exceptions(model_ref) -%} - {{ return(adapter.dispatch('filter_exceptions', 'dbt_project_evaluator')(model_ref)) }} +{% macro filter_exceptions(model_name) -%} + {{ return(adapter.dispatch('filter_exceptions', 'dbt_project_evaluator')(model_name)) }} {%- endmacro %} -{% macro default__filter_exceptions(model_ref) %} +{% macro default__filter_exceptions(model_name) %} {% set query_filters %} select column_name, id_to_exclude from {{ ref('dbt_project_evaluator_exceptions') }} - where fct_name = '{{ model_ref.name }}' + where fct_name = '{{ model_name }}' {% endset %} {% if execute %} diff --git a/macros/find_all_hard_coded_references.sql b/macros/find_all_hard_coded_references.sql index 731ff25c..d908d806 100644 --- a/macros/find_all_hard_coded_references.sql +++ b/macros/find_all_hard_coded_references.sql @@ -6,7 +6,7 @@ {%- set all_hard_coded_references_list = [] -%} - {% if node.resource_type == 'model' %} + {% if node.resource_type == 'model' and node.language == "sql" %} {% if execute %} {%- set model_raw_sql = node.raw_sql or node.raw_code -%} diff --git a/macros/on-run-end/print_dbt_project_evaluator_issues.sql b/macros/on-run-end/print_dbt_project_evaluator_issues.sql index 9079d0e2..802c926c 100644 --- a/macros/on-run-end/print_dbt_project_evaluator_issues.sql +++ b/macros/on-run-end/print_dbt_project_evaluator_issues.sql @@ -10,9 +10,9 @@ {{ print("\n-- " ~ result.node.alias ~ " --") }} {% set unique_id_model_checked = result.node.depends_on.nodes[0] %} - {% set name_model_checked = unique_id_model_checked.split('.')[-1] %} {% set model_details = graph["nodes"][unique_id_model_checked] %} + {% set name_model_checked = model_details.alias %} {% set model_schema = model_details.schema %} {% set model_database = model_details.database %} {% set db_schema = model_database ~ "." ~ model_schema if model_database else model_schema %} diff --git a/macros/recursive_dag.sql b/macros/recursive_dag.sql index 150f4451..7b58578c 100644 --- a/macros/recursive_dag.sql +++ b/macros/recursive_dag.sql @@ -27,6 +27,7 @@ all_relationships ( parent_file_path, parent_directory_path, parent_file_name, + parent_is_excluded, child_id, child, child_resource_type, @@ -38,6 +39,7 @@ all_relationships ( child_file_path, child_directory_path, child_file_name, + child_is_excluded, distance, path, is_dependent_on_chain_of_views @@ -55,6 +57,7 @@ all_relationships ( file_path as parent_file_path, directory_path as parent_directory_path, file_name as parent_file_name, + is_excluded as parent_is_excluded, resource_id as child_id, resource_name as child, resource_type as child_resource_type, @@ -66,6 +69,7 @@ all_relationships ( file_path as child_file_path, directory_path as child_directory_path, file_name as child_file_name, + is_excluded as child_is_excluded, 0 as distance, {{ dbt.array_construct(['resource_name']) }} as path, cast(null as boolean) as is_dependent_on_chain_of_views @@ -88,6 +92,7 @@ all_relationships ( all_relationships.parent_file_path as parent_file_path, all_relationships.parent_directory_path as parent_directory_path, all_relationships.parent_file_name as parent_file_name, + all_relationships.parent_is_excluded as parent_is_excluded, direct_relationships.resource_id as child_id, direct_relationships.resource_name as child, direct_relationships.resource_type as child_resource_type, @@ -99,6 +104,7 @@ all_relationships ( direct_relationships.file_path as child_file_path, direct_relationships.directory_path as child_directory_path, direct_relationships.file_name as child_file_name, + direct_relationships.is_excluded as child_is_excluded, all_relationships.distance+1 as distance, {{ dbt.array_append('all_relationships.path', 'direct_relationships.resource_name') }} as path, case @@ -155,6 +161,7 @@ with direct_relationships as ( is_public as child_is_public, access as child_access + is_excluded as child_is_excluded from direct_relationships ) @@ -165,6 +172,7 @@ with direct_relationships as ( child_materialized, child_is_public, child_access, + child_is_excluded, 0 as distance, {{ dbt.array_construct(['resource_name']) }} as path, cast(null as boolean) as is_dependent_on_chain_of_views @@ -180,6 +188,7 @@ with direct_relationships as ( direct_relationships.materialized as child_materialized, direct_relationships.is_public as child_is_public, direct_relationships.access as child_access, + direct_relationships.is_excluded as child_is_excluded, cte_{{i - 1}}.distance+1 as distance, {{ dbt.array_append(prev_cte_path, 'direct_relationships.resource_name') }} as path, case @@ -221,6 +230,7 @@ with direct_relationships as ( parent.file_path as parent_file_path, parent.directory_path as parent_directory_path, parent.file_name as parent_file_name, + parent.is_excluded as parent_is_excluded, child.resource_id as child_id, child.resource_name as child, child.resource_type as child_resource_type, @@ -232,6 +242,7 @@ with direct_relationships as ( child.file_path as child_file_path, child.directory_path as child_directory_path, child.file_name as child_file_name, + child.is_excluded as child_is_excluded, all_relationships_unioned.distance, all_relationships_unioned.path, all_relationships_unioned.is_dependent_on_chain_of_views diff --git a/macros/set_is_excluded.sql b/macros/set_is_excluded.sql new file mode 100644 index 00000000..f7067da1 --- /dev/null +++ b/macros/set_is_excluded.sql @@ -0,0 +1,39 @@ +{% macro set_is_excluded(resource, resource_type) %} + {{ return(adapter.dispatch('set_is_excluded', 'dbt_project_evaluator')(resource, resource_type)) }} +{% endmacro %} + +{% macro default__set_is_excluded(resource, resource_type) %} + + {% set re = modules.re %} + {%- set ns = namespace(exclude=false) -%} + + {% if resource_type == 'node' %} + {%- set resource_path = resource.original_file_path | replace("\\","\\\\") -%} + {% elif resource_type == 'source' %} + {%- set resource_path = resource.original_file_path | replace("\\","\\\\") ~ ":" ~ resource.fqn[-2] ~ "." ~ resource.fqn[-1] -%} + {% else %} + {{ exceptions.raise_compiler_error( + "`set_is_excluded()` macro does not support resource type: " ~ resource_type + ) }} + {% endif %} + + + {#- we exclude the resource if it is from the current project and matches the pattern -#} + {%- for exclude_paths_pattern in var('exclude_paths_from_project',[]) -%} + {%- set matched_path = re.search(exclude_paths_pattern, resource_path, re.IGNORECASE) -%} + {%- if matched_path and resource.package_name == project_name %} + {% set ns.exclude = true %} + {%- endif -%} + {%- endfor -%} + + {#- we exclude the resource if the package if it is listed in `exclude_packages` or if it is "all" -#} + {%- if ( + resource.package_name != project_name) + and (resource.package_name in var('exclude_packages',[]) or 'all' in var('exclude_packages',[])) + -%} + {% set ns.exclude = true %} + {%- endif -%} + + {{ return(ns.exclude) }} + +{% endmacro %} diff --git a/macros/unpack/get_metric_values.sql b/macros/unpack/get_metric_values.sql index 80ac8e8f..f10e9ef9 100644 --- a/macros/unpack/get_metric_values.sql +++ b/macros/unpack/get_metric_values.sql @@ -9,17 +9,6 @@ {%- set values = [] -%} {%- for node in nodes_list -%} - - {% set metric_filters %} - {%- if node.filters|length -%} - {%- for filt in node.filters %} - '{{ filt.field }}'||'{{ filt.operator }}'||'''{{ dbt.escape_single_quotes(filt.value) }}''' - {% if not loop.last %}|| ' - ' ||{% endif %} - {% endfor -%} - {%- else -%} - '' - {% endif -%} - {% endset %} {%- set values_line = [ @@ -29,13 +18,16 @@ wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")), "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as boolean)", wrap_string_with_quotes(node.type), - wrap_string_with_quotes(node.model.identifier), wrap_string_with_quotes(dbt.escape_single_quotes(node.label)), - wrap_string_with_quotes(node.sql), - wrap_string_with_quotes(node.timestamp), wrap_string_with_quotes(node.package_name), - wrap_string_with_quotes(node.dimensions|join(' - ')), - metric_filters, + wrap_string_with_quotes(node.filter), + wrap_string_with_quotes(node.type_params.measure.name), + wrap_string_with_quotes(node.type_params.measure.alias), + wrap_string_with_quotes(node.type_params.numerator), + wrap_string_with_quotes(node.type_params.denominator), + wrap_string_with_quotes(node.type_params.expr), + wrap_string_with_quotes(node.type_params.window), + wrap_string_with_quotes(node.type_params.grain_to_date), wrap_string_with_quotes(node.meta | tojson) ] %} diff --git a/macros/unpack/get_node_values.sql b/macros/unpack/get_node_values.sql index d37f5b66..7c9db32c 100644 --- a/macros/unpack/get_node_values.sql +++ b/macros/unpack/get_node_values.sql @@ -12,6 +12,8 @@ {%- set hard_coded_references = dbt_project_evaluator.find_all_hard_coded_references(node) -%} {%- set contract = node.contract.enforced if node.contract else false -%} + {%- set exclude_node = dbt_project_evaluator.set_is_excluded(node, resource_type="node") -%} + {%- set values_line = [ @@ -37,7 +39,8 @@ wrap_string_with_quotes(node.meta | tojson), wrap_string_with_quotes(dbt.escape_single_quotes(hard_coded_references)), wrap_string_with_quotes(node.get('depends_on',{}).get('macros',[]) | tojson), - "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.test_metadata) | trim ~ " as boolean)" + "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.test_metadata) | trim ~ " as boolean)", + "cast(" ~ exclude_node ~ " as boolean)", ] %} diff --git a/macros/unpack/get_source_values.sql b/macros/unpack/get_source_values.sql index 7a30ac63..b8d3abad 100644 --- a/macros/unpack/get_source_values.sql +++ b/macros/unpack/get_source_values.sql @@ -10,6 +10,8 @@ {%- for node in nodes_list -%} + {%- set exclude_source = dbt_project_evaluator.set_is_excluded(node, resource_type="source") -%} + {%- set values_line = [ wrap_string_with_quotes(node.unique_id), @@ -27,7 +29,8 @@ wrap_string_with_quotes(node.package_name), wrap_string_with_quotes(node.loader), wrap_string_with_quotes(node.identifier), - wrap_string_with_quotes(node.meta | tojson) + wrap_string_with_quotes(node.meta | tojson), + "cast(" ~ exclude_source ~ " as boolean)", ] %} diff --git a/mkdocs.yml b/mkdocs.yml index 6f927b0e..ed205fa5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,8 @@ theme: # Palette toggle for light mode - media: "(prefers-color-scheme: light)" scheme: default + primary: custom + accent: custom toggle: icon: material/brightness-7 name: Switch to dark mode @@ -16,6 +18,8 @@ theme: # Palette toggle for dark mode - media: "(prefers-color-scheme: dark)" scheme: slate + primary: custom + accent: custom toggle: icon: material/brightness-4 name: Switch to light mode @@ -72,9 +76,10 @@ nav: - Performance: rules/performance.md - Governance: rules/governance.md - Customization: - - Disabling models: customization/customization.md - Overriding variables: customization/overriding-variables.md + - Disabling checks: customization/customization.md - Configuring exceptions to the rules: customization/exceptions.md + - Excluding packages and models/sources based on path: customization/excluding-packages-and-paths.md - Display issues in the logs: customization/issues-in-log.md - Run in CI Check: ci-check.md - Querying the DAG: querying-the-dag.md diff --git a/models/marts/.DS_Store b/models/marts/.DS_Store new file mode 100644 index 00000000..e02ca497 Binary files /dev/null and b/models/marts/.DS_Store differ diff --git a/models/marts/core/int_all_graph_resources.sql b/models/marts/core/int_all_graph_resources.sql index 14b4c964..58b71953 100644 --- a/models/marts/core/int_all_graph_resources.sql +++ b/models/marts/core/int_all_graph_resources.sql @@ -94,16 +94,22 @@ joined as ( unioned_with_calc.meta, unioned_with_calc.macro_dependencies, unioned_with_calc.metric_type, - unioned_with_calc.model, unioned_with_calc.label, - unioned_with_calc.sql, - unioned_with_calc.timestamp as timestamp, + unioned_with_calc.metric_filter, + unioned_with_calc.metric_measure, + unioned_with_calc.metric_measure_alias, + unioned_with_calc.numerator, + unioned_with_calc.denominator, + unioned_with_calc.expr, + unioned_with_calc.metric_window, + unioned_with_calc.grain_to_date, unioned_with_calc.source_name, -- NULL for non-source resources unioned_with_calc.is_source_described, unioned_with_calc.loaded_at_field, unioned_with_calc.loader, unioned_with_calc.identifier, - unioned_with_calc.hard_coded_references -- NULL for non-model resources + unioned_with_calc.hard_coded_references, -- NULL for non-model resources + unioned_with_calc.is_excluded -- NULL for metrics and exposures from unioned_with_calc left join naming_convention_prefixes diff --git a/models/marts/core/int_direct_relationships.sql b/models/marts/core/int_direct_relationships.sql index df464260..f138ae5b 100644 --- a/models/marts/core/int_direct_relationships.sql +++ b/models/marts/core/int_direct_relationships.sql @@ -13,7 +13,8 @@ all_graph_resources as ( materialized, is_public, access, - source_name + source_name, + is_excluded from {{ ref('int_all_graph_resources') }} ), diff --git a/models/marts/dag/fct_direct_join_to_source.sql b/models/marts/dag/fct_direct_join_to_source.sql index ebb32cbe..da51cbeb 100644 --- a/models/marts/dag/fct_direct_join_to_source.sql +++ b/models/marts/dag/fct_direct_join_to_source.sql @@ -6,6 +6,8 @@ with direct_model_relationships as ( from {{ ref('int_all_dag_relationships') }} where child_resource_type = 'model' and distance = 1 + and not parent_is_excluded + and not child_is_excluded ), model_and_source_joined as ( @@ -25,7 +27,11 @@ model_and_source_joined as ( final as ( select - direct_model_relationships.* + direct_model_relationships.parent, + direct_model_relationships.parent_resource_type, + direct_model_relationships.child, + direct_model_relationships.child_resource_type, + direct_model_relationships.distance from direct_model_relationships inner join model_and_source_joined on direct_model_relationships.child = model_and_source_joined.child @@ -35,4 +41,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_duplicate_sources.sql b/models/marts/dag/fct_duplicate_sources.sql index 820a59eb..b0fd5d54 100644 --- a/models/marts/dag/fct_duplicate_sources.sql +++ b/models/marts/dag/fct_duplicate_sources.sql @@ -8,6 +8,7 @@ with sources as ( end as source_db_location from {{ ref('int_all_graph_resources') }} where resource_type = 'source' + and not is_excluded -- we order the CTE so that listagg returns values correctly sorted for some warehouses order by 1, 2 ), diff --git a/models/marts/dag/fct_hard_coded_references.sql b/models/marts/dag/fct_hard_coded_references.sql index a7769453..e3e6ccc9 100644 --- a/models/marts/dag/fct_hard_coded_references.sql +++ b/models/marts/dag/fct_hard_coded_references.sql @@ -3,6 +3,7 @@ with models as ( select * from {{ ref('int_all_graph_resources') }} where resource_type = 'model' + and not is_excluded ), final as ( @@ -15,4 +16,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql b/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql index fa354955..e6a1cd2f 100644 --- a/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql +++ b/models/marts/dag/fct_marts_or_intermediate_dependent_on_source.sql @@ -4,6 +4,8 @@ with direct_relationships as ( * from {{ ref('int_all_dag_relationships') }} where distance = 1 + and not parent_is_excluded + and not child_is_excluded ), final as ( select @@ -17,4 +19,4 @@ final as ( ) select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_model_fanout.sql b/models/marts/dag/fct_model_fanout.sql index 9d1a39db..8688ddc8 100644 --- a/models/marts/dag/fct_model_fanout.sql +++ b/models/marts/dag/fct_model_fanout.sql @@ -2,6 +2,8 @@ with all_dag_relationships as ( select * from {{ ref('int_all_dag_relationships') }} + where not parent_is_excluded + and not child_is_excluded ), -- find all models without children @@ -46,4 +48,4 @@ model_fanout_agg as ( select * from model_fanout_agg -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_multiple_sources_joined.sql b/models/marts/dag/fct_multiple_sources_joined.sql index e2b6bb4a..1a6c877a 100644 --- a/models/marts/dag/fct_multiple_sources_joined.sql +++ b/models/marts/dag/fct_multiple_sources_joined.sql @@ -6,6 +6,8 @@ with direct_source_relationships as ( from {{ ref('int_all_dag_relationships') }} where distance = 1 and parent_resource_type = 'source' + and not parent_is_excluded + and not child_is_excluded -- we order the CTE so that listagg returns values correctly sorted for some warehouses order by 1, 2 ), @@ -25,4 +27,4 @@ multiple_sources_joined as ( select * from multiple_sources_joined -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_rejoining_of_upstream_concepts.sql b/models/marts/dag/fct_rejoining_of_upstream_concepts.sql index 318ae778..12cd0ad7 100644 --- a/models/marts/dag/fct_rejoining_of_upstream_concepts.sql +++ b/models/marts/dag/fct_rejoining_of_upstream_concepts.sql @@ -4,6 +4,8 @@ with all_relationships as ( from {{ ref('int_all_dag_relationships') }} where parent_resource_type not in ('exposures', 'metrics') and child_resource_type not in ('exposures', 'metrics') + and not parent_is_excluded + and not child_is_excluded ), -- all parent/child relationships where the parent is BOTH the direct parent of the child and the second level parent of the child @@ -64,4 +66,4 @@ final_filtered as ( select * from final_filtered -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_root_models.sql b/models/marts/dag/fct_root_models.sql index 0ea8e4ff..d86d2005 100644 --- a/models/marts/dag/fct_root_models.sql +++ b/models/marts/dag/fct_root_models.sql @@ -5,6 +5,11 @@ with model_relationships as ( * from {{ ref('int_all_dag_relationships') }} where child_resource_type = 'model' + -- only filter out excluded children nodes + -- filtering parents could result in incorrectly flagging nodes that depend on excluded nodes + and not child_is_excluded + -- exclude required time spine + and child != 'metricflow_time_spine' ), final as ( @@ -17,4 +22,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_source_fanout.sql b/models/marts/dag/fct_source_fanout.sql index c2675e43..4c4eda99 100644 --- a/models/marts/dag/fct_source_fanout.sql +++ b/models/marts/dag/fct_source_fanout.sql @@ -6,6 +6,8 @@ with direct_source_relationships as ( where distance = 1 and parent_resource_type = 'source' and child_resource_type = 'model' + and not parent_is_excluded + and not child_is_excluded -- we order the CTE so that listagg returns values correctly sorted for some warehouses order by child ), @@ -25,4 +27,4 @@ source_fanout as ( select * from source_fanout -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql b/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql index 321da88d..aa411f12 100644 --- a/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql +++ b/models/marts/dag/fct_staging_dependent_on_marts_or_intermediate.sql @@ -7,6 +7,8 @@ with direct_model_relationships as ( where distance = 1 and parent_resource_type = 'model' and child_resource_type = 'model' + and not parent_is_excluded + and not child_is_excluded ), final as ( select @@ -20,4 +22,4 @@ final as ( ) select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_staging_dependent_on_staging.sql b/models/marts/dag/fct_staging_dependent_on_staging.sql index fbbe0e01..a69574ad 100644 --- a/models/marts/dag/fct_staging_dependent_on_staging.sql +++ b/models/marts/dag/fct_staging_dependent_on_staging.sql @@ -5,6 +5,8 @@ with direct_model_relationships as ( from {{ ref('int_all_dag_relationships') }} where parent_resource_type in ('model', 'snapshot') and child_resource_type in ('model', 'snapshot') + and not parent_is_excluded + and not child_is_excluded and distance = 1 ), @@ -21,4 +23,4 @@ bending_connections as ( select * from bending_connections -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/dag/fct_unused_sources.sql b/models/marts/dag/fct_unused_sources.sql index 6803dbef..7dfe583f 100644 --- a/models/marts/dag/fct_unused_sources.sql +++ b/models/marts/dag/fct_unused_sources.sql @@ -5,6 +5,8 @@ with source_relationships as ( * from {{ ref('int_all_dag_relationships') }} where parent_resource_type = 'source' + and not parent_is_excluded + and not child_is_excluded ), final as ( @@ -17,4 +19,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/documentation/fct_documentation_coverage.sql b/models/marts/documentation/fct_documentation_coverage.sql index 1df17322..39dd9ce3 100644 --- a/models/marts/documentation/fct_documentation_coverage.sql +++ b/models/marts/documentation/fct_documentation_coverage.sql @@ -3,6 +3,7 @@ with models as ( select * from {{ ref('int_all_graph_resources') }} where resource_type = 'model' + and not is_excluded ), conversion as ( diff --git a/models/marts/documentation/fct_undocumented_models.sql b/models/marts/documentation/fct_undocumented_models.sql index 60ae4b43..edf10a97 100644 --- a/models/marts/documentation/fct_undocumented_models.sql +++ b/models/marts/documentation/fct_undocumented_models.sql @@ -2,6 +2,7 @@ with all_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), @@ -18,4 +19,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/performance/fct_chained_views_dependencies.sql b/models/marts/performance/fct_chained_views_dependencies.sql index 42527100..95b7ae7a 100644 --- a/models/marts/performance/fct_chained_views_dependencies.sql +++ b/models/marts/performance/fct_chained_views_dependencies.sql @@ -3,6 +3,8 @@ with all_relationships as ( * from {{ ref('int_all_dag_relationships') }} where distance <> 0 + and not parent_is_excluded + and not child_is_excluded ), final as ( @@ -18,6 +20,6 @@ final as ( select * from final -{{ filter_exceptions(this) }} +{{ filter_exceptions(model.name) }} order by distance desc \ No newline at end of file diff --git a/models/marts/performance/fct_exposure_parents_materializations.sql b/models/marts/performance/fct_exposure_parents_materializations.sql index 87bd7683..0f519a2e 100644 --- a/models/marts/performance/fct_exposure_parents_materializations.sql +++ b/models/marts/performance/fct_exposure_parents_materializations.sql @@ -13,6 +13,8 @@ direct_exposure_relationships as ( parent_resource_type = 'source' ) ) + -- no test on child_is_excluded because exposures are never excluded + and not parent_is_excluded ), final as ( @@ -29,4 +31,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/structure/fct_model_directories.sql b/models/marts/structure/fct_model_directories.sql index 933f5230..1beb90db 100644 --- a/models/marts/structure/fct_model_directories.sql +++ b/models/marts/structure/fct_model_directories.sql @@ -55,6 +55,7 @@ innappropriate_subdirectories_non_staging_models as ( on folders.model_type = all_graph_resources.model_type -- either appropriate folder_name is not in the current_directory_path or a inappropriate folder name is closer to the file_name where all_graph_resources.model_type <> all_graph_resources.model_type_folder + and not is_excluded ), unioned as ( @@ -65,6 +66,6 @@ unioned as ( select * from unioned -{{ filter_exceptions(this) }} +{{ filter_exceptions(model.name) }} diff --git a/models/marts/structure/fct_model_naming_conventions.sql b/models/marts/structure/fct_model_naming_conventions.sql index 8f4bedca..d12b7c6a 100644 --- a/models/marts/structure/fct_model_naming_conventions.sql +++ b/models/marts/structure/fct_model_naming_conventions.sql @@ -3,6 +3,9 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded + -- exclude required metricflow time spine + and resource_name != 'metricflow_time_spine' ), naming_convention_prefixes as ( @@ -51,4 +54,4 @@ inappropriate_model_names as ( select * from inappropriate_model_names -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/structure/fct_source_directories.sql b/models/marts/structure/fct_source_directories.sql index a8e3c152..bc1b73e0 100644 --- a/models/marts/structure/fct_source_directories.sql +++ b/models/marts/structure/fct_source_directories.sql @@ -6,6 +6,7 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), -- find all sources that are definied in a .yml file NOT in their subdirectory @@ -22,4 +23,4 @@ inappropriate_subdirectories_sources as ( select * from inappropriate_subdirectories_sources -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/structure/fct_test_directories.sql b/models/marts/structure/fct_test_directories.sql index 3ca1c65e..0e6a3609 100644 --- a/models/marts/structure/fct_test_directories.sql +++ b/models/marts/structure/fct_test_directories.sql @@ -4,6 +4,7 @@ with resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), @@ -84,4 +85,4 @@ different_directories as ( select * from different_directories -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/tests/fct_missing_primary_key_tests.sql b/models/marts/tests/fct_missing_primary_key_tests.sql index 4c55a334..37bd0c29 100644 --- a/models/marts/tests/fct_missing_primary_key_tests.sql +++ b/models/marts/tests/fct_missing_primary_key_tests.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = 'my_alias', + ) +}} + with tests as ( @@ -15,4 +21,4 @@ final as ( select * from final -{{ filter_exceptions(this) }} \ No newline at end of file +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/tests/intermediate/int_model_test_summary.sql b/models/marts/tests/intermediate/int_model_test_summary.sql index 80ede484..54116d25 100644 --- a/models/marts/tests/intermediate/int_model_test_summary.sql +++ b/models/marts/tests/intermediate/int_model_test_summary.sql @@ -2,6 +2,7 @@ with all_graph_resources as ( select * from {{ ref('int_all_graph_resources') }} + where not is_excluded ), relationships as ( diff --git a/models/staging/graph/stg_metrics.sql b/models/staging/graph/stg_metrics.sql index fd921892..dd37b9a1 100644 --- a/models/staging/graph/stg_metrics.sql +++ b/models/staging/graph/stg_metrics.sql @@ -21,13 +21,16 @@ select cast(null as {{ dbt.type_string() }}) as file_path, cast(True as boolean) as is_described, cast(null as {{ dbt.type_string() }}) as metric_type, - cast(null as {{ dbt.type_string() }}) as model, cast(null as {{ dbt.type_string() }}) as label, - cast(null as {{ dbt.type_string() }}) as sql, - cast(null as {{ dbt.type_string() }}) as timestamp, cast(null as {{ dbt.type_string() }}) as package_name, - cast(null as {{ dbt.type_string() }}) as dimensions, - cast(null as {{ dbt.type_string() }}) as filters, + cast(null as {{ dbt.type_string() }}) as metric_filter, + cast(null as {{ dbt.type_string() }}) as metric_measure, + cast(null as {{ dbt.type_string() }}) as metric_measure_alias, + cast(null as {{ dbt.type_string() }}) as numerator, + cast(null as {{ dbt.type_string() }}) as denominator, + cast(null as {{ dbt.type_string() }}) as expr, + cast(null as {{ dbt.type_string() }}) as metric_window, + cast(null as {{ dbt.type_string() }}) as grain_to_date, cast(null as {{ dbt.type_string() }}) as meta from dummy_cte diff --git a/models/staging/graph/stg_nodes.sql b/models/staging/graph/stg_nodes.sql index 29985b2f..a5d5ecfe 100644 --- a/models/staging/graph/stg_nodes.sql +++ b/models/staging/graph/stg_nodes.sql @@ -38,7 +38,8 @@ select cast(null as {{ dbt.type_string() }}) as meta, cast(null as {{ dbt.type_string() }}) as hard_coded_references, cast(null as {{ dbt.type_string() }}) as macro_dependencies, - cast(True as boolean) as is_generic_test + cast(True as boolean) as is_generic_test, + cast(True as boolean) as is_excluded from dummy_cte where false \ No newline at end of file diff --git a/models/staging/graph/stg_sources.sql b/models/staging/graph/stg_sources.sql index 052652ff..ed018971 100644 --- a/models/staging/graph/stg_sources.sql +++ b/models/staging/graph/stg_sources.sql @@ -31,7 +31,8 @@ select cast(null as {{ dbt.type_string() }}) as package_name, cast(null as {{ dbt.type_string() }}) as loader, cast(null as {{ dbt.type_string() }}) as identifier, - cast(null as {{ dbt.type_string() }}) as meta + cast(null as {{ dbt.type_string() }}) as meta, + cast(True as boolean) as is_excluded from dummy_cte where false \ No newline at end of file