From 38572a39ce72681264e55cc0ee24e709ceaa16ae Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Mon, 18 Sep 2023 14:47:30 -0700 Subject: [PATCH 01/11] Add models for source documentation testing --- models/marts/documentation/documentation.yml | 10 +++++++++ .../fct_undocumented_source_nodes.sql | 22 +++++++++++++++++++ .../fct_undocumented_sources.sql | 22 +++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 models/marts/documentation/fct_undocumented_source_nodes.sql create mode 100644 models/marts/documentation/fct_undocumented_sources.sql diff --git a/models/marts/documentation/documentation.yml b/models/marts/documentation/documentation.yml index c2f95f74..c8710cee 100644 --- a/models/marts/documentation/documentation.yml +++ b/models/marts/documentation/documentation.yml @@ -16,5 +16,15 @@ models: - name: fct_undocumented_models description: > This model contains all models that do not have a description configured in a YML file. + tests: + - is_empty + - name: fct_undocumented_source_nodes + description: > + This model contains all source nodes that do not have a description configured in a YML file. + tests: + - is_empty + - name: fct_undocumented_sources + description: > + This model contains all sources that do not have a description configured in a YML file. tests: - is_empty \ No newline at end of file diff --git a/models/marts/documentation/fct_undocumented_source_nodes.sql b/models/marts/documentation/fct_undocumented_source_nodes.sql new file mode 100644 index 00000000..189462a4 --- /dev/null +++ b/models/marts/documentation/fct_undocumented_source_nodes.sql @@ -0,0 +1,22 @@ +with + +all_resources as ( + select * from {{ ref('int_all_graph_resources') }} + where not is_excluded + +), + +final as ( + + select + resource_name, + model_type + + from all_resources + where not is_described and resource_type = 'source' + +) + +select * from final + +{{ filter_exceptions(model.name) }} \ No newline at end of file diff --git a/models/marts/documentation/fct_undocumented_sources.sql b/models/marts/documentation/fct_undocumented_sources.sql new file mode 100644 index 00000000..5c65138c --- /dev/null +++ b/models/marts/documentation/fct_undocumented_sources.sql @@ -0,0 +1,22 @@ +with + +all_resources as ( + select * from {{ ref('int_all_graph_resources') }} + where not is_excluded + +), + +final as ( + + select + resource_name, + model_type + + from all_resources + where not is_source_described and resource_type = 'source' + +) + +select * from final + +{{ filter_exceptions(model.name) }} \ No newline at end of file From cc19783162ad7e5f948662277fc63dba136a6ffb Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Mon, 18 Sep 2023 14:53:49 -0700 Subject: [PATCH 02/11] First pass at documentation --- docs/rules.md | 2 ++ docs/rules/documentation.md | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/rules.md b/docs/rules.md index 8a71cc9d..4abee19b 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -23,6 +23,8 @@ hide: |Testing |[Test Coverage](../rules/testing/#test-coverage) |`fct_test_coverage`| |Documentation |[Undocumented Models](../rules/documentation/#undocumented-models) |`fct_undocumented_models`| |Documentation |[Documentation Coverage](../rules/documentation/#documentation-coverage) |`fct_documentation_coverage`| +|Documentation |[Undocumented Source Nodes](../rules/documentation/#undocumented-source-nodes) |`fct_undocumented_source_nodes`| +|Documentation |[Undocumented Sources](../rules/documentation/#undocumented-sources) |`fct_documentation_sources`| |Structure |[Test Directories](../rules/structure/#test-directories) |`fct_test_directories`| |Structure |[Model Naming Conventions](../rules/structure/#model-naming-conventions) |`fct_model_naming_conventions`| |Structure |[Source Directories](../rules/structure/#source-directories) |`fct_source_directories`| diff --git a/docs/rules/documentation.md b/docs/rules/documentation.md index 63466071..049e6738 100644 --- a/docs/rules/documentation.md +++ b/docs/rules/documentation.md @@ -46,3 +46,29 @@ function in the model's `.yml` entry. !!! note "Tip" We recommend that every model in your dbt project has at minimum a model-level description. This ensures that each model's purpose is clear to other developers and stakeholders when viewing the dbt docs site. Missing documentation should be addressed first for marts models, then for the rest of your project, to ensure that stakeholders in the organization can understand the data which is surfaced to them. + +## Undocumented Source Nodes + +`fct_undocumented_source_nodes` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/documentation/fct_undocumented_source_nodes.sql)) lists every source node with no description configured. + +**Reason to Flag** + +Good documentation for your dbt sources will help contributors to your project understand how and when data is loaded into your warehouse. + +**How to Remediate** + +Apply a text [description](https://docs.getdbt.com/docs/building-a-dbt-project/documentation) in the table's `.yml` entry, or create a [docs block](https://docs.getdbt.com/docs/building-a-dbt-project/documentation#using-docs-blocks) in a markdown file, and use the `{{ doc() }}` +function in the table's `.yml` entry. + +## Undocumented Sources + +`fct_undocumented_sources` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/documentation/fct_undocumented_sources.sql)) lists every source with no description configured. + +**Reason to Flag** + +Good documentation for your dbt sources will help contributors to your project understand how and when data is loaded into your warehouse. + +**How to Remediate** + +Apply a text [description](https://docs.getdbt.com/docs/building-a-dbt-project/documentation) in the source's `.yml` entry, or create a [docs block](https://docs.getdbt.com/docs/building-a-dbt-project/documentation#using-docs-blocks) in a markdown file, and use the `{{ doc() }}` +function in the source's `.yml` entry. \ No newline at end of file From 180602d7ce797e6d32b2123cc5b7c8ee68631a77 Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Tue, 19 Sep 2023 10:54:34 -0700 Subject: [PATCH 03/11] Change naming from node to table --- docs/rules/documentation.md | 4 ++-- models/marts/documentation/documentation.yml | 4 ++-- ...ed_source_nodes.sql => fct_undocumented_source_tables.sql} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename models/marts/documentation/{fct_undocumented_source_nodes.sql => fct_undocumented_source_tables.sql} (100%) diff --git a/docs/rules/documentation.md b/docs/rules/documentation.md index 049e6738..b08c837a 100644 --- a/docs/rules/documentation.md +++ b/docs/rules/documentation.md @@ -47,9 +47,9 @@ function in the model's `.yml` entry. We recommend that every model in your dbt project has at minimum a model-level description. This ensures that each model's purpose is clear to other developers and stakeholders when viewing the dbt docs site. Missing documentation should be addressed first for marts models, then for the rest of your project, to ensure that stakeholders in the organization can understand the data which is surfaced to them. -## Undocumented Source Nodes +## Undocumented Source Tables -`fct_undocumented_source_nodes` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/documentation/fct_undocumented_source_nodes.sql)) lists every source node with no description configured. +`fct_undocumented_source_tables` ([source](https://github.com/dbt-labs/dbt-project-evaluator/tree/main/models/marts/documentation/fct_undocumented_source_tables.sql)) lists every source table with no description configured. **Reason to Flag** diff --git a/models/marts/documentation/documentation.yml b/models/marts/documentation/documentation.yml index c8710cee..38e9ce06 100644 --- a/models/marts/documentation/documentation.yml +++ b/models/marts/documentation/documentation.yml @@ -18,9 +18,9 @@ models: This model contains all models that do not have a description configured in a YML file. tests: - is_empty - - name: fct_undocumented_source_nodes + - name: fct_undocumented_source_tables description: > - This model contains all source nodes that do not have a description configured in a YML file. + This model contains all source tables that do not have a description configured in a YML file. tests: - is_empty - name: fct_undocumented_sources diff --git a/models/marts/documentation/fct_undocumented_source_nodes.sql b/models/marts/documentation/fct_undocumented_source_tables.sql similarity index 100% rename from models/marts/documentation/fct_undocumented_source_nodes.sql rename to models/marts/documentation/fct_undocumented_source_tables.sql From 47d6a0a7810caa1376f94bd8185f066ac5c352e8 Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Tue, 19 Sep 2023 11:05:02 -0700 Subject: [PATCH 04/11] Change naming from node to table --- docs/rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rules.md b/docs/rules.md index 4abee19b..cc39851e 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -23,7 +23,7 @@ hide: |Testing |[Test Coverage](../rules/testing/#test-coverage) |`fct_test_coverage`| |Documentation |[Undocumented Models](../rules/documentation/#undocumented-models) |`fct_undocumented_models`| |Documentation |[Documentation Coverage](../rules/documentation/#documentation-coverage) |`fct_documentation_coverage`| -|Documentation |[Undocumented Source Nodes](../rules/documentation/#undocumented-source-nodes) |`fct_undocumented_source_nodes`| +|Documentation |[Undocumented Source Nodes](../rules/documentation/#undocumented-source-tables) |`fct_undocumented_source_tables`| |Documentation |[Undocumented Sources](../rules/documentation/#undocumented-sources) |`fct_documentation_sources`| |Structure |[Test Directories](../rules/structure/#test-directories) |`fct_test_directories`| |Structure |[Model Naming Conventions](../rules/structure/#model-naming-conventions) |`fct_model_naming_conventions`| From 840cd681e1da109efc55aeed9b39f88766833f99 Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Tue, 19 Sep 2023 11:05:23 -0700 Subject: [PATCH 05/11] Change naming from node to table --- docs/rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rules.md b/docs/rules.md index cc39851e..1adc6060 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -23,7 +23,7 @@ hide: |Testing |[Test Coverage](../rules/testing/#test-coverage) |`fct_test_coverage`| |Documentation |[Undocumented Models](../rules/documentation/#undocumented-models) |`fct_undocumented_models`| |Documentation |[Documentation Coverage](../rules/documentation/#documentation-coverage) |`fct_documentation_coverage`| -|Documentation |[Undocumented Source Nodes](../rules/documentation/#undocumented-source-tables) |`fct_undocumented_source_tables`| +|Documentation |[Undocumented Source Tables](../rules/documentation/#undocumented-source-tables) |`fct_undocumented_source_tables`| |Documentation |[Undocumented Sources](../rules/documentation/#undocumented-sources) |`fct_documentation_sources`| |Structure |[Test Directories](../rules/structure/#test-directories) |`fct_test_directories`| |Structure |[Model Naming Conventions](../rules/structure/#model-naming-conventions) |`fct_model_naming_conventions`| From 0fe5a9ec1203c90c985391f3dc0dd23351836c58 Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Tue, 19 Sep 2023 11:16:54 -0700 Subject: [PATCH 06/11] Add example source yml to docs --- docs/rules/documentation.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/rules/documentation.md b/docs/rules/documentation.md index b08c837a..74297ead 100644 --- a/docs/rules/documentation.md +++ b/docs/rules/documentation.md @@ -59,6 +59,13 @@ Good documentation for your dbt sources will help contributors to your project u Apply a text [description](https://docs.getdbt.com/docs/building-a-dbt-project/documentation) in the table's `.yml` entry, or create a [docs block](https://docs.getdbt.com/docs/building-a-dbt-project/documentation#using-docs-blocks) in a markdown file, and use the `{{ doc() }}` function in the table's `.yml` entry. +``` +sources: + - name: my_source + tables: + - name: my_table + description: This is the source table description +``` ## Undocumented Sources @@ -71,4 +78,11 @@ Good documentation for your dbt sources will help contributors to your project u **How to Remediate** Apply a text [description](https://docs.getdbt.com/docs/building-a-dbt-project/documentation) in the source's `.yml` entry, or create a [docs block](https://docs.getdbt.com/docs/building-a-dbt-project/documentation#using-docs-blocks) in a markdown file, and use the `{{ doc() }}` -function in the source's `.yml` entry. \ No newline at end of file +function in the source's `.yml` entry. +``` +sources: + - name: my_source + description: This is the source description + tables: + - name: my_table +``` \ No newline at end of file From d69127318b9dacea66753f892f7f45b223e07cef Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Tue, 19 Sep 2023 12:40:43 -0700 Subject: [PATCH 07/11] Add integration tests --- .../models/staging/source_1/source.yml | 1 + integration_tests/seeds/docs/docs_seeds.yml | 20 ++++++++++++++++++- .../test_fct_undocumented_source_tables.csv | 6 ++++++ .../docs/test_fct_undocumented_sources.csv | 2 ++ 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv create mode 100644 integration_tests/seeds/docs/test_fct_undocumented_sources.csv diff --git a/integration_tests/models/staging/source_1/source.yml b/integration_tests/models/staging/source_1/source.yml index 289c264c..f4e9495e 100644 --- a/integration_tests/models/staging/source_1/source.yml +++ b/integration_tests/models/staging/source_1/source.yml @@ -2,6 +2,7 @@ version: 2 sources: - name: source_1 + description: this is source 1. schema: real_schema # database: real_database tables: diff --git a/integration_tests/seeds/docs/docs_seeds.yml b/integration_tests/seeds/docs/docs_seeds.yml index 643b0e08..b011f388 100644 --- a/integration_tests/seeds/docs/docs_seeds.yml +++ b/integration_tests/seeds/docs/docs_seeds.yml @@ -30,4 +30,22 @@ seeds: - staging_documentation_coverage_pct - intermediate_documentation_coverage_pct - marts_documentation_coverage_pct - - other_documentation_coverage_pct \ No newline at end of file + - other_documentation_coverage_pct + + - name: test_fct_undocumented_source_tables + config: + tags: + - docs + tests: + - dbt_utils.equality: + name: equality_fct_undocumented_source_tables + compare_model: ref('fct_undocumented_source_tables') + + - name: test_fct_undocumented_sources + config: + tags: + - docs + tests: + - dbt_utils.equality: + name: equality_fct_undocumented_sources + compare_model: ref('fct_undocumented_sources') \ No newline at end of file diff --git a/integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv b/integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv new file mode 100644 index 00000000..7939f28a --- /dev/null +++ b/integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv @@ -0,0 +1,6 @@ +resource_name,model_type +source_1.table_4, +source_1.table_5, +source_2.table_3, +source_1.raw_table_5, +source_1.table_2 \ No newline at end of file diff --git a/integration_tests/seeds/docs/test_fct_undocumented_sources.csv b/integration_tests/seeds/docs/test_fct_undocumented_sources.csv new file mode 100644 index 00000000..3333cc20 --- /dev/null +++ b/integration_tests/seeds/docs/test_fct_undocumented_sources.csv @@ -0,0 +1,2 @@ +resource_name,model_type +source_2.table_3, \ No newline at end of file From 91fbe86b4e5e8f91abc89def1db172609029fde2 Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Tue, 19 Sep 2023 12:52:48 -0700 Subject: [PATCH 08/11] Remove unnesscessary model_type col --- .../seeds/docs/test_fct_undocumented_source_tables.csv | 10 +++++----- .../seeds/docs/test_fct_undocumented_sources.csv | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv b/integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv index 7939f28a..0aa8f5b2 100644 --- a/integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv +++ b/integration_tests/seeds/docs/test_fct_undocumented_source_tables.csv @@ -1,6 +1,6 @@ -resource_name,model_type -source_1.table_4, -source_1.table_5, -source_2.table_3, -source_1.raw_table_5, +resource_name +source_1.table_4 +source_1.table_5 +source_2.table_3 +source_1.raw_table_5 source_1.table_2 \ No newline at end of file diff --git a/integration_tests/seeds/docs/test_fct_undocumented_sources.csv b/integration_tests/seeds/docs/test_fct_undocumented_sources.csv index 3333cc20..0af60a19 100644 --- a/integration_tests/seeds/docs/test_fct_undocumented_sources.csv +++ b/integration_tests/seeds/docs/test_fct_undocumented_sources.csv @@ -1,2 +1,2 @@ -resource_name,model_type -source_2.table_3, \ No newline at end of file +resource_name +source_2.table_3 \ No newline at end of file From 867d091b6df14d8943a375089a992a54372198ce Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Wed, 20 Sep 2023 09:45:56 -0700 Subject: [PATCH 09/11] Remove unnesscessary model_type col (again) --- models/marts/documentation/fct_undocumented_source_tables.sql | 3 +-- models/marts/documentation/fct_undocumented_sources.sql | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/models/marts/documentation/fct_undocumented_source_tables.sql b/models/marts/documentation/fct_undocumented_source_tables.sql index 189462a4..57558298 100644 --- a/models/marts/documentation/fct_undocumented_source_tables.sql +++ b/models/marts/documentation/fct_undocumented_source_tables.sql @@ -9,8 +9,7 @@ all_resources as ( final as ( select - resource_name, - model_type + resource_name from all_resources where not is_described and resource_type = 'source' diff --git a/models/marts/documentation/fct_undocumented_sources.sql b/models/marts/documentation/fct_undocumented_sources.sql index 5c65138c..51cce40d 100644 --- a/models/marts/documentation/fct_undocumented_sources.sql +++ b/models/marts/documentation/fct_undocumented_sources.sql @@ -9,8 +9,7 @@ all_resources as ( final as ( select - resource_name, - model_type + resource_name from all_resources where not is_source_described and resource_type = 'source' From a415a797cfc7dd61f8efe0e2ad1baa02a6ef2414 Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Wed, 20 Sep 2023 09:50:27 -0700 Subject: [PATCH 10/11] Use source name for source-grain descriptions --- .../seeds/docs/test_fct_undocumented_sources.csv | 2 +- models/marts/documentation/fct_undocumented_sources.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/seeds/docs/test_fct_undocumented_sources.csv b/integration_tests/seeds/docs/test_fct_undocumented_sources.csv index 0af60a19..e1f3ba7a 100644 --- a/integration_tests/seeds/docs/test_fct_undocumented_sources.csv +++ b/integration_tests/seeds/docs/test_fct_undocumented_sources.csv @@ -1,2 +1,2 @@ resource_name -source_2.table_3 \ No newline at end of file +source_2 \ No newline at end of file diff --git a/models/marts/documentation/fct_undocumented_sources.sql b/models/marts/documentation/fct_undocumented_sources.sql index 51cce40d..bb82b5d1 100644 --- a/models/marts/documentation/fct_undocumented_sources.sql +++ b/models/marts/documentation/fct_undocumented_sources.sql @@ -8,8 +8,8 @@ all_resources as ( final as ( - select - resource_name + select distinct + source_name from all_resources where not is_source_described and resource_type = 'source' From 7ecc826d6fda10617b165daa23ad4f030829b937 Mon Sep 17 00:00:00 2001 From: Elize Papineau Date: Wed, 20 Sep 2023 09:54:37 -0700 Subject: [PATCH 11/11] Update seed column name --- integration_tests/seeds/docs/test_fct_undocumented_sources.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/seeds/docs/test_fct_undocumented_sources.csv b/integration_tests/seeds/docs/test_fct_undocumented_sources.csv index e1f3ba7a..c4bc5d34 100644 --- a/integration_tests/seeds/docs/test_fct_undocumented_sources.csv +++ b/integration_tests/seeds/docs/test_fct_undocumented_sources.csv @@ -1,2 +1,2 @@ -resource_name +source_name source_2 \ No newline at end of file