resolve conflicts

.github/workflows/mkdocs_main.yml

@@ -10,12 +10,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
 
@@ -30,4 +30,4 @@ jobs:
       run: |
         git config --global user.name "GitHub Actions Bot"
         git config --global user.email "github-actions[bot]@users.noreply.github.com"
-        mike deploy --push --message "Deployed by GitHub Actions" main
+        mike deploy --push --message "Deployed by GitHub Actions" main

.github/workflows/mkdocs_release.yml

@@ -10,12 +10,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
 
@@ -41,4 +41,4 @@ jobs:
         git config user.name "GitHub Actions"
         git config user.email "[email protected]"
         mike deploy --push --message "Deploy docs for release ${{ github.event.release.tag_name }}" --update-alias $MAJOR_MINOR_VERSION latest
-        mike set-default --push latest
+        mike set-default --push latest

.github/workflows/stale.yml

@@ -0,0 +1,30 @@
+# **what?**
+# For issues that have been open for awhile without activity, label
+# them as stale with a warning that they will be closed out. If
+# anyone comments to keep the issue open, it will automatically
+# remove the stale label and keep it open.
+
+# Stale label rules:
+# awaiting_response, more_information_needed -> 90 days
+# good_first_issue, help_wanted -> 360 days (a year)
+# tech_debt -> 720 (2 years)
+# all else defaults -> 180 days (6 months)
+
+# **why?**
+# To keep the repo in a clean state from issues that aren't relevant anymore
+
+# **when?**
+# Once a day
+
+name: "Close stale issues and PRs"
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main

.github/workflows/triage-labels.yml

@@ -0,0 +1,31 @@
+# **what?**
+# When we triage issues, we sometimes need more information from the issue creator.  In
+# those cases we remove the `triage` label and add the `awaiting_response` label.  Once we
+# receive a response in the form of a comment, we want the `awaiting_response` label removed
+# in favor of the `triage` label so we are aware that the issue needs action.
+
+# **why?**
+# To help with out team triage issue tracking
+
+# **when?**
+# This will run when a comment is added to an issue and that issue has the `awaiting_response` label.
+
+name: Update Triage Label
+
+on: issue_comment
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  issues: write
+
+jobs:
+  triage_label:
+    if: contains(github.event.issue.labels.*.name, 'awaiting_response')
+    uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main
+    with:
+      add_label: "triage"
+      remove_label: "awaiting_response"
+    secrets: inherit

.gitignore

@@ -6,4 +6,5 @@ dbt_packages/
 .vscode
 integration_tests/state/
 site/
-env/
+env/
+.DS_Store

dbt_project.yml

@@ -2,7 +2,7 @@ name: 'dbt_project_evaluator'
 version: '1.0.0'
 config-version: 2
 
-require-dbt-version: [">=1.5.0", "<2.0.0"]
+require-dbt-version: [">=1.6.0-rc1", "<2.0.0"]
 
 model-paths: ["models"]
 analysis-paths: ["analysis"]

docs/contributing.md

@@ -24,6 +24,7 @@ Docs are then automatically pushed to the website as part of our CI/CD process.
         "markdownlint.config": {
             "ul-indent": {"indent": 4},
             "MD036": false,
+            "MD046": false,
         }
         ```
 

docs/customization/customization.md

@@ -1,7 +1,14 @@
-# Disabling Models
+# Disabling checks from the package
 
-If there is a particular model or set of models that you *do not want this package to execute*, you can
-disable these models as you would any other model in your `dbt_project.yml` file
+!!! note
+
+    This section is describing how to completely deactivate tests from the package.
+    If you are looking to deactivate models/sources from being tested, you can look at [excluding packages and paths](excluding-packages-and-paths.md)
+
+All the tests done as part of the package are tied to `fct` models.
+
+If there is a particular test or set of tests that you *do not want this package to execute*, you can
+disable the corresponding `fct` models as you would any other model in your `dbt_project.yml` file
 
 ``` yaml title="dbt_project.yml"
 models:
@@ -14,5 +21,4 @@ models:
         # disable single DAG model
         fct_model_fanout:
           +enabled: false
-
 ```

docs/customization/excluding-packages-and-paths.md

@@ -0,0 +1,57 @@
+# Excluding packages or sources/models based on their path
+
+!!! note
+
+    This section is describing how to entirely exclude models/sources and packages to be evaluated.
+    If you want to document exceptions to the rules, see the section [on exceptions](exceptions.md)
+    and if you want to deactivate entire tests you can follow instructions from [this page](customization.md)
+
+There might be cases where you want to exclude models/sources from being tested:
+
+- they could come from a package for which you have no control over
+- you might be refactoring your project and wanting to exclude entire folders to follow best-practices in the new models
+
+In that case, this package provides the ability to exclude whole packages and/or models and sources based on their path
+
+## Configuration
+
+The variables `exclude_packages` and `exclude_paths_from_project` allow you to define a list of regex patterns to exclude from being reported as errors.
+
+- `exclude_packages` accepts a list of package names to exclude from the tool. To exclude all packages except the current project, you can set it to `["all"]`
+- `exclude_paths_from_project` accepts a list of regular expressions of paths to exclude for the current project
+    - **for models**, the regex provided will try to match the pattern in the string `<path/to/model.sql>`, allowing to exclude packages, but also whole folders or individual models
+    - **for sources**, the regex will try to match the pattern in `<path/to/sources.yml>:<source_name>.<source_table_name>` *(the pattern is different than for models because the path itself doesn't let us exclude individual sources)*
+
+!!! note
+
+    We currently don't allow excluding metrics and exposures, as if those need to be entirely excluded they could be deactivated from the project.
+
+    If you have a specific use case requiring this ability, please raise a GitHub issue to explain the situation you'd like to solve and we can revisit this decision !
+
+### Example to exclude a whole package
+
+```yaml title="dbt_project.yml"
+vars:
+  exclude_packages: ["upstream_package"]
+```
+
+### Example to exclude models/sources in a given path
+
+```yaml title="dbt_project.yml"
+vars:
+  exclude_paths_from_project: ["/models/legacy/"]
+```
+
+### Example to exclude both a package and models/sources in 2 different paths
+
+```yaml title="dbt_project.yml"
+vars:
+  exclude_packages: ["upstream_package"]
+  exclude_paths_from_project: ["/models/legacy/", "/my_date_spine.sql"]
+```
+
+## Tips and tricks
+
+Regular expressions are very powerful but can become complex. After defining your value for `exclude_paths_from_project`, we recommend running the package and inspecting the model `int_all_graph_resources`, checking if the value in the column `is_excluded` matches your expectation.
+
+A useful tool to debug regular expression is [regex101](https://regex101.com/). You can provide a pattern and a list of strings to see which ones actually match the pattern.

docs/rules.md

@@ -12,7 +12,7 @@ hide:
 |Modeling                                      |[Rejoining of Upstream Concepts](../rules/modeling/#rejoining-of-upstream-concepts)                                    |`fct_rejoining_of_upstream_concepts`|
 |Modeling                                      |[Model Fanout](../rules/modeling/#model-fanout)                                                                        |`fct_model_fanout`|
 |Modeling                                      |[Downstream Models Dependent on Source](../rules/modeling/#downstream-models-dependent-on-source)                      |`fct_marts_or_intermediate_dependent_on_source`|
-|Modeling                                      |[Direct Join to Source](/..rules/modeling/#direct-join-to-source)                                                      |`fct_direct_join_to_source`|
+|Modeling                                      |[Direct Join to Source](../rules/modeling/#direct-join-to-source)                                                      |`fct_direct_join_to_source`|
 |Modeling                                      |[Duplicate Sources](../rules/modeling/#duplicate-sources)                                                              |`fct_duplicate_sources`|
 |Modeling                                      |[Hard Coded References](../rules/modeling/#hard-coded-references)                                                      |`fct_hard_coded_references`|
 |Modeling                                      |[Multiple Sources Joined](../rules/modeling/#multiple-sources-joined)                                                  |`fct_multiple_sources_joined`|

integration_tests/dbt_project.yml

@@ -59,6 +59,8 @@ seeds:
 
 vars:
   # ensure integration tests run successfully when there are 0 of a given model type (extra)
+  exclude_packages: []
+  exclude_paths_from_project: ["/to_exclude/","source_3.table_6"]
   model_types: ['staging', 'intermediate', 'marts', 'other', 'extra', 'new_model_type']
   # dummy variable used for testing fct_hard_coded_references
   my_table_reference: 'grace_table'

integration_tests/models/marts/exposures.yml

@@ -13,6 +13,7 @@ exposures:
 
     depends_on:
       - ref('fct_model_6')
+      - ref('fct_model_9')
       - ref('dim_model_7')
       - ref('dim_model_7')
       - source('source_1', 'table_1')

integration_tests/models/marts/int_model_5.sql

@@ -1,2 +1,3 @@
-{{ ref('int_model_4') }}
-{{ ref('stg_model_1') }}
+-- {{ ref('int_model_4') }}
+-- {{ ref('stg_model_1') }}
+select 1 as id

integration_tests/models/marts/intermediate/dim_model_7.sql

@@ -1,2 +1,9 @@
-{{ ref('stg_model_4') }}
-{{ ref('int_model_5') }}
+
+{{
+    config(
+        materialized = 'table',
+    )
+}}
+
+select * from {{ ref('stg_model_4') }}
+-- {{ ref('int_model_5') }}

integration_tests/models/marts/intermediate/to_exclude/int_model_10.sql

@@ -0,0 +1,3 @@
+-- depends on: {{ ref('fct_model_9') }}
+
+select 1 as id

integration_tests/models/marts/intermediate/to_exclude/int_model_11.sql

@@ -0,0 +1,3 @@
+-- depends on: {{ ref('fct_model_9') }}
+
+select 1 as id

integration_tests/models/marts/metrics.yml

@@ -1,29 +1,37 @@
 version: 2
 
-metrics:
-  - name: new_customers
-    label: New Customers marked 'paying'
-    model: ref('dim_model_7')
-    description: "The number of paid customers using the product"
 
-    calculation_method: count
-    expression: user_id # superfluous here, but shown as an example
+semantic_models:
+  - name: my_model 
+    model: ref('dim_model_7') 
+    description: "Transaction fact table at the transaction level. This table contains one row per transaction and includes the transaction timestamp."
+    defaults:
+      agg_time_dimension: date_day
+
+    entities: # Entities included in the table are defined here. MetricFlow will use these columns as join keys.
+      - name: dim
+        type: primary
+        expr: id
 
-    timestamp: signup_date
-    time_grains: [day, week, month]
+    dimensions: # dimensions are qualitative values such as names, dates, or geographical data. They provide context to metrics and allow "metric by group" data slicing.
+      - name: id
+        type: categorical
 
-    dimensions:
-      - plan
-      - country
+      - name: date_day
+        type: time
+        type_params:
+          time_granularity: day
 
-    filters:
-      - field: is_paying
-        operator: '='
-        value: 'true'
-      - field: company_name
-        operator: '!='
-        value: "Acme', Inc"
+    measures: # Measures are columns we perform an aggregation over. Measures are inputs to metrics.
+      - name: total_count
+        description: "The total count."
+        agg: count
+        expr: 1
 
-    meta: 
-      team: "Finance"
-      refresh_rate: "Bob's weekly run"
+metrics:
+  - name: new_metric
+    label: "New Metric"
+    type: simple 
+    type_params:
+  # Specify the measure you are creating a proxy for. 
+      measure: total_count 

integration_tests/models/reports/report_1.sql

@@ -1 +1,3 @@
-{{ ref('fct_model_6') }}
+-- {{ ref('fct_model_6') }}
+
+select true

integration_tests/models/reports/reports.yml

@@ -2,8 +2,10 @@ version: 2
 
 models:
   - name: report_1
-    description: tom cruise's minorty report
+    description: tom cruise's minority report
     access: public
+    config:
+      materialized: table
   - name: report_2
     access: public
     config:

integration_tests/models/staging/source_1/source.yml

@@ -17,4 +17,9 @@ sources:
     schema: real_schema_2
     # database: real_database
     tables:
-      - name: table_3
+      - name: table_3
+
+  - name: source_3
+    schema: real_schema_3
+    tables:
+      - name: table_6

integration_tests/models/utils/_utils_models.yml

@@ -0,0 +1,8 @@
+models:
+  - name: metricflow_time_spine
+    description: default time spine for metricflow 
+    columns:
+      - name: date_day
+        tests:
+          - unique
+          - not_null