Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: check_exclude_cols option #10488

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20240726-134448.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Adding check exclude cols option for snapshots
time: 2024-07-26T13:44:48.494062+01:00
custom:
Author: willmaclean
Issue: "10438"
24 changes: 20 additions & 4 deletions core/dbt/artifacts/resources/v1/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,36 +17,52 @@ class SnapshotConfig(NodeConfig):
updated_at: Optional[str] = None
# Not using Optional because of serialization issues with a Union of str and List[str]
check_cols: Union[str, List[str], None] = None
check_exclude_cols: Union[str, List[str], None] = None

def final_validate(self):
if not self.strategy or not self.unique_key:
raise ValidationError(
"Snapshots must be configured with a 'strategy' and 'unique_key'."
)
if self.strategy == "check":
if not self.check_cols:
if self.check_cols and self.check_exclude_cols:
raise ValidationError(
"A snapshot configured with the check strategy cannot "
"specify both check_cols and check_exclude_cols."
)
if not self.check_cols and not self.check_exclude_cols:
raise ValidationError(
"A snapshot configured with the check strategy must "
"specify a check_cols configuration."
"specify a check_cols or check_exclude_cols configuration."
)
if isinstance(self.check_cols, str) and self.check_cols != "all":
raise ValidationError(
f"Invalid value for 'check_cols': {self.check_cols}. "
"Expected 'all' or a list of strings."
)
if isinstance(self.check_exclude_cols, str):
raise ValidationError(
f"Invalid value for 'check_exclude_cols': {self.check_exclude_cols}. "
"Expected a list of strings."
)
elif self.strategy == "timestamp":
if not self.updated_at:
raise ValidationError(
"A snapshot configured with the timestamp strategy "
"must specify an updated_at configuration."
)
if self.check_cols:
raise ValidationError("A 'timestamp' snapshot should not have 'check_cols'")
if self.check_cols or self.check_exclude_cols:
raise ValidationError(
"A 'timestamp' snapshot should not have 'check_cols' or 'check_exclude_cols"
)
# If the strategy is not 'check' or 'timestamp' it's a custom strategy,
# formerly supported with GenericSnapshotConfig

if self.materialized and self.materialized != "snapshot":
raise ValidationError("A snapshot must have a materialized value of 'snapshot'")
# def r():
# raise Exception()
# self.check_exclude_cols = lambda: r()

# Called by "calculate_node_config_dict" in ContextConfigGenerator
def finalize_and_validate(self):
Expand Down
2 changes: 1 addition & 1 deletion dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
git+https://github.com/dbt-labs/dbt-adapters.git@main
git+https://github.com/willmaclean/dbt-adapters.git@main
git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter
git+https://github.com/dbt-labs/dbt-common.git@main
git+https://github.com/dbt-labs/dbt-postgres.git@main
Expand Down
Loading